56 files changed, 6974 insertions, 456 deletions

diff --git a/lib/Target/ARM/ARMBaseInstrInfo.cpp b/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 0a0b0ea18540..1995f79fa373 100644
--- a/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -312,7 +312,7 @@ ARMBaseInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                                MachineBasicBlock *FBB,
                              const SmallVectorImpl<MachineOperand> &Cond) const {
   // FIXME this should probably have a DebugLoc argument
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
 
   ARMFunctionInfo *AFI = MBB.getParent()->getInfo<ARMFunctionInfo>();
   int BOpc   = !AFI->isThumbFunction()
@@ -653,7 +653,7 @@ ARMBaseInstrInfo::copyRegToReg(MachineBasicBlock &MBB,
                                unsigned DestReg, unsigned SrcReg,
                                const TargetRegisterClass *DestRC,
                                const TargetRegisterClass *SrcRC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   // tGPR is used sometimes in ARM instructions that need to avoid using
@@ -715,7 +715,7 @@ void ARMBaseInstrInfo::
 storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                     unsigned SrcReg, bool isKill, int FI,
                     const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = *MF.getFrameInfo();
@@ -769,7 +769,7 @@ void ARMBaseInstrInfo::
 loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                      unsigned DestReg, int FI,
                      const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = *MF.getFrameInfo();
diff --git a/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index b380c954d606..f1625469085e 100644
--- a/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -1277,8 +1277,7 @@ emitPrologue(MachineFunction &MF) const {
   unsigned VARegSaveSize = AFI->getVarArgsRegSaveSize();
   unsigned NumBytes = MFI->getStackSize();
   const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
-  DebugLoc dl = (MBBI != MBB.end() ?
-                 MBBI->getDebugLoc() : DebugLoc::getUnknownLoc());
+  DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   // Determine the sizes of each callee-save spill areas and record which frame
   // belongs to which callee-save spill areas.
diff --git a/lib/Target/ARM/ARMConstantIslandPass.cpp b/lib/Target/ARM/ARMConstantIslandPass.cpp
index 1c5bd42d63da..13d8b74014c5 100644
--- a/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -399,8 +399,8 @@ void ARMConstantIslands::DoInitialPlacement(MachineFunction &MF,
     // aligned.
     assert((Size & 3) == 0 && "CP Entry not multiple of 4 bytes!");
     MachineInstr *CPEMI =
-      BuildMI(BB, DebugLoc::getUnknownLoc(), TII->get(ARM::CONSTPOOL_ENTRY))
-                           .addImm(i).addConstantPoolIndex(i).addImm(Size);
+      BuildMI(BB, DebugLoc(), TII->get(ARM::CONSTPOOL_ENTRY))
+        .addImm(i).addConstantPoolIndex(i).addImm(Size);
     CPEMIs.push_back(CPEMI);
 
     // Add a new CPEntry, but no corresponding CPUser yet.
@@ -721,7 +721,7 @@ MachineBasicBlock *ARMConstantIslands::SplitBlockBeforeInstr(MachineInstr *MI) {
   // There doesn't seem to be meaningful DebugInfo available; this doesn't
   // correspond to anything in the source.
   unsigned Opc = isThumb ? (isThumb2 ? ARM::t2B : ARM::tB) : ARM::B;
-  BuildMI(OrigBB, DebugLoc::getUnknownLoc(), TII->get(Opc)).addMBB(NewBB);
+  BuildMI(OrigBB, DebugLoc(), TII->get(Opc)).addMBB(NewBB);
   NumSplit++;
 
   // Update the CFG.  All succs of OrigBB are now succs of NewBB.
@@ -1103,8 +1103,7 @@ void ARMConstantIslands::CreateNewWater(unsigned CPUserIndex,
     // targets will be exchanged, and the altered branch may be out of
     // range, so the machinery has to know about it.
     int UncondBr = isThumb ? ((isThumb2) ? ARM::t2B : ARM::tB) : ARM::B;
-    BuildMI(UserMBB, DebugLoc::getUnknownLoc(),
-            TII->get(UncondBr)).addMBB(NewMBB);
+    BuildMI(UserMBB, DebugLoc(), TII->get(UncondBr)).addMBB(NewMBB);
     unsigned MaxDisp = getUnconditionalBrDisp(UncondBr);
     ImmBranches.push_back(ImmBranch(&UserMBB->back(),
                           MaxDisp, false, UncondBr));
@@ -1244,8 +1243,7 @@ bool ARMConstantIslands::HandleConstantPoolUser(MachineFunction &MF,
   // Now that we have an island to add the CPE to, clone the original CPE and
   // add it to the island.
   U.HighWaterMark = NewIsland;
-  U.CPEMI = BuildMI(NewIsland, DebugLoc::getUnknownLoc(),
-                    TII->get(ARM::CONSTPOOL_ENTRY))
+  U.CPEMI = BuildMI(NewIsland, DebugLoc(), TII->get(ARM::CONSTPOOL_ENTRY))
                 .addImm(ID).addConstantPoolIndex(CPI).addImm(Size);
   CPEntries[CPI].push_back(CPEntry(U.CPEMI, ID, 1));
   NumCPEs++;
@@ -1446,12 +1444,11 @@ ARMConstantIslands::FixUpConditionalBr(MachineFunction &MF, ImmBranch &Br) {
 
   // Insert a new conditional branch and a new unconditional branch.
   // Also update the ImmBranch as well as adding a new entry for the new branch.
-  BuildMI(MBB, DebugLoc::getUnknownLoc(),
-          TII->get(MI->getOpcode()))
+  BuildMI(MBB, DebugLoc(), TII->get(MI->getOpcode()))
     .addMBB(NextBB).addImm(CC).addReg(CCReg);
   Br.MI = &MBB->back();
   BBSizes[MBB->getNumber()] += TII->GetInstSizeInBytes(&MBB->back());
-  BuildMI(MBB, DebugLoc::getUnknownLoc(), TII->get(Br.UncondBr)).addMBB(DestBB);
+  BuildMI(MBB, DebugLoc(), TII->get(Br.UncondBr)).addMBB(DestBB);
   BBSizes[MBB->getNumber()] += TII->GetInstSizeInBytes(&MBB->back());
   unsigned MaxDisp = getUnconditionalBrDisp(Br.UncondBr);
   ImmBranches.push_back(ImmBranch(&MBB->back(), MaxDisp, false, Br.UncondBr));
@@ -1809,7 +1806,7 @@ AdjustJTTargetBlockForward(MachineBasicBlock *BB, MachineBasicBlock *JTBB)
   // There doesn't seem to be meaningful DebugInfo available; this doesn't
   // correspond directly to anything in the source.
   assert (isThumb2 && "Adjusting for TB[BH] but not in Thumb2?");
-  BuildMI(NewBB, DebugLoc::getUnknownLoc(), TII->get(ARM::t2B)).addMBB(BB);
+  BuildMI(NewBB, DebugLoc(), TII->get(ARM::t2B)).addMBB(BB);
 
   // Update internal data structures to account for the newly inserted MBB.
   MF.RenumberBlocks(NewBB);
diff --git a/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 89c776909078..cf55377bc677 100644
--- a/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Target/TargetAsmParser.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 using namespace llvm;
@@ -46,11 +47,11 @@ private:
 
   bool Error(SMLoc L, const Twine &Msg) { return Parser.Error(L, Msg); }
 
-  bool MaybeParseRegister(ARMOperand &Op, bool ParseWriteBack);
+  bool MaybeParseRegister(OwningPtr<ARMOperand> &Op, bool ParseWriteBack);
 
-  bool ParseRegisterList(ARMOperand &Op);
+  bool ParseRegisterList(OwningPtr<ARMOperand> &Op);
 
-  bool ParseMemory(ARMOperand &Op);
+  bool ParseMemory(OwningPtr<ARMOperand> &Op);
 
   bool ParseMemoryOffsetReg(bool &Negative,
                             bool &OffsetRegShifted,
@@ -58,11 +59,12 @@ private:
                             const MCExpr *&ShiftAmount,
                             const MCExpr *&Offset,
                             bool &OffsetIsReg,
-                            int &OffsetRegNum);
+                            int &OffsetRegNum,
+                            SMLoc &E);
 
-  bool ParseShift(enum ShiftType &St, const MCExpr *&ShiftAmount);
+  bool ParseShift(enum ShiftType &St, const MCExpr *&ShiftAmount, SMLoc &E);
 
-  bool ParseOperand(ARMOperand &Op);
+  bool ParseOperand(OwningPtr<ARMOperand> &Op);
 
   bool ParseDirectiveWord(unsigned Size, SMLoc L);
 
@@ -104,13 +106,17 @@ public:
 /// ARMOperand - Instances of this class represent a parsed ARM machine
 /// instruction.
 struct ARMOperand : public MCParsedAsmOperand {
-  enum {
+private:
+  ARMOperand() {}
+public:
+  enum KindTy {
     Token,
     Register,
     Immediate,
     Memory
   } Kind;
 
+  SMLoc StartLoc, EndLoc;
 
   union {
     struct {
@@ -126,7 +132,7 @@ struct ARMOperand : public MCParsedAsmOperand {
     struct {
       const MCExpr *Val;
     } Imm;
-
+    
     // This is for all forms of ARM address expressions
     struct {
       unsigned BaseRegNum;
@@ -144,6 +150,34 @@ struct ARMOperand : public MCParsedAsmOperand {
     } Mem;
 
   };
+  
+  ARMOperand(KindTy K, SMLoc S, SMLoc E)
+    : Kind(K), StartLoc(S), EndLoc(E) {}
+  
+  ARMOperand(const ARMOperand &o) : MCParsedAsmOperand() {
+    Kind = o.Kind;
+    StartLoc = o.StartLoc;
+    EndLoc = o.EndLoc;
+    switch (Kind) {
+    case Token:
+    Tok = o.Tok;
+      break;
+    case Register:
+      Reg = o.Reg;
+      break;
+    case Immediate:
+      Imm = o.Imm;
+      break;
+    case Memory:
+      Mem = o.Mem;
+      break;
+    }
+  }
+  
+  /// getStartLoc - Get the location of the first token of this operand.
+  SMLoc getStartLoc() const { return StartLoc; }
+  /// getEndLoc - Get the location of the last token of this operand.
+  SMLoc getEndLoc() const { return EndLoc; }
 
   StringRef getToken() const {
     assert(Kind == Token && "Invalid access!");
@@ -169,48 +203,60 @@ struct ARMOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::CreateReg(getReg()));
   }
 
-  static ARMOperand CreateToken(StringRef Str) {
-    ARMOperand Res;
-    Res.Kind = Token;
-    Res.Tok.Data = Str.data();
-    Res.Tok.Length = Str.size();
-    return Res;
+  static void CreateToken(OwningPtr<ARMOperand> &Op, StringRef Str,
+                          SMLoc S) {
+    Op.reset(new ARMOperand);
+    Op->Kind = Token;
+    Op->Tok.Data = Str.data();
+    Op->Tok.Length = Str.size();
+    Op->StartLoc = S;
+    Op->EndLoc = S;
   }
 
-  static ARMOperand CreateReg(unsigned RegNum, bool Writeback) {
-    ARMOperand Res;
-    Res.Kind = Register;
-    Res.Reg.RegNum = RegNum;
-    Res.Reg.Writeback = Writeback;
-    return Res;
+  static void CreateReg(OwningPtr<ARMOperand> &Op, unsigned RegNum, 
+                        bool Writeback, SMLoc S, SMLoc E) {
+    Op.reset(new ARMOperand);
+    Op->Kind = Register;
+    Op->Reg.RegNum = RegNum;
+    Op->Reg.Writeback = Writeback;
+    
+    Op->StartLoc = S;
+    Op->EndLoc = E;
   }
 
-  static ARMOperand CreateImm(const MCExpr *Val) {
-    ARMOperand Res;
-    Res.Kind = Immediate;
-    Res.Imm.Val = Val;
-    return Res;
+  static void CreateImm(OwningPtr<ARMOperand> &Op, const MCExpr *Val,
+                        SMLoc S, SMLoc E) {
+    Op.reset(new ARMOperand);
+    Op->Kind = Immediate;
+    Op->Imm.Val = Val;
+    
+    Op->StartLoc = S;
+    Op->EndLoc = E;
   }
 
-  static ARMOperand CreateMem(unsigned BaseRegNum, bool OffsetIsReg,
-                              const MCExpr *Offset, unsigned OffsetRegNum,
-                              bool OffsetRegShifted, enum ShiftType ShiftType,
-                              const MCExpr *ShiftAmount, bool Preindexed,
-                              bool Postindexed, bool Negative, bool Writeback) {
-    ARMOperand Res;
-    Res.Kind = Memory;
-    Res.Mem.BaseRegNum = BaseRegNum;
-    Res.Mem.OffsetIsReg = OffsetIsReg;
-    Res.Mem.Offset = Offset;
-    Res.Mem.OffsetRegNum = OffsetRegNum;
-    Res.Mem.OffsetRegShifted = OffsetRegShifted;
-    Res.Mem.ShiftType = ShiftType;
-    Res.Mem.ShiftAmount = ShiftAmount;
-    Res.Mem.Preindexed = Preindexed;
-    Res.Mem.Postindexed = Postindexed;
-    Res.Mem.Negative = Negative;
-    Res.Mem.Writeback = Writeback;
-    return Res;
+  static void CreateMem(OwningPtr<ARMOperand> &Op,
+                        unsigned BaseRegNum, bool OffsetIsReg,
+                        const MCExpr *Offset, unsigned OffsetRegNum,
+                        bool OffsetRegShifted, enum ShiftType ShiftType,
+                        const MCExpr *ShiftAmount, bool Preindexed,
+                        bool Postindexed, bool Negative, bool Writeback,
+                        SMLoc S, SMLoc E) {
+    Op.reset(new ARMOperand);
+    Op->Kind = Memory;
+    Op->Mem.BaseRegNum = BaseRegNum;
+    Op->Mem.OffsetIsReg = OffsetIsReg;
+    Op->Mem.Offset = Offset;
+    Op->Mem.OffsetRegNum = OffsetRegNum;
+    Op->Mem.OffsetRegShifted = OffsetRegShifted;
+    Op->Mem.ShiftType = ShiftType;
+    Op->Mem.ShiftAmount = ShiftAmount;
+    Op->Mem.Preindexed = Preindexed;
+    Op->Mem.Postindexed = Postindexed;
+    Op->Mem.Negative = Negative;
+    Op->Mem.Writeback = Writeback;
+    
+    Op->StartLoc = S;
+    Op->EndLoc = E;
   }
 };
 
@@ -221,7 +267,9 @@ struct ARMOperand : public MCParsedAsmOperand {
 /// and false is returned.  Else true is returned and no token is eaten.
 /// TODO this is likely to change to allow different register types and or to
 /// parse for a specific register type.
-bool ARMAsmParser::MaybeParseRegister(ARMOperand &Op, bool ParseWriteBack) {
+bool ARMAsmParser::MaybeParseRegister
+  (OwningPtr<ARMOperand> &Op, bool ParseWriteBack) {
+  SMLoc S, E;
   const AsmToken &Tok = Parser.getTok();
   assert(Tok.is(AsmToken::Identifier) && "Token is not an Identifier");
 
@@ -232,27 +280,35 @@ bool ARMAsmParser::MaybeParseRegister(ARMOperand &Op, bool ParseWriteBack) {
   RegNum = MatchRegisterName(Tok.getString());
   if (RegNum == -1)
     return true;
+  
+  S = Tok.getLoc();
+  
   Parser.Lex(); // Eat identifier token.
+    
+  E = Parser.getTok().getLoc();
 
   bool Writeback = false;
   if (ParseWriteBack) {
     const AsmToken &ExclaimTok = Parser.getTok();
     if (ExclaimTok.is(AsmToken::Exclaim)) {
+      E = ExclaimTok.getLoc();
       Writeback = true;
       Parser.Lex(); // Eat exclaim token
     }
   }
 
-  Op = ARMOperand::CreateReg(RegNum, Writeback);
+  ARMOperand::CreateReg(Op, RegNum, Writeback, S, E);
 
   return false;
 }
 
 /// Parse a register list, return false if successful else return true or an 
 /// error.  The first token must be a '{' when called.
-bool ARMAsmParser::ParseRegisterList(ARMOperand &Op) {
+bool ARMAsmParser::ParseRegisterList(OwningPtr<ARMOperand> &Op) {
+  SMLoc S, E;
   assert(Parser.getTok().is(AsmToken::LCurly) &&
          "Token is not an Left Curly Brace");
+  S = Parser.getTok().getLoc();
   Parser.Lex(); // Eat left curly brace token.
 
   const AsmToken &RegTok = Parser.getTok();
@@ -290,6 +346,7 @@ bool ARMAsmParser::ParseRegisterList(ARMOperand &Op) {
   const AsmToken &RCurlyTok = Parser.getTok();
   if (RCurlyTok.isNot(AsmToken::RCurly))
     return Error(RCurlyTok.getLoc(), "'}' expected");
+  E = RCurlyTok.getLoc();
   Parser.Lex(); // Eat left curly brace token.
 
   return false;
@@ -299,9 +356,11 @@ bool ARMAsmParser::ParseRegisterList(ARMOperand &Op) {
 /// or an error.  The first token must be a '[' when called.
 /// TODO Only preindexing and postindexing addressing are started, unindexed
 /// with option, etc are still to do.
-bool ARMAsmParser::ParseMemory(ARMOperand &Op) {
+bool ARMAsmParser::ParseMemory(OwningPtr<ARMOperand> &Op) {
+  SMLoc S, E;
   assert(Parser.getTok().is(AsmToken::LBrac) &&
          "Token is not an Left Bracket");
+  S = Parser.getTok().getLoc();
   Parser.Lex(); // Eat left bracket token.
 
   const AsmToken &BaseRegTok = Parser.getTok();
@@ -309,7 +368,7 @@ bool ARMAsmParser::ParseMemory(ARMOperand &Op) {
     return Error(BaseRegTok.getLoc(), "register expected");
   if (MaybeParseRegister(Op, false))
     return Error(BaseRegTok.getLoc(), "register expected");
-  int BaseRegNum = Op.getReg();
+  int BaseRegNum = Op->getReg();
 
   bool Preindexed = false;
   bool Postindexed = false;
@@ -329,21 +388,23 @@ bool ARMAsmParser::ParseMemory(ARMOperand &Op) {
     const MCExpr *ShiftAmount;
     const MCExpr *Offset;
     if(ParseMemoryOffsetReg(Negative, OffsetRegShifted, ShiftType, ShiftAmount,
-                            Offset, OffsetIsReg, OffsetRegNum))
+                            Offset, OffsetIsReg, OffsetRegNum, E))
       return true;
     const AsmToken &RBracTok = Parser.getTok();
     if (RBracTok.isNot(AsmToken::RBrac))
       return Error(RBracTok.getLoc(), "']' expected");
+    E = RBracTok.getLoc();
     Parser.Lex(); // Eat right bracket token.
 
     const AsmToken &ExclaimTok = Parser.getTok();
     if (ExclaimTok.is(AsmToken::Exclaim)) {
+      E = ExclaimTok.getLoc();
       Writeback = true;
       Parser.Lex(); // Eat exclaim token
     }
-    Op = ARMOperand::CreateMem(BaseRegNum, OffsetIsReg, Offset, OffsetRegNum,
-                               OffsetRegShifted, ShiftType, ShiftAmount,
-                               Preindexed, Postindexed, Negative, Writeback);
+    ARMOperand::CreateMem(Op, BaseRegNum, OffsetIsReg, Offset, OffsetRegNum,
+                          OffsetRegShifted, ShiftType, ShiftAmount,
+                          Preindexed, Postindexed, Negative, Writeback, S, E);
     return false;
   }
   // The "[Rn" we have so far was not followed by a comma.
@@ -352,6 +413,7 @@ bool ARMAsmParser::ParseMemory(ARMOperand &Op) {
     // the "[Rn".
     Postindexed = true;
     Writeback = true;
+    E = Tok.getLoc();
     Parser.Lex(); // Eat right bracket token.
 
     int OffsetRegNum = 0;
@@ -366,13 +428,14 @@ bool ARMAsmParser::ParseMemory(ARMOperand &Op) {
 	return Error(NextTok.getLoc(), "',' expected");
       Parser.Lex(); // Eat comma token.
       if(ParseMemoryOffsetReg(Negative, OffsetRegShifted, ShiftType,
-                              ShiftAmount, Offset, OffsetIsReg, OffsetRegNum))
+                              ShiftAmount, Offset, OffsetIsReg, OffsetRegNum, 
+                              E))
         return true;
     }
 
-    Op = ARMOperand::CreateMem(BaseRegNum, OffsetIsReg, Offset, OffsetRegNum,
-                               OffsetRegShifted, ShiftType, ShiftAmount,
-                               Preindexed, Postindexed, Negative, Writeback);
+    ARMOperand::CreateMem(Op, BaseRegNum, OffsetIsReg, Offset, OffsetRegNum,
+                          OffsetRegShifted, ShiftType, ShiftAmount,
+                          Preindexed, Postindexed, Negative, Writeback, S, E);
     return false;
   }
 
@@ -387,18 +450,20 @@ bool ARMAsmParser::ParseMemory(ARMOperand &Op) {
 ///   #offset
 /// we return false on success or an error otherwise.
 bool ARMAsmParser::ParseMemoryOffsetReg(bool &Negative,
-					bool &OffsetRegShifted,
+                                        bool &OffsetRegShifted,
                                         enum ShiftType &ShiftType,
                                         const MCExpr *&ShiftAmount,
                                         const MCExpr *&Offset,
                                         bool &OffsetIsReg,
-                                        int &OffsetRegNum) {
-  ARMOperand Op;
+                                        int &OffsetRegNum,
+                                        SMLoc &E) {
+  OwningPtr<ARMOperand> Op;
   Negative = false;
   OffsetRegShifted = false;
   OffsetIsReg = false;
   OffsetRegNum = -1;
   const AsmToken &NextTok = Parser.getTok();
+  E = NextTok.getLoc();
   if (NextTok.is(AsmToken::Plus))
     Parser.Lex(); // Eat plus token.
   else if (NextTok.is(AsmToken::Minus)) {
@@ -409,8 +474,10 @@ bool ARMAsmParser::ParseMemoryOffsetReg(bool &Negative,
   const AsmToken &OffsetRegTok = Parser.getTok();
   if (OffsetRegTok.is(AsmToken::Identifier)) {
     OffsetIsReg = !MaybeParseRegister(Op, false);
-    if (OffsetIsReg)
-      OffsetRegNum = Op.getReg();
+    if (OffsetIsReg) {
+      E = Op->getEndLoc();
+      OffsetRegNum = Op->getReg();
+    }
   }
   // If we parsed a register as the offset then their can be a shift after that
   if (OffsetRegNum != -1) {
@@ -420,7 +487,7 @@ bool ARMAsmParser::ParseMemoryOffsetReg(bool &Negative,
       Parser.Lex(); // Eat comma token.
 
       const AsmToken &Tok = Parser.getTok();
-      if (ParseShift(ShiftType, ShiftAmount))
+      if (ParseShift(ShiftType, ShiftAmount, E))
 	return Error(Tok.getLoc(), "shift expected");
       OffsetRegShifted = true;
     }
@@ -430,10 +497,12 @@ bool ARMAsmParser::ParseMemoryOffsetReg(bool &Negative,
     const AsmToken &HashTok = Parser.getTok();
     if (HashTok.isNot(AsmToken::Hash))
       return Error(HashTok.getLoc(), "'#' expected");
+    
     Parser.Lex(); // Eat hash token.
 
     if (getParser().ParseExpression(Offset))
      return true;
+    E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
   }
   return false;
 }
@@ -442,7 +511,9 @@ bool ARMAsmParser::ParseMemoryOffsetReg(bool &Negative,
 ///   ( lsl | lsr | asr | ror ) , # shift_amount
 ///   rrx
 /// and returns true if it parses a shift otherwise it returns false.
-bool ARMAsmParser::ParseShift(ShiftType &St, const MCExpr *&ShiftAmount) {
+bool ARMAsmParser::ParseShift(ShiftType &St, 
+                              const MCExpr *&ShiftAmount, 
+                              SMLoc &E) {
   const AsmToken &Tok = Parser.getTok();
   if (Tok.isNot(AsmToken::Identifier))
     return true;
@@ -550,7 +621,9 @@ MatchInstruction(const SmallVectorImpl<MCParsedAsmOperand*> &Operands,
 
 /// Parse a arm instruction operand.  For now this parses the operand regardless
 /// of the mnemonic.
-bool ARMAsmParser::ParseOperand(ARMOperand &Op) {
+bool ARMAsmParser::ParseOperand(OwningPtr<ARMOperand> &Op) {
+  SMLoc S, E;
+  
   switch (getLexer().getKind()) {
   case AsmToken::Identifier:
     if (!MaybeParseRegister(Op, true))
@@ -558,9 +631,11 @@ bool ARMAsmParser::ParseOperand(ARMOperand &Op) {
     // This was not a register so parse other operands that start with an
     // identifier (like labels) as expressions and create them as immediates.
     const MCExpr *IdVal;
+    S = Parser.getTok().getLoc();
     if (getParser().ParseExpression(IdVal))
       return true;
-    Op = ARMOperand::CreateImm(IdVal);
+    E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
+    ARMOperand::CreateImm(Op, IdVal, S, E);
     return false;
   case AsmToken::LBrac:
     return ParseMemory(Op);
@@ -569,11 +644,13 @@ bool ARMAsmParser::ParseOperand(ARMOperand &Op) {
   case AsmToken::Hash:
     // #42 -> immediate.
     // TODO: ":lower16:" and ":upper16:" modifiers after # before immediate
+    S = Parser.getTok().getLoc();
     Parser.Lex();
     const MCExpr *ImmVal;
     if (getParser().ParseExpression(ImmVal))
       return true;
-    Op = ARMOperand::CreateImm(ImmVal);
+    E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
+    ARMOperand::CreateImm(Op, ImmVal, S, E);
     return false;
   default:
     return Error(Parser.getTok().getLoc(), "unexpected token in operand");
@@ -583,22 +660,25 @@ bool ARMAsmParser::ParseOperand(ARMOperand &Op) {
 /// Parse an arm instruction mnemonic followed by its operands.
 bool ARMAsmParser::ParseInstruction(const StringRef &Name, SMLoc NameLoc,
                                SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
-  Operands.push_back(new ARMOperand(ARMOperand::CreateToken(Name)));
+  OwningPtr<ARMOperand> Op;
+  ARMOperand::CreateToken(Op, Name, NameLoc);
+  
+  Operands.push_back(Op.take());
 
   SMLoc Loc = Parser.getTok().getLoc();
   if (getLexer().isNot(AsmToken::EndOfStatement)) {
 
     // Read the first operand.
-    ARMOperand Op;
+    OwningPtr<ARMOperand> Op;
     if (ParseOperand(Op)) return true;
-    Operands.push_back(new ARMOperand(Op));
+    Operands.push_back(Op.take());
 
     while (getLexer().is(AsmToken::Comma)) {
       Parser.Lex();  // Eat the comma.
 
       // Parse and remember the operand.
       if (ParseOperand(Op)) return true;
-      Operands.push_back(new ARMOperand(Op));
+      Operands.push_back(Op.take());
     }
   }
   return false;
diff --git a/lib/Target/ARM/Disassembler/ARMDisassembler.cpp b/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
new file mode 100644
index 000000000000..04313400b8d9
--- /dev/null
+++ b/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
@@ -0,0 +1,532 @@
+//===- ARMDisassembler.cpp - Disassembler for ARM/Thumb ISA -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is part of the ARM Disassembler.
+// It contains code to implement the public interfaces of ARMDisassembler and
+// ThumbDisassembler, both of which are instances of MCDisassembler.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "arm-disassembler"
+
+#include "ARMDisassembler.h"
+#include "ARMDisassemblerCore.h"
+
+#include "llvm/MC/MCInst.h"
+#include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/MemoryObject.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+
+/// ARMGenDecoderTables.inc - ARMDecoderTables.inc is tblgen'ed from
+/// ARMDecoderEmitter.cpp TableGen backend.  It contains:
+///
+/// o Mappings from opcode to ARM/Thumb instruction format
+///
+/// o static uint16_t decodeInstruction(uint32_t insn) - the decoding function
+/// for an ARM instruction.
+///
+/// o static uint16_t decodeThumbInstruction(field_t insn) - the decoding
+/// function for a Thumb instruction.
+///
+#include "../ARMGenDecoderTables.inc"
+
+namespace llvm {
+
+/// showBitVector - Use the raw_ostream to log a diagnostic message describing
+/// the inidividual bits of the instruction.
+///
+static inline void showBitVector(raw_ostream &os, const uint32_t &insn) {
+  // Split the bit position markers into more than one lines to fit 80 columns.
+  os << " 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11"
+     << " 10  9  8  7  6  5  4  3  2  1  0 \n";
+  os << "---------------------------------------------------------------"
+     << "----------------------------------\n";
+  os << '|';
+  for (unsigned i = 32; i != 0; --i) {
+    if (insn >> (i - 1) & 0x01)
+      os << " 1";
+    else
+      os << " 0";
+    os << (i%4 == 1 ? '|' : ':');
+  }
+  os << '\n';
+  // Split the bit position markers into more than one lines to fit 80 columns.
+  os << "---------------------------------------------------------------"
+     << "----------------------------------\n";
+  os << '\n';
+}
+
+/// decodeARMInstruction is a decorator function which tries special cases of
+/// instruction matching before calling the auto-generated decoder function.
+static unsigned decodeARMInstruction(uint32_t &insn) {
+  if (slice(insn, 31, 28) == 15)
+    goto AutoGenedDecoder;
+
+  // Special case processing, if any, goes here....
+
+  // LLVM combines the offset mode of A8.6.197 & A8.6.198 into STRB.
+  // The insufficient encoding information of the combined instruction confuses
+  // the decoder wrt BFC/BFI.  Therefore, we try to recover here.
+  // For BFC, Inst{27-21} = 0b0111110 & Inst{6-0} = 0b0011111.
+  // For BFI, Inst{27-21} = 0b0111110 & Inst{6-4} = 0b001 & Inst{3-0} =! 0b1111.
+  if (slice(insn, 27, 21) == 0x3e && slice(insn, 6, 4) == 1) {
+    if (slice(insn, 3, 0) == 15)
+      return ARM::BFC;
+    else
+      return ARM::BFI;
+  }
+
+  // Ditto for ADDSrs, which is a super-instruction for A8.6.7 & A8.6.8.
+  // As a result, the decoder fails to decode UMULL properly.
+  if (slice(insn, 27, 21) == 0x04 && slice(insn, 7, 4) == 9) {
+    return ARM::UMULL;
+  }
+
+  // Ditto for STR_PRE, which is a super-instruction for A8.6.194 & A8.6.195.
+  // As a result, the decoder fails to decode SBFX properly.
+  if (slice(insn, 27, 21) == 0x3d && slice(insn, 6, 4) == 5)
+    return ARM::SBFX;
+
+  // And STRB_PRE, which is a super-instruction for A8.6.197 & A8.6.198.
+  // As a result, the decoder fails to decode UBFX properly.
+  if (slice(insn, 27, 21) == 0x3f && slice(insn, 6, 4) == 5)
+    return ARM::UBFX;
+
+  // Ditto for STRT, which is a super-instruction for A8.6.210 Encoding A1 & A2.
+  // As a result, the decoder fails to deocode SSAT properly.
+  if (slice(insn, 27, 21) == 0x35 && slice(insn, 5, 4) == 1)
+    return slice(insn, 6, 6) == 0 ? ARM::SSATlsl : ARM::SSATasr;
+
+  // Ditto for RSCrs, which is a super-instruction for A8.6.146 & A8.6.147.
+  // As a result, the decoder fails to decode STRHT/LDRHT/LDRSHT/LDRSBT.
+  if (slice(insn, 27, 24) == 0) {
+    switch (slice(insn, 21, 20)) {
+    case 2:
+      switch (slice(insn, 7, 4)) {
+      case 11:
+        return ARM::STRHT;
+      default:
+        break; // fallthrough
+      }
+      break;
+    case 3:
+      switch (slice(insn, 7, 4)) {
+      case 11:
+        return ARM::LDRHT;
+      case 13:
+        return ARM::LDRSBT;
+      case 15:
+        return ARM::LDRSHT;
+      default:
+        break; // fallthrough
+      }
+      break;
+    default:
+      break;   // fallthrough
+    }
+  }
+
+  // Ditto for SBCrs, which is a super-instruction for A8.6.152 & A8.6.153.
+  // As a result, the decoder fails to decode STRH_Post/LDRD_POST/STRD_POST
+  // properly.
+  if (slice(insn, 27, 25) == 0 && slice(insn, 20, 20) == 0) {
+    unsigned PW = slice(insn, 24, 24) << 1 | slice(insn, 21, 21);
+    switch (slice(insn, 7, 4)) {
+    case 11:
+      switch (PW) {
+      case 2: // Offset
+        return ARM::STRH;
+      case 3: // Pre-indexed
+        return ARM::STRH_PRE;
+      case 0: // Post-indexed
+        return ARM::STRH_POST;
+      default:
+        break; // fallthrough
+      }
+      break;
+    case 13:
+      switch (PW) {
+      case 2: // Offset
+        return ARM::LDRD;
+      case 3: // Pre-indexed
+        return ARM::LDRD_PRE;
+      case 0: // Post-indexed
+        return ARM::LDRD_POST;
+      default:
+        break; // fallthrough
+      }
+      break;
+    case 15:
+      switch (PW) {
+      case 2: // Offset
+        return ARM::STRD;
+      case 3: // Pre-indexed
+        return ARM::STRD_PRE;
+      case 0: // Post-indexed
+        return ARM::STRD_POST;
+      default:
+        break; // fallthrough
+      }
+      break;
+    default:
+      break; // fallthrough
+    }
+  }
+
+  // Ditto for SBCSSrs, which is a super-instruction for A8.6.152 & A8.6.153.
+  // As a result, the decoder fails to decode LDRH_POST/LDRSB_POST/LDRSH_POST
+  // properly.
+  if (slice(insn, 27, 25) == 0 && slice(insn, 20, 20) == 1) {
+    unsigned PW = slice(insn, 24, 24) << 1 | slice(insn, 21, 21);
+    switch (slice(insn, 7, 4)) {
+    case 11:
+      switch (PW) {
+      case 2: // Offset
+        return ARM::LDRH;
+      case 3: // Pre-indexed
+        return ARM::LDRH_PRE;
+      case 0: // Post-indexed
+        return ARM::LDRH_POST;
+      default:
+        break; // fallthrough
+      }
+      break;
+    case 13:
+      switch (PW) {
+      case 2: // Offset
+        return ARM::LDRSB;
+      case 3: // Pre-indexed
+        return ARM::LDRSB_PRE;
+      case 0: // Post-indexed
+        return ARM::LDRSB_POST;
+      default:
+        break; // fallthrough
+      }
+      break;
+    case 15:
+      switch (PW) {
+      case 2: // Offset
+        return ARM::LDRSH;
+      case 3: // Pre-indexed
+        return ARM::LDRSH_PRE;
+      case 0: // Post-indexed
+        return ARM::LDRSH_POST;
+      default:
+        break; // fallthrough
+      }
+      break;
+    default:
+      break; // fallthrough
+    }
+  }
+
+AutoGenedDecoder:
+  // Calling the auto-generated decoder function.
+  return decodeInstruction(insn);
+}
+
+// Helper function for special case handling of LDR (literal) and friends.
+// See, for example, A6.3.7 Load word: Table A6-18 Load word.
+// See A8.6.57 T3, T4 & A8.6.60 T2 and friends for why we morphed the opcode
+// before returning it.
+static unsigned T2Morph2LoadLiteral(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    return Opcode; // Return unmorphed opcode.
+
+  case ARM::t2LDRDi8:
+    return ARM::t2LDRDpci;
+
+  case ARM::t2LDR_POST:   case ARM::t2LDR_PRE:
+  case ARM::t2LDRi12:     case ARM::t2LDRi8:
+  case ARM::t2LDRs:
+    return ARM::t2LDRpci;
+
+  case ARM::t2LDRB_POST:  case ARM::t2LDRB_PRE:
+  case ARM::t2LDRBi12:    case ARM::t2LDRBi8:
+  case ARM::t2LDRBs:
+    return ARM::t2LDRBpci;
+
+  case ARM::t2LDRH_POST:  case ARM::t2LDRH_PRE:
+  case ARM::t2LDRHi12:    case ARM::t2LDRHi8:
+  case ARM::t2LDRHs:
+    return ARM::t2LDRHpci;
+
+  case ARM::t2LDRSB_POST:  case ARM::t2LDRSB_PRE:
+  case ARM::t2LDRSBi12:    case ARM::t2LDRSBi8:
+  case ARM::t2LDRSBs:
+    return ARM::t2LDRSBpci;
+
+  case ARM::t2LDRSH_POST:  case ARM::t2LDRSH_PRE:
+  case ARM::t2LDRSHi12:    case ARM::t2LDRSHi8:
+  case ARM::t2LDRSHs:
+    return ARM::t2LDRSHpci;
+  }
+}
+
+/// decodeThumbSideEffect is a decorator function which can potentially twiddle
+/// the instruction or morph the returned opcode under Thumb2.
+///
+/// First it checks whether the insn is a NEON or VFP instr; if true, bit
+/// twiddling could be performed on insn to turn it into an ARM NEON/VFP
+/// equivalent instruction and decodeInstruction is called with the transformed
+/// insn.
+///
+/// Next, there is special handling for Load byte/halfword/word instruction by
+/// checking whether Rn=0b1111 and call T2Morph2LoadLiteral() on the decoded
+/// Thumb2 instruction.  See comments below for further details.
+///
+/// Finally, one last check is made to see whether the insn is a NEON/VFP and
+/// decodeInstruction(insn) is invoked on the original insn.
+///
+/// Otherwise, decodeThumbInstruction is called with the original insn.
+static unsigned decodeThumbSideEffect(bool IsThumb2, uint32_t &insn) {
+  if (IsThumb2) {
+    uint16_t op1 = slice(insn, 28, 27);
+    uint16_t op2 = slice(insn, 26, 20);
+
+    // A6.3 32-bit Thumb instruction encoding
+    // Table A6-9 32-bit Thumb instruction encoding
+
+    // The coprocessor instructions of interest are transformed to their ARM
+    // equivalents.
+
+    // --------- Transform Begin Marker ---------
+    if ((op1 == 1 || op1 == 3) && slice(op2, 6, 4) == 7) {
+      // A7.4 Advanced SIMD data-processing instructions
+      // U bit of Thumb corresponds to Inst{24} of ARM.
+      uint16_t U = slice(op1, 1, 1);
+
+      // Inst{28-24} of ARM = {1,0,0,1,U};
+      uint16_t bits28_24 = 9 << 1 | U;
+      DEBUG(showBitVector(errs(), insn));
+      setSlice(insn, 28, 24, bits28_24);
+      return decodeInstruction(insn);
+    }
+
+    if (op1 == 3 && slice(op2, 6, 4) == 1 && slice(op2, 0, 0) == 0) {
+      // A7.7 Advanced SIMD element or structure load/store instructions
+      // Inst{27-24} of Thumb = 0b1001
+      // Inst{27-24} of ARM   = 0b0100
+      DEBUG(showBitVector(errs(), insn));
+      setSlice(insn, 27, 24, 4);
+      return decodeInstruction(insn);
+    }
+    // --------- Transform End Marker ---------
+
+    // See, for example, A6.3.7 Load word: Table A6-18 Load word.
+    // See A8.6.57 T3, T4 & A8.6.60 T2 and friends for why we morphed the opcode
+    // before returning it to our caller.
+    if (op1 == 3 && slice(op2, 6, 5) == 0 && slice(op2, 0, 0) == 1
+        && slice(insn, 19, 16) == 15)
+      return T2Morph2LoadLiteral(decodeThumbInstruction(insn));
+
+    // One last check for NEON/VFP instructions.
+    if ((op1 == 1 || op1 == 3) && slice(op2, 6, 6) == 1)
+      return decodeInstruction(insn);
+
+    // Fall through.
+  }
+
+  return decodeThumbInstruction(insn);
+}
+
+static inline bool Thumb2PreloadOpcodeNoPCI(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    return false;
+  case ARM::t2PLDi12:   case ARM::t2PLDi8:
+  case ARM::t2PLDr:     case ARM::t2PLDs:
+  case ARM::t2PLDWi12:  case ARM::t2PLDWi8:
+  case ARM::t2PLDWr:    case ARM::t2PLDWs:
+  case ARM::t2PLIi12:   case ARM::t2PLIi8:
+  case ARM::t2PLIr:     case ARM::t2PLIs:
+    return true;
+  }
+}
+
+static inline unsigned T2Morph2Preload2PCI(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    return 0;
+  case ARM::t2PLDi12:   case ARM::t2PLDi8:
+  case ARM::t2PLDr:     case ARM::t2PLDs:
+    return ARM::t2PLDpci;
+  case ARM::t2PLDWi12:  case ARM::t2PLDWi8:
+  case ARM::t2PLDWr:    case ARM::t2PLDWs:
+    return ARM::t2PLDWpci;
+  case ARM::t2PLIi12:   case ARM::t2PLIi8:
+  case ARM::t2PLIr:     case ARM::t2PLIs:
+    return ARM::t2PLIpci;
+  }
+}
+
+//
+// Public interface for the disassembler
+//
+
+bool ARMDisassembler::getInstruction(MCInst &MI,
+                                     uint64_t &Size,
+                                     const MemoryObject &Region,
+                                     uint64_t Address,
+                                     raw_ostream &os) const {
+  // The machine instruction.
+  uint32_t insn;
+
+  // We want to read exactly 4 bytes of data.
+  if (Region.readBytes(Address, 4, (uint8_t*)&insn, NULL) == -1)
+    return false;
+
+  unsigned Opcode = decodeARMInstruction(insn);
+  ARMFormat Format = ARMFormats[Opcode];
+  Size = 4;
+
+  DEBUG({
+      errs() << "Opcode=" << Opcode << " Name=" << ARMUtils::OpcodeName(Opcode)
+             << " Format=" << stringForARMFormat(Format) << '(' << (int)Format
+             << ")\n";
+      showBitVector(errs(), insn);
+    });
+
+  ARMBasicMCBuilder *Builder = CreateMCBuilder(Opcode, Format);
+
+  if (!Builder)
+    return false;
+
+  if (!Builder->Build(MI, insn))
+    return false;
+
+  delete Builder;
+
+  return true;
+}
+
+bool ThumbDisassembler::getInstruction(MCInst &MI,
+                                       uint64_t &Size,
+                                       const MemoryObject &Region,
+                                       uint64_t Address,
+                                       raw_ostream &os) const {
+  // The machine instruction.
+  uint32_t insn = 0;
+  uint32_t insn1 = 0;
+
+  // A6.1 Thumb instruction set encoding
+  //
+  // If bits [15:11] of the halfword being decoded take any of the following
+  // values, the halfword is the first halfword of a 32-bit instruction:
+  // o 0b11101
+  // o 0b11110
+  // o 0b11111.
+  //
+  // Otherwise, the halfword is a 16-bit instruction.
+
+  // Read 2 bytes of data first.
+  if (Region.readBytes(Address, 2, (uint8_t*)&insn, NULL) == -1)
+    return false;
+
+  unsigned bits15_11 = slice(insn, 15, 11);
+  bool IsThumb2 = false;
+
+  // 32-bit instructions if the bits [15:11] of the halfword matches
+  // { 0b11101 /* 0x1D */, 0b11110 /* 0x1E */, ob11111 /* 0x1F */ }.
+  if (bits15_11 == 0x1D || bits15_11 == 0x1E || bits15_11 == 0x1F) {
+    IsThumb2 = true;
+    if (Region.readBytes(Address + 2, 2, (uint8_t*)&insn1, NULL) == -1)
+      return false;
+    insn = (insn << 16 | insn1);
+  }
+
+  // The insn could potentially be bit-twiddled in order to be decoded as an ARM
+  // NEON/VFP opcode.  In such case, the modified insn is later disassembled as
+  // an ARM NEON/VFP instruction.
+  //
+  // This is a short term solution for lack of encoding bits specified for the
+  // Thumb2 NEON/VFP instructions.  The long term solution could be adding some
+  // infrastructure to have each instruction support more than one encodings.
+  // Which encoding is used would be based on which subtarget the compiler/
+  // disassembler is working with at the time.  This would allow the sharing of
+  // the NEON patterns between ARM and Thumb2, as well as potential greater
+  // sharing between the regular ARM instructions and the 32-bit wide Thumb2
+  // instructions as well.
+  unsigned Opcode = decodeThumbSideEffect(IsThumb2, insn);
+
+  // A8.6.117/119/120/121.
+  // PLD/PLDW/PLI instructions with Rn==15 is transformed to the pci variant.
+  if (Thumb2PreloadOpcodeNoPCI(Opcode) && slice(insn, 19, 16) == 15)
+    Opcode = T2Morph2Preload2PCI(Opcode);
+
+  ARMFormat Format = ARMFormats[Opcode];
+  Size = IsThumb2 ? 4 : 2;
+
+  DEBUG({
+      errs() << "Opcode=" << Opcode << " Name=" << ARMUtils::OpcodeName(Opcode)
+             << " Format=" << stringForARMFormat(Format) << '(' << (int)Format
+             << ")\n";
+      showBitVector(errs(), insn);
+    });
+
+  ARMBasicMCBuilder *Builder = CreateMCBuilder(Opcode, Format);
+  Builder->setSession(const_cast<Session *>(&SO));
+
+  if (!Builder)
+    return false;
+
+  if (!Builder->Build(MI, insn))
+    return false;
+
+  delete Builder;
+
+  return true;
+}
+
+// A8.6.50
+static unsigned short CountITSize(unsigned ITMask) {
+  // First count the trailing zeros of the IT mask.
+  unsigned TZ = CountTrailingZeros_32(ITMask);
+  assert(TZ <= 3 && "Encoding error");
+  return (4 - TZ);
+}
+
+/// Init ITState.
+void Session::InitIT(unsigned short bits7_0) {
+  ITCounter = CountITSize(slice(bits7_0, 3, 0));
+  ITState = bits7_0;
+}
+
+/// Update ITState if necessary.
+void Session::UpdateIT() {
+  assert(ITCounter);
+  --ITCounter;
+  if (ITCounter == 0)
+    ITState = 0;
+  else {
+    unsigned short NewITState4_0 = slice(ITState, 4, 0) << 1;
+    setSlice(ITState, 4, 0, NewITState4_0);
+  }
+}
+
+static MCDisassembler *createARMDisassembler(const Target &T) {
+  return new ARMDisassembler;
+}
+
+static MCDisassembler *createThumbDisassembler(const Target &T) {
+  return new ThumbDisassembler;
+}
+
+extern "C" void LLVMInitializeARMDisassembler() { 
+  // Register the disassembler.
+  TargetRegistry::RegisterMCDisassembler(TheARMTarget, 
+                                         createARMDisassembler);
+  TargetRegistry::RegisterMCDisassembler(TheThumbTarget,
+                                         createThumbDisassembler);
+}
+
+} // namespace llvm
diff --git a/lib/Target/ARM/Disassembler/ARMDisassembler.h b/lib/Target/ARM/Disassembler/ARMDisassembler.h
new file mode 100644
index 000000000000..44592e0f1567
--- /dev/null
+++ b/lib/Target/ARM/Disassembler/ARMDisassembler.h
@@ -0,0 +1,91 @@
+//===- ARMDisassembler.h - Disassembler for ARM/Thumb ISA -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is part of the ARM Disassembler.
+// It contains the header for ARMDisassembler and ThumbDisassembler, both are
+// subclasses of MCDisassembler.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ARMDISASSEMBLER_H
+#define ARMDISASSEMBLER_H
+
+#include "llvm/MC/MCDisassembler.h"
+
+namespace llvm {
+  
+class MCInst;
+class MemoryObject;
+class raw_ostream;
+  
+/// ARMDisassembler - ARM disassembler for all ARM platforms.
+class ARMDisassembler : public MCDisassembler {
+public:
+  /// Constructor     - Initializes the disassembler.
+  ///
+  ARMDisassembler() :
+    MCDisassembler() {
+  }
+
+  ~ARMDisassembler() {
+  }
+
+  /// getInstruction - See MCDisassembler.
+  bool getInstruction(MCInst &instr,
+                      uint64_t &size,
+                      const MemoryObject &region,
+                      uint64_t address,
+                      raw_ostream &vStream) const;
+private:
+};
+
+// Forward declaration.
+class ARMBasicMCBuilder;
+
+/// Session - Keep track of the IT Block progression.
+class Session {
+  friend class ARMBasicMCBuilder;
+public:
+  Session() : ITCounter(0), ITState(0) {}
+  ~Session() {}
+  /// InitIT - Initializes ITCounter/ITState.
+  void InitIT(unsigned short bits7_0);
+  /// UpdateIT - Updates ITCounter/ITState as IT Block progresses.
+  void UpdateIT();
+
+private:
+  unsigned ITCounter; // Possible values: 0, 1, 2, 3, 4.
+  unsigned ITState;   // A2.5.2 Consists of IT[7:5] and IT[4:0] initially.
+};
+
+/// ThumbDisassembler - Thumb disassembler for all ARM platforms.
+class ThumbDisassembler : public MCDisassembler {
+public:
+  /// Constructor     - Initializes the disassembler.
+  ///
+  ThumbDisassembler() :
+    MCDisassembler(), SO() {
+  }
+
+  ~ThumbDisassembler() {
+  }
+
+  /// getInstruction - See MCDisassembler.
+  bool getInstruction(MCInst &instr,
+                      uint64_t &size,
+                      const MemoryObject &region,
+                      uint64_t address,
+                      raw_ostream &vStream) const;
+private:
+  Session SO;
+};
+
+} // namespace llvm
+  
+#endif
diff --git a/lib/Target/ARM/Disassembler/ARMDisassemblerCore.cpp b/lib/Target/ARM/Disassembler/ARMDisassemblerCore.cpp
new file mode 100644
index 000000000000..db921ef0b628
--- /dev/null
+++ b/lib/Target/ARM/Disassembler/ARMDisassemblerCore.cpp
@@ -0,0 +1,3252 @@
+//===- ARMDisassemblerCore.cpp - ARM disassembler helpers -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is part of the ARM Disassembler.
+// It contains code to represent the core concepts of Builder and DisassembleFP
+// to solve the problem of disassembling an ARM instr.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ARMDisassemblerCore.h"
+#include "ARMAddressingModes.h"
+
+/// ARMGenInstrInfo.inc - ARMGenInstrInfo.inc contains the static const
+/// TargetInstrDesc ARMInsts[] definition and the TargetOperandInfo[]'s
+/// describing the operand info for each ARMInsts[i].
+///
+/// Together with an instruction's encoding format, we can take advantage of the
+/// NumOperands and the OpInfo fields of the target instruction description in
+/// the quest to build out the MCOperand list for an MCInst.
+///
+/// The general guideline is that with a known format, the number of dst and src
+/// operands are well-known.  The dst is built first, followed by the src
+/// operand(s).  The operands not yet used at this point are for the Implicit
+/// Uses and Defs by this instr.  For the Uses part, the pred:$p operand is
+/// defined with two components:
+///
+/// def pred {	// Operand PredicateOperand
+///   ValueType Type = OtherVT;
+///   string PrintMethod = "printPredicateOperand";
+///   string AsmOperandLowerMethod = ?;
+///   dag MIOperandInfo = (ops i32imm, CCR);
+///   AsmOperandClass ParserMatchClass = ImmAsmOperand;
+///   dag DefaultOps = (ops (i32 14), (i32 zero_reg));
+/// }
+///
+/// which is manifested by the TargetOperandInfo[] of:
+///
+/// { 0, 0|(1<<TOI::Predicate), 0 },
+/// { ARM::CCRRegClassID, 0|(1<<TOI::Predicate), 0 }
+///
+/// So the first predicate MCOperand corresponds to the immediate part of the
+/// ARM condition field (Inst{31-28}), and the second predicate MCOperand
+/// corresponds to a register kind of ARM::CPSR.
+///
+/// For the Defs part, in the simple case of only cc_out:$s, we have:
+///
+/// def cc_out {	// Operand OptionalDefOperand
+///   ValueType Type = OtherVT;
+///   string PrintMethod = "printSBitModifierOperand";
+///   string AsmOperandLowerMethod = ?;
+///   dag MIOperandInfo = (ops CCR);
+///   AsmOperandClass ParserMatchClass = ImmAsmOperand;
+///   dag DefaultOps = (ops (i32 zero_reg));
+/// }
+///
+/// which is manifested by the one TargetOperandInfo of:
+///
+/// { ARM::CCRRegClassID, 0|(1<<TOI::OptionalDef), 0 }
+///
+/// And this maps to one MCOperand with the regsiter kind of ARM::CPSR.
+#include "ARMGenInstrInfo.inc"
+
+using namespace llvm;
+
+const char *ARMUtils::OpcodeName(unsigned Opcode) {
+  return ARMInsts[Opcode].Name;
+}
+
+// Return the register enum Based on RegClass and the raw register number.
+// For DRegPair, see comments below.
+// FIXME: Auto-gened?
+static unsigned getRegisterEnum(unsigned RegClassID, unsigned RawRegister,
+                                bool DRegPair = false) {
+
+  if (DRegPair && RegClassID == ARM::QPRRegClassID) {
+    // LLVM expects { Dd, Dd+1 } to form a super register; this is not specified
+    // in the ARM Architecture Manual as far as I understand it (A8.6.307).
+    // Therefore, we morph the RegClassID to be the sub register class and don't
+    // subsequently transform the RawRegister encoding when calculating RegNum.
+    //
+    // See also ARMinstPrinter::printOperand() wrt "dregpair" modifier part
+    // where this workaround is meant for.
+    RegClassID = ARM::DPRRegClassID;
+  }
+
+  // See also decodeNEONRd(), decodeNEONRn(), decodeNEONRm().
+  unsigned RegNum =
+    RegClassID == ARM::QPRRegClassID ? RawRegister >> 1 : RawRegister;
+
+  switch (RegNum) {
+  default:
+    break;
+  case 0:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R0;
+    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
+    case ARM::DPR_VFP2RegClassID:
+      return ARM::D0;
+    case ARM::QPRRegClassID: case ARM::QPR_8RegClassID:
+    case ARM::QPR_VFP2RegClassID:
+      return ARM::Q0;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S0;
+    }
+    break;
+  case 1:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R1;
+    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
+    case ARM::DPR_VFP2RegClassID:
+      return ARM::D1;
+    case ARM::QPRRegClassID: case ARM::QPR_8RegClassID:
+    case ARM::QPR_VFP2RegClassID:
+      return ARM::Q1;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S1;
+    }
+    break;
+  case 2:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R2;
+    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
+    case ARM::DPR_VFP2RegClassID:
+      return ARM::D2;
+    case ARM::QPRRegClassID: case ARM::QPR_8RegClassID:
+    case ARM::QPR_VFP2RegClassID:
+      return ARM::Q2;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S2;
+    }
+    break;
+  case 3:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R3;
+    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
+    case ARM::DPR_VFP2RegClassID:
+      return ARM::D3;
+    case ARM::QPRRegClassID: case ARM::QPR_8RegClassID:
+    case ARM::QPR_VFP2RegClassID:
+      return ARM::Q3;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S3;
+    }
+    break;
+  case 4:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R4;
+    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
+    case ARM::DPR_VFP2RegClassID:
+      return ARM::D4;
+    case ARM::QPRRegClassID: case ARM::QPR_VFP2RegClassID: return ARM::Q4;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S4;
+    }
+    break;
+  case 5:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R5;
+    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
+    case ARM::DPR_VFP2RegClassID:
+      return ARM::D5;
+    case ARM::QPRRegClassID: case ARM::QPR_VFP2RegClassID: return ARM::Q5;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S5;
+    }
+    break;
+  case 6:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R6;
+    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
+    case ARM::DPR_VFP2RegClassID:
+      return ARM::D6;
+    case ARM::QPRRegClassID: case ARM::QPR_VFP2RegClassID: return ARM::Q6;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S6;
+    }
+    break;
+  case 7:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: case ARM::tGPRRegClassID: return ARM::R7;
+    case ARM::DPRRegClassID: case ARM::DPR_8RegClassID:
+    case ARM::DPR_VFP2RegClassID:
+      return ARM::D7;
+    case ARM::QPRRegClassID: case ARM::QPR_VFP2RegClassID: return ARM::Q7;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S7;
+    }
+    break;
+  case 8:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: return ARM::R8;
+    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D8;
+    case ARM::QPRRegClassID: return ARM::Q8;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S8;
+    }
+    break;
+  case 9:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: return ARM::R9;
+    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D9;
+    case ARM::QPRRegClassID: return ARM::Q9;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S9;
+    }
+    break;
+  case 10:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: return ARM::R10;
+    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D10;
+    case ARM::QPRRegClassID: return ARM::Q10;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S10;
+    }
+    break;
+  case 11:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: return ARM::R11;
+    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D11;
+    case ARM::QPRRegClassID: return ARM::Q11;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S11;
+    }
+    break;
+  case 12:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: return ARM::R12;
+    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D12;
+    case ARM::QPRRegClassID: return ARM::Q12;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S12;
+    }
+    break;
+  case 13:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: return ARM::SP;
+    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D13;
+    case ARM::QPRRegClassID: return ARM::Q13;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S13;
+    }
+    break;
+  case 14:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: return ARM::LR;
+    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D14;
+    case ARM::QPRRegClassID: return ARM::Q14;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S14;
+    }
+    break;
+  case 15:
+    switch (RegClassID) {
+    case ARM::GPRRegClassID: return ARM::PC;
+    case ARM::DPRRegClassID: case ARM::DPR_VFP2RegClassID: return ARM::D15;
+    case ARM::QPRRegClassID: return ARM::Q15;
+    case ARM::SPRRegClassID: case ARM::SPR_8RegClassID: return ARM::S15;
+    }
+    break;
+  case 16:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D16;
+    case ARM::SPRRegClassID: return ARM::S16;
+    }
+    break;
+  case 17:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D17;
+    case ARM::SPRRegClassID: return ARM::S17;
+    }
+    break;
+  case 18:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D18;
+    case ARM::SPRRegClassID: return ARM::S18;
+    }
+    break;
+  case 19:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D19;
+    case ARM::SPRRegClassID: return ARM::S19;
+    }
+    break;
+  case 20:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D20;
+    case ARM::SPRRegClassID: return ARM::S20;
+    }
+    break;
+  case 21:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D21;
+    case ARM::SPRRegClassID: return ARM::S21;
+    }
+    break;
+  case 22:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D22;
+    case ARM::SPRRegClassID: return ARM::S22;
+    }
+    break;
+  case 23:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D23;
+    case ARM::SPRRegClassID: return ARM::S23;
+    }
+    break;
+  case 24:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D24;
+    case ARM::SPRRegClassID: return ARM::S24;
+    }
+    break;
+  case 25:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D25;
+    case ARM::SPRRegClassID: return ARM::S25;
+    }
+    break;
+  case 26:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D26;
+    case ARM::SPRRegClassID: return ARM::S26;
+    }
+    break;
+  case 27:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D27;
+    case ARM::SPRRegClassID: return ARM::S27;
+    }
+    break;
+  case 28:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D28;
+    case ARM::SPRRegClassID: return ARM::S28;
+    }
+    break;
+  case 29:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D29;
+    case ARM::SPRRegClassID: return ARM::S29;
+    }
+    break;
+  case 30:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D30;
+    case ARM::SPRRegClassID: return ARM::S30;
+    }
+    break;
+  case 31:
+    switch (RegClassID) {
+    case ARM::DPRRegClassID: return ARM::D31;
+    case ARM::SPRRegClassID: return ARM::S31;
+    }
+    break;
+  }
+  assert(0 && "Invalid (RegClassID, RawRegister) combination");
+  return 0;
+}
+
+///////////////////////////////
+//                           //
+//     Utility Functions     //
+//                           //
+///////////////////////////////
+
+// Extract/Decode Rd: Inst{15-12}.
+static inline unsigned decodeRd(uint32_t insn) {
+  return (insn >> ARMII::RegRdShift) & ARMII::GPRRegMask;
+}
+
+// Extract/Decode Rn: Inst{19-16}.
+static inline unsigned decodeRn(uint32_t insn) {
+  return (insn >> ARMII::RegRnShift) & ARMII::GPRRegMask;
+}
+
+// Extract/Decode Rm: Inst{3-0}.
+static inline unsigned decodeRm(uint32_t insn) {
+  return (insn & ARMII::GPRRegMask);
+}
+
+// Extract/Decode Rs: Inst{11-8}.
+static inline unsigned decodeRs(uint32_t insn) {
+  return (insn >> ARMII::RegRsShift) & ARMII::GPRRegMask;
+}
+
+static inline unsigned getCondField(uint32_t insn) {
+  return (insn >> ARMII::CondShift);
+}
+
+static inline unsigned getIBit(uint32_t insn) {
+  return (insn >> ARMII::I_BitShift) & 1;
+}
+
+static inline unsigned getAM3IBit(uint32_t insn) {
+  return (insn >> ARMII::AM3_I_BitShift) & 1;
+}
+
+static inline unsigned getPBit(uint32_t insn) {
+  return (insn >> ARMII::P_BitShift) & 1;
+}
+
+static inline unsigned getUBit(uint32_t insn) {
+  return (insn >> ARMII::U_BitShift) & 1;
+}
+
+static inline unsigned getPUBits(uint32_t insn) {
+  return (insn >> ARMII::U_BitShift) & 3;
+}
+
+static inline unsigned getSBit(uint32_t insn) {
+  return (insn >> ARMII::S_BitShift) & 1;
+}
+
+static inline unsigned getWBit(uint32_t insn) {
+  return (insn >> ARMII::W_BitShift) & 1;
+}
+
+static inline unsigned getDBit(uint32_t insn) {
+  return (insn >> ARMII::D_BitShift) & 1;
+}
+
+static inline unsigned getNBit(uint32_t insn) {
+  return (insn >> ARMII::N_BitShift) & 1;
+}
+
+static inline unsigned getMBit(uint32_t insn) {
+  return (insn >> ARMII::M_BitShift) & 1;
+}
+
+// See A8.4 Shifts applied to a register.
+//     A8.4.2 Register controlled shifts.
+//
+// getShiftOpcForBits - getShiftOpcForBits translates from the ARM encoding bits
+// into llvm enums for shift opcode.  The API clients should pass in the value
+// encoded with two bits, so the assert stays to signal a wrong API usage.
+//
+// A8-12: DecodeRegShift()
+static inline ARM_AM::ShiftOpc getShiftOpcForBits(unsigned bits) {
+  switch (bits) {
+  default: assert(0 && "No such value"); return ARM_AM::no_shift;
+  case 0:  return ARM_AM::lsl;
+  case 1:  return ARM_AM::lsr;
+  case 2:  return ARM_AM::asr;
+  case 3:  return ARM_AM::ror;
+  }
+}
+
+// See A8.4 Shifts applied to a register.
+//     A8.4.1 Constant shifts.
+//
+// getImmShiftSE - getImmShiftSE translates from the raw ShiftOpc and raw Imm5
+// encodings into the intended ShiftOpc and shift amount.
+//
+// A8-11: DecodeImmShift()
+static inline void getImmShiftSE(ARM_AM::ShiftOpc &ShOp, unsigned &ShImm) {
+  // If type == 0b11 and imm5 == 0, we have an rrx, instead.
+  if (ShOp == ARM_AM::ror && ShImm == 0)
+    ShOp = ARM_AM::rrx;
+  // If (lsr or asr) and imm5 == 0, shift amount is 32.
+  if ((ShOp == ARM_AM::lsr || ShOp == ARM_AM::asr) && ShImm == 0)
+    ShImm = 32;
+}
+
+// getAMSubModeForBits - getAMSubModeForBits translates from the ARM encoding
+// bits Inst{24-23} (P(24) and U(23)) into llvm enums for AMSubMode.  The API
+// clients should pass in the value encoded with two bits, so the assert stays
+// to signal a wrong API usage.
+static inline ARM_AM::AMSubMode getAMSubModeForBits(unsigned bits) {
+  switch (bits) {
+  default: assert(0 && "No such value"); return ARM_AM::bad_am_submode;
+  case 1:  return ARM_AM::ia;   // P=0 U=1
+  case 3:  return ARM_AM::ib;   // P=1 U=1
+  case 0:  return ARM_AM::da;   // P=0 U=0
+  case 2:  return ARM_AM::db;   // P=1 U=0
+  }
+}
+
+////////////////////////////////////////////
+//                                        //
+//    Disassemble function definitions    //
+//                                        //
+////////////////////////////////////////////
+
+/// There is a separate Disassemble*Frm function entry for disassembly of an ARM
+/// instr into a list of MCOperands in the appropriate order, with possible dst,
+/// followed by possible src(s).
+///
+/// The processing of the predicate, and the 'S' modifier bit, if MI modifies
+/// the CPSR, is factored into ARMBasicMCBuilder's method named
+/// TryPredicateAndSBitModifier.
+
+static bool DisassemblePseudo(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  if (Opcode == ARM::Int_MemBarrierV7 || Opcode == ARM::Int_SyncBarrierV7)
+    return true;
+
+  assert(0 && "Unexpected pseudo instruction!");
+  return false;
+}
+
+// Multiply Instructions.
+// MLA, MLS, SMLABB, SMLABT, SMLATB, SMLATT, SMLAWB, SMLAWT, SMMLA, SMMLS:
+//     Rd{19-16} Rn{3-0} Rm{11-8} Ra{15-12}
+//
+// MUL, SMMUL, SMULBB, SMULBT, SMULTB, SMULTT, SMULWB, SMULWT:
+//     Rd{19-16} Rn{3-0} Rm{11-8}
+//
+// SMLAL, SMULL, UMAAL, UMLAL, UMULL, SMLALBB, SMLALBT, SMLALTB, SMLALTT:
+//     RdLo{15-12} RdHi{19-16} Rn{3-0} Rm{11-8}
+//
+// The mapping of the multiply registers to the "regular" ARM registers, where
+// there are convenience decoder functions, is:
+//
+// Inst{15-12} => Rd
+// Inst{19-16} => Rn
+// Inst{3-0} => Rm
+// Inst{11-8} => Rs
+static bool DisassembleMulFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  unsigned short NumDefs = TID.getNumDefs();
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumDefs > 0 && "NumDefs should be greater than 0 for MulFrm");
+  assert(NumOps >= 3
+         && OpInfo[0].RegClass == ARM::GPRRegClassID
+         && OpInfo[1].RegClass == ARM::GPRRegClassID
+         && OpInfo[2].RegClass == ARM::GPRRegClassID
+         && "Expect three register operands");
+
+  // Instructions with two destination registers have RdLo{15-12} first.
+  if (NumDefs == 2) {
+    assert(NumOps >= 4 && OpInfo[3].RegClass == ARM::GPRRegClassID &&
+           "Expect 4th register operand");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRd(insn))));
+    ++OpIdx;
+  }
+
+  // The destination register: RdHi{19-16} or Rd{19-16}.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+
+  // The two src regsiters: Rn{3-0}, then Rm{11-8}.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRm(insn))));
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRs(insn))));
+  OpIdx += 3;
+
+  // Many multiply instructions (e.g., MLA) have three src registers.
+  // The third register operand is Ra{15-12}.
+  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::GPRRegClassID) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRd(insn))));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+// Helper routines for disassembly of coprocessor instructions.
+
+static bool LdStCopOpcode(unsigned Opcode) {
+  if ((Opcode >= ARM::LDC2L_OFFSET && Opcode <= ARM::LDC_PRE) ||
+      (Opcode >= ARM::STC2L_OFFSET && Opcode <= ARM::STC_PRE))
+    return true;
+  return false;
+}
+static bool CoprocessorOpcode(unsigned Opcode) {
+  if (LdStCopOpcode(Opcode))
+    return true;
+
+  switch (Opcode) {
+  default:
+    return false;
+  case ARM::CDP:  case ARM::CDP2:
+  case ARM::MCR:  case ARM::MCR2:  case ARM::MRC:  case ARM::MRC2:
+  case ARM::MCRR: case ARM::MCRR2: case ARM::MRRC: case ARM::MRRC2:
+    return true;
+  }
+}
+static inline unsigned GetCoprocessor(uint32_t insn) {
+  return slice(insn, 11, 8);
+}
+static inline unsigned GetCopOpc1(uint32_t insn, bool CDP) {
+  return CDP ? slice(insn, 23, 20) : slice(insn, 23, 21);
+}
+static inline unsigned GetCopOpc2(uint32_t insn) {
+  return slice(insn, 7, 5);
+}
+static inline unsigned GetCopOpc(uint32_t insn) {
+  return slice(insn, 7, 4);
+}
+// Most of the operands are in immediate forms, except Rd and Rn, which are ARM
+// core registers.
+//
+// CDP, CDP2:                cop opc1 CRd CRn CRm opc2
+//
+// MCR, MCR2, MRC, MRC2:     cop opc1 Rd CRn CRm opc2
+//
+// MCRR, MCRR2, MRRC, MRRc2: cop opc Rd Rn CRm
+//
+// LDC_OFFSET, LDC_PRE, LDC_POST: cop CRd Rn R0 [+/-]imm8:00
+// and friends
+// STC_OFFSET, STC_PRE, STC_POST: cop CRd Rn R0 [+/-]imm8:00
+// and friends
+//                                        <-- addrmode2 -->
+//
+// LDC_OPTION:                    cop CRd Rn imm8
+// and friends
+// STC_OPTION:                    cop CRd Rn imm8
+// and friends
+//
+static bool DisassembleCoprocessor(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  assert(NumOps >= 5 && "Num of operands >= 5 for coprocessor instr");
+
+  unsigned &OpIdx = NumOpsAdded;
+  bool OneCopOpc = (Opcode == ARM::MCRR || Opcode == ARM::MCRR2 ||
+                    Opcode == ARM::MRRC || Opcode == ARM::MRRC2);
+  // CDP/CDP2 has no GPR operand; the opc1 operand is also wider (Inst{23-20}).
+  bool NoGPR = (Opcode == ARM::CDP || Opcode == ARM::CDP2);
+  bool LdStCop = LdStCopOpcode(Opcode);
+
+  OpIdx = 0;
+
+  MI.addOperand(MCOperand::CreateImm(GetCoprocessor(insn)));
+
+  if (LdStCop) {
+    // Unindex if P:W = 0b00 --> _OPTION variant
+    unsigned PW = getPBit(insn) << 1 | getWBit(insn);
+
+    MI.addOperand(MCOperand::CreateImm(decodeRd(insn)));
+
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+
+    if (PW) {
+      MI.addOperand(MCOperand::CreateReg(0));
+      ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
+      unsigned Offset = ARM_AM::getAM2Opc(AddrOpcode, slice(insn, 7, 0) << 2,
+                                          ARM_AM::no_shift);
+      MI.addOperand(MCOperand::CreateImm(Offset));
+      OpIdx = 5;
+    } else {
+      MI.addOperand(MCOperand::CreateImm(slice(insn, 7, 0)));
+      OpIdx = 4;
+    }
+  } else {
+    MI.addOperand(MCOperand::CreateImm(OneCopOpc ? GetCopOpc(insn)
+                                                 : GetCopOpc1(insn, NoGPR)));
+
+    MI.addOperand(NoGPR ? MCOperand::CreateImm(decodeRd(insn))
+                        : MCOperand::CreateReg(
+                            getRegisterEnum(ARM::GPRRegClassID,
+                                            decodeRd(insn))));
+
+    MI.addOperand(OneCopOpc ? MCOperand::CreateReg(
+                                getRegisterEnum(ARM::GPRRegClassID,
+                                                decodeRn(insn)))
+                            : MCOperand::CreateImm(decodeRn(insn)));
+
+    MI.addOperand(MCOperand::CreateImm(decodeRm(insn)));
+
+    OpIdx = 5;
+
+    if (!OneCopOpc) {
+      MI.addOperand(MCOperand::CreateImm(GetCopOpc2(insn)));
+      ++OpIdx;
+    }
+  }
+
+  return true;
+}
+
+// Branch Instructions.
+// BLr9: SignExtend(Imm24:'00', 32)
+// Bcc, BLr9_pred: SignExtend(Imm24:'00', 32) Pred0 Pred1
+// SMC: ZeroExtend(imm4, 32)
+// SVC: ZeroExtend(Imm24, 32)
+//
+// Various coprocessor instructions are assigned BrFrm arbitrarily.
+// Delegates to DisassembleCoprocessor() helper function.
+//
+// MRS/MRSsys: Rd
+// MSR/MSRsys: Rm mask=Inst{19-16}
+// BXJ:        Rm
+// MSRi/MSRsysi: so_imm
+// SRSW/SRS: addrmode4:$addr mode_imm
+// RFEW/RFE: addrmode4:$addr Rn
+static bool DisassembleBrFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  if (CoprocessorOpcode(Opcode))
+    return DisassembleCoprocessor(MI, Opcode, insn, NumOps, NumOpsAdded);
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  // MRS and MRSsys take one GPR reg Rd.
+  if (Opcode == ARM::MRS || Opcode == ARM::MRSsys) {
+    assert(NumOps >= 1 && OpInfo[0].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRd(insn))));
+    NumOpsAdded = 1;
+    return true;
+  }
+  // BXJ takes one GPR reg Rm.
+  if (Opcode == ARM::BXJ) {
+    assert(NumOps >= 1 && OpInfo[0].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+    NumOpsAdded = 1;
+    return true;
+  }
+  // MSR and MSRsys take one GPR reg Rm, followed by the mask.
+  if (Opcode == ARM::MSR || Opcode == ARM::MSRsys) {
+    assert(NumOps >= 1 && OpInfo[0].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 19, 16)));
+    NumOpsAdded = 2;
+    return true;
+  }
+  // MSRi and MSRsysi take one so_imm operand, followed by the mask.
+  if (Opcode == ARM::MSRi || Opcode == ARM::MSRsysi) {
+    // SOImm is 4-bit rotate amount in bits 11-8 with 8-bit imm in bits 7-0.
+    // A5.2.4 Rotate amount is twice the numeric value of Inst{11-8}.
+    // See also ARMAddressingModes.h: getSOImmValImm() and getSOImmValRot().
+    unsigned Rot = (insn >> ARMII::SoRotImmShift) & 0xF;
+    unsigned Imm = insn & 0xFF;
+    MI.addOperand(MCOperand::CreateImm(ARM_AM::rotr32(Imm, 2*Rot)));
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 19, 16)));
+    NumOpsAdded = 2;
+    return true;
+  }
+  // SRSW and SRS requires addrmode4:$addr for ${addr:submode}, followed by the
+  // mode immediate (Inst{4-0}).
+  if (Opcode == ARM::SRSW || Opcode == ARM::SRS ||
+      Opcode == ARM::RFEW || Opcode == ARM::RFE) {
+    // ARMInstPrinter::printAddrMode4Operand() prints special mode string
+    // if the base register is SP; so don't set ARM::SP.
+    MI.addOperand(MCOperand::CreateReg(0));
+    ARM_AM::AMSubMode SubMode = getAMSubModeForBits(getPUBits(insn));
+    MI.addOperand(MCOperand::CreateImm(ARM_AM::getAM4ModeImm(SubMode)));
+
+    if (Opcode == ARM::SRSW || Opcode == ARM::SRS)
+      MI.addOperand(MCOperand::CreateImm(slice(insn, 4, 0)));
+    else
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                         decodeRn(insn))));
+    NumOpsAdded = 3;
+    return true;
+  }
+
+  assert((Opcode == ARM::Bcc || Opcode == ARM::BLr9 || Opcode == ARM::BLr9_pred
+          || Opcode == ARM::SMC || Opcode == ARM::SVC) &&
+         "Unexpected Opcode");
+
+  assert(NumOps >= 1 && OpInfo[0].RegClass == 0 && "Reg operand expected");
+
+  int Imm32 = 0;
+  if (Opcode == ARM::SMC) {
+    // ZeroExtend(imm4, 32) where imm24 = Inst{3-0}.
+    Imm32 = slice(insn, 3, 0);
+  } else if (Opcode == ARM::SVC) {
+    // ZeroExtend(imm24, 32) where imm24 = Inst{23-0}.
+    Imm32 = slice(insn, 23, 0);
+  } else {
+    // SignExtend(imm24:'00', 32) where imm24 = Inst{23-0}.
+    unsigned Imm26 = slice(insn, 23, 0) << 2;
+    //Imm32 = signextend<signed int, 26>(Imm26);
+    Imm32 = SignExtend32<26>(Imm26);
+
+    // When executing an ARM instruction, PC reads as the address of the current
+    // instruction plus 8.  The assembler subtracts 8 from the difference
+    // between the branch instruction and the target address, disassembler has
+    // to add 8 to compensate.
+    Imm32 += 8;
+  }
+
+  MI.addOperand(MCOperand::CreateImm(Imm32));
+  NumOpsAdded = 1;
+
+  return true;
+}
+
+// Misc. Branch Instructions.
+// BR_JTadd, BR_JTr, BR_JTm
+// BLXr9, BXr9
+// BRIND, BX_RET
+static bool DisassembleBrMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  // BX_RET has only two predicate operands, do an early return.
+  if (Opcode == ARM::BX_RET)
+    return true;
+
+  // BLXr9 and BRIND take one GPR reg.
+  if (Opcode == ARM::BLXr9 || Opcode == ARM::BRIND) {
+    assert(NumOps >= 1 && OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+    OpIdx = 1;
+    return true;
+  }
+
+  // BR_JTadd is an ADD with Rd = PC, (Rn, Rm) as the target and index regs.
+  if (Opcode == ARM::BR_JTadd) {
+    // InOperandList with GPR:$target and GPR:$idx regs.
+
+    assert(NumOps == 4 && "Expect 4 operands");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+
+    // Fill in the two remaining imm operands to signify build completion.
+    MI.addOperand(MCOperand::CreateImm(0));
+    MI.addOperand(MCOperand::CreateImm(0));
+
+    OpIdx = 4;
+    return true;
+  }
+
+  // BR_JTr is a MOV with Rd = PC, and Rm as the source register.
+  if (Opcode == ARM::BR_JTr) {
+    // InOperandList with GPR::$target reg.
+
+    assert(NumOps == 3 && "Expect 3 operands");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+
+    // Fill in the two remaining imm operands to signify build completion.
+    MI.addOperand(MCOperand::CreateImm(0));
+    MI.addOperand(MCOperand::CreateImm(0));
+
+    OpIdx = 3;
+    return true;
+  }
+
+  // BR_JTm is an LDR with Rt = PC.
+  if (Opcode == ARM::BR_JTm) {
+    // This is the reg/reg form, with base reg followed by +/- reg shop imm.
+    // See also ARMAddressingModes.h (Addressing Mode #2).
+
+    assert(NumOps == 5 && getIBit(insn) == 1 && "Expect 5 operands && I-bit=1");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+
+    ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
+
+    // Disassemble the offset reg (Rm), shift type, and immediate shift length.
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+    // Inst{6-5} encodes the shift opcode.
+    ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 6, 5));
+    // Inst{11-7} encodes the imm5 shift amount.
+    unsigned ShImm = slice(insn, 11, 7);
+
+    // A8.4.1.  Possible rrx or shift amount of 32...
+    getImmShiftSE(ShOp, ShImm);
+    MI.addOperand(MCOperand::CreateImm(
+                    ARM_AM::getAM2Opc(AddrOpcode, ShImm, ShOp)));
+
+    // Fill in the two remaining imm operands to signify build completion.
+    MI.addOperand(MCOperand::CreateImm(0));
+    MI.addOperand(MCOperand::CreateImm(0));
+
+    OpIdx = 5;
+    return true;
+  }
+
+  assert(0 && "Unexpected BrMiscFrm Opcode");
+  return false;
+}
+
+static inline uint32_t getBFCInvMask(uint32_t insn) {
+  uint32_t lsb = slice(insn, 11, 7);
+  uint32_t msb = slice(insn, 20, 16);
+  uint32_t Val = 0;
+  assert(lsb <= msb && "Encoding error: lsb > msb");
+  for (uint32_t i = lsb; i <= msb; ++i)
+    Val |= (1 << i);
+  return ~Val;
+}
+
+static inline bool SaturateOpcode(unsigned Opcode) {
+  switch (Opcode) {
+  case ARM::SSATlsl: case ARM::SSATasr: case ARM::SSAT16:
+  case ARM::USATlsl: case ARM::USATasr: case ARM::USAT16:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static inline unsigned decodeSaturatePos(unsigned Opcode, uint32_t insn) {
+  switch (Opcode) {
+  case ARM::SSATlsl:
+  case ARM::SSATasr:
+    return slice(insn, 20, 16) + 1;
+  case ARM::SSAT16:
+    return slice(insn, 19, 16) + 1;
+  case ARM::USATlsl:
+  case ARM::USATasr:
+    return slice(insn, 20, 16);
+  case ARM::USAT16:
+    return slice(insn, 19, 16);
+  default:
+    assert(0 && "Invalid opcode passed in");
+    return 0;
+  }
+}
+
+// A major complication is the fact that some of the saturating add/subtract
+// operations have Rd Rm Rn, instead of the "normal" Rd Rn Rm.
+// They are QADD, QDADD, QDSUB, and QSUB.
+static bool DisassembleDPFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  unsigned short NumDefs = TID.getNumDefs();
+  bool isUnary = isUnaryDP(TID.TSFlags);
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  // Disassemble register def if there is one.
+  if (NumDefs && (OpInfo[OpIdx].RegClass == ARM::GPRRegClassID)) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRd(insn))));
+    ++OpIdx;
+  }
+
+  // Now disassemble the src operands.
+  if (OpIdx >= NumOps)
+    return false;
+
+  // SSAT/SSAT16/USAT/USAT16 has imm operand after Rd.
+  if (SaturateOpcode(Opcode)) {
+    MI.addOperand(MCOperand::CreateImm(decodeSaturatePos(Opcode, insn)));
+
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+
+    if (Opcode == ARM::SSAT16 || Opcode == ARM::USAT16) {
+      OpIdx += 2;
+      return true;
+    }
+
+    // For SSAT operand reg (Rm) has been disassembled above.
+    // Now disassemble the shift amount.
+
+    // Inst{11-7} encodes the imm5 shift amount.
+    unsigned ShAmt = slice(insn, 11, 7);
+
+    // A8.6.183.  Possible ASR shift amount of 32...
+    if (Opcode == ARM::SSATasr && ShAmt == 0)
+      ShAmt = 32;
+
+    MI.addOperand(MCOperand::CreateImm(ShAmt));
+
+    OpIdx += 3;
+    return true;
+  }
+
+  // Special-case handling of BFC/BFI/SBFX/UBFX.
+  if (Opcode == ARM::BFC || Opcode == ARM::BFI) {
+    // TIED_TO operand skipped for BFC and Inst{3-0} (Reg) for BFI.
+    MI.addOperand(MCOperand::CreateReg(Opcode == ARM::BFC ? 0
+                                       : getRegisterEnum(ARM::GPRRegClassID,
+                                                         decodeRm(insn))));
+    MI.addOperand(MCOperand::CreateImm(getBFCInvMask(insn)));
+    OpIdx += 2;
+    return true;
+  }
+  if (Opcode == ARM::SBFX || Opcode == ARM::UBFX) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 11, 7)));
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 20, 16) + 1));
+    OpIdx += 3;
+    return true;
+  }
+
+  bool RmRn = (Opcode == ARM::QADD || Opcode == ARM::QDADD ||
+               Opcode == ARM::QDSUB || Opcode == ARM::QSUB);
+
+  // BinaryDP has an Rn operand.
+  if (!isUnary) {
+    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(
+                    getRegisterEnum(ARM::GPRRegClassID,
+                                    RmRn ? decodeRm(insn) : decodeRn(insn))));
+    ++OpIdx;
+  }
+
+  // If this is a two-address operand, skip it, e.g., MOVCCr operand 1.
+  if (isUnary && (TID.getOperandConstraint(OpIdx, TOI::TIED_TO) != -1)) {
+    MI.addOperand(MCOperand::CreateReg(0));
+    ++OpIdx;
+  }
+
+  // Now disassemble operand 2.
+  if (OpIdx >= NumOps)
+    return false;
+
+  if (OpInfo[OpIdx].RegClass == ARM::GPRRegClassID) {
+    // We have a reg/reg form.
+    // Assert disabled because saturating operations, e.g., A8.6.127 QASX, are
+    // routed here as well.
+    // assert(getIBit(insn) == 0 && "I_Bit != '0' reg/reg form");
+    MI.addOperand(MCOperand::CreateReg(
+                    getRegisterEnum(ARM::GPRRegClassID,
+                                    RmRn? decodeRn(insn) : decodeRm(insn))));
+    ++OpIdx;
+  } else if (Opcode == ARM::MOVi16 || Opcode == ARM::MOVTi16) {
+    // We have an imm16 = imm4:imm12 (imm4=Inst{19:16}, imm12 = Inst{11:0}).
+    assert(getIBit(insn) == 1 && "I_Bit != '1' reg/imm form");
+    unsigned Imm16 = slice(insn, 19, 16) << 12 | slice(insn, 11, 0);
+    MI.addOperand(MCOperand::CreateImm(Imm16));
+    ++OpIdx;
+  } else {
+    // We have a reg/imm form.
+    // SOImm is 4-bit rotate amount in bits 11-8 with 8-bit imm in bits 7-0.
+    // A5.2.4 Rotate amount is twice the numeric value of Inst{11-8}.
+    // See also ARMAddressingModes.h: getSOImmValImm() and getSOImmValRot().
+    assert(getIBit(insn) == 1 && "I_Bit != '1' reg/imm form");
+    unsigned Rot = (insn >> ARMII::SoRotImmShift) & 0xF;
+    unsigned Imm = insn & 0xFF;
+    MI.addOperand(MCOperand::CreateImm(ARM_AM::rotr32(Imm, 2*Rot)));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+static bool DisassembleDPSoRegFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  unsigned short NumDefs = TID.getNumDefs();
+  bool isUnary = isUnaryDP(TID.TSFlags);
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  // Disassemble register def if there is one.
+  if (NumDefs && (OpInfo[OpIdx].RegClass == ARM::GPRRegClassID)) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRd(insn))));
+    ++OpIdx;
+  }
+
+  // Disassemble the src operands.
+  if (OpIdx >= NumOps)
+    return false;
+
+  // BinaryDP has an Rn operand.
+  if (!isUnary) {
+    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    ++OpIdx;
+  }
+
+  // If this is a two-address operand, skip it, e.g., MOVCCs operand 1.
+  if (isUnary && (TID.getOperandConstraint(OpIdx, TOI::TIED_TO) != -1)) {
+    MI.addOperand(MCOperand::CreateReg(0));
+    ++OpIdx;
+  }
+
+  // Disassemble operand 2, which consists of three components.
+  if (OpIdx + 2 >= NumOps)
+    return false;
+
+  assert((OpInfo[OpIdx].RegClass == ARM::GPRRegClassID) &&
+         (OpInfo[OpIdx+1].RegClass == ARM::GPRRegClassID) &&
+         (OpInfo[OpIdx+2].RegClass == 0) &&
+         "Expect 3 reg operands");
+
+  // Register-controlled shifts have Inst{7} = 0 and Inst{4} = 1.
+  unsigned Rs = slice(insn, 4, 4);
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRm(insn))));
+  if (Rs) {
+    // Register-controlled shifts: [Rm, Rs, shift].
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRs(insn))));
+    // Inst{6-5} encodes the shift opcode.
+    ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 6, 5));
+    MI.addOperand(MCOperand::CreateImm(ARM_AM::getSORegOpc(ShOp, 0)));
+  } else {
+    // Constant shifts: [Rm, reg0, shift_imm].
+    MI.addOperand(MCOperand::CreateReg(0)); // NoRegister
+    // Inst{6-5} encodes the shift opcode.
+    ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 6, 5));
+    // Inst{11-7} encodes the imm5 shift amount.
+    unsigned ShImm = slice(insn, 11, 7);
+
+    // A8.4.1.  Possible rrx or shift amount of 32...
+    getImmShiftSE(ShOp, ShImm);
+    MI.addOperand(MCOperand::CreateImm(ARM_AM::getSORegOpc(ShOp, ShImm)));
+  }
+  OpIdx += 3;
+
+  return true;
+}
+
+static bool DisassembleLdStFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, bool isStore) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  unsigned short NumDefs = TID.getNumDefs();
+  bool isPrePost = isPrePostLdSt(TID.TSFlags);
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(((!isStore && NumDefs > 0) || (isStore && (NumDefs == 0 || isPrePost)))
+         && "Invalid arguments");
+
+  // Operand 0 of a pre- and post-indexed store is the address base writeback.
+  if (isPrePost && isStore) {
+    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    ++OpIdx;
+  }
+
+  // Disassemble the dst/src operand.
+  if (OpIdx >= NumOps)
+    return false;
+
+  assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+         "Reg operand expected");
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  ++OpIdx;
+
+  // After dst of a pre- and post-indexed load is the address base writeback.
+  if (isPrePost && !isStore) {
+    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    ++OpIdx;
+  }
+
+  // Disassemble the base operand.
+  if (OpIdx >= NumOps)
+    return false;
+
+  assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+         "Reg operand expected");
+  assert((!isPrePost || (TID.getOperandConstraint(OpIdx, TOI::TIED_TO) != -1))
+         && "Index mode or tied_to operand expected");
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+  ++OpIdx;
+
+  // For reg/reg form, base reg is followed by +/- reg shop imm.
+  // For immediate form, it is followed by +/- imm12.
+  // See also ARMAddressingModes.h (Addressing Mode #2).
+  if (OpIdx + 1 >= NumOps)
+    return false;
+
+  assert((OpInfo[OpIdx].RegClass == ARM::GPRRegClassID) &&
+         (OpInfo[OpIdx+1].RegClass == 0) &&
+         "Expect 1 reg operand followed by 1 imm operand");
+
+  ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
+  if (getIBit(insn) == 0) {
+    MI.addOperand(MCOperand::CreateReg(0));
+
+    // Disassemble the 12-bit immediate offset.
+    unsigned Imm12 = slice(insn, 11, 0);
+    unsigned Offset = ARM_AM::getAM2Opc(AddrOpcode, Imm12, ARM_AM::no_shift);
+    MI.addOperand(MCOperand::CreateImm(Offset));
+  } else {
+    // Disassemble the offset reg (Rm), shift type, and immediate shift length.
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+    // Inst{6-5} encodes the shift opcode.
+    ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 6, 5));
+    // Inst{11-7} encodes the imm5 shift amount.
+    unsigned ShImm = slice(insn, 11, 7);
+
+    // A8.4.1.  Possible rrx or shift amount of 32...
+    getImmShiftSE(ShOp, ShImm);
+    MI.addOperand(MCOperand::CreateImm(
+                    ARM_AM::getAM2Opc(AddrOpcode, ShImm, ShOp)));
+  }
+  OpIdx += 2;
+
+  return true;
+}
+
+static bool DisassembleLdFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+  return DisassembleLdStFrm(MI, Opcode, insn, NumOps, NumOpsAdded, false);
+}
+
+static bool DisassembleStFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+  return DisassembleLdStFrm(MI, Opcode, insn, NumOps, NumOpsAdded, true);
+}
+
+static bool HasDualReg(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    return false;
+  case ARM::LDRD: case ARM::LDRD_PRE: case ARM::LDRD_POST:
+  case ARM::STRD: case ARM::STRD_PRE: case ARM::STRD_POST:
+    return true;
+  }  
+}
+
+static bool DisassembleLdStMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, bool isStore) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  unsigned short NumDefs = TID.getNumDefs();
+  bool isPrePost = isPrePostLdSt(TID.TSFlags);
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(((!isStore && NumDefs > 0) || (isStore && (NumDefs == 0 || isPrePost)))
+         && "Invalid arguments");
+
+  // Operand 0 of a pre- and post-indexed store is the address base writeback.
+  if (isPrePost && isStore) {
+    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    ++OpIdx;
+  }
+
+  bool DualReg = HasDualReg(Opcode);
+
+  // Disassemble the dst/src operand.
+  if (OpIdx >= NumOps)
+    return false;
+
+  assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+         "Reg operand expected");
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  ++OpIdx;
+
+  // Fill in LDRD and STRD's second operand.
+  if (DualReg) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRd(insn) + 1)));
+    ++OpIdx;
+  }
+
+  // After dst of a pre- and post-indexed load is the address base writeback.
+  if (isPrePost && !isStore) {
+    assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    ++OpIdx;
+  }
+
+  // Disassemble the base operand.
+  if (OpIdx >= NumOps)
+    return false;
+
+  assert(OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+         "Reg operand expected");
+  assert((!isPrePost || (TID.getOperandConstraint(OpIdx, TOI::TIED_TO) != -1))
+         && "Index mode or tied_to operand expected");
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+  ++OpIdx;
+
+  // For reg/reg form, base reg is followed by +/- reg.
+  // For immediate form, it is followed by +/- imm8.
+  // See also ARMAddressingModes.h (Addressing Mode #3).
+  if (OpIdx + 1 >= NumOps)
+    return false;
+
+  assert((OpInfo[OpIdx].RegClass == ARM::GPRRegClassID) &&
+         (OpInfo[OpIdx+1].RegClass == 0) &&
+         "Expect 1 reg operand followed by 1 imm operand");
+
+  ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
+  if (getAM3IBit(insn) == 1) {
+    MI.addOperand(MCOperand::CreateReg(0));
+
+    // Disassemble the 8-bit immediate offset.
+    unsigned Imm4H = (insn >> ARMII::ImmHiShift) & 0xF;
+    unsigned Imm4L = insn & 0xF;
+    unsigned Offset = ARM_AM::getAM3Opc(AddrOpcode, (Imm4H << 4) | Imm4L);
+    MI.addOperand(MCOperand::CreateImm(Offset));
+  } else {
+    // Disassemble the offset reg (Rm).
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+    unsigned Offset = ARM_AM::getAM3Opc(AddrOpcode, 0);
+    MI.addOperand(MCOperand::CreateImm(Offset));
+  }
+  OpIdx += 2;
+
+  return true;
+}
+
+static bool DisassembleLdMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+  return DisassembleLdStMiscFrm(MI, Opcode, insn, NumOps, NumOpsAdded, false);
+}
+
+static bool DisassembleStMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+  return DisassembleLdStMiscFrm(MI, Opcode, insn, NumOps, NumOpsAdded, true);
+}
+
+// The algorithm for disassembly of LdStMulFrm is different from others because
+// it explicitly populates the two predicate operands after operand 0 (the base)
+// and operand 1 (the AM4 mode imm).  After operand 3, we need to populate the
+// reglist with each affected register encoded as an MCOperand.
+static bool DisassembleLdStMulFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(NumOps >= 5 && "LdStMulFrm expects NumOps >= 5");
+
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  unsigned Base = getRegisterEnum(ARM::GPRRegClassID, decodeRn(insn));
+
+  // Writeback to base, if necessary.
+  if (Opcode == ARM::LDM_UPD || Opcode == ARM::STM_UPD) {
+    MI.addOperand(MCOperand::CreateReg(Base));
+    ++OpIdx;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(Base));
+
+  ARM_AM::AMSubMode SubMode = getAMSubModeForBits(getPUBits(insn));
+  MI.addOperand(MCOperand::CreateImm(ARM_AM::getAM4ModeImm(SubMode)));
+
+  // Handling the two predicate operands before the reglist.
+  int64_t CondVal = insn >> ARMII::CondShift;
+  MI.addOperand(MCOperand::CreateImm(CondVal == 0xF ? 0xE : CondVal));
+  MI.addOperand(MCOperand::CreateReg(ARM::CPSR));
+
+  OpIdx += 4;
+
+  // Fill the variadic part of reglist.
+  unsigned RegListBits = insn & ((1 << 16) - 1);
+  for (unsigned i = 0; i < 16; ++i) {
+    if ((RegListBits >> i) & 1) {
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                         i)));
+      ++OpIdx;
+    }
+  }
+
+  return true;
+}
+
+// LDREX, LDREXB, LDREXH: Rd Rn
+// LDREXD:                Rd Rd+1 Rn
+// STREX, STREXB, STREXH: Rd Rm Rn
+// STREXD:                Rd Rm Rm+1 Rn
+//
+// SWP, SWPB:             Rd Rm Rn
+static bool DisassembleLdStExFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2
+         && OpInfo[0].RegClass == ARM::GPRRegClassID
+         && OpInfo[1].RegClass == ARM::GPRRegClassID
+         && "Expect 2 reg operands");
+
+  bool isStore = slice(insn, 20, 20) == 0;
+  bool isDW = (Opcode == ARM::LDREXD || Opcode == ARM::STREXD);
+
+  // Add the destination operand.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  ++OpIdx;
+
+  // Store register Exclusive needs a source operand.
+  if (isStore) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+    ++OpIdx;
+
+    if (isDW) {
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                         decodeRm(insn)+1)));
+      ++OpIdx;
+    }
+  } else if (isDW) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRd(insn)+1)));
+    ++OpIdx;
+  }
+
+  // Finally add the pointer operand.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+  ++OpIdx;
+
+  return true;
+}
+
+// Misc. Arithmetic Instructions.
+// CLZ: Rd Rm
+// PKHBT, PKHTB: Rd Rn Rm , LSL/ASR #imm5
+// RBIT, REV, REV16, REVSH: Rd Rm
+static bool DisassembleArithMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2
+         && OpInfo[0].RegClass == ARM::GPRRegClassID
+         && OpInfo[1].RegClass == ARM::GPRRegClassID
+         && "Expect 2 reg operands");
+
+  bool ThreeReg = NumOps > 2 && OpInfo[2].RegClass == ARM::GPRRegClassID;
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  ++OpIdx;
+
+  if (ThreeReg) {
+    assert(NumOps >= 4 && "Expect >= 4 operands");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    ++OpIdx;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRm(insn))));
+  ++OpIdx;
+
+  // If there is still an operand info left which is an immediate operand, add
+  // an additional imm5 LSL/ASR operand.
+  if (ThreeReg && OpInfo[OpIdx].RegClass == 0
+      && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
+    // Extract the 5-bit immediate field Inst{11-7}.
+    unsigned ShiftAmt = (insn >> ARMII::ShiftShift) & 0x1F;
+    MI.addOperand(MCOperand::CreateImm(ShiftAmt));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+// Extend instructions.
+// SXT* and UXT*: Rd [Rn] Rm [rot_imm].
+// The 2nd operand register is Rn and the 3rd operand regsiter is Rm for the
+// three register operand form.  Otherwise, Rn=0b1111 and only Rm is used.
+static bool DisassembleExtFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2
+         && OpInfo[0].RegClass == ARM::GPRRegClassID
+         && OpInfo[1].RegClass == ARM::GPRRegClassID
+         && "Expect 2 reg operands");
+
+  bool ThreeReg = NumOps > 2 && OpInfo[2].RegClass == ARM::GPRRegClassID;
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  ++OpIdx;
+
+  if (ThreeReg) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    ++OpIdx;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRm(insn))));
+  ++OpIdx;
+
+  // If there is still an operand info left which is an immediate operand, add
+  // an additional rotate immediate operand.
+  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == 0
+      && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
+    // Extract the 2-bit rotate field Inst{11-10}.
+    unsigned rot = (insn >> ARMII::ExtRotImmShift) & 3;
+    // Rotation by 8, 16, or 24 bits.
+    MI.addOperand(MCOperand::CreateImm(rot << 3));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+/////////////////////////////////////
+//                                 //
+//    Utility Functions For VFP    //
+//                                 //
+/////////////////////////////////////
+
+// Extract/Decode Dd/Sd:
+//
+// SP => d = UInt(Vd:D)
+// DP => d = UInt(D:Vd)
+static unsigned decodeVFPRd(uint32_t insn, bool isSPVFP) {
+  return isSPVFP ? (decodeRd(insn) << 1 | getDBit(insn))
+                 : (decodeRd(insn) | getDBit(insn) << 4);
+}
+
+// Extract/Decode Dn/Sn:
+//
+// SP => n = UInt(Vn:N)
+// DP => n = UInt(N:Vn)
+static unsigned decodeVFPRn(uint32_t insn, bool isSPVFP) {
+  return isSPVFP ? (decodeRn(insn) << 1 | getNBit(insn))
+                 : (decodeRn(insn) | getNBit(insn) << 4);
+}
+
+// Extract/Decode Dm/Sm:
+//
+// SP => m = UInt(Vm:M)
+// DP => m = UInt(M:Vm)
+static unsigned decodeVFPRm(uint32_t insn, bool isSPVFP) {
+  return isSPVFP ? (decodeRm(insn) << 1 | getMBit(insn))
+                 : (decodeRm(insn) | getMBit(insn) << 4);
+}
+
+// A7.5.1
+#if 0
+static uint64_t VFPExpandImm(unsigned char byte, unsigned N) {
+  assert(N == 32 || N == 64);
+
+  uint64_t Result;
+  unsigned bit6 = slice(byte, 6, 6);
+  if (N == 32) {
+    Result = slice(byte, 7, 7) << 31 | slice(byte, 5, 0) << 19;
+    if (bit6)
+      Result |= 0x1f << 25;
+    else
+      Result |= 0x1 << 30;
+  } else {
+    Result = (uint64_t)slice(byte, 7, 7) << 63 |
+             (uint64_t)slice(byte, 5, 0) << 48;
+    if (bit6)
+      Result |= 0xffL << 54;
+    else
+      Result |= 0x1L << 62;
+  }
+  return Result;
+}
+#endif
+
+// VFP Unary Format Instructions:
+//
+// VCMP[E]ZD, VCMP[E]ZS: compares one floating-point register with zero
+// VCVTDS, VCVTSD: converts between double-precision and single-precision
+// The rest of the instructions have homogeneous [VFP]Rd and [VFP]Rm registers.
+static bool DisassembleVFPUnaryFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(NumOps >= 1 && "VFPUnaryFrm expects NumOps >= 1");
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  unsigned RegClass = OpInfo[OpIdx].RegClass;
+  assert((RegClass == ARM::SPRRegClassID || RegClass == ARM::DPRRegClassID) &&
+         "Reg operand expected");
+  bool isSP = (RegClass == ARM::SPRRegClassID);
+
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(RegClass, decodeVFPRd(insn, isSP))));
+  ++OpIdx;
+
+  // Early return for compare with zero instructions.
+  if (Opcode == ARM::VCMPEZD || Opcode == ARM::VCMPEZS
+      || Opcode == ARM::VCMPZD || Opcode == ARM::VCMPZS)
+    return true;
+
+  RegClass = OpInfo[OpIdx].RegClass;
+  assert((RegClass == ARM::SPRRegClassID || RegClass == ARM::DPRRegClassID) &&
+         "Reg operand expected");
+  isSP = (RegClass == ARM::SPRRegClassID);
+
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(RegClass, decodeVFPRm(insn, isSP))));
+  ++OpIdx;
+
+  return true;
+}
+
+// All the instructions have homogeneous [VFP]Rd, [VFP]Rn, and [VFP]Rm regs.
+// Some of them have operand constraints which tie the first operand in the
+// InOperandList to that of the dst.  As far as asm printing is concerned, this
+// tied_to operand is simply skipped.
+static bool DisassembleVFPBinaryFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(NumOps >= 3 && "VFPBinaryFrm expects NumOps >= 3");
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  unsigned RegClass = OpInfo[OpIdx].RegClass;
+  assert((RegClass == ARM::SPRRegClassID || RegClass == ARM::DPRRegClassID) &&
+         "Reg operand expected");
+  bool isSP = (RegClass == ARM::SPRRegClassID);
+
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(RegClass, decodeVFPRd(insn, isSP))));
+  ++OpIdx;
+
+  // Skip tied_to operand constraint.
+  if (TID.getOperandConstraint(OpIdx, TOI::TIED_TO) != -1) {
+    assert(NumOps >= 4 && "Expect >=4 operands");
+    MI.addOperand(MCOperand::CreateReg(0));
+    ++OpIdx;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(RegClass, decodeVFPRn(insn, isSP))));
+  ++OpIdx;
+
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(RegClass, decodeVFPRm(insn, isSP))));
+  ++OpIdx;
+
+  return true;
+}
+
+// A8.6.295 vcvt (floating-point <-> integer)
+// Int to FP: VSITOD, VSITOS, VUITOD, VUITOS
+// FP to Int: VTOSI[Z|R]D, VTOSI[Z|R]S, VTOUI[Z|R]D, VTOUI[Z|R]S
+// 
+// A8.6.297 vcvt (floating-point and fixed-point)
+// Dd|Sd Dd|Sd(TIED_TO) #fbits(= 16|32 - UInt(imm4:i))
+static bool DisassembleVFPConv1Frm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(NumOps >= 2 && "VFPConv1Frm expects NumOps >= 2");
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+
+  bool SP = slice(insn, 8, 8) == 0; // A8.6.295 & A8.6.297
+  bool fixed_point = slice(insn, 17, 17) == 1; // A8.6.297
+  unsigned RegClassID = SP ? ARM::SPRRegClassID : ARM::DPRRegClassID;
+
+  if (fixed_point) {
+    // A8.6.297
+    assert(NumOps >= 3 && "Expect >= 3 operands");
+    int size = slice(insn, 7, 7) == 0 ? 16 : 32;
+    int fbits = size - (slice(insn,3,0) << 1 | slice(insn,5,5));
+    MI.addOperand(MCOperand::CreateReg(
+                    getRegisterEnum(RegClassID,
+                                    decodeVFPRd(insn, SP))));
+
+    assert(TID.getOperandConstraint(1, TOI::TIED_TO) != -1 &&
+           "Tied to operand expected");
+    MI.addOperand(MI.getOperand(0));
+
+    assert(OpInfo[2].RegClass == 0 && !OpInfo[2].isPredicate() &&
+           !OpInfo[2].isOptionalDef() && "Imm operand expected");
+    MI.addOperand(MCOperand::CreateImm(fbits));
+
+    NumOpsAdded = 3;
+  } else {
+    // A8.6.295
+    // The Rd (destination) and Rm (source) bits have different interpretations
+    // depending on their single-precisonness.
+    unsigned d, m;
+    if (slice(insn, 18, 18) == 1) { // to_integer operation
+      d = decodeVFPRd(insn, true /* Is Single Precision */);
+      MI.addOperand(MCOperand::CreateReg(
+                      getRegisterEnum(ARM::SPRRegClassID, d)));
+      m = decodeVFPRm(insn, SP);
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(RegClassID, m)));
+    } else {
+      d = decodeVFPRd(insn, SP);
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(RegClassID, d)));
+      m = decodeVFPRm(insn, true /* Is Single Precision */);
+      MI.addOperand(MCOperand::CreateReg(
+                      getRegisterEnum(ARM::SPRRegClassID, m)));
+    }
+    NumOpsAdded = 2;
+  }
+
+  return true;
+}
+
+// VMOVRS - A8.6.330
+// Rt => Rd; Sn => UInt(Vn:N)
+static bool DisassembleVFPConv2Frm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(NumOps >= 2 && "VFPConv2Frm expects NumOps >= 2");
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::SPRRegClassID,
+                                                     decodeVFPRn(insn, true))));
+  NumOpsAdded = 2;
+  return true;
+}
+
+// VMOVRRD - A8.6.332
+// Rt => Rd; Rt2 => Rn; Dm => UInt(M:Vm)
+//
+// VMOVRRS - A8.6.331
+// Rt => Rd; Rt2 => Rn; Sm => UInt(Vm:M); Sm1 = Sm+1
+static bool DisassembleVFPConv3Frm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(NumOps >= 3 && "VFPConv3Frm expects NumOps >= 3");
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+  OpIdx = 2;
+
+  if (OpInfo[OpIdx].RegClass == ARM::SPRRegClassID) {
+    unsigned Sm = decodeVFPRm(insn, true);
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::SPRRegClassID,
+                                                       Sm)));
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::SPRRegClassID,
+                                                       Sm+1)));
+    OpIdx += 2;
+  } else {
+    MI.addOperand(MCOperand::CreateReg(
+                    getRegisterEnum(ARM::DPRRegClassID,
+                                    decodeVFPRm(insn, false))));
+    ++OpIdx;
+  }
+  return true;
+}
+
+// VMOVSR - A8.6.330
+// Rt => Rd; Sn => UInt(Vn:N)
+static bool DisassembleVFPConv4Frm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(NumOps >= 2 && "VFPConv4Frm expects NumOps >= 2");
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::SPRRegClassID,
+                                                     decodeVFPRn(insn, true))));
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  NumOpsAdded = 2;
+  return true;
+}
+
+// VMOVDRR - A8.6.332
+// Rt => Rd; Rt2 => Rn; Dm => UInt(M:Vm)
+//
+// VMOVRRS - A8.6.331
+// Rt => Rd; Rt2 => Rn; Sm => UInt(Vm:M); Sm1 = Sm+1
+static bool DisassembleVFPConv5Frm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(NumOps >= 3 && "VFPConv5Frm expects NumOps >= 3");
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  if (OpInfo[OpIdx].RegClass == ARM::SPRRegClassID) {
+    unsigned Sm = decodeVFPRm(insn, true);
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::SPRRegClassID,
+                                                       Sm)));
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::SPRRegClassID,
+                                                       Sm+1)));
+    OpIdx += 2;
+  } else {
+    MI.addOperand(MCOperand::CreateReg(
+                    getRegisterEnum(ARM::DPRRegClassID,
+                                    decodeVFPRm(insn, false))));
+    ++OpIdx;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+  OpIdx += 2;
+  return true;
+}
+
+// VFP Load/Store Instructions.
+// VLDRD, VLDRS, VSTRD, VSTRS
+static bool DisassembleVFPLdStFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(NumOps >= 3 && "VFPLdStFrm expects NumOps >= 3");
+
+  bool isSPVFP = (Opcode == ARM::VLDRS || Opcode == ARM::VSTRS) ? true : false;
+  unsigned RegClassID = isSPVFP ? ARM::SPRRegClassID : ARM::DPRRegClassID;
+
+  // Extract Dd/Sd for operand 0.
+  unsigned RegD = decodeVFPRd(insn, isSPVFP);
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(RegClassID, RegD)));
+
+  unsigned Base = getRegisterEnum(ARM::GPRRegClassID, decodeRn(insn));
+  MI.addOperand(MCOperand::CreateReg(Base));
+
+  // Next comes the AM5 Opcode.
+  ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
+  unsigned char Imm8 = insn & 0xFF;
+  MI.addOperand(MCOperand::CreateImm(ARM_AM::getAM5Opc(AddrOpcode, Imm8)));
+
+  NumOpsAdded = 3;
+
+  return true;
+}
+
+// VFP Load/Store Multiple Instructions.
+// This is similar to the algorithm for LDM/STM in that operand 0 (the base) and
+// operand 1 (the AM5 mode imm) is followed by two predicate operands.  It is
+// followed by a reglist of either DPR(s) or SPR(s).
+//
+// VLDMD[_UPD], VLDMS[_UPD], VSTMD[_UPD], VSTMS[_UPD]
+static bool DisassembleVFPLdStMulFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(NumOps >= 5 && "VFPLdStMulFrm expects NumOps >= 5");
+
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  unsigned Base = getRegisterEnum(ARM::GPRRegClassID, decodeRn(insn));
+
+  // Writeback to base, if necessary.
+  if (Opcode == ARM::VLDMD_UPD || Opcode == ARM::VLDMS_UPD ||
+      Opcode == ARM::VSTMD_UPD || Opcode == ARM::VSTMS_UPD) {
+    MI.addOperand(MCOperand::CreateReg(Base));
+    ++OpIdx;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(Base));
+
+  // Next comes the AM5 Opcode.
+  ARM_AM::AMSubMode SubMode = getAMSubModeForBits(getPUBits(insn));
+  unsigned char Imm8 = insn & 0xFF;
+  MI.addOperand(MCOperand::CreateImm(ARM_AM::getAM5Opc(SubMode, Imm8)));
+
+  // Handling the two predicate operands before the reglist.
+  int64_t CondVal = insn >> ARMII::CondShift;
+  MI.addOperand(MCOperand::CreateImm(CondVal == 0xF ? 0xE : CondVal));
+  MI.addOperand(MCOperand::CreateReg(ARM::CPSR));
+
+  OpIdx += 4;
+
+  bool isSPVFP = (Opcode == ARM::VLDMS || Opcode == ARM::VLDMS_UPD ||
+     Opcode == ARM::VSTMS || Opcode == ARM::VSTMS_UPD) ? true : false;
+  unsigned RegClassID = isSPVFP ? ARM::SPRRegClassID : ARM::DPRRegClassID;
+
+  // Extract Dd/Sd.
+  unsigned RegD = decodeVFPRd(insn, isSPVFP);
+
+  // Fill the variadic part of reglist.
+  unsigned Regs = isSPVFP ? Imm8 : Imm8/2;
+  for (unsigned i = 0; i < Regs; ++i) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(RegClassID,
+                                                       RegD + i)));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+// Misc. VFP Instructions.
+// FMSTAT (vmrs with Rt=0b1111, i.e., to apsr_nzcv and no register operand)
+// FCONSTD (DPR and a VFPf64Imm operand)
+// FCONSTS (SPR and a VFPf32Imm operand)
+// VMRS/VMSR (GPR operand)
+static bool DisassembleVFPMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  if (Opcode == ARM::FMSTAT)
+    return true;
+
+  assert(NumOps >= 2 && "VFPMiscFrm expects >=2 operands");
+
+  unsigned RegEnum = 0;
+  switch (OpInfo[0].RegClass) {
+  case ARM::DPRRegClassID:
+    RegEnum = getRegisterEnum(ARM::DPRRegClassID, decodeVFPRd(insn, false));
+    break;
+  case ARM::SPRRegClassID:
+    RegEnum = getRegisterEnum(ARM::SPRRegClassID, decodeVFPRd(insn, true));
+    break;
+  case ARM::GPRRegClassID:
+    RegEnum = getRegisterEnum(ARM::GPRRegClassID, decodeRd(insn));
+    break;
+  default:
+    assert(0 && "Invalid reg class id");
+    return false;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(RegEnum));
+  ++OpIdx;
+
+  // Extract/decode the f64/f32 immediate.
+  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == 0
+        && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
+    // The asm syntax specifies the before-expanded <imm>.
+    // Not VFPExpandImm(slice(insn,19,16) << 4 | slice(insn, 3, 0),
+    //                  Opcode == ARM::FCONSTD ? 64 : 32)
+    MI.addOperand(MCOperand::CreateImm(slice(insn,19,16)<<4 | slice(insn,3,0)));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+// DisassembleThumbFrm() is defined in ThumbDisassemblerCore.h file.
+#include "ThumbDisassemblerCore.h"
+
+/////////////////////////////////////////////////////
+//                                                 //
+//     Utility Functions For ARM Advanced SIMD     //
+//                                                 //
+/////////////////////////////////////////////////////
+
+// The following NEON namings are based on A8.6.266 VABA, VABAL.  Notice that
+// A8.6.303 VDUP (ARM core register)'s D/Vd pair is the N/Vn pair of VABA/VABAL.
+
+// A7.3 Register encoding
+
+// Extract/Decode NEON D/Vd:
+//
+// Note that for quadword, Qd = UInt(D:Vd<3:1>) = Inst{22:15-13}, whereas for
+// doubleword, Dd = UInt(D:Vd).  We compensate for this difference by
+// handling it in the getRegisterEnum() utility function.
+// D = Inst{22}, Vd = Inst{15-12}
+static unsigned decodeNEONRd(uint32_t insn) {
+  return ((insn >> ARMII::NEON_D_BitShift) & 1) << 4
+    | (insn >> ARMII::NEON_RegRdShift) & ARMII::NEONRegMask;
+}
+
+// Extract/Decode NEON N/Vn:
+//
+// Note that for quadword, Qn = UInt(N:Vn<3:1>) = Inst{7:19-17}, whereas for
+// doubleword, Dn = UInt(N:Vn).  We compensate for this difference by
+// handling it in the getRegisterEnum() utility function.
+// N = Inst{7}, Vn = Inst{19-16}
+static unsigned decodeNEONRn(uint32_t insn) {
+  return ((insn >> ARMII::NEON_N_BitShift) & 1) << 4
+    | (insn >> ARMII::NEON_RegRnShift) & ARMII::NEONRegMask;
+}
+
+// Extract/Decode NEON M/Vm:
+//
+// Note that for quadword, Qm = UInt(M:Vm<3:1>) = Inst{5:3-1}, whereas for
+// doubleword, Dm = UInt(M:Vm).  We compensate for this difference by
+// handling it in the getRegisterEnum() utility function.
+// M = Inst{5}, Vm = Inst{3-0}
+static unsigned decodeNEONRm(uint32_t insn) {
+  return ((insn >> ARMII::NEON_M_BitShift) & 1) << 4
+    | (insn >> ARMII::NEON_RegRmShift) & ARMII::NEONRegMask;
+}
+
+namespace {
+enum ElemSize {
+  ESizeNA = 0,
+  ESize8 = 8,
+  ESize16 = 16,
+  ESize32 = 32,
+  ESize64 = 64
+};
+} // End of unnamed namespace
+
+// size        field -> Inst{11-10}
+// index_align field -> Inst{7-4}
+//
+// The Lane Index interpretation depends on the Data Size:
+//   8  (encoded as size = 0b00) -> Index = index_align[3:1]
+//   16 (encoded as size = 0b01) -> Index = index_align[3:2]
+//   32 (encoded as size = 0b10) -> Index = index_align[3]
+//
+// Ref: A8.6.317 VLD4 (single 4-element structure to one lane).
+static unsigned decodeLaneIndex(uint32_t insn) {
+  unsigned size = insn >> 10 & 3;
+  assert((size == 0 || size == 1 || size == 2) &&
+         "Encoding error: size should be either 0, 1, or 2");
+
+  unsigned index_align = insn >> 4 & 0xF;
+  return (index_align >> 1) >> size;
+}
+
+// imm64 = AdvSIMDExpandImm(op, cmode, i:imm3:imm4)
+// op = Inst{5}, cmode = Inst{11-8}
+// i = Inst{24} (ARM architecture)
+// imm3 = Inst{18-16}, imm4 = Inst{3-0}
+// Ref: Table A7-15 Modified immediate values for Advanced SIMD instructions.
+static uint64_t decodeN1VImm(uint32_t insn, ElemSize esize) {
+  unsigned char cmode = (insn >> 8) & 0xF;
+  unsigned char Imm8 = ((insn >> 24) & 1) << 7 |
+                       ((insn >> 16) & 7) << 4 |
+                       (insn & 0xF);
+  uint64_t Imm64 = 0;
+
+  switch (esize) {
+  case ESize8:
+    Imm64 = Imm8;
+    break;
+  case ESize16:
+    Imm64 = Imm8 << 8*(cmode >> 1 & 1);
+    break;
+  case ESize32: {
+    if (cmode == 12)
+      Imm64 = (Imm8 << 8) | 0xFF;
+    else if (cmode == 13)
+      Imm64 = (Imm8 << 16) | 0xFFFF;
+    else {
+      // Imm8 to be shifted left by how many bytes...
+      Imm64 = Imm8 << 8*(cmode >> 1 & 3);
+    }
+    break;
+  }
+  case ESize64: {
+    for (unsigned i = 0; i < 8; ++i)
+      if ((Imm8 >> i) & 1)
+        Imm64 |= 0xFF << 8*i;
+    break;
+  }
+  default:
+    assert(0 && "Unreachable code!");
+    return 0;
+  }
+
+  return Imm64;
+}
+
+// A8.6.339 VMUL, VMULL (by scalar)
+// ESize16 => m = Inst{2-0} (Vm<2:0>) D0-D7
+// ESize32 => m = Inst{3-0} (Vm<3:0>) D0-D15
+static unsigned decodeRestrictedDm(uint32_t insn, ElemSize esize) {
+  switch (esize) {
+  case ESize16:
+    return insn & 7;
+  case ESize32:
+    return insn & 0xF;
+  default:
+    assert(0 && "Unreachable code!");
+    return 0;
+  }
+}
+
+// A8.6.339 VMUL, VMULL (by scalar)
+// ESize16 => index = Inst{5:3} (M:Vm<3>) D0-D7
+// ESize32 => index = Inst{5}   (M)       D0-D15
+static unsigned decodeRestrictedDmIndex(uint32_t insn, ElemSize esize) {
+  switch (esize) {
+  case ESize16:
+    return (((insn >> 5) & 1) << 1) | ((insn >> 3) & 1);
+  case ESize32:
+    return (insn >> 5) & 1;
+  default:
+    assert(0 && "Unreachable code!");
+    return 0;
+  }
+}
+
+// A8.6.296 VCVT (between floating-point and fixed-point, Advanced SIMD)
+// (64 - <fbits>) is encoded as imm6, i.e., Inst{21-16}.
+static unsigned decodeVCVTFractionBits(uint32_t insn) {
+  return 64 - ((insn >> 16) & 0x3F);
+}
+
+// A8.6.302 VDUP (scalar)
+// ESize8  => index = Inst{19-17}
+// ESize16 => index = Inst{19-18}
+// ESize32 => index = Inst{19}
+static unsigned decodeNVLaneDupIndex(uint32_t insn, ElemSize esize) {
+  switch (esize) {
+  case ESize8:
+    return (insn >> 17) & 7;
+  case ESize16:
+    return (insn >> 18) & 3;
+  case ESize32:
+    return (insn >> 19) & 1;
+  default:
+    assert(0 && "Unspecified element size!");
+    return 0;
+  }
+}
+
+// A8.6.328 VMOV (ARM core register to scalar)
+// A8.6.329 VMOV (scalar to ARM core register)
+// ESize8  => index = Inst{21:6-5}
+// ESize16 => index = Inst{21:6}
+// ESize32 => index = Inst{21}
+static unsigned decodeNVLaneOpIndex(uint32_t insn, ElemSize esize) {
+  switch (esize) {
+  case ESize8:
+    return ((insn >> 21) & 1) << 2 | ((insn >> 5) & 3);
+  case ESize16:
+    return ((insn >> 21) & 1) << 1 | ((insn >> 6) & 1);
+  case ESize32:
+    return ((insn >> 21) & 1);
+  default:
+    assert(0 && "Unspecified element size!");
+    return 0;
+  }
+}
+
+// Imm6 = Inst{21-16}, L = Inst{7}
+//
+// LeftShift == true (A8.6.367 VQSHL, A8.6.387 VSLI):
+// case L:imm6 of
+//   '0001xxx' => esize = 8; shift_amount = imm6 - 8
+//   '001xxxx' => esize = 16; shift_amount = imm6 - 16
+//   '01xxxxx' => esize = 32; shift_amount = imm6 - 32
+//   '1xxxxxx' => esize = 64; shift_amount = imm6
+//
+// LeftShift == false (A8.6.376 VRSHR, A8.6.368 VQSHRN):
+// case L:imm6 of
+//   '0001xxx' => esize = 8; shift_amount = 16 - imm6
+//   '001xxxx' => esize = 16; shift_amount = 32 - imm6
+//   '01xxxxx' => esize = 32; shift_amount = 64 - imm6
+//   '1xxxxxx' => esize = 64; shift_amount = 64 - imm6
+//
+static unsigned decodeNVSAmt(uint32_t insn, bool LeftShift) {
+  ElemSize esize = ESizeNA;
+  unsigned L = (insn >> 7) & 1;
+  unsigned imm6 = (insn >> 16) & 0x3F;
+  if (L == 0) {
+    if (imm6 >> 3 == 1)
+      esize = ESize8;
+    else if (imm6 >> 4 == 1)
+      esize = ESize16;
+    else if (imm6 >> 5 == 1)
+      esize = ESize32;
+    else
+      assert(0 && "Wrong encoding of Inst{7:21-16}!");
+  } else
+    esize = ESize64;
+
+  if (LeftShift)
+    return esize == ESize64 ? imm6 : (imm6 - esize);
+  else
+    return esize == ESize64 ? (esize - imm6) : (2*esize - imm6);
+}
+
+// A8.6.305 VEXT
+// Imm4 = Inst{11-8}
+static unsigned decodeN3VImm(uint32_t insn) {
+  return (insn >> 8) & 0xF;
+}
+
+// VLD*
+//   D[d] D[d2] ... Rn [TIED_TO Rn] align [Rm]
+// VLD*LN*
+//   D[d] D[d2] ... Rn [TIED_TO Rn] align [Rm] TIED_TO ... imm(idx)
+// VST*
+//   Rn [TIED_TO Rn] align [Rm] D[d] D[d2] ...
+// VST*LN*
+//   Rn [TIED_TO Rn] align [Rm] D[d] D[d2] ... [imm(idx)]
+//
+// Correctly set VLD*/VST*'s TIED_TO GPR, as the asm printer needs it.
+static bool DisassembleNLdSt0(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, bool Store, bool DblSpaced) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+
+  // At least one DPR register plus addressing mode #6.
+  assert(NumOps >= 3 && "Expect >= 3 operands");
+
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  // We have homogeneous NEON registers for Load/Store.
+  unsigned RegClass = 0;
+
+  // Double-spaced registers have increments of 2.
+  unsigned Inc = DblSpaced ? 2 : 1;
+
+  unsigned Rn = decodeRn(insn);
+  unsigned Rm = decodeRm(insn);
+  unsigned Rd = decodeNEONRd(insn);
+
+  // A7.7.1 Advanced SIMD addressing mode.
+  bool WB = Rm != 15;
+
+  // LLVM Addressing Mode #6.
+  unsigned RmEnum = 0;
+  if (WB && Rm != 13)
+    RmEnum = getRegisterEnum(ARM::GPRRegClassID, Rm);
+
+  if (Store) {
+    // Consume possible WB, AddrMode6, possible increment reg, the DPR/QPR's,
+    // then possible lane index.
+    assert(OpIdx < NumOps && OpInfo[0].RegClass == ARM::GPRRegClassID &&
+           "Reg operand expected");
+
+    if (WB) {
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                         Rn)));
+      ++OpIdx;
+    }
+
+    assert((OpIdx+1) < NumOps && OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+           OpInfo[OpIdx + 1].RegClass == 0 && "Addrmode #6 Operands expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       Rn)));
+    MI.addOperand(MCOperand::CreateImm(0)); // Alignment ignored?
+    OpIdx += 2;
+
+    if (WB) {
+      MI.addOperand(MCOperand::CreateReg(RmEnum));
+      ++OpIdx;
+    }
+
+    assert(OpIdx < NumOps &&
+           (OpInfo[OpIdx].RegClass == ARM::DPRRegClassID ||
+            OpInfo[OpIdx].RegClass == ARM::QPRRegClassID) &&
+           "Reg operand expected");
+
+    RegClass = OpInfo[OpIdx].RegClass;
+    while (OpIdx < NumOps && OpInfo[OpIdx].RegClass == RegClass) {
+      if (Opcode >= ARM::VST1q16 && Opcode <= ARM::VST1q8)
+        MI.addOperand(MCOperand::CreateReg(getRegisterEnum(RegClass,Rd,true)));
+      else
+        MI.addOperand(MCOperand::CreateReg(getRegisterEnum(RegClass,Rd)));
+      Rd += Inc;
+      ++OpIdx;
+    }
+
+    // Handle possible lane index.
+    if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == 0
+        && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
+      MI.addOperand(MCOperand::CreateImm(decodeLaneIndex(insn)));
+      ++OpIdx;
+    }
+
+  } else {
+    // Consume the DPR/QPR's, possible WB, AddrMode6, possible incrment reg,
+    // possible TIED_TO DPR/QPR's (ignored), then possible lane index.
+    RegClass = OpInfo[0].RegClass;
+
+    while (OpIdx < NumOps && OpInfo[OpIdx].RegClass == RegClass) {
+      if (Opcode >= ARM::VLD1q16 && Opcode <= ARM::VLD1q8)
+        MI.addOperand(MCOperand::CreateReg(getRegisterEnum(RegClass,Rd,true)));
+      else
+        MI.addOperand(MCOperand::CreateReg(getRegisterEnum(RegClass,Rd)));
+      Rd += Inc;
+      ++OpIdx;
+    }
+
+    if (WB) {
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                         Rn)));
+      ++OpIdx;
+    }
+
+    assert((OpIdx+1) < NumOps && OpInfo[OpIdx].RegClass == ARM::GPRRegClassID &&
+           OpInfo[OpIdx + 1].RegClass == 0 && "Addrmode #6 Operands expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       Rn)));
+    MI.addOperand(MCOperand::CreateImm(0)); // Alignment ignored?
+    OpIdx += 2;
+
+    if (WB) {
+      MI.addOperand(MCOperand::CreateReg(RmEnum));
+      ++OpIdx;
+    }
+
+    while (OpIdx < NumOps && OpInfo[OpIdx].RegClass == RegClass) {
+      assert(TID.getOperandConstraint(OpIdx, TOI::TIED_TO) != -1 &&
+             "Tied to operand expected");
+      MI.addOperand(MCOperand::CreateReg(0));
+      ++OpIdx;
+    }
+
+    // Handle possible lane index.
+    if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == 0
+        && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
+      MI.addOperand(MCOperand::CreateImm(decodeLaneIndex(insn)));
+      ++OpIdx;
+    }
+  }
+
+  return true;
+}
+
+// A7.7
+// If L (Inst{21}) == 0, store instructions.
+// Find out about double-spaced-ness of the Opcode and pass it on to
+// DisassembleNLdSt0().
+static bool DisassembleNLdSt(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const StringRef Name = ARMInsts[Opcode].Name;
+  bool DblSpaced = false;
+
+  if (Name.find("LN") != std::string::npos) {
+    // To one lane instructions.
+    // See, for example, 8.6.317 VLD4 (single 4-element structure to one lane).
+
+    // <size> == 16 && Inst{5} == 1 --> DblSpaced = true
+    if (Name.endswith("16") || Name.endswith("16_UPD"))
+      DblSpaced = slice(insn, 5, 5) == 1;
+
+    // <size> == 32 && Inst{6} == 1 --> DblSpaced = true
+    if (Name.endswith("32") || Name.endswith("32_UPD"))
+      DblSpaced = slice(insn, 6, 6) == 1;
+
+  } else {
+    // Multiple n-element structures with type encoded as Inst{11-8}.
+    // See, for example, A8.6.316 VLD4 (multiple 4-element structures).
+
+    // n == 2 && type == 0b1001 -> DblSpaced = true
+    if (Name.startswith("VST2") || Name.startswith("VLD2"))
+      DblSpaced = slice(insn, 11, 8) == 9;
+    
+    // n == 3 && type == 0b0101 -> DblSpaced = true
+    if (Name.startswith("VST3") || Name.startswith("VLD3"))
+      DblSpaced = slice(insn, 11, 8) == 5;
+    
+    // n == 4 && type == 0b0001 -> DblSpaced = true
+    if (Name.startswith("VST4") || Name.startswith("VLD4"))
+      DblSpaced = slice(insn, 11, 8) == 1;
+    
+  }
+  return DisassembleNLdSt0(MI, Opcode, insn, NumOps, NumOpsAdded,
+                           slice(insn, 21, 21) == 0, DblSpaced);
+}
+
+// VMOV (immediate)
+//   Qd/Dd imm
+static bool DisassembleN1RegModImmFrm(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+
+  assert(NumOps >= 2 &&
+         (OpInfo[0].RegClass == ARM::DPRRegClassID ||
+          OpInfo[0].RegClass == ARM::QPRRegClassID) &&
+         (OpInfo[1].RegClass == 0) &&
+         "Expect 1 reg operand followed by 1 imm operand");
+
+  // Qd/Dd = Inst{22:15-12} => NEON Rd
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(OpInfo[0].RegClass,
+                                                     decodeNEONRd(insn))));
+
+  ElemSize esize = ESizeNA;
+  switch (Opcode) {
+  case ARM::VMOVv8i8:
+  case ARM::VMOVv16i8:
+    esize = ESize8;
+    break;
+  case ARM::VMOVv4i16:
+  case ARM::VMOVv8i16:
+    esize = ESize16;
+    break;
+  case ARM::VMOVv2i32:
+  case ARM::VMOVv4i32:
+    esize = ESize32;
+    break;
+  case ARM::VMOVv1i64:
+  case ARM::VMOVv2i64:
+    esize = ESize64;
+  default:
+    assert(0 && "Unreachable code!");
+    return false;
+  }
+
+  // One register and a modified immediate value.
+  // Add the imm operand.
+  MI.addOperand(MCOperand::CreateImm(decodeN1VImm(insn, esize)));
+
+  NumOpsAdded = 2;
+  return true;
+}
+
+namespace {
+enum N2VFlag {
+  N2V_None,
+  N2V_VectorDupLane,
+  N2V_VectorConvert_Between_Float_Fixed
+};
+} // End of unnamed namespace
+
+// Vector Convert [between floating-point and fixed-point]
+//   Qd/Dd Qm/Dm [fbits]
+//
+// Vector Duplicate Lane (from scalar to all elements) Instructions.
+// VDUPLN16d, VDUPLN16q, VDUPLN32d, VDUPLN32q, VDUPLN8d, VDUPLN8q:
+//   Qd/Dd Dm index
+//
+// Vector Move Long:
+//   Qd Dm
+// 
+// Vector Move Narrow:
+//   Dd Qm
+//
+// Others
+static bool DisassembleNVdVmOptImm(MCInst &MI, unsigned Opc, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, N2VFlag Flag = N2V_None) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opc];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+
+  assert(NumOps >= 2 &&
+         (OpInfo[0].RegClass == ARM::DPRRegClassID ||
+          OpInfo[0].RegClass == ARM::QPRRegClassID) &&
+         (OpInfo[1].RegClass == ARM::DPRRegClassID ||
+          OpInfo[1].RegClass == ARM::QPRRegClassID) &&
+         "Expect >= 2 operands and first 2 as reg operands");
+
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  ElemSize esize = ESizeNA;
+  if (Flag == N2V_VectorDupLane) {
+    // VDUPLN has its index embedded.  Its size can be inferred from the Opcode.
+    assert(Opc >= ARM::VDUPLN16d && Opc <= ARM::VDUPLN8q &&
+           "Unexpected Opcode");
+    esize = (Opc == ARM::VDUPLN8d || Opc == ARM::VDUPLN8q) ? ESize8
+       : ((Opc == ARM::VDUPLN16d || Opc == ARM::VDUPLN16q) ? ESize16
+                                                           : ESize32);
+  }
+
+  // Qd/Dd = Inst{22:15-12} => NEON Rd
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(OpInfo[OpIdx].RegClass,
+                                                     decodeNEONRd(insn))));
+  ++OpIdx;
+
+  // VPADAL...
+  if (TID.getOperandConstraint(OpIdx, TOI::TIED_TO) != -1) {
+    // TIED_TO operand.
+    MI.addOperand(MCOperand::CreateReg(0));
+    ++OpIdx;
+  }
+
+  // Dm = Inst{5:3-0} => NEON Rm
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(OpInfo[OpIdx].RegClass,
+                                                     decodeNEONRm(insn))));
+  ++OpIdx;
+
+  // VZIP and others have two TIED_TO reg operands.
+  int Idx;
+  while (OpIdx < NumOps &&
+         (Idx = TID.getOperandConstraint(OpIdx, TOI::TIED_TO)) != -1) {
+    // Add TIED_TO operand.
+    MI.addOperand(MI.getOperand(Idx));
+    ++OpIdx;
+  }
+
+  // Add the imm operand, if required.
+  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == 0
+      && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
+
+    unsigned imm = 0xFFFFFFFF;
+
+    if (Flag == N2V_VectorDupLane)
+      imm = decodeNVLaneDupIndex(insn, esize);
+    if (Flag == N2V_VectorConvert_Between_Float_Fixed)
+      imm = decodeVCVTFractionBits(insn);
+
+    assert(imm != 0xFFFFFFFF && "Internal error");
+    MI.addOperand(MCOperand::CreateImm(imm));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+static bool DisassembleN2RegFrm(MCInst &MI, unsigned Opc, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  return DisassembleNVdVmOptImm(MI, Opc, insn, NumOps, NumOpsAdded);
+}
+static bool DisassembleNVCVTFrm(MCInst &MI, unsigned Opc, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  return DisassembleNVdVmOptImm(MI, Opc, insn, NumOps, NumOpsAdded,
+                                N2V_VectorConvert_Between_Float_Fixed);
+}
+static bool DisassembleNVecDupLnFrm(MCInst &MI, unsigned Opc, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  return DisassembleNVdVmOptImm(MI, Opc, insn, NumOps, NumOpsAdded,
+                                N2V_VectorDupLane);
+}
+
+// Vector Shift [Accumulate] Instructions.
+// Qd/Dd [Qd/Dd (TIED_TO)] Qm/Dm ShiftAmt
+//
+// Vector Shift Left Long (with maximum shift count) Instructions.
+// VSHLLi16, VSHLLi32, VSHLLi8: Qd Dm imm (== size)
+//
+static bool DisassembleNVectorShift(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, bool LeftShift) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+
+  assert(NumOps >= 3 &&
+         (OpInfo[0].RegClass == ARM::DPRRegClassID ||
+          OpInfo[0].RegClass == ARM::QPRRegClassID) &&
+         (OpInfo[1].RegClass == ARM::DPRRegClassID ||
+          OpInfo[1].RegClass == ARM::QPRRegClassID) &&
+         "Expect >= 3 operands and first 2 as reg operands");
+
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  // Qd/Dd = Inst{22:15-12} => NEON Rd
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(OpInfo[OpIdx].RegClass,
+                                                     decodeNEONRd(insn))));
+  ++OpIdx;
+
+  if (TID.getOperandConstraint(OpIdx, TOI::TIED_TO) != -1) {
+    // TIED_TO operand.
+    MI.addOperand(MCOperand::CreateReg(0));
+    ++OpIdx;
+  }
+
+  assert((OpInfo[OpIdx].RegClass == ARM::DPRRegClassID ||
+          OpInfo[OpIdx].RegClass == ARM::QPRRegClassID) &&
+         "Reg operand expected");
+
+  // Qm/Dm = Inst{5:3-0} => NEON Rm
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(OpInfo[OpIdx].RegClass,
+                                                     decodeNEONRm(insn))));
+  ++OpIdx;
+
+  assert(OpInfo[OpIdx].RegClass == 0 && "Imm operand expected");
+
+  // Add the imm operand.
+  
+  // VSHLL has maximum shift count as the imm, inferred from its size.
+  unsigned Imm;
+  switch (Opcode) {
+  default:
+    Imm = decodeNVSAmt(insn, LeftShift);
+    break;
+  case ARM::VSHLLi8:
+    Imm = 8;
+    break;
+  case ARM::VSHLLi16:
+    Imm = 16;
+    break;
+  case ARM::VSHLLi32:
+    Imm = 32;
+    break;
+  }
+  MI.addOperand(MCOperand::CreateImm(Imm));
+  ++OpIdx;
+
+  return true;
+}
+
+// Left shift instructions.
+static bool DisassembleN2RegVecShLFrm(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  return DisassembleNVectorShift(MI, Opcode, insn, NumOps, NumOpsAdded, true);
+}
+// Right shift instructions have different shift amount interpretation.
+static bool DisassembleN2RegVecShRFrm(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  return DisassembleNVectorShift(MI, Opcode, insn, NumOps, NumOpsAdded, false);
+}
+
+namespace {
+enum N3VFlag {
+  N3V_None,
+  N3V_VectorExtract,
+  N3V_VectorShift,
+  N3V_Multiply_By_Scalar
+};
+} // End of unnamed namespace
+
+// NEON Three Register Instructions with Optional Immediate Operand
+//
+// Vector Extract Instructions.
+// Qd/Dd Qn/Dn Qm/Dm imm4
+//
+// Vector Shift (Register) Instructions.
+// Qd/Dd Qm/Dm Qn/Dn (notice the order of m, n)
+//
+// Vector Multiply [Accumulate/Subtract] [Long] By Scalar Instructions.
+// Qd/Dd Qn/Dn RestrictedDm index
+//
+// Others
+static bool DisassembleNVdVnVmOptImm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, N3VFlag Flag = N3V_None) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+
+  // No checking for OpInfo[2] because of MOVDneon/MOVQ with only two regs.
+  assert(NumOps >= 3 &&
+         (OpInfo[0].RegClass == ARM::DPRRegClassID ||
+          OpInfo[0].RegClass == ARM::QPRRegClassID) &&
+         (OpInfo[1].RegClass == ARM::DPRRegClassID ||
+          OpInfo[1].RegClass == ARM::QPRRegClassID) &&
+         "Expect >= 3 operands and first 2 as reg operands");
+
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  bool VdVnVm = Flag == N3V_VectorShift ? false : true;
+  bool IsImm4 = Flag == N3V_VectorExtract ? true : false;
+  bool IsDmRestricted = Flag == N3V_Multiply_By_Scalar ? true : false;
+  ElemSize esize = ESizeNA;
+  if (Flag == N3V_Multiply_By_Scalar) {
+    unsigned size = (insn >> 20) & 3;
+    if (size == 1) esize = ESize16;
+    if (size == 2) esize = ESize32;
+    assert (esize == ESize16 || esize == ESize32);
+  }
+
+  // Qd/Dd = Inst{22:15-12} => NEON Rd
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(OpInfo[OpIdx].RegClass,
+                                                     decodeNEONRd(insn))));
+  ++OpIdx;
+
+  // VABA, VABAL, VBSLd, VBSLq, ...
+  if (TID.getOperandConstraint(OpIdx, TOI::TIED_TO) != -1) {
+    // TIED_TO operand.
+    MI.addOperand(MCOperand::CreateReg(0));
+    ++OpIdx;
+  }
+
+  // Dn = Inst{7:19-16} => NEON Rn
+  // or
+  // Dm = Inst{5:3-0} => NEON Rm
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(OpInfo[OpIdx].RegClass,
+                                  VdVnVm ? decodeNEONRn(insn)
+                                         : decodeNEONRm(insn))));
+  ++OpIdx;
+
+  // Special case handling for VMOVDneon and VMOVQ because they are marked as
+  // N3RegFrm.
+  if (Opcode == ARM::VMOVDneon || Opcode == ARM::VMOVQ)
+    return true;
+  
+  // Dm = Inst{5:3-0} => NEON Rm
+  // or
+  // Dm is restricted to D0-D7 if size is 16, D0-D15 otherwise
+  // or
+  // Dn = Inst{7:19-16} => NEON Rn
+  unsigned m = VdVnVm ? (IsDmRestricted ? decodeRestrictedDm(insn, esize)
+                                        : decodeNEONRm(insn))
+                      : decodeNEONRn(insn);
+
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(OpInfo[OpIdx].RegClass, m)));
+  ++OpIdx;
+
+  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == 0
+      && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
+    // Add the imm operand.
+    unsigned Imm = 0;
+    if (IsImm4)
+      Imm = decodeN3VImm(insn);
+    else if (IsDmRestricted)
+      Imm = decodeRestrictedDmIndex(insn, esize);
+    else {
+      assert(0 && "Internal error: unreachable code!");
+      return false;
+    }
+
+    MI.addOperand(MCOperand::CreateImm(Imm));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+static bool DisassembleN3RegFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  return DisassembleNVdVnVmOptImm(MI, Opcode, insn, NumOps, NumOpsAdded);
+}
+static bool DisassembleN3RegVecShFrm(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  return DisassembleNVdVnVmOptImm(MI, Opcode, insn, NumOps, NumOpsAdded,
+                                  N3V_VectorShift);
+}
+static bool DisassembleNVecExtractFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  return DisassembleNVdVnVmOptImm(MI, Opcode, insn, NumOps, NumOpsAdded,
+                                  N3V_VectorExtract);
+}
+static bool DisassembleNVecMulScalarFrm(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  return DisassembleNVdVnVmOptImm(MI, Opcode, insn, NumOps, NumOpsAdded,
+                                  N3V_Multiply_By_Scalar);
+}
+
+// Vector Table Lookup
+//
+// VTBL1, VTBX1: Dd [Dd(TIED_TO)] Dn Dm
+// VTBL2, VTBX2: Dd [Dd(TIED_TO)] Dn Dn+1 Dm
+// VTBL3, VTBX3: Dd [Dd(TIED_TO)] Dn Dn+1 Dn+2 Dm
+// VTBL4, VTBX4: Dd [Dd(TIED_TO)] Dn Dn+1 Dn+2 Dn+3 Dm
+static bool DisassembleNVTBLFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+
+  assert(NumOps >= 3 &&
+         OpInfo[0].RegClass == ARM::DPRRegClassID &&
+         OpInfo[1].RegClass == ARM::DPRRegClassID &&
+         OpInfo[2].RegClass == ARM::DPRRegClassID &&
+         "Expect >= 3 operands and first 3 as reg operands");
+
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  unsigned Rn = decodeNEONRn(insn);
+
+  // {Dn} encoded as len = 0b00
+  // {Dn Dn+1} encoded as len = 0b01
+  // {Dn Dn+1 Dn+2 } encoded as len = 0b10
+  // {Dn Dn+1 Dn+2 Dn+3} encoded as len = 0b11
+  unsigned Len = slice(insn, 9, 8) + 1;
+
+  // Dd (the destination vector)
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::DPRRegClassID,
+                                                     decodeNEONRd(insn))));
+  ++OpIdx;
+
+  // Process tied_to operand constraint.
+  int Idx;
+  if ((Idx = TID.getOperandConstraint(OpIdx, TOI::TIED_TO)) != -1) {
+    MI.addOperand(MI.getOperand(Idx));
+    ++OpIdx;
+  }
+
+  // Do the <list> now.
+  for (unsigned i = 0; i < Len; ++i) {
+    assert(OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::DPRRegClassID &&
+           "Reg operand expected");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::DPRRegClassID,
+                                                       Rn + i)));
+    ++OpIdx;
+  }
+
+  // Dm (the index vector)
+  assert(OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::DPRRegClassID &&
+         "Reg operand (index vector) expected");
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::DPRRegClassID,
+                                                     decodeNEONRm(insn))));
+  ++OpIdx;
+
+  return true;
+}
+
+static bool DisassembleNEONFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+  assert(0 && "Unreachable code!");
+  return false;
+}
+
+// Vector Get Lane (move scalar to ARM core register) Instructions.
+// VGETLNi32, VGETLNs16, VGETLNs8, VGETLNu16, VGETLNu8: Rt Dn index
+static bool DisassembleNEONGetLnFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  unsigned short NumDefs = TID.getNumDefs();
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+
+  assert(NumDefs == 1 && NumOps >= 3 &&
+         OpInfo[0].RegClass == ARM::GPRRegClassID &&
+         OpInfo[1].RegClass == ARM::DPRRegClassID &&
+         OpInfo[2].RegClass == 0 &&
+         "Expect >= 3 operands with one dst operand");
+
+  ElemSize esize =
+    Opcode == ARM::VGETLNi32 ? ESize32
+      : ((Opcode == ARM::VGETLNs16 || Opcode == ARM::VGETLNu16) ? ESize16
+                                                                : ESize32);
+
+  // Rt = Inst{15-12} => ARM Rd
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+
+  // Dn = Inst{7:19-16} => NEON Rn
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::DPRRegClassID,
+                                                     decodeNEONRn(insn))));
+
+  MI.addOperand(MCOperand::CreateImm(decodeNVLaneOpIndex(insn, esize)));
+
+  NumOpsAdded = 3;
+  return true;
+}
+
+// Vector Set Lane (move ARM core register to scalar) Instructions.
+// VSETLNi16, VSETLNi32, VSETLNi8: Dd Dd (TIED_TO) Rt index
+static bool DisassembleNEONSetLnFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  unsigned short NumDefs = TID.getNumDefs();
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+
+  assert(NumDefs == 1 && NumOps >= 3 &&
+         OpInfo[0].RegClass == ARM::DPRRegClassID &&
+         OpInfo[1].RegClass == ARM::DPRRegClassID &&
+         TID.getOperandConstraint(1, TOI::TIED_TO) != -1 &&
+         OpInfo[2].RegClass == ARM::GPRRegClassID &&
+         OpInfo[3].RegClass == 0 &&
+         "Expect >= 3 operands with one dst operand");
+
+  ElemSize esize =
+    Opcode == ARM::VSETLNi8 ? ESize8
+                            : (Opcode == ARM::VSETLNi16 ? ESize16
+                                                        : ESize32);
+
+  // Dd = Inst{7:19-16} => NEON Rn
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::DPRRegClassID,
+                                                     decodeNEONRn(insn))));
+
+  // TIED_TO operand.
+  MI.addOperand(MCOperand::CreateReg(0));
+
+  // Rt = Inst{15-12} => ARM Rd
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+
+  MI.addOperand(MCOperand::CreateImm(decodeNVLaneOpIndex(insn, esize)));
+
+  NumOpsAdded = 4;
+  return true;
+}
+
+// Vector Duplicate Instructions (from ARM core register to all elements).
+// VDUP8d, VDUP16d, VDUP32d, VDUP8q, VDUP16q, VDUP32q: Qd/Dd Rt
+static bool DisassembleNEONDupFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  assert(NumOps >= 2 &&
+         (OpInfo[0].RegClass == ARM::DPRRegClassID ||
+          OpInfo[0].RegClass == ARM::QPRRegClassID) &&
+         OpInfo[1].RegClass == ARM::GPRRegClassID &&
+         "Expect >= 2 operands and first 2 as reg operand");
+
+  unsigned RegClass = OpInfo[0].RegClass;
+
+  // Qd/Dd = Inst{7:19-16} => NEON Rn
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(RegClass,
+                                                     decodeNEONRn(insn))));
+
+  // Rt = Inst{15-12} => ARM Rd
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+
+  NumOpsAdded = 2;
+  return true;
+}
+
+// A8.6.41 DMB
+// A8.6.42 DSB
+// A8.6.49 ISB
+static inline bool MemBarrierInstr(uint32_t insn) {
+  unsigned op7_4 = slice(insn, 7, 4);
+  if (slice(insn, 31, 20) == 0xf57 && (op7_4 >= 4 && op7_4 <= 6))
+    return true;
+
+  return false;
+}
+
+static inline bool PreLoadOpcode(unsigned Opcode) {
+  switch(Opcode) {
+  case ARM::PLDi:  case ARM::PLDr:
+  case ARM::PLDWi: case ARM::PLDWr:
+  case ARM::PLIi:  case ARM::PLIr:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static bool DisassemblePreLoadFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  // Preload Data/Instruction requires either 2 or 4 operands.
+  // PLDi, PLDWi, PLIi:                Rn [+/-]imm12 add = (U == '1')
+  // PLDr[a|m], PLDWr[a|m], PLIr[a|m]: Rn Rm addrmode2_opc
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+
+  if (Opcode == ARM::PLDi || Opcode == ARM::PLDWi || Opcode == ARM::PLIi) {
+    unsigned Imm12 = slice(insn, 11, 0);
+    bool Negative = getUBit(insn) == 0;
+    int Offset = Negative ? -1 - Imm12 : 1 * Imm12;
+    MI.addOperand(MCOperand::CreateImm(Offset));
+    NumOpsAdded = 2;
+  } else {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+
+    ARM_AM::AddrOpc AddrOpcode = getUBit(insn) ? ARM_AM::add : ARM_AM::sub;
+
+    // Inst{6-5} encodes the shift opcode.
+    ARM_AM::ShiftOpc ShOp = getShiftOpcForBits(slice(insn, 6, 5));
+    // Inst{11-7} encodes the imm5 shift amount.
+    unsigned ShImm = slice(insn, 11, 7);
+
+    // A8.4.1.  Possible rrx or shift amount of 32...
+    getImmShiftSE(ShOp, ShImm);
+    MI.addOperand(MCOperand::CreateImm(
+                    ARM_AM::getAM2Opc(AddrOpcode, ShImm, ShOp)));
+    NumOpsAdded = 3;
+  }
+
+  return true;
+}
+
+static bool DisassembleMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  if (MemBarrierInstr(insn))
+    return true;
+
+  switch (Opcode) {
+  case ARM::CLREX:
+  case ARM::NOP:
+  case ARM::TRAP:
+  case ARM::YIELD:
+  case ARM::WFE:
+  case ARM::WFI:
+  case ARM::SEV:
+  case ARM::SETENDBE:
+  case ARM::SETENDLE:
+    return true;
+  default:
+    break;
+  }
+
+  // CPS has a singleton $opt operand that contains the following information:
+  // opt{4-0} = mode from Inst{4-0}
+  // opt{5} = changemode from Inst{17}
+  // opt{8-6} = AIF from Inst{8-6}
+  // opt{10-9} = imod from Inst{19-18} with 0b10 as enable and 0b11 as disable
+  if (Opcode == ARM::CPS) {
+    unsigned Option = slice(insn, 4, 0) | slice(insn, 17, 17) << 5 |
+      slice(insn, 8, 6) << 6 | slice(insn, 19, 18) << 9;
+    MI.addOperand(MCOperand::CreateImm(Option));
+    NumOpsAdded = 1;
+    return true;
+  }
+
+  // DBG has its option specified in Inst{3-0}.
+  if (Opcode == ARM::DBG) {
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 3, 0)));
+    NumOpsAdded = 1;
+    return true;
+  }
+
+  // BKPT takes an imm32 val equal to ZeroExtend(Inst{19-8:3-0}).
+  if (Opcode == ARM::BKPT) {
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 19, 8) << 4 |
+                                       slice(insn, 3, 0)));
+    NumOpsAdded = 1;
+    return true;
+  }
+
+  if (PreLoadOpcode(Opcode))
+    return DisassemblePreLoadFrm(MI, Opcode, insn, NumOps, NumOpsAdded);
+
+  assert(0 && "Unexpected misc instruction!");
+  return false;
+}
+
+static bool DisassembleThumbMiscFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO) {
+
+  assert(0 && "Unexpected thumb misc. instruction!");
+  return false;
+}
+
+/// FuncPtrs - FuncPtrs maps ARMFormat to its corresponding DisassembleFP.
+/// We divide the disassembly task into different categories, with each one
+/// corresponding to a specific instruction encoding format.  There could be
+/// exceptions when handling a specific format, and that is why the Opcode is
+/// also present in the function prototype.
+static const DisassembleFP FuncPtrs[] = {
+  &DisassemblePseudo,
+  &DisassembleMulFrm,
+  &DisassembleBrFrm,
+  &DisassembleBrMiscFrm,
+  &DisassembleDPFrm,
+  &DisassembleDPSoRegFrm,
+  &DisassembleLdFrm,
+  &DisassembleStFrm,
+  &DisassembleLdMiscFrm,
+  &DisassembleStMiscFrm,
+  &DisassembleLdStMulFrm,
+  &DisassembleLdStExFrm,
+  &DisassembleArithMiscFrm,
+  &DisassembleExtFrm,
+  &DisassembleVFPUnaryFrm,
+  &DisassembleVFPBinaryFrm,
+  &DisassembleVFPConv1Frm,
+  &DisassembleVFPConv2Frm,
+  &DisassembleVFPConv3Frm,
+  &DisassembleVFPConv4Frm,
+  &DisassembleVFPConv5Frm,
+  &DisassembleVFPLdStFrm,
+  &DisassembleVFPLdStMulFrm,
+  &DisassembleVFPMiscFrm,
+  &DisassembleThumbFrm,
+  &DisassembleNEONFrm,
+  &DisassembleNEONGetLnFrm,
+  &DisassembleNEONSetLnFrm,
+  &DisassembleNEONDupFrm,
+  &DisassembleMiscFrm,
+  &DisassembleThumbMiscFrm,
+
+  // VLD and VST (including one lane) Instructions.
+  &DisassembleNLdSt,
+
+  // A7.4.6 One register and a modified immediate value
+  // 1-Register Instructions with imm.
+  // LLVM only defines VMOVv instructions.
+  &DisassembleN1RegModImmFrm,
+
+  // 2-Register Instructions with no imm.
+  &DisassembleN2RegFrm,
+
+  // 2-Register Instructions with imm (vector convert float/fixed point).
+  &DisassembleNVCVTFrm,
+
+  // 2-Register Instructions with imm (vector dup lane).
+  &DisassembleNVecDupLnFrm,
+
+  // Vector Shift Left Instructions.
+  &DisassembleN2RegVecShLFrm,
+
+  // Vector Shift Righ Instructions, which has different interpretation of the
+  // shift amount from the imm6 field.
+  &DisassembleN2RegVecShRFrm,
+
+  // 3-Register Data-Processing Instructions.
+  &DisassembleN3RegFrm,
+
+  // Vector Shift (Register) Instructions.
+  // D:Vd M:Vm N:Vn (notice that M:Vm is the first operand)
+  &DisassembleN3RegVecShFrm,
+
+  // Vector Extract Instructions.
+  &DisassembleNVecExtractFrm,
+
+  // Vector [Saturating Rounding Doubling] Multiply [Accumulate/Subtract] [Long]
+  // By Scalar Instructions.
+  &DisassembleNVecMulScalarFrm,
+
+  // Vector Table Lookup uses byte indexes in a control vector to look up byte
+  // values in a table and generate a new vector.
+  &DisassembleNVTBLFrm,
+
+  NULL
+};
+
+/// BuildIt - BuildIt performs the build step for this ARM Basic MC Builder.
+/// The general idea is to set the Opcode for the MCInst, followed by adding
+/// the appropriate MCOperands to the MCInst.  ARM Basic MC Builder delegates
+/// to the Format-specific disassemble function for disassembly, followed by
+/// TryPredicateAndSBitModifier() to do PredicateOperand and OptionalDefOperand
+/// which follow the Dst/Src Operands.
+bool ARMBasicMCBuilder::BuildIt(MCInst &MI, uint32_t insn) {
+  // Stage 1 sets the Opcode.
+  MI.setOpcode(Opcode);
+  // If the number of operands is zero, we're done!
+  if (NumOps == 0)
+    return true;
+
+  // Stage 2 calls the format-specific disassemble function to build the operand
+  // list.
+  if (Disasm == NULL)
+    return false;
+  unsigned NumOpsAdded = 0;
+  bool OK = (*Disasm)(MI, Opcode, insn, NumOps, NumOpsAdded, this);
+
+  if (!OK) return false;
+  if (NumOpsAdded >= NumOps)
+    return true;
+
+  // Stage 3 deals with operands unaccounted for after stage 2 is finished.
+  // FIXME: Should this be done selectively?
+  return TryPredicateAndSBitModifier(MI, Opcode, insn, NumOps - NumOpsAdded);
+}
+
+bool ARMBasicMCBuilder::TryPredicateAndSBitModifier(MCInst& MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOpsRemaining) {
+
+  assert(NumOpsRemaining > 0 && "Invalid argument");
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  const std::string &Name = ARMInsts[Opcode].Name;
+  unsigned Idx = MI.getNumOperands();
+
+  // First, we check whether this instr specifies the PredicateOperand through
+  // a pair of TargetOperandInfos with isPredicate() property.
+  if (NumOpsRemaining >= 2 &&
+      OpInfo[Idx].isPredicate() && OpInfo[Idx+1].isPredicate() &&
+      OpInfo[Idx].RegClass == 0 && OpInfo[Idx+1].RegClass == ARM::CCRRegClassID)
+  {
+    // If we are inside an IT block, get the IT condition bits maintained via
+    // ARMBasicMCBuilder::ITState[7:0], through ARMBasicMCBuilder::GetITCond().
+    // See also A2.5.2.
+    if (InITBlock())
+      MI.addOperand(MCOperand::CreateImm(GetITCond()));
+    else {
+      if (Name.length() > 1 && Name[0] == 't') {
+        // Thumb conditional branch instructions have their cond field embedded,
+        // like ARM.
+        //
+        // A8.6.16 B
+        if (Name == "t2Bcc")
+          MI.addOperand(MCOperand::CreateImm(slice(insn, 25, 22)));
+        else if (Name == "tBcc")
+          MI.addOperand(MCOperand::CreateImm(slice(insn, 11, 8)));
+        else
+          MI.addOperand(MCOperand::CreateImm(ARMCC::AL));
+      } else {
+        // ARM Instructions.  Check condition field.
+        int64_t CondVal = getCondField(insn);
+        if (CondVal == 0xF)
+          MI.addOperand(MCOperand::CreateImm(ARMCC::AL));
+        else
+          MI.addOperand(MCOperand::CreateImm(CondVal));
+      }
+    }
+    MI.addOperand(MCOperand::CreateReg(ARM::CPSR));
+    Idx += 2;
+    NumOpsRemaining -= 2;
+    if (NumOpsRemaining == 0)
+      return true;
+  }
+
+  // Next, if OptionalDefOperand exists, we check whether the 'S' bit is set.
+  if (OpInfo[Idx].isOptionalDef() && OpInfo[Idx].RegClass==ARM::CCRRegClassID) {
+    MI.addOperand(MCOperand::CreateReg(getSBit(insn) == 1 ? ARM::CPSR : 0));
+    --NumOpsRemaining;
+  }
+
+  if (NumOpsRemaining == 0)
+    return true;
+  else
+    return false;
+}
+
+/// RunBuildAfterHook - RunBuildAfterHook performs operations deemed necessary
+/// after BuildIt is finished.
+bool ARMBasicMCBuilder::RunBuildAfterHook(bool Status, MCInst &MI,
+    uint32_t insn) {
+
+  if (!SP) return Status;
+
+  if (Opcode == ARM::t2IT)
+    SP->InitIT(slice(insn, 7, 0));
+  else if (InITBlock())
+    SP->UpdateIT();
+
+  return Status;
+}
+
+/// Opcode, Format, and NumOperands make up an ARM Basic MCBuilder.
+ARMBasicMCBuilder::ARMBasicMCBuilder(unsigned opc, ARMFormat format,
+                                     unsigned short num)
+  : Opcode(opc), Format(format), NumOps(num), SP(0) {
+  unsigned Idx = (unsigned)format;
+  assert(Idx < (array_lengthof(FuncPtrs) - 1) && "Unknown format");
+  Disasm = FuncPtrs[Idx];
+}
+
+/// CreateMCBuilder - Return an ARMBasicMCBuilder that can build up the MC
+/// infrastructure of an MCInst given the Opcode and Format of the instr.
+/// Return NULL if it fails to create/return a proper builder.  API clients
+/// are responsible for freeing up of the allocated memory.  Cacheing can be
+/// performed by the API clients to improve performance.
+ARMBasicMCBuilder *llvm::CreateMCBuilder(unsigned Opcode, ARMFormat Format) {
+
+  return new ARMBasicMCBuilder(Opcode, Format,
+                               ARMInsts[Opcode].getNumOperands());
+}
diff --git a/lib/Target/ARM/Disassembler/ARMDisassemblerCore.h b/lib/Target/ARM/Disassembler/ARMDisassemblerCore.h
new file mode 100644
index 000000000000..307523037089
--- /dev/null
+++ b/lib/Target/ARM/Disassembler/ARMDisassemblerCore.h
@@ -0,0 +1,248 @@
+//===- ARMDisassemblerCore.h - ARM disassembler helpers ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is part of the ARM Disassembler.
+//
+// The first part defines the enumeration type of ARM instruction format, which
+// specifies the encoding used by the instruction, as well as a helper function
+// to convert the enums to printable char strings.
+//
+// It also contains code to represent the concepts of Builder and DisassembleFP
+// to solve the problem of disassembling an ARM instr.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ARMDISASSEMBLERCORE_H
+#define ARMDISASSEMBLERCORE_H
+
+#include "llvm/MC/MCInst.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "ARMInstrInfo.h"
+#include "ARMDisassembler.h"
+
+namespace llvm {
+
+class ARMUtils {
+public:
+  static const char *OpcodeName(unsigned Opcode);
+};
+
+/////////////////////////////////////////////////////
+//                                                 //
+//  Enums and Utilities for ARM Instruction Format //
+//                                                 //
+/////////////////////////////////////////////////////
+
+#define ARM_FORMATS                   \
+  ENTRY(ARM_FORMAT_PSEUDO,         0) \
+  ENTRY(ARM_FORMAT_MULFRM,         1) \
+  ENTRY(ARM_FORMAT_BRFRM,          2) \
+  ENTRY(ARM_FORMAT_BRMISCFRM,      3) \
+  ENTRY(ARM_FORMAT_DPFRM,          4) \
+  ENTRY(ARM_FORMAT_DPSOREGFRM,     5) \
+  ENTRY(ARM_FORMAT_LDFRM,          6) \
+  ENTRY(ARM_FORMAT_STFRM,          7) \
+  ENTRY(ARM_FORMAT_LDMISCFRM,      8) \
+  ENTRY(ARM_FORMAT_STMISCFRM,      9) \
+  ENTRY(ARM_FORMAT_LDSTMULFRM,    10) \
+  ENTRY(ARM_FORMAT_LDSTEXFRM,     11) \
+  ENTRY(ARM_FORMAT_ARITHMISCFRM,  12) \
+  ENTRY(ARM_FORMAT_EXTFRM,        13) \
+  ENTRY(ARM_FORMAT_VFPUNARYFRM,   14) \
+  ENTRY(ARM_FORMAT_VFPBINARYFRM,  15) \
+  ENTRY(ARM_FORMAT_VFPCONV1FRM,   16) \
+  ENTRY(ARM_FORMAT_VFPCONV2FRM,   17) \
+  ENTRY(ARM_FORMAT_VFPCONV3FRM,   18) \
+  ENTRY(ARM_FORMAT_VFPCONV4FRM,   19) \
+  ENTRY(ARM_FORMAT_VFPCONV5FRM,   20) \
+  ENTRY(ARM_FORMAT_VFPLDSTFRM,    21) \
+  ENTRY(ARM_FORMAT_VFPLDSTMULFRM, 22) \
+  ENTRY(ARM_FORMAT_VFPMISCFRM,    23) \
+  ENTRY(ARM_FORMAT_THUMBFRM,      24) \
+  ENTRY(ARM_FORMAT_NEONFRM,       25) \
+  ENTRY(ARM_FORMAT_NEONGETLNFRM,  26) \
+  ENTRY(ARM_FORMAT_NEONSETLNFRM,  27) \
+  ENTRY(ARM_FORMAT_NEONDUPFRM,    28) \
+  ENTRY(ARM_FORMAT_MISCFRM,       29) \
+  ENTRY(ARM_FORMAT_THUMBMISCFRM,  30) \
+  ENTRY(ARM_FORMAT_NLdSt,         31) \
+  ENTRY(ARM_FORMAT_N1RegModImm,   32) \
+  ENTRY(ARM_FORMAT_N2Reg,         33) \
+  ENTRY(ARM_FORMAT_NVCVT,         34) \
+  ENTRY(ARM_FORMAT_NVecDupLn,     35) \
+  ENTRY(ARM_FORMAT_N2RegVecShL,   36) \
+  ENTRY(ARM_FORMAT_N2RegVecShR,   37) \
+  ENTRY(ARM_FORMAT_N3Reg,         38) \
+  ENTRY(ARM_FORMAT_N3RegVecSh,    39) \
+  ENTRY(ARM_FORMAT_NVecExtract,   40) \
+  ENTRY(ARM_FORMAT_NVecMulScalar, 41) \
+  ENTRY(ARM_FORMAT_NVTBL,         42)
+
+// ARM instruction format specifies the encoding used by the instruction.
+#define ENTRY(n, v) n = v,
+typedef enum {
+  ARM_FORMATS
+  ARM_FORMAT_NA
+} ARMFormat;
+#undef ENTRY
+
+// Converts enum to const char*.
+static const inline char *stringForARMFormat(ARMFormat form) {
+#define ENTRY(n, v) case n: return #n;
+  switch(form) {
+    ARM_FORMATS
+  case ARM_FORMAT_NA:
+  default:
+    return "";
+  }
+#undef ENTRY
+}
+
+/// Expands on the enum definitions from ARMBaseInstrInfo.h.
+/// They are being used by the disassembler implementation.
+namespace ARMII {
+  enum {
+    NEONRegMask = 15,
+    GPRRegMask = 15,
+    NEON_RegRdShift = 12,
+    NEON_D_BitShift = 22,
+    NEON_RegRnShift = 16,
+    NEON_N_BitShift = 7,
+    NEON_RegRmShift = 0,
+    NEON_M_BitShift = 5
+  };
+}
+
+/// Utility function for extracting [From, To] bits from a uint32_t.
+static inline unsigned slice(uint32_t Bits, unsigned From, unsigned To) {
+  assert(From < 32 && To < 32 && From >= To);
+  return (Bits >> To) & ((1 << (From - To + 1)) - 1);
+}
+
+/// Utility function for setting [From, To] bits to Val for a uint32_t.
+static inline void setSlice(uint32_t &Bits, unsigned From, unsigned To,
+                            uint32_t Val) {
+  assert(From < 32 && To < 32 && From >= To);
+  uint32_t Mask = ((1 << (From - To + 1)) - 1);
+  Bits &= ~(Mask << To);
+  Bits |= (Val & Mask) << To;
+}
+
+/// Various utilities for checking the target specific flags.
+
+/// A unary data processing instruction doesn't have an Rn operand.
+static inline bool isUnaryDP(unsigned TSFlags) {
+  return (TSFlags & ARMII::UnaryDP);
+}
+
+/// This four-bit field describes the addressing mode used.
+/// See also ARMBaseInstrInfo.h.
+static inline unsigned getAddrMode(unsigned TSFlags) {
+  return (TSFlags & ARMII::AddrModeMask);
+}
+
+/// {IndexModePre, IndexModePost}
+/// Only valid for load and store ops.
+/// See also ARMBaseInstrInfo.h.
+static inline unsigned getIndexMode(unsigned TSFlags) {
+  return (TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift;
+}
+
+/// Pre-/post-indexed operations define an extra $base_wb in the OutOperandList.
+static inline bool isPrePostLdSt(unsigned TSFlags) {
+  return (TSFlags & ARMII::IndexModeMask) != 0;
+}
+
+// Forward declaration.
+class ARMBasicMCBuilder;
+
+// Builder Object is mostly ignored except in some Thumb disassemble functions.
+typedef ARMBasicMCBuilder *BO;
+
+/// DisassembleFP - DisassembleFP points to a function that disassembles an insn
+/// and builds the MCOperand list upon disassembly.  It returns false on failure
+/// or true on success.  The number of operands added is updated upon success.
+typedef bool (*DisassembleFP)(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO Builder);
+
+/// ARMBasicMCBuilder - ARMBasicMCBuilder represents an ARM MCInst builder that
+/// knows how to build up the MCOperand list.
+class ARMBasicMCBuilder {
+  unsigned Opcode;
+  ARMFormat Format;
+  unsigned short NumOps;
+  DisassembleFP Disasm;
+  Session *SP;
+
+public:
+  ARMBasicMCBuilder(ARMBasicMCBuilder &B)
+    : Opcode(B.Opcode), Format(B.Format), NumOps(B.NumOps), Disasm(B.Disasm),
+      SP(B.SP)
+  {}
+
+  /// Opcode, Format, and NumOperands make up an ARM Basic MCBuilder.
+  ARMBasicMCBuilder(unsigned opc, ARMFormat format, unsigned short num);
+
+  virtual ~ARMBasicMCBuilder() {}
+
+  void setSession(Session *sp) {
+    SP = sp;
+  }
+
+  /// TryPredicateAndSBitModifier - TryPredicateAndSBitModifier tries to process
+  /// the possible Predicate and SBitModifier, to build the remaining MCOperand
+  /// constituents.
+  bool TryPredicateAndSBitModifier(MCInst& MI, unsigned Opcode,
+      uint32_t insn, unsigned short NumOpsRemaning);
+
+  /// InITBlock - InITBlock returns true if we are inside an IT block.
+  bool InITBlock() {
+    if (SP)
+      return SP->ITCounter > 0;
+
+    return false;
+  }
+
+  /// Build - Build delegates to BuildIt to perform the heavy liftling.  After
+  /// that, it invokes RunBuildAfterHook where some housekeepings can be done.
+  virtual bool Build(MCInst &MI, uint32_t insn) {
+    bool Status = BuildIt(MI, insn);
+    return RunBuildAfterHook(Status, MI, insn);
+  }
+
+  /// BuildIt - BuildIt performs the build step for this ARM Basic MC Builder.
+  /// The general idea is to set the Opcode for the MCInst, followed by adding
+  /// the appropriate MCOperands to the MCInst.  ARM Basic MC Builder delegates
+  /// to the Format-specific disassemble function for disassembly, followed by
+  /// TryPredicateAndSBitModifier() for PredicateOperand and OptionalDefOperand
+  /// which follow the Dst/Src Operands.
+  virtual bool BuildIt(MCInst &MI, uint32_t insn);
+
+  /// RunBuildAfterHook - RunBuildAfterHook performs operations deemed necessary
+  /// after BuildIt is finished.
+  virtual bool RunBuildAfterHook(bool Status, MCInst &MI, uint32_t insn);
+
+private:
+  /// Get condition of the current IT instruction.
+  unsigned GetITCond() {
+    assert(SP);
+    return slice(SP->ITState, 7, 4);
+  }
+};
+
+/// CreateMCBuilder - Return an ARMBasicMCBuilder that can build up the MC
+/// infrastructure of an MCInst given the Opcode and Format of the instr.
+/// Return NULL if it fails to create/return a proper builder.  API clients
+/// are responsible for freeing up of the allocated memory.  Cacheing can be
+/// performed by the API clients to improve performance.
+extern ARMBasicMCBuilder *CreateMCBuilder(unsigned Opcode, ARMFormat Format);
+
+} // namespace llvm
+
+#endif
diff --git a/lib/Target/ARM/Disassembler/Makefile b/lib/Target/ARM/Disassembler/Makefile
new file mode 100644
index 000000000000..031b6aca5a48
--- /dev/null
+++ b/lib/Target/ARM/Disassembler/Makefile
@@ -0,0 +1,16 @@
+##===- lib/Target/ARM/Disassembler/Makefile ----------------*- Makefile -*-===##
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+##===----------------------------------------------------------------------===##
+
+LEVEL = ../../../..
+LIBRARYNAME = LLVMARMDisassembler
+
+# Hack: we need to include 'main' arm target directory to grab private headers
+CPPFLAGS = -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+
+include $(LEVEL)/Makefile.common
diff --git a/lib/Target/ARM/Disassembler/ThumbDisassemblerCore.h b/lib/Target/ARM/Disassembler/ThumbDisassemblerCore.h
new file mode 100644
index 000000000000..481f25d6f486
--- /dev/null
+++ b/lib/Target/ARM/Disassembler/ThumbDisassemblerCore.h
@@ -0,0 +1,2187 @@
+//===- ThumbDisassemblerCore.h - Thumb disassembler helpers -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is part of the ARM Disassembler.
+// It contains code for disassembling a Thumb instr.  It is to be included by
+// ARMDisassemblerCore.cpp because it contains the static DisassembleThumbFrm()
+// function which acts as the dispatcher to disassemble a Thumb instruction.
+//
+//===----------------------------------------------------------------------===//
+
+///////////////////////////////
+//                           //
+//     Utility Functions     //
+//                           //
+///////////////////////////////
+
+// Utilities for 16-bit Thumb instructions.
+/*
+15 14 13 12 11 10  9  8  7  6  5  4  3  2  1  0
+               [  tRt ]
+                      [ tRm ]  [ tRn ]  [ tRd ]
+                         D  [   Rm   ]  [  Rd ]
+
+                      [ imm3]
+               [    imm5    ]
+                   i     [    imm5   ]
+                            [       imm7      ]
+                         [       imm8         ]
+               [             imm11            ]
+
+            [   cond  ]
+*/
+
+// Extract tRt: Inst{10-8}.
+static inline unsigned getT1tRt(uint32_t insn) {
+  return slice(insn, 10, 8);
+}
+
+// Extract tRm: Inst{8-6}.
+static inline unsigned getT1tRm(uint32_t insn) {
+  return slice(insn, 8, 6);
+}
+
+// Extract tRn: Inst{5-3}.
+static inline unsigned getT1tRn(uint32_t insn) {
+  return slice(insn, 5, 3);
+}
+
+// Extract tRd: Inst{2-0}.
+static inline unsigned getT1tRd(uint32_t insn) {
+  return slice(insn, 2, 0);
+}
+
+// Extract [D:Rd]: Inst{7:2-0}.
+static inline unsigned getT1Rd(uint32_t insn) {
+  return slice(insn, 7, 7) << 3 | slice(insn, 2, 0);
+}
+
+// Extract Rm: Inst{6-3}.
+static inline unsigned getT1Rm(uint32_t insn) {
+  return slice(insn, 6, 3);
+}
+
+// Extract imm3: Inst{8-6}.
+static inline unsigned getT1Imm3(uint32_t insn) {
+  return slice(insn, 8, 6);
+}
+
+// Extract imm5: Inst{10-6}.
+static inline unsigned getT1Imm5(uint32_t insn) {
+  return slice(insn, 10, 6);
+}
+
+// Extract i:imm5: Inst{9:7-3}.
+static inline unsigned getT1Imm6(uint32_t insn) {
+  return slice(insn, 9, 9) << 5 | slice(insn, 7, 3);
+}
+
+// Extract imm7: Inst{6-0}.
+static inline unsigned getT1Imm7(uint32_t insn) {
+  return slice(insn, 6, 0);
+}
+
+// Extract imm8: Inst{7-0}.
+static inline unsigned getT1Imm8(uint32_t insn) {
+  return slice(insn, 7, 0);
+}
+
+// Extract imm11: Inst{10-0}.
+static inline unsigned getT1Imm11(uint32_t insn) {
+  return slice(insn, 10, 0);
+}
+
+// Extract cond: Inst{11-8}.
+static inline unsigned getT1Cond(uint32_t insn) {
+  return slice(insn, 11, 8);
+}
+
+static inline bool IsGPR(unsigned RegClass) {
+  return RegClass == ARM::GPRRegClassID;
+}
+
+// Utilities for 32-bit Thumb instructions.
+
+// Extract imm4: Inst{19-16}.
+static inline unsigned getImm4(uint32_t insn) {
+  return slice(insn, 19, 16);
+}
+
+// Extract imm3: Inst{14-12}.
+static inline unsigned getImm3(uint32_t insn) {
+  return slice(insn, 14, 12);
+}
+
+// Extract imm8: Inst{7-0}.
+static inline unsigned getImm8(uint32_t insn) {
+  return slice(insn, 7, 0);
+}
+
+// A8.6.61 LDRB (immediate, Thumb) and friends
+// +/-: Inst{9}
+// imm8: Inst{7-0}
+static inline int decodeImm8(uint32_t insn) {
+  int Offset = getImm8(insn);
+  return slice(insn, 9, 9) ? Offset : -Offset;
+}
+
+// Extract imm12: Inst{11-0}.
+static inline unsigned getImm12(uint32_t insn) {
+  return slice(insn, 11, 0);
+}
+
+// A8.6.63 LDRB (literal) and friends
+// +/-: Inst{23}
+// imm12: Inst{11-0}
+static inline int decodeImm12(uint32_t insn) {
+  int Offset = getImm12(insn);
+  return slice(insn, 23, 23) ? Offset : -Offset;
+}
+
+// Extract imm2: Inst{7-6}.
+static inline unsigned getImm2(uint32_t insn) {
+  return slice(insn, 7, 6);
+}
+
+// For BFI, BFC, t2SBFX, and t2UBFX.
+// Extract lsb: Inst{14-12:7-6}.
+static inline unsigned getLsb(uint32_t insn) {
+  return getImm3(insn) << 2 | getImm2(insn);
+}
+
+// For BFI and BFC.
+// Extract msb: Inst{4-0}.
+static inline unsigned getMsb(uint32_t insn) {
+  return slice(insn, 4, 0);
+}
+
+// For t2SBFX and t2UBFX.
+// Extract widthminus1: Inst{4-0}.
+static inline unsigned getWidthMinus1(uint32_t insn) {
+  return slice(insn, 4, 0);
+}
+
+// For t2ADDri12 and t2SUBri12.
+// imm12 = i:imm3:imm8;
+static inline unsigned getIImm3Imm8(uint32_t insn) {
+  return slice(insn, 26, 26) << 11 | getImm3(insn) << 8 | getImm8(insn);
+}
+
+// For t2MOVi16 and t2MOVTi16.
+// imm16 = imm4:i:imm3:imm8;
+static inline unsigned getImm16(uint32_t insn) {
+  return getImm4(insn) << 12 | slice(insn, 26, 26) << 11 |
+    getImm3(insn) << 8 | getImm8(insn);
+}
+
+// Inst{5-4} encodes the shift type.
+static inline unsigned getShiftTypeBits(uint32_t insn) {
+  return slice(insn, 5, 4);
+}
+
+// Inst{14-12}:Inst{7-6} encodes the imm5 shift amount.
+static inline unsigned getShiftAmtBits(uint32_t insn) {
+  return getImm3(insn) << 2 | getImm2(insn);
+}
+
+// A8.6.17 BFC
+// Encoding T1 ARMv6T2, ARMv7
+// LLVM-specific encoding for #<lsb> and #<width>
+static inline uint32_t getBitfieldInvMask(uint32_t insn) {
+  uint32_t lsb = getImm3(insn) << 2 | getImm2(insn);
+  uint32_t msb = getMsb(insn);
+  uint32_t Val = 0;
+  assert(lsb <= msb && "Encoding error: lsb > msb");
+  for (uint32_t i = lsb; i <= msb; ++i)
+    Val |= (1 << i);
+  return ~Val;
+}
+
+// A8.4 Shifts applied to a register
+// A8.4.1 Constant shifts
+// A8.4.3 Pseudocode details of instruction-specified shifts and rotates
+//
+// decodeImmShift() returns the shift amount and the the shift opcode.
+// Note that, as of Jan-06-2010, LLVM does not support rrx shifted operands yet.
+static inline unsigned decodeImmShift(unsigned bits2, unsigned imm5,
+                                      ARM_AM::ShiftOpc &ShOp) {
+
+  assert(imm5 < 32 && "Invalid imm5 argument");
+  switch (bits2) {
+  default: assert(0 && "No such value");
+  case 0:
+    ShOp = ARM_AM::lsl;
+    return imm5;
+  case 1:
+    ShOp = ARM_AM::lsr;
+    return (imm5 == 0 ? 32 : imm5);
+  case 2:
+    ShOp = ARM_AM::asr;
+    return (imm5 == 0 ? 32 : imm5);
+  case 3:
+    ShOp = (imm5 == 0 ? ARM_AM::rrx : ARM_AM::ror);
+    return (imm5 == 0 ? 1 : imm5);
+  }
+}
+
+// A6.3.2 Modified immediate constants in Thumb instructions
+//
+// ThumbExpandImm() returns the modified immediate constant given an imm12 for
+// Thumb data-processing instructions with modified immediate.
+// See also A6.3.1 Data-processing (modified immediate).
+static inline unsigned ThumbExpandImm(unsigned imm12) {
+  assert(imm12 <= 0xFFF && "Invalid imm12 argument");
+
+  // If the leading two bits is 0b00, the modified immediate constant is
+  // obtained by splatting the low 8 bits into the first byte, every other byte,
+  // or every byte of a 32-bit value.
+  //
+  // Otherwise, a rotate right of '1':imm12<6:0> by the amount imm12<11:7> is
+  // performed.
+
+  if (slice(imm12, 11, 10) == 0) {
+    unsigned short control = slice(imm12, 9, 8);
+    unsigned imm8 = slice(imm12, 7, 0);
+    switch (control) {
+    default:
+      assert(0 && "No such value");
+      return 0;
+    case 0:
+      return imm8;
+    case 1:
+      return imm8 << 16 | imm8;
+    case 2:
+      return imm8 << 24 | imm8 << 8;
+    case 3:
+      return imm8 << 24 | imm8 << 16 | imm8 << 8 | imm8;
+    }
+  } else {
+    // A rotate is required.
+    unsigned Val = 1 << 7 | slice(imm12, 6, 0);
+    unsigned Amt = slice(imm12, 11, 7);
+    return ARM_AM::rotr32(Val, Amt);
+  }
+}
+
+static inline int decodeImm32_B_EncodingT3(uint32_t insn) {
+  bool S = slice(insn, 26, 26);
+  bool J1 = slice(insn, 13, 13);
+  bool J2 = slice(insn, 11, 11);
+  unsigned Imm21 = slice(insn, 21, 16) << 12 | slice(insn, 10, 0) << 1;
+  if (S) Imm21 |= 1 << 20;
+  if (J2) Imm21 |= 1 << 19;
+  if (J1) Imm21 |= 1 << 18;
+
+  return SignExtend32<21>(Imm21);
+}
+
+static inline int decodeImm32_B_EncodingT4(uint32_t insn) {
+  unsigned S = slice(insn, 26, 26);
+  bool I1 = slice(insn, 13, 13) == S;
+  bool I2 = slice(insn, 11, 11) == S;
+  unsigned Imm25 = slice(insn, 25, 16) << 12 | slice(insn, 10, 0) << 1;
+  if (S) Imm25 |= 1 << 24;
+  if (I1) Imm25 |= 1 << 23;
+  if (I2) Imm25 |= 1 << 22;
+
+  return SignExtend32<25>(Imm25);
+}
+
+static inline int decodeImm32_BL(uint32_t insn) {
+  unsigned S = slice(insn, 26, 26);
+  bool I1 = slice(insn, 13, 13) == S;
+  bool I2 = slice(insn, 11, 11) == S;
+  unsigned Imm25 = slice(insn, 25, 16) << 12 | slice(insn, 10, 0) << 1;
+  if (S) Imm25 |= 1 << 24;
+  if (I1) Imm25 |= 1 << 23;
+  if (I2) Imm25 |= 1 << 22;
+
+  return SignExtend32<25>(Imm25);
+}
+
+static inline int decodeImm32_BLX(uint32_t insn) {
+  unsigned S = slice(insn, 26, 26);
+  bool I1 = slice(insn, 13, 13) == S;
+  bool I2 = slice(insn, 11, 11) == S;
+  unsigned Imm25 = slice(insn, 25, 16) << 12 | slice(insn, 10, 1) << 2;
+  if (S) Imm25 |= 1 << 24;
+  if (I1) Imm25 |= 1 << 23;
+  if (I2) Imm25 |= 1 << 22;
+
+  return SignExtend32<25>(Imm25);
+}
+
+// See, for example, A8.6.221 SXTAB16.
+static inline unsigned decodeRotate(uint32_t insn) {
+  unsigned rotate = slice(insn, 5, 4);
+  return rotate << 3;
+}
+
+///////////////////////////////////////////////
+//                                           //
+// Thumb1 instruction disassembly functions. //
+//                                           //
+///////////////////////////////////////////////
+
+// See "Utilities for 16-bit Thumb instructions" for register naming convention.
+
+// A6.2.1 Shift (immediate), add, subtract, move, and compare
+//
+// shift immediate:         tRd CPSR tRn imm5
+// add/sub register:        tRd CPSR tRn tRm
+// add/sub 3-bit immediate: tRd CPSR tRn imm3
+// add/sub 8-bit immediate: tRt CPSR tRt(TIED_TO) imm8
+// mov/cmp immediate:       tRt [CPSR] imm8 (CPSR present for mov)
+//
+// Special case:
+// tMOVSr:                  tRd tRn
+static bool DisassembleThumb1General(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO Builder) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::tGPRRegClassID
+         && "Invalid arguments");
+
+  bool Imm3 = (Opcode == ARM::tADDi3 || Opcode == ARM::tSUBi3);
+
+  // Use Rt implies use imm8.
+  bool UseRt = (Opcode == ARM::tADDi8 || Opcode == ARM::tSUBi8 ||
+                Opcode == ARM::tMOVi8 || Opcode == ARM::tCMPi8);
+
+  // Add the destination operand.
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(ARM::tGPRRegClassID,
+                                  UseRt ? getT1tRt(insn) : getT1tRd(insn))));
+  ++OpIdx;
+
+  // Check whether the next operand to be added is a CCR Register.
+  if (OpInfo[OpIdx].RegClass == ARM::CCRRegClassID) {
+    assert(OpInfo[OpIdx].isOptionalDef() && "Optional def operand expected");
+    MI.addOperand(MCOperand::CreateReg(Builder->InITBlock() ? 0 : ARM::CPSR));
+    ++OpIdx;
+  }
+
+  // Check whether the next operand to be added is a Thumb1 Register.
+  assert(OpIdx < NumOps && "More operands expected");
+  if (OpInfo[OpIdx].RegClass == ARM::tGPRRegClassID) {
+    // For UseRt, the reg operand is tied to the first reg operand.
+    MI.addOperand(MCOperand::CreateReg(
+                    getRegisterEnum(ARM::tGPRRegClassID,
+                                    UseRt ? getT1tRt(insn) : getT1tRn(insn))));
+    ++OpIdx;
+  }
+
+  // Special case for tMOVSr.
+  if (OpIdx == NumOps)
+    return true;
+
+  // The next available operand is either a reg operand or an imm operand.
+  if (OpInfo[OpIdx].RegClass == ARM::tGPRRegClassID) {
+    // Three register operand instructions.
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                       getT1tRm(insn))));
+  } else {
+    assert(OpInfo[OpIdx].RegClass == 0 &&
+           !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()
+           && "Pure imm operand expected");
+    MI.addOperand(MCOperand::CreateImm(UseRt ? getT1Imm8(insn)
+                                             : (Imm3 ? getT1Imm3(insn)
+                                                     : getT1Imm5(insn))));
+  }
+  ++OpIdx;
+
+  return true;
+}
+
+// A6.2.2 Data-processing
+//
+// tCMPr, tTST, tCMN: tRd tRn
+// tMVN, tRSB:        tRd CPSR tRn
+// Others:            tRd CPSR tRd(TIED_TO) tRn
+static bool DisassembleThumb1DP(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO Builder) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::tGPRRegClassID &&
+         (OpInfo[1].RegClass == ARM::CCRRegClassID
+          || OpInfo[1].RegClass == ARM::tGPRRegClassID)
+         && "Invalid arguments");
+
+  // Add the destination operand.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                     getT1tRd(insn))));
+  ++OpIdx;
+
+  // Check whether the next operand to be added is a CCR Register.
+  if (OpInfo[OpIdx].RegClass == ARM::CCRRegClassID) {
+    assert(OpInfo[OpIdx].isOptionalDef() && "Optional def operand expected");
+    MI.addOperand(MCOperand::CreateReg(Builder->InITBlock() ? 0 : ARM::CPSR));
+    ++OpIdx;
+  }
+
+  // We have either { tRd(TIED_TO), tRn } or { tRn } remaining.
+  // Process the TIED_TO operand first.
+
+  assert(OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::tGPRRegClassID
+         && "Thumb reg operand expected");
+  int Idx;
+  if ((Idx = TID.getOperandConstraint(OpIdx, TOI::TIED_TO)) != -1) {
+    // The reg operand is tied to the first reg operand.
+    MI.addOperand(MI.getOperand(Idx));
+    ++OpIdx;
+  }
+
+  // Process possible next reg operand.
+  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::tGPRRegClassID) {
+    // Add tRn operand.
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                       getT1tRn(insn))));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+// A6.2.3 Special data instructions and branch and exchange
+//
+// tADDhirr: Rd Rd(TIED_TO) Rm
+// tCMPhir:  Rd Rm
+// tMOVr, tMOVgpr2gpr, tMOVgpr2tgpr, tMOVtgpr2gpr: Rd|tRd Rm|tRn
+// tBX_RET: 0 operand
+// tBX_RET_vararg: Rm
+// tBLXr_r9: Rm
+static bool DisassembleThumb1Special(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  // tBX_RET has 0 operand.
+  if (NumOps == 0)
+    return true;
+
+  // BX/BLX has 1 reg operand: Rm.
+  if (NumOps == 1) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       getT1Rm(insn))));
+    NumOpsAdded = 1;
+    return true;
+  }
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  // Add the destination operand.
+  unsigned RegClass = OpInfo[OpIdx].RegClass;
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(RegClass,
+                                  IsGPR(RegClass) ? getT1Rd(insn)
+                                                  : getT1tRd(insn))));
+  ++OpIdx;
+
+  // We have either { Rd(TIED_TO), Rm } or { Rm|tRn } remaining.
+  // Process the TIED_TO operand first.
+
+  assert(OpIdx < NumOps && "More operands expected");
+  int Idx;
+  if ((Idx = TID.getOperandConstraint(OpIdx, TOI::TIED_TO)) != -1) {
+    // The reg operand is tied to the first reg operand.
+    MI.addOperand(MI.getOperand(Idx));
+    ++OpIdx;
+  }
+
+  // The next reg operand is either Rm or tRn.
+  assert(OpIdx < NumOps && "More operands expected");
+  RegClass = OpInfo[OpIdx].RegClass;
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(RegClass,
+                                  IsGPR(RegClass) ? getT1Rm(insn)
+                                                  : getT1tRn(insn))));
+  ++OpIdx;
+
+  return true;
+}
+
+// A8.6.59 LDR (literal)
+//
+// tLDRpci: tRt imm8*4
+static bool DisassembleThumb1LdPC(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::tGPRRegClassID &&
+         (OpInfo[1].RegClass == 0 &&
+          !OpInfo[1].isPredicate() &&
+          !OpInfo[1].isOptionalDef())
+         && "Invalid arguments");
+
+  // Add the destination operand.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                     getT1tRt(insn))));
+
+  // And the (imm8 << 2) operand.
+  MI.addOperand(MCOperand::CreateImm(getT1Imm8(insn) << 2));
+
+  NumOpsAdded = 2;
+
+  return true;
+}
+
+// Thumb specific addressing modes (see ARMInstrThumb.td):
+//
+// t_addrmode_rr := reg + reg
+//
+// t_addrmode_s4 := reg + reg
+//                  reg + imm5 * 4
+//
+// t_addrmode_s2 := reg + reg
+//                  reg + imm5 * 2
+//
+// t_addrmode_s1 := reg + reg
+//                  reg + imm5
+//
+// t_addrmode_sp := sp + imm8 * 4
+//
+
+// A6.2.4 Load/store single data item
+//
+// Load/Store Register (reg|imm):      tRd tRn imm5 tRm
+// Load Register Signed Byte|Halfword: tRd tRn tRm
+static bool DisassembleThumb1LdSt(unsigned opA, MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  // Table A6-5 16-bit Thumb Load/store instructions
+  // opA = 0b0101 for STR/LDR (register) and friends.
+  // Otherwise, we have STR/LDR (immediate) and friends.
+  bool Imm5 = (opA != 5);
+
+  assert(NumOps >= 2
+         && OpInfo[0].RegClass == ARM::tGPRRegClassID
+         && OpInfo[1].RegClass == ARM::tGPRRegClassID
+         && "Expect >= 2 operands and first two as thumb reg operands");
+
+  // Add the destination reg and the base reg.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                     getT1tRd(insn))));
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                     getT1tRn(insn))));
+  OpIdx = 2;
+
+  // We have either { imm5, tRm } or { tRm } remaining.
+  // Process the imm5 first.  Note that STR/LDR (register) should skip the imm5
+  // offset operand for t_addrmode_s[1|2|4].
+
+  assert(OpIdx < NumOps && "More operands expected");
+
+  if (OpInfo[OpIdx].RegClass == 0 && !OpInfo[OpIdx].isPredicate() &&
+      !OpInfo[OpIdx].isOptionalDef()) {
+
+    MI.addOperand(MCOperand::CreateImm(Imm5 ? getT1Imm5(insn) : 0));
+    ++OpIdx;
+  }
+
+  // The next reg operand is tRm, the offset.
+  assert(OpIdx < NumOps && OpInfo[OpIdx].RegClass == ARM::tGPRRegClassID
+         && "Thumb reg operand expected");
+  MI.addOperand(MCOperand::CreateReg(Imm5 ? 0
+                                          : getRegisterEnum(ARM::tGPRRegClassID,
+                                                            getT1tRm(insn))));
+  ++OpIdx;
+
+  return true;
+}
+
+// A6.2.4 Load/store single data item
+//
+// Load/Store Register SP relative: tRt ARM::SP imm8
+static bool DisassembleThumb1LdStSP(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  assert((Opcode == ARM::tLDRspi || Opcode == ARM::tSTRspi)
+         && "Invalid opcode");
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  assert(NumOps >= 3 &&
+         OpInfo[0].RegClass == ARM::tGPRRegClassID &&
+         OpInfo[1].RegClass == ARM::GPRRegClassID &&
+         (OpInfo[2].RegClass == 0 &&
+          !OpInfo[2].isPredicate() &&
+          !OpInfo[2].isOptionalDef())
+         && "Invalid arguments");
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                     getT1tRt(insn))));
+  MI.addOperand(MCOperand::CreateReg(ARM::SP));
+  MI.addOperand(MCOperand::CreateImm(getT1Imm8(insn)));
+  NumOpsAdded = 3;
+  return true;
+}
+
+// Table A6-1 16-bit Thumb instruction encoding
+// A8.6.10 ADR
+//
+// tADDrPCi: tRt imm8
+static bool DisassembleThumb1AddPCi(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  assert(Opcode == ARM::tADDrPCi && "Invalid opcode");
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::tGPRRegClassID &&
+         (OpInfo[1].RegClass == 0 &&
+          !OpInfo[1].isPredicate() &&
+          !OpInfo[1].isOptionalDef())
+         && "Invalid arguments");
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                     getT1tRt(insn))));
+  MI.addOperand(MCOperand::CreateImm(getT1Imm8(insn)));
+  NumOpsAdded = 2;
+  return true;
+}
+
+// Table A6-1 16-bit Thumb instruction encoding
+// A8.6.8 ADD (SP plus immediate)
+//
+// tADDrSPi: tRt ARM::SP imm8
+static bool DisassembleThumb1AddSPi(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  assert(Opcode == ARM::tADDrSPi && "Invalid opcode");
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  assert(NumOps >= 3 &&
+         OpInfo[0].RegClass == ARM::tGPRRegClassID &&
+         OpInfo[1].RegClass == ARM::GPRRegClassID &&
+         (OpInfo[2].RegClass == 0 &&
+          !OpInfo[2].isPredicate() &&
+          !OpInfo[2].isOptionalDef())
+         && "Invalid arguments");
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                     getT1tRt(insn))));
+  MI.addOperand(MCOperand::CreateReg(ARM::SP));
+  MI.addOperand(MCOperand::CreateImm(getT1Imm8(insn)));
+  NumOpsAdded = 3;
+  return true;
+}
+
+// tPUSH, tPOP: Pred-Imm Pred-CCR register_list
+//
+// where register_list = low registers + [lr] for PUSH or
+//                       low registers + [pc] for POP
+//
+// "low registers" is specified by Inst{7-0}
+// lr|pc is specified by Inst{8}
+static bool DisassembleThumb1PushPop(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  assert((Opcode == ARM::tPUSH || Opcode == ARM::tPOP) && "Invalid opcode");
+
+  unsigned &OpIdx = NumOpsAdded;
+
+  // Handling the two predicate operands before the reglist.
+  MI.addOperand(MCOperand::CreateImm(ARMCC::AL));
+  MI.addOperand(MCOperand::CreateReg(ARM::CPSR));
+  OpIdx = 2;
+
+  // Fill the variadic part of reglist.
+  unsigned RegListBits = slice(insn, 8, 8) << (Opcode == ARM::tPUSH ? 14 : 15)
+    | slice(insn, 7, 0);
+  for (unsigned i = 0; i < 16; ++i) {
+    if ((RegListBits >> i) & 1) {
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                         i)));
+      ++OpIdx;
+    }
+  }
+
+  return true;
+}
+
+// A6.2.5 Miscellaneous 16-bit instructions
+// Delegate to DisassembleThumb1PushPop() for tPUSH & tPOP.
+//
+// tADDspi, tSUBspi: ARM::SP ARM::SP(TIED_TO) imm7
+// t2IT:             firstcond=Inst{7-4} mask=Inst{3-0}
+// tCBNZ, tCBZ:      tRd imm6*2
+// tBKPT:            imm8
+// tNOP, tSEV, tYIELD, tWFE, tWFI:
+//   no operand (except predicate pair)
+// tSETENDBE, tSETENDLE, :
+//   no operand
+// Others:           tRd tRn
+static bool DisassembleThumb1Misc(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  if (NumOps == 0)
+    return true;
+
+  if (Opcode == ARM::tPUSH || Opcode == ARM::tPOP)
+    return DisassembleThumb1PushPop(MI, Opcode, insn, NumOps, NumOpsAdded);
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  // Predicate operands are handled elsewhere.
+  if (NumOps == 2 &&
+      OpInfo[0].isPredicate() && OpInfo[1].isPredicate() &&
+      OpInfo[0].RegClass == 0 && OpInfo[1].RegClass == ARM::CCRRegClassID) {
+    return true;
+  }
+
+  if (Opcode == ARM::tADDspi || Opcode == ARM::tSUBspi) {
+    // Special case handling for tADDspi and tSUBspi.
+    // A8.6.8 ADD (SP plus immediate) & A8.6.215 SUB (SP minus immediate)
+    MI.addOperand(MCOperand::CreateReg(ARM::SP));
+    MI.addOperand(MCOperand::CreateReg(ARM::SP));
+    MI.addOperand(MCOperand::CreateImm(getT1Imm7(insn)));
+    NumOpsAdded = 3;
+    return true;
+  }
+
+  if (Opcode == ARM::t2IT) {
+    // Special case handling for If-Then.
+    // A8.6.50 IT
+    // Tag the (firstcond[0] bit << 4) along with mask.
+
+    // firstcond
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 7, 4)));
+
+    // firstcond[0] and mask
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 4, 0)));
+    NumOpsAdded = 2;
+    return true;
+  }
+
+  if (Opcode == ARM::tBKPT) {
+    MI.addOperand(MCOperand::CreateImm(getT1Imm8(insn))); // breakpoint value
+    NumOpsAdded = 1;
+    return true;
+  }
+
+  // CPS has a singleton $opt operand that contains the following information:
+  // opt{4-0} = don't care
+  // opt{5} = 0 (false)
+  // opt{8-6} = AIF from Inst{2-0}
+  // opt{10-9} = 1:imod from Inst{4} with 0b10 as enable and 0b11 as disable
+  if (Opcode == ARM::tCPS) {
+    unsigned Option = slice(insn, 2, 0) << 6 | slice(insn, 4, 4) << 9 | 1 << 10;
+    MI.addOperand(MCOperand::CreateImm(Option));
+    NumOpsAdded = 1;
+    return true;
+  }
+
+  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::tGPRRegClassID &&
+         (OpInfo[1].RegClass==0 || OpInfo[1].RegClass==ARM::tGPRRegClassID)
+         && "Expect >=2 operands");
+
+  // Add the destination operand.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                     getT1tRd(insn))));
+
+  if (OpInfo[1].RegClass == ARM::tGPRRegClassID) {
+    // Two register instructions.
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                       getT1tRn(insn))));
+  } else {
+    // CBNZ, CBZ
+    assert((Opcode == ARM::tCBNZ || Opcode == ARM::tCBZ) && "Invalid opcode");
+    MI.addOperand(MCOperand::CreateImm(getT1Imm6(insn) * 2));
+  }
+
+  NumOpsAdded = 2;
+
+  return true;
+}
+
+// A8.6.53  LDM / LDMIA
+// A8.6.189 STM / STMIA
+//
+// tLDM_UPD/tSTM_UPD: tRt tRt AM4ModeImm Pred-Imm Pred-CCR register_list
+// tLDM:              tRt AM4ModeImm Pred-Imm Pred-CCR register_list
+static bool DisassembleThumb1LdStMul(bool Ld, MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  assert((Opcode == ARM::tLDM || Opcode == ARM::tLDM_UPD ||
+          Opcode == ARM::tSTM_UPD) && "Invalid opcode");
+
+  unsigned &OpIdx = NumOpsAdded;
+
+  unsigned tRt = getT1tRt(insn);
+  unsigned RegListBits = slice(insn, 7, 0);
+
+  OpIdx = 0;
+
+  // WB register, if necessary.
+  if (Opcode == ARM::tLDM_UPD || Opcode == ARM::tSTM_UPD) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       tRt)));
+    ++OpIdx;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     tRt)));
+  ++OpIdx;
+
+  // A8.6.53 LDM / LDMIA / LDMFD - Encoding T1
+  MI.addOperand(MCOperand::CreateImm(ARM_AM::getAM4ModeImm(ARM_AM::ia)));
+  ++OpIdx;
+
+  // Handling the two predicate operands before the reglist.
+  MI.addOperand(MCOperand::CreateImm(ARMCC::AL));
+  MI.addOperand(MCOperand::CreateReg(ARM::CPSR));
+  OpIdx += 2;
+
+  // Fill the variadic part of reglist.
+  for (unsigned i = 0; i < 8; ++i) {
+    if ((RegListBits >> i) & 1) {
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::tGPRRegClassID,
+                                                         i)));
+      ++OpIdx;
+    }
+  }
+
+  return true;
+}
+
+static bool DisassembleThumb1LdMul(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+  return DisassembleThumb1LdStMul(true, MI, Opcode, insn, NumOps, NumOpsAdded);
+}
+
+static bool DisassembleThumb1StMul(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+  return DisassembleThumb1LdStMul(false, MI, Opcode, insn, NumOps, NumOpsAdded);
+}
+
+// A8.6.16 B Encoding T1
+// cond = Inst{11-8} & imm8 = Inst{7-0}
+// imm32 = SignExtend(imm8:'0', 32)
+//
+// tBcc: offset Pred-Imm Pred-CCR
+// tSVC: imm8 Pred-Imm Pred-CCR
+// tTRAP: 0 operand (early return)
+static bool DisassembleThumb1CondBr(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  if (Opcode == ARM::tTRAP)
+    return true;
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  assert(NumOps == 3 && OpInfo[0].RegClass == 0 &&
+         OpInfo[1].isPredicate() && OpInfo[2].RegClass == ARM::CCRRegClassID
+         && "Exactly 3 operands expected");
+
+  unsigned Imm8 = getT1Imm8(insn);
+  MI.addOperand(MCOperand::CreateImm(
+                  Opcode == ARM::tBcc ? SignExtend32<9>(Imm8 << 1) + 4
+                                      : (int)Imm8));
+
+  // Predicate operands by ARMBasicMCBuilder::TryPredicateAndSBitModifier().
+  NumOpsAdded = 1;
+
+  return true;
+}
+
+// A8.6.16 B Encoding T2
+// imm11 = Inst{10-0}
+// imm32 = SignExtend(imm11:'0', 32)
+//
+// tB: offset
+static bool DisassembleThumb1Br(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  assert(NumOps == 1 && OpInfo[0].RegClass == 0 && "1 imm operand expected");
+
+  unsigned Imm11 = getT1Imm11(insn);
+
+  // When executing a Thumb instruction, PC reads as the address of the current
+  // instruction plus 4.  The assembler subtracts 4 from the difference between
+  // the branch instruction and the target address, disassembler has to add 4 to
+  // to compensate.
+  MI.addOperand(MCOperand::CreateImm(SignExtend32<12>(Imm11 << 1) + 4));
+
+  NumOpsAdded = 1;
+
+  return true;
+
+}
+
+// See A6.2 16-bit Thumb instruction encoding for instruction classes
+// corresponding to op.
+//
+// Table A6-1 16-bit Thumb instruction encoding (abridged)
+// op		Instruction or instruction class
+// ------	--------------------------------------------------------------------
+// 00xxxx	Shift (immediate), add, subtract, move, and compare on page A6-7
+// 010000	Data-processing on page A6-8
+// 010001	Special data instructions and branch and exchange on page A6-9
+// 01001x	Load from Literal Pool, see LDR (literal) on page A8-122
+// 0101xx	Load/store single data item on page A6-10
+// 011xxx
+// 100xxx
+// 10100x	Generate PC-relative address, see ADR on page A8-32
+// 10101x	Generate SP-relative address, see ADD (SP plus immediate) on page A8-28
+// 1011xx	Miscellaneous 16-bit instructions on page A6-11
+// 11000x	Store multiple registers, see STM / STMIA / STMEA on page A8-374
+// 11001x	Load multiple registers, see LDM / LDMIA / LDMFD on page A8-110 a
+// 1101xx	Conditional branch, and Supervisor Call on page A6-13
+// 11100x	Unconditional Branch, see B on page A8-44
+//
+static bool DisassembleThumb1(uint16_t op,
+    MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO Builder) {
+
+  unsigned op1 = slice(op, 5, 4);
+  unsigned op2 = slice(op, 3, 2);
+  unsigned op3 = slice(op, 1, 0);
+  unsigned opA = slice(op, 5, 2);
+  switch (op1) {
+  case 0:
+    // A6.2.1 Shift (immediate), add, subtract, move, and compare
+    return DisassembleThumb1General(MI, Opcode, insn, NumOps, NumOpsAdded,
+                                    Builder);
+  case 1:
+    switch (op2) {
+    case 0:
+      switch (op3) {
+      case 0:
+        // A6.2.2 Data-processing
+        return DisassembleThumb1DP(MI, Opcode, insn, NumOps, NumOpsAdded,
+                                   Builder);
+      case 1:
+        // A6.2.3 Special data instructions and branch and exchange
+        return DisassembleThumb1Special(MI, Opcode, insn, NumOps, NumOpsAdded);
+      default:
+        // A8.6.59 LDR (literal)
+        return DisassembleThumb1LdPC(MI, Opcode, insn, NumOps, NumOpsAdded);
+      }
+      break;
+    default:
+      // A6.2.4 Load/store single data item
+      return DisassembleThumb1LdSt(opA, MI, Opcode, insn, NumOps, NumOpsAdded);
+      break;
+    }
+    break;
+  case 2:
+    switch (op2) {
+    case 0:
+      // A6.2.4 Load/store single data item
+      return DisassembleThumb1LdSt(opA, MI, Opcode, insn, NumOps, NumOpsAdded);
+    case 1:
+      // A6.2.4 Load/store single data item
+      return DisassembleThumb1LdStSP(MI, Opcode, insn, NumOps, NumOpsAdded);
+    case 2:
+      if (op3 <= 1) {
+        // A8.6.10 ADR
+        return DisassembleThumb1AddPCi(MI, Opcode, insn, NumOps, NumOpsAdded);
+      } else {
+        // A8.6.8 ADD (SP plus immediate)
+        return DisassembleThumb1AddSPi(MI, Opcode, insn, NumOps, NumOpsAdded);
+      }
+    default:
+      // A6.2.5 Miscellaneous 16-bit instructions
+      return DisassembleThumb1Misc(MI, Opcode, insn, NumOps, NumOpsAdded);
+    }
+    break;
+  case 3:
+    switch (op2) {
+    case 0:
+      if (op3 <= 1) {
+        // A8.6.189 STM / STMIA / STMEA
+        return DisassembleThumb1StMul(MI, Opcode, insn, NumOps, NumOpsAdded);
+      } else {
+        // A8.6.53 LDM / LDMIA / LDMFD
+        return DisassembleThumb1LdMul(MI, Opcode, insn, NumOps, NumOpsAdded);
+      }
+    case 1:
+      // A6.2.6 Conditional branch, and Supervisor Call
+      return DisassembleThumb1CondBr(MI, Opcode, insn, NumOps, NumOpsAdded);
+    case 2:
+      // Unconditional Branch, see B on page A8-44
+      return DisassembleThumb1Br(MI, Opcode, insn, NumOps, NumOpsAdded);
+    default:
+      assert(0 && "Unreachable code");
+      break;
+    }
+    break;
+  default:
+    assert(0 && "Unreachable code");
+    break;
+  }
+
+  return false;
+}
+
+///////////////////////////////////////////////
+//                                           //
+// Thumb2 instruction disassembly functions. //
+//                                           //
+///////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////
+//                                                       //
+// Note: the register naming follows the ARM convention! //
+//                                                       //
+///////////////////////////////////////////////////////////
+
+static inline bool Thumb2SRSOpcode(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    return false;
+  case ARM::t2SRSDBW: case ARM::t2SRSDB:
+  case ARM::t2SRSIAW: case ARM::t2SRSIA:
+    return true;
+  }
+}
+
+static inline bool Thumb2RFEOpcode(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    return false;
+  case ARM::t2RFEDBW: case ARM::t2RFEDB:
+  case ARM::t2RFEIAW: case ARM::t2RFEIA:
+    return true;
+  }
+}
+
+// t2SRS[IA|DB]W/t2SRS[IA|DB]: mode_imm = Inst{4-0}
+static bool DisassembleThumb2SRS(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+  MI.addOperand(MCOperand::CreateImm(slice(insn, 4, 0)));
+  NumOpsAdded = 1;
+  return true;
+}
+
+// t2RFE[IA|DB]W/t2RFE[IA|DB]: Rn
+static bool DisassembleThumb2RFE(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+  NumOpsAdded = 1;
+  return true;
+}
+
+static bool DisassembleThumb2LdStMul(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  if (Thumb2SRSOpcode(Opcode))
+    return DisassembleThumb2SRS(MI, Opcode, insn, NumOps, NumOpsAdded);
+
+  if (Thumb2RFEOpcode(Opcode))
+    return DisassembleThumb2RFE(MI, Opcode, insn, NumOps, NumOpsAdded);
+
+  assert((Opcode == ARM::t2LDM || Opcode == ARM::t2LDM_UPD ||
+          Opcode == ARM::t2STM || Opcode == ARM::t2STM_UPD)
+         && "Invalid opcode");
+  assert(NumOps >= 5 && "Thumb2 LdStMul expects NumOps >= 5");
+
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  unsigned Base = getRegisterEnum(ARM::GPRRegClassID, decodeRn(insn));
+
+  // Writeback to base.
+  if (Opcode == ARM::t2LDM_UPD || Opcode == ARM::t2STM_UPD) {
+    MI.addOperand(MCOperand::CreateReg(Base));
+    ++OpIdx;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(Base));
+  ++OpIdx;
+
+  ARM_AM::AMSubMode SubMode = getAMSubModeForBits(getPUBits(insn));
+  MI.addOperand(MCOperand::CreateImm(ARM_AM::getAM4ModeImm(SubMode)));
+  ++OpIdx;
+
+  // Handling the two predicate operands before the reglist.
+  MI.addOperand(MCOperand::CreateImm(ARMCC::AL));
+  MI.addOperand(MCOperand::CreateReg(ARM::CPSR));
+  OpIdx += 2;
+
+  // Fill the variadic part of reglist.
+  unsigned RegListBits = insn & ((1 << 16) - 1);
+  for (unsigned i = 0; i < 16; ++i) {
+    if ((RegListBits >> i) & 1) {
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                         i)));
+      ++OpIdx;
+    }
+  }
+
+  return true;
+}
+
+// t2LDREX: Rd Rn
+// t2LDREXD: Rd Rs Rn
+// t2LDREXB, t2LDREXH: Rd Rn
+// t2STREX: Rs Rd Rn
+// t2STREXD: Rm Rd Rs Rn
+// t2STREXB, t2STREXH: Rm Rd Rn
+static bool DisassembleThumb2LdStEx(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2
+         && OpInfo[0].RegClass == ARM::GPRRegClassID
+         && OpInfo[1].RegClass == ARM::GPRRegClassID
+         && "Expect >=2 operands and first two as reg operands");
+
+  bool isStore = (ARM::t2STREX <= Opcode && Opcode <= ARM::t2STREXH);
+  bool isSW = (Opcode == ARM::t2LDREX || Opcode == ARM::t2STREX);
+  bool isDW = (Opcode == ARM::t2LDREXD || Opcode == ARM::t2STREXD);
+
+  // Add the destination operand for store.
+  if (isStore) {
+    MI.addOperand(MCOperand::CreateReg(
+                    getRegisterEnum(ARM::GPRRegClassID,
+                                    isSW ? decodeRs(insn) : decodeRm(insn))));
+    ++OpIdx;
+  }
+
+  // Source operand for store and destination operand for load.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  ++OpIdx;
+
+  // Thumb2 doubleword complication: with an extra source/destination operand.
+  if (isDW) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRs(insn))));
+    ++OpIdx;
+  }
+
+  // Finally add the pointer operand.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+  ++OpIdx;
+
+  return true;
+}
+
+// LLVM, as of Jan-05-2010, does not output <Rt2>, i.e., Rs, in the asm.
+// Whereas the ARM Arch. Manual does not require that t2 = t+1 like in ARM ISA.
+//
+// t2LDRDi8: Rd Rs Rn imm8s4 (offset mode)
+// t2LDRDpci: Rd Rs imm8s4 (Not decoded, prefer the generic t2LDRDi8 version)
+// t2STRDi8: Rd Rs Rn imm8s4 (offset mode)
+//
+// Ditto for t2LDRD_PRE, t2LDRD_POST, t2STRD_PRE, t2STRD_POST, which are for
+// disassembly only and do not have a tied_to writeback base register operand.
+static bool DisassembleThumb2LdStDual(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  assert(NumOps >= 4
+         && OpInfo[0].RegClass == ARM::GPRRegClassID
+         && OpInfo[1].RegClass == ARM::GPRRegClassID
+         && OpInfo[2].RegClass == ARM::GPRRegClassID
+         && OpInfo[3].RegClass == 0
+         && "Expect >= 4 operands and first 3 as reg operands");
+
+  // Add the <Rt> <Rt2> operands.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRs(insn))));
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+
+  // Finally add (+/-)imm8*4, depending on the U bit.
+  int Offset = getImm8(insn) * 4;
+  if (getUBit(insn) == 0)
+    Offset = -Offset;
+  MI.addOperand(MCOperand::CreateImm(Offset));
+  NumOpsAdded = 4;
+
+  return true;
+}
+
+// PC-based defined for Codegen, which do not get decoded by design:
+//
+// t2TBB, t2TBH: Rm immDontCare immDontCare
+//
+// Generic version defined for disassembly:
+//
+// t2TBBgen, t2TBHgen: Rn Rm Pred-Imm Pred-CCR
+static bool DisassembleThumb2TB(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  assert(NumOps >= 2 && "Expect >= 2 operands");
+
+  // The generic version of TBB/TBH needs a base register.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+  // Add the index register.
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRm(insn))));
+  NumOpsAdded = 2;
+
+  return true;
+}
+
+static inline bool Thumb2ShiftOpcode(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    return false;
+  case ARM::t2MOVCClsl: case ARM::t2MOVCClsr:
+  case ARM::t2MOVCCasr: case ARM::t2MOVCCror:
+  case ARM::t2LSLri:    case ARM::t2LSRri:
+  case ARM::t2ASRri:    case ARM::t2RORri:
+    return true;
+  }
+}
+
+// A6.3.11 Data-processing (shifted register)
+//
+// Two register operands (Rn=0b1111 no 1st operand reg): Rs Rm
+// Two register operands (Rs=0b1111 no dst operand reg): Rn Rm
+// Three register operands: Rs Rn Rm
+// Three register operands: (Rn=0b1111 Conditional Move) Rs Ro(TIED_TO) Rm
+//
+// Constant shifts t2_so_reg is a 2-operand unit corresponding to the Thumb2
+// register with shift forms: (Rm, ConstantShiftSpecifier).
+// Constant shift specifier: Imm = (ShOp | ShAmt<<3).
+//
+// There are special instructions, like t2MOVsra_flag and t2MOVsrl_flag, which
+// only require two register operands: Rd, Rm in ARM Reference Manual terms, and
+// nothing else, because the shift amount is already specified.
+// Similar case holds for t2MOVrx, t2ADDrr, ..., etc.
+static bool DisassembleThumb2DPSoReg(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  // Special case handling.
+  if (Opcode == ARM::t2BR_JT) {
+    assert(NumOps == 4
+           && OpInfo[0].RegClass == ARM::GPRRegClassID
+           && OpInfo[1].RegClass == ARM::GPRRegClassID
+           && OpInfo[2].RegClass == 0
+           && OpInfo[3].RegClass == 0
+           && "Exactlt 4 operands expect and first two as reg operands");
+    // Only need to populate the src reg operand.
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+    MI.addOperand(MCOperand::CreateReg(0));
+    MI.addOperand(MCOperand::CreateImm(0));
+    MI.addOperand(MCOperand::CreateImm(0));
+    NumOpsAdded = 4;
+    return true;
+  }
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2
+         && OpInfo[0].RegClass == ARM::GPRRegClassID
+         && OpInfo[1].RegClass == ARM::GPRRegClassID
+         && "Expect >= 2 operands and first two as reg operands");
+
+  bool ThreeReg = (NumOps > 2 && OpInfo[2].RegClass == ARM::GPRRegClassID);
+  bool NoDstReg = (decodeRs(insn) == 0xF);
+
+  // Build the register operands, followed by the constant shift specifier.
+
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(ARM::GPRRegClassID,
+                                  NoDstReg ? decodeRn(insn) : decodeRs(insn))));
+  ++OpIdx;
+
+  if (ThreeReg) {
+    int Idx;
+    if ((Idx = TID.getOperandConstraint(OpIdx, TOI::TIED_TO)) != -1) {
+      // Process tied_to operand constraint.
+      MI.addOperand(MI.getOperand(Idx));
+    } else {
+      assert(!NoDstReg && "Internal error");
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                         decodeRn(insn))));
+    }
+    ++OpIdx;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRm(insn))));
+  ++OpIdx;
+
+  if (NumOps == OpIdx)
+    return true;
+
+  if (OpInfo[OpIdx].RegClass == 0 && !OpInfo[OpIdx].isPredicate()
+      && !OpInfo[OpIdx].isOptionalDef()) {
+
+    if (Thumb2ShiftOpcode(Opcode))
+      MI.addOperand(MCOperand::CreateImm(getShiftAmtBits(insn)));
+    else {
+      // Build the constant shift specifier operand.
+      unsigned bits2 = getShiftTypeBits(insn);
+      unsigned imm5 = getShiftAmtBits(insn);
+      ARM_AM::ShiftOpc ShOp = ARM_AM::no_shift;
+      unsigned ShAmt = decodeImmShift(bits2, imm5, ShOp);
+
+      // PKHBT/PKHTB are special in that we need the decodeImmShift() call to
+      // decode the shift amount from raw imm5 and bits2, but we DO NOT need
+      // to encode the ShOp, as it's in the asm string already.
+      if (Opcode == ARM::t2PKHBT || Opcode == ARM::t2PKHTB)
+        MI.addOperand(MCOperand::CreateImm(ShAmt));
+      else
+        MI.addOperand(MCOperand::CreateImm(ARM_AM::getSORegOpc(ShOp, ShAmt)));
+    }
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+// A6.3.1 Data-processing (modified immediate)
+//
+// Two register operands: Rs Rn ModImm
+// One register operands (Rs=0b1111 no explicit dest reg): Rn ModImm
+// One register operands (Rn=0b1111 no explicit src reg): Rs ModImm - {t2MOVi, t2MVNi}
+//
+// ModImm = ThumbExpandImm(i:imm3:imm8)
+static bool DisassembleThumb2DPModImm(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::GPRRegClassID
+         && "Expect >= 2 operands and first one as reg operand");
+
+  bool TwoReg = (OpInfo[1].RegClass == ARM::GPRRegClassID);
+  bool NoDstReg = (decodeRs(insn) == 0xF);
+
+  // Build the register operands, followed by the modified immediate.
+
+  MI.addOperand(MCOperand::CreateReg(
+                  getRegisterEnum(ARM::GPRRegClassID,
+                                  NoDstReg ? decodeRn(insn) : decodeRs(insn))));
+  ++OpIdx;
+
+  if (TwoReg) {
+    assert(!NoDstReg && "Internal error");
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    ++OpIdx;
+  }
+
+  // The modified immediate operand should come next.
+  assert(OpIdx < NumOps && OpInfo[OpIdx].RegClass == 0 &&
+         !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()
+         && "Pure imm operand expected");
+
+  // i:imm3:imm8
+  // A6.3.2 Modified immediate constants in Thumb instructions
+  unsigned imm12 = getIImm3Imm8(insn);
+  MI.addOperand(MCOperand::CreateImm(ThumbExpandImm(imm12)));
+  ++OpIdx;
+
+  return true;
+}
+
+static inline bool Thumb2SaturateOpcode(unsigned Opcode) {
+  switch (Opcode) {
+  case ARM::t2SSATlsl: case ARM::t2SSATasr: case ARM::t2SSAT16:
+  case ARM::t2USATlsl: case ARM::t2USATasr: case ARM::t2USAT16:
+    return true;
+  default:
+    return false;
+  }
+}
+
+static inline unsigned decodeThumb2SaturatePos(unsigned Opcode, uint32_t insn) {
+  switch (Opcode) {
+  case ARM::t2SSATlsl:
+  case ARM::t2SSATasr:
+    return slice(insn, 4, 0) + 1;
+  case ARM::t2SSAT16:
+    return slice(insn, 3, 0) + 1;
+  case ARM::t2USATlsl:
+  case ARM::t2USATasr:
+    return slice(insn, 4, 0);
+  case ARM::t2USAT16:
+    return slice(insn, 3, 0);
+  default:
+    assert(0 && "Invalid opcode passed in");
+    return 0;
+  }
+}
+
+// A6.3.3 Data-processing (plain binary immediate)
+//
+// o t2ADDri12, t2SUBri12: Rs Rn imm12
+// o t2LEApcrel (ADR): Rs imm12
+// o t2BFC (BFC): Rs Ro(TIED_TO) bf_inv_mask_imm
+// o t2BFI (BFI) (Currently not defined in LLVM as of Jan-07-2010)
+// o t2MOVi16: Rs imm16
+// o t2MOVTi16: Rs imm16
+// o t2SBFX (SBFX): Rs Rn lsb width
+// o t2UBFX (UBFX): Rs Rn lsb width
+// o t2BFI (BFI): Rs Rn lsb width
+//
+// [Signed|Unsigned] Saturate [16]
+//
+// o t2SSAT[lsl|asr], t2USAT[lsl|asr]: Rs sat_pos Rn shamt
+// o t2SSAT16, t2USAT16: Rs sat_pos Rn
+static bool DisassembleThumb2DPBinImm(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2 && OpInfo[0].RegClass == ARM::GPRRegClassID
+         && "Expect >= 2 operands and first one as reg operand");
+
+  bool TwoReg = (OpInfo[1].RegClass == ARM::GPRRegClassID);
+
+  // Build the register operand(s), followed by the immediate(s).
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRs(insn))));
+  ++OpIdx;
+
+  // t2SSAT/t2SSAT16/t2USAT/t2USAT16 has imm operand after Rd.
+  if (Thumb2SaturateOpcode(Opcode)) {
+    MI.addOperand(MCOperand::CreateImm(decodeThumb2SaturatePos(Opcode, insn)));
+
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+
+    if (Opcode == ARM::t2SSAT16 || Opcode == ARM::t2USAT16) {
+      OpIdx += 2;
+      return true;
+    }
+
+    // For SSAT operand reg (Rn) has been disassembled above.
+    // Now disassemble the shift amount.
+
+    // Inst{14-12:7-6} encodes the imm5 shift amount.
+    unsigned ShAmt = slice(insn, 14, 12) << 2 | slice(insn, 7, 6);
+
+    MI.addOperand(MCOperand::CreateImm(ShAmt));
+
+    OpIdx += 3;
+    return true;
+  }
+
+  if (TwoReg) {
+    assert(NumOps >= 3 && "Expect >= 3 operands");
+    int Idx;
+    if ((Idx = TID.getOperandConstraint(OpIdx, TOI::TIED_TO)) != -1) {
+      // Process tied_to operand constraint.
+      MI.addOperand(MI.getOperand(Idx));
+    } else {
+      // Add src reg operand.
+      MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                         decodeRn(insn))));
+    }
+    ++OpIdx;
+  }
+
+  assert(OpInfo[OpIdx].RegClass == 0 && !OpInfo[OpIdx].isPredicate()
+         && !OpInfo[OpIdx].isOptionalDef()
+         && "Pure imm operand expected");
+
+  // Pre-increment OpIdx.
+  ++OpIdx;
+
+  if (Opcode == ARM::t2ADDri12 || Opcode == ARM::t2SUBri12
+      || Opcode == ARM::t2LEApcrel)
+    MI.addOperand(MCOperand::CreateImm(getIImm3Imm8(insn)));
+  else if (Opcode == ARM::t2MOVi16 || Opcode == ARM::t2MOVTi16)
+    MI.addOperand(MCOperand::CreateImm(getImm16(insn)));
+  else if (Opcode == ARM::t2BFC)
+    MI.addOperand(MCOperand::CreateImm(getBitfieldInvMask(insn)));
+  else {
+    // Handle the case of: lsb width
+    assert((Opcode == ARM::t2SBFX || Opcode == ARM::t2UBFX ||
+            Opcode == ARM::t2BFI) && "Invalid opcode");
+    MI.addOperand(MCOperand::CreateImm(getLsb(insn)));
+    if (Opcode == ARM::t2BFI) {
+      assert(getMsb(insn) >= getLsb(insn) && "Encoding error");
+      MI.addOperand(MCOperand::CreateImm(getMsb(insn) - getLsb(insn) + 1));
+    } else
+      MI.addOperand(MCOperand::CreateImm(getWidthMinus1(insn) + 1));
+
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+// A6.3.4 Table A6-15 Miscellaneous control instructions
+// A8.6.41 DMB
+// A8.6.42 DSB
+// A8.6.49 ISB
+static inline bool t2MiscCtrlInstr(uint32_t insn) {
+  if (slice(insn, 31, 20) == 0xf3b && slice(insn, 15, 14) == 2 &&
+      slice(insn, 12, 12) == 0)
+    return true;
+
+  return false;
+}
+
+// A6.3.4 Branches and miscellaneous control
+//
+// A8.6.16 B
+// Branches: t2B, t2Bcc -> imm operand
+//
+// Branches: t2TPsoft -> no operand
+//
+// A8.6.23 BL, BLX (immediate)
+// Branches (defined in ARMInstrThumb.td): tBLr9, tBLXi_r9 -> imm operand
+//
+// A8.6.26
+// t2BXJ -> Rn
+//
+// Miscellaneous control: t2Int_MemBarrierV7 (and its t2DMB variants),
+// t2Int_SyncBarrierV7 (and its t2DSB varianst), t2ISBsy, t2CLREX
+//   -> no operand (except pred-imm pred-ccr for CLREX, memory barrier variants)
+//
+// Hint: t2NOP, t2YIELD, t2WFE, t2WFI, t2SEV
+//   -> no operand (except pred-imm pred-ccr)
+//
+// t2DBG -> imm4 = Inst{3-0}
+//
+// t2MRS/t2MRSsys -> Rs
+// t2MSR/t2MSRsys -> Rn mask=Inst{11-8}
+// t2SMC -> imm4 = Inst{19-16}
+static bool DisassembleThumb2BrMiscCtrl(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  if (NumOps == 0)
+    return true;
+
+  if (t2MiscCtrlInstr(insn))
+    return true;
+
+  switch (Opcode) {
+  case ARM::t2CLREX:
+  case ARM::t2NOP:
+  case ARM::t2YIELD:
+  case ARM::t2WFE:
+  case ARM::t2WFI:
+  case ARM::t2SEV:
+    return true;
+  default:
+    break;
+  }
+
+  // CPS has a singleton $opt operand that contains the following information:
+  // opt{4-0} = mode from Inst{4-0}
+  // opt{5} = changemode from Inst{8}
+  // opt{8-6} = AIF from Inst{7-5}
+  // opt{10-9} = imod from Inst{10-9} with 0b10 as enable and 0b11 as disable
+  if (Opcode == ARM::t2CPS) {
+    unsigned Option = slice(insn, 4, 0) | slice(insn, 8, 8) << 5 |
+      slice(insn, 7, 5) << 6 | slice(insn, 10, 9) << 9;
+    MI.addOperand(MCOperand::CreateImm(Option));
+    NumOpsAdded = 1;
+    return true;
+  }
+
+  // DBG has its option specified in Inst{3-0}.
+  if (Opcode == ARM::t2DBG) {
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 3, 0)));
+    NumOpsAdded = 1;
+    return true;
+  }
+
+  // MRS and MRSsys take one GPR reg Rs.
+  if (Opcode == ARM::t2MRS || Opcode == ARM::t2MRSsys) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRs(insn))));
+    NumOpsAdded = 1;
+    return true;
+  }
+  // BXJ takes one GPR reg Rn.
+  if (Opcode == ARM::t2BXJ) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    NumOpsAdded = 1;
+    return true;
+  }
+  // MSR and MSRsys take one GPR reg Rn, followed by the mask.
+  if (Opcode == ARM::t2MSR || Opcode == ARM::t2MSRsys || Opcode == ARM::t2BXJ) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 11, 8)));
+    NumOpsAdded = 2;
+    return true;
+  }
+  // SMC take imm4.
+  if (Opcode == ARM::t2SMC) {
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 19, 16)));
+    NumOpsAdded = 1;
+    return true;
+  }
+
+  // Add the imm operand.
+  int Offset = 0;
+
+  switch (Opcode) {
+  default:
+    assert(0 && "Unreachable code");
+    return false;
+  case ARM::t2B:
+    Offset = decodeImm32_B_EncodingT4(insn);
+    break;
+  case ARM::t2Bcc:
+    Offset = decodeImm32_B_EncodingT3(insn);
+    break;
+  case ARM::tBLr9:
+    Offset = decodeImm32_BL(insn);
+    break;
+  case ARM::tBLXi_r9:
+    Offset = decodeImm32_BLX(insn);
+    break;
+  }
+  // When executing a Thumb instruction, PC reads as the address of the current
+  // instruction plus 4.  The assembler subtracts 4 from the difference between
+  // the branch instruction and the target address, disassembler has to add 4 to
+  // to compensate.
+  MI.addOperand(MCOperand::CreateImm(Offset + 4));
+
+  NumOpsAdded = 1;
+
+  return true;
+}
+
+static inline bool Thumb2PreloadOpcode(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    return false;
+  case ARM::t2PLDi12:   case ARM::t2PLDi8:   case ARM::t2PLDpci:
+  case ARM::t2PLDr:     case ARM::t2PLDs:
+  case ARM::t2PLDWi12:  case ARM::t2PLDWi8:  case ARM::t2PLDWpci:
+  case ARM::t2PLDWr:    case ARM::t2PLDWs:
+  case ARM::t2PLIi12:   case ARM::t2PLIi8:   case ARM::t2PLIpci:
+  case ARM::t2PLIr:     case ARM::t2PLIs:
+    return true;
+  }
+}
+
+static bool DisassembleThumb2PreLoad(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  // Preload Data/Instruction requires either 2 or 3 operands.
+  // t2PLDi12, t2PLDi8, t2PLDpci: Rn [+/-]imm12/imm8
+  // t2PLDr:                      Rn Rm
+  // t2PLDs:                      Rn Rm imm2=Inst{5-4}
+  // Same pattern applies for t2PLDW* and t2PLI*.
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2 &&
+         OpInfo[0].RegClass == ARM::GPRRegClassID &&
+         "Expect >= 2 operands and first one as reg operand");
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+  ++OpIdx;
+
+  if (OpInfo[OpIdx].RegClass == ARM::GPRRegClassID) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRm(insn))));
+  } else {
+    assert(OpInfo[OpIdx].RegClass == 0 && !OpInfo[OpIdx].isPredicate()
+           && !OpInfo[OpIdx].isOptionalDef()
+           && "Pure imm operand expected");
+    int Offset = 0;
+    if (Opcode == ARM::t2PLDpci || Opcode == ARM::t2PLDWpci ||
+             Opcode == ARM::t2PLIpci) {
+      bool Negative = slice(insn, 23, 23) == 0;
+      unsigned Imm12 = getImm12(insn);
+      Offset = Negative ? -1 - Imm12 : 1 * Imm12;      
+    } else if (Opcode == ARM::t2PLDi8 || Opcode == ARM::t2PLDWi8 ||
+               Opcode == ARM::t2PLIi8) {
+      // A8.6.117 Encoding T2: add = FALSE
+      unsigned Imm8 = getImm8(insn);
+      Offset = -1 - Imm8;
+    } else // The i12 forms.  See, for example, A8.6.117 Encoding T1.
+      Offset = decodeImm12(insn);
+    MI.addOperand(MCOperand::CreateImm(Offset));
+  }
+  ++OpIdx;
+
+  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == 0 &&
+      !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
+    // Fills in the shift amount for t2PLDs, t2PLDWs, t2PLIs.
+    MI.addOperand(MCOperand::CreateImm(slice(insn, 5, 4)));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+// A8.6.63 LDRB (literal)
+// A8.6.79 LDRSB (literal)
+// A8.6.75 LDRH (literal)
+// A8.6.83 LDRSH (literal)
+// A8.6.59 LDR (literal)
+//
+// These instrs calculate an address from the PC value and an immediate offset.
+// Rd Rn=PC (+/-)imm12 (+ if Inst{23} == 0b1)
+static bool DisassembleThumb2Ldpci(MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  assert(NumOps >= 2 &&
+         OpInfo[0].RegClass == ARM::GPRRegClassID &&
+         OpInfo[1].RegClass == 0 &&
+         "Expect >= 2 operands, first as reg, and second as imm operand");
+
+  // Build the register operand, followed by the (+/-)imm12 immediate.
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRd(insn))));
+
+  MI.addOperand(MCOperand::CreateImm(decodeImm12(insn)));
+
+  NumOpsAdded = 2;
+
+  return true;
+}
+
+// A6.3.10 Store single data item
+// A6.3.9 Load byte, memory hints
+// A6.3.8 Load halfword, memory hints
+// A6.3.7 Load word
+//
+// For example,
+//
+// t2LDRi12:   Rd Rn (+)imm12
+// t2LDRi8:    Rd Rn (+/-)imm8 (+ if Inst{9} == 0b1)
+// t2LDRs:     Rd Rn Rm ConstantShiftSpecifier (see also DisassembleThumb2DPSoReg)
+// t2LDR_POST: Rd Rn Rn(TIED_TO) (+/-)imm8 (+ if Inst{9} == 0b1)
+// t2LDR_PRE:  Rd Rn Rn(TIED_TO) (+/-)imm8 (+ if Inst{9} == 0b1)
+//
+// t2STRi12:   Rd Rn (+)imm12
+// t2STRi8:    Rd Rn (+/-)imm8 (+ if Inst{9} == 0b1)
+// t2STRs:     Rd Rn Rm ConstantShiftSpecifier (see also DisassembleThumb2DPSoReg)
+// t2STR_POST: Rn Rd Rn(TIED_TO) (+/-)imm8 (+ if Inst{9} == 0b1)
+// t2STR_PRE:  Rn Rd Rn(TIED_TO) (+/-)imm8 (+ if Inst{9} == 0b1)
+//
+// Note that for indexed modes, the Rn(TIED_TO) operand needs to be populated
+// correctly, as LLVM AsmPrinter depends on it.  For indexed stores, the first
+// operand is Rn; for all the other instructions, Rd is the first operand.
+//
+// Delegates to DisassembleThumb2PreLoad() for preload data/instruction.
+// Delegates to DisassembleThumb2Ldpci() for load * literal operations.
+static bool DisassembleThumb2LdSt(bool Load, MCInst &MI, unsigned Opcode,
+    uint32_t insn, unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  unsigned Rn = decodeRn(insn);
+
+  if (Thumb2PreloadOpcode(Opcode))
+    return DisassembleThumb2PreLoad(MI, Opcode, insn, NumOps, NumOpsAdded);
+
+  // See, for example, A6.3.7 Load word: Table A6-18 Load word.
+  if (Load && Rn == 15)
+    return DisassembleThumb2Ldpci(MI, Opcode, insn, NumOps, NumOpsAdded);
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 3 &&
+         OpInfo[0].RegClass == ARM::GPRRegClassID &&
+         OpInfo[1].RegClass == ARM::GPRRegClassID &&
+         "Expect >= 3 operands and first two as reg operands");
+
+  bool ThreeReg = (OpInfo[2].RegClass == ARM::GPRRegClassID);
+  bool TIED_TO = ThreeReg && TID.getOperandConstraint(2, TOI::TIED_TO) != -1;
+  bool Imm12 = !ThreeReg && slice(insn, 23, 23) == 1; // ARMInstrThumb2.td
+
+  // Build the register operands, followed by the immediate.
+  unsigned R0, R1, R2 = 0;
+  unsigned Rd = decodeRd(insn);
+  int Imm = 0;
+
+  if (!Load && TIED_TO) {
+    R0 = Rn;
+    R1 = Rd;
+  } else {
+    R0 = Rd;
+    R1 = Rn;
+  }
+  if (ThreeReg) {
+    if (TIED_TO) {
+      R2 = Rn;
+      Imm = decodeImm8(insn);
+    } else {
+      R2 = decodeRm(insn);
+      // See, for example, A8.6.64 LDRB (register).
+      // And ARMAsmPrinter::printT2AddrModeSoRegOperand().
+      // LSL is the default shift opc, and LLVM does not expect it to be encoded
+      // as part of the immediate operand.
+      // Imm = ARM_AM::getSORegOpc(ARM_AM::lsl, slice(insn, 5, 4));
+      Imm = slice(insn, 5, 4);
+    }
+  } else {
+    if (Imm12)
+      Imm = getImm12(insn);
+    else
+      Imm = decodeImm8(insn);
+  }
+  
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID, R0)));
+  ++OpIdx;
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID, R1)));
+  ++OpIdx;
+
+  if (ThreeReg) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,R2)));
+    ++OpIdx;
+  }
+
+  assert(OpInfo[OpIdx].RegClass == 0 && !OpInfo[OpIdx].isPredicate()
+         && !OpInfo[OpIdx].isOptionalDef()
+         && "Pure imm operand expected");
+
+  MI.addOperand(MCOperand::CreateImm(Imm));
+  ++OpIdx;
+
+  return true;
+}
+
+// A6.3.12 Data-processing (register)
+//
+// Two register operands [rotate]:   Rs Rm [rotation(= (rotate:'000'))]
+// Three register operands only:     Rs Rn Rm
+// Three register operands [rotate]: Rs Rn Rm [rotation(= (rotate:'000'))]
+//
+// Parallel addition and subtraction 32-bit Thumb instructions: Rs Rn Rm
+//
+// Miscellaneous operations: Rs [Rn] Rm
+static bool DisassembleThumb2DPReg(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetInstrDesc &TID = ARMInsts[Opcode];
+  const TargetOperandInfo *OpInfo = TID.OpInfo;
+  unsigned &OpIdx = NumOpsAdded;
+
+  OpIdx = 0;
+
+  assert(NumOps >= 2 &&
+         OpInfo[0].RegClass == ARM::GPRRegClassID &&
+         OpInfo[1].RegClass == ARM::GPRRegClassID &&
+         "Expect >= 2 operands and first two as reg operands");
+
+  // Build the register operands, followed by the optional rotation amount.
+
+  bool ThreeReg = NumOps > 2 && OpInfo[2].RegClass == ARM::GPRRegClassID;
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRs(insn))));
+  ++OpIdx;
+
+  if (ThreeReg) {
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRn(insn))));
+    ++OpIdx;
+  }
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRm(insn))));
+  ++OpIdx;
+
+  if (OpIdx < NumOps && OpInfo[OpIdx].RegClass == 0
+      && !OpInfo[OpIdx].isPredicate() && !OpInfo[OpIdx].isOptionalDef()) {
+    // Add the rotation amount immediate.
+    MI.addOperand(MCOperand::CreateImm(decodeRotate(insn)));
+    ++OpIdx;
+  }
+
+  return true;
+}
+
+// A6.3.16 Multiply, multiply accumulate, and absolute difference
+//
+// t2MLA, t2MLS, t2SMMLA, t2SMMLS: Rs Rn Rm Ra=Inst{15-12}
+// t2MUL, t2SMMUL:                 Rs Rn Rm
+// t2SMLA[BB|BT|TB|TT|WB|WT]:      Rs Rn Rm Ra=Inst{15-12}
+// t2SMUL[BB|BT|TB|TT|WB|WT]:      Rs Rn Rm
+//
+// Dual halfword multiply: t2SMUAD[X], t2SMUSD[X], t2SMLAD[X], t2SMLSD[X]:
+//   Rs Rn Rm Ra=Inst{15-12}
+//
+// Unsigned Sum of Absolute Differences [and Accumulate]
+//    Rs Rn Rm [Ra=Inst{15-12}]
+static bool DisassembleThumb2Mul(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  assert(NumOps >= 3 &&
+         OpInfo[0].RegClass == ARM::GPRRegClassID &&
+         OpInfo[1].RegClass == ARM::GPRRegClassID &&
+         OpInfo[2].RegClass == ARM::GPRRegClassID &&
+         "Expect >= 3 operands and first three as reg operands");
+
+  // Build the register operands.
+
+  bool FourReg = NumOps > 3 && OpInfo[3].RegClass == ARM::GPRRegClassID;
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRs(insn))));
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRm(insn))));
+
+  if (FourReg)
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRd(insn))));
+
+  NumOpsAdded = FourReg ? 4 : 3;
+
+  return true;
+}
+
+// A6.3.17 Long multiply, long multiply accumulate, and divide
+//
+// t2SMULL, t2UMULL, t2SMLAL, t2UMLAL, t2UMAAL: RdLo RdHi Rn Rm
+// where RdLo = Inst{15-12} and RdHi = Inst{11-8}
+//
+// Halfword multiple accumulate long: t2SMLAL<x><y>: RdLo RdHi Rn Rm
+// where RdLo = Inst{15-12} and RdHi = Inst{11-8}
+//
+// Dual halfword multiple: t2SMLALD[X], t2SMLSLD[X]: RdLo RdHi Rn Rm
+// where RdLo = Inst{15-12} and RdHi = Inst{11-8}
+//
+// Signed/Unsigned divide: t2SDIV, t2UDIV: Rs Rn Rm
+static bool DisassembleThumb2LongMul(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  const TargetOperandInfo *OpInfo = ARMInsts[Opcode].OpInfo;
+
+  assert(NumOps >= 3 &&
+         OpInfo[0].RegClass == ARM::GPRRegClassID &&
+         OpInfo[1].RegClass == ARM::GPRRegClassID &&
+         OpInfo[2].RegClass == ARM::GPRRegClassID &&
+         "Expect >= 3 operands and first three as reg operands");
+
+  bool FourReg = NumOps > 3 && OpInfo[3].RegClass == ARM::GPRRegClassID;
+
+  // Build the register operands.
+
+  if (FourReg)
+    MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                       decodeRd(insn))));
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRs(insn))));
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRn(insn))));
+
+  MI.addOperand(MCOperand::CreateReg(getRegisterEnum(ARM::GPRRegClassID,
+                                                     decodeRm(insn))));
+
+  if (FourReg)
+    NumOpsAdded = 4;
+  else
+    NumOpsAdded = 3;
+
+  return true;
+}
+
+// See A6.3 32-bit Thumb instruction encoding for instruction classes
+// corresponding to (op1, op2, op).
+//
+// Table A6-9 32-bit Thumb instruction encoding
+// op1	op2		op	Instruction class, see
+// ---	-------	--	------------------------------------------------------------
+// 01	00xx0xx	-	Load/store multiple on page A6-23
+// 		00xx1xx	-	Load/store dual, load/store exclusive, table branch on page A6-24
+// 		01xxxxx	-	Data-processing (shifted register) on page A6-31
+// 		1xxxxxx	-	Coprocessor instructions on page A6-40
+// 10	x0xxxxx	0	Data-processing (modified immediate) on page A6-15
+// 		x1xxxxx	0	Data-processing (plain binary immediate) on page A6-19
+// 		-		1	Branches and miscellaneous control on page A6-20
+// 11	000xxx0	-	Store single data item on page A6-30
+// 		001xxx0	-	Advanced SIMD element or structure load/store instructions on page A7-27
+// 		00xx001 -	Load byte, memory hints on page A6-28
+// 		00xx011	-	Load halfword, memory hints on page A6-26
+// 		00xx101	-	Load word on page A6-25
+// 		00xx111	-	UNDEFINED
+// 		010xxxx	-	Data-processing (register) on page A6-33
+// 		0110xxx	-	Multiply, multiply accumulate, and absolute difference on page A6-38
+// 		0111xxx	-	Long multiply, long multiply accumulate, and divide on page A6-39
+// 		1xxxxxx	-	Coprocessor instructions on page A6-40
+//
+static bool DisassembleThumb2(uint16_t op1, uint16_t op2, uint16_t op,
+    MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded) {
+
+  switch (op1) {
+  case 1:
+    if (slice(op2, 6, 5) == 0) {
+      if (slice(op2, 2, 2) == 0) {
+        // Load/store multiple.
+        return DisassembleThumb2LdStMul(MI, Opcode, insn, NumOps, NumOpsAdded);
+      }
+
+      // Load/store dual, load/store exclusive, table branch, otherwise.
+      assert(slice(op2, 2, 2) == 1 && "Encoding error");
+      if ((ARM::t2LDREX <= Opcode && Opcode <= ARM::t2LDREXH) ||
+          (ARM::t2STREX <= Opcode && Opcode <= ARM::t2STREXH)) {
+        // Load/store exclusive.
+        return DisassembleThumb2LdStEx(MI, Opcode, insn, NumOps, NumOpsAdded);
+      }
+      if (Opcode == ARM::t2LDRDi8 ||
+          Opcode == ARM::t2LDRD_PRE || Opcode == ARM::t2LDRD_POST ||
+          Opcode == ARM::t2STRDi8 ||
+          Opcode == ARM::t2STRD_PRE || Opcode == ARM::t2STRD_POST) {
+        // Load/store dual.
+        return DisassembleThumb2LdStDual(MI, Opcode, insn, NumOps, NumOpsAdded);
+      }
+      if (Opcode == ARM::t2TBBgen || Opcode == ARM::t2TBHgen) {
+        // Table branch.
+        return DisassembleThumb2TB(MI, Opcode, insn, NumOps, NumOpsAdded);
+      }
+    } else if (slice(op2, 6, 5) == 1) {
+      // Data-processing (shifted register).
+      return DisassembleThumb2DPSoReg(MI, Opcode, insn, NumOps, NumOpsAdded);
+    }
+
+    // FIXME: A6.3.18 Coprocessor instructions
+    // But see ThumbDisassembler::getInstruction().
+
+    break;
+  case 2:
+    if (op == 0) {
+      if (slice(op2, 5, 5) == 0) {
+        // Data-processing (modified immediate)
+        return DisassembleThumb2DPModImm(MI, Opcode, insn, NumOps, NumOpsAdded);
+      } else {
+        // Data-processing (plain binary immediate)
+        return DisassembleThumb2DPBinImm(MI, Opcode, insn, NumOps, NumOpsAdded);
+      }
+    } else {
+      // Branches and miscellaneous control on page A6-20.
+      return DisassembleThumb2BrMiscCtrl(MI, Opcode, insn, NumOps, NumOpsAdded);
+    }
+
+    break;
+  case 3:
+    switch (slice(op2, 6, 5)) {
+    case 0:
+      // Load/store instructions...
+      if (slice(op2, 0, 0) == 0) {
+        if (slice(op2, 4, 4) == 0) {
+          // Store single data item on page A6-30
+          return DisassembleThumb2LdSt(false, MI,Opcode,insn,NumOps,NumOpsAdded);
+        } else {
+          // FIXME: Advanced SIMD element or structure load/store instructions.
+          // But see ThumbDisassembler::getInstruction().
+          ;
+        }
+      } else {
+        // Table A6-9 32-bit Thumb instruction encoding: Load byte|halfword|word
+        return DisassembleThumb2LdSt(true, MI,Opcode,insn,NumOps,NumOpsAdded);
+      }
+      break;
+    case 1:
+      if (slice(op2, 4, 4) == 0) {
+        // A6.3.12 Data-processing (register)
+        return DisassembleThumb2DPReg(MI, Opcode, insn, NumOps, NumOpsAdded);
+      } else if (slice(op2, 3, 3) == 0) {
+        // A6.3.16 Multiply, multiply accumulate, and absolute difference
+        return DisassembleThumb2Mul(MI, Opcode, insn, NumOps, NumOpsAdded);
+      } else {
+        // A6.3.17 Long multiply, long multiply accumulate, and divide
+        return DisassembleThumb2LongMul(MI, Opcode, insn, NumOps, NumOpsAdded);
+      }
+      break;
+    default:
+      // FIXME: A6.3.18 Coprocessor instructions
+      // But see ThumbDisassembler::getInstruction().
+      ;
+      break;
+    }
+
+    break;
+  default:
+    assert(0 && "Encoding error for Thumb2 instruction!");
+    break;
+  }
+
+  return false;
+}
+
+static bool DisassembleThumbFrm(MCInst &MI, unsigned Opcode, uint32_t insn,
+    unsigned short NumOps, unsigned &NumOpsAdded, BO Builder) {
+
+  uint16_t HalfWord = slice(insn, 31, 16);
+
+  if (HalfWord == 0) {
+    // A6.2 16-bit Thumb instruction encoding
+    // op = bits[15:10]
+    uint16_t op = slice(insn, 15, 10);
+    return DisassembleThumb1(op, MI, Opcode, insn, NumOps, NumOpsAdded,
+                             Builder);
+  }
+
+  unsigned bits15_11 = slice(HalfWord, 15, 11);
+
+  // A6.1 Thumb instruction set encoding
+  assert((bits15_11 == 0x1D || bits15_11 == 0x1E || bits15_11 == 0x1F) &&
+         "Bits [15:11] of first halfword of a Thumb2 instruction out of range");
+
+  // A6.3 32-bit Thumb instruction encoding
+  
+  uint16_t op1 = slice(HalfWord, 12, 11);
+  uint16_t op2 = slice(HalfWord, 10, 4);
+  uint16_t op = slice(insn, 15, 15);
+
+  return DisassembleThumb2(op1, op2, op, MI, Opcode, insn, NumOps, NumOpsAdded);
+}
diff --git a/lib/Target/ARM/Makefile b/lib/Target/ARM/Makefile
index a8dd38cb362e..b7ed14abed78 100644
--- a/lib/Target/ARM/Makefile
+++ b/lib/Target/ARM/Makefile
@@ -16,8 +16,9 @@ BUILT_SOURCES = ARMGenRegisterInfo.h.inc ARMGenRegisterNames.inc \
                 ARMGenRegisterInfo.inc ARMGenInstrNames.inc \
                 ARMGenInstrInfo.inc ARMGenAsmWriter.inc \
                 ARMGenDAGISel.inc ARMGenSubtarget.inc \
-                ARMGenCodeEmitter.inc ARMGenCallingConv.inc
+                ARMGenCodeEmitter.inc ARMGenCallingConv.inc \
+		ARMGenDecoderTables.inc
 
-DIRS = AsmPrinter AsmParser TargetInfo
+DIRS = AsmPrinter AsmParser Disassembler TargetInfo
 
 include $(LEVEL)/Makefile.common
diff --git a/lib/Target/ARM/Thumb1InstrInfo.cpp b/lib/Target/ARM/Thumb1InstrInfo.cpp
index ad98839ce262..b10c3f74c70a 100644
--- a/lib/Target/ARM/Thumb1InstrInfo.cpp
+++ b/lib/Target/ARM/Thumb1InstrInfo.cpp
@@ -37,7 +37,7 @@ bool Thumb1InstrInfo::copyRegToReg(MachineBasicBlock &MBB,
                                    unsigned DestReg, unsigned SrcReg,
                                    const TargetRegisterClass *DestRC,
                                    const TargetRegisterClass *SrcRC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (DestRC == ARM::GPRRegisterClass) {
@@ -98,7 +98,7 @@ void Thumb1InstrInfo::
 storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                     unsigned SrcReg, bool isKill, int FI,
                     const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   assert((RC == ARM::tGPRRegisterClass ||
@@ -125,7 +125,7 @@ void Thumb1InstrInfo::
 loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                      unsigned DestReg, int FI,
                      const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   assert((RC == ARM::tGPRRegisterClass ||
@@ -154,7 +154,7 @@ spillCalleeSavedRegisters(MachineBasicBlock &MBB,
   if (CSI.empty())
     return false;
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, get(ARM::tPUSH));
diff --git a/lib/Target/ARM/Thumb1RegisterInfo.cpp b/lib/Target/ARM/Thumb1RegisterInfo.cpp
index 99c38b1f4d4b..a5dfcb34f7bc 100644
--- a/lib/Target/ARM/Thumb1RegisterInfo.cpp
+++ b/lib/Target/ARM/Thumb1RegisterInfo.cpp
@@ -398,7 +398,7 @@ Thumb1RegisterInfo::saveScavengerRegister(MachineBasicBlock &MBB,
   // off the frame pointer (if, for example, there are alloca() calls in
   // the function, the offset will be negative. Use R12 instead since that's
   // a call clobbered register that we know won't be used in Thumb1 mode.
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   BuildMI(MBB, I, DL, TII.get(ARM::tMOVtgpr2gpr)).
     addReg(ARM::R12, RegState::Define).addReg(Reg, RegState::Kill);
 
@@ -685,8 +685,7 @@ void Thumb1RegisterInfo::emitPrologue(MachineFunction &MF) const {
   unsigned VARegSaveSize = AFI->getVarArgsRegSaveSize();
   unsigned NumBytes = MFI->getStackSize();
   const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
-  DebugLoc dl = (MBBI != MBB.end() ?
-                 MBBI->getDebugLoc() : DebugLoc::getUnknownLoc());
+  DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   // Thumb add/sub sp, imm8 instructions implicitly multiply the offset by 4.
   NumBytes = (NumBytes + 3) & ~3;
diff --git a/lib/Target/ARM/Thumb2InstrInfo.cpp b/lib/Target/ARM/Thumb2InstrInfo.cpp
index 55163f9b820c..de4605669315 100644
--- a/lib/Target/ARM/Thumb2InstrInfo.cpp
+++ b/lib/Target/ARM/Thumb2InstrInfo.cpp
@@ -41,7 +41,7 @@ Thumb2InstrInfo::copyRegToReg(MachineBasicBlock &MBB,
                               unsigned DestReg, unsigned SrcReg,
                               const TargetRegisterClass *DestRC,
                               const TargetRegisterClass *SrcRC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (DestRC == ARM::GPRRegisterClass &&
@@ -66,7 +66,7 @@ void Thumb2InstrInfo::
 storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                     unsigned SrcReg, bool isKill, int FI,
                     const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (RC == ARM::GPRRegisterClass || RC == ARM::tGPRRegisterClass) {
@@ -90,7 +90,7 @@ void Thumb2InstrInfo::
 loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                      unsigned DestReg, int FI,
                      const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (RC == ARM::GPRRegisterClass || RC == ARM::tGPRRegisterClass) {
diff --git a/lib/Target/Alpha/AlphaISelLowering.cpp b/lib/Target/Alpha/AlphaISelLowering.cpp
index 5d8310ec840b..94c6f80c0361 100644
--- a/lib/Target/Alpha/AlphaISelLowering.cpp
+++ b/lib/Target/Alpha/AlphaISelLowering.cpp
@@ -471,8 +471,7 @@ AlphaTargetLowering::LowerReturn(SDValue Chain,
 
   SDValue Copy = DAG.getCopyToReg(Chain, dl, Alpha::R26,
                                   DAG.getNode(AlphaISD::GlobalRetAddr,
-                                              DebugLoc::getUnknownLoc(),
-                                              MVT::i64),
+                                              DebugLoc(), MVT::i64),
                                   SDValue());
   switch (Outs.size()) {
   default:
@@ -740,8 +739,7 @@ SDValue AlphaTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
                              SA2, NULL, 0, MVT::i32, false, false, 0);
   }
   case ISD::RETURNADDR:
-    return DAG.getNode(AlphaISD::GlobalRetAddr, DebugLoc::getUnknownLoc(),
-                       MVT::i64);
+    return DAG.getNode(AlphaISD::GlobalRetAddr, DebugLoc(), MVT::i64);
       //FIXME: implement
   case ISD::FRAMEADDR:          break;
   }
diff --git a/lib/Target/Alpha/AlphaInstrInfo.cpp b/lib/Target/Alpha/AlphaInstrInfo.cpp
index d539e082118b..ba403e22ce35 100644
--- a/lib/Target/Alpha/AlphaInstrInfo.cpp
+++ b/lib/Target/Alpha/AlphaInstrInfo.cpp
@@ -112,7 +112,7 @@ unsigned AlphaInstrInfo::InsertBranch(MachineBasicBlock &MBB,
                                       MachineBasicBlock *FBB,
                             const SmallVectorImpl<MachineOperand> &Cond) const {
   // FIXME this should probably have a DebugLoc argument
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
   assert((Cond.size() == 2 || Cond.size() == 0) && 
          "Alpha branch conditions have two components!");
@@ -153,7 +153,7 @@ bool AlphaInstrInfo::copyRegToReg(MachineBasicBlock &MBB,
     return false;
   }
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   if (DestRC == Alpha::GPRCRegisterClass) {
@@ -185,7 +185,7 @@ AlphaInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   //     << FrameIdx << "\n";
   //BuildMI(MBB, MI, Alpha::WTF, 0).addReg(SrcReg);
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   if (RC == Alpha::F4RCRegisterClass)
@@ -211,7 +211,7 @@ AlphaInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                         const TargetRegisterClass *RC) const {
   //cerr << "Trying to load " << getPrettyName(DestReg) << " to "
   //     << FrameIdx << "\n";
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   if (RC == Alpha::F4RCRegisterClass)
@@ -398,7 +398,7 @@ unsigned AlphaInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
 
 void AlphaInstrInfo::insertNoop(MachineBasicBlock &MBB, 
                                 MachineBasicBlock::iterator MI) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
   BuildMI(MBB, MI, DL, get(Alpha::BISr), Alpha::R31)
     .addReg(Alpha::R31)
diff --git a/lib/Target/Alpha/AlphaLLRP.cpp b/lib/Target/Alpha/AlphaLLRP.cpp
index 0c51bc554be9..34be470f03e3 100644
--- a/lib/Target/Alpha/AlphaLLRP.cpp
+++ b/lib/Target/Alpha/AlphaLLRP.cpp
@@ -49,7 +49,7 @@ namespace {
       const TargetInstrInfo *TII = F.getTarget().getInstrInfo();
       bool Changed = false;
       MachineInstr* prev[3] = {0,0,0};
-      DebugLoc dl = DebugLoc::getUnknownLoc();
+      DebugLoc dl;
       unsigned count = 0;
       for (MachineFunction::iterator FI = F.begin(), FE = F.end();
            FI != FE; ++FI) {
diff --git a/lib/Target/Alpha/AlphaRegisterInfo.cpp b/lib/Target/Alpha/AlphaRegisterInfo.cpp
index 55eec3ae4450..16a23cc120fb 100644
--- a/lib/Target/Alpha/AlphaRegisterInfo.cpp
+++ b/lib/Target/Alpha/AlphaRegisterInfo.cpp
@@ -207,8 +207,7 @@ void AlphaRegisterInfo::emitPrologue(MachineFunction &MF) const {
   MachineBasicBlock &MBB = MF.front();   // Prolog goes in entry BB
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  DebugLoc dl = (MBBI != MBB.end() ?
-                 MBBI->getDebugLoc() : DebugLoc::getUnknownLoc());
+  DebugLoc dl = (MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc());
   bool FP = hasFP(MF);
 
   //handle GOP offset
diff --git a/lib/Target/Blackfin/BlackfinInstrInfo.cpp b/lib/Target/Blackfin/BlackfinInstrInfo.cpp
index 3fd5d4dc0bf1..cf1901ba917b 100644
--- a/lib/Target/Blackfin/BlackfinInstrInfo.cpp
+++ b/lib/Target/Blackfin/BlackfinInstrInfo.cpp
@@ -106,7 +106,7 @@ InsertBranch(MachineBasicBlock &MBB,
              MachineBasicBlock *FBB,
              const SmallVectorImpl<MachineOperand> &Cond) const {
   // FIXME this should probably have a DebugLoc operand
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
 
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
@@ -116,7 +116,7 @@ InsertBranch(MachineBasicBlock &MBB,
   if (Cond.empty()) {
     // Unconditional branch?
     assert(!FBB && "Unconditional branch with multiple successors!");
-    BuildMI(&MBB, dl, get(BF::JUMPa)).addMBB(TBB);
+    BuildMI(&MBB, DL, get(BF::JUMPa)).addMBB(TBB);
     return 1;
   }
 
@@ -139,27 +139,27 @@ bool BlackfinInstrInfo::copyRegToReg(MachineBasicBlock &MBB,
                                      unsigned SrcReg,
                                      const TargetRegisterClass *DestRC,
                                      const TargetRegisterClass *SrcRC) const {
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
 
   if (inClass(BF::ALLRegClass, DestReg, DestRC) &&
       inClass(BF::ALLRegClass, SrcReg,  SrcRC)) {
-    BuildMI(MBB, I, dl, get(BF::MOVE), DestReg).addReg(SrcReg);
+    BuildMI(MBB, I, DL, get(BF::MOVE), DestReg).addReg(SrcReg);
     return true;
   }
 
   if (inClass(BF::D16RegClass, DestReg, DestRC) &&
       inClass(BF::D16RegClass, SrcReg,  SrcRC)) {
-    BuildMI(MBB, I, dl, get(BF::SLL16i), DestReg).addReg(SrcReg).addImm(0);
+    BuildMI(MBB, I, DL, get(BF::SLL16i), DestReg).addReg(SrcReg).addImm(0);
     return true;
   }
 
   if (inClass(BF::AnyCCRegClass, SrcReg, SrcRC) &&
       inClass(BF::DRegClass, DestReg, DestRC)) {
     if (inClass(BF::NotCCRegClass, SrcReg, SrcRC)) {
-      BuildMI(MBB, I, dl, get(BF::MOVENCC_z), DestReg).addReg(SrcReg);
-      BuildMI(MBB, I, dl, get(BF::BITTGL), DestReg).addReg(DestReg).addImm(0);
+      BuildMI(MBB, I, DL, get(BF::MOVENCC_z), DestReg).addReg(SrcReg);
+      BuildMI(MBB, I, DL, get(BF::BITTGL), DestReg).addReg(DestReg).addImm(0);
     } else {
-      BuildMI(MBB, I, dl, get(BF::MOVECC_zext), DestReg).addReg(SrcReg);
+      BuildMI(MBB, I, DL, get(BF::MOVECC_zext), DestReg).addReg(SrcReg);
     }
     return true;
   }
@@ -167,21 +167,21 @@ bool BlackfinInstrInfo::copyRegToReg(MachineBasicBlock &MBB,
   if (inClass(BF::AnyCCRegClass, DestReg, DestRC) &&
       inClass(BF::DRegClass, SrcReg,  SrcRC)) {
     if (inClass(BF::NotCCRegClass, DestReg, DestRC))
-      BuildMI(MBB, I, dl, get(BF::SETEQri_not), DestReg).addReg(SrcReg);
+      BuildMI(MBB, I, DL, get(BF::SETEQri_not), DestReg).addReg(SrcReg);
     else
-      BuildMI(MBB, I, dl, get(BF::MOVECC_nz), DestReg).addReg(SrcReg);
+      BuildMI(MBB, I, DL, get(BF::MOVECC_nz), DestReg).addReg(SrcReg);
     return true;
   }
 
   if (inClass(BF::NotCCRegClass, DestReg, DestRC) &&
       inClass(BF::JustCCRegClass, SrcReg,  SrcRC)) {
-    BuildMI(MBB, I, dl, get(BF::MOVE_ncccc), DestReg).addReg(SrcReg);
+    BuildMI(MBB, I, DL, get(BF::MOVE_ncccc), DestReg).addReg(SrcReg);
     return true;
   }
 
   if (inClass(BF::JustCCRegClass, DestReg, DestRC) &&
       inClass(BF::NotCCRegClass, SrcReg,  SrcRC)) {
-    BuildMI(MBB, I, dl, get(BF::MOVE_ccncc), DestReg).addReg(SrcReg);
+    BuildMI(MBB, I, DL, get(BF::MOVE_ccncc), DestReg).addReg(SrcReg);
     return true;
   }
 
@@ -197,8 +197,7 @@ BlackfinInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
                                        bool isKill,
                                        int FI,
                                        const TargetRegisterClass *RC) const {
-  DebugLoc DL = I != MBB.end() ?
-    I->getDebugLoc() : DebugLoc::getUnknownLoc();
+  DebugLoc DL = I != MBB.end() ? I->getDebugLoc() : DebugLoc();
 
   if (inClass(BF::DPRegClass, SrcReg, RC)) {
     BuildMI(MBB, I, DL, get(BF::STORE32fi))
@@ -244,8 +243,7 @@ BlackfinInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                         unsigned DestReg,
                                         int FI,
                                         const TargetRegisterClass *RC) const {
-  DebugLoc DL = I != MBB.end() ?
-    I->getDebugLoc() : DebugLoc::getUnknownLoc();
+  DebugLoc DL = I != MBB.end() ? I->getDebugLoc() : DebugLoc();
   if (inClass(BF::DPRegClass, DestReg, RC)) {
     BuildMI(MBB, I, DL, get(BF::LOAD32fi), DestReg)
       .addFrameIndex(FI)
diff --git a/lib/Target/Blackfin/BlackfinRegisterInfo.cpp b/lib/Target/Blackfin/BlackfinRegisterInfo.cpp
index 84dc9cac504a..6fd610fa3b53 100644
--- a/lib/Target/Blackfin/BlackfinRegisterInfo.cpp
+++ b/lib/Target/Blackfin/BlackfinRegisterInfo.cpp
@@ -384,9 +384,7 @@ void BlackfinRegisterInfo::emitPrologue(MachineFunction &MF) const {
   MachineBasicBlock &MBB = MF.front();   // Prolog goes in entry BB
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  DebugLoc dl = (MBBI != MBB.end()
-                 ? MBBI->getDebugLoc()
-                 : DebugLoc::getUnknownLoc());
+  DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   int FrameSize = MFI->getStackSize();
   if (FrameSize%4) {
diff --git a/lib/Target/CellSPU/SPUInstrInfo.cpp b/lib/Target/CellSPU/SPUInstrInfo.cpp
index 86825c81861b..423da3b4dbeb 100644
--- a/lib/Target/CellSPU/SPUInstrInfo.cpp
+++ b/lib/Target/CellSPU/SPUInstrInfo.cpp
@@ -262,7 +262,7 @@ bool SPUInstrInfo::copyRegToReg(MachineBasicBlock &MBB,
   // we instruction select bitconvert i64 -> f64 as a noop for example, so our
   // types have no specific meaning.
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   if (DestRC == SPU::R8CRegisterClass) {
@@ -317,7 +317,7 @@ SPUInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
     llvm_unreachable("Unknown regclass!");
   }
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
   addFrameReference(BuildMI(MBB, MI, DL, get(opc))
                     .addReg(SrcReg, getKillRegState(isKill)), FrameIdx);
@@ -351,7 +351,7 @@ SPUInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
     llvm_unreachable("Unknown regclass in loadRegFromStackSlot!");
   }
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
   addFrameReference(BuildMI(MBB, MI, DL, get(opc), DestReg), FrameIdx);
 }
@@ -553,7 +553,7 @@ SPUInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                            MachineBasicBlock *FBB,
                            const SmallVectorImpl<MachineOperand> &Cond) const {
   // FIXME this should probably have a DebugLoc argument
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
   assert((Cond.size() == 2 || Cond.size() == 0) &&
diff --git a/lib/Target/CellSPU/SPURegisterInfo.cpp b/lib/Target/CellSPU/SPURegisterInfo.cpp
index f3071f2e5697..ad034ebcd839 100644
--- a/lib/Target/CellSPU/SPURegisterInfo.cpp
+++ b/lib/Target/CellSPU/SPURegisterInfo.cpp
@@ -452,8 +452,7 @@ void SPURegisterInfo::emitPrologue(MachineFunction &MF) const
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MachineModuleInfo *MMI = MFI->getMachineModuleInfo();
-  DebugLoc dl = (MBBI != MBB.end() ?
-                 MBBI->getDebugLoc() : DebugLoc::getUnknownLoc());
+  DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   // Prepare for debug frame info.
   bool hasDebugInfo = MMI && MMI->hasDebugInfo();
diff --git a/lib/Target/MBlaze/MBlazeInstrInfo.cpp b/lib/Target/MBlaze/MBlazeInstrInfo.cpp
index a7e8eb7d554f..01f317471ee7 100644
--- a/lib/Target/MBlaze/MBlazeInstrInfo.cpp
+++ b/lib/Target/MBlaze/MBlazeInstrInfo.cpp
@@ -106,7 +106,7 @@ isStoreToStackSlot(const MachineInstr *MI, int &FrameIndex) const {
 /// instruction.
 void MBlazeInstrInfo::
 insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
   BuildMI(MBB, MI, DL, get(MBlaze::NOP));
 }
@@ -116,8 +116,8 @@ copyRegToReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
              unsigned DestReg, unsigned SrcReg,
              const TargetRegisterClass *DestRC,
              const TargetRegisterClass *SrcRC) const {
-  DebugLoc dl = DebugLoc::getUnknownLoc();
-  llvm::BuildMI(MBB, I, dl, get(MBlaze::ADD), DestReg)
+  DebugLoc DL;
+  llvm::BuildMI(MBB, I, DL, get(MBlaze::ADD), DestReg)
       .addReg(SrcReg).addReg(MBlaze::R0);
   return true;
 }
@@ -126,8 +126,8 @@ void MBlazeInstrInfo::
 storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                     unsigned SrcReg, bool isKill, int FI,
                     const TargetRegisterClass *RC) const {
-  DebugLoc dl = DebugLoc::getUnknownLoc();
-  BuildMI(MBB, I, dl, get(MBlaze::SWI)).addReg(SrcReg,getKillRegState(isKill))
+  DebugLoc DL;
+  BuildMI(MBB, I, DL, get(MBlaze::SWI)).addReg(SrcReg,getKillRegState(isKill))
     .addImm(0).addFrameIndex(FI);
 }
 
@@ -135,8 +135,8 @@ void MBlazeInstrInfo::
 loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                      unsigned DestReg, int FI,
                      const TargetRegisterClass *RC) const {
-  DebugLoc dl = DebugLoc::getUnknownLoc();
-  BuildMI(MBB, I, dl, get(MBlaze::LWI), DestReg)
+  DebugLoc DL;
+  BuildMI(MBB, I, DL, get(MBlaze::LWI), DestReg)
       .addImm(0).addFrameIndex(FI);
 }
 
@@ -185,11 +185,9 @@ unsigned MBlazeInstrInfo::
 InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
              MachineBasicBlock *FBB,
              const SmallVectorImpl<MachineOperand> &Cond) const {
-  DebugLoc dl = DebugLoc::getUnknownLoc();
-
   // Can only insert uncond branches so far.
   assert(Cond.empty() && !FBB && TBB && "Can only handle uncond branches!");
-  BuildMI(&MBB, dl, get(MBlaze::BRI)).addMBB(TBB);
+  BuildMI(&MBB, DebugLoc(), get(MBlaze::BRI)).addMBB(TBB);
   return 1;
 }
 
diff --git a/lib/Target/MBlaze/MBlazeRegisterInfo.cpp b/lib/Target/MBlaze/MBlazeRegisterInfo.cpp
index 6d528a2488cf..a12310a2bd32 100644
--- a/lib/Target/MBlaze/MBlazeRegisterInfo.cpp
+++ b/lib/Target/MBlaze/MBlazeRegisterInfo.cpp
@@ -302,8 +302,7 @@ emitPrologue(MachineFunction &MF) const {
   MachineFrameInfo *MFI    = MF.getFrameInfo();
   MBlazeFunctionInfo *MBlazeFI = MF.getInfo<MBlazeFunctionInfo>();
   MachineBasicBlock::iterator MBBI = MBB.begin();
-  DebugLoc dl = (MBBI != MBB.end() ?
-                 MBBI->getDebugLoc() : DebugLoc::getUnknownLoc());
+  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   // Get the right frame order for MBlaze.
   adjustMBlazeStackFrame(MF);
@@ -319,13 +318,13 @@ emitPrologue(MachineFunction &MF) const {
   int RAOffset = MBlazeFI->getRAStackOffset();
 
   // Adjust stack : addi R1, R1, -imm
-  BuildMI(MBB, MBBI, dl, TII.get(MBlaze::ADDI), MBlaze::R1)
+  BuildMI(MBB, MBBI, DL, TII.get(MBlaze::ADDI), MBlaze::R1)
       .addReg(MBlaze::R1).addImm(-StackSize);
 
   // Save the return address only if the function isnt a leaf one.
   // swi  R15, R1, stack_loc
   if (MFI->hasCalls()) {
-    BuildMI(MBB, MBBI, dl, TII.get(MBlaze::SWI))
+    BuildMI(MBB, MBBI, DL, TII.get(MBlaze::SWI))
         .addReg(MBlaze::R15).addImm(RAOffset).addReg(MBlaze::R1);
   }
 
@@ -333,11 +332,11 @@ emitPrologue(MachineFunction &MF) const {
   // to point to the stack pointer
   if (hasFP(MF)) {
     // swi  R19, R1, stack_loc
-    BuildMI(MBB, MBBI, dl, TII.get(MBlaze::SWI))
+    BuildMI(MBB, MBBI, DL, TII.get(MBlaze::SWI))
       .addReg(MBlaze::R19).addImm(FPOffset).addReg(MBlaze::R1);
 
     // add R19, R1, R0
-    BuildMI(MBB, MBBI, dl, TII.get(MBlaze::ADD), MBlaze::R19)
+    BuildMI(MBB, MBBI, DL, TII.get(MBlaze::ADD), MBlaze::R19)
       .addReg(MBlaze::R1).addReg(MBlaze::R0);
   }
 }
diff --git a/lib/Target/MSP430/MSP430InstrInfo.cpp b/lib/Target/MSP430/MSP430InstrInfo.cpp
index e584770dd497..03819041067c 100644
--- a/lib/Target/MSP430/MSP430InstrInfo.cpp
+++ b/lib/Target/MSP430/MSP430InstrInfo.cpp
@@ -33,7 +33,7 @@ void MSP430InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
                                           MachineBasicBlock::iterator MI,
                                     unsigned SrcReg, bool isKill, int FrameIdx,
                                     const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = *MF.getFrameInfo();
@@ -60,7 +60,7 @@ void MSP430InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                            MachineBasicBlock::iterator MI,
                                            unsigned DestReg, int FrameIdx,
                                            const TargetRegisterClass *RC) const{
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = *MF.getFrameInfo();
@@ -86,7 +86,7 @@ bool MSP430InstrInfo::copyRegToReg(MachineBasicBlock &MBB,
                                    unsigned DestReg, unsigned SrcReg,
                                    const TargetRegisterClass *DestRC,
                                    const TargetRegisterClass *SrcRC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (DestRC == SrcRC) {
@@ -134,7 +134,7 @@ MSP430InstrInfo::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
   if (CSI.empty())
     return false;
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   MachineFunction &MF = *MBB.getParent();
@@ -158,7 +158,7 @@ MSP430InstrInfo::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
   if (CSI.empty())
     return false;
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   for (unsigned i = 0, e = CSI.size(); i != e; ++i)
@@ -323,7 +323,7 @@ MSP430InstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                               MachineBasicBlock *FBB,
                             const SmallVectorImpl<MachineOperand> &Cond) const {
   // FIXME this should probably have a DebugLoc operand
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
 
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
@@ -333,18 +333,18 @@ MSP430InstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
   if (Cond.empty()) {
     // Unconditional branch?
     assert(!FBB && "Unconditional branch with multiple successors!");
-    BuildMI(&MBB, dl, get(MSP430::JMP)).addMBB(TBB);
+    BuildMI(&MBB, DL, get(MSP430::JMP)).addMBB(TBB);
     return 1;
   }
 
   // Conditional branch.
   unsigned Count = 0;
-  BuildMI(&MBB, dl, get(MSP430::JCC)).addMBB(TBB).addImm(Cond[0].getImm());
+  BuildMI(&MBB, DL, get(MSP430::JCC)).addMBB(TBB).addImm(Cond[0].getImm());
   ++Count;
 
   if (FBB) {
     // Two-way Conditional branch. Insert the second branch.
-    BuildMI(&MBB, dl, get(MSP430::JMP)).addMBB(FBB);
+    BuildMI(&MBB, DL, get(MSP430::JMP)).addMBB(FBB);
     ++Count;
   }
   return Count;
diff --git a/lib/Target/MSP430/MSP430RegisterInfo.cpp b/lib/Target/MSP430/MSP430RegisterInfo.cpp
index daac68324cb3..d91783a80c83 100644
--- a/lib/Target/MSP430/MSP430RegisterInfo.cpp
+++ b/lib/Target/MSP430/MSP430RegisterInfo.cpp
@@ -283,8 +283,7 @@ void MSP430RegisterInfo::emitPrologue(MachineFunction &MF) const {
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MSP430MachineFunctionInfo *MSP430FI = MF.getInfo<MSP430MachineFunctionInfo>();
   MachineBasicBlock::iterator MBBI = MBB.begin();
-  DebugLoc DL = (MBBI != MBB.end() ? MBBI->getDebugLoc() :
-                 DebugLoc::getUnknownLoc());
+  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   // Get the number of bytes to allocate from the FrameInfo.
   uint64_t StackSize = MFI->getStackSize();
diff --git a/lib/Target/Mips/MipsInstrInfo.cpp b/lib/Target/Mips/MipsInstrInfo.cpp
index 85cf064b9f52..dbd3c24dc1cd 100644
--- a/lib/Target/Mips/MipsInstrInfo.cpp
+++ b/lib/Target/Mips/MipsInstrInfo.cpp
@@ -123,7 +123,7 @@ isStoreToStackSlot(const MachineInstr *MI, int &FrameIndex) const
 void MipsInstrInfo::
 insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const 
 {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
   BuildMI(MBB, MI, DL, get(Mips::NOP));
 }
@@ -133,7 +133,7 @@ copyRegToReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
              unsigned DestReg, unsigned SrcReg,
              const TargetRegisterClass *DestRC,
              const TargetRegisterClass *SrcRC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   
   if (I != MBB.end()) DL = I->getDebugLoc();
 
@@ -191,7 +191,7 @@ void MipsInstrInfo::
 storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                     unsigned SrcReg, bool isKill, int FI, 
                     const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (RC == Mips::CPURegsRegisterClass) 
@@ -225,7 +225,7 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                      unsigned DestReg, int FI,
                      const TargetRegisterClass *RC) const 
 {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (RC == Mips::CPURegsRegisterClass) 
@@ -523,7 +523,7 @@ InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
              MachineBasicBlock *FBB,
              const SmallVectorImpl<MachineOperand> &Cond) const {
   // FIXME this should probably have a DebugLoc argument
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
   assert((Cond.size() == 3 || Cond.size() == 2 || Cond.size() == 0) &&
diff --git a/lib/Target/Mips/MipsRegisterInfo.cpp b/lib/Target/Mips/MipsRegisterInfo.cpp
index f3c87bc5554a..f43e69b35457 100644
--- a/lib/Target/Mips/MipsRegisterInfo.cpp
+++ b/lib/Target/Mips/MipsRegisterInfo.cpp
@@ -397,8 +397,7 @@ emitPrologue(MachineFunction &MF) const
   MachineFrameInfo *MFI    = MF.getFrameInfo();
   MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
   MachineBasicBlock::iterator MBBI = MBB.begin();
-  DebugLoc dl = (MBBI != MBB.end() ?
-                 MBBI->getDebugLoc() : DebugLoc::getUnknownLoc());
+  DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
   bool isPIC = (MF.getTarget().getRelocationModel() == Reloc::PIC_);
 
   // Get the right frame order for Mips.
diff --git a/lib/Target/PIC16/PIC16DebugInfo.cpp b/lib/Target/PIC16/PIC16DebugInfo.cpp
index da4e02787691..a223540170a7 100644
--- a/lib/Target/PIC16/PIC16DebugInfo.cpp
+++ b/lib/Target/PIC16/PIC16DebugInfo.cpp
@@ -256,15 +256,11 @@ void PIC16DbgInfo::BeginFunction(const MachineFunction &MF) {
 ///
 void PIC16DbgInfo::ChangeDebugLoc(const MachineFunction &MF,  
                                   const DebugLoc &DL, bool IsInBeginFunction) {
-  if (! EmitDebugDirectives) return;
-  assert (! DL.isUnknown()  && "can't change to invalid debug loc");
-
-  DILocation Loc = MF.getDILocation(DL);
-  MDNode *CU = Loc.getScope().getNode();
-  unsigned line = Loc.getLineNumber();
+  if (!EmitDebugDirectives) return;
+  assert(!DL.isUnknown() && "can't change to invalid debug loc");
 
-  SwitchToCU(CU);
-  SwitchToLine(line, IsInBeginFunction);
+  SwitchToCU(DL.getScope(MF.getFunction()->getContext()));
+  SwitchToLine(DL.getLine(), IsInBeginFunction);
 }
 
 /// SwitchToLine - Emit line directive for a new line.
diff --git a/lib/Target/PIC16/PIC16InstrInfo.cpp b/lib/Target/PIC16/PIC16InstrInfo.cpp
index da16e8383c31..365e8b20b7a1 100644
--- a/lib/Target/PIC16/PIC16InstrInfo.cpp
+++ b/lib/Target/PIC16/PIC16InstrInfo.cpp
@@ -72,7 +72,7 @@ void PIC16InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
                                          unsigned SrcReg, bool isKill, int FI,
                                          const TargetRegisterClass *RC) const {
   PIC16TargetLowering *PTLI = TM.getTargetLowering();
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   const Function *Func = MBB.getParent()->getFunction();
@@ -114,7 +114,7 @@ void PIC16InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                           unsigned DestReg, int FI,
                                           const TargetRegisterClass *RC) const {
   PIC16TargetLowering *PTLI = TM.getTargetLowering();
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   const Function *Func = MBB.getParent()->getFunction();
@@ -154,7 +154,7 @@ bool PIC16InstrInfo::copyRegToReg (MachineBasicBlock &MBB,
                                    unsigned DestReg, unsigned SrcReg,
                                    const TargetRegisterClass *DestRC,
                                    const TargetRegisterClass *SrcRC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (DestRC == PIC16::FSR16RegisterClass) {
@@ -202,7 +202,7 @@ InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
   if (FBB == 0) { // One way branch.
     if (Cond.empty()) {
       // Unconditional branch?
-      DebugLoc dl = DebugLoc::getUnknownLoc();
+      DebugLoc dl;
       BuildMI(&MBB, dl, get(PIC16::br_uncond)).addMBB(TBB);
     }
     return 1;
diff --git a/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 4f88d35deaf8..1e323849d1ef 100644
--- a/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -215,7 +215,7 @@ void PPCDAGToDAGISel::InsertVRSaveCode(MachineFunction &Fn) {
   
   const TargetInstrInfo &TII = *TM.getInstrInfo();
   MachineBasicBlock &EntryBB = *Fn.begin();
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
   // Emit the following code into the entry block:
   // InVRSAVE = MFVRSAVE
   // UpdatedVRSAVE = UPDATE_VRSAVE InVRSAVE
@@ -253,7 +253,7 @@ SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
     // Insert the set of GlobalBaseReg into the first MBB of the function
     MachineBasicBlock &FirstMBB = MF->front();
     MachineBasicBlock::iterator MBBI = FirstMBB.begin();
-    DebugLoc dl = DebugLoc::getUnknownLoc();
+    DebugLoc dl;
 
     if (PPCLowering.getPointerTy() == MVT::i32) {
       GlobalBaseReg = RegInfo->createVirtualRegister(PPC::GPRCRegisterClass);
diff --git a/lib/Target/PowerPC/PPCISelLowering.cpp b/lib/Target/PowerPC/PPCISelLowering.cpp
index e67666d80481..dda530eef434 100644
--- a/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -1122,7 +1122,7 @@ SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
     // With PIC, the first instruction is actually "GR+hi(&G)".
     Hi = DAG.getNode(ISD::ADD, dl, PtrVT,
                      DAG.getNode(PPCISD::GlobalBaseReg,
-                                 DebugLoc::getUnknownLoc(), PtrVT), Hi);
+                                 DebugLoc(), PtrVT), Hi);
   }
 
   Lo = DAG.getNode(ISD::ADD, dl, PtrVT, Hi, Lo);
@@ -1155,7 +1155,7 @@ SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) {
     // With PIC, the first instruction is actually "GR+hi(&G)".
     Hi = DAG.getNode(ISD::ADD, dl, PtrVT,
                      DAG.getNode(PPCISD::GlobalBaseReg,
-                                 DebugLoc::getUnknownLoc(), PtrVT), Hi);
+                                 DebugLoc(), PtrVT), Hi);
   }
 
   Lo = DAG.getNode(ISD::ADD, dl, PtrVT, Hi, Lo);
@@ -1192,7 +1192,7 @@ SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) {
     // With PIC, the first instruction is actually "GR+hi(&G)".
     Hi = DAG.getNode(ISD::ADD, DL, PtrVT,
                      DAG.getNode(PPCISD::GlobalBaseReg,
-                                 DebugLoc::getUnknownLoc(), PtrVT), Hi);
+                                 DebugLoc(), PtrVT), Hi);
   }
 
   return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo);
@@ -1233,7 +1233,7 @@ SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op,
     // With PIC, the first instruction is actually "GR+hi(&G)".
     Hi = DAG.getNode(ISD::ADD, dl, PtrVT,
                      DAG.getNode(PPCISD::GlobalBaseReg,
-                                 DebugLoc::getUnknownLoc(), PtrVT), Hi);
+                                 DebugLoc(), PtrVT), Hi);
   }
 
   Lo = DAG.getNode(ISD::ADD, dl, PtrVT, Hi, Lo);
@@ -5540,15 +5540,18 @@ PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
 }
 
 /// getOptimalMemOpType - Returns the target specific optimal type for load
-/// and store operations as a result of memset, memcpy, and memmove lowering.
-/// If DstAlign is zero that means it's safe to destination alignment can
-/// satisfy any constraint. Similarly if SrcAlign is zero it means there
-/// isn't a need to check it against alignment requirement, probably because
-/// the source does not need to be loaded. It returns EVT::Other if
-/// SelectionDAG should be responsible for determining it.
+/// and store operations as a result of memset, memcpy, and memmove
+/// lowering. If DstAlign is zero that means it's safe to destination
+/// alignment can satisfy any constraint. Similarly if SrcAlign is zero it
+/// means there isn't a need to check it against alignment requirement,
+/// probably because the source does not need to be loaded. If
+/// 'NonScalarIntSafe' is true, that means it's safe to return a
+/// non-scalar-integer type, e.g. empty string source, constant, or loaded
+/// from memory. It returns EVT::Other if SelectionDAG should be responsible
+/// for determining it.
 EVT PPCTargetLowering::getOptimalMemOpType(uint64_t Size,
                                            unsigned DstAlign, unsigned SrcAlign,
-                                           bool SafeToUseFP,
+                                           bool NonScalarIntSafe,
                                            SelectionDAG &DAG) const {
   if (this->PPCSubTarget.isPPC64()) {
     return MVT::i64;
diff --git a/lib/Target/PowerPC/PPCISelLowering.h b/lib/Target/PowerPC/PPCISelLowering.h
index 19fefab2d3a0..f816bddaf5a3 100644
--- a/lib/Target/PowerPC/PPCISelLowering.h
+++ b/lib/Target/PowerPC/PPCISelLowering.h
@@ -348,15 +348,19 @@ namespace llvm {
     virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const;
     
     /// getOptimalMemOpType - Returns the target specific optimal type for load
-    /// and store operations as a result of memset, memcpy, and memmove lowering.
-    /// If DstAlign is zero that means it's safe to destination alignment can
-    /// satisfy any constraint. Similarly if SrcAlign is zero it means there
-    /// isn't a need to check it against alignment requirement, probably because
-    /// the source does not need to be loaded. It returns EVT::Other if
-    /// SelectionDAG should be responsible for determining it.
-    virtual EVT getOptimalMemOpType(uint64_t Size,
-                                    unsigned DstAlign, unsigned SrcAlign,
-                                    bool SafeToUseFP, SelectionDAG &DAG) const;
+    /// and store operations as a result of memset, memcpy, and memmove
+    /// lowering. If DstAlign is zero that means it's safe to destination
+    /// alignment can satisfy any constraint. Similarly if SrcAlign is zero it
+    /// means there isn't a need to check it against alignment requirement,
+    /// probably because the source does not need to be loaded. If
+    /// 'NonScalarIntSafe' is true, that means it's safe to return a
+    /// non-scalar-integer type, e.g. empty string source, constant, or loaded
+    /// from memory. It returns EVT::Other if SelectionDAG should be responsible
+    /// for determining it.
+    virtual EVT
+    getOptimalMemOpType(uint64_t Size,
+                        unsigned DstAlign, unsigned SrcAlign,
+                        bool NonScalarIntSafe, SelectionDAG &DAG) const;
 
     /// getFunctionAlignment - Return the Log2 alignment of this function.
     virtual unsigned getFunctionAlignment(const Function *F) const;
diff --git a/lib/Target/PowerPC/PPCInstrInfo.cpp b/lib/Target/PowerPC/PPCInstrInfo.cpp
index 82c637efaf25..6b0a282af09c 100644
--- a/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -199,7 +199,7 @@ PPCInstrInfo::commuteInstruction(MachineInstr *MI, bool NewMI) const {
 
 void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB, 
                               MachineBasicBlock::iterator MI) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   BuildMI(MBB, MI, DL, get(PPC::NOP));
@@ -317,7 +317,7 @@ PPCInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                            MachineBasicBlock *FBB,
                            const SmallVectorImpl<MachineOperand> &Cond) const {
   // FIXME this should probably have a DebugLoc argument
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
   assert((Cond.size() == 2 || Cond.size() == 0) && 
@@ -350,7 +350,7 @@ bool PPCInstrInfo::copyRegToReg(MachineBasicBlock &MBB,
     return false;
   }
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   if (DestRC == PPC::GPRCRegisterClass) {
@@ -380,7 +380,7 @@ PPCInstrInfo::StoreRegToStackSlot(MachineFunction &MF,
                                   int FrameIdx,
                                   const TargetRegisterClass *RC,
                                   SmallVectorImpl<MachineInstr*> &NewMIs) const{
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (RC == PPC::GPRCRegisterClass) {
     if (SrcReg != PPC::LR) {
       NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::STW))
@@ -635,7 +635,7 @@ PPCInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                    const TargetRegisterClass *RC) const {
   MachineFunction &MF = *MBB.getParent();
   SmallVector<MachineInstr*, 4> NewMIs;
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
   LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
   for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
diff --git a/lib/Target/PowerPC/PPCRegisterInfo.cpp b/lib/Target/PowerPC/PPCRegisterInfo.cpp
index 44c5fe672f86..9363aa8b7566 100644
--- a/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -1281,7 +1281,7 @@ PPCRegisterInfo::emitPrologue(MachineFunction &MF) const {
   MachineBasicBlock::iterator MBBI = MBB.begin();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MachineModuleInfo *MMI = MFI->getMachineModuleInfo();
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
   bool needsFrameMoves = (MMI && MMI->hasDebugInfo()) ||
        !MF.getFunction()->doesNotThrow() ||
        UnwindTablesMandatory;
@@ -1521,7 +1521,7 @@ void PPCRegisterInfo::emitEpilogue(MachineFunction &MF,
                                    MachineBasicBlock &MBB) const {
   MachineBasicBlock::iterator MBBI = prior(MBB.end());
   unsigned RetOpcode = MBBI->getOpcode();
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
 
   assert( (RetOpcode == PPC::BLR ||
            RetOpcode == PPC::TCRETURNri ||
diff --git a/lib/Target/Sparc/DelaySlotFiller.cpp b/lib/Target/Sparc/DelaySlotFiller.cpp
index 15b26c29872f..9e148ada8853 100644
--- a/lib/Target/Sparc/DelaySlotFiller.cpp
+++ b/lib/Target/Sparc/DelaySlotFiller.cpp
@@ -68,7 +68,7 @@ bool Filler::runOnMachineBasicBlock(MachineBasicBlock &MBB) {
     if (I->getDesc().hasDelaySlot()) {
       MachineBasicBlock::iterator J = I;
       ++J;
-      BuildMI(MBB, J, DebugLoc::getUnknownLoc(), TII->get(SP::NOP));
+      BuildMI(MBB, J, DebugLoc(), TII->get(SP::NOP));
       ++FilledSlots;
       Changed = true;
     }
diff --git a/lib/Target/Sparc/SparcInstrInfo.cpp b/lib/Target/Sparc/SparcInstrInfo.cpp
index 8667bca7fe96..e494d7ddf9f8 100644
--- a/lib/Target/Sparc/SparcInstrInfo.cpp
+++ b/lib/Target/Sparc/SparcInstrInfo.cpp
@@ -111,7 +111,7 @@ SparcInstrInfo::InsertBranch(MachineBasicBlock &MBB,MachineBasicBlock *TBB,
                              MachineBasicBlock *FBB,
                              const SmallVectorImpl<MachineOperand> &Cond)const{
   // FIXME this should probably take a DebugLoc argument
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
   // Can only insert uncond branches so far.
   assert(Cond.empty() && !FBB && TBB && "Can only handle uncond branches!");
   BuildMI(&MBB, dl, get(SP::BA)).addMBB(TBB);
@@ -128,7 +128,7 @@ bool SparcInstrInfo::copyRegToReg(MachineBasicBlock &MBB,
     return false;
   }
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (DestRC == SP::IntRegsRegisterClass)
@@ -149,7 +149,7 @@ void SparcInstrInfo::
 storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                     unsigned SrcReg, bool isKill, int FI,
                     const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   // On the order of operands here: think "[FrameIdx + 0] = SrcReg".
@@ -170,7 +170,7 @@ void SparcInstrInfo::
 loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                      unsigned DestReg, int FI,
                      const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (RC == SP::IntRegsRegisterClass)
@@ -253,7 +253,7 @@ unsigned SparcInstrInfo::getGlobalBaseReg(MachineFunction *MF) const
   GlobalBaseReg = RegInfo.createVirtualRegister(&SP::IntRegsRegClass);
 
 
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
 
   BuildMI(FirstMBB, MBBI, dl, get(SP::GETPCX), GlobalBaseReg);
   SparcFI->setGlobalBaseReg(GlobalBaseReg);
diff --git a/lib/Target/Sparc/SparcRegisterInfo.cpp b/lib/Target/Sparc/SparcRegisterInfo.cpp
index 740e3bc14f2b..08373bb83869 100644
--- a/lib/Target/Sparc/SparcRegisterInfo.cpp
+++ b/lib/Target/Sparc/SparcRegisterInfo.cpp
@@ -125,8 +125,7 @@ void SparcRegisterInfo::emitPrologue(MachineFunction &MF) const {
   MachineBasicBlock &MBB = MF.front();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MachineBasicBlock::iterator MBBI = MBB.begin();
-  DebugLoc dl = (MBBI != MBB.end() ?
-                 MBBI->getDebugLoc() : DebugLoc::getUnknownLoc());
+  DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   // Get the number of bytes to allocate from the FrameInfo
   int NumBytes = (int) MFI->getStackSize();
diff --git a/lib/Target/SystemZ/SystemZInstrInfo.cpp b/lib/Target/SystemZ/SystemZInstrInfo.cpp
index 06f01e7b32b8..c92caa4e9ec0 100644
--- a/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -62,7 +62,7 @@ void SystemZInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
                                           MachineBasicBlock::iterator MI,
                                     unsigned SrcReg, bool isKill, int FrameIdx,
                                     const TargetRegisterClass *RC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   unsigned Opc = 0;
@@ -91,7 +91,7 @@ void SystemZInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                            MachineBasicBlock::iterator MI,
                                            unsigned DestReg, int FrameIdx,
                                            const TargetRegisterClass *RC) const{
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   unsigned Opc = 0;
@@ -120,7 +120,7 @@ bool SystemZInstrInfo::copyRegToReg(MachineBasicBlock &MBB,
                                     unsigned DestReg, unsigned SrcReg,
                                     const TargetRegisterClass *DestRC,
                                     const TargetRegisterClass *SrcRC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   // Determine if DstRC and SrcRC have a common superclass.
@@ -273,7 +273,7 @@ SystemZInstrInfo::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
   if (CSI.empty())
     return false;
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   MachineFunction &MF = *MBB.getParent();
@@ -347,7 +347,7 @@ SystemZInstrInfo::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
   if (CSI.empty())
     return false;
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
 
   MachineFunction &MF = *MBB.getParent();
@@ -521,7 +521,7 @@ SystemZInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                                MachineBasicBlock *FBB,
                             const SmallVectorImpl<MachineOperand> &Cond) const {
   // FIXME: this should probably have a DebugLoc operand
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
   assert((Cond.size() == 1 || Cond.size() == 0) &&
@@ -530,19 +530,19 @@ SystemZInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
   if (Cond.empty()) {
     // Unconditional branch?
     assert(!FBB && "Unconditional branch with multiple successors!");
-    BuildMI(&MBB, dl, get(SystemZ::JMP)).addMBB(TBB);
+    BuildMI(&MBB, DL, get(SystemZ::JMP)).addMBB(TBB);
     return 1;
   }
 
   // Conditional branch.
   unsigned Count = 0;
   SystemZCC::CondCodes CC = (SystemZCC::CondCodes)Cond[0].getImm();
-  BuildMI(&MBB, dl, getBrCond(CC)).addMBB(TBB);
+  BuildMI(&MBB, DL, getBrCond(CC)).addMBB(TBB);
   ++Count;
 
   if (FBB) {
     // Two-way Conditional branch. Insert the second branch.
-    BuildMI(&MBB, dl, get(SystemZ::JMP)).addMBB(FBB);
+    BuildMI(&MBB, DL, get(SystemZ::JMP)).addMBB(FBB);
     ++Count;
   }
   return Count;
diff --git a/lib/Target/SystemZ/SystemZRegisterInfo.cpp b/lib/Target/SystemZ/SystemZRegisterInfo.cpp
index ca2fe6f920ba..302c418d1ec9 100644
--- a/lib/Target/SystemZ/SystemZRegisterInfo.cpp
+++ b/lib/Target/SystemZ/SystemZRegisterInfo.cpp
@@ -194,8 +194,7 @@ void emitSPUpdate(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI,
     Chunk = (1LL << 15) - 1;
   }
 
-  DebugLoc DL = (MBBI != MBB.end() ? MBBI->getDebugLoc() :
-                 DebugLoc::getUnknownLoc());
+  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   while (Offset) {
     uint64_t ThisVal = (Offset > Chunk) ? Chunk : Offset;
@@ -215,8 +214,7 @@ void SystemZRegisterInfo::emitPrologue(MachineFunction &MF) const {
   SystemZMachineFunctionInfo *SystemZMFI =
     MF.getInfo<SystemZMachineFunctionInfo>();
   MachineBasicBlock::iterator MBBI = MBB.begin();
-  DebugLoc DL = (MBBI != MBB.end() ? MBBI->getDebugLoc() :
-                 DebugLoc::getUnknownLoc());
+  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   // Get the number of bytes to allocate from the FrameInfo.
   // Note that area for callee-saved stuff is already allocated, thus we need to
diff --git a/lib/Target/X86/Disassembler/X86Disassembler.cpp b/lib/Target/X86/Disassembler/X86Disassembler.cpp
index 7b7b5cb4f574..7328dc0ba8d7 100644
--- a/lib/Target/X86/Disassembler/X86Disassembler.cpp
+++ b/lib/Target/X86/Disassembler/X86Disassembler.cpp
@@ -21,8 +21,8 @@
 #include "llvm/MC/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/Target/TargetRegistry.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/MemoryObject.h"
-#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include "X86GenRegisterNames.inc"
@@ -30,6 +30,14 @@
 using namespace llvm;
 using namespace llvm::X86Disassembler;
 
+void x86DisassemblerDebug(const char *file,
+                          unsigned line,
+                          const char *s) {
+  dbgs() << file << ":" << line << ": " << s;
+}
+
+#define debug(s) DEBUG(x86DisassemblerDebug(__FILE__, __LINE__, s));
+
 namespace llvm {  
   
 // Fill-ins to make the compiler happy.  These constants are never actually
@@ -50,8 +58,8 @@ extern Target TheX86_32Target, TheX86_64Target;
 
 }
 
-static void translateInstruction(MCInst &target,
-                                 InternalInstruction &source);
+static bool translateInstruction(MCInst &target,
+                                InternalInstruction &source);
 
 X86GenericDisassembler::X86GenericDisassembler(DisassemblerMode mode) :
     MCDisassembler(),
@@ -106,14 +114,13 @@ bool X86GenericDisassembler::getInstruction(MCInst &instr,
                               address,
                               fMode);
 
-  if(ret) {
+  if (ret) {
     size = internalInstr.readerCursor - address;
     return false;
   }
   else {
     size = internalInstr.length;
-    translateInstruction(instr, internalInstr);
-    return true;
+    return !translateInstruction(instr, internalInstr);
   }
 }
 
@@ -151,29 +158,35 @@ static void translateImmediate(MCInst &mcInst, uint64_t immediate) {
 /// @param mcInst       - The MCInst to append to.
 /// @param insn         - The internal instruction to extract the R/M field
 ///                       from.
-static void translateRMRegister(MCInst &mcInst,
+/// @return             - 0 on success; -1 otherwise
+static bool translateRMRegister(MCInst &mcInst,
                                 InternalInstruction &insn) {
-  assert(insn.eaBase != EA_BASE_sib && insn.eaBase != EA_BASE_sib64 && 
-         "A R/M register operand may not have a SIB byte");
+  if (insn.eaBase == EA_BASE_sib || insn.eaBase == EA_BASE_sib64) {
+    debug("A R/M register operand may not have a SIB byte");
+    return true;
+  }
   
   switch (insn.eaBase) {
+  default:
+    debug("Unexpected EA base register");
+    return true;
   case EA_BASE_NONE:
-    llvm_unreachable("EA_BASE_NONE for ModR/M base");
-    break;
+    debug("EA_BASE_NONE for ModR/M base");
+    return true;
 #define ENTRY(x) case EA_BASE_##x:
   ALL_EA_BASES
 #undef ENTRY
-    llvm_unreachable("A R/M register operand may not have a base; "
-                     "the operand must be a register.");
-    break;
-#define ENTRY(x)                                                        \
+    debug("A R/M register operand may not have a base; "
+          "the operand must be a register.");
+    return true;
+#define ENTRY(x)                                                      \
   case EA_REG_##x:                                                    \
     mcInst.addOperand(MCOperand::CreateReg(X86::x)); break;
   ALL_REGS
 #undef ENTRY
-  default:
-    llvm_unreachable("Unexpected EA base register");
   }
+  
+  return false;
 }
 
 /// translateRMMemory - Translates a memory operand stored in the Mod and R/M
@@ -186,7 +199,8 @@ static void translateRMRegister(MCInst &mcInst,
 /// @param sr           - Whether or not to emit the segment register.  The
 ///                       LEA instruction does not expect a segment-register
 ///                       operand.
-static void translateRMMemory(MCInst &mcInst,
+/// @return             - 0 on success; nonzero otherwise
+static bool translateRMMemory(MCInst &mcInst,
                               InternalInstruction &insn,
                               bool sr) {
   // Addresses in an MCInst are represented as five operands:
@@ -211,7 +225,8 @@ static void translateRMMemory(MCInst &mcInst,
     if (insn.sibBase != SIB_BASE_NONE) {
       switch (insn.sibBase) {
       default:
-        llvm_unreachable("Unexpected sibBase");
+        debug("Unexpected sibBase");
+        return true;
 #define ENTRY(x)                                          \
       case SIB_BASE_##x:                                  \
         baseReg = MCOperand::CreateReg(X86::x); break;
@@ -225,7 +240,8 @@ static void translateRMMemory(MCInst &mcInst,
     if (insn.sibIndex != SIB_INDEX_NONE) {
       switch (insn.sibIndex) {
       default:
-        llvm_unreachable("Unexpected sibIndex");
+        debug("Unexpected sibIndex");
+        return true;
 #define ENTRY(x)                                          \
       case SIB_INDEX_##x:                                 \
         indexReg = MCOperand::CreateReg(X86::x); break;
@@ -241,9 +257,10 @@ static void translateRMMemory(MCInst &mcInst,
   } else {
     switch (insn.eaBase) {
     case EA_BASE_NONE:
-      assert(insn.eaDisplacement != EA_DISP_NONE && 
-             "EA_BASE_NONE and EA_DISP_NONE for ModR/M base");
-      
+      if (insn.eaDisplacement == EA_DISP_NONE) {
+        debug("EA_BASE_NONE and EA_DISP_NONE for ModR/M base");
+        return true;
+      }
       if (insn.mode == MODE_64BIT)
         baseReg = MCOperand::CreateReg(X86::RIP); // Section 2.2.1.6
       else
@@ -271,8 +288,8 @@ static void translateRMMemory(MCInst &mcInst,
       indexReg = MCOperand::CreateReg(0);
       switch (insn.eaBase) {
       default:
-        llvm_unreachable("Unexpected eaBase");
-        break;
+        debug("Unexpected eaBase");
+        return true;
         // Here, we will use the fill-ins defined above.  However,
         //   BX_SI, BX_DI, BP_SI, and BP_DI are all handled above and
         //   sib and sib64 were handled in the top-level if, so they're only
@@ -285,9 +302,9 @@ static void translateRMMemory(MCInst &mcInst,
 #define ENTRY(x) case EA_REG_##x:
       ALL_REGS
 #undef ENTRY
-        llvm_unreachable("A R/M memory operand may not be a register; "
-                         "the base field must be a base.");
-            break;
+        debug("A R/M memory operand may not be a register; "
+              "the base field must be a base.");
+        return true;
       }
     }
     
@@ -315,6 +332,8 @@ static void translateRMMemory(MCInst &mcInst,
   
   if (sr)
     mcInst.addOperand(segmentReg);
+  
+  return false;
 }
 
 /// translateRM - Translates an operand stored in the R/M (and possibly SIB)
@@ -324,12 +343,14 @@ static void translateRMMemory(MCInst &mcInst,
 /// @param operand      - The operand, as stored in the descriptor table.
 /// @param insn         - The instruction to extract Mod, R/M, and SIB fields
 ///                       from.
-static void translateRM(MCInst &mcInst,
-                        OperandSpecifier &operand,
-                        InternalInstruction &insn) {
+/// @return             - 0 on success; nonzero otherwise
+static bool translateRM(MCInst &mcInst,
+                       OperandSpecifier &operand,
+                       InternalInstruction &insn) {
   switch (operand.type) {
   default:
-    llvm_unreachable("Unexpected type for a R/M operand");
+    debug("Unexpected type for a R/M operand");
+    return true;
   case TYPE_R8:
   case TYPE_R16:
   case TYPE_R32:
@@ -345,8 +366,7 @@ static void translateRM(MCInst &mcInst,
   case TYPE_DEBUGREG:
   case TYPE_CR32:
   case TYPE_CR64:
-    translateRMRegister(mcInst, insn);
-    break;
+    return translateRMRegister(mcInst, insn);
   case TYPE_M:
   case TYPE_M8:
   case TYPE_M16:
@@ -364,11 +384,9 @@ static void translateRM(MCInst &mcInst,
   case TYPE_M1616:
   case TYPE_M1632:
   case TYPE_M1664:
-    translateRMMemory(mcInst, insn, true);
-    break;
+    return translateRMMemory(mcInst, insn, true);
   case TYPE_LEA:
-    translateRMMemory(mcInst, insn, false);
-    break;
+    return translateRMMemory(mcInst, insn, false);
   }
 }
   
@@ -377,11 +395,17 @@ static void translateRM(MCInst &mcInst,
 ///
 /// @param mcInst       - The MCInst to append to.
 /// @param stackPos     - The stack position to translate.
-static void translateFPRegister(MCInst &mcInst,
-                                uint8_t stackPos) {
-  assert(stackPos < 8 && "Invalid FP stack position");
+/// @return             - 0 on success; nonzero otherwise.
+static bool translateFPRegister(MCInst &mcInst,
+                               uint8_t stackPos) {
+  if (stackPos >= 8) {
+    debug("Invalid FP stack position");
+    return true;
+  }
   
   mcInst.addOperand(MCOperand::CreateReg(X86::ST0 + stackPos));
+
+  return false;
 }
 
 /// translateOperand - Translates an operand stored in an internal instruction 
@@ -390,25 +414,27 @@ static void translateFPRegister(MCInst &mcInst,
 /// @param mcInst       - The MCInst to append to.
 /// @param operand      - The operand, as stored in the descriptor table.
 /// @param insn         - The internal instruction.
-static void translateOperand(MCInst &mcInst,
-                             OperandSpecifier &operand,
-                             InternalInstruction &insn) {
+/// @return             - false on success; true otherwise.
+static bool translateOperand(MCInst &mcInst,
+                            OperandSpecifier &operand,
+                            InternalInstruction &insn) {
   switch (operand.encoding) {
   default:
-    llvm_unreachable("Unhandled operand encoding during translation");
+    debug("Unhandled operand encoding during translation");
+    return true;
   case ENCODING_REG:
     translateRegister(mcInst, insn.reg);
-    break;
+    return false;
   case ENCODING_RM:
-    translateRM(mcInst, operand, insn);
-    break;
+    return translateRM(mcInst, operand, insn);
   case ENCODING_CB:
   case ENCODING_CW:
   case ENCODING_CD:
   case ENCODING_CP:
   case ENCODING_CO:
   case ENCODING_CT:
-    llvm_unreachable("Translation of code offsets isn't supported.");
+    debug("Translation of code offsets isn't supported.");
+    return true;
   case ENCODING_IB:
   case ENCODING_IW:
   case ENCODING_ID:
@@ -417,24 +443,22 @@ static void translateOperand(MCInst &mcInst,
   case ENCODING_Ia:
     translateImmediate(mcInst, 
                        insn.immediates[insn.numImmediatesTranslated++]);
-    break;
+    return false;
   case ENCODING_RB:
   case ENCODING_RW:
   case ENCODING_RD:
   case ENCODING_RO:
     translateRegister(mcInst, insn.opcodeRegister);
-    break;
+    return false;
   case ENCODING_I:
-    translateFPRegister(mcInst, insn.opcodeModifier);
-    break;
+    return translateFPRegister(mcInst, insn.opcodeModifier);
   case ENCODING_Rv:
     translateRegister(mcInst, insn.opcodeRegister);
-    break;
+    return false;
   case ENCODING_DUP:
-    translateOperand(mcInst,
-                     insn.spec->operands[operand.type - TYPE_DUP0],
-                     insn);
-    break;
+    return translateOperand(mcInst,
+                            insn.spec->operands[operand.type - TYPE_DUP0],
+                            insn);
   }
 }
   
@@ -443,9 +467,13 @@ static void translateOperand(MCInst &mcInst,
 ///
 /// @param mcInst       - The MCInst to populate with the instruction's data.
 /// @param insn         - The internal instruction.
-static void translateInstruction(MCInst &mcInst,
-                                 InternalInstruction &insn) {  
-  assert(insn.spec);
+/// @return             - false on success; true otherwise.
+static bool translateInstruction(MCInst &mcInst,
+                                InternalInstruction &insn) {  
+  if (!insn.spec) {
+    debug("Instruction has no specification");
+    return true;
+  }
   
   mcInst.setOpcode(insn.instructionID);
   
@@ -454,9 +482,14 @@ static void translateInstruction(MCInst &mcInst,
   insn.numImmediatesTranslated = 0;
   
   for (index = 0; index < X86_MAX_OPERANDS; ++index) {
-    if (insn.spec->operands[index].encoding != ENCODING_NONE)                
-      translateOperand(mcInst, insn.spec->operands[index], insn);
+    if (insn.spec->operands[index].encoding != ENCODING_NONE) {
+      if (translateOperand(mcInst, insn.spec->operands[index], insn)) {
+        return true;
+      }
+    }
   }
+  
+  return false;
 }
 
 static MCDisassembler *createX86_32Disassembler(const Target &T) {
diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
index 4f02ed46e38b..db694bc2f3c5 100644
--- a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
+++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
@@ -13,7 +13,6 @@
  *
  *===----------------------------------------------------------------------===*/
 
-#include <assert.h>   /* for assert()     */
 #include <stdarg.h>   /* for va_*()       */
 #include <stdio.h>    /* for vsnprintf()  */
 #include <stdlib.h>   /* for exit()       */
@@ -26,17 +25,20 @@
 #define TRUE  1
 #define FALSE 0
 
+typedef int8_t bool;
+
 #ifdef __GNUC__
 #define NORETURN __attribute__((noreturn))
 #else
 #define NORETURN
 #endif
 
-#define unreachable(s)                                      \
-  do {                                                      \
-    fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, s);  \
-    exit(-1);                                               \
-  } while (0);
+#ifndef NDEBUG
+#define debug(s) do { x86DisassemblerDebug(__FILE__, __LINE__, s); } while (0)
+#else
+#define debug(s) do { } while (0)
+#endif
+
 
 /*
  * contextForAttrs - Client for the instruction context table.  Takes a set of
@@ -84,7 +86,6 @@ static int modRMRequired(OpcodeType type,
   return decision->opcodeDecisions[insnContext].modRMDecisions[opcode].
     modrm_type != MODRM_ONEENTRY;
   
-  unreachable("Unknown opcode type");
   return 0;
 }
 
@@ -96,16 +97,18 @@ static int modRMRequired(OpcodeType type,
  * @param insnContext - See modRMRequired().
  * @param opcode      - See modRMRequired().
  * @param modRM       - The ModR/M byte if required, or any value if not.
+ * @return            - The UID of the instruction, or 0 on failure.
  */
 static InstrUID decode(OpcodeType type,
-                               InstructionContext insnContext,
-                               uint8_t opcode,
-                               uint8_t modRM) {
+                       InstructionContext insnContext,
+                       uint8_t opcode,
+                       uint8_t modRM) {
   struct ModRMDecision* dec;
   
   switch (type) {
   default:
-    unreachable("Unknown opcode type");
+    debug("Unknown opcode type");
+    return 0;
   case ONEBYTE:
     dec = &ONEBYTE_SYM.opcodeDecisions[insnContext].modRMDecisions[opcode];
     break;
@@ -122,7 +125,8 @@ static InstrUID decode(OpcodeType type,
   
   switch (dec->modrm_type) {
   default:
-    unreachable("Corrupt table!  Unknown modrm_type");
+    debug("Corrupt table!  Unknown modrm_type");
+    return 0;
   case MODRM_ONEENTRY:
     return dec->instructionIDs[0];
   case MODRM_SPLITRM:
@@ -133,8 +137,6 @@ static InstrUID decode(OpcodeType type,
   case MODRM_FULL:
     return dec->instructionIDs[modRM];
   }
-  
-  return 0;
 }
 
 /*
@@ -342,7 +344,8 @@ static int readPrefixes(struct InternalInstruction* insn) {
         insn->segmentOverride = SEG_OVERRIDE_GS;
         break;
       default:
-        unreachable("Unhandled override");
+        debug("Unhandled override");
+        return -1;
       }
       if (prefixGroups[1])
         dbgprintf(insn, "Redundant Group 2 prefix");
@@ -376,7 +379,7 @@ static int readPrefixes(struct InternalInstruction* insn) {
     if ((byte & 0xf0) == 0x40) {
       uint8_t opcodeByte;
       
-      if(lookAtByte(insn, &opcodeByte) || ((opcodeByte & 0xf0) == 0x40)) {
+      if (lookAtByte(insn, &opcodeByte) || ((opcodeByte & 0xf0) == 0x40)) {
         dbgprintf(insn, "Redundant REX prefix");
         return -1;
       }
@@ -540,17 +543,17 @@ static int getIDWithAttrMask(uint16_t* instructionID,
 static BOOL is16BitEquvalent(const char* orig, const char* equiv) {
   off_t i;
   
-  for(i = 0;; i++) {
-    if(orig[i] == '\0' && equiv[i] == '\0')
+  for (i = 0;; i++) {
+    if (orig[i] == '\0' && equiv[i] == '\0')
       return TRUE;
-    if(orig[i] == '\0' || equiv[i] == '\0')
+    if (orig[i] == '\0' || equiv[i] == '\0')
       return FALSE;
-    if(orig[i] != equiv[i]) {
-      if((orig[i] == 'Q' || orig[i] == 'L') && equiv[i] == 'W')
+    if (orig[i] != equiv[i]) {
+      if ((orig[i] == 'Q' || orig[i] == 'L') && equiv[i] == 'W')
         continue;
-      if((orig[i] == '6' || orig[i] == '3') && equiv[i] == '1')
+      if ((orig[i] == '6' || orig[i] == '3') && equiv[i] == '1')
         continue;
-      if((orig[i] == '4' || orig[i] == '2') && equiv[i] == '6')
+      if ((orig[i] == '4' || orig[i] == '2') && equiv[i] == '6')
         continue;
       return FALSE;
     }
@@ -567,17 +570,17 @@ static BOOL is16BitEquvalent(const char* orig, const char* equiv) {
 static BOOL is64BitEquivalent(const char* orig, const char* equiv) {
   off_t i;
   
-  for(i = 0;; i++) {
-    if(orig[i] == '\0' && equiv[i] == '\0')
+  for (i = 0;; i++) {
+    if (orig[i] == '\0' && equiv[i] == '\0')
       return TRUE;
-    if(orig[i] == '\0' || equiv[i] == '\0')
+    if (orig[i] == '\0' || equiv[i] == '\0')
       return FALSE;
-    if(orig[i] != equiv[i]) {
-      if((orig[i] == 'W' || orig[i] == 'L') && equiv[i] == 'Q')
+    if (orig[i] != equiv[i]) {
+      if ((orig[i] == 'W' || orig[i] == 'L') && equiv[i] == 'Q')
         continue;
-      if((orig[i] == '1' || orig[i] == '3') && equiv[i] == '6')
+      if ((orig[i] == '1' || orig[i] == '3') && equiv[i] == '6')
         continue;
-      if((orig[i] == '6' || orig[i] == '2') && equiv[i] == '4')
+      if ((orig[i] == '6' || orig[i] == '2') && equiv[i] == '4')
         continue;
       return FALSE;
     }
@@ -615,7 +618,7 @@ static int getID(struct InternalInstruction* insn) {
   else if (isPrefixAtLocation(insn, 0xf2, insn->necessaryPrefixLocation))
     attrMask |= ATTR_XD;
   
-  if(getIDWithAttrMask(&instructionID, insn, attrMask))
+  if (getIDWithAttrMask(&instructionID, insn, attrMask))
     return -1;
   
   /* The following clauses compensate for limitations of the tables. */
@@ -792,7 +795,8 @@ static int readSIB(struct InternalInstruction* insn) {
                        SIB_BASE_EBP : SIB_BASE_RBP);
       break;
     case 0x3:
-      unreachable("Cannot have Mod = 0b11 and a SIB byte");
+      debug("Cannot have Mod = 0b11 and a SIB byte");
+      return -1;
     }
     break;
   default:
@@ -903,7 +907,7 @@ static int readModRM(struct InternalInstruction* insn) {
       if (rm == 0x6) {
         insn->eaBase = EA_BASE_NONE;
         insn->eaDisplacement = EA_DISP_16;
-        if(readDisplacement(insn))
+        if (readDisplacement(insn))
           return -1;
       } else {
         insn->eaBase = (EABase)(insn->eaBaseBase + rm);
@@ -913,18 +917,18 @@ static int readModRM(struct InternalInstruction* insn) {
     case 0x1:
       insn->eaBase = (EABase)(insn->eaBaseBase + rm);
       insn->eaDisplacement = EA_DISP_8;
-      if(readDisplacement(insn))
+      if (readDisplacement(insn))
         return -1;
       break;
     case 0x2:
       insn->eaBase = (EABase)(insn->eaBaseBase + rm);
       insn->eaDisplacement = EA_DISP_16;
-      if(readDisplacement(insn))
+      if (readDisplacement(insn))
         return -1;
       break;
     case 0x3:
       insn->eaBase = (EABase)(insn->eaRegBase + rm);
-      if(readDisplacement(insn))
+      if (readDisplacement(insn))
         return -1;
       break;
     }
@@ -942,13 +946,13 @@ static int readModRM(struct InternalInstruction* insn) {
         insn->eaBase = (insn->addressSize == 4 ? 
                         EA_BASE_sib : EA_BASE_sib64);
         readSIB(insn);
-        if(readDisplacement(insn))
+        if (readDisplacement(insn))
           return -1;
         break;
       case 0x5:
         insn->eaBase = EA_BASE_NONE;
         insn->eaDisplacement = EA_DISP_32;
-        if(readDisplacement(insn))
+        if (readDisplacement(insn))
           return -1;
         break;
       default:
@@ -964,12 +968,12 @@ static int readModRM(struct InternalInstruction* insn) {
       case 0xc:   /* in case REXW.b is set */
         insn->eaBase = EA_BASE_sib;
         readSIB(insn);
-        if(readDisplacement(insn))
+        if (readDisplacement(insn))
           return -1;
         break;
       default:
         insn->eaBase = (EABase)(insn->eaBaseBase + rm);
-        if(readDisplacement(insn))
+        if (readDisplacement(insn))
           return -1;
         break;
       }
@@ -993,11 +997,13 @@ static int readModRM(struct InternalInstruction* insn) {
     *valid = 1;                                           \
     switch (type) {                                       \
     default:                                              \
-      unreachable("Unhandled register type");             \
+      debug("Unhandled register type");                   \
+      *valid = 0;                                         \
+      return 0;                                           \
     case TYPE_Rv:                                         \
       return base + index;                                \
     case TYPE_R8:                                         \
-      if(insn->rexPrefix &&                               \
+      if (insn->rexPrefix &&                              \
          index >= 4 && index <= 7) {                      \
         return prefix##_SPL + (index - 4);                \
       } else {                                            \
@@ -1017,23 +1023,23 @@ static int readModRM(struct InternalInstruction* insn) {
     case TYPE_MM64:                                       \
     case TYPE_MM32:                                       \
     case TYPE_MM:                                         \
-      if(index > 7)                                       \
+      if (index > 7)                                      \
         *valid = 0;                                       \
       return prefix##_MM0 + index;                        \
     case TYPE_SEGMENTREG:                                 \
-      if(index > 5)                                       \
+      if (index > 5)                                      \
         *valid = 0;                                       \
       return prefix##_ES + index;                         \
     case TYPE_DEBUGREG:                                   \
-      if(index > 7)                                       \
+      if (index > 7)                                      \
         *valid = 0;                                       \
       return prefix##_DR0 + index;                        \
     case TYPE_CR32:                                       \
-      if(index > 7)                                       \
+      if (index > 7)                                      \
         *valid = 0;                                       \
       return prefix##_ECR0 + index;                       \
     case TYPE_CR64:                                       \
-      if(index > 8)                                       \
+      if (index > 8)                                      \
         *valid = 0;                                       \
       return prefix##_RCR0 + index;                       \
     }                                                     \
@@ -1050,6 +1056,7 @@ static int readModRM(struct InternalInstruction* insn) {
  * @param index - The existing value of the field as reported by readModRM().
  * @param valid - The address of a uint8_t.  The target is set to 1 if the
  *                field is valid for the register class; 0 if not.
+ * @return      - The proper value.
  */
 GENERIC_FIXUP_FUNC(fixupRegValue, insn->regBase,    MODRM_REG)
 GENERIC_FIXUP_FUNC(fixupRMValue,  insn->eaRegBase,  EA_REG)
@@ -1071,7 +1078,8 @@ static int fixupReg(struct InternalInstruction *insn,
   
   switch ((OperandEncoding)op->encoding) {
   default:
-    unreachable("Expected a REG or R/M encoding in fixupReg");
+    debug("Expected a REG or R/M encoding in fixupReg");
+    return -1;
   case ENCODING_REG:
     insn->reg = (Reg)fixupRegValue(insn,
                                    (OperandType)op->type,
@@ -1102,26 +1110,29 @@ static int fixupReg(struct InternalInstruction *insn,
  * @param insn    - The instruction whose opcode field is to be read.
  * @param inModRM - Indicates that the opcode field is to be read from the
  *                  ModR/M extension; useful for escape opcodes
+ * @return        - 0 on success; nonzero otherwise.
  */
-static void readOpcodeModifier(struct InternalInstruction* insn) {
+static int readOpcodeModifier(struct InternalInstruction* insn) {
   dbgprintf(insn, "readOpcodeModifier()");
   
   if (insn->consumedOpcodeModifier)
-    return;
+    return 0;
   
   insn->consumedOpcodeModifier = TRUE;
   
-  switch(insn->spec->modifierType) {
+  switch (insn->spec->modifierType) {
   default:
-    unreachable("Unknown modifier type.");
+    debug("Unknown modifier type.");
+    return -1;
   case MODIFIER_NONE:
-    unreachable("No modifier but an operand expects one.");
+    debug("No modifier but an operand expects one.");
+    return -1;
   case MODIFIER_OPCODE:
     insn->opcodeModifier = insn->opcode - insn->spec->modifierBase;
-    break;
+    return 0;
   case MODIFIER_MODRM:
     insn->opcodeModifier = insn->modRM - insn->spec->modifierBase;
-    break;
+    return 0;
   }  
 }
 
@@ -1134,11 +1145,13 @@ static void readOpcodeModifier(struct InternalInstruction* insn) {
  * @param size  - The width (in bytes) of the register being specified.
  *                1 means AL and friends, 2 means AX, 4 means EAX, and 8 means
  *                RAX.
+ * @return      - 0 on success; nonzero otherwise.
  */
-static void readOpcodeRegister(struct InternalInstruction* insn, uint8_t size) {
+static int readOpcodeRegister(struct InternalInstruction* insn, uint8_t size) {
   dbgprintf(insn, "readOpcodeRegister()");
 
-  readOpcodeModifier(insn);
+  if (readOpcodeModifier(insn))
+    return -1;
   
   if (size == 0)
     size = insn->registerSize;
@@ -1147,9 +1160,9 @@ static void readOpcodeRegister(struct InternalInstruction* insn, uint8_t size) {
   case 1:
     insn->opcodeRegister = (Reg)(MODRM_REG_AL + ((bFromREX(insn->rexPrefix) << 3) 
                                                   | insn->opcodeModifier));
-    if(insn->rexPrefix && 
-       insn->opcodeRegister >= MODRM_REG_AL + 0x4 &&
-       insn->opcodeRegister < MODRM_REG_AL + 0x8) {
+    if (insn->rexPrefix && 
+        insn->opcodeRegister >= MODRM_REG_AL + 0x4 &&
+        insn->opcodeRegister < MODRM_REG_AL + 0x8) {
       insn->opcodeRegister = (Reg)(MODRM_REG_SPL
                                    + (insn->opcodeRegister - MODRM_REG_AL - 4));
     }
@@ -1161,7 +1174,7 @@ static void readOpcodeRegister(struct InternalInstruction* insn, uint8_t size) {
                                     | insn->opcodeModifier));
     break;
   case 4:
-    insn->opcodeRegister = (Reg)(MODRM_REG_EAX +
+    insn->opcodeRegister = (Reg)(MODRM_REG_EAX
                                  + ((bFromREX(insn->rexPrefix) << 3) 
                                     | insn->opcodeModifier));
     break;
@@ -1171,6 +1184,8 @@ static void readOpcodeRegister(struct InternalInstruction* insn, uint8_t size) {
                                     | insn->opcodeModifier));
     break;
   }
+  
+  return 0;
 }
 
 /*
@@ -1190,8 +1205,10 @@ static int readImmediate(struct InternalInstruction* insn, uint8_t size) {
   
   dbgprintf(insn, "readImmediate()");
   
-  if (insn->numImmediatesConsumed == 2)
-    unreachable("Already consumed two immediates");
+  if (insn->numImmediatesConsumed == 2) {
+    debug("Already consumed two immediates");
+    return -1;
+  }
   
   if (size == 0)
     size = insn->immediateSize;
@@ -1274,29 +1291,35 @@ static int readOperands(struct InternalInstruction* insn) {
         return -1;
       break;
     case ENCODING_Iv:
-      readImmediate(insn, insn->immediateSize);
-      break;
+      if (readImmediate(insn, insn->immediateSize))
+        return -1;
     case ENCODING_Ia:
-      readImmediate(insn, insn->addressSize);
+      if (readImmediate(insn, insn->addressSize))
+        return -1;
       break;
     case ENCODING_RB:
-      readOpcodeRegister(insn, 1);
+      if (readOpcodeRegister(insn, 1))
+        return -1;
       break;
     case ENCODING_RW:
-      readOpcodeRegister(insn, 2);
+      if (readOpcodeRegister(insn, 2))
+        return -1;
       break;
     case ENCODING_RD:
-      readOpcodeRegister(insn, 4);
+      if (readOpcodeRegister(insn, 4))
+        return -1;
       break;
     case ENCODING_RO:
-      readOpcodeRegister(insn, 8);
+      if (readOpcodeRegister(insn, 8))
+        return -1;
       break;
     case ENCODING_Rv:
-      readOpcodeRegister(insn, 0);
+      if (readOpcodeRegister(insn, 0))
+        return -1;
       break;
     case ENCODING_I:
-      readOpcodeModifier(insn);
-      break;
+      if (readOpcodeModifier(insn))
+        return -1;
     case ENCODING_DUP:
       break;
     default:
diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
index c03c07a3895c..462cf68ccdb2 100644
--- a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
+++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
@@ -508,6 +508,17 @@ int decodeInstruction(struct InternalInstruction* insn,
                       uint64_t startLoc,
                       DisassemblerMode mode);
 
+/* x86DisassemblerDebug - C-accessible function for printing a message to
+ *   debugs()
+ * @param file  - The name of the file printing the debug message.
+ * @param line  - The line number that printed the debug message.
+ * @param s     - The message to print.
+ */
+  
+void x86DisassemblerDebug(const char *file,
+                          unsigned line,
+                          const char *s);
+
 #ifdef __cplusplus 
 }
 #endif
diff --git a/lib/Target/X86/X86.td b/lib/Target/X86/X86.td
index 6b62795a6e14..89cc84ffb3c6 100644
--- a/lib/Target/X86/X86.td
+++ b/lib/Target/X86/X86.td
@@ -74,6 +74,8 @@ def FeatureFMA4    : SubtargetFeature<"fma4", "HasFMA4", "true",
 def FeatureVectorUAMem : SubtargetFeature<"vector-unaligned-mem",
                                           "HasVectorUAMem", "true",
                  "Allow unaligned memory operands on vector/SIMD instructions">;
+def FeatureAES     : SubtargetFeature<"aes", "HasAES", "true",
+                                      "Enable AES instructions">;
 
 //===----------------------------------------------------------------------===//
 // X86 processors supported.
@@ -101,11 +103,17 @@ def : Proc<"nocona",          [FeatureSSE3,   Feature64Bit, FeatureSlowBTMem]>;
 def : Proc<"core2",           [FeatureSSSE3,  Feature64Bit, FeatureSlowBTMem]>;
 def : Proc<"penryn",          [FeatureSSE41,  Feature64Bit, FeatureSlowBTMem]>;
 def : Proc<"atom",            [FeatureSSE3,   Feature64Bit, FeatureSlowBTMem]>;
+// "Arrandale" along with corei3 and corei5
 def : Proc<"corei7",          [FeatureSSE42,  Feature64Bit, FeatureSlowBTMem,
-                               FeatureFastUAMem]>;
+                               FeatureFastUAMem, FeatureAES]>;
 def : Proc<"nehalem",         [FeatureSSE42,  Feature64Bit, FeatureSlowBTMem,
                                FeatureFastUAMem]>;
+// Westmere is a similar machine to nehalem with some additional features.
+// Westmere is the corei3/i5/i7 path from nehalem to sandybridge
+def : Proc<"westmere",         [FeatureSSE42,  Feature64Bit, FeatureSlowBTMem,
+                                FeatureFastUAMem, FeatureAES]>;
 // Sandy Bridge does not have FMA
+// FIXME: Wikipedia says it does... it should have AES as well.
 def : Proc<"sandybridge",     [FeatureSSE42,  FeatureAVX,   Feature64Bit]>;
 
 def : Proc<"k6",              [FeatureMMX]>;
diff --git a/lib/Target/X86/X86FloatingPointRegKill.cpp b/lib/Target/X86/X86FloatingPointRegKill.cpp
index 6a117dde9bde..541083f6fffd 100644
--- a/lib/Target/X86/X86FloatingPointRegKill.cpp
+++ b/lib/Target/X86/X86FloatingPointRegKill.cpp
@@ -129,7 +129,7 @@ bool FPRegKiller::runOnMachineFunction(MachineFunction &MF) {
     }
     // Finally, if we found any FP code, emit the FP_REG_KILL instruction.
     if (ContainsFPCode) {
-      BuildMI(*MBB, MBBI->getFirstTerminator(), DebugLoc::getUnknownLoc(),
+      BuildMI(*MBB, MBBI->getFirstTerminator(), DebugLoc(),
               MF.getTarget().getInstrInfo()->get(X86::FP_REG_KILL));
       ++NumFPKill;
       Changed = true;
diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp
index 1c0ed7e6327c..da45dac807f6 100644
--- a/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -542,7 +542,7 @@ void X86DAGToDAGISel::EmitSpecialCodeForMain(MachineBasicBlock *BB,
                                              MachineFrameInfo *MFI) {
   const TargetInstrInfo *TII = TM.getInstrInfo();
   if (Subtarget->isTargetCygMing())
-    BuildMI(BB, DebugLoc::getUnknownLoc(),
+    BuildMI(BB, DebugLoc(),
             TII->get(X86::CALLpcrel32)).addExternalSymbol("__main");
 }
 
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index b24d5a1707a0..527e19b60882 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -1071,18 +1071,21 @@ unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const {
 /// If DstAlign is zero that means it's safe to destination alignment can
 /// satisfy any constraint. Similarly if SrcAlign is zero it means there
 /// isn't a need to check it against alignment requirement, probably because
-/// the source does not need to be loaded. It returns EVT::Other if
-/// SelectionDAG should be responsible for determining it.
+/// the source does not need to be loaded. If 'NonScalarIntSafe' is true, that
+/// means it's safe to return a non-scalar-integer type, e.g. constant string
+/// source or loaded from memory. It returns EVT::Other if SelectionDAG should
+/// be responsible for determining it.
 EVT
 X86TargetLowering::getOptimalMemOpType(uint64_t Size,
                                        unsigned DstAlign, unsigned SrcAlign,
-                                       bool SafeToUseFP,
+                                       bool NonScalarIntSafe,
                                        SelectionDAG &DAG) const {
   // FIXME: This turns off use of xmm stores for memset/memcpy on targets like
   // linux.  This is because the stack realignment code can't handle certain
   // cases like PR2962.  This should be removed when PR2962 is fixed.
   const Function *F = DAG.getMachineFunction().getFunction();
-  if (!F->hasFnAttr(Attribute::NoImplicitFloat)) {
+  if (NonScalarIntSafe &&
+      !F->hasFnAttr(Attribute::NoImplicitFloat)) {
     if (Size >= 16 &&
         (Subtarget->isUnalignedMemAccessFast() ||
          ((DstAlign == 0 || DstAlign >= 16) &&
@@ -1090,10 +1093,9 @@ X86TargetLowering::getOptimalMemOpType(uint64_t Size,
         Subtarget->getStackAlignment() >= 16) {
       if (Subtarget->hasSSE2())
         return MVT::v4i32;
-      if (SafeToUseFP && Subtarget->hasSSE1())
+      if (Subtarget->hasSSE1())
         return MVT::v4f32;
-    } else if (SafeToUseFP &&
-               Size >= 8 &&
+    } else if (Size >= 8 &&
                !Subtarget->is64Bit() &&
                Subtarget->getStackAlignment() >= 8 &&
                Subtarget->hasSSE2())
@@ -1147,8 +1149,7 @@ SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
   if (!Subtarget->is64Bit())
     // This doesn't have DebugLoc associated with it, but is not really the
     // same as a Register.
-    return DAG.getNode(X86ISD::GlobalBaseReg, DebugLoc::getUnknownLoc(),
-                       getPointerTy());
+    return DAG.getNode(X86ISD::GlobalBaseReg, DebugLoc(), getPointerTy());
   return Table;
 }
 
@@ -1929,8 +1930,7 @@ X86TargetLowering::LowerCall(SDValue Chain, SDValue Callee,
     if (!isTailCall) {
       Chain = DAG.getCopyToReg(Chain, dl, X86::EBX,
                                DAG.getNode(X86ISD::GlobalBaseReg,
-                                           DebugLoc::getUnknownLoc(),
-                                           getPointerTy()),
+                                           DebugLoc(), getPointerTy()),
                                InFlag);
       InFlag = Chain.getValue(1);
     } else {
@@ -5059,7 +5059,7 @@ X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) {
   if (OpFlag) {
     Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
                          DAG.getNode(X86ISD::GlobalBaseReg,
-                                     DebugLoc::getUnknownLoc(), getPointerTy()),
+                                     DebugLoc(), getPointerTy()),
                          Result);
   }
 
@@ -5092,7 +5092,7 @@ SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) {
   if (OpFlag) {
     Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
                          DAG.getNode(X86ISD::GlobalBaseReg,
-                                     DebugLoc::getUnknownLoc(), getPointerTy()),
+                                     DebugLoc(), getPointerTy()),
                          Result);
   }
 
@@ -5128,8 +5128,7 @@ X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) {
       !Subtarget->is64Bit()) {
     Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
                          DAG.getNode(X86ISD::GlobalBaseReg,
-                                     DebugLoc::getUnknownLoc(),
-                                     getPointerTy()),
+                                     DebugLoc(), getPointerTy()),
                          Result);
   }
 
@@ -5251,8 +5250,7 @@ LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
   DebugLoc dl = GA->getDebugLoc();  // ? function entry point might be better
   SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
                                      DAG.getNode(X86ISD::GlobalBaseReg,
-                                                 DebugLoc::getUnknownLoc(),
-                                                 PtrVT), InFlag);
+                                                 DebugLoc(), PtrVT), InFlag);
   InFlag = Chain.getValue(1);
 
   return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD);
@@ -5274,7 +5272,7 @@ static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
   DebugLoc dl = GA->getDebugLoc();
   // Get the Thread Pointer
   SDValue Base = DAG.getNode(X86ISD::SegmentBaseAddress,
-                             DebugLoc::getUnknownLoc(), PtrVT,
+                             DebugLoc(), PtrVT,
                              DAG.getRegister(is64Bit? X86::FS : X86::GS,
                                              MVT::i32));
 
diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h
index 4549cba6f397..2c2a5fbb8032 100644
--- a/lib/Target/X86/X86ISelLowering.h
+++ b/lib/Target/X86/X86ISelLowering.h
@@ -417,15 +417,19 @@ namespace llvm {
     virtual unsigned getByValTypeAlignment(const Type *Ty) const;
 
     /// getOptimalMemOpType - Returns the target specific optimal type for load
-    /// and store operations as a result of memset, memcpy, and memmove lowering.
-    /// If DstAlign is zero that means it's safe to destination alignment can
-    /// satisfy any constraint. Similarly if SrcAlign is zero it means there
-    /// isn't a need to check it against alignment requirement, probably because
-    /// the source does not need to be loaded. It returns EVT::Other if
-    /// SelectionDAG should be responsible for determining it.
-    virtual EVT getOptimalMemOpType(uint64_t Size,
-                                    unsigned DstAlign, unsigned SrcAlign,
-                                    bool SafeToUseFP, SelectionDAG &DAG) const;
+    /// and store operations as a result of memset, memcpy, and memmove
+    /// lowering. If DstAlign is zero that means it's safe to destination
+    /// alignment can satisfy any constraint. Similarly if SrcAlign is zero it
+    /// means there isn't a need to check it against alignment requirement,
+    /// probably because the source does not need to be loaded. If
+    /// 'NonScalarIntSafe' is true, that means it's safe to return a
+    /// non-scalar-integer type, e.g. empty string source, constant, or loaded
+    /// from memory. It returns EVT::Other if SelectionDAG should be responsible
+    /// for determining it.
+    virtual EVT
+    getOptimalMemOpType(uint64_t Size,
+                        unsigned DstAlign, unsigned SrcAlign,
+                        bool NonScalarIntSafe, SelectionDAG &DAG) const;
 
     /// allowsUnalignedMemoryAccesses - Returns true if the target allows
     /// unaligned memory accesses. of the specified type.
diff --git a/lib/Target/X86/X86InstrFormats.td b/lib/Target/X86/X86InstrFormats.td
index d25ec260491e..cbe4c8256299 100644
--- a/lib/Target/X86/X86InstrFormats.td
+++ b/lib/Target/X86/X86InstrFormats.td
@@ -311,6 +311,21 @@ class SS42AI<bits<8> o, Format F, dag outs, dag ins, string asm,
       : Ii8<o, F, outs, ins, asm, pattern, SSEPackedInt>, TA,
         Requires<[HasSSE42]>;
 
+// AES Instruction Templates:
+//
+// AES8I
+// FIXME: Verify these, they appear to use the same encoding as the SSE4.2 T8
+// and TA encodings.
+class AES8I<bits<8> o, Format F, dag outs, dag ins, string asm,
+            list<dag>pattern>
+      : I<o, F, outs, ins, asm, pattern, SSEPackedInt>, T8,
+        Requires<[HasAES]>;
+
+class AESAI<bits<8> o, Format F, dag outs, dag ins, string asm,
+            list<dag> pattern>
+      : Ii8<o, F, outs, ins, asm, pattern, SSEPackedInt>, TA,
+        Requires<[HasAES]>;
+
 // X86-64 Instruction templates...
 //
 
diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp
index c0c9d98ffea8..fcb9947423de 100644
--- a/lib/Target/X86/X86InstrInfo.cpp
+++ b/lib/Target/X86/X86InstrInfo.cpp
@@ -1803,7 +1803,7 @@ X86InstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
                            MachineBasicBlock *FBB,
                            const SmallVectorImpl<MachineOperand> &Cond) const {
   // FIXME this should probably have a DebugLoc operand
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
   assert((Cond.size() == 1 || Cond.size() == 0) &&
@@ -2107,7 +2107,7 @@ void X86InstrInfo::storeRegToAddr(MachineFunction &MF, unsigned SrcReg,
                                   SmallVectorImpl<MachineInstr*> &NewMIs) const {
   bool isAligned = (*MMOBegin)->getAlignment() >= 16;
   unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, TM);
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc));
   for (unsigned i = 0, e = Addr.size(); i != e; ++i)
     MIB.addOperand(Addr[i]);
@@ -2202,7 +2202,7 @@ void X86InstrInfo::loadRegFromAddr(MachineFunction &MF, unsigned DestReg,
                                  SmallVectorImpl<MachineInstr*> &NewMIs) const {
   bool isAligned = (*MMOBegin)->getAlignment() >= 16;
   unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, TM);
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   MachineInstrBuilder MIB = BuildMI(MF, DL, get(Opc), DestReg);
   for (unsigned i = 0, e = Addr.size(); i != e; ++i)
     MIB.addOperand(Addr[i]);
diff --git a/lib/Target/X86/X86InstrInfo.td b/lib/Target/X86/X86InstrInfo.td
index 8fccc8a37aca..65b7ec023d81 100644
--- a/lib/Target/X86/X86InstrInfo.td
+++ b/lib/Target/X86/X86InstrInfo.td
@@ -330,6 +330,7 @@ def OptForSize   : Predicate<"OptForSize">;
 def OptForSpeed  : Predicate<"!OptForSize">;
 def FastBTMem    : Predicate<"!Subtarget->isBTMemSlow()">;
 def CallImmAddr  : Predicate<"Subtarget->IsLegalToCallImmediateAddr(TM)">;
+def HasAES       : Predicate<"Subtarget->hasAES()">;
 
 //===----------------------------------------------------------------------===//
 // X86 Instruction Format Definitions.
diff --git a/lib/Target/X86/X86InstrSSE.td b/lib/Target/X86/X86InstrSSE.td
index dadc2a663b57..11f7e27c4fc3 100644
--- a/lib/Target/X86/X86InstrSSE.td
+++ b/lib/Target/X86/X86InstrSSE.td
@@ -3848,53 +3848,6 @@ def : Pat<(v2i64 (X86pcmpgtq VR128:$src1, VR128:$src2)),
 def : Pat<(v2i64 (X86pcmpgtq VR128:$src1, (memop addr:$src2))),
           (PCMPGTQrm VR128:$src1, addr:$src2)>;
 
-// TODO: These should be AES as a feature set.
-defm AESIMC          : SS42I_binop_rm_int<0xDB, "aesimc",
-                       int_x86_aesni_aesimc>;
-defm AESENC          : SS42I_binop_rm_int<0xDC, "aesenc",
-                       int_x86_aesni_aesenc>;
-defm AESENCLAST      : SS42I_binop_rm_int<0xDD, "aesenclast",
-                       int_x86_aesni_aesenclast>;
-defm AESDEC          : SS42I_binop_rm_int<0xDE, "aesdec",
-                       int_x86_aesni_aesdec>;
-defm AESDECLAST      : SS42I_binop_rm_int<0xDF, "aesdeclast",
-                       int_x86_aesni_aesdeclast>;
-
-def : Pat<(v2i64 (int_x86_aesni_aesimc VR128:$src1, VR128:$src2)),
-          (AESIMCrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesimc VR128:$src1, (memop addr:$src2))),
-          (AESIMCrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesenc VR128:$src1, VR128:$src2)),
-          (AESENCrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesenc VR128:$src1, (memop addr:$src2))),
-          (AESENCrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesenclast VR128:$src1, VR128:$src2)),
-          (AESENCLASTrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesenclast VR128:$src1, (memop addr:$src2))),
-          (AESENCLASTrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesdec VR128:$src1, VR128:$src2)),
-          (AESDECrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesdec VR128:$src1, (memop addr:$src2))),
-          (AESDECrm VR128:$src1, addr:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesdeclast VR128:$src1, VR128:$src2)),
-          (AESDECLASTrr VR128:$src1, VR128:$src2)>;
-def : Pat<(v2i64 (int_x86_aesni_aesdeclast VR128:$src1, (memop addr:$src2))),
-          (AESDECLASTrm VR128:$src1, addr:$src2)>;
-
-def AESKEYGENASSIST128rr : SS42AI<0xDF, MRMSrcReg, (outs VR128:$dst),
-  (ins VR128:$src1, i32i8imm:$src2),
-  "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-  [(set VR128:$dst,
-    (int_x86_aesni_aeskeygenassist VR128:$src1, imm:$src2))]>,
-  OpSize;
-def AESKEYGENASSIST128rm : SS42AI<0xDF, MRMSrcMem, (outs VR128:$dst),
-  (ins i128mem:$src1, i32i8imm:$src2),
-  "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-  [(set VR128:$dst,
-    (int_x86_aesni_aeskeygenassist (bitconvert (memopv2i64 addr:$src1)),
-                                    imm:$src2))]>,
-  OpSize;
-
 // crc intrinsic instruction
 // This set of instructions are only rm, the only difference is the size
 // of r and m.
@@ -4056,3 +4009,81 @@ defm PCMPESTRIC : SS42AI_pcmpestri<int_x86_sse42_pcmpestric128>;
 defm PCMPESTRIO : SS42AI_pcmpestri<int_x86_sse42_pcmpestrio128>;
 defm PCMPESTRIS : SS42AI_pcmpestri<int_x86_sse42_pcmpestris128>;
 defm PCMPESTRIZ : SS42AI_pcmpestri<int_x86_sse42_pcmpestriz128>;
+
+//===----------------------------------------------------------------------===//
+// AES-NI Instructions
+//===----------------------------------------------------------------------===//
+
+let Constraints = "$src1 = $dst" in {
+  multiclass AESI_binop_rm_int<bits<8> opc, string OpcodeStr,
+                                Intrinsic IntId128, bit Commutable = 0> {
+    def rr : AES8I<opc, MRMSrcReg, (outs VR128:$dst),
+                   (ins VR128:$src1, VR128:$src2),
+                   !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
+                   [(set VR128:$dst, (IntId128 VR128:$src1, VR128:$src2))]>,
+                   OpSize {
+      let isCommutable = Commutable;
+    }
+    def rm : AES8I<opc, MRMSrcMem, (outs VR128:$dst),
+                   (ins VR128:$src1, i128mem:$src2),
+                   !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
+                   [(set VR128:$dst,
+                     (IntId128 VR128:$src1,
+                      (bitconvert (memopv16i8 addr:$src2))))]>, OpSize;
+  }
+}
+
+defm AESENC          : AESI_binop_rm_int<0xDC, "aesenc",
+                       int_x86_aesni_aesenc>;
+defm AESENCLAST      : AESI_binop_rm_int<0xDD, "aesenclast",
+                       int_x86_aesni_aesenclast>;
+defm AESDEC          : AESI_binop_rm_int<0xDE, "aesdec",
+                       int_x86_aesni_aesdec>;
+defm AESDECLAST      : AESI_binop_rm_int<0xDF, "aesdeclast",
+                       int_x86_aesni_aesdeclast>;
+
+def : Pat<(v2i64 (int_x86_aesni_aesenc VR128:$src1, VR128:$src2)),
+          (AESENCrr VR128:$src1, VR128:$src2)>;
+def : Pat<(v2i64 (int_x86_aesni_aesenc VR128:$src1, (memop addr:$src2))),
+          (AESENCrm VR128:$src1, addr:$src2)>;
+def : Pat<(v2i64 (int_x86_aesni_aesenclast VR128:$src1, VR128:$src2)),
+          (AESENCLASTrr VR128:$src1, VR128:$src2)>;
+def : Pat<(v2i64 (int_x86_aesni_aesenclast VR128:$src1, (memop addr:$src2))),
+          (AESENCLASTrm VR128:$src1, addr:$src2)>;
+def : Pat<(v2i64 (int_x86_aesni_aesdec VR128:$src1, VR128:$src2)),
+          (AESDECrr VR128:$src1, VR128:$src2)>;
+def : Pat<(v2i64 (int_x86_aesni_aesdec VR128:$src1, (memop addr:$src2))),
+          (AESDECrm VR128:$src1, addr:$src2)>;
+def : Pat<(v2i64 (int_x86_aesni_aesdeclast VR128:$src1, VR128:$src2)),
+          (AESDECLASTrr VR128:$src1, VR128:$src2)>;
+def : Pat<(v2i64 (int_x86_aesni_aesdeclast VR128:$src1, (memop addr:$src2))),
+          (AESDECLASTrm VR128:$src1, addr:$src2)>;
+
+def AESIMCrr : AES8I<0xDB, MRMSrcReg, (outs VR128:$dst),
+  (ins VR128:$src1),
+  "aesimc\t{$src1, $dst|$dst, $src1}",
+  [(set VR128:$dst,
+    (int_x86_aesni_aesimc VR128:$src1))]>,
+  OpSize;
+
+def AESIMCrm : AES8I<0xDB, MRMSrcMem, (outs VR128:$dst),
+  (ins i128mem:$src1),
+  "aesimc\t{$src1, $dst|$dst, $src1}",
+  [(set VR128:$dst,
+    (int_x86_aesni_aesimc (bitconvert (memopv2i64 addr:$src1))))]>,
+  OpSize;
+
+def AESKEYGENASSIST128rr : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst),
+  (ins VR128:$src1, i32i8imm:$src2),
+  "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+  [(set VR128:$dst,
+    (int_x86_aesni_aeskeygenassist VR128:$src1, imm:$src2))]>,
+  OpSize;
+def AESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst),
+  (ins i128mem:$src1, i32i8imm:$src2),
+  "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+  [(set VR128:$dst,
+    (int_x86_aesni_aeskeygenassist (bitconvert (memopv2i64 addr:$src1)),
+                                    imm:$src2))]>,
+  OpSize;
+
diff --git a/lib/Target/X86/X86Subtarget.cpp b/lib/Target/X86/X86Subtarget.cpp
index 8a0cde49aea6..09a26858eb7e 100644
--- a/lib/Target/X86/X86Subtarget.cpp
+++ b/lib/Target/X86/X86Subtarget.cpp
@@ -259,6 +259,7 @@ void X86Subtarget::AutoDetectSubtargetFeatures() {
 
   HasFMA3 = IsIntel && ((ECX >> 12) & 0x1);
   HasAVX = ((ECX >> 28) & 0x1);
+  HasAES = IsIntel && ((ECX >> 25) & 0x1);
 
   if (IsIntel || IsAMD) {
     // Determine if bit test memory instructions are slow.
@@ -286,6 +287,7 @@ X86Subtarget::X86Subtarget(const std::string &TT, const std::string &FS,
   , HasX86_64(false)
   , HasSSE4A(false)
   , HasAVX(false)
+  , HasAES(false)
   , HasFMA3(false)
   , HasFMA4(false)
   , IsBTMemSlow(false)
diff --git a/lib/Target/X86/X86Subtarget.h b/lib/Target/X86/X86Subtarget.h
index bf30154625bf..8a873f04df4e 100644
--- a/lib/Target/X86/X86Subtarget.h
+++ b/lib/Target/X86/X86Subtarget.h
@@ -69,6 +69,9 @@ protected:
   /// HasAVX - Target has AVX instructions
   bool HasAVX;
 
+  /// HasAES - Target has AES instructions
+  bool HasAES;
+
   /// HasFMA3 - Target has 3-operand fused multiply-add
   bool HasFMA3;
 
@@ -148,6 +151,7 @@ public:
   bool has3DNow() const { return X863DNowLevel >= ThreeDNow; }
   bool has3DNowA() const { return X863DNowLevel >= ThreeDNowA; }
   bool hasAVX() const { return HasAVX; }
+  bool hasAES() const { return HasAES; }
   bool hasFMA3() const { return HasFMA3; }
   bool hasFMA4() const { return HasFMA4; }
   bool isBTMemSlow() const { return IsBTMemSlow; }
diff --git a/lib/Target/XCore/XCoreInstrInfo.cpp b/lib/Target/XCore/XCoreInstrInfo.cpp
index 54df33c50647..ae3f16c2dfe4 100644
--- a/lib/Target/XCore/XCoreInstrInfo.cpp
+++ b/lib/Target/XCore/XCoreInstrInfo.cpp
@@ -301,7 +301,7 @@ XCoreInstrInfo::InsertBranch(MachineBasicBlock &MBB,MachineBasicBlock *TBB,
                              MachineBasicBlock *FBB,
                              const SmallVectorImpl<MachineOperand> &Cond)const{
   // FIXME there should probably be a DebugLoc argument here
-  DebugLoc dl = DebugLoc::getUnknownLoc();
+  DebugLoc dl;
   // Shouldn't be a fall through.
   assert(TBB && "InsertBranch must not be told to insert a fallthrough");
   assert((Cond.size() == 2 || Cond.size() == 0) &&
@@ -362,7 +362,7 @@ bool XCoreInstrInfo::copyRegToReg(MachineBasicBlock &MBB,
                                   unsigned DestReg, unsigned SrcReg,
                                   const TargetRegisterClass *DestRC,
                                   const TargetRegisterClass *SrcRC) const {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
 
   if (DestRC == SrcRC) {
@@ -397,7 +397,7 @@ void XCoreInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
                                          int FrameIndex,
                                          const TargetRegisterClass *RC) const
 {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
   BuildMI(MBB, I, DL, get(XCore::STWFI))
     .addReg(SrcReg, getKillRegState(isKill))
@@ -410,7 +410,7 @@ void XCoreInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                           unsigned DestReg, int FrameIndex,
                                           const TargetRegisterClass *RC) const
 {
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
   BuildMI(MBB, I, DL, get(XCore::LDWFI), DestReg)
     .addFrameIndex(FrameIndex)
@@ -431,7 +431,7 @@ bool XCoreInstrInfo::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
   
   bool emitFrameMoves = XCoreRegisterInfo::needsFrameMoves(*MF);
 
-  DebugLoc DL = DebugLoc::getUnknownLoc();
+  DebugLoc DL;
   if (MI != MBB.end()) DL = MI->getDebugLoc();
   
   for (std::vector<CalleeSavedInfo>::const_iterator it = CSI.begin();
diff --git a/lib/Target/XCore/XCoreRegisterInfo.cpp b/lib/Target/XCore/XCoreRegisterInfo.cpp
index 88925047ccd7..1631e7399922 100644
--- a/lib/Target/XCore/XCoreRegisterInfo.cpp
+++ b/lib/Target/XCore/XCoreRegisterInfo.cpp
@@ -414,8 +414,7 @@ void XCoreRegisterInfo::emitPrologue(MachineFunction &MF) const {
   MachineFrameInfo *MFI = MF.getFrameInfo();
   MachineModuleInfo *MMI = MFI->getMachineModuleInfo();
   XCoreFunctionInfo *XFI = MF.getInfo<XCoreFunctionInfo>();
-  DebugLoc dl = (MBBI != MBB.end() ?
-                 MBBI->getDebugLoc() : DebugLoc::getUnknownLoc());
+  DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   bool FP = hasFP(MF);