vendor/llvm-project/llvmorg-13-init-16847-g88e66fa60ae5vendor/llvm-project/llvmorg-12.0.1-rc2-0-ge7dac564cd0evendor/llvm-project/llvmorg-12.0.1-0-gfed41342a82f

47 files changed, 1444 insertions, 1070 deletions

diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index 9d304910ba4e..00bdd127e3c2 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -612,7 +612,7 @@ struct MatchableInfo {
   /// operator< - Compare two matchables.
   bool operator<(const MatchableInfo &RHS) const {
     // The primary comparator is the instruction mnemonic.
-    if (int Cmp = Mnemonic.compare_lower(RHS.Mnemonic))
+    if (int Cmp = Mnemonic.compare_insensitive(RHS.Mnemonic))
       return Cmp == -1;
 
     if (AsmOperands.size() != RHS.AsmOperands.size())
@@ -1100,8 +1100,8 @@ bool MatchableInfo::validate(StringRef CommentDelimiter, bool IsAlias) const {
 static std::string getEnumNameForToken(StringRef Str) {
   std::string Res;
 
-  for (StringRef::iterator it = Str.begin(), ie = Str.end(); it != ie; ++it) {
-    switch (*it) {
+  for (char C : Str) {
+    switch (C) {
     case '*': Res += "_STAR_"; break;
     case '%': Res += "_PCT_"; break;
     case ':': Res += "_COLON_"; break;
@@ -1112,10 +1112,10 @@ static std::string getEnumNameForToken(StringRef Str) {
     case '-': Res += "_MINUS_"; break;
     case '#': Res += "_HASH_"; break;
     default:
-      if (isAlnum(*it))
-        Res += *it;
+      if (isAlnum(C))
+        Res += C;
       else
-        Res += "_" + utostr((unsigned) *it) + "_";
+        Res += "_" + utostr((unsigned)C) + "_";
     }
   }
 
@@ -1322,9 +1322,8 @@ buildRegisterClasses(SmallPtrSetImpl<Record*> &SingletonRegisters) {
   }
 
   // Populate the map for individual registers.
-  for (std::map<Record*, RegisterSet>::iterator it = RegisterMap.begin(),
-         ie = RegisterMap.end(); it != ie; ++it)
-    RegisterClasses[it->first] = RegisterSetClasses[it->second];
+  for (auto &It : RegisterMap)
+    RegisterClasses[It.first] = RegisterSetClasses[It.second];
 
   // Name the register classes which correspond to singleton registers.
   for (Record *Rec : SingletonRegisters) {
@@ -1529,9 +1528,8 @@ void AsmMatcherInfo::buildInfo() {
     // matchables.
     std::vector<Record*> AllInstAliases =
       Records.getAllDerivedDefinitions("InstAlias");
-    for (unsigned i = 0, e = AllInstAliases.size(); i != e; ++i) {
-      auto Alias = std::make_unique<CodeGenInstAlias>(AllInstAliases[i],
-                                                       Target);
+    for (Record *InstAlias : AllInstAliases) {
+      auto Alias = std::make_unique<CodeGenInstAlias>(InstAlias, Target);
 
       // If the tblgen -match-prefix option is specified (for tblgen hackers),
       // filter the set of instruction aliases we consider, based on the target
@@ -2726,7 +2724,7 @@ static void emitMnemonicAliasVariant(raw_ostream &OS,const AsmMatcherInfo &Info,
     StringRef AsmVariantName = R->getValueAsString("AsmVariantName");
     if (AsmVariantName != AsmParserVariantName)
       continue;
-    AliasesFromMnemonic[std::string(R->getValueAsString("FromMnemonic"))]
+    AliasesFromMnemonic[R->getValueAsString("FromMnemonic").lower()]
         .push_back(R);
   }
   if (AliasesFromMnemonic.empty())
@@ -2751,10 +2749,14 @@ static void emitMnemonicAliasVariant(raw_ostream &OS,const AsmMatcherInfo &Info,
       // If this unconditionally matches, remember it for later and diagnose
       // duplicates.
       if (FeatureMask.empty()) {
-        if (AliasWithNoPredicate != -1) {
-          // We can't have two aliases from the same mnemonic with no predicate.
-          PrintError(ToVec[AliasWithNoPredicate]->getLoc(),
-                     "two MnemonicAliases with the same 'from' mnemonic!");
+        if (AliasWithNoPredicate != -1 &&
+            R->getValueAsString("ToMnemonic") !=
+                ToVec[AliasWithNoPredicate]->getValueAsString("ToMnemonic")) {
+          // We can't have two different aliases from the same mnemonic with no
+          // predicate.
+          PrintError(
+              ToVec[AliasWithNoPredicate]->getLoc(),
+              "two different MnemonicAliases with the same 'from' mnemonic!");
           PrintFatalError(R->getLoc(), "this is the other MnemonicAlias.");
         }
 
@@ -2768,7 +2770,7 @@ static void emitMnemonicAliasVariant(raw_ostream &OS,const AsmMatcherInfo &Info,
         MatchCode += "else ";
       MatchCode += "if (" + FeatureMask + ")\n";
       MatchCode += "  Mnemonic = \"";
-      MatchCode += R->getValueAsString("ToMnemonic");
+      MatchCode += R->getValueAsString("ToMnemonic").lower();
       MatchCode += "\";\n";
     }
 
@@ -2777,7 +2779,7 @@ static void emitMnemonicAliasVariant(raw_ostream &OS,const AsmMatcherInfo &Info,
       if (!MatchCode.empty())
         MatchCode += "else\n  ";
       MatchCode += "Mnemonic = \"";
-      MatchCode += R->getValueAsString("ToMnemonic");
+      MatchCode += R->getValueAsString("ToMnemonic").lower();
       MatchCode += "\";\n";
     }
 
@@ -2883,14 +2885,10 @@ static void emitCustomOperandParsing(raw_ostream &OS, CodeGenTarget &Target,
 
     OS << OMI.OperandMask;
     OS << " /* ";
-    bool printComma = false;
+    ListSeparator LS;
     for (int i = 0, e = 31; i !=e; ++i)
-      if (OMI.OperandMask & (1 << i)) {
-        if (printComma)
-          OS << ", ";
-        OS << i;
-        printComma = true;
-      }
+      if (OMI.OperandMask & (1 << i))
+        OS << LS << i;
     OS << " */, ";
 
     OS << OMI.CI->Name;
@@ -3513,12 +3511,9 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
           OS << '_' << MI->RequiredFeatures[i]->TheDef->getName();
 
       OS << ", { ";
-      for (unsigned i = 0, e = MI->AsmOperands.size(); i != e; ++i) {
-        const MatchableInfo::AsmOperand &Op = MI->AsmOperands[i];
-
-        if (i) OS << ", ";
-        OS << Op.Class->Name;
-      }
+      ListSeparator LS;
+      for (const MatchableInfo::AsmOperand &Op : MI->AsmOperands)
+        OS << LS << Op.Class->Name;
       OS << " }, },\n";
     }
 
diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp
index 92df204475b9..94fd8f7e92b4 100644
--- a/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -994,8 +994,7 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
         });
       }
 
-      for (auto I = ReqFeatures.cbegin(); I != ReqFeatures.cend(); I++) {
-        Record *R = *I;
+      for (Record *const R : ReqFeatures) {
         const DagInit *D = R->getValueAsDag("AssemblerCondDag");
         std::string CombineType = D->getOperator()->getAsString();
         if (CombineType != "any_of" && CombineType != "all_of")
diff --git a/llvm/utils/TableGen/AsmWriterInst.h b/llvm/utils/TableGen/AsmWriterInst.h
index 366c9eca664f..fe2b934e266f 100644
--- a/llvm/utils/TableGen/AsmWriterInst.h
+++ b/llvm/utils/TableGen/AsmWriterInst.h
@@ -66,7 +66,8 @@ namespace llvm {
     bool operator!=(const AsmWriterOperand &Other) const {
       if (OperandType != Other.OperandType || Str != Other.Str) return true;
       if (OperandType == isMachineInstrOperand)
-        return MIOpNo != Other.MIOpNo || MiModifier != Other.MiModifier;
+        return MIOpNo != Other.MIOpNo || MiModifier != Other.MiModifier ||
+               PCRel != Other.PCRel;
       return false;
     }
     bool operator==(const AsmWriterOperand &Other) const {
diff --git a/llvm/utils/TableGen/Attributes.cpp b/llvm/utils/TableGen/Attributes.cpp
index f3f875e8ce0b..5deac4b34bf2 100644
--- a/llvm/utils/TableGen/Attributes.cpp
+++ b/llvm/utils/TableGen/Attributes.cpp
@@ -25,6 +25,7 @@ public:
 private:
   void emitTargetIndependentNames(raw_ostream &OS);
   void emitFnAttrCompatCheck(raw_ostream &OS, bool IsStringAttr);
+  void emitAttributeProperties(raw_ostream &OF);
 
   RecordKeeper &Records;
 };
@@ -58,7 +59,20 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   Emit({"StrBoolAttr"}, "ATTRIBUTE_STRBOOL");
 
   OS << "#undef ATTRIBUTE_ALL\n";
-  OS << "#endif\n";
+  OS << "#endif\n\n";
+
+  OS << "#ifdef GET_ATTR_ENUM\n";
+  OS << "#undef GET_ATTR_ENUM\n";
+  unsigned Value = 1; // Leave zero for AttrKind::None.
+  for (StringRef KindName : {"EnumAttr", "TypeAttr", "IntAttr"}) {
+    OS << "First" << KindName << " = " << Value << ",\n";
+    for (auto A : Records.getAllDerivedDefinitions(KindName)) {
+      OS << A->getName() << " = " << Value << ",\n";
+      Value++;
+    }
+    OS << "Last" << KindName << " = " << (Value - 1) << ",\n";
+  }
+  OS << "#endif\n\n";
 }
 
 void Attributes::emitFnAttrCompatCheck(raw_ostream &OS, bool IsStringAttr) {
@@ -96,9 +110,26 @@ void Attributes::emitFnAttrCompatCheck(raw_ostream &OS, bool IsStringAttr) {
   OS << "#endif\n";
 }
 
+void Attributes::emitAttributeProperties(raw_ostream &OS) {
+  OS << "#ifdef GET_ATTR_PROP_TABLE\n";
+  OS << "#undef GET_ATTR_PROP_TABLE\n";
+  OS << "static const uint8_t AttrPropTable[] = {\n";
+  for (StringRef KindName : {"EnumAttr", "TypeAttr", "IntAttr"}) {
+    for (auto A : Records.getAllDerivedDefinitions(KindName)) {
+      OS << "0";
+      for (Init *P : *A->getValueAsListInit("Properties"))
+        OS << " | AttributeProperty::" << cast<DefInit>(P)->getDef()->getName();
+      OS << ",\n";
+    }
+  }
+  OS << "};\n";
+  OS << "#endif\n";
+}
+
 void Attributes::emit(raw_ostream &OS) {
   emitTargetIndependentNames(OS);
   emitFnAttrCompatCheck(OS, false);
+  emitAttributeProperties(OS);
 }
 
 namespace llvm {
diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp
index 9e997483d21d..127ae6247bd9 100644
--- a/llvm/utils/TableGen/CallingConvEmitter.cpp
+++ b/llvm/utils/TableGen/CallingConvEmitter.cpp
@@ -131,10 +131,9 @@ void CallingConvEmitter::EmitAction(Record *Action,
         O << IndentStr << "static const MCPhysReg RegList" << ++Counter
           << "[] = {\n";
         O << IndentStr << "  ";
-        for (unsigned i = 0, e = RegList->size(); i != e; ++i) {
-          if (i != 0) O << ", ";
-          O << getQualifiedName(RegList->getElementAsRecord(i));
-        }
+        ListSeparator LS;
+        for (unsigned i = 0, e = RegList->size(); i != e; ++i)
+          O << LS << getQualifiedName(RegList->getElementAsRecord(i));
         O << "\n" << IndentStr << "};\n";
         O << IndentStr << "if (unsigned Reg = State.AllocateReg(RegList"
           << Counter << ")) {\n";
@@ -162,19 +161,17 @@ void CallingConvEmitter::EmitAction(Record *Action,
         O << IndentStr << "static const MCPhysReg RegList" << RegListNumber
           << "[] = {\n";
         O << IndentStr << "  ";
-        for (unsigned i = 0, e = RegList->size(); i != e; ++i) {
-          if (i != 0) O << ", ";
-          O << getQualifiedName(RegList->getElementAsRecord(i));
-        }
+        ListSeparator LS;
+        for (unsigned i = 0, e = RegList->size(); i != e; ++i)
+          O << LS << getQualifiedName(RegList->getElementAsRecord(i));
         O << "\n" << IndentStr << "};\n";
 
         O << IndentStr << "static const MCPhysReg RegList"
           << ShadowRegListNumber << "[] = {\n";
         O << IndentStr << "  ";
-        for (unsigned i = 0, e = ShadowRegList->size(); i != e; ++i) {
-          if (i != 0) O << ", ";
-          O << getQualifiedName(ShadowRegList->getElementAsRecord(i));
-        }
+        ListSeparator LSS;
+        for (unsigned i = 0, e = ShadowRegList->size(); i != e; ++i)
+          O << LSS << getQualifiedName(ShadowRegList->getElementAsRecord(i));
         O << "\n" << IndentStr << "};\n";
 
         O << IndentStr << "if (unsigned Reg = State.AllocateReg(RegList"
@@ -220,10 +217,9 @@ void CallingConvEmitter::EmitAction(Record *Action,
       O << IndentStr << "static const MCPhysReg ShadowRegList"
           << ShadowRegListNumber << "[] = {\n";
       O << IndentStr << "  ";
-      for (unsigned i = 0, e = ShadowRegList->size(); i != e; ++i) {
-        if (i != 0) O << ", ";
-        O << getQualifiedName(ShadowRegList->getElementAsRecord(i));
-      }
+      ListSeparator LS;
+      for (unsigned i = 0, e = ShadowRegList->size(); i != e; ++i)
+        O << LS << getQualifiedName(ShadowRegList->getElementAsRecord(i));
       O << "\n" << IndentStr << "};\n";
 
       O << IndentStr << "unsigned Offset" << ++Counter
diff --git a/llvm/utils/TableGen/CodeBeadsGen.cpp b/llvm/utils/TableGen/CodeBeadsGen.cpp
new file mode 100644
index 000000000000..18a6d6d19eb2
--- /dev/null
+++ b/llvm/utils/TableGen/CodeBeadsGen.cpp
@@ -0,0 +1,137 @@
+//===---------- CodeBeadsGen.cpp - Code Beads Generator -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// CodeBeads are data fields carrying auxiliary information for instructions.
+//
+// Under the hood it's simply implemented by a `bits` field (with arbitrary
+// length) in each TG instruction description, where this TG backend will
+// generate a helper function to access it.
+//
+// This is especially useful for expressing variable length encoding
+// instructions and complex addressing modes. Since in those cases each
+// instruction is usually associated with large amount of information like
+// addressing mode details used on a specific operand. Instead of retreating to
+// ad-hoc methods to figure out these information when encoding an instruction,
+// CodeBeads provide a clean table for the instruction encoder to lookup.
+//===----------------------------------------------------------------------===//
+
+#include "CodeGenTarget.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
+#include <map>
+#include <string>
+#include <vector>
+using namespace llvm;
+
+namespace {
+
+class CodeBeadsGen {
+  RecordKeeper &Records;
+
+public:
+  CodeBeadsGen(RecordKeeper &R) : Records(R) {}
+  void run(raw_ostream &OS);
+};
+
+void CodeBeadsGen::run(raw_ostream &OS) {
+  CodeGenTarget Target(Records);
+  std::vector<Record *> Insts = Records.getAllDerivedDefinitions("Instruction");
+
+  // For little-endian instruction bit encodings, reverse the bit order
+  Target.reverseBitsForLittleEndianEncoding();
+
+  ArrayRef<const CodeGenInstruction *> NumberedInstructions =
+      Target.getInstructionsByEnumValue();
+
+  // Emit function declaration
+  OS << "const uint8_t *llvm::" << Target.getInstNamespace();
+  OS << "::getMCInstrBeads(unsigned Opcode) {\n";
+
+  // First, get the maximum bit length among all beads. And do some
+  // simple validation
+  unsigned MaxBitLength = 0;
+
+  for (const CodeGenInstruction *CGI : NumberedInstructions) {
+    Record *R = CGI->TheDef;
+    if (!R->getValue("Beads"))
+      continue;
+
+    BitsInit *BI = R->getValueAsBitsInit("Beads");
+    if (!BI->isComplete()) {
+      PrintFatalError(R->getLoc(), "Record `" + R->getName() +
+                                       "', bit field 'Beads' is not complete");
+    }
+
+    MaxBitLength = std::max(MaxBitLength, BI->getNumBits());
+  }
+
+  // Number of bytes
+  unsigned Parts = MaxBitLength / 8;
+
+  // Emit instruction base values
+  OS << "  static const uint8_t InstBits[][" << Parts << "] = {\n";
+  for (const CodeGenInstruction *CGI : NumberedInstructions) {
+    Record *R = CGI->TheDef;
+
+    if (R->getValueAsString("Namespace") == "TargetOpcode" ||
+        !R->getValue("Beads")) {
+      OS << "\t{ 0x0 },\t// ";
+      if (R->getValueAsBit("isPseudo"))
+        OS << "(Pseudo) ";
+      OS << R->getName() << "\n";
+      continue;
+    }
+
+    BitsInit *BI = R->getValueAsBitsInit("Beads");
+
+    // Convert to byte array:
+    // [dcba] -> [a][b][c][d]
+    OS << "\t{";
+    for (unsigned p = 0; p < Parts; ++p) {
+      unsigned Right = 8 * p;
+      unsigned Left = Right + 8;
+
+      uint8_t Value = 0;
+      for (unsigned i = Right; i != Left; ++i) {
+        unsigned Shift = i % 8;
+        if (auto *B = dyn_cast<BitInit>(BI->getBit(i))) {
+          Value |= (static_cast<uint8_t>(B->getValue()) << Shift);
+        } else {
+          PrintFatalError(R->getLoc(), "Record `" + R->getName() +
+                                           "', bit 'Beads[" + Twine(i) +
+                                           "]' is not defined");
+        }
+      }
+
+      if (p)
+        OS << ',';
+      OS << " 0x";
+      OS.write_hex(Value);
+      OS << "";
+    }
+    OS << " }," << '\t' << "// " << R->getName() << "\n";
+  }
+  OS << "\t{ 0x0 }\n  };\n";
+
+  // Emit initial function code
+  OS << "  return InstBits[Opcode];\n"
+     << "}\n\n";
+}
+
+} // End anonymous namespace
+
+namespace llvm {
+
+void EmitCodeBeads(RecordKeeper &RK, raw_ostream &OS) {
+  emitSourceFileHeader("Machine Code Beads", OS);
+  CodeBeadsGen(RK).run(OS);
+}
+
+} // namespace llvm
diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp
index 53bf953b13cf..ffc2878d3508 100644
--- a/llvm/utils/TableGen/CodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/CodeEmitterGen.cpp
@@ -272,7 +272,7 @@ std::string CodeEmitterGen::getInstructionCase(Record *R,
       EncodingInfoByHwMode EBM(DI->getDef(), HWM);
       Case += "      switch (HwMode) {\n";
       Case += "      default: llvm_unreachable(\"Unhandled HwMode\");\n";
-      for (auto &KV : EBM.Map) {
+      for (auto &KV : EBM) {
         Case += "      case " + itostr(KV.first) + ": {\n";
         Case += getInstructionCaseForEncoding(R, KV.second, Target);
         Case += "      break;\n";
@@ -409,7 +409,7 @@ void CodeEmitterGen::run(raw_ostream &o) {
     if (const RecordVal *RV = R->getValue("EncodingInfos")) {
       if (DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
         EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-        for (auto &KV : EBM.Map) {
+        for (auto &KV : EBM) {
           BitsInit *BI = KV.second->getValueAsBitsInit("Inst");
           BitWidth = std::max(BitWidth, BI->getNumBits());
           HwModes.insert(KV.first);
@@ -461,9 +461,7 @@ void CodeEmitterGen::run(raw_ostream &o) {
   std::map<std::string, std::vector<std::string>> CaseMap;
 
   // Construct all cases statement for each opcode
-  for (std::vector<Record*>::iterator IC = Insts.begin(), EC = Insts.end();
-        IC != EC; ++IC) {
-    Record *R = *IC;
+  for (Record *R : Insts) {
     if (R->getValueAsString("Namespace") == "TargetOpcode" ||
         R->getValueAsBit("isPseudo"))
       continue;
@@ -483,8 +481,8 @@ void CodeEmitterGen::run(raw_ostream &o) {
       << "    Inst = Inst.zext(" << BitWidth << ");\n"
       << "  if (Scratch.getBitWidth() != " << BitWidth << ")\n"
       << "    Scratch = Scratch.zext(" << BitWidth << ");\n"
-      << "  LoadIntFromMemory(Inst, (uint8_t *)&InstBits[opcode * " << NumWords
-      << "], " << NumBytes << ");\n"
+      << "  LoadIntFromMemory(Inst, (const uint8_t *)&InstBits[opcode * "
+      << NumWords << "], " << NumBytes << ");\n"
       << "  APInt &Value = Inst;\n"
       << "  APInt &op = Scratch;\n"
       << "  switch (opcode) {\n";
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
index 1ca4a68eb155..c1a3a34d928b 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenDAGPatterns.h"
-#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
@@ -111,10 +110,8 @@ bool TypeSetByHwMode::insert(const ValueTypeByHwMode &VVT) {
   bool ContainsDefault = false;
   MVT DT = MVT::Other;
 
-  SmallDenseSet<unsigned, 4> Modes;
   for (const auto &P : VVT) {
     unsigned M = P.first;
-    Modes.insert(M);
     // Make sure there exists a set for each specific mode from VVT.
     Changed |= getOrCreate(M).insert(P.second).second;
     // Cache VVT's default mode.
@@ -128,7 +125,7 @@ bool TypeSetByHwMode::insert(const ValueTypeByHwMode &VVT) {
   // modes in "this" that do not exist in VVT.
   if (ContainsDefault)
     for (auto &I : *this)
-      if (!Modes.count(I.first))
+      if (!VVT.hasMode(I.first))
         Changed |= I.second.insert(DT).second;
 
   return Changed;
@@ -205,11 +202,9 @@ void TypeSetByHwMode::writeToStream(const SetType &S, raw_ostream &OS) {
   array_pod_sort(Types.begin(), Types.end());
 
   OS << '[';
-  for (unsigned i = 0, e = Types.size(); i != e; ++i) {
-    OS << ValueTypeByHwMode::getMVTName(Types[i]);
-    if (i != e-1)
-      OS << ' ';
-  }
+  ListSeparator LS(" ");
+  for (const MVT &T : Types)
+    OS << LS << ValueTypeByHwMode::getMVTName(T);
   OS << ']';
 }
 
@@ -226,7 +221,7 @@ bool TypeSetByHwMode::operator==(const TypeSetByHwMode &VTS) const {
   if (HaveDefault != VTSHaveDefault)
     return false;
 
-  SmallDenseSet<unsigned, 4> Modes;
+  SmallSet<unsigned, 4> Modes;
   for (auto &I : *this)
     Modes.insert(I.first);
   for (const auto &I : VTS)
@@ -470,7 +465,8 @@ bool TypeInfer::EnforceSmallerThan(TypeSetByHwMode &Small,
 
   assert(Small.hasDefault() && Big.hasDefault());
 
-  std::vector<unsigned> Modes = union_modes(Small, Big);
+  SmallVector<unsigned, 4> Modes;
+  union_modes(Small, Big, Modes);
 
   // 1. Only allow integer or floating point types and make sure that
   //    both sides are both integer or both floating point.
@@ -577,7 +573,9 @@ bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec,
   if (Elem.empty())
     Changed |= EnforceScalar(Elem);
 
-  for (unsigned M : union_modes(Vec, Elem)) {
+  SmallVector<unsigned, 4> Modes;
+  union_modes(Vec, Elem, Modes);
+  for (unsigned M : Modes) {
     TypeSetByHwMode::SetType &V = Vec.get(M);
     TypeSetByHwMode::SetType &E = Elem.get(M);
 
@@ -585,7 +583,7 @@ bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec,
     Changed |= berase_if(E, isVector);  // Vector = !scalar
     assert(!V.empty() && !E.empty());
 
-    SmallSet<MVT,4> VT, ST;
+    MachineValueTypeSet VT, ST;
     // Collect element types from the "vector" set.
     for (MVT T : V)
       VT.insert(T.getVectorElementType());
@@ -632,7 +630,7 @@ bool TypeInfer::EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec,
       return false;
     if (B.getVectorElementType() != P.getVectorElementType())
       return false;
-    return B.getVectorNumElements() < P.getVectorNumElements();
+    return B.getVectorMinNumElements() < P.getVectorMinNumElements();
   };
 
   /// Return true if S has no element (vector type) that T is a sub-vector of,
@@ -660,7 +658,9 @@ bool TypeInfer::EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec,
   if (Sub.empty())
     Changed |= EnforceVector(Sub);
 
-  for (unsigned M : union_modes(Vec, Sub)) {
+  SmallVector<unsigned, 4> Modes;
+  union_modes(Vec, Sub, Modes);
+  for (unsigned M : Modes) {
     TypeSetByHwMode::SetType &S = Sub.get(M);
     TypeSetByHwMode::SetType &V = Vec.get(M);
 
@@ -696,19 +696,25 @@ bool TypeInfer::EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W) {
   // An actual vector type cannot have 0 elements, so we can treat scalars
   // as zero-length vectors. This way both vectors and scalars can be
   // processed identically.
-  auto NoLength = [](const SmallSet<unsigned,2> &Lengths, MVT T) -> bool {
-    return !Lengths.count(T.isVector() ? T.getVectorNumElements() : 0);
+  auto NoLength = [](const SmallDenseSet<ElementCount> &Lengths,
+                     MVT T) -> bool {
+    return !Lengths.count(T.isVector() ? T.getVectorElementCount()
+                                       : ElementCount::getNull());
   };
 
-  for (unsigned M : union_modes(V, W)) {
+  SmallVector<unsigned, 4> Modes;
+  union_modes(V, W, Modes);
+  for (unsigned M : Modes) {
     TypeSetByHwMode::SetType &VS = V.get(M);
     TypeSetByHwMode::SetType &WS = W.get(M);
 
-    SmallSet<unsigned,2> VN, WN;
+    SmallDenseSet<ElementCount> VN, WN;
     for (MVT T : VS)
-      VN.insert(T.isVector() ? T.getVectorNumElements() : 0);
+      VN.insert(T.isVector() ? T.getVectorElementCount()
+                             : ElementCount::getNull());
     for (MVT T : WS)
-      WN.insert(T.isVector() ? T.getVectorNumElements() : 0);
+      WN.insert(T.isVector() ? T.getVectorElementCount()
+                             : ElementCount::getNull());
 
     Changed |= berase_if(VS, std::bind(NoLength, WN, std::placeholders::_1));
     Changed |= berase_if(WS, std::bind(NoLength, VN, std::placeholders::_1));
@@ -716,6 +722,15 @@ bool TypeInfer::EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W) {
   return Changed;
 }
 
+namespace {
+struct TypeSizeComparator {
+  bool operator()(const TypeSize &LHS, const TypeSize &RHS) const {
+    return std::make_tuple(LHS.isScalable(), LHS.getKnownMinValue()) <
+           std::make_tuple(RHS.isScalable(), RHS.getKnownMinValue());
+  }
+};
+} // end anonymous namespace
+
 /// 1. Ensure that for each type T in A, there exists a type U in B,
 ///    such that T and U have equal size in bits.
 /// 2. Ensure that for each type U in B, there exists a type T in A
@@ -730,14 +745,18 @@ bool TypeInfer::EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B) {
   if (B.empty())
     Changed |= EnforceAny(B);
 
-  auto NoSize = [](const SmallSet<TypeSize, 2> &Sizes, MVT T) -> bool {
+  typedef SmallSet<TypeSize, 2, TypeSizeComparator> TypeSizeSet;
+
+  auto NoSize = [](const TypeSizeSet &Sizes, MVT T) -> bool {
     return !Sizes.count(T.getSizeInBits());
   };
 
-  for (unsigned M : union_modes(A, B)) {
+  SmallVector<unsigned, 4> Modes;
+  union_modes(A, B, Modes);
+  for (unsigned M : Modes) {
     TypeSetByHwMode::SetType &AS = A.get(M);
     TypeSetByHwMode::SetType &BS = B.get(M);
-    SmallSet<TypeSize, 2> AN, BN;
+    TypeSizeSet AN, BN;
 
     for (MVT T : AS)
       AN.insert(T.getSizeInBits());
@@ -837,7 +856,11 @@ TypeInfer::ValidateOnExit::~ValidateOnExit() {
               "(use -print-records with llvm-tblgen to see all "
               "expanded records).\n";
     Infer.TP.dump();
-    llvm_unreachable(nullptr);
+    dbgs() << "Generated from record:\n";
+    Infer.TP.getRecord()->dump();
+    PrintFatalError(Infer.TP.getRecord()->getLoc(),
+                    "Type set is empty for each HW mode in '" +
+                        Infer.TP.getRecord()->getName() + "'");
   }
 }
 #endif
@@ -980,12 +1003,9 @@ std::string TreePredicateFn::getPredCode() const {
       Code += "unsigned AddrSpace = cast<MemSDNode>(N)->getAddressSpace();\n"
         " if (";
 
-      bool First = true;
+      ListSeparator LS(" && ");
       for (Init *Val : AddressSpaces->getValues()) {
-        if (First)
-          First = false;
-        else
-          Code += " && ";
+        Code += LS;
 
         IntInit *IntVal = dyn_cast<IntInit>(Val);
         if (!IntVal) {
@@ -1015,33 +1035,33 @@ std::string TreePredicateFn::getPredCode() const {
   }
 
   if (isAtomic() && isAtomicOrderingMonotonic())
-    Code += "if (cast<AtomicSDNode>(N)->getOrdering() != "
+    Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != "
             "AtomicOrdering::Monotonic) return false;\n";
   if (isAtomic() && isAtomicOrderingAcquire())
-    Code += "if (cast<AtomicSDNode>(N)->getOrdering() != "
+    Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != "
             "AtomicOrdering::Acquire) return false;\n";
   if (isAtomic() && isAtomicOrderingRelease())
-    Code += "if (cast<AtomicSDNode>(N)->getOrdering() != "
+    Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != "
             "AtomicOrdering::Release) return false;\n";
   if (isAtomic() && isAtomicOrderingAcquireRelease())
-    Code += "if (cast<AtomicSDNode>(N)->getOrdering() != "
+    Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != "
             "AtomicOrdering::AcquireRelease) return false;\n";
   if (isAtomic() && isAtomicOrderingSequentiallyConsistent())
-    Code += "if (cast<AtomicSDNode>(N)->getOrdering() != "
+    Code += "if (cast<AtomicSDNode>(N)->getMergedOrdering() != "
             "AtomicOrdering::SequentiallyConsistent) return false;\n";
 
   if (isAtomic() && isAtomicOrderingAcquireOrStronger())
-    Code += "if (!isAcquireOrStronger(cast<AtomicSDNode>(N)->getOrdering())) "
+    Code += "if (!isAcquireOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) "
             "return false;\n";
   if (isAtomic() && isAtomicOrderingWeakerThanAcquire())
-    Code += "if (isAcquireOrStronger(cast<AtomicSDNode>(N)->getOrdering())) "
+    Code += "if (isAcquireOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) "
             "return false;\n";
 
   if (isAtomic() && isAtomicOrderingReleaseOrStronger())
-    Code += "if (!isReleaseOrStronger(cast<AtomicSDNode>(N)->getOrdering())) "
+    Code += "if (!isReleaseOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) "
             "return false;\n";
   if (isAtomic() && isAtomicOrderingWeakerThanRelease())
-    Code += "if (isReleaseOrStronger(cast<AtomicSDNode>(N)->getOrdering())) "
+    Code += "if (isReleaseOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) "
             "return false;\n";
 
   if (isLoad() || isStore()) {
@@ -1242,7 +1262,7 @@ StringRef TreePredicateFn::getImmType() const {
 StringRef TreePredicateFn::getImmTypeIdentifier() const {
   if (immCodeUsesAPInt())
     return "APInt";
-  else if (immCodeUsesAPFloat())
+  if (immCodeUsesAPFloat())
     return "APFloat";
   return "I64";
 }
@@ -1423,24 +1443,50 @@ getPatternComplexity(const CodeGenDAGPatterns &CGP) const {
   return getPatternSize(getSrcPattern(), CGP) + getAddedComplexity();
 }
 
+void PatternToMatch::getPredicateRecords(
+    SmallVectorImpl<Record *> &PredicateRecs) const {
+  for (Init *I : Predicates->getValues()) {
+    if (DefInit *Pred = dyn_cast<DefInit>(I)) {
+      Record *Def = Pred->getDef();
+      if (!Def->isSubClassOf("Predicate")) {
+#ifndef NDEBUG
+        Def->dump();
+#endif
+        llvm_unreachable("Unknown predicate type!");
+      }
+      PredicateRecs.push_back(Def);
+    }
+  }
+  // Sort so that different orders get canonicalized to the same string.
+  llvm::sort(PredicateRecs, LessRecord());
+}
+
 /// getPredicateCheck - Return a single string containing all of this
 /// pattern's predicates concatenated with "&&" operators.
 ///
 std::string PatternToMatch::getPredicateCheck() const {
-  SmallVector<const Predicate*,4> PredList;
-  for (const Predicate &P : Predicates) {
-    if (!P.getCondString().empty())
-      PredList.push_back(&P);
+  SmallVector<Record *, 4> PredicateRecs;
+  getPredicateRecords(PredicateRecs);
+
+  SmallString<128> PredicateCheck;
+  for (Record *Pred : PredicateRecs) {
+    StringRef CondString = Pred->getValueAsString("CondString");
+    if (CondString.empty())
+      continue;
+    if (!PredicateCheck.empty())
+      PredicateCheck += " && ";
+    PredicateCheck += "(";
+    PredicateCheck += CondString;
+    PredicateCheck += ")";
   }
-  llvm::sort(PredList, deref<std::less<>>());
 
-  std::string Check;
-  for (unsigned i = 0, e = PredList.size(); i != e; ++i) {
-    if (i != 0)
-      Check += " && ";
-    Check += '(' + PredList[i]->getCondString() + ')';
+  if (!HwModeFeatures.empty()) {
+    if (!PredicateCheck.empty())
+      PredicateCheck += " && ";
+    PredicateCheck += HwModeFeatures;
   }
-  return Check;
+
+  return std::string(PredicateCheck);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1791,7 +1837,7 @@ static unsigned GetNumNodeResults(Record *Operator, CodeGenDAGPatterns &CDP) {
       // The number of results of a fragment with alternative records is the
       // maximum number of results across all alternatives.
       unsigned NumResults = 0;
-      for (auto T : PFRec->getTrees())
+      for (const auto &T : PFRec->getTrees())
         NumResults = std::max(NumResults, T->getNumTypes());
       return NumResults;
     }
@@ -1857,9 +1903,9 @@ void TreePatternNode::print(raw_ostream &OS) const {
   if (!isLeaf()) {
     if (getNumChildren() != 0) {
       OS << " ";
-      getChild(0)->print(OS);
-      for (unsigned i = 1, e = getNumChildren(); i != e; ++i) {
-        OS << ", ";
+      ListSeparator LS;
+      for (unsigned i = 0, e = getNumChildren(); i != e; ++i) {
+        OS << LS;
         getChild(i)->print(OS);
       }
     }
@@ -2013,10 +2059,13 @@ void TreePatternNode::InlinePatternFragments(
       if (ChildAlternatives[i].empty())
         return;
 
-      for (auto NewChild : ChildAlternatives[i])
-        assert((Child->getPredicateCalls().empty() ||
-                NewChild->getPredicateCalls() == Child->getPredicateCalls()) &&
-               "Non-empty child predicate clobbered!");
+      assert((Child->getPredicateCalls().empty() ||
+              llvm::all_of(ChildAlternatives[i],
+                           [&](const TreePatternNodePtr &NewChild) {
+                             return NewChild->getPredicateCalls() ==
+                                    Child->getPredicateCalls();
+                           })) &&
+             "Non-empty child predicate clobbered!");
     }
 
     // The end result is an all-pairs construction of the resultant pattern.
@@ -2088,7 +2137,7 @@ void TreePatternNode::InlinePatternFragments(
   }
 
   // Loop over all fragment alternatives.
-  for (auto Alternative : Frag->getTrees()) {
+  for (const auto &Alternative : Frag->getTrees()) {
     TreePatternNodePtr FragTree = Alternative->clone();
 
     if (!PredFn.isAlwaysTrue())
@@ -2637,6 +2686,8 @@ static bool OnlyOnRHSOfCommutative(TreePatternNode *N) {
     return true;
   if (N->isLeaf() && isa<IntInit>(N->getLeafValue()))
     return true;
+  if (isImmAllOnesAllZerosMatch(N))
+    return true;
   return false;
 }
 
@@ -2802,7 +2853,8 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit,
         ParseTreePattern(Dag->getArg(0), Dag->getArgNameStr(0));
 
     // Apply the type cast.
-    assert(New->getNumTypes() == 1 && "FIXME: Unhandled");
+    if (New->getNumTypes() != 1)
+      error("Type cast can only have one type!");
     const CodeGenHwModes &CGH = getDAGPatterns().getTargetInfo().getHwModes();
     New->UpdateNodeType(0, getValueTypeByHwMode(Operator, CGH), *this);
 
@@ -3030,9 +3082,10 @@ InferAllTypes(const StringMap<SmallVector<TreePatternNode*,1> > *InNamedTypes) {
 void TreePattern::print(raw_ostream &OS) const {
   OS << getRecord()->getName();
   if (!Args.empty()) {
-    OS << "(" << Args[0];
-    for (unsigned i = 1, e = Args.size(); i != e; ++i)
-      OS << ", " << Args[i];
+    OS << "(";
+    ListSeparator LS;
+    for (const std::string &Arg : Args)
+      OS << LS << Arg;
     OS << ")";
   }
   OS << ": ";
@@ -3070,15 +3123,15 @@ CodeGenDAGPatterns::CodeGenDAGPatterns(RecordKeeper &R,
   ParsePatternFragments(/*OutFrags*/true);
   ParsePatterns();
 
+  // Generate variants.  For example, commutative patterns can match
+  // multiple ways.  Add them to PatternsToMatch as well.
+  GenerateVariants();
+
   // Break patterns with parameterized types into a series of patterns,
   // where each one has a fixed type and is predicated on the conditions
   // of the associated HW mode.
   ExpandHwModeBasedTypes();
 
-  // Generate variants.  For example, commutative patterns can match
-  // multiple ways.  Add them to PatternsToMatch as well.
-  GenerateVariants();
-
   // Infer instruction flags.  For example, we can detect loads,
   // stores, and side effects in many cases by examining an
   // instruction's pattern.
@@ -3201,7 +3254,7 @@ void CodeGenDAGPatterns::ParsePatternFragments(bool OutFrags) {
     // it.
     Record *Transform = Frag->getValueAsDef("OperandTransform");
     if (!getSDNodeTransform(Transform).second.empty())    // not noop xform?
-      for (auto T : P->getTrees())
+      for (const auto &T : P->getTrees())
         T->setTransformFn(Transform);
   }
 
@@ -3470,6 +3523,9 @@ private:
     if (N->getNumChildren() != 1 || !N->getChild(0)->isLeaf())
       return false;
 
+    if (N->getOperator()->isSubClassOf("ComplexPattern"))
+      return false;
+
     const SDNodeInfo &OpInfo = CDP.getSDNodeInfo(N->getOperator());
     if (OpInfo.getNumResults() != 1 || OpInfo.getNumOperands() != 1)
       return false;
@@ -3671,9 +3727,9 @@ void CodeGenDAGPatterns::parseInstructionPattern(
     TreePatternNodePtr Pat = I.getTree(j);
     if (Pat->getNumTypes() != 0) {
       raw_svector_ostream OS(TypesString);
+      ListSeparator LS;
       for (unsigned k = 0, ke = Pat->getNumTypes(); k != ke; ++k) {
-        if (k > 0)
-          OS << ", ";
+        OS << LS;
         Pat->getExtType(k).writeToStream(OS);
       }
       I.error("Top-level forms in instruction pattern should have"
@@ -3904,20 +3960,6 @@ static void FindNames(TreePatternNode *P,
   }
 }
 
-std::vector<Predicate> CodeGenDAGPatterns::makePredList(ListInit *L) {
-  std::vector<Predicate> Preds;
-  for (Init *I : L->getValues()) {
-    if (DefInit *Pred = dyn_cast<DefInit>(I))
-      Preds.push_back(Pred->getDef());
-    else
-      llvm_unreachable("Non-def on the list");
-  }
-
-  // Sort so that different orders get canonicalized to the same string.
-  llvm::sort(Preds);
-  return Preds;
-}
-
 void CodeGenDAGPatterns::AddPatternToMatch(TreePattern *Pattern,
                                            PatternToMatch &&PTM) {
   // Do some sanity checking on the pattern we're about to match.
@@ -3958,7 +4000,7 @@ void CodeGenDAGPatterns::AddPatternToMatch(TreePattern *Pattern,
         SrcNames[Entry.first].second == 1)
       Pattern->error("Pattern has dead named input: $" + Entry.first);
 
-  PatternsToMatch.push_back(PTM);
+  PatternsToMatch.push_back(std::move(PTM));
 }
 
 void CodeGenDAGPatterns::InferInstructionFlags() {
@@ -4030,8 +4072,7 @@ void CodeGenDAGPatterns::InferInstructionFlags() {
 /// Verify instruction flags against pattern node properties.
 void CodeGenDAGPatterns::VerifyInstructionFlags() {
   unsigned Errors = 0;
-  for (ptm_iterator I = ptm_begin(), E = ptm_end(); I != E; ++I) {
-    const PatternToMatch &PTM = *I;
+  for (const PatternToMatch &PTM : ptms()) {
     SmallVector<Record*, 8> Instrs;
     getInstructionsInTree(PTM.getDstPattern(), Instrs);
     if (Instrs.empty())
@@ -4172,7 +4213,7 @@ void CodeGenDAGPatterns::ParseOnePattern(Record *TheDef,
     // resolve cases where the input type is known to be a pointer type (which
     // is considered resolved), but the result knows it needs to be 32- or
     // 64-bits.  Infer the other way for good measure.
-    for (auto T : Pattern.getTrees())
+    for (const auto &T : Pattern.getTrees())
       for (unsigned i = 0, e = std::min(Result.getOnlyTree()->getNumTypes(),
                                         T->getNumTypes());
          i != e; ++i) {
@@ -4226,11 +4267,10 @@ void CodeGenDAGPatterns::ParseOnePattern(Record *TheDef,
   // will lead to a contradiction, which is not an error however, but
   // a sign that this pattern will simply never match.
   if (Temp.getOnlyTree()->hasPossibleType())
-    for (auto T : Pattern.getTrees())
+    for (const auto &T : Pattern.getTrees())
       if (T->hasPossibleType())
         AddPatternToMatch(&Pattern,
-                          PatternToMatch(TheDef, makePredList(Preds),
-                                         T, Temp.getOnlyTree(),
+                          PatternToMatch(TheDef, Preds, T, Temp.getOnlyTree(),
                                          InstImpResults, Complexity,
                                          TheDef->getID()));
 }
@@ -4281,32 +4321,29 @@ static void collectModes(std::set<unsigned> &Modes, const TreePatternNode *N) {
 
 void CodeGenDAGPatterns::ExpandHwModeBasedTypes() {
   const CodeGenHwModes &CGH = getTargetInfo().getHwModes();
-  std::map<unsigned,std::vector<Predicate>> ModeChecks;
-  std::vector<PatternToMatch> Copy = PatternsToMatch;
-  PatternsToMatch.clear();
+  std::vector<PatternToMatch> Copy;
+  PatternsToMatch.swap(Copy);
 
-  auto AppendPattern = [this, &ModeChecks](PatternToMatch &P, unsigned Mode) {
-    TreePatternNodePtr NewSrc = P.SrcPattern->clone();
-    TreePatternNodePtr NewDst = P.DstPattern->clone();
+  auto AppendPattern = [this](PatternToMatch &P, unsigned Mode,
+                              StringRef Check) {
+    TreePatternNodePtr NewSrc = P.getSrcPattern()->clone();
+    TreePatternNodePtr NewDst = P.getDstPattern()->clone();
     if (!NewSrc->setDefaultMode(Mode) || !NewDst->setDefaultMode(Mode)) {
       return;
     }
 
-    std::vector<Predicate> Preds = P.Predicates;
-    const std::vector<Predicate> &MC = ModeChecks[Mode];
-    llvm::append_range(Preds, MC);
-    PatternsToMatch.emplace_back(P.getSrcRecord(), Preds, std::move(NewSrc),
-                                 std::move(NewDst), P.getDstRegs(),
-                                 P.getAddedComplexity(), Record::getNewUID(),
-                                 Mode);
+    PatternsToMatch.emplace_back(P.getSrcRecord(), P.getPredicates(),
+                                 std::move(NewSrc), std::move(NewDst),
+                                 P.getDstRegs(), P.getAddedComplexity(),
+                                 Record::getNewUID(), Mode, Check);
   };
 
   for (PatternToMatch &P : Copy) {
     TreePatternNodePtr SrcP = nullptr, DstP = nullptr;
-    if (P.SrcPattern->hasProperTypeByHwMode())
-      SrcP = P.SrcPattern;
-    if (P.DstPattern->hasProperTypeByHwMode())
-      DstP = P.DstPattern;
+    if (P.getSrcPattern()->hasProperTypeByHwMode())
+      SrcP = P.getSrcPatternShared();
+    if (P.getDstPattern()->hasProperTypeByHwMode())
+      DstP = P.getDstPatternShared();
     if (!SrcP && !DstP) {
       PatternsToMatch.push_back(P);
       continue;
@@ -4329,31 +4366,27 @@ void CodeGenDAGPatterns::ExpandHwModeBasedTypes() {
     // duplicated patterns with different predicate checks, construct the
     // default check as a negation of all predicates that are actually present
     // in the source/destination patterns.
-    std::vector<Predicate> DefaultPred;
+    SmallString<128> DefaultCheck;
 
     for (unsigned M : Modes) {
       if (M == DefaultMode)
         continue;
-      if (ModeChecks.find(M) != ModeChecks.end())
-        continue;
 
       // Fill the map entry for this mode.
       const HwMode &HM = CGH.getMode(M);
-      ModeChecks[M].emplace_back(Predicate(HM.Features, true));
+      AppendPattern(P, M, "(MF->getSubtarget().checkFeatures(\"" + HM.Features + "\"))");
 
       // Add negations of the HM's predicates to the default predicate.
-      DefaultPred.emplace_back(Predicate(HM.Features, false));
-    }
-
-    for (unsigned M : Modes) {
-      if (M == DefaultMode)
-        continue;
-      AppendPattern(P, M);
+      if (!DefaultCheck.empty())
+        DefaultCheck += " && ";
+      DefaultCheck += "(!(MF->getSubtarget().checkFeatures(\"";
+      DefaultCheck += HM.Features;
+      DefaultCheck += "\")))";
     }
 
     bool HasDefault = Modes.count(DefaultMode);
     if (HasDefault)
-      AppendPattern(P, DefaultMode);
+      AppendPattern(P, DefaultMode, DefaultCheck);
   }
 }
 
@@ -4635,17 +4668,7 @@ void CodeGenDAGPatterns::GenerateVariants() {
   // intentionally do not reconsider these.  Any variants of added patterns have
   // already been added.
   //
-  const unsigned NumOriginalPatterns = PatternsToMatch.size();
-  BitVector MatchedPatterns(NumOriginalPatterns);
-  std::vector<BitVector> MatchedPredicates(NumOriginalPatterns,
-                                           BitVector(NumOriginalPatterns));
-
-  typedef std::pair<MultipleUseVarSet, std::vector<TreePatternNodePtr>>
-      DepsAndVariants;
-  std::map<unsigned, DepsAndVariants> PatternsWithVariants;
-
-  // Collect patterns with more than one variant.
-  for (unsigned i = 0; i != NumOriginalPatterns; ++i) {
+  for (unsigned i = 0, e = PatternsToMatch.size(); i != e; ++i) {
     MultipleUseVarSet DepVars;
     std::vector<TreePatternNodePtr> Variants;
     FindDepVars(PatternsToMatch[i].getSrcPattern(), DepVars);
@@ -4655,6 +4678,9 @@ void CodeGenDAGPatterns::GenerateVariants() {
     GenerateVariantsOf(PatternsToMatch[i].getSrcPatternShared(), Variants,
                        *this, DepVars);
 
+    assert(PatternsToMatch[i].getHwModeFeatures().empty() &&
+           "HwModes should not have been expanded yet!");
+
     assert(!Variants.empty() && "Must create at least original variant!");
     if (Variants.size() == 1) // No additional variants for this pattern.
       continue;
@@ -4662,40 +4688,8 @@ void CodeGenDAGPatterns::GenerateVariants() {
     LLVM_DEBUG(errs() << "FOUND VARIANTS OF: ";
                PatternsToMatch[i].getSrcPattern()->dump(); errs() << "\n");
 
-    PatternsWithVariants[i] = std::make_pair(DepVars, Variants);
-
-    // Cache matching predicates.
-    if (MatchedPatterns[i])
-      continue;
-
-    const std::vector<Predicate> &Predicates =
-        PatternsToMatch[i].getPredicates();
-
-    BitVector &Matches = MatchedPredicates[i];
-    MatchedPatterns.set(i);
-    Matches.set(i);
-
-    // Don't test patterns that have already been cached - it won't match.
-    for (unsigned p = 0; p != NumOriginalPatterns; ++p)
-      if (!MatchedPatterns[p])
-        Matches[p] = (Predicates == PatternsToMatch[p].getPredicates());
-
-    // Copy this to all the matching patterns.
-    for (int p = Matches.find_first(); p != -1; p = Matches.find_next(p))
-      if (p != (int)i) {
-        MatchedPatterns.set(p);
-        MatchedPredicates[p] = Matches;
-      }
-  }
-
-  for (auto it : PatternsWithVariants) {
-    unsigned i = it.first;
-    const MultipleUseVarSet &DepVars = it.second.first;
-    const std::vector<TreePatternNodePtr> &Variants = it.second.second;
-
     for (unsigned v = 0, e = Variants.size(); v != e; ++v) {
       TreePatternNodePtr Variant = Variants[v];
-      BitVector &Matches = MatchedPredicates[i];
 
       LLVM_DEBUG(errs() << "  VAR#" << v << ": "; Variant->dump();
                  errs() << "\n");
@@ -4704,7 +4698,8 @@ void CodeGenDAGPatterns::GenerateVariants() {
       bool AlreadyExists = false;
       for (unsigned p = 0, e = PatternsToMatch.size(); p != e; ++p) {
         // Skip if the top level predicates do not match.
-        if (!Matches[p])
+        if ((i != p) && (PatternsToMatch[i].getPredicates() !=
+                         PatternsToMatch[p].getPredicates()))
           continue;
         // Check to see if this variant already exists.
         if (Variant->isIsomorphicTo(PatternsToMatch[p].getSrcPattern(),
@@ -4718,16 +4713,13 @@ void CodeGenDAGPatterns::GenerateVariants() {
       if (AlreadyExists) continue;
 
       // Otherwise, add it to the list of patterns we have.
-      PatternsToMatch.push_back(PatternToMatch(
+      PatternsToMatch.emplace_back(
           PatternsToMatch[i].getSrcRecord(), PatternsToMatch[i].getPredicates(),
           Variant, PatternsToMatch[i].getDstPatternShared(),
           PatternsToMatch[i].getDstRegs(),
-          PatternsToMatch[i].getAddedComplexity(), Record::getNewUID()));
-      MatchedPredicates.push_back(Matches);
-
-      // Add a new match the same as this pattern.
-      for (auto &P : MatchedPredicates)
-        P.push_back(P[i]);
+          PatternsToMatch[i].getAddedComplexity(), Record::getNewUID(),
+          PatternsToMatch[i].getForceMode(),
+          PatternsToMatch[i].getHwModeFeatures());
     }
 
     LLVM_DEBUG(errs() << "\n");
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/CodeGenDAGPatterns.h
index bc939fe9acc1..a69f1e2e3030 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.h
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.h
@@ -200,9 +200,7 @@ struct TypeSetByHwMode : public InfoByHwMode<MachineValueTypeSet> {
   TypeSetByHwMode(ArrayRef<ValueTypeByHwMode> VTList);
 
   SetType &getOrCreate(unsigned Mode) {
-    if (hasMode(Mode))
-      return get(Mode);
-    return Map.insert({Mode,SetType()}).first->second;
+    return Map[Mode];
   }
 
   bool isValueTypeByHwMode(bool AllowEmpty) const;
@@ -1049,89 +1047,43 @@ public:
   TreePatternNodePtr getResultPattern() const { return ResultPattern; }
 };
 
-/// This class represents a condition that has to be satisfied for a pattern
-/// to be tried. It is a generalization of a class "Pattern" from Target.td:
-/// in addition to the Target.td's predicates, this class can also represent
-/// conditions associated with HW modes. Both types will eventually become
-/// strings containing C++ code to be executed, the difference is in how
-/// these strings are generated.
-class Predicate {
-public:
-  Predicate(Record *R, bool C = true) : Def(R), IfCond(C), IsHwMode(false) {
-    assert(R->isSubClassOf("Predicate") &&
-           "Predicate objects should only be created for records derived"
-           "from Predicate class");
-  }
-  Predicate(StringRef FS, bool C = true) : Def(nullptr), Features(FS.str()),
-    IfCond(C), IsHwMode(true) {}
-
-  /// Return a string which contains the C++ condition code that will serve
-  /// as a predicate during instruction selection.
-  std::string getCondString() const {
-    // The string will excute in a subclass of SelectionDAGISel.
-    // Cast to std::string explicitly to avoid ambiguity with StringRef.
-    std::string C = IsHwMode
-                        ? std::string("MF->getSubtarget().checkFeatures(\"" +
-                                      Features + "\")")
-                        : std::string(Def->getValueAsString("CondString"));
-    if (C.empty())
-      return "";
-    return IfCond ? C : "!("+C+')';
-  }
-
-  bool operator==(const Predicate &P) const {
-    return IfCond == P.IfCond && IsHwMode == P.IsHwMode && Def == P.Def;
-  }
-  bool operator<(const Predicate &P) const {
-    if (IsHwMode != P.IsHwMode)
-      return IsHwMode < P.IsHwMode;
-    assert(!Def == !P.Def && "Inconsistency between Def and IsHwMode");
-    if (IfCond != P.IfCond)
-      return IfCond < P.IfCond;
-    if (Def)
-      return LessRecord()(Def, P.Def);
-    return Features < P.Features;
-  }
-  Record *Def;            ///< Predicate definition from .td file, null for
-                          ///< HW modes.
-  std::string Features;   ///< Feature string for HW mode.
-  bool IfCond;            ///< The boolean value that the condition has to
-                          ///< evaluate to for this predicate to be true.
-  bool IsHwMode;          ///< Does this predicate correspond to a HW mode?
-};
-
 /// PatternToMatch - Used by CodeGenDAGPatterns to keep tab of patterns
 /// processed to produce isel.
 class PatternToMatch {
-public:
-  PatternToMatch(Record *srcrecord, std::vector<Predicate> preds,
-                 TreePatternNodePtr src, TreePatternNodePtr dst,
-                 std::vector<Record *> dstregs, int complexity,
-                 unsigned uid, unsigned setmode = 0)
-      : SrcRecord(srcrecord), SrcPattern(src), DstPattern(dst),
-        Predicates(std::move(preds)), Dstregs(std::move(dstregs)),
-        AddedComplexity(complexity), ID(uid), ForceMode(setmode) {}
-
   Record          *SrcRecord;   // Originating Record for the pattern.
+  ListInit        *Predicates;  // Top level predicate conditions to match.
   TreePatternNodePtr SrcPattern;      // Source pattern to match.
   TreePatternNodePtr DstPattern;      // Resulting pattern.
-  std::vector<Predicate> Predicates;  // Top level predicate conditions
-                                      // to match.
   std::vector<Record*> Dstregs; // Physical register defs being matched.
+  std::string      HwModeFeatures;
   int              AddedComplexity; // Add to matching pattern complexity.
   unsigned         ID;          // Unique ID for the record.
   unsigned         ForceMode;   // Force this mode in type inference when set.
 
+public:
+  PatternToMatch(Record *srcrecord, ListInit *preds, TreePatternNodePtr src,
+                 TreePatternNodePtr dst, std::vector<Record *> dstregs,
+                 int complexity, unsigned uid, unsigned setmode = 0,
+                 const Twine &hwmodefeatures = "")
+      : SrcRecord(srcrecord), Predicates(preds), SrcPattern(src),
+        DstPattern(dst), Dstregs(std::move(dstregs)),
+        HwModeFeatures(hwmodefeatures.str()), AddedComplexity(complexity),
+        ID(uid), ForceMode(setmode) {}
+
   Record          *getSrcRecord()  const { return SrcRecord; }
+  ListInit        *getPredicates() const { return Predicates; }
   TreePatternNode *getSrcPattern() const { return SrcPattern.get(); }
   TreePatternNodePtr getSrcPatternShared() const { return SrcPattern; }
   TreePatternNode *getDstPattern() const { return DstPattern.get(); }
   TreePatternNodePtr getDstPatternShared() const { return DstPattern; }
   const std::vector<Record*> &getDstRegs() const { return Dstregs; }
+  StringRef   getHwModeFeatures() const { return HwModeFeatures; }
   int         getAddedComplexity() const { return AddedComplexity; }
-  const std::vector<Predicate> &getPredicates() const { return Predicates; }
+  unsigned getID() const { return ID; }
+  unsigned getForceMode() const { return ForceMode; }
 
   std::string getPredicateCheck() const;
+  void getPredicateRecords(SmallVectorImpl<Record *> &PredicateRecs) const;
 
   /// Compute the complexity metric for the input pattern.  This roughly
   /// corresponds to the number of nodes that are covered.
@@ -1289,8 +1241,6 @@ private:
   void GenerateVariants();
   void VerifyInstructionFlags();
 
-  std::vector<Predicate> makePredList(ListInit *L);
-
   void ParseOnePattern(Record *TheDef,
                        TreePattern &Pattern, TreePattern &Result,
                        const std::vector<Record *> &InstImpResults);
diff --git a/llvm/utils/TableGen/CodeGenInstruction.cpp b/llvm/utils/TableGen/CodeGenInstruction.cpp
index 960fe08677f7..3933ce6e1106 100644
--- a/llvm/utils/TableGen/CodeGenInstruction.cpp
+++ b/llvm/utils/TableGen/CodeGenInstruction.cpp
@@ -177,17 +177,17 @@ bool CGIOperandList::hasOperandNamed(StringRef Name, unsigned &OpIdx) const {
 }
 
 std::pair<unsigned,unsigned>
-CGIOperandList::ParseOperandName(const std::string &Op, bool AllowWholeOp) {
+CGIOperandList::ParseOperandName(StringRef Op, bool AllowWholeOp) {
   if (Op.empty() || Op[0] != '$')
     PrintFatalError(TheDef->getLoc(),
                     TheDef->getName() + ": Illegal operand name: '" + Op + "'");
 
-  std::string OpName = Op.substr(1);
-  std::string SubOpName;
+  StringRef OpName = Op.substr(1);
+  StringRef SubOpName;
 
   // Check to see if this is $foo.bar.
-  std::string::size_type DotIdx = OpName.find_first_of('.');
-  if (DotIdx != std::string::npos) {
+  StringRef::size_type DotIdx = OpName.find_first_of('.');
+  if (DotIdx != StringRef::npos) {
     SubOpName = OpName.substr(DotIdx+1);
     if (SubOpName.empty())
       PrintFatalError(TheDef->getLoc(),
@@ -231,16 +231,16 @@ CGIOperandList::ParseOperandName(const std::string &Op, bool AllowWholeOp) {
   return std::make_pair(0U, 0U);
 }
 
-static void ParseConstraint(const std::string &CStr, CGIOperandList &Ops,
+static void ParseConstraint(StringRef CStr, CGIOperandList &Ops,
                             Record *Rec) {
   // EARLY_CLOBBER: @early $reg
-  std::string::size_type wpos = CStr.find_first_of(" \t");
-  std::string::size_type start = CStr.find_first_not_of(" \t");
-  std::string Tok = CStr.substr(start, wpos - start);
+  StringRef::size_type wpos = CStr.find_first_of(" \t");
+  StringRef::size_type start = CStr.find_first_not_of(" \t");
+  StringRef Tok = CStr.substr(start, wpos - start);
   if (Tok == "@earlyclobber") {
-    std::string Name = CStr.substr(wpos+1);
+    StringRef Name = CStr.substr(wpos+1);
     wpos = Name.find_first_not_of(" \t");
-    if (wpos == std::string::npos)
+    if (wpos == StringRef::npos)
       PrintFatalError(
         Rec->getLoc(), "Illegal format for @earlyclobber constraint in '" +
         Rec->getName() + "': '" + CStr + "'");
@@ -258,8 +258,8 @@ static void ParseConstraint(const std::string &CStr, CGIOperandList &Ops,
   }
 
   // Only other constraint is "TIED_TO" for now.
-  std::string::size_type pos = CStr.find_first_of('=');
-  if (pos == std::string::npos)
+  StringRef::size_type pos = CStr.find_first_of('=');
+  if (pos == StringRef::npos)
     PrintFatalError(
       Rec->getLoc(), "Unrecognized constraint '" + CStr +
       "' in '" + Rec->getName() + "'");
@@ -267,20 +267,19 @@ static void ParseConstraint(const std::string &CStr, CGIOperandList &Ops,
 
   // TIED_TO: $src1 = $dst
   wpos = CStr.find_first_of(" \t", start);
-  if (wpos == std::string::npos || wpos > pos)
+  if (wpos == StringRef::npos || wpos > pos)
     PrintFatalError(
       Rec->getLoc(), "Illegal format for tied-to constraint in '" +
       Rec->getName() + "': '" + CStr + "'");
-  std::string LHSOpName =
-      std::string(StringRef(CStr).substr(start, wpos - start));
+  StringRef LHSOpName = CStr.substr(start, wpos - start);
   std::pair<unsigned,unsigned> LHSOp = Ops.ParseOperandName(LHSOpName, false);
 
   wpos = CStr.find_first_not_of(" \t", pos + 1);
-  if (wpos == std::string::npos)
+  if (wpos == StringRef::npos)
     PrintFatalError(
       Rec->getLoc(), "Illegal format for tied-to constraint: '" + CStr + "'");
 
-  std::string RHSOpName = std::string(StringRef(CStr).substr(wpos));
+  StringRef RHSOpName = CStr.substr(wpos);
   std::pair<unsigned,unsigned> RHSOp = Ops.ParseOperandName(RHSOpName, false);
 
   // Sort the operands into order, which should put the output one
@@ -325,29 +324,27 @@ static void ParseConstraint(const std::string &CStr, CGIOperandList &Ops,
   Ops[SrcOp.first].Constraints[SrcOp.second] = NewConstraint;
 }
 
-static void ParseConstraints(const std::string &CStr, CGIOperandList &Ops,
-                             Record *Rec) {
+static void ParseConstraints(StringRef CStr, CGIOperandList &Ops, Record *Rec) {
   if (CStr.empty()) return;
 
-  const std::string delims(",");
-  std::string::size_type bidx, eidx;
+  StringRef delims(",");
+  StringRef::size_type bidx, eidx;
 
   bidx = CStr.find_first_not_of(delims);
-  while (bidx != std::string::npos) {
+  while (bidx != StringRef::npos) {
     eidx = CStr.find_first_of(delims, bidx);
-    if (eidx == std::string::npos)
-      eidx = CStr.length();
+    if (eidx == StringRef::npos)
+      eidx = CStr.size();
 
     ParseConstraint(CStr.substr(bidx, eidx - bidx), Ops, Rec);
     bidx = CStr.find_first_not_of(delims, eidx);
   }
 }
 
-void CGIOperandList::ProcessDisableEncoding(std::string DisableEncoding) {
+void CGIOperandList::ProcessDisableEncoding(StringRef DisableEncoding) {
   while (1) {
-    std::pair<StringRef, StringRef> P = getToken(DisableEncoding, " ,\t");
-    std::string OpName = std::string(P.first);
-    DisableEncoding = std::string(P.second);
+    StringRef OpName;
+    std::tie(OpName, DisableEncoding) = getToken(DisableEncoding, " ,\t");
     if (OpName.empty()) break;
 
     // Figure out which operand this is.
@@ -427,12 +424,11 @@ CodeGenInstruction::CodeGenInstruction(Record *R)
   hasChain_Inferred = false;
 
   // Parse Constraints.
-  ParseConstraints(std::string(R->getValueAsString("Constraints")), Operands,
-                   R);
+  ParseConstraints(R->getValueAsString("Constraints"), Operands, R);
 
   // Parse the DisableEncoding field.
   Operands.ProcessDisableEncoding(
-      std::string(R->getValueAsString("DisableEncoding")));
+      R->getValueAsString("DisableEncoding"));
 
   // First check for a ComplexDeprecationPredicate.
   if (R->getValue("ComplexDeprecationPredicate")) {
diff --git a/llvm/utils/TableGen/CodeGenInstruction.h b/llvm/utils/TableGen/CodeGenInstruction.h
index af851a11676b..e35d2191ea45 100644
--- a/llvm/utils/TableGen/CodeGenInstruction.h
+++ b/llvm/utils/TableGen/CodeGenInstruction.h
@@ -182,7 +182,7 @@ template <typename T> class ArrayRef;
     /// where $foo is a whole operand and $foo.bar refers to a suboperand.
     /// This aborts if the name is invalid.  If AllowWholeOp is true, references
     /// to operands with suboperands are allowed, otherwise not.
-    std::pair<unsigned,unsigned> ParseOperandName(const std::string &Op,
+    std::pair<unsigned,unsigned> ParseOperandName(StringRef Op,
                                                   bool AllowWholeOp = true);
 
     /// getFlattenedOperandNumber - Flatten a operand/suboperand pair into a
@@ -211,7 +211,7 @@ template <typename T> class ArrayRef;
       return false;
     }
 
-    void ProcessDisableEncoding(std::string Value);
+    void ProcessDisableEncoding(StringRef Value);
   };
 
 
diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.h b/llvm/utils/TableGen/CodeGenIntrinsics.h
index c469f662a42d..dbfad3bf6b17 100644
--- a/llvm/utils/TableGen/CodeGenIntrinsics.h
+++ b/llvm/utils/TableGen/CodeGenIntrinsics.h
@@ -120,6 +120,9 @@ struct CodeGenIntrinsic {
   /// True if the intrinsic is marked as noduplicate.
   bool isNoDuplicate;
 
+  /// True if the intrinsic is marked as nomerge.
+  bool isNoMerge;
+
   /// True if the intrinsic is no-return.
   bool isNoReturn;
 
diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp
index 289a20a96f02..6f718acbac3e 100644
--- a/llvm/utils/TableGen/CodeGenMapTable.cpp
+++ b/llvm/utils/TableGen/CodeGenMapTable.cpp
@@ -318,11 +318,10 @@ Record *MapTableEmitter::getInstrForColumn(Record *KeyInstr,
   ListInit *ColFields = InstrMapDesc.getColFields();
   Record *MatchInstr = nullptr;
 
-  for (unsigned i = 0, e = RelatedInstrVec.size(); i < e; i++) {
+  for (llvm::Record *CurInstr : RelatedInstrVec) {
     bool MatchFound = true;
-    Record *CurInstr = RelatedInstrVec[i];
     for (unsigned j = 0, endCF = ColFields->size();
-        (j < endCF) && MatchFound; j++) {
+         (j < endCF) && MatchFound; j++) {
       Init *ColFieldJ = ColFields->getElement(j);
       Init *CurInstrInit = CurInstr->getValue(ColFieldJ)->getValue();
       std::string CurInstrVal = CurInstrInit->getAsUnquotedString();
@@ -342,8 +341,9 @@ Record *MapTableEmitter::getInstrForColumn(Record *KeyInstr,
         }
 
         PrintFatalError("Multiple matches found for `" + KeyInstr->getName() +
-              "', for the relation `" + InstrMapDesc.getName() + "', row fields [" +
-              KeyValueStr + "], column `" + CurValueCol->getAsString() + "'");
+                        "', for the relation `" + InstrMapDesc.getName() +
+                        "', row fields [" + KeyValueStr + "], column `" +
+                        CurValueCol->getAsString() + "'");
       }
       MatchInstr = CurInstr;
     }
diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/CodeGenRegisters.cpp
index f9a7ba6bba80..930b7742103e 100644
--- a/llvm/utils/TableGen/CodeGenRegisters.cpp
+++ b/llvm/utils/TableGen/CodeGenRegisters.cpp
@@ -154,14 +154,11 @@ void CodeGenSubRegIndex::computeConcatTransitiveClosure() {
 //===----------------------------------------------------------------------===//
 
 CodeGenRegister::CodeGenRegister(Record *R, unsigned Enum)
-  : TheDef(R),
-    EnumValue(Enum),
-    CostPerUse(R->getValueAsInt("CostPerUse")),
-    CoveredBySubRegs(R->getValueAsBit("CoveredBySubRegs")),
-    HasDisjunctSubRegs(false),
-    SubRegsComplete(false),
-    SuperRegsComplete(false),
-    TopoSig(~0u) {
+    : TheDef(R), EnumValue(Enum),
+      CostPerUse(R->getValueAsListOfInts("CostPerUse")),
+      CoveredBySubRegs(R->getValueAsBit("CoveredBySubRegs")),
+      HasDisjunctSubRegs(false), SubRegsComplete(false),
+      SuperRegsComplete(false), TopoSig(~0u) {
   Artificial = R->getValueAsBit("isArtificial");
 }
 
@@ -314,18 +311,17 @@ CodeGenRegister::computeSubRegs(CodeGenRegBank &RegBank) {
 
     // Look at the possible compositions of Idx.
     // They may not all be supported by SR.
-    for (CodeGenSubRegIndex::CompMap::const_iterator I = Comps.begin(),
-           E = Comps.end(); I != E; ++I) {
-      SubRegMap::const_iterator SRI = Map.find(I->first);
+    for (auto Comp : Comps) {
+      SubRegMap::const_iterator SRI = Map.find(Comp.first);
       if (SRI == Map.end())
         continue; // Idx + I->first doesn't exist in SR.
       // Add I->second as a name for the subreg SRI->second, assuming it is
       // orphaned, and the name isn't already used for something else.
-      if (SubRegs.count(I->second) || !Orphans.erase(SRI->second))
+      if (SubRegs.count(Comp.second) || !Orphans.erase(SRI->second))
         continue;
       // We found a new name for the orphaned sub-register.
-      SubRegs.insert(std::make_pair(I->second, SRI->second));
-      Indices.push_back(I->second);
+      SubRegs.insert(std::make_pair(Comp.second, SRI->second));
+      Indices.push_back(Comp.second);
     }
   }
 
@@ -531,13 +527,13 @@ void CodeGenRegister::computeSecondarySubRegs(CodeGenRegBank &RegBank) {
   for (unsigned i = 0, e = NewSubRegs.size(); i != e; ++i) {
     CodeGenSubRegIndex *NewIdx = NewSubRegs[i].first;
     CodeGenRegister *NewSubReg = NewSubRegs[i].second;
-    for (SubRegMap::const_iterator SI = NewSubReg->SubRegs.begin(),
-           SE = NewSubReg->SubRegs.end(); SI != SE; ++SI) {
-      CodeGenSubRegIndex *SubIdx = getSubRegIndex(SI->second);
+    for (auto SubReg : NewSubReg->SubRegs) {
+      CodeGenSubRegIndex *SubIdx = getSubRegIndex(SubReg.second);
       if (!SubIdx)
         PrintFatalError(TheDef->getLoc(), "No SubRegIndex for " +
-                        SI->second->getName() + " in " + getName());
-      NewIdx->addComposite(SI->first, SubIdx);
+                                              SubReg.second->getName() +
+                                              " in " + getName());
+      NewIdx->addComposite(SubReg.first, SubIdx);
     }
   }
 }
@@ -550,24 +546,23 @@ void CodeGenRegister::computeSuperRegs(CodeGenRegBank &RegBank) {
 
   // Make sure all sub-registers have been visited first, so the super-reg
   // lists will be topologically ordered.
-  for (SubRegMap::const_iterator I = SubRegs.begin(), E = SubRegs.end();
-       I != E; ++I)
-    I->second->computeSuperRegs(RegBank);
+  for (auto SubReg : SubRegs)
+    SubReg.second->computeSuperRegs(RegBank);
 
   // Now add this as a super-register on all sub-registers.
   // Also compute the TopoSigId in post-order.
   TopoSigId Id;
-  for (SubRegMap::const_iterator I = SubRegs.begin(), E = SubRegs.end();
-       I != E; ++I) {
+  for (auto SubReg : SubRegs) {
     // Topological signature computed from SubIdx, TopoId(SubReg).
     // Loops and idempotent indices have TopoSig = ~0u.
-    Id.push_back(I->first->EnumValue);
-    Id.push_back(I->second->TopoSig);
+    Id.push_back(SubReg.first->EnumValue);
+    Id.push_back(SubReg.second->TopoSig);
 
     // Don't add duplicate entries.
-    if (!I->second->SuperRegs.empty() && I->second->SuperRegs.back() == this)
+    if (!SubReg.second->SuperRegs.empty() &&
+        SubReg.second->SuperRegs.back() == this)
       continue;
-    I->second->SuperRegs.push_back(this);
+    SubReg.second->SuperRegs.push_back(this);
   }
   TopoSig = RegBank.getTopoSig(Id);
 }
@@ -582,17 +577,15 @@ CodeGenRegister::addSubRegsPreOrder(SetVector<const CodeGenRegister*> &OSet,
       SR->addSubRegsPreOrder(OSet, RegBank);
   }
   // Add any secondary sub-registers that weren't part of the explicit tree.
-  for (SubRegMap::const_iterator I = SubRegs.begin(), E = SubRegs.end();
-       I != E; ++I)
-    OSet.insert(I->second);
+  for (auto SubReg : SubRegs)
+    OSet.insert(SubReg.second);
 }
 
 // Get the sum of this register's unit weights.
 unsigned CodeGenRegister::getWeight(const CodeGenRegBank &RegBank) const {
   unsigned Weight = 0;
-  for (RegUnitList::iterator I = RegUnits.begin(), E = RegUnits.end();
-       I != E; ++I) {
-    Weight += RegBank.getRegUnit(*I).Weight;
+  for (unsigned RegUnit : RegUnits) {
+    Weight += RegBank.getRegUnit(RegUnit).Weight;
   }
   return Weight;
 }
@@ -646,16 +639,18 @@ struct TupleExpander : SetTheory::Expander {
       std::string Name;
       Record *Proto = Lists[0][n];
       std::vector<Init*> Tuple;
-      unsigned CostPerUse = 0;
       for (unsigned i = 0; i != Dim; ++i) {
         Record *Reg = Lists[i][n];
         if (i) Name += '_';
         Name += Reg->getName();
         Tuple.push_back(DefInit::get(Reg));
-        CostPerUse = std::max(CostPerUse,
-                              unsigned(Reg->getValueAsInt("CostPerUse")));
       }
 
+      // Take the cost list of the first register in the tuple.
+      ListInit *CostList = Proto->getValueAsListInit("CostPerUse");
+      SmallVector<Init *, 2> CostPerUse;
+      CostPerUse.insert(CostPerUse.end(), CostList->begin(), CostList->end());
+
       StringInit *AsmName = StringInit::get("");
       if (!RegNames.empty()) {
         if (RegNames.size() <= n)
@@ -697,7 +692,7 @@ struct TupleExpander : SetTheory::Expander {
 
         // CostPerUse is aggregated from all Tuple members.
         if (Field == "CostPerUse")
-          RV.setValue(IntInit::get(CostPerUse));
+          RV.setValue(ListInit::get(CostPerUse, CostList->getElementType()));
 
         // Composite registers are always covered by sub-registers.
         if (Field == "CoveredBySubRegs")
@@ -797,7 +792,7 @@ CodeGenRegisterClass::CodeGenRegisterClass(CodeGenRegBank &RegBank, Record *R)
     RI.RegSize = RI.SpillSize = Size ? Size
                                      : VTs[0].getSimple().getSizeInBits();
     RI.SpillAlignment = R->getValueAsInt("Alignment");
-    RSI.Map.insert({DefaultMode, RI});
+    RSI.insertRegSizeForMode(DefaultMode, RI);
   }
 
   CopyCost = R->getValueAsInt("CopyCost");
@@ -838,7 +833,10 @@ void CodeGenRegisterClass::inheritProperties(CodeGenRegBank &RegBank) {
   Namespace = Super.Namespace;
   VTs = Super.VTs;
   CopyCost = Super.CopyCost;
-  Allocatable = Super.Allocatable;
+  // Check for allocatable superclasses.
+  Allocatable = any_of(SuperClasses, [&](const CodeGenRegisterClass *S) {
+    return S->Allocatable;
+  });
   AltOrderSelect = Super.AltOrderSelect;
   AllocationPriority = Super.AllocationPriority;
   GeneratePressureSet |= Super.GeneratePressureSet;
@@ -1396,19 +1394,17 @@ void CodeGenRegBank::computeComposites() {
     TopoSigs.set(Reg1.getTopoSig());
 
     const CodeGenRegister::SubRegMap &SRM1 = Reg1.getSubRegs();
-    for (CodeGenRegister::SubRegMap::const_iterator i1 = SRM1.begin(),
-         e1 = SRM1.end(); i1 != e1; ++i1) {
-      CodeGenSubRegIndex *Idx1 = i1->first;
-      CodeGenRegister *Reg2 = i1->second;
+    for (auto I1 : SRM1) {
+      CodeGenSubRegIndex *Idx1 = I1.first;
+      CodeGenRegister *Reg2 = I1.second;
       // Ignore identity compositions.
       if (&Reg1 == Reg2)
         continue;
       const CodeGenRegister::SubRegMap &SRM2 = Reg2->getSubRegs();
       // Try composing Idx1 with another SubRegIndex.
-      for (CodeGenRegister::SubRegMap::const_iterator i2 = SRM2.begin(),
-           e2 = SRM2.end(); i2 != e2; ++i2) {
-        CodeGenSubRegIndex *Idx2 = i2->first;
-        CodeGenRegister *Reg3 = i2->second;
+      for (auto I2 : SRM2) {
+        CodeGenSubRegIndex *Idx2 = I2.first;
+        CodeGenRegister *Reg3 = I2.second;
         // Ignore identity compositions.
         if (Reg2 == Reg3)
           continue;
@@ -1425,7 +1421,7 @@ void CodeGenRegBank::computeComposites() {
                          " and " + Idx2->getQualifiedName() +
                          " compose ambiguously as " + Prev->getQualifiedName() +
                          " or " + Idx3->getQualifiedName());
-        }          
+        }
       }
     }
   }
@@ -1728,13 +1724,12 @@ static bool normalizeWeight(CodeGenRegister *Reg,
 
   bool Changed = false;
   const CodeGenRegister::SubRegMap &SRM = Reg->getSubRegs();
-  for (CodeGenRegister::SubRegMap::const_iterator SRI = SRM.begin(),
-         SRE = SRM.end(); SRI != SRE; ++SRI) {
-    if (SRI->second == Reg)
+  for (auto SRI : SRM) {
+    if (SRI.second == Reg)
       continue; // self-cycles happen
 
-    Changed |= normalizeWeight(SRI->second, UberSets, RegSets,
-                               NormalRegs, NormalUnits, RegBank);
+    Changed |= normalizeWeight(SRI.second, UberSets, RegSets, NormalRegs,
+                               NormalUnits, RegBank);
   }
   // Postorder register normalization.
 
@@ -2064,15 +2059,14 @@ void CodeGenRegBank::computeRegUnitLaneMasks() {
     // Iterate through SubRegisters.
     typedef CodeGenRegister::SubRegMap SubRegMap;
     const SubRegMap &SubRegs = Register.getSubRegs();
-    for (SubRegMap::const_iterator S = SubRegs.begin(),
-         SE = SubRegs.end(); S != SE; ++S) {
-      CodeGenRegister *SubReg = S->second;
+    for (auto S : SubRegs) {
+      CodeGenRegister *SubReg = S.second;
       // Ignore non-leaf subregisters, their lane masks are fully covered by
       // the leaf subregisters anyway.
       if (!SubReg->getSubRegs().empty())
         continue;
-      CodeGenSubRegIndex *SubRegIndex = S->first;
-      const CodeGenRegister *SubRegister = S->second;
+      CodeGenSubRegIndex *SubRegIndex = S.first;
+      const CodeGenRegister *SubRegister = S.second;
       LaneBitmask LaneMask = SubRegIndex->LaneMask;
       // Distribute LaneMask to Register Units touched.
       for (unsigned SUI : SubRegister->getRegUnits()) {
@@ -2194,10 +2188,9 @@ void CodeGenRegBank::inferSubClassWithSubReg(CodeGenRegisterClass *RC) {
     if (R->Artificial)
       continue;
     const CodeGenRegister::SubRegMap &SRM = R->getSubRegs();
-    for (CodeGenRegister::SubRegMap::const_iterator I = SRM.begin(),
-         E = SRM.end(); I != E; ++I) {
-      if (!I->first->Artificial)
-        SRSets[I->first].push_back(R);
+    for (auto I : SRM) {
+      if (!I.first->Artificial)
+        SRSets[I.first].push_back(R);
     }
   }
 
@@ -2422,9 +2415,8 @@ BitVector CodeGenRegBank::computeCoveredRegisters(ArrayRef<Record*> Regs) {
       // This new super-register is covered by its sub-registers.
       bool AllSubsInSet = true;
       const CodeGenRegister::SubRegMap &SRM = Super->getSubRegs();
-      for (CodeGenRegister::SubRegMap::const_iterator I = SRM.begin(),
-             E = SRM.end(); I != E; ++I)
-        if (!Set.count(I->second)) {
+      for (auto I : SRM)
+        if (!Set.count(I.second)) {
           AllSubsInSet = false;
           break;
         }
diff --git a/llvm/utils/TableGen/CodeGenRegisters.h b/llvm/utils/TableGen/CodeGenRegisters.h
index 5228e6518fe5..6a0696011a40 100644
--- a/llvm/utils/TableGen/CodeGenRegisters.h
+++ b/llvm/utils/TableGen/CodeGenRegisters.h
@@ -151,7 +151,7 @@ namespace llvm {
   struct CodeGenRegister {
     Record *TheDef;
     unsigned EnumValue;
-    unsigned CostPerUse;
+    std::vector<int64_t> CostPerUse;
     bool CoveredBySubRegs;
     bool HasDisjunctSubRegs;
     bool Artificial;
@@ -738,9 +738,8 @@ namespace llvm {
     // Get the sum of unit weights.
     unsigned getRegUnitSetWeight(const std::vector<unsigned> &Units) const {
       unsigned Weight = 0;
-      for (std::vector<unsigned>::const_iterator
-             I = Units.begin(), E = Units.end(); I != E; ++I)
-        Weight += getRegUnit(*I).Weight;
+      for (unsigned Unit : Units)
+        Weight += getRegUnit(Unit).Weight;
       return Weight;
     }
 
diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/CodeGenSchedule.cpp
index b20eb6eff422..ee52b2e7ab9f 100644
--- a/llvm/utils/TableGen/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/CodeGenSchedule.cpp
@@ -708,10 +708,10 @@ void CodeGenSchedModels::collectSchedRW() {
 /// Compute a SchedWrite name from a sequence of writes.
 std::string CodeGenSchedModels::genRWName(ArrayRef<unsigned> Seq, bool IsRead) {
   std::string Name("(");
-  for (auto I = Seq.begin(), E = Seq.end(); I != E; ++I) {
-    if (I != Seq.begin())
-      Name += '_';
-    Name += getSchedRW(*I, IsRead).Name;
+  ListSeparator LS("_");
+  for (unsigned I : Seq) {
+    Name += LS;
+    Name += getSchedRW(I, IsRead).Name;
   }
   Name += ')';
   return Name;
@@ -921,11 +921,10 @@ void CodeGenSchedModels::collectSchedClasses() {
       ProcIndices.push_back(0);
       LLVM_DEBUG({
         dbgs() << "SchedRW machine model for " << InstName;
-        for (IdxIter WI = SC.Writes.begin(), WE = SC.Writes.end(); WI != WE;
-             ++WI)
-          dbgs() << " " << SchedWrites[*WI].Name;
-        for (IdxIter RI = SC.Reads.begin(), RE = SC.Reads.end(); RI != RE; ++RI)
-          dbgs() << " " << SchedReads[*RI].Name;
+        for (unsigned int Write : SC.Writes)
+          dbgs() << " " << SchedWrites[Write].Name;
+        for (unsigned int Read : SC.Reads)
+          dbgs() << " " << SchedReads[Read].Name;
         dbgs() << '\n';
       });
     }
@@ -990,10 +989,10 @@ CodeGenSchedModels::createSchedClassName(Record *ItinClassDef,
 std::string CodeGenSchedModels::createSchedClassName(const RecVec &InstDefs) {
 
   std::string Name;
-  for (RecIter I = InstDefs.begin(), E = InstDefs.end(); I != E; ++I) {
-    if (I != InstDefs.begin())
-      Name += '_';
-    Name += (*I)->getName();
+  ListSeparator LS("_");
+  for (const Record *InstDef : InstDefs) {
+    Name += LS;
+    Name += InstDef->getName();
   }
   return Name;
 }
@@ -1557,12 +1556,11 @@ pushVariant(const TransVariant &VInfo, bool IsRead) {
     // sequence (add to the current operand's sequence).
     SmallVectorImpl<unsigned> &Seq = RWSequences.back();
     IdxVec ExpandedRWs;
-    for (IdxIter RWI = SelectedRWs.begin(), RWE = SelectedRWs.end();
-         RWI != RWE; ++RWI) {
+    for (unsigned int SelectedRW : SelectedRWs) {
       if (IsRead)
-        ExpandedRWs.push_back(*RWI);
+        ExpandedRWs.push_back(SelectedRW);
       else
-        SchedModels.expandRWSequence(*RWI, ExpandedRWs, IsRead);
+        SchedModels.expandRWSequence(SelectedRW, ExpandedRWs, IsRead);
     }
     llvm::append_range(Seq, ExpandedRWs);
   }
@@ -1576,9 +1574,8 @@ bool PredTransitions::substituteVariantOperand(
     const SmallVectorImpl<unsigned> &RWSeq, bool IsRead, unsigned StartIdx) {
   bool Subst = false;
   // Visit each original RW within the current sequence.
-  for (SmallVectorImpl<unsigned>::const_iterator
-         RWI = RWSeq.begin(), RWE = RWSeq.end(); RWI != RWE; ++RWI) {
-    const CodeGenSchedRW &SchedRW = SchedModels.getSchedRW(*RWI, IsRead);
+  for (unsigned int RWI : RWSeq) {
+    const CodeGenSchedRW &SchedRW = SchedModels.getSchedRW(RWI, IsRead);
     // Push this RW on all partial PredTransitions or distribute variants.
     // New PredTransitions may be pushed within this loop which should not be
     // revisited (TransEnd must be loop invariant).
@@ -1593,9 +1590,9 @@ bool PredTransitions::substituteVariantOperand(
         pushVariant(IV, IsRead);
       if (IntersectingVariants.empty()) {
         if (IsRead)
-          TransVec[TransIdx].ReadSequences.back().push_back(*RWI);
+          TransVec[TransIdx].ReadSequences.back().push_back(RWI);
         else
-          TransVec[TransIdx].WriteSequences.back().push_back(*RWI);
+          TransVec[TransIdx].WriteSequences.back().push_back(RWI);
         continue;
       } else {
         Subst = true;
@@ -1620,26 +1617,23 @@ bool PredTransitions::substituteVariants(const PredTransition &Trans) {
   TransVec.emplace_back(Trans.PredTerm, Trans.ProcIndex);
 
   // Visit each original write sequence.
-  for (SmallVectorImpl<SmallVector<unsigned,4>>::const_iterator
-         WSI = Trans.WriteSequences.begin(), WSE = Trans.WriteSequences.end();
-       WSI != WSE; ++WSI) {
+  for (const auto &WriteSequence : Trans.WriteSequences) {
     // Push a new (empty) write sequence onto all partial Transitions.
     for (std::vector<PredTransition>::iterator I =
            TransVec.begin() + StartIdx, E = TransVec.end(); I != E; ++I) {
       I->WriteSequences.emplace_back();
     }
-    Subst |= substituteVariantOperand(*WSI, /*IsRead=*/false, StartIdx);
+    Subst |=
+        substituteVariantOperand(WriteSequence, /*IsRead=*/false, StartIdx);
   }
   // Visit each original read sequence.
-  for (SmallVectorImpl<SmallVector<unsigned,4>>::const_iterator
-         RSI = Trans.ReadSequences.begin(), RSE = Trans.ReadSequences.end();
-       RSI != RSE; ++RSI) {
+  for (const auto &ReadSequence : Trans.ReadSequences) {
     // Push a new (empty) read sequence onto all partial Transitions.
     for (std::vector<PredTransition>::iterator I =
            TransVec.begin() + StartIdx, E = TransVec.end(); I != E; ++I) {
       I->ReadSequences.emplace_back();
     }
-    Subst |= substituteVariantOperand(*RSI, /*IsRead=*/true, StartIdx);
+    Subst |= substituteVariantOperand(ReadSequence, /*IsRead=*/true, StartIdx);
   }
   return Subst;
 }
@@ -1676,33 +1670,32 @@ static void inferFromTransitions(ArrayRef<PredTransition> LastTransitions,
                                  CodeGenSchedModels &SchedModels) {
   // For each PredTransition, create a new CodeGenSchedTransition, which usually
   // requires creating a new SchedClass.
-  for (ArrayRef<PredTransition>::iterator
-         I = LastTransitions.begin(), E = LastTransitions.end(); I != E; ++I) {
+  for (const auto &LastTransition : LastTransitions) {
     // Variant expansion (substituteVariants) may create unconditional
     // transitions. We don't need to build sched classes for them.
-    if (I->PredTerm.empty())
+    if (LastTransition.PredTerm.empty())
       continue;
     IdxVec OperWritesVariant, OperReadsVariant;
-    addSequences(SchedModels, I->WriteSequences, OperWritesVariant, false);
-    addSequences(SchedModels, I->ReadSequences, OperReadsVariant, true);
+    addSequences(SchedModels, LastTransition.WriteSequences, OperWritesVariant,
+                 false);
+    addSequences(SchedModels, LastTransition.ReadSequences, OperReadsVariant,
+                 true);
     CodeGenSchedTransition SCTrans;
 
     // Transition should not contain processor indices already assigned to
     // InstRWs in this scheduling class.
     const CodeGenSchedClass &FromSC = SchedModels.getSchedClass(FromClassIdx);
-    if (FromSC.InstRWProcIndices.count(I->ProcIndex))
+    if (FromSC.InstRWProcIndices.count(LastTransition.ProcIndex))
       continue;
-    SCTrans.ProcIndex = I->ProcIndex;
+    SCTrans.ProcIndex = LastTransition.ProcIndex;
     SCTrans.ToClassIdx =
         SchedModels.addSchedClass(/*ItinClassDef=*/nullptr, OperWritesVariant,
-                                  OperReadsVariant, I->ProcIndex);
+                                  OperReadsVariant, LastTransition.ProcIndex);
 
     // The final PredTerm is unique set of predicates guarding the transition.
     RecVec Preds;
-    transform(I->PredTerm, std::back_inserter(Preds),
-              [](const PredCheck &P) {
-                return P.Predicate;
-              });
+    transform(LastTransition.PredTerm, std::back_inserter(Preds),
+              [](const PredCheck &P) { return P.Predicate; });
     Preds.erase(std::unique(Preds.begin(), Preds.end()), Preds.end());
     dumpTransition(SchedModels, FromSC, SCTrans, Preds);
     SCTrans.PredTerm = std::move(Preds);
@@ -1791,11 +1784,10 @@ void CodeGenSchedModels::inferFromRW(ArrayRef<unsigned> OperWrites,
 // Check if any processor resource group contains all resource records in
 // SubUnits.
 bool CodeGenSchedModels::hasSuperGroup(RecVec &SubUnits, CodeGenProcModel &PM) {
-  for (unsigned i = 0, e = PM.ProcResourceDefs.size(); i < e; ++i) {
-    if (!PM.ProcResourceDefs[i]->isSubClassOf("ProcResGroup"))
+  for (Record *ProcResourceDef : PM.ProcResourceDefs) {
+    if (!ProcResourceDef->isSubClassOf("ProcResGroup"))
       continue;
-    RecVec SuperUnits =
-      PM.ProcResourceDefs[i]->getValueAsListOfDefs("Resources");
+    RecVec SuperUnits = ProcResourceDef->getValueAsListOfDefs("Resources");
     RecIter RI = SubUnits.begin(), RE = SubUnits.end();
     for ( ; RI != RE; ++RI) {
       if (!is_contained(SuperUnits, *RI)) {
@@ -1951,27 +1943,26 @@ void CodeGenSchedModels::collectProcResources() {
     llvm::sort(PM.ReadAdvanceDefs, LessRecord());
     llvm::sort(PM.ProcResourceDefs, LessRecord());
     LLVM_DEBUG(
-        PM.dump();
-        dbgs() << "WriteResDefs: "; for (RecIter RI = PM.WriteResDefs.begin(),
-                                         RE = PM.WriteResDefs.end();
-                                         RI != RE; ++RI) {
-          if ((*RI)->isSubClassOf("WriteRes"))
-            dbgs() << (*RI)->getValueAsDef("WriteType")->getName() << " ";
+        PM.dump(); dbgs() << "WriteResDefs: "; for (auto WriteResDef
+                                                    : PM.WriteResDefs) {
+          if (WriteResDef->isSubClassOf("WriteRes"))
+            dbgs() << WriteResDef->getValueAsDef("WriteType")->getName() << " ";
           else
-            dbgs() << (*RI)->getName() << " ";
+            dbgs() << WriteResDef->getName() << " ";
         } dbgs() << "\nReadAdvanceDefs: ";
-        for (RecIter RI = PM.ReadAdvanceDefs.begin(),
-             RE = PM.ReadAdvanceDefs.end();
-             RI != RE; ++RI) {
-          if ((*RI)->isSubClassOf("ReadAdvance"))
-            dbgs() << (*RI)->getValueAsDef("ReadType")->getName() << " ";
+        for (Record *ReadAdvanceDef
+             : PM.ReadAdvanceDefs) {
+          if (ReadAdvanceDef->isSubClassOf("ReadAdvance"))
+            dbgs() << ReadAdvanceDef->getValueAsDef("ReadType")->getName()
+                   << " ";
           else
-            dbgs() << (*RI)->getName() << " ";
+            dbgs() << ReadAdvanceDef->getName() << " ";
         } dbgs()
         << "\nProcResourceDefs: ";
-        for (RecIter RI = PM.ProcResourceDefs.begin(),
-             RE = PM.ProcResourceDefs.end();
-             RI != RE; ++RI) { dbgs() << (*RI)->getName() << " "; } dbgs()
+        for (Record *ProcResourceDef
+             : PM.ProcResourceDefs) {
+          dbgs() << ProcResourceDef->getName() << " ";
+        } dbgs()
         << '\n');
     verifyProcResourceGroups(PM);
   }
@@ -2073,23 +2064,20 @@ void CodeGenSchedModels::collectRWResources(unsigned RWIdx, bool IsRead,
         addReadAdvance(SchedRW.TheDef, Idx);
     }
   }
-  for (RecIter AI = SchedRW.Aliases.begin(), AE = SchedRW.Aliases.end();
-       AI != AE; ++AI) {
+  for (auto *Alias : SchedRW.Aliases) {
     IdxVec AliasProcIndices;
-    if ((*AI)->getValueInit("SchedModel")->isComplete()) {
+    if (Alias->getValueInit("SchedModel")->isComplete()) {
       AliasProcIndices.push_back(
-        getProcModel((*AI)->getValueAsDef("SchedModel")).Index);
-    }
-    else
+          getProcModel(Alias->getValueAsDef("SchedModel")).Index);
+    } else
       AliasProcIndices = ProcIndices;
-    const CodeGenSchedRW &AliasRW = getSchedRW((*AI)->getValueAsDef("AliasRW"));
+    const CodeGenSchedRW &AliasRW = getSchedRW(Alias->getValueAsDef("AliasRW"));
     assert(AliasRW.IsRead == IsRead && "cannot alias reads to writes");
 
     IdxVec ExpandedRWs;
     expandRWSequence(AliasRW.Index, ExpandedRWs, IsRead);
-    for (IdxIter SI = ExpandedRWs.begin(), SE = ExpandedRWs.end();
-         SI != SE; ++SI) {
-      collectRWResources(*SI, IsRead, AliasProcIndices);
+    for (unsigned int ExpandedRW : ExpandedRWs) {
+      collectRWResources(ExpandedRW, IsRead, AliasProcIndices);
     }
   }
 }
@@ -2179,9 +2167,8 @@ void CodeGenSchedModels::addWriteRes(Record *ProcWriteResDef, unsigned PIdx) {
 
   // Visit ProcResourceKinds referenced by the newly discovered WriteRes.
   RecVec ProcResDefs = ProcWriteResDef->getValueAsListOfDefs("ProcResources");
-  for (RecIter WritePRI = ProcResDefs.begin(), WritePRE = ProcResDefs.end();
-       WritePRI != WritePRE; ++WritePRI) {
-    addProcResource(*WritePRI, ProcModels[PIdx], ProcWriteResDef->getLoc());
+  for (auto *ProcResDef : ProcResDefs) {
+    addProcResource(ProcResDef, ProcModels[PIdx], ProcWriteResDef->getLoc());
   }
 }
 
@@ -2259,28 +2246,21 @@ void CodeGenSchedClass::dump(const CodeGenSchedModels* SchedModels) const {
 
 void PredTransitions::dump() const {
   dbgs() << "Expanded Variants:\n";
-  for (std::vector<PredTransition>::const_iterator
-         TI = TransVec.begin(), TE = TransVec.end(); TI != TE; ++TI) {
+  for (const auto &TI : TransVec) {
     dbgs() << "{";
-    for (SmallVectorImpl<PredCheck>::const_iterator
-           PCI = TI->PredTerm.begin(), PCE = TI->PredTerm.end();
-         PCI != PCE; ++PCI) {
-      if (PCI != TI->PredTerm.begin())
-        dbgs() << ", ";
-      dbgs() << SchedModels.getSchedRW(PCI->RWIdx, PCI->IsRead).Name
-             << ":" << PCI->Predicate->getName();
-    }
+    ListSeparator LS;
+    for (const PredCheck &PC : TI.PredTerm)
+      dbgs() << LS << SchedModels.getSchedRW(PC.RWIdx, PC.IsRead).Name << ":"
+             << PC.Predicate->getName();
     dbgs() << "},\n  => {";
-    for (SmallVectorImpl<SmallVector<unsigned,4>>::const_iterator
-           WSI = TI->WriteSequences.begin(), WSE = TI->WriteSequences.end();
+    for (SmallVectorImpl<SmallVector<unsigned, 4>>::const_iterator
+             WSI = TI.WriteSequences.begin(),
+             WSE = TI.WriteSequences.end();
          WSI != WSE; ++WSI) {
       dbgs() << "(";
-      for (SmallVectorImpl<unsigned>::const_iterator
-             WI = WSI->begin(), WE = WSI->end(); WI != WE; ++WI) {
-        if (WI != WSI->begin())
-          dbgs() << ", ";
-        dbgs() << SchedModels.getSchedWrite(*WI).Name;
-      }
+      ListSeparator LS;
+      for (unsigned N : *WSI)
+        dbgs() << LS << SchedModels.getSchedWrite(N).Name;
       dbgs() << "),";
     }
     dbgs() << "}\n";
diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/CodeGenTarget.cpp
index 8f6d212df5ec..7311819f77ff 100644
--- a/llvm/utils/TableGen/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/CodeGenTarget.cpp
@@ -99,6 +99,8 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::v64i8:    return "MVT::v64i8";
   case MVT::v128i8:   return "MVT::v128i8";
   case MVT::v256i8:   return "MVT::v256i8";
+  case MVT::v512i8:   return "MVT::v512i8";
+  case MVT::v1024i8:  return "MVT::v1024i8";
   case MVT::v1i16:    return "MVT::v1i16";
   case MVT::v2i16:    return "MVT::v2i16";
   case MVT::v3i16:    return "MVT::v3i16";
@@ -108,11 +110,15 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::v32i16:   return "MVT::v32i16";
   case MVT::v64i16:   return "MVT::v64i16";
   case MVT::v128i16:  return "MVT::v128i16";
+  case MVT::v256i16:  return "MVT::v256i16";
+  case MVT::v512i16:  return "MVT::v512i16";
   case MVT::v1i32:    return "MVT::v1i32";
   case MVT::v2i32:    return "MVT::v2i32";
   case MVT::v3i32:    return "MVT::v3i32";
   case MVT::v4i32:    return "MVT::v4i32";
   case MVT::v5i32:    return "MVT::v5i32";
+  case MVT::v6i32:    return "MVT::v6i32";
+  case MVT::v7i32:    return "MVT::v7i32";
   case MVT::v8i32:    return "MVT::v8i32";
   case MVT::v16i32:   return "MVT::v16i32";
   case MVT::v32i32:   return "MVT::v32i32";
@@ -124,6 +130,7 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::v2048i32: return "MVT::v2048i32";
   case MVT::v1i64:    return "MVT::v1i64";
   case MVT::v2i64:    return "MVT::v2i64";
+  case MVT::v3i64:    return "MVT::v3i64";
   case MVT::v4i64:    return "MVT::v4i64";
   case MVT::v8i64:    return "MVT::v8i64";
   case MVT::v16i64:   return "MVT::v16i64";
@@ -132,6 +139,7 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::v128i64:  return "MVT::v128i64";
   case MVT::v256i64:  return "MVT::v256i64";
   case MVT::v1i128:   return "MVT::v1i128";
+  case MVT::v1f16:    return "MVT::v1f16";
   case MVT::v2f16:    return "MVT::v2f16";
   case MVT::v3f16:    return "MVT::v3f16";
   case MVT::v4f16:    return "MVT::v4f16";
@@ -140,6 +148,8 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::v32f16:   return "MVT::v32f16";
   case MVT::v64f16:   return "MVT::v64f16";
   case MVT::v128f16:  return "MVT::v128f16";
+  case MVT::v256f16:  return "MVT::v256f16";
+  case MVT::v512f16:  return "MVT::v512f16";
   case MVT::v2bf16:   return "MVT::v2bf16";
   case MVT::v3bf16:   return "MVT::v3bf16";
   case MVT::v4bf16:   return "MVT::v4bf16";
@@ -153,6 +163,8 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::v3f32:    return "MVT::v3f32";
   case MVT::v4f32:    return "MVT::v4f32";
   case MVT::v5f32:    return "MVT::v5f32";
+  case MVT::v6f32:    return "MVT::v6f32";
+  case MVT::v7f32:    return "MVT::v7f32";
   case MVT::v8f32:    return "MVT::v8f32";
   case MVT::v16f32:   return "MVT::v16f32";
   case MVT::v32f32:   return "MVT::v32f32";
@@ -164,6 +176,7 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::v2048f32: return "MVT::v2048f32";
   case MVT::v1f64:    return "MVT::v1f64";
   case MVT::v2f64:    return "MVT::v2f64";
+  case MVT::v3f64:    return "MVT::v3f64";
   case MVT::v4f64:    return "MVT::v4f64";
   case MVT::v8f64:    return "MVT::v8f64";
   case MVT::v16f64:   return "MVT::v16f64";
@@ -209,6 +222,7 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::nxv8f16:  return "MVT::nxv8f16";
   case MVT::nxv16f16: return "MVT::nxv16f16";
   case MVT::nxv32f16: return "MVT::nxv32f16";
+  case MVT::nxv1bf16:  return "MVT::nxv1bf16";
   case MVT::nxv2bf16:  return "MVT::nxv2bf16";
   case MVT::nxv4bf16:  return "MVT::nxv4bf16";
   case MVT::nxv8bf16:  return "MVT::nxv8bf16";
@@ -251,9 +265,9 @@ CodeGenTarget::CodeGenTarget(RecordKeeper &records)
   : Records(records), CGH(records) {
   std::vector<Record*> Targets = Records.getAllDerivedDefinitions("Target");
   if (Targets.size() == 0)
-    PrintFatalError("ERROR: No 'Target' subclasses defined!");
+    PrintFatalError("No 'Target' subclasses defined!");
   if (Targets.size() != 1)
-    PrintFatalError("ERROR: Multiple subclasses of Target defined!");
+    PrintFatalError("Multiple subclasses of Target defined!");
   TargetRec = Targets[0];
 }
 
@@ -656,6 +670,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
   isWillReturn = false;
   isCold = false;
   isNoDuplicate = false;
+  isNoMerge = false;
   isConvergent = false;
   isSpeculatable = false;
   hasSideEffects = false;
@@ -845,6 +860,8 @@ void CodeGenIntrinsic::setProperty(Record *R) {
     canThrow = true;
   else if (R->getName() == "IntrNoDuplicate")
     isNoDuplicate = true;
+  else if (R->getName() == "IntrNoMerge")
+    isNoMerge = true;
   else if (R->getName() == "IntrConvergent")
     isConvergent = true;
   else if (R->getName() == "IntrNoReturn")
diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp
index 32ed0bf98743..2f211e2958fa 100644
--- a/llvm/utils/TableGen/DAGISelEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelEmitter.cpp
@@ -115,7 +115,7 @@ struct PatternSortingPredicate {
     // pattern may have been resolved into multiple match patterns due to
     // alternative fragments.  To ensure deterministic output, always use
     // std::stable_sort with this predicate.
-    return LHS->ID < RHS->ID;
+    return LHS->getID() < RHS->getID();
   }
 };
 } // End anonymous namespace
@@ -153,9 +153,8 @@ void DAGISelEmitter::run(raw_ostream &OS) {
   // Add all the patterns to a temporary list so we can sort them.
   Records.startTimer("Sort patterns");
   std::vector<const PatternToMatch*> Patterns;
-  for (CodeGenDAGPatterns::ptm_iterator I = CGP.ptm_begin(), E = CGP.ptm_end();
-       I != E; ++I)
-    Patterns.push_back(&*I);
+  for (const PatternToMatch &PTM : CGP.ptms())
+    Patterns.push_back(&PTM);
 
   // We want to process the matches in order of minimal cost.  Sort the patterns
   // so the least cost one is at the start.
@@ -164,9 +163,9 @@ void DAGISelEmitter::run(raw_ostream &OS) {
   // Convert each variant of each pattern into a Matcher.
   Records.startTimer("Convert to matchers");
   std::vector<Matcher*> PatternMatchers;
-  for (unsigned i = 0, e = Patterns.size(); i != e; ++i) {
+  for (const PatternToMatch *PTM : Patterns) {
     for (unsigned Variant = 0; ; ++Variant) {
-      if (Matcher *M = ConvertPatternToMatcher(*Patterns[i], Variant, CGP))
+      if (Matcher *M = ConvertPatternToMatcher(*PTM, Variant, CGP))
         PatternMatchers.push_back(M);
       else
         break;
diff --git a/llvm/utils/TableGen/DAGISelMatcher.cpp b/llvm/utils/TableGen/DAGISelMatcher.cpp
index bebd205ad58f..e436a931a9f5 100644
--- a/llvm/utils/TableGen/DAGISelMatcher.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcher.cpp
@@ -421,3 +421,15 @@ bool CheckImmAllZerosVMatcher::isContradictoryImpl(const Matcher *M) const {
   // AllOnes is contradictory.
   return isa<CheckImmAllOnesVMatcher>(M);
 }
+
+bool CheckCondCodeMatcher::isContradictoryImpl(const Matcher *M) const {
+  if (const auto *CCCM = dyn_cast<CheckCondCodeMatcher>(M))
+    return CCCM->getCondCodeName() != getCondCodeName();
+  return false;
+}
+
+bool CheckChild2CondCodeMatcher::isContradictoryImpl(const Matcher *M) const {
+  if (const auto *CCCCM = dyn_cast<CheckChild2CondCodeMatcher>(M))
+    return CCCCM->getCondCodeName() != getCondCodeName();
+  return false;
+}
diff --git a/llvm/utils/TableGen/DAGISelMatcher.h b/llvm/utils/TableGen/DAGISelMatcher.h
index ff9a0cb335d1..77280acaf4ca 100644
--- a/llvm/utils/TableGen/DAGISelMatcher.h
+++ b/llvm/utils/TableGen/DAGISelMatcher.h
@@ -635,6 +635,7 @@ private:
   bool isEqualImpl(const Matcher *M) const override {
     return cast<CheckCondCodeMatcher>(M)->CondCodeName == CondCodeName;
   }
+  bool isContradictoryImpl(const Matcher *M) const override;
 };
 
 /// CheckChild2CondCodeMatcher - This checks to see if child 2 node is a
@@ -656,6 +657,7 @@ private:
   bool isEqualImpl(const Matcher *M) const override {
     return cast<CheckChild2CondCodeMatcher>(M)->CondCodeName == CondCodeName;
   }
+  bool isContradictoryImpl(const Matcher *M) const override;
 };
 
 /// CheckValueTypeMatcher - This checks to see if the current node is a
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 03528a46aea7..a552998e4eee 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -173,7 +173,7 @@ static std::string GetPatFromTreePatternNode(const TreePatternNode *N) {
   return str;
 }
 
-static size_t GetVBRSize(unsigned Val) {
+static unsigned GetVBRSize(unsigned Val) {
   if (Val <= 127) return 1;
 
   unsigned NumBytes = 0;
@@ -186,7 +186,7 @@ static size_t GetVBRSize(unsigned Val) {
 
 /// EmitVBRValue - Emit the specified value as a VBR, returning the number of
 /// bytes emitted.
-static uint64_t EmitVBRValue(uint64_t Val, raw_ostream &OS) {
+static unsigned EmitVBRValue(uint64_t Val, raw_ostream &OS) {
   if (Val <= 127) {
     OS << Val << ", ";
     return 1;
@@ -206,6 +206,18 @@ static uint64_t EmitVBRValue(uint64_t Val, raw_ostream &OS) {
   return NumBytes+1;
 }
 
+/// Emit the specified signed value as a VBR. To improve compression we encode
+/// positive numbers shifted left by 1 and negative numbers negated and shifted
+/// left by 1 with bit 0 set.
+static unsigned EmitSignedVBRValue(uint64_t Val, raw_ostream &OS) {
+  if ((int64_t)Val >= 0)
+    Val = Val << 1;
+  else
+    Val = (-Val << 1) | 1;
+
+  return EmitVBRValue(Val, OS);
+}
+
 // This is expensive and slow.
 static std::string getIncludePath(const Record *R) {
   std::string str;
@@ -255,7 +267,7 @@ SizeMatcher(Matcher *N, raw_ostream &OS) {
     assert(SM->getNext() == nullptr && "Scope matcher should not have next");
     unsigned Size = 1; // Count the kind.
     for (unsigned i = 0, e = SM->getNumChildren(); i != e; ++i) {
-      const size_t ChildSize = SizeMatcherList(SM->getChild(i), OS);
+      const unsigned ChildSize = SizeMatcherList(SM->getChild(i), OS);
       assert(ChildSize != 0 && "Matcher cannot have child of size 0");
       SM->getChild(i)->setSize(ChildSize);
       Size += GetVBRSize(ChildSize) + ChildSize; // Count VBR and child size.
@@ -283,7 +295,7 @@ SizeMatcher(Matcher *N, raw_ostream &OS) {
         Child = cast<SwitchTypeMatcher>(N)->getCaseMatcher(i);
         ++Size; // Count the child's type.
       }
-      const size_t ChildSize = SizeMatcherList(Child, OS);
+      const unsigned ChildSize = SizeMatcherList(Child, OS);
       assert(ChildSize != 0 && "Matcher cannot have child of size 0");
       Child->setSize(ChildSize);
       Size += GetVBRSize(ChildSize) + ChildSize; // Count VBR and child size.
@@ -381,9 +393,8 @@ EmitMatcher(const Matcher *N, const unsigned Indent, unsigned CurrentIdx,
           OS.indent(Indent);
       }
 
-      size_t ChildSize = SM->getChild(i)->getSize();
-      size_t VBRSize = GetVBRSize(ChildSize);
-      EmitVBRValue(ChildSize, OS);
+      unsigned ChildSize = SM->getChild(i)->getSize();
+      unsigned VBRSize = EmitVBRValue(ChildSize, OS);
       if (!OmitComments) {
         OS << "/*->" << CurrentIdx + VBRSize + ChildSize << "*/";
         if (i == 0)
@@ -534,7 +545,7 @@ EmitMatcher(const Matcher *N, const unsigned Indent, unsigned CurrentIdx,
                      "/*SwitchOpcode*/ " : "/*SwitchType*/ ");
       }
 
-      size_t ChildSize = Child->getSize();
+      unsigned ChildSize = Child->getSize();
       CurrentIdx += EmitVBRValue(ChildSize, OS) + IdxSize;
       if (const SwitchOpcodeMatcher *SOM = dyn_cast<SwitchOpcodeMatcher>(N))
         OS << "TARGET_VAL(" << SOM->getCaseOpcode(i).getEnumName() << "),";
@@ -580,15 +591,16 @@ EmitMatcher(const Matcher *N, const unsigned Indent, unsigned CurrentIdx,
 
   case Matcher::CheckInteger: {
     OS << "OPC_CheckInteger, ";
-    unsigned Bytes=1+EmitVBRValue(cast<CheckIntegerMatcher>(N)->getValue(), OS);
+    unsigned Bytes =
+        1 + EmitSignedVBRValue(cast<CheckIntegerMatcher>(N)->getValue(), OS);
     OS << '\n';
     return Bytes;
   }
   case Matcher::CheckChildInteger: {
     OS << "OPC_CheckChild" << cast<CheckChildIntegerMatcher>(N)->getChildNo()
        << "Integer, ";
-    unsigned Bytes=1+EmitVBRValue(cast<CheckChildIntegerMatcher>(N)->getValue(),
-                                  OS);
+    unsigned Bytes = 1 + EmitSignedVBRValue(
+                             cast<CheckChildIntegerMatcher>(N)->getValue(), OS);
     OS << '\n';
     return Bytes;
   }
@@ -656,16 +668,16 @@ EmitMatcher(const Matcher *N, const unsigned Indent, unsigned CurrentIdx,
     int64_t Val = cast<EmitIntegerMatcher>(N)->getValue();
     OS << "OPC_EmitInteger, "
        << getEnumName(cast<EmitIntegerMatcher>(N)->getVT()) << ", ";
-    unsigned Bytes = 2+EmitVBRValue(Val, OS);
+    unsigned Bytes = 2 + EmitSignedVBRValue(Val, OS);
     OS << '\n';
     return Bytes;
   }
   case Matcher::EmitStringInteger: {
     const std::string &Val = cast<EmitStringIntegerMatcher>(N)->getValue();
     // These should always fit into 7 bits.
-    OS << "OPC_EmitInteger, "
-      << getEnumName(cast<EmitStringIntegerMatcher>(N)->getVT()) << ", "
-      << Val << ",\n";
+    OS << "OPC_EmitStringInteger, "
+       << getEnumName(cast<EmitStringIntegerMatcher>(N)->getVT()) << ", " << Val
+       << ",\n";
     return 3;
   }
 
@@ -886,14 +898,13 @@ void MatcherTableEmitter::EmitNodePredicatesFunction(
   for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
     // Emit the predicate code corresponding to this pattern.
     const TreePredicateFn PredFn = Preds[i];
-
     assert(!PredFn.isAlwaysTrue() && "No code in this predicate");
-    OS << "  case " << i << ": {\n";
-    for (auto *SimilarPred :
-             NodePredicatesByCodeToRun[PredFn.getCodeToRunOnSDNode()])
-      OS << "    // " << TreePredicateFn(SimilarPred).getFnName() <<'\n';
+    std::string PredFnCodeStr = PredFn.getCodeToRunOnSDNode();
 
-    OS << PredFn.getCodeToRunOnSDNode() << "\n  }\n";
+    OS << "  case " << i << ": {\n";
+    for (auto *SimilarPred : NodePredicatesByCodeToRun[PredFnCodeStr])
+      OS << "    // " << TreePredicateFn(SimilarPred).getFnName() << '\n';
+    OS << PredFnCodeStr << "\n  }\n";
   }
   OS << "  }\n";
   OS << "}\n";
diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
index f7415b87e1c0..2625595cf9d5 100644
--- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
@@ -163,7 +163,7 @@ MatcherGen::MatcherGen(const PatternToMatch &pattern,
   PatWithNoTypes->RemoveAllTypes();
 
   // If there are types that are manifestly known, infer them.
-  InferPossibleTypes(Pattern.ForceMode);
+  InferPossibleTypes(Pattern.getForceMode());
 }
 
 /// InferPossibleTypes - As we emit the pattern, we end up generating type
@@ -576,7 +576,7 @@ bool MatcherGen::EmitMatcherCode(unsigned Variant) {
 
   // Emit the matcher for the pattern structure and types.
   EmitMatchCode(Pattern.getSrcPattern(), PatWithNoTypes.get(),
-                Pattern.ForceMode);
+                Pattern.getForceMode());
 
   // If the pattern has a predicate on it (e.g. only enabled when a subtarget
   // feature is around, do the check).
diff --git a/llvm/utils/TableGen/DFAEmitter.cpp b/llvm/utils/TableGen/DFAEmitter.cpp
index 781cb0636fa1..27161d261e85 100644
--- a/llvm/utils/TableGen/DFAEmitter.cpp
+++ b/llvm/utils/TableGen/DFAEmitter.cpp
@@ -19,7 +19,6 @@
 // to the NFA.
 //
 //===----------------------------------------------------------------------===//
-#define DEBUG_TYPE "dfa-emitter"
 
 #include "DFAEmitter.h"
 #include "CodeGenTarget.h"
@@ -39,6 +38,8 @@
 #include <string>
 #include <vector>
 
+#define DEBUG_TYPE "dfa-emitter"
+
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
@@ -376,11 +377,9 @@ void CustomDfaEmitter::printActionValue(action_type A, raw_ostream &OS) {
   const ActionTuple &AT = Actions[A];
   if (AT.size() > 1)
     OS << "std::make_tuple(";
-  bool First = true;
+  ListSeparator LS;
   for (const auto &SingleAction : AT) {
-    if (!First)
-      OS << ", ";
-    First = false;
+    OS << LS;
     SingleAction.print(OS);
   }
   if (AT.size() > 1)
diff --git a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
index 40d9385e5e9e..9cbdbc19c206 100644
--- a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
+++ b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
@@ -14,8 +14,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define DEBUG_TYPE "dfa-emitter"
-
 #include "CodeGenSchedule.h"
 #include "CodeGenTarget.h"
 #include "DFAEmitter.h"
@@ -34,6 +32,8 @@
 #include <unordered_map>
 #include <vector>
 
+#define DEBUG_TYPE "dfa-emitter"
+
 using namespace llvm;
 
 // We use a uint64_t to represent a resource bitmask.
@@ -157,8 +157,8 @@ int DFAPacketizerEmitter::collectAllComboFuncs(ArrayRef<Record *> ComboFuncList)
       uint64_t ComboResources = ComboBit;
       LLVM_DEBUG(dbgs() << "      combo: " << ComboFuncName << ":0x"
                         << Twine::utohexstr(ComboResources) << "\n");
-      for (unsigned k = 0, M = FuncList.size(); k < M; ++k) {
-        std::string FuncName = std::string(FuncList[k]->getName());
+      for (auto *K : FuncList) {
+        std::string FuncName = std::string(K->getName());
         uint64_t FuncResources = FUNameToBitsMap[FuncName];
         LLVM_DEBUG(dbgs() << "        " << FuncName << ":0x"
                           << Twine::utohexstr(FuncResources) << "\n");
diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp
index c9daa9b24155..b21bf369d18e 100644
--- a/llvm/utils/TableGen/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/DirectiveEmitter.cpp
@@ -633,6 +633,43 @@ void GenerateFlangClauseUnparse(const DirectiveLanguage &DirLang,
   }
 }
 
+// Generate check in the Enter functions for clauses classes.
+void GenerateFlangClauseCheckPrototypes(const DirectiveLanguage &DirLang,
+                                        raw_ostream &OS) {
+
+  IfDefScope Scope("GEN_FLANG_CLAUSE_CHECK_ENTER", OS);
+
+  OS << "\n";
+  for (const auto &C : DirLang.getClauses()) {
+    Clause Clause{C};
+    OS << "void Enter(const parser::" << DirLang.getFlangClauseBaseClass()
+       << "::" << Clause.getFormattedParserClassName() << " &);\n";
+  }
+}
+
+// Generate the mapping for clauses between the parser class and the
+// corresponding clause Kind
+void GenerateFlangClauseParserKindMap(const DirectiveLanguage &DirLang,
+                                      raw_ostream &OS) {
+
+  IfDefScope Scope("GEN_FLANG_CLAUSE_PARSER_KIND_MAP", OS);
+
+  OS << "\n";
+  for (const auto &C : DirLang.getClauses()) {
+    Clause Clause{C};
+    OS << "if constexpr (std::is_same_v<A, parser::"
+       << DirLang.getFlangClauseBaseClass()
+       << "::" << Clause.getFormattedParserClassName();
+    OS << ">)\n";
+    OS << "  return llvm::" << DirLang.getCppNamespace()
+       << "::Clause::" << DirLang.getClausePrefix() << Clause.getFormattedName()
+       << ";\n";
+  }
+
+  OS << "llvm_unreachable(\"Invalid " << DirLang.getName()
+     << " Parser clause\");\n";
+}
+
 // Generate the implementation section for the enumeration in the directive
 // language
 void EmitDirectivesFlangImpl(const DirectiveLanguage &DirLang,
@@ -649,6 +686,10 @@ void EmitDirectivesFlangImpl(const DirectiveLanguage &DirLang,
   GenerateFlangClauseDump(DirLang, OS);
 
   GenerateFlangClauseUnparse(DirLang, OS);
+
+  GenerateFlangClauseCheckPrototypes(DirLang, OS);
+
+  GenerateFlangClauseParserKindMap(DirLang, OS);
 }
 
 void GenerateClauseClassMacro(const DirectiveLanguage &DirLang,
@@ -717,37 +758,11 @@ void GenerateClauseClassMacro(const DirectiveLanguage &DirLang,
   OS << "#undef CLAUSE\n";
 }
 
-// Generate the implementation section for the enumeration in the directive
-// language.
-void EmitDirectivesGen(RecordKeeper &Records, raw_ostream &OS) {
-  const auto DirLang = DirectiveLanguage{Records};
-  if (DirLang.HasValidityErrors())
-    return;
-
-  EmitDirectivesFlangImpl(DirLang, OS);
-
-  GenerateClauseClassMacro(DirLang, OS);
-}
-
-// Generate the implementation for the enumeration in the directive
+// Generate the implemenation for the enumeration in the directive
 // language. This code can be included in library.
-void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
-  const auto DirLang = DirectiveLanguage{Records};
-  if (DirLang.HasValidityErrors())
-    return;
-
-  if (!DirLang.getIncludeHeader().empty())
-    OS << "#include \"" << DirLang.getIncludeHeader() << "\"\n\n";
-
-  OS << "#include \"llvm/ADT/StringRef.h\"\n";
-  OS << "#include \"llvm/ADT/StringSwitch.h\"\n";
-  OS << "#include \"llvm/Support/ErrorHandling.h\"\n";
-  OS << "\n";
-  OS << "using namespace llvm;\n";
-  llvm::SmallVector<StringRef, 2> Namespaces;
-  llvm::SplitString(DirLang.getCppNamespace(), Namespaces, "::");
-  for (auto Ns : Namespaces)
-    OS << "using namespace " << Ns << ";\n";
+void EmitDirectivesBasicImpl(const DirectiveLanguage &DirLang,
+                             raw_ostream &OS) {
+  IfDefScope Scope("GEN_DIRECTIVES_IMPL", OS);
 
   // getDirectiveKind(StringRef Str)
   GenerateGetKind(DirLang.getDirectives(), OS, "Directive", DirLang,
@@ -773,4 +788,18 @@ void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
   GenerateIsAllowedClause(DirLang, OS);
 }
 
+// Generate the implemenation section for the enumeration in the directive
+// language.
+void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
+  const auto DirLang = DirectiveLanguage{Records};
+  if (DirLang.HasValidityErrors())
+    return;
+
+  EmitDirectivesFlangImpl(DirLang, OS);
+
+  GenerateClauseClassMacro(DirLang, OS);
+
+  EmitDirectivesBasicImpl(DirLang, OS);
+}
+
 } // namespace llvm
diff --git a/llvm/utils/TableGen/ExegesisEmitter.cpp b/llvm/utils/TableGen/ExegesisEmitter.cpp
index 4e532c371691..77654cbc92fd 100644
--- a/llvm/utils/TableGen/ExegesisEmitter.cpp
+++ b/llvm/utils/TableGen/ExegesisEmitter.cpp
@@ -98,9 +98,9 @@ ExegesisEmitter::ExegesisEmitter(RecordKeeper &RK)
     : Records(RK), PfmCounterNameTable(collectPfmCounters(RK)) {
   std::vector<Record *> Targets = Records.getAllDerivedDefinitions("Target");
   if (Targets.size() == 0)
-    PrintFatalError("ERROR: No 'Target' subclasses defined!");
+    PrintFatalError("No 'Target' subclasses defined!");
   if (Targets.size() != 1)
-    PrintFatalError("ERROR: Multiple subclasses of Target defined!");
+    PrintFatalError("Multiple subclasses of Target defined!");
   Target = std::string(Targets[0]->getName());
 }
 
diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp
index 0729ab70d696..d64262124308 100644
--- a/llvm/utils/TableGen/FastISelEmitter.cpp
+++ b/llvm/utils/TableGen/FastISelEmitter.cpp
@@ -290,9 +290,11 @@ struct OperandsSignature {
   }
 
   void PrintParameters(raw_ostream &OS) const {
+    ListSeparator LS;
     for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
+      OS << LS;
       if (Operands[i].isReg()) {
-        OS << "unsigned Op" << i << ", bool Op" << i << "IsKill";
+        OS << "unsigned Op" << i;
       } else if (Operands[i].isImm()) {
         OS << "uint64_t imm" << i;
       } else if (Operands[i].isFP()) {
@@ -300,31 +302,25 @@ struct OperandsSignature {
       } else {
         llvm_unreachable("Unknown operand kind!");
       }
-      if (i + 1 != e)
-        OS << ", ";
     }
   }
 
   void PrintArguments(raw_ostream &OS,
                       const std::vector<std::string> &PR) const {
     assert(PR.size() == Operands.size());
-    bool PrintedArg = false;
+    ListSeparator LS;
     for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
       if (PR[i] != "")
         // Implicit physical register operand.
         continue;
 
-      if (PrintedArg)
-        OS << ", ";
+      OS << LS;
       if (Operands[i].isReg()) {
-        OS << "Op" << i << ", Op" << i << "IsKill";
-        PrintedArg = true;
+        OS << "Op" << i;
       } else if (Operands[i].isImm()) {
         OS << "imm" << i;
-        PrintedArg = true;
       } else if (Operands[i].isFP()) {
         OS << "f" << i;
-        PrintedArg = true;
       } else {
         llvm_unreachable("Unknown operand kind!");
       }
@@ -332,9 +328,11 @@ struct OperandsSignature {
   }
 
   void PrintArguments(raw_ostream &OS) const {
+    ListSeparator LS;
     for (unsigned i = 0, e = Operands.size(); i != e; ++i) {
+      OS << LS;
       if (Operands[i].isReg()) {
-        OS << "Op" << i << ", Op" << i << "IsKill";
+        OS << "Op" << i;
       } else if (Operands[i].isImm()) {
         OS << "imm" << i;
       } else if (Operands[i].isFP()) {
@@ -342,8 +340,6 @@ struct OperandsSignature {
       } else {
         llvm_unreachable("Unknown operand kind!");
       }
-      if (i + 1 != e)
-        OS << ", ";
     }
   }
 
@@ -620,10 +616,10 @@ void FastISelMap::printImmediatePredicates(raw_ostream &OS) {
     return;
 
   OS << "\n// FastEmit Immediate Predicate functions.\n";
-  for (ImmPredicateSet::iterator I = ImmediatePredicates.begin(),
-       E = ImmediatePredicates.end(); I != E; ++I) {
-    OS << "static bool " << I->getFnName() << "(int64_t Imm) {\n";
-    OS << I->getImmediatePredicateCode() << "\n}\n";
+  for (auto ImmediatePredicate : ImmediatePredicates) {
+    OS << "static bool " << ImmediatePredicate.getFnName()
+       << "(int64_t Imm) {\n";
+    OS << ImmediatePredicate.getImmediatePredicateCode() << "\n}\n";
   }
 
   OS << "\n\n";
@@ -677,7 +673,7 @@ void FastISelMap::emitInstructionCode(raw_ostream &OS,
       OS << ");\n";
     } else {
       OS << "extractsubreg(" << RetVTName
-         << ", Op0, Op0IsKill, " << Memo.SubRegNo << ");\n";
+         << ", Op0, " << Memo.SubRegNo << ");\n";
     }
 
     if (!PredicateCheck.empty()) {
@@ -695,29 +691,25 @@ void FastISelMap::emitInstructionCode(raw_ostream &OS,
 
 void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
   // Now emit code for all the patterns that we collected.
-  for (OperandsOpcodeTypeRetPredMap::const_iterator OI = SimplePatterns.begin(),
-       OE = SimplePatterns.end(); OI != OE; ++OI) {
-    const OperandsSignature &Operands = OI->first;
-    const OpcodeTypeRetPredMap &OTM = OI->second;
+  for (const auto &SimplePattern : SimplePatterns) {
+    const OperandsSignature &Operands = SimplePattern.first;
+    const OpcodeTypeRetPredMap &OTM = SimplePattern.second;
 
-    for (OpcodeTypeRetPredMap::const_iterator I = OTM.begin(), E = OTM.end();
-         I != E; ++I) {
-      const std::string &Opcode = I->first;
-      const TypeRetPredMap &TM = I->second;
+    for (const auto &I : OTM) {
+      const std::string &Opcode = I.first;
+      const TypeRetPredMap &TM = I.second;
 
       OS << "// FastEmit functions for " << Opcode << ".\n";
       OS << "\n";
 
       // Emit one function for each opcode,type pair.
-      for (TypeRetPredMap::const_iterator TI = TM.begin(), TE = TM.end();
-           TI != TE; ++TI) {
-        MVT::SimpleValueType VT = TI->first;
-        const RetPredMap &RM = TI->second;
+      for (const auto &TI : TM) {
+        MVT::SimpleValueType VT = TI.first;
+        const RetPredMap &RM = TI.second;
         if (RM.size() != 1) {
-          for (RetPredMap::const_iterator RI = RM.begin(), RE = RM.end();
-               RI != RE; ++RI) {
-            MVT::SimpleValueType RetVT = RI->first;
-            const PredMap &PM = RI->second;
+          for (const auto &RI : RM) {
+            MVT::SimpleValueType RetVT = RI.first;
+            const PredMap &PM = RI.second;
 
             OS << "unsigned fastEmit_" << getLegalCName(Opcode) << "_"
                << getLegalCName(std::string(getName(VT))) << "_"
@@ -739,9 +731,8 @@ void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
             OS << ", ";
           Operands.PrintParameters(OS);
           OS << ") {\nswitch (RetVT.SimpleTy) {\n";
-          for (RetPredMap::const_iterator RI = RM.begin(), RE = RM.end();
-               RI != RE; ++RI) {
-            MVT::SimpleValueType RetVT = RI->first;
+          for (const auto &RI : RM) {
+            MVT::SimpleValueType RetVT = RI.first;
             OS << "  case " << getName(RetVT) << ": return fastEmit_"
                << getLegalCName(Opcode) << "_"
                << getLegalCName(std::string(getName(VT))) << "_"
@@ -783,9 +774,8 @@ void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
       Operands.PrintParameters(OS);
       OS << ") {\n";
       OS << "  switch (VT.SimpleTy) {\n";
-      for (TypeRetPredMap::const_iterator TI = TM.begin(), TE = TM.end();
-           TI != TE; ++TI) {
-        MVT::SimpleValueType VT = TI->first;
+      for (const auto &TI : TM) {
+        MVT::SimpleValueType VT = TI.first;
         std::string TypeName = std::string(getName(VT));
         OS << "  case " << TypeName << ": return fastEmit_"
            << getLegalCName(Opcode) << "_" << getLegalCName(TypeName) << "_";
@@ -850,9 +840,8 @@ void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
     }
 
     OS << "  switch (Opcode) {\n";
-    for (OpcodeTypeRetPredMap::const_iterator I = OTM.begin(), E = OTM.end();
-         I != E; ++I) {
-      const std::string &Opcode = I->first;
+    for (const auto &I : OTM) {
+      const std::string &Opcode = I.first;
 
       OS << "  case " << Opcode << ": return fastEmit_"
          << getLegalCName(Opcode) << "_";
diff --git a/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp b/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
index 01b39df055ad..c5dd1e626696 100644
--- a/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
+++ b/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
@@ -973,7 +973,13 @@ emitDecoderFunction(formatted_raw_ostream &OS, DecoderSet &Decoders,
     << "Address, const void *Decoder, bool &DecodeComplete) {\n";
   Indentation += 2;
   OS.indent(Indentation) << "DecodeComplete = true;\n";
-  OS.indent(Indentation) << "InsnType tmp;\n";
+  // TODO: When InsnType is large, using uint64_t limits all fields to 64 bits
+  // It would be better for emitBinaryParser to use a 64-bit tmp whenever
+  // possible but fall back to an InsnType-sized tmp for truly large fields.
+  OS.indent(Indentation) << "using TmpType = "
+                            "std::conditional_t<std::is_integral<InsnType>::"
+                            "value, InsnType, uint64_t>;\n";
+  OS.indent(Indentation) << "TmpType tmp;\n";
   OS.indent(Indentation) << "switch (Idx) {\n";
   OS.indent(Indentation) << "default: llvm_unreachable(\"Invalid index!\");\n";
   unsigned Index = 0;
@@ -1107,18 +1113,24 @@ void FilterChooser::emitBinaryParser(raw_ostream &o, unsigned &Indentation,
                                      bool &OpHasCompleteDecoder) const {
   const std::string &Decoder = OpInfo.Decoder;
 
-  if (OpInfo.numFields() != 1 || OpInfo.InitValue != 0) {
+  bool UseInsertBits = OpInfo.numFields() != 1 || OpInfo.InitValue != 0;
+
+  if (UseInsertBits) {
     o.indent(Indentation) << "tmp = 0x";
     o.write_hex(OpInfo.InitValue);
     o << ";\n";
   }
 
   for (const EncodingField &EF : OpInfo) {
-    o.indent(Indentation) << "tmp ";
-    if (OpInfo.numFields() != 1 || OpInfo.InitValue != 0) o << '|';
-    o << "= fieldFromInstruction"
-      << "(insn, " << EF.Base << ", " << EF.Width << ')';
-    if (OpInfo.numFields() != 1 || EF.Offset != 0)
+    o.indent(Indentation);
+    if (UseInsertBits)
+      o << "insertBits(tmp, ";
+    else
+      o << "tmp = ";
+    o << "fieldFromInstruction(insn, " << EF.Base << ", " << EF.Width << ')';
+    if (UseInsertBits)
+      o << ", " << EF.Offset << ", " << EF.Width << ')';
+    else if (EF.Offset != 0)
       o << " << " << EF.Offset;
     o << ";\n";
   }
@@ -1210,14 +1222,9 @@ bool FilterChooser::emitPredicateMatch(raw_ostream &o, unsigned &Indentation,
     if (IsOr)
       o << "(";
 
-    bool First = true;
+    ListSeparator LS(IsOr ? " || " : " && ");
     for (auto *Arg : D->getArgs()) {
-      if (!First) {
-        if (IsOr)
-          o << " || ";
-        else
-          o << " && ";
-      }
+      o << LS;
       if (auto *NotArg = dyn_cast<DagInit>(Arg)) {
         if (NotArg->getOperator()->getAsString() != "not" ||
             NotArg->getNumArgs() != 1)
@@ -1230,8 +1237,6 @@ bool FilterChooser::emitPredicateMatch(raw_ostream &o, unsigned &Indentation,
         PrintFatalError(Pred->getLoc(), "Invalid AssemblerCondDag!");
       o << "Bits[" << Emitter->PredicateNamespace << "::" << Arg->getAsString()
         << "]";
-
-      First = false;
     }
 
     if (IsOr)
@@ -1250,7 +1255,7 @@ bool FilterChooser::doesOpcodeNeedPredicate(unsigned Opc) const {
     if (!Pred->getValue("AssemblerMatcherPredicate"))
       continue;
 
-    if (dyn_cast<DagInit>(Pred->getValue("AssemblerCondDag")->getValue()))
+    if (isa<DagInit>(Pred->getValue("AssemblerCondDag")->getValue()))
       return true;
   }
   return false;
@@ -1501,13 +1506,13 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
   if (AllowMixed && !Greedy) {
     assert(numInstructions == 3);
 
-    for (unsigned i = 0; i < Opcodes.size(); ++i) {
+    for (auto Opcode : Opcodes) {
       std::vector<unsigned> StartBits;
       std::vector<unsigned> EndBits;
       std::vector<uint64_t> FieldVals;
       insn_t Insn;
 
-      insnWithID(Insn, Opcodes[i].EncodingID);
+      insnWithID(Insn, Opcode.EncodingID);
 
       // Look for islands of undecoded bits of any instruction.
       if (getIslands(StartBits, EndBits, FieldVals, Insn) > 0) {
@@ -1769,13 +1774,13 @@ void FilterChooser::emitTableEntries(DecoderTableInfo &TableInfo) const {
 
   dumpStack(errs(), "\t\t");
 
-  for (unsigned i = 0; i < Opcodes.size(); ++i) {
+  for (auto Opcode : Opcodes) {
     errs() << '\t';
-    emitNameWithID(errs(), Opcodes[i].EncodingID);
+    emitNameWithID(errs(), Opcode.EncodingID);
     errs() << " ";
     dumpBits(
         errs(),
-        getBitsField(*AllInstructions[Opcodes[i].EncodingID].EncodingDef, "Inst"));
+        getBitsField(*AllInstructions[Opcode.EncodingID].EncodingDef, "Inst"));
     errs() << '\n';
   }
 }
@@ -2149,27 +2154,22 @@ static void emitFieldFromInstruction(formatted_raw_ostream &OS) {
   OS << "// Helper functions for extracting fields from encoded instructions.\n"
      << "// InsnType must either be integral or an APInt-like object that "
         "must:\n"
-     << "// * Have a static const max_size_in_bits equal to the number of bits "
-        "in the\n"
-     << "//   encoding.\n"
      << "// * be default-constructible and copy-constructible\n"
      << "// * be constructible from a uint64_t\n"
      << "// * be constructible from an APInt (this can be private)\n"
-     << "// * Support getBitsSet(loBit, hiBit)\n"
-     << "// * be convertible to uint64_t\n"
-     << "// * Support the ~, &, ==, !=, and |= operators with other objects of "
+     << "// * Support insertBits(bits, startBit, numBits)\n"
+     << "// * Support extractBitsAsZExtValue(numBits, startBit)\n"
+     << "// * be convertible to bool\n"
+     << "// * Support the ~, &, ==, and != operators with other objects of "
         "the same type\n"
-     << "// * Support shift (<<, >>) with signed and unsigned integers on the "
-        "RHS\n"
      << "// * Support put (<<) to raw_ostream&\n"
      << "template <typename InsnType>\n"
      << "#if defined(_MSC_VER) && !defined(__clang__)\n"
      << "__declspec(noinline)\n"
      << "#endif\n"
-     << "static InsnType fieldFromInstruction(InsnType insn, unsigned "
-        "startBit,\n"
-     << "                                     unsigned numBits, "
-        "std::true_type) {\n"
+     << "static std::enable_if_t<std::is_integral<InsnType>::value, InsnType>\n"
+     << "fieldFromInstruction(const InsnType &insn, unsigned startBit,\n"
+     << "                     unsigned numBits) {\n"
      << "  assert(startBit + numBits <= 64 && \"Cannot support >64-bit "
         "extractions!\");\n"
      << "  assert(startBit + numBits <= (sizeof(InsnType) * 8) &&\n"
@@ -2183,22 +2183,32 @@ static void emitFieldFromInstruction(formatted_raw_ostream &OS) {
      << "}\n"
      << "\n"
      << "template <typename InsnType>\n"
-     << "static InsnType fieldFromInstruction(InsnType insn, unsigned "
-        "startBit,\n"
-     << "                                     unsigned numBits, "
-        "std::false_type) {\n"
-     << "  assert(startBit + numBits <= InsnType::max_size_in_bits && "
-        "\"Instruction field out of bounds!\");\n"
-     << "  InsnType fieldMask = InsnType::getBitsSet(0, numBits);\n"
-     << "  return (insn >> startBit) & fieldMask;\n"
+     << "static std::enable_if_t<!std::is_integral<InsnType>::value, "
+        "uint64_t>\n"
+     << "fieldFromInstruction(const InsnType &insn, unsigned startBit,\n"
+     << "                     unsigned numBits) {\n"
+     << "  return insn.extractBitsAsZExtValue(numBits, startBit);\n"
+     << "}\n\n";
+}
+
+// emitInsertBits - Emit the templated helper function insertBits().
+static void emitInsertBits(formatted_raw_ostream &OS) {
+  OS << "// Helper function for inserting bits extracted from an encoded "
+        "instruction into\n"
+     << "// a field.\n"
+     << "template <typename InsnType>\n"
+     << "static std::enable_if_t<std::is_integral<InsnType>::value>\n"
+     << "insertBits(InsnType &field, InsnType bits, unsigned startBit, "
+        "unsigned numBits) {\n"
+     << "  assert(startBit + numBits <= sizeof field * 8);\n"
+     << "  field |= (InsnType)bits << startBit;\n"
      << "}\n"
      << "\n"
      << "template <typename InsnType>\n"
-     << "static InsnType fieldFromInstruction(InsnType insn, unsigned "
-        "startBit,\n"
-     << "                                     unsigned numBits) {\n"
-     << "  return fieldFromInstruction(insn, startBit, numBits, "
-        "std::is_integral<InsnType>());\n"
+     << "static std::enable_if_t<!std::is_integral<InsnType>::value>\n"
+     << "insertBits(InsnType &field, uint64_t bits, unsigned startBit, "
+        "unsigned numBits) {\n"
+     << "  field.insertBits(bits, startBit, numBits);\n"
      << "}\n\n";
 }
 
@@ -2401,6 +2411,7 @@ void FixedLenDecoderEmitter::run(raw_ostream &o) {
   OS << "namespace llvm {\n\n";
 
   emitFieldFromInstruction(OS);
+  emitInsertBits(OS);
 
   Target.reverseBitsForLittleEndianEncoding();
 
@@ -2418,7 +2429,7 @@ void FixedLenDecoderEmitter::run(raw_ostream &o) {
       if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
         const CodeGenHwModes &HWM = Target.getHwModes();
         EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-        for (auto &KV : EBM.Map)
+        for (auto &KV : EBM)
           HwModeNames.insert(HWM.getMode(KV.first).Name);
       }
     }
@@ -2436,7 +2447,7 @@ void FixedLenDecoderEmitter::run(raw_ostream &o) {
       if (DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
         const CodeGenHwModes &HWM = Target.getHwModes();
         EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-        for (auto &KV : EBM.Map) {
+        for (auto &KV : EBM) {
           NumberedEncodings.emplace_back(KV.second, NumberedInstruction,
                                          HWM.getMode(KV.first).Name);
           HwModeNames.insert(HWM.getMode(KV.first).Name);
diff --git a/llvm/utils/TableGen/GICombinerEmitter.cpp b/llvm/utils/TableGen/GICombinerEmitter.cpp
index ab00cff63998..c03cd371ef9d 100644
--- a/llvm/utils/TableGen/GICombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GICombinerEmitter.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/TableGen/Error.h"
@@ -241,13 +242,12 @@ public:
     bool Progressed = false;
     SmallSet<GIMatchDagEdge *, 20> EdgesToRemove;
     while (!EdgesRemaining.empty()) {
-      for (auto EI = EdgesRemaining.begin(), EE = EdgesRemaining.end();
-           EI != EE; ++EI) {
-        if (Visited.count((*EI)->getFromMI())) {
-          if (Roots.count((*EI)->getToMI()))
+      for (auto *EI : EdgesRemaining) {
+        if (Visited.count(EI->getFromMI())) {
+          if (Roots.count(EI->getToMI()))
             PrintError(TheDef.getLoc(), "One or more roots are unnecessary");
-          Visited.insert((*EI)->getToMI());
-          EdgesToRemove.insert(*EI);
+          Visited.insert(EI->getToMI());
+          EdgesToRemove.insert(EI);
           Progressed = true;
         }
       }
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index 0a6985a3eac2..693073672fc1 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -90,7 +90,7 @@ std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) {
 }
 
 /// Get the opcode used to check this predicate.
-std::string getMatchOpcodeForPredicate(const TreePredicateFn &Predicate) {
+std::string getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) {
   return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate";
 }
 
@@ -118,7 +118,9 @@ public:
       return;
     }
     if (Ty.isVector()) {
-      OS << "GILLT_v" << Ty.getNumElements() << "s" << Ty.getScalarSizeInBits();
+      OS << (Ty.isScalable() ? "GILLT_nxv" : "GILLT_v")
+         << Ty.getElementCount().getKnownMinValue() << "s"
+         << Ty.getScalarSizeInBits();
       return;
     }
     if (Ty.isPointer()) {
@@ -136,7 +138,10 @@ public:
       return;
     }
     if (Ty.isVector()) {
-      OS << "LLT::vector(" << Ty.getNumElements() << ", "
+      OS << "LLT::vector("
+         << (Ty.isScalable() ? "ElementCount::getScalable("
+                             : "ElementCount::getFixed(")
+         << Ty.getElementCount().getKnownMinValue() << "), "
          << Ty.getScalarSizeInBits() << ")";
       return;
     }
@@ -169,10 +174,21 @@ public:
     if (Ty.isPointer() && Ty.getAddressSpace() != Other.Ty.getAddressSpace())
       return Ty.getAddressSpace() < Other.Ty.getAddressSpace();
 
-    if (Ty.isVector() && Ty.getNumElements() != Other.Ty.getNumElements())
-      return Ty.getNumElements() < Other.Ty.getNumElements();
+    if (Ty.isVector() && Ty.getElementCount() != Other.Ty.getElementCount())
+      return std::make_tuple(Ty.isScalable(),
+                             Ty.getElementCount().getKnownMinValue()) <
+             std::make_tuple(Other.Ty.isScalable(),
+                             Other.Ty.getElementCount().getKnownMinValue());
 
-    return Ty.getSizeInBits() < Other.Ty.getSizeInBits();
+    assert((!Ty.isVector() || Ty.isScalable() == Other.Ty.isScalable()) &&
+           "Unexpected mismatch of scalable property");
+    return Ty.isVector()
+               ? std::make_tuple(Ty.isScalable(),
+                                 Ty.getSizeInBits().getKnownMinSize()) <
+                     std::make_tuple(Other.Ty.isScalable(),
+                                     Other.Ty.getSizeInBits().getKnownMinSize())
+               : Ty.getSizeInBits().getFixedSize() <
+                     Other.Ty.getSizeInBits().getFixedSize();
   }
 
   bool operator==(const LLTCodeGen &B) const { return Ty == B.Ty; }
@@ -187,12 +203,9 @@ class InstructionMatcher;
 static Optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT) {
   MVT VT(SVT);
 
-  if (VT.isScalableVector())
-    return None;
-
-  if (VT.isFixedLengthVector() && VT.getVectorNumElements() != 1)
+  if (VT.isVector() && !VT.getVectorElementCount().isScalar())
     return LLTCodeGen(
-        LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits()));
+        LLT::vector(VT.getVectorElementCount(), VT.getScalarSizeInBits()));
 
   if (VT.isInteger() || VT.isFloatingPoint())
     return LLTCodeGen(LLT::scalar(VT.getSizeInBits()));
@@ -1562,6 +1575,40 @@ public:
   }
 };
 
+/// Generates code to check that this operand is an immediate whose value meets
+/// an immediate predicate.
+class OperandImmPredicateMatcher : public OperandPredicateMatcher {
+protected:
+  TreePredicateFn Predicate;
+
+public:
+  OperandImmPredicateMatcher(unsigned InsnVarID, unsigned OpIdx,
+                             const TreePredicateFn &Predicate)
+      : OperandPredicateMatcher(IPM_ImmPredicate, InsnVarID, OpIdx),
+        Predicate(Predicate) {}
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return OperandPredicateMatcher::isIdentical(B) &&
+           Predicate.getOrigPatFragRecord() ==
+               cast<OperandImmPredicateMatcher>(&B)
+                   ->Predicate.getOrigPatFragRecord();
+  }
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_ImmPredicate;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override {
+    Table << MatchTable::Opcode("GIM_CheckImmOperandPredicate")
+          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+          << MatchTable::Comment("MO") << MatchTable::IntValue(OpIdx)
+          << MatchTable::Comment("Predicate")
+          << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
+          << MatchTable::LineBreak;
+  }
+};
+
 /// Generates code to check that a set of predicates match for a particular
 /// operand.
 class OperandMatcher : public PredicateListMatcher<OperandPredicateMatcher> {
@@ -1924,7 +1971,7 @@ public:
 
   void emitPredicateOpcodes(MatchTable &Table,
                             RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode(getMatchOpcodeForPredicate(Predicate))
+    Table << MatchTable::Opcode(getMatchOpcodeForImmPredicate(Predicate))
           << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
           << MatchTable::Comment("Predicate")
           << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
@@ -3536,7 +3583,7 @@ private:
   const CodeGenInstruction *getEquivNode(Record &Equiv,
                                          const TreePatternNode *N) const;
 
-  Error importRulePredicates(RuleMatcher &M, ArrayRef<Predicate> Predicates);
+  Error importRulePredicates(RuleMatcher &M, ArrayRef<Record *> Predicates);
   Expected<InstructionMatcher &>
   createAndImportSelDAGMatcher(RuleMatcher &Rule,
                                InstructionMatcher &InsnMatcher,
@@ -3623,6 +3670,10 @@ private:
   Optional<const CodeGenRegisterClass *>
   inferRegClassFromPattern(TreePatternNode *N);
 
+  /// Return the size of the MemoryVT in this predicate, if possible.
+  Optional<unsigned>
+  getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate);
+
   // Add builtin predicates.
   Expected<InstructionMatcher &>
   addBuiltinPredicates(const Record *SrcGIEquivOrNull,
@@ -3723,19 +3774,30 @@ GlobalISelEmitter::GlobalISelEmitter(RecordKeeper &RK)
 
 //===- Emitter ------------------------------------------------------------===//
 
-Error
-GlobalISelEmitter::importRulePredicates(RuleMatcher &M,
-                                        ArrayRef<Predicate> Predicates) {
-  for (const Predicate &P : Predicates) {
-    if (!P.Def || P.getCondString().empty())
+Error GlobalISelEmitter::importRulePredicates(RuleMatcher &M,
+                                              ArrayRef<Record *> Predicates) {
+  for (Record *Pred : Predicates) {
+    if (Pred->getValueAsString("CondString").empty())
       continue;
-    declareSubtargetFeature(P.Def);
-    M.addRequiredFeature(P.Def);
+    declareSubtargetFeature(Pred);
+    M.addRequiredFeature(Pred);
   }
 
   return Error::success();
 }
 
+Optional<unsigned> GlobalISelEmitter::getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate) {
+  Optional<LLTCodeGen> MemTyOrNone =
+      MVTToLLT(getValueType(Predicate.getMemoryVT()));
+
+  if (!MemTyOrNone)
+    return None;
+
+  // Align so unusual types like i1 don't get rounded down.
+  return llvm::alignTo(
+      static_cast<unsigned>(MemTyOrNone->get().getSizeInBits()), 8);
+}
+
 Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
     const Record *SrcGIEquivOrNull, const TreePredicateFn &Predicate,
     InstructionMatcher &InsnMatcher, bool &HasAddedMatcher) {
@@ -3775,9 +3837,18 @@ Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
 
   if (Predicate.isStore()) {
     if (Predicate.isTruncStore()) {
-      // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size.
-      InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
-          0, MemoryVsLLTSizePredicateMatcher::LessThan, 0);
+      if (Predicate.getMemoryVT() != nullptr) {
+        // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size.
+        auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
+        if (!MemSizeInBits)
+          return failedImport("MemVT could not be converted to LLT");
+
+        InsnMatcher.addPredicate<MemorySizePredicateMatcher>(0, *MemSizeInBits /
+                                                                    8);
+      } else {
+        InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
+            0, MemoryVsLLTSizePredicateMatcher::LessThan, 0);
+      }
       return InsnMatcher;
     }
     if (Predicate.isNonTruncStore()) {
@@ -3804,19 +3875,12 @@ Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
 
   if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
     if (Predicate.getMemoryVT() != nullptr) {
-      Optional<LLTCodeGen> MemTyOrNone =
-          MVTToLLT(getValueType(Predicate.getMemoryVT()));
-
-      if (!MemTyOrNone)
+      auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
+      if (!MemSizeInBits)
         return failedImport("MemVT could not be converted to LLT");
 
-      // MMO's work in bytes so we must take care of unusual types like i1
-      // don't round down.
-      unsigned MemSizeInBits =
-          llvm::alignTo(MemTyOrNone->get().getSizeInBits(), 8);
-
       InsnMatcher.addPredicate<MemorySizePredicateMatcher>(0,
-                                                           MemSizeInBits / 8);
+                                                           *MemSizeInBits / 8);
       return InsnMatcher;
     }
   }
@@ -4153,6 +4217,17 @@ Error GlobalISelEmitter::importChildMatcher(
       }
       if (SrcChild->getOperator()->getName() == "timm") {
         OM.addPredicate<ImmOperandMatcher>();
+
+        // Add predicates, if any
+        for (const TreePredicateCall &Call : SrcChild->getPredicateCalls()) {
+          const TreePredicateFn &Predicate = Call.Fn;
+
+          // Only handle immediate patterns for now
+          if (Predicate.isImmediatePattern()) {
+            OM.addPredicate<OperandImmPredicateMatcher>(Predicate);
+          }
+        }
+
         return Error::success();
       }
     }
@@ -4464,7 +4539,6 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer(
     return failedImport(
         "Dst pattern child def is an unsupported tablegen class");
   }
-
   return failedImport("Dst pattern child is an unsupported kind");
 }
 
@@ -5043,7 +5117,9 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
                                   "  =>  " +
                                   llvm::to_string(*P.getDstPattern()));
 
-  if (auto Error = importRulePredicates(M, P.getPredicates()))
+  SmallVector<Record *, 4> Predicates;
+  P.getPredicateRecords(Predicates);
+  if (auto Error = importRulePredicates(M, Predicates))
     return std::move(Error);
 
   // Next, analyze the pattern operators.
@@ -5351,7 +5427,7 @@ void GlobalISelEmitter::emitCxxPredicateFns(
     StringRef AdditionalDeclarations,
     std::function<bool(const Record *R)> Filter) {
   std::vector<const Record *> MatchedRecords;
-  const auto &Defs = RK.getAllDerivedDefinitions("PatFrag");
+  const auto &Defs = RK.getAllDerivedDefinitions("PatFrags");
   std::copy_if(Defs.begin(), Defs.end(), std::back_inserter(MatchedRecords),
                [&](Record *Record) {
                  return !Record->getValueAsString(CodeFieldName).empty() &&
@@ -5595,9 +5671,17 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
       RK.getAllDerivedDefinitions("GIComplexOperandMatcher");
   llvm::sort(ComplexPredicates, orderByName);
 
-  std::vector<Record *> CustomRendererFns =
-      RK.getAllDerivedDefinitions("GICustomOperandRenderer");
-  llvm::sort(CustomRendererFns, orderByName);
+  std::vector<StringRef> CustomRendererFns;
+  transform(RK.getAllDerivedDefinitions("GICustomOperandRenderer"),
+            std::back_inserter(CustomRendererFns), [](const auto &Record) {
+              return Record->getValueAsString("RendererFn");
+            });
+  // Sort and remove duplicates to get a list of unique renderer functions, in
+  // case some were mentioned more than once.
+  llvm::sort(CustomRendererFns);
+  CustomRendererFns.erase(
+      std::unique(CustomRendererFns.begin(), CustomRendererFns.end()),
+      CustomRendererFns.end());
 
   unsigned MaxTemporaries = 0;
   for (const auto &Rule : Rules)
@@ -5675,13 +5759,6 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
     "(const " << Target.getName() << "Subtarget *)&MF.getSubtarget(), &MF);\n"
     "}\n";
 
-  if (Target.getName() == "X86" || Target.getName() == "AArch64") {
-    // TODO: Implement PGSO.
-    OS << "static bool shouldOptForSize(const MachineFunction *MF) {\n";
-    OS << "    return MF->getFunction().hasOptSize();\n";
-    OS << "}\n\n";
-  }
-
   SubtargetFeatureInfo::emitComputeAvailableFeatures(
       Target.getName(), "InstructionSelector",
       "computeAvailableFunctionFeatures", FunctionFeatures, OS,
@@ -5791,17 +5868,15 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
   OS << "// Custom renderers.\n"
      << "enum {\n"
      << "  GICR_Invalid,\n";
-  for (const auto &Record : CustomRendererFns)
-    OS << "  GICR_" << Record->getValueAsString("RendererFn") << ",\n";
+  for (const auto &Fn : CustomRendererFns)
+    OS << "  GICR_" << Fn << ",\n";
   OS << "};\n";
 
   OS << Target.getName() << "InstructionSelector::CustomRendererFn\n"
      << Target.getName() << "InstructionSelector::CustomRenderers[] = {\n"
      << "  nullptr, // GICR_Invalid\n";
-  for (const auto &Record : CustomRendererFns)
-    OS << "  &" << Target.getName()
-       << "InstructionSelector::" << Record->getValueAsString("RendererFn")
-       << ", // " << Record->getName() << "\n";
+  for (const auto &Fn : CustomRendererFns)
+    OS << "  &" << Target.getName() << "InstructionSelector::" << Fn << ",\n";
   OS << "};\n\n";
 
   llvm::stable_sort(Rules, [&](const RuleMatcher &A, const RuleMatcher &B) {
diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/InfoByHwMode.cpp
index 7cd1b0f08132..3d236b828032 100644
--- a/llvm/utils/TableGen/InfoByHwMode.cpp
+++ b/llvm/utils/TableGen/InfoByHwMode.cpp
@@ -91,13 +91,10 @@ void ValueTypeByHwMode::writeToStream(raw_ostream &OS) const {
   llvm::sort(Pairs, deref<std::less<PairType>>());
 
   OS << '{';
-  for (unsigned i = 0, e = Pairs.size(); i != e; ++i) {
-    const PairType *P = Pairs[i];
-    OS << '(' << getModeName(P->first)
-       << ':' << getMVTName(P->second).str() << ')';
-    if (i != e-1)
-      OS << ',';
-  }
+  ListSeparator LS(",");
+  for (const PairType *P : Pairs)
+    OS << LS << '(' << getModeName(P->first) << ':'
+       << getMVTName(P->second).str() << ')';
   OS << '}';
 }
 
@@ -183,12 +180,9 @@ void RegSizeInfoByHwMode::writeToStream(raw_ostream &OS) const {
   llvm::sort(Pairs, deref<std::less<PairType>>());
 
   OS << '{';
-  for (unsigned i = 0, e = Pairs.size(); i != e; ++i) {
-    const PairType *P = Pairs[i];
-    OS << '(' << getModeName(P->first) << ':' << P->second << ')';
-    if (i != e-1)
-      OS << ',';
-  }
+  ListSeparator LS(",");
+  for (const PairType *P : Pairs)
+    OS << LS << '(' << getModeName(P->first) << ':' << P->second << ')';
   OS << '}';
 }
 
diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/InfoByHwMode.h
index d92e5901a7f3..c97add687ca2 100644
--- a/llvm/utils/TableGen/InfoByHwMode.h
+++ b/llvm/utils/TableGen/InfoByHwMode.h
@@ -15,10 +15,10 @@
 #define LLVM_UTILS_TABLEGEN_INFOBYHWMODE_H
 
 #include "CodeGenHwModes.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/MachineValueType.h"
 
 #include <map>
-#include <set>
 #include <string>
 #include <vector>
 
@@ -37,10 +37,10 @@ enum : unsigned {
 };
 
 template <typename InfoT>
-std::vector<unsigned> union_modes(const InfoByHwMode<InfoT> &A,
-                                  const InfoByHwMode<InfoT> &B) {
-  std::vector<unsigned> V;
-  std::set<unsigned> U;
+void union_modes(const InfoByHwMode<InfoT> &A,
+                 const InfoByHwMode<InfoT> &B,
+                 SmallVectorImpl<unsigned> &Modes) {
+  SmallSet<unsigned, 4> U;
   for (const auto &P : A)
     U.insert(P.first);
   for (const auto &P : B)
@@ -49,12 +49,11 @@ std::vector<unsigned> union_modes(const InfoByHwMode<InfoT> &A,
   bool HasDefault = false;
   for (unsigned M : U)
     if (M != DefaultMode)
-      V.push_back(M);
+      Modes.push_back(M);
     else
       HasDefault = true;
   if (HasDefault)
-    V.push_back(DefaultMode);
-  return V;
+    Modes.push_back(DefaultMode);
 }
 
 template <typename InfoT>
@@ -114,6 +113,7 @@ struct InfoByHwMode {
     Map.insert(std::make_pair(DefaultMode, I));
   }
 
+protected:
   MapType Map;
 };
 
@@ -178,6 +178,10 @@ struct RegSizeInfoByHwMode : public InfoByHwMode<RegSizeInfo> {
   bool hasStricterSpillThan(const RegSizeInfoByHwMode &I) const;
 
   void writeToStream(raw_ostream &OS) const;
+
+  void insertRegSizeForMode(unsigned Mode, RegSizeInfo Info) {
+    Map.insert(std::make_pair(Mode, Info));
+  }
 };
 
 raw_ostream &operator<<(raw_ostream &OS, const ValueTypeByHwMode &T);
diff --git a/llvm/utils/TableGen/InstrDocsEmitter.cpp b/llvm/utils/TableGen/InstrDocsEmitter.cpp
index 66744bf9ecef..bc391227edd1 100644
--- a/llvm/utils/TableGen/InstrDocsEmitter.cpp
+++ b/llvm/utils/TableGen/InstrDocsEmitter.cpp
@@ -141,13 +141,9 @@ void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS) {
     FLAG(isAuthenticated)
     if (!FlagStrings.empty()) {
       OS << "Flags: ";
-      bool IsFirst = true;
-      for (auto FlagString : FlagStrings) {
-        if (!IsFirst)
-          OS << ", ";
-        OS << "``" << FlagString << "``";
-        IsFirst = false;
-      }
+      ListSeparator LS;
+      for (auto FlagString : FlagStrings)
+        OS << LS << "``" << FlagString << "``";
       OS << "\n\n";
     }
 
@@ -192,26 +188,18 @@ void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS) {
     // Implicit definitions.
     if (!II->ImplicitDefs.empty()) {
       OS << "Implicit defs: ";
-      bool IsFirst = true;
-      for (Record *Def : II->ImplicitDefs) {
-        if (!IsFirst)
-          OS << ", ";
-        OS << "``" << Def->getName() << "``";
-        IsFirst = false;
-      }
+      ListSeparator LS;
+      for (Record *Def : II->ImplicitDefs)
+        OS << LS << "``" << Def->getName() << "``";
       OS << "\n\n";
     }
 
     // Implicit uses.
     if (!II->ImplicitUses.empty()) {
       OS << "Implicit uses: ";
-      bool IsFirst = true;
-      for (Record *Use : II->ImplicitUses) {
-        if (!IsFirst)
-          OS << ", ";
-        OS << "``" << Use->getName() << "``";
-        IsFirst = false;
-      }
+      ListSeparator LS;
+      for (Record *Use : II->ImplicitUses)
+        OS << LS << "``" << Use->getName() << "``";
       OS << "\n\n";
     }
 
@@ -220,13 +208,9 @@ void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS) {
         II->TheDef->getValueAsListOfDefs("Predicates");
     if (!Predicates.empty()) {
       OS << "Predicates: ";
-      bool IsFirst = true;
-      for (Record *P : Predicates) {
-        if (!IsFirst)
-          OS << ", ";
-        OS << "``" << P->getName() << "``";
-        IsFirst = false;
-      }
+      ListSeparator LS;
+      for (Record *P : Predicates)
+        OS << LS << "``" << P->getName() << "``";
       OS << "\n\n";
     }
   }
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index 9ff385faec56..aee887a906e5 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -19,6 +19,7 @@
 #include "SequenceToOffsetTable.h"
 #include "TableGenBackends.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/raw_ostream.h"
@@ -27,6 +28,7 @@
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cassert>
 #include <cstdint>
+#include <iterator>
 #include <map>
 #include <string>
 #include <utility>
@@ -87,6 +89,13 @@ private:
   void emitOperandNameMappings(raw_ostream &OS, const CodeGenTarget &Target,
             ArrayRef<const CodeGenInstruction*> NumberedInstructions);
 
+  void emitLogicalOperandSizeMappings(
+      raw_ostream &OS, StringRef Namespace,
+      ArrayRef<const CodeGenInstruction *> NumberedInstructions);
+  void emitLogicalOperandTypeMappings(
+      raw_ostream &OS, StringRef Namespace,
+      ArrayRef<const CodeGenInstruction *> NumberedInstructions);
+
   // Operand information.
   void EmitOperandInfo(raw_ostream &OS, OperandInfoMapTy &OperandInfoIDs);
   std::vector<std::string> GetOperandInfo(const CodeGenInstruction &Inst);
@@ -442,6 +451,182 @@ void InstrInfoEmitter::emitOperandTypeMappings(
   OS << "#endif // GET_INSTRINFO_OPERAND_TYPE\n\n";
 }
 
+void InstrInfoEmitter::emitLogicalOperandSizeMappings(
+    raw_ostream &OS, StringRef Namespace,
+    ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
+  std::map<std::vector<unsigned>, unsigned> LogicalOpSizeMap;
+
+  std::map<unsigned, std::vector<std::string>> InstMap;
+
+  size_t LogicalOpListSize = 0U;
+  std::vector<unsigned> LogicalOpList;
+  for (const auto *Inst : NumberedInstructions) {
+    if (!Inst->TheDef->getValueAsBit("UseLogicalOperandMappings"))
+      continue;
+
+    LogicalOpList.clear();
+    llvm::transform(Inst->Operands, std::back_inserter(LogicalOpList),
+                    [](const CGIOperandList::OperandInfo &Op) -> unsigned {
+                      auto *MIOI = Op.MIOperandInfo;
+                      if (!MIOI || MIOI->getNumArgs() == 0)
+                        return 1;
+                      return MIOI->getNumArgs();
+                    });
+    LogicalOpListSize = std::max(LogicalOpList.size(), LogicalOpListSize);
+
+    auto I =
+        LogicalOpSizeMap.insert({LogicalOpList, LogicalOpSizeMap.size()}).first;
+    InstMap[I->second].push_back(
+        (Namespace + "::" + Inst->TheDef->getName()).str());
+  }
+
+  OS << "#ifdef GET_INSTRINFO_LOGICAL_OPERAND_SIZE_MAP\n";
+  OS << "#undef GET_INSTRINFO_LOGICAL_OPERAND_SIZE_MAP\n";
+  OS << "namespace llvm {\n";
+  OS << "namespace " << Namespace << " {\n";
+  OS << "LLVM_READONLY static unsigned\n";
+  OS << "getLogicalOperandSize(uint16_t Opcode, uint16_t LogicalOpIdx) {\n";
+  if (!InstMap.empty()) {
+    std::vector<const std::vector<unsigned> *> LogicalOpSizeList(
+        LogicalOpSizeMap.size());
+    for (auto &P : LogicalOpSizeMap) {
+      LogicalOpSizeList[P.second] = &P.first;
+    }
+    OS << "  static const unsigned SizeMap[][" << LogicalOpListSize
+       << "] = {\n";
+    for (auto &R : LogicalOpSizeList) {
+      const auto &Row = *R;
+      OS << "   {";
+      int i;
+      for (i = 0; i < static_cast<int>(Row.size()); ++i) {
+        OS << Row[i] << ", ";
+      }
+      for (; i < static_cast<int>(LogicalOpListSize); ++i) {
+        OS << "0, ";
+      }
+      OS << "}, ";
+      OS << "\n";
+    }
+    OS << "  };\n";
+
+    OS << "  switch (Opcode) {\n";
+    OS << "  default: return LogicalOpIdx;\n";
+    for (auto &P : InstMap) {
+      auto OpMapIdx = P.first;
+      const auto &Insts = P.second;
+      for (const auto &Inst : Insts) {
+        OS << "  case " << Inst << ":\n";
+      }
+      OS << "    return SizeMap[" << OpMapIdx << "][LogicalOpIdx];\n";
+    }
+    OS << "  }\n";
+  } else {
+    OS << "  return LogicalOpIdx;\n";
+  }
+  OS << "}\n";
+
+  OS << "LLVM_READONLY static inline unsigned\n";
+  OS << "getLogicalOperandIdx(uint16_t Opcode, uint16_t LogicalOpIdx) {\n";
+  OS << "  auto S = 0U;\n";
+  OS << "  for (auto i = 0U; i < LogicalOpIdx; ++i)\n";
+  OS << "    S += getLogicalOperandSize(Opcode, i);\n";
+  OS << "  return S;\n";
+  OS << "}\n";
+
+  OS << "} // end namespace " << Namespace << "\n";
+  OS << "} // end namespace llvm\n";
+  OS << "#endif // GET_INSTRINFO_LOGICAL_OPERAND_SIZE_MAP\n\n";
+}
+
+void InstrInfoEmitter::emitLogicalOperandTypeMappings(
+    raw_ostream &OS, StringRef Namespace,
+    ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
+  std::map<std::vector<std::string>, unsigned> LogicalOpTypeMap;
+
+  std::map<unsigned, std::vector<std::string>> InstMap;
+
+  size_t OpTypeListSize = 0U;
+  std::vector<std::string> LogicalOpTypeList;
+  for (const auto *Inst : NumberedInstructions) {
+    if (!Inst->TheDef->getValueAsBit("UseLogicalOperandMappings"))
+      continue;
+
+    LogicalOpTypeList.clear();
+    for (const auto &Op : Inst->Operands) {
+      auto *OpR = Op.Rec;
+      if ((OpR->isSubClassOf("Operand") ||
+           OpR->isSubClassOf("RegisterOperand") ||
+           OpR->isSubClassOf("RegisterClass")) &&
+          !OpR->isAnonymous()) {
+        LogicalOpTypeList.push_back(
+            (Namespace + "::OpTypes::" + Op.Rec->getName()).str());
+      } else {
+        LogicalOpTypeList.push_back("-1");
+      }
+    }
+    OpTypeListSize = std::max(LogicalOpTypeList.size(), OpTypeListSize);
+
+    auto I =
+        LogicalOpTypeMap.insert({LogicalOpTypeList, LogicalOpTypeMap.size()})
+            .first;
+    InstMap[I->second].push_back(
+        (Namespace + "::" + Inst->TheDef->getName()).str());
+  }
+
+  OS << "#ifdef GET_INSTRINFO_LOGICAL_OPERAND_TYPE_MAP\n";
+  OS << "#undef GET_INSTRINFO_LOGICAL_OPERAND_TYPE_MAP\n";
+  OS << "namespace llvm {\n";
+  OS << "namespace " << Namespace << " {\n";
+  OS << "LLVM_READONLY static int\n";
+  OS << "getLogicalOperandType(uint16_t Opcode, uint16_t LogicalOpIdx) {\n";
+  if (!InstMap.empty()) {
+    std::vector<const std::vector<std::string> *> LogicalOpTypeList(
+        LogicalOpTypeMap.size());
+    for (auto &P : LogicalOpTypeMap) {
+      LogicalOpTypeList[P.second] = &P.first;
+    }
+    OS << "  static const int TypeMap[][" << OpTypeListSize << "] = {\n";
+    for (int r = 0, rs = LogicalOpTypeList.size(); r < rs; ++r) {
+      const auto &Row = *LogicalOpTypeList[r];
+      OS << "   {";
+      int i, s = Row.size();
+      for (i = 0; i < s; ++i) {
+        if (i > 0)
+          OS << ", ";
+        OS << Row[i];
+      }
+      for (; i < static_cast<int>(OpTypeListSize); ++i) {
+        if (i > 0)
+          OS << ", ";
+        OS << "-1";
+      }
+      OS << "}";
+      if (r != rs - 1)
+        OS << ",";
+      OS << "\n";
+    }
+    OS << "  };\n";
+
+    OS << "  switch (Opcode) {\n";
+    OS << "  default: return -1;\n";
+    for (auto &P : InstMap) {
+      auto OpMapIdx = P.first;
+      const auto &Insts = P.second;
+      for (const auto &Inst : Insts) {
+        OS << "  case " << Inst << ":\n";
+      }
+      OS << "    return TypeMap[" << OpMapIdx << "][LogicalOpIdx];\n";
+    }
+    OS << "  }\n";
+  } else {
+    OS << "  return -1;\n";
+  }
+  OS << "}\n";
+  OS << "} // end namespace " << Namespace << "\n";
+  OS << "} // end namespace llvm\n";
+  OS << "#endif // GET_INSTRINFO_LOGICAL_OPERAND_TYPE_MAP\n\n";
+}
+
 void InstrInfoEmitter::emitMCIIHelperMethods(raw_ostream &OS,
                                              StringRef TargetName) {
   RecVec TIIPredicates = Records.getAllDerivedDefinitions("TIIPredicate");
@@ -726,6 +911,12 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   Records.startTimer("Emit operand type mappings");
   emitOperandTypeMappings(OS, Target, NumberedInstructions);
 
+  Records.startTimer("Emit logical operand size mappings");
+  emitLogicalOperandSizeMappings(OS, TargetName, NumberedInstructions);
+
+  Records.startTimer("Emit logical operand type mappings");
+  emitLogicalOperandTypeMappings(OS, TargetName, NumberedInstructions);
+
   Records.startTimer("Emit helper methods");
   emitMCIIHelperMethods(OS, TargetName);
 }
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index 4be0d90a45d2..3d1d258e342e 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -378,7 +378,7 @@ static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes,
     MVT VVT = VT;
     if (VVT.isScalableVector())
       Sig.push_back(IIT_SCALABLE_VEC);
-    switch (VVT.getVectorNumElements()) {
+    switch (VVT.getVectorMinNumElements()) {
     default: PrintFatalError("unhandled vector type width in intrinsic!");
     case 1: Sig.push_back(IIT_V1); break;
     case 2: Sig.push_back(IIT_V2); break;
@@ -584,6 +584,9 @@ struct AttributeComparator {
     if (L->isNoDuplicate != R->isNoDuplicate)
       return R->isNoDuplicate;
 
+    if (L->isNoMerge != R->isNoMerge)
+      return R->isNoMerge;
+
     if (L->isNoReturn != R->isNoReturn)
       return R->isNoReturn;
 
@@ -638,13 +641,13 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
       std::max(maxArgAttrs, unsigned(intrinsic.ArgumentAttributes.size()));
     unsigned &N = UniqAttributes[&intrinsic];
     if (N) continue;
-    assert(AttrNum < 256 && "Too many unique attributes for table!");
     N = ++AttrNum;
+    assert(N < 65536 && "Too many unique attributes for table!");
   }
 
   // Emit an array of AttributeList.  Most intrinsics will have at least one
   // entry, for the function itself (index ~1), which is usually nounwind.
-  OS << "  static const uint8_t IntrinsicsToAttributesMap[] = {\n";
+  OS << "  static const uint16_t IntrinsicsToAttributesMap[] = {\n";
 
   for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
     const CodeGenIntrinsic &intrinsic = Ints[i];
@@ -659,240 +662,165 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
   OS << "  if (id != 0) {\n";
   OS << "    switch(IntrinsicsToAttributesMap[id - 1]) {\n";
   OS << "    default: llvm_unreachable(\"Invalid attribute number\");\n";
-  for (UniqAttrMapTy::const_iterator I = UniqAttributes.begin(),
-       E = UniqAttributes.end(); I != E; ++I) {
-    OS << "    case " << I->second << ": {\n";
+  for (auto UniqAttribute : UniqAttributes) {
+    OS << "    case " << UniqAttribute.second << ": {\n";
 
-    const CodeGenIntrinsic &intrinsic = *(I->first);
+    const CodeGenIntrinsic &Intrinsic = *(UniqAttribute.first);
 
     // Keep track of the number of attributes we're writing out.
     unsigned numAttrs = 0;
 
     // The argument attributes are alreadys sorted by argument index.
-    unsigned ai = 0, ae = intrinsic.ArgumentAttributes.size();
-    if (ae) {
-      while (ai != ae) {
-        unsigned attrIdx = intrinsic.ArgumentAttributes[ai].Index;
+    unsigned Ai = 0, Ae = Intrinsic.ArgumentAttributes.size();
+    if (Ae) {
+      while (Ai != Ae) {
+        unsigned AttrIdx = Intrinsic.ArgumentAttributes[Ai].Index;
 
-        OS << "      const Attribute::AttrKind AttrParam" << attrIdx << "[]= {";
-        bool addComma = false;
+        OS << "      const Attribute::AttrKind AttrParam" << AttrIdx << "[]= {";
+        ListSeparator LS(",");
 
         bool AllValuesAreZero = true;
         SmallVector<uint64_t, 8> Values;
         do {
-          switch (intrinsic.ArgumentAttributes[ai].Kind) {
+          switch (Intrinsic.ArgumentAttributes[Ai].Kind) {
           case CodeGenIntrinsic::NoCapture:
-            if (addComma)
-              OS << ",";
-            OS << "Attribute::NoCapture";
-            addComma = true;
+            OS << LS << "Attribute::NoCapture";
             break;
           case CodeGenIntrinsic::NoAlias:
-            if (addComma)
-              OS << ",";
-            OS << "Attribute::NoAlias";
-            addComma = true;
+            OS << LS << "Attribute::NoAlias";
             break;
           case CodeGenIntrinsic::NoUndef:
-            if (addComma)
-              OS << ",";
-            OS << "Attribute::NoUndef";
-            addComma = true;
+            OS << LS << "Attribute::NoUndef";
             break;
           case CodeGenIntrinsic::Returned:
-            if (addComma)
-              OS << ",";
-            OS << "Attribute::Returned";
-            addComma = true;
+            OS << LS << "Attribute::Returned";
             break;
           case CodeGenIntrinsic::ReadOnly:
-            if (addComma)
-              OS << ",";
-            OS << "Attribute::ReadOnly";
-            addComma = true;
+            OS << LS << "Attribute::ReadOnly";
             break;
           case CodeGenIntrinsic::WriteOnly:
-            if (addComma)
-              OS << ",";
-            OS << "Attribute::WriteOnly";
-            addComma = true;
+            OS << LS << "Attribute::WriteOnly";
             break;
           case CodeGenIntrinsic::ReadNone:
-            if (addComma)
-              OS << ",";
-            OS << "Attribute::ReadNone";
-            addComma = true;
+            OS << LS << "Attribute::ReadNone";
             break;
           case CodeGenIntrinsic::ImmArg:
-            if (addComma)
-              OS << ',';
-            OS << "Attribute::ImmArg";
-            addComma = true;
+            OS << LS << "Attribute::ImmArg";
             break;
           case CodeGenIntrinsic::Alignment:
-            if (addComma)
-              OS << ',';
-            OS << "Attribute::Alignment";
-            addComma = true;
+            OS << LS << "Attribute::Alignment";
             break;
           }
-          uint64_t V = intrinsic.ArgumentAttributes[ai].Value;
+          uint64_t V = Intrinsic.ArgumentAttributes[Ai].Value;
           Values.push_back(V);
           AllValuesAreZero &= (V == 0);
 
-          ++ai;
-        } while (ai != ae && intrinsic.ArgumentAttributes[ai].Index == attrIdx);
+          ++Ai;
+        } while (Ai != Ae && Intrinsic.ArgumentAttributes[Ai].Index == AttrIdx);
         OS << "};\n";
 
         // Generate attribute value array if not all attribute values are zero.
         if (!AllValuesAreZero) {
-          OS << "      const uint64_t AttrValParam" << attrIdx << "[]= {";
-          addComma = false;
-          for (const auto V : Values) {
-            if (addComma)
-              OS << ',';
-            OS << V;
-            addComma = true;
-          }
+          OS << "      const uint64_t AttrValParam" << AttrIdx << "[]= {";
+          ListSeparator LSV(",");
+          for (const auto V : Values)
+            OS << LSV << V;
           OS << "};\n";
         }
 
         OS << "      AS[" << numAttrs++ << "] = AttributeList::get(C, "
-           << attrIdx << ", AttrParam" << attrIdx;
+           << AttrIdx << ", AttrParam" << AttrIdx;
         if (!AllValuesAreZero)
-          OS << ", AttrValParam" << attrIdx;
+          OS << ", AttrValParam" << AttrIdx;
         OS << ");\n";
       }
     }
 
-    if (!intrinsic.canThrow ||
-        (intrinsic.ModRef != CodeGenIntrinsic::ReadWriteMem &&
-         !intrinsic.hasSideEffects) ||
-        intrinsic.isNoReturn || intrinsic.isNoSync || intrinsic.isNoFree ||
-        intrinsic.isWillReturn || intrinsic.isCold || intrinsic.isNoDuplicate ||
-        intrinsic.isConvergent || intrinsic.isSpeculatable) {
+    if (!Intrinsic.canThrow ||
+        (Intrinsic.ModRef != CodeGenIntrinsic::ReadWriteMem &&
+         !Intrinsic.hasSideEffects) ||
+        Intrinsic.isNoReturn || Intrinsic.isNoSync || Intrinsic.isNoFree ||
+        Intrinsic.isWillReturn || Intrinsic.isCold || Intrinsic.isNoDuplicate ||
+        Intrinsic.isNoMerge || Intrinsic.isConvergent ||
+        Intrinsic.isSpeculatable) {
       OS << "      const Attribute::AttrKind Atts[] = {";
-      bool addComma = false;
-      if (!intrinsic.canThrow) {
-        OS << "Attribute::NoUnwind";
-        addComma = true;
-      }
-      if (intrinsic.isNoReturn) {
-        if (addComma)
-          OS << ",";
-        OS << "Attribute::NoReturn";
-        addComma = true;
-      }
-      if (intrinsic.isNoSync) {
-        if (addComma)
-          OS << ",";
-        OS << "Attribute::NoSync";
-        addComma = true;
-      }
-      if (intrinsic.isNoFree) {
-        if (addComma)
-          OS << ",";
-        OS << "Attribute::NoFree";
-        addComma = true;
-      }
-      if (intrinsic.isWillReturn) {
-        if (addComma)
-          OS << ",";
-        OS << "Attribute::WillReturn";
-        addComma = true;
-      }
-      if (intrinsic.isCold) {
-        if (addComma)
-          OS << ",";
-        OS << "Attribute::Cold";
-        addComma = true;
-      }
-      if (intrinsic.isNoDuplicate) {
-        if (addComma)
-          OS << ",";
-        OS << "Attribute::NoDuplicate";
-        addComma = true;
-      }
-      if (intrinsic.isConvergent) {
-        if (addComma)
-          OS << ",";
-        OS << "Attribute::Convergent";
-        addComma = true;
-      }
-      if (intrinsic.isSpeculatable) {
-        if (addComma)
-          OS << ",";
-        OS << "Attribute::Speculatable";
-        addComma = true;
-      }
+      ListSeparator LS(",");
+      if (!Intrinsic.canThrow)
+        OS << LS << "Attribute::NoUnwind";
+      if (Intrinsic.isNoReturn)
+        OS << LS << "Attribute::NoReturn";
+      if (Intrinsic.isNoSync)
+        OS << LS << "Attribute::NoSync";
+      if (Intrinsic.isNoFree)
+        OS << LS << "Attribute::NoFree";
+      if (Intrinsic.isWillReturn)
+        OS << LS << "Attribute::WillReturn";
+      if (Intrinsic.isCold)
+        OS << LS << "Attribute::Cold";
+      if (Intrinsic.isNoDuplicate)
+        OS << LS << "Attribute::NoDuplicate";
+      if (Intrinsic.isNoMerge)
+        OS << LS << "Attribute::NoMerge";
+      if (Intrinsic.isConvergent)
+        OS << LS << "Attribute::Convergent";
+      if (Intrinsic.isSpeculatable)
+        OS << LS << "Attribute::Speculatable";
 
-      switch (intrinsic.ModRef) {
+      switch (Intrinsic.ModRef) {
       case CodeGenIntrinsic::NoMem:
-        if (intrinsic.hasSideEffects)
+        if (Intrinsic.hasSideEffects)
           break;
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::ReadNone";
         break;
       case CodeGenIntrinsic::ReadArgMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::ReadOnly,";
         OS << "Attribute::ArgMemOnly";
         break;
       case CodeGenIntrinsic::ReadMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::ReadOnly";
         break;
       case CodeGenIntrinsic::ReadInaccessibleMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::ReadOnly,";
         OS << "Attribute::InaccessibleMemOnly";
         break;
       case CodeGenIntrinsic::ReadInaccessibleMemOrArgMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::ReadOnly,";
         OS << "Attribute::InaccessibleMemOrArgMemOnly";
         break;
       case CodeGenIntrinsic::WriteArgMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::WriteOnly,";
         OS << "Attribute::ArgMemOnly";
         break;
       case CodeGenIntrinsic::WriteMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::WriteOnly";
         break;
       case CodeGenIntrinsic::WriteInaccessibleMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::WriteOnly,";
         OS << "Attribute::InaccessibleMemOnly";
         break;
       case CodeGenIntrinsic::WriteInaccessibleMemOrArgMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::WriteOnly,";
         OS << "Attribute::InaccessibleMemOrArgMemOnly";
         break;
       case CodeGenIntrinsic::ReadWriteArgMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::ArgMemOnly";
         break;
       case CodeGenIntrinsic::ReadWriteInaccessibleMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::InaccessibleMemOnly";
         break;
       case CodeGenIntrinsic::ReadWriteInaccessibleMemOrArgMem:
-        if (addComma)
-          OS << ",";
+        OS << LS;
         OS << "Attribute::InaccessibleMemOrArgMemOnly";
         break;
       case CodeGenIntrinsic::ReadWriteMem:
@@ -977,25 +905,25 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
   OS << "  StringRef TargetPrefix(TargetPrefixStr);\n\n";
 
   // Note: this could emit significantly better code if we cared.
-  for (BIMTy::iterator I = BuiltinMap.begin(), E = BuiltinMap.end();I != E;++I){
+  for (auto &I : BuiltinMap) {
     OS << "  ";
-    if (!I->first.empty())
-      OS << "if (TargetPrefix == \"" << I->first << "\") ";
+    if (!I.first.empty())
+      OS << "if (TargetPrefix == \"" << I.first << "\") ";
     else
       OS << "/* Target Independent Builtins */ ";
     OS << "{\n";
 
     // Emit the comparisons for this target prefix.
-    OS << "    static const BuiltinEntry " << I->first << "Names[] = {\n";
-    for (const auto &P : I->second) {
+    OS << "    static const BuiltinEntry " << I.first << "Names[] = {\n";
+    for (const auto &P : I.second) {
       OS << "      {Intrinsic::" << P.second << ", "
          << Table.GetOrAddStringOffset(P.first) << "}, // " << P.first << "\n";
     }
     OS << "    };\n";
-    OS << "    auto I = std::lower_bound(std::begin(" << I->first << "Names),\n";
-    OS << "                              std::end(" << I->first << "Names),\n";
+    OS << "    auto I = std::lower_bound(std::begin(" << I.first << "Names),\n";
+    OS << "                              std::end(" << I.first << "Names),\n";
     OS << "                              BuiltinNameStr);\n";
-    OS << "    if (I != std::end(" << I->first << "Names) &&\n";
+    OS << "    if (I != std::end(" << I.first << "Names) &&\n";
     OS << "        I->getName() == BuiltinNameStr)\n";
     OS << "      return I->IntrinID;\n";
     OS << "  }\n";
diff --git a/llvm/utils/TableGen/OptParserEmitter.cpp b/llvm/utils/TableGen/OptParserEmitter.cpp
index 8e6c05885e5b..0809432dfd0d 100644
--- a/llvm/utils/TableGen/OptParserEmitter.cpp
+++ b/llvm/utils/TableGen/OptParserEmitter.cpp
@@ -251,7 +251,7 @@ void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) {
 
     // Prefix values.
     OS << ", {";
-    for (StringRef PrefixKey : Prefix.first)
+    for (const auto &PrefixKey : Prefix.first)
       OS << "\"" << PrefixKey << "\" COMMA ";
     OS << "nullptr})\n";
   }
diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index e05409db67d0..6acb630299c1 100644
--- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -108,6 +108,11 @@ addDagOperandMapping(Record *Rec, DagInit *Dag, CodeGenInstruction &Insn,
       OperandMap[BaseIdx + i].Kind = OpData::Imm;
       OperandMap[BaseIdx + i].Data.Imm = II->getValue();
       ++OpsAdded;
+    } else if (auto *BI = dyn_cast<BitsInit>(Dag->getArg(i))) {
+      auto *II = cast<IntInit>(BI->convertInitializerTo(IntRecTy::get()));
+      OperandMap[BaseIdx + i].Kind = OpData::Imm;
+      OperandMap[BaseIdx + i].Data.Imm = II->getValue();
+      ++OpsAdded;
     } else if (DagInit *SubDag = dyn_cast<DagInit>(Dag->getArg(i))) {
       // Just add the operands recursively. This is almost certainly
       // a constant value for a complex operand (> 1 MI operand).
diff --git a/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp b/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp
index 183c8f9494db..e931801f82a4 100644
--- a/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp
@@ -273,8 +273,8 @@ static bool verifyDagOpCount(CodeGenInstruction &Inst, DagInit *Dag,
   // The Instruction might have tied operands so the Dag might have
   //  a fewer operand count.
   unsigned RealCount = Inst.Operands.size();
-  for (unsigned i = 0; i < Inst.Operands.size(); i++)
-    if (Inst.Operands[i].getTiedRegister() != -1)
+  for (const auto &Operand : Inst.Operands)
+    if (Operand.getTiedRegister() != -1)
       --RealCount;
 
   if (Dag->getNumArgs() != RealCount)
diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index dce7594dec2f..037fad207ac7 100644
--- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -305,9 +305,8 @@ EmitRegUnitPressure(raw_ostream &OS, const CodeGenRegBank &RegBank,
   for (unsigned i = 0, e = NumRCUnitSets; i != e; ++i) {
     ArrayRef<unsigned> PSetIDs = RegBank.getRCPressureSetIDs(i);
     PSets[i].reserve(PSetIDs.size());
-    for (ArrayRef<unsigned>::iterator PSetI = PSetIDs.begin(),
-           PSetE = PSetIDs.end(); PSetI != PSetE; ++PSetI) {
-      PSets[i].push_back(RegBank.getRegPressureSet(*PSetI).Order);
+    for (unsigned PSetID : PSetIDs) {
+      PSets[i].push_back(RegBank.getRegPressureSet(PSetID).Order);
     }
     llvm::sort(PSets[i]);
     PSetsSeqs.add(PSets[i]);
@@ -399,11 +398,9 @@ void RegisterInfoEmitter::EmitRegMappingTables(
     return;
 
   // Now we know maximal length of number list. Append -1's, where needed
-  for (DwarfRegNumsVecTy::iterator I = DwarfRegNums.begin(),
-                                   E = DwarfRegNums.end();
-       I != E; ++I)
-    for (unsigned i = I->second.size(), e = maxLength; i != e; ++i)
-      I->second.push_back(-1);
+  for (auto &DwarfRegNum : DwarfRegNums)
+    for (unsigned I = DwarfRegNum.second.size(), E = maxLength; I != E; ++I)
+      DwarfRegNum.second.push_back(-1);
 
   StringRef Namespace = Regs.front().TheDef->getValueAsString("Namespace");
 
@@ -411,10 +408,10 @@ void RegisterInfoEmitter::EmitRegMappingTables(
 
   // Emit reverse information about the dwarf register numbers.
   for (unsigned j = 0; j < 2; ++j) {
-    for (unsigned i = 0, e = maxLength; i != e; ++i) {
+    for (unsigned I = 0, E = maxLength; I != E; ++I) {
       OS << "extern const MCRegisterInfo::DwarfLLVMRegPair " << Namespace;
       OS << (j == 0 ? "DwarfFlavour" : "EHFlavour");
-      OS << i << "Dwarf2L[]";
+      OS << I << "Dwarf2L[]";
 
       if (!isCtor) {
         OS << " = {\n";
@@ -422,17 +419,15 @@ void RegisterInfoEmitter::EmitRegMappingTables(
         // Store the mapping sorted by the LLVM reg num so lookup can be done
         // with a binary search.
         std::map<uint64_t, Record*> Dwarf2LMap;
-        for (DwarfRegNumsVecTy::iterator
-               I = DwarfRegNums.begin(), E = DwarfRegNums.end(); I != E; ++I) {
-          int DwarfRegNo = I->second[i];
+        for (auto &DwarfRegNum : DwarfRegNums) {
+          int DwarfRegNo = DwarfRegNum.second[I];
           if (DwarfRegNo < 0)
             continue;
-          Dwarf2LMap[DwarfRegNo] = I->first;
+          Dwarf2LMap[DwarfRegNo] = DwarfRegNum.first;
         }
 
-        for (std::map<uint64_t, Record*>::iterator
-               I = Dwarf2LMap.begin(), E = Dwarf2LMap.end(); I != E; ++I)
-          OS << "  { " << I->first << "U, " << getQualifiedName(I->second)
+        for (auto &I : Dwarf2LMap)
+          OS << "  { " << I.first << "U, " << getQualifiedName(I.second)
              << " },\n";
 
         OS << "};\n";
@@ -443,11 +438,10 @@ void RegisterInfoEmitter::EmitRegMappingTables(
       // We have to store the size in a const global, it's used in multiple
       // places.
       OS << "extern const unsigned " << Namespace
-         << (j == 0 ? "DwarfFlavour" : "EHFlavour") << i << "Dwarf2LSize";
+         << (j == 0 ? "DwarfFlavour" : "EHFlavour") << I << "Dwarf2LSize";
       if (!isCtor)
         OS << " = array_lengthof(" << Namespace
-           << (j == 0 ? "DwarfFlavour" : "EHFlavour") << i
-           << "Dwarf2L);\n\n";
+           << (j == 0 ? "DwarfFlavour" : "EHFlavour") << I << "Dwarf2L);\n\n";
       else
         OS << ";\n\n";
     }
@@ -486,13 +480,12 @@ void RegisterInfoEmitter::EmitRegMappingTables(
         OS << " = {\n";
         // Store the mapping sorted by the Dwarf reg num so lookup can be done
         // with a binary search.
-        for (DwarfRegNumsVecTy::iterator
-               I = DwarfRegNums.begin(), E = DwarfRegNums.end(); I != E; ++I) {
-          int RegNo = I->second[i];
+        for (auto &DwarfRegNum : DwarfRegNums) {
+          int RegNo = DwarfRegNum.second[i];
           if (RegNo == -1) // -1 is the default value, don't emit a mapping.
             continue;
 
-          OS << "  { " << getQualifiedName(I->first) << ", " << RegNo
+          OS << "  { " << getQualifiedName(DwarfRegNum.first) << ", " << RegNo
              << "U },\n";
         }
         OS << "};\n";
@@ -1029,9 +1022,10 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target,
     ArrayRef<const CodeGenRegister*> Roots = RegBank.getRegUnit(i).getRoots();
     assert(!Roots.empty() && "All regunits must have a root register.");
     assert(Roots.size() <= 2 && "More than two roots not supported yet.");
-    OS << "  { " << getQualifiedName(Roots.front()->TheDef);
-    for (unsigned r = 1; r != Roots.size(); ++r)
-      OS << ", " << getQualifiedName(Roots[r]->TheDef);
+    OS << "  { ";
+    ListSeparator LS;
+    for (const CodeGenRegister *R : Roots)
+      OS << LS << getQualifiedName(R->TheDef);
     OS << " },\n";
   }
   OS << "};\n\n";
@@ -1083,12 +1077,15 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target,
 
   for (const auto &RC : RegisterClasses) {
     assert(isInt<8>(RC.CopyCost) && "Copy cost too large.");
+    uint32_t RegSize = 0;
+    if (RC.RSI.isSimple())
+      RegSize = RC.RSI.getSimple().RegSize;
     OS << "  { " << RC.getName() << ", " << RC.getName() << "Bits, "
-       << RegClassStrings.get(RC.getName()) << ", "
-       << RC.getOrder().size() << ", sizeof(" << RC.getName() << "Bits), "
-       << RC.getQualifiedName() + "RegClassID" << ", "
-       << RC.CopyCost << ", "
-       << ( RC.Allocatable ? "true" : "false" ) << " },\n";
+       << RegClassStrings.get(RC.getName()) << ", " << RC.getOrder().size()
+       << ", sizeof(" << RC.getName() << "Bits), "
+       << RC.getQualifiedName() + "RegClassID"
+       << ", " << RegSize << ", " << RC.CopyCost << ", "
+       << (RC.Allocatable ? "true" : "false") << " },\n";
   }
 
   OS << "};\n\n";
@@ -1441,19 +1438,52 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
 
   // Emit extra information about registers.
   const std::string &TargetName = std::string(Target.getName());
-  OS << "\nstatic const TargetRegisterInfoDesc "
-     << TargetName << "RegInfoDesc[] = { // Extra Descriptors\n";
-  OS << "  { 0, false },\n";
-
   const auto &Regs = RegBank.getRegisters();
+  unsigned NumRegCosts = 1;
+  for (const auto &Reg : Regs)
+    NumRegCosts = std::max((size_t)NumRegCosts, Reg.CostPerUse.size());
+
+  std::vector<unsigned> AllRegCostPerUse;
+  llvm::BitVector InAllocClass(Regs.size() + 1, false);
+  AllRegCostPerUse.insert(AllRegCostPerUse.end(), NumRegCosts, 0);
+
+  // Populate the vector RegCosts with the CostPerUse list of the registers
+  // in the order they are read. Have at most NumRegCosts entries for
+  // each register. Fill with zero for values which are not explicitly given.
   for (const auto &Reg : Regs) {
-    OS << "  { ";
-    OS << Reg.CostPerUse << ", "
-       << ( AllocatableRegs.count(Reg.TheDef) != 0 ? "true" : "false" )
-       << " },\n";
+    auto Costs = Reg.CostPerUse;
+    AllRegCostPerUse.insert(AllRegCostPerUse.end(), Costs.begin(), Costs.end());
+    if (NumRegCosts > Costs.size())
+      AllRegCostPerUse.insert(AllRegCostPerUse.end(),
+                              NumRegCosts - Costs.size(), 0);
+
+    if (AllocatableRegs.count(Reg.TheDef))
+      InAllocClass.set(Reg.EnumValue);
   }
-  OS << "};\n";      // End of register descriptors...
 
+  // Emit the cost values as a 1D-array after grouping them by their indices,
+  // i.e. the costs for all registers corresponds to index 0, 1, 2, etc.
+  // Size of the emitted array should be NumRegCosts * (Regs.size() + 1).
+  OS << "\nstatic const uint8_t "
+     << "CostPerUseTable[] = { \n";
+  for (unsigned int I = 0; I < NumRegCosts; ++I) {
+    for (unsigned J = I, E = AllRegCostPerUse.size(); J < E; J += NumRegCosts)
+      OS << AllRegCostPerUse[J] << ", ";
+  }
+  OS << "};\n\n";
+
+  OS << "\nstatic const bool "
+     << "InAllocatableClassTable[] = { \n";
+  for (unsigned I = 0, E = InAllocClass.size(); I < E; ++I) {
+    OS << (InAllocClass[I] ? "true" : "false") << ", ";
+  }
+  OS << "};\n\n";
+
+  OS << "\nstatic const TargetRegisterInfoDesc " << TargetName
+     << "RegInfoDesc = { // Extra Descriptors\n";
+  OS << "CostPerUseTable, " << NumRegCosts << ", "
+     << "InAllocatableClassTable";
+  OS << "};\n\n"; // End of register descriptors...
 
   std::string ClassName = Target.getName().str() + "GenRegisterInfo";
 
@@ -1513,10 +1543,11 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
 
   EmitRegMappingTables(OS, Regs, true);
 
-  OS << ClassName << "::\n" << ClassName
+  OS << ClassName << "::\n"
+     << ClassName
      << "(unsigned RA, unsigned DwarfFlavour, unsigned EHFlavour,\n"
         "      unsigned PC, unsigned HwMode)\n"
-     << "  : TargetRegisterInfo(" << TargetName << "RegInfoDesc"
+     << "  : TargetRegisterInfo(&" << TargetName << "RegInfoDesc"
      << ", RegisterClasses, RegisterClasses+" << RegisterClasses.size() << ",\n"
      << "             SubRegIndexNameTable, SubRegIndexLaneMaskTable,\n"
      << "             ";
@@ -1679,7 +1710,10 @@ void RegisterInfoEmitter::debugDump(raw_ostream &OS) {
 
   for (const CodeGenRegister &R : RegBank.getRegisters()) {
     OS << "Register " << R.getName() << ":\n";
-    OS << "\tCostPerUse: " << R.CostPerUse << '\n';
+    OS << "\tCostPerUse: ";
+    for (const auto &Cost : R.CostPerUse)
+      OS << Cost << " ";
+    OS << '\n';
     OS << "\tCoveredBySubregs: " << R.CoveredBySubRegs << '\n';
     OS << "\tHasDisjunctSubRegs: " << R.HasDisjunctSubRegs << '\n';
     for (std::pair<CodeGenSubRegIndex*,CodeGenRegister*> P : R.getSubRegs()) {
diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp
index 912d43b2caa1..7803848aafc4 100644
--- a/llvm/utils/TableGen/SearchableTableEmitter.cpp
+++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp
@@ -348,17 +348,13 @@ void SearchableTableEmitter::emitLookupFunction(const GenericTable &Table,
       IndexRowsStorage.push_back(Entry.first);
 
       OS << "    { ";
-      bool NeedComma = false;
+      ListSeparator LS;
       for (const auto &Field : Index.Fields) {
-        if (NeedComma)
-          OS << ", ";
-        NeedComma = true;
-
         std::string Repr = primaryRepresentation(
             Index.Loc, Field, Entry.first->getValueInit(Field.Name));
         if (isa<StringRecTy>(Field.RecType))
           Repr = StringRef(Repr).upper();
-        OS << Repr;
+        OS << LS << Repr;
       }
       OS << ", " << Entry.second << " },\n";
     }
@@ -414,13 +410,9 @@ void SearchableTableEmitter::emitLookupFunction(const GenericTable &Table,
   }
   OS << "  };\n";
   OS << "  KeyType Key = {";
-  bool NeedComma = false;
+  ListSeparator LS;
   for (const auto &Field : Index.Fields) {
-    if (NeedComma)
-      OS << ", ";
-    NeedComma = true;
-
-    OS << Field.Name;
+    OS << LS << Field.Name;
     if (isa<StringRecTy>(Field.RecType)) {
       OS << ".upper()";
       if (IsPrimary)
@@ -482,15 +474,10 @@ void SearchableTableEmitter::emitLookupDeclaration(const GenericTable &Table,
                                                    raw_ostream &OS) {
   OS << "const " << Table.CppTypeName << " *" << Index.Name << "(";
 
-  bool NeedComma = false;
-  for (const auto &Field : Index.Fields) {
-    if (NeedComma)
-      OS << ", ";
-    NeedComma = true;
-
-    OS << searchableFieldType(Table, Index, Field, TypeInArgument) << " "
+  ListSeparator LS;
+  for (const auto &Field : Index.Fields)
+    OS << LS << searchableFieldType(Table, Index, Field, TypeInArgument) << " "
        << Field.Name;
-  }
   OS << ")";
 }
 
@@ -518,15 +505,11 @@ void SearchableTableEmitter::emitGenericTable(const GenericTable &Table,
     Record *Entry = Table.Entries[i];
     OS << "  { ";
 
-    bool NeedComma = false;
-    for (const auto &Field : Table.Fields) {
-      if (NeedComma)
-        OS << ", ";
-      NeedComma = true;
-
-      OS << primaryRepresentation(Table.Locs[0], Field,
+    ListSeparator LS;
+    for (const auto &Field : Table.Fields)
+      OS << LS
+         << primaryRepresentation(Table.Locs[0], Field,
                                   Entry->getValueInit(Field.Name));
-    }
 
     OS << " }, // " << i << "\n";
   }
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 7d2b4b929df3..c88fe949c25c 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -179,8 +179,8 @@ void SubtargetEmitter::Enumeration(raw_ostream &OS,
 static void printFeatureMask(raw_ostream &OS, RecVec &FeatureList,
                              const DenseMap<Record *, unsigned> &FeatureMap) {
   std::array<uint64_t, MAX_SUBTARGET_WORDS> Mask = {};
-  for (unsigned j = 0, M = FeatureList.size(); j < M; ++j) {
-    unsigned Bit = FeatureMap.lookup(FeatureList[j]);
+  for (const Record *Feature : FeatureList) {
+    unsigned Bit = FeatureMap.lookup(Feature);
     Mask[Bit / 64] |= 1ULL << (Bit % 64);
   }
 
@@ -215,10 +215,8 @@ unsigned SubtargetEmitter::FeatureKeyValues(
 
   // For each feature
   unsigned NumFeatures = 0;
-  for (unsigned i = 0, N = FeatureList.size(); i < N; ++i) {
+  for (const Record *Feature : FeatureList) {
     // Next feature
-    Record *Feature = FeatureList[i];
-
     StringRef Name = Feature->getName();
     StringRef CommandLineName = Feature->getValueAsString("Name");
     StringRef Desc = Feature->getValueAsString("Desc");
@@ -344,13 +342,12 @@ void SubtargetEmitter::FormItineraryOperandCycleString(Record *ItinData,
     ItinData->getValueAsListOfInts("OperandCycles");
 
   // For each operand cycle
-  unsigned N = NOperandCycles = OperandCycleList.size();
-  for (unsigned i = 0; i < N;) {
+  NOperandCycles = OperandCycleList.size();
+  ListSeparator LS;
+  for (int OCycle : OperandCycleList) {
     // Next operand cycle
-    const int OCycle = OperandCycleList[i];
-
+    ItinString += LS;
     ItinString += "  " + itostr(OCycle);
-    if (++i < N) ItinString += ", ";
   }
 }
 
@@ -361,13 +358,14 @@ void SubtargetEmitter::FormItineraryBypassString(const std::string &Name,
   RecVec BypassList = ItinData->getValueAsListOfDefs("Bypasses");
   unsigned N = BypassList.size();
   unsigned i = 0;
-  for (; i < N;) {
+  ListSeparator LS;
+  for (; i < N; ++i) {
+    ItinString += LS;
     ItinString += Name + "Bypass::" + BypassList[i]->getName().str();
-    if (++i < NOperandCycles) ItinString += ", ";
   }
-  for (; i < NOperandCycles;) {
+  for (; i < NOperandCycles; ++i) {
+    ItinString += LS;
     ItinString += " 0";
-    if (++i < NOperandCycles) ItinString += ", ";
   }
 }
 
@@ -995,6 +993,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
     SCDesc.NumMicroOps = 0;
     SCDesc.BeginGroup = false;
     SCDesc.EndGroup = false;
+    SCDesc.RetireOOO = false;
     SCDesc.WriteProcResIdx = 0;
     SCDesc.WriteLatencyIdx = 0;
     SCDesc.ReadAdvanceIdx = 0;
@@ -1097,6 +1096,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
         SCDesc.EndGroup |= WriteRes->getValueAsBit("EndGroup");
         SCDesc.BeginGroup |= WriteRes->getValueAsBit("SingleIssue");
         SCDesc.EndGroup |= WriteRes->getValueAsBit("SingleIssue");
+        SCDesc.RetireOOO |= WriteRes->getValueAsBit("RetireOOO");
 
         // Create an entry for each ProcResource listed in WriteRes.
         RecVec PRVec = WriteRes->getValueAsListOfDefs("ProcResources");
@@ -1295,7 +1295,7 @@ void SubtargetEmitter::EmitSchedClassTables(SchedClassTables &SchedTables,
     std::vector<MCSchedClassDesc> &SCTab =
       SchedTables.ProcSchedClasses[1 + (PI - SchedModels.procModelBegin())];
 
-    OS << "\n// {Name, NumMicroOps, BeginGroup, EndGroup,"
+    OS << "\n// {Name, NumMicroOps, BeginGroup, EndGroup, RetireOOO,"
        << " WriteProcResIdx,#, WriteLatencyIdx,#, ReadAdvanceIdx,#}\n";
     OS << "static const llvm::MCSchedClassDesc "
        << PI->ModelName << "SchedClasses[] = {\n";
@@ -1306,7 +1306,7 @@ void SubtargetEmitter::EmitSchedClassTables(SchedClassTables &SchedTables,
            && "invalid class not first");
     OS << "  {DBGFIELD(\"InvalidSchedClass\")  "
        << MCSchedClassDesc::InvalidNumMicroOps
-       << ", false, false,  0, 0,  0, 0,  0, 0},\n";
+       << ", false, false, false, 0, 0,  0, 0,  0, 0},\n";
 
     for (unsigned SCIdx = 1, SCEnd = SCTab.size(); SCIdx != SCEnd; ++SCIdx) {
       MCSchedClassDesc &MCDesc = SCTab[SCIdx];
@@ -1317,6 +1317,7 @@ void SubtargetEmitter::EmitSchedClassTables(SchedClassTables &SchedTables,
       OS << MCDesc.NumMicroOps
          << ", " << ( MCDesc.BeginGroup ? "true" : "false" )
          << ", " << ( MCDesc.EndGroup ? "true" : "false" )
+         << ", " << ( MCDesc.RetireOOO ? "true" : "false" )
          << ", " << format("%2d", MCDesc.WriteProcResIdx)
          << ", " << MCDesc.NumWriteProcResEntries
          << ", " << format("%2d", MCDesc.WriteLatencyIdx)
diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/llvm/utils/TableGen/SubtargetFeatureInfo.cpp
index 105ed82c9d02..33a22776f2df 100644
--- a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp
+++ b/llvm/utils/TableGen/SubtargetFeatureInfo.cpp
@@ -130,14 +130,9 @@ void SubtargetFeatureInfo::emitComputeAssemblerAvailableFeatures(
     if (IsOr)
       OS << "(";
 
-    bool First = true;
+    ListSeparator LS(IsOr ? " || " : " && ");
     for (auto *Arg : D->getArgs()) {
-      if (!First) {
-        if (IsOr)
-          OS << " || ";
-        else
-          OS << " && ";
-      }
+      OS << LS;
       if (auto *NotArg = dyn_cast<DagInit>(Arg)) {
         if (NotArg->getOperator()->getAsString() != "not" ||
             NotArg->getNumArgs() != 1)
@@ -149,8 +144,6 @@ void SubtargetFeatureInfo::emitComputeAssemblerAvailableFeatures(
           !cast<DefInit>(Arg)->getDef()->isSubClassOf("SubtargetFeature"))
         PrintFatalError(SFI.TheDef->getLoc(), "Invalid AssemblerCondDag!");
       OS << "FB[" << TargetName << "::" << Arg->getAsString() << "]";
-
-      First = false;
     }
 
     if (IsOr)
diff --git a/llvm/utils/TableGen/TableGen.cpp b/llvm/utils/TableGen/TableGen.cpp
index 6d851da34731..24c11c8bc831 100644
--- a/llvm/utils/TableGen/TableGen.cpp
+++ b/llvm/utils/TableGen/TableGen.cpp
@@ -25,6 +25,7 @@ enum ActionType {
   NullBackend,
   DumpJSON,
   GenEmitter,
+  GenCodeBeads,
   GenRegisterInfo,
   GenInstrInfo,
   GenInstrDocs,
@@ -56,7 +57,6 @@ enum ActionType {
   GenAutomata,
   GenDirectivesEnumDecl,
   GenDirectivesEnumImpl,
-  GenDirectivesEnumGen,
 };
 
 namespace llvm {
@@ -81,6 +81,8 @@ cl::opt<ActionType> Action(
         clEnumValN(DumpJSON, "dump-json",
                    "Dump all records as machine-readable JSON"),
         clEnumValN(GenEmitter, "gen-emitter", "Generate machine code emitter"),
+        clEnumValN(GenCodeBeads, "gen-code-beads",
+                   "Generate machine code beads"),
         clEnumValN(GenRegisterInfo, "gen-register-info",
                    "Generate registers and register classes info"),
         clEnumValN(GenInstrInfo, "gen-instr-info",
@@ -136,9 +138,7 @@ cl::opt<ActionType> Action(
         clEnumValN(GenDirectivesEnumDecl, "gen-directive-decl",
                    "Generate directive related declaration code (header file)"),
         clEnumValN(GenDirectivesEnumImpl, "gen-directive-impl",
-                   "Generate directive related implementation code"),
-        clEnumValN(GenDirectivesEnumGen, "gen-directive-gen",
-                   "Generate directive related implementation code part")));
+                   "Generate directive related implementation code")));
 
 cl::OptionCategory PrintEnumsCat("Options for -print-enums");
 cl::opt<std::string> Class("class", cl::desc("Print Enum list for this class"),
@@ -161,6 +161,9 @@ bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   case GenEmitter:
     EmitCodeEmitter(Records, OS);
     break;
+  case GenCodeBeads:
+    EmitCodeBeads(Records, OS);
+    break;
   case GenRegisterInfo:
     EmitRegisterInfo(Records, OS);
     break;
@@ -269,9 +272,6 @@ bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   case GenDirectivesEnumImpl:
     EmitDirectivesImpl(Records, OS);
     break;
-  case GenDirectivesEnumGen:
-    EmitDirectivesGen(Records, OS);
-    break;
   }
 
   return false;
diff --git a/llvm/utils/TableGen/TableGenBackends.h b/llvm/utils/TableGen/TableGenBackends.h
index 92204f39f8fa..71db8dc77b05 100644
--- a/llvm/utils/TableGen/TableGenBackends.h
+++ b/llvm/utils/TableGen/TableGenBackends.h
@@ -67,6 +67,7 @@ void EmitAsmMatcher(RecordKeeper &RK, raw_ostream &OS);
 void EmitAsmWriter(RecordKeeper &RK, raw_ostream &OS);
 void EmitCallingConv(RecordKeeper &RK, raw_ostream &OS);
 void EmitCodeEmitter(RecordKeeper &RK, raw_ostream &OS);
+void EmitCodeBeads(RecordKeeper &RK, raw_ostream &OS);
 void EmitDAGISel(RecordKeeper &RK, raw_ostream &OS);
 void EmitDFAPacketizer(RecordKeeper &RK, raw_ostream &OS);
 void EmitDisassembler(RecordKeeper &RK, raw_ostream &OS);
@@ -92,7 +93,6 @@ void EmitExegesis(RecordKeeper &RK, raw_ostream &OS);
 void EmitAutomata(RecordKeeper &RK, raw_ostream &OS);
 void EmitDirectivesDecl(RecordKeeper &RK, raw_ostream &OS);
 void EmitDirectivesImpl(RecordKeeper &RK, raw_ostream &OS);
-void EmitDirectivesGen(RecordKeeper &RK, raw_ostream &OS);
 
 } // End llvm namespace
 
diff --git a/llvm/utils/TableGen/X86DisassemblerTables.cpp b/llvm/utils/TableGen/X86DisassemblerTables.cpp
index 331664f875b7..89069ec3e4ff 100644
--- a/llvm/utils/TableGen/X86DisassemblerTables.cpp
+++ b/llvm/utils/TableGen/X86DisassemblerTables.cpp
@@ -102,7 +102,8 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_64BIT_ADSIZE:
     return (noPrefix && inheritsFrom(child, IC_64BIT_OPSIZE_ADSIZE, noPrefix));
   case IC_64BIT_OPSIZE_ADSIZE:
-    return false;
+    return (noPrefix &&
+            inheritsFrom(child, IC_64BIT_VEX_OPSIZE_ADSIZE, noPrefix));
   case IC_XD:
     return inheritsFrom(child, IC_64BIT_XD);
   case IC_XS:
@@ -123,10 +124,11 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_64BIT_OPSIZE:
     return inheritsFrom(child, IC_64BIT_REXW_OPSIZE) ||
            (!AdSize64 && inheritsFrom(child, IC_64BIT_OPSIZE_ADSIZE)) ||
-           (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE));
+           (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE)) ||
+           (!AdSize64 && inheritsFrom(child, IC_64BIT_VEX_OPSIZE_ADSIZE));
   case IC_64BIT_XD:
-    return(inheritsFrom(child, IC_64BIT_REXW_XD) ||
-           (!AdSize64 && inheritsFrom(child, IC_64BIT_XD_ADSIZE)));
+    return (inheritsFrom(child, IC_64BIT_REXW_XD) ||
+            (!AdSize64 && inheritsFrom(child, IC_64BIT_XD_ADSIZE)));
   case IC_64BIT_XS:
     return(inheritsFrom(child, IC_64BIT_REXW_XS) ||
            (!AdSize64 && inheritsFrom(child, IC_64BIT_XS_ADSIZE)));
@@ -156,7 +158,12 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_VEX_OPSIZE:
     return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_VEX_L_W_OPSIZE)) ||
            (VEX_WIG && inheritsFrom(child, IC_VEX_W_OPSIZE)) ||
-           (VEX_LIG && inheritsFrom(child, IC_VEX_L_OPSIZE));
+           (VEX_LIG && inheritsFrom(child, IC_VEX_L_OPSIZE)) ||
+           inheritsFrom(child, IC_64BIT_VEX_OPSIZE);
+  case IC_64BIT_VEX_OPSIZE:
+    return inheritsFrom(child, IC_64BIT_VEX_OPSIZE_ADSIZE);
+  case IC_64BIT_VEX_OPSIZE_ADSIZE:
+    return false;
   case IC_VEX_W:
     return VEX_LIG && inheritsFrom(child, IC_VEX_L_W);
   case IC_VEX_W_XS:
@@ -698,8 +705,8 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2,
         ModRMDecision.push_back(decision.instructionIDs[index]);
       break;
     case MODRM_FULL:
-      for (unsigned index = 0; index < 256; ++index)
-        ModRMDecision.push_back(decision.instructionIDs[index]);
+      for (unsigned short InstructionID : decision.instructionIDs)
+        ModRMDecision.push_back(InstructionID);
       break;
   }
 
@@ -710,10 +717,9 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2,
     ModRMTableNum += ModRMDecision.size();
     o1 << "/*Table" << EntryNumber << "*/\n";
     i1++;
-    for (std::vector<unsigned>::const_iterator I = ModRMDecision.begin(),
-           E = ModRMDecision.end(); I != E; ++I) {
-      o1.indent(i1 * 2) << format("0x%hx", *I) << ", /*"
-                        << InstructionSpecifiers[*I].name << "*/\n";
+    for (unsigned I : ModRMDecision) {
+      o1.indent(i1 * 2) << format("0x%hx", I) << ", /*"
+                        << InstructionSpecifiers[I].name << "*/\n";
     }
     i1--;
   }
@@ -743,6 +749,7 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2,
   // We assume that the index can fit into uint16_t.
   assert(sEntryNumber < 65536U &&
          "Index into ModRMDecision is too large for uint16_t!");
+  (void)sEntryNumber;
 
   ++sTableNumber;
 }
@@ -823,12 +830,9 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o,
   for (unsigned Index = 0; Index < NumInstructions; ++Index) {
     OperandListTy OperandList;
 
-    for (unsigned OperandIndex = 0; OperandIndex < X86_MAX_OPERANDS;
-         ++OperandIndex) {
-      OperandEncoding Encoding = (OperandEncoding)InstructionSpecifiers[Index]
-                                 .operands[OperandIndex].encoding;
-      OperandType Type = (OperandType)InstructionSpecifiers[Index]
-                         .operands[OperandIndex].type;
+    for (auto Operand : InstructionSpecifiers[Index].operands) {
+      OperandEncoding Encoding = (OperandEncoding)Operand.encoding;
+      OperandType Type = (OperandType)Operand.type;
       OperandList.push_back(std::make_pair(Encoding, Type));
     }
     unsigned &N = OperandSets[OperandList];
@@ -856,12 +860,9 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o,
     i++;
 
     OperandListTy OperandList;
-    for (unsigned OperandIndex = 0; OperandIndex < X86_MAX_OPERANDS;
-         ++OperandIndex) {
-      OperandEncoding Encoding = (OperandEncoding)InstructionSpecifiers[index]
-                                 .operands[OperandIndex].encoding;
-      OperandType Type = (OperandType)InstructionSpecifiers[index]
-                         .operands[OperandIndex].type;
+    for (auto Operand : InstructionSpecifiers[index].operands) {
+      OperandEncoding Encoding = (OperandEncoding)Operand.encoding;
+      OperandType Type = (OperandType)Operand.type;
       OperandList.push_back(std::make_pair(Encoding, Type));
     }
     o.indent(i * 2) << (OperandSets[OperandList] - 1) << ",\n";
@@ -887,6 +888,9 @@ void DisassemblerTables::emitContextTable(raw_ostream &o, unsigned &i) const {
     if ((index & ATTR_EVEX) || (index & ATTR_VEX) || (index & ATTR_VEXL)) {
       if (index & ATTR_EVEX)
         o << "IC_EVEX";
+      else if ((index & (ATTR_64BIT | ATTR_VEXL | ATTR_REXW | ATTR_OPSIZE)) ==
+               (ATTR_64BIT | ATTR_OPSIZE))
+        o << "IC_64BIT_VEX";
       else
         o << "IC_VEX";
 
@@ -898,9 +902,13 @@ void DisassemblerTables::emitContextTable(raw_ostream &o, unsigned &i) const {
       if (index & ATTR_REXW)
         o << "_W";
 
-      if (index & ATTR_OPSIZE)
+      if (index & ATTR_OPSIZE) {
         o << "_OPSIZE";
-      else if (index & ATTR_XD)
+        if ((index & (ATTR_64BIT | ATTR_EVEX | ATTR_VEX | ATTR_VEXL |
+                      ATTR_REXW | ATTR_ADSIZE)) ==
+            (ATTR_64BIT | ATTR_VEX | ATTR_ADSIZE))
+          o << "_ADSIZE";
+      } else if (index & ATTR_XD)
         o << "_XD";
       else if (index & ATTR_XS)
         o << "_XS";
@@ -914,8 +922,7 @@ void DisassemblerTables::emitContextTable(raw_ostream &o, unsigned &i) const {
         if (index & ATTR_EVEXB)
           o << "_B";
       }
-    }
-    else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XS))
+    } else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XS))
       o << "IC_64BIT_REXW_XS";
     else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XD))
       o << "IC_64BIT_REXW_XD";
diff --git a/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp b/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
index 6dc7e31e0dab..009dc036cf97 100644
--- a/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
@@ -30,10 +30,13 @@ class X86EVEX2VEXTablesEmitter {
   std::map<uint64_t, std::vector<const CodeGenInstruction *>> VEXInsts;
 
   typedef std::pair<const CodeGenInstruction *, const CodeGenInstruction *> Entry;
+  typedef std::pair<StringRef, StringRef> Predicate;
 
   // Represent both compress tables
   std::vector<Entry> EVEX2VEX128;
   std::vector<Entry> EVEX2VEX256;
+  // Represent predicates of VEX instructions.
+  std::vector<Predicate> EVEX2VEXPredicates;
 
 public:
   X86EVEX2VEXTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {}
@@ -45,6 +48,9 @@ private:
   // Prints the given table as a C++ array of type
   // X86EvexToVexCompressTableEntry
   void printTable(const std::vector<Entry> &Table, raw_ostream &OS);
+  // Prints function which checks target feature specific predicate.
+  void printCheckPredicate(const std::vector<Predicate> &Predicates,
+                           raw_ostream &OS);
 };
 
 void X86EVEX2VEXTablesEmitter::printTable(const std::vector<Entry> &Table,
@@ -67,6 +73,19 @@ void X86EVEX2VEXTablesEmitter::printTable(const std::vector<Entry> &Table,
   OS << "};\n\n";
 }
 
+void X86EVEX2VEXTablesEmitter::printCheckPredicate(
+    const std::vector<Predicate> &Predicates, raw_ostream &OS) {
+  OS << "static bool CheckVEXInstPredicate"
+     << "(MachineInstr &MI, const X86Subtarget *Subtarget) {\n"
+     << "  unsigned Opc = MI.getOpcode();\n"
+     << "  switch (Opc) {\n"
+     << "    default: return true;\n";
+  for (auto Pair : Predicates)
+    OS << "    case X86::" << Pair.first << ": return " << Pair.second << ";\n";
+  OS << "  }\n"
+     << "}\n\n";
+}
+
 // Return true if the 2 BitsInits are equal
 // Calculates the integer value residing BitsInit object
 static inline uint64_t getValueFromBitsInit(const BitsInit *B) {
@@ -169,6 +188,18 @@ private:
 };
 
 void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) {
+  auto getPredicates = [&](const CodeGenInstruction *Inst) {
+    std::vector<Record *> PredicatesRecords =
+        Inst->TheDef->getValueAsListOfDefs("Predicates");
+    // Currently we only do AVX related checks and assume each instruction
+    // has one and only one AVX related predicates.
+    for (unsigned i = 0, e = PredicatesRecords.size(); i != e; ++i)
+      if (PredicatesRecords[i]->getName().startswith("HasAVX"))
+        return PredicatesRecords[i]->getValueAsString("CondString");
+    llvm_unreachable(
+        "Instruction with checkPredicate set must have one predicate!");
+  };
+
   emitSourceFileHeader("X86 EVEX2VEX tables", OS);
 
   ArrayRef<const CodeGenInstruction *> NumberedInstructions =
@@ -222,11 +253,18 @@ void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) {
       EVEX2VEX256.push_back(std::make_pair(EVEXInst, VEXInst)); // {0,1}
     else
       EVEX2VEX128.push_back(std::make_pair(EVEXInst, VEXInst)); // {0,0}
+
+    // Adding predicate check to EVEX2VEXPredicates table when needed.
+    if (VEXInst->TheDef->getValueAsBit("checkVEXPredicate"))
+      EVEX2VEXPredicates.push_back(
+          std::make_pair(EVEXInst->TheDef->getName(), getPredicates(VEXInst)));
   }
 
   // Print both tables
   printTable(EVEX2VEX128, OS);
   printTable(EVEX2VEX256, OS);
+  // Print CheckVEXInstPredicate function.
+  printCheckPredicate(EVEX2VEXPredicates, OS);
 }
 }
 
diff --git a/llvm/utils/TableGen/X86RecognizableInstr.cpp b/llvm/utils/TableGen/X86RecognizableInstr.cpp
index e4b7c05cfb88..c2ca3791ac36 100644
--- a/llvm/utils/TableGen/X86RecognizableInstr.cpp
+++ b/llvm/utils/TableGen/X86RecognizableInstr.cpp
@@ -125,13 +125,7 @@ RecognizableInstr::RecognizableInstr(DisassemblerTables &tables,
     return;
   }
 
-  // Special case since there is no attribute class for 64-bit and VEX
-  if (Name == "VMASKMOVDQU64") {
-    ShouldBeEmitted = false;
-    return;
-  }
-
-  ShouldBeEmitted  = true;
+  ShouldBeEmitted = true;
 }
 
 void RecognizableInstr::processInstr(DisassemblerTables &tables,
@@ -267,6 +261,11 @@ InstructionContext RecognizableInstr::insnContext() const {
       insnContext = IC_VEX_L_OPSIZE;
     else if (OpPrefix == X86Local::PD && HasVEX_W)
       insnContext = IC_VEX_W_OPSIZE;
+    else if (OpPrefix == X86Local::PD && Is64Bit &&
+             AdSize == X86Local::AdSize32)
+      insnContext = IC_64BIT_VEX_OPSIZE_ADSIZE;
+    else if (OpPrefix == X86Local::PD && Is64Bit)
+      insnContext = IC_64BIT_VEX_OPSIZE;
     else if (OpPrefix == X86Local::PD)
       insnContext = IC_VEX_OPSIZE;
     else if (HasVEX_LPrefix && OpPrefix == X86Local::XS)