vendor/llvm-project/llvmorg-17-init-19304-gd0b54bb50e51

87 files changed, 9691 insertions, 7298 deletions

diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index c13e5b5deff6..1c195200a888 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -95,7 +95,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CodeGenInstAlias.h"
 #include "CodeGenInstruction.h"
+#include "CodeGenRegisters.h"
 #include "CodeGenTarget.h"
 #include "SubtargetFeatureInfo.h"
 #include "Types.h"
@@ -105,7 +107,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Config/llvm-config.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -505,9 +506,9 @@ struct MatchableInfo {
   PointerUnion<const CodeGenInstruction*, const CodeGenInstAlias*> DefRec;
 
   const CodeGenInstruction *getResultInst() const {
-    if (DefRec.is<const CodeGenInstruction*>())
-      return DefRec.get<const CodeGenInstruction*>();
-    return DefRec.get<const CodeGenInstAlias*>()->ResultInst;
+    if (isa<const CodeGenInstruction *>(DefRec))
+      return cast<const CodeGenInstruction *>(DefRec);
+    return cast<const CodeGenInstAlias *>(DefRec)->ResultInst;
   }
 
   /// ResOperands - This is the operand list that should be built for the result
@@ -533,7 +534,7 @@ struct MatchableInfo {
   std::string ConversionFnKind;
 
   /// If this instruction is deprecated in some form.
-  bool HasDeprecation;
+  bool HasDeprecation = false;
 
   /// If this is an alias, this is use to determine whether or not to using
   /// the conversion function defined by the instruction's AsmMatchConverter
@@ -563,11 +564,11 @@ struct MatchableInfo {
         ConversionFnKind(RHS.ConversionFnKind),
         HasDeprecation(RHS.HasDeprecation),
         UseInstAsmMatchConverter(RHS.UseInstAsmMatchConverter) {
-    assert(!DefRec.is<const CodeGenInstAlias *>());
+    assert(!isa<const CodeGenInstAlias *>(DefRec));
   }
 
   ~MatchableInfo() {
-    delete DefRec.dyn_cast<const CodeGenInstAlias*>();
+    delete dyn_cast_if_present<const CodeGenInstAlias *>(DefRec);
   }
 
   // Two-operand aliases clone from the main matchable, but mark the second
@@ -629,6 +630,17 @@ struct MatchableInfo {
         return false;
     }
 
+    // For X86 AVX/AVX512 instructions, we prefer vex encoding because the
+    // vex encoding size is smaller. Since X86InstrSSE.td is included ahead
+    // of X86InstrAVX512.td, the AVX instruction ID is less than AVX512 ID.
+    // We use the ID to sort AVX instruction before AVX512 instruction in
+    // matching table.
+    if (TheDef->isSubClassOf("Instruction") &&
+        TheDef->getValueAsBit("HasPositionOrder") &&
+        RHS.TheDef->isSubClassOf("Instruction") &&
+        RHS.TheDef->getValueAsBit("HasPositionOrder"))
+      return TheDef->getID() < RHS.TheDef->getID();
+
     // Give matches that require more features higher precedence. This is useful
     // because we cannot define AssemblerPredicates with the negation of
     // processor features. For example, ARM v6 "nop" may be either a HINT or
@@ -638,15 +650,6 @@ struct MatchableInfo {
     if (RequiredFeatures.size() != RHS.RequiredFeatures.size())
       return RequiredFeatures.size() > RHS.RequiredFeatures.size();
 
-    // For X86 AVX/AVX512 instructions, we prefer vex encoding because the
-    // vex encoding size is smaller. Since X86InstrSSE.td is included ahead
-    // of X86InstrAVX512.td, the AVX instruction ID is less than AVX512 ID.
-    // We use the ID to sort AVX instruction before AVX512 instruction in
-    // matching table.
-    if (TheDef->isSubClassOf("Instruction") &&
-        TheDef->getValueAsBit("HasPositionOrder"))
-      return TheDef->getID() < RHS.TheDef->getID();
-
     return false;
   }
 
@@ -1613,13 +1616,13 @@ void AsmMatcherInfo::buildInfo() {
       else
         OperandName = Token.substr(1);
 
-      if (II->DefRec.is<const CodeGenInstruction*>())
+      if (isa<const CodeGenInstruction *>(II->DefRec))
         buildInstructionOperandReference(II.get(), OperandName, i);
       else
         buildAliasOperandReference(II.get(), OperandName, Op);
     }
 
-    if (II->DefRec.is<const CodeGenInstruction*>()) {
+    if (isa<const CodeGenInstruction *>(II->DefRec)) {
       II->buildInstructionResultOperands();
       // If the instruction has a two-operand alias, build up the
       // matchable here. We'll add them in bulk at the end to avoid
@@ -1682,7 +1685,7 @@ void AsmMatcherInfo::
 buildInstructionOperandReference(MatchableInfo *II,
                                  StringRef OperandName,
                                  unsigned AsmOpIdx) {
-  const CodeGenInstruction &CGI = *II->DefRec.get<const CodeGenInstruction*>();
+  const CodeGenInstruction &CGI = *cast<const CodeGenInstruction *>(II->DefRec);
   const CGIOperandList &Operands = CGI.Operands;
   MatchableInfo::AsmOperand *Op = &II->AsmOperands[AsmOpIdx];
 
@@ -1745,7 +1748,7 @@ buildInstructionOperandReference(MatchableInfo *II,
 void AsmMatcherInfo::buildAliasOperandReference(MatchableInfo *II,
                                                 StringRef OperandName,
                                                 MatchableInfo::AsmOperand &Op) {
-  const CodeGenInstAlias &CGA = *II->DefRec.get<const CodeGenInstAlias*>();
+  const CodeGenInstAlias &CGA = *cast<const CodeGenInstAlias *>(II->DefRec);
 
   // Set up the operand class.
   for (unsigned i = 0, e = CGA.ResultOperands.size(); i != e; ++i)
@@ -1818,7 +1821,7 @@ void MatchableInfo::buildInstructionResultOperands() {
 }
 
 void MatchableInfo::buildAliasResultOperands(bool AliasConstraintsAreChecked) {
-  const CodeGenInstAlias &CGA = *DefRec.get<const CodeGenInstAlias*>();
+  const CodeGenInstAlias &CGA = *cast<const CodeGenInstAlias *>(DefRec);
   const CodeGenInstruction *ResultInst = getResultInst();
 
   // Map of:  $reg -> #lastref
@@ -2924,7 +2927,7 @@ emitCustomOperandParsing(raw_ostream &OS, CodeGenTarget &Target,
 
   // Emit the operand class switch to call the correct custom parser for
   // the found operand class.
-  OS << "OperandMatchResultTy " << Target.getName() << ClassName << "::\n"
+  OS << "ParseStatus " << Target.getName() << ClassName << "::\n"
      << "tryCustomParseOperand(OperandVector"
      << " &Operands,\n                      unsigned MCK) {\n\n"
      << "  switch(MCK) {\n";
@@ -2937,15 +2940,15 @@ emitCustomOperandParsing(raw_ostream &OS, CodeGenTarget &Target,
   }
 
   OS << "  default:\n";
-  OS << "    return MatchOperand_NoMatch;\n";
+  OS << "    return ParseStatus::NoMatch;\n";
   OS << "  }\n";
-  OS << "  return MatchOperand_NoMatch;\n";
+  OS << "  return ParseStatus::NoMatch;\n";
   OS << "}\n\n";
 
   // Emit the static custom operand parser. This code is very similar with
   // the other matcher. Also use MatchResultTy here just in case we go for
   // a better error handling.
-  OS << "OperandMatchResultTy " << Target.getName() << ClassName << "::\n"
+  OS << "ParseStatus " << Target.getName() << ClassName << "::\n"
      << "MatchOperandParserImpl(OperandVector"
      << " &Operands,\n                       StringRef Mnemonic,\n"
      << "                       bool ParseForAllFeatures) {\n";
@@ -2976,7 +2979,7 @@ emitCustomOperandParsing(raw_ostream &OS, CodeGenTarget &Target,
   }
 
   OS << "  if (MnemonicRange.first == MnemonicRange.second)\n";
-  OS << "    return MatchOperand_NoMatch;\n\n";
+  OS << "    return ParseStatus::NoMatch;\n\n";
 
   OS << "  for (const OperandMatchEntry *it = MnemonicRange.first,\n"
      << "       *ie = MnemonicRange.second; it != ie; ++it) {\n";
@@ -3002,14 +3005,13 @@ emitCustomOperandParsing(raw_ostream &OS, CodeGenTarget &Target,
   if (ParserName.empty())
     ParserName = "tryCustomParseOperand";
   OS << "    // call custom parse method to handle the operand\n";
-  OS << "    OperandMatchResultTy Result = " << ParserName
-     << "(Operands, it->Class);\n";
-  OS << "    if (Result != MatchOperand_NoMatch)\n";
+  OS << "    ParseStatus Result = " << ParserName << "(Operands, it->Class);\n";
+  OS << "    if (!Result.isNoMatch())\n";
   OS << "      return Result;\n";
   OS << "  }\n\n";
 
   OS << "  // Okay, we had no match.\n";
-  OS << "  return MatchOperand_NoMatch;\n";
+  OS << "  return ParseStatus::NoMatch;\n";
   OS << "}\n\n";
 }
 
@@ -3202,6 +3204,8 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
   Record *AsmParser = Target.getAsmParser();
   StringRef ClassName = AsmParser->getValueAsString("AsmParserClassName");
 
+  emitSourceFileHeader("Assembly Matcher Source Fragment", OS);
+
   // Compute the information on the instructions to match.
   AsmMatcherInfo Info(AsmParser, Target, Records);
   Info.buildInfo();
@@ -3303,12 +3307,12 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
 
 
   if (!Info.OperandMatchInfo.empty()) {
-    OS << "  OperandMatchResultTy MatchOperandParserImpl(\n";
+    OS << "  ParseStatus MatchOperandParserImpl(\n";
     OS << "    OperandVector &Operands,\n";
     OS << "    StringRef Mnemonic,\n";
     OS << "    bool ParseForAllFeatures = false);\n";
 
-    OS << "  OperandMatchResultTy tryCustomParseOperand(\n";
+    OS << "  ParseStatus tryCustomParseOperand(\n";
     OS << "    OperandVector &Operands,\n";
     OS << "    unsigned MCK);\n\n";
   }
@@ -3999,11 +4003,5 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
   OS << "#endif // GET_MNEMONIC_CHECKER\n\n";
 }
 
-namespace llvm {
-
-void EmitAsmMatcher(RecordKeeper &RK, raw_ostream &OS) {
-  emitSourceFileHeader("Assembly Matcher Source Fragment", OS);
-  AsmMatcherEmitter(RK).run(OS);
-}
-
-} // end namespace llvm
+static TableGen::Emitter::OptClass<AsmMatcherEmitter>
+    X("gen-asm-matcher", "Generate assembly instruction matcher");
diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp
index f2e4d15a2c75..92e71910a800 100644
--- a/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AsmWriterInst.h"
+#include "CodeGenInstAlias.h"
 #include "CodeGenInstruction.h"
 #include "CodeGenRegisters.h"
 #include "CodeGenTarget.h"
@@ -994,7 +995,10 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
 
       for (Record *const R : ReqFeatures) {
         const DagInit *D = R->getValueAsDag("AssemblerCondDag");
-        std::string CombineType = D->getOperator()->getAsString();
+        auto *Op = dyn_cast<DefInit>(D->getOperator());
+        if (!Op)
+          PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!");
+        StringRef CombineType = Op->getDef()->getName();
         if (CombineType != "any_of" && CombineType != "all_of")
           PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!");
         if (D->getNumArgs() == 0)
@@ -1002,7 +1006,7 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
         bool IsOr = CombineType == "any_of";
         // Change (any_of FeatureAll, (any_of ...)) to (any_of FeatureAll, ...).
         if (IsOr && D->getNumArgs() == 2 && isa<DagInit>(D->getArg(1))) {
-          DagInit *RHS = dyn_cast<DagInit>(D->getArg(1));
+          DagInit *RHS = cast<DagInit>(D->getArg(1));
           SmallVector<Init *> Args{D->getArg(0)};
           SmallVector<StringInit *> ArgNames{D->getArgName(0)};
           for (unsigned i = 0, e = RHS->getNumArgs(); i != e; ++i) {
@@ -1298,17 +1302,12 @@ void AsmWriterEmitter::run(raw_ostream &O) {
   std::vector<std::vector<std::string>> TableDrivenOperandPrinters;
   unsigned BitsLeft = 0;
   unsigned AsmStrBits = 0;
+  emitSourceFileHeader("Assembly Writer Source Fragment", O);
   EmitGetMnemonic(O, TableDrivenOperandPrinters, BitsLeft, AsmStrBits);
   EmitPrintInstruction(O, TableDrivenOperandPrinters, BitsLeft, AsmStrBits);
   EmitGetRegisterName(O);
   EmitPrintAliasInstruction(O);
 }
 
-namespace llvm {
-
-void EmitAsmWriter(RecordKeeper &RK, raw_ostream &OS) {
-  emitSourceFileHeader("Assembly Writer Source Fragment", OS);
-  AsmWriterEmitter(RK).run(OS);
-}
-
-} // end namespace llvm
+static TableGen::Emitter::OptClass<AsmWriterEmitter>
+    X("gen-asm-writer", "Generate assembly writer");
diff --git a/llvm/utils/TableGen/AsmWriterInst.cpp b/llvm/utils/TableGen/AsmWriterInst.cpp
index 4a78108d6f4a..c9558593e142 100644
--- a/llvm/utils/TableGen/AsmWriterInst.cpp
+++ b/llvm/utils/TableGen/AsmWriterInst.cpp
@@ -12,7 +12,6 @@
 
 #include "AsmWriterInst.h"
 #include "CodeGenInstruction.h"
-#include "CodeGenTarget.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
diff --git a/llvm/utils/TableGen/AsmWriterInst.h b/llvm/utils/TableGen/AsmWriterInst.h
index fe2b934e266f..9c93e82b611b 100644
--- a/llvm/utils/TableGen/AsmWriterInst.h
+++ b/llvm/utils/TableGen/AsmWriterInst.h
@@ -21,7 +21,6 @@
 
 namespace llvm {
   class CodeGenInstruction;
-  class Record;
 
   struct AsmWriterOperand {
     enum OpType {
diff --git a/llvm/utils/TableGen/Attributes.cpp b/llvm/utils/TableGen/Attributes.cpp
index 735c53dd6fcf..474042a3e9a3 100644
--- a/llvm/utils/TableGen/Attributes.cpp
+++ b/llvm/utils/TableGen/Attributes.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <vector>
 using namespace llvm;
 
@@ -17,7 +18,7 @@ namespace {
 class Attributes {
 public:
   Attributes(RecordKeeper &R) : Records(R) {}
-  void emit(raw_ostream &OS);
+  void run(raw_ostream &OS);
 
 private:
   void emitTargetIndependentNames(raw_ostream &OS);
@@ -54,6 +55,7 @@ void Attributes::emitTargetIndependentNames(raw_ostream &OS) {
   // Emit attribute enums in the same order llvm::Attribute::operator< expects.
   Emit({"EnumAttr", "TypeAttr", "IntAttr"}, "ATTRIBUTE_ENUM");
   Emit({"StrBoolAttr"}, "ATTRIBUTE_STRBOOL");
+  Emit({"ComplexStrAttr"}, "ATTRIBUTE_COMPLEXSTR");
 
   OS << "#undef ATTRIBUTE_ALL\n";
   OS << "#endif\n\n";
@@ -123,16 +125,11 @@ void Attributes::emitAttributeProperties(raw_ostream &OS) {
   OS << "#endif\n";
 }
 
-void Attributes::emit(raw_ostream &OS) {
+void Attributes::run(raw_ostream &OS) {
   emitTargetIndependentNames(OS);
   emitFnAttrCompatCheck(OS, false);
   emitAttributeProperties(OS);
 }
 
-namespace llvm {
-
-void EmitAttributes(RecordKeeper &RK, raw_ostream &OS) {
-  Attributes(RK).emit(OS);
-}
-
-} // End llvm namespace.
+static TableGen::Emitter::OptClass<Attributes> X("gen-attrs",
+                                                 "Generate attributes");
diff --git a/llvm/utils/TableGen/CTagsEmitter.cpp b/llvm/utils/TableGen/CTagsEmitter.cpp
index fe62d6a9b67f..b8e27d057d95 100644
--- a/llvm/utils/TableGen/CTagsEmitter.cpp
+++ b/llvm/utils/TableGen/CTagsEmitter.cpp
@@ -12,12 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SourceMgr.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <algorithm>
-#include <string>
 #include <vector>
 using namespace llvm;
 
@@ -86,8 +86,5 @@ void CTagsEmitter::run(raw_ostream &OS) {
     T.emit(OS);
 }
 
-namespace llvm {
-
-void EmitCTags(RecordKeeper &RK, raw_ostream &OS) { CTagsEmitter(RK).run(OS); }
-
-} // End llvm namespace.
+static TableGen::Emitter::OptClass<CTagsEmitter>
+    X("gen-ctags", "Generate ctags-compatible index");
diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp
index e8ec90e9c078..de3810b2e227 100644
--- a/llvm/utils/TableGen/CallingConvEmitter.cpp
+++ b/llvm/utils/TableGen/CallingConvEmitter.cpp
@@ -15,14 +15,16 @@
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
+#include <deque>
+
 using namespace llvm;
 
 namespace {
 class CallingConvEmitter {
   RecordKeeper &Records;
-  unsigned Counter;
+  unsigned Counter = 0u;
   std::string CurrentAction;
-  bool SwiftAction;
+  bool SwiftAction = false;
 
   std::map<std::string, std::set<std::string>> AssignedRegsMap;
   std::map<std::string, std::set<std::string>> AssignedSwiftRegsMap;
@@ -41,7 +43,9 @@ private:
 } // End anonymous namespace
 
 void CallingConvEmitter::run(raw_ostream &O) {
-  std::vector<Record*> CCs = Records.getAllDerivedDefinitions("CallingConv");
+  emitSourceFileHeader("Calling Convention Implementation Fragment", O);
+
+  std::vector<Record *> CCs = Records.getAllDerivedDefinitions("CallingConv");
 
   // Emit prototypes for all of the non-custom CC's so that they can forward ref
   // each other.
@@ -247,7 +251,7 @@ void CallingConvEmitter::EmitAction(Record *Action,
       int Size = Action->getValueAsInt("Size");
       int Align = Action->getValueAsInt("Align");
 
-      O << IndentStr << "unsigned Offset" << ++Counter
+      O << IndentStr << "int64_t Offset" << ++Counter
         << " = State.AllocateStack(";
       if (Size)
         O << Size << ", ";
@@ -283,7 +287,7 @@ void CallingConvEmitter::EmitAction(Record *Action,
         O << LS << getQualifiedName(ShadowRegList->getElementAsRecord(i));
       O << "\n" << IndentStr << "};\n";
 
-      O << IndentStr << "unsigned Offset" << ++Counter
+      O << IndentStr << "int64_t Offset" << ++Counter
         << " = State.AllocateStack(" << Size << ", Align(" << Align << "), "
         << "ShadowRegList" << ShadowRegListNumber << ");\n";
       O << IndentStr << "State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset"
@@ -426,11 +430,5 @@ void CallingConvEmitter::EmitArgRegisterLists(raw_ostream &O) {
   }
 }
 
-namespace llvm {
-
-void EmitCallingConv(RecordKeeper &RK, raw_ostream &OS) {
-  emitSourceFileHeader("Calling Convention Implementation Fragment", OS);
-  CallingConvEmitter(RK).run(OS);
-}
-
-} // End llvm namespace
+static TableGen::Emitter::OptClass<CallingConvEmitter>
+    X("gen-callingconv", "Generate calling convention descriptions");
diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp
index dc4fd589eaa8..48ed319bf06f 100644
--- a/llvm/utils/TableGen/CodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/CodeEmitterGen.cpp
@@ -7,15 +7,25 @@
 //===----------------------------------------------------------------------===//
 //
 // CodeEmitterGen uses the descriptions of instructions and their fields to
-// construct an automated code emitter: a function that, given a MachineInstr,
-// returns the (currently, 32-bit unsigned) value of the instruction.
+// construct an automated code emitter: a function called
+// getBinaryCodeForInstr() that, given a MCInst, returns the value of the
+// instruction - either as an uint64_t or as an APInt, depending on the
+// maximum bit width of all Inst definitions.
+//
+// In addition, it generates another function called getOperandBitOffset()
+// that, given a MCInst and an operand index, returns the minimum of indices of
+// all bits that carry some portion of the respective operand. When the target's
+// encodeInstruction() stores the instruction in a little-endian byte order, the
+// returned value is the offset of the start of the operand in the encoded
+// instruction. Other targets might need to adjust the returned value according
+// to their encodeInstruction() implementation.
 //
 //===----------------------------------------------------------------------===//
 
+#include "CodeGenHwModes.h"
 #include "CodeGenInstruction.h"
 #include "CodeGenTarget.h"
-#include "SubtargetFeatureInfo.h"
-#include "Types.h"
+#include "InfoByHwMode.h"
 #include "VarLenCodeEmitterGen.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -46,19 +56,24 @@ public:
 
 private:
   int getVariableBit(const std::string &VarName, BitsInit *BI, int bit);
-  std::string getInstructionCase(Record *R, CodeGenTarget &Target);
-  std::string getInstructionCaseForEncoding(Record *R, Record *EncodingDef,
-                                            CodeGenTarget &Target);
+  std::pair<std::string, std::string>
+  getInstructionCases(Record *R, CodeGenTarget &Target);
+  void addInstructionCasesForEncoding(Record *R, Record *EncodingDef,
+                                      CodeGenTarget &Target, std::string &Case,
+                                      std::string &BitOffsetCase);
   bool addCodeToMergeInOperand(Record *R, BitsInit *BI,
-                               const std::string &VarName, unsigned &NumberedOp,
-                               std::set<unsigned> &NamedOpIndices,
-                               std::string &Case, CodeGenTarget &Target);
+                               const std::string &VarName, std::string &Case,
+                               std::string &BitOffsetCase,
+                               CodeGenTarget &Target);
 
   void emitInstructionBaseValues(
       raw_ostream &o, ArrayRef<const CodeGenInstruction *> NumberedInstructions,
       CodeGenTarget &Target, int HwMode = -1);
-  unsigned BitWidth;
-  bool UseAPInt;
+  void
+  emitCaseMap(raw_ostream &o,
+              const std::map<std::string, std::vector<std::string>> &CaseMap);
+  unsigned BitWidth = 0u;
+  bool UseAPInt = false;
 };
 
 // If the VarBitInit at position 'bit' matches the specified variable then
@@ -80,9 +95,8 @@ int CodeEmitterGen::getVariableBit(const std::string &VarName,
 // Returns true if it succeeds, false if an error.
 bool CodeEmitterGen::addCodeToMergeInOperand(Record *R, BitsInit *BI,
                                              const std::string &VarName,
-                                             unsigned &NumberedOp,
-                                             std::set<unsigned> &NamedOpIndices,
                                              std::string &Case,
+                                             std::string &BitOffsetCase,
                                              CodeGenTarget &Target) {
   CodeGenInstruction &CGI = Target.getInstruction(R);
 
@@ -113,52 +127,8 @@ bool CodeEmitterGen::addCodeToMergeInOperand(Record *R, BitsInit *BI,
     // Get the machine operand number for the indicated operand.
     OpIdx = CGI.Operands[OpIdx].MIOperandNo;
   } else {
-    // Fall back to positional lookup. By default, we now disable positional
-    // lookup (and print an error, below), but even so, we'll do the lookup to
-    // help print a helpful diagnostic message.
-    //
-    // TODO: When we remove useDeprecatedPositionallyEncodedOperands, delete all
-    // this code, just leaving a "no operand named X in record Y" error.
-
-    unsigned NumberOps = CGI.Operands.size();
-    /// If this operand is not supposed to be emitted by the
-    /// generated emitter, skip it.
-    while (NumberedOp < NumberOps &&
-           (CGI.Operands.isFlatOperandNotEmitted(NumberedOp) ||
-              (!NamedOpIndices.empty() && NamedOpIndices.count(
-                CGI.Operands.getSubOperandNumber(NumberedOp).first)))) {
-      ++NumberedOp;
-    }
-
-    if (NumberedOp >=
-        CGI.Operands.back().MIOperandNo + CGI.Operands.back().MINumOperands) {
-      if (!Target.getInstructionSet()->getValueAsBit(
-              "useDeprecatedPositionallyEncodedOperands")) {
-        PrintError(R, Twine("No operand named ") + VarName + " in record " +
-                          R->getName() +
-                          " (would've given 'too few operands' error with "
-                          "useDeprecatedPositionallyEncodedOperands=true)");
-      } else {
-        PrintError(R, "Too few operands in record " + R->getName() +
-                          " (no match for variable " + VarName + ")");
-      }
-      return false;
-    }
-
-    OpIdx = NumberedOp++;
-
-    if (!Target.getInstructionSet()->getValueAsBit(
-            "useDeprecatedPositionallyEncodedOperands")) {
-      std::pair<unsigned, unsigned> SO =
-          CGI.Operands.getSubOperandNumber(OpIdx);
-      std::string OpName = CGI.Operands[SO.first].Name;
-      PrintError(R, Twine("No operand named ") + VarName + " in record " +
-                        R->getName() + " (would've used positional operand #" +
-                        Twine(SO.first) + " ('" + OpName + "') sub-op #" +
-                        Twine(SO.second) +
-                        " with useDeprecatedPositionallyEncodedOperands=true)");
-      return false;
-    }
+    PrintError(R, Twine("No operand named ") + VarName + " in record " + R->getName());
+    return false;
   }
 
   if (CGI.Operands.isFlatOperandNotEmitted(OpIdx)) {
@@ -222,6 +192,7 @@ bool CodeEmitterGen::addCodeToMergeInOperand(Record *R, BitsInit *BI,
     ++numOperandLits;
   }
 
+  unsigned BitOffset = -1;
   for (; bit >= 0; ) {
     int varBit = getVariableBit(VarName, BI, bit);
     
@@ -230,7 +201,7 @@ bool CodeEmitterGen::addCodeToMergeInOperand(Record *R, BitsInit *BI,
       --bit;
       continue;
     }
-    
+
     // Figure out the consecutive range of bits covered by this operand, in
     // order to generate better encoding code.
     int beginInstBit = bit;
@@ -249,6 +220,7 @@ bool CodeEmitterGen::addCodeToMergeInOperand(Record *R, BitsInit *BI,
     unsigned loBit = beginVarBit - N + 1;
     unsigned hiBit = loBit + N;
     unsigned loInstBit = beginInstBit - N + 1;
+    BitOffset = loInstBit;
     if (UseAPInt) {
       std::string extractStr;
       if (N >= 64) {
@@ -290,65 +262,71 @@ bool CodeEmitterGen::addCodeToMergeInOperand(Record *R, BitsInit *BI,
       }
     }
   }
+
+  if (BitOffset != (unsigned)-1) {
+    BitOffsetCase += "      case " + utostr(OpIdx) + ":\n";
+    BitOffsetCase += "        // op: " + VarName + "\n";
+    BitOffsetCase += "        return " + utostr(BitOffset) + ";\n";
+  }
+
   return true;
 }
 
-std::string CodeEmitterGen::getInstructionCase(Record *R,
-                                               CodeGenTarget &Target) {
-  std::string Case;
+std::pair<std::string, std::string>
+CodeEmitterGen::getInstructionCases(Record *R, CodeGenTarget &Target) {
+  std::string Case, BitOffsetCase;
+
+  auto append = [&](const char *S) {
+    Case += S;
+    BitOffsetCase += S;
+  };
+
   if (const RecordVal *RV = R->getValue("EncodingInfos")) {
     if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
       const CodeGenHwModes &HWM = Target.getHwModes();
       EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-      Case += "      switch (HwMode) {\n";
-      Case += "      default: llvm_unreachable(\"Unhandled HwMode\");\n";
+      append("      switch (HwMode) {\n");
+      append("      default: llvm_unreachable(\"Unhandled HwMode\");\n");
       for (auto &KV : EBM) {
-        Case += "      case " + itostr(KV.first) + ": {\n";
-        Case += getInstructionCaseForEncoding(R, KV.second, Target);
-        Case += "      break;\n";
-        Case += "      }\n";
+        append(("      case " + itostr(KV.first) + ": {\n").c_str());
+        addInstructionCasesForEncoding(R, KV.second, Target, Case,
+                                       BitOffsetCase);
+        append("      break;\n");
+        append("      }\n");
       }
-      Case += "      }\n";
-      return Case;
+      append("      }\n");
+      return std::make_pair(std::move(Case), std::move(BitOffsetCase));
     }
   }
-  return getInstructionCaseForEncoding(R, R, Target);
+  addInstructionCasesForEncoding(R, R, Target, Case, BitOffsetCase);
+  return std::make_pair(std::move(Case), std::move(BitOffsetCase));
 }
 
-std::string CodeEmitterGen::getInstructionCaseForEncoding(Record *R, Record *EncodingDef,
-                                                          CodeGenTarget &Target) {
-  std::string Case;
+void CodeEmitterGen::addInstructionCasesForEncoding(
+    Record *R, Record *EncodingDef, CodeGenTarget &Target, std::string &Case,
+    std::string &BitOffsetCase) {
   BitsInit *BI = EncodingDef->getValueAsBitsInit("Inst");
-  unsigned NumberedOp = 0;
-  std::set<unsigned> NamedOpIndices;
-
-  // Collect the set of operand indices that might correspond to named
-  // operand, and skip these when assigning operands based on position.
-  if (Target.getInstructionSet()->
-       getValueAsBit("noNamedPositionallyEncodedOperands")) {
-    CodeGenInstruction &CGI = Target.getInstruction(R);
-    for (const RecordVal &RV : R->getValues()) {
-      unsigned OpIdx;
-      if (!CGI.Operands.hasOperandNamed(RV.getName(), OpIdx))
-        continue;
-
-      NamedOpIndices.insert(OpIdx);
-    }
-  }
 
   // Loop over all of the fields in the instruction, determining which are the
   // operands to the instruction.
   bool Success = true;
+  size_t OrigBitOffsetCaseSize = BitOffsetCase.size();
+  BitOffsetCase += "      switch (OpNum) {\n";
+  size_t BitOffsetCaseSizeBeforeLoop = BitOffsetCase.size();
   for (const RecordVal &RV : EncodingDef->getValues()) {
     // Ignore fixed fields in the record, we're looking for values like:
     //    bits<5> RST = { ?, ?, ?, ?, ? };
     if (RV.isNonconcreteOK() || RV.getValue()->isComplete())
       continue;
 
-    Success &=
-        addCodeToMergeInOperand(R, BI, std::string(RV.getName()), NumberedOp,
-                                NamedOpIndices, Case, Target);
+    Success &= addCodeToMergeInOperand(R, BI, std::string(RV.getName()), Case,
+                                       BitOffsetCase, Target);
   }
+  // Avoid empty switches.
+  if (BitOffsetCase.size() == BitOffsetCaseSizeBeforeLoop)
+    BitOffsetCase.resize(OrigBitOffsetCaseSize);
+  else
+    BitOffsetCase += "      }\n";
 
   if (!Success) {
     // Dump the record, so we can see what's going on...
@@ -367,8 +345,6 @@ std::string CodeEmitterGen::getInstructionCaseForEncoding(Record *R, Record *Enc
     Case += ", STI";
     Case += ");\n";
   }
-  
-  return Case;
 }
 
 static void emitInstBits(raw_ostream &OS, const APInt &Bits) {
@@ -419,7 +395,29 @@ void CodeEmitterGen::emitInstructionBaseValues(
   o << "    UINT64_C(0)\n  };\n";
 }
 
+void CodeEmitterGen::emitCaseMap(
+    raw_ostream &o,
+    const std::map<std::string, std::vector<std::string>> &CaseMap) {
+  std::map<std::string, std::vector<std::string>>::const_iterator IE, EE;
+  for (IE = CaseMap.begin(), EE = CaseMap.end(); IE != EE; ++IE) {
+    const std::string &Case = IE->first;
+    const std::vector<std::string> &InstList = IE->second;
+
+    for (int i = 0, N = InstList.size(); i < N; i++) {
+      if (i)
+        o << "\n";
+      o << "    case " << InstList[i] << ":";
+    }
+    o << " {\n";
+    o << Case;
+    o << "      break;\n"
+      << "    }\n";
+  }
+}
+
 void CodeEmitterGen::run(raw_ostream &o) {
+  emitSourceFileHeader("Machine Code Emitter", o);
+
   CodeGenTarget Target(Records);
   std::vector<Record*> Insts = Records.getAllDerivedDefinitions("Instruction");
 
@@ -498,6 +496,7 @@ void CodeEmitterGen::run(raw_ostream &o) {
 
     // Map to accumulate all the cases.
     std::map<std::string, std::vector<std::string>> CaseMap;
+    std::map<std::string, std::vector<std::string>> BitOffsetCaseMap;
 
     // Construct all cases statement for each opcode
     for (Record *R : Insts) {
@@ -506,9 +505,11 @@ void CodeEmitterGen::run(raw_ostream &o) {
         continue;
       std::string InstName =
           (R->getValueAsString("Namespace") + "::" + R->getName()).str();
-      std::string Case = getInstructionCase(R, Target);
+      std::string Case, BitOffsetCase;
+      std::tie(Case, BitOffsetCase) = getInstructionCases(R, Target);
 
-      CaseMap[Case].push_back(std::move(InstName));
+      CaseMap[Case].push_back(InstName);
+      BitOffsetCaseMap[BitOffsetCase].push_back(std::move(InstName));
     }
 
     // Emit initial function code
@@ -531,21 +532,7 @@ void CodeEmitterGen::run(raw_ostream &o) {
     }
 
     // Emit each case statement
-    std::map<std::string, std::vector<std::string>>::iterator IE, EE;
-    for (IE = CaseMap.begin(), EE = CaseMap.end(); IE != EE; ++IE) {
-      const std::string &Case = IE->first;
-      std::vector<std::string> &InstList = IE->second;
-
-      for (int i = 0, N = InstList.size(); i < N; i++) {
-        if (i)
-          o << "\n";
-        o << "    case " << InstList[i] << ":";
-      }
-      o << " {\n";
-      o << Case;
-      o << "      break;\n"
-        << "    }\n";
-    }
+    emitCaseMap(o, CaseMap);
 
     // Default case: unhandled opcode
     o << "  default:\n"
@@ -559,16 +546,27 @@ void CodeEmitterGen::run(raw_ostream &o) {
     else
       o << "  return Value;\n";
     o << "}\n\n";
+
+    o << "#ifdef GET_OPERAND_BIT_OFFSET\n"
+      << "#undef GET_OPERAND_BIT_OFFSET\n\n"
+      << "uint32_t " << Target.getName()
+      << "MCCodeEmitter::getOperandBitOffset(const MCInst &MI,\n"
+      << "    unsigned OpNum,\n"
+      << "    const MCSubtargetInfo &STI) const {\n"
+      << "  switch (MI.getOpcode()) {\n";
+    emitCaseMap(o, BitOffsetCaseMap);
+    o << "  }\n"
+      << "  std::string msg;\n"
+      << "  raw_string_ostream Msg(msg);\n"
+      << "  Msg << \"Not supported instr[opcode]: \" << MI << \"[\" << OpNum "
+         "<< \"]\";\n"
+      << "  report_fatal_error(Msg.str().c_str());\n"
+      << "}\n\n"
+      << "#endif // GET_OPERAND_BIT_OFFSET\n\n";
   }
 }
 
 } // end anonymous namespace
 
-namespace llvm {
-
-void EmitCodeEmitter(RecordKeeper &RK, raw_ostream &OS) {
-  emitSourceFileHeader("Machine Code Emitter", OS);
-  CodeEmitterGen(RK).run(OS);
-}
-
-} // end namespace llvm
+static TableGen::Emitter::OptClass<CodeEmitterGen>
+    X("gen-emitter", "Generate machine code emitter");
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
index dd04778e2dbe..e481f7e38e6a 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -13,6 +13,7 @@
 
 #include "CodeGenDAGPatterns.h"
 #include "CodeGenInstruction.h"
+#include "CodeGenRegisters.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
@@ -82,10 +83,12 @@ void MachineValueTypeSet::writeToStream(raw_ostream &OS) const {
 // inference will apply to each mode separately.
 
 TypeSetByHwMode::TypeSetByHwMode(ArrayRef<ValueTypeByHwMode> VTList) {
-  for (const ValueTypeByHwMode &VVT : VTList) {
+  // Take the address space from the first type in the list.
+  if (!VTList.empty())
+    AddrSpace = VTList[0].PtrAddrSpace;
+
+  for (const ValueTypeByHwMode &VVT : VTList)
     insert(VVT);
-    AddrSpaces.push_back(VVT.PtrAddrSpace);
-  }
 }
 
 bool TypeSetByHwMode::isValueTypeByHwMode(bool AllowEmpty) const {
@@ -102,13 +105,11 @@ ValueTypeByHwMode TypeSetByHwMode::getValueTypeByHwMode() const {
   assert(isValueTypeByHwMode(true) &&
          "The type set has multiple types for at least one HW mode");
   ValueTypeByHwMode VVT;
-  auto ASI = AddrSpaces.begin();
+  VVT.PtrAddrSpace = AddrSpace;
 
   for (const auto &I : *this) {
     MVT T = I.second.empty() ? MVT::Other : *I.second.begin();
     VVT.getOrCreateTypeForMode(I.first, T);
-    if (ASI != AddrSpaces.end())
-      VVT.PtrAddrSpace = *ASI++;
   }
   return VVT;
 }
@@ -217,7 +218,7 @@ bool TypeSetByHwMode::operator==(const TypeSetByHwMode &VTS) const {
   bool IsSimple = isSimple();
   bool VTSIsSimple = VTS.isSimple();
   if (IsSimple && VTSIsSimple)
-    return *begin() == *VTS.begin();
+    return getSimple() == VTS.getSimple();
 
   // Speedup: We have a default if the set is simple.
   bool HaveDefault = IsSimple || hasDefault();
@@ -354,21 +355,18 @@ bool TypeSetByHwMode::intersect(SetType &Out, const SetType &In) {
 }
 
 bool TypeSetByHwMode::validate() const {
-#ifndef NDEBUG
   if (empty())
     return true;
   bool AllEmpty = true;
   for (const auto &I : *this)
     AllEmpty &= I.second.empty();
   return !AllEmpty;
-#endif
-  return true;
 }
 
 // --- TypeInfer
 
 bool TypeInfer::MergeInTypeInfo(TypeSetByHwMode &Out,
-                                const TypeSetByHwMode &In) {
+                                const TypeSetByHwMode &In) const {
   ValidateOnExit _1(Out, *this);
   In.validate();
   if (In.empty() || Out == In || TP.hasError())
@@ -809,72 +807,57 @@ bool TypeInfer::EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B) {
   return Changed;
 }
 
-void TypeInfer::expandOverloads(TypeSetByHwMode &VTS) {
+void TypeInfer::expandOverloads(TypeSetByHwMode &VTS) const {
   ValidateOnExit _1(VTS, *this);
   const TypeSetByHwMode &Legal = getLegalTypes();
-  assert(Legal.isDefaultOnly() && "Default-mode only expected");
-  const TypeSetByHwMode::SetType &LegalTypes = Legal.get(DefaultMode);
+  assert(Legal.isSimple() && "Default-mode only expected");
+  const TypeSetByHwMode::SetType &LegalTypes = Legal.getSimple();
 
   for (auto &I : VTS)
     expandOverloads(I.second, LegalTypes);
 }
 
 void TypeInfer::expandOverloads(TypeSetByHwMode::SetType &Out,
-                                const TypeSetByHwMode::SetType &Legal) {
-  std::set<MVT> Ovs;
-  for (MVT T : Out) {
-    if (!T.isOverloaded())
-      continue;
-
-    Ovs.insert(T);
-    // MachineValueTypeSet allows iteration and erasing.
-    Out.erase(T);
-  }
-
-  for (MVT Ov : Ovs) {
-    switch (Ov.SimpleTy) {
-      case MVT::iPTRAny:
-        Out.insert(MVT::iPTR);
-        return;
-      case MVT::iAny:
-        for (MVT T : MVT::integer_valuetypes())
-          if (Legal.count(T))
-            Out.insert(T);
-        for (MVT T : MVT::integer_fixedlen_vector_valuetypes())
-          if (Legal.count(T))
-            Out.insert(T);
-        for (MVT T : MVT::integer_scalable_vector_valuetypes())
-          if (Legal.count(T))
-            Out.insert(T);
-        return;
-      case MVT::fAny:
-        for (MVT T : MVT::fp_valuetypes())
-          if (Legal.count(T))
-            Out.insert(T);
-        for (MVT T : MVT::fp_fixedlen_vector_valuetypes())
-          if (Legal.count(T))
-            Out.insert(T);
-        for (MVT T : MVT::fp_scalable_vector_valuetypes())
-          if (Legal.count(T))
-            Out.insert(T);
-        return;
-      case MVT::vAny:
-        for (MVT T : MVT::vector_valuetypes())
-          if (Legal.count(T))
-            Out.insert(T);
-        return;
-      case MVT::Any:
-        for (MVT T : MVT::all_valuetypes())
-          if (Legal.count(T))
-            Out.insert(T);
-        return;
-      default:
-        break;
-    }
+                                const TypeSetByHwMode::SetType &Legal) const {
+  if (Out.count(MVT::iPTRAny)) {
+    Out.erase(MVT::iPTRAny);
+    Out.insert(MVT::iPTR);
+  } else if (Out.count(MVT::iAny)) {
+    Out.erase(MVT::iAny);
+    for (MVT T : MVT::integer_valuetypes())
+      if (Legal.count(T))
+        Out.insert(T);
+    for (MVT T : MVT::integer_fixedlen_vector_valuetypes())
+      if (Legal.count(T))
+        Out.insert(T);
+    for (MVT T : MVT::integer_scalable_vector_valuetypes())
+      if (Legal.count(T))
+        Out.insert(T);
+  } else if (Out.count(MVT::fAny)) {
+    Out.erase(MVT::fAny);
+    for (MVT T : MVT::fp_valuetypes())
+      if (Legal.count(T))
+        Out.insert(T);
+    for (MVT T : MVT::fp_fixedlen_vector_valuetypes())
+      if (Legal.count(T))
+        Out.insert(T);
+    for (MVT T : MVT::fp_scalable_vector_valuetypes())
+      if (Legal.count(T))
+        Out.insert(T);
+  } else if (Out.count(MVT::vAny)) {
+    Out.erase(MVT::vAny);
+    for (MVT T : MVT::vector_valuetypes())
+      if (Legal.count(T))
+        Out.insert(T);
+  } else if (Out.count(MVT::Any)) {
+    Out.erase(MVT::Any);
+    for (MVT T : MVT::all_valuetypes())
+      if (Legal.count(T))
+        Out.insert(T);
   }
 }
 
-const TypeSetByHwMode &TypeInfer::getLegalTypes() {
+const TypeSetByHwMode &TypeInfer::getLegalTypes() const {
   if (!LegalTypesCached) {
     TypeSetByHwMode::SetType &LegalTypes = LegalCache.getOrCreate(DefaultMode);
     // Stuff all types from all modes into the default mode.
@@ -883,26 +866,26 @@ const TypeSetByHwMode &TypeInfer::getLegalTypes() {
       LegalTypes.insert(I.second);
     LegalTypesCached = true;
   }
-  assert(LegalCache.isDefaultOnly() && "Default-mode only expected");
+  assert(LegalCache.isSimple() && "Default-mode only expected");
   return LegalCache;
 }
 
-#ifndef NDEBUG
 TypeInfer::ValidateOnExit::~ValidateOnExit() {
   if (Infer.Validate && !VTS.validate()) {
-    dbgs() << "Type set is empty for each HW mode:\n"
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+    errs() << "Type set is empty for each HW mode:\n"
               "possible type contradiction in the pattern below "
               "(use -print-records with llvm-tblgen to see all "
               "expanded records).\n";
     Infer.TP.dump();
-    dbgs() << "Generated from record:\n";
+    errs() << "Generated from record:\n";
     Infer.TP.getRecord()->dump();
+#endif
     PrintFatalError(Infer.TP.getRecord()->getLoc(),
                     "Type set is empty for each HW mode in '" +
                         Infer.TP.getRecord()->getName() + "'");
   }
 }
-#endif
 
 
 //===----------------------------------------------------------------------===//
@@ -1512,6 +1495,9 @@ void PatternToMatch::getPredicateRecords(
   }
   // Sort so that different orders get canonicalized to the same string.
   llvm::sort(PredicateRecs, LessRecord());
+  // Remove duplicate predicates.
+  PredicateRecs.erase(std::unique(PredicateRecs.begin(), PredicateRecs.end()),
+                      PredicateRecs.end());
 }
 
 /// getPredicateCheck - Return a single string containing all of this
@@ -1522,22 +1508,17 @@ std::string PatternToMatch::getPredicateCheck() const {
   getPredicateRecords(PredicateRecs);
 
   SmallString<128> PredicateCheck;
+  raw_svector_ostream OS(PredicateCheck);
+  ListSeparator LS(" && ");
   for (Record *Pred : PredicateRecs) {
     StringRef CondString = Pred->getValueAsString("CondString");
     if (CondString.empty())
       continue;
-    if (!PredicateCheck.empty())
-      PredicateCheck += " && ";
-    PredicateCheck += "(";
-    PredicateCheck += CondString;
-    PredicateCheck += ")";
+    OS << LS << '(' << CondString << ')';
   }
 
-  if (!HwModeFeatures.empty()) {
-    if (!PredicateCheck.empty())
-      PredicateCheck += " && ";
-    PredicateCheck += HwModeFeatures;
-  }
+  if (!HwModeFeatures.empty())
+    OS << LS << HwModeFeatures;
 
   return std::string(PredicateCheck);
 }
@@ -1792,7 +1773,7 @@ bool TreePatternNode::ContainsUnresolvedType(TreePattern &TP) const {
 
 bool TreePatternNode::hasProperTypeByHwMode() const {
   for (const TypeSetByHwMode &S : Types)
-    if (!S.isDefaultOnly())
+    if (!S.isSimple())
       return true;
   for (const TreePatternNodePtr &C : Children)
     if (C->hasProperTypeByHwMode())
@@ -1880,7 +1861,7 @@ static unsigned GetNumNodeResults(Record *Operator, CodeGenDAGPatterns &CDP) {
     return 0;  // All return nothing.
 
   if (Operator->isSubClassOf("Intrinsic"))
-    return CDP.getIntrinsic(Operator).IS.RetVTs.size();
+    return CDP.getIntrinsic(Operator).IS.RetTys.size();
 
   if (Operator->isSubClassOf("SDNode"))
     return CDP.getSDNodeInfo(Operator).getNumResults();
@@ -1996,7 +1977,17 @@ void TreePatternNode::dump() const {
 bool TreePatternNode::isIsomorphicTo(const TreePatternNode *N,
                                      const MultipleUseVarSet &DepVars) const {
   if (N == this) return true;
-  if (N->isLeaf() != isLeaf() || getExtTypes() != N->getExtTypes() ||
+  if (N->isLeaf() != isLeaf())
+    return false;
+
+  // Check operator of non-leaves early since it can be cheaper than checking
+  // types.
+  if (!isLeaf())
+    if (N->getOperator() != getOperator() ||
+        N->getNumChildren() != getNumChildren())
+      return false;
+
+  if (getExtTypes() != N->getExtTypes() ||
       getPredicateCalls() != N->getPredicateCalls() ||
       getTransformFn() != N->getTransformFn())
     return false;
@@ -2004,16 +1995,13 @@ bool TreePatternNode::isIsomorphicTo(const TreePatternNode *N,
   if (isLeaf()) {
     if (DefInit *DI = dyn_cast<DefInit>(getLeafValue())) {
       if (DefInit *NDI = dyn_cast<DefInit>(N->getLeafValue())) {
-        return ((DI->getDef() == NDI->getDef())
-                && (DepVars.find(getName()) == DepVars.end()
-                    || getName() == N->getName()));
+        return ((DI->getDef() == NDI->getDef()) &&
+                (!DepVars.contains(getName()) || getName() == N->getName()));
       }
     }
     return getLeafValue() == N->getLeafValue();
   }
 
-  if (N->getOperator() != getOperator() ||
-      N->getNumChildren() != getNumChildren()) return false;
   for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
     if (!getChild(i)->isIsomorphicTo(N->getChild(i), DepVars))
       return false;
@@ -2025,19 +2013,20 @@ bool TreePatternNode::isIsomorphicTo(const TreePatternNode *N,
 TreePatternNodePtr TreePatternNode::clone() const {
   TreePatternNodePtr New;
   if (isLeaf()) {
-    New = std::make_shared<TreePatternNode>(getLeafValue(), getNumTypes());
+    New = makeIntrusiveRefCnt<TreePatternNode>(getLeafValue(), getNumTypes());
   } else {
     std::vector<TreePatternNodePtr> CChildren;
     CChildren.reserve(Children.size());
     for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
       CChildren.push_back(getChild(i)->clone());
-    New = std::make_shared<TreePatternNode>(getOperator(), std::move(CChildren),
-                                            getNumTypes());
+    New = makeIntrusiveRefCnt<TreePatternNode>(
+        getOperator(), std::move(CChildren), getNumTypes());
   }
   New->setName(getName());
   New->setNamesAsPredicateArg(getNamesAsPredicateArg());
   New->Types = Types;
   New->setPredicateCalls(getPredicateCalls());
+  New->setGISelFlagsRecord(getGISelFlagsRecord());
   New->setTransformFn(getTransformFn());
   return New;
 }
@@ -2085,14 +2074,13 @@ void TreePatternNode::SubstituteFormalArguments(
 /// fragments, return the set of inlined versions (this can be more than
 /// one if a PatFrags record has multiple alternatives).
 void TreePatternNode::InlinePatternFragments(
-  TreePatternNodePtr T, TreePattern &TP,
-  std::vector<TreePatternNodePtr> &OutAlternatives) {
+    TreePattern &TP, std::vector<TreePatternNodePtr> &OutAlternatives) {
 
   if (TP.hasError())
     return;
 
   if (isLeaf()) {
-    OutAlternatives.push_back(T);  // nothing to do.
+    OutAlternatives.push_back(this); // nothing to do.
     return;
   }
 
@@ -2100,16 +2088,16 @@ void TreePatternNode::InlinePatternFragments(
 
   if (!Op->isSubClassOf("PatFrags")) {
     if (getNumChildren() == 0) {
-      OutAlternatives.push_back(T);
+      OutAlternatives.push_back(this);
       return;
     }
 
     // Recursively inline children nodes.
-    std::vector<std::vector<TreePatternNodePtr> > ChildAlternatives;
-    ChildAlternatives.resize(getNumChildren());
+    std::vector<std::vector<TreePatternNodePtr>> ChildAlternatives(
+        getNumChildren());
     for (unsigned i = 0, e = getNumChildren(); i != e; ++i) {
       TreePatternNodePtr Child = getChildShared(i);
-      Child->InlinePatternFragments(Child, TP, ChildAlternatives[i]);
+      Child->InlinePatternFragments(TP, ChildAlternatives[i]);
       // If there are no alternatives for any child, there are no
       // alternatives for this expression as whole.
       if (ChildAlternatives[i].empty())
@@ -2125,21 +2113,22 @@ void TreePatternNode::InlinePatternFragments(
     }
 
     // The end result is an all-pairs construction of the resultant pattern.
-    std::vector<unsigned> Idxs;
-    Idxs.resize(ChildAlternatives.size());
+    std::vector<unsigned> Idxs(ChildAlternatives.size());
     bool NotDone;
     do {
       // Create the variant and add it to the output list.
       std::vector<TreePatternNodePtr> NewChildren;
+      NewChildren.reserve(ChildAlternatives.size());
       for (unsigned i = 0, e = ChildAlternatives.size(); i != e; ++i)
         NewChildren.push_back(ChildAlternatives[i][Idxs[i]]);
-      TreePatternNodePtr R = std::make_shared<TreePatternNode>(
+      TreePatternNodePtr R = makeIntrusiveRefCnt<TreePatternNode>(
           getOperator(), std::move(NewChildren), getNumTypes());
 
       // Copy over properties.
       R->setName(getName());
       R->setNamesAsPredicateArg(getNamesAsPredicateArg());
       R->setPredicateCalls(getPredicateCalls());
+      R->setGISelFlagsRecord(getGISelFlagsRecord());
       R->setTransformFn(getTransformFn());
       for (unsigned i = 0, e = getNumTypes(); i != e; ++i)
         R->setType(i, getExtType(i));
@@ -2170,7 +2159,7 @@ void TreePatternNode::InlinePatternFragments(
   TreePattern *Frag = TP.getDAGPatterns().getPatternFragment(Op);
 
   // Verify that we are passing the right number of operands.
-  if (Frag->getNumArgs() != Children.size()) {
+  if (Frag->getNumArgs() != getNumChildren()) {
     TP.error("'" + Op->getName() + "' fragment requires " +
              Twine(Frag->getNumArgs()) + " operands!");
     return;
@@ -2209,13 +2198,16 @@ void TreePatternNode::InlinePatternFragments(
     for (unsigned i = 0, e = FragTree->getNumTypes(); i != e; ++i)
       FragTree->UpdateNodeType(i, getExtType(i), TP);
 
+    if (Op->isSubClassOf("GISelFlags"))
+      FragTree->setGISelFlagsRecord(Op);
+
     // Transfer in the old predicates.
     for (const TreePredicateCall &Pred : getPredicateCalls())
       FragTree->addPredicateCall(Pred);
 
     // The fragment we inlined could have recursive inlining that is needed.  See
     // if there are any pattern fragments in it and inline them as needed.
-    FragTree->InlinePatternFragments(FragTree, TP, OutAlternatives);
+    FragTree->InlinePatternFragments(TP, OutAlternatives);
   }
 }
 
@@ -2408,10 +2400,10 @@ bool TreePatternNode::NodeHasProperty(SDNP Property,
       return Int->hasProperty(Property);
   }
 
-  if (!Operator->isSubClassOf("SDPatternOperator"))
+  if (!getOperator()->isSubClassOf("SDPatternOperator"))
     return false;
 
-  return CGP.getSDNodeInfo(Operator).hasProperty(Property);
+  return CGP.getSDNodeInfo(getOperator()).hasProperty(Property);
 }
 
 
@@ -2522,11 +2514,12 @@ bool TreePatternNode::ApplyTypeConstraints(TreePattern &TP, bool NotRegisters) {
     bool MadeChange = false;
 
     // Apply the result type to the node.
-    unsigned NumRetVTs = Int->IS.RetVTs.size();
-    unsigned NumParamVTs = Int->IS.ParamVTs.size();
+    unsigned NumRetVTs = Int->IS.RetTys.size();
+    unsigned NumParamVTs = Int->IS.ParamTys.size();
 
     for (unsigned i = 0, e = NumRetVTs; i != e; ++i)
-      MadeChange |= UpdateNodeType(i, Int->IS.RetVTs[i], TP);
+      MadeChange |= UpdateNodeType(
+          i, getValueType(Int->IS.RetTys[i]->getValueAsDef("VT")), TP);
 
     if (getNumChildren() != NumParamVTs + 1) {
       TP.error("Intrinsic '" + Int->Name + "' expects " + Twine(NumParamVTs) +
@@ -2540,9 +2533,10 @@ bool TreePatternNode::ApplyTypeConstraints(TreePattern &TP, bool NotRegisters) {
     for (unsigned i = 0, e = getNumChildren()-1; i != e; ++i) {
       MadeChange |= getChild(i+1)->ApplyTypeConstraints(TP, NotRegisters);
 
-      MVT::SimpleValueType OpVT = Int->IS.ParamVTs[i];
-      assert(getChild(i+1)->getNumTypes() == 1 && "Unhandled case");
-      MadeChange |= getChild(i+1)->UpdateNodeType(0, OpVT, TP);
+      MVT::SimpleValueType OpVT =
+          getValueType(Int->IS.ParamTys[i]->getValueAsDef("VT"));
+      assert(getChild(i + 1)->getNumTypes() == 1 && "Unhandled case");
+      MadeChange |= getChild(i + 1)->UpdateNodeType(0, OpVT, TP);
     }
     return MadeChange;
   }
@@ -2872,7 +2866,7 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit,
         OpName);
 
     // Input argument?
-    TreePatternNodePtr Res = std::make_shared<TreePatternNode>(DI, 1);
+    TreePatternNodePtr Res = makeIntrusiveRefCnt<TreePatternNode>(DI, 1);
     if (R->getName() == "node" && !OpName.empty()) {
       if (OpName.empty())
         error("'node' argument requires a name to match with operand list");
@@ -2887,7 +2881,7 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit,
   if (isa<UnsetInit>(TheInit)) {
     if (OpName.empty())
       error("'?' argument requires a name to match with operand list");
-    TreePatternNodePtr Res = std::make_shared<TreePatternNode>(TheInit, 1);
+    TreePatternNodePtr Res = makeIntrusiveRefCnt<TreePatternNode>(TheInit, 1);
     Args.push_back(std::string(OpName));
     Res->setName(OpName);
     return Res;
@@ -2898,7 +2892,7 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit,
       error("Constant int or bit argument should not have a name!");
     if (isa<BitInit>(TheInit))
       TheInit = TheInit->convertInitializerTo(IntRecTy::get(RK));
-    return std::make_shared<TreePatternNode>(TheInit, 1);
+    return makeIntrusiveRefCnt<TreePatternNode>(TheInit, 1);
   }
 
   if (BitsInit *BI = dyn_cast<BitsInit>(TheInit)) {
@@ -2906,16 +2900,20 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit,
     Init *II = BI->convertInitializerTo(IntRecTy::get(RK));
     if (!II || !isa<IntInit>(II))
       error("Bits value must be constants!");
-    return ParseTreePattern(II, OpName);
+    return II ? ParseTreePattern(II, OpName) : nullptr;
   }
 
   DagInit *Dag = dyn_cast<DagInit>(TheInit);
   if (!Dag) {
     TheInit->print(errs());
     error("Pattern has unexpected init kind!");
+    return nullptr;
   }
   DefInit *OpDef = dyn_cast<DefInit>(Dag->getOperator());
-  if (!OpDef) error("Pattern has unexpected operator type!");
+  if (!OpDef) {
+    error("Pattern has unexpected operator type!");
+    return nullptr;
+  }
   Record *Operator = OpDef->getDef();
 
   if (Operator->isSubClassOf("ValueType")) {
@@ -2994,7 +2992,7 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit,
 
     // If this intrinsic returns void, it must have side-effects and thus a
     // chain.
-    if (Int.IS.RetVTs.empty())
+    if (Int.IS.RetTys.empty())
       Operator = getDAGPatterns().get_intrinsic_void_sdnode();
     else if (!Int.ME.doesNotAccessMemory() || Int.hasSideEffects)
       // Has side-effects, requires chain.
@@ -3002,7 +3000,7 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit,
     else // Otherwise, no chain.
       Operator = getDAGPatterns().get_intrinsic_wo_chain_sdnode();
 
-    Children.insert(Children.begin(), std::make_shared<TreePatternNode>(
+    Children.insert(Children.begin(), makeIntrusiveRefCnt<TreePatternNode>(
                                           IntInit::get(RK, IID), 1));
   }
 
@@ -3027,9 +3025,8 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit,
     }
   }
 
-  TreePatternNodePtr Result =
-      std::make_shared<TreePatternNode>(Operator, std::move(Children),
-                                        NumResults);
+  TreePatternNodePtr Result = makeIntrusiveRefCnt<TreePatternNode>(
+      Operator, std::move(Children), NumResults);
   Result->setName(OpName);
 
   if (Dag->getName()) {
@@ -3069,11 +3066,9 @@ static bool SimplifyTree(TreePatternNodePtr &N) {
 
   // Walk all children.
   bool MadeChange = false;
-  for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) {
-    TreePatternNodePtr Child = N->getChildShared(i);
-    MadeChange |= SimplifyTree(Child);
-    N->setChild(i, std::move(Child));
-  }
+  for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i)
+    MadeChange |= SimplifyTree(N->getChildSharedPtr(i));
+
   return MadeChange;
 }
 
@@ -3492,7 +3487,8 @@ void CodeGenDAGPatterns::FindPatternInputsAndOutputs(
       DefInit *Val = dyn_cast<DefInit>(Dest->getLeafValue());
       if (!Val || !Val->getDef()->isSubClassOf("Register"))
         I.error("implicitly defined value should be a register!");
-      InstImpResults.push_back(Val->getDef());
+      if (Val)
+        InstImpResults.push_back(Val->getDef());
     }
     return;
   }
@@ -3913,7 +3909,7 @@ void CodeGenDAGPatterns::parseInstructionPattern(
       OpNode->setTransformFn(nullptr);
       std::vector<TreePatternNodePtr> Children;
       Children.push_back(OpNode);
-      OpNode = std::make_shared<TreePatternNode>(Xform, std::move(Children),
+      OpNode = makeIntrusiveRefCnt<TreePatternNode>(Xform, std::move(Children),
                                                  OpNode->getNumTypes());
     }
 
@@ -3924,7 +3920,7 @@ void CodeGenDAGPatterns::parseInstructionPattern(
     I.error("Input operand $" + InstInputs.begin()->first +
             " occurs in pattern but not in operands list!");
 
-  TreePatternNodePtr ResultPattern = std::make_shared<TreePatternNode>(
+  TreePatternNodePtr ResultPattern = makeIntrusiveRefCnt<TreePatternNode>(
       I.getRecord(), std::move(ResultNodeOperands),
       GetNumNodeResults(I.getRecord(), *this));
   // Copy fully inferred output node types to instruction result pattern.
@@ -3948,9 +3944,8 @@ void CodeGenDAGPatterns::parseInstructionPattern(
   // Create and insert the instruction.
   // FIXME: InstImpResults should not be part of DAGInstruction.
   Record *R = I.getRecord();
-  DAGInsts.emplace(std::piecewise_construct, std::forward_as_tuple(R),
-                   std::forward_as_tuple(Results, Operands, InstImpResults,
-                                         SrcPattern, ResultPattern));
+  DAGInsts.try_emplace(R, std::move(Results), std::move(Operands),
+                       std::move(InstImpResults), SrcPattern, ResultPattern);
 
   LLVM_DEBUG(I.dump());
 }
@@ -3990,9 +3985,8 @@ void CodeGenDAGPatterns::ParseInstructions() {
       }
 
       // Create and insert the instruction.
-      std::vector<Record*> ImpResults;
-      Instructions.insert(std::make_pair(Instr,
-                            DAGInstruction(Results, Operands, ImpResults)));
+      Instructions.try_emplace(Instr, std::move(Results), std::move(Operands),
+                               std::vector<Record *>());
       continue;  // no pattern.
     }
 
@@ -4246,7 +4240,7 @@ static TreePatternNodePtr PromoteXForms(TreePatternNodePtr N) {
       N->setTransformFn(nullptr);
       std::vector<TreePatternNodePtr> Children;
       Children.push_back(PromoteXForms(N));
-      return std::make_shared<TreePatternNode>(Xform, std::move(Children),
+      return makeIntrusiveRefCnt<TreePatternNode>(Xform, std::move(Children),
                                                N->getNumTypes());
   }
 
@@ -4342,13 +4336,24 @@ void CodeGenDAGPatterns::ParseOnePattern(Record *TheDef,
   // that register class does not accept that type, the type inference
   // will lead to a contradiction, which is not an error however, but
   // a sign that this pattern will simply never match.
-  if (Temp.getOnlyTree()->hasPossibleType())
-    for (const auto &T : Pattern.getTrees())
+  if (Temp.getOnlyTree()->hasPossibleType()) {
+    for (const auto &T : Pattern.getTrees()) {
       if (T->hasPossibleType())
         AddPatternToMatch(&Pattern,
                           PatternToMatch(TheDef, Preds, T, Temp.getOnlyTree(),
                                          InstImpResults, Complexity,
                                          TheDef->getID()));
+    }
+  } else {
+    // Show a message about a dropped pattern with some info to make it
+    // easier to identify it in the .td files.
+    LLVM_DEBUG({
+      dbgs() << "Dropping: ";
+      Pattern.dump();
+      Temp.getOnlyTree()->dump();
+      dbgs() << "\n";
+    });
+  }
 }
 
 void CodeGenDAGPatterns::ParsePatterns() {
@@ -4397,6 +4402,9 @@ static void collectModes(std::set<unsigned> &Modes, const TreePatternNode *N) {
 
 void CodeGenDAGPatterns::ExpandHwModeBasedTypes() {
   const CodeGenHwModes &CGH = getTargetInfo().getHwModes();
+  if (CGH.getNumModeIds() == 1)
+    return;
+
   std::vector<PatternToMatch> Copy;
   PatternsToMatch.swap(Copy);
 
@@ -4411,15 +4419,15 @@ void CodeGenDAGPatterns::ExpandHwModeBasedTypes() {
     PatternsToMatch.emplace_back(P.getSrcRecord(), P.getPredicates(),
                                  std::move(NewSrc), std::move(NewDst),
                                  P.getDstRegs(), P.getAddedComplexity(),
-                                 Record::getNewUID(Records), Mode, Check);
+                                 Record::getNewUID(Records), Check);
   };
 
   for (PatternToMatch &P : Copy) {
-    TreePatternNodePtr SrcP = nullptr, DstP = nullptr;
+    const TreePatternNode *SrcP = nullptr, *DstP = nullptr;
     if (P.getSrcPattern()->hasProperTypeByHwMode())
-      SrcP = P.getSrcPatternShared();
+      SrcP = P.getSrcPattern();
     if (P.getDstPattern()->hasProperTypeByHwMode())
-      DstP = P.getDstPatternShared();
+      DstP = P.getDstPattern();
     if (!SrcP && !DstP) {
       PatternsToMatch.push_back(P);
       continue;
@@ -4427,9 +4435,9 @@ void CodeGenDAGPatterns::ExpandHwModeBasedTypes() {
 
     std::set<unsigned> Modes;
     if (SrcP)
-      collectModes(Modes, SrcP.get());
+      collectModes(Modes, SrcP);
     if (DstP)
-      collectModes(Modes, DstP.get());
+      collectModes(Modes, DstP);
 
     // The predicate for the default mode needs to be constructed for each
     // pattern separately.
@@ -4450,14 +4458,14 @@ void CodeGenDAGPatterns::ExpandHwModeBasedTypes() {
 
       // Fill the map entry for this mode.
       const HwMode &HM = CGH.getMode(M);
-      AppendPattern(P, M, "(MF->getSubtarget().checkFeatures(\"" + HM.Features + "\"))");
+      AppendPattern(P, M, HM.Predicates);
 
       // Add negations of the HM's predicates to the default predicate.
       if (!DefaultCheck.empty())
         DefaultCheck += " && ";
-      DefaultCheck += "(!(MF->getSubtarget().checkFeatures(\"";
-      DefaultCheck += HM.Features;
-      DefaultCheck += "\")))";
+      DefaultCheck += "!(";
+      DefaultCheck += HM.Predicates;
+      DefaultCheck += ")";
     }
 
     bool HasDefault = Modes.count(DefaultMode);
@@ -4518,8 +4526,7 @@ static void CombineChildVariants(
       return;
 
   // The end result is an all-pairs construction of the resultant pattern.
-  std::vector<unsigned> Idxs;
-  Idxs.resize(ChildVariants.size());
+  std::vector<unsigned> Idxs(ChildVariants.size());
   bool NotDone;
   do {
 #ifndef NDEBUG
@@ -4533,15 +4540,17 @@ static void CombineChildVariants(
 #endif
     // Create the variant and add it to the output list.
     std::vector<TreePatternNodePtr> NewChildren;
+    NewChildren.reserve(ChildVariants.size());
     for (unsigned i = 0, e = ChildVariants.size(); i != e; ++i)
       NewChildren.push_back(ChildVariants[i][Idxs[i]]);
-    TreePatternNodePtr R = std::make_shared<TreePatternNode>(
+    TreePatternNodePtr R = makeIntrusiveRefCnt<TreePatternNode>(
         Orig->getOperator(), std::move(NewChildren), Orig->getNumTypes());
 
     // Copy over properties.
     R->setName(Orig->getName());
     R->setNamesAsPredicateArg(Orig->getNamesAsPredicateArg());
     R->setPredicateCalls(Orig->getPredicateCalls());
+    R->setGISelFlagsRecord(Orig->getGISelFlagsRecord());
     R->setTransformFn(Orig->getTransformFn());
     for (unsigned i = 0, e = Orig->getNumTypes(); i != e; ++i)
       R->setType(i, Orig->getExtType(i));
@@ -4679,8 +4688,8 @@ static void GenerateVariantsOf(TreePatternNodePtr N,
   }
 
   // Compute permutations of all children.
-  std::vector<std::vector<TreePatternNodePtr>> ChildVariants;
-  ChildVariants.resize(N->getNumChildren());
+  std::vector<std::vector<TreePatternNodePtr>> ChildVariants(
+      N->getNumChildren());
   for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i)
     GenerateVariantsOf(N->getChildShared(i), ChildVariants[i], CDP, DepVars);
 
@@ -4708,17 +4717,10 @@ static void GenerateVariantsOf(TreePatternNodePtr N,
     }
     // Consider the commuted order.
     if (NoRegisters) {
-      std::vector<std::vector<TreePatternNodePtr>> Variants;
-      unsigned i = 0;
-      if (isCommIntrinsic)
-        Variants.push_back(std::move(ChildVariants[i++])); // Intrinsic id.
-      Variants.push_back(std::move(ChildVariants[i + 1]));
-      Variants.push_back(std::move(ChildVariants[i]));
-      i += 2;
-      // Remaining operands are not commuted.
-      for (; i != N->getNumChildren(); ++i)
-        Variants.push_back(std::move(ChildVariants[i]));
-      CombineChildVariants(N, Variants, OutVariants, CDP, DepVars);
+      // Swap the first two operands after the intrinsic id, if present.
+      unsigned i = isCommIntrinsic ? 1 : 0;
+      std::swap(ChildVariants[i], ChildVariants[i + 1]);
+      CombineChildVariants(N, ChildVariants, OutVariants, CDP, DepVars);
     }
   }
 }
@@ -4788,7 +4790,6 @@ void CodeGenDAGPatterns::GenerateVariants() {
           Variant, PatternsToMatch[i].getDstPatternShared(),
           PatternsToMatch[i].getDstRegs(),
           PatternsToMatch[i].getAddedComplexity(), Record::getNewUID(Records),
-          PatternsToMatch[i].getForceMode(),
           PatternsToMatch[i].getHwModeFeatures());
     }
 
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/CodeGenDAGPatterns.h
index ec35e6680088..2611fe06f55c 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.h
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.h
@@ -17,12 +17,16 @@
 #include "CodeGenIntrinsics.h"
 #include "CodeGenTarget.h"
 #include "SDNodeProperties.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
+#include "llvm/TableGen/Record.h"
 #include <algorithm>
 #include <array>
 #include <functional>
@@ -32,7 +36,6 @@
 
 namespace llvm {
 
-class Record;
 class Init;
 class ListInit;
 class DagInit;
@@ -42,7 +45,7 @@ class TreePatternNode;
 class CodeGenDAGPatterns;
 
 /// Shared pointer for TreePatternNode.
-using TreePatternNodePtr = std::shared_ptr<TreePatternNode>;
+using TreePatternNodePtr = IntrusiveRefCntPtr<TreePatternNode>;
 
 /// This represents a set of MVTs. Since the underlying type for the MVT
 /// is uint8_t, there are at most 256 values. To reduce the number of memory
@@ -191,7 +194,7 @@ raw_ostream &operator<<(raw_ostream &OS, const MachineValueTypeSet &T);
 
 struct TypeSetByHwMode : public InfoByHwMode<MachineValueTypeSet> {
   using SetType = MachineValueTypeSet;
-  SmallVector<unsigned, 16> AddrSpaces;
+  unsigned AddrSpace = std::numeric_limits<unsigned>::max();
 
   TypeSetByHwMode() = default;
   TypeSetByHwMode(const TypeSetByHwMode &VTS) = default;
@@ -211,22 +214,17 @@ struct TypeSetByHwMode : public InfoByHwMode<MachineValueTypeSet> {
 
   LLVM_ATTRIBUTE_ALWAYS_INLINE
   bool isMachineValueType() const {
-    return isDefaultOnly() && Map.begin()->second.size() == 1;
+    return isSimple() && getSimple().size() == 1;
   }
 
   LLVM_ATTRIBUTE_ALWAYS_INLINE
   MVT getMachineValueType() const {
     assert(isMachineValueType());
-    return *Map.begin()->second.begin();
+    return *getSimple().begin();
   }
 
   bool isPossible() const;
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
-  bool isDefaultOnly() const {
-    return Map.size() == 1 && Map.begin()->first == DefaultMode;
-  }
-
   bool isPointer() const {
     return getValueTypeByHwMode().isPointer();
   }
@@ -259,7 +257,7 @@ private:
 raw_ostream &operator<<(raw_ostream &OS, const TypeSetByHwMode &T);
 
 struct TypeInfer {
-  TypeInfer(TreePattern &T) : TP(T), ForceMode(0) {}
+  TypeInfer(TreePattern &T) : TP(T) {}
 
   bool isConcrete(const TypeSetByHwMode &VTS, bool AllowEmpty) const {
     return VTS.isValueTypeByHwMode(AllowEmpty);
@@ -274,11 +272,11 @@ struct TypeInfer {
   /// expand*) is to return "true" if a change has been made, "false"
   /// otherwise.
 
-  bool MergeInTypeInfo(TypeSetByHwMode &Out, const TypeSetByHwMode &In);
-  bool MergeInTypeInfo(TypeSetByHwMode &Out, MVT::SimpleValueType InVT) {
+  bool MergeInTypeInfo(TypeSetByHwMode &Out, const TypeSetByHwMode &In) const;
+  bool MergeInTypeInfo(TypeSetByHwMode &Out, MVT::SimpleValueType InVT) const {
     return MergeInTypeInfo(Out, TypeSetByHwMode(InVT));
   }
-  bool MergeInTypeInfo(TypeSetByHwMode &Out, ValueTypeByHwMode InVT) {
+  bool MergeInTypeInfo(TypeSetByHwMode &Out, ValueTypeByHwMode InVT) const {
     return MergeInTypeInfo(Out, TypeSetByHwMode(InVT));
   }
 
@@ -332,19 +330,16 @@ struct TypeInfer {
 
   /// For each overloaded type (i.e. of form *Any), replace it with the
   /// corresponding subset of legal, specific types.
-  void expandOverloads(TypeSetByHwMode &VTS);
+  void expandOverloads(TypeSetByHwMode &VTS) const;
   void expandOverloads(TypeSetByHwMode::SetType &Out,
-                       const TypeSetByHwMode::SetType &Legal);
+                       const TypeSetByHwMode::SetType &Legal) const;
 
   struct ValidateOnExit {
-    ValidateOnExit(TypeSetByHwMode &T, TypeInfer &TI) : Infer(TI), VTS(T) {}
-  #ifndef NDEBUG
+    ValidateOnExit(const TypeSetByHwMode &T, const TypeInfer &TI)
+        : Infer(TI), VTS(T) {}
     ~ValidateOnExit();
-  #else
-    ~ValidateOnExit() {}  // Empty destructor with NDEBUG.
-  #endif
-    TypeInfer &Infer;
-    TypeSetByHwMode &VTS;
+    const TypeInfer &Infer;
+    const TypeSetByHwMode &VTS;
   };
 
   struct SuppressValidation {
@@ -359,16 +354,14 @@ struct TypeInfer {
   };
 
   TreePattern &TP;
-  unsigned ForceMode;     // Mode to use when set.
-  bool CodeGen = false;   // Set during generation of matcher code.
   bool Validate = true;   // Indicate whether to validate types.
 
 private:
-  const TypeSetByHwMode &getLegalTypes();
+  const TypeSetByHwMode &getLegalTypes() const;
 
   /// Cached legal types (in default mode).
-  bool LegalTypesCached = false;
-  TypeSetByHwMode LegalCache;
+  mutable bool LegalTypesCached = false;
+  mutable TypeSetByHwMode LegalCache;
 };
 
 /// Set type used to track multiply used variables in patterns
@@ -632,7 +625,7 @@ struct TreePredicateCall {
   }
 };
 
-class TreePatternNode {
+class TreePatternNode : public RefCountedBase<TreePatternNode> {
   /// The type of each node result.  Before and during type inference, each
   /// result may be a set of possible types.  After (successful) type inference,
   /// each is a single concrete type.
@@ -641,13 +634,10 @@ class TreePatternNode {
   /// The index of each result in results of the pattern.
   std::vector<unsigned> ResultPerm;
 
-  /// Operator - The Record for the operator if this is an interior node (not
-  /// a leaf).
-  Record *Operator;
-
-  /// Val - The init value (e.g. the "GPRC" record, or "7") for a leaf.
-  ///
-  Init *Val;
+  /// OperatorOrVal - The Record for the operator if this is an interior node
+  /// (not a leaf) or the init value (e.g. the "GPRC" record, or "7") for a
+  /// leaf.
+  PointerUnion<Record *, Init *> OperatorOrVal;
 
   /// Name - The name given to this node with the :$foo notation.
   ///
@@ -665,17 +655,20 @@ class TreePatternNode {
 
   std::vector<TreePatternNodePtr> Children;
 
+  /// If this was instantiated from a PatFrag node, and the PatFrag was derived
+  /// from "GISelFlags": the original Record derived from GISelFlags.
+  const Record *GISelFlags = nullptr;
+
 public:
   TreePatternNode(Record *Op, std::vector<TreePatternNodePtr> Ch,
                   unsigned NumResults)
-      : Operator(Op), Val(nullptr), TransformFn(nullptr),
-        Children(std::move(Ch)) {
+      : OperatorOrVal(Op), TransformFn(nullptr), Children(std::move(Ch)) {
     Types.resize(NumResults);
     ResultPerm.resize(NumResults);
     std::iota(ResultPerm.begin(), ResultPerm.end(), 0);
   }
-  TreePatternNode(Init *val, unsigned NumResults)    // leaf ctor
-    : Operator(nullptr), Val(val), TransformFn(nullptr) {
+  TreePatternNode(Init *val, unsigned NumResults) // leaf ctor
+      : OperatorOrVal(val), TransformFn(nullptr) {
     Types.resize(NumResults);
     ResultPerm.resize(NumResults);
     std::iota(ResultPerm.begin(), ResultPerm.end(), 0);
@@ -695,7 +688,7 @@ public:
     NamesAsPredicateArg.push_back(N);
   }
 
-  bool isLeaf() const { return Val != nullptr; }
+  bool isLeaf() const { return isa<Init *>(OperatorOrVal); }
 
   // Type accessors.
   unsigned getNumTypes() const { return Types.size(); }
@@ -723,14 +716,26 @@ public:
   unsigned getResultIndex(unsigned ResNo) const { return ResultPerm[ResNo]; }
   void setResultIndex(unsigned ResNo, unsigned RI) { ResultPerm[ResNo] = RI; }
 
-  Init *getLeafValue() const { assert(isLeaf()); return Val; }
-  Record *getOperator() const { assert(!isLeaf()); return Operator; }
+  Init *getLeafValue() const {
+    assert(isLeaf());
+    return cast<Init *>(OperatorOrVal);
+  }
+  Record *getOperator() const {
+    assert(!isLeaf());
+    return cast<Record *>(OperatorOrVal);
+  }
 
   unsigned getNumChildren() const { return Children.size(); }
-  TreePatternNode *getChild(unsigned N) const { return Children[N].get(); }
+  const TreePatternNode *getChild(unsigned N) const {
+    return Children[N].get();
+  }
+  TreePatternNode *getChild(unsigned N) { return Children[N].get(); }
   const TreePatternNodePtr &getChildShared(unsigned N) const {
     return Children[N];
   }
+  TreePatternNodePtr &getChildSharedPtr(unsigned N) {
+    return Children[N];
+  }
   void setChild(unsigned i, TreePatternNodePtr N) { Children[i] = N; }
 
   /// hasChild - Return true if N is any of our children.
@@ -794,6 +799,9 @@ public:
   /// marked isCommutative.
   bool isCommutativeIntrinsic(const CodeGenDAGPatterns &CDP) const;
 
+  void setGISelFlagsRecord(const Record *R) { GISelFlags = R; }
+  const Record *getGISelFlagsRecord() const { return GISelFlags; }
+
   void print(raw_ostream &OS) const;
   void dump() const;
 
@@ -818,11 +826,10 @@ public:   // Higher level manipulation routines.
   void
   SubstituteFormalArguments(std::map<std::string, TreePatternNodePtr> &ArgMap);
 
-  /// InlinePatternFragments - If this pattern refers to any pattern
+  /// InlinePatternFragments - If \p T pattern refers to any pattern
   /// fragments, return the set of inlined versions (this can be more than
   /// one if a PatFrags record has multiple alternatives).
-  void InlinePatternFragments(TreePatternNodePtr T,
-                              TreePattern &TP,
+  void InlinePatternFragments(TreePattern &TP,
                               std::vector<TreePatternNodePtr> &OutAlternatives);
 
   /// ApplyTypeConstraints - Apply all of the type constraints relevant to
@@ -860,7 +867,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, const TreePatternNode &TPN) {
   return OS;
 }
 
-
 /// TreePattern - Represent a pattern, used for instructions, pattern
 /// fragments, etc.
 ///
@@ -950,10 +956,10 @@ public:
   /// PatFrags references.  This may increase the number of trees in the
   /// pattern if a PatFrags has multiple alternatives.
   void InlinePatternFragments() {
-    std::vector<TreePatternNodePtr> Copy = Trees;
-    Trees.clear();
-    for (unsigned i = 0, e = Copy.size(); i != e; ++i)
-      Copy[i]->InlinePatternFragments(Copy[i], *this, Trees);
+    std::vector<TreePatternNodePtr> Copy;
+    Trees.swap(Copy);
+    for (const TreePatternNodePtr &C : Copy)
+      C->InlinePatternFragments(*this, Trees);
   }
 
   /// InferAllTypes - Infer/propagate as many types throughout the expression
@@ -1023,13 +1029,14 @@ class DAGInstruction {
   TreePatternNodePtr ResultPattern;
 
 public:
-  DAGInstruction(const std::vector<Record*> &results,
-                 const std::vector<Record*> &operands,
-                 const std::vector<Record*> &impresults,
+  DAGInstruction(std::vector<Record *> &&results,
+                 std::vector<Record *> &&operands,
+                 std::vector<Record *> &&impresults,
                  TreePatternNodePtr srcpattern = nullptr,
                  TreePatternNodePtr resultpattern = nullptr)
-    : Results(results), Operands(operands), ImpResults(impresults),
-      SrcPattern(srcpattern), ResultPattern(resultpattern) {}
+      : Results(std::move(results)), Operands(std::move(operands)),
+        ImpResults(std::move(impresults)), SrcPattern(srcpattern),
+        ResultPattern(resultpattern) {}
 
   unsigned getNumResults() const { return Results.size(); }
   unsigned getNumOperands() const { return Operands.size(); }
@@ -1066,17 +1073,16 @@ class PatternToMatch {
   std::string      HwModeFeatures;
   int              AddedComplexity; // Add to matching pattern complexity.
   unsigned         ID;          // Unique ID for the record.
-  unsigned         ForceMode;   // Force this mode in type inference when set.
 
 public:
   PatternToMatch(Record *srcrecord, ListInit *preds, TreePatternNodePtr src,
                  TreePatternNodePtr dst, std::vector<Record *> dstregs,
-                 int complexity, unsigned uid, unsigned setmode = 0,
+                 int complexity, unsigned uid,
                  const Twine &hwmodefeatures = "")
       : SrcRecord(srcrecord), Predicates(preds), SrcPattern(src),
         DstPattern(dst), Dstregs(std::move(dstregs)),
         HwModeFeatures(hwmodefeatures.str()), AddedComplexity(complexity),
-        ID(uid), ForceMode(setmode) {}
+        ID(uid) {}
 
   Record          *getSrcRecord()  const { return SrcRecord; }
   ListInit        *getPredicates() const { return Predicates; }
@@ -1088,7 +1094,6 @@ public:
   StringRef   getHwModeFeatures() const { return HwModeFeatures; }
   int         getAddedComplexity() const { return AddedComplexity; }
   unsigned getID() const { return ID; }
-  unsigned getForceMode() const { return ForceMode; }
 
   std::string getPredicateCheck() const;
   void getPredicateRecords(SmallVectorImpl<Record *> &PredicateRecs) const;
diff --git a/llvm/utils/TableGen/CodeGenHwModes.cpp b/llvm/utils/TableGen/CodeGenHwModes.cpp
index 2fec46c44100..2171507f4c63 100644
--- a/llvm/utils/TableGen/CodeGenHwModes.cpp
+++ b/llvm/utils/TableGen/CodeGenHwModes.cpp
@@ -21,6 +21,19 @@ StringRef CodeGenHwModes::DefaultModeName = "DefaultMode";
 HwMode::HwMode(Record *R) {
   Name = R->getName();
   Features = std::string(R->getValueAsString("Features"));
+
+  std::vector<Record *> PredicateRecs = R->getValueAsListOfDefs("Predicates");
+  SmallString<128> PredicateCheck;
+  raw_svector_ostream OS(PredicateCheck);
+  ListSeparator LS(" && ");
+  for (Record *Pred : PredicateRecs) {
+    StringRef CondString = Pred->getValueAsString("CondString");
+    if (CondString.empty())
+      continue;
+    OS << LS << '(' << CondString << ')';
+  }
+
+  Predicates = std::string(PredicateCheck);
 }
 
 LLVM_DUMP_METHOD
@@ -38,7 +51,7 @@ HwModeSelect::HwModeSelect(Record *R, CodeGenHwModes &CGH) {
     report_fatal_error("error in target description.");
   }
   for (unsigned i = 0, e = Modes.size(); i != e; ++i) {
-    unsigned ModeId = CGH.getHwModeId(Modes[i]->getName());
+    unsigned ModeId = CGH.getHwModeId(Modes[i]);
     Items.push_back(std::make_pair(ModeId, Objects[i]));
   }
 }
@@ -52,34 +65,26 @@ void HwModeSelect::dump() const {
 }
 
 CodeGenHwModes::CodeGenHwModes(RecordKeeper &RK) : Records(RK) {
-  std::vector<Record*> MRs = Records.getAllDerivedDefinitions("HwMode");
-  // The default mode needs a definition in the .td sources for TableGen
-  // to accept references to it. We need to ignore the definition here.
-  for (auto I = MRs.begin(), E = MRs.end(); I != E; ++I) {
-    if ((*I)->getName() != DefaultModeName)
+  for (Record *R : Records.getAllDerivedDefinitions("HwMode")) {
+    // The default mode needs a definition in the .td sources for TableGen
+    // to accept references to it. We need to ignore the definition here.
+    if (R->getName() == DefaultModeName)
       continue;
-    MRs.erase(I);
-    break;
-  }
-
-  for (Record *R : MRs) {
     Modes.emplace_back(R);
-    unsigned NewId = Modes.size();
-    ModeIds.insert(std::make_pair(Modes[NewId-1].Name, NewId));
+    ModeIds.insert(std::make_pair(R, Modes.size()));
   }
 
-  std::vector<Record*> MSs = Records.getAllDerivedDefinitions("HwModeSelect");
-  for (Record *R : MSs) {
+  for (Record *R : Records.getAllDerivedDefinitions("HwModeSelect")) {
     auto P = ModeSelects.emplace(std::make_pair(R, HwModeSelect(R, *this)));
     assert(P.second);
     (void)P;
   }
 }
 
-unsigned CodeGenHwModes::getHwModeId(StringRef Name) const {
-  if (Name == DefaultModeName)
+unsigned CodeGenHwModes::getHwModeId(Record *R) const {
+  if (R->getName() == DefaultModeName)
     return DefaultMode;
-  auto F = ModeIds.find(Name);
+  auto F = ModeIds.find(R);
   assert(F != ModeIds.end() && "Unknown mode name");
   return F->second;
 }
@@ -101,7 +106,7 @@ void CodeGenHwModes::dump() const {
 
   dbgs() << "ModeIds: {\n";
   for (const auto &P : ModeIds)
-    dbgs() << "  " << P.first() << " -> " << P.second << '\n';
+    dbgs() << "  " << P.first->getName() << " -> " << P.second << '\n';
   dbgs() << "}\n";
 
   dbgs() << "ModeSelects: {\n";
diff --git a/llvm/utils/TableGen/CodeGenHwModes.h b/llvm/utils/TableGen/CodeGenHwModes.h
index 55507cbca37d..09d20ad85c5e 100644
--- a/llvm/utils/TableGen/CodeGenHwModes.h
+++ b/llvm/utils/TableGen/CodeGenHwModes.h
@@ -11,10 +11,12 @@
 #ifndef LLVM_UTILS_TABLEGEN_CODEGENHWMODES_H
 #define LLVM_UTILS_TABLEGEN_CODEGENHWMODES_H
 
-#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
 #include <cassert>
 #include <map>
 #include <string>
+#include <utility>
 #include <vector>
 
 // HwModeId -> list of predicates (definition)
@@ -29,6 +31,7 @@ namespace llvm {
     HwMode(Record *R);
     StringRef Name;
     std::string Features;
+    std::string Predicates;
     void dump() const;
   };
 
@@ -44,7 +47,7 @@ namespace llvm {
     static StringRef DefaultModeName;
 
     CodeGenHwModes(RecordKeeper &R);
-    unsigned getHwModeId(StringRef Name) const;
+    unsigned getHwModeId(Record *R) const;
     const HwMode &getMode(unsigned Id) const {
       assert(Id != 0 && "Mode id of 0 is reserved for the default mode");
       return Modes[Id-1];
@@ -55,7 +58,7 @@ namespace llvm {
 
   private:
     RecordKeeper &Records;
-    StringMap<unsigned> ModeIds;  // HwMode (string) -> HwModeId
+    DenseMap<Record *, unsigned> ModeIds;  // HwMode Record -> HwModeId
     std::vector<HwMode> Modes;
     std::map<Record*,HwModeSelect> ModeSelects;
   };
diff --git a/llvm/utils/TableGen/CodeGenInstAlias.cpp b/llvm/utils/TableGen/CodeGenInstAlias.cpp
new file mode 100644
index 000000000000..8634d45eafc7
--- /dev/null
+++ b/llvm/utils/TableGen/CodeGenInstAlias.cpp
@@ -0,0 +1,283 @@
+//===- CodeGenInstAlias.cpp - CodeGen InstAlias Class Wrapper -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the CodeGenInstAlias class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CodeGenInstAlias.h"
+#include "CodeGenInstruction.h"
+#include "CodeGenRegisters.h"
+#include "CodeGenTarget.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+
+using namespace llvm;
+
+/// tryAliasOpMatch - This is a helper function for the CodeGenInstAlias
+/// constructor.  It checks if an argument in an InstAlias pattern matches
+/// the corresponding operand of the instruction.  It returns true on a
+/// successful match, with ResOp set to the result operand to be used.
+bool CodeGenInstAlias::tryAliasOpMatch(DagInit *Result, unsigned AliasOpNo,
+                                       Record *InstOpRec, bool hasSubOps,
+                                       ArrayRef<SMLoc> Loc, CodeGenTarget &T,
+                                       ResultOperand &ResOp) {
+  Init *Arg = Result->getArg(AliasOpNo);
+  DefInit *ADI = dyn_cast<DefInit>(Arg);
+  Record *ResultRecord = ADI ? ADI->getDef() : nullptr;
+
+  if (ADI && ADI->getDef() == InstOpRec) {
+    // If the operand is a record, it must have a name, and the record type
+    // must match up with the instruction's argument type.
+    if (!Result->getArgName(AliasOpNo))
+      PrintFatalError(Loc, "result argument #" + Twine(AliasOpNo) +
+                               " must have a name!");
+    ResOp = ResultOperand(std::string(Result->getArgNameStr(AliasOpNo)),
+                          ResultRecord);
+    return true;
+  }
+
+  // For register operands, the source register class can be a subclass
+  // of the instruction register class, not just an exact match.
+  if (InstOpRec->isSubClassOf("RegisterOperand"))
+    InstOpRec = InstOpRec->getValueAsDef("RegClass");
+
+  if (ADI && ADI->getDef()->isSubClassOf("RegisterOperand"))
+    ADI = ADI->getDef()->getValueAsDef("RegClass")->getDefInit();
+
+  if (ADI && ADI->getDef()->isSubClassOf("RegisterClass")) {
+    if (!InstOpRec->isSubClassOf("RegisterClass"))
+      return false;
+    if (!T.getRegisterClass(InstOpRec).hasSubClass(
+            &T.getRegisterClass(ADI->getDef())))
+      return false;
+    ResOp = ResultOperand(std::string(Result->getArgNameStr(AliasOpNo)),
+                          ResultRecord);
+    return true;
+  }
+
+  // Handle explicit registers.
+  if (ADI && ADI->getDef()->isSubClassOf("Register")) {
+    if (InstOpRec->isSubClassOf("OptionalDefOperand")) {
+      DagInit *DI = InstOpRec->getValueAsDag("MIOperandInfo");
+      // The operand info should only have a single (register) entry. We
+      // want the register class of it.
+      InstOpRec = cast<DefInit>(DI->getArg(0))->getDef();
+    }
+
+    if (!InstOpRec->isSubClassOf("RegisterClass"))
+      return false;
+
+    if (!T.getRegisterClass(InstOpRec).contains(
+            T.getRegBank().getReg(ADI->getDef())))
+      PrintFatalError(Loc, "fixed register " + ADI->getDef()->getName() +
+                               " is not a member of the " +
+                               InstOpRec->getName() + " register class!");
+
+    if (Result->getArgName(AliasOpNo))
+      PrintFatalError(Loc, "result fixed register argument must "
+                           "not have a name!");
+
+    ResOp = ResultOperand(ResultRecord);
+    return true;
+  }
+
+  // Handle "zero_reg" for optional def operands.
+  if (ADI && ADI->getDef()->getName() == "zero_reg") {
+
+    // Check if this is an optional def.
+    // Tied operands where the source is a sub-operand of a complex operand
+    // need to represent both operands in the alias destination instruction.
+    // Allow zero_reg for the tied portion. This can and should go away once
+    // the MC representation of things doesn't use tied operands at all.
+    // if (!InstOpRec->isSubClassOf("OptionalDefOperand"))
+    //  throw TGError(Loc, "reg0 used for result that is not an "
+    //                "OptionalDefOperand!");
+
+    ResOp = ResultOperand(static_cast<Record *>(nullptr));
+    return true;
+  }
+
+  // Literal integers.
+  if (IntInit *II = dyn_cast<IntInit>(Arg)) {
+    if (hasSubOps || !InstOpRec->isSubClassOf("Operand"))
+      return false;
+    // Integer arguments can't have names.
+    if (Result->getArgName(AliasOpNo))
+      PrintFatalError(Loc, "result argument #" + Twine(AliasOpNo) +
+                               " must not have a name!");
+    ResOp = ResultOperand(II->getValue());
+    return true;
+  }
+
+  // Bits<n> (also used for 0bxx literals)
+  if (BitsInit *BI = dyn_cast<BitsInit>(Arg)) {
+    if (hasSubOps || !InstOpRec->isSubClassOf("Operand"))
+      return false;
+    if (!BI->isComplete())
+      return false;
+    // Convert the bits init to an integer and use that for the result.
+    IntInit *II = dyn_cast_or_null<IntInit>(
+        BI->convertInitializerTo(IntRecTy::get(BI->getRecordKeeper())));
+    if (!II)
+      return false;
+    ResOp = ResultOperand(II->getValue());
+    return true;
+  }
+
+  // If both are Operands with the same MVT, allow the conversion. It's
+  // up to the user to make sure the values are appropriate, just like
+  // for isel Pat's.
+  if (InstOpRec->isSubClassOf("Operand") && ADI &&
+      ADI->getDef()->isSubClassOf("Operand")) {
+    // FIXME: What other attributes should we check here? Identical
+    // MIOperandInfo perhaps?
+    if (InstOpRec->getValueInit("Type") != ADI->getDef()->getValueInit("Type"))
+      return false;
+    ResOp = ResultOperand(std::string(Result->getArgNameStr(AliasOpNo)),
+                          ADI->getDef());
+    return true;
+  }
+
+  return false;
+}
+
+unsigned CodeGenInstAlias::ResultOperand::getMINumOperands() const {
+  if (!isRecord())
+    return 1;
+
+  Record *Rec = getRecord();
+  if (!Rec->isSubClassOf("Operand"))
+    return 1;
+
+  DagInit *MIOpInfo = Rec->getValueAsDag("MIOperandInfo");
+  if (MIOpInfo->getNumArgs() == 0) {
+    // Unspecified, so it defaults to 1
+    return 1;
+  }
+
+  return MIOpInfo->getNumArgs();
+}
+
+CodeGenInstAlias::CodeGenInstAlias(Record *R, CodeGenTarget &T) : TheDef(R) {
+  Result = R->getValueAsDag("ResultInst");
+  AsmString = std::string(R->getValueAsString("AsmString"));
+
+  // Verify that the root of the result is an instruction.
+  DefInit *DI = dyn_cast<DefInit>(Result->getOperator());
+  if (!DI || !DI->getDef()->isSubClassOf("Instruction"))
+    PrintFatalError(R->getLoc(),
+                    "result of inst alias should be an instruction");
+
+  ResultInst = &T.getInstruction(DI->getDef());
+
+  // NameClass - If argument names are repeated, we need to verify they have
+  // the same class.
+  StringMap<Record *> NameClass;
+  for (unsigned i = 0, e = Result->getNumArgs(); i != e; ++i) {
+    DefInit *ADI = dyn_cast<DefInit>(Result->getArg(i));
+    if (!ADI || !Result->getArgName(i))
+      continue;
+    // Verify we don't have something like: (someinst GR16:$foo, GR32:$foo)
+    // $foo can exist multiple times in the result list, but it must have the
+    // same type.
+    Record *&Entry = NameClass[Result->getArgNameStr(i)];
+    if (Entry && Entry != ADI->getDef())
+      PrintFatalError(R->getLoc(), "result value $" + Result->getArgNameStr(i) +
+                                       " is both " + Entry->getName() +
+                                       " and " + ADI->getDef()->getName() +
+                                       "!");
+    Entry = ADI->getDef();
+  }
+
+  // Decode and validate the arguments of the result.
+  unsigned AliasOpNo = 0;
+  for (unsigned i = 0, e = ResultInst->Operands.size(); i != e; ++i) {
+
+    // Tied registers don't have an entry in the result dag unless they're part
+    // of a complex operand, in which case we include them anyways, as we
+    // don't have any other way to specify the whole operand.
+    if (ResultInst->Operands[i].MINumOperands == 1 &&
+        ResultInst->Operands[i].getTiedRegister() != -1) {
+      // Tied operands of different RegisterClass should be explicit within an
+      // instruction's syntax and so cannot be skipped.
+      int TiedOpNum = ResultInst->Operands[i].getTiedRegister();
+      if (ResultInst->Operands[i].Rec->getName() ==
+          ResultInst->Operands[TiedOpNum].Rec->getName())
+        continue;
+    }
+
+    if (AliasOpNo >= Result->getNumArgs())
+      PrintFatalError(R->getLoc(), "not enough arguments for instruction!");
+
+    Record *InstOpRec = ResultInst->Operands[i].Rec;
+    unsigned NumSubOps = ResultInst->Operands[i].MINumOperands;
+    ResultOperand ResOp(static_cast<int64_t>(0));
+    if (tryAliasOpMatch(Result, AliasOpNo, InstOpRec, (NumSubOps > 1),
+                        R->getLoc(), T, ResOp)) {
+      // If this is a simple operand, or a complex operand with a custom match
+      // class, then we can match is verbatim.
+      if (NumSubOps == 1 || (InstOpRec->getValue("ParserMatchClass") &&
+                             InstOpRec->getValueAsDef("ParserMatchClass")
+                                     ->getValueAsString("Name") != "Imm")) {
+        ResultOperands.push_back(ResOp);
+        ResultInstOperandIndex.push_back(std::make_pair(i, -1));
+        ++AliasOpNo;
+
+        // Otherwise, we need to match each of the suboperands individually.
+      } else {
+        DagInit *MIOI = ResultInst->Operands[i].MIOperandInfo;
+        for (unsigned SubOp = 0; SubOp != NumSubOps; ++SubOp) {
+          Record *SubRec = cast<DefInit>(MIOI->getArg(SubOp))->getDef();
+
+          // Take care to instantiate each of the suboperands with the correct
+          // nomenclature: $foo.bar
+          ResultOperands.emplace_back(
+              Result->getArgName(AliasOpNo)->getAsUnquotedString() + "." +
+                  MIOI->getArgName(SubOp)->getAsUnquotedString(),
+              SubRec);
+          ResultInstOperandIndex.push_back(std::make_pair(i, SubOp));
+        }
+        ++AliasOpNo;
+      }
+      continue;
+    }
+
+    // If the argument did not match the instruction operand, and the operand
+    // is composed of multiple suboperands, try matching the suboperands.
+    if (NumSubOps > 1) {
+      DagInit *MIOI = ResultInst->Operands[i].MIOperandInfo;
+      for (unsigned SubOp = 0; SubOp != NumSubOps; ++SubOp) {
+        if (AliasOpNo >= Result->getNumArgs())
+          PrintFatalError(R->getLoc(), "not enough arguments for instruction!");
+        Record *SubRec = cast<DefInit>(MIOI->getArg(SubOp))->getDef();
+        if (tryAliasOpMatch(Result, AliasOpNo, SubRec, false, R->getLoc(), T,
+                            ResOp)) {
+          ResultOperands.push_back(ResOp);
+          ResultInstOperandIndex.push_back(std::make_pair(i, SubOp));
+          ++AliasOpNo;
+        } else {
+          PrintFatalError(
+              R->getLoc(),
+              "result argument #" + Twine(AliasOpNo) +
+                  " does not match instruction operand class " +
+                  (SubOp == 0 ? InstOpRec->getName() : SubRec->getName()));
+        }
+      }
+      continue;
+    }
+    PrintFatalError(R->getLoc(),
+                    "result argument #" + Twine(AliasOpNo) +
+                        " does not match instruction operand class " +
+                        InstOpRec->getName());
+  }
+
+  if (AliasOpNo != Result->getNumArgs())
+    PrintFatalError(R->getLoc(), "too many operands for instruction!");
+}
diff --git a/llvm/utils/TableGen/CodeGenInstAlias.h b/llvm/utils/TableGen/CodeGenInstAlias.h
new file mode 100644
index 000000000000..2a05273e7270
--- /dev/null
+++ b/llvm/utils/TableGen/CodeGenInstAlias.h
@@ -0,0 +1,105 @@
+//===- CodeGenInstAlias.h - InstAlias Class Wrapper -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a wrapper class for the 'InstAlias' TableGen class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UTILS_TABLEGEN_CODEGENINSTALIAS_H
+#define LLVM_UTILS_TABLEGEN_CODEGENINSTALIAS_H
+
+#include "llvm/ADT/StringRef.h"
+#include <cassert>
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace llvm {
+
+template <typename T> class ArrayRef;
+class CodeGenInstruction;
+class CodeGenTarget;
+class DagInit;
+class SMLoc;
+class Record;
+
+/// CodeGenInstAlias - This represents an InstAlias definition.
+class CodeGenInstAlias {
+public:
+  Record *TheDef; // The actual record defining this InstAlias.
+
+  /// AsmString - The format string used to emit a .s file for the
+  /// instruction.
+  std::string AsmString;
+
+  /// Result - The result instruction.
+  DagInit *Result;
+
+  /// ResultInst - The instruction generated by the alias (decoded from
+  /// Result).
+  CodeGenInstruction *ResultInst;
+
+  struct ResultOperand {
+  private:
+    std::string Name;
+    Record *R = nullptr;
+    int64_t Imm = 0;
+
+  public:
+    enum { K_Record, K_Imm, K_Reg } Kind;
+
+    ResultOperand(std::string N, Record *r)
+        : Name(std::move(N)), R(r), Kind(K_Record) {}
+    ResultOperand(int64_t I) : Imm(I), Kind(K_Imm) {}
+    ResultOperand(Record *r) : R(r), Kind(K_Reg) {}
+
+    bool isRecord() const { return Kind == K_Record; }
+    bool isImm() const { return Kind == K_Imm; }
+    bool isReg() const { return Kind == K_Reg; }
+
+    StringRef getName() const {
+      assert(isRecord());
+      return Name;
+    }
+    Record *getRecord() const {
+      assert(isRecord());
+      return R;
+    }
+    int64_t getImm() const {
+      assert(isImm());
+      return Imm;
+    }
+    Record *getRegister() const {
+      assert(isReg());
+      return R;
+    }
+
+    unsigned getMINumOperands() const;
+  };
+
+  /// ResultOperands - The decoded operands for the result instruction.
+  std::vector<ResultOperand> ResultOperands;
+
+  /// ResultInstOperandIndex - For each operand, this vector holds a pair of
+  /// indices to identify the corresponding operand in the result
+  /// instruction.  The first index specifies the operand and the second
+  /// index specifies the suboperand.  If there are no suboperands or if all
+  /// of them are matched by the operand, the second value should be -1.
+  std::vector<std::pair<unsigned, int>> ResultInstOperandIndex;
+
+  CodeGenInstAlias(Record *R, CodeGenTarget &T);
+
+  bool tryAliasOpMatch(DagInit *Result, unsigned AliasOpNo, Record *InstOpRec,
+                       bool hasSubOps, ArrayRef<SMLoc> Loc, CodeGenTarget &T,
+                       ResultOperand &ResOp);
+};
+
+} // namespace llvm
+
+#endif // LLVM_UTILS_TABLEGEN_CODEGENINSTALIAS_H
diff --git a/llvm/utils/TableGen/CodeGenInstruction.cpp b/llvm/utils/TableGen/CodeGenInstruction.cpp
index 238c6a1b6ba8..8662b6fb52da 100644
--- a/llvm/utils/TableGen/CodeGenInstruction.cpp
+++ b/llvm/utils/TableGen/CodeGenInstruction.cpp
@@ -13,7 +13,6 @@
 #include "CodeGenInstruction.h"
 #include "CodeGenTarget.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include <set>
@@ -176,7 +175,7 @@ CGIOperandList::CGIOperandList(Record *R) : TheDef(R) {
         }
 
         OpInfo.SubOpNames[j] = SubArgName;
-        SubOpAliases[SubArgName] = std::make_pair(MIOperandNo, j);
+        SubOpAliases[SubArgName] = std::make_pair(i, j);
       }
     } else if (!EncoderMethod.empty()) {
       // If we have no explicit sub-op dag, but have an top-level encoder
@@ -592,266 +591,3 @@ bool CodeGenInstruction::isOperandImpl(StringRef OpListName, unsigned i,
   return Constraint->getDef()->isSubClassOf("TypedOperand") &&
          Constraint->getDef()->getValueAsBit(PropertyName);
 }
-
-//===----------------------------------------------------------------------===//
-/// CodeGenInstAlias Implementation
-//===----------------------------------------------------------------------===//
-
-/// tryAliasOpMatch - This is a helper function for the CodeGenInstAlias
-/// constructor.  It checks if an argument in an InstAlias pattern matches
-/// the corresponding operand of the instruction.  It returns true on a
-/// successful match, with ResOp set to the result operand to be used.
-bool CodeGenInstAlias::tryAliasOpMatch(DagInit *Result, unsigned AliasOpNo,
-                                       Record *InstOpRec, bool hasSubOps,
-                                       ArrayRef<SMLoc> Loc, CodeGenTarget &T,
-                                       ResultOperand &ResOp) {
-  Init *Arg = Result->getArg(AliasOpNo);
-  DefInit *ADI = dyn_cast<DefInit>(Arg);
-  Record *ResultRecord = ADI ? ADI->getDef() : nullptr;
-
-  if (ADI && ADI->getDef() == InstOpRec) {
-    // If the operand is a record, it must have a name, and the record type
-    // must match up with the instruction's argument type.
-    if (!Result->getArgName(AliasOpNo))
-      PrintFatalError(Loc, "result argument #" + Twine(AliasOpNo) +
-                           " must have a name!");
-    ResOp = ResultOperand(std::string(Result->getArgNameStr(AliasOpNo)),
-                          ResultRecord);
-    return true;
-  }
-
-  // For register operands, the source register class can be a subclass
-  // of the instruction register class, not just an exact match.
-  if (InstOpRec->isSubClassOf("RegisterOperand"))
-    InstOpRec = InstOpRec->getValueAsDef("RegClass");
-
-  if (ADI && ADI->getDef()->isSubClassOf("RegisterOperand"))
-    ADI = ADI->getDef()->getValueAsDef("RegClass")->getDefInit();
-
-  if (ADI && ADI->getDef()->isSubClassOf("RegisterClass")) {
-    if (!InstOpRec->isSubClassOf("RegisterClass"))
-      return false;
-    if (!T.getRegisterClass(InstOpRec)
-              .hasSubClass(&T.getRegisterClass(ADI->getDef())))
-      return false;
-    ResOp = ResultOperand(std::string(Result->getArgNameStr(AliasOpNo)),
-                          ResultRecord);
-    return true;
-  }
-
-  // Handle explicit registers.
-  if (ADI && ADI->getDef()->isSubClassOf("Register")) {
-    if (InstOpRec->isSubClassOf("OptionalDefOperand")) {
-      DagInit *DI = InstOpRec->getValueAsDag("MIOperandInfo");
-      // The operand info should only have a single (register) entry. We
-      // want the register class of it.
-      InstOpRec = cast<DefInit>(DI->getArg(0))->getDef();
-    }
-
-    if (!InstOpRec->isSubClassOf("RegisterClass"))
-      return false;
-
-    if (!T.getRegisterClass(InstOpRec)
-        .contains(T.getRegBank().getReg(ADI->getDef())))
-      PrintFatalError(Loc, "fixed register " + ADI->getDef()->getName() +
-                      " is not a member of the " + InstOpRec->getName() +
-                      " register class!");
-
-    if (Result->getArgName(AliasOpNo))
-      PrintFatalError(Loc, "result fixed register argument must "
-                      "not have a name!");
-
-    ResOp = ResultOperand(ResultRecord);
-    return true;
-  }
-
-  // Handle "zero_reg" for optional def operands.
-  if (ADI && ADI->getDef()->getName() == "zero_reg") {
-
-    // Check if this is an optional def.
-    // Tied operands where the source is a sub-operand of a complex operand
-    // need to represent both operands in the alias destination instruction.
-    // Allow zero_reg for the tied portion. This can and should go away once
-    // the MC representation of things doesn't use tied operands at all.
-    //if (!InstOpRec->isSubClassOf("OptionalDefOperand"))
-    //  throw TGError(Loc, "reg0 used for result that is not an "
-    //                "OptionalDefOperand!");
-
-    ResOp = ResultOperand(static_cast<Record*>(nullptr));
-    return true;
-  }
-
-  // Literal integers.
-  if (IntInit *II = dyn_cast<IntInit>(Arg)) {
-    if (hasSubOps || !InstOpRec->isSubClassOf("Operand"))
-      return false;
-    // Integer arguments can't have names.
-    if (Result->getArgName(AliasOpNo))
-      PrintFatalError(Loc, "result argument #" + Twine(AliasOpNo) +
-                      " must not have a name!");
-    ResOp = ResultOperand(II->getValue());
-    return true;
-  }
-
-  // Bits<n> (also used for 0bxx literals)
-  if (BitsInit *BI = dyn_cast<BitsInit>(Arg)) {
-    if (hasSubOps || !InstOpRec->isSubClassOf("Operand"))
-      return false;
-    if (!BI->isComplete())
-      return false;
-    // Convert the bits init to an integer and use that for the result.
-    IntInit *II = dyn_cast_or_null<IntInit>(
-        BI->convertInitializerTo(IntRecTy::get(BI->getRecordKeeper())));
-    if (!II)
-      return false;
-    ResOp = ResultOperand(II->getValue());
-    return true;
-  }
-
-  // If both are Operands with the same MVT, allow the conversion. It's
-  // up to the user to make sure the values are appropriate, just like
-  // for isel Pat's.
-  if (InstOpRec->isSubClassOf("Operand") && ADI &&
-      ADI->getDef()->isSubClassOf("Operand")) {
-    // FIXME: What other attributes should we check here? Identical
-    // MIOperandInfo perhaps?
-    if (InstOpRec->getValueInit("Type") != ADI->getDef()->getValueInit("Type"))
-      return false;
-    ResOp = ResultOperand(std::string(Result->getArgNameStr(AliasOpNo)),
-                          ADI->getDef());
-    return true;
-  }
-
-  return false;
-}
-
-unsigned CodeGenInstAlias::ResultOperand::getMINumOperands() const {
-  if (!isRecord())
-    return 1;
-
-  Record *Rec = getRecord();
-  if (!Rec->isSubClassOf("Operand"))
-    return 1;
-
-  DagInit *MIOpInfo = Rec->getValueAsDag("MIOperandInfo");
-  if (MIOpInfo->getNumArgs() == 0) {
-    // Unspecified, so it defaults to 1
-    return 1;
-  }
-
-  return MIOpInfo->getNumArgs();
-}
-
-CodeGenInstAlias::CodeGenInstAlias(Record *R, CodeGenTarget &T)
-    : TheDef(R) {
-  Result = R->getValueAsDag("ResultInst");
-  AsmString = std::string(R->getValueAsString("AsmString"));
-
-  // Verify that the root of the result is an instruction.
-  DefInit *DI = dyn_cast<DefInit>(Result->getOperator());
-  if (!DI || !DI->getDef()->isSubClassOf("Instruction"))
-    PrintFatalError(R->getLoc(),
-                    "result of inst alias should be an instruction");
-
-  ResultInst = &T.getInstruction(DI->getDef());
-
-  // NameClass - If argument names are repeated, we need to verify they have
-  // the same class.
-  StringMap<Record*> NameClass;
-  for (unsigned i = 0, e = Result->getNumArgs(); i != e; ++i) {
-    DefInit *ADI = dyn_cast<DefInit>(Result->getArg(i));
-    if (!ADI || !Result->getArgName(i))
-      continue;
-    // Verify we don't have something like: (someinst GR16:$foo, GR32:$foo)
-    // $foo can exist multiple times in the result list, but it must have the
-    // same type.
-    Record *&Entry = NameClass[Result->getArgNameStr(i)];
-    if (Entry && Entry != ADI->getDef())
-      PrintFatalError(R->getLoc(), "result value $" + Result->getArgNameStr(i) +
-                      " is both " + Entry->getName() + " and " +
-                      ADI->getDef()->getName() + "!");
-    Entry = ADI->getDef();
-  }
-
-  // Decode and validate the arguments of the result.
-  unsigned AliasOpNo = 0;
-  for (unsigned i = 0, e = ResultInst->Operands.size(); i != e; ++i) {
-
-    // Tied registers don't have an entry in the result dag unless they're part
-    // of a complex operand, in which case we include them anyways, as we
-    // don't have any other way to specify the whole operand.
-    if (ResultInst->Operands[i].MINumOperands == 1 &&
-        ResultInst->Operands[i].getTiedRegister() != -1) {
-      // Tied operands of different RegisterClass should be explicit within an
-      // instruction's syntax and so cannot be skipped.
-      int TiedOpNum = ResultInst->Operands[i].getTiedRegister();
-      if (ResultInst->Operands[i].Rec->getName() ==
-          ResultInst->Operands[TiedOpNum].Rec->getName())
-        continue;
-    }
-
-    if (AliasOpNo >= Result->getNumArgs())
-      PrintFatalError(R->getLoc(), "not enough arguments for instruction!");
-
-    Record *InstOpRec = ResultInst->Operands[i].Rec;
-    unsigned NumSubOps = ResultInst->Operands[i].MINumOperands;
-    ResultOperand ResOp(static_cast<int64_t>(0));
-    if (tryAliasOpMatch(Result, AliasOpNo, InstOpRec, (NumSubOps > 1),
-                        R->getLoc(), T, ResOp)) {
-      // If this is a simple operand, or a complex operand with a custom match
-      // class, then we can match is verbatim.
-      if (NumSubOps == 1 ||
-          (InstOpRec->getValue("ParserMatchClass") &&
-           InstOpRec->getValueAsDef("ParserMatchClass")
-             ->getValueAsString("Name") != "Imm")) {
-        ResultOperands.push_back(ResOp);
-        ResultInstOperandIndex.push_back(std::make_pair(i, -1));
-        ++AliasOpNo;
-
-      // Otherwise, we need to match each of the suboperands individually.
-      } else {
-         DagInit *MIOI = ResultInst->Operands[i].MIOperandInfo;
-         for (unsigned SubOp = 0; SubOp != NumSubOps; ++SubOp) {
-          Record *SubRec = cast<DefInit>(MIOI->getArg(SubOp))->getDef();
-
-          // Take care to instantiate each of the suboperands with the correct
-          // nomenclature: $foo.bar
-          ResultOperands.emplace_back(
-            Result->getArgName(AliasOpNo)->getAsUnquotedString() + "." +
-            MIOI->getArgName(SubOp)->getAsUnquotedString(), SubRec);
-          ResultInstOperandIndex.push_back(std::make_pair(i, SubOp));
-         }
-         ++AliasOpNo;
-      }
-      continue;
-    }
-
-    // If the argument did not match the instruction operand, and the operand
-    // is composed of multiple suboperands, try matching the suboperands.
-    if (NumSubOps > 1) {
-      DagInit *MIOI = ResultInst->Operands[i].MIOperandInfo;
-      for (unsigned SubOp = 0; SubOp != NumSubOps; ++SubOp) {
-        if (AliasOpNo >= Result->getNumArgs())
-          PrintFatalError(R->getLoc(), "not enough arguments for instruction!");
-        Record *SubRec = cast<DefInit>(MIOI->getArg(SubOp))->getDef();
-        if (tryAliasOpMatch(Result, AliasOpNo, SubRec, false,
-                            R->getLoc(), T, ResOp)) {
-          ResultOperands.push_back(ResOp);
-          ResultInstOperandIndex.push_back(std::make_pair(i, SubOp));
-          ++AliasOpNo;
-        } else {
-          PrintFatalError(R->getLoc(), "result argument #" + Twine(AliasOpNo) +
-                        " does not match instruction operand class " +
-                        (SubOp == 0 ? InstOpRec->getName() :SubRec->getName()));
-        }
-      }
-      continue;
-    }
-    PrintFatalError(R->getLoc(), "result argument #" + Twine(AliasOpNo) +
-                    " does not match instruction operand class " +
-                    InstOpRec->getName());
-  }
-
-  if (AliasOpNo != Result->getNumArgs())
-    PrintFatalError(R->getLoc(), "too many operands for instruction!");
-}
diff --git a/llvm/utils/TableGen/CodeGenInstruction.h b/llvm/utils/TableGen/CodeGenInstruction.h
index 72626caada56..ee7a1696bab9 100644
--- a/llvm/utils/TableGen/CodeGenInstruction.h
+++ b/llvm/utils/TableGen/CodeGenInstruction.h
@@ -16,15 +16,13 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/MachineValueType.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include <cassert>
 #include <string>
 #include <utility>
 #include <vector>
 
 namespace llvm {
-class SMLoc;
-template <typename T> class ArrayRef;
   class Record;
   class DagInit;
   class CodeGenTarget;
@@ -340,71 +338,6 @@ template <typename T> class ArrayRef;
     bool isOperandImpl(StringRef OpListName, unsigned i,
                        StringRef PropertyName) const;
   };
-
-
-  /// CodeGenInstAlias - This represents an InstAlias definition.
-  class CodeGenInstAlias {
-  public:
-    Record *TheDef;            // The actual record defining this InstAlias.
-
-    /// AsmString - The format string used to emit a .s file for the
-    /// instruction.
-    std::string AsmString;
-
-    /// Result - The result instruction.
-    DagInit *Result;
-
-    /// ResultInst - The instruction generated by the alias (decoded from
-    /// Result).
-    CodeGenInstruction *ResultInst;
-
-
-    struct ResultOperand {
-    private:
-      std::string Name;
-      Record *R = nullptr;
-      int64_t Imm = 0;
-
-    public:
-      enum {
-        K_Record,
-        K_Imm,
-        K_Reg
-      } Kind;
-
-      ResultOperand(std::string N, Record *r)
-          : Name(std::move(N)), R(r), Kind(K_Record) {}
-      ResultOperand(int64_t I) : Imm(I), Kind(K_Imm) {}
-      ResultOperand(Record *r) : R(r), Kind(K_Reg) {}
-
-      bool isRecord() const { return Kind == K_Record; }
-      bool isImm() const { return Kind == K_Imm; }
-      bool isReg() const { return Kind == K_Reg; }
-
-      StringRef getName() const { assert(isRecord()); return Name; }
-      Record *getRecord() const { assert(isRecord()); return R; }
-      int64_t getImm() const { assert(isImm()); return Imm; }
-      Record *getRegister() const { assert(isReg()); return R; }
-
-      unsigned getMINumOperands() const;
-    };
-
-    /// ResultOperands - The decoded operands for the result instruction.
-    std::vector<ResultOperand> ResultOperands;
-
-    /// ResultInstOperandIndex - For each operand, this vector holds a pair of
-    /// indices to identify the corresponding operand in the result
-    /// instruction.  The first index specifies the operand and the second
-    /// index specifies the suboperand.  If there are no suboperands or if all
-    /// of them are matched by the operand, the second value should be -1.
-    std::vector<std::pair<unsigned, int> > ResultInstOperandIndex;
-
-    CodeGenInstAlias(Record *R, CodeGenTarget &T);
-
-    bool tryAliasOpMatch(DagInit *Result, unsigned AliasOpNo,
-                         Record *InstOpRec, bool hasSubOps, ArrayRef<SMLoc> Loc,
-                         CodeGenTarget &T, ResultOperand &ResOp);
-  };
-}
+} // namespace llvm
 
 #endif
diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/CodeGenIntrinsics.cpp
new file mode 100644
index 000000000000..7cb86ad95266
--- /dev/null
+++ b/llvm/utils/TableGen/CodeGenIntrinsics.cpp
@@ -0,0 +1,270 @@
+//===- CodeGenIntrinsics.cpp - Intrinsic Class Wrapper --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a wrapper class for the 'Intrinsic' TableGen class.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CodeGenIntrinsics.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+#include <algorithm>
+#include <cassert>
+using namespace llvm;
+
+//===----------------------------------------------------------------------===//
+// CodeGenIntrinsic Implementation
+//===----------------------------------------------------------------------===//
+
+CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) {
+  std::vector<Record *> IntrProperties =
+      RC.getAllDerivedDefinitions("IntrinsicProperty");
+
+  std::vector<Record *> DefaultProperties;
+  for (Record *Rec : IntrProperties)
+    if (Rec->getValueAsBit("IsDefault"))
+      DefaultProperties.push_back(Rec);
+
+  std::vector<Record *> Defs = RC.getAllDerivedDefinitions("Intrinsic");
+  Intrinsics.reserve(Defs.size());
+
+  for (unsigned I = 0, e = Defs.size(); I != e; ++I)
+    Intrinsics.push_back(CodeGenIntrinsic(Defs[I], DefaultProperties));
+
+  llvm::sort(Intrinsics,
+             [](const CodeGenIntrinsic &LHS, const CodeGenIntrinsic &RHS) {
+               return std::tie(LHS.TargetPrefix, LHS.Name) <
+                      std::tie(RHS.TargetPrefix, RHS.Name);
+             });
+  Targets.push_back({"", 0, 0});
+  for (size_t I = 0, E = Intrinsics.size(); I < E; ++I)
+    if (Intrinsics[I].TargetPrefix != Targets.back().Name) {
+      Targets.back().Count = I - Targets.back().Offset;
+      Targets.push_back({Intrinsics[I].TargetPrefix, I, 0});
+    }
+  Targets.back().Count = Intrinsics.size() - Targets.back().Offset;
+}
+
+CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
+                                   std::vector<Record *> DefaultProperties) {
+  TheDef = R;
+  std::string DefName = std::string(R->getName());
+  ArrayRef<SMLoc> DefLoc = R->getLoc();
+  Properties = 0;
+  isOverloaded = false;
+  isCommutative = false;
+  canThrow = false;
+  isNoReturn = false;
+  isNoCallback = false;
+  isNoSync = false;
+  isNoFree = false;
+  isWillReturn = false;
+  isCold = false;
+  isNoDuplicate = false;
+  isNoMerge = false;
+  isConvergent = false;
+  isSpeculatable = false;
+  hasSideEffects = false;
+  isStrictFP = false;
+
+  if (DefName.size() <= 4 || DefName.substr(0, 4) != "int_")
+    PrintFatalError(DefLoc,
+                    "Intrinsic '" + DefName + "' does not start with 'int_'!");
+
+  EnumName = DefName.substr(4);
+
+  if (R->getValue(
+          "ClangBuiltinName")) // Ignore a missing ClangBuiltinName field.
+    ClangBuiltinName = std::string(R->getValueAsString("ClangBuiltinName"));
+  if (R->getValue("MSBuiltinName")) // Ignore a missing MSBuiltinName field.
+    MSBuiltinName = std::string(R->getValueAsString("MSBuiltinName"));
+
+  TargetPrefix = std::string(R->getValueAsString("TargetPrefix"));
+  Name = std::string(R->getValueAsString("LLVMName"));
+
+  if (Name == "") {
+    // If an explicit name isn't specified, derive one from the DefName.
+    Name = "llvm.";
+
+    for (unsigned i = 0, e = EnumName.size(); i != e; ++i)
+      Name += (EnumName[i] == '_') ? '.' : EnumName[i];
+  } else {
+    // Verify it starts with "llvm.".
+    if (Name.size() <= 5 || Name.substr(0, 5) != "llvm.")
+      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
+                                  "'s name does not start with 'llvm.'!");
+  }
+
+  // If TargetPrefix is specified, make sure that Name starts with
+  // "llvm.<targetprefix>.".
+  if (!TargetPrefix.empty()) {
+    if (Name.size() < 6 + TargetPrefix.size() ||
+        Name.substr(5, 1 + TargetPrefix.size()) != (TargetPrefix + "."))
+      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
+                                  "' does not start with 'llvm." +
+                                  TargetPrefix + ".'!");
+  }
+
+  if (auto *Types = R->getValue("Types")) {
+    auto *TypeList = cast<ListInit>(Types->getValue());
+    isOverloaded = R->getValueAsBit("isOverloaded");
+
+    unsigned I = 0;
+    for (unsigned E = R->getValueAsListInit("RetTypes")->size(); I < E; ++I)
+      IS.RetTys.push_back(TypeList->getElementAsRecord(I));
+
+    for (unsigned E = TypeList->size(); I < E; ++I)
+      IS.ParamTys.push_back(TypeList->getElementAsRecord(I));
+  }
+
+  // Parse the intrinsic properties.
+  ListInit *PropList = R->getValueAsListInit("IntrProperties");
+  for (unsigned i = 0, e = PropList->size(); i != e; ++i) {
+    Record *Property = PropList->getElementAsRecord(i);
+    assert(Property->isSubClassOf("IntrinsicProperty") &&
+           "Expected a property!");
+
+    setProperty(Property);
+  }
+
+  // Set default properties to true.
+  setDefaultProperties(R, DefaultProperties);
+
+  // Also record the SDPatternOperator Properties.
+  Properties = parseSDPatternOperatorProperties(R);
+
+  // Sort the argument attributes for later benefit.
+  for (auto &Attrs : ArgumentAttributes)
+    llvm::sort(Attrs);
+}
+
+void CodeGenIntrinsic::setDefaultProperties(
+    Record *R, std::vector<Record *> DefaultProperties) {
+  // opt-out of using default attributes.
+  if (R->getValueAsBit("DisableDefaultAttributes"))
+    return;
+
+  for (Record *Rec : DefaultProperties)
+    setProperty(Rec);
+}
+
+void CodeGenIntrinsic::setProperty(Record *R) {
+  if (R->getName() == "IntrNoMem")
+    ME = MemoryEffects::none();
+  else if (R->getName() == "IntrReadMem") {
+    if (ME.onlyWritesMemory())
+      PrintFatalError(TheDef->getLoc(),
+                      Twine("IntrReadMem cannot be used after IntrNoMem or "
+                            "IntrWriteMem. Default is ReadWrite"));
+    ME &= MemoryEffects::readOnly();
+  } else if (R->getName() == "IntrWriteMem") {
+    if (ME.onlyReadsMemory())
+      PrintFatalError(TheDef->getLoc(),
+                      Twine("IntrWriteMem cannot be used after IntrNoMem or "
+                            "IntrReadMem. Default is ReadWrite"));
+    ME &= MemoryEffects::writeOnly();
+  } else if (R->getName() == "IntrArgMemOnly")
+    ME &= MemoryEffects::argMemOnly();
+  else if (R->getName() == "IntrInaccessibleMemOnly")
+    ME &= MemoryEffects::inaccessibleMemOnly();
+  else if (R->getName() == "IntrInaccessibleMemOrArgMemOnly")
+    ME &= MemoryEffects::inaccessibleOrArgMemOnly();
+  else if (R->getName() == "Commutative")
+    isCommutative = true;
+  else if (R->getName() == "Throws")
+    canThrow = true;
+  else if (R->getName() == "IntrNoDuplicate")
+    isNoDuplicate = true;
+  else if (R->getName() == "IntrNoMerge")
+    isNoMerge = true;
+  else if (R->getName() == "IntrConvergent")
+    isConvergent = true;
+  else if (R->getName() == "IntrNoReturn")
+    isNoReturn = true;
+  else if (R->getName() == "IntrNoCallback")
+    isNoCallback = true;
+  else if (R->getName() == "IntrNoSync")
+    isNoSync = true;
+  else if (R->getName() == "IntrNoFree")
+    isNoFree = true;
+  else if (R->getName() == "IntrWillReturn")
+    isWillReturn = !isNoReturn;
+  else if (R->getName() == "IntrCold")
+    isCold = true;
+  else if (R->getName() == "IntrSpeculatable")
+    isSpeculatable = true;
+  else if (R->getName() == "IntrHasSideEffects")
+    hasSideEffects = true;
+  else if (R->getName() == "IntrStrictFP")
+    isStrictFP = true;
+  else if (R->isSubClassOf("NoCapture")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    addArgAttribute(ArgNo, NoCapture);
+  } else if (R->isSubClassOf("NoAlias")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    addArgAttribute(ArgNo, NoAlias);
+  } else if (R->isSubClassOf("NoUndef")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    addArgAttribute(ArgNo, NoUndef);
+  } else if (R->isSubClassOf("NonNull")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    addArgAttribute(ArgNo, NonNull);
+  } else if (R->isSubClassOf("Returned")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    addArgAttribute(ArgNo, Returned);
+  } else if (R->isSubClassOf("ReadOnly")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    addArgAttribute(ArgNo, ReadOnly);
+  } else if (R->isSubClassOf("WriteOnly")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    addArgAttribute(ArgNo, WriteOnly);
+  } else if (R->isSubClassOf("ReadNone")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    addArgAttribute(ArgNo, ReadNone);
+  } else if (R->isSubClassOf("ImmArg")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    addArgAttribute(ArgNo, ImmArg);
+  } else if (R->isSubClassOf("Align")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    uint64_t Align = R->getValueAsInt("Align");
+    addArgAttribute(ArgNo, Alignment, Align);
+  } else if (R->isSubClassOf("Dereferenceable")) {
+    unsigned ArgNo = R->getValueAsInt("ArgNo");
+    uint64_t Bytes = R->getValueAsInt("Bytes");
+    addArgAttribute(ArgNo, Dereferenceable, Bytes);
+  } else
+    llvm_unreachable("Unknown property!");
+}
+
+bool CodeGenIntrinsic::isParamAPointer(unsigned ParamIdx) const {
+  if (ParamIdx >= IS.ParamTys.size())
+    return false;
+  return (IS.ParamTys[ParamIdx]->isSubClassOf("LLVMQualPointerType") ||
+          IS.ParamTys[ParamIdx]->isSubClassOf("LLVMAnyPointerType"));
+}
+
+bool CodeGenIntrinsic::isParamImmArg(unsigned ParamIdx) const {
+  // Convert argument index to attribute index starting from `FirstArgIndex`.
+  ++ParamIdx;
+  if (ParamIdx >= ArgumentAttributes.size())
+    return false;
+  ArgAttribute Val{ImmArg, 0};
+  return std::binary_search(ArgumentAttributes[ParamIdx].begin(),
+                            ArgumentAttributes[ParamIdx].end(), Val);
+}
+
+void CodeGenIntrinsic::addArgAttribute(unsigned Idx, ArgAttrKind AK,
+                                       uint64_t V) {
+  if (Idx >= ArgumentAttributes.size())
+    ArgumentAttributes.resize(Idx + 1);
+  ArgumentAttributes[Idx].emplace_back(AK, V);
+}
diff --git a/llvm/utils/TableGen/CodeGenIntrinsics.h b/llvm/utils/TableGen/CodeGenIntrinsics.h
index 0558918b3028..f3452f5acea8 100644
--- a/llvm/utils/TableGen/CodeGenIntrinsics.h
+++ b/llvm/utils/TableGen/CodeGenIntrinsics.h
@@ -1,4 +1,4 @@
-//===- CodeGenIntrinsic.h - Intrinsic Class Wrapper ------------*- C++ -*--===//
+//===- CodeGenIntrinsics.h - Intrinsic Class Wrapper -----------*- C++ -*--===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -15,9 +15,9 @@
 
 #include "SDNodeProperties.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/ModRef.h"
 #include <string>
+#include <tuple>
 #include <vector>
 
 namespace llvm {
@@ -42,19 +42,13 @@ struct CodeGenIntrinsic {
     /// only populated when in the context of a target .td file. When building
     /// Intrinsics.td, this isn't available, because we don't know the target
     /// pointer size.
-    std::vector<MVT::SimpleValueType> RetVTs;
-
-    /// The records for each return type.
-    std::vector<Record *> RetTypeDefs;
+    std::vector<Record *> RetTys;
 
     /// The MVT::SimpleValueType for each parameter type. Note that this list is
     /// only populated when in the context of a target .td file.  When building
     /// Intrinsics.td, this isn't available, because we don't know the target
     /// pointer size.
-    std::vector<MVT::SimpleValueType> ParamVTs;
-
-    /// The records for each parameter type.
-    std::vector<Record *> ParamTypeDefs;
+    std::vector<Record *> ParamTys;
   };
 
   IntrinsicSignature IS;
@@ -109,6 +103,9 @@ struct CodeGenIntrinsic {
   // True if the intrinsic is marked as speculatable.
   bool isSpeculatable;
 
+  // True if the intrinsic is marked as strictfp.
+  bool isStrictFP;
+
   enum ArgAttrKind {
     NoCapture,
     NoAlias,
@@ -119,7 +116,8 @@ struct CodeGenIntrinsic {
     WriteOnly,
     ReadNone,
     ImmArg,
-    Alignment
+    Alignment,
+    Dereferenceable
   };
 
   struct ArgAttribute {
diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp
index 02695942f5c1..fd375735dfd2 100644
--- a/llvm/utils/TableGen/CodeGenMapTable.cpp
+++ b/llvm/utils/TableGen/CodeGenMapTable.cpp
@@ -78,6 +78,7 @@
 #include "CodeGenInstruction.h"
 #include "CodeGenTarget.h"
 #include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 using namespace llvm;
 typedef std::map<std::string, std::vector<Record*> > InstrRelMapTy;
 
diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/CodeGenRegisters.cpp
index 8ad8a7a5bc9b..5c45290a0657 100644
--- a/llvm/utils/TableGen/CodeGenRegisters.cpp
+++ b/llvm/utils/TableGen/CodeGenRegisters.cpp
@@ -872,7 +872,7 @@ bool CodeGenRegisterClass::hasType(const ValueTypeByHwMode &VT) const {
   // If VT is not identical to any of this class's types, but is a simple
   // type, check if any of the types for this class contain it under some
   // mode.
-  // The motivating example came from RISCV, where (likely because of being
+  // The motivating example came from RISC-V, where (likely because of being
   // guarded by "64-bit" predicate), the type of X5 was {*:[i64]}, but the
   // type in GRC was {*:[i32], m1:[i64]}.
   if (VT.isSimple()) {
@@ -1659,8 +1659,8 @@ static void computeUberSets(std::vector<UberRegSet> &UberSets,
          "register enum value mismatch");
 
   // For simplicitly make the SetID the same as EnumValue.
-  IntEqClasses UberSetIDs(Registers.size()+1);
-  std::set<unsigned> AllocatableRegs;
+  IntEqClasses UberSetIDs(Registers.size() + 1);
+  BitVector AllocatableRegs(Registers.size() + 1);
   for (auto &RegClass : RegBank.getRegClasses()) {
     if (!RegClass.Allocatable)
       continue;
@@ -1672,16 +1672,16 @@ static void computeUberSets(std::vector<UberRegSet> &UberSets,
     unsigned USetID = UberSetIDs.findLeader((*Regs.begin())->EnumValue);
     assert(USetID && "register number 0 is invalid");
 
-    AllocatableRegs.insert((*Regs.begin())->EnumValue);
+    AllocatableRegs.set((*Regs.begin())->EnumValue);
     for (const CodeGenRegister *CGR : llvm::drop_begin(Regs)) {
-      AllocatableRegs.insert(CGR->EnumValue);
+      AllocatableRegs.set(CGR->EnumValue);
       UberSetIDs.join(USetID, CGR->EnumValue);
     }
   }
   // Combine non-allocatable regs.
   for (const auto &Reg : Registers) {
     unsigned RegNum = Reg.EnumValue;
-    if (AllocatableRegs.count(RegNum))
+    if (AllocatableRegs.test(RegNum))
       continue;
 
     UberSetIDs.join(0, RegNum);
@@ -1704,7 +1704,6 @@ static void computeUberSets(std::vector<UberRegSet> &UberSets,
 
     UberRegSet *USet = &UberSets[USetID];
     USet->Regs.push_back(&Reg);
-    sortAndUniqueRegisters(USet->Regs);
     RegSets[i++] = USet;
   }
 }
diff --git a/llvm/utils/TableGen/CodeGenRegisters.h b/llvm/utils/TableGen/CodeGenRegisters.h
index 765425ed68cb..15f08d1431f9 100644
--- a/llvm/utils/TableGen/CodeGenRegisters.h
+++ b/llvm/utils/TableGen/CodeGenRegisters.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_UTILS_TABLEGEN_CODEGENREGISTERS_H
 #define LLVM_UTILS_TABLEGEN_CODEGENREGISTERS_H
 
+#include "CodeGenHwModes.h"
 #include "InfoByHwMode.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
@@ -32,8 +33,11 @@
 #include <cassert>
 #include <cstdint>
 #include <deque>
+#include <functional>
 #include <list>
 #include <map>
+#include <memory>
+#include <optional>
 #include <string>
 #include <utility>
 #include <vector>
@@ -41,7 +45,6 @@
 namespace llvm {
 
   class CodeGenRegBank;
-  template <typename T, typename Vector, typename Set> class SetVector;
 
   /// Used to encode a step in a register lane mask transformation.
   /// Mask the bits specified in Mask, then rotate them Rol bits to the left
@@ -147,14 +150,15 @@ namespace llvm {
   };
 
   /// CodeGenRegister - Represents a register definition.
-  struct CodeGenRegister {
+  class CodeGenRegister {
+  public:
     Record *TheDef;
     unsigned EnumValue;
     std::vector<int64_t> CostPerUse;
-    bool CoveredBySubRegs;
-    bool HasDisjunctSubRegs;
-    bool Artificial;
-    bool Constant;
+    bool CoveredBySubRegs = true;
+    bool HasDisjunctSubRegs = false;
+    bool Artificial = true;
+    bool Constant = false;
 
     // Map SubRegIndex -> Register.
     typedef std::map<CodeGenSubRegIndex *, CodeGenRegister *,
diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/CodeGenSchedule.cpp
index 441a088c1731..04219a6e54d9 100644
--- a/llvm/utils/TableGen/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/CodeGenSchedule.cpp
@@ -298,12 +298,12 @@ processSTIPredicate(STIPredicateFunction &Fn,
     RecVec Classes = Def->getValueAsListOfDefs("Classes");
     for (const Record *EC : Classes) {
       const Record *Pred = EC->getValueAsDef("Predicate");
-      if (Predicate2Index.find(Pred) == Predicate2Index.end())
+      if (!Predicate2Index.contains(Pred))
         Predicate2Index[Pred] = NumUniquePredicates++;
 
       RecVec Opcodes = EC->getValueAsListOfDefs("Opcodes");
       for (const Record *Opcode : Opcodes) {
-        if (Opcode2Index.find(Opcode) == Opcode2Index.end()) {
+        if (!Opcode2Index.contains(Opcode)) {
           Opcode2Index[Opcode] = OpcodeMappings.size();
           OpcodeMappings.emplace_back(Opcode, OpcodeInfo());
         }
@@ -370,11 +370,11 @@ processSTIPredicate(STIPredicateFunction &Fn,
                const std::pair<APInt, APInt> &RhsMasks = OpcodeMasks[RhsIdx];
 
                auto LessThan = [](const APInt &Lhs, const APInt &Rhs) {
-                 unsigned LhsCountPopulation = Lhs.countPopulation();
-                 unsigned RhsCountPopulation = Rhs.countPopulation();
+                 unsigned LhsCountPopulation = Lhs.popcount();
+                 unsigned RhsCountPopulation = Rhs.popcount();
                  return ((LhsCountPopulation < RhsCountPopulation) ||
                          ((LhsCountPopulation == RhsCountPopulation) &&
-                          (Lhs.countLeadingZeros() > Rhs.countLeadingZeros())));
+                          (Lhs.countl_zero() > Rhs.countl_zero())));
                };
 
                if (LhsMasks.first != RhsMasks.first)
diff --git a/llvm/utils/TableGen/CodeGenSchedule.h b/llvm/utils/TableGen/CodeGenSchedule.h
index bbf5381ad086..76ef1e439530 100644
--- a/llvm/utils/TableGen/CodeGenSchedule.h
+++ b/llvm/utils/TableGen/CodeGenSchedule.h
@@ -15,10 +15,17 @@
 #define LLVM_UTILS_TABLEGEN_CODEGENSCHEDULE_H
 
 #include "llvm/ADT/APInt.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/SetTheory.h"
+#include <cassert>
+#include <string>
+#include <utility>
+#include <vector>
 
 namespace llvm {
 
diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/CodeGenTarget.cpp
index b7240f01300c..fbdc0499a8cf 100644
--- a/llvm/utils/TableGen/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/CodeGenTarget.cpp
@@ -15,13 +15,17 @@
 
 #include "CodeGenTarget.h"
 #include "CodeGenInstruction.h"
-#include "CodeGenIntrinsics.h"
+#include "CodeGenRegisters.h"
 #include "CodeGenSchedule.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include <algorithm>
+#include <iterator>
+#include <tuple>
 using namespace llvm;
 
 cl::OptionCategory AsmParserCat("Options for -gen-asm-parser");
@@ -77,6 +81,7 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) {
   case MVT::ppcf128:  return "MVT::ppcf128";
   case MVT::x86mmx:   return "MVT::x86mmx";
   case MVT::x86amx:   return "MVT::x86amx";
+  case MVT::aarch64svcount:   return "MVT::aarch64svcount";
   case MVT::i64x8:    return "MVT::i64x8";
   case MVT::Glue:     return "MVT::Glue";
   case MVT::isVoid:   return "MVT::isVoid";
@@ -427,6 +432,10 @@ const CodeGenRegister *CodeGenTarget::getRegisterByName(StringRef Name) const {
   return getRegBank().getRegistersByName().lookup(Name);
 }
 
+const CodeGenRegisterClass &CodeGenTarget::getRegisterClass(Record *R) const {
+  return *getRegBank().getRegClass(R);
+}
+
 std::vector<ValueTypeByHwMode> CodeGenTarget::getRegisterVTs(Record *R)
       const {
   const CodeGenRegister *Reg = getRegBank().getReg(R);
@@ -635,318 +644,3 @@ ComplexPattern::ComplexPattern(Record *R) {
                                        "'!");
     }
 }
-
-//===----------------------------------------------------------------------===//
-// CodeGenIntrinsic Implementation
-//===----------------------------------------------------------------------===//
-
-CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) {
-  std::vector<Record *> IntrProperties =
-      RC.getAllDerivedDefinitions("IntrinsicProperty");
-
-  std::vector<Record *> DefaultProperties;
-  for (Record *Rec : IntrProperties)
-    if (Rec->getValueAsBit("IsDefault"))
-      DefaultProperties.push_back(Rec);
-
-  std::vector<Record *> Defs = RC.getAllDerivedDefinitions("Intrinsic");
-  Intrinsics.reserve(Defs.size());
-
-  for (unsigned I = 0, e = Defs.size(); I != e; ++I)
-    Intrinsics.push_back(CodeGenIntrinsic(Defs[I], DefaultProperties));
-
-  llvm::sort(Intrinsics,
-             [](const CodeGenIntrinsic &LHS, const CodeGenIntrinsic &RHS) {
-               return std::tie(LHS.TargetPrefix, LHS.Name) <
-                      std::tie(RHS.TargetPrefix, RHS.Name);
-             });
-  Targets.push_back({"", 0, 0});
-  for (size_t I = 0, E = Intrinsics.size(); I < E; ++I)
-    if (Intrinsics[I].TargetPrefix != Targets.back().Name) {
-      Targets.back().Count = I - Targets.back().Offset;
-      Targets.push_back({Intrinsics[I].TargetPrefix, I, 0});
-    }
-  Targets.back().Count = Intrinsics.size() - Targets.back().Offset;
-}
-
-CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
-                                   std::vector<Record *> DefaultProperties) {
-  TheDef = R;
-  std::string DefName = std::string(R->getName());
-  ArrayRef<SMLoc> DefLoc = R->getLoc();
-  Properties = 0;
-  isOverloaded = false;
-  isCommutative = false;
-  canThrow = false;
-  isNoReturn = false;
-  isNoCallback = false;
-  isNoSync = false;
-  isNoFree = false;
-  isWillReturn = false;
-  isCold = false;
-  isNoDuplicate = false;
-  isNoMerge = false;
-  isConvergent = false;
-  isSpeculatable = false;
-  hasSideEffects = false;
-
-  if (DefName.size() <= 4 || DefName.substr(0, 4) != "int_")
-    PrintFatalError(DefLoc,
-                    "Intrinsic '" + DefName + "' does not start with 'int_'!");
-
-  EnumName = DefName.substr(4);
-
-  if (R->getValue("ClangBuiltinName"))  // Ignore a missing ClangBuiltinName field.
-    ClangBuiltinName = std::string(R->getValueAsString("ClangBuiltinName"));
-  if (R->getValue("MSBuiltinName"))   // Ignore a missing MSBuiltinName field.
-    MSBuiltinName = std::string(R->getValueAsString("MSBuiltinName"));
-
-  TargetPrefix = std::string(R->getValueAsString("TargetPrefix"));
-  Name = std::string(R->getValueAsString("LLVMName"));
-
-  if (Name == "") {
-    // If an explicit name isn't specified, derive one from the DefName.
-    Name = "llvm.";
-
-    for (unsigned i = 0, e = EnumName.size(); i != e; ++i)
-      Name += (EnumName[i] == '_') ? '.' : EnumName[i];
-  } else {
-    // Verify it starts with "llvm.".
-    if (Name.size() <= 5 || Name.substr(0, 5) != "llvm.")
-      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
-                                  "'s name does not start with 'llvm.'!");
-  }
-
-  // If TargetPrefix is specified, make sure that Name starts with
-  // "llvm.<targetprefix>.".
-  if (!TargetPrefix.empty()) {
-    if (Name.size() < 6+TargetPrefix.size() ||
-        Name.substr(5, 1 + TargetPrefix.size()) != (TargetPrefix + "."))
-      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
-                                  "' does not start with 'llvm." +
-                                  TargetPrefix + ".'!");
-  }
-
-  ListInit *RetTypes = R->getValueAsListInit("RetTypes");
-  ListInit *ParamTypes = R->getValueAsListInit("ParamTypes");
-
-  // First collate a list of overloaded types.
-  std::vector<MVT::SimpleValueType> OverloadedVTs;
-  for (ListInit *TypeList : {RetTypes, ParamTypes}) {
-    for (unsigned i = 0, e = TypeList->size(); i != e; ++i) {
-      Record *TyEl = TypeList->getElementAsRecord(i);
-      assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!");
-
-      if (TyEl->isSubClassOf("LLVMMatchType"))
-        continue;
-
-      MVT::SimpleValueType VT = getValueType(TyEl->getValueAsDef("VT"));
-      if (MVT(VT).isOverloaded()) {
-        OverloadedVTs.push_back(VT);
-        isOverloaded = true;
-      }
-    }
-  }
-
-  // Parse the list of return types.
-  ListInit *TypeList = RetTypes;
-  for (unsigned i = 0, e = TypeList->size(); i != e; ++i) {
-    Record *TyEl = TypeList->getElementAsRecord(i);
-    assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!");
-    MVT::SimpleValueType VT;
-    if (TyEl->isSubClassOf("LLVMMatchType")) {
-      unsigned MatchTy = TyEl->getValueAsInt("Number");
-      assert(MatchTy < OverloadedVTs.size() &&
-             "Invalid matching number!");
-      VT = OverloadedVTs[MatchTy];
-      // It only makes sense to use the extended and truncated vector element
-      // variants with iAny types; otherwise, if the intrinsic is not
-      // overloaded, all the types can be specified directly.
-      assert(((!TyEl->isSubClassOf("LLVMExtendedType") &&
-               !TyEl->isSubClassOf("LLVMTruncatedType")) ||
-              VT == MVT::iAny || VT == MVT::vAny) &&
-             "Expected iAny or vAny type");
-    } else {
-      VT = getValueType(TyEl->getValueAsDef("VT"));
-    }
-
-    // Reject invalid types.
-    if (VT == MVT::isVoid)
-      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
-                                  " has void in result type list!");
-
-    IS.RetVTs.push_back(VT);
-    IS.RetTypeDefs.push_back(TyEl);
-  }
-
-  // Parse the list of parameter types.
-  TypeList = ParamTypes;
-  for (unsigned i = 0, e = TypeList->size(); i != e; ++i) {
-    Record *TyEl = TypeList->getElementAsRecord(i);
-    assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!");
-    MVT::SimpleValueType VT;
-    if (TyEl->isSubClassOf("LLVMMatchType")) {
-      unsigned MatchTy = TyEl->getValueAsInt("Number");
-      if (MatchTy >= OverloadedVTs.size()) {
-        PrintError(R->getLoc(),
-                   "Parameter #" + Twine(i) + " has out of bounds matching "
-                   "number " + Twine(MatchTy));
-        PrintFatalError(DefLoc,
-                        Twine("ParamTypes is ") + TypeList->getAsString());
-      }
-      VT = OverloadedVTs[MatchTy];
-      // It only makes sense to use the extended and truncated vector element
-      // variants with iAny types; otherwise, if the intrinsic is not
-      // overloaded, all the types can be specified directly.
-      assert(((!TyEl->isSubClassOf("LLVMExtendedType") &&
-               !TyEl->isSubClassOf("LLVMTruncatedType")) ||
-              VT == MVT::iAny || VT == MVT::vAny) &&
-             "Expected iAny or vAny type");
-    } else
-      VT = getValueType(TyEl->getValueAsDef("VT"));
-
-    // Reject invalid types.
-    if (VT == MVT::isVoid && i != e-1 /*void at end means varargs*/)
-      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
-                                  " has void in result type list!");
-
-    IS.ParamVTs.push_back(VT);
-    IS.ParamTypeDefs.push_back(TyEl);
-  }
-
-  // Parse the intrinsic properties.
-  ListInit *PropList = R->getValueAsListInit("IntrProperties");
-  for (unsigned i = 0, e = PropList->size(); i != e; ++i) {
-    Record *Property = PropList->getElementAsRecord(i);
-    assert(Property->isSubClassOf("IntrinsicProperty") &&
-           "Expected a property!");
-
-    setProperty(Property);
-  }
-
-  // Set default properties to true.
-  setDefaultProperties(R, DefaultProperties);
-
-  // Also record the SDPatternOperator Properties.
-  Properties = parseSDPatternOperatorProperties(R);
-
-  // Sort the argument attributes for later benefit.
-  for (auto &Attrs : ArgumentAttributes)
-    llvm::sort(Attrs);
-}
-
-void CodeGenIntrinsic::setDefaultProperties(
-    Record *R, std::vector<Record *> DefaultProperties) {
-  // opt-out of using default attributes.
-  if (R->getValueAsBit("DisableDefaultAttributes"))
-    return;
-
-  for (Record *Rec : DefaultProperties)
-    setProperty(Rec);
-}
-
-void CodeGenIntrinsic::setProperty(Record *R) {
-  if (R->getName() == "IntrNoMem")
-    ME = MemoryEffects::none();
-  else if (R->getName() == "IntrReadMem") {
-    if (ME.onlyWritesMemory())
-      PrintFatalError(TheDef->getLoc(),
-                      Twine("IntrReadMem cannot be used after IntrNoMem or "
-                            "IntrWriteMem. Default is ReadWrite"));
-    ME &= MemoryEffects::readOnly();
-  } else if (R->getName() == "IntrWriteMem") {
-    if (ME.onlyReadsMemory())
-      PrintFatalError(TheDef->getLoc(),
-                      Twine("IntrWriteMem cannot be used after IntrNoMem or "
-                            "IntrReadMem. Default is ReadWrite"));
-    ME &= MemoryEffects::writeOnly();
-  } else if (R->getName() == "IntrArgMemOnly")
-    ME &= MemoryEffects::argMemOnly();
-  else if (R->getName() == "IntrInaccessibleMemOnly")
-    ME &= MemoryEffects::inaccessibleMemOnly();
-  else if (R->getName() == "IntrInaccessibleMemOrArgMemOnly")
-    ME &= MemoryEffects::inaccessibleOrArgMemOnly();
-  else if (R->getName() == "Commutative")
-    isCommutative = true;
-  else if (R->getName() == "Throws")
-    canThrow = true;
-  else if (R->getName() == "IntrNoDuplicate")
-    isNoDuplicate = true;
-  else if (R->getName() == "IntrNoMerge")
-    isNoMerge = true;
-  else if (R->getName() == "IntrConvergent")
-    isConvergent = true;
-  else if (R->getName() == "IntrNoReturn")
-    isNoReturn = true;
-  else if (R->getName() == "IntrNoCallback")
-    isNoCallback = true;
-  else if (R->getName() == "IntrNoSync")
-    isNoSync = true;
-  else if (R->getName() == "IntrNoFree")
-    isNoFree = true;
-  else if (R->getName() == "IntrWillReturn")
-    isWillReturn = !isNoReturn;
-  else if (R->getName() == "IntrCold")
-    isCold = true;
-  else if (R->getName() == "IntrSpeculatable")
-    isSpeculatable = true;
-  else if (R->getName() == "IntrHasSideEffects")
-    hasSideEffects = true;
-  else if (R->isSubClassOf("NoCapture")) {
-    unsigned ArgNo = R->getValueAsInt("ArgNo");
-    addArgAttribute(ArgNo, NoCapture);
-  } else if (R->isSubClassOf("NoAlias")) {
-    unsigned ArgNo = R->getValueAsInt("ArgNo");
-    addArgAttribute(ArgNo, NoAlias);
-  } else if (R->isSubClassOf("NoUndef")) {
-    unsigned ArgNo = R->getValueAsInt("ArgNo");
-    addArgAttribute(ArgNo, NoUndef);
-  } else if (R->isSubClassOf("NonNull")) {
-    unsigned ArgNo = R->getValueAsInt("ArgNo");
-    addArgAttribute(ArgNo, NonNull);
-  } else if (R->isSubClassOf("Returned")) {
-    unsigned ArgNo = R->getValueAsInt("ArgNo");
-    addArgAttribute(ArgNo, Returned);
-  } else if (R->isSubClassOf("ReadOnly")) {
-    unsigned ArgNo = R->getValueAsInt("ArgNo");
-    addArgAttribute(ArgNo, ReadOnly);
-  } else if (R->isSubClassOf("WriteOnly")) {
-    unsigned ArgNo = R->getValueAsInt("ArgNo");
-    addArgAttribute(ArgNo, WriteOnly);
-  } else if (R->isSubClassOf("ReadNone")) {
-    unsigned ArgNo = R->getValueAsInt("ArgNo");
-    addArgAttribute(ArgNo, ReadNone);
-  } else if (R->isSubClassOf("ImmArg")) {
-    unsigned ArgNo = R->getValueAsInt("ArgNo");
-    addArgAttribute(ArgNo, ImmArg);
-  } else if (R->isSubClassOf("Align")) {
-    unsigned ArgNo = R->getValueAsInt("ArgNo");
-    uint64_t Align = R->getValueAsInt("Align");
-    addArgAttribute(ArgNo, Alignment, Align);
-  } else
-    llvm_unreachable("Unknown property!");
-}
-
-bool CodeGenIntrinsic::isParamAPointer(unsigned ParamIdx) const {
-  if (ParamIdx >= IS.ParamVTs.size())
-    return false;
-  MVT ParamType = MVT(IS.ParamVTs[ParamIdx]);
-  return ParamType == MVT::iPTR || ParamType == MVT::iPTRAny;
-}
-
-bool CodeGenIntrinsic::isParamImmArg(unsigned ParamIdx) const {
-  // Convert argument index to attribute index starting from `FirstArgIndex`.
-  ++ParamIdx;
-  if (ParamIdx >= ArgumentAttributes.size())
-    return false;
-  ArgAttribute Val{ImmArg, 0};
-  return std::binary_search(ArgumentAttributes[ParamIdx].begin(),
-                            ArgumentAttributes[ParamIdx].end(), Val);
-}
-
-void CodeGenIntrinsic::addArgAttribute(unsigned Idx, ArgAttrKind AK,
-                                       uint64_t V) {
-  if (Idx >= ArgumentAttributes.size())
-    ArgumentAttributes.resize(Idx + 1);
-  ArgumentAttributes[Idx].emplace_back(AK, V);
-}
diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/CodeGenTarget.h
index 6846e6b5c77a..2ba3af724d36 100644
--- a/llvm/utils/TableGen/CodeGenTarget.h
+++ b/llvm/utils/TableGen/CodeGenTarget.h
@@ -17,18 +17,29 @@
 #define LLVM_UTILS_TABLEGEN_CODEGENTARGET_H
 
 #include "CodeGenHwModes.h"
-#include "CodeGenRegisters.h"
 #include "InfoByHwMode.h"
 #include "SDNodeProperties.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachineValueType.h"
+#include <cassert>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
 
 namespace llvm {
 
 class RecordKeeper;
 class Record;
 class CodeGenInstruction;
-struct CodeGenRegister;
+class CodeGenRegBank;
+class CodeGenRegister;
+class CodeGenRegisterClass;
 class CodeGenSchedModels;
-class CodeGenTarget;
+class CodeGenSubRegIndex;
 
 /// getValueType - Return the MVT::SimpleValueType that the specified TableGen
 /// record corresponds to.
@@ -122,9 +133,7 @@ public:
     return RegAltNameIndices;
   }
 
-  const CodeGenRegisterClass &getRegisterClass(Record *R) const {
-    return *getRegBank().getRegClass(R);
-  }
+  const CodeGenRegisterClass &getRegisterClass(Record *R) const;
 
   /// getRegisterVTs - Find the union of all possible SimpleValueTypes for the
   /// specified physical register.
diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp
index a18d6a6b8854..9d9b69f4cfbd 100644
--- a/llvm/utils/TableGen/CompressInstEmitter.cpp
+++ b/llvm/utils/TableGen/CompressInstEmitter.cpp
@@ -65,6 +65,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenInstruction.h"
+#include "CodeGenRegisters.h"
 #include "CodeGenTarget.h"
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/SmallVector.h"
@@ -902,10 +903,5 @@ void CompressInstEmitter::run(raw_ostream &o) {
   emitCompressInstEmitter(o, EmitterType::CheckCompress);
 }
 
-namespace llvm {
-
-void EmitCompressInst(RecordKeeper &RK, raw_ostream &OS) {
-  CompressInstEmitter(RK).run(OS);
-}
-
-} // namespace llvm
+static TableGen::Emitter::OptClass<CompressInstEmitter>
+    X("gen-compress-inst-emitter", "Generate compressed instructions.");
diff --git a/llvm/utils/TableGen/DAGISelEmitter.cpp b/llvm/utils/TableGen/DAGISelEmitter.cpp
index d012a0172a8f..eaf7f7f9f0a3 100644
--- a/llvm/utils/TableGen/DAGISelEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelEmitter.cpp
@@ -12,6 +12,7 @@
 
 #include "CodeGenDAGPatterns.h"
 #include "CodeGenInstruction.h"
+#include "CodeGenTarget.h"
 #include "DAGISelMatcher.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/TableGen/Record.h"
@@ -123,6 +124,7 @@ struct PatternSortingPredicate {
 
 
 void DAGISelEmitter::run(raw_ostream &OS) {
+  Records.startTimer("Parse patterns");
   emitSourceFileHeader("DAG Instruction Selector for the " +
                        CGP.getTargetInfo().getName().str() + " target", OS);
 
@@ -163,7 +165,7 @@ void DAGISelEmitter::run(raw_ostream &OS) {
 
   // Convert each variant of each pattern into a Matcher.
   Records.startTimer("Convert to matchers");
-  std::vector<Matcher*> PatternMatchers;
+  SmallVector<Matcher *, 0> PatternMatchers;
   for (const PatternToMatch *PTM : Patterns) {
     for (unsigned Variant = 0; ; ++Variant) {
       if (Matcher *M = ConvertPatternToMatcher(*PTM, Variant, CGP))
@@ -174,7 +176,7 @@ void DAGISelEmitter::run(raw_ostream &OS) {
   }
 
   std::unique_ptr<Matcher> TheMatcher =
-    std::make_unique<ScopeMatcher>(PatternMatchers);
+      std::make_unique<ScopeMatcher>(std::move(PatternMatchers));
 
   Records.startTimer("Optimize matchers");
   OptimizeMatcher(TheMatcher, CGP);
@@ -185,11 +187,5 @@ void DAGISelEmitter::run(raw_ostream &OS) {
   EmitMatcherTable(TheMatcher.get(), CGP, OS);
 }
 
-namespace llvm {
-
-void EmitDAGISel(RecordKeeper &RK, raw_ostream &OS) {
-  RK.startTimer("Parse patterns");
-  DAGISelEmitter(RK).run(OS);
-}
-
-} // End llvm namespace
+static TableGen::Emitter::OptClass<DAGISelEmitter>
+    X("gen-dag-isel", "Generate a DAG instruction selector");
diff --git a/llvm/utils/TableGen/DAGISelMatcher.cpp b/llvm/utils/TableGen/DAGISelMatcher.cpp
index e436a931a9f5..0609f006763b 100644
--- a/llvm/utils/TableGen/DAGISelMatcher.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcher.cpp
@@ -8,6 +8,8 @@
 
 #include "DAGISelMatcher.h"
 #include "CodeGenDAGPatterns.h"
+#include "CodeGenInstruction.h"
+#include "CodeGenRegisters.h"
 #include "CodeGenTarget.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Record.h"
@@ -290,7 +292,7 @@ void EmitNodeXFormMatcher::printImpl(raw_ostream &OS, unsigned indent) const {
 void EmitNodeMatcherCommon::printImpl(raw_ostream &OS, unsigned indent) const {
   OS.indent(indent);
   OS << (isa<MorphNodeToMatcher>(this) ? "MorphNodeTo: " : "EmitNode: ")
-     << OpcodeName << ": <todo flags> ";
+     << CGI.Namespace << "::" << CGI.TheDef->getName() << ": <todo flags> ";
 
   for (unsigned i = 0, e = VTs.size(); i != e; ++i)
     OS << ' ' << getEnumName(VTs[i]);
@@ -315,10 +317,9 @@ bool CheckOpcodeMatcher::isEqualImpl(const Matcher *M) const {
 
 bool EmitNodeMatcherCommon::isEqualImpl(const Matcher *m) const {
   const EmitNodeMatcherCommon *M = cast<EmitNodeMatcherCommon>(m);
-  return M->OpcodeName == OpcodeName && M->VTs == VTs &&
-         M->Operands == Operands && M->HasChain == HasChain &&
-         M->HasInGlue == HasInGlue && M->HasOutGlue == HasOutGlue &&
-         M->HasMemRefs == HasMemRefs &&
+  return &M->CGI == &CGI && M->VTs == VTs && M->Operands == Operands &&
+         M->HasChain == HasChain && M->HasInGlue == HasInGlue &&
+         M->HasOutGlue == HasOutGlue && M->HasMemRefs == HasMemRefs &&
          M->NumFixedArityOperands == NumFixedArityOperands;
 }
 
diff --git a/llvm/utils/TableGen/DAGISelMatcher.h b/llvm/utils/TableGen/DAGISelMatcher.h
index 77280acaf4ca..e3cf847edd12 100644
--- a/llvm/utils/TableGen/DAGISelMatcher.h
+++ b/llvm/utils/TableGen/DAGISelMatcher.h
@@ -12,12 +12,18 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/MachineValueType.h"
+#include <cassert>
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <utility>
 
 namespace llvm {
-  struct CodeGenRegister;
+  class CodeGenRegister;
   class CodeGenDAGPatterns;
+  class CodeGenInstruction;
   class Matcher;
   class PatternToMatch;
   class raw_ostream;
@@ -41,7 +47,7 @@ class Matcher {
   // The next matcher node that is executed after this one.  Null if this is the
   // last stage of a match.
   std::unique_ptr<Matcher> Next;
-  size_t Size; // Size in bytes of matcher and all its children (if any).
+  size_t Size = 0; // Size in bytes of matcher and all its children (if any).
   virtual void anchor();
 public:
   enum KindTy {
@@ -189,9 +195,8 @@ protected:
 class ScopeMatcher : public Matcher {
   SmallVector<Matcher*, 4> Children;
 public:
-  ScopeMatcher(ArrayRef<Matcher *> children)
-    : Matcher(Scope), Children(children.begin(), children.end()) {
-  }
+  ScopeMatcher(SmallVectorImpl<Matcher *> &&children)
+      : Matcher(Scope), Children(std::move(children)) {}
   ~ScopeMatcher() override;
 
   unsigned getNumChildren() const { return Children.size(); }
@@ -473,8 +478,9 @@ private:
 class SwitchOpcodeMatcher : public Matcher {
   SmallVector<std::pair<const SDNodeInfo*, Matcher*>, 8> Cases;
 public:
-  SwitchOpcodeMatcher(ArrayRef<std::pair<const SDNodeInfo*, Matcher*> > cases)
-    : Matcher(SwitchOpcode), Cases(cases.begin(), cases.end()) {}
+  SwitchOpcodeMatcher(
+      SmallVectorImpl<std::pair<const SDNodeInfo *, Matcher *>> &&cases)
+      : Matcher(SwitchOpcode), Cases(std::move(cases)) {}
   ~SwitchOpcodeMatcher() override;
 
   static bool classof(const Matcher *N) {
@@ -523,8 +529,9 @@ private:
 class SwitchTypeMatcher : public Matcher {
   SmallVector<std::pair<MVT::SimpleValueType, Matcher*>, 8> Cases;
 public:
-  SwitchTypeMatcher(ArrayRef<std::pair<MVT::SimpleValueType, Matcher*> > cases)
-  : Matcher(SwitchType), Cases(cases.begin(), cases.end()) {}
+  SwitchTypeMatcher(
+      SmallVectorImpl<std::pair<MVT::SimpleValueType, Matcher *>> &&cases)
+      : Matcher(SwitchType), Cases(std::move(cases)) {}
   ~SwitchTypeMatcher() override;
 
   static bool classof(const Matcher *N) {
@@ -991,7 +998,7 @@ private:
 /// EmitNodeMatcherCommon - Common class shared between EmitNode and
 /// MorphNodeTo.
 class EmitNodeMatcherCommon : public Matcher {
-  std::string OpcodeName;
+  const CodeGenInstruction &CGI;
   const SmallVector<MVT::SimpleValueType, 3> VTs;
   const SmallVector<unsigned, 6> Operands;
   bool HasChain, HasInGlue, HasOutGlue, HasMemRefs;
@@ -1001,18 +1008,17 @@ class EmitNodeMatcherCommon : public Matcher {
   /// operands in the root of the pattern.  The rest are appended to this node.
   int NumFixedArityOperands;
 public:
-  EmitNodeMatcherCommon(const std::string &opcodeName,
+  EmitNodeMatcherCommon(const CodeGenInstruction &cgi,
                         ArrayRef<MVT::SimpleValueType> vts,
-                        ArrayRef<unsigned> operands,
-                        bool hasChain, bool hasInGlue, bool hasOutGlue,
-                        bool hasmemrefs,
+                        ArrayRef<unsigned> operands, bool hasChain,
+                        bool hasInGlue, bool hasOutGlue, bool hasmemrefs,
                         int numfixedarityoperands, bool isMorphNodeTo)
-    : Matcher(isMorphNodeTo ? MorphNodeTo : EmitNode), OpcodeName(opcodeName),
-      VTs(vts.begin(), vts.end()), Operands(operands.begin(), operands.end()),
-      HasChain(hasChain), HasInGlue(hasInGlue), HasOutGlue(hasOutGlue),
-      HasMemRefs(hasmemrefs), NumFixedArityOperands(numfixedarityoperands) {}
+      : Matcher(isMorphNodeTo ? MorphNodeTo : EmitNode), CGI(cgi),
+        VTs(vts.begin(), vts.end()), Operands(operands.begin(), operands.end()),
+        HasChain(hasChain), HasInGlue(hasInGlue), HasOutGlue(hasOutGlue),
+        HasMemRefs(hasmemrefs), NumFixedArityOperands(numfixedarityoperands) {}
 
-  const std::string &getOpcodeName() const { return OpcodeName; }
+  const CodeGenInstruction &getInstruction() const { return CGI; }
 
   unsigned getNumVTs() const { return VTs.size(); }
   MVT::SimpleValueType getVT(unsigned i) const {
@@ -1031,8 +1037,8 @@ public:
 
 
   bool hasChain() const { return HasChain; }
-  bool hasInFlag() const { return HasInGlue; }
-  bool hasOutFlag() const { return HasOutGlue; }
+  bool hasInGlue() const { return HasInGlue; }
+  bool hasOutGlue() const { return HasOutGlue; }
   bool hasMemRefs() const { return HasMemRefs; }
   int getNumFixedArityOperands() const { return NumFixedArityOperands; }
 
@@ -1050,16 +1056,15 @@ class EmitNodeMatcher : public EmitNodeMatcherCommon {
   void anchor() override;
   unsigned FirstResultSlot;
 public:
-  EmitNodeMatcher(const std::string &opcodeName,
+  EmitNodeMatcher(const CodeGenInstruction &cgi,
                   ArrayRef<MVT::SimpleValueType> vts,
-                  ArrayRef<unsigned> operands,
-                  bool hasChain, bool hasInFlag, bool hasOutFlag,
-                  bool hasmemrefs,
-                  int numfixedarityoperands, unsigned firstresultslot)
-  : EmitNodeMatcherCommon(opcodeName, vts, operands, hasChain,
-                          hasInFlag, hasOutFlag, hasmemrefs,
-                          numfixedarityoperands, false),
-    FirstResultSlot(firstresultslot) {}
+                  ArrayRef<unsigned> operands, bool hasChain, bool hasInGlue,
+                  bool hasOutGlue, bool hasmemrefs, int numfixedarityoperands,
+                  unsigned firstresultslot)
+      : EmitNodeMatcherCommon(cgi, vts, operands, hasChain, hasInGlue,
+                              hasOutGlue, hasmemrefs, numfixedarityoperands,
+                              false),
+        FirstResultSlot(firstresultslot) {}
 
   unsigned getFirstResultSlot() const { return FirstResultSlot; }
 
@@ -1073,17 +1078,15 @@ class MorphNodeToMatcher : public EmitNodeMatcherCommon {
   void anchor() override;
   const PatternToMatch &Pattern;
 public:
-  MorphNodeToMatcher(const std::string &opcodeName,
+  MorphNodeToMatcher(const CodeGenInstruction &cgi,
                      ArrayRef<MVT::SimpleValueType> vts,
-                     ArrayRef<unsigned> operands,
-                     bool hasChain, bool hasInFlag, bool hasOutFlag,
-                     bool hasmemrefs,
+                     ArrayRef<unsigned> operands, bool hasChain, bool hasInGlue,
+                     bool hasOutGlue, bool hasmemrefs,
                      int numfixedarityoperands, const PatternToMatch &pattern)
-    : EmitNodeMatcherCommon(opcodeName, vts, operands, hasChain,
-                            hasInFlag, hasOutFlag, hasmemrefs,
-                            numfixedarityoperands, true),
-      Pattern(pattern) {
-  }
+      : EmitNodeMatcherCommon(cgi, vts, operands, hasChain, hasInGlue,
+                              hasOutGlue, hasmemrefs, numfixedarityoperands,
+                              true),
+        Pattern(pattern) {}
 
   const PatternToMatch &getPattern() const { return Pattern; }
 
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 777e75dcd929..28d4d585f3dd 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -11,7 +11,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenDAGPatterns.h"
+#include "CodeGenInstruction.h"
+#include "CodeGenRegisters.h"
+#include "CodeGenTarget.h"
 #include "DAGISelMatcher.h"
+#include "SDNodeProperties.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/StringMap.h"
@@ -80,9 +84,8 @@ class MatcherTableEmitter {
   }
 
 public:
-  MatcherTableEmitter(const CodeGenDAGPatterns &cgp) : CGP(cgp) {
-    OpcodeCounts.assign(Matcher::HighestKind+1, 0);
-  }
+  MatcherTableEmitter(const CodeGenDAGPatterns &cgp)
+      : CGP(cgp), OpcodeCounts(Matcher::HighestKind + 1, 0) {}
 
   unsigned EmitMatcherList(const Matcher *N, const unsigned Indent,
                            unsigned StartIdx, raw_ostream &OS);
@@ -772,11 +775,13 @@ EmitMatcher(const Matcher *N, const unsigned Indent, unsigned CurrentIdx,
     if (CompressVTs)
       OS << EN->getNumVTs();
 
-    OS << ", TARGET_VAL(" << EN->getOpcodeName() << "), 0";
+    const CodeGenInstruction &CGI = EN->getInstruction();
+    OS << ", TARGET_VAL(" << CGI.Namespace << "::" << CGI.TheDef->getName()
+       << "), 0";
 
     if (EN->hasChain())   OS << "|OPFL_Chain";
-    if (EN->hasInFlag())  OS << "|OPFL_GlueInput";
-    if (EN->hasOutFlag()) OS << "|OPFL_GlueOutput";
+    if (EN->hasInGlue())  OS << "|OPFL_GlueInput";
+    if (EN->hasOutGlue()) OS << "|OPFL_GlueOutput";
     if (EN->hasMemRefs()) OS << "|OPFL_MemRefs";
     if (EN->getNumFixedArityOperands() != -1)
       OS << "|OPFL_Variadic" << EN->getNumFixedArityOperands();
diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
index 44bff4c67ab3..f773f7c77a77 100644
--- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp
@@ -9,7 +9,10 @@
 #include "CodeGenDAGPatterns.h"
 #include "CodeGenInstruction.h"
 #include "CodeGenRegisters.h"
+#include "CodeGenTarget.h"
 #include "DAGISelMatcher.h"
+#include "InfoByHwMode.h"
+#include "SDNodeProperties.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/TableGen/Error.h"
@@ -19,8 +22,8 @@ using namespace llvm;
 
 
 /// getRegisterValueType - Look up and return the ValueType of the specified
-/// register. If the register is a member of multiple register classes which
-/// have different associated types, return MVT::Other.
+/// register. If the register is a member of multiple register classes, they
+/// must all have the same type.
 static MVT::SimpleValueType getRegisterValueType(Record *R,
                                                  const CodeGenTarget &T) {
   bool FoundRC = false;
@@ -34,15 +37,15 @@ static MVT::SimpleValueType getRegisterValueType(Record *R,
     if (!FoundRC) {
       FoundRC = true;
       const ValueTypeByHwMode &VVT = RC.getValueTypeNum(0);
-      if (VVT.isSimple())
-        VT = VVT.getSimple().SimpleTy;
+      assert(VVT.isSimple());
+      VT = VVT.getSimple().SimpleTy;
       continue;
     }
 
 #ifndef NDEBUG
     // If this occurs in multiple register classes, they all have to agree.
-    const ValueTypeByHwMode &T = RC.getValueTypeNum(0);
-    assert((!T.isSimple() || T.getSimple().SimpleTy == VT) &&
+    const ValueTypeByHwMode &VVT = RC.getValueTypeNum(0);
+    assert(VVT.isSimple() && VVT.getSimple().SimpleTy == VT &&
            "ValueType mismatch between register classes for this register");
 #endif
   }
@@ -107,15 +110,13 @@ namespace {
     Matcher *GetMatcher() const { return TheMatcher; }
   private:
     void AddMatcher(Matcher *NewNode);
-    void InferPossibleTypes(unsigned ForceMode);
+    void InferPossibleTypes();
 
     // Matcher Generation.
-    void EmitMatchCode(const TreePatternNode *N, TreePatternNode *NodeNoTypes,
-                       unsigned ForceMode);
+    void EmitMatchCode(const TreePatternNode *N, TreePatternNode *NodeNoTypes);
     void EmitLeafMatchCode(const TreePatternNode *N);
     void EmitOperatorMatchCode(const TreePatternNode *N,
-                               TreePatternNode *NodeNoTypes,
-                               unsigned ForceMode);
+                               TreePatternNode *NodeNoTypes);
 
     /// If this is the first time a node with unique identifier Name has been
     /// seen, record it. Otherwise, emit a check to make sure this is the same
@@ -164,19 +165,17 @@ MatcherGen::MatcherGen(const PatternToMatch &pattern,
   PatWithNoTypes->RemoveAllTypes();
 
   // If there are types that are manifestly known, infer them.
-  InferPossibleTypes(Pattern.getForceMode());
+  InferPossibleTypes();
 }
 
 /// InferPossibleTypes - As we emit the pattern, we end up generating type
 /// checks and applying them to the 'PatWithNoTypes' tree.  As we do this, we
 /// want to propagate implied types as far throughout the tree as possible so
 /// that we avoid doing redundant type checks.  This does the type propagation.
-void MatcherGen::InferPossibleTypes(unsigned ForceMode) {
+void MatcherGen::InferPossibleTypes() {
   // TP - Get *SOME* tree pattern, we don't care which.  It is only used for
   // diagnostics, which we know are impossible at this point.
   TreePattern &TP = *CGP.pf_begin()->second;
-  TP.getInfer().CodeGen = true;
-  TP.getInfer().ForceMode = ForceMode;
 
   bool MadeChange = true;
   while (MadeChange)
@@ -278,7 +277,8 @@ void MatcherGen::EmitLeafMatchCode(const TreePatternNode *N) {
     return;
   }
 
-  if (LeafRec->getName() == "immAllOnesV") {
+  if (LeafRec->getName() == "immAllOnesV" ||
+      LeafRec->getName() == "immAllZerosV") {
     // If this is the root of the dag we're matching, we emit a redundant opcode
     // check to ensure that this gets folded into the normal top-level
     // OpcodeSwitch.
@@ -288,19 +288,11 @@ void MatcherGen::EmitLeafMatchCode(const TreePatternNode *N) {
       const SDNodeInfo &NI = CGP.getSDNodeInfo(CGP.getSDNodeNamed(Name));
       AddMatcher(new CheckOpcodeMatcher(NI));
     }
-    return AddMatcher(new CheckImmAllOnesVMatcher());
-  }
-  if (LeafRec->getName() == "immAllZerosV") {
-    // If this is the root of the dag we're matching, we emit a redundant opcode
-    // check to ensure that this gets folded into the normal top-level
-    // OpcodeSwitch.
-    if (N == Pattern.getSrcPattern()) {
-      MVT VT = N->getSimpleType(0);
-      StringRef Name = VT.isScalableVector() ? "splat_vector" : "build_vector";
-      const SDNodeInfo &NI = CGP.getSDNodeInfo(CGP.getSDNodeNamed(Name));
-      AddMatcher(new CheckOpcodeMatcher(NI));
-    }
-    return AddMatcher(new CheckImmAllZerosVMatcher());
+    if (LeafRec->getName() == "immAllOnesV")
+      AddMatcher(new CheckImmAllOnesVMatcher());
+    else
+      AddMatcher(new CheckImmAllZerosVMatcher());
+    return;
   }
 
   errs() << "Unknown leaf kind: " << *N << "\n";
@@ -308,8 +300,7 @@ void MatcherGen::EmitLeafMatchCode(const TreePatternNode *N) {
 }
 
 void MatcherGen::EmitOperatorMatchCode(const TreePatternNode *N,
-                                       TreePatternNode *NodeNoTypes,
-                                       unsigned ForceMode) {
+                                       TreePatternNode *NodeNoTypes) {
   assert(!N->isLeaf() && "Not an operator?");
 
   if (N->getOperator()->isSubClassOf("ComplexPattern")) {
@@ -347,7 +338,8 @@ void MatcherGen::EmitOperatorMatchCode(const TreePatternNode *N,
       N->getChild(1)->isLeaf() && N->getChild(1)->getPredicateCalls().empty() &&
       N->getPredicateCalls().empty()) {
     if (IntInit *II = dyn_cast<IntInit>(N->getChild(1)->getLeafValue())) {
-      if (!isPowerOf2_32(II->getValue())) {  // Don't bother with single bits.
+      if (!llvm::has_single_bit<uint32_t>(
+              II->getValue())) { // Don't bother with single bits.
         // If this is at the root of the pattern, we emit a redundant
         // CheckOpcode so that the following checks get factored properly under
         // a single opcode check.
@@ -362,7 +354,7 @@ void MatcherGen::EmitOperatorMatchCode(const TreePatternNode *N,
 
         // Match the LHS of the AND as appropriate.
         AddMatcher(new MoveChildMatcher(0));
-        EmitMatchCode(N->getChild(0), NodeNoTypes->getChild(0), ForceMode);
+        EmitMatchCode(N->getChild(0), NodeNoTypes->getChild(0));
         AddMatcher(new MoveParentMatcher());
         return;
       }
@@ -461,7 +453,7 @@ void MatcherGen::EmitOperatorMatchCode(const TreePatternNode *N,
     // Get the code suitable for matching this child.  Move to the child, check
     // it then move back to the parent.
     AddMatcher(new MoveChildMatcher(OpNo));
-    EmitMatchCode(N->getChild(i), NodeNoTypes->getChild(i), ForceMode);
+    EmitMatchCode(N->getChild(i), NodeNoTypes->getChild(i));
     AddMatcher(new MoveParentMatcher());
   }
 }
@@ -502,8 +494,7 @@ bool MatcherGen::recordUniqueNode(ArrayRef<std::string> Names) {
 }
 
 void MatcherGen::EmitMatchCode(const TreePatternNode *N,
-                               TreePatternNode *NodeNoTypes,
-                               unsigned ForceMode) {
+                               TreePatternNode *NodeNoTypes) {
   // If N and NodeNoTypes don't agree on a type, then this is a case where we
   // need to do a type check.  Emit the check, apply the type to NodeNoTypes and
   // reinfer any correlated types.
@@ -512,7 +503,7 @@ void MatcherGen::EmitMatchCode(const TreePatternNode *N,
   for (unsigned i = 0, e = NodeNoTypes->getNumTypes(); i != e; ++i) {
     if (NodeNoTypes->getExtType(i) == N->getExtType(i)) continue;
     NodeNoTypes->setType(i, N->getExtType(i));
-    InferPossibleTypes(ForceMode);
+    InferPossibleTypes();
     ResultsToTypeCheck.push_back(i);
   }
 
@@ -534,7 +525,7 @@ void MatcherGen::EmitMatchCode(const TreePatternNode *N,
   if (N->isLeaf())
     EmitLeafMatchCode(N);
   else
-    EmitOperatorMatchCode(N, NodeNoTypes, ForceMode);
+    EmitOperatorMatchCode(N, NodeNoTypes);
 
   // If there are node predicates for this node, generate their checks.
   for (unsigned i = 0, e = N->getPredicateCalls().size(); i != e; ++i) {
@@ -576,13 +567,13 @@ bool MatcherGen::EmitMatcherCode(unsigned Variant) {
   }
 
   // Emit the matcher for the pattern structure and types.
-  EmitMatchCode(Pattern.getSrcPattern(), PatWithNoTypes.get(),
-                Pattern.getForceMode());
+  EmitMatchCode(Pattern.getSrcPattern(), PatWithNoTypes.get());
 
   // If the pattern has a predicate on it (e.g. only enabled when a subtarget
   // feature is around, do the check).
-  if (!Pattern.getPredicateCheck().empty())
-    AddMatcher(new CheckPatternPredicateMatcher(Pattern.getPredicateCheck()));
+  std::string PredicateCheck = Pattern.getPredicateCheck();
+  if (!PredicateCheck.empty())
+    AddMatcher(new CheckPatternPredicateMatcher(PredicateCheck));
 
   // Now that we've completed the structural type match, emit any ComplexPattern
   // checks (e.g. addrmode matches).  We emit this after the structural match
@@ -605,16 +596,17 @@ bool MatcherGen::EmitMatcherCode(unsigned Variant) {
     // Get the slot we recorded the value in from the name on the node.
     unsigned RecNodeEntry = MatchedComplexPatterns[i].second;
 
-    const ComplexPattern &CP = *N->getComplexPatternInfo(CGP);
+    const ComplexPattern *CP = N->getComplexPatternInfo(CGP);
+    assert(CP && "Not a valid ComplexPattern!");
 
     // Emit a CheckComplexPat operation, which does the match (aborting if it
     // fails) and pushes the matched operands onto the recorded nodes list.
-    AddMatcher(new CheckComplexPatMatcher(CP, RecNodeEntry,
-                                          N->getName(), NextRecordedOperandNo));
+    AddMatcher(new CheckComplexPatMatcher(*CP, RecNodeEntry, N->getName(),
+                                          NextRecordedOperandNo));
 
     // Record the right number of operands.
-    NextRecordedOperandNo += CP.getNumOperands();
-    if (CP.hasProperty(SDNPHasChain)) {
+    NextRecordedOperandNo += CP->getNumOperands();
+    if (CP->hasProperty(SDNPHasChain)) {
       // If the complex pattern has a chain, then we need to keep track of the
       // fact that we just recorded a chain input.  The chain input will be
       // matched as the last operand of the predicate if it was successful.
@@ -697,12 +689,12 @@ void MatcherGen::EmitResultLeafAsOperand(const TreePatternNode *N,
     }
 
     if (Def->getName() == "undef_tied_input") {
-      std::array<MVT::SimpleValueType, 1> ResultVTs = {{ N->getSimpleType(0) }};
-      std::array<unsigned, 0> InstOps;
+      MVT::SimpleValueType ResultVT = N->getSimpleType(0);
       auto IDOperandNo = NextRecordedOperandNo++;
-      AddMatcher(new EmitNodeMatcher("TargetOpcode::IMPLICIT_DEF",
-                                     ResultVTs, InstOps, false, false, false,
-                                     false, -1, IDOperandNo));
+      Record *ImpDef = Def->getRecords().getDef("IMPLICIT_DEF");
+      CodeGenInstruction &II = CGP.getTargetInfo().getInstruction(ImpDef);
+      AddMatcher(new EmitNodeMatcher(II, ResultVT, std::nullopt, false, false,
+                                     false, false, -1, IDOperandNo));
       ResultOps.push_back(IDOperandNo);
       return;
     }
@@ -983,11 +975,9 @@ EmitResultInstructionAsOperand(const TreePatternNode *N,
   assert((!ResultVTs.empty() || TreeHasOutGlue || NodeHasChain) &&
          "Node has no result");
 
-  AddMatcher(new EmitNodeMatcher(II.Namespace.str()+"::"+II.TheDef->getName().str(),
-                                 ResultVTs, InstOps,
-                                 NodeHasChain, TreeHasInGlue, TreeHasOutGlue,
-                                 NodeHasMemRefs, NumFixedArityOperands,
-                                 NextRecordedOperandNo));
+  AddMatcher(new EmitNodeMatcher(II, ResultVTs, InstOps, NodeHasChain,
+                                 TreeHasInGlue, TreeHasOutGlue, NodeHasMemRefs,
+                                 NumFixedArityOperands, NextRecordedOperandNo));
 
   // The non-chain and non-glue results of the newly emitted node get recorded.
   for (unsigned i = 0, e = ResultVTs.size(); i != e; ++i) {
diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
index 4273bd69b87d..bf2a24241e84 100644
--- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
@@ -10,8 +10,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "DAGISelMatcher.h"
 #include "CodeGenDAGPatterns.h"
+#include "DAGISelMatcher.h"
+#include "SDNodeProperties.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -25,8 +26,9 @@ static void ContractNodes(std::unique_ptr<Matcher> &MatcherPtr,
                           const CodeGenDAGPatterns &CGP) {
   // If we reached the end of the chain, we're done.
   Matcher *N = MatcherPtr.get();
-  if (!N) return;
-  
+  if (!N)
+    return;
+
   // If we have a scope node, walk down all of the children.
   if (ScopeMatcher *Scope = dyn_cast<ScopeMatcher>(N)) {
     for (unsigned i = 0, e = Scope->getNumChildren(); i != e; ++i) {
@@ -36,31 +38,31 @@ static void ContractNodes(std::unique_ptr<Matcher> &MatcherPtr,
     }
     return;
   }
-  
+
   // If we found a movechild node with a node that comes in a 'foochild' form,
   // transform it.
   if (MoveChildMatcher *MC = dyn_cast<MoveChildMatcher>(N)) {
     Matcher *New = nullptr;
     if (RecordMatcher *RM = dyn_cast<RecordMatcher>(MC->getNext()))
-      if (MC->getChildNo() < 8)  // Only have RecordChild0...7
+      if (MC->getChildNo() < 8) // Only have RecordChild0...7
         New = new RecordChildMatcher(MC->getChildNo(), RM->getWhatFor(),
                                      RM->getResultNo());
 
     if (CheckTypeMatcher *CT = dyn_cast<CheckTypeMatcher>(MC->getNext()))
-      if (MC->getChildNo() < 8 &&  // Only have CheckChildType0...7
-          CT->getResNo() == 0)     // CheckChildType checks res #0
+      if (MC->getChildNo() < 8 && // Only have CheckChildType0...7
+          CT->getResNo() == 0)    // CheckChildType checks res #0
         New = new CheckChildTypeMatcher(MC->getChildNo(), CT->getType());
 
     if (CheckSameMatcher *CS = dyn_cast<CheckSameMatcher>(MC->getNext()))
-      if (MC->getChildNo() < 4)  // Only have CheckChildSame0...3
+      if (MC->getChildNo() < 4) // Only have CheckChildSame0...3
         New = new CheckChildSameMatcher(MC->getChildNo(), CS->getMatchNumber());
 
     if (CheckIntegerMatcher *CI = dyn_cast<CheckIntegerMatcher>(MC->getNext()))
-      if (MC->getChildNo() < 5)  // Only have CheckChildInteger0...4
+      if (MC->getChildNo() < 5) // Only have CheckChildInteger0...4
         New = new CheckChildIntegerMatcher(MC->getChildNo(), CI->getValue());
 
     if (auto *CCC = dyn_cast<CheckCondCodeMatcher>(MC->getNext()))
-      if (MC->getChildNo() == 2)  // Only have CheckChild2CondCode
+      if (MC->getChildNo() == 2) // Only have CheckChild2CondCode
         New = new CheckChild2CondCodeMatcher(CCC->getCondCodeName());
 
     if (New) {
@@ -72,11 +74,10 @@ static void ContractNodes(std::unique_ptr<Matcher> &MatcherPtr,
       return ContractNodes(MatcherPtr, CGP);
     }
   }
-  
+
   // Zap movechild -> moveparent.
   if (MoveChildMatcher *MC = dyn_cast<MoveChildMatcher>(N))
-    if (MoveParentMatcher *MP = 
-          dyn_cast<MoveParentMatcher>(MC->getNext())) {
+    if (MoveParentMatcher *MP = dyn_cast<MoveParentMatcher>(MC->getNext())) {
       MatcherPtr.reset(MP->takeNext());
       return ContractNodes(MatcherPtr, CGP);
     }
@@ -84,19 +85,19 @@ static void ContractNodes(std::unique_ptr<Matcher> &MatcherPtr,
   // Turn EmitNode->CompleteMatch into MorphNodeTo if we can.
   if (EmitNodeMatcher *EN = dyn_cast<EmitNodeMatcher>(N))
     if (CompleteMatchMatcher *CM =
-          dyn_cast<CompleteMatchMatcher>(EN->getNext())) {
+            dyn_cast<CompleteMatchMatcher>(EN->getNext())) {
       // We can only use MorphNodeTo if the result values match up.
       unsigned RootResultFirst = EN->getFirstResultSlot();
       bool ResultsMatch = true;
       for (unsigned i = 0, e = CM->getNumResults(); i != e; ++i)
-        if (CM->getResult(i) != RootResultFirst+i)
+        if (CM->getResult(i) != RootResultFirst + i)
           ResultsMatch = false;
-      
+
       // If the selected node defines a subset of the glue/chain results, we
       // can't use MorphNodeTo.  For example, we can't use MorphNodeTo if the
       // matched pattern has a chain but the root node doesn't.
       const PatternToMatch &Pattern = CM->getPattern();
-      
+
       if (!EN->hasChain() &&
           Pattern.getSrcPattern()->NodeHasProperty(SDNPHasChain, CGP))
         ResultsMatch = false;
@@ -107,40 +108,35 @@ static void ContractNodes(std::unique_ptr<Matcher> &MatcherPtr,
       // NOTE: Strictly speaking, we don't have to check for glue here
       // because the code in the pattern generator doesn't handle it right.  We
       // do it anyway for thoroughness.
-      if (!EN->hasOutFlag() &&
+      if (!EN->hasOutGlue() &&
           Pattern.getSrcPattern()->NodeHasProperty(SDNPOutGlue, CGP))
         ResultsMatch = false;
-      
-      
+
+#if 0
       // If the root result node defines more results than the source root node
       // *and* has a chain or glue input, then we can't match it because it
       // would end up replacing the extra result with the chain/glue.
-#if 0
       if ((EN->hasGlue() || EN->hasChain()) &&
           EN->getNumNonChainGlueVTs() > ... need to get no results reliably ...)
         ResultMatch = false;
 #endif
-          
+
       if (ResultsMatch) {
         const SmallVectorImpl<MVT::SimpleValueType> &VTs = EN->getVTList();
         const SmallVectorImpl<unsigned> &Operands = EN->getOperandList();
-        MatcherPtr.reset(new MorphNodeToMatcher(EN->getOpcodeName(),
-                                                VTs, Operands,
-                                                EN->hasChain(), EN->hasInFlag(),
-                                                EN->hasOutFlag(),
-                                                EN->hasMemRefs(),
-                                                EN->getNumFixedArityOperands(),
-                                                Pattern));
+        MatcherPtr.reset(new MorphNodeToMatcher(
+            EN->getInstruction(), VTs, Operands, EN->hasChain(),
+            EN->hasInGlue(), EN->hasOutGlue(), EN->hasMemRefs(),
+            EN->getNumFixedArityOperands(), Pattern));
         return;
       }
 
       // FIXME2: Kill off all the SelectionDAG::SelectNodeTo and getMachineNode
       // variants.
     }
-  
+
   ContractNodes(N->getNextPtr(), CGP);
-  
-  
+
   // If we have a CheckType/CheckChildType/Record node followed by a
   // CheckOpcode, invert the two nodes.  We prefer to do structural checks
   // before type checks, as this opens opportunities for factoring on targets
@@ -152,7 +148,7 @@ static void ContractNodes(std::unique_ptr<Matcher> &MatcherPtr,
     Matcher *CheckType = MatcherPtr.release();
     Matcher *CheckOpcode = CheckType->takeNext();
     Matcher *Tail = CheckOpcode->takeNext();
-    
+
     // Relink them.
     MatcherPtr.reset(CheckOpcode);
     CheckOpcode->setNext(CheckType);
@@ -171,7 +167,6 @@ static Matcher *FindNodeWithKind(Matcher *M, Matcher::KindTy Kind) {
   return nullptr;
 }
 
-
 /// FactorNodes - Turn matches like this:
 ///   Scope
 ///     OPC_CheckType i32
@@ -191,7 +186,8 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
   while (!Scope) {
     // If we reached the end of the chain, we're done.
     Matcher *N = RebindableMatcherPtr->get();
-    if (!N) return;
+    if (!N)
+      return;
 
     // If this is not a push node, just scan for one.
     Scope = dyn_cast<ScopeMatcher>(N);
@@ -199,78 +195,73 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
       RebindableMatcherPtr = &(N->getNextPtr());
   }
   std::unique_ptr<Matcher> &MatcherPtr = *RebindableMatcherPtr;
-  
+
   // Okay, pull together the children of the scope node into a vector so we can
   // inspect it more easily.
-  SmallVector<Matcher*, 32> OptionsToMatch;
-  
+  SmallVector<Matcher *, 32> OptionsToMatch;
+
   for (unsigned i = 0, e = Scope->getNumChildren(); i != e; ++i) {
     // Factor the subexpression.
     std::unique_ptr<Matcher> Child(Scope->takeChild(i));
     FactorNodes(Child);
-    
-    if (Child) {
-      // If the child is a ScopeMatcher we can just merge its contents.
-      if (auto *SM = dyn_cast<ScopeMatcher>(Child.get())) {
-        for (unsigned j = 0, e = SM->getNumChildren(); j != e; ++j)
-          OptionsToMatch.push_back(SM->takeChild(j));
-      } else {
-        OptionsToMatch.push_back(Child.release());
-      }
+
+    // If the child is a ScopeMatcher we can just merge its contents.
+    if (auto *SM = dyn_cast<ScopeMatcher>(Child.get())) {
+      for (unsigned j = 0, e = SM->getNumChildren(); j != e; ++j)
+        OptionsToMatch.push_back(SM->takeChild(j));
+    } else {
+      OptionsToMatch.push_back(Child.release());
     }
   }
-  
-  SmallVector<Matcher*, 32> NewOptionsToMatch;
-  
+
   // Loop over options to match, merging neighboring patterns with identical
   // starting nodes into a shared matcher.
-  for (unsigned OptionIdx = 0, e = OptionsToMatch.size(); OptionIdx != e;) {
+  auto E = OptionsToMatch.end();
+  for (auto I = OptionsToMatch.begin(); I != E; ++I) {
+    // If there are no other matchers left, there's nothing to merge with.
+    auto J = std::next(I);
+    if (J == E)
+      break;
+
+    // Remember where we started. We'll use this to move non-equal elements.
+    auto K = J;
+
     // Find the set of matchers that start with this node.
-    Matcher *Optn = OptionsToMatch[OptionIdx++];
+    Matcher *Optn = *I;
 
-    if (OptionIdx == e) {
-      NewOptionsToMatch.push_back(Optn);
-      continue;
-    }
-    
     // See if the next option starts with the same matcher.  If the two
     // neighbors *do* start with the same matcher, we can factor the matcher out
     // of at least these two patterns.  See what the maximal set we can merge
     // together is.
-    SmallVector<Matcher*, 8> EqualMatchers;
+    SmallVector<Matcher *, 8> EqualMatchers;
     EqualMatchers.push_back(Optn);
-    
+
     // Factor all of the known-equal matchers after this one into the same
     // group.
-    while (OptionIdx != e && OptionsToMatch[OptionIdx]->isEqual(Optn))
-      EqualMatchers.push_back(OptionsToMatch[OptionIdx++]);
+    while (J != E && (*J)->isEqual(Optn))
+      EqualMatchers.push_back(*J++);
 
     // If we found a non-equal matcher, see if it is contradictory with the
     // current node.  If so, we know that the ordering relation between the
     // current sets of nodes and this node don't matter.  Look past it to see if
     // we can merge anything else into this matching group.
-    unsigned Scan = OptionIdx;
-    while (true) {
-      // If we ran out of stuff to scan, we're done.
-      if (Scan == e) break;
-      
-      Matcher *ScanMatcher = OptionsToMatch[Scan];
-      
+    while (J != E) {
+      Matcher *ScanMatcher = *J;
+
       // If we found an entry that matches out matcher, merge it into the set to
       // handle.
       if (Optn->isEqual(ScanMatcher)) {
-        // If is equal after all, add the option to EqualMatchers and remove it
-        // from OptionsToMatch.
+        // It is equal after all, add the option to EqualMatchers.
         EqualMatchers.push_back(ScanMatcher);
-        OptionsToMatch.erase(OptionsToMatch.begin()+Scan);
-        --e;
+        ++J;
         continue;
       }
-      
+
       // If the option we're checking for contradicts the start of the list,
-      // skip over it.
+      // move it earlier in OptionsToMatch for the next iteration of the outer
+      // loop. Then continue searching for equal or contradictory matchers.
       if (Optn->isContradictory(ScanMatcher)) {
-        ++Scan;
+        *K++ = *J++;
         continue;
       }
 
@@ -279,38 +270,47 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
       // or the same as what we're looking for.  If so, reorder it.
       if (Optn->isSimplePredicateOrRecordNode()) {
         Matcher *M2 = FindNodeWithKind(ScanMatcher, Optn->getKind());
-        if (M2 && M2 != ScanMatcher &&
-            M2->canMoveBefore(ScanMatcher) &&
+        if (M2 && M2 != ScanMatcher && M2->canMoveBefore(ScanMatcher) &&
             (M2->isEqual(Optn) || M2->isContradictory(Optn))) {
           Matcher *MatcherWithoutM2 = ScanMatcher->unlinkNode(M2);
           M2->setNext(MatcherWithoutM2);
-          OptionsToMatch[Scan] = M2;
+          *J = M2;
           continue;
         }
       }
-      
+
       // Otherwise, we don't know how to handle this entry, we have to bail.
       break;
     }
-      
-    if (Scan != e &&
-        // Don't print it's obvious nothing extra could be merged anyway.
-        Scan+1 != e) {
+
+    if (J != E &&
+        // Don't print if it's obvious nothing extract could be merged anyway.
+        std::next(J) != E) {
       LLVM_DEBUG(errs() << "Couldn't merge this:\n"; Optn->print(errs(), 4);
                  errs() << "into this:\n";
-                 OptionsToMatch[Scan]->print(errs(), 4);
-                 if (Scan + 1 != e) OptionsToMatch[Scan + 1]->printOne(errs());
-                 if (Scan + 2 < e) OptionsToMatch[Scan + 2]->printOne(errs());
+                 (*J)->print(errs(), 4);
+                 (*std::next(J))->printOne(errs());
+                 if (std::next(J, 2) != E) (*std::next(J, 2))->printOne(errs());
                  errs() << "\n");
     }
-    
+
+    // If we removed any equal matchers, we may need to slide the rest of the
+    // elements down for the next iteration of the outer loop.
+    if (J != K) {
+      while (J != E)
+        *K++ = *J++;
+
+      // Update end pointer for outer loop.
+      E = K;
+    }
+
     // If we only found one option starting with this matcher, no factoring is
-    // possible.
+    // possible. Put the Matcher back in OptionsToMatch.
     if (EqualMatchers.size() == 1) {
-      NewOptionsToMatch.push_back(EqualMatchers[0]);
+      *I = EqualMatchers[0];
       continue;
     }
-    
+
     // Factor these checks by pulling the first node off each entry and
     // discarding it.  Take the first one off the first entry to reuse.
     Matcher *Shared = Optn;
@@ -322,42 +322,49 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
       Matcher *Tmp = EqualMatchers[i]->takeNext();
       delete EqualMatchers[i];
       EqualMatchers[i] = Tmp;
+      assert(!Optn == !Tmp && "Expected all to be null if any are null");
     }
-    
-    Shared->setNext(new ScopeMatcher(EqualMatchers));
 
-    // Recursively factor the newly created node.
-    FactorNodes(Shared->getNextPtr());
-    
-    NewOptionsToMatch.push_back(Shared);
+    if (EqualMatchers[0]) {
+      Shared->setNext(new ScopeMatcher(std::move(EqualMatchers)));
+
+      // Recursively factor the newly created node.
+      FactorNodes(Shared->getNextPtr());
+    }
+
+    // Put the new Matcher where we started in OptionsToMatch.
+    *I = Shared;
   }
-  
+
+  // Trim the array to match the updated end.
+  if (E != OptionsToMatch.end())
+    OptionsToMatch.erase(E, OptionsToMatch.end());
+
   // If we're down to a single pattern to match, then we don't need this scope
   // anymore.
-  if (NewOptionsToMatch.size() == 1) {
-    MatcherPtr.reset(NewOptionsToMatch[0]);
+  if (OptionsToMatch.size() == 1) {
+    MatcherPtr.reset(OptionsToMatch[0]);
     return;
   }
-  
-  if (NewOptionsToMatch.empty()) {
+
+  if (OptionsToMatch.empty()) {
     MatcherPtr.reset();
     return;
   }
-  
+
   // If our factoring failed (didn't achieve anything) see if we can simplify in
   // other ways.
-  
+
   // Check to see if all of the leading entries are now opcode checks.  If so,
   // we can convert this Scope to be a OpcodeSwitch instead.
   bool AllOpcodeChecks = true, AllTypeChecks = true;
-  for (unsigned i = 0, e = NewOptionsToMatch.size(); i != e; ++i) {
+  for (unsigned i = 0, e = OptionsToMatch.size(); i != e; ++i) {
     // Check to see if this breaks a series of CheckOpcodeMatchers.
-    if (AllOpcodeChecks &&
-        !isa<CheckOpcodeMatcher>(NewOptionsToMatch[i])) {
+    if (AllOpcodeChecks && !isa<CheckOpcodeMatcher>(OptionsToMatch[i])) {
 #if 0
       if (i > 3) {
         errs() << "FAILING OPC #" << i << "\n";
-        NewOptionsToMatch[i]->dump();
+        OptionsToMatch[i]->dump();
       }
 #endif
       AllOpcodeChecks = false;
@@ -365,9 +372,8 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
 
     // Check to see if this breaks a series of CheckTypeMatcher's.
     if (AllTypeChecks) {
-      CheckTypeMatcher *CTM =
-        cast_or_null<CheckTypeMatcher>(FindNodeWithKind(NewOptionsToMatch[i],
-                                                        Matcher::CheckType));
+      CheckTypeMatcher *CTM = cast_or_null<CheckTypeMatcher>(
+          FindNodeWithKind(OptionsToMatch[i], Matcher::CheckType));
       if (!CTM ||
           // iPTR checks could alias any other case without us knowing, don't
           // bother with them.
@@ -376,66 +382,66 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
           CTM->getResNo() != 0 ||
           // If the CheckType isn't at the start of the list, see if we can move
           // it there.
-          !CTM->canMoveBefore(NewOptionsToMatch[i])) {
+          !CTM->canMoveBefore(OptionsToMatch[i])) {
 #if 0
         if (i > 3 && AllTypeChecks) {
           errs() << "FAILING TYPE #" << i << "\n";
-          NewOptionsToMatch[i]->dump();
+          OptionsToMatch[i]->dump();
         }
 #endif
         AllTypeChecks = false;
       }
     }
   }
-  
+
   // If all the options are CheckOpcode's, we can form the SwitchOpcode, woot.
   if (AllOpcodeChecks) {
     StringSet<> Opcodes;
-    SmallVector<std::pair<const SDNodeInfo*, Matcher*>, 8> Cases;
-    for (unsigned i = 0, e = NewOptionsToMatch.size(); i != e; ++i) {
-      CheckOpcodeMatcher *COM = cast<CheckOpcodeMatcher>(NewOptionsToMatch[i]);
+    SmallVector<std::pair<const SDNodeInfo *, Matcher *>, 8> Cases;
+    for (unsigned i = 0, e = OptionsToMatch.size(); i != e; ++i) {
+      CheckOpcodeMatcher *COM = cast<CheckOpcodeMatcher>(OptionsToMatch[i]);
       assert(Opcodes.insert(COM->getOpcode().getEnumName()).second &&
              "Duplicate opcodes not factored?");
       Cases.push_back(std::make_pair(&COM->getOpcode(), COM->takeNext()));
       delete COM;
     }
-    
-    MatcherPtr.reset(new SwitchOpcodeMatcher(Cases));
+
+    MatcherPtr.reset(new SwitchOpcodeMatcher(std::move(Cases)));
     return;
   }
-  
+
   // If all the options are CheckType's, we can form the SwitchType, woot.
   if (AllTypeChecks) {
     DenseMap<unsigned, unsigned> TypeEntry;
-    SmallVector<std::pair<MVT::SimpleValueType, Matcher*>, 8> Cases;
-    for (unsigned i = 0, e = NewOptionsToMatch.size(); i != e; ++i) {
-      Matcher* M = FindNodeWithKind(NewOptionsToMatch[i], Matcher::CheckType);
+    SmallVector<std::pair<MVT::SimpleValueType, Matcher *>, 8> Cases;
+    for (unsigned i = 0, e = OptionsToMatch.size(); i != e; ++i) {
+      Matcher *M = FindNodeWithKind(OptionsToMatch[i], Matcher::CheckType);
       assert(M && isa<CheckTypeMatcher>(M) && "Unknown Matcher type");
 
       auto *CTM = cast<CheckTypeMatcher>(M);
-      Matcher *MatcherWithoutCTM = NewOptionsToMatch[i]->unlinkNode(CTM);
+      Matcher *MatcherWithoutCTM = OptionsToMatch[i]->unlinkNode(CTM);
       MVT::SimpleValueType CTMTy = CTM->getType();
       delete CTM;
 
       unsigned &Entry = TypeEntry[CTMTy];
       if (Entry != 0) {
         // If we have unfactored duplicate types, then we should factor them.
-        Matcher *PrevMatcher = Cases[Entry-1].second;
+        Matcher *PrevMatcher = Cases[Entry - 1].second;
         if (ScopeMatcher *SM = dyn_cast<ScopeMatcher>(PrevMatcher)) {
-          SM->setNumChildren(SM->getNumChildren()+1);
-          SM->resetChild(SM->getNumChildren()-1, MatcherWithoutCTM);
+          SM->setNumChildren(SM->getNumChildren() + 1);
+          SM->resetChild(SM->getNumChildren() - 1, MatcherWithoutCTM);
           continue;
         }
-        
-        Matcher *Entries[2] = { PrevMatcher, MatcherWithoutCTM };
-        Cases[Entry-1].second = new ScopeMatcher(Entries);
+
+        SmallVector<Matcher *, 2> Entries = {PrevMatcher, MatcherWithoutCTM};
+        Cases[Entry - 1].second = new ScopeMatcher(std::move(Entries));
         continue;
       }
-      
-      Entry = Cases.size()+1;
+
+      Entry = Cases.size() + 1;
       Cases.push_back(std::make_pair(CTMTy, MatcherWithoutCTM));
     }
-    
+
     // Make sure we recursively factor any scopes we may have created.
     for (auto &M : Cases) {
       if (ScopeMatcher *SM = dyn_cast<ScopeMatcher>(M.second)) {
@@ -447,7 +453,7 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
     }
 
     if (Cases.size() != 1) {
-      MatcherPtr.reset(new SwitchTypeMatcher(Cases));
+      MatcherPtr.reset(new SwitchTypeMatcher(std::move(Cases)));
     } else {
       // If we factored and ended up with one case, create it now.
       MatcherPtr.reset(new CheckTypeMatcher(Cases[0].first, 0));
@@ -455,17 +461,15 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
     }
     return;
   }
-  
 
   // Reassemble the Scope node with the adjusted children.
-  Scope->setNumChildren(NewOptionsToMatch.size());
-  for (unsigned i = 0, e = NewOptionsToMatch.size(); i != e; ++i)
-    Scope->resetChild(i, NewOptionsToMatch[i]);
+  Scope->setNumChildren(OptionsToMatch.size());
+  for (unsigned i = 0, e = OptionsToMatch.size(); i != e; ++i)
+    Scope->resetChild(i, OptionsToMatch[i]);
 }
 
-void
-llvm::OptimizeMatcher(std::unique_ptr<Matcher> &MatcherPtr,
-                      const CodeGenDAGPatterns &CGP) {
+void llvm::OptimizeMatcher(std::unique_ptr<Matcher> &MatcherPtr,
+                           const CodeGenDAGPatterns &CGP) {
   ContractNodes(MatcherPtr, CGP);
   FactorNodes(MatcherPtr);
 }
diff --git a/llvm/utils/TableGen/DFAEmitter.cpp b/llvm/utils/TableGen/DFAEmitter.cpp
index 705908226fa1..54ad81cbebe8 100644
--- a/llvm/utils/TableGen/DFAEmitter.cpp
+++ b/llvm/utils/TableGen/DFAEmitter.cpp
@@ -22,13 +22,13 @@
 
 #include "DFAEmitter.h"
 #include "SequenceToOffsetTable.h"
-#include "TableGenBackends.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/UniqueVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <cassert>
 #include <cstdint>
 #include <deque>
@@ -370,10 +370,5 @@ void CustomDfaEmitter::printActionValue(action_type A, raw_ostream &OS) {
     OS << ")";
 }
 
-namespace llvm {
-
-void EmitAutomata(RecordKeeper &RK, raw_ostream &OS) {
-  AutomatonEmitter(RK).run(OS);
-}
-
-} // namespace llvm
+static TableGen::Emitter::OptClass<AutomatonEmitter>
+    X("gen-automata", "Generate generic automata");
diff --git a/llvm/utils/TableGen/DFAEmitter.h b/llvm/utils/TableGen/DFAEmitter.h
index 44e5d97d544f..c831a65a73cd 100644
--- a/llvm/utils/TableGen/DFAEmitter.h
+++ b/llvm/utils/TableGen/DFAEmitter.h
@@ -21,6 +21,8 @@
 #include "llvm/ADT/UniqueVector.h"
 #include <map>
 #include <set>
+#include <utility>
+#include <vector>
 
 namespace llvm {
 
diff --git a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
index 6704d747f715..64c7884616a5 100644
--- a/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
+++ b/llvm/utils/TableGen/DFAPacketizerEmitter.cpp
@@ -24,6 +24,7 @@
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cassert>
 #include <cstdint>
+#include <deque>
 #include <map>
 #include <set>
 #include <string>
@@ -205,6 +206,7 @@ void DFAPacketizerEmitter::createScheduleClasses(unsigned ItineraryIdx,
 // Run the worklist algorithm to generate the DFA.
 //
 void DFAPacketizerEmitter::run(raw_ostream &OS) {
+  emitSourceFileHeader("Target DFA Packetizer Tables", OS);
   OS << "\n"
      << "#include \"llvm/CodeGen/DFAPacketizer.h\"\n";
   OS << "namespace llvm {\n";
@@ -352,11 +354,5 @@ void DFAPacketizerEmitter::emitForItineraries(
      << "\n}\n\n";
 }
 
-namespace llvm {
-
-void EmitDFAPacketizer(RecordKeeper &RK, raw_ostream &OS) {
-  emitSourceFileHeader("Target DFA Packetizer Tables", OS);
-  DFAPacketizerEmitter(RK).run(OS);
-}
-
-} // end namespace llvm
+static TableGen::Emitter::OptClass<DFAPacketizerEmitter>
+    X("gen-dfa-packetizer", "Generate DFA Packetizer for VLIW targets");
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index 44c1df3e9ac4..b294c66007f8 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -17,8 +17,8 @@
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/DXILOperationCommon.h"
-#include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 using namespace llvm;
 using namespace llvm::dxil;
@@ -26,8 +26,8 @@ using namespace llvm::dxil;
 namespace {
 
 struct DXILShaderModel {
-  int Major;
-  int Minor;
+  int Major = 0;
+  int Minor = 0;
 };
 
 struct DXILParam {
@@ -56,12 +56,13 @@ struct DXILOperationData {
                                  // memory,ro=only reads from memory
   StringRef Intrinsic; // The llvm intrinsic map to DXILOp. Default is "" which
                        // means no map exist
-  bool IsDeriv;        // whether this is some kind of derivative
-  bool IsGradient;               // whether this requires a gradient calculation
-  bool IsFeedback;               // whether this is a sampler feedback op
-  bool IsWave; // whether this requires in-wave, cross-lane functionality
-  bool RequiresUniformInputs; // whether this operation requires that all
-                              // of its inputs are uniform across the wave
+  bool IsDeriv = false;    // whether this is some kind of derivative
+  bool IsGradient = false; // whether this requires a gradient calculation
+  bool IsFeedback = false; // whether this is a sampler feedback op
+  bool IsWave = false;     // whether this requires in-wave, cross-lane functionality
+  bool RequiresUniformInputs = false; // whether this operation requires that
+                                      // all of its inputs are uniform across
+                                      // the wave
   SmallVector<StringRef, 4>
       ShaderStages; // shader stages to which this applies, empty for all.
   DXILShaderModel ShaderModel;           // minimum shader model required
@@ -322,7 +323,7 @@ static void emitDXILOperationTable(std::vector<DXILOperationData> &DXILOps,
   for (auto &DXILOp : DXILOps) {
     OpStrings.add(DXILOp.DXILOp.str());
 
-    if (ClassSet.find(DXILOp.DXILClass) != ClassSet.end())
+    if (ClassSet.contains(DXILOp.DXILClass))
       continue;
     ClassSet.insert(DXILOp.DXILClass);
     OpClassStrings.add(getDXILOpClassName(DXILOp.DXILClass));
@@ -411,9 +412,7 @@ static void emitDXILOperationTable(std::vector<DXILOperationData> &DXILOps,
   OS << "}\n ";
 }
 
-namespace llvm {
-
-void EmitDXILOperation(RecordKeeper &Records, raw_ostream &OS) {
+static void EmitDXILOperation(RecordKeeper &Records, raw_ostream &OS) {
   std::vector<Record *> Ops = Records.getAllDerivedDefinitions("dxil_op");
   OS << "// Generated code, do not edit.\n";
   OS << "\n";
@@ -439,4 +438,5 @@ void EmitDXILOperation(RecordKeeper &Records, raw_ostream &OS) {
   OS << "\n";
 }
 
-} // namespace llvm
+static TableGen::Emitter::Opt X("gen-dxil-operation", EmitDXILOperation,
+                                "Generate DXIL operation information");
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 8f816744370c..607f19653c7a 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -11,9 +11,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CodeGenHwModes.h"
 #include "CodeGenInstruction.h"
 #include "CodeGenTarget.h"
 #include "InfoByHwMode.h"
+#include "TableGenBackends.h"
 #include "VarLenCodeEmitterGen.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -2028,193 +2030,11 @@ populateInstruction(CodeGenTarget &Target, const Record &EncodingDef,
   if (IsVarLenInst) {
     parseVarLenInstOperand(EncodingDef, InsnOperands, CGI);
   } else {
-    std::map<std::string, std::vector<OperandInfo>> NumberedInsnOperands;
-    std::set<std::string> NumberedInsnOperandsNoTie;
-    bool SupportPositionalDecoding =
-        Target.getInstructionSet()->getValueAsBit(
-            "useDeprecatedPositionallyEncodedOperands") &&
-        Target.getInstructionSet()->getValueAsBit(
-            "decodePositionallyEncodedOperands");
-    if (SupportPositionalDecoding) {
-      const std::vector<RecordVal> &Vals = Def.getValues();
-      unsigned NumberedOp = 0;
-
-      std::set<unsigned> NamedOpIndices;
-      if (Target.getInstructionSet()->getValueAsBit(
-              "noNamedPositionallyEncodedOperands"))
-        // Collect the set of operand indices that might correspond to named
-        // operand, and skip these when assigning operands based on position.
-        for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
-          unsigned OpIdx;
-          if (!CGI.Operands.hasOperandNamed(Vals[i].getName(), OpIdx))
-            continue;
-
-          NamedOpIndices.insert(OpIdx);
-        }
-
-      for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
-        // Ignore fixed fields in the record, we're looking for values like:
-        //    bits<5> RST = { ?, ?, ?, ?, ? };
-        if (Vals[i].isNonconcreteOK() || Vals[i].getValue()->isComplete())
-          continue;
-
-        // Determine if Vals[i] actually contributes to the Inst encoding.
-        unsigned bi = 0;
-        for (; bi < Bits.getNumBits(); ++bi) {
-          VarInit *Var = nullptr;
-          VarBitInit *BI = dyn_cast<VarBitInit>(Bits.getBit(bi));
-          if (BI)
-            Var = dyn_cast<VarInit>(BI->getBitVar());
-          else
-            Var = dyn_cast<VarInit>(Bits.getBit(bi));
-
-          if (Var && Var->getName() == Vals[i].getName())
-            break;
-        }
-
-        if (bi == Bits.getNumBits())
-          continue;
-
-        // Skip variables that correspond to explicitly-named operands.
-        unsigned OpIdx;
-        std::pair<unsigned, unsigned> SubOp;
-        if (CGI.Operands.hasSubOperandAlias(Vals[i].getName(), SubOp) ||
-            CGI.Operands.hasOperandNamed(Vals[i].getName(), OpIdx))
-          continue;
-
-        // Get the bit range for this operand:
-        unsigned bitStart = bi++, bitWidth = 1;
-        for (; bi < Bits.getNumBits(); ++bi) {
-          VarInit *Var = nullptr;
-          VarBitInit *BI = dyn_cast<VarBitInit>(Bits.getBit(bi));
-          if (BI)
-            Var = dyn_cast<VarInit>(BI->getBitVar());
-          else
-            Var = dyn_cast<VarInit>(Bits.getBit(bi));
-
-          if (!Var)
-            break;
-
-          if (Var->getName() != Vals[i].getName())
-            break;
-
-          ++bitWidth;
-        }
-
-        unsigned NumberOps = CGI.Operands.size();
-        while (NumberedOp < NumberOps &&
-               (CGI.Operands.isFlatOperandNotEmitted(NumberedOp) ||
-                (!NamedOpIndices.empty() &&
-                 NamedOpIndices.count(
-                     CGI.Operands.getSubOperandNumber(NumberedOp).first))))
-          ++NumberedOp;
-
-        OpIdx = NumberedOp++;
-
-        // OpIdx now holds the ordered operand number of Vals[i].
-        std::pair<unsigned, unsigned> SO =
-            CGI.Operands.getSubOperandNumber(OpIdx);
-        const std::string &Name = CGI.Operands[SO.first].Name;
-
-        LLVM_DEBUG(dbgs() << "Numbered operand mapping for " << Def.getName()
-                          << ": " << Name << "(" << SO.first << ", "
-                          << SO.second << ") => " << Vals[i].getName() << "\n");
-
-        std::string Decoder;
-        Record *TypeRecord = CGI.Operands[SO.first].Rec;
-
-        RecordVal *DecoderString = TypeRecord->getValue("DecoderMethod");
-        StringInit *String =
-            DecoderString ? dyn_cast<StringInit>(DecoderString->getValue())
-                          : nullptr;
-        if (String && String->getValue() != "")
-          Decoder = std::string(String->getValue());
-
-        if (Decoder == "" && CGI.Operands[SO.first].MIOperandInfo &&
-            CGI.Operands[SO.first].MIOperandInfo->getNumArgs()) {
-          Init *Arg = CGI.Operands[SO.first].MIOperandInfo->getArg(SO.second);
-          if (DefInit *DI = cast<DefInit>(Arg))
-            TypeRecord = DI->getDef();
-        }
-
-        bool isReg = false;
-        if (TypeRecord->isSubClassOf("RegisterOperand"))
-          TypeRecord = TypeRecord->getValueAsDef("RegClass");
-        if (TypeRecord->isSubClassOf("RegisterClass")) {
-          Decoder = "Decode" + TypeRecord->getName().str() + "RegisterClass";
-          isReg = true;
-        } else if (TypeRecord->isSubClassOf("PointerLikeRegClass")) {
-          Decoder = "DecodePointerLikeRegClass" +
-                    utostr(TypeRecord->getValueAsInt("RegClassKind"));
-          isReg = true;
-        }
-
-        DecoderString = TypeRecord->getValue("DecoderMethod");
-        String = DecoderString ? dyn_cast<StringInit>(DecoderString->getValue())
-                               : nullptr;
-        if (!isReg && String && String->getValue() != "")
-          Decoder = std::string(String->getValue());
-
-        RecordVal *HasCompleteDecoderVal =
-            TypeRecord->getValue("hasCompleteDecoder");
-        BitInit *HasCompleteDecoderBit =
-            HasCompleteDecoderVal
-                ? dyn_cast<BitInit>(HasCompleteDecoderVal->getValue())
-                : nullptr;
-        bool HasCompleteDecoder =
-            HasCompleteDecoderBit ? HasCompleteDecoderBit->getValue() : true;
-
-        OperandInfo OpInfo(Decoder, HasCompleteDecoder);
-        OpInfo.addField(bitStart, bitWidth, 0);
-
-        NumberedInsnOperands[Name].push_back(OpInfo);
-
-        // FIXME: For complex operands with custom decoders we can't handle tied
-        // sub-operands automatically. Skip those here and assume that this is
-        // fixed up elsewhere.
-        if (CGI.Operands[SO.first].MIOperandInfo &&
-            CGI.Operands[SO.first].MIOperandInfo->getNumArgs() > 1 && String &&
-            String->getValue() != "")
-          NumberedInsnOperandsNoTie.insert(Name);
-      }
-    }
-
     // For each operand, see if we can figure out where it is encoded.
     for (const auto &Op : InOutOperands) {
       Init *OpInit = Op.first;
       StringRef OpName = Op.second;
 
-      if (SupportPositionalDecoding) {
-        if (!NumberedInsnOperands[std::string(OpName)].empty()) {
-          llvm::append_range(InsnOperands,
-                             NumberedInsnOperands[std::string(OpName)]);
-          continue;
-        }
-        if (!NumberedInsnOperands[TiedNames[std::string(OpName)]].empty()) {
-          if (!NumberedInsnOperandsNoTie.count(
-                  TiedNames[std::string(OpName)])) {
-            // Figure out to which (sub)operand we're tied.
-            unsigned i =
-                CGI.Operands.getOperandNamed(TiedNames[std::string(OpName)]);
-            int tiedTo = CGI.Operands[i].getTiedRegister();
-            if (tiedTo == -1) {
-              i = CGI.Operands.getOperandNamed(OpName);
-              tiedTo = CGI.Operands[i].getTiedRegister();
-            }
-
-            if (tiedTo != -1) {
-              std::pair<unsigned, unsigned> SO =
-                  CGI.Operands.getSubOperandNumber(tiedTo);
-
-              InsnOperands.push_back(
-                  NumberedInsnOperands[TiedNames[std::string(OpName)]]
-                                      [SO.second]);
-            }
-          }
-          continue;
-        }
-      }
-
       // We're ready to find the instruction encoding locations for this operand.
 
       // First, find the operand type ("OpInit"), and sub-op names
@@ -2597,11 +2417,11 @@ void DecoderEmitter::run(raw_ostream &o) {
   formatted_raw_ostream OS(o);
   OS << "#include \"llvm/MC/MCInst.h\"\n";
   OS << "#include \"llvm/MC/MCSubtargetInfo.h\"\n";
-  OS << "#include \"llvm/MC/SubtargetFeature.h\"\n";
   OS << "#include \"llvm/Support/DataTypes.h\"\n";
   OS << "#include \"llvm/Support/Debug.h\"\n";
   OS << "#include \"llvm/Support/LEB128.h\"\n";
   OS << "#include \"llvm/Support/raw_ostream.h\"\n";
+  OS << "#include \"llvm/TargetParser/SubtargetFeature.h\"\n";
   OS << "#include <assert.h>\n";
   OS << '\n';
   OS << "namespace llvm {\n\n";
diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp
index f32fbe3e25cd..67033c6290ca 100644
--- a/llvm/utils/TableGen/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/DirectiveEmitter.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 using namespace llvm;
 
@@ -36,14 +37,12 @@ private:
   StringRef Name;
   raw_ostream &OS;
 };
-} // end anonymous namespace
-
-namespace llvm {
+} // namespace
 
 // Generate enum class
-void GenerateEnumClass(const std::vector<Record *> &Records, raw_ostream &OS,
-                       StringRef Enum, StringRef Prefix,
-                       const DirectiveLanguage &DirLang) {
+static void GenerateEnumClass(const std::vector<Record *> &Records,
+                              raw_ostream &OS, StringRef Enum, StringRef Prefix,
+                              const DirectiveLanguage &DirLang) {
   OS << "\n";
   OS << "enum class " << Enum << " {\n";
   for (const auto &R : Records) {
@@ -73,9 +72,10 @@ void GenerateEnumClass(const std::vector<Record *> &Records, raw_ostream &OS,
 
 // Generate enums for values that clauses can take.
 // Also generate function declarations for get<Enum>Name(StringRef Str).
-void GenerateEnumClauseVal(const std::vector<Record *> &Records,
-                           raw_ostream &OS, const DirectiveLanguage &DirLang,
-                           std::string &EnumHelperFuncs) {
+static void GenerateEnumClauseVal(const std::vector<Record *> &Records,
+                                  raw_ostream &OS,
+                                  const DirectiveLanguage &DirLang,
+                                  std::string &EnumHelperFuncs) {
   for (const auto &R : Records) {
     Clause C{R};
     const auto &ClauseVals = C.getClauseVals();
@@ -117,9 +117,9 @@ void GenerateEnumClauseVal(const std::vector<Record *> &Records,
   }
 }
 
-bool HasDuplicateClauses(const std::vector<Record *> &Clauses,
-                         const Directive &Directive,
-                         llvm::StringSet<> &CrtClauses) {
+static bool HasDuplicateClauses(const std::vector<Record *> &Clauses,
+                                const Directive &Directive,
+                                llvm::StringSet<> &CrtClauses) {
   bool HasError = false;
   for (const auto &C : Clauses) {
     VersionedClause VerClause{C};
@@ -136,7 +136,8 @@ bool HasDuplicateClauses(const std::vector<Record *> &Clauses,
 // Check for duplicate clauses in lists. Clauses cannot appear twice in the
 // three allowed list. Also, since required implies allowed, clauses cannot
 // appear in both the allowedClauses and requiredClauses lists.
-bool HasDuplicateClausesInDirectives(const std::vector<Record *> &Directives) {
+static bool
+HasDuplicateClausesInDirectives(const std::vector<Record *> &Directives) {
   bool HasDuplicate = false;
   for (const auto &D : Directives) {
     Directive Dir{D};
@@ -175,7 +176,7 @@ bool DirectiveLanguage::HasValidityErrors() const {
 
 // Generate the declaration section for the enumeration in the directive
 // language
-void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) {
+static void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) {
   const auto DirLang = DirectiveLanguage{Records};
   if (DirLang.HasValidityErrors())
     return;
@@ -245,9 +246,10 @@ void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) {
 }
 
 // Generate function implementation for get<Enum>Name(StringRef Str)
-void GenerateGetName(const std::vector<Record *> &Records, raw_ostream &OS,
-                     StringRef Enum, const DirectiveLanguage &DirLang,
-                     StringRef Prefix) {
+static void GenerateGetName(const std::vector<Record *> &Records,
+                            raw_ostream &OS, StringRef Enum,
+                            const DirectiveLanguage &DirLang,
+                            StringRef Prefix) {
   OS << "\n";
   OS << "llvm::StringRef llvm::" << DirLang.getCppNamespace() << "::get"
      << DirLang.getName() << Enum << "Name(" << Enum << " Kind) {\n";
@@ -269,9 +271,10 @@ void GenerateGetName(const std::vector<Record *> &Records, raw_ostream &OS,
 }
 
 // Generate function implementation for get<Enum>Kind(StringRef Str)
-void GenerateGetKind(const std::vector<Record *> &Records, raw_ostream &OS,
-                     StringRef Enum, const DirectiveLanguage &DirLang,
-                     StringRef Prefix, bool ImplicitAsUnknown) {
+static void GenerateGetKind(const std::vector<Record *> &Records,
+                            raw_ostream &OS, StringRef Enum,
+                            const DirectiveLanguage &DirLang, StringRef Prefix,
+                            bool ImplicitAsUnknown) {
 
   auto DefaultIt = llvm::find_if(
       Records, [](Record *R) { return R->getValueAsBit("isDefault") == true; });
@@ -303,8 +306,8 @@ void GenerateGetKind(const std::vector<Record *> &Records, raw_ostream &OS,
 }
 
 // Generate function implementation for get<ClauseVal>Kind(StringRef Str)
-void GenerateGetKindClauseVal(const DirectiveLanguage &DirLang,
-                              raw_ostream &OS) {
+static void GenerateGetKindClauseVal(const DirectiveLanguage &DirLang,
+                                     raw_ostream &OS) {
   for (const auto &R : DirLang.getClauses()) {
     Clause C{R};
     const auto &ClauseVals = C.getClauseVals();
@@ -359,10 +362,11 @@ void GenerateGetKindClauseVal(const DirectiveLanguage &DirLang,
   }
 }
 
-void GenerateCaseForVersionedClauses(const std::vector<Record *> &Clauses,
-                                     raw_ostream &OS, StringRef DirectiveName,
-                                     const DirectiveLanguage &DirLang,
-                                     llvm::StringSet<> &Cases) {
+static void
+GenerateCaseForVersionedClauses(const std::vector<Record *> &Clauses,
+                                raw_ostream &OS, StringRef DirectiveName,
+                                const DirectiveLanguage &DirLang,
+                                llvm::StringSet<> &Cases) {
   for (const auto &C : Clauses) {
     VersionedClause VerClause{C};
 
@@ -378,8 +382,8 @@ void GenerateCaseForVersionedClauses(const std::vector<Record *> &Clauses,
 }
 
 // Generate the isAllowedClauseForDirective function implementation.
-void GenerateIsAllowedClause(const DirectiveLanguage &DirLang,
-                             raw_ostream &OS) {
+static void GenerateIsAllowedClause(const DirectiveLanguage &DirLang,
+                                    raw_ostream &OS) {
   OS << "\n";
   OS << "bool llvm::" << DirLang.getCppNamespace()
      << "::isAllowedClauseForDirective("
@@ -432,9 +436,10 @@ void GenerateIsAllowedClause(const DirectiveLanguage &DirLang,
 }
 
 // Generate a simple enum set with the give clauses.
-void GenerateClauseSet(const std::vector<Record *> &Clauses, raw_ostream &OS,
-                       StringRef ClauseSetPrefix, Directive &Dir,
-                       const DirectiveLanguage &DirLang) {
+static void GenerateClauseSet(const std::vector<Record *> &Clauses,
+                              raw_ostream &OS, StringRef ClauseSetPrefix,
+                              Directive &Dir,
+                              const DirectiveLanguage &DirLang) {
 
   OS << "\n";
   OS << "  static " << DirLang.getClauseEnumSetClass() << " " << ClauseSetPrefix
@@ -450,8 +455,8 @@ void GenerateClauseSet(const std::vector<Record *> &Clauses, raw_ostream &OS,
 }
 
 // Generate an enum set for the 4 kinds of clauses linked to a directive.
-void GenerateDirectiveClauseSets(const DirectiveLanguage &DirLang,
-                                 raw_ostream &OS) {
+static void GenerateDirectiveClauseSets(const DirectiveLanguage &DirLang,
+                                        raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_SETS", OS);
 
@@ -490,8 +495,8 @@ void GenerateDirectiveClauseSets(const DirectiveLanguage &DirLang,
 // Generate a map of directive (key) with DirectiveClauses struct as values.
 // The struct holds the 4 sets of enumeration for the 4 kinds of clauses
 // allowances (allowed, allowed once, allowed exclusive and required).
-void GenerateDirectiveClauseMap(const DirectiveLanguage &DirLang,
-                                raw_ostream &OS) {
+static void GenerateDirectiveClauseMap(const DirectiveLanguage &DirLang,
+                                       raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_MAP", OS);
 
@@ -525,8 +530,8 @@ void GenerateDirectiveClauseMap(const DirectiveLanguage &DirLang,
 // If the clause does not hold a value, an EMPTY_CLASS is used.
 // If the clause class is generic then a WRAPPER_CLASS is used. When the value
 // is optional, the value class is wrapped into a std::optional.
-void GenerateFlangClauseParserClass(const DirectiveLanguage &DirLang,
-                                    raw_ostream &OS) {
+static void GenerateFlangClauseParserClass(const DirectiveLanguage &DirLang,
+                                           raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_CLAUSE_PARSER_CLASSES", OS);
 
@@ -553,8 +558,8 @@ void GenerateFlangClauseParserClass(const DirectiveLanguage &DirLang,
 }
 
 // Generate a list of the different clause classes for Flang.
-void GenerateFlangClauseParserClassList(const DirectiveLanguage &DirLang,
-                                        raw_ostream &OS) {
+static void GenerateFlangClauseParserClassList(const DirectiveLanguage &DirLang,
+                                               raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_CLAUSE_PARSER_CLASSES_LIST", OS);
 
@@ -566,8 +571,8 @@ void GenerateFlangClauseParserClassList(const DirectiveLanguage &DirLang,
 }
 
 // Generate dump node list for the clauses holding a generic class name.
-void GenerateFlangClauseDump(const DirectiveLanguage &DirLang,
-                             raw_ostream &OS) {
+static void GenerateFlangClauseDump(const DirectiveLanguage &DirLang,
+                                    raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_DUMP_PARSE_TREE_CLAUSES", OS);
 
@@ -581,8 +586,8 @@ void GenerateFlangClauseDump(const DirectiveLanguage &DirLang,
 
 // Generate Unparse functions for clauses classes in the Flang parse-tree
 // If the clause is a non-generic class, no entry is generated.
-void GenerateFlangClauseUnparse(const DirectiveLanguage &DirLang,
-                                raw_ostream &OS) {
+static void GenerateFlangClauseUnparse(const DirectiveLanguage &DirLang,
+                                       raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_CLAUSE_UNPARSE", OS);
 
@@ -633,8 +638,8 @@ void GenerateFlangClauseUnparse(const DirectiveLanguage &DirLang,
 }
 
 // Generate check in the Enter functions for clauses classes.
-void GenerateFlangClauseCheckPrototypes(const DirectiveLanguage &DirLang,
-                                        raw_ostream &OS) {
+static void GenerateFlangClauseCheckPrototypes(const DirectiveLanguage &DirLang,
+                                               raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_CLAUSE_CHECK_ENTER", OS);
 
@@ -648,8 +653,8 @@ void GenerateFlangClauseCheckPrototypes(const DirectiveLanguage &DirLang,
 
 // Generate the mapping for clauses between the parser class and the
 // corresponding clause Kind
-void GenerateFlangClauseParserKindMap(const DirectiveLanguage &DirLang,
-                                      raw_ostream &OS) {
+static void GenerateFlangClauseParserKindMap(const DirectiveLanguage &DirLang,
+                                             raw_ostream &OS) {
 
   IfDefScope Scope("GEN_FLANG_CLAUSE_PARSER_KIND_MAP", OS);
 
@@ -669,15 +674,15 @@ void GenerateFlangClauseParserKindMap(const DirectiveLanguage &DirLang,
      << " Parser clause\");\n";
 }
 
-bool compareClauseName(Record *R1, Record *R2) {
+static bool compareClauseName(Record *R1, Record *R2) {
   Clause C1{R1};
   Clause C2{R2};
   return (C1.getName() > C2.getName());
 }
 
 // Generate the parser for the clauses.
-void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
-                                raw_ostream &OS) {
+static void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
+                                       raw_ostream &OS) {
   std::vector<Record *> Clauses = DirLang.getClauses();
   // Sort clauses in reverse alphabetical order so with clauses with same
   // beginning, the longer option is tried before.
@@ -715,6 +720,8 @@ void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
     if (Clause.isValueOptional())
       OS << "maybe(";
     OS << "parenthesized(";
+    if (Clause.isValueList())
+      OS << "nonemptyList(";
 
     if (!Clause.getPrefix().empty())
       OS << "\"" << Clause.getPrefix() << ":\" >> ";
@@ -735,6 +742,8 @@ void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
     OS << Parser;
     if (!Clause.getPrefix().empty() && Clause.isPrefixOptional())
       OS << " || " << Parser;
+    if (Clause.isValueList()) // close nonemptyList(.
+      OS << ")";
     OS << ")"; // close parenthesized(.
 
     if (Clause.isValueOptional()) // close maybe(.
@@ -750,8 +759,8 @@ void GenerateFlangClausesParser(const DirectiveLanguage &DirLang,
 
 // Generate the implementation section for the enumeration in the directive
 // language
-void EmitDirectivesFlangImpl(const DirectiveLanguage &DirLang,
-                             raw_ostream &OS) {
+static void EmitDirectivesFlangImpl(const DirectiveLanguage &DirLang,
+                                    raw_ostream &OS) {
 
   GenerateDirectiveClauseSets(DirLang, OS);
 
@@ -772,8 +781,8 @@ void EmitDirectivesFlangImpl(const DirectiveLanguage &DirLang,
   GenerateFlangClausesParser(DirLang, OS);
 }
 
-void GenerateClauseClassMacro(const DirectiveLanguage &DirLang,
-                              raw_ostream &OS) {
+static void GenerateClauseClassMacro(const DirectiveLanguage &DirLang,
+                                     raw_ostream &OS) {
   // Generate macros style information for legacy code in clang
   IfDefScope Scope("GEN_CLANG_CLAUSE_CLASS", OS);
 
@@ -870,7 +879,7 @@ void EmitDirectivesBasicImpl(const DirectiveLanguage &DirLang,
 
 // Generate the implemenation section for the enumeration in the directive
 // language.
-void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
+static void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
   const auto DirLang = DirectiveLanguage{Records};
   if (DirLang.HasValidityErrors())
     return;
@@ -882,4 +891,10 @@ void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) {
   EmitDirectivesBasicImpl(DirLang, OS);
 }
 
-} // namespace llvm
+static TableGen::Emitter::Opt
+    X("gen-directive-decl", EmitDirectivesDecl,
+      "Generate directive related declaration code (header file)");
+
+static TableGen::Emitter::Opt
+    Y("gen-directive-impl", EmitDirectivesImpl,
+      "Generate directive related implementation code");
diff --git a/llvm/utils/TableGen/DisassemblerEmitter.cpp b/llvm/utils/TableGen/DisassemblerEmitter.cpp
index dfa4b30ee569..92f3721507e5 100644
--- a/llvm/utils/TableGen/DisassemblerEmitter.cpp
+++ b/llvm/utils/TableGen/DisassemblerEmitter.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenTarget.h"
+#include "TableGenBackends.h"
 #include "WebAssemblyDisassemblerEmitter.h"
 #include "X86DisassemblerTables.h"
 #include "X86RecognizableInstr.h"
@@ -93,12 +94,7 @@ using namespace llvm::X86Disassembler;
 /// X86RecognizableInstr.cpp contains the implementation for a single
 ///   instruction.
 
-namespace llvm {
-
-extern void EmitDecoder(RecordKeeper &RK, raw_ostream &OS,
-                        const std::string &PredicateNamespace);
-
-void EmitDisassembler(RecordKeeper &Records, raw_ostream &OS) {
+static void EmitDisassembler(RecordKeeper &Records, raw_ostream &OS) {
   CodeGenTarget Target(Records);
   emitSourceFileHeader(" * " + Target.getName().str() + " Disassembler", OS);
 
@@ -135,4 +131,5 @@ void EmitDisassembler(RecordKeeper &Records, raw_ostream &OS) {
   EmitDecoder(Records, OS, PredicateNamespace);
 }
 
-} // end namespace llvm
+static TableGen::Emitter::Opt X("gen-disassembler", EmitDisassembler,
+                                "Generate disassembler");
diff --git a/llvm/utils/TableGen/ExegesisEmitter.cpp b/llvm/utils/TableGen/ExegesisEmitter.cpp
index bc8ccdac557b..736f1220be14 100644
--- a/llvm/utils/TableGen/ExegesisEmitter.cpp
+++ b/llvm/utils/TableGen/ExegesisEmitter.cpp
@@ -202,10 +202,5 @@ void ExegesisEmitter::run(raw_ostream &OS) const {
 
 } // end anonymous namespace
 
-namespace llvm {
-
-void EmitExegesis(RecordKeeper &RK, raw_ostream &OS) {
-  ExegesisEmitter(RK).run(OS);
-}
-
-} // end namespace llvm
+static TableGen::Emitter::OptClass<ExegesisEmitter>
+    X("gen-exegesis", "Generate llvm-exegesis tables");
diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp
index 0a88f67be168..3f3a63de0c0c 100644
--- a/llvm/utils/TableGen/FastISelEmitter.cpp
+++ b/llvm/utils/TableGen/FastISelEmitter.cpp
@@ -18,6 +18,9 @@
 
 #include "CodeGenDAGPatterns.h"
 #include "CodeGenInstruction.h"
+#include "CodeGenRegisters.h"
+#include "CodeGenTarget.h"
+#include "InfoByHwMode.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/TableGen/Error.h"
@@ -854,9 +857,7 @@ void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
   // TODO: SignaturesWithConstantForms should be empty here.
 }
 
-namespace llvm {
-
-void EmitFastISel(RecordKeeper &RK, raw_ostream &OS) {
+static void EmitFastISel(RecordKeeper &RK, raw_ostream &OS) {
   CodeGenDAGPatterns CGP(RK);
   const CodeGenTarget &Target = CGP.getTargetInfo();
   emitSourceFileHeader("\"Fast\" Instruction Selector for the " +
@@ -872,4 +873,5 @@ void EmitFastISel(RecordKeeper &RK, raw_ostream &OS) {
   F.printFunctionDefinitions(OS);
 }
 
-} // End llvm namespace
+static TableGen::Emitter::Opt X("gen-fast-isel", EmitFastISel,
+                                "Generate a \"fast\" instruction selector");
diff --git a/llvm/utils/TableGen/GICombinerEmitter.cpp b/llvm/utils/TableGen/GICombinerEmitter.cpp
index 2ae313081a6f..ec26024b6518 100644
--- a/llvm/utils/TableGen/GICombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GICombinerEmitter.cpp
@@ -14,7 +14,11 @@
 #include "CodeGenTarget.h"
 #include "GlobalISel/CodeExpander.h"
 #include "GlobalISel/CodeExpansions.h"
+#include "GlobalISel/CombinerUtils.h"
 #include "GlobalISel/GIMatchDag.h"
+#include "GlobalISel/GIMatchDagEdge.h"
+#include "GlobalISel/GIMatchDagInstr.h"
+#include "GlobalISel/GIMatchDagOperands.h"
 #include "GlobalISel/GIMatchDagPredicate.h"
 #include "GlobalISel/GIMatchTree.h"
 #include "llvm/ADT/SmallSet.h"
@@ -24,6 +28,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/StringMatcher.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cstdint>
@@ -38,14 +43,14 @@ STATISTIC(NumPatternTotalStatistic, "Total number of patterns");
 
 cl::OptionCategory
     GICombinerEmitterCat("Options for -gen-global-isel-combiner");
-static cl::list<std::string>
+cl::list<std::string>
     SelectedCombiners("combiners", cl::desc("Emit the specified combiners"),
                       cl::cat(GICombinerEmitterCat), cl::CommaSeparated);
 static cl::opt<bool> ShowExpansions(
     "gicombiner-show-expansions",
     cl::desc("Use C++ comments to indicate occurence of code expansion"),
     cl::cat(GICombinerEmitterCat));
-static cl::opt<bool> StopAfterParse(
+cl::opt<bool> StopAfterParse(
     "gicombiner-stop-after-parse",
     cl::desc("Stop processing after parsing rules and dump state"),
     cl::cat(GICombinerEmitterCat));
@@ -277,55 +282,6 @@ public:
   }
 };
 
-/// A convenience function to check that an Init refers to a specific def. This
-/// is primarily useful for testing for defs and similar in DagInit's since
-/// DagInit's support any type inside them.
-static bool isSpecificDef(const Init &N, StringRef Def) {
-  if (const DefInit *OpI = dyn_cast<DefInit>(&N))
-    if (OpI->getDef()->getName() == Def)
-      return true;
-  return false;
-}
-
-/// A convenience function to check that an Init refers to a def that is a
-/// subclass of the given class and coerce it to a def if it is. This is
-/// primarily useful for testing for subclasses of GIMatchKind and similar in
-/// DagInit's since DagInit's support any type inside them.
-static Record *getDefOfSubClass(const Init &N, StringRef Cls) {
-  if (const DefInit *OpI = dyn_cast<DefInit>(&N))
-    if (OpI->getDef()->isSubClassOf(Cls))
-      return OpI->getDef();
-  return nullptr;
-}
-
-/// A convenience function to check that an Init refers to a dag whose operator
-/// is a specific def and coerce it to a dag if it is. This is primarily useful
-/// for testing for subclasses of GIMatchKind and similar in DagInit's since
-/// DagInit's support any type inside them.
-static const DagInit *getDagWithSpecificOperator(const Init &N,
-                                                 StringRef Name) {
-  if (const DagInit *I = dyn_cast<DagInit>(&N))
-    if (I->getNumArgs() > 0)
-      if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator()))
-        if (OpI->getDef()->getName() == Name)
-          return I;
-  return nullptr;
-}
-
-/// A convenience function to check that an Init refers to a dag whose operator
-/// is a def that is a subclass of the given class and coerce it to a dag if it
-/// is. This is primarily useful for testing for subclasses of GIMatchKind and
-/// similar in DagInit's since DagInit's support any type inside them.
-static const DagInit *getDagWithOperatorOfSubClass(const Init &N,
-                                                   StringRef Cls) {
-  if (const DagInit *I = dyn_cast<DagInit>(&N))
-    if (I->getNumArgs() > 0)
-      if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator()))
-        if (OpI->getDef()->isSubClassOf(Cls))
-          return I;
-  return nullptr;
-}
-
 StringRef makeNameForAnonInstr(CombineRule &Rule) {
   return insertStrTab(to_string(
       format("__anon%" PRIu64 "_%u", Rule.getID(), Rule.allocUID())));
@@ -1062,8 +1018,14 @@ void GICombinerEmitter::run(raw_ostream &OS) {
 
 //===----------------------------------------------------------------------===//
 
-namespace llvm {
-void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS) {
+static void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS) {
+  PrintWarning(
+      "'-gen-global-isel-combiner' is deprecated and will be removed soon; "
+      "please use '-gen-global-isel-combiner-match-table' instead");
+  PrintNote(
+      "See "
+      "https://discourse.llvm.org/t/rfc-matchtable-based-globalisel-combiners");
+
   CodeGenTarget Target(RK);
   emitSourceFileHeader("Global Combiner", OS);
 
@@ -1078,4 +1040,5 @@ void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS) {
   NumPatternTotalStatistic = NumPatternTotal;
 }
 
-} // namespace llvm
+static TableGen::Emitter::Opt X("gen-global-isel-combiner", EmitGICombiner,
+                                "Generate GlobalISel combiner");
diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.h b/llvm/utils/TableGen/GlobalISel/CodeExpander.h
index 1291eb1ad940..0b1e6ceab52c 100644
--- a/llvm/utils/TableGen/GlobalISel/CodeExpander.h
+++ b/llvm/utils/TableGen/GlobalISel/CodeExpander.h
@@ -1,4 +1,4 @@
-//===- CodeExpander.h - Expand variables in a string ----------------------===//
+//===- CodeExpander.h - Expand variables in a string ------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/utils/TableGen/GlobalISel/CombinerUtils.h b/llvm/utils/TableGen/GlobalISel/CombinerUtils.h
new file mode 100644
index 000000000000..394c43e3fa83
--- /dev/null
+++ b/llvm/utils/TableGen/GlobalISel/CombinerUtils.h
@@ -0,0 +1,72 @@
+//===- CombinerUtils.h ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Utility functions used by both Combiner backends.
+/// TODO: Can remove when MatchDAG-based backend is removed.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UTILS_TABLEGEN_COMBINERUTILS_H
+#define LLVM_UTILS_TABLEGEN_COMBINERUTILS_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/TableGen/Record.h"
+
+namespace llvm {
+
+/// A convenience function to check that an Init refers to a specific def. This
+/// is primarily useful for testing for defs and similar in DagInit's since
+/// DagInit's support any type inside them.
+inline bool isSpecificDef(const Init &N, StringRef Def) {
+  if (const DefInit *OpI = dyn_cast<DefInit>(&N))
+    if (OpI->getDef()->getName() == Def)
+      return true;
+  return false;
+}
+
+/// A convenience function to check that an Init refers to a def that is a
+/// subclass of the given class and coerce it to a def if it is. This is
+/// primarily useful for testing for subclasses of GIMatchKind and similar in
+/// DagInit's since DagInit's support any type inside them.
+inline Record *getDefOfSubClass(const Init &N, StringRef Cls) {
+  if (const DefInit *OpI = dyn_cast<DefInit>(&N))
+    if (OpI->getDef()->isSubClassOf(Cls))
+      return OpI->getDef();
+  return nullptr;
+}
+
+/// A convenience function to check that an Init refers to a dag whose operator
+/// is a specific def and coerce it to a dag if it is. This is primarily useful
+/// for testing for subclasses of GIMatchKind and similar in DagInit's since
+/// DagInit's support any type inside them.
+inline const DagInit *getDagWithSpecificOperator(const Init &N,
+                                                 StringRef Name) {
+  if (const DagInit *I = dyn_cast<DagInit>(&N))
+    if (I->getNumArgs() > 0)
+      if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator()))
+        if (OpI->getDef()->getName() == Name)
+          return I;
+  return nullptr;
+}
+
+/// A convenience function to check that an Init refers to a dag whose operator
+/// is a def that is a subclass of the given class and coerce it to a dag if it
+/// is. This is primarily useful for testing for subclasses of GIMatchKind and
+/// similar in DagInit's since DagInit's support any type inside them.
+inline const DagInit *getDagWithOperatorOfSubClass(const Init &N,
+                                                   StringRef Cls) {
+  if (const DagInit *I = dyn_cast<DagInit>(&N))
+    if (I->getNumArgs() > 0)
+      if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator()))
+        if (OpI->getDef()->isSubClassOf(Cls))
+          return I;
+  return nullptr;
+}
+} // namespace llvm
+
+#endif
diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDag.h b/llvm/utils/TableGen/GlobalISel/GIMatchDag.h
index 4c3c610aff74..c566dd73f709 100644
--- a/llvm/utils/TableGen/GlobalISel/GIMatchDag.h
+++ b/llvm/utils/TableGen/GlobalISel/GIMatchDag.h
@@ -1,4 +1,4 @@
-//===- GIMatchDag.h - Represent a DAG to be matched -----------------------===//
+//===- GIMatchDag.h - Represent a DAG to be matched -------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h
index 8e845ff0a51e..e76ef1b4a3aa 100644
--- a/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h
+++ b/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h
@@ -1,4 +1,4 @@
-//===- GIMatchDagEdge.h - Represent a shared operand list for nodes -------===//
+//===- GIMatchDagEdge.h - Represent node shared operand lists ---*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h
index 5e60448b30c1..d2c746dda9e9 100644
--- a/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h
+++ b/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h
@@ -1,4 +1,4 @@
-//===- GIMatchDagInstr.h - Represent a instruction to be matched ----------===//
+//===- GIMatchDagInstr.h - Represent instruction to be matched --*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h
index c2d30574231d..ae7190cb7296 100644
--- a/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h
+++ b/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h
@@ -1,4 +1,4 @@
-//===- GIMatchDagOperands.h - Represent a shared operand list for nodes ---===//
+//===- GIMatchDagOperands.h - Represent operand lists for nodes -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h
index 96fef21b7627..952cbdb24f54 100644
--- a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h
+++ b/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h
@@ -1,4 +1,4 @@
-//===- GIMatchDagPredicate - Represent a predicate to check ---------------===//
+//===- GIMatchDagPredicate - Represent a predicate to check -----*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp
index d98884493e84..23697fd9e2e2 100644
--- a/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp
+++ b/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp
@@ -89,20 +89,20 @@ GIMatchTreeBuilderLeafInfo::GIMatchTreeBuilderLeafInfo(
       TraversableEdges(MatchDag.getNumEdges()),
       TestablePredicates(MatchDag.getNumPredicates()) {
   // Number all the predicates in this DAG
-  for (auto &P : enumerate(MatchDag.predicates())) {
-    PredicateIDs.insert(std::make_pair(P.value(), P.index()));
+  for (const auto &[Idx, P] : enumerate(MatchDag.predicates())) {
+    PredicateIDs.insert(std::make_pair(P, Idx));
   }
 
   // Number all the predicate dependencies in this DAG and set up a bitvector
   // for each predicate indicating the unsatisfied dependencies.
-  for (auto &Dep : enumerate(MatchDag.predicate_edges())) {
-    PredicateDepIDs.insert(std::make_pair(Dep.value(), Dep.index()));
+  for (const auto &[Idx, Dep] : enumerate(MatchDag.predicate_edges())) {
+    PredicateDepIDs.insert(std::make_pair(Dep, Idx));
   }
   UnsatisfiedPredDepsForPred.resize(MatchDag.getNumPredicates(),
                                     BitVector(PredicateDepIDs.size()));
-  for (auto &Dep : enumerate(MatchDag.predicate_edges())) {
-    unsigned ID = PredicateIDs.lookup(Dep.value()->getPredicate());
-    UnsatisfiedPredDepsForPred[ID].set(Dep.index());
+  for (const auto &[Idx, Dep] : enumerate(MatchDag.predicate_edges())) {
+    unsigned ID = PredicateIDs.lookup(Dep->getPredicate());
+    UnsatisfiedPredDepsForPred[ID].set(Idx);
   }
 }
 
@@ -134,10 +134,10 @@ void GIMatchTreeBuilderLeafInfo::declareInstr(const GIMatchDagInstr *Instr, unsi
   // Mark the dependencies that are now satisfied as a result of this
   // instruction and mark any predicates whose dependencies are fully
   // satisfied.
-  for (auto &Dep : enumerate(MatchDag.predicate_edges())) {
+  for (const auto &Dep : enumerate(MatchDag.predicate_edges())) {
     if (Dep.value()->getRequiredMI() == Instr &&
         Dep.value()->getRequiredMO() == nullptr) {
-      for (auto &DepsFor : enumerate(UnsatisfiedPredDepsForPred)) {
+      for (const auto &DepsFor : enumerate(UnsatisfiedPredDepsForPred)) {
         DepsFor.value().reset(Dep.index());
         if (DepsFor.value().none())
           TestablePredicates.set(DepsFor.index());
@@ -157,10 +157,9 @@ void GIMatchTreeBuilderLeafInfo::declareOperand(unsigned InstrID,
   // When an operand becomes reachable, we potentially activate some traversals.
   // Record the edges that can now be followed as a result of this
   // instruction.
-  for (auto &E : enumerate(MatchDag.edges())) {
-    if (E.value()->getFromMI() == Instr &&
-        E.value()->getFromMO()->getIdx() == OpIdx) {
-      TraversableEdges.set(E.index());
+  for (const auto &[Idx, E] : enumerate(MatchDag.edges())) {
+    if (E->getFromMI() == Instr && E->getFromMO()->getIdx() == OpIdx) {
+      TraversableEdges.set(Idx);
     }
   }
 
@@ -168,10 +167,10 @@ void GIMatchTreeBuilderLeafInfo::declareOperand(unsigned InstrID,
   // Clear the dependencies that are now satisfied as a result of this
   // operand and activate any predicates whose dependencies are fully
   // satisfied.
-  for (auto &Dep : enumerate(MatchDag.predicate_edges())) {
+  for (const auto &Dep : enumerate(MatchDag.predicate_edges())) {
     if (Dep.value()->getRequiredMI() == Instr && Dep.value()->getRequiredMO() &&
         Dep.value()->getRequiredMO()->getIdx() == OpIdx) {
-      for (auto &DepsFor : enumerate(UnsatisfiedPredDepsForPred)) {
+      for (const auto &DepsFor : enumerate(UnsatisfiedPredDepsForPred)) {
         DepsFor.value().reset(Dep.index());
         if (DepsFor.value().none())
           TestablePredicates.set(DepsFor.index());
@@ -231,25 +230,6 @@ void GIMatchTreeBuilder::runStep() {
                dbgs() << "\n");
 #endif // ifndef NDEBUG
 
-  // Check for unreachable rules. Rules are unreachable if they are preceeded by
-  // a fully tested rule.
-  // Note: This is only true for the current algorithm, if we allow the
-  //       algorithm to compare equally valid rules then they will become
-  //       reachable.
-  {
-    auto FullyTestedLeafI = Leaves.end();
-    for (auto LeafI = Leaves.begin(), LeafE = Leaves.end();
-         LeafI != LeafE; ++LeafI) {
-      if (LeafI->isFullyTraversed() && LeafI->isFullyTested())
-        FullyTestedLeafI = LeafI;
-      else if (FullyTestedLeafI != Leaves.end()) {
-        PrintError("Leaf " + LeafI->getName() + " is unreachable");
-        PrintNote("Leaf " + FullyTestedLeafI->getName() +
-                  " will have already matched");
-      }
-    }
-  }
-
   LLVM_DEBUG(dbgs() << "  Eliminating redundant partitioners:\n");
   filterRedundantPartitioners();
   LLVM_DEBUG(dbgs() << "  Partitioners remaining:\n");
@@ -339,9 +319,9 @@ void GIMatchTreeBuilder::runStep() {
          "Must always partition into at least one partition");
 
   TreeNode->setNumChildren(Partitioner->getNumPartitions());
-  for (auto &C : enumerate(TreeNode->children())) {
-    SubtreeBuilders.emplace_back(&C.value(), NextInstrID);
-    Partitioner->applyForPartition(C.index(), *this, SubtreeBuilders.back());
+  for (const auto &[Idx, Child] : enumerate(TreeNode->children())) {
+    SubtreeBuilders.emplace_back(&Child, NextInstrID);
+    Partitioner->applyForPartition(Idx, *this, SubtreeBuilders.back());
   }
 
   TreeNode->setPartitioner(std::move(Partitioner));
@@ -536,22 +516,22 @@ void GIMatchTreeOpcodePartitioner::applyForPartition(
 
   BitVector PossibleLeaves = getPossibleLeavesForPartition(PartitionIdx);
   // Consume any predicates we handled.
-  for (auto &EnumeratedLeaf : enumerate(Builder.getPossibleLeaves())) {
-    if (!PossibleLeaves[EnumeratedLeaf.index()])
+  for (const auto &[Index, EnumeratedLeaf] :
+       enumerate(Builder.getPossibleLeaves())) {
+    if (!PossibleLeaves[Index])
       continue;
 
-    auto &Leaf = EnumeratedLeaf.value();
-    const auto &TestedPredicatesForLeaf =
-        TestedPredicates[EnumeratedLeaf.index()];
+    const auto &TestedPredicatesForLeaf = TestedPredicates[Index];
 
     for (unsigned PredIdx : TestedPredicatesForLeaf.set_bits()) {
-      LLVM_DEBUG(dbgs() << "    " << Leaf.getName() << " tested predicate #"
-                        << PredIdx << " of " << TestedPredicatesForLeaf.size()
-                        << " " << *Leaf.getPredicate(PredIdx) << "\n");
-      Leaf.RemainingPredicates.reset(PredIdx);
-      Leaf.TestablePredicates.reset(PredIdx);
+      LLVM_DEBUG(dbgs() << "    " << EnumeratedLeaf.getName()
+                        << " tested predicate #" << PredIdx << " of "
+                        << TestedPredicatesForLeaf.size() << " "
+                        << *EnumeratedLeaf.getPredicate(PredIdx) << "\n");
+      EnumeratedLeaf.RemainingPredicates.reset(PredIdx);
+      EnumeratedLeaf.TestablePredicates.reset(PredIdx);
     }
-    SubBuilder.addLeaf(Leaf);
+    SubBuilder.addLeaf(EnumeratedLeaf);
   }
 
   // Nothing to do, we don't know anything about this instruction as a result
@@ -571,11 +551,11 @@ void GIMatchTreeOpcodePartitioner::applyForPartition(
     if (!InstrInfo)
       continue;
     const GIMatchDagInstr *Instr = InstrInfo->getInstrNode();
-    for (auto &E : enumerate(Leaf.getMatchDag().edges())) {
-      if (E.value()->getFromMI() == Instr &&
-          E.value()->getFromMO()->getIdx() < CGI->Operands.size()) {
-        ReferencedOperands.resize(E.value()->getFromMO()->getIdx() + 1);
-        ReferencedOperands.set(E.value()->getFromMO()->getIdx());
+    for (const auto &E : Leaf.getMatchDag().edges()) {
+      if (E->getFromMI() == Instr &&
+          E->getFromMO()->getIdx() < CGI->Operands.size()) {
+        ReferencedOperands.resize(E->getFromMO()->getIdx() + 1);
+        ReferencedOperands.set(E->getFromMO()->getIdx());
       }
     }
   }
@@ -682,12 +662,7 @@ void GIMatchTreeVRegDefPartitioner::repartition(
       WantsEdge = true;
     }
 
-    bool isNotReg = false;
-    if (!WantsEdge && isNotReg) {
-      // If this leaf doesn't have an edge and we _don't_ want a register,
-      // then add it to partition 0.
-      addToPartition(false, Leaf.index());
-    } else if (!WantsEdge) {
+    if (!WantsEdge) {
       // If this leaf doesn't have an edge and we don't know what we want,
       // then add it to partition 0 and 1.
       addToPartition(false, Leaf.index());
@@ -715,16 +690,16 @@ void GIMatchTreeVRegDefPartitioner::applyForPartition(
 
   std::vector<BitVector> TraversedEdgesByNewLeaves;
   // Consume any edges we handled.
-  for (auto &EnumeratedLeaf : enumerate(Builder.getPossibleLeaves())) {
-    if (!PossibleLeaves[EnumeratedLeaf.index()])
+  for (const auto &[Index, EnumeratedLeaf] :
+       enumerate(Builder.getPossibleLeaves())) {
+    if (!PossibleLeaves[Index])
       continue;
 
-    auto &Leaf = EnumeratedLeaf.value();
-    const auto &TraversedEdgesForLeaf = TraversedEdges[EnumeratedLeaf.index()];
+    const auto &TraversedEdgesForLeaf = TraversedEdges[Index];
     TraversedEdgesByNewLeaves.push_back(TraversedEdgesForLeaf);
-    Leaf.RemainingEdges.reset(TraversedEdgesForLeaf);
-    Leaf.TraversableEdges.reset(TraversedEdgesForLeaf);
-    SubBuilder.addLeaf(Leaf);
+    EnumeratedLeaf.RemainingEdges.reset(TraversedEdgesForLeaf);
+    EnumeratedLeaf.TraversableEdges.reset(TraversedEdgesForLeaf);
+    SubBuilder.addLeaf(EnumeratedLeaf);
   }
 
   // Nothing to do. The only thing we know is that it isn't a vreg-def.
@@ -734,7 +709,7 @@ void GIMatchTreeVRegDefPartitioner::applyForPartition(
   NewInstrID = SubBuilder.allocInstrID();
 
   GIMatchTreeBuilder::LeafVec &NewLeaves = SubBuilder.getPossibleLeaves();
-  for (const auto I : zip(NewLeaves, TraversedEdgesByNewLeaves)) {
+  for (const auto &I : zip(NewLeaves, TraversedEdgesByNewLeaves)) {
     auto &Leaf = std::get<0>(I);
     auto &TraversedEdgesForLeaf = std::get<1>(I);
     GIMatchTreeInstrInfo *InstrInfo = Leaf.getInstrInfo(InstrID);
diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchTree.h b/llvm/utils/TableGen/GlobalISel/GIMatchTree.h
index 0ce4060fe7b4..c65423ddacdb 100644
--- a/llvm/utils/TableGen/GlobalISel/GIMatchTree.h
+++ b/llvm/utils/TableGen/GlobalISel/GIMatchTree.h
@@ -390,7 +390,7 @@ protected:
   /// The leaves that the resulting decision tree will distinguish.
   LeafVec Leaves;
   /// The tree node being constructed.
-  GIMatchTree *TreeNode;
+  GIMatchTree *TreeNode = nullptr;
   /// The builders for each subtree resulting from the current decision.
   std::vector<GIMatchTreeBuilder> SubtreeBuilders;
   /// The possible partitioners we could apply right now.
diff --git a/llvm/utils/TableGen/GlobalISelCombinerMatchTableEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerMatchTableEmitter.cpp
new file mode 100644
index 000000000000..3ae66ed01b3a
--- /dev/null
+++ b/llvm/utils/TableGen/GlobalISelCombinerMatchTableEmitter.cpp
@@ -0,0 +1,1575 @@
+//===- GlobalISelCombinerMatchTableEmitter.cpp - --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Generate a combiner implementation for GlobalISel from a declarative
+/// syntax using GlobalISelMatchTable.
+///
+//===----------------------------------------------------------------------===//
+
+#include "CodeGenInstruction.h"
+#include "CodeGenTarget.h"
+#include "GlobalISel/CodeExpander.h"
+#include "GlobalISel/CodeExpansions.h"
+#include "GlobalISel/CombinerUtils.h"
+#include "GlobalISelMatchTable.h"
+#include "GlobalISelMatchTableExecutorEmitter.h"
+#include "SubtargetFeatureInfo.h"
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/StringMatcher.h"
+#include "llvm/TableGen/TableGenBackend.h"
+#include <cstdint>
+
+using namespace llvm;
+using namespace llvm::gi;
+
+#define DEBUG_TYPE "gicombiner-matchtable-emitter"
+
+extern cl::list<std::string> SelectedCombiners;
+extern cl::opt<bool> StopAfterParse;
+
+namespace {
+constexpr StringLiteral CXXApplyPrefix = "GICXXCustomAction_CombineApply";
+constexpr StringLiteral CXXPredPrefix = "GICXXPred_MI_Predicate_";
+
+std::string getIsEnabledPredicateEnumName(unsigned CombinerRuleID) {
+  return "GICXXPred_Simple_IsRule" + to_string(CombinerRuleID) + "Enabled";
+}
+
+void declareInstExpansion(CodeExpansions &CE, const InstructionMatcher &IM,
+                          StringRef Name) {
+  CE.declare(Name, "State.MIs[" + to_string(IM.getInsnVarID()) + "]");
+}
+
+void declareOperandExpansion(CodeExpansions &CE, const OperandMatcher &OM,
+                             StringRef Name) {
+  CE.declare(Name, "State.MIs[" + to_string(OM.getInsnVarID()) +
+                       "]->getOperand(" + to_string(OM.getOpIdx()) + ")");
+}
+
+//===- MatchData Handling -------------------------------------------------===//
+
+/// Represents MatchData defined by the match stage and required by the apply
+/// stage.
+///
+/// This allows the plumbing of arbitrary data from C++ predicates between the
+/// stages.
+///
+/// When this class is initially created, it only has a pattern symbol and a
+/// type. When all of the MatchDatas declarations of a given pattern have been
+/// parsed, `AssignVariables` must be called to assign storage variable names to
+/// each MatchDataInfo.
+class MatchDataInfo {
+  StringRef PatternSymbol;
+  StringRef Type;
+  std::string VarName;
+
+public:
+  static constexpr StringLiteral StructTypeName = "MatchInfosTy";
+  static constexpr StringLiteral StructName = "MatchInfos";
+
+  MatchDataInfo(StringRef PatternSymbol, StringRef Type)
+      : PatternSymbol(PatternSymbol), Type(Type.trim()) {}
+
+  StringRef getPatternSymbol() const { return PatternSymbol; };
+  StringRef getType() const { return Type; };
+
+  bool hasVariableName() const { return !VarName.empty(); }
+  void setVariableName(StringRef Name) { VarName = Name; }
+  StringRef getVariableName() const;
+
+  std::string getQualifiedVariableName() const {
+    return StructName.str() + "." + getVariableName().str();
+  }
+
+  void print(raw_ostream &OS) const;
+  void dump() const { print(dbgs()); }
+};
+
+StringRef MatchDataInfo::getVariableName() const {
+  assert(hasVariableName());
+  return VarName;
+}
+
+void MatchDataInfo::print(raw_ostream &OS) const {
+  OS << "(MatchDataInfo pattern_symbol:" << PatternSymbol << " type:'" << Type
+     << "' var_name:" << (VarName.empty() ? "<unassigned>" : VarName) << ")";
+}
+
+/// Pool of type -> variables used to emit MatchData variables declarations.
+///
+/// e.g. if the map contains "int64_t" -> ["MD0", "MD1"], then two variable
+/// declarations must be emitted: `int64_t MD0` and `int64_t MD1`.
+///
+/// This has a static lifetime and will outlive all the `MatchDataInfo` objects
+/// by design. It needs to persist after all `CombineRuleBuilder` objects died
+/// so we can emit the variable declarations.
+StringMap<std::vector<std::string>> AllMatchDataVars;
+
+// Assign variable names to all MatchDatas used by a pattern. This must be
+// called after all MatchData decls have been parsed inside a rule.
+//
+// Requires an array of MatchDataInfo so we can handle cases where a pattern
+// uses multiple instances of the same MatchData type.
+void AssignMatchDataVariables(MutableArrayRef<MatchDataInfo> Infos) {
+  static unsigned NextVarID = 0;
+
+  StringMap<unsigned> SeenTypes;
+  for (auto &I : Infos) {
+    unsigned &NumSeen = SeenTypes[I.getType()];
+    auto &ExistingVars = AllMatchDataVars[I.getType()];
+
+    if (NumSeen == ExistingVars.size())
+      ExistingVars.push_back("MDInfo" + to_string(NextVarID++));
+
+    I.setVariableName(ExistingVars[NumSeen++]);
+  }
+}
+
+//===- C++ Predicates Handling --------------------------------------------===//
+
+/// Entry into the static pool of all CXX Predicate code. This contains the
+/// fully expanded C++ code.
+///
+/// Each CXXPattern creates a new entry in the pool to store its data, even
+/// after the pattern is destroyed.
+///
+/// Note that CXXPattern trims C++ code, so the Code is already expected to be
+/// free of leading/trailing whitespace.
+struct CXXPredicateCode {
+  CXXPredicateCode(std::string Code, unsigned ID)
+      : Code(Code), ID(ID), BaseEnumName("GICombiner" + to_string(ID)) {
+    assert(StringRef(Code).trim() == Code &&
+           "Code was expected to be trimmed!");
+  }
+
+  const std::string Code;
+  const unsigned ID;
+  const std::string BaseEnumName;
+
+  bool needsUnreachable() const {
+    return !StringRef(Code).starts_with("return");
+  }
+
+  std::string getEnumNameWithPrefix(StringRef Prefix) const {
+    return Prefix.str() + BaseEnumName;
+  }
+};
+
+using CXXPredicateCodePool =
+    DenseMap<hash_code, std::unique_ptr<CXXPredicateCode>>;
+CXXPredicateCodePool AllCXXMatchCode;
+CXXPredicateCodePool AllCXXApplyCode;
+
+/// Gets an instance of `CXXPredicateCode` for \p Code, or returns an already
+/// existing one.
+const CXXPredicateCode &getOrInsert(CXXPredicateCodePool &Pool,
+                                    std::string Code) {
+  // Check if we already have an identical piece of code, if not, create an
+  // entry in the pool.
+  const auto CodeHash = hash_value(Code);
+  if (auto It = Pool.find(CodeHash); It != Pool.end())
+    return *It->second;
+
+  const auto ID = Pool.size();
+  auto OwnedData = std::make_unique<CXXPredicateCode>(std::move(Code), ID);
+  const auto &DataRef = *OwnedData;
+  Pool[CodeHash] = std::move(OwnedData);
+  return DataRef;
+}
+
+/// Sorts a `CXXPredicateCodePool` by their IDs and returns it.
+std::vector<const CXXPredicateCode *>
+getSorted(const CXXPredicateCodePool &Pool) {
+  std::vector<const CXXPredicateCode *> Out;
+  std::transform(Pool.begin(), Pool.end(), std::back_inserter(Out),
+                 [&](auto &Elt) { return Elt.second.get(); });
+  sort(Out, [](const auto *A, const auto *B) { return A->ID < B->ID; });
+  return Out;
+}
+
+//===- Pattern Base Class -------------------------------------------------===//
+
+// An abstract pattern found in a combine rule. This can be an apply or match
+// pattern.
+class Pattern {
+public:
+  enum {
+    K_AnyOpcode,
+    K_Inst,
+    K_CXX,
+  };
+
+  virtual ~Pattern() = default;
+
+  unsigned getKind() const { return Kind; }
+  const char *getKindName() const;
+
+  bool hasName() const { return !Name.empty(); }
+  StringRef getName() const { return Name; }
+
+  virtual void print(raw_ostream &OS, bool PrintName = true) const = 0;
+  void dump() const { return print(dbgs()); }
+
+protected:
+  Pattern(unsigned Kind, StringRef Name) : Kind(Kind), Name(Name.str()) {
+    assert(!Name.empty() && "unnamed pattern!");
+  }
+
+  void printImpl(raw_ostream &OS, bool PrintName,
+                 function_ref<void()> ContentPrinter) const;
+
+private:
+  unsigned Kind;
+
+  // Note: if this ever changes to a StringRef (e.g. allocated in a pool or
+  // something), CombineRuleBuilder::verify() needs to be updated as well.
+  // It currently checks that the StringRef in the PatternMap references this.
+  std::string Name;
+};
+
+const char *Pattern::getKindName() const {
+  switch (Kind) {
+  case K_AnyOpcode:
+    return "AnyOpcodePattern";
+  case K_Inst:
+    return "InstructionPattern";
+  case K_CXX:
+    return "CXXPattern";
+  }
+
+  llvm_unreachable("unknown pattern kind!");
+}
+
+void Pattern::printImpl(raw_ostream &OS, bool PrintName,
+                        function_ref<void()> ContentPrinter) const {
+  OS << "(" << getKindName() << " ";
+  if (PrintName)
+    OS << "name:" << getName() << " ";
+  ContentPrinter();
+  OS << ")";
+}
+
+//===- AnyOpcodePattern ---------------------------------------------------===//
+
+/// `wip_match_opcode` patterns.
+/// This matches one or more opcodes, and does not check any operands
+/// whatsoever.
+class AnyOpcodePattern : public Pattern {
+public:
+  AnyOpcodePattern(StringRef Name) : Pattern(K_AnyOpcode, Name) {}
+
+  static bool classof(const Pattern *P) { return P->getKind() == K_AnyOpcode; }
+
+  void addOpcode(const CodeGenInstruction *I) { Insts.push_back(I); }
+  const auto &insts() const { return Insts; }
+
+  void print(raw_ostream &OS, bool PrintName = true) const override;
+
+private:
+  SmallVector<const CodeGenInstruction *, 4> Insts;
+};
+
+void AnyOpcodePattern::print(raw_ostream &OS, bool PrintName) const {
+  printImpl(OS, PrintName, [&OS, this]() {
+    OS << "["
+       << join(map_range(Insts,
+                         [](const auto *I) { return I->TheDef->getName(); }),
+               ", ")
+       << "]";
+  });
+}
+
+//===- InstructionPattern -------------------------------------------------===//
+
+/// Matches an instruction, e.g. `G_ADD $x, $y, $z`.
+///
+/// This pattern is simply CodeGenInstruction + a list of operands.
+class InstructionPattern : public Pattern {
+public:
+  struct Operand {
+    std::string Name;
+    bool IsDef = false;
+  };
+
+  InstructionPattern(const CodeGenInstruction &I, StringRef Name)
+      : Pattern(K_Inst, Name), I(I) {}
+
+  static bool classof(const Pattern *P) { return P->getKind() == K_Inst; }
+
+  const auto &operands() const { return Operands; }
+  void addOperand(StringRef Name);
+  unsigned getNumDefs() const { return I.Operands.NumDefs; }
+
+  const CodeGenInstruction &getInst() const { return I; }
+  StringRef getInstName() const { return I.TheDef->getName(); }
+
+  void reportUnreachable(ArrayRef<SMLoc> Locs) const;
+  bool checkSemantics(ArrayRef<SMLoc> Loc) const;
+
+  void print(raw_ostream &OS, bool PrintName = true) const override;
+
+private:
+  const CodeGenInstruction &I;
+  SmallVector<Operand, 4> Operands;
+};
+
+void InstructionPattern::addOperand(StringRef Name) {
+  const bool IsDef = Operands.size() < getNumDefs();
+  Operands.emplace_back(Operand{Name.str(), IsDef});
+}
+
+void InstructionPattern::reportUnreachable(ArrayRef<SMLoc> Locs) const {
+  PrintError(Locs, "Instruction pattern '" + getName() +
+                       "' is unreachable from the pattern root!");
+}
+
+bool InstructionPattern::checkSemantics(ArrayRef<SMLoc> Loc) const {
+  unsigned NumExpectedOperands = I.Operands.size();
+  if (NumExpectedOperands != Operands.size()) {
+
+    PrintError(Loc, "'" + getInstName() + "' expected " +
+                        Twine(NumExpectedOperands) + " operands, got " +
+                        Twine(Operands.size()));
+    return false;
+  }
+  return true;
+}
+
+void InstructionPattern::print(raw_ostream &OS, bool PrintName) const {
+  printImpl(OS, PrintName, [&OS, this]() {
+    OS << "inst:" << I.TheDef->getName() << " operands:["
+       << join(map_range(Operands,
+                         [](const auto &O) {
+                           return (O.IsDef ? "<def>" : "") + O.Name;
+                         }),
+               ", ")
+       << "]";
+  });
+}
+
+//===- CXXPattern ---------------------------------------------------------===//
+
+/// Raw C++ code which may need some expansions.
+///
+///   e.g. [{ return isFooBux(${src}.getReg()); }]
+///
+/// For the expanded code, \see CXXPredicateCode. CXXPredicateCode objects are
+/// created through `expandCode`.
+///
+/// \see CodeExpander and \see CodeExpansions for more information on code
+/// expansions.
+///
+/// This object has two purposes:
+///   - Represent C++ code as a pattern entry.
+///   - Be a factory for expanded C++ code.
+///     - It's immutable and only holds the raw code so we can expand the same
+///       CXX pattern multiple times if we need to.
+///
+/// Note that the code is always trimmed in the constructor, so leading and
+/// trailing whitespaces are removed. This removes bloat in the output, avoids
+/// formatting issues, but also allows us to check things like
+/// `.startswith("return")` trivially without worrying about spaces.
+class CXXPattern : public Pattern {
+public:
+  CXXPattern(const StringInit &Code, StringRef Name, bool IsApply)
+      : CXXPattern(Code.getAsUnquotedString(), Name, IsApply) {}
+
+  CXXPattern(StringRef Code, StringRef Name, bool IsApply)
+      : Pattern(K_CXX, Name), IsApply(IsApply), RawCode(Code.trim().str()) {}
+
+  static bool classof(const Pattern *P) { return P->getKind() == K_CXX; }
+
+  bool isApply() const { return IsApply; }
+  StringRef getRawCode() const { return RawCode; }
+
+  /// Expands raw code, replacing things such as `${foo}` with their
+  /// substitution in \p CE.
+  ///
+  /// \param CE     Map of Code Expansions
+  /// \param Locs   SMLocs for the Code Expander, in case it needs to emit
+  ///               diagnostics.
+  /// \return A CXXPredicateCode object that contains the expanded code. Note
+  /// that this may or may not insert a new object. All CXXPredicateCode objects
+  /// are held in a set to avoid emitting duplicate C++ code.
+  const CXXPredicateCode &expandCode(const CodeExpansions &CE,
+                                     ArrayRef<SMLoc> Locs) const;
+
+  void print(raw_ostream &OS, bool PrintName = true) const override;
+
+private:
+  bool IsApply;
+  std::string RawCode;
+};
+
+const CXXPredicateCode &CXXPattern::expandCode(const CodeExpansions &CE,
+                                               ArrayRef<SMLoc> Locs) const {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpander Expander(RawCode, CE, Locs, /*ShowExpansions*/ false);
+  Expander.emit(OS);
+  return getOrInsert(IsApply ? AllCXXApplyCode : AllCXXMatchCode,
+                     std::move(Result));
+}
+
+void CXXPattern::print(raw_ostream &OS, bool PrintName) const {
+  printImpl(OS, PrintName, [&OS, this] {
+    OS << (IsApply ? "apply" : "match") << " code:\"";
+    printEscapedString(getRawCode(), OS);
+    OS << "\"";
+  });
+}
+
+//===- CombineRuleBuilder -------------------------------------------------===//
+
+/// Helper for CombineRuleBuilder.
+///
+/// Represents information about an operand.
+/// Operands with no MatchPat are considered live-in to the pattern.
+struct OperandTableEntry {
+  // The matcher pattern that defines this operand.
+  // null for live-ins.
+  InstructionPattern *MatchPat = nullptr;
+  // The apply pattern that (re)defines this operand.
+  // This can only be non-null if MatchPat is.
+  InstructionPattern *ApplyPat = nullptr;
+
+  bool isLiveIn() const { return !MatchPat; }
+};
+
+/// Parses combine rule and builds a small intermediate representation to tie
+/// patterns together and emit RuleMatchers to match them. This may emit more
+/// than one RuleMatcher, e.g. for `wip_match_opcode`.
+///
+/// Memory management for `Pattern` objects is done through `std::unique_ptr`.
+/// In most cases, there are two stages to a pattern's lifetime:
+///   - Creation in a `parse` function
+///     - The unique_ptr is stored in a variable, and may be destroyed if the
+///       pattern is found to be semantically invalid.
+///   - Ownership transfer into a `PatternMap`
+///     - Once a pattern is moved into either the map of Match or Apply
+///       patterns, it is known to be valid and it never moves back.
+class CombineRuleBuilder {
+public:
+  using PatternMap = MapVector<StringRef, std::unique_ptr<Pattern>>;
+
+  CombineRuleBuilder(const CodeGenTarget &CGT,
+                     SubtargetFeatureInfoMap &SubtargetFeatures,
+                     Record &RuleDef, unsigned ID,
+                     std::vector<RuleMatcher> &OutRMs)
+      : CGT(CGT), SubtargetFeatures(SubtargetFeatures), RuleDef(RuleDef),
+        RuleID(ID), OutRMs(OutRMs) {}
+
+  /// Parses all fields in the RuleDef record.
+  bool parseAll();
+
+  /// Emits all RuleMatchers into the vector of RuleMatchers passed in the
+  /// constructor.
+  bool emitRuleMatchers();
+
+  void print(raw_ostream &OS) const;
+  void dump() const { print(dbgs()); }
+
+  /// Debug-only verification of invariants.
+  void verify() const;
+
+private:
+  void PrintError(Twine Msg) const { ::PrintError(RuleDef.getLoc(), Msg); }
+
+  /// Adds the expansions from \see MatchDatas to \p CE.
+  void declareAllMatchDatasExpansions(CodeExpansions &CE) const;
+
+  /// Adds \p P to \p IM, expanding its code using \p CE.
+  void addCXXPredicate(InstructionMatcher &IM, const CodeExpansions &CE,
+                       const CXXPattern &P);
+
+  /// Generates a name for anonymous patterns.
+  ///
+  /// e.g. (G_ADD $x, $y, $z):$foo is a pattern named "foo", but if ":$foo" is
+  /// absent, then the pattern is anonymous and this is used to assign it a
+  /// name.
+  std::string makeAnonPatName(StringRef Prefix) const;
+  mutable unsigned AnonIDCnt = 0;
+
+  /// Creates a new RuleMatcher with some boilerplate
+  /// settings/actions/predicates, and and adds it to \p OutRMs.
+  /// \see addFeaturePredicates too.
+  ///
+  /// \param AdditionalComment Comment string to be added to the
+  ///        `DebugCommentAction`.
+  RuleMatcher &addRuleMatcher(Twine AdditionalComment = "");
+  bool addFeaturePredicates(RuleMatcher &M);
+
+  bool findRoots();
+  bool buildOperandsTable();
+
+  bool parseDefs(DagInit &Def);
+  bool parseMatch(DagInit &Match);
+  bool parseApply(DagInit &Apply);
+
+  std::unique_ptr<Pattern> parseInstructionMatcher(const Init &Arg,
+                                                   StringRef PatName);
+  std::unique_ptr<Pattern> parseWipMatchOpcodeMatcher(const Init &Arg,
+                                                      StringRef PatName);
+
+  bool emitMatchPattern(CodeExpansions &CE, const InstructionPattern &IP);
+  bool emitMatchPattern(CodeExpansions &CE, const AnyOpcodePattern &AOP);
+
+  bool emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M);
+
+  // Recursively visits InstructionPattern from P to build up the
+  // RuleMatcher/InstructionMatcher. May create new InstructionMatchers as
+  // needed.
+  bool emitInstructionMatchPattern(CodeExpansions &CE, RuleMatcher &M,
+                                   InstructionMatcher &IM,
+                                   const InstructionPattern &P,
+                                   DenseSet<const Pattern *> &SeenPats);
+
+  const CodeGenTarget &CGT;
+  SubtargetFeatureInfoMap &SubtargetFeatures;
+  Record &RuleDef;
+  const unsigned RuleID;
+  std::vector<RuleMatcher> &OutRMs;
+
+  // For InstructionMatcher::addOperand
+  unsigned AllocatedTemporariesBaseID = 0;
+
+  /// The root of the pattern.
+  StringRef RootName;
+
+  /// These maps have ownership of the actual Pattern objects.
+  /// They both map a Pattern's name to the Pattern instance.
+  PatternMap MatchPats;
+  PatternMap ApplyPats;
+
+  /// Set by findRoots.
+  Pattern *MatchRoot = nullptr;
+
+  MapVector<StringRef, OperandTableEntry> OperandTable;
+  SmallVector<MatchDataInfo, 2> MatchDatas;
+};
+
+bool CombineRuleBuilder::parseAll() {
+  if (!parseDefs(*RuleDef.getValueAsDag("Defs")))
+    return false;
+  if (!parseMatch(*RuleDef.getValueAsDag("Match")))
+    return false;
+  if (!parseApply(*RuleDef.getValueAsDag("Apply")))
+    return false;
+  if (!buildOperandsTable())
+    return false;
+  if (!findRoots())
+    return false;
+  LLVM_DEBUG(verify());
+  return true;
+}
+
+bool CombineRuleBuilder::emitRuleMatchers() {
+  assert(MatchRoot);
+  CodeExpansions CE;
+  declareAllMatchDatasExpansions(CE);
+
+  switch (MatchRoot->getKind()) {
+  case Pattern::K_AnyOpcode: {
+    if (!emitMatchPattern(CE, *cast<AnyOpcodePattern>(MatchRoot)))
+      return false;
+    break;
+  }
+  case Pattern::K_Inst:
+    if (!emitMatchPattern(CE, *cast<InstructionPattern>(MatchRoot)))
+      return false;
+    break;
+  case Pattern::K_CXX:
+    PrintError("C++ code cannot be the root of a pattern!");
+    return false;
+  default:
+    llvm_unreachable("unknown pattern kind!");
+  }
+
+  return true;
+}
+
+void CombineRuleBuilder::print(raw_ostream &OS) const {
+  OS << "(CombineRule name:" << RuleDef.getName() << " id:" << RuleID
+     << " root:" << RootName << "\n";
+
+  OS << "  (MatchDatas ";
+  if (MatchDatas.empty())
+    OS << "<empty>)\n";
+  else {
+    OS << "\n";
+    for (const auto &MD : MatchDatas) {
+      OS << "    ";
+      MD.print(OS);
+      OS << "\n";
+    }
+    OS << "  )\n";
+  }
+
+  const auto DumpPats = [&](StringRef Name, const PatternMap &Pats) {
+    OS << "  (" << Name << " ";
+    if (Pats.empty()) {
+      OS << "<empty>)\n";
+      return;
+    }
+
+    OS << "\n";
+    for (const auto &[Name, Pat] : Pats) {
+      OS << "    ";
+      if (Pat.get() == MatchRoot)
+        OS << "<root>";
+      OS << Name << ":";
+      Pat->print(OS, /*PrintName=*/false);
+      OS << "\n";
+    }
+    OS << "  )\n";
+  };
+
+  DumpPats("MatchPats", MatchPats);
+  DumpPats("ApplyPats", ApplyPats);
+
+  OS << "  (OperandTable ";
+  if (OperandTable.empty())
+    OS << "<empty>)\n";
+  else {
+    OS << "\n";
+    for (const auto &[Key, Val] : OperandTable) {
+      OS << "    [" << Key;
+      if (const auto *P = Val.MatchPat)
+        OS << " match_pat:" << P->getName();
+      if (const auto *P = Val.ApplyPat)
+        OS << " apply_pat:" << P->getName();
+      if (Val.isLiveIn())
+        OS << " live-in";
+      OS << "]\n";
+    }
+    OS << "  )\n";
+  }
+
+  OS << ")\n";
+}
+
+void CombineRuleBuilder::verify() const {
+  const auto VerifyPats = [&](const PatternMap &Pats) {
+    for (const auto &[Name, Pat] : Pats) {
+      if (!Pat)
+        PrintFatalError("null pattern in pattern map!");
+
+      if (Name != Pat->getName()) {
+        Pat->dump();
+        PrintFatalError("Pattern name mismatch! Map name: " + Name +
+                        ", Pat name: " + Pat->getName());
+      }
+
+      // As an optimization, the PatternMaps don't re-allocate the PatternName
+      // string. They simply reference the std::string inside Pattern. Ensure
+      // this is the case to avoid memory issues.
+      if (Name.data() != Pat->getName().data()) {
+        dbgs() << "Map StringRef: '" << Name << "' @ "
+               << (const void *)Name.data() << "\n";
+        dbgs() << "Pat String: '" << Pat->getName() << "' @ "
+               << (const void *)Pat->getName().data() << "\n";
+        PrintFatalError("StringRef stored in the PatternMap is not referencing "
+                        "the same string as its Pattern!");
+      }
+    }
+  };
+
+  VerifyPats(MatchPats);
+  VerifyPats(ApplyPats);
+
+  for (const auto &[Name, Op] : OperandTable) {
+    if (Op.ApplyPat && !Op.MatchPat) {
+      dump();
+      PrintFatalError("Operand " + Name +
+                      " has an apply pattern, but no match pattern!");
+    }
+  }
+}
+
+bool CombineRuleBuilder::addFeaturePredicates(RuleMatcher &M) {
+  if (!RuleDef.getValue("Predicates"))
+    return true;
+
+  ListInit *Preds = RuleDef.getValueAsListInit("Predicates");
+  for (Init *I : Preds->getValues()) {
+    if (DefInit *Pred = dyn_cast<DefInit>(I)) {
+      Record *Def = Pred->getDef();
+      if (!Def->isSubClassOf("Predicate")) {
+        ::PrintError(Def->getLoc(), "Unknown 'Predicate' Type");
+        return false;
+      }
+
+      if (Def->getValueAsString("CondString").empty())
+        continue;
+
+      if (SubtargetFeatures.count(Def) == 0) {
+        SubtargetFeatures.emplace(
+            Def, SubtargetFeatureInfo(Def, SubtargetFeatures.size()));
+      }
+
+      M.addRequiredFeature(Def);
+    }
+  }
+
+  return true;
+}
+
+void CombineRuleBuilder::declareAllMatchDatasExpansions(
+    CodeExpansions &CE) const {
+  for (const auto &MD : MatchDatas)
+    CE.declare(MD.getPatternSymbol(), MD.getQualifiedVariableName());
+}
+
+void CombineRuleBuilder::addCXXPredicate(InstructionMatcher &IM,
+                                         const CodeExpansions &CE,
+                                         const CXXPattern &P) {
+  const auto &ExpandedCode = P.expandCode(CE, RuleDef.getLoc());
+  IM.addPredicate<GenericInstructionPredicateMatcher>(
+      ExpandedCode.getEnumNameWithPrefix(CXXPredPrefix));
+}
+
+std::string CombineRuleBuilder::makeAnonPatName(StringRef Prefix) const {
+  return to_string("__anon_pat_" + Prefix + "_" + to_string(RuleID) + "_" +
+                   to_string(AnonIDCnt++));
+}
+
+RuleMatcher &CombineRuleBuilder::addRuleMatcher(Twine AdditionalComment) {
+  auto &RM = OutRMs.emplace_back(RuleDef.getLoc());
+  addFeaturePredicates(RM);
+  RM.addRequiredSimplePredicate(getIsEnabledPredicateEnumName(RuleID));
+  const std::string AdditionalCommentStr = AdditionalComment.str();
+  RM.addAction<DebugCommentAction>(
+      "Combiner Rule #" + to_string(RuleID) + ": " + RuleDef.getName().str() +
+      (AdditionalCommentStr.empty() ? "" : "; " + AdditionalCommentStr));
+  return RM;
+}
+
+bool CombineRuleBuilder::findRoots() {
+  // Look by pattern name, e.g.
+  //    (G_FNEG $x, $y):$root
+  if (auto It = MatchPats.find(RootName); It != MatchPats.end()) {
+    MatchRoot = It->second.get();
+    return true;
+  }
+
+  // Look by def:
+  //    (G_FNEG $root, $y)
+  auto It = OperandTable.find(RootName);
+  if (It == OperandTable.end()) {
+    PrintError("Cannot find root '" + RootName + "' in match patterns!");
+    return false;
+  }
+
+  if (!It->second.MatchPat) {
+    PrintError("Cannot use live-in operand '" + RootName +
+               "' as match pattern root!");
+    return false;
+  }
+
+  MatchRoot = It->second.MatchPat;
+  return true;
+}
+
+bool CombineRuleBuilder::buildOperandsTable() {
+  // Walk each instruction pattern
+  for (auto &[_, P] : MatchPats) {
+    auto *IP = dyn_cast<InstructionPattern>(P.get());
+    if (!IP)
+      continue;
+    for (const auto &Operand : IP->operands()) {
+      // Create an entry, no matter if it's a use or a def.
+      auto &Entry = OperandTable[Operand.Name];
+
+      // We only need to do additional checking on defs, though.
+      if (!Operand.IsDef)
+        continue;
+
+      if (Entry.MatchPat) {
+        PrintError("Operand '" + Operand.Name +
+                   "' is defined multiple times in the 'match' patterns");
+        return false;
+      }
+      Entry.MatchPat = IP;
+    }
+  }
+
+  for (auto &[_, P] : ApplyPats) {
+    auto *IP = dyn_cast<InstructionPattern>(P.get());
+    if (!IP)
+      continue;
+    for (const auto &Operand : IP->operands()) {
+      // Create an entry, no matter if it's a use or a def.
+      auto &Entry = OperandTable[Operand.Name];
+
+      // We only need to do additional checking on defs, though.
+      if (!Operand.IsDef)
+        continue;
+
+      if (!Entry.MatchPat) {
+        PrintError("Cannot define live-in operand '" + Operand.Name +
+                   "' in the 'apply' pattern");
+        return false;
+      }
+      if (Entry.ApplyPat) {
+        PrintError("Operand '" + Operand.Name +
+                   "' is defined multiple times in the 'apply' patterns");
+        return false;
+      }
+      Entry.ApplyPat = IP;
+    }
+  }
+
+  return true;
+}
+
+bool CombineRuleBuilder::parseDefs(DagInit &Def) {
+  if (Def.getOperatorAsDef(RuleDef.getLoc())->getName() != "defs") {
+    PrintError("Expected defs operator");
+    return false;
+  }
+
+  SmallVector<StringRef> Roots;
+  for (unsigned I = 0, E = Def.getNumArgs(); I < E; ++I) {
+    if (isSpecificDef(*Def.getArg(I), "root")) {
+      Roots.emplace_back(Def.getArgNameStr(I));
+      continue;
+    }
+
+    // Subclasses of GIDefMatchData should declare that this rule needs to pass
+    // data from the match stage to the apply stage, and ensure that the
+    // generated matcher has a suitable variable for it to do so.
+    if (Record *MatchDataRec =
+            getDefOfSubClass(*Def.getArg(I), "GIDefMatchData")) {
+      MatchDatas.emplace_back(Def.getArgNameStr(I),
+                              MatchDataRec->getValueAsString("Type"));
+      continue;
+    }
+
+    // Otherwise emit an appropriate error message.
+    if (getDefOfSubClass(*Def.getArg(I), "GIDefKind"))
+      PrintError("This GIDefKind not implemented in tablegen");
+    else if (getDefOfSubClass(*Def.getArg(I), "GIDefKindWithArgs"))
+      PrintError("This GIDefKindWithArgs not implemented in tablegen");
+    else
+      PrintError("Expected a subclass of GIDefKind or a sub-dag whose "
+                 "operator is of type GIDefKindWithArgs");
+    return false;
+  }
+
+  if (Roots.size() != 1) {
+    PrintError("Combine rules must have exactly one root");
+    return false;
+  }
+
+  RootName = Roots.front();
+
+  // Assign variables to all MatchDatas.
+  AssignMatchDataVariables(MatchDatas);
+  return true;
+}
+
+bool CombineRuleBuilder::parseMatch(DagInit &Match) {
+  if (Match.getOperatorAsDef(RuleDef.getLoc())->getName() != "match") {
+    PrintError("Expected match operator");
+    return false;
+  }
+
+  if (Match.getNumArgs() == 0) {
+    PrintError("Matcher is empty");
+    return false;
+  }
+
+  // The match section consists of a list of matchers and predicates. Parse each
+  // one and add the equivalent GIMatchDag nodes, predicates, and edges.
+  bool HasOpcodeMatcher = false;
+  for (unsigned I = 0; I < Match.getNumArgs(); ++I) {
+    Init *Arg = Match.getArg(I);
+    std::string Name = Match.getArgName(I)
+                           ? Match.getArgName(I)->getValue().str()
+                           : makeAnonPatName("match");
+
+    if (MatchPats.contains(Name)) {
+      PrintError("'" + Name + "' match pattern defined more than once!");
+      return false;
+    }
+
+    if (auto Pat = parseInstructionMatcher(*Arg, Name)) {
+      MatchPats[Pat->getName()] = std::move(Pat);
+      continue;
+    }
+
+    if (auto Pat = parseWipMatchOpcodeMatcher(*Arg, Name)) {
+      if (HasOpcodeMatcher) {
+        PrintError("wip_opcode_match can only be present once");
+        return false;
+      }
+      HasOpcodeMatcher = true;
+      MatchPats[Pat->getName()] = std::move(Pat);
+      continue;
+    }
+
+    // Parse arbitrary C++ code
+    if (const auto *StringI = dyn_cast<StringInit>(Arg)) {
+      auto CXXPat =
+          std::make_unique<CXXPattern>(*StringI, Name, /*IsApply*/ false);
+      if (!CXXPat->getRawCode().contains("return ")) {
+        PrintWarning(RuleDef.getLoc(),
+                     "'match' C++ code does not seem to return!");
+      }
+      MatchPats[CXXPat->getName()] = std::move(CXXPat);
+      continue;
+    }
+
+    // TODO: don't print this on, e.g. bad operand count in inst pat
+    PrintError("Expected a subclass of GIMatchKind or a sub-dag whose "
+               "operator is either of a GIMatchKindWithArgs or Instruction");
+    PrintNote("Pattern was `" + Arg->getAsString() + "'");
+    return false;
+  }
+
+  return true;
+}
+
+bool CombineRuleBuilder::parseApply(DagInit &Apply) {
+  // Currently we only support C++ :(
+  if (Apply.getOperatorAsDef(RuleDef.getLoc())->getName() != "apply") {
+    PrintError("Expected 'apply' operator in Apply DAG");
+    return false;
+  }
+
+  if (Apply.getNumArgs() != 1) {
+    PrintError("Expected exactly 1 argument in 'apply'");
+    return false;
+  }
+
+  const StringInit *Code = dyn_cast<StringInit>(Apply.getArg(0));
+  auto Pat = std::make_unique<CXXPattern>(*Code, makeAnonPatName("apply"),
+                                          /*IsApply*/ true);
+  ApplyPats[Pat->getName()] = std::move(Pat);
+  return true;
+}
+
+std::unique_ptr<Pattern>
+CombineRuleBuilder::parseInstructionMatcher(const Init &Arg, StringRef Name) {
+  const DagInit *Matcher = getDagWithOperatorOfSubClass(Arg, "Instruction");
+  if (!Matcher)
+    return nullptr;
+
+  auto &Instr = CGT.getInstruction(Matcher->getOperatorAsDef(RuleDef.getLoc()));
+  auto Pat = std::make_unique<InstructionPattern>(Instr, Name);
+
+  for (const auto &NameInit : Matcher->getArgNames())
+    Pat->addOperand(NameInit->getAsUnquotedString());
+
+  if (!Pat->checkSemantics(RuleDef.getLoc()))
+    return nullptr;
+
+  return std::move(Pat);
+}
+
+std::unique_ptr<Pattern>
+CombineRuleBuilder::parseWipMatchOpcodeMatcher(const Init &Arg,
+                                               StringRef Name) {
+  const DagInit *Matcher = getDagWithSpecificOperator(Arg, "wip_match_opcode");
+  if (!Matcher)
+    return nullptr;
+
+  if (Matcher->getNumArgs() == 0) {
+    PrintError("Empty wip_match_opcode");
+    return nullptr;
+  }
+
+  // Each argument is an opcode that can match.
+  auto Result = std::make_unique<AnyOpcodePattern>(Name);
+  for (const auto &Arg : Matcher->getArgs()) {
+    Record *OpcodeDef = getDefOfSubClass(*Arg, "Instruction");
+    if (OpcodeDef) {
+      Result->addOpcode(&CGT.getInstruction(OpcodeDef));
+      continue;
+    }
+
+    PrintError("Arguments to wip_match_opcode must be instructions");
+    return nullptr;
+  }
+
+  return std::move(Result);
+}
+
+bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
+                                          const InstructionPattern &IP) {
+  auto &M = addRuleMatcher();
+  InstructionMatcher &IM = M.addInstructionMatcher("root");
+  declareInstExpansion(CE, IM, IP.getName());
+
+  DenseSet<const Pattern *> SeenPats;
+  if (!emitInstructionMatchPattern(CE, M, IM, IP, SeenPats))
+    return false;
+
+  // Emit remaining patterns
+  for (auto &[_, Pat] : MatchPats) {
+    if (SeenPats.contains(Pat.get()))
+      continue;
+
+    switch (Pat->getKind()) {
+    case Pattern::K_AnyOpcode:
+      PrintError("wip_match_opcode can not be used with instruction patterns!");
+      return false;
+    case Pattern::K_Inst:
+      cast<InstructionPattern>(Pat.get())->reportUnreachable(RuleDef.getLoc());
+      return false;
+    case Pattern::K_CXX: {
+      addCXXPredicate(IM, CE, *cast<CXXPattern>(Pat.get()));
+      continue;
+    }
+    default:
+      llvm_unreachable("unknown pattern kind!");
+    }
+  }
+
+  return emitApplyPatterns(CE, M);
+}
+
+bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
+                                          const AnyOpcodePattern &AOP) {
+
+  for (const CodeGenInstruction *CGI : AOP.insts()) {
+    auto &M = addRuleMatcher("wip_match_opcode alternative '" +
+                             CGI->TheDef->getName() + "'");
+
+    InstructionMatcher &IM = M.addInstructionMatcher(AOP.getName());
+    declareInstExpansion(CE, IM, AOP.getName());
+    // declareInstExpansion needs to be identical, otherwise we need to create a
+    // CodeExpansions object here instead.
+    assert(IM.getInsnVarID() == 0);
+
+    IM.addPredicate<InstructionOpcodeMatcher>(CGI);
+
+    // Emit remaining patterns.
+    for (auto &[_, Pat] : MatchPats) {
+      if (Pat.get() == &AOP)
+        continue;
+
+      switch (Pat->getKind()) {
+      case Pattern::K_AnyOpcode:
+        PrintError("wip_match_opcode can only be present once!");
+        return false;
+      case Pattern::K_Inst:
+        cast<InstructionPattern>(Pat.get())->reportUnreachable(
+            RuleDef.getLoc());
+        return false;
+      case Pattern::K_CXX: {
+        addCXXPredicate(IM, CE, *cast<CXXPattern>(Pat.get()));
+        break;
+      }
+      default:
+        llvm_unreachable("unknown pattern kind!");
+      }
+    }
+
+    if (!emitApplyPatterns(CE, M))
+      return false;
+  }
+
+  return true;
+}
+
+bool CombineRuleBuilder::emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M) {
+  for (auto &[_, Pat] : ApplyPats) {
+    switch (Pat->getKind()) {
+    case Pattern::K_AnyOpcode:
+    case Pattern::K_Inst:
+      llvm_unreachable("Unsupported pattern kind in output pattern!");
+    case Pattern::K_CXX: {
+      CXXPattern *CXXPat = cast<CXXPattern>(Pat.get());
+      const auto &ExpandedCode = CXXPat->expandCode(CE, RuleDef.getLoc());
+      M.addAction<CustomCXXAction>(
+          ExpandedCode.getEnumNameWithPrefix(CXXApplyPrefix));
+      continue;
+    }
+    default:
+      llvm_unreachable("Unknown pattern kind!");
+    }
+  }
+
+  return true;
+}
+
+bool CombineRuleBuilder::emitInstructionMatchPattern(
+    CodeExpansions &CE, RuleMatcher &M, InstructionMatcher &IM,
+    const InstructionPattern &P, DenseSet<const Pattern *> &SeenPats) {
+  if (SeenPats.contains(&P))
+    return true;
+
+  SeenPats.insert(&P);
+
+  IM.addPredicate<InstructionOpcodeMatcher>(&P.getInst());
+  declareInstExpansion(CE, IM, P.getName());
+
+  unsigned OpIdx = 0;
+  for (auto &O : P.operands()) {
+    auto &OpTableEntry = OperandTable.find(O.Name)->second;
+
+    OperandMatcher &OM =
+        IM.addOperand(OpIdx++, O.Name, AllocatedTemporariesBaseID++);
+    declareOperandExpansion(CE, OM, O.Name);
+
+    if (O.IsDef)
+      continue;
+
+    if (InstructionPattern *DefPat = OpTableEntry.MatchPat) {
+      auto InstOpM = OM.addPredicate<InstructionOperandMatcher>(M, O.Name);
+      if (!InstOpM) {
+        // TODO: copy-pasted from GlobalISelEmitter.cpp. Is it still relevant
+        // here?
+        PrintError("Nested instruction '" + DefPat->getName() +
+                   "' cannot be the same as another operand '" + O.Name + "'");
+        return false;
+      }
+
+      if (!emitInstructionMatchPattern(CE, M, (*InstOpM)->getInsnMatcher(),
+                                       *DefPat, SeenPats))
+        return false;
+    }
+  }
+
+  return true;
+}
+
+//===- GICombinerEmitter --------------------------------------------------===//
+
+/// This class is essentially the driver. It fetches all TableGen records, calls
+/// CombineRuleBuilder to build the MatchTable's RuleMatchers, then creates the
+/// MatchTable & emits it. It also handles emitting all the supporting code such
+/// as the list of LLTs, the CXXPredicates, etc.
+class GICombinerEmitter final : public GlobalISelMatchTableExecutorEmitter {
+  RecordKeeper &Records;
+  StringRef Name;
+  const CodeGenTarget &Target;
+  Record *Combiner;
+  unsigned NextRuleID = 0;
+
+  // List all combine rules (ID, name) imported.
+  // Note that the combiner rule ID is different from the RuleMatcher ID. The
+  // latter is internal to the MatchTable, the former is the canonical ID of the
+  // combine rule used to disable/enable it.
+  std::vector<std::pair<unsigned, std::string>> AllCombineRules;
+
+  MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules);
+
+  void emitRuleConfigImpl(raw_ostream &OS);
+
+  void emitAdditionalImpl(raw_ostream &OS) override;
+
+  void emitMIPredicateFns(raw_ostream &OS) override;
+  void emitI64ImmPredicateFns(raw_ostream &OS) override;
+  void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
+  void emitAPIntImmPredicateFns(raw_ostream &OS) override;
+  void emitTestSimplePredicate(raw_ostream &OS) override;
+  void emitRunCustomAction(raw_ostream &OS) override;
+
+  void emitAdditionalTemporariesDecl(raw_ostream &OS,
+                                     StringRef Indent) override;
+
+  const CodeGenTarget &getTarget() const override { return Target; }
+  StringRef getClassName() const override {
+    return Combiner->getValueAsString("Classname");
+  }
+
+  std::string getRuleConfigClassName() const {
+    return getClassName().str() + "RuleConfig";
+  }
+
+  void gatherRules(std::vector<RuleMatcher> &Rules,
+                   const std::vector<Record *> &&RulesAndGroups);
+
+public:
+  explicit GICombinerEmitter(RecordKeeper &RK, const CodeGenTarget &Target,
+                             StringRef Name, Record *Combiner);
+  ~GICombinerEmitter() {}
+
+  void run(raw_ostream &OS);
+};
+
+void GICombinerEmitter::emitRuleConfigImpl(raw_ostream &OS) {
+  OS << "struct " << getRuleConfigClassName() << " {\n"
+     << "  SparseBitVector<> DisabledRules;\n\n"
+     << "  bool isRuleEnabled(unsigned RuleID) const;\n"
+     << "  bool parseCommandLineOption();\n"
+     << "  bool setRuleEnabled(StringRef RuleIdentifier);\n"
+     << "  bool setRuleDisabled(StringRef RuleIdentifier);\n"
+     << "};\n\n";
+
+  std::vector<std::pair<std::string, std::string>> Cases;
+  Cases.reserve(AllCombineRules.size());
+
+  for (const auto &[ID, Name] : AllCombineRules)
+    Cases.emplace_back(Name, "return " + to_string(ID) + ";\n");
+
+  OS << "static std::optional<uint64_t> getRuleIdxForIdentifier(StringRef "
+        "RuleIdentifier) {\n"
+     << "  uint64_t I;\n"
+     << "  // getAtInteger(...) returns false on success\n"
+     << "  bool Parsed = !RuleIdentifier.getAsInteger(0, I);\n"
+     << "  if (Parsed)\n"
+     << "    return I;\n\n"
+     << "#ifndef NDEBUG\n";
+  StringMatcher Matcher("RuleIdentifier", Cases, OS);
+  Matcher.Emit();
+  OS << "#endif // ifndef NDEBUG\n\n"
+     << "  return std::nullopt;\n"
+     << "}\n";
+
+  OS << "static std::optional<std::pair<uint64_t, uint64_t>> "
+        "getRuleRangeForIdentifier(StringRef RuleIdentifier) {\n"
+     << "  std::pair<StringRef, StringRef> RangePair = "
+        "RuleIdentifier.split('-');\n"
+     << "  if (!RangePair.second.empty()) {\n"
+     << "    const auto First = "
+        "getRuleIdxForIdentifier(RangePair.first);\n"
+     << "    const auto Last = "
+        "getRuleIdxForIdentifier(RangePair.second);\n"
+     << "    if (!First || !Last)\n"
+     << "      return std::nullopt;\n"
+     << "    if (First >= Last)\n"
+     << "      report_fatal_error(\"Beginning of range should be before "
+        "end of range\");\n"
+     << "    return {{*First, *Last + 1}};\n"
+     << "  }\n"
+     << "  if (RangePair.first == \"*\") {\n"
+     << "    return {{0, " << AllCombineRules.size() << "}};\n"
+     << "  }\n"
+     << "  const auto I = getRuleIdxForIdentifier(RangePair.first);\n"
+     << "  if (!I)\n"
+     << "    return std::nullopt;\n"
+     << "  return {{*I, *I + 1}};\n"
+     << "}\n\n";
+
+  for (bool Enabled : {true, false}) {
+    OS << "bool " << getRuleConfigClassName() << "::setRule"
+       << (Enabled ? "Enabled" : "Disabled") << "(StringRef RuleIdentifier) {\n"
+       << "  auto MaybeRange = getRuleRangeForIdentifier(RuleIdentifier);\n"
+       << "  if (!MaybeRange)\n"
+       << "    return false;\n"
+       << "  for (auto I = MaybeRange->first; I < MaybeRange->second; ++I)\n"
+       << "    DisabledRules." << (Enabled ? "reset" : "set") << "(I);\n"
+       << "  return true;\n"
+       << "}\n\n";
+  }
+
+  OS << "static std::vector<std::string> " << Name << "Option;\n"
+     << "static cl::list<std::string> " << Name << "DisableOption(\n"
+     << "    \"" << Name.lower() << "-disable-rule\",\n"
+     << "    cl::desc(\"Disable one or more combiner rules temporarily in "
+     << "the " << Name << " pass\"),\n"
+     << "    cl::CommaSeparated,\n"
+     << "    cl::Hidden,\n"
+     << "    cl::cat(GICombinerOptionCategory),\n"
+     << "    cl::callback([](const std::string &Str) {\n"
+     << "      " << Name << "Option.push_back(Str);\n"
+     << "    }));\n"
+     << "static cl::list<std::string> " << Name << "OnlyEnableOption(\n"
+     << "    \"" << Name.lower() << "-only-enable-rule\",\n"
+     << "    cl::desc(\"Disable all rules in the " << Name
+     << " pass then re-enable the specified ones\"),\n"
+     << "    cl::Hidden,\n"
+     << "    cl::cat(GICombinerOptionCategory),\n"
+     << "    cl::callback([](const std::string &CommaSeparatedArg) {\n"
+     << "      StringRef Str = CommaSeparatedArg;\n"
+     << "      " << Name << "Option.push_back(\"*\");\n"
+     << "      do {\n"
+     << "        auto X = Str.split(\",\");\n"
+     << "        " << Name << "Option.push_back((\"!\" + X.first).str());\n"
+     << "        Str = X.second;\n"
+     << "      } while (!Str.empty());\n"
+     << "    }));\n"
+     << "\n\n"
+     << "bool " << getRuleConfigClassName()
+     << "::isRuleEnabled(unsigned RuleID) const {\n"
+     << "    return  !DisabledRules.test(RuleID);\n"
+     << "}\n"
+     << "bool " << getRuleConfigClassName() << "::parseCommandLineOption() {\n"
+     << "  for (StringRef Identifier : " << Name << "Option) {\n"
+     << "    bool Enabled = Identifier.consume_front(\"!\");\n"
+     << "    if (Enabled && !setRuleEnabled(Identifier))\n"
+     << "      return false;\n"
+     << "    if (!Enabled && !setRuleDisabled(Identifier))\n"
+     << "      return false;\n"
+     << "  }\n"
+     << "  return true;\n"
+     << "}\n\n";
+}
+
+void GICombinerEmitter::emitAdditionalImpl(raw_ostream &OS) {
+  OS << "bool " << getClassName()
+     << "::tryCombineAll(MachineInstr &I) const {\n"
+     << "  const TargetSubtargetInfo &ST = MF.getSubtarget();\n"
+     << "  const PredicateBitset AvailableFeatures = "
+        "getAvailableFeatures();\n"
+     << "  NewMIVector OutMIs;\n"
+     << "  State.MIs.clear();\n"
+     << "  State.MIs.push_back(&I);\n"
+     << "  " << MatchDataInfo::StructName << " = "
+     << MatchDataInfo::StructTypeName << "();\n\n"
+     << "  if (executeMatchTable(*this, OutMIs, State, ExecInfo"
+     << ", getMatchTable(), *ST.getInstrInfo(), MRI, "
+        "*MRI.getTargetRegisterInfo(), *ST.getRegBankInfo(), AvailableFeatures"
+     << ", /*CoverageInfo*/ nullptr)) {\n"
+     << "    return true;\n"
+     << "  }\n\n"
+     << "  return false;\n"
+     << "}\n\n";
+}
+
+void GICombinerEmitter::emitMIPredicateFns(raw_ostream &OS) {
+  auto MatchCode = getSorted(AllCXXMatchCode);
+  emitMIPredicateFnsImpl<const CXXPredicateCode *>(
+      OS, "", ArrayRef<const CXXPredicateCode *>(MatchCode),
+      [](const CXXPredicateCode *C) -> StringRef { return C->BaseEnumName; },
+      [](const CXXPredicateCode *C) -> StringRef { return C->Code; });
+}
+
+void GICombinerEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
+  // Unused, but still needs to be called.
+  emitImmPredicateFnsImpl<unsigned>(
+      OS, "I64", "int64_t", {}, [](unsigned) { return ""; },
+      [](unsigned) { return ""; });
+}
+
+void GICombinerEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {
+  // Unused, but still needs to be called.
+  emitImmPredicateFnsImpl<unsigned>(
+      OS, "APFloat", "const APFloat &", {}, [](unsigned) { return ""; },
+      [](unsigned) { return ""; });
+}
+
+void GICombinerEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {
+  // Unused, but still needs to be called.
+  emitImmPredicateFnsImpl<unsigned>(
+      OS, "APInt", "const APInt &", {}, [](unsigned) { return ""; },
+      [](unsigned) { return ""; });
+}
+
+void GICombinerEmitter::emitTestSimplePredicate(raw_ostream &OS) {
+  if (!AllCombineRules.empty()) {
+    OS << "enum {\n";
+    std::string EnumeratorSeparator = " = GICXXPred_Invalid + 1,\n";
+    // To avoid emitting a switch, we expect that all those rules are in order.
+    // That way we can just get the RuleID from the enum by subtracting
+    // (GICXXPred_Invalid + 1).
+    unsigned ExpectedID = 0;
+    (void)ExpectedID;
+    for (const auto &[ID, _] : AllCombineRules) {
+      assert(ExpectedID++ == ID && "combine rules are not ordered!");
+      OS << "  " << getIsEnabledPredicateEnumName(ID) << EnumeratorSeparator;
+      EnumeratorSeparator = ",\n";
+    }
+    OS << "};\n\n";
+  }
+
+  OS << "bool " << getClassName()
+     << "::testSimplePredicate(unsigned Predicate) const {\n"
+     << "    return RuleConfig.isRuleEnabled(Predicate - "
+        "GICXXPred_Invalid - "
+        "1);\n"
+     << "}\n";
+}
+
+void GICombinerEmitter::emitRunCustomAction(raw_ostream &OS) {
+  const auto ApplyCode = getSorted(AllCXXApplyCode);
+
+  if (!ApplyCode.empty()) {
+    OS << "enum {\n";
+    std::string EnumeratorSeparator = " = GICXXCustomAction_Invalid + 1,\n";
+    for (const auto &Apply : ApplyCode) {
+      OS << "  " << Apply->getEnumNameWithPrefix(CXXApplyPrefix)
+         << EnumeratorSeparator;
+      EnumeratorSeparator = ",\n";
+    }
+    OS << "};\n";
+  }
+
+  OS << "void " << getClassName()
+     << "::runCustomAction(unsigned ApplyID, const MatcherState &State) const "
+        "{\n";
+  if (!ApplyCode.empty()) {
+    OS << "  switch(ApplyID) {\n";
+    for (const auto &Apply : ApplyCode) {
+      OS << "  case " << Apply->getEnumNameWithPrefix(CXXApplyPrefix) << ":{\n"
+         << "    " << Apply->Code << "\n"
+         << "    return;\n";
+      OS << "  }\n";
+    }
+    OS << "}\n";
+  }
+  OS << "  llvm_unreachable(\"Unknown Apply Action\");\n"
+     << "}\n";
+}
+
+void GICombinerEmitter::emitAdditionalTemporariesDecl(raw_ostream &OS,
+                                                      StringRef Indent) {
+  OS << Indent << "struct " << MatchDataInfo::StructTypeName << " {\n";
+  for (const auto &[Type, VarNames] : AllMatchDataVars) {
+    assert(!VarNames.empty() && "Cannot have no vars for this type!");
+    OS << Indent << "  " << Type << " " << join(VarNames, ", ") << ";\n";
+  }
+  OS << Indent << "};\n"
+     << Indent << "mutable " << MatchDataInfo::StructTypeName << " "
+     << MatchDataInfo::StructName << ";\n\n";
+}
+
+GICombinerEmitter::GICombinerEmitter(RecordKeeper &RK,
+                                     const CodeGenTarget &Target,
+                                     StringRef Name, Record *Combiner)
+    : Records(RK), Name(Name), Target(Target), Combiner(Combiner) {}
+
+MatchTable
+GICombinerEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules) {
+  std::vector<Matcher *> InputRules;
+  for (Matcher &Rule : Rules)
+    InputRules.push_back(&Rule);
+
+  unsigned CurrentOrdering = 0;
+  StringMap<unsigned> OpcodeOrder;
+  for (RuleMatcher &Rule : Rules) {
+    const StringRef Opcode = Rule.getOpcode();
+    assert(!Opcode.empty() && "Didn't expect an undefined opcode");
+    if (OpcodeOrder.count(Opcode) == 0)
+      OpcodeOrder[Opcode] = CurrentOrdering++;
+  }
+
+  llvm::stable_sort(InputRules, [&OpcodeOrder](const Matcher *A,
+                                               const Matcher *B) {
+    auto *L = static_cast<const RuleMatcher *>(A);
+    auto *R = static_cast<const RuleMatcher *>(B);
+    return std::make_tuple(OpcodeOrder[L->getOpcode()], L->getNumOperands()) <
+           std::make_tuple(OpcodeOrder[R->getOpcode()], R->getNumOperands());
+  });
+
+  for (Matcher *Rule : InputRules)
+    Rule->optimize();
+
+  std::vector<std::unique_ptr<Matcher>> MatcherStorage;
+  std::vector<Matcher *> OptRules =
+      optimizeRules<GroupMatcher>(InputRules, MatcherStorage);
+
+  for (Matcher *Rule : OptRules)
+    Rule->optimize();
+
+  OptRules = optimizeRules<SwitchMatcher>(OptRules, MatcherStorage);
+
+  return MatchTable::buildTable(OptRules, /*WithCoverage*/ false,
+                                /*IsCombiner*/ true);
+}
+
+/// Recurse into GICombineGroup's and flatten the ruleset into a simple list.
+void GICombinerEmitter::gatherRules(
+    std::vector<RuleMatcher> &ActiveRules,
+    const std::vector<Record *> &&RulesAndGroups) {
+  for (Record *R : RulesAndGroups) {
+    if (R->isValueUnset("Rules")) {
+      AllCombineRules.emplace_back(NextRuleID, R->getName().str());
+      CombineRuleBuilder CRB(Target, SubtargetFeatures, *R, NextRuleID++,
+                             ActiveRules);
+
+      if (!CRB.parseAll())
+        continue;
+
+      if (StopAfterParse) {
+        CRB.print(outs());
+        continue;
+      }
+
+      if (!CRB.emitRuleMatchers())
+        continue;
+    } else
+      gatherRules(ActiveRules, R->getValueAsListOfDefs("Rules"));
+  }
+}
+
+void GICombinerEmitter::run(raw_ostream &OS) {
+  Records.startTimer("Gather rules");
+  std::vector<RuleMatcher> Rules;
+  gatherRules(Rules, Combiner->getValueAsListOfDefs("Rules"));
+  if (ErrorsPrinted)
+    PrintFatalError(Combiner->getLoc(), "Failed to parse one or more rules");
+
+  Records.startTimer("Creating Match Table");
+  unsigned MaxTemporaries = 0;
+  for (const auto &Rule : Rules)
+    MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns());
+
+  const MatchTable Table = buildMatchTable(Rules);
+
+  Records.startTimer("Emit combiner");
+
+  emitSourceFileHeader(getClassName().str() + " Combiner Match Table", OS);
+
+  // Unused
+  std::vector<StringRef> CustomRendererFns;
+  // Unused, but hack to avoid empty declarator
+  std::vector<LLTCodeGen> TypeObjects = {LLTCodeGen(LLT::scalar(1))};
+  // Unused
+  std::vector<Record *> ComplexPredicates;
+
+  // GET_GICOMBINER_DEPS, which pulls in extra dependencies.
+  OS << "#ifdef GET_GICOMBINER_DEPS\n"
+     << "#include \"llvm/ADT/SparseBitVector.h\"\n"
+     << "namespace llvm {\n"
+     << "extern cl::OptionCategory GICombinerOptionCategory;\n"
+     << "} // end namespace llvm\n"
+     << "#endif // ifdef GET_GICOMBINER_DEPS\n\n";
+
+  // GET_GICOMBINER_TYPES, which needs to be included before the declaration of
+  // the class.
+  OS << "#ifdef GET_GICOMBINER_TYPES\n";
+  emitRuleConfigImpl(OS);
+  OS << "#endif // ifdef GET_GICOMBINER_TYPES\n\n";
+  emitPredicateBitset(OS, "GET_GICOMBINER_TYPES");
+
+  // GET_GICOMBINER_CLASS_MEMBERS, which need to be included inside the class.
+  emitPredicatesDecl(OS, "GET_GICOMBINER_CLASS_MEMBERS");
+  emitTemporariesDecl(OS, "GET_GICOMBINER_CLASS_MEMBERS");
+
+  // GET_GICOMBINER_IMPL, which needs to be included outside the class.
+  emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexPredicates,
+                   CustomRendererFns, "GET_GICOMBINER_IMPL");
+
+  // GET_GICOMBINER_CONSTRUCTOR_INITS, which are in the constructor's
+  // initializer list.
+  emitPredicatesInit(OS, "GET_GICOMBINER_CONSTRUCTOR_INITS");
+  emitTemporariesInit(OS, MaxTemporaries, "GET_GICOMBINER_CONSTRUCTOR_INITS");
+}
+
+} // end anonymous namespace
+
+//===----------------------------------------------------------------------===//
+
+static void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS) {
+  CodeGenTarget Target(RK);
+
+  if (SelectedCombiners.empty())
+    PrintFatalError("No combiners selected with -combiners");
+  for (const auto &Combiner : SelectedCombiners) {
+    Record *CombinerDef = RK.getDef(Combiner);
+    if (!CombinerDef)
+      PrintFatalError("Could not find " + Combiner);
+    GICombinerEmitter(RK, Target, Combiner, CombinerDef).run(OS);
+  }
+}
+
+static TableGen::Emitter::Opt X("gen-global-isel-combiner-matchtable",
+                                EmitGICombiner,
+                                "Generate GlobalISel combiner Match Table");
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index c79c79948a80..3bdcfec06e24 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -8,7 +8,7 @@
 //
 /// \file
 /// This tablegen backend emits code for use by the GlobalISel instruction
-/// selector. See include/llvm/CodeGen/TargetGlobalISel.td.
+/// selector. See include/llvm/Target/GlobalISel/Target.td.
 ///
 /// This file analyzes the patterns recognized by the SelectionDAGISel tablegen
 /// backend, filters out the ones that are unsupported, maps
@@ -31,28 +31,39 @@
 
 #include "CodeGenDAGPatterns.h"
 #include "CodeGenInstruction.h"
+#include "CodeGenIntrinsics.h"
+#include "CodeGenRegisters.h"
+#include "CodeGenTarget.h"
+#include "GlobalISelMatchTable.h"
+#include "GlobalISelMatchTableExecutorEmitter.h"
+#include "InfoByHwMode.h"
 #include "SubtargetFeatureInfo.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/Support/CodeGenCoverage.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
-#include "llvm/Support/LowLevelTypeImpl.h"
-#include "llvm/Support/MachineValueType.h"
+#include "llvm/Support/SaveAndRestore.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <numeric>
 #include <string>
+
 using namespace llvm;
+using namespace llvm::gi;
+
+using action_iterator = RuleMatcher::action_iterator;
 
 #define DEBUG_TYPE "gisel-emitter"
 
 STATISTIC(NumPatternTotal, "Total number of patterns");
 STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG");
 STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped");
-STATISTIC(NumPatternsTested, "Number of patterns executed according to coverage information");
-STATISTIC(NumPatternEmitted, "Number of patterns emitted");
+STATISTIC(NumPatternsTested,
+          "Number of patterns executed according to coverage information");
 
 cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel");
 
@@ -78,140 +89,6 @@ static cl::opt<bool> OptimizeMatchTable(
     cl::init(true), cl::cat(GlobalISelEmitterCat));
 
 namespace {
-//===- Helper functions ---------------------------------------------------===//
-
-/// Get the name of the enum value used to number the predicate function.
-std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) {
-  if (Predicate.hasGISelPredicateCode())
-    return "GIPFP_MI_" + Predicate.getFnName();
-  return "GIPFP_" + Predicate.getImmTypeIdentifier().str() + "_" +
-         Predicate.getFnName();
-}
-
-/// Get the opcode used to check this predicate.
-std::string getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) {
-  return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate";
-}
-
-/// This class stands in for LLT wherever we want to tablegen-erate an
-/// equivalent at compiler run-time.
-class LLTCodeGen {
-private:
-  LLT Ty;
-
-public:
-  LLTCodeGen() = default;
-  LLTCodeGen(const LLT &Ty) : Ty(Ty) {}
-
-  std::string getCxxEnumValue() const {
-    std::string Str;
-    raw_string_ostream OS(Str);
-
-    emitCxxEnumValue(OS);
-    return Str;
-  }
-
-  void emitCxxEnumValue(raw_ostream &OS) const {
-    if (Ty.isScalar()) {
-      OS << "GILLT_s" << Ty.getSizeInBits();
-      return;
-    }
-    if (Ty.isVector()) {
-      OS << (Ty.isScalable() ? "GILLT_nxv" : "GILLT_v")
-         << Ty.getElementCount().getKnownMinValue() << "s"
-         << Ty.getScalarSizeInBits();
-      return;
-    }
-    if (Ty.isPointer()) {
-      OS << "GILLT_p" << Ty.getAddressSpace();
-      if (Ty.getSizeInBits() > 0)
-        OS << "s" << Ty.getSizeInBits();
-      return;
-    }
-    llvm_unreachable("Unhandled LLT");
-  }
-
-  void emitCxxConstructorCall(raw_ostream &OS) const {
-    if (Ty.isScalar()) {
-      OS << "LLT::scalar(" << Ty.getSizeInBits() << ")";
-      return;
-    }
-    if (Ty.isVector()) {
-      OS << "LLT::vector("
-         << (Ty.isScalable() ? "ElementCount::getScalable("
-                             : "ElementCount::getFixed(")
-         << Ty.getElementCount().getKnownMinValue() << "), "
-         << Ty.getScalarSizeInBits() << ")";
-      return;
-    }
-    if (Ty.isPointer() && Ty.getSizeInBits() > 0) {
-      OS << "LLT::pointer(" << Ty.getAddressSpace() << ", "
-         << Ty.getSizeInBits() << ")";
-      return;
-    }
-    llvm_unreachable("Unhandled LLT");
-  }
-
-  const LLT &get() const { return Ty; }
-
-  /// This ordering is used for std::unique() and llvm::sort(). There's no
-  /// particular logic behind the order but either A < B or B < A must be
-  /// true if A != B.
-  bool operator<(const LLTCodeGen &Other) const {
-    if (Ty.isValid() != Other.Ty.isValid())
-      return Ty.isValid() < Other.Ty.isValid();
-    if (!Ty.isValid())
-      return false;
-
-    if (Ty.isVector() != Other.Ty.isVector())
-      return Ty.isVector() < Other.Ty.isVector();
-    if (Ty.isScalar() != Other.Ty.isScalar())
-      return Ty.isScalar() < Other.Ty.isScalar();
-    if (Ty.isPointer() != Other.Ty.isPointer())
-      return Ty.isPointer() < Other.Ty.isPointer();
-
-    if (Ty.isPointer() && Ty.getAddressSpace() != Other.Ty.getAddressSpace())
-      return Ty.getAddressSpace() < Other.Ty.getAddressSpace();
-
-    if (Ty.isVector() && Ty.getElementCount() != Other.Ty.getElementCount())
-      return std::make_tuple(Ty.isScalable(),
-                             Ty.getElementCount().getKnownMinValue()) <
-             std::make_tuple(Other.Ty.isScalable(),
-                             Other.Ty.getElementCount().getKnownMinValue());
-
-    assert((!Ty.isVector() || Ty.isScalable() == Other.Ty.isScalable()) &&
-           "Unexpected mismatch of scalable property");
-    return Ty.isVector()
-               ? std::make_tuple(Ty.isScalable(),
-                                 Ty.getSizeInBits().getKnownMinValue()) <
-                     std::make_tuple(
-                         Other.Ty.isScalable(),
-                         Other.Ty.getSizeInBits().getKnownMinValue())
-               : Ty.getSizeInBits().getFixedValue() <
-                     Other.Ty.getSizeInBits().getFixedValue();
-  }
-
-  bool operator==(const LLTCodeGen &B) const { return Ty == B.Ty; }
-};
-
-// Track all types that are used so we can emit the corresponding enum.
-std::set<LLTCodeGen> KnownTypes;
-
-class InstructionMatcher;
-/// Convert an MVT to an equivalent LLT if possible, or the invalid LLT() for
-/// MVTs that don't map cleanly to an LLT (e.g., iPTR, *any, ...).
-static std::optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT) {
-  MVT VT(SVT);
-
-  if (VT.isVector() && !VT.getVectorElementCount().isScalar())
-    return LLTCodeGen(
-        LLT::vector(VT.getVectorElementCount(), VT.getScalarSizeInBits()));
-
-  if (VT.isInteger() || VT.isFloatingPoint())
-    return LLTCodeGen(LLT::scalar(VT.getSizeInBits()));
-
-  return std::nullopt;
-}
 
 static std::string explainPredicates(const TreePatternNode *N) {
   std::string Explanation;
@@ -401,3150 +278,10 @@ static Record *getInitValueAsRegClass(Init *V) {
   return nullptr;
 }
 
-std::string
-getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset) {
-  std::string Name = "GIFBS";
-  for (const auto &Feature : FeatureBitset)
-    Name += ("_" + Feature->getName()).str();
-  return Name;
-}
-
 static std::string getScopedName(unsigned Scope, const std::string &Name) {
   return ("pred:" + Twine(Scope) + ":" + Name).str();
 }
 
-//===- MatchTable Helpers -------------------------------------------------===//
-
-class MatchTable;
-
-/// A record to be stored in a MatchTable.
-///
-/// This class represents any and all output that may be required to emit the
-/// MatchTable. Instances  are most often configured to represent an opcode or
-/// value that will be emitted to the table with some formatting but it can also
-/// represent commas, comments, and other formatting instructions.
-struct MatchTableRecord {
-  enum RecordFlagsBits {
-    MTRF_None = 0x0,
-    /// Causes EmitStr to be formatted as comment when emitted.
-    MTRF_Comment = 0x1,
-    /// Causes the record value to be followed by a comma when emitted.
-    MTRF_CommaFollows = 0x2,
-    /// Causes the record value to be followed by a line break when emitted.
-    MTRF_LineBreakFollows = 0x4,
-    /// Indicates that the record defines a label and causes an additional
-    /// comment to be emitted containing the index of the label.
-    MTRF_Label = 0x8,
-    /// Causes the record to be emitted as the index of the label specified by
-    /// LabelID along with a comment indicating where that label is.
-    MTRF_JumpTarget = 0x10,
-    /// Causes the formatter to add a level of indentation before emitting the
-    /// record.
-    MTRF_Indent = 0x20,
-    /// Causes the formatter to remove a level of indentation after emitting the
-    /// record.
-    MTRF_Outdent = 0x40,
-  };
-
-  /// When MTRF_Label or MTRF_JumpTarget is used, indicates a label id to
-  /// reference or define.
-  unsigned LabelID;
-  /// The string to emit. Depending on the MTRF_* flags it may be a comment, a
-  /// value, a label name.
-  std::string EmitStr;
-
-private:
-  /// The number of MatchTable elements described by this record. Comments are 0
-  /// while values are typically 1. Values >1 may occur when we need to emit
-  /// values that exceed the size of a MatchTable element.
-  unsigned NumElements;
-
-public:
-  /// A bitfield of RecordFlagsBits flags.
-  unsigned Flags;
-
-  /// The actual run-time value, if known
-  int64_t RawValue;
-
-  MatchTableRecord(std::optional<unsigned> LabelID_, StringRef EmitStr,
-                   unsigned NumElements, unsigned Flags,
-                   int64_t RawValue = std::numeric_limits<int64_t>::min())
-      : LabelID(LabelID_.value_or(~0u)), EmitStr(EmitStr),
-        NumElements(NumElements), Flags(Flags), RawValue(RawValue) {
-    assert((!LabelID_ || LabelID != ~0u) &&
-           "This value is reserved for non-labels");
-  }
-  MatchTableRecord(const MatchTableRecord &Other) = default;
-  MatchTableRecord(MatchTableRecord &&Other) = default;
-
-  /// Useful if a Match Table Record gets optimized out
-  void turnIntoComment() {
-    Flags |= MTRF_Comment;
-    Flags &= ~MTRF_CommaFollows;
-    NumElements = 0;
-  }
-
-  /// For Jump Table generation purposes
-  bool operator<(const MatchTableRecord &Other) const {
-    return RawValue < Other.RawValue;
-  }
-  int64_t getRawValue() const { return RawValue; }
-
-  void emit(raw_ostream &OS, bool LineBreakNextAfterThis,
-            const MatchTable &Table) const;
-  unsigned size() const { return NumElements; }
-};
-
-class Matcher;
-
-/// Holds the contents of a generated MatchTable to enable formatting and the
-/// necessary index tracking needed to support GIM_Try.
-class MatchTable {
-  /// An unique identifier for the table. The generated table will be named
-  /// MatchTable${ID}.
-  unsigned ID;
-  /// The records that make up the table. Also includes comments describing the
-  /// values being emitted and line breaks to format it.
-  std::vector<MatchTableRecord> Contents;
-  /// The currently defined labels.
-  DenseMap<unsigned, unsigned> LabelMap;
-  /// Tracks the sum of MatchTableRecord::NumElements as the table is built.
-  unsigned CurrentSize = 0;
-  /// A unique identifier for a MatchTable label.
-  unsigned CurrentLabelID = 0;
-  /// Determines if the table should be instrumented for rule coverage tracking.
-  bool IsWithCoverage;
-
-public:
-  static MatchTableRecord LineBreak;
-  static MatchTableRecord Comment(StringRef Comment) {
-    return MatchTableRecord(std::nullopt, Comment, 0,
-                            MatchTableRecord::MTRF_Comment);
-  }
-  static MatchTableRecord Opcode(StringRef Opcode, int IndentAdjust = 0) {
-    unsigned ExtraFlags = 0;
-    if (IndentAdjust > 0)
-      ExtraFlags |= MatchTableRecord::MTRF_Indent;
-    if (IndentAdjust < 0)
-      ExtraFlags |= MatchTableRecord::MTRF_Outdent;
-
-    return MatchTableRecord(std::nullopt, Opcode, 1,
-                            MatchTableRecord::MTRF_CommaFollows | ExtraFlags);
-  }
-  static MatchTableRecord NamedValue(StringRef NamedValue) {
-    return MatchTableRecord(std::nullopt, NamedValue, 1,
-                            MatchTableRecord::MTRF_CommaFollows);
-  }
-  static MatchTableRecord NamedValue(StringRef NamedValue, int64_t RawValue) {
-    return MatchTableRecord(std::nullopt, NamedValue, 1,
-                            MatchTableRecord::MTRF_CommaFollows, RawValue);
-  }
-  static MatchTableRecord NamedValue(StringRef Namespace,
-                                     StringRef NamedValue) {
-    return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
-                            1, MatchTableRecord::MTRF_CommaFollows);
-  }
-  static MatchTableRecord NamedValue(StringRef Namespace, StringRef NamedValue,
-                                     int64_t RawValue) {
-    return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
-                            1, MatchTableRecord::MTRF_CommaFollows, RawValue);
-  }
-  static MatchTableRecord IntValue(int64_t IntValue) {
-    return MatchTableRecord(std::nullopt, llvm::to_string(IntValue), 1,
-                            MatchTableRecord::MTRF_CommaFollows);
-  }
-  static MatchTableRecord Label(unsigned LabelID) {
-    return MatchTableRecord(LabelID, "Label " + llvm::to_string(LabelID), 0,
-                            MatchTableRecord::MTRF_Label |
-                                MatchTableRecord::MTRF_Comment |
-                                MatchTableRecord::MTRF_LineBreakFollows);
-  }
-  static MatchTableRecord JumpTarget(unsigned LabelID) {
-    return MatchTableRecord(LabelID, "Label " + llvm::to_string(LabelID), 1,
-                            MatchTableRecord::MTRF_JumpTarget |
-                                MatchTableRecord::MTRF_Comment |
-                                MatchTableRecord::MTRF_CommaFollows);
-  }
-
-  static MatchTable buildTable(ArrayRef<Matcher *> Rules, bool WithCoverage);
-
-  MatchTable(bool WithCoverage, unsigned ID = 0)
-      : ID(ID), IsWithCoverage(WithCoverage) {}
-
-  bool isWithCoverage() const { return IsWithCoverage; }
-
-  void push_back(const MatchTableRecord &Value) {
-    if (Value.Flags & MatchTableRecord::MTRF_Label)
-      defineLabel(Value.LabelID);
-    Contents.push_back(Value);
-    CurrentSize += Value.size();
-  }
-
-  unsigned allocateLabelID() { return CurrentLabelID++; }
-
-  void defineLabel(unsigned LabelID) {
-    LabelMap.insert(std::make_pair(LabelID, CurrentSize));
-  }
-
-  unsigned getLabelIndex(unsigned LabelID) const {
-    const auto I = LabelMap.find(LabelID);
-    assert(I != LabelMap.end() && "Use of undeclared label");
-    return I->second;
-  }
-
-  void emitUse(raw_ostream &OS) const { OS << "MatchTable" << ID; }
-
-  void emitDeclaration(raw_ostream &OS) const {
-    unsigned Indentation = 4;
-    OS << "  constexpr static int64_t MatchTable" << ID << "[] = {";
-    LineBreak.emit(OS, true, *this);
-    OS << std::string(Indentation, ' ');
-
-    for (auto I = Contents.begin(), E = Contents.end(); I != E;
-         ++I) {
-      bool LineBreakIsNext = false;
-      const auto &NextI = std::next(I);
-
-      if (NextI != E) {
-        if (NextI->EmitStr == "" &&
-            NextI->Flags == MatchTableRecord::MTRF_LineBreakFollows)
-          LineBreakIsNext = true;
-      }
-
-      if (I->Flags & MatchTableRecord::MTRF_Indent)
-        Indentation += 2;
-
-      I->emit(OS, LineBreakIsNext, *this);
-      if (I->Flags & MatchTableRecord::MTRF_LineBreakFollows)
-        OS << std::string(Indentation, ' ');
-
-      if (I->Flags & MatchTableRecord::MTRF_Outdent)
-        Indentation -= 2;
-    }
-    OS << "};\n";
-  }
-};
-
-MatchTableRecord MatchTable::LineBreak = {
-    std::nullopt, "" /* Emit String */, 0 /* Elements */,
-    MatchTableRecord::MTRF_LineBreakFollows};
-
-void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis,
-                            const MatchTable &Table) const {
-  bool UseLineComment =
-      LineBreakIsNextAfterThis || (Flags & MTRF_LineBreakFollows);
-  if (Flags & (MTRF_JumpTarget | MTRF_CommaFollows))
-    UseLineComment = false;
-
-  if (Flags & MTRF_Comment)
-    OS << (UseLineComment ? "// " : "/*");
-
-  OS << EmitStr;
-  if (Flags & MTRF_Label)
-    OS << ": @" << Table.getLabelIndex(LabelID);
-
-  if ((Flags & MTRF_Comment) && !UseLineComment)
-    OS << "*/";
-
-  if (Flags & MTRF_JumpTarget) {
-    if (Flags & MTRF_Comment)
-      OS << " ";
-    OS << Table.getLabelIndex(LabelID);
-  }
-
-  if (Flags & MTRF_CommaFollows) {
-    OS << ",";
-    if (!LineBreakIsNextAfterThis && !(Flags & MTRF_LineBreakFollows))
-      OS << " ";
-  }
-
-  if (Flags & MTRF_LineBreakFollows)
-    OS << "\n";
-}
-
-MatchTable &operator<<(MatchTable &Table, const MatchTableRecord &Value) {
-  Table.push_back(Value);
-  return Table;
-}
-
-//===- Matchers -----------------------------------------------------------===//
-
-class OperandMatcher;
-class MatchAction;
-class PredicateMatcher;
-
-class Matcher {
-public:
-  virtual ~Matcher() = default;
-  virtual void optimize() {}
-  virtual void emit(MatchTable &Table) = 0;
-
-  virtual bool hasFirstCondition() const = 0;
-  virtual const PredicateMatcher &getFirstCondition() const = 0;
-  virtual std::unique_ptr<PredicateMatcher> popFirstCondition() = 0;
-};
-
-MatchTable MatchTable::buildTable(ArrayRef<Matcher *> Rules,
-                                  bool WithCoverage) {
-  MatchTable Table(WithCoverage);
-  for (Matcher *Rule : Rules)
-    Rule->emit(Table);
-
-  return Table << MatchTable::Opcode("GIM_Reject") << MatchTable::LineBreak;
-}
-
-class GroupMatcher final : public Matcher {
-  /// Conditions that form a common prefix of all the matchers contained.
-  SmallVector<std::unique_ptr<PredicateMatcher>, 1> Conditions;
-
-  /// All the nested matchers, sharing a common prefix.
-  std::vector<Matcher *> Matchers;
-
-  /// An owning collection for any auxiliary matchers created while optimizing
-  /// nested matchers contained.
-  std::vector<std::unique_ptr<Matcher>> MatcherStorage;
-
-public:
-  /// Add a matcher to the collection of nested matchers if it meets the
-  /// requirements, and return true. If it doesn't, do nothing and return false.
-  ///
-  /// Expected to preserve its argument, so it could be moved out later on.
-  bool addMatcher(Matcher &Candidate);
-
-  /// Mark the matcher as fully-built and ensure any invariants expected by both
-  /// optimize() and emit(...) methods. Generally, both sequences of calls
-  /// are expected to lead to a sensible result:
-  ///
-  /// addMatcher(...)*; finalize(); optimize(); emit(...); and
-  /// addMatcher(...)*; finalize(); emit(...);
-  ///
-  /// or generally
-  ///
-  /// addMatcher(...)*; finalize(); { optimize()*; emit(...); }*
-  ///
-  /// Multiple calls to optimize() are expected to be handled gracefully, though
-  /// optimize() is not expected to be idempotent. Multiple calls to finalize()
-  /// aren't generally supported. emit(...) is expected to be non-mutating and
-  /// producing the exact same results upon repeated calls.
-  ///
-  /// addMatcher() calls after the finalize() call are not supported.
-  ///
-  /// finalize() and optimize() are both allowed to mutate the contained
-  /// matchers, so moving them out after finalize() is not supported.
-  void finalize();
-  void optimize() override;
-  void emit(MatchTable &Table) override;
-
-  /// Could be used to move out the matchers added previously, unless finalize()
-  /// has been already called. If any of the matchers are moved out, the group
-  /// becomes safe to destroy, but not safe to re-use for anything else.
-  iterator_range<std::vector<Matcher *>::iterator> matchers() {
-    return make_range(Matchers.begin(), Matchers.end());
-  }
-  size_t size() const { return Matchers.size(); }
-  bool empty() const { return Matchers.empty(); }
-
-  std::unique_ptr<PredicateMatcher> popFirstCondition() override {
-    assert(!Conditions.empty() &&
-           "Trying to pop a condition from a condition-less group");
-    std::unique_ptr<PredicateMatcher> P = std::move(Conditions.front());
-    Conditions.erase(Conditions.begin());
-    return P;
-  }
-  const PredicateMatcher &getFirstCondition() const override {
-    assert(!Conditions.empty() &&
-           "Trying to get a condition from a condition-less group");
-    return *Conditions.front();
-  }
-  bool hasFirstCondition() const override { return !Conditions.empty(); }
-
-private:
-  /// See if a candidate matcher could be added to this group solely by
-  /// analyzing its first condition.
-  bool candidateConditionMatches(const PredicateMatcher &Predicate) const;
-};
-
-class SwitchMatcher : public Matcher {
-  /// All the nested matchers, representing distinct switch-cases. The first
-  /// conditions (as Matcher::getFirstCondition() reports) of all the nested
-  /// matchers must share the same type and path to a value they check, in other
-  /// words, be isIdenticalDownToValue, but have different values they check
-  /// against.
-  std::vector<Matcher *> Matchers;
-
-  /// The representative condition, with a type and a path (InsnVarID and OpIdx
-  /// in most cases)  shared by all the matchers contained.
-  std::unique_ptr<PredicateMatcher> Condition = nullptr;
-
-  /// Temporary set used to check that the case values don't repeat within the
-  /// same switch.
-  std::set<MatchTableRecord> Values;
-
-  /// An owning collection for any auxiliary matchers created while optimizing
-  /// nested matchers contained.
-  std::vector<std::unique_ptr<Matcher>> MatcherStorage;
-
-public:
-  bool addMatcher(Matcher &Candidate);
-
-  void finalize();
-  void emit(MatchTable &Table) override;
-
-  iterator_range<std::vector<Matcher *>::iterator> matchers() {
-    return make_range(Matchers.begin(), Matchers.end());
-  }
-  size_t size() const { return Matchers.size(); }
-  bool empty() const { return Matchers.empty(); }
-
-  std::unique_ptr<PredicateMatcher> popFirstCondition() override {
-    // SwitchMatcher doesn't have a common first condition for its cases, as all
-    // the cases only share a kind of a value (a type and a path to it) they
-    // match, but deliberately differ in the actual value they match.
-    llvm_unreachable("Trying to pop a condition from a condition-less group");
-  }
-  const PredicateMatcher &getFirstCondition() const override {
-    llvm_unreachable("Trying to pop a condition from a condition-less group");
-  }
-  bool hasFirstCondition() const override { return false; }
-
-private:
-  /// See if the predicate type has a Switch-implementation for it.
-  static bool isSupportedPredicateType(const PredicateMatcher &Predicate);
-
-  bool candidateConditionMatches(const PredicateMatcher &Predicate) const;
-
-  /// emit()-helper
-  static void emitPredicateSpecificOpcodes(const PredicateMatcher &P,
-                                           MatchTable &Table);
-};
-
-/// Generates code to check that a match rule matches.
-class RuleMatcher : public Matcher {
-public:
-  using ActionList = std::list<std::unique_ptr<MatchAction>>;
-  using action_iterator = ActionList::iterator;
-
-protected:
-  /// A list of matchers that all need to succeed for the current rule to match.
-  /// FIXME: This currently supports a single match position but could be
-  /// extended to support multiple positions to support div/rem fusion or
-  /// load-multiple instructions.
-  using MatchersTy = std::vector<std::unique_ptr<InstructionMatcher>> ;
-  MatchersTy Matchers;
-
-  /// A list of actions that need to be taken when all predicates in this rule
-  /// have succeeded.
-  ActionList Actions;
-
-  using DefinedInsnVariablesMap = std::map<InstructionMatcher *, unsigned>;
-
-  /// A map of instruction matchers to the local variables
-  DefinedInsnVariablesMap InsnVariableIDs;
-
-  using MutatableInsnSet = SmallPtrSet<InstructionMatcher *, 4>;
-
-  // The set of instruction matchers that have not yet been claimed for mutation
-  // by a BuildMI.
-  MutatableInsnSet MutatableInsns;
-
-  /// A map of named operands defined by the matchers that may be referenced by
-  /// the renderers.
-  StringMap<OperandMatcher *> DefinedOperands;
-
-  /// A map of anonymous physical register operands defined by the matchers that
-  /// may be referenced by the renderers.
-  DenseMap<Record *, OperandMatcher *> PhysRegOperands;
-
-  /// ID for the next instruction variable defined with implicitlyDefineInsnVar()
-  unsigned NextInsnVarID;
-
-  /// ID for the next output instruction allocated with allocateOutputInsnID()
-  unsigned NextOutputInsnID;
-
-  /// ID for the next temporary register ID allocated with allocateTempRegID()
-  unsigned NextTempRegID;
-
-  std::vector<Record *> RequiredFeatures;
-  std::vector<std::unique_ptr<PredicateMatcher>> EpilogueMatchers;
-
-  ArrayRef<SMLoc> SrcLoc;
-
-  typedef std::tuple<Record *, unsigned, unsigned>
-      DefinedComplexPatternSubOperand;
-  typedef StringMap<DefinedComplexPatternSubOperand>
-      DefinedComplexPatternSubOperandMap;
-  /// A map of Symbolic Names to ComplexPattern sub-operands.
-  DefinedComplexPatternSubOperandMap ComplexSubOperands;
-  /// A map used to for multiple referenced error check of ComplexSubOperand.
-  /// ComplexSubOperand can't be referenced multiple from different operands,
-  /// however multiple references from same operand are allowed since that is
-  /// how 'same operand checks' are generated.
-  StringMap<std::string> ComplexSubOperandsParentName;
-
-  uint64_t RuleID;
-  static uint64_t NextRuleID;
-
-public:
-  RuleMatcher(ArrayRef<SMLoc> SrcLoc)
-      : NextInsnVarID(0), NextOutputInsnID(0), NextTempRegID(0), SrcLoc(SrcLoc),
-        RuleID(NextRuleID++) {}
-  RuleMatcher(RuleMatcher &&Other) = default;
-  RuleMatcher &operator=(RuleMatcher &&Other) = default;
-
-  uint64_t getRuleID() const { return RuleID; }
-
-  InstructionMatcher &addInstructionMatcher(StringRef SymbolicName);
-  void addRequiredFeature(Record *Feature);
-  const std::vector<Record *> &getRequiredFeatures() const;
-
-  template <class Kind, class... Args> Kind &addAction(Args &&... args);
-  template <class Kind, class... Args>
-  action_iterator insertAction(action_iterator InsertPt, Args &&... args);
-
-  /// Define an instruction without emitting any code to do so.
-  unsigned implicitlyDefineInsnVar(InstructionMatcher &Matcher);
-
-  unsigned getInsnVarID(InstructionMatcher &InsnMatcher) const;
-  DefinedInsnVariablesMap::const_iterator defined_insn_vars_begin() const {
-    return InsnVariableIDs.begin();
-  }
-  DefinedInsnVariablesMap::const_iterator defined_insn_vars_end() const {
-    return InsnVariableIDs.end();
-  }
-  iterator_range<typename DefinedInsnVariablesMap::const_iterator>
-  defined_insn_vars() const {
-    return make_range(defined_insn_vars_begin(), defined_insn_vars_end());
-  }
-
-  MutatableInsnSet::const_iterator mutatable_insns_begin() const {
-    return MutatableInsns.begin();
-  }
-  MutatableInsnSet::const_iterator mutatable_insns_end() const {
-    return MutatableInsns.end();
-  }
-  iterator_range<typename MutatableInsnSet::const_iterator>
-  mutatable_insns() const {
-    return make_range(mutatable_insns_begin(), mutatable_insns_end());
-  }
-  void reserveInsnMatcherForMutation(InstructionMatcher *InsnMatcher) {
-    bool R = MutatableInsns.erase(InsnMatcher);
-    assert(R && "Reserving a mutatable insn that isn't available");
-    (void)R;
-  }
-
-  action_iterator actions_begin() { return Actions.begin(); }
-  action_iterator actions_end() { return Actions.end(); }
-  iterator_range<action_iterator> actions() {
-    return make_range(actions_begin(), actions_end());
-  }
-
-  void defineOperand(StringRef SymbolicName, OperandMatcher &OM);
-
-  void definePhysRegOperand(Record *Reg, OperandMatcher &OM);
-
-  Error defineComplexSubOperand(StringRef SymbolicName, Record *ComplexPattern,
-                                unsigned RendererID, unsigned SubOperandID,
-                                StringRef ParentSymbolicName) {
-    std::string ParentName(ParentSymbolicName);
-    if (ComplexSubOperands.count(SymbolicName)) {
-      const std::string &RecordedParentName =
-          ComplexSubOperandsParentName[SymbolicName];
-      if (RecordedParentName != ParentName)
-        return failedImport("Error: Complex suboperand " + SymbolicName +
-                            " referenced by different operands: " +
-                            RecordedParentName + " and " + ParentName + ".");
-      // Complex suboperand referenced more than once from same the operand is
-      // used to generate 'same operand check'. Emitting of
-      // GIR_ComplexSubOperandRenderer for them is already handled.
-      return Error::success();
-    }
-
-    ComplexSubOperands[SymbolicName] =
-        std::make_tuple(ComplexPattern, RendererID, SubOperandID);
-    ComplexSubOperandsParentName[SymbolicName] = ParentName;
-
-    return Error::success();
-  }
-
-  std::optional<DefinedComplexPatternSubOperand>
-  getComplexSubOperand(StringRef SymbolicName) const {
-    const auto &I = ComplexSubOperands.find(SymbolicName);
-    if (I == ComplexSubOperands.end())
-      return std::nullopt;
-    return I->second;
-  }
-
-  InstructionMatcher &getInstructionMatcher(StringRef SymbolicName) const;
-  const OperandMatcher &getOperandMatcher(StringRef Name) const;
-  const OperandMatcher &getPhysRegOperandMatcher(Record *) const;
-
-  void optimize() override;
-  void emit(MatchTable &Table) override;
-
-  /// Compare the priority of this object and B.
-  ///
-  /// Returns true if this object is more important than B.
-  bool isHigherPriorityThan(const RuleMatcher &B) const;
-
-  /// Report the maximum number of temporary operands needed by the rule
-  /// matcher.
-  unsigned countRendererFns() const;
-
-  std::unique_ptr<PredicateMatcher> popFirstCondition() override;
-  const PredicateMatcher &getFirstCondition() const override;
-  LLTCodeGen getFirstConditionAsRootType();
-  bool hasFirstCondition() const override;
-  unsigned getNumOperands() const;
-  StringRef getOpcode() const;
-
-  // FIXME: Remove this as soon as possible
-  InstructionMatcher &insnmatchers_front() const { return *Matchers.front(); }
-
-  unsigned allocateOutputInsnID() { return NextOutputInsnID++; }
-  unsigned allocateTempRegID() { return NextTempRegID++; }
-
-  iterator_range<MatchersTy::iterator> insnmatchers() {
-    return make_range(Matchers.begin(), Matchers.end());
-  }
-  bool insnmatchers_empty() const { return Matchers.empty(); }
-  void insnmatchers_pop_front() { Matchers.erase(Matchers.begin()); }
-};
-
-uint64_t RuleMatcher::NextRuleID = 0;
-
-using action_iterator = RuleMatcher::action_iterator;
-
-template <class PredicateTy> class PredicateListMatcher {
-private:
-  /// Template instantiations should specialize this to return a string to use
-  /// for the comment emitted when there are no predicates.
-  std::string getNoPredicateComment() const;
-
-protected:
-  using PredicatesTy = std::deque<std::unique_ptr<PredicateTy>>;
-  PredicatesTy Predicates;
-
-  /// Track if the list of predicates was manipulated by one of the optimization
-  /// methods.
-  bool Optimized = false;
-
-public:
-  typename PredicatesTy::iterator predicates_begin() {
-    return Predicates.begin();
-  }
-  typename PredicatesTy::iterator predicates_end() {
-    return Predicates.end();
-  }
-  iterator_range<typename PredicatesTy::iterator> predicates() {
-    return make_range(predicates_begin(), predicates_end());
-  }
-  typename PredicatesTy::size_type predicates_size() const {
-    return Predicates.size();
-  }
-  bool predicates_empty() const { return Predicates.empty(); }
-
-  std::unique_ptr<PredicateTy> predicates_pop_front() {
-    std::unique_ptr<PredicateTy> Front = std::move(Predicates.front());
-    Predicates.pop_front();
-    Optimized = true;
-    return Front;
-  }
-
-  void prependPredicate(std::unique_ptr<PredicateTy> &&Predicate) {
-    Predicates.push_front(std::move(Predicate));
-  }
-
-  void eraseNullPredicates() {
-    const auto NewEnd =
-        std::stable_partition(Predicates.begin(), Predicates.end(),
-                              std::logical_not<std::unique_ptr<PredicateTy>>());
-    if (NewEnd != Predicates.begin()) {
-      Predicates.erase(Predicates.begin(), NewEnd);
-      Optimized = true;
-    }
-  }
-
-  /// Emit MatchTable opcodes that tests whether all the predicates are met.
-  template <class... Args>
-  void emitPredicateListOpcodes(MatchTable &Table, Args &&... args) {
-    if (Predicates.empty() && !Optimized) {
-      Table << MatchTable::Comment(getNoPredicateComment())
-            << MatchTable::LineBreak;
-      return;
-    }
-
-    for (const auto &Predicate : predicates())
-      Predicate->emitPredicateOpcodes(Table, std::forward<Args>(args)...);
-  }
-
-  /// Provide a function to avoid emitting certain predicates. This is used to
-  /// defer some predicate checks until after others
-  using PredicateFilterFunc = std::function<bool(const PredicateTy&)>;
-
-  /// Emit MatchTable opcodes for predicates which satisfy \p
-  /// ShouldEmitPredicate. This should be called multiple times to ensure all
-  /// predicates are eventually added to the match table.
-  template <class... Args>
-  void emitFilteredPredicateListOpcodes(PredicateFilterFunc ShouldEmitPredicate,
-                                        MatchTable &Table, Args &&... args) {
-    if (Predicates.empty() && !Optimized) {
-      Table << MatchTable::Comment(getNoPredicateComment())
-            << MatchTable::LineBreak;
-      return;
-    }
-
-    for (const auto &Predicate : predicates()) {
-      if (ShouldEmitPredicate(*Predicate))
-        Predicate->emitPredicateOpcodes(Table, std::forward<Args>(args)...);
-    }
-  }
-};
-
-class PredicateMatcher {
-public:
-  /// This enum is used for RTTI and also defines the priority that is given to
-  /// the predicate when generating the matcher code. Kinds with higher priority
-  /// must be tested first.
-  ///
-  /// The relative priority of OPM_LLT, OPM_RegBank, and OPM_MBB do not matter
-  /// but OPM_Int must have priority over OPM_RegBank since constant integers
-  /// are represented by a virtual register defined by a G_CONSTANT instruction.
-  ///
-  /// Note: The relative priority between IPM_ and OPM_ does not matter, they
-  /// are currently not compared between each other.
-  enum PredicateKind {
-    IPM_Opcode,
-    IPM_NumOperands,
-    IPM_ImmPredicate,
-    IPM_Imm,
-    IPM_AtomicOrderingMMO,
-    IPM_MemoryLLTSize,
-    IPM_MemoryVsLLTSize,
-    IPM_MemoryAddressSpace,
-    IPM_MemoryAlignment,
-    IPM_VectorSplatImm,
-    IPM_NoUse,
-    IPM_GenericPredicate,
-    OPM_SameOperand,
-    OPM_ComplexPattern,
-    OPM_IntrinsicID,
-    OPM_CmpPredicate,
-    OPM_Instruction,
-    OPM_Int,
-    OPM_LiteralInt,
-    OPM_LLT,
-    OPM_PointerToAny,
-    OPM_RegBank,
-    OPM_MBB,
-    OPM_RecordNamedOperand,
-  };
-
-protected:
-  PredicateKind Kind;
-  unsigned InsnVarID;
-  unsigned OpIdx;
-
-public:
-  PredicateMatcher(PredicateKind Kind, unsigned InsnVarID, unsigned OpIdx = ~0)
-      : Kind(Kind), InsnVarID(InsnVarID), OpIdx(OpIdx) {}
-
-  unsigned getInsnVarID() const { return InsnVarID; }
-  unsigned getOpIdx() const { return OpIdx; }
-
-  virtual ~PredicateMatcher() = default;
-  /// Emit MatchTable opcodes that check the predicate for the given operand.
-  virtual void emitPredicateOpcodes(MatchTable &Table,
-                                    RuleMatcher &Rule) const = 0;
-
-  PredicateKind getKind() const { return Kind; }
-
-  bool dependsOnOperands() const {
-    // Custom predicates really depend on the context pattern of the
-    // instruction, not just the individual instruction. This therefore
-    // implicitly depends on all other pattern constraints.
-    return Kind == IPM_GenericPredicate;
-  }
-
-  virtual bool isIdentical(const PredicateMatcher &B) const {
-    return B.getKind() == getKind() && InsnVarID == B.InsnVarID &&
-           OpIdx == B.OpIdx;
-  }
-
-  virtual bool isIdenticalDownToValue(const PredicateMatcher &B) const {
-    return hasValue() && PredicateMatcher::isIdentical(B);
-  }
-
-  virtual MatchTableRecord getValue() const {
-    assert(hasValue() && "Can not get a value of a value-less predicate!");
-    llvm_unreachable("Not implemented yet");
-  }
-  virtual bool hasValue() const { return false; }
-
-  /// Report the maximum number of temporary operands needed by the predicate
-  /// matcher.
-  virtual unsigned countRendererFns() const { return 0; }
-};
-
-/// Generates code to check a predicate of an operand.
-///
-/// Typical predicates include:
-/// * Operand is a particular register.
-/// * Operand is assigned a particular register bank.
-/// * Operand is an MBB.
-class OperandPredicateMatcher : public PredicateMatcher {
-public:
-  OperandPredicateMatcher(PredicateKind Kind, unsigned InsnVarID,
-                          unsigned OpIdx)
-      : PredicateMatcher(Kind, InsnVarID, OpIdx) {}
-  virtual ~OperandPredicateMatcher() {}
-
-  /// Compare the priority of this object and B.
-  ///
-  /// Returns true if this object is more important than B.
-  virtual bool isHigherPriorityThan(const OperandPredicateMatcher &B) const;
-};
-
-template <>
-std::string
-PredicateListMatcher<OperandPredicateMatcher>::getNoPredicateComment() const {
-  return "No operand predicates";
-}
-
-/// Generates code to check that a register operand is defined by the same exact
-/// one as another.
-class SameOperandMatcher : public OperandPredicateMatcher {
-  std::string MatchingName;
-  unsigned OrigOpIdx;
-
-public:
-  SameOperandMatcher(unsigned InsnVarID, unsigned OpIdx, StringRef MatchingName,
-                     unsigned OrigOpIdx)
-      : OperandPredicateMatcher(OPM_SameOperand, InsnVarID, OpIdx),
-        MatchingName(MatchingName), OrigOpIdx(OrigOpIdx) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_SameOperand;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override;
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return OperandPredicateMatcher::isIdentical(B) &&
-           OrigOpIdx == cast<SameOperandMatcher>(&B)->OrigOpIdx &&
-           MatchingName == cast<SameOperandMatcher>(&B)->MatchingName;
-  }
-};
-
-/// Generates code to check that an operand is a particular LLT.
-class LLTOperandMatcher : public OperandPredicateMatcher {
-protected:
-  LLTCodeGen Ty;
-
-public:
-  static std::map<LLTCodeGen, unsigned> TypeIDValues;
-
-  static void initTypeIDValuesMap() {
-    TypeIDValues.clear();
-
-    unsigned ID = 0;
-    for (const LLTCodeGen &LLTy : KnownTypes)
-      TypeIDValues[LLTy] = ID++;
-  }
-
-  LLTOperandMatcher(unsigned InsnVarID, unsigned OpIdx, const LLTCodeGen &Ty)
-      : OperandPredicateMatcher(OPM_LLT, InsnVarID, OpIdx), Ty(Ty) {
-    KnownTypes.insert(Ty);
-  }
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_LLT;
-  }
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return OperandPredicateMatcher::isIdentical(B) &&
-           Ty == cast<LLTOperandMatcher>(&B)->Ty;
-  }
-  MatchTableRecord getValue() const override {
-    const auto VI = TypeIDValues.find(Ty);
-    if (VI == TypeIDValues.end())
-      return MatchTable::NamedValue(getTy().getCxxEnumValue());
-    return MatchTable::NamedValue(getTy().getCxxEnumValue(), VI->second);
-  }
-  bool hasValue() const override {
-    if (TypeIDValues.size() != KnownTypes.size())
-      initTypeIDValuesMap();
-    return TypeIDValues.count(Ty);
-  }
-
-  LLTCodeGen getTy() const { return Ty; }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckType") << MatchTable::Comment("MI")
-          << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
-          << MatchTable::IntValue(OpIdx) << MatchTable::Comment("Type")
-          << getValue() << MatchTable::LineBreak;
-  }
-};
-
-std::map<LLTCodeGen, unsigned> LLTOperandMatcher::TypeIDValues;
-
-/// Generates code to check that an operand is a pointer to any address space.
-///
-/// In SelectionDAG, the types did not describe pointers or address spaces. As a
-/// result, iN is used to describe a pointer of N bits to any address space and
-/// PatFrag predicates are typically used to constrain the address space. There's
-/// no reliable means to derive the missing type information from the pattern so
-/// imported rules must test the components of a pointer separately.
-///
-/// If SizeInBits is zero, then the pointer size will be obtained from the
-/// subtarget.
-class PointerToAnyOperandMatcher : public OperandPredicateMatcher {
-protected:
-  unsigned SizeInBits;
-
-public:
-  PointerToAnyOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
-                             unsigned SizeInBits)
-      : OperandPredicateMatcher(OPM_PointerToAny, InsnVarID, OpIdx),
-        SizeInBits(SizeInBits) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_PointerToAny;
-  }
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return OperandPredicateMatcher::isIdentical(B) &&
-           SizeInBits == cast<PointerToAnyOperandMatcher>(&B)->SizeInBits;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckPointerToAny")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-          << MatchTable::Comment("SizeInBits")
-          << MatchTable::IntValue(SizeInBits) << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to record named operand in RecordedOperands list at StoreIdx.
-/// Predicates with 'let PredicateCodeUsesOperands = 1' get RecordedOperands as
-/// an argument to predicate's c++ code once all operands have been matched.
-class RecordNamedOperandMatcher : public OperandPredicateMatcher {
-protected:
-  unsigned StoreIdx;
-  std::string Name;
-
-public:
-  RecordNamedOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
-                            unsigned StoreIdx, StringRef Name)
-      : OperandPredicateMatcher(OPM_RecordNamedOperand, InsnVarID, OpIdx),
-        StoreIdx(StoreIdx), Name(Name) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_RecordNamedOperand;
-  }
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return OperandPredicateMatcher::isIdentical(B) &&
-           StoreIdx == cast<RecordNamedOperandMatcher>(&B)->StoreIdx &&
-           Name == cast<RecordNamedOperandMatcher>(&B)->Name;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_RecordNamedOperand")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-          << MatchTable::Comment("StoreIdx") << MatchTable::IntValue(StoreIdx)
-          << MatchTable::Comment("Name : " + Name) << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that an operand is a particular target constant.
-class ComplexPatternOperandMatcher : public OperandPredicateMatcher {
-protected:
-  const OperandMatcher &Operand;
-  const Record &TheDef;
-
-  unsigned getAllocatedTemporariesBaseID() const;
-
-public:
-  bool isIdentical(const PredicateMatcher &B) const override { return false; }
-
-  ComplexPatternOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
-                               const OperandMatcher &Operand,
-                               const Record &TheDef)
-      : OperandPredicateMatcher(OPM_ComplexPattern, InsnVarID, OpIdx),
-        Operand(Operand), TheDef(TheDef) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_ComplexPattern;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    unsigned ID = getAllocatedTemporariesBaseID();
-    Table << MatchTable::Opcode("GIM_CheckComplexPattern")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-          << MatchTable::Comment("Renderer") << MatchTable::IntValue(ID)
-          << MatchTable::NamedValue(("GICP_" + TheDef.getName()).str())
-          << MatchTable::LineBreak;
-  }
-
-  unsigned countRendererFns() const override {
-    return 1;
-  }
-};
-
-/// Generates code to check that an operand is in a particular register bank.
-class RegisterBankOperandMatcher : public OperandPredicateMatcher {
-protected:
-  const CodeGenRegisterClass &RC;
-
-public:
-  RegisterBankOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
-                             const CodeGenRegisterClass &RC)
-      : OperandPredicateMatcher(OPM_RegBank, InsnVarID, OpIdx), RC(RC) {}
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return OperandPredicateMatcher::isIdentical(B) &&
-           RC.getDef() == cast<RegisterBankOperandMatcher>(&B)->RC.getDef();
-  }
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_RegBank;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckRegBankForClass")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-          << MatchTable::Comment("RC")
-          << MatchTable::NamedValue(RC.getQualifiedName() + "RegClassID")
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that an operand is a basic block.
-class MBBOperandMatcher : public OperandPredicateMatcher {
-public:
-  MBBOperandMatcher(unsigned InsnVarID, unsigned OpIdx)
-      : OperandPredicateMatcher(OPM_MBB, InsnVarID, OpIdx) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_MBB;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckIsMBB") << MatchTable::Comment("MI")
-          << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
-          << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
-  }
-};
-
-class ImmOperandMatcher : public OperandPredicateMatcher {
-public:
-  ImmOperandMatcher(unsigned InsnVarID, unsigned OpIdx)
-      : OperandPredicateMatcher(IPM_Imm, InsnVarID, OpIdx) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_Imm;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckIsImm") << MatchTable::Comment("MI")
-          << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
-          << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that an operand is a G_CONSTANT with a particular
-/// int.
-class ConstantIntOperandMatcher : public OperandPredicateMatcher {
-protected:
-  int64_t Value;
-
-public:
-  ConstantIntOperandMatcher(unsigned InsnVarID, unsigned OpIdx, int64_t Value)
-      : OperandPredicateMatcher(OPM_Int, InsnVarID, OpIdx), Value(Value) {}
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return OperandPredicateMatcher::isIdentical(B) &&
-           Value == cast<ConstantIntOperandMatcher>(&B)->Value;
-  }
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_Int;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckConstantInt")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-          << MatchTable::IntValue(Value) << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that an operand is a raw int (where MO.isImm() or
-/// MO.isCImm() is true).
-class LiteralIntOperandMatcher : public OperandPredicateMatcher {
-protected:
-  int64_t Value;
-
-public:
-  LiteralIntOperandMatcher(unsigned InsnVarID, unsigned OpIdx, int64_t Value)
-      : OperandPredicateMatcher(OPM_LiteralInt, InsnVarID, OpIdx),
-        Value(Value) {}
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return OperandPredicateMatcher::isIdentical(B) &&
-           Value == cast<LiteralIntOperandMatcher>(&B)->Value;
-  }
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_LiteralInt;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckLiteralInt")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-          << MatchTable::IntValue(Value) << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that an operand is an CmpInst predicate
-class CmpPredicateOperandMatcher : public OperandPredicateMatcher {
-protected:
-  std::string PredName;
-
-public:
-  CmpPredicateOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
-                             std::string P)
-    : OperandPredicateMatcher(OPM_CmpPredicate, InsnVarID, OpIdx), PredName(P) {}
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return OperandPredicateMatcher::isIdentical(B) &&
-           PredName == cast<CmpPredicateOperandMatcher>(&B)->PredName;
-  }
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_CmpPredicate;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckCmpPredicate")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-          << MatchTable::Comment("Predicate")
-          << MatchTable::NamedValue("CmpInst", PredName)
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that an operand is an intrinsic ID.
-class IntrinsicIDOperandMatcher : public OperandPredicateMatcher {
-protected:
-  const CodeGenIntrinsic *II;
-
-public:
-  IntrinsicIDOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
-                            const CodeGenIntrinsic *II)
-      : OperandPredicateMatcher(OPM_IntrinsicID, InsnVarID, OpIdx), II(II) {}
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return OperandPredicateMatcher::isIdentical(B) &&
-           II == cast<IntrinsicIDOperandMatcher>(&B)->II;
-  }
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_IntrinsicID;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckIntrinsicID")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-          << MatchTable::NamedValue("Intrinsic::" + II->EnumName)
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that this operand is an immediate whose value meets
-/// an immediate predicate.
-class OperandImmPredicateMatcher : public OperandPredicateMatcher {
-protected:
-  TreePredicateFn Predicate;
-
-public:
-  OperandImmPredicateMatcher(unsigned InsnVarID, unsigned OpIdx,
-                             const TreePredicateFn &Predicate)
-      : OperandPredicateMatcher(IPM_ImmPredicate, InsnVarID, OpIdx),
-        Predicate(Predicate) {}
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return OperandPredicateMatcher::isIdentical(B) &&
-           Predicate.getOrigPatFragRecord() ==
-               cast<OperandImmPredicateMatcher>(&B)
-                   ->Predicate.getOrigPatFragRecord();
-  }
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_ImmPredicate;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckImmOperandPredicate")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("MO") << MatchTable::IntValue(OpIdx)
-          << MatchTable::Comment("Predicate")
-          << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that a set of predicates match for a particular
-/// operand.
-class OperandMatcher : public PredicateListMatcher<OperandPredicateMatcher> {
-protected:
-  InstructionMatcher &Insn;
-  unsigned OpIdx;
-  std::string SymbolicName;
-
-  /// The index of the first temporary variable allocated to this operand. The
-  /// number of allocated temporaries can be found with
-  /// countRendererFns().
-  unsigned AllocatedTemporariesBaseID;
-
-public:
-  OperandMatcher(InstructionMatcher &Insn, unsigned OpIdx,
-                 const std::string &SymbolicName,
-                 unsigned AllocatedTemporariesBaseID)
-      : Insn(Insn), OpIdx(OpIdx), SymbolicName(SymbolicName),
-        AllocatedTemporariesBaseID(AllocatedTemporariesBaseID) {}
-
-  bool hasSymbolicName() const { return !SymbolicName.empty(); }
-  StringRef getSymbolicName() const { return SymbolicName; }
-  void setSymbolicName(StringRef Name) {
-    assert(SymbolicName.empty() && "Operand already has a symbolic name");
-    SymbolicName = std::string(Name);
-  }
-
-  /// Construct a new operand predicate and add it to the matcher.
-  template <class Kind, class... Args>
-  std::optional<Kind *> addPredicate(Args &&...args) {
-    if (isSameAsAnotherOperand())
-      return std::nullopt;
-    Predicates.emplace_back(std::make_unique<Kind>(
-        getInsnVarID(), getOpIdx(), std::forward<Args>(args)...));
-    return static_cast<Kind *>(Predicates.back().get());
-  }
-
-  unsigned getOpIdx() const { return OpIdx; }
-  unsigned getInsnVarID() const;
-
-  std::string getOperandExpr(unsigned InsnVarID) const {
-    return "State.MIs[" + llvm::to_string(InsnVarID) + "]->getOperand(" +
-           llvm::to_string(OpIdx) + ")";
-  }
-
-  InstructionMatcher &getInstructionMatcher() const { return Insn; }
-
-  Error addTypeCheckPredicate(const TypeSetByHwMode &VTy,
-                              bool OperandIsAPointer);
-
-  /// Emit MatchTable opcodes that test whether the instruction named in
-  /// InsnVarID matches all the predicates and all the operands.
-  void emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule) {
-    if (!Optimized) {
-      std::string Comment;
-      raw_string_ostream CommentOS(Comment);
-      CommentOS << "MIs[" << getInsnVarID() << "] ";
-      if (SymbolicName.empty())
-        CommentOS << "Operand " << OpIdx;
-      else
-        CommentOS << SymbolicName;
-      Table << MatchTable::Comment(Comment) << MatchTable::LineBreak;
-    }
-
-    emitPredicateListOpcodes(Table, Rule);
-  }
-
-  /// Compare the priority of this object and B.
-  ///
-  /// Returns true if this object is more important than B.
-  bool isHigherPriorityThan(OperandMatcher &B) {
-    // Operand matchers involving more predicates have higher priority.
-    if (predicates_size() > B.predicates_size())
-      return true;
-    if (predicates_size() < B.predicates_size())
-      return false;
-
-    // This assumes that predicates are added in a consistent order.
-    for (auto &&Predicate : zip(predicates(), B.predicates())) {
-      if (std::get<0>(Predicate)->isHigherPriorityThan(*std::get<1>(Predicate)))
-        return true;
-      if (std::get<1>(Predicate)->isHigherPriorityThan(*std::get<0>(Predicate)))
-        return false;
-    }
-
-    return false;
-  };
-
-  /// Report the maximum number of temporary operands needed by the operand
-  /// matcher.
-  unsigned countRendererFns() {
-    return std::accumulate(
-        predicates().begin(), predicates().end(), 0,
-        [](unsigned A,
-           const std::unique_ptr<OperandPredicateMatcher> &Predicate) {
-          return A + Predicate->countRendererFns();
-        });
-  }
-
-  unsigned getAllocatedTemporariesBaseID() const {
-    return AllocatedTemporariesBaseID;
-  }
-
-  bool isSameAsAnotherOperand() {
-    for (const auto &Predicate : predicates())
-      if (isa<SameOperandMatcher>(Predicate))
-        return true;
-    return false;
-  }
-};
-
-Error OperandMatcher::addTypeCheckPredicate(const TypeSetByHwMode &VTy,
-                                            bool OperandIsAPointer) {
-  if (!VTy.isMachineValueType())
-    return failedImport("unsupported typeset");
-
-  if (VTy.getMachineValueType() == MVT::iPTR && OperandIsAPointer) {
-    addPredicate<PointerToAnyOperandMatcher>(0);
-    return Error::success();
-  }
-
-  auto OpTyOrNone = MVTToLLT(VTy.getMachineValueType().SimpleTy);
-  if (!OpTyOrNone)
-    return failedImport("unsupported type");
-
-  if (OperandIsAPointer)
-    addPredicate<PointerToAnyOperandMatcher>(OpTyOrNone->get().getSizeInBits());
-  else if (VTy.isPointer())
-    addPredicate<LLTOperandMatcher>(LLT::pointer(VTy.getPtrAddrSpace(),
-                                                 OpTyOrNone->get().getSizeInBits()));
-  else
-    addPredicate<LLTOperandMatcher>(*OpTyOrNone);
-  return Error::success();
-}
-
-unsigned ComplexPatternOperandMatcher::getAllocatedTemporariesBaseID() const {
-  return Operand.getAllocatedTemporariesBaseID();
-}
-
-/// Generates code to check a predicate on an instruction.
-///
-/// Typical predicates include:
-/// * The opcode of the instruction is a particular value.
-/// * The nsw/nuw flag is/isn't set.
-class InstructionPredicateMatcher : public PredicateMatcher {
-public:
-  InstructionPredicateMatcher(PredicateKind Kind, unsigned InsnVarID)
-      : PredicateMatcher(Kind, InsnVarID) {}
-  virtual ~InstructionPredicateMatcher() {}
-
-  /// Compare the priority of this object and B.
-  ///
-  /// Returns true if this object is more important than B.
-  virtual bool
-  isHigherPriorityThan(const InstructionPredicateMatcher &B) const {
-    return Kind < B.Kind;
-  };
-};
-
-template <>
-std::string
-PredicateListMatcher<PredicateMatcher>::getNoPredicateComment() const {
-  return "No instruction predicates";
-}
-
-/// Generates code to check the opcode of an instruction.
-class InstructionOpcodeMatcher : public InstructionPredicateMatcher {
-protected:
-  // Allow matching one to several, similar opcodes that share properties. This
-  // is to handle patterns where one SelectionDAG operation maps to multiple
-  // GlobalISel ones (e.g. G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC). The first
-  // is treated as the canonical opcode.
-  SmallVector<const CodeGenInstruction *, 2> Insts;
-
-  static DenseMap<const CodeGenInstruction *, unsigned> OpcodeValues;
-
-
-  MatchTableRecord getInstValue(const CodeGenInstruction *I) const {
-    const auto VI = OpcodeValues.find(I);
-    if (VI != OpcodeValues.end())
-      return MatchTable::NamedValue(I->Namespace, I->TheDef->getName(),
-                                    VI->second);
-    return MatchTable::NamedValue(I->Namespace, I->TheDef->getName());
-  }
-
-public:
-  static void initOpcodeValuesMap(const CodeGenTarget &Target) {
-    OpcodeValues.clear();
-
-    unsigned OpcodeValue = 0;
-    for (const CodeGenInstruction *I : Target.getInstructionsByEnumValue())
-      OpcodeValues[I] = OpcodeValue++;
-  }
-
-  InstructionOpcodeMatcher(unsigned InsnVarID,
-                           ArrayRef<const CodeGenInstruction *> I)
-      : InstructionPredicateMatcher(IPM_Opcode, InsnVarID),
-        Insts(I.begin(), I.end()) {
-    assert((Insts.size() == 1 || Insts.size() == 2) &&
-           "unexpected number of opcode alternatives");
-  }
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_Opcode;
-  }
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return InstructionPredicateMatcher::isIdentical(B) &&
-           Insts == cast<InstructionOpcodeMatcher>(&B)->Insts;
-  }
-
-  bool hasValue() const override {
-    return Insts.size() == 1 && OpcodeValues.count(Insts[0]);
-  }
-
-  // TODO: This is used for the SwitchMatcher optimization. We should be able to
-  // return a list of the opcodes to match.
-  MatchTableRecord getValue() const override {
-    assert(Insts.size() == 1);
-
-    const CodeGenInstruction *I = Insts[0];
-    const auto VI = OpcodeValues.find(I);
-    if (VI != OpcodeValues.end())
-      return MatchTable::NamedValue(I->Namespace, I->TheDef->getName(),
-                                    VI->second);
-    return MatchTable::NamedValue(I->Namespace, I->TheDef->getName());
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    StringRef CheckType = Insts.size() == 1 ?
-                          "GIM_CheckOpcode" : "GIM_CheckOpcodeIsEither";
-    Table << MatchTable::Opcode(CheckType) << MatchTable::Comment("MI")
-          << MatchTable::IntValue(InsnVarID);
-
-    for (const CodeGenInstruction *I : Insts)
-      Table << getInstValue(I);
-    Table << MatchTable::LineBreak;
-  }
-
-  /// Compare the priority of this object and B.
-  ///
-  /// Returns true if this object is more important than B.
-  bool
-  isHigherPriorityThan(const InstructionPredicateMatcher &B) const override {
-    if (InstructionPredicateMatcher::isHigherPriorityThan(B))
-      return true;
-    if (B.InstructionPredicateMatcher::isHigherPriorityThan(*this))
-      return false;
-
-    // Prioritize opcodes for cosmetic reasons in the generated source. Although
-    // this is cosmetic at the moment, we may want to drive a similar ordering
-    // using instruction frequency information to improve compile time.
-    if (const InstructionOpcodeMatcher *BO =
-            dyn_cast<InstructionOpcodeMatcher>(&B))
-      return Insts[0]->TheDef->getName() < BO->Insts[0]->TheDef->getName();
-
-    return false;
-  };
-
-  bool isConstantInstruction() const {
-    return Insts.size() == 1 && Insts[0]->TheDef->getName() == "G_CONSTANT";
-  }
-
-  // The first opcode is the canonical opcode, and later are alternatives.
-  StringRef getOpcode() const {
-    return Insts[0]->TheDef->getName();
-  }
-
-  ArrayRef<const CodeGenInstruction *> getAlternativeOpcodes() {
-    return Insts;
-  }
-
-  bool isVariadicNumOperands() const {
-    // If one is variadic, they all should be.
-    return Insts[0]->Operands.isVariadic;
-  }
-
-  StringRef getOperandType(unsigned OpIdx) const {
-    // Types expected to be uniform for all alternatives.
-    return Insts[0]->Operands[OpIdx].OperandType;
-  }
-};
-
-DenseMap<const CodeGenInstruction *, unsigned>
-    InstructionOpcodeMatcher::OpcodeValues;
-
-class InstructionNumOperandsMatcher final : public InstructionPredicateMatcher {
-  unsigned NumOperands = 0;
-
-public:
-  InstructionNumOperandsMatcher(unsigned InsnVarID, unsigned NumOperands)
-      : InstructionPredicateMatcher(IPM_NumOperands, InsnVarID),
-        NumOperands(NumOperands) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_NumOperands;
-  }
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return InstructionPredicateMatcher::isIdentical(B) &&
-           NumOperands == cast<InstructionNumOperandsMatcher>(&B)->NumOperands;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckNumOperands")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("Expected")
-          << MatchTable::IntValue(NumOperands) << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that this instruction is a constant whose value
-/// meets an immediate predicate.
-///
-/// Immediates are slightly odd since they are typically used like an operand
-/// but are represented as an operator internally. We typically write simm8:$src
-/// in a tablegen pattern, but this is just syntactic sugar for
-/// (imm:i32)<<P:Predicate_simm8>>:$imm which more directly describes the nodes
-/// that will be matched and the predicate (which is attached to the imm
-/// operator) that will be tested. In SelectionDAG this describes a
-/// ConstantSDNode whose internal value will be tested using the simm8 predicate.
-///
-/// The corresponding GlobalISel representation is %1 = G_CONSTANT iN Value. In
-/// this representation, the immediate could be tested with an
-/// InstructionMatcher, InstructionOpcodeMatcher, OperandMatcher, and a
-/// OperandPredicateMatcher-subclass to check the Value meets the predicate but
-/// there are two implementation issues with producing that matcher
-/// configuration from the SelectionDAG pattern:
-/// * ImmLeaf is a PatFrag whose root is an InstructionMatcher. This means that
-///   were we to sink the immediate predicate to the operand we would have to
-///   have two partial implementations of PatFrag support, one for immediates
-///   and one for non-immediates.
-/// * At the point we handle the predicate, the OperandMatcher hasn't been
-///   created yet. If we were to sink the predicate to the OperandMatcher we
-///   would also have to complicate (or duplicate) the code that descends and
-///   creates matchers for the subtree.
-/// Overall, it's simpler to handle it in the place it was found.
-class InstructionImmPredicateMatcher : public InstructionPredicateMatcher {
-protected:
-  TreePredicateFn Predicate;
-
-public:
-  InstructionImmPredicateMatcher(unsigned InsnVarID,
-                                 const TreePredicateFn &Predicate)
-      : InstructionPredicateMatcher(IPM_ImmPredicate, InsnVarID),
-        Predicate(Predicate) {}
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return InstructionPredicateMatcher::isIdentical(B) &&
-           Predicate.getOrigPatFragRecord() ==
-               cast<InstructionImmPredicateMatcher>(&B)
-                   ->Predicate.getOrigPatFragRecord();
-  }
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_ImmPredicate;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode(getMatchOpcodeForImmPredicate(Predicate))
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("Predicate")
-          << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that a memory instruction has a atomic ordering
-/// MachineMemoryOperand.
-class AtomicOrderingMMOPredicateMatcher : public InstructionPredicateMatcher {
-public:
-  enum AOComparator {
-    AO_Exactly,
-    AO_OrStronger,
-    AO_WeakerThan,
-  };
-
-protected:
-  StringRef Order;
-  AOComparator Comparator;
-
-public:
-  AtomicOrderingMMOPredicateMatcher(unsigned InsnVarID, StringRef Order,
-                                    AOComparator Comparator = AO_Exactly)
-      : InstructionPredicateMatcher(IPM_AtomicOrderingMMO, InsnVarID),
-        Order(Order), Comparator(Comparator) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_AtomicOrderingMMO;
-  }
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    if (!InstructionPredicateMatcher::isIdentical(B))
-      return false;
-    const auto &R = *cast<AtomicOrderingMMOPredicateMatcher>(&B);
-    return Order == R.Order && Comparator == R.Comparator;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    StringRef Opcode = "GIM_CheckAtomicOrdering";
-
-    if (Comparator == AO_OrStronger)
-      Opcode = "GIM_CheckAtomicOrderingOrStrongerThan";
-    if (Comparator == AO_WeakerThan)
-      Opcode = "GIM_CheckAtomicOrderingWeakerThan";
-
-    Table << MatchTable::Opcode(Opcode) << MatchTable::Comment("MI")
-          << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Order")
-          << MatchTable::NamedValue(("(int64_t)AtomicOrdering::" + Order).str())
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that the size of an MMO is exactly N bytes.
-class MemorySizePredicateMatcher : public InstructionPredicateMatcher {
-protected:
-  unsigned MMOIdx;
-  uint64_t Size;
-
-public:
-  MemorySizePredicateMatcher(unsigned InsnVarID, unsigned MMOIdx, unsigned Size)
-      : InstructionPredicateMatcher(IPM_MemoryLLTSize, InsnVarID),
-        MMOIdx(MMOIdx), Size(Size) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_MemoryLLTSize;
-  }
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return InstructionPredicateMatcher::isIdentical(B) &&
-           MMOIdx == cast<MemorySizePredicateMatcher>(&B)->MMOIdx &&
-           Size == cast<MemorySizePredicateMatcher>(&B)->Size;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckMemorySizeEqualTo")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
-          << MatchTable::Comment("Size") << MatchTable::IntValue(Size)
-          << MatchTable::LineBreak;
-  }
-};
-
-class MemoryAddressSpacePredicateMatcher : public InstructionPredicateMatcher {
-protected:
-  unsigned MMOIdx;
-  SmallVector<unsigned, 4> AddrSpaces;
-
-public:
-  MemoryAddressSpacePredicateMatcher(unsigned InsnVarID, unsigned MMOIdx,
-                                     ArrayRef<unsigned> AddrSpaces)
-      : InstructionPredicateMatcher(IPM_MemoryAddressSpace, InsnVarID),
-        MMOIdx(MMOIdx), AddrSpaces(AddrSpaces.begin(), AddrSpaces.end()) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_MemoryAddressSpace;
-  }
-  bool isIdentical(const PredicateMatcher &B) const override {
-    if (!InstructionPredicateMatcher::isIdentical(B))
-      return false;
-    auto *Other = cast<MemoryAddressSpacePredicateMatcher>(&B);
-    return MMOIdx == Other->MMOIdx && AddrSpaces == Other->AddrSpaces;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckMemoryAddressSpace")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
-        // Encode number of address spaces to expect.
-          << MatchTable::Comment("NumAddrSpace")
-          << MatchTable::IntValue(AddrSpaces.size());
-    for (unsigned AS : AddrSpaces)
-      Table << MatchTable::Comment("AddrSpace") << MatchTable::IntValue(AS);
-
-    Table << MatchTable::LineBreak;
-  }
-};
-
-class MemoryAlignmentPredicateMatcher : public InstructionPredicateMatcher {
-protected:
-  unsigned MMOIdx;
-  int MinAlign;
-
-public:
-  MemoryAlignmentPredicateMatcher(unsigned InsnVarID, unsigned MMOIdx,
-                                  int MinAlign)
-      : InstructionPredicateMatcher(IPM_MemoryAlignment, InsnVarID),
-        MMOIdx(MMOIdx), MinAlign(MinAlign) {
-    assert(MinAlign > 0);
-  }
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_MemoryAlignment;
-  }
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    if (!InstructionPredicateMatcher::isIdentical(B))
-      return false;
-    auto *Other = cast<MemoryAlignmentPredicateMatcher>(&B);
-    return MMOIdx == Other->MMOIdx && MinAlign == Other->MinAlign;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckMemoryAlignment")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
-          << MatchTable::Comment("MinAlign") << MatchTable::IntValue(MinAlign)
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that the size of an MMO is less-than, equal-to, or
-/// greater than a given LLT.
-class MemoryVsLLTSizePredicateMatcher : public InstructionPredicateMatcher {
-public:
-  enum RelationKind {
-    GreaterThan,
-    EqualTo,
-    LessThan,
-  };
-
-protected:
-  unsigned MMOIdx;
-  RelationKind Relation;
-  unsigned OpIdx;
-
-public:
-  MemoryVsLLTSizePredicateMatcher(unsigned InsnVarID, unsigned MMOIdx,
-                                  enum RelationKind Relation,
-                                  unsigned OpIdx)
-      : InstructionPredicateMatcher(IPM_MemoryVsLLTSize, InsnVarID),
-        MMOIdx(MMOIdx), Relation(Relation), OpIdx(OpIdx) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_MemoryVsLLTSize;
-  }
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return InstructionPredicateMatcher::isIdentical(B) &&
-           MMOIdx == cast<MemoryVsLLTSizePredicateMatcher>(&B)->MMOIdx &&
-           Relation == cast<MemoryVsLLTSizePredicateMatcher>(&B)->Relation &&
-           OpIdx == cast<MemoryVsLLTSizePredicateMatcher>(&B)->OpIdx;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode(Relation == EqualTo
-                                    ? "GIM_CheckMemorySizeEqualToLLT"
-                                    : Relation == GreaterThan
-                                          ? "GIM_CheckMemorySizeGreaterThanLLT"
-                                          : "GIM_CheckMemorySizeLessThanLLT")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
-          << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx)
-          << MatchTable::LineBreak;
-  }
-};
-
-// Matcher for immAllOnesV/immAllZerosV
-class VectorSplatImmPredicateMatcher : public InstructionPredicateMatcher {
-public:
-  enum SplatKind {
-    AllZeros,
-    AllOnes
-  };
-
-private:
-  SplatKind Kind;
-
-public:
-  VectorSplatImmPredicateMatcher(unsigned InsnVarID, SplatKind K)
-      : InstructionPredicateMatcher(IPM_VectorSplatImm, InsnVarID), Kind(K) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_VectorSplatImm;
-  }
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return InstructionPredicateMatcher::isIdentical(B) &&
-           Kind == static_cast<const VectorSplatImmPredicateMatcher &>(B).Kind;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    if (Kind == AllOnes)
-      Table << MatchTable::Opcode("GIM_CheckIsBuildVectorAllOnes");
-    else
-      Table << MatchTable::Opcode("GIM_CheckIsBuildVectorAllZeros");
-
-    Table << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID);
-    Table << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check an arbitrary C++ instruction predicate.
-class GenericInstructionPredicateMatcher : public InstructionPredicateMatcher {
-protected:
-  TreePredicateFn Predicate;
-
-public:
-  GenericInstructionPredicateMatcher(unsigned InsnVarID,
-                                     TreePredicateFn Predicate)
-      : InstructionPredicateMatcher(IPM_GenericPredicate, InsnVarID),
-        Predicate(Predicate) {}
-
-  static bool classof(const InstructionPredicateMatcher *P) {
-    return P->getKind() == IPM_GenericPredicate;
-  }
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return InstructionPredicateMatcher::isIdentical(B) &&
-           Predicate ==
-               static_cast<const GenericInstructionPredicateMatcher &>(B)
-                   .Predicate;
-  }
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckCxxInsnPredicate")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::Comment("FnId")
-          << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check for the absence of use of the result.
-// TODO? Generalize this to support checking for one use.
-class NoUsePredicateMatcher : public InstructionPredicateMatcher {
-public:
-  NoUsePredicateMatcher(unsigned InsnVarID)
-      : InstructionPredicateMatcher(IPM_NoUse, InsnVarID) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == IPM_NoUse;
-  }
-
-  bool isIdentical(const PredicateMatcher &B) const override {
-    return InstructionPredicateMatcher::isIdentical(B);
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIM_CheckHasNoUse")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to check that a set of predicates and operands match for a
-/// particular instruction.
-///
-/// Typical predicates include:
-/// * Has a specific opcode.
-/// * Has an nsw/nuw flag or doesn't.
-class InstructionMatcher final : public PredicateListMatcher<PredicateMatcher> {
-protected:
-  typedef std::vector<std::unique_ptr<OperandMatcher>> OperandVec;
-
-  RuleMatcher &Rule;
-
-  /// The operands to match. All rendered operands must be present even if the
-  /// condition is always true.
-  OperandVec Operands;
-  bool NumOperandsCheck = true;
-
-  std::string SymbolicName;
-  unsigned InsnVarID;
-
-  /// PhysRegInputs - List list has an entry for each explicitly specified
-  /// physreg input to the pattern.  The first elt is the Register node, the
-  /// second is the recorded slot number the input pattern match saved it in.
-  SmallVector<std::pair<Record *, unsigned>, 2> PhysRegInputs;
-
-public:
-  InstructionMatcher(RuleMatcher &Rule, StringRef SymbolicName,
-                     bool NumOpsCheck = true)
-      : Rule(Rule), NumOperandsCheck(NumOpsCheck), SymbolicName(SymbolicName) {
-    // We create a new instruction matcher.
-    // Get a new ID for that instruction.
-    InsnVarID = Rule.implicitlyDefineInsnVar(*this);
-  }
-
-  /// Construct a new instruction predicate and add it to the matcher.
-  template <class Kind, class... Args>
-  std::optional<Kind *> addPredicate(Args &&...args) {
-    Predicates.emplace_back(
-        std::make_unique<Kind>(getInsnVarID(), std::forward<Args>(args)...));
-    return static_cast<Kind *>(Predicates.back().get());
-  }
-
-  RuleMatcher &getRuleMatcher() const { return Rule; }
-
-  unsigned getInsnVarID() const { return InsnVarID; }
-
-  /// Add an operand to the matcher.
-  OperandMatcher &addOperand(unsigned OpIdx, const std::string &SymbolicName,
-                             unsigned AllocatedTemporariesBaseID) {
-    Operands.emplace_back(new OperandMatcher(*this, OpIdx, SymbolicName,
-                                             AllocatedTemporariesBaseID));
-    if (!SymbolicName.empty())
-      Rule.defineOperand(SymbolicName, *Operands.back());
-
-    return *Operands.back();
-  }
-
-  OperandMatcher &getOperand(unsigned OpIdx) {
-    auto I = llvm::find_if(Operands,
-                           [&OpIdx](const std::unique_ptr<OperandMatcher> &X) {
-                             return X->getOpIdx() == OpIdx;
-                           });
-    if (I != Operands.end())
-      return **I;
-    llvm_unreachable("Failed to lookup operand");
-  }
-
-  OperandMatcher &addPhysRegInput(Record *Reg, unsigned OpIdx,
-                                  unsigned TempOpIdx) {
-    assert(SymbolicName.empty());
-    OperandMatcher *OM = new OperandMatcher(*this, OpIdx, "", TempOpIdx);
-    Operands.emplace_back(OM);
-    Rule.definePhysRegOperand(Reg, *OM);
-    PhysRegInputs.emplace_back(Reg, OpIdx);
-    return *OM;
-  }
-
-  ArrayRef<std::pair<Record *, unsigned>> getPhysRegInputs() const {
-    return PhysRegInputs;
-  }
-
-  StringRef getSymbolicName() const { return SymbolicName; }
-  unsigned getNumOperands() const { return Operands.size(); }
-  OperandVec::iterator operands_begin() { return Operands.begin(); }
-  OperandVec::iterator operands_end() { return Operands.end(); }
-  iterator_range<OperandVec::iterator> operands() {
-    return make_range(operands_begin(), operands_end());
-  }
-  OperandVec::const_iterator operands_begin() const { return Operands.begin(); }
-  OperandVec::const_iterator operands_end() const { return Operands.end(); }
-  iterator_range<OperandVec::const_iterator> operands() const {
-    return make_range(operands_begin(), operands_end());
-  }
-  bool operands_empty() const { return Operands.empty(); }
-
-  void pop_front() { Operands.erase(Operands.begin()); }
-
-  void optimize();
-
-  /// Emit MatchTable opcodes that test whether the instruction named in
-  /// InsnVarName matches all the predicates and all the operands.
-  void emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule) {
-    if (NumOperandsCheck)
-      InstructionNumOperandsMatcher(InsnVarID, getNumOperands())
-          .emitPredicateOpcodes(Table, Rule);
-
-    // First emit all instruction level predicates need to be verified before we
-    // can verify operands.
-    emitFilteredPredicateListOpcodes(
-      [](const PredicateMatcher &P) {
-        return !P.dependsOnOperands();
-      }, Table, Rule);
-
-    // Emit all operand constraints.
-    for (const auto &Operand : Operands)
-      Operand->emitPredicateOpcodes(Table, Rule);
-
-    // All of the tablegen defined predicates should now be matched. Now emit
-    // any custom predicates that rely on all generated checks.
-    emitFilteredPredicateListOpcodes(
-      [](const PredicateMatcher &P) {
-        return P.dependsOnOperands();
-      }, Table, Rule);
-  }
-
-  /// Compare the priority of this object and B.
-  ///
-  /// Returns true if this object is more important than B.
-  bool isHigherPriorityThan(InstructionMatcher &B) {
-    // Instruction matchers involving more operands have higher priority.
-    if (Operands.size() > B.Operands.size())
-      return true;
-    if (Operands.size() < B.Operands.size())
-      return false;
-
-    for (auto &&P : zip(predicates(), B.predicates())) {
-      auto L = static_cast<InstructionPredicateMatcher *>(std::get<0>(P).get());
-      auto R = static_cast<InstructionPredicateMatcher *>(std::get<1>(P).get());
-      if (L->isHigherPriorityThan(*R))
-        return true;
-      if (R->isHigherPriorityThan(*L))
-        return false;
-    }
-
-    for (auto Operand : zip(Operands, B.Operands)) {
-      if (std::get<0>(Operand)->isHigherPriorityThan(*std::get<1>(Operand)))
-        return true;
-      if (std::get<1>(Operand)->isHigherPriorityThan(*std::get<0>(Operand)))
-        return false;
-    }
-
-    return false;
-  };
-
-  /// Report the maximum number of temporary operands needed by the instruction
-  /// matcher.
-  unsigned countRendererFns() {
-    return std::accumulate(
-               predicates().begin(), predicates().end(), 0,
-               [](unsigned A,
-                  const std::unique_ptr<PredicateMatcher> &Predicate) {
-                 return A + Predicate->countRendererFns();
-               }) +
-           std::accumulate(
-               Operands.begin(), Operands.end(), 0,
-               [](unsigned A, const std::unique_ptr<OperandMatcher> &Operand) {
-                 return A + Operand->countRendererFns();
-               });
-  }
-
-  InstructionOpcodeMatcher &getOpcodeMatcher() {
-    for (auto &P : predicates())
-      if (auto *OpMatcher = dyn_cast<InstructionOpcodeMatcher>(P.get()))
-        return *OpMatcher;
-    llvm_unreachable("Didn't find an opcode matcher");
-  }
-
-  bool isConstantInstruction() {
-    return getOpcodeMatcher().isConstantInstruction();
-  }
-
-  StringRef getOpcode() { return getOpcodeMatcher().getOpcode(); }
-};
-
-StringRef RuleMatcher::getOpcode() const {
-  return Matchers.front()->getOpcode();
-}
-
-unsigned RuleMatcher::getNumOperands() const {
-  return Matchers.front()->getNumOperands();
-}
-
-LLTCodeGen RuleMatcher::getFirstConditionAsRootType() {
-  InstructionMatcher &InsnMatcher = *Matchers.front();
-  if (!InsnMatcher.predicates_empty())
-    if (const auto *TM =
-            dyn_cast<LLTOperandMatcher>(&**InsnMatcher.predicates_begin()))
-      if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0)
-        return TM->getTy();
-  return {};
-}
-
-/// Generates code to check that the operand is a register defined by an
-/// instruction that matches the given instruction matcher.
-///
-/// For example, the pattern:
-///   (set $dst, (G_MUL (G_ADD $src1, $src2), $src3))
-/// would use an InstructionOperandMatcher for operand 1 of the G_MUL to match
-/// the:
-///   (G_ADD $src1, $src2)
-/// subpattern.
-class InstructionOperandMatcher : public OperandPredicateMatcher {
-protected:
-  std::unique_ptr<InstructionMatcher> InsnMatcher;
-
-public:
-  InstructionOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
-                            RuleMatcher &Rule, StringRef SymbolicName,
-                            bool NumOpsCheck = true)
-      : OperandPredicateMatcher(OPM_Instruction, InsnVarID, OpIdx),
-        InsnMatcher(new InstructionMatcher(Rule, SymbolicName, NumOpsCheck)) {}
-
-  static bool classof(const PredicateMatcher *P) {
-    return P->getKind() == OPM_Instruction;
-  }
-
-  InstructionMatcher &getInsnMatcher() const { return *InsnMatcher; }
-
-  void emitCaptureOpcodes(MatchTable &Table, RuleMatcher &Rule) const {
-    const unsigned NewInsnVarID = InsnMatcher->getInsnVarID();
-    Table << MatchTable::Opcode("GIM_RecordInsn")
-          << MatchTable::Comment("DefineMI")
-          << MatchTable::IntValue(NewInsnVarID) << MatchTable::Comment("MI")
-          << MatchTable::IntValue(getInsnVarID())
-          << MatchTable::Comment("OpIdx") << MatchTable::IntValue(getOpIdx())
-          << MatchTable::Comment("MIs[" + llvm::to_string(NewInsnVarID) + "]")
-          << MatchTable::LineBreak;
-  }
-
-  void emitPredicateOpcodes(MatchTable &Table,
-                            RuleMatcher &Rule) const override {
-    emitCaptureOpcodes(Table, Rule);
-    InsnMatcher->emitPredicateOpcodes(Table, Rule);
-  }
-
-  bool isHigherPriorityThan(const OperandPredicateMatcher &B) const override {
-    if (OperandPredicateMatcher::isHigherPriorityThan(B))
-      return true;
-    if (B.OperandPredicateMatcher::isHigherPriorityThan(*this))
-      return false;
-
-    if (const InstructionOperandMatcher *BP =
-            dyn_cast<InstructionOperandMatcher>(&B))
-      if (InsnMatcher->isHigherPriorityThan(*BP->InsnMatcher))
-        return true;
-    return false;
-  }
-
-  /// Report the maximum number of temporary operands needed by the predicate
-  /// matcher.
-  unsigned countRendererFns() const override {
-    return InsnMatcher->countRendererFns();
-  }
-};
-
-void InstructionMatcher::optimize() {
-  SmallVector<std::unique_ptr<PredicateMatcher>, 8> Stash;
-  const auto &OpcMatcher = getOpcodeMatcher();
-
-  Stash.push_back(predicates_pop_front());
-  if (Stash.back().get() == &OpcMatcher) {
-    if (NumOperandsCheck && OpcMatcher.isVariadicNumOperands())
-      Stash.emplace_back(
-          new InstructionNumOperandsMatcher(InsnVarID, getNumOperands()));
-    NumOperandsCheck = false;
-
-    for (auto &OM : Operands)
-      for (auto &OP : OM->predicates())
-        if (isa<IntrinsicIDOperandMatcher>(OP)) {
-          Stash.push_back(std::move(OP));
-          OM->eraseNullPredicates();
-          break;
-        }
-  }
-
-  if (InsnVarID > 0) {
-    assert(!Operands.empty() && "Nested instruction is expected to def a vreg");
-    for (auto &OP : Operands[0]->predicates())
-      OP.reset();
-    Operands[0]->eraseNullPredicates();
-  }
-  for (auto &OM : Operands) {
-    for (auto &OP : OM->predicates())
-      if (isa<LLTOperandMatcher>(OP))
-        Stash.push_back(std::move(OP));
-    OM->eraseNullPredicates();
-  }
-  while (!Stash.empty())
-    prependPredicate(Stash.pop_back_val());
-}
-
-//===- Actions ------------------------------------------------------------===//
-class OperandRenderer {
-public:
-  enum RendererKind {
-    OR_Copy,
-    OR_CopyOrAddZeroReg,
-    OR_CopySubReg,
-    OR_CopyPhysReg,
-    OR_CopyConstantAsImm,
-    OR_CopyFConstantAsFPImm,
-    OR_Imm,
-    OR_SubRegIndex,
-    OR_Register,
-    OR_TempRegister,
-    OR_ComplexPattern,
-    OR_Custom,
-    OR_CustomOperand
-  };
-
-protected:
-  RendererKind Kind;
-
-public:
-  OperandRenderer(RendererKind Kind) : Kind(Kind) {}
-  virtual ~OperandRenderer() {}
-
-  RendererKind getKind() const { return Kind; }
-
-  virtual void emitRenderOpcodes(MatchTable &Table,
-                                 RuleMatcher &Rule) const = 0;
-};
-
-/// A CopyRenderer emits code to copy a single operand from an existing
-/// instruction to the one being built.
-class CopyRenderer : public OperandRenderer {
-protected:
-  unsigned NewInsnID;
-  /// The name of the operand.
-  const StringRef SymbolicName;
-
-public:
-  CopyRenderer(unsigned NewInsnID, StringRef SymbolicName)
-      : OperandRenderer(OR_Copy), NewInsnID(NewInsnID),
-        SymbolicName(SymbolicName) {
-    assert(!SymbolicName.empty() && "Cannot copy from an unspecified source");
-  }
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_Copy;
-  }
-
-  StringRef getSymbolicName() const { return SymbolicName; }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
-    unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
-    Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID")
-          << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID")
-          << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
-          << MatchTable::IntValue(Operand.getOpIdx())
-          << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
-  }
-};
-
-/// A CopyRenderer emits code to copy a virtual register to a specific physical
-/// register.
-class CopyPhysRegRenderer : public OperandRenderer {
-protected:
-  unsigned NewInsnID;
-  Record *PhysReg;
-
-public:
-  CopyPhysRegRenderer(unsigned NewInsnID, Record *Reg)
-      : OperandRenderer(OR_CopyPhysReg), NewInsnID(NewInsnID),
-        PhysReg(Reg) {
-    assert(PhysReg);
-  }
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_CopyPhysReg;
-  }
-
-  Record *getPhysReg() const { return PhysReg; }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    const OperandMatcher &Operand = Rule.getPhysRegOperandMatcher(PhysReg);
-    unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
-    Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID")
-          << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID")
-          << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
-          << MatchTable::IntValue(Operand.getOpIdx())
-          << MatchTable::Comment(PhysReg->getName())
-          << MatchTable::LineBreak;
-  }
-};
-
-/// A CopyOrAddZeroRegRenderer emits code to copy a single operand from an
-/// existing instruction to the one being built. If the operand turns out to be
-/// a 'G_CONSTANT 0' then it replaces the operand with a zero register.
-class CopyOrAddZeroRegRenderer : public OperandRenderer {
-protected:
-  unsigned NewInsnID;
-  /// The name of the operand.
-  const StringRef SymbolicName;
-  const Record *ZeroRegisterDef;
-
-public:
-  CopyOrAddZeroRegRenderer(unsigned NewInsnID,
-                           StringRef SymbolicName, Record *ZeroRegisterDef)
-      : OperandRenderer(OR_CopyOrAddZeroReg), NewInsnID(NewInsnID),
-        SymbolicName(SymbolicName), ZeroRegisterDef(ZeroRegisterDef) {
-    assert(!SymbolicName.empty() && "Cannot copy from an unspecified source");
-  }
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_CopyOrAddZeroReg;
-  }
-
-  StringRef getSymbolicName() const { return SymbolicName; }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
-    unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
-    Table << MatchTable::Opcode("GIR_CopyOrAddZeroReg")
-          << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
-          << MatchTable::Comment("OldInsnID")
-          << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
-          << MatchTable::IntValue(Operand.getOpIdx())
-          << MatchTable::NamedValue(
-                 (ZeroRegisterDef->getValue("Namespace")
-                      ? ZeroRegisterDef->getValueAsString("Namespace")
-                      : ""),
-                 ZeroRegisterDef->getName())
-          << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
-  }
-};
-
-/// A CopyConstantAsImmRenderer emits code to render a G_CONSTANT instruction to
-/// an extended immediate operand.
-class CopyConstantAsImmRenderer : public OperandRenderer {
-protected:
-  unsigned NewInsnID;
-  /// The name of the operand.
-  const std::string SymbolicName;
-  bool Signed;
-
-public:
-  CopyConstantAsImmRenderer(unsigned NewInsnID, StringRef SymbolicName)
-      : OperandRenderer(OR_CopyConstantAsImm), NewInsnID(NewInsnID),
-        SymbolicName(SymbolicName), Signed(true) {}
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_CopyConstantAsImm;
-  }
-
-  StringRef getSymbolicName() const { return SymbolicName; }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
-    unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
-    Table << MatchTable::Opcode(Signed ? "GIR_CopyConstantAsSImm"
-                                       : "GIR_CopyConstantAsUImm")
-          << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
-          << MatchTable::Comment("OldInsnID")
-          << MatchTable::IntValue(OldInsnVarID)
-          << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
-  }
-};
-
-/// A CopyFConstantAsFPImmRenderer emits code to render a G_FCONSTANT
-/// instruction to an extended immediate operand.
-class CopyFConstantAsFPImmRenderer : public OperandRenderer {
-protected:
-  unsigned NewInsnID;
-  /// The name of the operand.
-  const std::string SymbolicName;
-
-public:
-  CopyFConstantAsFPImmRenderer(unsigned NewInsnID, StringRef SymbolicName)
-      : OperandRenderer(OR_CopyFConstantAsFPImm), NewInsnID(NewInsnID),
-        SymbolicName(SymbolicName) {}
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_CopyFConstantAsFPImm;
-  }
-
-  StringRef getSymbolicName() const { return SymbolicName; }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
-    unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
-    Table << MatchTable::Opcode("GIR_CopyFConstantAsFPImm")
-          << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
-          << MatchTable::Comment("OldInsnID")
-          << MatchTable::IntValue(OldInsnVarID)
-          << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
-  }
-};
-
-/// A CopySubRegRenderer emits code to copy a single register operand from an
-/// existing instruction to the one being built and indicate that only a
-/// subregister should be copied.
-class CopySubRegRenderer : public OperandRenderer {
-protected:
-  unsigned NewInsnID;
-  /// The name of the operand.
-  const StringRef SymbolicName;
-  /// The subregister to extract.
-  const CodeGenSubRegIndex *SubReg;
-
-public:
-  CopySubRegRenderer(unsigned NewInsnID, StringRef SymbolicName,
-                     const CodeGenSubRegIndex *SubReg)
-      : OperandRenderer(OR_CopySubReg), NewInsnID(NewInsnID),
-        SymbolicName(SymbolicName), SubReg(SubReg) {}
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_CopySubReg;
-  }
-
-  StringRef getSymbolicName() const { return SymbolicName; }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
-    unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
-    Table << MatchTable::Opcode("GIR_CopySubReg")
-          << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
-          << MatchTable::Comment("OldInsnID")
-          << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
-          << MatchTable::IntValue(Operand.getOpIdx())
-          << MatchTable::Comment("SubRegIdx")
-          << MatchTable::IntValue(SubReg->EnumValue)
-          << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
-  }
-};
-
-/// Adds a specific physical register to the instruction being built.
-/// This is typically useful for WZR/XZR on AArch64.
-class AddRegisterRenderer : public OperandRenderer {
-protected:
-  unsigned InsnID;
-  const Record *RegisterDef;
-  bool IsDef;
-  const CodeGenTarget &Target;
-
-public:
-  AddRegisterRenderer(unsigned InsnID, const CodeGenTarget &Target,
-                      const Record *RegisterDef, bool IsDef = false)
-      : OperandRenderer(OR_Register), InsnID(InsnID), RegisterDef(RegisterDef),
-        IsDef(IsDef), Target(Target) {}
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_Register;
-  }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIR_AddRegister")
-          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID);
-    if (RegisterDef->getName() != "zero_reg") {
-      Table << MatchTable::NamedValue(
-                   (RegisterDef->getValue("Namespace")
-                        ? RegisterDef->getValueAsString("Namespace")
-                        : ""),
-                   RegisterDef->getName());
-    } else {
-      Table << MatchTable::NamedValue(Target.getRegNamespace(), "NoRegister");
-    }
-    Table << MatchTable::Comment("AddRegisterRegFlags");
-
-    // TODO: This is encoded as a 64-bit element, but only 16 or 32-bits are
-    // really needed for a physical register reference. We can pack the
-    // register and flags in a single field.
-    if (IsDef)
-      Table << MatchTable::NamedValue("RegState::Define");
-    else
-      Table << MatchTable::IntValue(0);
-    Table << MatchTable::LineBreak;
-  }
-};
-
-/// Adds a specific temporary virtual register to the instruction being built.
-/// This is used to chain instructions together when emitting multiple
-/// instructions.
-class TempRegRenderer : public OperandRenderer {
-protected:
-  unsigned InsnID;
-  unsigned TempRegID;
-  const CodeGenSubRegIndex *SubRegIdx;
-  bool IsDef;
-  bool IsDead;
-
-public:
-  TempRegRenderer(unsigned InsnID, unsigned TempRegID, bool IsDef = false,
-                  const CodeGenSubRegIndex *SubReg = nullptr,
-                  bool IsDead = false)
-      : OperandRenderer(OR_Register), InsnID(InsnID), TempRegID(TempRegID),
-        SubRegIdx(SubReg), IsDef(IsDef), IsDead(IsDead) {}
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_TempRegister;
-  }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    if (SubRegIdx) {
-      assert(!IsDef);
-      Table << MatchTable::Opcode("GIR_AddTempSubRegister");
-    } else
-      Table << MatchTable::Opcode("GIR_AddTempRegister");
-
-    Table << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-          << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
-          << MatchTable::Comment("TempRegFlags");
-
-    if (IsDef) {
-      SmallString<32> RegFlags;
-      RegFlags += "RegState::Define";
-      if (IsDead)
-        RegFlags += "|RegState::Dead";
-      Table << MatchTable::NamedValue(RegFlags);
-    } else
-      Table << MatchTable::IntValue(0);
-
-    if (SubRegIdx)
-      Table << MatchTable::NamedValue(SubRegIdx->getQualifiedName());
-    Table << MatchTable::LineBreak;
-  }
-};
-
-/// Adds a specific immediate to the instruction being built.
-class ImmRenderer : public OperandRenderer {
-protected:
-  unsigned InsnID;
-  int64_t Imm;
-
-public:
-  ImmRenderer(unsigned InsnID, int64_t Imm)
-      : OperandRenderer(OR_Imm), InsnID(InsnID), Imm(Imm) {}
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_Imm;
-  }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID")
-          << MatchTable::IntValue(InsnID) << MatchTable::Comment("Imm")
-          << MatchTable::IntValue(Imm) << MatchTable::LineBreak;
-  }
-};
-
-/// Adds an enum value for a subreg index to the instruction being built.
-class SubRegIndexRenderer : public OperandRenderer {
-protected:
-  unsigned InsnID;
-  const CodeGenSubRegIndex *SubRegIdx;
-
-public:
-  SubRegIndexRenderer(unsigned InsnID, const CodeGenSubRegIndex *SRI)
-      : OperandRenderer(OR_SubRegIndex), InsnID(InsnID), SubRegIdx(SRI) {}
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_SubRegIndex;
-  }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID")
-          << MatchTable::IntValue(InsnID) << MatchTable::Comment("SubRegIndex")
-          << MatchTable::IntValue(SubRegIdx->EnumValue)
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Adds operands by calling a renderer function supplied by the ComplexPattern
-/// matcher function.
-class RenderComplexPatternOperand : public OperandRenderer {
-private:
-  unsigned InsnID;
-  const Record &TheDef;
-  /// The name of the operand.
-  const StringRef SymbolicName;
-  /// The renderer number. This must be unique within a rule since it's used to
-  /// identify a temporary variable to hold the renderer function.
-  unsigned RendererID;
-  /// When provided, this is the suboperand of the ComplexPattern operand to
-  /// render. Otherwise all the suboperands will be rendered.
-  std::optional<unsigned> SubOperand;
-
-  unsigned getNumOperands() const {
-    return TheDef.getValueAsDag("Operands")->getNumArgs();
-  }
-
-public:
-  RenderComplexPatternOperand(unsigned InsnID, const Record &TheDef,
-                              StringRef SymbolicName, unsigned RendererID,
-                              std::optional<unsigned> SubOperand = std::nullopt)
-      : OperandRenderer(OR_ComplexPattern), InsnID(InsnID), TheDef(TheDef),
-        SymbolicName(SymbolicName), RendererID(RendererID),
-        SubOperand(SubOperand) {}
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_ComplexPattern;
-  }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode(SubOperand ? "GIR_ComplexSubOperandRenderer"
-                                           : "GIR_ComplexRenderer")
-          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-          << MatchTable::Comment("RendererID")
-          << MatchTable::IntValue(RendererID);
-    if (SubOperand)
-      Table << MatchTable::Comment("SubOperand")
-            << MatchTable::IntValue(*SubOperand);
-    Table << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
-  }
-};
-
-class CustomRenderer : public OperandRenderer {
-protected:
-  unsigned InsnID;
-  const Record &Renderer;
-  /// The name of the operand.
-  const std::string SymbolicName;
-
-public:
-  CustomRenderer(unsigned InsnID, const Record &Renderer,
-                 StringRef SymbolicName)
-      : OperandRenderer(OR_Custom), InsnID(InsnID), Renderer(Renderer),
-        SymbolicName(SymbolicName) {}
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_Custom;
-  }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
-    unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
-    Table << MatchTable::Opcode("GIR_CustomRenderer")
-          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-          << MatchTable::Comment("OldInsnID")
-          << MatchTable::IntValue(OldInsnVarID)
-          << MatchTable::Comment("Renderer")
-          << MatchTable::NamedValue(
-                 "GICR_" + Renderer.getValueAsString("RendererFn").str())
-          << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
-  }
-};
-
-class CustomOperandRenderer : public OperandRenderer {
-protected:
-  unsigned InsnID;
-  const Record &Renderer;
-  /// The name of the operand.
-  const std::string SymbolicName;
-
-public:
-  CustomOperandRenderer(unsigned InsnID, const Record &Renderer,
-                        StringRef SymbolicName)
-      : OperandRenderer(OR_CustomOperand), InsnID(InsnID), Renderer(Renderer),
-        SymbolicName(SymbolicName) {}
-
-  static bool classof(const OperandRenderer *R) {
-    return R->getKind() == OR_CustomOperand;
-  }
-
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    const OperandMatcher &OpdMatcher = Rule.getOperandMatcher(SymbolicName);
-    Table << MatchTable::Opcode("GIR_CustomOperandRenderer")
-          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-          << MatchTable::Comment("OldInsnID")
-          << MatchTable::IntValue(OpdMatcher.getInsnVarID())
-          << MatchTable::Comment("OpIdx")
-          << MatchTable::IntValue(OpdMatcher.getOpIdx())
-          << MatchTable::Comment("OperandRenderer")
-          << MatchTable::NamedValue(
-            "GICR_" + Renderer.getValueAsString("RendererFn").str())
-          << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
-  }
-};
-
-/// An action taken when all Matcher predicates succeeded for a parent rule.
-///
-/// Typical actions include:
-/// * Changing the opcode of an instruction.
-/// * Adding an operand to an instruction.
-class MatchAction {
-public:
-  virtual ~MatchAction() {}
-
-  /// Emit the MatchTable opcodes to implement the action.
-  virtual void emitActionOpcodes(MatchTable &Table,
-                                 RuleMatcher &Rule) const = 0;
-};
-
-/// Generates a comment describing the matched rule being acted upon.
-class DebugCommentAction : public MatchAction {
-private:
-  std::string S;
-
-public:
-  DebugCommentAction(StringRef S) : S(std::string(S)) {}
-
-  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    Table << MatchTable::Comment(S) << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to build an instruction or mutate an existing instruction
-/// into the desired instruction when this is possible.
-class BuildMIAction : public MatchAction {
-private:
-  unsigned InsnID;
-  const CodeGenInstruction *I;
-  InstructionMatcher *Matched;
-  std::vector<std::unique_ptr<OperandRenderer>> OperandRenderers;
-
-  /// True if the instruction can be built solely by mutating the opcode.
-  bool canMutate(RuleMatcher &Rule, const InstructionMatcher *Insn) const {
-    if (!Insn)
-      return false;
-
-    if (OperandRenderers.size() != Insn->getNumOperands())
-      return false;
-
-    for (const auto &Renderer : enumerate(OperandRenderers)) {
-      if (const auto *Copy = dyn_cast<CopyRenderer>(&*Renderer.value())) {
-        const OperandMatcher &OM = Rule.getOperandMatcher(Copy->getSymbolicName());
-        if (Insn != &OM.getInstructionMatcher() ||
-            OM.getOpIdx() != Renderer.index())
-          return false;
-      } else
-        return false;
-    }
-
-    return true;
-  }
-
-public:
-  BuildMIAction(unsigned InsnID, const CodeGenInstruction *I)
-      : InsnID(InsnID), I(I), Matched(nullptr) {}
-
-  unsigned getInsnID() const { return InsnID; }
-  const CodeGenInstruction *getCGI() const { return I; }
-
-  void chooseInsnToMutate(RuleMatcher &Rule) {
-    for (auto *MutateCandidate : Rule.mutatable_insns()) {
-      if (canMutate(Rule, MutateCandidate)) {
-        // Take the first one we're offered that we're able to mutate.
-        Rule.reserveInsnMatcherForMutation(MutateCandidate);
-        Matched = MutateCandidate;
-        return;
-      }
-    }
-  }
-
-  template <class Kind, class... Args>
-  Kind &addRenderer(Args&&... args) {
-    OperandRenderers.emplace_back(
-        std::make_unique<Kind>(InsnID, std::forward<Args>(args)...));
-    return *static_cast<Kind *>(OperandRenderers.back().get());
-  }
-
-  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    if (Matched) {
-      assert(canMutate(Rule, Matched) &&
-             "Arranged to mutate an insn that isn't mutatable");
-
-      unsigned RecycleInsnID = Rule.getInsnVarID(*Matched);
-      Table << MatchTable::Opcode("GIR_MutateOpcode")
-            << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-            << MatchTable::Comment("RecycleInsnID")
-            << MatchTable::IntValue(RecycleInsnID)
-            << MatchTable::Comment("Opcode")
-            << MatchTable::NamedValue(I->Namespace, I->TheDef->getName())
-            << MatchTable::LineBreak;
-
-      if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) {
-        for (auto *Def : I->ImplicitDefs) {
-          auto Namespace = Def->getValue("Namespace")
-                               ? Def->getValueAsString("Namespace")
-                               : "";
-          Table << MatchTable::Opcode("GIR_AddImplicitDef")
-                << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-                << MatchTable::NamedValue(Namespace, Def->getName())
-                << MatchTable::LineBreak;
-        }
-        for (auto *Use : I->ImplicitUses) {
-          auto Namespace = Use->getValue("Namespace")
-                               ? Use->getValueAsString("Namespace")
-                               : "";
-          Table << MatchTable::Opcode("GIR_AddImplicitUse")
-                << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-                << MatchTable::NamedValue(Namespace, Use->getName())
-                << MatchTable::LineBreak;
-        }
-      }
-      return;
-    }
-
-    // TODO: Simple permutation looks like it could be almost as common as
-    //       mutation due to commutative operations.
-
-    Table << MatchTable::Opcode("GIR_BuildMI") << MatchTable::Comment("InsnID")
-          << MatchTable::IntValue(InsnID) << MatchTable::Comment("Opcode")
-          << MatchTable::NamedValue(I->Namespace, I->TheDef->getName())
-          << MatchTable::LineBreak;
-    for (const auto &Renderer : OperandRenderers)
-      Renderer->emitRenderOpcodes(Table, Rule);
-
-    if (I->mayLoad || I->mayStore) {
-      Table << MatchTable::Opcode("GIR_MergeMemOperands")
-            << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-            << MatchTable::Comment("MergeInsnID's");
-      // Emit the ID's for all the instructions that are matched by this rule.
-      // TODO: Limit this to matched instructions that mayLoad/mayStore or have
-      //       some other means of having a memoperand. Also limit this to
-      //       emitted instructions that expect to have a memoperand too. For
-      //       example, (G_SEXT (G_LOAD x)) that results in separate load and
-      //       sign-extend instructions shouldn't put the memoperand on the
-      //       sign-extend since it has no effect there.
-      std::vector<unsigned> MergeInsnIDs;
-      for (const auto &IDMatcherPair : Rule.defined_insn_vars())
-        MergeInsnIDs.push_back(IDMatcherPair.second);
-      llvm::sort(MergeInsnIDs);
-      for (const auto &MergeInsnID : MergeInsnIDs)
-        Table << MatchTable::IntValue(MergeInsnID);
-      Table << MatchTable::NamedValue("GIU_MergeMemOperands_EndOfList")
-            << MatchTable::LineBreak;
-    }
-
-    // FIXME: This is a hack but it's sufficient for ISel. We'll need to do
-    //        better for combines. Particularly when there are multiple match
-    //        roots.
-    if (InsnID == 0)
-      Table << MatchTable::Opcode("GIR_EraseFromParent")
-            << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-            << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to constrain the operands of an output instruction to the
-/// register classes specified by the definition of that instruction.
-class ConstrainOperandsToDefinitionAction : public MatchAction {
-  unsigned InsnID;
-
-public:
-  ConstrainOperandsToDefinitionAction(unsigned InsnID) : InsnID(InsnID) {}
-
-  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIR_ConstrainSelectedInstOperands")
-          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to constrain the specified operand of an output instruction
-/// to the specified register class.
-class ConstrainOperandToRegClassAction : public MatchAction {
-  unsigned InsnID;
-  unsigned OpIdx;
-  const CodeGenRegisterClass &RC;
-
-public:
-  ConstrainOperandToRegClassAction(unsigned InsnID, unsigned OpIdx,
-                                   const CodeGenRegisterClass &RC)
-      : InsnID(InsnID), OpIdx(OpIdx), RC(RC) {}
-
-  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIR_ConstrainOperandRC")
-          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-          << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-          << MatchTable::NamedValue(RC.getQualifiedName() + "RegClassID")
-          << MatchTable::LineBreak;
-  }
-};
-
-/// Generates code to create a temporary register which can be used to chain
-/// instructions together.
-class MakeTempRegisterAction : public MatchAction {
-private:
-  LLTCodeGen Ty;
-  unsigned TempRegID;
-
-public:
-  MakeTempRegisterAction(const LLTCodeGen &Ty, unsigned TempRegID)
-      : Ty(Ty), TempRegID(TempRegID) {
-    KnownTypes.insert(Ty);
-  }
-
-  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    Table << MatchTable::Opcode("GIR_MakeTempReg")
-          << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
-          << MatchTable::Comment("TypeID")
-          << MatchTable::NamedValue(Ty.getCxxEnumValue())
-          << MatchTable::LineBreak;
-  }
-};
-
-InstructionMatcher &RuleMatcher::addInstructionMatcher(StringRef SymbolicName) {
-  Matchers.emplace_back(new InstructionMatcher(*this, SymbolicName));
-  MutatableInsns.insert(Matchers.back().get());
-  return *Matchers.back();
-}
-
-void RuleMatcher::addRequiredFeature(Record *Feature) {
-  RequiredFeatures.push_back(Feature);
-}
-
-const std::vector<Record *> &RuleMatcher::getRequiredFeatures() const {
-  return RequiredFeatures;
-}
-
-// Emplaces an action of the specified Kind at the end of the action list.
-//
-// Returns a reference to the newly created action.
-//
-// Like std::vector::emplace_back(), may invalidate all iterators if the new
-// size exceeds the capacity. Otherwise, only invalidates the past-the-end
-// iterator.
-template <class Kind, class... Args>
-Kind &RuleMatcher::addAction(Args &&... args) {
-  Actions.emplace_back(std::make_unique<Kind>(std::forward<Args>(args)...));
-  return *static_cast<Kind *>(Actions.back().get());
-}
-
-// Emplaces an action of the specified Kind before the given insertion point.
-//
-// Returns an iterator pointing at the newly created instruction.
-//
-// Like std::vector::insert(), may invalidate all iterators if the new size
-// exceeds the capacity. Otherwise, only invalidates the iterators from the
-// insertion point onwards.
-template <class Kind, class... Args>
-action_iterator RuleMatcher::insertAction(action_iterator InsertPt,
-                                          Args &&... args) {
-  return Actions.emplace(InsertPt,
-                         std::make_unique<Kind>(std::forward<Args>(args)...));
-}
-
-unsigned RuleMatcher::implicitlyDefineInsnVar(InstructionMatcher &Matcher) {
-  unsigned NewInsnVarID = NextInsnVarID++;
-  InsnVariableIDs[&Matcher] = NewInsnVarID;
-  return NewInsnVarID;
-}
-
-unsigned RuleMatcher::getInsnVarID(InstructionMatcher &InsnMatcher) const {
-  const auto &I = InsnVariableIDs.find(&InsnMatcher);
-  if (I != InsnVariableIDs.end())
-    return I->second;
-  llvm_unreachable("Matched Insn was not captured in a local variable");
-}
-
-void RuleMatcher::defineOperand(StringRef SymbolicName, OperandMatcher &OM) {
-  if (DefinedOperands.find(SymbolicName) == DefinedOperands.end()) {
-    DefinedOperands[SymbolicName] = &OM;
-    return;
-  }
-
-  // If the operand is already defined, then we must ensure both references in
-  // the matcher have the exact same node.
-  OM.addPredicate<SameOperandMatcher>(
-      OM.getSymbolicName(), getOperandMatcher(OM.getSymbolicName()).getOpIdx());
-}
-
-void RuleMatcher::definePhysRegOperand(Record *Reg, OperandMatcher &OM) {
-  if (PhysRegOperands.find(Reg) == PhysRegOperands.end()) {
-    PhysRegOperands[Reg] = &OM;
-    return;
-  }
-}
-
-InstructionMatcher &
-RuleMatcher::getInstructionMatcher(StringRef SymbolicName) const {
-  for (const auto &I : InsnVariableIDs)
-    if (I.first->getSymbolicName() == SymbolicName)
-      return *I.first;
-  llvm_unreachable(
-      ("Failed to lookup instruction " + SymbolicName).str().c_str());
-}
-
-const OperandMatcher &
-RuleMatcher::getPhysRegOperandMatcher(Record *Reg) const {
-  const auto &I = PhysRegOperands.find(Reg);
-
-  if (I == PhysRegOperands.end()) {
-    PrintFatalError(SrcLoc, "Register " + Reg->getName() +
-                    " was not declared in matcher");
-  }
-
-  return *I->second;
-}
-
-const OperandMatcher &
-RuleMatcher::getOperandMatcher(StringRef Name) const {
-  const auto &I = DefinedOperands.find(Name);
-
-  if (I == DefinedOperands.end())
-    PrintFatalError(SrcLoc, "Operand " + Name + " was not declared in matcher");
-
-  return *I->second;
-}
-
-void RuleMatcher::emit(MatchTable &Table) {
-  if (Matchers.empty())
-    llvm_unreachable("Unexpected empty matcher!");
-
-  // The representation supports rules that require multiple roots such as:
-  //    %ptr(p0) = ...
-  //    %elt0(s32) = G_LOAD %ptr
-  //    %1(p0) = G_ADD %ptr, 4
-  //    %elt1(s32) = G_LOAD p0 %1
-  // which could be usefully folded into:
-  //    %ptr(p0) = ...
-  //    %elt0(s32), %elt1(s32) = TGT_LOAD_PAIR %ptr
-  // on some targets but we don't need to make use of that yet.
-  assert(Matchers.size() == 1 && "Cannot handle multi-root matchers yet");
-
-  unsigned LabelID = Table.allocateLabelID();
-  Table << MatchTable::Opcode("GIM_Try", +1)
-        << MatchTable::Comment("On fail goto")
-        << MatchTable::JumpTarget(LabelID)
-        << MatchTable::Comment(("Rule ID " + Twine(RuleID) + " //").str())
-        << MatchTable::LineBreak;
-
-  if (!RequiredFeatures.empty()) {
-    Table << MatchTable::Opcode("GIM_CheckFeatures")
-          << MatchTable::NamedValue(getNameForFeatureBitset(RequiredFeatures))
-          << MatchTable::LineBreak;
-  }
-
-  Matchers.front()->emitPredicateOpcodes(Table, *this);
-
-  // We must also check if it's safe to fold the matched instructions.
-  if (InsnVariableIDs.size() >= 2) {
-    // Invert the map to create stable ordering (by var names)
-    SmallVector<unsigned, 2> InsnIDs;
-    for (const auto &Pair : InsnVariableIDs) {
-      // Skip the root node since it isn't moving anywhere. Everything else is
-      // sinking to meet it.
-      if (Pair.first == Matchers.front().get())
-        continue;
-
-      InsnIDs.push_back(Pair.second);
-    }
-    llvm::sort(InsnIDs);
-
-    for (const auto &InsnID : InsnIDs) {
-      // Reject the difficult cases until we have a more accurate check.
-      Table << MatchTable::Opcode("GIM_CheckIsSafeToFold")
-            << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-            << MatchTable::LineBreak;
-
-      // FIXME: Emit checks to determine it's _actually_ safe to fold and/or
-      //        account for unsafe cases.
-      //
-      //        Example:
-      //          MI1--> %0 = ...
-      //                 %1 = ... %0
-      //          MI0--> %2 = ... %0
-      //          It's not safe to erase MI1. We currently handle this by not
-      //          erasing %0 (even when it's dead).
-      //
-      //        Example:
-      //          MI1--> %0 = load volatile @a
-      //                 %1 = load volatile @a
-      //          MI0--> %2 = ... %0
-      //          It's not safe to sink %0's def past %1. We currently handle
-      //          this by rejecting all loads.
-      //
-      //        Example:
-      //          MI1--> %0 = load @a
-      //                 %1 = store @a
-      //          MI0--> %2 = ... %0
-      //          It's not safe to sink %0's def past %1. We currently handle
-      //          this by rejecting all loads.
-      //
-      //        Example:
-      //                   G_CONDBR %cond, @BB1
-      //                 BB0:
-      //          MI1-->   %0 = load @a
-      //                   G_BR @BB1
-      //                 BB1:
-      //          MI0-->   %2 = ... %0
-      //          It's not always safe to sink %0 across control flow. In this
-      //          case it may introduce a memory fault. We currentl handle this
-      //          by rejecting all loads.
-    }
-  }
-
-  for (const auto &PM : EpilogueMatchers)
-    PM->emitPredicateOpcodes(Table, *this);
-
-  for (const auto &MA : Actions)
-    MA->emitActionOpcodes(Table, *this);
-
-  if (Table.isWithCoverage())
-    Table << MatchTable::Opcode("GIR_Coverage") << MatchTable::IntValue(RuleID)
-          << MatchTable::LineBreak;
-  else
-    Table << MatchTable::Comment(("GIR_Coverage, " + Twine(RuleID) + ",").str())
-          << MatchTable::LineBreak;
-
-  Table << MatchTable::Opcode("GIR_Done", -1) << MatchTable::LineBreak
-        << MatchTable::Label(LabelID);
-  ++NumPatternEmitted;
-}
-
-bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const {
-  // Rules involving more match roots have higher priority.
-  if (Matchers.size() > B.Matchers.size())
-    return true;
-  if (Matchers.size() < B.Matchers.size())
-    return false;
-
-  for (auto Matcher : zip(Matchers, B.Matchers)) {
-    if (std::get<0>(Matcher)->isHigherPriorityThan(*std::get<1>(Matcher)))
-      return true;
-    if (std::get<1>(Matcher)->isHigherPriorityThan(*std::get<0>(Matcher)))
-      return false;
-  }
-
-  return false;
-}
-
-unsigned RuleMatcher::countRendererFns() const {
-  return std::accumulate(
-      Matchers.begin(), Matchers.end(), 0,
-      [](unsigned A, const std::unique_ptr<InstructionMatcher> &Matcher) {
-        return A + Matcher->countRendererFns();
-      });
-}
-
-bool OperandPredicateMatcher::isHigherPriorityThan(
-    const OperandPredicateMatcher &B) const {
-  // Generally speaking, an instruction is more important than an Int or a
-  // LiteralInt because it can cover more nodes but theres an exception to
-  // this. G_CONSTANT's are less important than either of those two because they
-  // are more permissive.
-
-  const InstructionOperandMatcher *AOM =
-      dyn_cast<InstructionOperandMatcher>(this);
-  const InstructionOperandMatcher *BOM =
-      dyn_cast<InstructionOperandMatcher>(&B);
-  bool AIsConstantInsn = AOM && AOM->getInsnMatcher().isConstantInstruction();
-  bool BIsConstantInsn = BOM && BOM->getInsnMatcher().isConstantInstruction();
-
-  if (AOM && BOM) {
-    // The relative priorities between a G_CONSTANT and any other instruction
-    // don't actually matter but this code is needed to ensure a strict weak
-    // ordering. This is particularly important on Windows where the rules will
-    // be incorrectly sorted without it.
-    if (AIsConstantInsn != BIsConstantInsn)
-      return AIsConstantInsn < BIsConstantInsn;
-    return false;
-  }
-
-  if (AOM && AIsConstantInsn && (B.Kind == OPM_Int || B.Kind == OPM_LiteralInt))
-    return false;
-  if (BOM && BIsConstantInsn && (Kind == OPM_Int || Kind == OPM_LiteralInt))
-    return true;
-
-  return Kind < B.Kind;
-}
-
-void SameOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
-                                              RuleMatcher &Rule) const {
-  const OperandMatcher &OtherOM = Rule.getOperandMatcher(MatchingName);
-  unsigned OtherInsnVarID = Rule.getInsnVarID(OtherOM.getInstructionMatcher());
-  assert(OtherInsnVarID == OtherOM.getInstructionMatcher().getInsnVarID());
-
-  Table << MatchTable::Opcode("GIM_CheckIsSameOperand")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx)
-        << MatchTable::Comment("OtherMI")
-        << MatchTable::IntValue(OtherInsnVarID)
-        << MatchTable::Comment("OtherOpIdx")
-        << MatchTable::IntValue(OtherOM.getOpIdx())
-        << MatchTable::LineBreak;
-}
-
 //===- GlobalISelEmitter class --------------------------------------------===//
 
 static Expected<LLTCodeGen> getInstResultType(const TreePatternNode *Dst) {
@@ -3554,8 +291,7 @@ static Expected<LLTCodeGen> getInstResultType(const TreePatternNode *Dst) {
 
   std::optional<LLTCodeGen> MaybeOpTy;
   if (ChildTypes.front().isMachineValueType()) {
-    MaybeOpTy =
-      MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy);
+    MaybeOpTy = MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy);
   }
 
   if (!MaybeOpTy)
@@ -3563,17 +299,34 @@ static Expected<LLTCodeGen> getInstResultType(const TreePatternNode *Dst) {
   return *MaybeOpTy;
 }
 
-class GlobalISelEmitter {
+class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {
 public:
   explicit GlobalISelEmitter(RecordKeeper &RK);
+
+  void emitAdditionalImpl(raw_ostream &OS) override;
+
+  void emitMIPredicateFns(raw_ostream &OS) override;
+  void emitI64ImmPredicateFns(raw_ostream &OS) override;
+  void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
+  void emitAPIntImmPredicateFns(raw_ostream &OS) override;
+  void emitTestSimplePredicate(raw_ostream &OS) override;
+  void emitRunCustomAction(raw_ostream &OS) override;
+
+  const CodeGenTarget &getTarget() const override { return Target; }
+  StringRef getClassName() const override { return ClassName; }
+
   void run(raw_ostream &OS);
 
 private:
+  std::string ClassName;
+
   const RecordKeeper &RK;
   const CodeGenDAGPatterns CGP;
   const CodeGenTarget &Target;
   CodeGenRegBank &CGRegs;
 
+  std::vector<Record *> AllPatFrags;
+
   /// Keep track of the equivalence between SDNodes and Instruction by mapping
   /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to
   /// check for attributes on the relation such as CheckMMOIsNonAtomic.
@@ -3594,9 +347,6 @@ private:
   /// This adds compatibility for RuleMatchers to use this for ordering rules.
   DenseMap<uint64_t, int> RuleMatcherScores;
 
-  // Map of predicates to their subtarget features.
-  SubtargetFeatureInfoMap SubtargetFeatures;
-
   // Rule coverage information.
   std::optional<CodeGenCoverage> RuleCoverage;
 
@@ -3635,24 +385,21 @@ private:
       const TreePatternNode *Src, const TreePatternNode *Dst);
   Expected<action_iterator> createAndImportSubInstructionRenderer(
       action_iterator InsertPt, RuleMatcher &M, const TreePatternNode *Dst,
-      unsigned TempReg);
+      const TreePatternNode *Src, unsigned TempReg);
   Expected<action_iterator>
   createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M,
                             const TreePatternNode *Dst);
 
-  Expected<action_iterator>
-  importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M,
-                             BuildMIAction &DstMIBuilder,
-                             const TreePatternNode *Dst);
+  Expected<action_iterator> importExplicitDefRenderers(
+      action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
+      const TreePatternNode *Src, const TreePatternNode *Dst);
 
-  Expected<action_iterator>
-  importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M,
-                             BuildMIAction &DstMIBuilder,
-                             const llvm::TreePatternNode *Dst);
-  Expected<action_iterator>
-  importExplicitUseRenderer(action_iterator InsertPt, RuleMatcher &Rule,
-                            BuildMIAction &DstMIBuilder,
-                            TreePatternNode *DstChild);
+  Expected<action_iterator> importExplicitUseRenderers(
+      action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
+      const llvm::TreePatternNode *Dst, const TreePatternNode *Src);
+  Expected<action_iterator> importExplicitUseRenderer(
+      action_iterator InsertPt, RuleMatcher &Rule, BuildMIAction &DstMIBuilder,
+      const TreePatternNode *DstChild, const TreePatternNode *Src);
   Error importDefaultOperandRenderers(action_iterator InsertPt, RuleMatcher &M,
                                       BuildMIAction &DstMIBuilder,
                                       DagInit *DefaultOps) const;
@@ -3660,16 +407,6 @@ private:
   importImplicitDefRenderers(BuildMIAction &DstMIBuilder,
                              const std::vector<Record *> &ImplicitDefs) const;
 
-  void emitCxxPredicateFns(raw_ostream &OS, StringRef CodeFieldName,
-                           StringRef TypeIdentifier, StringRef ArgType,
-                           StringRef ArgName, StringRef AdditionalArgs,
-                           StringRef AdditionalDeclarations,
-                           std::function<bool(const Record *R)> Filter);
-  void emitImmPredicateFns(raw_ostream &OS, StringRef TypeIdentifier,
-                           StringRef ArgType,
-                           std::function<bool(const Record *R)> Filter);
-  void emitMIPredicateFns(raw_ostream &OS);
-
   /// Analyze pattern \p P, returning a matcher for it if possible.
   /// Otherwise, return an Error explaining why we don't support it.
   Expected<RuleMatcher> runOnPattern(const PatternToMatch &P);
@@ -3685,25 +422,25 @@ private:
   /// If no register class is found, return std::nullopt.
   std::optional<const CodeGenRegisterClass *>
   inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty,
-                                 TreePatternNode *SuperRegNode,
-                                 TreePatternNode *SubRegIdxNode);
+                                 const TreePatternNode *SuperRegNode,
+                                 const TreePatternNode *SubRegIdxNode);
   std::optional<CodeGenSubRegIndex *>
-  inferSubRegIndexForNode(TreePatternNode *SubRegIdxNode);
+  inferSubRegIndexForNode(const TreePatternNode *SubRegIdxNode);
 
   /// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode.
   /// Return std::nullopt if no such class exists.
   std::optional<const CodeGenRegisterClass *>
   inferSuperRegisterClass(const TypeSetByHwMode &Ty,
-                          TreePatternNode *SubRegIdxNode);
+                          const TreePatternNode *SubRegIdxNode);
 
   /// Return the CodeGenRegisterClass associated with \p Leaf if it has one.
   std::optional<const CodeGenRegisterClass *>
-  getRegClassFromLeaf(TreePatternNode *Leaf);
+  getRegClassFromLeaf(const TreePatternNode *Leaf);
 
   /// Return a CodeGenRegisterClass for \p N if one can be found. Return
   /// std::nullopt otherwise.
   std::optional<const CodeGenRegisterClass *>
-  inferRegClassFromPattern(TreePatternNode *N);
+  inferRegClassFromPattern(const TreePatternNode *N);
 
   /// Return the size of the MemoryVT in this predicate, if possible.
   std::optional<unsigned>
@@ -3714,39 +451,10 @@ private:
   addBuiltinPredicates(const Record *SrcGIEquivOrNull,
                        const TreePredicateFn &Predicate,
                        InstructionMatcher &InsnMatcher, bool &HasAddedMatcher);
-
-public:
-  /// Takes a sequence of \p Rules and group them based on the predicates
-  /// they share. \p MatcherStorage is used as a memory container
-  /// for the group that are created as part of this process.
-  ///
-  /// What this optimization does looks like if GroupT = GroupMatcher:
-  /// Output without optimization:
-  /// \verbatim
-  /// # R1
-  ///  # predicate A
-  ///  # predicate B
-  ///  ...
-  /// # R2
-  ///  # predicate A // <-- effectively this is going to be checked twice.
-  ///                //     Once in R1 and once in R2.
-  ///  # predicate C
-  /// \endverbatim
-  /// Output with optimization:
-  /// \verbatim
-  /// # Group1_2
-  ///  # predicate A // <-- Check is now shared.
-  ///  # R1
-  ///   # predicate B
-  ///  # R2
-  ///   # predicate C
-  /// \endverbatim
-  template <class GroupT>
-  static std::vector<Matcher *> optimizeRules(
-      ArrayRef<Matcher *> Rules,
-      std::vector<std::unique_ptr<Matcher>> &MatcherStorage);
 };
 
+StringRef getPatFragPredicateEnumName(Record *R) { return R->getName(); }
+
 void GlobalISelEmitter::gatherOpcodeValues() {
   InstructionOpcodeMatcher::initOpcodeValuesMap(Target);
 }
@@ -3766,15 +474,15 @@ void GlobalISelEmitter::gatherNodeEquivs() {
     if (!SelDAGEquiv)
       continue;
     ComplexPatternEquivs[SelDAGEquiv] = Equiv;
- }
+  }
 
- assert(SDNodeXFormEquivs.empty());
- for (Record *Equiv : RK.getAllDerivedDefinitions("GISDNodeXFormEquiv")) {
-   Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent");
-   if (!SelDAGEquiv)
-     continue;
-   SDNodeXFormEquivs[SelDAGEquiv] = Equiv;
- }
+  assert(SDNodeXFormEquivs.empty());
+  for (Record *Equiv : RK.getAllDerivedDefinitions("GISDNodeXFormEquiv")) {
+    Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent");
+    if (!SelDAGEquiv)
+      continue;
+    SDNodeXFormEquivs[SelDAGEquiv] = Equiv;
+  }
 }
 
 Record *GlobalISelEmitter::findNodeEquiv(Record *N) const {
@@ -3806,8 +514,10 @@ GlobalISelEmitter::getEquivNode(Record &Equiv, const TreePatternNode *N) const {
 }
 
 GlobalISelEmitter::GlobalISelEmitter(RecordKeeper &RK)
-    : RK(RK), CGP(RK), Target(CGP.getTargetInfo()),
-      CGRegs(Target.getRegBank()) {}
+    : GlobalISelMatchTableExecutorEmitter(), RK(RK), CGP(RK),
+      Target(CGP.getTargetInfo()), CGRegs(Target.getRegBank()) {
+  ClassName = Target.getName().str() + "InstructionSelector";
+}
 
 //===- Emitter ------------------------------------------------------------===//
 
@@ -3900,6 +610,7 @@ Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
     }
   }
 
+  assert(SrcGIEquivOrNull != nullptr && "Invalid SrcGIEquivOrNull value");
   // No check required. We already did it by swapping the opcode.
   if (!SrcGIEquivOrNull->isValueUnset("IfSignExtend") &&
       Predicate.isSignExtLoad())
@@ -3985,13 +696,12 @@ Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
 Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
     RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
     const TreePatternNode *Src, unsigned &TempOpIdx) {
+  const auto SavedFlags = Rule.setGISelFlags(Src->getGISelFlagsRecord());
+
   Record *SrcGIEquivOrNull = nullptr;
   const CodeGenInstruction *SrcGIOrNull = nullptr;
 
   // Start with the defined operands (i.e., the results of the root operator).
-  if (Src->getExtTypes().size() > 1)
-    return failedImport("Src pattern has multiple results");
-
   if (Src->isLeaf()) {
     Init *SrcInit = Src->getLeafValue();
     if (isa<IntInit>(SrcInit)) {
@@ -4070,12 +780,14 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
   }
 
   bool IsAtomic = false;
-  if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic"))
+  if (SrcGIEquivOrNull &&
+      SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic"))
     InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("NotAtomic");
-  else if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) {
+  else if (SrcGIEquivOrNull &&
+           SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) {
     IsAtomic = true;
     InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
-      "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
+        "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
   }
 
   if (Src->isLeaf()) {
@@ -4107,7 +819,7 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
     bool IsFCmp = SrcGIOrNull->TheDef->getName() == "G_FCMP";
 
     if (IsFCmp || SrcGIOrNull->TheDef->getName() == "G_ICMP") {
-      TreePatternNode *SrcChild = Src->getChild(NumChildren - 1);
+      const TreePatternNode *SrcChild = Src->getChild(NumChildren - 1);
       if (SrcChild->isLeaf()) {
         DefInit *DI = dyn_cast<DefInit>(SrcChild->getLeafValue());
         Record *CCDef = DI ? DI->getDef() : nullptr;
@@ -4115,9 +827,9 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
           return failedImport("Unable to handle CondCode");
 
         OperandMatcher &OM =
-          InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx);
-        StringRef PredType = IsFCmp ? CCDef->getValueAsString("FCmpPredicate") :
-                                      CCDef->getValueAsString("ICmpPredicate");
+            InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx);
+        StringRef PredType = IsFCmp ? CCDef->getValueAsString("FCmpPredicate")
+                                    : CCDef->getValueAsString("ICmpPredicate");
 
         if (!PredType.empty()) {
           OM.addPredicate<CmpPredicateOperandMatcher>(std::string(PredType));
@@ -4135,8 +847,8 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
     if (IsAtomic && SrcGIOrNull->TheDef->getName() == "G_STORE") {
       assert(NumChildren == 2 && "wrong operands for atomic store");
 
-      TreePatternNode *PtrChild = Src->getChild(0);
-      TreePatternNode *ValueChild = Src->getChild(1);
+      const TreePatternNode *PtrChild = Src->getChild(0);
+      const TreePatternNode *ValueChild = Src->getChild(1);
 
       if (auto Error = importChildMatcher(Rule, InsnMatcher, PtrChild, true,
                                           false, 1, TempOpIdx))
@@ -4157,7 +869,7 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
       return failedImport("Expected IntInit containing intrinsic ID)");
 
     for (unsigned i = 0; i != NumChildren; ++i) {
-      TreePatternNode *SrcChild = Src->getChild(i);
+      const TreePatternNode *SrcChild = Src->getChild(i);
 
       // We need to determine the meaning of a literal integer based on the
       // context. If this is a field required to be an immediate (such as an
@@ -4169,8 +881,9 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
       bool OperandIsImmArg = SrcGIOrNull->isInOperandImmArg(i);
 
       // SelectionDAG allows pointers to be represented with iN since it doesn't
-      // distinguish between pointers and integers but they are different types in GlobalISel.
-      // Coerce integers to pointers to address space 0 if the context indicates a pointer.
+      // distinguish between pointers and integers but they are different types
+      // in GlobalISel. Coerce integers to pointers to address space 0 if the
+      // context indicates a pointer.
       //
       bool OperandIsAPointer = SrcGIOrNull->isInOperandAPointer(i);
 
@@ -4327,7 +1040,8 @@ Error GlobalISelEmitter::importChildMatcher(
       // This isn't strictly true. If the user were to provide exactly the same
       // matchers as the original operand then we could allow it. However, it's
       // simpler to not permit the redundant specification.
-      return failedImport("Nested instruction cannot be the same as another operand");
+      return failedImport(
+          "Nested instruction cannot be the same as another operand");
     }
 
     // Map the node to a gMIR instruction.
@@ -4377,11 +1091,11 @@ Error GlobalISelEmitter::importChildMatcher(
     if (ChildRec->isSubClassOf("Register")) {
       // This just be emitted as a copy to the specific register.
       ValueTypeByHwMode VT = ChildTypes.front().getValueTypeByHwMode();
-      const CodeGenRegisterClass *RC
-        = CGRegs.getMinimalPhysRegClass(ChildRec, &VT);
+      const CodeGenRegisterClass *RC =
+          CGRegs.getMinimalPhysRegClass(ChildRec, &VT);
       if (!RC) {
         return failedImport(
-          "Could not determine physical register class of pattern source");
+            "Could not determine physical register class of pattern source");
       }
 
       OM.addPredicate<RegisterBankOperandMatcher>(*RC);
@@ -4416,10 +1130,10 @@ Error GlobalISelEmitter::importChildMatcher(
 
       ValueTypeByHwMode VTy = ChildTypes.front().getValueTypeByHwMode();
 
-      const CodeGenInstruction &BuildVector
-        = Target.getInstruction(RK.getDef("G_BUILD_VECTOR"));
-      const CodeGenInstruction &BuildVectorTrunc
-        = Target.getInstruction(RK.getDef("G_BUILD_VECTOR_TRUNC"));
+      const CodeGenInstruction &BuildVector =
+          Target.getInstruction(RK.getDef("G_BUILD_VECTOR"));
+      const CodeGenInstruction &BuildVectorTrunc =
+          Target.getInstruction(RK.getDef("G_BUILD_VECTOR_TRUNC"));
 
       // Treat G_BUILD_VECTOR as the canonical opcode, and G_BUILD_VECTOR_TRUNC
       // as an alternative.
@@ -4451,7 +1165,7 @@ Error GlobalISelEmitter::importChildMatcher(
 
 Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer(
     action_iterator InsertPt, RuleMatcher &Rule, BuildMIAction &DstMIBuilder,
-    TreePatternNode *DstChild) {
+    const TreePatternNode *DstChild, const TreePatternNode *Src) {
 
   const auto &SubOperand = Rule.getComplexSubOperand(DstChild->getName());
   if (SubOperand) {
@@ -4516,18 +1230,19 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer(
         return OpTy.takeError();
 
       unsigned TempRegID = Rule.allocateTempRegID();
-      InsertPt = Rule.insertAction<MakeTempRegisterAction>(
-          InsertPt, *OpTy, TempRegID);
+      InsertPt =
+          Rule.insertAction<MakeTempRegisterAction>(InsertPt, *OpTy, TempRegID);
       DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID);
 
       auto InsertPtOrError = createAndImportSubInstructionRenderer(
-          ++InsertPt, Rule, DstChild, TempRegID);
+          ++InsertPt, Rule, DstChild, Src, TempRegID);
       if (auto Error = InsertPtOrError.takeError())
         return std::move(Error);
       return InsertPtOrError.get();
     }
 
-    return failedImport("Dst pattern child isn't a leaf node or an MBB" + llvm::to_string(*DstChild));
+    return failedImport("Dst pattern child isn't a leaf node or an MBB" +
+                        llvm::to_string(*DstChild));
   }
 
   // It could be a specific immediate in which case we should just check for
@@ -4593,6 +1308,16 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer(
     return failedImport(
         "Dst pattern child def is an unsupported tablegen class");
   }
+
+  // Handle the case where the MVT/register class is omitted in the dest pattern
+  // but MVT exists in the source pattern.
+  if (isa<UnsetInit>(DstChild->getLeafValue())) {
+    for (unsigned NumOp = 0; NumOp < Src->getNumChildren(); NumOp++)
+      if (Src->getChild(NumOp)->getName() == DstChild->getName()) {
+        DstMIBuilder.addRenderer<CopyRenderer>(Src->getChild(NumOp)->getName());
+        return InsertPt;
+      }
+  }
   return failedImport("Dst pattern child is an unsupported kind");
 }
 
@@ -4612,18 +1337,19 @@ Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
         &Target.getInstruction(RK.getDef("COPY")));
     BuildMIAction &CopyToPhysRegMIBuilder =
         *static_cast<BuildMIAction *>(InsertPt->get());
-    CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(Target,
-                                                            PhysInput.first,
-                                                            true);
+    CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(
+        Target, PhysInput.first, true);
     CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(PhysInput.first);
   }
 
-  if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst)
-                       .takeError())
+  if (auto Error =
+          importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Src, Dst)
+              .takeError())
     return std::move(Error);
 
-  if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst)
-                       .takeError())
+  if (auto Error =
+          importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst, Src)
+              .takeError())
     return std::move(Error);
 
   return DstMIBuilder;
@@ -4632,7 +1358,7 @@ Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
 Expected<action_iterator>
 GlobalISelEmitter::createAndImportSubInstructionRenderer(
     const action_iterator InsertPt, RuleMatcher &M, const TreePatternNode *Dst,
-    unsigned TempRegID) {
+    const TreePatternNode *Src, unsigned TempRegID) {
   auto InsertPtOrError = createInstructionRenderer(InsertPt, M, Dst);
 
   // TODO: Assert there's exactly one result.
@@ -4646,8 +1372,8 @@ GlobalISelEmitter::createAndImportSubInstructionRenderer(
   // Assign the result to TempReg.
   DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, true);
 
-  InsertPtOrError =
-      importExplicitUseRenderers(InsertPtOrError.get(), M, DstMIBuilder, Dst);
+  InsertPtOrError = importExplicitUseRenderers(InsertPtOrError.get(), M,
+                                               DstMIBuilder, Dst, Src);
   if (auto Error = InsertPtOrError.takeError())
     return std::move(Error);
 
@@ -4685,19 +1411,19 @@ GlobalISelEmitter::createAndImportSubInstructionRenderer(
     auto SuperClass = inferRegClassFromPattern(Dst->getChild(0));
     if (!SuperClass)
       return failedImport(
-        "Cannot infer register class from EXTRACT_SUBREG operand #0");
+          "Cannot infer register class from EXTRACT_SUBREG operand #0");
 
     auto SubIdx = inferSubRegIndexForNode(Dst->getChild(1));
     if (!SubIdx)
       return failedImport("EXTRACT_SUBREG child #1 is not a subreg index");
 
     const auto SrcRCDstRCPair =
-      (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
+        (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
     assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
     M.insertAction<ConstrainOperandToRegClassAction>(
-      InsertPt, DstMIBuilder.getInsnID(), 0, *SrcRCDstRCPair->second);
+        InsertPt, DstMIBuilder.getInsnID(), 0, *SrcRCDstRCPair->second);
     M.insertAction<ConstrainOperandToRegClassAction>(
-      InsertPt, DstMIBuilder.getInsnID(), 1, *SrcRCDstRCPair->first);
+        InsertPt, DstMIBuilder.getInsnID(), 1, *SrcRCDstRCPair->first);
 
     // We're done with this pattern!  It's eligible for GISel emission; return
     // it.
@@ -4710,37 +1436,37 @@ GlobalISelEmitter::createAndImportSubInstructionRenderer(
     auto SubClass = inferRegClassFromPattern(Dst->getChild(1));
     if (!SubClass)
       return failedImport(
-        "Cannot infer register class from SUBREG_TO_REG child #1");
-    auto SuperClass = inferSuperRegisterClass(Dst->getExtType(0),
-                                              Dst->getChild(2));
+          "Cannot infer register class from SUBREG_TO_REG child #1");
+    auto SuperClass =
+        inferSuperRegisterClass(Dst->getExtType(0), Dst->getChild(2));
     if (!SuperClass)
       return failedImport(
-        "Cannot infer register class for SUBREG_TO_REG operand #0");
+          "Cannot infer register class for SUBREG_TO_REG operand #0");
     M.insertAction<ConstrainOperandToRegClassAction>(
-      InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass);
+        InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass);
     M.insertAction<ConstrainOperandToRegClassAction>(
-      InsertPt, DstMIBuilder.getInsnID(), 2, **SubClass);
+        InsertPt, DstMIBuilder.getInsnID(), 2, **SubClass);
     return InsertPtOrError.get();
   }
 
   if (OpName == "REG_SEQUENCE") {
     auto SuperClass = inferRegClassFromPattern(Dst->getChild(0));
     M.insertAction<ConstrainOperandToRegClassAction>(
-      InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass);
+        InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass);
 
     unsigned Num = Dst->getNumChildren();
     for (unsigned I = 1; I != Num; I += 2) {
-      TreePatternNode *SubRegChild = Dst->getChild(I + 1);
+      const TreePatternNode *SubRegChild = Dst->getChild(I + 1);
 
       auto SubIdx = inferSubRegIndexForNode(SubRegChild);
       if (!SubIdx)
         return failedImport("REG_SEQUENCE child is not a subreg index");
 
       const auto SrcRCDstRCPair =
-        (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
+          (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
       assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
       M.insertAction<ConstrainOperandToRegClassAction>(
-        InsertPt, DstMIBuilder.getInsnID(), I, *SrcRCDstRCPair->second);
+          InsertPt, DstMIBuilder.getInsnID(), I, *SrcRCDstRCPair->second);
     }
 
     return InsertPtOrError.get();
@@ -4774,22 +1500,22 @@ Expected<action_iterator> GlobalISelEmitter::createInstructionRenderer(
 
 Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
     action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
-    const TreePatternNode *Dst) {
+    const TreePatternNode *Src, const TreePatternNode *Dst) {
   const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
-  const unsigned NumDefs = DstI->Operands.NumDefs;
-  if (NumDefs == 0)
+  const unsigned SrcNumDefs = Src->getExtTypes().size();
+  const unsigned DstNumDefs = DstI->Operands.NumDefs;
+  if (DstNumDefs == 0)
     return InsertPt;
 
-  DstMIBuilder.addRenderer<CopyRenderer>(DstI->Operands[0].Name);
+  for (unsigned I = 0; I < SrcNumDefs; ++I)
+    DstMIBuilder.addRenderer<CopyRenderer>(DstI->Operands[I].Name);
 
   // Some instructions have multiple defs, but are missing a type entry
   // (e.g. s_cc_out operands).
-  if (Dst->getExtTypes().size() < NumDefs)
+  if (Dst->getExtTypes().size() < DstNumDefs)
     return failedImport("unhandled discarded def");
 
-  // Patterns only handle a single result, so any result after the first is an
-  // implicitly dead def.
-  for (unsigned I = 1; I < NumDefs; ++I) {
+  for (unsigned I = SrcNumDefs; I < DstNumDefs; ++I) {
     const TypeSetByHwMode &ExtTy = Dst->getExtType(I);
     if (!ExtTy.isMachineValueType())
       return failedImport("unsupported typeset");
@@ -4800,7 +1526,7 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
 
     unsigned TempRegID = M.allocateTempRegID();
     InsertPt =
-      M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTy, TempRegID);
+        M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTy, TempRegID);
     DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, true, nullptr, true);
   }
 
@@ -4809,7 +1535,7 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
 
 Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
     action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
-    const llvm::TreePatternNode *Dst) {
+    const llvm::TreePatternNode *Dst, const llvm::TreePatternNode *Src) {
   const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
   CodeGenInstruction *OrigDstI = &Target.getInstruction(Dst->getOperator());
 
@@ -4825,7 +1551,7 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
       return failedImport("EXTRACT_SUBREG child #1 is not a subreg index");
 
     CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
-    TreePatternNode *ValChild = Dst->getChild(0);
+    const TreePatternNode *ValChild = Dst->getChild(0);
     if (!ValChild->isLeaf()) {
       // We really have to handle the source instruction, and then insert a
       // copy from the subregister.
@@ -4834,11 +1560,11 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
         return ExtractSrcTy.takeError();
 
       unsigned TempRegID = M.allocateTempRegID();
-      InsertPt = M.insertAction<MakeTempRegisterAction>(
-        InsertPt, *ExtractSrcTy, TempRegID);
+      InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, *ExtractSrcTy,
+                                                        TempRegID);
 
       auto InsertPtOrError = createAndImportSubInstructionRenderer(
-        ++InsertPt, M, ValChild, TempRegID);
+          ++InsertPt, M, ValChild, Src, TempRegID);
       if (auto Error = InsertPtOrError.takeError())
         return std::move(Error);
 
@@ -4855,15 +1581,22 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
     CodeGenRegisterClass *RC = CGRegs.getRegClass(RCDef);
 
     const auto SrcRCDstRCPair =
-      RC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx);
+        RC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx);
     if (SrcRCDstRCPair) {
       assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
       if (SrcRCDstRCPair->first != RC)
         return failedImport("EXTRACT_SUBREG requires an additional COPY");
     }
 
-    DstMIBuilder.addRenderer<CopySubRegRenderer>(Dst->getChild(0)->getName(),
-                                                 SubIdx);
+    StringRef RegOperandName = Dst->getChild(0)->getName();
+    if (const auto &SubOperand = M.getComplexSubOperand(RegOperandName)) {
+      DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
+          *std::get<0>(*SubOperand), RegOperandName, std::get<1>(*SubOperand),
+          std::get<2>(*SubOperand), SubIdx);
+      return InsertPt;
+    }
+
+    DstMIBuilder.addRenderer<CopySubRegRenderer>(RegOperandName, SubIdx);
     return InsertPt;
   }
 
@@ -4880,15 +1613,15 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
       return failedImport("Malformed REG_SEQUENCE");
 
     for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) {
-      TreePatternNode *ValChild = Dst->getChild(I);
-      TreePatternNode *SubRegChild = Dst->getChild(I + 1);
+      const TreePatternNode *ValChild = Dst->getChild(I);
+      const TreePatternNode *SubRegChild = Dst->getChild(I + 1);
 
       if (DefInit *SubRegInit =
               dyn_cast<DefInit>(SubRegChild->getLeafValue())) {
         CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
 
         auto InsertPtOrError =
-            importExplicitUseRenderer(InsertPt, M, DstMIBuilder, ValChild);
+            importExplicitUseRenderer(InsertPt, M, DstMIBuilder, ValChild, Src);
         if (auto Error = InsertPtOrError.takeError())
           return std::move(Error);
         InsertPt = InsertPtOrError.get();
@@ -4949,15 +1682,15 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
 
       const CGIOperandList::OperandInfo &DstIOperand = DstI->Operands[InstOpNo];
       DagInit *DefaultOps = DstIOperand.Rec->getValueAsDag("DefaultOps");
-      if (auto Error = importDefaultOperandRenderers(
-            InsertPt, M, DstMIBuilder, DefaultOps))
+      if (auto Error = importDefaultOperandRenderers(InsertPt, M, DstMIBuilder,
+                                                     DefaultOps))
         return std::move(Error);
       ++NumDefaultOps;
       continue;
     }
 
     auto InsertPtOrError = importExplicitUseRenderer(InsertPt, M, DstMIBuilder,
-                                                     Dst->getChild(Child));
+                                                     Dst->getChild(Child), Src);
     if (auto Error = InsertPtOrError.takeError())
       return std::move(Error);
     InsertPt = InsertPtOrError.get();
@@ -4985,8 +1718,7 @@ Error GlobalISelEmitter::importDefaultOperandRenderers(
       if (const DefInit *DefaultDagOperator =
               dyn_cast<DefInit>(DefaultDagOp->getOperator())) {
         if (DefaultDagOperator->getDef()->isSubClassOf("ValueType")) {
-          OpTyOrNone = MVTToLLT(getValueType(
-                                  DefaultDagOperator->getDef()));
+          OpTyOrNone = MVTToLLT(getValueType(DefaultDagOperator->getDef()));
           DefaultOp = DefaultDagOp->getArg(0);
         }
       }
@@ -4999,10 +1731,10 @@ Error GlobalISelEmitter::importDefaultOperandRenderers(
         M.insertAction<MakeTempRegisterAction>(InsertPt, *OpTyOrNone,
                                                TempRegID);
         InsertPt = M.insertAction<BuildMIAction>(
-          InsertPt, M.allocateOutputInsnID(),
-          &Target.getInstruction(RK.getDef("IMPLICIT_DEF")));
-        BuildMIAction &IDMIBuilder = *static_cast<BuildMIAction *>(
-          InsertPt->get());
+            InsertPt, M.allocateOutputInsnID(),
+            &Target.getInstruction(RK.getDef("IMPLICIT_DEF")));
+        BuildMIAction &IDMIBuilder =
+            *static_cast<BuildMIAction *>(InsertPt->get());
         IDMIBuilder.addRenderer<TempRegRenderer>(TempRegID);
         DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID);
       } else {
@@ -5031,7 +1763,7 @@ Error GlobalISelEmitter::importImplicitDefRenderers(
 }
 
 std::optional<const CodeGenRegisterClass *>
-GlobalISelEmitter::getRegClassFromLeaf(TreePatternNode *Leaf) {
+GlobalISelEmitter::getRegClassFromLeaf(const TreePatternNode *Leaf) {
   assert(Leaf && "Expected node?");
   assert(Leaf->isLeaf() && "Expected leaf?");
   Record *RCRec = getInitValueAsRegClass(Leaf->getLeafValue());
@@ -5044,7 +1776,7 @@ GlobalISelEmitter::getRegClassFromLeaf(TreePatternNode *Leaf) {
 }
 
 std::optional<const CodeGenRegisterClass *>
-GlobalISelEmitter::inferRegClassFromPattern(TreePatternNode *N) {
+GlobalISelEmitter::inferRegClassFromPattern(const TreePatternNode *N) {
   if (!N)
     return std::nullopt;
 
@@ -5076,13 +1808,13 @@ GlobalISelEmitter::inferRegClassFromPattern(TreePatternNode *N) {
   if (IsRegSequence || InstName == "COPY_TO_REGCLASS") {
     // If we have a COPY_TO_REGCLASS, then we need to handle it specially. It
     // has the desired register class as the first child.
-    TreePatternNode *RCChild = N->getChild(IsRegSequence ? 0 : 1);
+    const TreePatternNode *RCChild = N->getChild(IsRegSequence ? 0 : 1);
     if (!RCChild->isLeaf())
       return std::nullopt;
     return getRegClassFromLeaf(RCChild);
   }
   if (InstName == "INSERT_SUBREG") {
-    TreePatternNode *Child0 = N->getChild(0);
+    const TreePatternNode *Child0 = N->getChild(0);
     assert(Child0->getNumTypes() == 1 && "Unexpected number of types!");
     const TypeSetByHwMode &VTy = Child0->getExtType(0);
     return inferSuperRegisterClassForNode(VTy, Child0, N->getChild(2));
@@ -5112,8 +1844,8 @@ GlobalISelEmitter::inferRegClassFromPattern(TreePatternNode *N) {
 }
 
 std::optional<const CodeGenRegisterClass *>
-GlobalISelEmitter::inferSuperRegisterClass(const TypeSetByHwMode &Ty,
-                                           TreePatternNode *SubRegIdxNode) {
+GlobalISelEmitter::inferSuperRegisterClass(
+    const TypeSetByHwMode &Ty, const TreePatternNode *SubRegIdxNode) {
   assert(SubRegIdxNode && "Expected subregister index node!");
   // We need a ValueTypeByHwMode for getSuperRegForSubReg.
   if (!Ty.isValueTypeByHwMode(false))
@@ -5137,8 +1869,8 @@ GlobalISelEmitter::inferSuperRegisterClass(const TypeSetByHwMode &Ty,
 
 std::optional<const CodeGenRegisterClass *>
 GlobalISelEmitter::inferSuperRegisterClassForNode(
-    const TypeSetByHwMode &Ty, TreePatternNode *SuperRegNode,
-    TreePatternNode *SubRegIdxNode) {
+    const TypeSetByHwMode &Ty, const TreePatternNode *SuperRegNode,
+    const TreePatternNode *SubRegIdxNode) {
   assert(SuperRegNode && "Expected super register node!");
   // Check if we already have a defined register class for the super register
   // node. If we do, then we should preserve that rather than inferring anything
@@ -5151,8 +1883,8 @@ GlobalISelEmitter::inferSuperRegisterClassForNode(
   return inferSuperRegisterClass(Ty, SubRegIdxNode);
 }
 
-std::optional<CodeGenSubRegIndex *>
-GlobalISelEmitter::inferSubRegIndexForNode(TreePatternNode *SubRegIdxNode) {
+std::optional<CodeGenSubRegIndex *> GlobalISelEmitter::inferSubRegIndexForNode(
+    const TreePatternNode *SubRegIdxNode) {
   if (!SubRegIdxNode->isLeaf())
     return std::nullopt;
 
@@ -5211,6 +1943,9 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
   // before their first use.)
   InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(Src->getName());
   unsigned TempOpIdx = 0;
+
+  const auto SavedFlags = M.setGISelFlags(P.getSrcRecord());
+
   auto InsnMatcherOrError =
       createAndImportSelDAGMatcher(M, InsnMatcherTemp, Src, TempOpIdx);
   if (auto Error = InsnMatcherOrError.takeError())
@@ -5297,7 +2032,8 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
     } else if (DstIName == "EXTRACT_SUBREG") {
       auto InferredClass = inferRegClassFromPattern(Dst->getChild(0));
       if (!InferredClass)
-        return failedImport("Could not infer class for EXTRACT_SUBREG operand #0");
+        return failedImport(
+            "Could not infer class for EXTRACT_SUBREG operand #0");
 
       // We can assume that a subregister is in the same bank as it's super
       // register.
@@ -5379,7 +2115,7 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
     auto SuperClass = inferRegClassFromPattern(Dst->getChild(0));
     if (!SuperClass)
       return failedImport(
-        "Cannot infer register class from EXTRACT_SUBREG operand #0");
+          "Cannot infer register class from EXTRACT_SUBREG operand #0");
 
     auto SubIdx = inferSubRegIndexForNode(Dst->getChild(1));
     if (!SubIdx)
@@ -5392,17 +2128,18 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
     // FIXME: This may introduce an extra copy if the chosen class doesn't
     //        actually contain the subregisters.
     assert(Src->getExtTypes().size() == 1 &&
-             "Expected Src of EXTRACT_SUBREG to have one result type");
+           "Expected Src of EXTRACT_SUBREG to have one result type");
 
     const auto SrcRCDstRCPair =
-      (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
+        (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
     if (!SrcRCDstRCPair) {
       return failedImport("subreg index is incompatible "
                           "with inferred reg class");
     }
 
     assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
-    M.addAction<ConstrainOperandToRegClassAction>(0, 0, *SrcRCDstRCPair->second);
+    M.addAction<ConstrainOperandToRegClassAction>(0, 0,
+                                                  *SrcRCDstRCPair->second);
     M.addAction<ConstrainOperandToRegClassAction>(0, 1, *SrcRCDstRCPair->first);
 
     // We're done with this pattern!  It's eligible for GISel emission; return
@@ -5470,7 +2207,7 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
         return failedImport("REG_SEQUENCE child is not a subreg index");
 
       const auto SrcRCDstRCPair =
-        (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
+          (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx);
 
       M.addAction<ConstrainOperandToRegClassAction>(0, I,
                                                     *SrcRCDstRCPair->second);
@@ -5487,126 +2224,6 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
   return std::move(M);
 }
 
-// Emit imm predicate table and an enum to reference them with.
-// The 'Predicate_' part of the name is redundant but eliminating it is more
-// trouble than it's worth.
-void GlobalISelEmitter::emitCxxPredicateFns(
-    raw_ostream &OS, StringRef CodeFieldName, StringRef TypeIdentifier,
-    StringRef ArgType, StringRef ArgName, StringRef AdditionalArgs,
-    StringRef AdditionalDeclarations,
-    std::function<bool(const Record *R)> Filter) {
-  std::vector<const Record *> MatchedRecords;
-  const auto &Defs = RK.getAllDerivedDefinitions("PatFrags");
-  std::copy_if(Defs.begin(), Defs.end(), std::back_inserter(MatchedRecords),
-               [&](Record *Record) {
-                 return !Record->getValueAsString(CodeFieldName).empty() &&
-                        Filter(Record);
-               });
-
-  if (!MatchedRecords.empty()) {
-    OS << "// PatFrag predicates.\n"
-       << "enum {\n";
-    std::string EnumeratorSeparator =
-        (" = GIPFP_" + TypeIdentifier + "_Invalid + 1,\n").str();
-    for (const auto *Record : MatchedRecords) {
-      OS << "  GIPFP_" << TypeIdentifier << "_Predicate_" << Record->getName()
-         << EnumeratorSeparator;
-      EnumeratorSeparator = ",\n";
-    }
-    OS << "};\n";
-  }
-
-  OS << "bool " << Target.getName() << "InstructionSelector::test" << ArgName
-     << "Predicate_" << TypeIdentifier << "(unsigned PredicateID, " << ArgType << " "
-     << ArgName << AdditionalArgs <<") const {\n"
-     << AdditionalDeclarations;
-  if (!AdditionalDeclarations.empty())
-    OS << "\n";
-  if (!MatchedRecords.empty())
-    OS << "  switch (PredicateID) {\n";
-  for (const auto *Record : MatchedRecords) {
-    OS << "  case GIPFP_" << TypeIdentifier << "_Predicate_"
-       << Record->getName() << ": {\n"
-       << "    " << Record->getValueAsString(CodeFieldName) << "\n"
-       << "    llvm_unreachable(\"" << CodeFieldName
-       << " should have returned\");\n"
-       << "    return false;\n"
-       << "  }\n";
-  }
-  if (!MatchedRecords.empty())
-    OS << "  }\n";
-  OS << "  llvm_unreachable(\"Unknown predicate\");\n"
-     << "  return false;\n"
-     << "}\n";
-}
-
-void GlobalISelEmitter::emitImmPredicateFns(
-    raw_ostream &OS, StringRef TypeIdentifier, StringRef ArgType,
-    std::function<bool(const Record *R)> Filter) {
-  return emitCxxPredicateFns(OS, "ImmediateCode", TypeIdentifier, ArgType,
-                             "Imm", "", "", Filter);
-}
-
-void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) {
-  return emitCxxPredicateFns(
-      OS, "GISelPredicateCode", "MI", "const MachineInstr &", "MI",
-      ", const std::array<const MachineOperand *, 3> &Operands",
-      "  const MachineFunction &MF = *MI.getParent()->getParent();\n"
-      "  const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
-      "  (void)MRI;",
-      [](const Record *R) { return true; });
-}
-
-template <class GroupT>
-std::vector<Matcher *> GlobalISelEmitter::optimizeRules(
-    ArrayRef<Matcher *> Rules,
-    std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
-
-  std::vector<Matcher *> OptRules;
-  std::unique_ptr<GroupT> CurrentGroup = std::make_unique<GroupT>();
-  assert(CurrentGroup->empty() && "Newly created group isn't empty!");
-  unsigned NumGroups = 0;
-
-  auto ProcessCurrentGroup = [&]() {
-    if (CurrentGroup->empty())
-      // An empty group is good to be reused:
-      return;
-
-    // If the group isn't large enough to provide any benefit, move all the
-    // added rules out of it and make sure to re-create the group to properly
-    // re-initialize it:
-    if (CurrentGroup->size() < 2)
-      append_range(OptRules, CurrentGroup->matchers());
-    else {
-      CurrentGroup->finalize();
-      OptRules.push_back(CurrentGroup.get());
-      MatcherStorage.emplace_back(std::move(CurrentGroup));
-      ++NumGroups;
-    }
-    CurrentGroup = std::make_unique<GroupT>();
-  };
-  for (Matcher *Rule : Rules) {
-    // Greedily add as many matchers as possible to the current group:
-    if (CurrentGroup->addMatcher(*Rule))
-      continue;
-
-    ProcessCurrentGroup();
-    assert(CurrentGroup->empty() && "A group wasn't properly re-initialized");
-
-    // Try to add the pending matcher to a newly created empty group:
-    if (!CurrentGroup->addMatcher(*Rule))
-      // If we couldn't add the matcher to an empty group, that group type
-      // doesn't support that kind of matchers at all, so just skip it:
-      OptRules.push_back(Rule);
-  }
-  ProcessCurrentGroup();
-
-  LLVM_DEBUG(dbgs() << "NumGroups: " << NumGroups << "\n");
-  (void) NumGroups;
-  assert(CurrentGroup->empty() && "The last group wasn't properly processed");
-  return OptRules;
-}
-
 MatchTable
 GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules,
                                    bool Optimize, bool WithCoverage) {
@@ -5649,32 +2266,101 @@ GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules,
   return MatchTable::buildTable(OptRules, WithCoverage);
 }
 
-void GroupMatcher::optimize() {
-  // Make sure we only sort by a specific predicate within a range of rules that
-  // all have that predicate checked against a specific value (not a wildcard):
-  auto F = Matchers.begin();
-  auto T = F;
-  auto E = Matchers.end();
-  while (T != E) {
-    while (T != E) {
-      auto *R = static_cast<RuleMatcher *>(*T);
-      if (!R->getFirstConditionAsRootType().get().isValid())
-        break;
-      ++T;
-    }
-    std::stable_sort(F, T, [](Matcher *A, Matcher *B) {
-      auto *L = static_cast<RuleMatcher *>(A);
-      auto *R = static_cast<RuleMatcher *>(B);
-      return L->getFirstConditionAsRootType() <
-             R->getFirstConditionAsRootType();
-    });
-    if (T != E)
-      F = ++T;
-  }
-  GlobalISelEmitter::optimizeRules<GroupMatcher>(Matchers, MatcherStorage)
-      .swap(Matchers);
-  GlobalISelEmitter::optimizeRules<SwitchMatcher>(Matchers, MatcherStorage)
-      .swap(Matchers);
+void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) {
+  OS << "bool " << getClassName()
+     << "::selectImpl(MachineInstr &I, CodeGenCoverage "
+        "&CoverageInfo) const {\n"
+     << "  const PredicateBitset AvailableFeatures = "
+        "getAvailableFeatures();\n"
+     << "  NewMIVector OutMIs;\n"
+     << "  State.MIs.clear();\n"
+     << "  State.MIs.push_back(&I);\n\n"
+     << "  if (executeMatchTable(*this, OutMIs, State, ExecInfo"
+     << ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures"
+     << ", &CoverageInfo)) {\n"
+     << "    return true;\n"
+     << "  }\n\n"
+     << "  return false;\n"
+     << "}\n\n";
+}
+
+void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) {
+  std::vector<Record *> MatchedRecords;
+  std::copy_if(AllPatFrags.begin(), AllPatFrags.end(),
+               std::back_inserter(MatchedRecords), [&](Record *R) {
+                 return !R->getValueAsString("GISelPredicateCode").empty();
+               });
+  emitMIPredicateFnsImpl<Record *>(
+      OS,
+      "  const MachineFunction &MF = *MI.getParent()->getParent();\n"
+      "  const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
+      "  const auto &Operands = State.RecordedOperands;\n"
+      "  (void)Operands;\n"
+      "  (void)MRI;",
+      ArrayRef<Record *>(MatchedRecords), &getPatFragPredicateEnumName,
+      [&](Record *R) { return R->getValueAsString("GISelPredicateCode"); },
+      "PatFrag predicates.");
+}
+
+void GlobalISelEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
+  std::vector<Record *> MatchedRecords;
+  std::copy_if(AllPatFrags.begin(), AllPatFrags.end(),
+               std::back_inserter(MatchedRecords), [&](Record *R) {
+                 bool Unset;
+                 return !R->getValueAsString("ImmediateCode").empty() &&
+                        !R->getValueAsBitOrUnset("IsAPFloat", Unset) &&
+                        !R->getValueAsBit("IsAPInt");
+               });
+  emitImmPredicateFnsImpl<Record *>(
+      OS, "I64", "int64_t", ArrayRef<Record *>(MatchedRecords),
+      &getPatFragPredicateEnumName,
+      [&](Record *R) { return R->getValueAsString("ImmediateCode"); },
+      "PatFrag predicates.");
+}
+
+void GlobalISelEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {
+  std::vector<Record *> MatchedRecords;
+  std::copy_if(AllPatFrags.begin(), AllPatFrags.end(),
+               std::back_inserter(MatchedRecords), [&](Record *R) {
+                 bool Unset;
+                 return !R->getValueAsString("ImmediateCode").empty() &&
+                        R->getValueAsBitOrUnset("IsAPFloat", Unset);
+               });
+  emitImmPredicateFnsImpl<Record *>(
+      OS, "APFloat", "const APFloat &", ArrayRef<Record *>(MatchedRecords),
+      &getPatFragPredicateEnumName,
+      [&](Record *R) { return R->getValueAsString("ImmediateCode"); },
+      "PatFrag predicates.");
+}
+
+void GlobalISelEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {
+  std::vector<Record *> MatchedRecords;
+  std::copy_if(AllPatFrags.begin(), AllPatFrags.end(),
+               std::back_inserter(MatchedRecords), [&](Record *R) {
+                 return !R->getValueAsString("ImmediateCode").empty() &&
+                        R->getValueAsBit("IsAPInt");
+               });
+  emitImmPredicateFnsImpl<Record *>(
+      OS, "APInt", "const APInt &", ArrayRef<Record *>(MatchedRecords),
+      &getPatFragPredicateEnumName,
+      [&](Record *R) { return R->getValueAsString("ImmediateCode"); },
+      "PatFrag predicates.");
+}
+
+void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) {
+  OS << "bool " << getClassName() << "::testSimplePredicate(unsigned) const {\n"
+     << "    llvm_unreachable(\"" + getClassName() +
+            " does not support simple predicates!\");\n"
+     << "  return false;\n"
+     << "}\n";
+}
+
+void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) {
+  OS << "void " << getClassName()
+     << "::runCustomAction(unsigned, const MatcherState&) const {\n"
+     << "    llvm_unreachable(\"" + getClassName() +
+            " does not support custom C++ actions!\");\n"
+     << "}\n";
 }
 
 void GlobalISelEmitter::run(raw_ostream &OS) {
@@ -5700,8 +2386,12 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
   // Track the GINodeEquiv definitions.
   gatherNodeEquivs();
 
-  emitSourceFileHeader(("Global Instruction Selector for the " +
-                       Target.getName() + " target").str(), OS);
+  AllPatFrags = RK.getAllDerivedDefinitions("PatFrags");
+
+  emitSourceFileHeader(
+      ("Global Instruction Selector for the " + Target.getName() + " target")
+          .str(),
+      OS);
   std::vector<RuleMatcher> Rules;
   // Look through the SelectionDAG patterns we found, possibly emitting some.
   for (const PatternToMatch &Pat : CGP.ptms()) {
@@ -5753,203 +2443,13 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
       std::unique(CustomRendererFns.begin(), CustomRendererFns.end()),
       CustomRendererFns.end());
 
-  unsigned MaxTemporaries = 0;
-  for (const auto &Rule : Rules)
-    MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns());
-
-  OS << "#ifdef GET_GLOBALISEL_PREDICATE_BITSET\n"
-     << "const unsigned MAX_SUBTARGET_PREDICATES = " << SubtargetFeatures.size()
-     << ";\n"
-     << "using PredicateBitset = "
-        "llvm::PredicateBitsetImpl<MAX_SUBTARGET_PREDICATES>;\n"
-     << "#endif // ifdef GET_GLOBALISEL_PREDICATE_BITSET\n\n";
-
-  OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n"
-     << "  mutable MatcherState State;\n"
-     << "  typedef "
-        "ComplexRendererFns("
-     << Target.getName()
-     << "InstructionSelector::*ComplexMatcherMemFn)(MachineOperand &) const;\n"
-
-     << "  typedef void(" << Target.getName()
-     << "InstructionSelector::*CustomRendererFn)(MachineInstrBuilder &, const "
-        "MachineInstr &, int) "
-        "const;\n"
-     << "  const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, "
-        "CustomRendererFn> "
-        "ISelInfo;\n";
-  OS << "  static " << Target.getName()
-     << "InstructionSelector::ComplexMatcherMemFn ComplexPredicateFns[];\n"
-     << "  static " << Target.getName()
-     << "InstructionSelector::CustomRendererFn CustomRenderers[];\n"
-     << "  bool testImmPredicate_I64(unsigned PredicateID, int64_t Imm) const "
-        "override;\n"
-     << "  bool testImmPredicate_APInt(unsigned PredicateID, const APInt &Imm) "
-        "const override;\n"
-     << "  bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat "
-        "&Imm) const override;\n"
-     << "  const int64_t *getMatchTable() const override;\n"
-     << "  bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI"
-        ", const std::array<const MachineOperand *, 3> &Operands) "
-        "const override;\n"
-     << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n\n";
-
-  OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n"
-     << ", State(" << MaxTemporaries << "),\n"
-     << "ISelInfo(TypeObjects, NumTypeObjects, FeatureBitsets"
-     << ", ComplexPredicateFns, CustomRenderers)\n"
-     << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n\n";
-
-  OS << "#ifdef GET_GLOBALISEL_IMPL\n";
-  SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(SubtargetFeatures,
-                                                           OS);
-
-  // Separate subtarget features by how often they must be recomputed.
-  SubtargetFeatureInfoMap ModuleFeatures;
-  std::copy_if(SubtargetFeatures.begin(), SubtargetFeatures.end(),
-               std::inserter(ModuleFeatures, ModuleFeatures.end()),
-               [](const SubtargetFeatureInfoMap::value_type &X) {
-                 return !X.second.mustRecomputePerFunction();
-               });
-  SubtargetFeatureInfoMap FunctionFeatures;
-  std::copy_if(SubtargetFeatures.begin(), SubtargetFeatures.end(),
-               std::inserter(FunctionFeatures, FunctionFeatures.end()),
-               [](const SubtargetFeatureInfoMap::value_type &X) {
-                 return X.second.mustRecomputePerFunction();
-               });
-
-  SubtargetFeatureInfo::emitComputeAvailableFeatures(
-    Target.getName(), "InstructionSelector", "computeAvailableModuleFeatures",
-      ModuleFeatures, OS);
-
-
-  OS << "void " << Target.getName() << "InstructionSelector"
-    "::setupGeneratedPerFunctionState(MachineFunction &MF) {\n"
-    "  AvailableFunctionFeatures = computeAvailableFunctionFeatures("
-    "(const " << Target.getName() << "Subtarget *)&MF.getSubtarget(), &MF);\n"
-    "}\n";
-
-  SubtargetFeatureInfo::emitComputeAvailableFeatures(
-      Target.getName(), "InstructionSelector",
-      "computeAvailableFunctionFeatures", FunctionFeatures, OS,
-      "const MachineFunction *MF");
-
-  // Emit a table containing the LLT objects needed by the matcher and an enum
+  // Create a table containing the LLT objects needed by the matcher and an enum
   // for the matcher to reference them with.
   std::vector<LLTCodeGen> TypeObjects;
   append_range(TypeObjects, KnownTypes);
   llvm::sort(TypeObjects);
-  OS << "// LLT Objects.\n"
-     << "enum {\n";
-  for (const auto &TypeObject : TypeObjects) {
-    OS << "  ";
-    TypeObject.emitCxxEnumValue(OS);
-    OS << ",\n";
-  }
-  OS << "};\n";
-  OS << "const static size_t NumTypeObjects = " << TypeObjects.size() << ";\n"
-     << "const static LLT TypeObjects[] = {\n";
-  for (const auto &TypeObject : TypeObjects) {
-    OS << "  ";
-    TypeObject.emitCxxConstructorCall(OS);
-    OS << ",\n";
-  }
-  OS << "};\n\n";
-
-  // Emit a table containing the PredicateBitsets objects needed by the matcher
-  // and an enum for the matcher to reference them with.
-  std::vector<std::vector<Record *>> FeatureBitsets;
-  FeatureBitsets.reserve(Rules.size());
-  for (auto &Rule : Rules)
-    FeatureBitsets.push_back(Rule.getRequiredFeatures());
-  llvm::sort(FeatureBitsets, [&](const std::vector<Record *> &A,
-                                 const std::vector<Record *> &B) {
-    if (A.size() < B.size())
-      return true;
-    if (A.size() > B.size())
-      return false;
-    for (auto Pair : zip(A, B)) {
-      if (std::get<0>(Pair)->getName() < std::get<1>(Pair)->getName())
-        return true;
-      if (std::get<0>(Pair)->getName() > std::get<1>(Pair)->getName())
-        return false;
-    }
-    return false;
-  });
-  FeatureBitsets.erase(
-      std::unique(FeatureBitsets.begin(), FeatureBitsets.end()),
-      FeatureBitsets.end());
-  OS << "// Feature bitsets.\n"
-     << "enum {\n"
-     << "  GIFBS_Invalid,\n";
-  for (const auto &FeatureBitset : FeatureBitsets) {
-    if (FeatureBitset.empty())
-      continue;
-    OS << "  " << getNameForFeatureBitset(FeatureBitset) << ",\n";
-  }
-  OS << "};\n"
-     << "const static PredicateBitset FeatureBitsets[] {\n"
-     << "  {}, // GIFBS_Invalid\n";
-  for (const auto &FeatureBitset : FeatureBitsets) {
-    if (FeatureBitset.empty())
-      continue;
-    OS << "  {";
-    for (const auto &Feature : FeatureBitset) {
-      const auto &I = SubtargetFeatures.find(Feature);
-      assert(I != SubtargetFeatures.end() && "Didn't import predicate?");
-      OS << I->second.getEnumBitName() << ", ";
-    }
-    OS << "},\n";
-  }
-  OS << "};\n\n";
-
-  // Emit complex predicate table and an enum to reference them with.
-  OS << "// ComplexPattern predicates.\n"
-     << "enum {\n"
-     << "  GICP_Invalid,\n";
-  for (const auto &Record : ComplexPredicates)
-    OS << "  GICP_" << Record->getName() << ",\n";
-  OS << "};\n"
-     << "// See constructor for table contents\n\n";
-
-  emitImmPredicateFns(OS, "I64", "int64_t", [](const Record *R) {
-    bool Unset;
-    return !R->getValueAsBitOrUnset("IsAPFloat", Unset) &&
-           !R->getValueAsBit("IsAPInt");
-  });
-  emitImmPredicateFns(OS, "APFloat", "const APFloat &", [](const Record *R) {
-    bool Unset;
-    return R->getValueAsBitOrUnset("IsAPFloat", Unset);
-  });
-  emitImmPredicateFns(OS, "APInt", "const APInt &", [](const Record *R) {
-    return R->getValueAsBit("IsAPInt");
-  });
-  emitMIPredicateFns(OS);
-  OS << "\n";
-
-  OS << Target.getName() << "InstructionSelector::ComplexMatcherMemFn\n"
-     << Target.getName() << "InstructionSelector::ComplexPredicateFns[] = {\n"
-     << "  nullptr, // GICP_Invalid\n";
-  for (const auto &Record : ComplexPredicates)
-    OS << "  &" << Target.getName()
-       << "InstructionSelector::" << Record->getValueAsString("MatcherFn")
-       << ", // " << Record->getName() << "\n";
-  OS << "};\n\n";
-
-  OS << "// Custom renderers.\n"
-     << "enum {\n"
-     << "  GICR_Invalid,\n";
-  for (const auto &Fn : CustomRendererFns)
-    OS << "  GICR_" << Fn << ",\n";
-  OS << "};\n";
-
-  OS << Target.getName() << "InstructionSelector::CustomRendererFn\n"
-     << Target.getName() << "InstructionSelector::CustomRenderers[] = {\n"
-     << "  nullptr, // GICR_Invalid\n";
-  for (const auto &Fn : CustomRendererFns)
-    OS << "  &" << Target.getName() << "InstructionSelector::" << Fn << ",\n";
-  OS << "};\n\n";
 
+  // Sort rules.
   llvm::stable_sort(Rules, [&](const RuleMatcher &A, const RuleMatcher &B) {
     int ScoreA = RuleMatcherScores[A.getRuleID()];
     int ScoreB = RuleMatcherScores[B.getRuleID()];
@@ -5966,53 +2466,21 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
     return false;
   });
 
-  OS << "bool " << Target.getName()
-     << "InstructionSelector::selectImpl(MachineInstr &I, CodeGenCoverage "
-        "&CoverageInfo) const {\n"
-     << "  MachineFunction &MF = *I.getParent()->getParent();\n"
-     << "  MachineRegisterInfo &MRI = MF.getRegInfo();\n"
-     << "  const PredicateBitset AvailableFeatures = getAvailableFeatures();\n"
-     << "  NewMIVector OutMIs;\n"
-     << "  State.MIs.clear();\n"
-     << "  State.MIs.push_back(&I);\n\n"
-     << "  if (executeMatchTable(*this, OutMIs, State, ISelInfo"
-     << ", getMatchTable(), TII, MRI, TRI, RBI, AvailableFeatures"
-     << ", CoverageInfo)) {\n"
-     << "    return true;\n"
-     << "  }\n\n"
-     << "  return false;\n"
-     << "}\n\n";
+  unsigned MaxTemporaries = 0;
+  for (const auto &Rule : Rules)
+    MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns());
 
+  // Build match table
   const MatchTable Table =
       buildMatchTable(Rules, OptimizeMatchTable, GenerateCoverage);
-  OS << "const int64_t *" << Target.getName()
-     << "InstructionSelector::getMatchTable() const {\n";
-  Table.emitDeclaration(OS);
-  OS << "  return ";
-  Table.emitUse(OS);
-  OS << ";\n}\n";
-  OS << "#endif // ifdef GET_GLOBALISEL_IMPL\n";
 
-  OS << "#ifdef GET_GLOBALISEL_PREDICATES_DECL\n"
-     << "PredicateBitset AvailableModuleFeatures;\n"
-     << "mutable PredicateBitset AvailableFunctionFeatures;\n"
-     << "PredicateBitset getAvailableFeatures() const {\n"
-     << "  return AvailableModuleFeatures | AvailableFunctionFeatures;\n"
-     << "}\n"
-     << "PredicateBitset\n"
-     << "computeAvailableModuleFeatures(const " << Target.getName()
-     << "Subtarget *Subtarget) const;\n"
-     << "PredicateBitset\n"
-     << "computeAvailableFunctionFeatures(const " << Target.getName()
-     << "Subtarget *Subtarget,\n"
-     << "                                 const MachineFunction *MF) const;\n"
-     << "void setupGeneratedPerFunctionState(MachineFunction &MF) override;\n"
-     << "#endif // ifdef GET_GLOBALISEL_PREDICATES_DECL\n";
-
-  OS << "#ifdef GET_GLOBALISEL_PREDICATES_INIT\n"
-     << "AvailableModuleFeatures(computeAvailableModuleFeatures(&STI)),\n"
-     << "AvailableFunctionFeatures()\n"
-     << "#endif // ifdef GET_GLOBALISEL_PREDICATES_INIT\n";
+  emitPredicateBitset(OS, "GET_GLOBALISEL_PREDICATE_BITSET");
+  emitTemporariesDecl(OS, "GET_GLOBALISEL_TEMPORARIES_DECL");
+  emitTemporariesInit(OS, MaxTemporaries, "GET_GLOBALISEL_TEMPORARIES_INIT");
+  emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexPredicates,
+                   CustomRendererFns, "GET_GLOBALISEL_IMPL");
+  emitPredicatesDecl(OS, "GET_GLOBALISEL_PREDICATES_DECL");
+  emitPredicatesInit(OS, "GET_GLOBALISEL_PREDICATES_INIT");
 }
 
 void GlobalISelEmitter::declareSubtargetFeature(Record *Predicate) {
@@ -6021,294 +2489,9 @@ void GlobalISelEmitter::declareSubtargetFeature(Record *Predicate) {
         Predicate, SubtargetFeatureInfo(Predicate, SubtargetFeatures.size()));
 }
 
-void RuleMatcher::optimize() {
-  for (auto &Item : InsnVariableIDs) {
-    InstructionMatcher &InsnMatcher = *Item.first;
-    for (auto &OM : InsnMatcher.operands()) {
-      // Complex Patterns are usually expensive and they relatively rarely fail
-      // on their own: more often we end up throwing away all the work done by a
-      // matching part of a complex pattern because some other part of the
-      // enclosing pattern didn't match. All of this makes it beneficial to
-      // delay complex patterns until the very end of the rule matching,
-      // especially for targets having lots of complex patterns.
-      for (auto &OP : OM->predicates())
-        if (isa<ComplexPatternOperandMatcher>(OP))
-          EpilogueMatchers.emplace_back(std::move(OP));
-      OM->eraseNullPredicates();
-    }
-    InsnMatcher.optimize();
-  }
-  llvm::sort(EpilogueMatchers, [](const std::unique_ptr<PredicateMatcher> &L,
-                                  const std::unique_ptr<PredicateMatcher> &R) {
-    return std::make_tuple(L->getKind(), L->getInsnVarID(), L->getOpIdx()) <
-           std::make_tuple(R->getKind(), R->getInsnVarID(), R->getOpIdx());
-  });
-}
-
-bool RuleMatcher::hasFirstCondition() const {
-  if (insnmatchers_empty())
-    return false;
-  InstructionMatcher &Matcher = insnmatchers_front();
-  if (!Matcher.predicates_empty())
-    return true;
-  for (auto &OM : Matcher.operands())
-    for (auto &OP : OM->predicates())
-      if (!isa<InstructionOperandMatcher>(OP))
-        return true;
-  return false;
-}
-
-const PredicateMatcher &RuleMatcher::getFirstCondition() const {
-  assert(!insnmatchers_empty() &&
-         "Trying to get a condition from an empty RuleMatcher");
-
-  InstructionMatcher &Matcher = insnmatchers_front();
-  if (!Matcher.predicates_empty())
-    return **Matcher.predicates_begin();
-  // If there is no more predicate on the instruction itself, look at its
-  // operands.
-  for (auto &OM : Matcher.operands())
-    for (auto &OP : OM->predicates())
-      if (!isa<InstructionOperandMatcher>(OP))
-        return *OP;
-
-  llvm_unreachable("Trying to get a condition from an InstructionMatcher with "
-                   "no conditions");
-}
-
-std::unique_ptr<PredicateMatcher> RuleMatcher::popFirstCondition() {
-  assert(!insnmatchers_empty() &&
-         "Trying to pop a condition from an empty RuleMatcher");
-
-  InstructionMatcher &Matcher = insnmatchers_front();
-  if (!Matcher.predicates_empty())
-    return Matcher.predicates_pop_front();
-  // If there is no more predicate on the instruction itself, look at its
-  // operands.
-  for (auto &OM : Matcher.operands())
-    for (auto &OP : OM->predicates())
-      if (!isa<InstructionOperandMatcher>(OP)) {
-        std::unique_ptr<PredicateMatcher> Result = std::move(OP);
-        OM->eraseNullPredicates();
-        return Result;
-      }
-
-  llvm_unreachable("Trying to pop a condition from an InstructionMatcher with "
-                   "no conditions");
-}
-
-bool GroupMatcher::candidateConditionMatches(
-    const PredicateMatcher &Predicate) const {
-
-  if (empty()) {
-    // Sharing predicates for nested instructions is not supported yet as we
-    // currently don't hoist the GIM_RecordInsn's properly, therefore we can
-    // only work on the original root instruction (InsnVarID == 0):
-    if (Predicate.getInsnVarID() != 0)
-      return false;
-    // ... otherwise an empty group can handle any predicate with no specific
-    // requirements:
-    return true;
-  }
-
-  const Matcher &Representative = **Matchers.begin();
-  const auto &RepresentativeCondition = Representative.getFirstCondition();
-  // ... if not empty, the group can only accomodate matchers with the exact
-  // same first condition:
-  return Predicate.isIdentical(RepresentativeCondition);
-}
-
-bool GroupMatcher::addMatcher(Matcher &Candidate) {
-  if (!Candidate.hasFirstCondition())
-    return false;
-
-  const PredicateMatcher &Predicate = Candidate.getFirstCondition();
-  if (!candidateConditionMatches(Predicate))
-    return false;
-
-  Matchers.push_back(&Candidate);
-  return true;
-}
-
-void GroupMatcher::finalize() {
-  assert(Conditions.empty() && "Already finalized?");
-  if (empty())
-    return;
-
-  Matcher &FirstRule = **Matchers.begin();
-  for (;;) {
-    // All the checks are expected to succeed during the first iteration:
-    for (const auto &Rule : Matchers)
-      if (!Rule->hasFirstCondition())
-        return;
-    const auto &FirstCondition = FirstRule.getFirstCondition();
-    for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
-      if (!Matchers[I]->getFirstCondition().isIdentical(FirstCondition))
-        return;
-
-    Conditions.push_back(FirstRule.popFirstCondition());
-    for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
-      Matchers[I]->popFirstCondition();
-  }
-}
-
-void GroupMatcher::emit(MatchTable &Table) {
-  unsigned LabelID = ~0U;
-  if (!Conditions.empty()) {
-    LabelID = Table.allocateLabelID();
-    Table << MatchTable::Opcode("GIM_Try", +1)
-          << MatchTable::Comment("On fail goto")
-          << MatchTable::JumpTarget(LabelID) << MatchTable::LineBreak;
-  }
-  for (auto &Condition : Conditions)
-    Condition->emitPredicateOpcodes(
-        Table, *static_cast<RuleMatcher *>(*Matchers.begin()));
-
-  for (const auto &M : Matchers)
-    M->emit(Table);
-
-  // Exit the group
-  if (!Conditions.empty())
-    Table << MatchTable::Opcode("GIM_Reject", -1) << MatchTable::LineBreak
-          << MatchTable::Label(LabelID);
-}
-
-bool SwitchMatcher::isSupportedPredicateType(const PredicateMatcher &P) {
-  return isa<InstructionOpcodeMatcher>(P) || isa<LLTOperandMatcher>(P);
-}
-
-bool SwitchMatcher::candidateConditionMatches(
-    const PredicateMatcher &Predicate) const {
-
-  if (empty()) {
-    // Sharing predicates for nested instructions is not supported yet as we
-    // currently don't hoist the GIM_RecordInsn's properly, therefore we can
-    // only work on the original root instruction (InsnVarID == 0):
-    if (Predicate.getInsnVarID() != 0)
-      return false;
-    // ... while an attempt to add even a root matcher to an empty SwitchMatcher
-    // could fail as not all the types of conditions are supported:
-    if (!isSupportedPredicateType(Predicate))
-      return false;
-    // ... or the condition might not have a proper implementation of
-    // getValue() / isIdenticalDownToValue() yet:
-    if (!Predicate.hasValue())
-      return false;
-    // ... otherwise an empty Switch can accomodate the condition with no
-    // further requirements:
-    return true;
-  }
-
-  const Matcher &CaseRepresentative = **Matchers.begin();
-  const auto &RepresentativeCondition = CaseRepresentative.getFirstCondition();
-  // Switch-cases must share the same kind of condition and path to the value it
-  // checks:
-  if (!Predicate.isIdenticalDownToValue(RepresentativeCondition))
-    return false;
-
-  const auto Value = Predicate.getValue();
-  // ... but be unique with respect to the actual value they check:
-  return Values.count(Value) == 0;
-}
-
-bool SwitchMatcher::addMatcher(Matcher &Candidate) {
-  if (!Candidate.hasFirstCondition())
-    return false;
-
-  const PredicateMatcher &Predicate = Candidate.getFirstCondition();
-  if (!candidateConditionMatches(Predicate))
-    return false;
-  const auto Value = Predicate.getValue();
-  Values.insert(Value);
-
-  Matchers.push_back(&Candidate);
-  return true;
-}
-
-void SwitchMatcher::finalize() {
-  assert(Condition == nullptr && "Already finalized");
-  assert(Values.size() == Matchers.size() && "Broken SwitchMatcher");
-  if (empty())
-    return;
-
-  llvm::stable_sort(Matchers, [](const Matcher *L, const Matcher *R) {
-    return L->getFirstCondition().getValue() <
-           R->getFirstCondition().getValue();
-  });
-  Condition = Matchers[0]->popFirstCondition();
-  for (unsigned I = 1, E = Values.size(); I < E; ++I)
-    Matchers[I]->popFirstCondition();
-}
-
-void SwitchMatcher::emitPredicateSpecificOpcodes(const PredicateMatcher &P,
-                                                 MatchTable &Table) {
-  assert(isSupportedPredicateType(P) && "Predicate type is not supported");
-
-  if (const auto *Condition = dyn_cast<InstructionOpcodeMatcher>(&P)) {
-    Table << MatchTable::Opcode("GIM_SwitchOpcode") << MatchTable::Comment("MI")
-          << MatchTable::IntValue(Condition->getInsnVarID());
-    return;
-  }
-  if (const auto *Condition = dyn_cast<LLTOperandMatcher>(&P)) {
-    Table << MatchTable::Opcode("GIM_SwitchType") << MatchTable::Comment("MI")
-          << MatchTable::IntValue(Condition->getInsnVarID())
-          << MatchTable::Comment("Op")
-          << MatchTable::IntValue(Condition->getOpIdx());
-    return;
-  }
-
-  llvm_unreachable("emitPredicateSpecificOpcodes is broken: can not handle a "
-                   "predicate type that is claimed to be supported");
-}
-
-void SwitchMatcher::emit(MatchTable &Table) {
-  assert(Values.size() == Matchers.size() && "Broken SwitchMatcher");
-  if (empty())
-    return;
-  assert(Condition != nullptr &&
-         "Broken SwitchMatcher, hasn't been finalized?");
-
-  std::vector<unsigned> LabelIDs(Values.size());
-  std::generate(LabelIDs.begin(), LabelIDs.end(),
-                [&Table]() { return Table.allocateLabelID(); });
-  const unsigned Default = Table.allocateLabelID();
-
-  const int64_t LowerBound = Values.begin()->getRawValue();
-  const int64_t UpperBound = Values.rbegin()->getRawValue() + 1;
-
-  emitPredicateSpecificOpcodes(*Condition, Table);
-
-  Table << MatchTable::Comment("[") << MatchTable::IntValue(LowerBound)
-        << MatchTable::IntValue(UpperBound) << MatchTable::Comment(")")
-        << MatchTable::Comment("default:") << MatchTable::JumpTarget(Default);
-
-  int64_t J = LowerBound;
-  auto VI = Values.begin();
-  for (unsigned I = 0, E = Values.size(); I < E; ++I) {
-    auto V = *VI++;
-    while (J++ < V.getRawValue())
-      Table << MatchTable::IntValue(0);
-    V.turnIntoComment();
-    Table << MatchTable::LineBreak << V << MatchTable::JumpTarget(LabelIDs[I]);
-  }
-  Table << MatchTable::LineBreak;
-
-  for (unsigned I = 0, E = Values.size(); I < E; ++I) {
-    Table << MatchTable::Label(LabelIDs[I]);
-    Matchers[I]->emit(Table);
-    Table << MatchTable::Opcode("GIM_Reject") << MatchTable::LineBreak;
-  }
-  Table << MatchTable::Label(Default);
-}
-
-unsigned OperandMatcher::getInsnVarID() const { return Insn.getInsnVarID(); }
-
 } // end anonymous namespace
 
 //===----------------------------------------------------------------------===//
 
-namespace llvm {
-void EmitGlobalISel(RecordKeeper &RK, raw_ostream &OS) {
-  GlobalISelEmitter(RK).run(OS);
-}
-} // End llvm namespace
+static TableGen::Emitter::OptClass<GlobalISelEmitter>
+    X("gen-global-isel", "Generate GlobalISel selector");
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp
new file mode 100644
index 000000000000..aab772f020a6
--- /dev/null
+++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp
@@ -0,0 +1,2019 @@
+//===- GlobalISelMatchTable.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "GlobalISelMatchTable.h"
+#include "CodeGenInstruction.h"
+#include "CodeGenRegisters.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Error.h"
+
+#define DEBUG_TYPE "gi-match-table"
+
+STATISTIC(NumPatternEmitted, "Number of patterns emitted");
+
+namespace llvm {
+namespace gi {
+
+namespace {
+
+Error failUnsupported(const Twine &Reason) {
+  return make_error<StringError>(Reason, inconvertibleErrorCode());
+}
+
+/// Get the name of the enum value used to number the predicate function.
+std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) {
+  if (Predicate.hasGISelPredicateCode())
+    return "GICXXPred_MI_" + Predicate.getFnName();
+  return "GICXXPred_" + Predicate.getImmTypeIdentifier().str() + "_" +
+         Predicate.getFnName();
+}
+
+std::string getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) {
+  return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate";
+}
+} // namespace
+
+//===- Helpers ------------------------------------------------------------===//
+
+std::string
+getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset) {
+  std::string Name = "GIFBS";
+  for (const auto &Feature : FeatureBitset)
+    Name += ("_" + Feature->getName()).str();
+  return Name;
+}
+
+template <class GroupT>
+std::vector<Matcher *>
+optimizeRules(ArrayRef<Matcher *> Rules,
+              std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
+
+  std::vector<Matcher *> OptRules;
+  std::unique_ptr<GroupT> CurrentGroup = std::make_unique<GroupT>();
+  assert(CurrentGroup->empty() && "Newly created group isn't empty!");
+  unsigned NumGroups = 0;
+
+  auto ProcessCurrentGroup = [&]() {
+    if (CurrentGroup->empty())
+      // An empty group is good to be reused:
+      return;
+
+    // If the group isn't large enough to provide any benefit, move all the
+    // added rules out of it and make sure to re-create the group to properly
+    // re-initialize it:
+    if (CurrentGroup->size() < 2)
+      append_range(OptRules, CurrentGroup->matchers());
+    else {
+      CurrentGroup->finalize();
+      OptRules.push_back(CurrentGroup.get());
+      MatcherStorage.emplace_back(std::move(CurrentGroup));
+      ++NumGroups;
+    }
+    CurrentGroup = std::make_unique<GroupT>();
+  };
+  for (Matcher *Rule : Rules) {
+    // Greedily add as many matchers as possible to the current group:
+    if (CurrentGroup->addMatcher(*Rule))
+      continue;
+
+    ProcessCurrentGroup();
+    assert(CurrentGroup->empty() && "A group wasn't properly re-initialized");
+
+    // Try to add the pending matcher to a newly created empty group:
+    if (!CurrentGroup->addMatcher(*Rule))
+      // If we couldn't add the matcher to an empty group, that group type
+      // doesn't support that kind of matchers at all, so just skip it:
+      OptRules.push_back(Rule);
+  }
+  ProcessCurrentGroup();
+
+  LLVM_DEBUG(dbgs() << "NumGroups: " << NumGroups << "\n");
+  (void)NumGroups;
+  assert(CurrentGroup->empty() && "The last group wasn't properly processed");
+  return OptRules;
+}
+
+template std::vector<Matcher *> optimizeRules<GroupMatcher>(
+    ArrayRef<Matcher *> Rules,
+    std::vector<std::unique_ptr<Matcher>> &MatcherStorage);
+
+template std::vector<Matcher *> optimizeRules<SwitchMatcher>(
+    ArrayRef<Matcher *> Rules,
+    std::vector<std::unique_ptr<Matcher>> &MatcherStorage);
+
+//===- Global Data --------------------------------------------------------===//
+
+std::set<LLTCodeGen> KnownTypes;
+
+//===- MatchTableRecord ---------------------------------------------------===//
+
+void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis,
+                            const MatchTable &Table) const {
+  bool UseLineComment =
+      LineBreakIsNextAfterThis || (Flags & MTRF_LineBreakFollows);
+  if (Flags & (MTRF_JumpTarget | MTRF_CommaFollows))
+    UseLineComment = false;
+
+  if (Flags & MTRF_Comment)
+    OS << (UseLineComment ? "// " : "/*");
+
+  OS << EmitStr;
+  if (Flags & MTRF_Label)
+    OS << ": @" << Table.getLabelIndex(LabelID);
+
+  if ((Flags & MTRF_Comment) && !UseLineComment)
+    OS << "*/";
+
+  if (Flags & MTRF_JumpTarget) {
+    if (Flags & MTRF_Comment)
+      OS << " ";
+    OS << Table.getLabelIndex(LabelID);
+  }
+
+  if (Flags & MTRF_CommaFollows) {
+    OS << ",";
+    if (!LineBreakIsNextAfterThis && !(Flags & MTRF_LineBreakFollows))
+      OS << " ";
+  }
+
+  if (Flags & MTRF_LineBreakFollows)
+    OS << "\n";
+}
+
+//===- MatchTable ---------------------------------------------------------===//
+
+MatchTableRecord MatchTable::LineBreak = {
+    std::nullopt, "" /* Emit String */, 0 /* Elements */,
+    MatchTableRecord::MTRF_LineBreakFollows};
+
+MatchTableRecord MatchTable::Comment(StringRef Comment) {
+  return MatchTableRecord(std::nullopt, Comment, 0,
+                          MatchTableRecord::MTRF_Comment);
+}
+
+MatchTableRecord MatchTable::Opcode(StringRef Opcode, int IndentAdjust) {
+  unsigned ExtraFlags = 0;
+  if (IndentAdjust > 0)
+    ExtraFlags |= MatchTableRecord::MTRF_Indent;
+  if (IndentAdjust < 0)
+    ExtraFlags |= MatchTableRecord::MTRF_Outdent;
+
+  return MatchTableRecord(std::nullopt, Opcode, 1,
+                          MatchTableRecord::MTRF_CommaFollows | ExtraFlags);
+}
+
+MatchTableRecord MatchTable::NamedValue(StringRef NamedValue) {
+  return MatchTableRecord(std::nullopt, NamedValue, 1,
+                          MatchTableRecord::MTRF_CommaFollows);
+}
+
+MatchTableRecord MatchTable::NamedValue(StringRef NamedValue,
+                                        int64_t RawValue) {
+  return MatchTableRecord(std::nullopt, NamedValue, 1,
+                          MatchTableRecord::MTRF_CommaFollows, RawValue);
+}
+
+MatchTableRecord MatchTable::NamedValue(StringRef Namespace,
+                                        StringRef NamedValue) {
+  return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
+                          1, MatchTableRecord::MTRF_CommaFollows);
+}
+
+MatchTableRecord MatchTable::NamedValue(StringRef Namespace,
+                                        StringRef NamedValue,
+                                        int64_t RawValue) {
+  return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
+                          1, MatchTableRecord::MTRF_CommaFollows, RawValue);
+}
+
+MatchTableRecord MatchTable::IntValue(int64_t IntValue) {
+  return MatchTableRecord(std::nullopt, llvm::to_string(IntValue), 1,
+                          MatchTableRecord::MTRF_CommaFollows);
+}
+
+MatchTableRecord MatchTable::Label(unsigned LabelID) {
+  return MatchTableRecord(LabelID, "Label " + llvm::to_string(LabelID), 0,
+                          MatchTableRecord::MTRF_Label |
+                              MatchTableRecord::MTRF_Comment |
+                              MatchTableRecord::MTRF_LineBreakFollows);
+}
+
+MatchTableRecord MatchTable::JumpTarget(unsigned LabelID) {
+  return MatchTableRecord(LabelID, "Label " + llvm::to_string(LabelID), 1,
+                          MatchTableRecord::MTRF_JumpTarget |
+                              MatchTableRecord::MTRF_Comment |
+                              MatchTableRecord::MTRF_CommaFollows);
+}
+
+void MatchTable::emitUse(raw_ostream &OS) const { OS << "MatchTable" << ID; }
+
+void MatchTable::emitDeclaration(raw_ostream &OS) const {
+  unsigned Indentation = 4;
+  OS << "  constexpr static int64_t MatchTable" << ID << "[] = {";
+  LineBreak.emit(OS, true, *this);
+  OS << std::string(Indentation, ' ');
+
+  for (auto I = Contents.begin(), E = Contents.end(); I != E; ++I) {
+    bool LineBreakIsNext = false;
+    const auto &NextI = std::next(I);
+
+    if (NextI != E) {
+      if (NextI->EmitStr == "" &&
+          NextI->Flags == MatchTableRecord::MTRF_LineBreakFollows)
+        LineBreakIsNext = true;
+    }
+
+    if (I->Flags & MatchTableRecord::MTRF_Indent)
+      Indentation += 2;
+
+    I->emit(OS, LineBreakIsNext, *this);
+    if (I->Flags & MatchTableRecord::MTRF_LineBreakFollows)
+      OS << std::string(Indentation, ' ');
+
+    if (I->Flags & MatchTableRecord::MTRF_Outdent)
+      Indentation -= 2;
+  }
+  OS << "};\n";
+}
+
+MatchTable MatchTable::buildTable(ArrayRef<Matcher *> Rules, bool WithCoverage,
+                                  bool IsCombiner) {
+  MatchTable Table(WithCoverage, IsCombiner);
+  for (Matcher *Rule : Rules)
+    Rule->emit(Table);
+
+  return Table << MatchTable::Opcode("GIM_Reject") << MatchTable::LineBreak;
+}
+
+//===- LLTCodeGen ---------------------------------------------------------===//
+
+std::string LLTCodeGen::getCxxEnumValue() const {
+  std::string Str;
+  raw_string_ostream OS(Str);
+
+  emitCxxEnumValue(OS);
+  return Str;
+}
+
+void LLTCodeGen::emitCxxEnumValue(raw_ostream &OS) const {
+  if (Ty.isScalar()) {
+    OS << "GILLT_s" << Ty.getSizeInBits();
+    return;
+  }
+  if (Ty.isVector()) {
+    OS << (Ty.isScalable() ? "GILLT_nxv" : "GILLT_v")
+       << Ty.getElementCount().getKnownMinValue() << "s"
+       << Ty.getScalarSizeInBits();
+    return;
+  }
+  if (Ty.isPointer()) {
+    OS << "GILLT_p" << Ty.getAddressSpace();
+    if (Ty.getSizeInBits() > 0)
+      OS << "s" << Ty.getSizeInBits();
+    return;
+  }
+  llvm_unreachable("Unhandled LLT");
+}
+
+void LLTCodeGen::emitCxxConstructorCall(raw_ostream &OS) const {
+  if (Ty.isScalar()) {
+    OS << "LLT::scalar(" << Ty.getSizeInBits() << ")";
+    return;
+  }
+  if (Ty.isVector()) {
+    OS << "LLT::vector("
+       << (Ty.isScalable() ? "ElementCount::getScalable("
+                           : "ElementCount::getFixed(")
+       << Ty.getElementCount().getKnownMinValue() << "), "
+       << Ty.getScalarSizeInBits() << ")";
+    return;
+  }
+  if (Ty.isPointer() && Ty.getSizeInBits() > 0) {
+    OS << "LLT::pointer(" << Ty.getAddressSpace() << ", " << Ty.getSizeInBits()
+       << ")";
+    return;
+  }
+  llvm_unreachable("Unhandled LLT");
+}
+
+/// This ordering is used for std::unique() and llvm::sort(). There's no
+/// particular logic behind the order but either A < B or B < A must be
+/// true if A != B.
+bool LLTCodeGen::operator<(const LLTCodeGen &Other) const {
+  if (Ty.isValid() != Other.Ty.isValid())
+    return Ty.isValid() < Other.Ty.isValid();
+  if (!Ty.isValid())
+    return false;
+
+  if (Ty.isVector() != Other.Ty.isVector())
+    return Ty.isVector() < Other.Ty.isVector();
+  if (Ty.isScalar() != Other.Ty.isScalar())
+    return Ty.isScalar() < Other.Ty.isScalar();
+  if (Ty.isPointer() != Other.Ty.isPointer())
+    return Ty.isPointer() < Other.Ty.isPointer();
+
+  if (Ty.isPointer() && Ty.getAddressSpace() != Other.Ty.getAddressSpace())
+    return Ty.getAddressSpace() < Other.Ty.getAddressSpace();
+
+  if (Ty.isVector() && Ty.getElementCount() != Other.Ty.getElementCount())
+    return std::make_tuple(Ty.isScalable(),
+                           Ty.getElementCount().getKnownMinValue()) <
+           std::make_tuple(Other.Ty.isScalable(),
+                           Other.Ty.getElementCount().getKnownMinValue());
+
+  assert((!Ty.isVector() || Ty.isScalable() == Other.Ty.isScalable()) &&
+         "Unexpected mismatch of scalable property");
+  return Ty.isVector()
+             ? std::make_tuple(Ty.isScalable(),
+                               Ty.getSizeInBits().getKnownMinValue()) <
+                   std::make_tuple(Other.Ty.isScalable(),
+                                   Other.Ty.getSizeInBits().getKnownMinValue())
+             : Ty.getSizeInBits().getFixedValue() <
+                   Other.Ty.getSizeInBits().getFixedValue();
+}
+
+//===- LLTCodeGen Helpers -------------------------------------------------===//
+
+std::optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT) {
+  MVT VT(SVT);
+
+  if (VT.isVector() && !VT.getVectorElementCount().isScalar())
+    return LLTCodeGen(
+        LLT::vector(VT.getVectorElementCount(), VT.getScalarSizeInBits()));
+
+  if (VT.isInteger() || VT.isFloatingPoint())
+    return LLTCodeGen(LLT::scalar(VT.getSizeInBits()));
+
+  return std::nullopt;
+}
+
+//===- Matcher ------------------------------------------------------------===//
+
+void Matcher::optimize() {}
+
+Matcher::~Matcher() {}
+
+//===- GroupMatcher -------------------------------------------------------===//
+
+bool GroupMatcher::candidateConditionMatches(
+    const PredicateMatcher &Predicate) const {
+
+  if (empty()) {
+    // Sharing predicates for nested instructions is not supported yet as we
+    // currently don't hoist the GIM_RecordInsn's properly, therefore we can
+    // only work on the original root instruction (InsnVarID == 0):
+    if (Predicate.getInsnVarID() != 0)
+      return false;
+    // ... otherwise an empty group can handle any predicate with no specific
+    // requirements:
+    return true;
+  }
+
+  const Matcher &Representative = **Matchers.begin();
+  const auto &RepresentativeCondition = Representative.getFirstCondition();
+  // ... if not empty, the group can only accomodate matchers with the exact
+  // same first condition:
+  return Predicate.isIdentical(RepresentativeCondition);
+}
+
+bool GroupMatcher::addMatcher(Matcher &Candidate) {
+  if (!Candidate.hasFirstCondition())
+    return false;
+
+  const PredicateMatcher &Predicate = Candidate.getFirstCondition();
+  if (!candidateConditionMatches(Predicate))
+    return false;
+
+  Matchers.push_back(&Candidate);
+  return true;
+}
+
+void GroupMatcher::finalize() {
+  assert(Conditions.empty() && "Already finalized?");
+  if (empty())
+    return;
+
+  Matcher &FirstRule = **Matchers.begin();
+  for (;;) {
+    // All the checks are expected to succeed during the first iteration:
+    for (const auto &Rule : Matchers)
+      if (!Rule->hasFirstCondition())
+        return;
+    const auto &FirstCondition = FirstRule.getFirstCondition();
+    for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
+      if (!Matchers[I]->getFirstCondition().isIdentical(FirstCondition))
+        return;
+
+    Conditions.push_back(FirstRule.popFirstCondition());
+    for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
+      Matchers[I]->popFirstCondition();
+  }
+}
+
+void GroupMatcher::emit(MatchTable &Table) {
+  unsigned LabelID = ~0U;
+  if (!Conditions.empty()) {
+    LabelID = Table.allocateLabelID();
+    Table << MatchTable::Opcode("GIM_Try", +1)
+          << MatchTable::Comment("On fail goto")
+          << MatchTable::JumpTarget(LabelID) << MatchTable::LineBreak;
+  }
+  for (auto &Condition : Conditions)
+    Condition->emitPredicateOpcodes(
+        Table, *static_cast<RuleMatcher *>(*Matchers.begin()));
+
+  for (const auto &M : Matchers)
+    M->emit(Table);
+
+  // Exit the group
+  if (!Conditions.empty())
+    Table << MatchTable::Opcode("GIM_Reject", -1) << MatchTable::LineBreak
+          << MatchTable::Label(LabelID);
+}
+
+void GroupMatcher::optimize() {
+  // Make sure we only sort by a specific predicate within a range of rules that
+  // all have that predicate checked against a specific value (not a wildcard):
+  auto F = Matchers.begin();
+  auto T = F;
+  auto E = Matchers.end();
+  while (T != E) {
+    while (T != E) {
+      auto *R = static_cast<RuleMatcher *>(*T);
+      if (!R->getFirstConditionAsRootType().get().isValid())
+        break;
+      ++T;
+    }
+    std::stable_sort(F, T, [](Matcher *A, Matcher *B) {
+      auto *L = static_cast<RuleMatcher *>(A);
+      auto *R = static_cast<RuleMatcher *>(B);
+      return L->getFirstConditionAsRootType() <
+             R->getFirstConditionAsRootType();
+    });
+    if (T != E)
+      F = ++T;
+  }
+  optimizeRules<GroupMatcher>(Matchers, MatcherStorage).swap(Matchers);
+  optimizeRules<SwitchMatcher>(Matchers, MatcherStorage).swap(Matchers);
+}
+
+//===- SwitchMatcher ------------------------------------------------------===//
+
+bool SwitchMatcher::isSupportedPredicateType(const PredicateMatcher &P) {
+  return isa<InstructionOpcodeMatcher>(P) || isa<LLTOperandMatcher>(P);
+}
+
+bool SwitchMatcher::candidateConditionMatches(
+    const PredicateMatcher &Predicate) const {
+
+  if (empty()) {
+    // Sharing predicates for nested instructions is not supported yet as we
+    // currently don't hoist the GIM_RecordInsn's properly, therefore we can
+    // only work on the original root instruction (InsnVarID == 0):
+    if (Predicate.getInsnVarID() != 0)
+      return false;
+    // ... while an attempt to add even a root matcher to an empty SwitchMatcher
+    // could fail as not all the types of conditions are supported:
+    if (!isSupportedPredicateType(Predicate))
+      return false;
+    // ... or the condition might not have a proper implementation of
+    // getValue() / isIdenticalDownToValue() yet:
+    if (!Predicate.hasValue())
+      return false;
+    // ... otherwise an empty Switch can accomodate the condition with no
+    // further requirements:
+    return true;
+  }
+
+  const Matcher &CaseRepresentative = **Matchers.begin();
+  const auto &RepresentativeCondition = CaseRepresentative.getFirstCondition();
+  // Switch-cases must share the same kind of condition and path to the value it
+  // checks:
+  if (!Predicate.isIdenticalDownToValue(RepresentativeCondition))
+    return false;
+
+  const auto Value = Predicate.getValue();
+  // ... but be unique with respect to the actual value they check:
+  return Values.count(Value) == 0;
+}
+
+bool SwitchMatcher::addMatcher(Matcher &Candidate) {
+  if (!Candidate.hasFirstCondition())
+    return false;
+
+  const PredicateMatcher &Predicate = Candidate.getFirstCondition();
+  if (!candidateConditionMatches(Predicate))
+    return false;
+  const auto Value = Predicate.getValue();
+  Values.insert(Value);
+
+  Matchers.push_back(&Candidate);
+  return true;
+}
+
+void SwitchMatcher::finalize() {
+  assert(Condition == nullptr && "Already finalized");
+  assert(Values.size() == Matchers.size() && "Broken SwitchMatcher");
+  if (empty())
+    return;
+
+  llvm::stable_sort(Matchers, [](const Matcher *L, const Matcher *R) {
+    return L->getFirstCondition().getValue() <
+           R->getFirstCondition().getValue();
+  });
+  Condition = Matchers[0]->popFirstCondition();
+  for (unsigned I = 1, E = Values.size(); I < E; ++I)
+    Matchers[I]->popFirstCondition();
+}
+
+void SwitchMatcher::emitPredicateSpecificOpcodes(const PredicateMatcher &P,
+                                                 MatchTable &Table) {
+  assert(isSupportedPredicateType(P) && "Predicate type is not supported");
+
+  if (const auto *Condition = dyn_cast<InstructionOpcodeMatcher>(&P)) {
+    Table << MatchTable::Opcode("GIM_SwitchOpcode") << MatchTable::Comment("MI")
+          << MatchTable::IntValue(Condition->getInsnVarID());
+    return;
+  }
+  if (const auto *Condition = dyn_cast<LLTOperandMatcher>(&P)) {
+    Table << MatchTable::Opcode("GIM_SwitchType") << MatchTable::Comment("MI")
+          << MatchTable::IntValue(Condition->getInsnVarID())
+          << MatchTable::Comment("Op")
+          << MatchTable::IntValue(Condition->getOpIdx());
+    return;
+  }
+
+  llvm_unreachable("emitPredicateSpecificOpcodes is broken: can not handle a "
+                   "predicate type that is claimed to be supported");
+}
+
+void SwitchMatcher::emit(MatchTable &Table) {
+  assert(Values.size() == Matchers.size() && "Broken SwitchMatcher");
+  if (empty())
+    return;
+  assert(Condition != nullptr &&
+         "Broken SwitchMatcher, hasn't been finalized?");
+
+  std::vector<unsigned> LabelIDs(Values.size());
+  std::generate(LabelIDs.begin(), LabelIDs.end(),
+                [&Table]() { return Table.allocateLabelID(); });
+  const unsigned Default = Table.allocateLabelID();
+
+  const int64_t LowerBound = Values.begin()->getRawValue();
+  const int64_t UpperBound = Values.rbegin()->getRawValue() + 1;
+
+  emitPredicateSpecificOpcodes(*Condition, Table);
+
+  Table << MatchTable::Comment("[") << MatchTable::IntValue(LowerBound)
+        << MatchTable::IntValue(UpperBound) << MatchTable::Comment(")")
+        << MatchTable::Comment("default:") << MatchTable::JumpTarget(Default);
+
+  int64_t J = LowerBound;
+  auto VI = Values.begin();
+  for (unsigned I = 0, E = Values.size(); I < E; ++I) {
+    auto V = *VI++;
+    while (J++ < V.getRawValue())
+      Table << MatchTable::IntValue(0);
+    V.turnIntoComment();
+    Table << MatchTable::LineBreak << V << MatchTable::JumpTarget(LabelIDs[I]);
+  }
+  Table << MatchTable::LineBreak;
+
+  for (unsigned I = 0, E = Values.size(); I < E; ++I) {
+    Table << MatchTable::Label(LabelIDs[I]);
+    Matchers[I]->emit(Table);
+    Table << MatchTable::Opcode("GIM_Reject") << MatchTable::LineBreak;
+  }
+  Table << MatchTable::Label(Default);
+}
+
+//===- RuleMatcher --------------------------------------------------------===//
+
+uint64_t RuleMatcher::NextRuleID = 0;
+
+StringRef RuleMatcher::getOpcode() const {
+  return Matchers.front()->getOpcode();
+}
+
+unsigned RuleMatcher::getNumOperands() const {
+  return Matchers.front()->getNumOperands();
+}
+
+LLTCodeGen RuleMatcher::getFirstConditionAsRootType() {
+  InstructionMatcher &InsnMatcher = *Matchers.front();
+  if (!InsnMatcher.predicates_empty())
+    if (const auto *TM =
+            dyn_cast<LLTOperandMatcher>(&**InsnMatcher.predicates_begin()))
+      if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0)
+        return TM->getTy();
+  return {};
+}
+
+void RuleMatcher::optimize() {
+  for (auto &Item : InsnVariableIDs) {
+    InstructionMatcher &InsnMatcher = *Item.first;
+    for (auto &OM : InsnMatcher.operands()) {
+      // Complex Patterns are usually expensive and they relatively rarely fail
+      // on their own: more often we end up throwing away all the work done by a
+      // matching part of a complex pattern because some other part of the
+      // enclosing pattern didn't match. All of this makes it beneficial to
+      // delay complex patterns until the very end of the rule matching,
+      // especially for targets having lots of complex patterns.
+      for (auto &OP : OM->predicates())
+        if (isa<ComplexPatternOperandMatcher>(OP))
+          EpilogueMatchers.emplace_back(std::move(OP));
+      OM->eraseNullPredicates();
+    }
+    InsnMatcher.optimize();
+  }
+  llvm::sort(EpilogueMatchers, [](const std::unique_ptr<PredicateMatcher> &L,
+                                  const std::unique_ptr<PredicateMatcher> &R) {
+    return std::make_tuple(L->getKind(), L->getInsnVarID(), L->getOpIdx()) <
+           std::make_tuple(R->getKind(), R->getInsnVarID(), R->getOpIdx());
+  });
+}
+
+bool RuleMatcher::hasFirstCondition() const {
+  if (insnmatchers_empty())
+    return false;
+  InstructionMatcher &Matcher = insnmatchers_front();
+  if (!Matcher.predicates_empty())
+    return true;
+  for (auto &OM : Matcher.operands())
+    for (auto &OP : OM->predicates())
+      if (!isa<InstructionOperandMatcher>(OP))
+        return true;
+  return false;
+}
+
+const PredicateMatcher &RuleMatcher::getFirstCondition() const {
+  assert(!insnmatchers_empty() &&
+         "Trying to get a condition from an empty RuleMatcher");
+
+  InstructionMatcher &Matcher = insnmatchers_front();
+  if (!Matcher.predicates_empty())
+    return **Matcher.predicates_begin();
+  // If there is no more predicate on the instruction itself, look at its
+  // operands.
+  for (auto &OM : Matcher.operands())
+    for (auto &OP : OM->predicates())
+      if (!isa<InstructionOperandMatcher>(OP))
+        return *OP;
+
+  llvm_unreachable("Trying to get a condition from an InstructionMatcher with "
+                   "no conditions");
+}
+
+std::unique_ptr<PredicateMatcher> RuleMatcher::popFirstCondition() {
+  assert(!insnmatchers_empty() &&
+         "Trying to pop a condition from an empty RuleMatcher");
+
+  InstructionMatcher &Matcher = insnmatchers_front();
+  if (!Matcher.predicates_empty())
+    return Matcher.predicates_pop_front();
+  // If there is no more predicate on the instruction itself, look at its
+  // operands.
+  for (auto &OM : Matcher.operands())
+    for (auto &OP : OM->predicates())
+      if (!isa<InstructionOperandMatcher>(OP)) {
+        std::unique_ptr<PredicateMatcher> Result = std::move(OP);
+        OM->eraseNullPredicates();
+        return Result;
+      }
+
+  llvm_unreachable("Trying to pop a condition from an InstructionMatcher with "
+                   "no conditions");
+}
+
+GISelFlags RuleMatcher::updateGISelFlag(GISelFlags CurFlags, const Record *R,
+                                        StringRef FlagName,
+                                        GISelFlags FlagBit) {
+  // If the value of a flag is unset, ignore it.
+  // If it's set, it always takes precedence over the existing value so
+  // clear/set the corresponding bit.
+  bool Unset = false;
+  bool Value = R->getValueAsBitOrUnset("GIIgnoreCopies", Unset);
+  if (!Unset)
+    return Value ? (CurFlags | FlagBit) : (CurFlags & ~FlagBit);
+  return CurFlags;
+}
+
+SaveAndRestore<GISelFlags> RuleMatcher::setGISelFlags(const Record *R) {
+  if (!R || !R->isSubClassOf("GISelFlags"))
+    return {Flags, Flags};
+
+  assert((R->isSubClassOf("PatFrags") || R->isSubClassOf("Pattern")) &&
+         "GISelFlags is only expected on Pattern/PatFrags!");
+
+  GISelFlags NewFlags =
+      updateGISelFlag(Flags, R, "GIIgnoreCopies", GISF_IgnoreCopies);
+  return {Flags, NewFlags};
+}
+
+Error RuleMatcher::defineComplexSubOperand(StringRef SymbolicName,
+                                           Record *ComplexPattern,
+                                           unsigned RendererID,
+                                           unsigned SubOperandID,
+                                           StringRef ParentSymbolicName) {
+  std::string ParentName(ParentSymbolicName);
+  if (ComplexSubOperands.count(SymbolicName)) {
+    const std::string &RecordedParentName =
+        ComplexSubOperandsParentName[SymbolicName];
+    if (RecordedParentName != ParentName)
+      return failUnsupported("Error: Complex suboperand " + SymbolicName +
+                             " referenced by different operands: " +
+                             RecordedParentName + " and " + ParentName + ".");
+    // Complex suboperand referenced more than once from same the operand is
+    // used to generate 'same operand check'. Emitting of
+    // GIR_ComplexSubOperandRenderer for them is already handled.
+    return Error::success();
+  }
+
+  ComplexSubOperands[SymbolicName] =
+      std::make_tuple(ComplexPattern, RendererID, SubOperandID);
+  ComplexSubOperandsParentName[SymbolicName] = ParentName;
+
+  return Error::success();
+}
+
+InstructionMatcher &RuleMatcher::addInstructionMatcher(StringRef SymbolicName) {
+  Matchers.emplace_back(new InstructionMatcher(*this, SymbolicName));
+  MutatableInsns.insert(Matchers.back().get());
+  return *Matchers.back();
+}
+
+void RuleMatcher::addRequiredSimplePredicate(StringRef PredName) {
+  RequiredSimplePredicates.push_back(PredName.str());
+}
+
+const std::vector<std::string> &RuleMatcher::getRequiredSimplePredicates() {
+  return RequiredSimplePredicates;
+}
+
+void RuleMatcher::addRequiredFeature(Record *Feature) {
+  RequiredFeatures.push_back(Feature);
+}
+
+const std::vector<Record *> &RuleMatcher::getRequiredFeatures() const {
+  return RequiredFeatures;
+}
+
+unsigned RuleMatcher::implicitlyDefineInsnVar(InstructionMatcher &Matcher) {
+  unsigned NewInsnVarID = NextInsnVarID++;
+  InsnVariableIDs[&Matcher] = NewInsnVarID;
+  return NewInsnVarID;
+}
+
+unsigned RuleMatcher::getInsnVarID(InstructionMatcher &InsnMatcher) const {
+  const auto &I = InsnVariableIDs.find(&InsnMatcher);
+  if (I != InsnVariableIDs.end())
+    return I->second;
+  llvm_unreachable("Matched Insn was not captured in a local variable");
+}
+
+void RuleMatcher::defineOperand(StringRef SymbolicName, OperandMatcher &OM) {
+  if (!DefinedOperands.contains(SymbolicName)) {
+    DefinedOperands[SymbolicName] = &OM;
+    return;
+  }
+
+  // If the operand is already defined, then we must ensure both references in
+  // the matcher have the exact same node.
+  RuleMatcher &RM = OM.getInstructionMatcher().getRuleMatcher();
+  OM.addPredicate<SameOperandMatcher>(
+      OM.getSymbolicName(), getOperandMatcher(OM.getSymbolicName()).getOpIdx(),
+      RM.getGISelFlags());
+}
+
+void RuleMatcher::definePhysRegOperand(Record *Reg, OperandMatcher &OM) {
+  if (!PhysRegOperands.contains(Reg)) {
+    PhysRegOperands[Reg] = &OM;
+    return;
+  }
+}
+
+InstructionMatcher &
+RuleMatcher::getInstructionMatcher(StringRef SymbolicName) const {
+  for (const auto &I : InsnVariableIDs)
+    if (I.first->getSymbolicName() == SymbolicName)
+      return *I.first;
+  llvm_unreachable(
+      ("Failed to lookup instruction " + SymbolicName).str().c_str());
+}
+
+const OperandMatcher &RuleMatcher::getPhysRegOperandMatcher(Record *Reg) const {
+  const auto &I = PhysRegOperands.find(Reg);
+
+  if (I == PhysRegOperands.end()) {
+    PrintFatalError(SrcLoc, "Register " + Reg->getName() +
+                                " was not declared in matcher");
+  }
+
+  return *I->second;
+}
+
+const OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) const {
+  const auto &I = DefinedOperands.find(Name);
+
+  if (I == DefinedOperands.end())
+    PrintFatalError(SrcLoc, "Operand " + Name + " was not declared in matcher");
+
+  return *I->second;
+}
+
+void RuleMatcher::emit(MatchTable &Table) {
+  if (Matchers.empty())
+    llvm_unreachable("Unexpected empty matcher!");
+
+  // The representation supports rules that require multiple roots such as:
+  //    %ptr(p0) = ...
+  //    %elt0(s32) = G_LOAD %ptr
+  //    %1(p0) = G_ADD %ptr, 4
+  //    %elt1(s32) = G_LOAD p0 %1
+  // which could be usefully folded into:
+  //    %ptr(p0) = ...
+  //    %elt0(s32), %elt1(s32) = TGT_LOAD_PAIR %ptr
+  // on some targets but we don't need to make use of that yet.
+  assert(Matchers.size() == 1 && "Cannot handle multi-root matchers yet");
+
+  unsigned LabelID = Table.allocateLabelID();
+  Table << MatchTable::Opcode("GIM_Try", +1)
+        << MatchTable::Comment("On fail goto")
+        << MatchTable::JumpTarget(LabelID)
+        << MatchTable::Comment(("Rule ID " + Twine(RuleID) + " //").str())
+        << MatchTable::LineBreak;
+
+  if (!RequiredFeatures.empty()) {
+    Table << MatchTable::Opcode("GIM_CheckFeatures")
+          << MatchTable::NamedValue(getNameForFeatureBitset(RequiredFeatures))
+          << MatchTable::LineBreak;
+  }
+
+  if (!RequiredSimplePredicates.empty()) {
+    for (const auto &Pred : RequiredSimplePredicates) {
+      Table << MatchTable::Opcode("GIM_CheckSimplePredicate")
+            << MatchTable::NamedValue(Pred) << MatchTable::LineBreak;
+    }
+  }
+
+  Matchers.front()->emitPredicateOpcodes(Table, *this);
+
+  // We must also check if it's safe to fold the matched instructions.
+  if (InsnVariableIDs.size() >= 2) {
+    // Invert the map to create stable ordering (by var names)
+    SmallVector<unsigned, 2> InsnIDs;
+    for (const auto &Pair : InsnVariableIDs) {
+      // Skip the root node since it isn't moving anywhere. Everything else is
+      // sinking to meet it.
+      if (Pair.first == Matchers.front().get())
+        continue;
+
+      InsnIDs.push_back(Pair.second);
+    }
+    llvm::sort(InsnIDs);
+
+    for (const auto &InsnID : InsnIDs) {
+      // Reject the difficult cases until we have a more accurate check.
+      Table << MatchTable::Opcode("GIM_CheckIsSafeToFold")
+            << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+            << MatchTable::LineBreak;
+
+      // FIXME: Emit checks to determine it's _actually_ safe to fold and/or
+      //        account for unsafe cases.
+      //
+      //        Example:
+      //          MI1--> %0 = ...
+      //                 %1 = ... %0
+      //          MI0--> %2 = ... %0
+      //          It's not safe to erase MI1. We currently handle this by not
+      //          erasing %0 (even when it's dead).
+      //
+      //        Example:
+      //          MI1--> %0 = load volatile @a
+      //                 %1 = load volatile @a
+      //          MI0--> %2 = ... %0
+      //          It's not safe to sink %0's def past %1. We currently handle
+      //          this by rejecting all loads.
+      //
+      //        Example:
+      //          MI1--> %0 = load @a
+      //                 %1 = store @a
+      //          MI0--> %2 = ... %0
+      //          It's not safe to sink %0's def past %1. We currently handle
+      //          this by rejecting all loads.
+      //
+      //        Example:
+      //                   G_CONDBR %cond, @BB1
+      //                 BB0:
+      //          MI1-->   %0 = load @a
+      //                   G_BR @BB1
+      //                 BB1:
+      //          MI0-->   %2 = ... %0
+      //          It's not always safe to sink %0 across control flow. In this
+      //          case it may introduce a memory fault. We currentl handle
+      //          this by rejecting all loads.
+    }
+  }
+
+  for (const auto &PM : EpilogueMatchers)
+    PM->emitPredicateOpcodes(Table, *this);
+
+  for (const auto &MA : Actions)
+    MA->emitActionOpcodes(Table, *this);
+
+  assert((Table.isWithCoverage() ? !Table.isCombiner() : true) &&
+         "Combiner tables don't support coverage!");
+  if (Table.isWithCoverage())
+    Table << MatchTable::Opcode("GIR_Coverage") << MatchTable::IntValue(RuleID)
+          << MatchTable::LineBreak;
+  else if (!Table.isCombiner())
+    Table << MatchTable::Comment(("GIR_Coverage, " + Twine(RuleID) + ",").str())
+          << MatchTable::LineBreak;
+
+  Table << MatchTable::Opcode("GIR_Done", -1) << MatchTable::LineBreak
+        << MatchTable::Label(LabelID);
+  ++NumPatternEmitted;
+}
+
+bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const {
+  // Rules involving more match roots have higher priority.
+  if (Matchers.size() > B.Matchers.size())
+    return true;
+  if (Matchers.size() < B.Matchers.size())
+    return false;
+
+  for (auto Matcher : zip(Matchers, B.Matchers)) {
+    if (std::get<0>(Matcher)->isHigherPriorityThan(*std::get<1>(Matcher)))
+      return true;
+    if (std::get<1>(Matcher)->isHigherPriorityThan(*std::get<0>(Matcher)))
+      return false;
+  }
+
+  return false;
+}
+
+unsigned RuleMatcher::countRendererFns() const {
+  return std::accumulate(
+      Matchers.begin(), Matchers.end(), 0,
+      [](unsigned A, const std::unique_ptr<InstructionMatcher> &Matcher) {
+        return A + Matcher->countRendererFns();
+      });
+}
+
+//===- PredicateMatcher ---------------------------------------------------===//
+
+PredicateMatcher::~PredicateMatcher() {}
+
+//===- OperandPredicateMatcher --------------------------------------------===//
+
+OperandPredicateMatcher::~OperandPredicateMatcher() {}
+
+bool OperandPredicateMatcher::isHigherPriorityThan(
+    const OperandPredicateMatcher &B) const {
+  // Generally speaking, an instruction is more important than an Int or a
+  // LiteralInt because it can cover more nodes but theres an exception to
+  // this. G_CONSTANT's are less important than either of those two because they
+  // are more permissive.
+
+  const InstructionOperandMatcher *AOM =
+      dyn_cast<InstructionOperandMatcher>(this);
+  const InstructionOperandMatcher *BOM =
+      dyn_cast<InstructionOperandMatcher>(&B);
+  bool AIsConstantInsn = AOM && AOM->getInsnMatcher().isConstantInstruction();
+  bool BIsConstantInsn = BOM && BOM->getInsnMatcher().isConstantInstruction();
+
+  if (AOM && BOM) {
+    // The relative priorities between a G_CONSTANT and any other instruction
+    // don't actually matter but this code is needed to ensure a strict weak
+    // ordering. This is particularly important on Windows where the rules will
+    // be incorrectly sorted without it.
+    if (AIsConstantInsn != BIsConstantInsn)
+      return AIsConstantInsn < BIsConstantInsn;
+    return false;
+  }
+
+  if (AOM && AIsConstantInsn && (B.Kind == OPM_Int || B.Kind == OPM_LiteralInt))
+    return false;
+  if (BOM && BIsConstantInsn && (Kind == OPM_Int || Kind == OPM_LiteralInt))
+    return true;
+
+  return Kind < B.Kind;
+}
+
+//===- SameOperandMatcher -------------------------------------------------===//
+
+void SameOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                              RuleMatcher &Rule) const {
+  const OperandMatcher &OtherOM = Rule.getOperandMatcher(MatchingName);
+  unsigned OtherInsnVarID = Rule.getInsnVarID(OtherOM.getInstructionMatcher());
+  assert(OtherInsnVarID == OtherOM.getInstructionMatcher().getInsnVarID());
+  const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
+  Table << MatchTable::Opcode(IgnoreCopies
+                                  ? "GIM_CheckIsSameOperandIgnoreCopies"
+                                  : "GIM_CheckIsSameOperand")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("OtherMI")
+        << MatchTable::IntValue(OtherInsnVarID)
+        << MatchTable::Comment("OtherOpIdx")
+        << MatchTable::IntValue(OtherOM.getOpIdx()) << MatchTable::LineBreak;
+}
+
+//===- LLTOperandMatcher --------------------------------------------------===//
+
+std::map<LLTCodeGen, unsigned> LLTOperandMatcher::TypeIDValues;
+
+MatchTableRecord LLTOperandMatcher::getValue() const {
+  const auto VI = TypeIDValues.find(Ty);
+  if (VI == TypeIDValues.end())
+    return MatchTable::NamedValue(getTy().getCxxEnumValue());
+  return MatchTable::NamedValue(getTy().getCxxEnumValue(), VI->second);
+}
+
+bool LLTOperandMatcher::hasValue() const {
+  if (TypeIDValues.size() != KnownTypes.size())
+    initTypeIDValuesMap();
+  return TypeIDValues.count(Ty);
+}
+
+void LLTOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                             RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckType") << MatchTable::Comment("MI")
+        << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
+        << MatchTable::IntValue(OpIdx) << MatchTable::Comment("Type")
+        << getValue() << MatchTable::LineBreak;
+}
+
+//===- PointerToAnyOperandMatcher -----------------------------------------===//
+
+void PointerToAnyOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                                      RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckPointerToAny")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("SizeInBits") << MatchTable::IntValue(SizeInBits)
+        << MatchTable::LineBreak;
+}
+
+//===- RecordNamedOperandMatcher ------------------------------------------===//
+
+void RecordNamedOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                                     RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_RecordNamedOperand")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("StoreIdx") << MatchTable::IntValue(StoreIdx)
+        << MatchTable::Comment("Name : " + Name) << MatchTable::LineBreak;
+}
+
+//===- ComplexPatternOperandMatcher ---------------------------------------===//
+
+void ComplexPatternOperandMatcher::emitPredicateOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  unsigned ID = getAllocatedTemporariesBaseID();
+  Table << MatchTable::Opcode("GIM_CheckComplexPattern")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("Renderer") << MatchTable::IntValue(ID)
+        << MatchTable::NamedValue(("GICP_" + TheDef.getName()).str())
+        << MatchTable::LineBreak;
+}
+
+unsigned ComplexPatternOperandMatcher::getAllocatedTemporariesBaseID() const {
+  return Operand.getAllocatedTemporariesBaseID();
+}
+
+//===- RegisterBankOperandMatcher -----------------------------------------===//
+
+bool RegisterBankOperandMatcher::isIdentical(const PredicateMatcher &B) const {
+  return OperandPredicateMatcher::isIdentical(B) &&
+         RC.getDef() == cast<RegisterBankOperandMatcher>(&B)->RC.getDef();
+}
+
+void RegisterBankOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                                      RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckRegBankForClass")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("RC")
+        << MatchTable::NamedValue(RC.getQualifiedName() + "RegClassID")
+        << MatchTable::LineBreak;
+}
+
+//===- MBBOperandMatcher --------------------------------------------------===//
+
+void MBBOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                             RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckIsMBB") << MatchTable::Comment("MI")
+        << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
+        << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
+}
+
+//===- ImmOperandMatcher --------------------------------------------------===//
+
+void ImmOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                             RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckIsImm") << MatchTable::Comment("MI")
+        << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
+        << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
+}
+
+//===- ConstantIntOperandMatcher ------------------------------------------===//
+
+void ConstantIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                                     RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckConstantInt")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::IntValue(Value) << MatchTable::LineBreak;
+}
+
+//===- LiteralIntOperandMatcher -------------------------------------------===//
+
+void LiteralIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                                    RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckLiteralInt")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::IntValue(Value) << MatchTable::LineBreak;
+}
+
+//===- CmpPredicateOperandMatcher -----------------------------------------===//
+
+void CmpPredicateOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                                      RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckCmpPredicate")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("Predicate")
+        << MatchTable::NamedValue("CmpInst", PredName) << MatchTable::LineBreak;
+}
+
+//===- IntrinsicIDOperandMatcher ------------------------------------------===//
+
+void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                                     RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckIntrinsicID")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::NamedValue("Intrinsic::" + II->EnumName)
+        << MatchTable::LineBreak;
+}
+
+//===- OperandImmPredicateMatcher -----------------------------------------===//
+
+void OperandImmPredicateMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                                      RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckImmOperandPredicate")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("MO") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("Predicate")
+        << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
+        << MatchTable::LineBreak;
+}
+
+//===- OperandMatcher -----------------------------------------------------===//
+
+std::string OperandMatcher::getOperandExpr(unsigned InsnVarID) const {
+  return "State.MIs[" + llvm::to_string(InsnVarID) + "]->getOperand(" +
+         llvm::to_string(OpIdx) + ")";
+}
+
+unsigned OperandMatcher::getInsnVarID() const { return Insn.getInsnVarID(); }
+
+void OperandMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                          RuleMatcher &Rule) {
+  if (!Optimized) {
+    std::string Comment;
+    raw_string_ostream CommentOS(Comment);
+    CommentOS << "MIs[" << getInsnVarID() << "] ";
+    if (SymbolicName.empty())
+      CommentOS << "Operand " << OpIdx;
+    else
+      CommentOS << SymbolicName;
+    Table << MatchTable::Comment(Comment) << MatchTable::LineBreak;
+  }
+
+  emitPredicateListOpcodes(Table, Rule);
+}
+
+bool OperandMatcher::isHigherPriorityThan(OperandMatcher &B) {
+  // Operand matchers involving more predicates have higher priority.
+  if (predicates_size() > B.predicates_size())
+    return true;
+  if (predicates_size() < B.predicates_size())
+    return false;
+
+  // This assumes that predicates are added in a consistent order.
+  for (auto &&Predicate : zip(predicates(), B.predicates())) {
+    if (std::get<0>(Predicate)->isHigherPriorityThan(*std::get<1>(Predicate)))
+      return true;
+    if (std::get<1>(Predicate)->isHigherPriorityThan(*std::get<0>(Predicate)))
+      return false;
+  }
+
+  return false;
+}
+
+unsigned OperandMatcher::countRendererFns() {
+  return std::accumulate(
+      predicates().begin(), predicates().end(), 0,
+      [](unsigned A,
+         const std::unique_ptr<OperandPredicateMatcher> &Predicate) {
+        return A + Predicate->countRendererFns();
+      });
+}
+
+Error OperandMatcher::addTypeCheckPredicate(const TypeSetByHwMode &VTy,
+                                            bool OperandIsAPointer) {
+  if (!VTy.isMachineValueType())
+    return failUnsupported("unsupported typeset");
+
+  if (VTy.getMachineValueType() == MVT::iPTR && OperandIsAPointer) {
+    addPredicate<PointerToAnyOperandMatcher>(0);
+    return Error::success();
+  }
+
+  auto OpTyOrNone = MVTToLLT(VTy.getMachineValueType().SimpleTy);
+  if (!OpTyOrNone)
+    return failUnsupported("unsupported type");
+
+  if (OperandIsAPointer)
+    addPredicate<PointerToAnyOperandMatcher>(OpTyOrNone->get().getSizeInBits());
+  else if (VTy.isPointer())
+    addPredicate<LLTOperandMatcher>(
+        LLT::pointer(VTy.getPtrAddrSpace(), OpTyOrNone->get().getSizeInBits()));
+  else
+    addPredicate<LLTOperandMatcher>(*OpTyOrNone);
+  return Error::success();
+}
+
+//===- InstructionOpcodeMatcher -------------------------------------------===//
+
+DenseMap<const CodeGenInstruction *, unsigned>
+    InstructionOpcodeMatcher::OpcodeValues;
+
+MatchTableRecord
+InstructionOpcodeMatcher::getInstValue(const CodeGenInstruction *I) const {
+  const auto VI = OpcodeValues.find(I);
+  if (VI != OpcodeValues.end())
+    return MatchTable::NamedValue(I->Namespace, I->TheDef->getName(),
+                                  VI->second);
+  return MatchTable::NamedValue(I->Namespace, I->TheDef->getName());
+}
+
+void InstructionOpcodeMatcher::initOpcodeValuesMap(
+    const CodeGenTarget &Target) {
+  OpcodeValues.clear();
+
+  unsigned OpcodeValue = 0;
+  for (const CodeGenInstruction *I : Target.getInstructionsByEnumValue())
+    OpcodeValues[I] = OpcodeValue++;
+}
+
+MatchTableRecord InstructionOpcodeMatcher::getValue() const {
+  assert(Insts.size() == 1);
+
+  const CodeGenInstruction *I = Insts[0];
+  const auto VI = OpcodeValues.find(I);
+  if (VI != OpcodeValues.end())
+    return MatchTable::NamedValue(I->Namespace, I->TheDef->getName(),
+                                  VI->second);
+  return MatchTable::NamedValue(I->Namespace, I->TheDef->getName());
+}
+
+void InstructionOpcodeMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                                    RuleMatcher &Rule) const {
+  StringRef CheckType =
+      Insts.size() == 1 ? "GIM_CheckOpcode" : "GIM_CheckOpcodeIsEither";
+  Table << MatchTable::Opcode(CheckType) << MatchTable::Comment("MI")
+        << MatchTable::IntValue(InsnVarID);
+
+  for (const CodeGenInstruction *I : Insts)
+    Table << getInstValue(I);
+  Table << MatchTable::LineBreak;
+}
+
+bool InstructionOpcodeMatcher::isHigherPriorityThan(
+    const InstructionPredicateMatcher &B) const {
+  if (InstructionPredicateMatcher::isHigherPriorityThan(B))
+    return true;
+  if (B.InstructionPredicateMatcher::isHigherPriorityThan(*this))
+    return false;
+
+  // Prioritize opcodes for cosmetic reasons in the generated source. Although
+  // this is cosmetic at the moment, we may want to drive a similar ordering
+  // using instruction frequency information to improve compile time.
+  if (const InstructionOpcodeMatcher *BO =
+          dyn_cast<InstructionOpcodeMatcher>(&B))
+    return Insts[0]->TheDef->getName() < BO->Insts[0]->TheDef->getName();
+
+  return false;
+}
+
+bool InstructionOpcodeMatcher::isConstantInstruction() const {
+  return Insts.size() == 1 && Insts[0]->TheDef->getName() == "G_CONSTANT";
+}
+
+StringRef InstructionOpcodeMatcher::getOpcode() const {
+  return Insts[0]->TheDef->getName();
+}
+
+bool InstructionOpcodeMatcher::isVariadicNumOperands() const {
+  // If one is variadic, they all should be.
+  return Insts[0]->Operands.isVariadic;
+}
+
+StringRef InstructionOpcodeMatcher::getOperandType(unsigned OpIdx) const {
+  // Types expected to be uniform for all alternatives.
+  return Insts[0]->Operands[OpIdx].OperandType;
+}
+
+//===- InstructionNumOperandsMatcher --------------------------------------===//
+
+void InstructionNumOperandsMatcher::emitPredicateOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckNumOperands")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("Expected") << MatchTable::IntValue(NumOperands)
+        << MatchTable::LineBreak;
+}
+
+//===- InstructionImmPredicateMatcher -------------------------------------===//
+
+bool InstructionImmPredicateMatcher::isIdentical(
+    const PredicateMatcher &B) const {
+  return InstructionPredicateMatcher::isIdentical(B) &&
+         Predicate.getOrigPatFragRecord() ==
+             cast<InstructionImmPredicateMatcher>(&B)
+                 ->Predicate.getOrigPatFragRecord();
+}
+
+void InstructionImmPredicateMatcher::emitPredicateOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode(getMatchOpcodeForImmPredicate(Predicate))
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("Predicate")
+        << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
+        << MatchTable::LineBreak;
+}
+
+//===- AtomicOrderingMMOPredicateMatcher ----------------------------------===//
+
+bool AtomicOrderingMMOPredicateMatcher::isIdentical(
+    const PredicateMatcher &B) const {
+  if (!InstructionPredicateMatcher::isIdentical(B))
+    return false;
+  const auto &R = *cast<AtomicOrderingMMOPredicateMatcher>(&B);
+  return Order == R.Order && Comparator == R.Comparator;
+}
+
+void AtomicOrderingMMOPredicateMatcher::emitPredicateOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  StringRef Opcode = "GIM_CheckAtomicOrdering";
+
+  if (Comparator == AO_OrStronger)
+    Opcode = "GIM_CheckAtomicOrderingOrStrongerThan";
+  if (Comparator == AO_WeakerThan)
+    Opcode = "GIM_CheckAtomicOrderingWeakerThan";
+
+  Table << MatchTable::Opcode(Opcode) << MatchTable::Comment("MI")
+        << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Order")
+        << MatchTable::NamedValue(("(int64_t)AtomicOrdering::" + Order).str())
+        << MatchTable::LineBreak;
+}
+
+//===- MemorySizePredicateMatcher -----------------------------------------===//
+
+void MemorySizePredicateMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                                      RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckMemorySizeEqualTo")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
+        << MatchTable::Comment("Size") << MatchTable::IntValue(Size)
+        << MatchTable::LineBreak;
+}
+
+//===- MemoryAddressSpacePredicateMatcher ---------------------------------===//
+
+bool MemoryAddressSpacePredicateMatcher::isIdentical(
+    const PredicateMatcher &B) const {
+  if (!InstructionPredicateMatcher::isIdentical(B))
+    return false;
+  auto *Other = cast<MemoryAddressSpacePredicateMatcher>(&B);
+  return MMOIdx == Other->MMOIdx && AddrSpaces == Other->AddrSpaces;
+}
+
+void MemoryAddressSpacePredicateMatcher::emitPredicateOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckMemoryAddressSpace")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("MMO")
+        << MatchTable::IntValue(MMOIdx)
+        // Encode number of address spaces to expect.
+        << MatchTable::Comment("NumAddrSpace")
+        << MatchTable::IntValue(AddrSpaces.size());
+  for (unsigned AS : AddrSpaces)
+    Table << MatchTable::Comment("AddrSpace") << MatchTable::IntValue(AS);
+
+  Table << MatchTable::LineBreak;
+}
+
+//===- MemoryAlignmentPredicateMatcher ------------------------------------===//
+
+bool MemoryAlignmentPredicateMatcher::isIdentical(
+    const PredicateMatcher &B) const {
+  if (!InstructionPredicateMatcher::isIdentical(B))
+    return false;
+  auto *Other = cast<MemoryAlignmentPredicateMatcher>(&B);
+  return MMOIdx == Other->MMOIdx && MinAlign == Other->MinAlign;
+}
+
+void MemoryAlignmentPredicateMatcher::emitPredicateOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckMemoryAlignment")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
+        << MatchTable::Comment("MinAlign") << MatchTable::IntValue(MinAlign)
+        << MatchTable::LineBreak;
+}
+
+//===- MemoryVsLLTSizePredicateMatcher ------------------------------------===//
+
+bool MemoryVsLLTSizePredicateMatcher::isIdentical(
+    const PredicateMatcher &B) const {
+  return InstructionPredicateMatcher::isIdentical(B) &&
+         MMOIdx == cast<MemoryVsLLTSizePredicateMatcher>(&B)->MMOIdx &&
+         Relation == cast<MemoryVsLLTSizePredicateMatcher>(&B)->Relation &&
+         OpIdx == cast<MemoryVsLLTSizePredicateMatcher>(&B)->OpIdx;
+}
+
+void MemoryVsLLTSizePredicateMatcher::emitPredicateOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode(
+               Relation == EqualTo       ? "GIM_CheckMemorySizeEqualToLLT"
+               : Relation == GreaterThan ? "GIM_CheckMemorySizeGreaterThanLLT"
+                                         : "GIM_CheckMemorySizeLessThanLLT")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
+        << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx)
+        << MatchTable::LineBreak;
+}
+
+//===- VectorSplatImmPredicateMatcher -------------------------------------===//
+
+void VectorSplatImmPredicateMatcher::emitPredicateOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  if (Kind == AllOnes)
+    Table << MatchTable::Opcode("GIM_CheckIsBuildVectorAllOnes");
+  else
+    Table << MatchTable::Opcode("GIM_CheckIsBuildVectorAllZeros");
+
+  Table << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID);
+  Table << MatchTable::LineBreak;
+}
+
+//===- GenericInstructionPredicateMatcher ---------------------------------===//
+
+GenericInstructionPredicateMatcher::GenericInstructionPredicateMatcher(
+    unsigned InsnVarID, TreePredicateFn Predicate)
+    : GenericInstructionPredicateMatcher(InsnVarID,
+                                         getEnumNameForPredicate(Predicate)) {}
+
+bool GenericInstructionPredicateMatcher::isIdentical(
+    const PredicateMatcher &B) const {
+  return InstructionPredicateMatcher::isIdentical(B) &&
+         EnumVal ==
+             static_cast<const GenericInstructionPredicateMatcher &>(B).EnumVal;
+}
+void GenericInstructionPredicateMatcher::emitPredicateOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIM_CheckCxxInsnPredicate")
+        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("FnId") << MatchTable::NamedValue(EnumVal)
+        << MatchTable::LineBreak;
+}
+
+//===- InstructionMatcher -------------------------------------------------===//
+
+OperandMatcher &
+InstructionMatcher::addOperand(unsigned OpIdx, const std::string &SymbolicName,
+                               unsigned AllocatedTemporariesBaseID) {
+  Operands.emplace_back(new OperandMatcher(*this, OpIdx, SymbolicName,
+                                           AllocatedTemporariesBaseID));
+  if (!SymbolicName.empty())
+    Rule.defineOperand(SymbolicName, *Operands.back());
+
+  return *Operands.back();
+}
+
+OperandMatcher &InstructionMatcher::getOperand(unsigned OpIdx) {
+  auto I = llvm::find_if(Operands,
+                         [&OpIdx](const std::unique_ptr<OperandMatcher> &X) {
+                           return X->getOpIdx() == OpIdx;
+                         });
+  if (I != Operands.end())
+    return **I;
+  llvm_unreachable("Failed to lookup operand");
+}
+
+OperandMatcher &InstructionMatcher::addPhysRegInput(Record *Reg, unsigned OpIdx,
+                                                    unsigned TempOpIdx) {
+  assert(SymbolicName.empty());
+  OperandMatcher *OM = new OperandMatcher(*this, OpIdx, "", TempOpIdx);
+  Operands.emplace_back(OM);
+  Rule.definePhysRegOperand(Reg, *OM);
+  PhysRegInputs.emplace_back(Reg, OpIdx);
+  return *OM;
+}
+
+void InstructionMatcher::emitPredicateOpcodes(MatchTable &Table,
+                                              RuleMatcher &Rule) {
+  if (NumOperandsCheck)
+    InstructionNumOperandsMatcher(InsnVarID, getNumOperands())
+        .emitPredicateOpcodes(Table, Rule);
+
+  // First emit all instruction level predicates need to be verified before we
+  // can verify operands.
+  emitFilteredPredicateListOpcodes(
+      [](const PredicateMatcher &P) { return !P.dependsOnOperands(); }, Table,
+      Rule);
+
+  // Emit all operand constraints.
+  for (const auto &Operand : Operands)
+    Operand->emitPredicateOpcodes(Table, Rule);
+
+  // All of the tablegen defined predicates should now be matched. Now emit
+  // any custom predicates that rely on all generated checks.
+  emitFilteredPredicateListOpcodes(
+      [](const PredicateMatcher &P) { return P.dependsOnOperands(); }, Table,
+      Rule);
+}
+
+bool InstructionMatcher::isHigherPriorityThan(InstructionMatcher &B) {
+  // Instruction matchers involving more operands have higher priority.
+  if (Operands.size() > B.Operands.size())
+    return true;
+  if (Operands.size() < B.Operands.size())
+    return false;
+
+  for (auto &&P : zip(predicates(), B.predicates())) {
+    auto L = static_cast<InstructionPredicateMatcher *>(std::get<0>(P).get());
+    auto R = static_cast<InstructionPredicateMatcher *>(std::get<1>(P).get());
+    if (L->isHigherPriorityThan(*R))
+      return true;
+    if (R->isHigherPriorityThan(*L))
+      return false;
+  }
+
+  for (auto Operand : zip(Operands, B.Operands)) {
+    if (std::get<0>(Operand)->isHigherPriorityThan(*std::get<1>(Operand)))
+      return true;
+    if (std::get<1>(Operand)->isHigherPriorityThan(*std::get<0>(Operand)))
+      return false;
+  }
+
+  return false;
+}
+
+unsigned InstructionMatcher::countRendererFns() {
+  return std::accumulate(
+             predicates().begin(), predicates().end(), 0,
+             [](unsigned A,
+                const std::unique_ptr<PredicateMatcher> &Predicate) {
+               return A + Predicate->countRendererFns();
+             }) +
+         std::accumulate(
+             Operands.begin(), Operands.end(), 0,
+             [](unsigned A, const std::unique_ptr<OperandMatcher> &Operand) {
+               return A + Operand->countRendererFns();
+             });
+}
+
+void InstructionMatcher::optimize() {
+  SmallVector<std::unique_ptr<PredicateMatcher>, 8> Stash;
+  const auto &OpcMatcher = getOpcodeMatcher();
+
+  Stash.push_back(predicates_pop_front());
+  if (Stash.back().get() == &OpcMatcher) {
+    if (NumOperandsCheck && OpcMatcher.isVariadicNumOperands() &&
+        getNumOperands() != 0)
+      Stash.emplace_back(
+          new InstructionNumOperandsMatcher(InsnVarID, getNumOperands()));
+    NumOperandsCheck = false;
+
+    for (auto &OM : Operands)
+      for (auto &OP : OM->predicates())
+        if (isa<IntrinsicIDOperandMatcher>(OP)) {
+          Stash.push_back(std::move(OP));
+          OM->eraseNullPredicates();
+          break;
+        }
+  }
+
+  if (InsnVarID > 0) {
+    assert(!Operands.empty() && "Nested instruction is expected to def a vreg");
+    for (auto &OP : Operands[0]->predicates())
+      OP.reset();
+    Operands[0]->eraseNullPredicates();
+  }
+  for (auto &OM : Operands) {
+    for (auto &OP : OM->predicates())
+      if (isa<LLTOperandMatcher>(OP))
+        Stash.push_back(std::move(OP));
+    OM->eraseNullPredicates();
+  }
+  while (!Stash.empty())
+    prependPredicate(Stash.pop_back_val());
+}
+
+//===- InstructionOperandMatcher ------------------------------------------===//
+
+void InstructionOperandMatcher::emitCaptureOpcodes(MatchTable &Table,
+                                                   RuleMatcher &Rule) const {
+  const unsigned NewInsnVarID = InsnMatcher->getInsnVarID();
+  const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
+  Table << MatchTable::Opcode(IgnoreCopies ? "GIM_RecordInsnIgnoreCopies"
+                                           : "GIM_RecordInsn")
+        << MatchTable::Comment("DefineMI") << MatchTable::IntValue(NewInsnVarID)
+        << MatchTable::Comment("MI") << MatchTable::IntValue(getInsnVarID())
+        << MatchTable::Comment("OpIdx") << MatchTable::IntValue(getOpIdx())
+        << MatchTable::Comment("MIs[" + llvm::to_string(NewInsnVarID) + "]")
+        << MatchTable::LineBreak;
+}
+
+bool InstructionOperandMatcher::isHigherPriorityThan(
+    const OperandPredicateMatcher &B) const {
+  if (OperandPredicateMatcher::isHigherPriorityThan(B))
+    return true;
+  if (B.OperandPredicateMatcher::isHigherPriorityThan(*this))
+    return false;
+
+  if (const InstructionOperandMatcher *BP =
+          dyn_cast<InstructionOperandMatcher>(&B))
+    if (InsnMatcher->isHigherPriorityThan(*BP->InsnMatcher))
+      return true;
+  return false;
+}
+
+//===- OperandRenderer ----------------------------------------------------===//
+
+OperandRenderer::~OperandRenderer() {}
+
+//===- CopyRenderer -------------------------------------------------------===//
+
+void CopyRenderer::emitRenderOpcodes(MatchTable &Table,
+                                     RuleMatcher &Rule) const {
+  const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
+  unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
+  Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID")
+        << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID")
+        << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
+        << MatchTable::IntValue(Operand.getOpIdx())
+        << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
+}
+
+//===- CopyPhysRegRenderer ------------------------------------------------===//
+
+void CopyPhysRegRenderer::emitRenderOpcodes(MatchTable &Table,
+                                            RuleMatcher &Rule) const {
+  const OperandMatcher &Operand = Rule.getPhysRegOperandMatcher(PhysReg);
+  unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
+  Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID")
+        << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID")
+        << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
+        << MatchTable::IntValue(Operand.getOpIdx())
+        << MatchTable::Comment(PhysReg->getName()) << MatchTable::LineBreak;
+}
+
+//===- CopyOrAddZeroRegRenderer -------------------------------------------===//
+
+void CopyOrAddZeroRegRenderer::emitRenderOpcodes(MatchTable &Table,
+                                                 RuleMatcher &Rule) const {
+  const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
+  unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
+  Table << MatchTable::Opcode("GIR_CopyOrAddZeroReg")
+        << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
+        << MatchTable::Comment("OldInsnID")
+        << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
+        << MatchTable::IntValue(Operand.getOpIdx())
+        << MatchTable::NamedValue(
+               (ZeroRegisterDef->getValue("Namespace")
+                    ? ZeroRegisterDef->getValueAsString("Namespace")
+                    : ""),
+               ZeroRegisterDef->getName())
+        << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
+}
+
+//===- CopyConstantAsImmRenderer ------------------------------------------===//
+
+void CopyConstantAsImmRenderer::emitRenderOpcodes(MatchTable &Table,
+                                                  RuleMatcher &Rule) const {
+  InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
+  unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
+  Table << MatchTable::Opcode(Signed ? "GIR_CopyConstantAsSImm"
+                                     : "GIR_CopyConstantAsUImm")
+        << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
+        << MatchTable::Comment("OldInsnID")
+        << MatchTable::IntValue(OldInsnVarID)
+        << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
+}
+
+//===- CopyFConstantAsFPImmRenderer ---------------------------------------===//
+
+void CopyFConstantAsFPImmRenderer::emitRenderOpcodes(MatchTable &Table,
+                                                     RuleMatcher &Rule) const {
+  InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
+  unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
+  Table << MatchTable::Opcode("GIR_CopyFConstantAsFPImm")
+        << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
+        << MatchTable::Comment("OldInsnID")
+        << MatchTable::IntValue(OldInsnVarID)
+        << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
+}
+
+//===- CopySubRegRenderer -------------------------------------------------===//
+
+void CopySubRegRenderer::emitRenderOpcodes(MatchTable &Table,
+                                           RuleMatcher &Rule) const {
+  const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
+  unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
+  Table << MatchTable::Opcode("GIR_CopySubReg")
+        << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
+        << MatchTable::Comment("OldInsnID")
+        << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
+        << MatchTable::IntValue(Operand.getOpIdx())
+        << MatchTable::Comment("SubRegIdx")
+        << MatchTable::IntValue(SubReg->EnumValue)
+        << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
+}
+
+//===- AddRegisterRenderer ------------------------------------------------===//
+
+void AddRegisterRenderer::emitRenderOpcodes(MatchTable &Table,
+                                            RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIR_AddRegister")
+        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID);
+  if (RegisterDef->getName() != "zero_reg") {
+    Table << MatchTable::NamedValue(
+        (RegisterDef->getValue("Namespace")
+             ? RegisterDef->getValueAsString("Namespace")
+             : ""),
+        RegisterDef->getName());
+  } else {
+    Table << MatchTable::NamedValue(Target.getRegNamespace(), "NoRegister");
+  }
+  Table << MatchTable::Comment("AddRegisterRegFlags");
+
+  // TODO: This is encoded as a 64-bit element, but only 16 or 32-bits are
+  // really needed for a physical register reference. We can pack the
+  // register and flags in a single field.
+  if (IsDef)
+    Table << MatchTable::NamedValue("RegState::Define");
+  else
+    Table << MatchTable::IntValue(0);
+  Table << MatchTable::LineBreak;
+}
+
+//===- TempRegRenderer ----------------------------------------------------===//
+
+void TempRegRenderer::emitRenderOpcodes(MatchTable &Table,
+                                        RuleMatcher &Rule) const {
+  if (SubRegIdx) {
+    assert(!IsDef);
+    Table << MatchTable::Opcode("GIR_AddTempSubRegister");
+  } else
+    Table << MatchTable::Opcode("GIR_AddTempRegister");
+
+  Table << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+        << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
+        << MatchTable::Comment("TempRegFlags");
+
+  if (IsDef) {
+    SmallString<32> RegFlags;
+    RegFlags += "RegState::Define";
+    if (IsDead)
+      RegFlags += "|RegState::Dead";
+    Table << MatchTable::NamedValue(RegFlags);
+  } else
+    Table << MatchTable::IntValue(0);
+
+  if (SubRegIdx)
+    Table << MatchTable::NamedValue(SubRegIdx->getQualifiedName());
+  Table << MatchTable::LineBreak;
+}
+
+//===- SubRegIndexRenderer ------------------------------------------------===//
+
+void SubRegIndexRenderer::emitRenderOpcodes(MatchTable &Table,
+                                            RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID")
+        << MatchTable::IntValue(InsnID) << MatchTable::Comment("SubRegIndex")
+        << MatchTable::IntValue(SubRegIdx->EnumValue) << MatchTable::LineBreak;
+}
+
+//===- RenderComplexPatternOperand ----------------------------------------===//
+
+void RenderComplexPatternOperand::emitRenderOpcodes(MatchTable &Table,
+                                                    RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode(
+               SubOperand ? (SubReg ? "GIR_ComplexSubOperandSubRegRenderer"
+                                    : "GIR_ComplexSubOperandRenderer")
+                          : "GIR_ComplexRenderer")
+        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+        << MatchTable::Comment("RendererID")
+        << MatchTable::IntValue(RendererID);
+  if (SubOperand)
+    Table << MatchTable::Comment("SubOperand")
+          << MatchTable::IntValue(*SubOperand);
+  if (SubReg)
+    Table << MatchTable::Comment("SubRegIdx")
+          << MatchTable::IntValue(SubReg->EnumValue);
+  Table << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
+}
+
+//===- CustomRenderer -----------------------------------------------------===//
+
+void CustomRenderer::emitRenderOpcodes(MatchTable &Table,
+                                       RuleMatcher &Rule) const {
+  InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
+  unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
+  Table << MatchTable::Opcode("GIR_CustomRenderer")
+        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+        << MatchTable::Comment("OldInsnID")
+        << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("Renderer")
+        << MatchTable::NamedValue("GICR_" +
+                                  Renderer.getValueAsString("RendererFn").str())
+        << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
+}
+
+//===- CustomOperandRenderer ----------------------------------------------===//
+
+void CustomOperandRenderer::emitRenderOpcodes(MatchTable &Table,
+                                              RuleMatcher &Rule) const {
+  const OperandMatcher &OpdMatcher = Rule.getOperandMatcher(SymbolicName);
+  Table << MatchTable::Opcode("GIR_CustomOperandRenderer")
+        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+        << MatchTable::Comment("OldInsnID")
+        << MatchTable::IntValue(OpdMatcher.getInsnVarID())
+        << MatchTable::Comment("OpIdx")
+        << MatchTable::IntValue(OpdMatcher.getOpIdx())
+        << MatchTable::Comment("OperandRenderer")
+        << MatchTable::NamedValue("GICR_" +
+                                  Renderer.getValueAsString("RendererFn").str())
+        << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
+}
+
+//===- CustomCXXAction ----------------------------------------------------===//
+
+void CustomCXXAction::emitActionOpcodes(MatchTable &Table,
+                                        RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIR_CustomAction")
+        << MatchTable::NamedValue(FnEnumName) << MatchTable::LineBreak;
+}
+
+//===- BuildMIAction ------------------------------------------------------===//
+
+bool BuildMIAction::canMutate(RuleMatcher &Rule,
+                              const InstructionMatcher *Insn) const {
+  if (!Insn)
+    return false;
+
+  if (OperandRenderers.size() != Insn->getNumOperands())
+    return false;
+
+  for (const auto &Renderer : enumerate(OperandRenderers)) {
+    if (const auto *Copy = dyn_cast<CopyRenderer>(&*Renderer.value())) {
+      const OperandMatcher &OM =
+          Rule.getOperandMatcher(Copy->getSymbolicName());
+      if (Insn != &OM.getInstructionMatcher() ||
+          OM.getOpIdx() != Renderer.index())
+        return false;
+    } else
+      return false;
+  }
+
+  return true;
+}
+
+void BuildMIAction::chooseInsnToMutate(RuleMatcher &Rule) {
+  for (auto *MutateCandidate : Rule.mutatable_insns()) {
+    if (canMutate(Rule, MutateCandidate)) {
+      // Take the first one we're offered that we're able to mutate.
+      Rule.reserveInsnMatcherForMutation(MutateCandidate);
+      Matched = MutateCandidate;
+      return;
+    }
+  }
+}
+
+void BuildMIAction::emitActionOpcodes(MatchTable &Table,
+                                      RuleMatcher &Rule) const {
+  if (Matched) {
+    assert(canMutate(Rule, Matched) &&
+           "Arranged to mutate an insn that isn't mutatable");
+
+    unsigned RecycleInsnID = Rule.getInsnVarID(*Matched);
+    Table << MatchTable::Opcode("GIR_MutateOpcode")
+          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+          << MatchTable::Comment("RecycleInsnID")
+          << MatchTable::IntValue(RecycleInsnID)
+          << MatchTable::Comment("Opcode")
+          << MatchTable::NamedValue(I->Namespace, I->TheDef->getName())
+          << MatchTable::LineBreak;
+
+    if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) {
+      for (auto *Def : I->ImplicitDefs) {
+        auto Namespace = Def->getValue("Namespace")
+                             ? Def->getValueAsString("Namespace")
+                             : "";
+        Table << MatchTable::Opcode("GIR_AddImplicitDef")
+              << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+              << MatchTable::NamedValue(Namespace, Def->getName())
+              << MatchTable::LineBreak;
+      }
+      for (auto *Use : I->ImplicitUses) {
+        auto Namespace = Use->getValue("Namespace")
+                             ? Use->getValueAsString("Namespace")
+                             : "";
+        Table << MatchTable::Opcode("GIR_AddImplicitUse")
+              << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+              << MatchTable::NamedValue(Namespace, Use->getName())
+              << MatchTable::LineBreak;
+      }
+    }
+    return;
+  }
+
+  // TODO: Simple permutation looks like it could be almost as common as
+  //       mutation due to commutative operations.
+
+  Table << MatchTable::Opcode("GIR_BuildMI") << MatchTable::Comment("InsnID")
+        << MatchTable::IntValue(InsnID) << MatchTable::Comment("Opcode")
+        << MatchTable::NamedValue(I->Namespace, I->TheDef->getName())
+        << MatchTable::LineBreak;
+  for (const auto &Renderer : OperandRenderers)
+    Renderer->emitRenderOpcodes(Table, Rule);
+
+  if (I->mayLoad || I->mayStore) {
+    Table << MatchTable::Opcode("GIR_MergeMemOperands")
+          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+          << MatchTable::Comment("MergeInsnID's");
+    // Emit the ID's for all the instructions that are matched by this rule.
+    // TODO: Limit this to matched instructions that mayLoad/mayStore or have
+    //       some other means of having a memoperand. Also limit this to
+    //       emitted instructions that expect to have a memoperand too. For
+    //       example, (G_SEXT (G_LOAD x)) that results in separate load and
+    //       sign-extend instructions shouldn't put the memoperand on the
+    //       sign-extend since it has no effect there.
+    std::vector<unsigned> MergeInsnIDs;
+    for (const auto &IDMatcherPair : Rule.defined_insn_vars())
+      MergeInsnIDs.push_back(IDMatcherPair.second);
+    llvm::sort(MergeInsnIDs);
+    for (const auto &MergeInsnID : MergeInsnIDs)
+      Table << MatchTable::IntValue(MergeInsnID);
+    Table << MatchTable::NamedValue("GIU_MergeMemOperands_EndOfList")
+          << MatchTable::LineBreak;
+  }
+
+  // FIXME: This is a hack but it's sufficient for ISel. We'll need to do
+  //        better for combines. Particularly when there are multiple match
+  //        roots.
+  if (InsnID == 0)
+    Table << MatchTable::Opcode("GIR_EraseFromParent")
+          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+          << MatchTable::LineBreak;
+}
+
+//===- ConstrainOperandToRegClassAction -----------------------------------===//
+
+void ConstrainOperandToRegClassAction::emitActionOpcodes(
+    MatchTable &Table, RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIR_ConstrainOperandRC")
+        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::NamedValue(RC.getQualifiedName() + "RegClassID")
+        << MatchTable::LineBreak;
+}
+
+//===- MakeTempRegisterAction ---------------------------------------------===//
+
+void MakeTempRegisterAction::emitActionOpcodes(MatchTable &Table,
+                                               RuleMatcher &Rule) const {
+  Table << MatchTable::Opcode("GIR_MakeTempReg")
+        << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
+        << MatchTable::Comment("TypeID")
+        << MatchTable::NamedValue(Ty.getCxxEnumValue())
+        << MatchTable::LineBreak;
+}
+
+} // namespace gi
+} // namespace llvm
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/GlobalISelMatchTable.h
new file mode 100644
index 000000000000..fcb3392226c1
--- /dev/null
+++ b/llvm/utils/TableGen/GlobalISelMatchTable.h
@@ -0,0 +1,2162 @@
+//===- GlobalISelMatchTable.h ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file contains the code related to the GlobalISel Match Table emitted by
+/// GlobalISelEmitter.cpp. The generated match table is interpreted at runtime
+/// by `GIMatchTableExecutorImpl.h` to match & apply ISel patterns.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H
+#define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLE_H
+
+#include "CodeGenDAGPatterns.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/LowLevelType.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/SaveAndRestore.h"
+#include <deque>
+#include <list>
+#include <map>
+#include <memory>
+#include <optional>
+#include <set>
+#include <string>
+#include <vector>
+
+namespace llvm {
+
+class raw_ostream;
+class Record;
+class SMLoc;
+class CodeGenRegisterClass;
+
+// Use a namespace to avoid conflicts because there's some fairly generic names
+// in there (e.g. Matcher).
+namespace gi {
+class MatchTable;
+class Matcher;
+class OperandMatcher;
+class MatchAction;
+class PredicateMatcher;
+class InstructionMatcher;
+
+enum {
+  GISF_IgnoreCopies = 0x1,
+};
+
+using GISelFlags = std::uint16_t;
+
+//===- Helper functions ---------------------------------------------------===//
+
+std::string getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset);
+
+/// Takes a sequence of \p Rules and group them based on the predicates
+/// they share. \p MatcherStorage is used as a memory container
+/// for the group that are created as part of this process.
+///
+/// What this optimization does looks like if GroupT = GroupMatcher:
+/// Output without optimization:
+/// \verbatim
+/// # R1
+///  # predicate A
+///  # predicate B
+///  ...
+/// # R2
+///  # predicate A // <-- effectively this is going to be checked twice.
+///                //     Once in R1 and once in R2.
+///  # predicate C
+/// \endverbatim
+/// Output with optimization:
+/// \verbatim
+/// # Group1_2
+///  # predicate A // <-- Check is now shared.
+///  # R1
+///   # predicate B
+///  # R2
+///   # predicate C
+/// \endverbatim
+template <class GroupT>
+std::vector<Matcher *>
+optimizeRules(ArrayRef<Matcher *> Rules,
+              std::vector<std::unique_ptr<Matcher>> &MatcherStorage);
+
+/// A record to be stored in a MatchTable.
+///
+/// This class represents any and all output that may be required to emit the
+/// MatchTable. Instances  are most often configured to represent an opcode or
+/// value that will be emitted to the table with some formatting but it can also
+/// represent commas, comments, and other formatting instructions.
+struct MatchTableRecord {
+  enum RecordFlagsBits {
+    MTRF_None = 0x0,
+    /// Causes EmitStr to be formatted as comment when emitted.
+    MTRF_Comment = 0x1,
+    /// Causes the record value to be followed by a comma when emitted.
+    MTRF_CommaFollows = 0x2,
+    /// Causes the record value to be followed by a line break when emitted.
+    MTRF_LineBreakFollows = 0x4,
+    /// Indicates that the record defines a label and causes an additional
+    /// comment to be emitted containing the index of the label.
+    MTRF_Label = 0x8,
+    /// Causes the record to be emitted as the index of the label specified by
+    /// LabelID along with a comment indicating where that label is.
+    MTRF_JumpTarget = 0x10,
+    /// Causes the formatter to add a level of indentation before emitting the
+    /// record.
+    MTRF_Indent = 0x20,
+    /// Causes the formatter to remove a level of indentation after emitting the
+    /// record.
+    MTRF_Outdent = 0x40,
+  };
+
+  /// When MTRF_Label or MTRF_JumpTarget is used, indicates a label id to
+  /// reference or define.
+  unsigned LabelID;
+  /// The string to emit. Depending on the MTRF_* flags it may be a comment, a
+  /// value, a label name.
+  std::string EmitStr;
+
+private:
+  /// The number of MatchTable elements described by this record. Comments are 0
+  /// while values are typically 1. Values >1 may occur when we need to emit
+  /// values that exceed the size of a MatchTable element.
+  unsigned NumElements;
+
+public:
+  /// A bitfield of RecordFlagsBits flags.
+  unsigned Flags;
+
+  /// The actual run-time value, if known
+  int64_t RawValue;
+
+  MatchTableRecord(std::optional<unsigned> LabelID_, StringRef EmitStr,
+                   unsigned NumElements, unsigned Flags,
+                   int64_t RawValue = std::numeric_limits<int64_t>::min())
+      : LabelID(LabelID_.value_or(~0u)), EmitStr(EmitStr),
+        NumElements(NumElements), Flags(Flags), RawValue(RawValue) {
+    assert((!LabelID_ || LabelID != ~0u) &&
+           "This value is reserved for non-labels");
+  }
+  MatchTableRecord(const MatchTableRecord &Other) = default;
+  MatchTableRecord(MatchTableRecord &&Other) = default;
+
+  /// Useful if a Match Table Record gets optimized out
+  void turnIntoComment() {
+    Flags |= MTRF_Comment;
+    Flags &= ~MTRF_CommaFollows;
+    NumElements = 0;
+  }
+
+  /// For Jump Table generation purposes
+  bool operator<(const MatchTableRecord &Other) const {
+    return RawValue < Other.RawValue;
+  }
+  int64_t getRawValue() const { return RawValue; }
+
+  void emit(raw_ostream &OS, bool LineBreakNextAfterThis,
+            const MatchTable &Table) const;
+  unsigned size() const { return NumElements; }
+};
+
+/// Holds the contents of a generated MatchTable to enable formatting and the
+/// necessary index tracking needed to support GIM_Try.
+class MatchTable {
+  /// An unique identifier for the table. The generated table will be named
+  /// MatchTable${ID}.
+  unsigned ID;
+  /// The records that make up the table. Also includes comments describing the
+  /// values being emitted and line breaks to format it.
+  std::vector<MatchTableRecord> Contents;
+  /// The currently defined labels.
+  DenseMap<unsigned, unsigned> LabelMap;
+  /// Tracks the sum of MatchTableRecord::NumElements as the table is built.
+  unsigned CurrentSize = 0;
+  /// A unique identifier for a MatchTable label.
+  unsigned CurrentLabelID = 0;
+  /// Determines if the table should be instrumented for rule coverage tracking.
+  bool IsWithCoverage;
+  /// Whether this table is for the GISel combiner.
+  bool IsCombinerTable;
+
+public:
+  static MatchTableRecord LineBreak;
+  static MatchTableRecord Comment(StringRef Comment);
+  static MatchTableRecord Opcode(StringRef Opcode, int IndentAdjust = 0);
+  static MatchTableRecord NamedValue(StringRef NamedValue);
+  static MatchTableRecord NamedValue(StringRef NamedValue, int64_t RawValue);
+  static MatchTableRecord NamedValue(StringRef Namespace, StringRef NamedValue);
+  static MatchTableRecord NamedValue(StringRef Namespace, StringRef NamedValue,
+                                     int64_t RawValue);
+  static MatchTableRecord IntValue(int64_t IntValue);
+  static MatchTableRecord Label(unsigned LabelID);
+  static MatchTableRecord JumpTarget(unsigned LabelID);
+
+  static MatchTable buildTable(ArrayRef<Matcher *> Rules, bool WithCoverage,
+                               bool IsCombiner = false);
+
+  MatchTable(bool WithCoverage, bool IsCombinerTable, unsigned ID = 0)
+      : ID(ID), IsWithCoverage(WithCoverage), IsCombinerTable(IsCombinerTable) {
+  }
+
+  bool isWithCoverage() const { return IsWithCoverage; }
+  bool isCombiner() const { return IsCombinerTable; }
+
+  void push_back(const MatchTableRecord &Value) {
+    if (Value.Flags & MatchTableRecord::MTRF_Label)
+      defineLabel(Value.LabelID);
+    Contents.push_back(Value);
+    CurrentSize += Value.size();
+  }
+
+  unsigned allocateLabelID() { return CurrentLabelID++; }
+
+  void defineLabel(unsigned LabelID) {
+    LabelMap.insert(std::make_pair(LabelID, CurrentSize));
+  }
+
+  unsigned getLabelIndex(unsigned LabelID) const {
+    const auto I = LabelMap.find(LabelID);
+    assert(I != LabelMap.end() && "Use of undeclared label");
+    return I->second;
+  }
+
+  void emitUse(raw_ostream &OS) const;
+  void emitDeclaration(raw_ostream &OS) const;
+};
+
+inline MatchTable &operator<<(MatchTable &Table,
+                              const MatchTableRecord &Value) {
+  Table.push_back(Value);
+  return Table;
+}
+
+/// This class stands in for LLT wherever we want to tablegen-erate an
+/// equivalent at compiler run-time.
+class LLTCodeGen {
+private:
+  LLT Ty;
+
+public:
+  LLTCodeGen() = default;
+  LLTCodeGen(const LLT &Ty) : Ty(Ty) {}
+
+  std::string getCxxEnumValue() const;
+
+  void emitCxxEnumValue(raw_ostream &OS) const;
+  void emitCxxConstructorCall(raw_ostream &OS) const;
+
+  const LLT &get() const { return Ty; }
+
+  /// This ordering is used for std::unique() and llvm::sort(). There's no
+  /// particular logic behind the order but either A < B or B < A must be
+  /// true if A != B.
+  bool operator<(const LLTCodeGen &Other) const;
+  bool operator==(const LLTCodeGen &B) const { return Ty == B.Ty; }
+};
+
+// Track all types that are used so we can emit the corresponding enum.
+extern std::set<LLTCodeGen> KnownTypes;
+
+/// Convert an MVT to an equivalent LLT if possible, or the invalid LLT() for
+/// MVTs that don't map cleanly to an LLT (e.g., iPTR, *any, ...).
+std::optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT);
+
+//===- Matchers -----------------------------------------------------------===//
+class Matcher {
+public:
+  virtual ~Matcher();
+  virtual void optimize();
+  virtual void emit(MatchTable &Table) = 0;
+
+  virtual bool hasFirstCondition() const = 0;
+  virtual const PredicateMatcher &getFirstCondition() const = 0;
+  virtual std::unique_ptr<PredicateMatcher> popFirstCondition() = 0;
+};
+
+class GroupMatcher final : public Matcher {
+  /// Conditions that form a common prefix of all the matchers contained.
+  SmallVector<std::unique_ptr<PredicateMatcher>, 1> Conditions;
+
+  /// All the nested matchers, sharing a common prefix.
+  std::vector<Matcher *> Matchers;
+
+  /// An owning collection for any auxiliary matchers created while optimizing
+  /// nested matchers contained.
+  std::vector<std::unique_ptr<Matcher>> MatcherStorage;
+
+public:
+  /// Add a matcher to the collection of nested matchers if it meets the
+  /// requirements, and return true. If it doesn't, do nothing and return false.
+  ///
+  /// Expected to preserve its argument, so it could be moved out later on.
+  bool addMatcher(Matcher &Candidate);
+
+  /// Mark the matcher as fully-built and ensure any invariants expected by both
+  /// optimize() and emit(...) methods. Generally, both sequences of calls
+  /// are expected to lead to a sensible result:
+  ///
+  /// addMatcher(...)*; finalize(); optimize(); emit(...); and
+  /// addMatcher(...)*; finalize(); emit(...);
+  ///
+  /// or generally
+  ///
+  /// addMatcher(...)*; finalize(); { optimize()*; emit(...); }*
+  ///
+  /// Multiple calls to optimize() are expected to be handled gracefully, though
+  /// optimize() is not expected to be idempotent. Multiple calls to finalize()
+  /// aren't generally supported. emit(...) is expected to be non-mutating and
+  /// producing the exact same results upon repeated calls.
+  ///
+  /// addMatcher() calls after the finalize() call are not supported.
+  ///
+  /// finalize() and optimize() are both allowed to mutate the contained
+  /// matchers, so moving them out after finalize() is not supported.
+  void finalize();
+  void optimize() override;
+  void emit(MatchTable &Table) override;
+
+  /// Could be used to move out the matchers added previously, unless finalize()
+  /// has been already called. If any of the matchers are moved out, the group
+  /// becomes safe to destroy, but not safe to re-use for anything else.
+  iterator_range<std::vector<Matcher *>::iterator> matchers() {
+    return make_range(Matchers.begin(), Matchers.end());
+  }
+  size_t size() const { return Matchers.size(); }
+  bool empty() const { return Matchers.empty(); }
+
+  std::unique_ptr<PredicateMatcher> popFirstCondition() override {
+    assert(!Conditions.empty() &&
+           "Trying to pop a condition from a condition-less group");
+    std::unique_ptr<PredicateMatcher> P = std::move(Conditions.front());
+    Conditions.erase(Conditions.begin());
+    return P;
+  }
+  const PredicateMatcher &getFirstCondition() const override {
+    assert(!Conditions.empty() &&
+           "Trying to get a condition from a condition-less group");
+    return *Conditions.front();
+  }
+  bool hasFirstCondition() const override { return !Conditions.empty(); }
+
+private:
+  /// See if a candidate matcher could be added to this group solely by
+  /// analyzing its first condition.
+  bool candidateConditionMatches(const PredicateMatcher &Predicate) const;
+};
+
+class SwitchMatcher : public Matcher {
+  /// All the nested matchers, representing distinct switch-cases. The first
+  /// conditions (as Matcher::getFirstCondition() reports) of all the nested
+  /// matchers must share the same type and path to a value they check, in other
+  /// words, be isIdenticalDownToValue, but have different values they check
+  /// against.
+  std::vector<Matcher *> Matchers;
+
+  /// The representative condition, with a type and a path (InsnVarID and OpIdx
+  /// in most cases)  shared by all the matchers contained.
+  std::unique_ptr<PredicateMatcher> Condition = nullptr;
+
+  /// Temporary set used to check that the case values don't repeat within the
+  /// same switch.
+  std::set<MatchTableRecord> Values;
+
+  /// An owning collection for any auxiliary matchers created while optimizing
+  /// nested matchers contained.
+  std::vector<std::unique_ptr<Matcher>> MatcherStorage;
+
+public:
+  bool addMatcher(Matcher &Candidate);
+
+  void finalize();
+  void emit(MatchTable &Table) override;
+
+  iterator_range<std::vector<Matcher *>::iterator> matchers() {
+    return make_range(Matchers.begin(), Matchers.end());
+  }
+  size_t size() const { return Matchers.size(); }
+  bool empty() const { return Matchers.empty(); }
+
+  std::unique_ptr<PredicateMatcher> popFirstCondition() override {
+    // SwitchMatcher doesn't have a common first condition for its cases, as all
+    // the cases only share a kind of a value (a type and a path to it) they
+    // match, but deliberately differ in the actual value they match.
+    llvm_unreachable("Trying to pop a condition from a condition-less group");
+  }
+
+  const PredicateMatcher &getFirstCondition() const override {
+    llvm_unreachable("Trying to pop a condition from a condition-less group");
+  }
+
+  bool hasFirstCondition() const override { return false; }
+
+private:
+  /// See if the predicate type has a Switch-implementation for it.
+  static bool isSupportedPredicateType(const PredicateMatcher &Predicate);
+
+  bool candidateConditionMatches(const PredicateMatcher &Predicate) const;
+
+  /// emit()-helper
+  static void emitPredicateSpecificOpcodes(const PredicateMatcher &P,
+                                           MatchTable &Table);
+};
+
+/// Generates code to check that a match rule matches.
+class RuleMatcher : public Matcher {
+public:
+  using ActionList = std::list<std::unique_ptr<MatchAction>>;
+  using action_iterator = ActionList::iterator;
+
+protected:
+  /// A list of matchers that all need to succeed for the current rule to match.
+  /// FIXME: This currently supports a single match position but could be
+  /// extended to support multiple positions to support div/rem fusion or
+  /// load-multiple instructions.
+  using MatchersTy = std::vector<std::unique_ptr<InstructionMatcher>>;
+  MatchersTy Matchers;
+
+  /// A list of actions that need to be taken when all predicates in this rule
+  /// have succeeded.
+  ActionList Actions;
+
+  using DefinedInsnVariablesMap = std::map<InstructionMatcher *, unsigned>;
+
+  /// A map of instruction matchers to the local variables
+  DefinedInsnVariablesMap InsnVariableIDs;
+
+  using MutatableInsnSet = SmallPtrSet<InstructionMatcher *, 4>;
+
+  // The set of instruction matchers that have not yet been claimed for mutation
+  // by a BuildMI.
+  MutatableInsnSet MutatableInsns;
+
+  /// A map of named operands defined by the matchers that may be referenced by
+  /// the renderers.
+  StringMap<OperandMatcher *> DefinedOperands;
+
+  /// A map of anonymous physical register operands defined by the matchers that
+  /// may be referenced by the renderers.
+  DenseMap<Record *, OperandMatcher *> PhysRegOperands;
+
+  /// ID for the next instruction variable defined with
+  /// implicitlyDefineInsnVar()
+  unsigned NextInsnVarID;
+
+  /// ID for the next output instruction allocated with allocateOutputInsnID()
+  unsigned NextOutputInsnID;
+
+  /// ID for the next temporary register ID allocated with allocateTempRegID()
+  unsigned NextTempRegID;
+
+  /// Current GISelFlags
+  GISelFlags Flags = 0;
+
+  std::vector<std::string> RequiredSimplePredicates;
+  std::vector<Record *> RequiredFeatures;
+  std::vector<std::unique_ptr<PredicateMatcher>> EpilogueMatchers;
+
+  ArrayRef<SMLoc> SrcLoc;
+
+  typedef std::tuple<Record *, unsigned, unsigned>
+      DefinedComplexPatternSubOperand;
+  typedef StringMap<DefinedComplexPatternSubOperand>
+      DefinedComplexPatternSubOperandMap;
+  /// A map of Symbolic Names to ComplexPattern sub-operands.
+  DefinedComplexPatternSubOperandMap ComplexSubOperands;
+  /// A map used to for multiple referenced error check of ComplexSubOperand.
+  /// ComplexSubOperand can't be referenced multiple from different operands,
+  /// however multiple references from same operand are allowed since that is
+  /// how 'same operand checks' are generated.
+  StringMap<std::string> ComplexSubOperandsParentName;
+
+  uint64_t RuleID;
+  static uint64_t NextRuleID;
+
+  GISelFlags updateGISelFlag(GISelFlags CurFlags, const Record *R,
+                             StringRef FlagName, GISelFlags FlagBit);
+
+public:
+  RuleMatcher(ArrayRef<SMLoc> SrcLoc)
+      : NextInsnVarID(0), NextOutputInsnID(0), NextTempRegID(0), SrcLoc(SrcLoc),
+        RuleID(NextRuleID++) {}
+  RuleMatcher(RuleMatcher &&Other) = default;
+  RuleMatcher &operator=(RuleMatcher &&Other) = default;
+
+  uint64_t getRuleID() const { return RuleID; }
+
+  InstructionMatcher &addInstructionMatcher(StringRef SymbolicName);
+  void addRequiredFeature(Record *Feature);
+  const std::vector<Record *> &getRequiredFeatures() const;
+
+  void addRequiredSimplePredicate(StringRef PredName);
+  const std::vector<std::string> &getRequiredSimplePredicates();
+
+  // Emplaces an action of the specified Kind at the end of the action list.
+  //
+  // Returns a reference to the newly created action.
+  //
+  // Like std::vector::emplace_back(), may invalidate all iterators if the new
+  // size exceeds the capacity. Otherwise, only invalidates the past-the-end
+  // iterator.
+  template <class Kind, class... Args> Kind &addAction(Args &&...args) {
+    Actions.emplace_back(std::make_unique<Kind>(std::forward<Args>(args)...));
+    return *static_cast<Kind *>(Actions.back().get());
+  }
+
+  // Emplaces an action of the specified Kind before the given insertion point.
+  //
+  // Returns an iterator pointing at the newly created instruction.
+  //
+  // Like std::vector::insert(), may invalidate all iterators if the new size
+  // exceeds the capacity. Otherwise, only invalidates the iterators from the
+  // insertion point onwards.
+  template <class Kind, class... Args>
+  action_iterator insertAction(action_iterator InsertPt, Args &&...args) {
+    return Actions.emplace(InsertPt,
+                           std::make_unique<Kind>(std::forward<Args>(args)...));
+  }
+
+  // Update the active GISelFlags based on the GISelFlags Record R.
+  // A SaveAndRestore object is returned so the old GISelFlags are restored
+  // at the end of the scope.
+  SaveAndRestore<GISelFlags> setGISelFlags(const Record *R);
+  GISelFlags getGISelFlags() const { return Flags; }
+
+  /// Define an instruction without emitting any code to do so.
+  unsigned implicitlyDefineInsnVar(InstructionMatcher &Matcher);
+
+  unsigned getInsnVarID(InstructionMatcher &InsnMatcher) const;
+  DefinedInsnVariablesMap::const_iterator defined_insn_vars_begin() const {
+    return InsnVariableIDs.begin();
+  }
+  DefinedInsnVariablesMap::const_iterator defined_insn_vars_end() const {
+    return InsnVariableIDs.end();
+  }
+  iterator_range<typename DefinedInsnVariablesMap::const_iterator>
+  defined_insn_vars() const {
+    return make_range(defined_insn_vars_begin(), defined_insn_vars_end());
+  }
+
+  MutatableInsnSet::const_iterator mutatable_insns_begin() const {
+    return MutatableInsns.begin();
+  }
+  MutatableInsnSet::const_iterator mutatable_insns_end() const {
+    return MutatableInsns.end();
+  }
+  iterator_range<typename MutatableInsnSet::const_iterator>
+  mutatable_insns() const {
+    return make_range(mutatable_insns_begin(), mutatable_insns_end());
+  }
+  void reserveInsnMatcherForMutation(InstructionMatcher *InsnMatcher) {
+    bool R = MutatableInsns.erase(InsnMatcher);
+    assert(R && "Reserving a mutatable insn that isn't available");
+    (void)R;
+  }
+
+  action_iterator actions_begin() { return Actions.begin(); }
+  action_iterator actions_end() { return Actions.end(); }
+  iterator_range<action_iterator> actions() {
+    return make_range(actions_begin(), actions_end());
+  }
+
+  void defineOperand(StringRef SymbolicName, OperandMatcher &OM);
+
+  void definePhysRegOperand(Record *Reg, OperandMatcher &OM);
+
+  Error defineComplexSubOperand(StringRef SymbolicName, Record *ComplexPattern,
+                                unsigned RendererID, unsigned SubOperandID,
+                                StringRef ParentSymbolicName);
+
+  std::optional<DefinedComplexPatternSubOperand>
+  getComplexSubOperand(StringRef SymbolicName) const {
+    const auto &I = ComplexSubOperands.find(SymbolicName);
+    if (I == ComplexSubOperands.end())
+      return std::nullopt;
+    return I->second;
+  }
+
+  InstructionMatcher &getInstructionMatcher(StringRef SymbolicName) const;
+  const OperandMatcher &getOperandMatcher(StringRef Name) const;
+  const OperandMatcher &getPhysRegOperandMatcher(Record *) const;
+
+  void optimize() override;
+  void emit(MatchTable &Table) override;
+
+  /// Compare the priority of this object and B.
+  ///
+  /// Returns true if this object is more important than B.
+  bool isHigherPriorityThan(const RuleMatcher &B) const;
+
+  /// Report the maximum number of temporary operands needed by the rule
+  /// matcher.
+  unsigned countRendererFns() const;
+
+  std::unique_ptr<PredicateMatcher> popFirstCondition() override;
+  const PredicateMatcher &getFirstCondition() const override;
+  LLTCodeGen getFirstConditionAsRootType();
+  bool hasFirstCondition() const override;
+  unsigned getNumOperands() const;
+  StringRef getOpcode() const;
+
+  // FIXME: Remove this as soon as possible
+  InstructionMatcher &insnmatchers_front() const { return *Matchers.front(); }
+
+  unsigned allocateOutputInsnID() { return NextOutputInsnID++; }
+  unsigned allocateTempRegID() { return NextTempRegID++; }
+
+  iterator_range<MatchersTy::iterator> insnmatchers() {
+    return make_range(Matchers.begin(), Matchers.end());
+  }
+  bool insnmatchers_empty() const { return Matchers.empty(); }
+  void insnmatchers_pop_front() { Matchers.erase(Matchers.begin()); }
+};
+
+template <class PredicateTy> class PredicateListMatcher {
+private:
+  /// Template instantiations should specialize this to return a string to use
+  /// for the comment emitted when there are no predicates.
+  std::string getNoPredicateComment() const;
+
+protected:
+  using PredicatesTy = std::deque<std::unique_ptr<PredicateTy>>;
+  PredicatesTy Predicates;
+
+  /// Track if the list of predicates was manipulated by one of the optimization
+  /// methods.
+  bool Optimized = false;
+
+public:
+  typename PredicatesTy::iterator predicates_begin() {
+    return Predicates.begin();
+  }
+  typename PredicatesTy::iterator predicates_end() { return Predicates.end(); }
+  iterator_range<typename PredicatesTy::iterator> predicates() {
+    return make_range(predicates_begin(), predicates_end());
+  }
+  typename PredicatesTy::size_type predicates_size() const {
+    return Predicates.size();
+  }
+  bool predicates_empty() const { return Predicates.empty(); }
+
+  std::unique_ptr<PredicateTy> predicates_pop_front() {
+    std::unique_ptr<PredicateTy> Front = std::move(Predicates.front());
+    Predicates.pop_front();
+    Optimized = true;
+    return Front;
+  }
+
+  void prependPredicate(std::unique_ptr<PredicateTy> &&Predicate) {
+    Predicates.push_front(std::move(Predicate));
+  }
+
+  void eraseNullPredicates() {
+    const auto NewEnd =
+        std::stable_partition(Predicates.begin(), Predicates.end(),
+                              std::logical_not<std::unique_ptr<PredicateTy>>());
+    if (NewEnd != Predicates.begin()) {
+      Predicates.erase(Predicates.begin(), NewEnd);
+      Optimized = true;
+    }
+  }
+
+  /// Emit MatchTable opcodes that tests whether all the predicates are met.
+  template <class... Args>
+  void emitPredicateListOpcodes(MatchTable &Table, Args &&...args) {
+    if (Predicates.empty() && !Optimized) {
+      Table << MatchTable::Comment(getNoPredicateComment())
+            << MatchTable::LineBreak;
+      return;
+    }
+
+    for (const auto &Predicate : predicates())
+      Predicate->emitPredicateOpcodes(Table, std::forward<Args>(args)...);
+  }
+
+  /// Provide a function to avoid emitting certain predicates. This is used to
+  /// defer some predicate checks until after others
+  using PredicateFilterFunc = std::function<bool(const PredicateTy &)>;
+
+  /// Emit MatchTable opcodes for predicates which satisfy \p
+  /// ShouldEmitPredicate. This should be called multiple times to ensure all
+  /// predicates are eventually added to the match table.
+  template <class... Args>
+  void emitFilteredPredicateListOpcodes(PredicateFilterFunc ShouldEmitPredicate,
+                                        MatchTable &Table, Args &&...args) {
+    if (Predicates.empty() && !Optimized) {
+      Table << MatchTable::Comment(getNoPredicateComment())
+            << MatchTable::LineBreak;
+      return;
+    }
+
+    for (const auto &Predicate : predicates()) {
+      if (ShouldEmitPredicate(*Predicate))
+        Predicate->emitPredicateOpcodes(Table, std::forward<Args>(args)...);
+    }
+  }
+};
+
+class PredicateMatcher {
+public:
+  /// This enum is used for RTTI and also defines the priority that is given to
+  /// the predicate when generating the matcher code. Kinds with higher priority
+  /// must be tested first.
+  ///
+  /// The relative priority of OPM_LLT, OPM_RegBank, and OPM_MBB do not matter
+  /// but OPM_Int must have priority over OPM_RegBank since constant integers
+  /// are represented by a virtual register defined by a G_CONSTANT instruction.
+  ///
+  /// Note: The relative priority between IPM_ and OPM_ does not matter, they
+  /// are currently not compared between each other.
+  enum PredicateKind {
+    IPM_Opcode,
+    IPM_NumOperands,
+    IPM_ImmPredicate,
+    IPM_Imm,
+    IPM_AtomicOrderingMMO,
+    IPM_MemoryLLTSize,
+    IPM_MemoryVsLLTSize,
+    IPM_MemoryAddressSpace,
+    IPM_MemoryAlignment,
+    IPM_VectorSplatImm,
+    IPM_NoUse,
+    IPM_GenericPredicate,
+    OPM_SameOperand,
+    OPM_ComplexPattern,
+    OPM_IntrinsicID,
+    OPM_CmpPredicate,
+    OPM_Instruction,
+    OPM_Int,
+    OPM_LiteralInt,
+    OPM_LLT,
+    OPM_PointerToAny,
+    OPM_RegBank,
+    OPM_MBB,
+    OPM_RecordNamedOperand,
+  };
+
+protected:
+  PredicateKind Kind;
+  unsigned InsnVarID;
+  unsigned OpIdx;
+
+public:
+  PredicateMatcher(PredicateKind Kind, unsigned InsnVarID, unsigned OpIdx = ~0)
+      : Kind(Kind), InsnVarID(InsnVarID), OpIdx(OpIdx) {}
+  virtual ~PredicateMatcher();
+
+  unsigned getInsnVarID() const { return InsnVarID; }
+  unsigned getOpIdx() const { return OpIdx; }
+
+  /// Emit MatchTable opcodes that check the predicate for the given operand.
+  virtual void emitPredicateOpcodes(MatchTable &Table,
+                                    RuleMatcher &Rule) const = 0;
+
+  PredicateKind getKind() const { return Kind; }
+
+  bool dependsOnOperands() const {
+    // Custom predicates really depend on the context pattern of the
+    // instruction, not just the individual instruction. This therefore
+    // implicitly depends on all other pattern constraints.
+    return Kind == IPM_GenericPredicate;
+  }
+
+  virtual bool isIdentical(const PredicateMatcher &B) const {
+    return B.getKind() == getKind() && InsnVarID == B.InsnVarID &&
+           OpIdx == B.OpIdx;
+  }
+
+  virtual bool isIdenticalDownToValue(const PredicateMatcher &B) const {
+    return hasValue() && PredicateMatcher::isIdentical(B);
+  }
+
+  virtual MatchTableRecord getValue() const {
+    assert(hasValue() && "Can not get a value of a value-less predicate!");
+    llvm_unreachable("Not implemented yet");
+  }
+  virtual bool hasValue() const { return false; }
+
+  /// Report the maximum number of temporary operands needed by the predicate
+  /// matcher.
+  virtual unsigned countRendererFns() const { return 0; }
+};
+
+/// Generates code to check a predicate of an operand.
+///
+/// Typical predicates include:
+/// * Operand is a particular register.
+/// * Operand is assigned a particular register bank.
+/// * Operand is an MBB.
+class OperandPredicateMatcher : public PredicateMatcher {
+public:
+  OperandPredicateMatcher(PredicateKind Kind, unsigned InsnVarID,
+                          unsigned OpIdx)
+      : PredicateMatcher(Kind, InsnVarID, OpIdx) {}
+  virtual ~OperandPredicateMatcher();
+
+  /// Compare the priority of this object and B.
+  ///
+  /// Returns true if this object is more important than B.
+  virtual bool isHigherPriorityThan(const OperandPredicateMatcher &B) const;
+};
+
+template <>
+inline std::string
+PredicateListMatcher<OperandPredicateMatcher>::getNoPredicateComment() const {
+  return "No operand predicates";
+}
+
+/// Generates code to check that a register operand is defined by the same exact
+/// one as another.
+class SameOperandMatcher : public OperandPredicateMatcher {
+  std::string MatchingName;
+  unsigned OrigOpIdx;
+
+  GISelFlags Flags;
+
+public:
+  SameOperandMatcher(unsigned InsnVarID, unsigned OpIdx, StringRef MatchingName,
+                     unsigned OrigOpIdx, GISelFlags Flags)
+      : OperandPredicateMatcher(OPM_SameOperand, InsnVarID, OpIdx),
+        MatchingName(MatchingName), OrigOpIdx(OrigOpIdx), Flags(Flags) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_SameOperand;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return OperandPredicateMatcher::isIdentical(B) &&
+           OrigOpIdx == cast<SameOperandMatcher>(&B)->OrigOpIdx &&
+           MatchingName == cast<SameOperandMatcher>(&B)->MatchingName;
+  }
+};
+
+/// Generates code to check that an operand is a particular LLT.
+class LLTOperandMatcher : public OperandPredicateMatcher {
+protected:
+  LLTCodeGen Ty;
+
+public:
+  static std::map<LLTCodeGen, unsigned> TypeIDValues;
+
+  static void initTypeIDValuesMap() {
+    TypeIDValues.clear();
+
+    unsigned ID = 0;
+    for (const LLTCodeGen &LLTy : KnownTypes)
+      TypeIDValues[LLTy] = ID++;
+  }
+
+  LLTOperandMatcher(unsigned InsnVarID, unsigned OpIdx, const LLTCodeGen &Ty)
+      : OperandPredicateMatcher(OPM_LLT, InsnVarID, OpIdx), Ty(Ty) {
+    KnownTypes.insert(Ty);
+  }
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_LLT;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return OperandPredicateMatcher::isIdentical(B) &&
+           Ty == cast<LLTOperandMatcher>(&B)->Ty;
+  }
+
+  MatchTableRecord getValue() const override;
+  bool hasValue() const override;
+
+  LLTCodeGen getTy() const { return Ty; }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that an operand is a pointer to any address space.
+///
+/// In SelectionDAG, the types did not describe pointers or address spaces. As a
+/// result, iN is used to describe a pointer of N bits to any address space and
+/// PatFrag predicates are typically used to constrain the address space.
+/// There's no reliable means to derive the missing type information from the
+/// pattern so imported rules must test the components of a pointer separately.
+///
+/// If SizeInBits is zero, then the pointer size will be obtained from the
+/// subtarget.
+class PointerToAnyOperandMatcher : public OperandPredicateMatcher {
+protected:
+  unsigned SizeInBits;
+
+public:
+  PointerToAnyOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                             unsigned SizeInBits)
+      : OperandPredicateMatcher(OPM_PointerToAny, InsnVarID, OpIdx),
+        SizeInBits(SizeInBits) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_PointerToAny;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return OperandPredicateMatcher::isIdentical(B) &&
+           SizeInBits == cast<PointerToAnyOperandMatcher>(&B)->SizeInBits;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to record named operand in RecordedOperands list at StoreIdx.
+/// Predicates with 'let PredicateCodeUsesOperands = 1' get RecordedOperands as
+/// an argument to predicate's c++ code once all operands have been matched.
+class RecordNamedOperandMatcher : public OperandPredicateMatcher {
+protected:
+  unsigned StoreIdx;
+  std::string Name;
+
+public:
+  RecordNamedOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                            unsigned StoreIdx, StringRef Name)
+      : OperandPredicateMatcher(OPM_RecordNamedOperand, InsnVarID, OpIdx),
+        StoreIdx(StoreIdx), Name(Name) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_RecordNamedOperand;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return OperandPredicateMatcher::isIdentical(B) &&
+           StoreIdx == cast<RecordNamedOperandMatcher>(&B)->StoreIdx &&
+           Name == cast<RecordNamedOperandMatcher>(&B)->Name;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that an operand is a particular target constant.
+class ComplexPatternOperandMatcher : public OperandPredicateMatcher {
+protected:
+  const OperandMatcher &Operand;
+  const Record &TheDef;
+
+  unsigned getAllocatedTemporariesBaseID() const;
+
+public:
+  bool isIdentical(const PredicateMatcher &B) const override { return false; }
+
+  ComplexPatternOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                               const OperandMatcher &Operand,
+                               const Record &TheDef)
+      : OperandPredicateMatcher(OPM_ComplexPattern, InsnVarID, OpIdx),
+        Operand(Operand), TheDef(TheDef) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_ComplexPattern;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+  unsigned countRendererFns() const override { return 1; }
+};
+
+/// Generates code to check that an operand is in a particular register bank.
+class RegisterBankOperandMatcher : public OperandPredicateMatcher {
+protected:
+  const CodeGenRegisterClass &RC;
+
+public:
+  RegisterBankOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                             const CodeGenRegisterClass &RC)
+      : OperandPredicateMatcher(OPM_RegBank, InsnVarID, OpIdx), RC(RC) {}
+
+  bool isIdentical(const PredicateMatcher &B) const override;
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_RegBank;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that an operand is a basic block.
+class MBBOperandMatcher : public OperandPredicateMatcher {
+public:
+  MBBOperandMatcher(unsigned InsnVarID, unsigned OpIdx)
+      : OperandPredicateMatcher(OPM_MBB, InsnVarID, OpIdx) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_MBB;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+class ImmOperandMatcher : public OperandPredicateMatcher {
+public:
+  ImmOperandMatcher(unsigned InsnVarID, unsigned OpIdx)
+      : OperandPredicateMatcher(IPM_Imm, InsnVarID, OpIdx) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_Imm;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that an operand is a G_CONSTANT with a particular
+/// int.
+class ConstantIntOperandMatcher : public OperandPredicateMatcher {
+protected:
+  int64_t Value;
+
+public:
+  ConstantIntOperandMatcher(unsigned InsnVarID, unsigned OpIdx, int64_t Value)
+      : OperandPredicateMatcher(OPM_Int, InsnVarID, OpIdx), Value(Value) {}
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return OperandPredicateMatcher::isIdentical(B) &&
+           Value == cast<ConstantIntOperandMatcher>(&B)->Value;
+  }
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_Int;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that an operand is a raw int (where MO.isImm() or
+/// MO.isCImm() is true).
+class LiteralIntOperandMatcher : public OperandPredicateMatcher {
+protected:
+  int64_t Value;
+
+public:
+  LiteralIntOperandMatcher(unsigned InsnVarID, unsigned OpIdx, int64_t Value)
+      : OperandPredicateMatcher(OPM_LiteralInt, InsnVarID, OpIdx),
+        Value(Value) {}
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return OperandPredicateMatcher::isIdentical(B) &&
+           Value == cast<LiteralIntOperandMatcher>(&B)->Value;
+  }
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_LiteralInt;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that an operand is an CmpInst predicate
+class CmpPredicateOperandMatcher : public OperandPredicateMatcher {
+protected:
+  std::string PredName;
+
+public:
+  CmpPredicateOperandMatcher(unsigned InsnVarID, unsigned OpIdx, std::string P)
+      : OperandPredicateMatcher(OPM_CmpPredicate, InsnVarID, OpIdx),
+        PredName(P) {}
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return OperandPredicateMatcher::isIdentical(B) &&
+           PredName == cast<CmpPredicateOperandMatcher>(&B)->PredName;
+  }
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_CmpPredicate;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that an operand is an intrinsic ID.
+class IntrinsicIDOperandMatcher : public OperandPredicateMatcher {
+protected:
+  const CodeGenIntrinsic *II;
+
+public:
+  IntrinsicIDOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                            const CodeGenIntrinsic *II)
+      : OperandPredicateMatcher(OPM_IntrinsicID, InsnVarID, OpIdx), II(II) {}
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return OperandPredicateMatcher::isIdentical(B) &&
+           II == cast<IntrinsicIDOperandMatcher>(&B)->II;
+  }
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_IntrinsicID;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that this operand is an immediate whose value meets
+/// an immediate predicate.
+class OperandImmPredicateMatcher : public OperandPredicateMatcher {
+protected:
+  TreePredicateFn Predicate;
+
+public:
+  OperandImmPredicateMatcher(unsigned InsnVarID, unsigned OpIdx,
+                             const TreePredicateFn &Predicate)
+      : OperandPredicateMatcher(IPM_ImmPredicate, InsnVarID, OpIdx),
+        Predicate(Predicate) {}
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return OperandPredicateMatcher::isIdentical(B) &&
+           Predicate.getOrigPatFragRecord() ==
+               cast<OperandImmPredicateMatcher>(&B)
+                   ->Predicate.getOrigPatFragRecord();
+  }
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_ImmPredicate;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that a set of predicates match for a particular
+/// operand.
+class OperandMatcher : public PredicateListMatcher<OperandPredicateMatcher> {
+protected:
+  InstructionMatcher &Insn;
+  unsigned OpIdx;
+  std::string SymbolicName;
+
+  /// The index of the first temporary variable allocated to this operand. The
+  /// number of allocated temporaries can be found with
+  /// countRendererFns().
+  unsigned AllocatedTemporariesBaseID;
+
+public:
+  OperandMatcher(InstructionMatcher &Insn, unsigned OpIdx,
+                 const std::string &SymbolicName,
+                 unsigned AllocatedTemporariesBaseID)
+      : Insn(Insn), OpIdx(OpIdx), SymbolicName(SymbolicName),
+        AllocatedTemporariesBaseID(AllocatedTemporariesBaseID) {}
+
+  bool hasSymbolicName() const { return !SymbolicName.empty(); }
+  StringRef getSymbolicName() const { return SymbolicName; }
+  void setSymbolicName(StringRef Name) {
+    assert(SymbolicName.empty() && "Operand already has a symbolic name");
+    SymbolicName = std::string(Name);
+  }
+
+  /// Construct a new operand predicate and add it to the matcher.
+  template <class Kind, class... Args>
+  std::optional<Kind *> addPredicate(Args &&...args) {
+    if (isSameAsAnotherOperand())
+      return std::nullopt;
+    Predicates.emplace_back(std::make_unique<Kind>(
+        getInsnVarID(), getOpIdx(), std::forward<Args>(args)...));
+    return static_cast<Kind *>(Predicates.back().get());
+  }
+
+  unsigned getOpIdx() const { return OpIdx; }
+  unsigned getInsnVarID() const;
+
+  std::string getOperandExpr(unsigned InsnVarID) const;
+
+  InstructionMatcher &getInstructionMatcher() const { return Insn; }
+
+  Error addTypeCheckPredicate(const TypeSetByHwMode &VTy,
+                              bool OperandIsAPointer);
+
+  /// Emit MatchTable opcodes that test whether the instruction named in
+  /// InsnVarID matches all the predicates and all the operands.
+  void emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule);
+
+  /// Compare the priority of this object and B.
+  ///
+  /// Returns true if this object is more important than B.
+  bool isHigherPriorityThan(OperandMatcher &B);
+
+  /// Report the maximum number of temporary operands needed by the operand
+  /// matcher.
+  unsigned countRendererFns();
+
+  unsigned getAllocatedTemporariesBaseID() const {
+    return AllocatedTemporariesBaseID;
+  }
+
+  bool isSameAsAnotherOperand() {
+    for (const auto &Predicate : predicates())
+      if (isa<SameOperandMatcher>(Predicate))
+        return true;
+    return false;
+  }
+};
+
+/// Generates code to check a predicate on an instruction.
+///
+/// Typical predicates include:
+/// * The opcode of the instruction is a particular value.
+/// * The nsw/nuw flag is/isn't set.
+class InstructionPredicateMatcher : public PredicateMatcher {
+public:
+  InstructionPredicateMatcher(PredicateKind Kind, unsigned InsnVarID)
+      : PredicateMatcher(Kind, InsnVarID) {}
+  virtual ~InstructionPredicateMatcher() {}
+
+  /// Compare the priority of this object and B.
+  ///
+  /// Returns true if this object is more important than B.
+  virtual bool
+  isHigherPriorityThan(const InstructionPredicateMatcher &B) const {
+    return Kind < B.Kind;
+  };
+};
+
+template <>
+inline std::string
+PredicateListMatcher<PredicateMatcher>::getNoPredicateComment() const {
+  return "No instruction predicates";
+}
+
+/// Generates code to check the opcode of an instruction.
+class InstructionOpcodeMatcher : public InstructionPredicateMatcher {
+protected:
+  // Allow matching one to several, similar opcodes that share properties. This
+  // is to handle patterns where one SelectionDAG operation maps to multiple
+  // GlobalISel ones (e.g. G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC). The first
+  // is treated as the canonical opcode.
+  SmallVector<const CodeGenInstruction *, 2> Insts;
+
+  static DenseMap<const CodeGenInstruction *, unsigned> OpcodeValues;
+
+  MatchTableRecord getInstValue(const CodeGenInstruction *I) const;
+
+public:
+  static void initOpcodeValuesMap(const CodeGenTarget &Target);
+
+  InstructionOpcodeMatcher(unsigned InsnVarID,
+                           ArrayRef<const CodeGenInstruction *> I)
+      : InstructionPredicateMatcher(IPM_Opcode, InsnVarID),
+        Insts(I.begin(), I.end()) {
+    assert((Insts.size() == 1 || Insts.size() == 2) &&
+           "unexpected number of opcode alternatives");
+  }
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_Opcode;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return InstructionPredicateMatcher::isIdentical(B) &&
+           Insts == cast<InstructionOpcodeMatcher>(&B)->Insts;
+  }
+
+  bool hasValue() const override {
+    return Insts.size() == 1 && OpcodeValues.count(Insts[0]);
+  }
+
+  // TODO: This is used for the SwitchMatcher optimization. We should be able to
+  // return a list of the opcodes to match.
+  MatchTableRecord getValue() const override;
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+
+  /// Compare the priority of this object and B.
+  ///
+  /// Returns true if this object is more important than B.
+  bool
+  isHigherPriorityThan(const InstructionPredicateMatcher &B) const override;
+
+  bool isConstantInstruction() const;
+
+  // The first opcode is the canonical opcode, and later are alternatives.
+  StringRef getOpcode() const;
+  ArrayRef<const CodeGenInstruction *> getAlternativeOpcodes() { return Insts; }
+  bool isVariadicNumOperands() const;
+  StringRef getOperandType(unsigned OpIdx) const;
+};
+
+class InstructionNumOperandsMatcher final : public InstructionPredicateMatcher {
+  unsigned NumOperands = 0;
+
+public:
+  InstructionNumOperandsMatcher(unsigned InsnVarID, unsigned NumOperands)
+      : InstructionPredicateMatcher(IPM_NumOperands, InsnVarID),
+        NumOperands(NumOperands) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_NumOperands;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return InstructionPredicateMatcher::isIdentical(B) &&
+           NumOperands == cast<InstructionNumOperandsMatcher>(&B)->NumOperands;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that this instruction is a constant whose value
+/// meets an immediate predicate.
+///
+/// Immediates are slightly odd since they are typically used like an operand
+/// but are represented as an operator internally. We typically write simm8:$src
+/// in a tablegen pattern, but this is just syntactic sugar for
+/// (imm:i32)<<P:Predicate_simm8>>:$imm which more directly describes the nodes
+/// that will be matched and the predicate (which is attached to the imm
+/// operator) that will be tested. In SelectionDAG this describes a
+/// ConstantSDNode whose internal value will be tested using the simm8
+/// predicate.
+///
+/// The corresponding GlobalISel representation is %1 = G_CONSTANT iN Value. In
+/// this representation, the immediate could be tested with an
+/// InstructionMatcher, InstructionOpcodeMatcher, OperandMatcher, and a
+/// OperandPredicateMatcher-subclass to check the Value meets the predicate but
+/// there are two implementation issues with producing that matcher
+/// configuration from the SelectionDAG pattern:
+/// * ImmLeaf is a PatFrag whose root is an InstructionMatcher. This means that
+///   were we to sink the immediate predicate to the operand we would have to
+///   have two partial implementations of PatFrag support, one for immediates
+///   and one for non-immediates.
+/// * At the point we handle the predicate, the OperandMatcher hasn't been
+///   created yet. If we were to sink the predicate to the OperandMatcher we
+///   would also have to complicate (or duplicate) the code that descends and
+///   creates matchers for the subtree.
+/// Overall, it's simpler to handle it in the place it was found.
+class InstructionImmPredicateMatcher : public InstructionPredicateMatcher {
+protected:
+  TreePredicateFn Predicate;
+
+public:
+  InstructionImmPredicateMatcher(unsigned InsnVarID,
+                                 const TreePredicateFn &Predicate)
+      : InstructionPredicateMatcher(IPM_ImmPredicate, InsnVarID),
+        Predicate(Predicate) {}
+
+  bool isIdentical(const PredicateMatcher &B) const override;
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_ImmPredicate;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that a memory instruction has a atomic ordering
+/// MachineMemoryOperand.
+class AtomicOrderingMMOPredicateMatcher : public InstructionPredicateMatcher {
+public:
+  enum AOComparator {
+    AO_Exactly,
+    AO_OrStronger,
+    AO_WeakerThan,
+  };
+
+protected:
+  StringRef Order;
+  AOComparator Comparator;
+
+public:
+  AtomicOrderingMMOPredicateMatcher(unsigned InsnVarID, StringRef Order,
+                                    AOComparator Comparator = AO_Exactly)
+      : InstructionPredicateMatcher(IPM_AtomicOrderingMMO, InsnVarID),
+        Order(Order), Comparator(Comparator) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_AtomicOrderingMMO;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override;
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that the size of an MMO is exactly N bytes.
+class MemorySizePredicateMatcher : public InstructionPredicateMatcher {
+protected:
+  unsigned MMOIdx;
+  uint64_t Size;
+
+public:
+  MemorySizePredicateMatcher(unsigned InsnVarID, unsigned MMOIdx, unsigned Size)
+      : InstructionPredicateMatcher(IPM_MemoryLLTSize, InsnVarID),
+        MMOIdx(MMOIdx), Size(Size) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_MemoryLLTSize;
+  }
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return InstructionPredicateMatcher::isIdentical(B) &&
+           MMOIdx == cast<MemorySizePredicateMatcher>(&B)->MMOIdx &&
+           Size == cast<MemorySizePredicateMatcher>(&B)->Size;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+class MemoryAddressSpacePredicateMatcher : public InstructionPredicateMatcher {
+protected:
+  unsigned MMOIdx;
+  SmallVector<unsigned, 4> AddrSpaces;
+
+public:
+  MemoryAddressSpacePredicateMatcher(unsigned InsnVarID, unsigned MMOIdx,
+                                     ArrayRef<unsigned> AddrSpaces)
+      : InstructionPredicateMatcher(IPM_MemoryAddressSpace, InsnVarID),
+        MMOIdx(MMOIdx), AddrSpaces(AddrSpaces.begin(), AddrSpaces.end()) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_MemoryAddressSpace;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override;
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+class MemoryAlignmentPredicateMatcher : public InstructionPredicateMatcher {
+protected:
+  unsigned MMOIdx;
+  int MinAlign;
+
+public:
+  MemoryAlignmentPredicateMatcher(unsigned InsnVarID, unsigned MMOIdx,
+                                  int MinAlign)
+      : InstructionPredicateMatcher(IPM_MemoryAlignment, InsnVarID),
+        MMOIdx(MMOIdx), MinAlign(MinAlign) {
+    assert(MinAlign > 0);
+  }
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_MemoryAlignment;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override;
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check that the size of an MMO is less-than, equal-to, or
+/// greater than a given LLT.
+class MemoryVsLLTSizePredicateMatcher : public InstructionPredicateMatcher {
+public:
+  enum RelationKind {
+    GreaterThan,
+    EqualTo,
+    LessThan,
+  };
+
+protected:
+  unsigned MMOIdx;
+  RelationKind Relation;
+  unsigned OpIdx;
+
+public:
+  MemoryVsLLTSizePredicateMatcher(unsigned InsnVarID, unsigned MMOIdx,
+                                  enum RelationKind Relation, unsigned OpIdx)
+      : InstructionPredicateMatcher(IPM_MemoryVsLLTSize, InsnVarID),
+        MMOIdx(MMOIdx), Relation(Relation), OpIdx(OpIdx) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_MemoryVsLLTSize;
+  }
+  bool isIdentical(const PredicateMatcher &B) const override;
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+// Matcher for immAllOnesV/immAllZerosV
+class VectorSplatImmPredicateMatcher : public InstructionPredicateMatcher {
+public:
+  enum SplatKind { AllZeros, AllOnes };
+
+private:
+  SplatKind Kind;
+
+public:
+  VectorSplatImmPredicateMatcher(unsigned InsnVarID, SplatKind K)
+      : InstructionPredicateMatcher(IPM_VectorSplatImm, InsnVarID), Kind(K) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_VectorSplatImm;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return InstructionPredicateMatcher::isIdentical(B) &&
+           Kind == static_cast<const VectorSplatImmPredicateMatcher &>(B).Kind;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check an arbitrary C++ instruction predicate.
+class GenericInstructionPredicateMatcher : public InstructionPredicateMatcher {
+protected:
+  std::string EnumVal;
+
+public:
+  GenericInstructionPredicateMatcher(unsigned InsnVarID,
+                                     TreePredicateFn Predicate);
+
+  GenericInstructionPredicateMatcher(unsigned InsnVarID,
+                                     const std::string &EnumVal)
+      : InstructionPredicateMatcher(IPM_GenericPredicate, InsnVarID),
+        EnumVal(EnumVal) {}
+
+  static bool classof(const InstructionPredicateMatcher *P) {
+    return P->getKind() == IPM_GenericPredicate;
+  }
+  bool isIdentical(const PredicateMatcher &B) const override;
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override;
+};
+
+/// Generates code to check for the absence of use of the result.
+// TODO? Generalize this to support checking for one use.
+class NoUsePredicateMatcher : public InstructionPredicateMatcher {
+public:
+  NoUsePredicateMatcher(unsigned InsnVarID)
+      : InstructionPredicateMatcher(IPM_NoUse, InsnVarID) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_NoUse;
+  }
+
+  bool isIdentical(const PredicateMatcher &B) const override {
+    return InstructionPredicateMatcher::isIdentical(B);
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override {
+    Table << MatchTable::Opcode("GIM_CheckHasNoUse")
+          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+          << MatchTable::LineBreak;
+  }
+};
+
+/// Generates code to check that a set of predicates and operands match for a
+/// particular instruction.
+///
+/// Typical predicates include:
+/// * Has a specific opcode.
+/// * Has an nsw/nuw flag or doesn't.
+class InstructionMatcher final : public PredicateListMatcher<PredicateMatcher> {
+protected:
+  typedef std::vector<std::unique_ptr<OperandMatcher>> OperandVec;
+
+  RuleMatcher &Rule;
+
+  /// The operands to match. All rendered operands must be present even if the
+  /// condition is always true.
+  OperandVec Operands;
+  bool NumOperandsCheck = true;
+
+  std::string SymbolicName;
+  unsigned InsnVarID;
+
+  /// PhysRegInputs - List list has an entry for each explicitly specified
+  /// physreg input to the pattern.  The first elt is the Register node, the
+  /// second is the recorded slot number the input pattern match saved it in.
+  SmallVector<std::pair<Record *, unsigned>, 2> PhysRegInputs;
+
+public:
+  InstructionMatcher(RuleMatcher &Rule, StringRef SymbolicName,
+                     bool NumOpsCheck = true)
+      : Rule(Rule), NumOperandsCheck(NumOpsCheck), SymbolicName(SymbolicName) {
+    // We create a new instruction matcher.
+    // Get a new ID for that instruction.
+    InsnVarID = Rule.implicitlyDefineInsnVar(*this);
+  }
+
+  /// Construct a new instruction predicate and add it to the matcher.
+  template <class Kind, class... Args>
+  std::optional<Kind *> addPredicate(Args &&...args) {
+    Predicates.emplace_back(
+        std::make_unique<Kind>(getInsnVarID(), std::forward<Args>(args)...));
+    return static_cast<Kind *>(Predicates.back().get());
+  }
+
+  RuleMatcher &getRuleMatcher() const { return Rule; }
+
+  unsigned getInsnVarID() const { return InsnVarID; }
+
+  /// Add an operand to the matcher.
+  OperandMatcher &addOperand(unsigned OpIdx, const std::string &SymbolicName,
+                             unsigned AllocatedTemporariesBaseID);
+  OperandMatcher &getOperand(unsigned OpIdx);
+  OperandMatcher &addPhysRegInput(Record *Reg, unsigned OpIdx,
+                                  unsigned TempOpIdx);
+
+  ArrayRef<std::pair<Record *, unsigned>> getPhysRegInputs() const {
+    return PhysRegInputs;
+  }
+
+  StringRef getSymbolicName() const { return SymbolicName; }
+  unsigned getNumOperands() const { return Operands.size(); }
+  OperandVec::iterator operands_begin() { return Operands.begin(); }
+  OperandVec::iterator operands_end() { return Operands.end(); }
+  iterator_range<OperandVec::iterator> operands() {
+    return make_range(operands_begin(), operands_end());
+  }
+  OperandVec::const_iterator operands_begin() const { return Operands.begin(); }
+  OperandVec::const_iterator operands_end() const { return Operands.end(); }
+  iterator_range<OperandVec::const_iterator> operands() const {
+    return make_range(operands_begin(), operands_end());
+  }
+  bool operands_empty() const { return Operands.empty(); }
+
+  void pop_front() { Operands.erase(Operands.begin()); }
+
+  void optimize();
+
+  /// Emit MatchTable opcodes that test whether the instruction named in
+  /// InsnVarName matches all the predicates and all the operands.
+  void emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule);
+
+  /// Compare the priority of this object and B.
+  ///
+  /// Returns true if this object is more important than B.
+  bool isHigherPriorityThan(InstructionMatcher &B);
+
+  /// Report the maximum number of temporary operands needed by the instruction
+  /// matcher.
+  unsigned countRendererFns();
+
+  InstructionOpcodeMatcher &getOpcodeMatcher() {
+    for (auto &P : predicates())
+      if (auto *OpMatcher = dyn_cast<InstructionOpcodeMatcher>(P.get()))
+        return *OpMatcher;
+    llvm_unreachable("Didn't find an opcode matcher");
+  }
+
+  bool isConstantInstruction() {
+    return getOpcodeMatcher().isConstantInstruction();
+  }
+
+  StringRef getOpcode() { return getOpcodeMatcher().getOpcode(); }
+};
+
+/// Generates code to check that the operand is a register defined by an
+/// instruction that matches the given instruction matcher.
+///
+/// For example, the pattern:
+///   (set $dst, (G_MUL (G_ADD $src1, $src2), $src3))
+/// would use an InstructionOperandMatcher for operand 1 of the G_MUL to match
+/// the:
+///   (G_ADD $src1, $src2)
+/// subpattern.
+class InstructionOperandMatcher : public OperandPredicateMatcher {
+protected:
+  std::unique_ptr<InstructionMatcher> InsnMatcher;
+
+  GISelFlags Flags;
+
+public:
+  InstructionOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                            RuleMatcher &Rule, StringRef SymbolicName,
+                            bool NumOpsCheck = true)
+      : OperandPredicateMatcher(OPM_Instruction, InsnVarID, OpIdx),
+        InsnMatcher(new InstructionMatcher(Rule, SymbolicName, NumOpsCheck)),
+        Flags(Rule.getGISelFlags()) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == OPM_Instruction;
+  }
+
+  InstructionMatcher &getInsnMatcher() const { return *InsnMatcher; }
+
+  void emitCaptureOpcodes(MatchTable &Table, RuleMatcher &Rule) const;
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override {
+    emitCaptureOpcodes(Table, Rule);
+    InsnMatcher->emitPredicateOpcodes(Table, Rule);
+  }
+
+  bool isHigherPriorityThan(const OperandPredicateMatcher &B) const override;
+
+  /// Report the maximum number of temporary operands needed by the predicate
+  /// matcher.
+  unsigned countRendererFns() const override {
+    return InsnMatcher->countRendererFns();
+  }
+};
+
+//===- Actions ------------------------------------------------------------===//
+class OperandRenderer {
+public:
+  enum RendererKind {
+    OR_Copy,
+    OR_CopyOrAddZeroReg,
+    OR_CopySubReg,
+    OR_CopyPhysReg,
+    OR_CopyConstantAsImm,
+    OR_CopyFConstantAsFPImm,
+    OR_Imm,
+    OR_SubRegIndex,
+    OR_Register,
+    OR_TempRegister,
+    OR_ComplexPattern,
+    OR_Custom,
+    OR_CustomOperand
+  };
+
+protected:
+  RendererKind Kind;
+
+public:
+  OperandRenderer(RendererKind Kind) : Kind(Kind) {}
+  virtual ~OperandRenderer();
+
+  RendererKind getKind() const { return Kind; }
+
+  virtual void emitRenderOpcodes(MatchTable &Table,
+                                 RuleMatcher &Rule) const = 0;
+};
+
+/// A CopyRenderer emits code to copy a single operand from an existing
+/// instruction to the one being built.
+class CopyRenderer : public OperandRenderer {
+protected:
+  unsigned NewInsnID;
+  /// The name of the operand.
+  const StringRef SymbolicName;
+
+public:
+  CopyRenderer(unsigned NewInsnID, StringRef SymbolicName)
+      : OperandRenderer(OR_Copy), NewInsnID(NewInsnID),
+        SymbolicName(SymbolicName) {
+    assert(!SymbolicName.empty() && "Cannot copy from an unspecified source");
+  }
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_Copy;
+  }
+
+  StringRef getSymbolicName() const { return SymbolicName; }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// A CopyRenderer emits code to copy a virtual register to a specific physical
+/// register.
+class CopyPhysRegRenderer : public OperandRenderer {
+protected:
+  unsigned NewInsnID;
+  Record *PhysReg;
+
+public:
+  CopyPhysRegRenderer(unsigned NewInsnID, Record *Reg)
+      : OperandRenderer(OR_CopyPhysReg), NewInsnID(NewInsnID), PhysReg(Reg) {
+    assert(PhysReg);
+  }
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_CopyPhysReg;
+  }
+
+  Record *getPhysReg() const { return PhysReg; }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// A CopyOrAddZeroRegRenderer emits code to copy a single operand from an
+/// existing instruction to the one being built. If the operand turns out to be
+/// a 'G_CONSTANT 0' then it replaces the operand with a zero register.
+class CopyOrAddZeroRegRenderer : public OperandRenderer {
+protected:
+  unsigned NewInsnID;
+  /// The name of the operand.
+  const StringRef SymbolicName;
+  const Record *ZeroRegisterDef;
+
+public:
+  CopyOrAddZeroRegRenderer(unsigned NewInsnID, StringRef SymbolicName,
+                           Record *ZeroRegisterDef)
+      : OperandRenderer(OR_CopyOrAddZeroReg), NewInsnID(NewInsnID),
+        SymbolicName(SymbolicName), ZeroRegisterDef(ZeroRegisterDef) {
+    assert(!SymbolicName.empty() && "Cannot copy from an unspecified source");
+  }
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_CopyOrAddZeroReg;
+  }
+
+  StringRef getSymbolicName() const { return SymbolicName; }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// A CopyConstantAsImmRenderer emits code to render a G_CONSTANT instruction to
+/// an extended immediate operand.
+class CopyConstantAsImmRenderer : public OperandRenderer {
+protected:
+  unsigned NewInsnID;
+  /// The name of the operand.
+  const std::string SymbolicName;
+  bool Signed;
+
+public:
+  CopyConstantAsImmRenderer(unsigned NewInsnID, StringRef SymbolicName)
+      : OperandRenderer(OR_CopyConstantAsImm), NewInsnID(NewInsnID),
+        SymbolicName(SymbolicName), Signed(true) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_CopyConstantAsImm;
+  }
+
+  StringRef getSymbolicName() const { return SymbolicName; }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// A CopyFConstantAsFPImmRenderer emits code to render a G_FCONSTANT
+/// instruction to an extended immediate operand.
+class CopyFConstantAsFPImmRenderer : public OperandRenderer {
+protected:
+  unsigned NewInsnID;
+  /// The name of the operand.
+  const std::string SymbolicName;
+
+public:
+  CopyFConstantAsFPImmRenderer(unsigned NewInsnID, StringRef SymbolicName)
+      : OperandRenderer(OR_CopyFConstantAsFPImm), NewInsnID(NewInsnID),
+        SymbolicName(SymbolicName) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_CopyFConstantAsFPImm;
+  }
+
+  StringRef getSymbolicName() const { return SymbolicName; }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// A CopySubRegRenderer emits code to copy a single register operand from an
+/// existing instruction to the one being built and indicate that only a
+/// subregister should be copied.
+class CopySubRegRenderer : public OperandRenderer {
+protected:
+  unsigned NewInsnID;
+  /// The name of the operand.
+  const StringRef SymbolicName;
+  /// The subregister to extract.
+  const CodeGenSubRegIndex *SubReg;
+
+public:
+  CopySubRegRenderer(unsigned NewInsnID, StringRef SymbolicName,
+                     const CodeGenSubRegIndex *SubReg)
+      : OperandRenderer(OR_CopySubReg), NewInsnID(NewInsnID),
+        SymbolicName(SymbolicName), SubReg(SubReg) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_CopySubReg;
+  }
+
+  StringRef getSymbolicName() const { return SymbolicName; }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// Adds a specific physical register to the instruction being built.
+/// This is typically useful for WZR/XZR on AArch64.
+class AddRegisterRenderer : public OperandRenderer {
+protected:
+  unsigned InsnID;
+  const Record *RegisterDef;
+  bool IsDef;
+  const CodeGenTarget &Target;
+
+public:
+  AddRegisterRenderer(unsigned InsnID, const CodeGenTarget &Target,
+                      const Record *RegisterDef, bool IsDef = false)
+      : OperandRenderer(OR_Register), InsnID(InsnID), RegisterDef(RegisterDef),
+        IsDef(IsDef), Target(Target) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_Register;
+  }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// Adds a specific temporary virtual register to the instruction being built.
+/// This is used to chain instructions together when emitting multiple
+/// instructions.
+class TempRegRenderer : public OperandRenderer {
+protected:
+  unsigned InsnID;
+  unsigned TempRegID;
+  const CodeGenSubRegIndex *SubRegIdx;
+  bool IsDef;
+  bool IsDead;
+
+public:
+  TempRegRenderer(unsigned InsnID, unsigned TempRegID, bool IsDef = false,
+                  const CodeGenSubRegIndex *SubReg = nullptr,
+                  bool IsDead = false)
+      : OperandRenderer(OR_Register), InsnID(InsnID), TempRegID(TempRegID),
+        SubRegIdx(SubReg), IsDef(IsDef), IsDead(IsDead) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_TempRegister;
+  }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// Adds a specific immediate to the instruction being built.
+class ImmRenderer : public OperandRenderer {
+protected:
+  unsigned InsnID;
+  int64_t Imm;
+
+public:
+  ImmRenderer(unsigned InsnID, int64_t Imm)
+      : OperandRenderer(OR_Imm), InsnID(InsnID), Imm(Imm) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_Imm;
+  }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
+    Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID")
+          << MatchTable::IntValue(InsnID) << MatchTable::Comment("Imm")
+          << MatchTable::IntValue(Imm) << MatchTable::LineBreak;
+  }
+};
+
+/// Adds an enum value for a subreg index to the instruction being built.
+class SubRegIndexRenderer : public OperandRenderer {
+protected:
+  unsigned InsnID;
+  const CodeGenSubRegIndex *SubRegIdx;
+
+public:
+  SubRegIndexRenderer(unsigned InsnID, const CodeGenSubRegIndex *SRI)
+      : OperandRenderer(OR_SubRegIndex), InsnID(InsnID), SubRegIdx(SRI) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_SubRegIndex;
+  }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// Adds operands by calling a renderer function supplied by the ComplexPattern
+/// matcher function.
+class RenderComplexPatternOperand : public OperandRenderer {
+private:
+  unsigned InsnID;
+  const Record &TheDef;
+  /// The name of the operand.
+  const StringRef SymbolicName;
+  /// The renderer number. This must be unique within a rule since it's used to
+  /// identify a temporary variable to hold the renderer function.
+  unsigned RendererID;
+  /// When provided, this is the suboperand of the ComplexPattern operand to
+  /// render. Otherwise all the suboperands will be rendered.
+  std::optional<unsigned> SubOperand;
+  /// The subregister to extract. Render the whole register if not specified.
+  const CodeGenSubRegIndex *SubReg;
+
+  unsigned getNumOperands() const {
+    return TheDef.getValueAsDag("Operands")->getNumArgs();
+  }
+
+public:
+  RenderComplexPatternOperand(unsigned InsnID, const Record &TheDef,
+                              StringRef SymbolicName, unsigned RendererID,
+                              std::optional<unsigned> SubOperand = std::nullopt,
+                              const CodeGenSubRegIndex *SubReg = nullptr)
+      : OperandRenderer(OR_ComplexPattern), InsnID(InsnID), TheDef(TheDef),
+        SymbolicName(SymbolicName), RendererID(RendererID),
+        SubOperand(SubOperand), SubReg(SubReg) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_ComplexPattern;
+  }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+class CustomRenderer : public OperandRenderer {
+protected:
+  unsigned InsnID;
+  const Record &Renderer;
+  /// The name of the operand.
+  const std::string SymbolicName;
+
+public:
+  CustomRenderer(unsigned InsnID, const Record &Renderer,
+                 StringRef SymbolicName)
+      : OperandRenderer(OR_Custom), InsnID(InsnID), Renderer(Renderer),
+        SymbolicName(SymbolicName) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_Custom;
+  }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+class CustomOperandRenderer : public OperandRenderer {
+protected:
+  unsigned InsnID;
+  const Record &Renderer;
+  /// The name of the operand.
+  const std::string SymbolicName;
+
+public:
+  CustomOperandRenderer(unsigned InsnID, const Record &Renderer,
+                        StringRef SymbolicName)
+      : OperandRenderer(OR_CustomOperand), InsnID(InsnID), Renderer(Renderer),
+        SymbolicName(SymbolicName) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_CustomOperand;
+  }
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// An action taken when all Matcher predicates succeeded for a parent rule.
+///
+/// Typical actions include:
+/// * Changing the opcode of an instruction.
+/// * Adding an operand to an instruction.
+class MatchAction {
+public:
+  virtual ~MatchAction() {}
+
+  /// Emit the MatchTable opcodes to implement the action.
+  virtual void emitActionOpcodes(MatchTable &Table,
+                                 RuleMatcher &Rule) const = 0;
+};
+
+/// Generates a comment describing the matched rule being acted upon.
+class DebugCommentAction : public MatchAction {
+private:
+  std::string S;
+
+public:
+  DebugCommentAction(StringRef S) : S(std::string(S)) {}
+
+  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
+    Table << MatchTable::Comment(S) << MatchTable::LineBreak;
+  }
+};
+
+class CustomCXXAction : public MatchAction {
+  std::string FnEnumName;
+
+public:
+  CustomCXXAction(StringRef FnEnumName) : FnEnumName(FnEnumName.str()) {}
+
+  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// Generates code to build an instruction or mutate an existing instruction
+/// into the desired instruction when this is possible.
+class BuildMIAction : public MatchAction {
+private:
+  unsigned InsnID;
+  const CodeGenInstruction *I;
+  InstructionMatcher *Matched;
+  std::vector<std::unique_ptr<OperandRenderer>> OperandRenderers;
+
+  /// True if the instruction can be built solely by mutating the opcode.
+  bool canMutate(RuleMatcher &Rule, const InstructionMatcher *Insn) const;
+
+public:
+  BuildMIAction(unsigned InsnID, const CodeGenInstruction *I)
+      : InsnID(InsnID), I(I), Matched(nullptr) {}
+
+  unsigned getInsnID() const { return InsnID; }
+  const CodeGenInstruction *getCGI() const { return I; }
+
+  void chooseInsnToMutate(RuleMatcher &Rule);
+
+  template <class Kind, class... Args> Kind &addRenderer(Args &&...args) {
+    OperandRenderers.emplace_back(
+        std::make_unique<Kind>(InsnID, std::forward<Args>(args)...));
+    return *static_cast<Kind *>(OperandRenderers.back().get());
+  }
+
+  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// Generates code to constrain the operands of an output instruction to the
+/// register classes specified by the definition of that instruction.
+class ConstrainOperandsToDefinitionAction : public MatchAction {
+  unsigned InsnID;
+
+public:
+  ConstrainOperandsToDefinitionAction(unsigned InsnID) : InsnID(InsnID) {}
+
+  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
+    Table << MatchTable::Opcode("GIR_ConstrainSelectedInstOperands")
+          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+          << MatchTable::LineBreak;
+  }
+};
+
+/// Generates code to constrain the specified operand of an output instruction
+/// to the specified register class.
+class ConstrainOperandToRegClassAction : public MatchAction {
+  unsigned InsnID;
+  unsigned OpIdx;
+  const CodeGenRegisterClass &RC;
+
+public:
+  ConstrainOperandToRegClassAction(unsigned InsnID, unsigned OpIdx,
+                                   const CodeGenRegisterClass &RC)
+      : InsnID(InsnID), OpIdx(OpIdx), RC(RC) {}
+
+  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+/// Generates code to create a temporary register which can be used to chain
+/// instructions together.
+class MakeTempRegisterAction : public MatchAction {
+private:
+  LLTCodeGen Ty;
+  unsigned TempRegID;
+
+public:
+  MakeTempRegisterAction(const LLTCodeGen &Ty, unsigned TempRegID)
+      : Ty(Ty), TempRegID(TempRegID) {
+    KnownTypes.insert(Ty);
+  }
+
+  void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
+};
+
+} // namespace gi
+} // namespace llvm
+
+#endif
diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp
new file mode 100644
index 000000000000..8dc422b140a5
--- /dev/null
+++ b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp
@@ -0,0 +1,267 @@
+//===- GlobalISelMatchTableExecutorEmitter.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "GlobalISelMatchTableExecutorEmitter.h"
+#include "GlobalISelMatchTable.h"
+
+using namespace llvm;
+using namespace llvm::gi;
+
+void GlobalISelMatchTableExecutorEmitter::emitSubtargetFeatureBitsetImpl(
+    raw_ostream &OS, ArrayRef<RuleMatcher> Rules) {
+  SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(SubtargetFeatures,
+                                                           OS);
+
+  // Separate subtarget features by how often they must be recomputed.
+  SubtargetFeatureInfoMap ModuleFeatures;
+  std::copy_if(SubtargetFeatures.begin(), SubtargetFeatures.end(),
+               std::inserter(ModuleFeatures, ModuleFeatures.end()),
+               [](const SubtargetFeatureInfoMap::value_type &X) {
+                 return !X.second.mustRecomputePerFunction();
+               });
+  SubtargetFeatureInfoMap FunctionFeatures;
+  std::copy_if(SubtargetFeatures.begin(), SubtargetFeatures.end(),
+               std::inserter(FunctionFeatures, FunctionFeatures.end()),
+               [](const SubtargetFeatureInfoMap::value_type &X) {
+                 return X.second.mustRecomputePerFunction();
+               });
+
+  SubtargetFeatureInfo::emitComputeAvailableFeatures(
+      getTarget().getName(), getClassName(), "computeAvailableModuleFeatures",
+      ModuleFeatures, OS);
+
+  OS << "void " << getClassName()
+     << "::setupGeneratedPerFunctionState(MachineFunction &MF) {\n"
+        "  AvailableFunctionFeatures = computeAvailableFunctionFeatures("
+        "(const "
+     << getTarget().getName()
+     << "Subtarget *)&MF.getSubtarget(), &MF);\n"
+        "}\n";
+
+  SubtargetFeatureInfo::emitComputeAvailableFeatures(
+      getTarget().getName(), getClassName(), "computeAvailableFunctionFeatures",
+      FunctionFeatures, OS, "const MachineFunction *MF");
+
+  // Emit a table containing the PredicateBitsets objects needed by the matcher
+  // and an enum for the matcher to reference them with.
+  std::vector<std::vector<Record *>> FeatureBitsets;
+  FeatureBitsets.reserve(Rules.size());
+  for (auto &Rule : Rules)
+    FeatureBitsets.push_back(Rule.getRequiredFeatures());
+  llvm::sort(FeatureBitsets, [&](const std::vector<Record *> &A,
+                                 const std::vector<Record *> &B) {
+    if (A.size() < B.size())
+      return true;
+    if (A.size() > B.size())
+      return false;
+    for (auto [First, Second] : zip(A, B)) {
+      if (First->getName() < Second->getName())
+        return true;
+      if (First->getName() > Second->getName())
+        return false;
+    }
+    return false;
+  });
+  FeatureBitsets.erase(
+      std::unique(FeatureBitsets.begin(), FeatureBitsets.end()),
+      FeatureBitsets.end());
+  OS << "// Feature bitsets.\n"
+     << "enum {\n"
+     << "  GIFBS_Invalid,\n";
+  for (const auto &FeatureBitset : FeatureBitsets) {
+    if (FeatureBitset.empty())
+      continue;
+    OS << "  " << getNameForFeatureBitset(FeatureBitset) << ",\n";
+  }
+  OS << "};\n"
+     << "const static PredicateBitset FeatureBitsets[] {\n"
+     << "  {}, // GIFBS_Invalid\n";
+  for (const auto &FeatureBitset : FeatureBitsets) {
+    if (FeatureBitset.empty())
+      continue;
+    OS << "  {";
+    for (const auto &Feature : FeatureBitset) {
+      const auto &I = SubtargetFeatures.find(Feature);
+      assert(I != SubtargetFeatures.end() && "Didn't import predicate?");
+      OS << I->second.getEnumBitName() << ", ";
+    }
+    OS << "},\n";
+  }
+  OS << "};\n\n";
+}
+
+void GlobalISelMatchTableExecutorEmitter::emitComplexPredicates(
+    raw_ostream &OS, ArrayRef<Record *> ComplexOperandMatchers) {
+  // Emit complex predicate table and an enum to reference them with.
+  OS << "// ComplexPattern predicates.\n"
+     << "enum {\n"
+     << "  GICP_Invalid,\n";
+  for (const auto &Record : ComplexOperandMatchers)
+    OS << "  GICP_" << Record->getName() << ",\n";
+  OS << "};\n"
+     << "// See constructor for table contents\n\n";
+
+  OS << getClassName() << "::ComplexMatcherMemFn\n"
+     << getClassName() << "::ComplexPredicateFns[] = {\n"
+     << "  nullptr, // GICP_Invalid\n";
+  for (const auto &Record : ComplexOperandMatchers)
+    OS << "  &" << getClassName()
+       << "::" << Record->getValueAsString("MatcherFn") << ", // "
+       << Record->getName() << "\n";
+  OS << "};\n\n";
+}
+
+void GlobalISelMatchTableExecutorEmitter::emitCustomOperandRenderers(
+    raw_ostream &OS, ArrayRef<StringRef> CustomOperandRenderers) {
+  OS << "// Custom renderers.\n"
+     << "enum {\n"
+     << "  GICR_Invalid,\n";
+  for (const auto &Fn : CustomOperandRenderers)
+    OS << "  GICR_" << Fn << ",\n";
+  OS << "};\n";
+
+  OS << getClassName() << "::CustomRendererFn\n"
+     << getClassName() << "::CustomRenderers[] = {\n"
+     << "  nullptr, // GICR_Invalid\n";
+  for (const auto &Fn : CustomOperandRenderers)
+    OS << "  &" << getClassName() << "::" << Fn << ",\n";
+  OS << "};\n\n";
+}
+
+void GlobalISelMatchTableExecutorEmitter::emitTypeObjects(
+    raw_ostream &OS, ArrayRef<LLTCodeGen> TypeObjects) {
+  OS << "// LLT Objects.\n"
+     << "enum {\n";
+  for (const auto &TypeObject : TypeObjects) {
+    OS << "  ";
+    TypeObject.emitCxxEnumValue(OS);
+    OS << ",\n";
+  }
+  OS << "};\n"
+     << "const static size_t NumTypeObjects = " << TypeObjects.size() << ";\n"
+     << "const static LLT TypeObjects[] = {\n";
+  for (const auto &TypeObject : TypeObjects) {
+    OS << "  ";
+    TypeObject.emitCxxConstructorCall(OS);
+    OS << ",\n";
+  }
+  OS << "};\n\n";
+}
+
+void GlobalISelMatchTableExecutorEmitter::emitMatchTable(
+    raw_ostream &OS, const MatchTable &Table) {
+  OS << "const int64_t *" << getClassName() << "::getMatchTable() const {\n";
+  Table.emitDeclaration(OS);
+  OS << "  return ";
+  Table.emitUse(OS);
+  OS << ";\n}\n";
+}
+
+void GlobalISelMatchTableExecutorEmitter::emitExecutorImpl(
+    raw_ostream &OS, const MatchTable &Table, ArrayRef<LLTCodeGen> TypeObjects,
+    ArrayRef<RuleMatcher> Rules, ArrayRef<Record *> ComplexOperandMatchers,
+    ArrayRef<StringRef> CustomOperandRenderers, StringRef IfDefName) {
+  OS << "#ifdef " << IfDefName << "\n";
+  emitTypeObjects(OS, TypeObjects);
+  emitSubtargetFeatureBitsetImpl(OS, Rules);
+  emitComplexPredicates(OS, ComplexOperandMatchers);
+  emitMIPredicateFns(OS);
+  emitI64ImmPredicateFns(OS);
+  emitAPFloatImmPredicateFns(OS);
+  emitAPIntImmPredicateFns(OS);
+  emitTestSimplePredicate(OS);
+  emitCustomOperandRenderers(OS, CustomOperandRenderers);
+  emitAdditionalImpl(OS);
+  emitRunCustomAction(OS);
+  emitMatchTable(OS, Table);
+  OS << "#endif // ifdef " << IfDefName << "\n\n";
+}
+
+void GlobalISelMatchTableExecutorEmitter::emitPredicateBitset(
+    raw_ostream &OS, StringRef IfDefName) {
+  OS << "#ifdef " << IfDefName << "\n"
+     << "const unsigned MAX_SUBTARGET_PREDICATES = " << SubtargetFeatures.size()
+     << ";\n"
+     << "using PredicateBitset = "
+        "llvm::PredicateBitsetImpl<MAX_SUBTARGET_PREDICATES>;\n"
+     << "#endif // ifdef " << IfDefName << "\n\n";
+}
+
+void GlobalISelMatchTableExecutorEmitter::emitTemporariesDecl(
+    raw_ostream &OS, StringRef IfDefName) {
+  OS << "#ifdef " << IfDefName << "\n"
+     << "  mutable MatcherState State;\n"
+     << "  typedef "
+        "ComplexRendererFns("
+     << getClassName() << "::*ComplexMatcherMemFn)(MachineOperand &) const;\n"
+
+     << "  typedef void(" << getClassName()
+     << "::*CustomRendererFn)(MachineInstrBuilder &, const "
+        "MachineInstr &, int) "
+        "const;\n"
+     << "  const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, "
+        "CustomRendererFn> "
+        "ExecInfo;\n"
+     << "  static " << getClassName()
+     << "::ComplexMatcherMemFn ComplexPredicateFns[];\n"
+     << "  static " << getClassName()
+     << "::CustomRendererFn CustomRenderers[];\n"
+     << "  bool testImmPredicate_I64(unsigned PredicateID, int64_t Imm) const "
+        "override;\n"
+     << "  bool testImmPredicate_APInt(unsigned PredicateID, const APInt &Imm) "
+        "const override;\n"
+     << "  bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat "
+        "&Imm) const override;\n"
+     << "  const int64_t *getMatchTable() const override;\n"
+     << "  bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI"
+        ", const MatcherState &State) "
+        "const override;\n"
+     << "  bool testSimplePredicate(unsigned PredicateID) const override;\n"
+     << "  void runCustomAction(unsigned FnID, const MatcherState &State) "
+        "const override;\n";
+  emitAdditionalTemporariesDecl(OS, "  ");
+  OS << "#endif // ifdef " << IfDefName << "\n\n";
+}
+
+void GlobalISelMatchTableExecutorEmitter::emitTemporariesInit(
+    raw_ostream &OS, unsigned MaxTemporaries, StringRef IfDefName) {
+  OS << "#ifdef " << IfDefName << "\n"
+     << ", State(" << MaxTemporaries << "),\n"
+     << "ExecInfo(TypeObjects, NumTypeObjects, FeatureBitsets"
+     << ", ComplexPredicateFns, CustomRenderers)\n"
+     << "#endif // ifdef " << IfDefName << "\n\n";
+
+  emitAdditionalTemporariesInit(OS);
+}
+
+void GlobalISelMatchTableExecutorEmitter::emitPredicatesDecl(
+    raw_ostream &OS, StringRef IfDefName) {
+  OS << "#ifdef " << IfDefName << "\n"
+     << "PredicateBitset AvailableModuleFeatures;\n"
+     << "mutable PredicateBitset AvailableFunctionFeatures;\n"
+     << "PredicateBitset getAvailableFeatures() const {\n"
+     << "  return AvailableModuleFeatures | AvailableFunctionFeatures;\n"
+     << "}\n"
+     << "PredicateBitset\n"
+     << "computeAvailableModuleFeatures(const " << getTarget().getName()
+     << "Subtarget *Subtarget) const;\n"
+     << "PredicateBitset\n"
+     << "computeAvailableFunctionFeatures(const " << getTarget().getName()
+     << "Subtarget *Subtarget,\n"
+     << "                                 const MachineFunction *MF) const;\n"
+     << "void setupGeneratedPerFunctionState(MachineFunction &MF) override;\n"
+     << "#endif // ifdef " << IfDefName << "\n";
+}
+
+void GlobalISelMatchTableExecutorEmitter::emitPredicatesInit(
+    raw_ostream &OS, StringRef IfDefName) {
+  OS << "#ifdef " << IfDefName << "\n"
+     << "AvailableModuleFeatures(computeAvailableModuleFeatures(&STI)),\n"
+     << "AvailableFunctionFeatures()\n"
+     << "#endif // ifdef " << IfDefName << "\n";
+}
diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h
new file mode 100644
index 000000000000..d526e08a96e3
--- /dev/null
+++ b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h
@@ -0,0 +1,228 @@
+//===- GlobalISelMatchTableExecutorEmitter.h ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file contains common code related to emitting
+/// GIMatchTableExecutor-derived classes.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H
+#define LLVM_UTILS_TABLEGEN_GLOBALISELMATCHTABLEEXECUTOREMITTER_H
+
+#include "SubtargetFeatureInfo.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include <functional>
+#include <vector>
+
+namespace llvm {
+class CodeGenTarget;
+
+namespace gi {
+class RuleMatcher;
+class LLTCodeGen;
+class MatchTable;
+} // namespace gi
+
+/// Abstract base class for TableGen backends that emit a
+/// `GIMatchTableExecutor`-derived class.
+class GlobalISelMatchTableExecutorEmitter {
+  /// Emits logic to check features required by \p Rules using the
+
+  /// SubtargetFeatures map.
+  void emitSubtargetFeatureBitsetImpl(raw_ostream &OS,
+                                      ArrayRef<gi::RuleMatcher> Rules);
+
+  /// Emits an enum + an array that stores references to
+  /// \p ComplexOperandMatchers.
+  void emitComplexPredicates(raw_ostream &OS,
+                             ArrayRef<Record *> ComplexOperandMatchers);
+
+  /// Emits an enum + an array that stores references to
+  /// \p CustomOperandRenderers.
+  void emitCustomOperandRenderers(raw_ostream &OS,
+                                  ArrayRef<StringRef> CustomOperandRenderers);
+
+  /// Emits an enum + an array to reference \p TypeObjects (LLTs) in the match
+  /// table.
+  void emitTypeObjects(raw_ostream &OS, ArrayRef<gi::LLTCodeGen> TypeObjects);
+
+  /// Emits the getMatchTable function which contains all of the match table's
+  /// opcodes.
+  void emitMatchTable(raw_ostream &OS, const gi::MatchTable &Table);
+
+  /// Helper function to emit `test` functions for the executor. This emits both
+  /// an enum to reference predicates in the MatchTable, and a function to
+  /// switch over the enum & execute the predicate's C++ code.
+  ///
+  /// \tparam PredicateObject An object representing a predicate to emit.
+  /// \param OS Output stream
+  /// \param TypeIdentifier Identifier used for the type of the predicate,
+  ///        e.g. `MI` for MachineInstrs.
+  /// \param ArgType Full type of the argument, e.g. `const MachineInstr &`
+  /// \param ArgName Name of the argument, e.g. `MI` for MachineInstrs.
+  /// \param AdditionalArgs Optional additional argument declarations.
+  /// \param AdditionalDeclarations Optional declarations to write at the start
+  ///        of the function, before switching over the predicates enum.
+  /// \param Predicates Predicates to emit.
+  /// \param GetPredEnumName Returns an enum name for a given predicate.
+  /// \param GetPredCode Returns the C++ code of a given predicate.
+  /// \param Comment Optional comment for the enum declaration.
+  template <typename PredicateObject>
+  void emitCxxPredicateFns(
+      raw_ostream &OS, StringRef TypeIdentifier, StringRef ArgType,
+      StringRef ArgName, StringRef AdditionalArgs,
+      StringRef AdditionalDeclarations, ArrayRef<PredicateObject> Predicates,
+      std::function<StringRef(PredicateObject)> GetPredEnumName,
+      std::function<StringRef(PredicateObject)> GetPredCode,
+      StringRef Comment) {
+    if (!Comment.empty())
+      OS << "// " << Comment << "\n";
+    if (!Predicates.empty()) {
+      OS << "enum {\n";
+      StringRef EnumeratorSeparator = " = GICXXPred_Invalid + 1,\n";
+      for (const auto &Pred : Predicates) {
+        OS << "  GICXXPred_" << TypeIdentifier << "_Predicate_"
+           << GetPredEnumName(Pred) << EnumeratorSeparator;
+        EnumeratorSeparator = ",\n";
+      }
+      OS << "};\n";
+    }
+
+    OS << "bool " << getClassName() << "::test" << ArgName << "Predicate_"
+       << TypeIdentifier << "(unsigned PredicateID, " << ArgType << " "
+       << ArgName << AdditionalArgs << ") const {\n"
+       << AdditionalDeclarations;
+    if (!AdditionalDeclarations.empty())
+      OS << "\n";
+    if (!Predicates.empty()) {
+      OS << "  switch (PredicateID) {\n";
+      for (const auto &Pred : Predicates) {
+        const auto Code = GetPredCode(Pred);
+        OS << "  case GICXXPred_" << TypeIdentifier << "_Predicate_"
+           << GetPredEnumName(Pred) << ": {\n"
+           << "    " << Code << "\n";
+        if (!StringRef(Code).ltrim().startswith("return")) {
+          OS << "    llvm_unreachable(\"" << GetPredEnumName(Pred)
+             << " should have returned\");\n";
+        }
+        OS << "  }\n";
+      }
+      OS << "  }\n";
+    }
+    OS << "  llvm_unreachable(\"Unknown predicate\");\n"
+       << "  return false;\n"
+       << "}\n";
+  }
+
+protected:
+  /// Emits `testMIPredicate_MI`.
+  /// \tparam PredicateObject An object representing a predicate to emit.
+  /// \param OS Output stream
+  /// \param AdditionalDecls Additional C++ variable declarations.
+  /// \param Predicates Predicates to emit.
+  /// \param GetPredEnumName Returns an enum name for a given predicate.
+  /// \param GetPredCode Returns the C++ code of a given predicate.
+  /// \param Comment Optional comment for the enum declaration.
+  template <typename PredicateObject>
+  void emitMIPredicateFnsImpl(
+      raw_ostream &OS, StringRef AdditionalDecls,
+      ArrayRef<PredicateObject> Predicates,
+      std::function<StringRef(PredicateObject)> GetPredEnumName,
+      std::function<StringRef(PredicateObject)> GetPredCode,
+      StringRef Comment = "") {
+    return emitCxxPredicateFns(
+        OS, "MI", "const MachineInstr &", "MI", ", const MatcherState &State",
+        AdditionalDecls, Predicates, GetPredEnumName, GetPredCode, Comment);
+  }
+
+  /// Helper function to emit the following executor functions:
+  ///   * testImmPredicate_I64      (TypeIdentifier=I64)
+  ///   * testImmPredicate_APInt    (TypeIdentifier=APInt)
+  ///   * testImmPredicate_APFloat  (TypeIdentifier=APFloat)
+  ///
+  /// \tparam PredicateObject An object representing a predicate to emit.
+  /// \param OS Output stream
+  /// \param TypeIdentifier Identifier used for the type of the predicate
+  /// \param ArgType Full type of the argument
+  /// \param Predicates Predicates to emit.
+  /// \param GetPredEnumName Returns an enum name for a given predicate.
+  /// \param GetPredCode Returns the C++ code of a given predicate.
+  /// \param Comment Optional comment for the enum declaration.
+  template <typename PredicateObject>
+  void emitImmPredicateFnsImpl(
+      raw_ostream &OS, StringRef TypeIdentifier, StringRef ArgType,
+      ArrayRef<PredicateObject> Predicates,
+      std::function<StringRef(PredicateObject)> GetPredEnumName,
+      std::function<StringRef(PredicateObject)> GetPredCode,
+      StringRef Comment = "") {
+    return emitCxxPredicateFns(OS, TypeIdentifier, ArgType, "Imm", "", "",
+                               Predicates, GetPredEnumName, GetPredCode,
+                               Comment);
+  }
+
+  GlobalISelMatchTableExecutorEmitter() = default;
+
+public:
+  virtual ~GlobalISelMatchTableExecutorEmitter() = default;
+
+  virtual const CodeGenTarget &getTarget() const = 0;
+
+  /// \returns the name of the class being emitted including any prefixes, e.g.
+  /// `AMDGPUInstructionSelector`.
+  virtual StringRef getClassName() const = 0;
+
+  /// Emit additional content in emitExecutorImpl
+  virtual void emitAdditionalImpl(raw_ostream &OS) {}
+
+  /// Emit additional content in emitTemporariesDecl.
+  virtual void emitAdditionalTemporariesDecl(raw_ostream &OS,
+                                             StringRef Indent) {}
+
+  /// Emit additional content in emitTemporariesInit.
+  virtual void emitAdditionalTemporariesInit(raw_ostream &OS) {}
+
+  /// Emit the `testMIPredicate_MI` function.
+  /// Note: `emitMIPredicateFnsImpl` can be used to do most of the work.
+  virtual void emitMIPredicateFns(raw_ostream &OS) = 0;
+
+  /// Emit the `testImmPredicate_I64` function.
+  /// Note: `emitImmPredicateFnsImpl` can be used to do most of the work.
+  virtual void emitI64ImmPredicateFns(raw_ostream &OS) = 0;
+
+  /// Emit the `testImmPredicate_APFloat` function.
+  /// Note: `emitImmPredicateFnsImpl` can be used to do most of the work.
+  virtual void emitAPFloatImmPredicateFns(raw_ostream &OS) = 0;
+
+  /// Emit the `testImmPredicate_APInt` function.
+  /// Note: `emitImmPredicateFnsImpl` can be used to do most of the work.
+  virtual void emitAPIntImmPredicateFns(raw_ostream &OS) = 0;
+  virtual void emitTestSimplePredicate(raw_ostream &OS) = 0;
+  virtual void emitRunCustomAction(raw_ostream &OS) = 0;
+
+  void emitExecutorImpl(raw_ostream &OS, const gi::MatchTable &Table,
+                        ArrayRef<gi::LLTCodeGen> TypeObjects,
+                        ArrayRef<gi::RuleMatcher> Rules,
+                        ArrayRef<Record *> ComplexOperandMatchers,
+                        ArrayRef<StringRef> CustomOperandRenderers,
+                        StringRef IfDefName);
+  void emitPredicateBitset(raw_ostream &OS, StringRef IfDefName);
+  void emitTemporariesDecl(raw_ostream &OS, StringRef IfDefName);
+  void emitTemporariesInit(raw_ostream &OS, unsigned MaxTemporaries,
+                           StringRef IfDefName);
+  void emitPredicatesDecl(raw_ostream &OS, StringRef IfDefName);
+  void emitPredicatesInit(raw_ostream &OS, StringRef IfDefName);
+
+  // Map of predicates to their subtarget features.
+  SubtargetFeatureInfoMap SubtargetFeatures;
+};
+} // namespace llvm
+
+#endif
diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/InfoByHwMode.cpp
index 73c4fbf0a5eb..4e9136e936af 100644
--- a/llvm/utils/TableGen/InfoByHwMode.cpp
+++ b/llvm/utils/TableGen/InfoByHwMode.cpp
@@ -17,7 +17,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-
+#include "llvm/TableGen/Record.h"
 #include <string>
 
 using namespace llvm;
@@ -65,8 +65,8 @@ MVT &ValueTypeByHwMode::getOrCreateTypeForMode(unsigned Mode, MVT Type) {
     return F->second;
   // If Mode is not in the map, look up the default mode. If it exists,
   // make a copy of it for Mode and return it.
-  auto D = Map.find(DefaultMode);
-  if (D != Map.end())
+  auto D = Map.begin();
+  if (D != Map.end() && D->first == DefaultMode)
     return Map.insert(std::make_pair(Mode, D->second)).first->second;
   // If default mode is not present either, use provided Type.
   return Map.insert(std::make_pair(Mode, Type)).first->second;
diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/InfoByHwMode.h
index 44927d0bf0df..b8a6645baca5 100644
--- a/llvm/utils/TableGen/InfoByHwMode.h
+++ b/llvm/utils/TableGen/InfoByHwMode.h
@@ -16,10 +16,16 @@
 
 #include "CodeGenHwModes.h"
 #include "llvm/ADT/SmallSet.h"
-#include "llvm/Support/MachineValueType.h"
-
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/MachineValueType.h"
+#include "llvm/Support/Compiler.h"
+#include <cassert>
+#include <limits>
 #include <map>
 #include <string>
+#include <tuple>
+#include <utility>
 
 namespace llvm {
 
@@ -38,18 +44,44 @@ template <typename InfoT>
 void union_modes(const InfoByHwMode<InfoT> &A,
                  const InfoByHwMode<InfoT> &B,
                  SmallVectorImpl<unsigned> &Modes) {
-  SmallSet<unsigned, 4> U;
-  for (const auto &P : A)
-    U.insert(P.first);
-  for (const auto &P : B)
-    U.insert(P.first);
-  // Make sure that the default mode is last on the list.
+  auto AI = A.begin();
+  auto BI = B.begin();
+
+  // Skip default mode, but remember if we had one.
   bool HasDefault = false;
-  for (unsigned M : U)
-    if (M != DefaultMode)
-      Modes.push_back(M);
-    else
-      HasDefault = true;
+  if (AI != A.end() && AI->first == DefaultMode) {
+    HasDefault = true;
+    ++AI;
+  }
+  if (BI != B.end() && BI->first == DefaultMode) {
+    HasDefault = true;
+    ++BI;
+  }
+
+  while (AI != A.end()) {
+    // If we're done with B, finish A.
+    if (BI == B.end()) {
+      for (; AI != A.end(); ++AI)
+        Modes.push_back(AI->first);
+      break;
+    }
+
+    if (BI->first < AI->first) {
+      Modes.push_back(BI->first);
+      ++BI;
+    } else {
+      Modes.push_back(AI->first);
+      if (AI->first == BI->first)
+        ++BI;
+      ++AI;
+    }
+  }
+
+  // Finish B.
+  for (; BI != B.end(); ++BI)
+    Modes.push_back(BI->first);
+
+  // Make sure that the default mode is last on the list.
   if (HasDefault)
     Modes.push_back(DefaultMode);
 }
@@ -78,20 +110,27 @@ struct InfoByHwMode {
   LLVM_ATTRIBUTE_ALWAYS_INLINE
   bool hasMode(unsigned M) const { return Map.find(M) != Map.end(); }
   LLVM_ATTRIBUTE_ALWAYS_INLINE
-  bool hasDefault() const { return hasMode(DefaultMode); }
+  bool hasDefault() const {
+    return !Map.empty() && Map.begin()->first == DefaultMode;
+  }
 
   InfoT &get(unsigned Mode) {
-    if (!hasMode(Mode)) {
-      assert(hasMode(DefaultMode));
-      Map.insert({Mode, Map.at(DefaultMode)});
-    }
-    return Map.at(Mode);
+    auto F = Map.find(Mode);
+    if (F != Map.end())
+      return F->second;
+
+    // Copy and insert the default mode which should be first.
+    assert(hasDefault());
+    auto P = Map.insert({Mode, Map.begin()->second});
+    return P.first->second;
   }
   const InfoT &get(unsigned Mode) const {
     auto F = Map.find(Mode);
-    if (Mode != DefaultMode && F == Map.end())
-      F = Map.find(DefaultMode);
-    assert(F != Map.end());
+    if (F != Map.end())
+      return F->second;
+    // Get the default mode which should be first.
+    F = Map.begin();
+    assert(F != Map.end() && F->first == DefaultMode);
     return F->second;
   }
 
@@ -100,7 +139,7 @@ struct InfoByHwMode {
     return Map.size() == 1 && Map.begin()->first == DefaultMode;
   }
   LLVM_ATTRIBUTE_ALWAYS_INLINE
-  InfoT getSimple() const {
+  const InfoT &getSimple() const {
     assert(isSimple());
     return Map.begin()->second;
   }
diff --git a/llvm/utils/TableGen/InstrDocsEmitter.cpp b/llvm/utils/TableGen/InstrDocsEmitter.cpp
index bc391227edd1..616e7b589288 100644
--- a/llvm/utils/TableGen/InstrDocsEmitter.cpp
+++ b/llvm/utils/TableGen/InstrDocsEmitter.cpp
@@ -21,25 +21,24 @@
 #include "CodeGenDAGPatterns.h"
 #include "CodeGenInstruction.h"
 #include "CodeGenTarget.h"
-#include "TableGenBackends.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <string>
 #include <vector>
 
 using namespace llvm;
 
-namespace llvm {
-
-void writeTitle(StringRef Str, raw_ostream &OS, char Kind = '-') {
-  OS << std::string(Str.size(), Kind) << "\n" << Str << "\n"
+static void writeTitle(StringRef Str, raw_ostream &OS, char Kind = '-') {
+  OS << std::string(Str.size(), Kind) << "\n"
+     << Str << "\n"
      << std::string(Str.size(), Kind) << "\n";
 }
 
-void writeHeader(StringRef Str, raw_ostream &OS, char Kind = '-') {
+static void writeHeader(StringRef Str, raw_ostream &OS, char Kind = '-') {
   OS << Str << "\n" << std::string(Str.size(), Kind) << "\n";
 }
 
-std::string escapeForRST(StringRef Str) {
+static std::string escapeForRST(StringRef Str) {
   std::string Result;
   Result.reserve(Str.size() + 4);
   for (char C : Str) {
@@ -55,7 +54,7 @@ std::string escapeForRST(StringRef Str) {
   return Result;
 }
 
-void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS) {
+static void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS) {
   CodeGenDAGPatterns CDP(RK);
   CodeGenTarget &Target = CDP.getTargetInfo();
   unsigned VariantCount = Target.getAsmParserVariantCount();
@@ -216,4 +215,5 @@ void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS) {
   }
 }
 
-} // end namespace llvm
+static TableGen::Emitter::Opt X("gen-instr-docs", EmitInstrDocs,
+                                "Generate instruction documentation");
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index 564c3ed64e26..cab9ecd4ea97 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -61,7 +61,9 @@ public:
 private:
   void emitEnums(raw_ostream &OS);
 
-  typedef std::map<std::vector<std::string>, unsigned> OperandInfoMapTy;
+  typedef std::vector<std::string> OperandInfoTy;
+  typedef std::vector<OperandInfoTy> OperandInfoListTy;
+  typedef std::map<OperandInfoTy, unsigned> OperandInfoMapTy;
 
   /// The keys of this map are maps which have OpName enum values as their keys
   /// and instruction operand indices as their values.  The values of this map
@@ -86,9 +88,8 @@ private:
   void emitFeatureVerifier(raw_ostream &OS, const CodeGenTarget &Target);
   void emitRecord(const CodeGenInstruction &Inst, unsigned Num,
                   Record *InstrInfo,
-                  std::map<std::vector<Record*>, unsigned> &EL,
-                  const OperandInfoMapTy &OpInfo,
-                  raw_ostream &OS);
+                  std::map<std::vector<Record *>, unsigned> &EL,
+                  const OperandInfoMapTy &OperandInfo, raw_ostream &OS);
   void emitOperandTypeMappings(
       raw_ostream &OS, const CodeGenTarget &Target,
       ArrayRef<const CodeGenInstruction *> NumberedInstructions);
@@ -108,27 +109,21 @@ private:
       ArrayRef<const CodeGenInstruction *> NumberedInstructions);
 
   // Operand information.
-  void EmitOperandInfo(raw_ostream &OS, OperandInfoMapTy &OperandInfoIDs);
-  std::vector<std::string> GetOperandInfo(const CodeGenInstruction &Inst);
+  unsigned CollectOperandInfo(OperandInfoListTy &OperandInfoList,
+                              OperandInfoMapTy &OperandInfoMap);
+  void EmitOperandInfo(raw_ostream &OS, OperandInfoListTy &OperandInfoList);
+  OperandInfoTy GetOperandInfo(const CodeGenInstruction &Inst);
 };
 
 } // end anonymous namespace
 
-static void PrintDefList(const std::vector<Record*> &Uses,
-                         unsigned Num, raw_ostream &OS) {
-  OS << "static const MCPhysReg ImplicitList" << Num << "[] = { ";
-  for (auto [Idx, U] : enumerate(Uses))
-    OS << (Idx ? ", " : "") << getQualifiedName(U);
-  OS << " };\n";
-}
-
 //===----------------------------------------------------------------------===//
 // Operand Info Emission.
 //===----------------------------------------------------------------------===//
 
-std::vector<std::string>
+InstrInfoEmitter::OperandInfoTy
 InstrInfoEmitter::GetOperandInfo(const CodeGenInstruction &Inst) {
-  std::vector<std::string> Result;
+  OperandInfoTy Result;
 
   for (auto &Op : Inst.Operands) {
     // Handle aggregate operands and normal operands the same way by expanding
@@ -215,24 +210,30 @@ InstrInfoEmitter::GetOperandInfo(const CodeGenInstruction &Inst) {
   return Result;
 }
 
-void InstrInfoEmitter::EmitOperandInfo(raw_ostream &OS,
-                                       OperandInfoMapTy &OperandInfoIDs) {
-  // ID #0 is for no operand info.
-  unsigned OperandListNum = 0;
-  OperandInfoIDs[std::vector<std::string>()] = ++OperandListNum;
-
-  OS << "\n";
+unsigned
+InstrInfoEmitter::CollectOperandInfo(OperandInfoListTy &OperandInfoList,
+                                     OperandInfoMapTy &OperandInfoMap) {
   const CodeGenTarget &Target = CDP.getTargetInfo();
+  unsigned Offset = 0;
   for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
-    std::vector<std::string> OperandInfo = GetOperandInfo(*Inst);
-    unsigned &N = OperandInfoIDs[OperandInfo];
-    if (N != 0) continue;
+    OperandInfoTy OperandInfo = GetOperandInfo(*Inst);
+    if (OperandInfoMap.insert({OperandInfo, Offset}).second) {
+      OperandInfoList.push_back(OperandInfo);
+      Offset += OperandInfo.size();
+    }
+  }
+  return Offset;
+}
 
-    N = ++OperandListNum;
-    OS << "static const MCOperandInfo OperandInfo" << N << "[] = { ";
-    for (const std::string &Info : OperandInfo)
-      OS << "{ " << Info << " }, ";
-    OS << "};\n";
+void InstrInfoEmitter::EmitOperandInfo(raw_ostream &OS,
+                                       OperandInfoListTy &OperandInfoList) {
+  unsigned Offset = 0;
+  for (auto &OperandInfo : OperandInfoList) {
+    OS << "    /* " << Offset << " */";
+    for (auto &Info : OperandInfo)
+      OS << " { " << Info << " },";
+    OS << '\n';
+    Offset += OperandInfo.size();
   }
 }
 
@@ -419,8 +420,7 @@ void InstrInfoEmitter::emitOperandTypeMappings(
     // Size the unsigned integer offset to save space.
     assert(OperandRecords.size() <= UINT32_MAX &&
            "Too many operands for offset table");
-    OS << ((OperandRecords.size() <= UINT16_MAX) ? "  const uint16_t"
-                                                 : "  const uint32_t");
+    OS << "  static const " << getMinimalTypeForRange(OperandRecords.size());
     OS << " Offsets[] = {\n";
     for (int I = 0, E = OperandOffsets.size(); I != E; ++I) {
       OS << "    /* " << getInstrName(I) << " */\n";
@@ -436,7 +436,8 @@ void InstrInfoEmitter::emitOperandTypeMappings(
     assert(EnumVal <= INT16_MAX &&
            "Too many operand types for operand types table");
     OS << "\n  using namespace OpTypes;\n";
-    OS << ((EnumVal <= INT8_MAX) ? "  const int8_t" : "  const int16_t");
+    OS << "  static";
+    OS << ((EnumVal <= INT8_MAX) ? " const int8_t" : " const int16_t");
     OS << " OpcodeOperandTypes[] = {\n    ";
     for (int I = 0, E = OperandRecords.size(), CurOffset = 0; I != E; ++I) {
       // We print each Opcode's operands in its own row.
@@ -732,23 +733,19 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS,
   std::map<Record *, SubtargetFeatureInfo, LessRecordByID> SubtargetFeatures;
   SubtargetFeatures.insert(All.begin(), All.end());
 
-  OS << "#ifdef ENABLE_INSTR_PREDICATE_VERIFIER\n"
-     << "#undef ENABLE_INSTR_PREDICATE_VERIFIER\n"
-     << "#include <sstream>\n\n";
-
-  OS << "namespace llvm {\n";
-  OS << "namespace " << Target.getName() << "_MC {\n\n";
+  OS << "#if defined(ENABLE_INSTR_PREDICATE_VERIFIER) && !defined(NDEBUG)\n"
+     << "#define GET_COMPUTE_FEATURES\n"
+     << "#endif\n";
+  OS << "#ifdef GET_COMPUTE_FEATURES\n"
+     << "#undef GET_COMPUTE_FEATURES\n"
+     << "namespace llvm {\n"
+     << "namespace " << Target.getName() << "_MC {\n\n";
 
   // Emit the subtarget feature enumeration.
   SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(SubtargetFeatures,
                                                            OS);
-
-  // Emit the name table for error messages.
-  OS << "#ifndef NDEBUG\n";
-  SubtargetFeatureInfo::emitNameTable(SubtargetFeatures, OS);
-  OS << "#endif // NDEBUG\n\n";
-
   // Emit the available features compute function.
+  OS << "inline ";
   SubtargetFeatureInfo::emitComputeAssemblerAvailableFeatures(
       Target.getName(), "", "computeAvailableFeatures", SubtargetFeatures, OS);
 
@@ -779,22 +776,21 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS,
   FeatureBitsets.erase(
       std::unique(FeatureBitsets.begin(), FeatureBitsets.end()),
       FeatureBitsets.end());
-  OS << "#ifndef NDEBUG\n"
-     << "// Feature bitsets.\n"
-     << "enum : " << getMinimalTypeForRange(FeatureBitsets.size()) << " {\n"
-     << "  CEFBS_None,\n";
+  OS << "inline FeatureBitset computeRequiredFeatures(unsigned Opcode) {\n"
+     << "  enum : " << getMinimalTypeForRange(FeatureBitsets.size()) << " {\n"
+     << "    CEFBS_None,\n";
   for (const auto &FeatureBitset : FeatureBitsets) {
     if (FeatureBitset.empty())
       continue;
-    OS << "  " << getNameForFeatureBitset(FeatureBitset) << ",\n";
+    OS << "    " << getNameForFeatureBitset(FeatureBitset) << ",\n";
   }
-  OS << "};\n\n"
-     << "static constexpr FeatureBitset FeatureBitsets[] = {\n"
-     << "  {}, // CEFBS_None\n";
+  OS << "  };\n\n"
+     << "  static constexpr FeatureBitset FeatureBitsets[] = {\n"
+     << "    {}, // CEFBS_None\n";
   for (const auto &FeatureBitset : FeatureBitsets) {
     if (FeatureBitset.empty())
       continue;
-    OS << "  {";
+    OS << "    {";
     for (const auto &Feature : FeatureBitset) {
       const auto &I = SubtargetFeatures.find(Feature);
       assert(I != SubtargetFeatures.end() && "Didn't import predicate?");
@@ -802,13 +798,7 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS,
     }
     OS << "},\n";
   }
-  OS << "};\n"
-     << "#endif // NDEBUG\n\n";
-
-  // Emit the predicate verifier.
-  OS << "void verifyInstructionPredicates(\n"
-     << "    unsigned Opcode, const FeatureBitset &Features) {\n"
-     << "#ifndef NDEBUG\n"
+  OS << "  };\n"
      << "  static " << getMinimalTypeForRange(FeatureBitsets.size())
      << " RequiredFeaturesRefs[] = {\n";
   unsigned InstIdx = 0;
@@ -827,12 +817,35 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS,
     OS << ", // " << Inst->TheDef->getName() << " = " << InstIdx << "\n";
     InstIdx++;
   }
-  OS << "  };\n\n";
-  OS << "  assert(Opcode < " << InstIdx << ");\n";
+  OS << "  };\n\n"
+     << "  assert(Opcode < " << InstIdx << ");\n"
+     << "  return FeatureBitsets[RequiredFeaturesRefs[Opcode]];\n"
+     << "}\n\n";
+
+  OS << "} // end namespace " << Target.getName() << "_MC\n"
+     << "} // end namespace llvm\n"
+     << "#endif // GET_COMPUTE_FEATURES\n\n";
+
+  OS << "#ifdef ENABLE_INSTR_PREDICATE_VERIFIER\n"
+     << "#undef ENABLE_INSTR_PREDICATE_VERIFIER\n"
+     << "#include <sstream>\n\n";
+
+  OS << "namespace llvm {\n";
+  OS << "namespace " << Target.getName() << "_MC {\n\n";
+
+  // Emit the name table for error messages.
+  OS << "#ifndef NDEBUG\n";
+  SubtargetFeatureInfo::emitNameTable(SubtargetFeatures, OS);
+  OS << "#endif // NDEBUG\n\n";
+
+  // Emit the predicate verifier.
+  OS << "void verifyInstructionPredicates(\n"
+     << "    unsigned Opcode, const FeatureBitset &Features) {\n"
+     << "#ifndef NDEBUG\n";
   OS << "  FeatureBitset AvailableFeatures = "
         "computeAvailableFeatures(Features);\n";
-  OS << "  const FeatureBitset &RequiredFeatures = "
-        "FeatureBitsets[RequiredFeaturesRefs[Opcode]];\n";
+  OS << "  FeatureBitset RequiredFeatures = "
+     << "computeRequiredFeatures(Opcode);\n";
   OS << "  FeatureBitset MissingFeatures =\n"
      << "      (AvailableFeatures & RequiredFeatures) ^\n"
      << "      RequiredFeatures;\n"
@@ -891,54 +904,90 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   emitSourceFileHeader("Target Instruction Enum Values and Descriptors", OS);
   emitEnums(OS);
 
-  OS << "#ifdef GET_INSTRINFO_MC_DESC\n";
-  OS << "#undef GET_INSTRINFO_MC_DESC\n";
-
-  OS << "namespace llvm {\n\n";
-
   CodeGenTarget &Target = CDP.getTargetInfo();
   const std::string &TargetName = std::string(Target.getName());
   Record *InstrInfo = Target.getInstructionSet();
 
-  // Keep track of all of the def lists we have emitted already.
-  std::map<std::vector<Record*>, unsigned> EmittedLists;
-  unsigned ListNumber = 0;
+  // Collect all of the operand info records.
+  Records.startTimer("Collect operand info");
+  OperandInfoListTy OperandInfoList;
+  OperandInfoMapTy OperandInfoMap;
+  unsigned OperandInfoSize =
+      CollectOperandInfo(OperandInfoList, OperandInfoMap);
 
-  // Emit all of the instruction's implicit uses and defs.
-  Records.startTimer("Emit uses/defs");
+  // Collect all of the instruction's implicit uses and defs.
+  Records.startTimer("Collect uses/defs");
+  std::map<std::vector<Record*>, unsigned> EmittedLists;
+  std::vector<std::vector<Record *>> ImplicitLists;
+  unsigned ImplicitListSize = 0;
   for (const CodeGenInstruction *II : Target.getInstructionsByEnumValue()) {
     std::vector<Record *> ImplicitOps = II->ImplicitUses;
     llvm::append_range(ImplicitOps, II->ImplicitDefs);
-    if (!ImplicitOps.empty()) {
-      unsigned &IL = EmittedLists[ImplicitOps];
-      if (!IL) {
-        IL = ++ListNumber;
-        PrintDefList(ImplicitOps, IL, OS);
-      }
+    if (EmittedLists.insert({ImplicitOps, ImplicitListSize}).second) {
+      ImplicitLists.push_back(ImplicitOps);
+      ImplicitListSize += ImplicitOps.size();
     }
   }
 
-  OperandInfoMapTy OperandInfoIDs;
+  ArrayRef<const CodeGenInstruction *> NumberedInstructions =
+      Target.getInstructionsByEnumValue();
+  OS << "#if defined(GET_INSTRINFO_MC_DESC) || "
+        "defined(GET_INSTRINFO_CTOR_DTOR)\n";
+  OS << "namespace llvm {\n\n";
 
-  // Emit all of the operand info records.
-  Records.startTimer("Emit operand info");
-  EmitOperandInfo(OS, OperandInfoIDs);
+  OS << "struct " << TargetName << "InstrTable {\n";
+  OS << "  MCInstrDesc Insts[" << NumberedInstructions.size() << "];\n";
+  OS << "  static_assert(alignof(MCInstrDesc) >= alignof(MCOperandInfo), "
+        "\"Unwanted padding between Insts and OperandInfo\");\n";
+  OS << "  MCOperandInfo OperandInfo[" << OperandInfoSize << "];\n";
+  OS << "  static_assert(alignof(MCOperandInfo) >= alignof(MCPhysReg), "
+        "\"Unwanted padding between OperandInfo and ImplicitOps\");\n";
+  OS << "  MCPhysReg ImplicitOps[" << std::max(ImplicitListSize, 1U) << "];\n";
+  OS << "};\n\n";
+
+  OS << "} // end namespace llvm\n";
+  OS << "#endif // defined(GET_INSTRINFO_MC_DESC) || "
+        "defined(GET_INSTRINFO_CTOR_DTOR)\n\n";
+
+  OS << "#ifdef GET_INSTRINFO_MC_DESC\n";
+  OS << "#undef GET_INSTRINFO_MC_DESC\n";
+  OS << "namespace llvm {\n\n";
 
   // Emit all of the MCInstrDesc records in reverse ENUM ordering.
   Records.startTimer("Emit InstrDesc records");
-  OS << "\nextern const MCInstrDesc " << TargetName << "Insts[] = {\n";
-  ArrayRef<const CodeGenInstruction*> NumberedInstructions =
-    Target.getInstructionsByEnumValue();
+  OS << "static_assert(sizeof(MCOperandInfo) % sizeof(MCPhysReg) == 0);\n";
+  OS << "static constexpr unsigned " << TargetName << "ImpOpBase = sizeof "
+     << TargetName << "InstrTable::OperandInfo / (sizeof(MCPhysReg));\n\n";
 
+  OS << "extern const " << TargetName << "InstrTable " << TargetName
+     << "Descs = {\n  {\n";
   SequenceToOffsetTable<std::string> InstrNames;
   unsigned Num = NumberedInstructions.size();
   for (const CodeGenInstruction *Inst : reverse(NumberedInstructions)) {
     // Keep a list of the instruction names.
     InstrNames.add(std::string(Inst->TheDef->getName()));
     // Emit the record into the table.
-    emitRecord(*Inst, --Num, InstrInfo, EmittedLists, OperandInfoIDs, OS);
+    emitRecord(*Inst, --Num, InstrInfo, EmittedLists, OperandInfoMap, OS);
   }
-  OS << "};\n\n";
+
+  OS << "  }, {\n";
+
+  // Emit all of the operand info records.
+  Records.startTimer("Emit operand info");
+  EmitOperandInfo(OS, OperandInfoList);
+
+  OS << "  }, {\n";
+
+  // Emit all of the instruction's implicit uses and defs.
+  Records.startTimer("Emit uses/defs");
+  for (auto &List : ImplicitLists) {
+    OS << "    /* " << EmittedLists[List] << " */";
+    for (auto &Reg : List)
+      OS << ' ' << getQualifiedName(Reg) << ',';
+    OS << '\n';
+  }
+
+  OS << "  }\n};\n\n";
 
   // Emit the array of instruction names.
   Records.startTimer("Emit instruction names");
@@ -1005,7 +1054,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   Records.startTimer("Emit initialization routine");
   OS << "static inline void Init" << TargetName
      << "MCInstrInfo(MCInstrInfo *II) {\n";
-  OS << "  II->InitMCInstrInfo(" << TargetName << "Insts, " << TargetName
+  OS << "  II->InitMCInstrInfo(" << TargetName << "Descs.Insts, " << TargetName
      << "InstrNameIndices, " << TargetName << "InstrNameData, ";
   if (HasDeprecationFeatures)
     OS << TargetName << "InstrDeprecationFeatures, ";
@@ -1053,7 +1102,8 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   OS << "#undef GET_INSTRINFO_CTOR_DTOR\n";
 
   OS << "namespace llvm {\n";
-  OS << "extern const MCInstrDesc " << TargetName << "Insts[];\n";
+  OS << "extern const " << TargetName << "InstrTable " << TargetName
+     << "Descs;\n";
   OS << "extern const unsigned " << TargetName << "InstrNameIndices[];\n";
   OS << "extern const char " << TargetName << "InstrNameData[];\n";
   if (HasDeprecationFeatures)
@@ -1067,7 +1117,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
         "CatchRetOpcode, unsigned ReturnOpcode)\n"
      << "  : TargetInstrInfo(CFSetupOpcode, CFDestroyOpcode, CatchRetOpcode, "
         "ReturnOpcode) {\n"
-     << "  InitMCInstrInfo(" << TargetName << "Insts, " << TargetName
+     << "  InitMCInstrInfo(" << TargetName << "Descs.Insts, " << TargetName
      << "InstrNameIndices, " << TargetName << "InstrNameData, ";
   if (HasDeprecationFeatures)
     OS << TargetName << "InstrDeprecationFeatures, ";
@@ -1101,27 +1151,34 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   emitFeatureVerifier(OS, Target);
 }
 
-void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num,
-                                  Record *InstrInfo,
-                         std::map<std::vector<Record*>, unsigned> &EmittedLists,
-                                  const OperandInfoMapTy &OpInfo,
-                                  raw_ostream &OS) {
+void InstrInfoEmitter::emitRecord(
+    const CodeGenInstruction &Inst, unsigned Num, Record *InstrInfo,
+    std::map<std::vector<Record *>, unsigned> &EmittedLists,
+    const OperandInfoMapTy &OperandInfoMap, raw_ostream &OS) {
   int MinOperands = 0;
   if (!Inst.Operands.empty())
     // Each logical operand can be multiple MI operands.
     MinOperands = Inst.Operands.back().MIOperandNo +
                   Inst.Operands.back().MINumOperands;
 
-  OS << "  { ";
-  OS << Num << ",\t" << MinOperands << ",\t"
-     << Inst.Operands.NumDefs << ",\t"
+  OS << "    { ";
+  OS << Num << ",\t" << MinOperands << ",\t" << Inst.Operands.NumDefs << ",\t"
      << Inst.TheDef->getValueAsInt("Size") << ",\t"
-     << SchedModels.getSchedClassIdx(Inst) << ",\t"
-     << Inst.ImplicitUses.size() << ",\t"
-     << Inst.ImplicitDefs.size() << ",\t0";
+     << SchedModels.getSchedClassIdx(Inst) << ",\t";
 
   CodeGenTarget &Target = CDP.getTargetInfo();
 
+  // Emit the implicit use/def list...
+  OS << Inst.ImplicitUses.size() << ",\t" << Inst.ImplicitDefs.size() << ",\t";
+  std::vector<Record *> ImplicitOps = Inst.ImplicitUses;
+  llvm::append_range(ImplicitOps, Inst.ImplicitDefs);
+  OS << Target.getName() << "ImpOpBase + " << EmittedLists[ImplicitOps]
+     << ",\t";
+
+  // Emit the operand info offset.
+  OperandInfoTy OperandInfo = GetOperandInfo(Inst);
+  OS << OperandInfoMap.find(OperandInfo)->second << ",\t0";
+
   // Emit all of the target independent flags...
   if (Inst.isPreISelOpcode)    OS << "|(1ULL<<MCID::PreISelOpcode)";
   if (Inst.isPseudo)           OS << "|(1ULL<<MCID::Pseudo)";
@@ -1181,22 +1238,7 @@ void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num,
   }
   OS << ", 0x";
   OS.write_hex(Value);
-  OS << "ULL, ";
-
-  // Emit the implicit use/def list...
-  std::vector<Record *> ImplicitOps = Inst.ImplicitUses;
-  llvm::append_range(ImplicitOps, Inst.ImplicitDefs);
-  if (ImplicitOps.empty())
-    OS << "nullptr, ";
-  else
-    OS << "ImplicitList" << EmittedLists[ImplicitOps] << ", ";
-
-  // Emit the operand info.
-  std::vector<std::string> OperandInfo = GetOperandInfo(Inst);
-  if (OperandInfo.empty())
-    OS << "nullptr";
-  else
-    OS << "OperandInfo" << OpInfo.find(OperandInfo)->second;
+  OS << "ULL";
 
   OS << " },  // Inst #" << Num << " = " << Inst.TheDef->getName() << "\n";
 }
@@ -1245,13 +1287,12 @@ void InstrInfoEmitter::emitEnums(raw_ostream &OS) {
   OS << "#endif // GET_INSTRINFO_SCHED_ENUM\n\n";
 }
 
-namespace llvm {
-
-void EmitInstrInfo(RecordKeeper &RK, raw_ostream &OS) {
+static void EmitInstrInfo(RecordKeeper &RK, raw_ostream &OS) {
   RK.startTimer("Analyze DAG patterns");
   InstrInfoEmitter(RK).run(OS);
   RK.startTimer("Emit map table");
   EmitMapTable(RK, OS);
 }
 
-} // end namespace llvm
+static TableGen::Emitter::Opt X("gen-instr-info", EmitInstrInfo,
+                                "Generate instruction descriptions");
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index 946a58417594..09aad78536fe 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -11,16 +11,29 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenIntrinsics.h"
-#include "CodeGenTarget.h"
 #include "SequenceToOffsetTable.h"
-#include "TableGenBackends.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/ModRef.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/StringToOffsetTable.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <algorithm>
+#include <array>
+#include <cassert>
+#include <map>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
 using namespace llvm;
 
 cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
@@ -39,6 +52,8 @@ public:
   void run(raw_ostream &OS, bool Enums);
 
   void EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
+  void EmitArgKind(raw_ostream &OS);
+  void EmitIITInfo(raw_ostream &OS);
   void EmitTargetInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
   void EmitIntrinsicToNameTable(const CodeGenIntrinsicTable &Ints,
                                 raw_ostream &OS);
@@ -63,7 +78,13 @@ void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) {
   if (Enums) {
     // Emit the enum information.
     EmitEnumInfo(Ints, OS);
+
+    // Emit ArgKind for Intrinsics.h.
+    EmitArgKind(OS);
   } else {
+    // Emit IIT_Info constants.
+    EmitIITInfo(OS);
+
     // Emit the target metadata.
     EmitTargetInfo(Ints, OS);
 
@@ -110,7 +131,9 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
   }
 
   // Generate a complete header for target specific intrinsics.
-  if (!IntrinsicPrefix.empty()) {
+  if (IntrinsicPrefix.empty()) {
+    OS << "#ifdef GET_INTRINSIC_ENUM_VALUES\n";
+  } else {
     std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();
     OS << "#ifndef LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n";
     OS << "#define LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n\n";
@@ -137,6 +160,7 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
   // Emit num_intrinsics into the target neutral enum.
   if (IntrinsicPrefix.empty()) {
     OS << "    num_intrinsics = " << (Ints.size() + 1) << "\n";
+    OS << "#endif\n\n";
   } else {
     OS << "}; // enum\n";
     OS << "} // namespace Intrinsic\n";
@@ -145,6 +169,41 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
   }
 }
 
+void IntrinsicEmitter::EmitArgKind(raw_ostream &OS) {
+  if (!IntrinsicPrefix.empty())
+    return;
+  OS << "// llvm::Intrinsic::IITDescriptor::ArgKind\n";
+  OS << "#ifdef GET_INTRINSIC_ARGKIND\n";
+  if (auto RecArgKind = Records.getDef("ArgKind")) {
+    for (auto &RV : RecArgKind->getValues())
+      OS << "    AK_" << RV.getName() << " = " << *RV.getValue() << ",\n";
+  } else {
+    OS << "#error \"ArgKind is not defined\"\n";
+  }
+  OS << "#endif\n\n";
+}
+
+void IntrinsicEmitter::EmitIITInfo(raw_ostream &OS) {
+  OS << "#ifdef GET_INTRINSIC_IITINFO\n";
+  std::array<StringRef, 256> RecsByNumber;
+  auto IIT_Base = Records.getAllDerivedDefinitionsIfDefined("IIT_Base");
+  for (auto Rec : IIT_Base) {
+    auto Number = Rec->getValueAsInt("Number");
+    assert(0 <= Number && Number < (int)RecsByNumber.size() &&
+           "IIT_Info.Number should be uint8_t");
+    assert(RecsByNumber[Number].empty() && "Duplicate IIT_Info.Number");
+    RecsByNumber[Number] = Rec->getName();
+  }
+  if (IIT_Base.size() > 0) {
+    for (unsigned I = 0, E = RecsByNumber.size(); I < E; ++I)
+      if (!RecsByNumber[I].empty())
+        OS << "  " << RecsByNumber[I] << " = " << I << ",\n";
+  } else {
+    OS << "#error \"class IIT_Base is not defined\"\n";
+  }
+  OS << "#endif\n\n";
+}
+
 void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
                                     raw_ostream &OS) {
   OS << "// Target mapping\n";
@@ -191,327 +250,16 @@ void IntrinsicEmitter::EmitIntrinsicToOverloadTable(
   OS << "#endif\n\n";
 }
 
-
-// NOTE: This must be kept in synch with the copy in lib/IR/Function.cpp!
-enum IIT_Info {
-  // Common values should be encoded with 0-15.
-  IIT_Done = 0,
-  IIT_I1 = 1,
-  IIT_I8 = 2,
-  IIT_I16 = 3,
-  IIT_I32 = 4,
-  IIT_I64 = 5,
-  IIT_F16 = 6,
-  IIT_F32 = 7,
-  IIT_F64 = 8,
-  IIT_V2 = 9,
-  IIT_V4 = 10,
-  IIT_V8 = 11,
-  IIT_V16 = 12,
-  IIT_V32 = 13,
-  IIT_PTR = 14,
-  IIT_ARG = 15,
-
-  // Values from 16+ are only encodable with the inefficient encoding.
-  IIT_V64 = 16,
-  IIT_MMX = 17,
-  IIT_TOKEN = 18,
-  IIT_METADATA = 19,
-  IIT_EMPTYSTRUCT = 20,
-  IIT_STRUCT2 = 21,
-  IIT_STRUCT3 = 22,
-  IIT_STRUCT4 = 23,
-  IIT_STRUCT5 = 24,
-  IIT_EXTEND_ARG = 25,
-  IIT_TRUNC_ARG = 26,
-  IIT_ANYPTR = 27,
-  IIT_V1 = 28,
-  IIT_VARARG = 29,
-  IIT_HALF_VEC_ARG = 30,
-  IIT_SAME_VEC_WIDTH_ARG = 31,
-  IIT_PTR_TO_ARG = 32,
-  IIT_PTR_TO_ELT = 33,
-  IIT_VEC_OF_ANYPTRS_TO_ELT = 34,
-  IIT_I128 = 35,
-  IIT_V512 = 36,
-  IIT_V1024 = 37,
-  IIT_STRUCT6 = 38,
-  IIT_STRUCT7 = 39,
-  IIT_STRUCT8 = 40,
-  IIT_F128 = 41,
-  IIT_VEC_ELEMENT = 42,
-  IIT_SCALABLE_VEC = 43,
-  IIT_SUBDIVIDE2_ARG = 44,
-  IIT_SUBDIVIDE4_ARG = 45,
-  IIT_VEC_OF_BITCASTS_TO_INT = 46,
-  IIT_V128 = 47,
-  IIT_BF16 = 48,
-  IIT_STRUCT9 = 49,
-  IIT_V256 = 50,
-  IIT_AMX = 51,
-  IIT_PPCF128 = 52,
-  IIT_V3 = 53,
-  IIT_EXTERNREF = 54,
-  IIT_FUNCREF = 55,
-  IIT_ANYPTR_TO_ELT = 56,
-  IIT_I2 = 57,
-  IIT_I4 = 58,
-};
-
-static void EncodeFixedValueType(MVT::SimpleValueType VT,
-                                 std::vector<unsigned char> &Sig) {
-  // clang-format off
-  if (MVT(VT).isInteger()) {
-    unsigned BitWidth = MVT(VT).getFixedSizeInBits();
-    switch (BitWidth) {
-    default: PrintFatalError("unhandled integer type width in intrinsic!");
-    case 1: return Sig.push_back(IIT_I1);
-    case 2: return Sig.push_back(IIT_I2);
-    case 4: return Sig.push_back(IIT_I4);
-    case 8: return Sig.push_back(IIT_I8);
-    case 16: return Sig.push_back(IIT_I16);
-    case 32: return Sig.push_back(IIT_I32);
-    case 64: return Sig.push_back(IIT_I64);
-    case 128: return Sig.push_back(IIT_I128);
-    }
-  }
-
-  switch (VT) {
-  default: PrintFatalError("unhandled MVT in intrinsic!");
-  case MVT::f16: return Sig.push_back(IIT_F16);
-  case MVT::bf16: return Sig.push_back(IIT_BF16);
-  case MVT::f32: return Sig.push_back(IIT_F32);
-  case MVT::f64: return Sig.push_back(IIT_F64);
-  case MVT::f128: return Sig.push_back(IIT_F128);
-  case MVT::ppcf128: return Sig.push_back(IIT_PPCF128);
-  case MVT::token: return Sig.push_back(IIT_TOKEN);
-  case MVT::Metadata: return Sig.push_back(IIT_METADATA);
-  case MVT::x86mmx: return Sig.push_back(IIT_MMX);
-  case MVT::x86amx: return Sig.push_back(IIT_AMX);
-  // MVT::OtherVT is used to mean the empty struct type here.
-  case MVT::Other: return Sig.push_back(IIT_EMPTYSTRUCT);
-  // MVT::isVoid is used to represent varargs here.
-  case MVT::isVoid: return Sig.push_back(IIT_VARARG);
-  case MVT::externref:
-    return Sig.push_back(IIT_EXTERNREF);
-  case MVT::funcref:
-    return Sig.push_back(IIT_FUNCREF);
-  }
-  // clang-format on
-}
-
-#if defined(_MSC_VER) && !defined(__clang__)
-#pragma optimize("",off) // MSVC 2015 optimizer can't deal with this function.
-#endif
-
-static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes,
-                            unsigned &NextArgCode,
-                            std::vector<unsigned char> &Sig,
-                            ArrayRef<unsigned char> Mapping) {
-
-  if (R->isSubClassOf("LLVMMatchType")) {
-    unsigned Number = Mapping[R->getValueAsInt("Number")];
-    assert(Number < ArgCodes.size() && "Invalid matching number!");
-    if (R->isSubClassOf("LLVMExtendedType"))
-      Sig.push_back(IIT_EXTEND_ARG);
-    else if (R->isSubClassOf("LLVMTruncatedType"))
-      Sig.push_back(IIT_TRUNC_ARG);
-    else if (R->isSubClassOf("LLVMHalfElementsVectorType"))
-      Sig.push_back(IIT_HALF_VEC_ARG);
-    else if (R->isSubClassOf("LLVMScalarOrSameVectorWidth")) {
-      Sig.push_back(IIT_SAME_VEC_WIDTH_ARG);
-      Sig.push_back((Number << 3) | ArgCodes[Number]);
-      MVT::SimpleValueType VT = getValueType(R->getValueAsDef("ElTy"));
-      EncodeFixedValueType(VT, Sig);
-      return;
-    }
-    else if (R->isSubClassOf("LLVMPointerTo"))
-      Sig.push_back(IIT_PTR_TO_ARG);
-    else if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) {
-      Sig.push_back(IIT_VEC_OF_ANYPTRS_TO_ELT);
-      // Encode overloaded ArgNo
-      Sig.push_back(NextArgCode++);
-      // Encode LLVMMatchType<Number> ArgNo
-      Sig.push_back(Number);
-      return;
-    } else if (R->isSubClassOf("LLVMAnyPointerToElt")) {
-      Sig.push_back(IIT_ANYPTR_TO_ELT);
-      // Encode overloaded ArgNo
-      Sig.push_back(NextArgCode++);
-      // Encode LLVMMatchType<Number> ArgNo
-      Sig.push_back(Number);
-      return;
-    } else if (R->isSubClassOf("LLVMPointerToElt"))
-      Sig.push_back(IIT_PTR_TO_ELT);
-    else if (R->isSubClassOf("LLVMVectorElementType"))
-      Sig.push_back(IIT_VEC_ELEMENT);
-    else if (R->isSubClassOf("LLVMSubdivide2VectorType"))
-      Sig.push_back(IIT_SUBDIVIDE2_ARG);
-    else if (R->isSubClassOf("LLVMSubdivide4VectorType"))
-      Sig.push_back(IIT_SUBDIVIDE4_ARG);
-    else if (R->isSubClassOf("LLVMVectorOfBitcastsToInt"))
-      Sig.push_back(IIT_VEC_OF_BITCASTS_TO_INT);
-    else
-      Sig.push_back(IIT_ARG);
-    return Sig.push_back((Number << 3) | 7 /*IITDescriptor::AK_MatchType*/);
-  }
-
-  MVT::SimpleValueType VT = getValueType(R->getValueAsDef("VT"));
-
-  unsigned Tmp = 0;
-  switch (VT) {
-  default: break;
-  case MVT::iPTRAny: ++Tmp; [[fallthrough]];
-  case MVT::vAny: ++Tmp;    [[fallthrough]];
-  case MVT::fAny: ++Tmp;    [[fallthrough]];
-  case MVT::iAny: ++Tmp;    [[fallthrough]];
-  case MVT::Any: {
-    // If this is an "any" valuetype, then the type is the type of the next
-    // type in the list specified to getIntrinsic().
-    Sig.push_back(IIT_ARG);
-
-    // Figure out what arg # this is consuming, and remember what kind it was.
-    assert(NextArgCode < ArgCodes.size() && ArgCodes[NextArgCode] == Tmp &&
-           "Invalid or no ArgCode associated with overloaded VT!");
-    unsigned ArgNo = NextArgCode++;
-
-    // Encode what sort of argument it must be in the low 3 bits of the ArgNo.
-    return Sig.push_back((ArgNo << 3) | Tmp);
-  }
-
-  case MVT::iPTR: {
-    unsigned AddrSpace = 0;
-    if (R->isSubClassOf("LLVMQualPointerType")) {
-      AddrSpace = R->getValueAsInt("AddrSpace");
-      assert(AddrSpace < 256 && "Address space exceeds 255");
-    }
-    if (AddrSpace) {
-      Sig.push_back(IIT_ANYPTR);
-      Sig.push_back(AddrSpace);
-    } else {
-      Sig.push_back(IIT_PTR);
-    }
-    return EncodeFixedType(R->getValueAsDef("ElTy"), ArgCodes, NextArgCode, Sig,
-                           Mapping);
-  }
-  }
-
-  if (MVT(VT).isVector()) {
-    MVT VVT = VT;
-    if (VVT.isScalableVector())
-      Sig.push_back(IIT_SCALABLE_VEC);
-    switch (VVT.getVectorMinNumElements()) {
-    default: PrintFatalError("unhandled vector type width in intrinsic!");
-    case 1: Sig.push_back(IIT_V1); break;
-    case 2: Sig.push_back(IIT_V2); break;
-    case 3: Sig.push_back(IIT_V3); break;
-    case 4: Sig.push_back(IIT_V4); break;
-    case 8: Sig.push_back(IIT_V8); break;
-    case 16: Sig.push_back(IIT_V16); break;
-    case 32: Sig.push_back(IIT_V32); break;
-    case 64: Sig.push_back(IIT_V64); break;
-    case 128: Sig.push_back(IIT_V128); break;
-    case 256: Sig.push_back(IIT_V256); break;
-    case 512: Sig.push_back(IIT_V512); break;
-    case 1024: Sig.push_back(IIT_V1024); break;
-    }
-
-    return EncodeFixedValueType(VVT.getVectorElementType().SimpleTy, Sig);
-  }
-
-  EncodeFixedValueType(VT, Sig);
-}
-
-static void UpdateArgCodes(Record *R, std::vector<unsigned char> &ArgCodes,
-                           unsigned int &NumInserted,
-                           SmallVectorImpl<unsigned char> &Mapping) {
-  if (R->isSubClassOf("LLVMMatchType")) {
-    if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) {
-      ArgCodes.push_back(3 /*vAny*/);
-      ++NumInserted;
-    } else if (R->isSubClassOf("LLVMAnyPointerToElt")) {
-      ArgCodes.push_back(4 /*iPTRAny*/);
-      ++NumInserted;
-    }
-    return;
-  }
-
-  unsigned Tmp = 0;
-  switch (getValueType(R->getValueAsDef("VT"))) {
-  default: break;
-  case MVT::iPTR:
-    UpdateArgCodes(R->getValueAsDef("ElTy"), ArgCodes, NumInserted, Mapping);
-    break;
-  case MVT::iPTRAny:
-    ++Tmp;
-    [[fallthrough]];
-  case MVT::vAny:
-    ++Tmp;
-    [[fallthrough]];
-  case MVT::fAny:
-    ++Tmp;
-    [[fallthrough]];
-  case MVT::iAny:
-    ++Tmp;
-    [[fallthrough]];
-  case MVT::Any:
-    unsigned OriginalIdx = ArgCodes.size() - NumInserted;
-    assert(OriginalIdx >= Mapping.size());
-    Mapping.resize(OriginalIdx+1);
-    Mapping[OriginalIdx] = ArgCodes.size();
-    ArgCodes.push_back(Tmp);
-    break;
-  }
-}
-
-#if defined(_MSC_VER) && !defined(__clang__)
-#pragma optimize("",on)
-#endif
-
 /// ComputeFixedEncoding - If we can encode the type signature for this
 /// intrinsic into 32 bits, return it.  If not, return ~0U.
 static void ComputeFixedEncoding(const CodeGenIntrinsic &Int,
                                  std::vector<unsigned char> &TypeSig) {
-  std::vector<unsigned char> ArgCodes;
-
-  // Add codes for any overloaded result VTs.
-  unsigned int NumInserted = 0;
-  SmallVector<unsigned char, 8> ArgMapping;
-  for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i)
-    UpdateArgCodes(Int.IS.RetTypeDefs[i], ArgCodes, NumInserted, ArgMapping);
-
-  // Add codes for any overloaded operand VTs.
-  for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i)
-    UpdateArgCodes(Int.IS.ParamTypeDefs[i], ArgCodes, NumInserted, ArgMapping);
-
-  unsigned NextArgCode = 0;
-  if (Int.IS.RetVTs.empty())
-    TypeSig.push_back(IIT_Done);
-  else if (Int.IS.RetVTs.size() == 1 &&
-           Int.IS.RetVTs[0] == MVT::isVoid)
-    TypeSig.push_back(IIT_Done);
-  else {
-    switch (Int.IS.RetVTs.size()) {
-      case 1: break;
-      case 2: TypeSig.push_back(IIT_STRUCT2); break;
-      case 3: TypeSig.push_back(IIT_STRUCT3); break;
-      case 4: TypeSig.push_back(IIT_STRUCT4); break;
-      case 5: TypeSig.push_back(IIT_STRUCT5); break;
-      case 6: TypeSig.push_back(IIT_STRUCT6); break;
-      case 7: TypeSig.push_back(IIT_STRUCT7); break;
-      case 8: TypeSig.push_back(IIT_STRUCT8); break;
-      case 9: TypeSig.push_back(IIT_STRUCT9); break;
-      default: llvm_unreachable("Unhandled case in struct");
+  if (auto *R = Int.TheDef->getValue("TypeSig")) {
+    for (auto &a : cast<ListInit>(R->getValue())->getValues()) {
+      for (auto &b : cast<ListInit>(a)->getValues())
+        TypeSig.push_back(cast<IntInit>(b)->getValue());
     }
-
-    for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i)
-      EncodeFixedType(Int.IS.RetTypeDefs[i], ArgCodes, NextArgCode, TypeSig,
-                      ArgMapping);
   }
-
-  for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i)
-    EncodeFixedType(Int.IS.ParamTypeDefs[i], ArgCodes, NextArgCode, TypeSig,
-                    ArgMapping);
 }
 
 static void printIITEntry(raw_ostream &OS, unsigned char X) {
@@ -640,6 +388,9 @@ std::optional<bool> compareFnAttributes(const CodeGenIntrinsic *L,
   if (L->hasSideEffects != R->hasSideEffects)
     return R->hasSideEffects;
 
+  if (L->isStrictFP != R->isStrictFP)
+    return R->isStrictFP;
+
   // Try to order by readonly/readnone attribute.
   uint32_t LK = L->ME.toIntValue();
   uint32_t RK = R->ME.toIntValue();
@@ -726,6 +477,10 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
           OS << "      Attribute::get(C, Attribute::Alignment, "
              << Attr.Value << "),\n";
           break;
+        case CodeGenIntrinsic::Dereferenceable:
+          OS << "      Attribute::get(C, Attribute::Dereferenceable, "
+             << Attr.Value << "),\n";
+          break;
         }
       }
       OS << "    });\n";
@@ -770,6 +525,8 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
       OS << "      Attribute::get(C, Attribute::Convergent),\n";
     if (Intrinsic.isSpeculatable)
       OS << "      Attribute::get(C, Attribute::Speculatable),\n";
+    if (Intrinsic.isStrictFP)
+      OS << "      Attribute::get(C, Attribute::StrictFP),\n";
 
     MemoryEffects ME = Intrinsic.ME;
     // TODO: IntrHasSideEffects should affect not only readnone intrinsics.
@@ -842,7 +599,8 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
         Intrinsic.isNoReturn || Intrinsic.isNoCallback || Intrinsic.isNoSync ||
         Intrinsic.isNoFree || Intrinsic.isWillReturn || Intrinsic.isCold ||
         Intrinsic.isNoDuplicate || Intrinsic.isNoMerge ||
-        Intrinsic.isConvergent || Intrinsic.isSpeculatable) {
+        Intrinsic.isConvergent || Intrinsic.isSpeculatable ||
+        Intrinsic.isStrictFP) {
       unsigned ID = UniqFnAttributes.find(&Intrinsic)->second;
       OS << "      AS[" << numAttrs++ << "] = {AttributeList::FunctionIndex, "
          << "getIntrinsicFnAttributeSet(C, " << ID << ")};\n";
@@ -952,10 +710,16 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
   OS << "#endif\n\n";
 }
 
-void llvm::EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS) {
+static void EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS) {
   IntrinsicEmitter(RK).run(OS, /*Enums=*/true);
 }
 
-void llvm::EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS) {
+static TableGen::Emitter::Opt X("gen-intrinsic-enums", EmitIntrinsicEnums,
+                                "Generate intrinsic enums");
+
+static void EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS) {
   IntrinsicEmitter(RK).run(OS, /*Enums=*/false);
 }
+
+static TableGen::Emitter::Opt Y("gen-intrinsic-impl", EmitIntrinsicImpl,
+                                "Generate intrinsic information");
diff --git a/llvm/utils/TableGen/OptParserEmitter.cpp b/llvm/utils/TableGen/OptParserEmitter.cpp
index d363191bd9b8..a04680b5d91e 100644
--- a/llvm/utils/TableGen/OptParserEmitter.cpp
+++ b/llvm/utils/TableGen/OptParserEmitter.cpp
@@ -64,7 +64,7 @@ class MarshallingInfo {
 public:
   static constexpr const char *MacroName = "OPTION_WITH_MARSHALLING";
   const Record &R;
-  bool ShouldAlwaysEmit;
+  bool ShouldAlwaysEmit = false;
   StringRef MacroPrefix;
   StringRef KeyPath;
   StringRef DefaultValue;
@@ -212,8 +212,7 @@ static MarshallingInfo createMarshallingInfo(const Record &R) {
 /// OptParserEmitter - This tablegen backend takes an input .td file
 /// describing a list of options and emits a data structure for parsing and
 /// working with those options when given an input command line.
-namespace llvm {
-void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) {
+static void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) {
   // Get the option groups and options.
   const std::vector<Record*> &Groups =
     Records.getAllDerivedDefinitions("OptionGroup");
@@ -499,4 +498,6 @@ void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) {
 
   OS << "\n";
 }
-} // end namespace llvm
+
+static TableGen::Emitter::Opt X("gen-opt-parser-defs", EmitOptParser,
+                                "Generate option definitions");
diff --git a/llvm/utils/TableGen/OptRSTEmitter.cpp b/llvm/utils/TableGen/OptRSTEmitter.cpp
index 03c7326e817a..87e755d943a1 100644
--- a/llvm/utils/TableGen/OptRSTEmitter.cpp
+++ b/llvm/utils/TableGen/OptRSTEmitter.cpp
@@ -10,13 +10,13 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 using namespace llvm;
 
 /// OptParserEmitter - This tablegen backend takes an input .td file
 /// describing a list of options and emits a RST man page.
-namespace llvm {
-void EmitOptRST(RecordKeeper &Records, raw_ostream &OS) {
+static void EmitOptRST(RecordKeeper &Records, raw_ostream &OS) {
   llvm::StringMap<std::vector<Record *>> OptionsByGroup;
   std::vector<Record *> OptionsWithoutGroup;
 
@@ -102,4 +102,6 @@ void EmitOptRST(RecordKeeper &Records, raw_ostream &OS) {
     }
   }
 }
-} // end namespace llvm
+
+static TableGen::Emitter::Opt X("gen-opt-rst", EmitOptRST,
+                                "Generate option RST");
diff --git a/llvm/utils/TableGen/PredicateExpander.cpp b/llvm/utils/TableGen/PredicateExpander.cpp
index b129401461b5..8f96d3307ded 100644
--- a/llvm/utils/TableGen/PredicateExpander.cpp
+++ b/llvm/utils/TableGen/PredicateExpander.cpp
@@ -12,6 +12,7 @@
 
 #include "PredicateExpander.h"
 #include "CodeGenSchedule.h" // Definition of STIPredicateFunction.
+#include "llvm/TableGen/Record.h"
 
 namespace llvm {
 
diff --git a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
index 6a1e1332d767..e07fb9188098 100644
--- a/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/llvm/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -313,10 +313,5 @@ void PseudoLoweringEmitter::run(raw_ostream &o) {
   emitLoweringEmitter(o);
 }
 
-namespace llvm {
-
-void EmitPseudoLowering(RecordKeeper &RK, raw_ostream &OS) {
-  PseudoLoweringEmitter(RK).run(OS);
-}
-
-} // End llvm namespace
+static TableGen::Emitter::OptClass<PseudoLoweringEmitter>
+    X("gen-pseudo-lowering", "Generate pseudo instruction lowering");
diff --git a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
index fa6508cbfc69..12174fd83f56 100644
--- a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp
@@ -1,4 +1,4 @@
-//===- RISCVTargetDefEmitter.cpp - Generate lists of RISCV CPUs -----------===//
+//===- RISCVTargetDefEmitter.cpp - Generate lists of RISC-V CPUs ----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -11,23 +11,23 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "TableGenBackends.h"
 #include "llvm/Support/RISCVISAInfo.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 
 using namespace llvm;
 
 using ISAInfoTy = llvm::Expected<std::unique_ptr<RISCVISAInfo>>;
 
 // We can generate march string from target features as what has been described
-// in RISCV ISA specification (version 20191213) 'Chapter 27. ISA Extension
+// in RISC-V ISA specification (version 20191213) 'Chapter 27. ISA Extension
 // Naming Conventions'.
 //
 // This is almost the same as RISCVFeatures::parseFeatureBits, except that we
 // get feature name from feature records instead of feature bits.
 static std::string getMArch(const Record &Rec) {
   std::vector<std::string> FeatureVector;
-  int XLen = 32;
+  unsigned XLen = 32;
 
   // Convert features to FeatureVector.
   for (auto *Feature : Rec.getValueAsListOfDefs("Features")) {
@@ -47,12 +47,11 @@ static std::string getMArch(const Record &Rec) {
   return (*ISAInfo)->toString();
 }
 
-void llvm::EmitRISCVTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
+static void EmitRISCVTargetDef(RecordKeeper &RK, raw_ostream &OS) {
   OS << "#ifndef PROC\n"
      << "#define PROC(ENUM, NAME, DEFAULT_MARCH)\n"
      << "#endif\n\n";
 
-  OS << "PROC(INVALID, {\"invalid\"}, {\"\"})\n";
   // Iterate on all definition records.
   for (const Record *Rec : RK.getAllDerivedDefinitions("RISCVProcessorModel")) {
     std::string MArch = Rec->getValueAsString("DefaultMarch").str();
@@ -80,3 +79,6 @@ void llvm::EmitRISCVTargetDef(const RecordKeeper &RK, raw_ostream &OS) {
 
   OS << "\n#undef TUNE_PROC\n";
 }
+
+static TableGen::Emitter::Opt X("gen-riscv-target-def", EmitRISCVTargetDef,
+                                "Generate the list of CPU for RISCV");
diff --git a/llvm/utils/TableGen/RegisterBankEmitter.cpp b/llvm/utils/TableGen/RegisterBankEmitter.cpp
index e6689b211a7d..2d23bf86b6ad 100644
--- a/llvm/utils/TableGen/RegisterBankEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterBankEmitter.cpp
@@ -11,15 +11,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CodeGenRegisters.h"
+#include "CodeGenTarget.h"
+#include "InfoByHwMode.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 
-#include "CodeGenRegisters.h"
-#include "CodeGenTarget.h"
-
 #define DEBUG_TYPE "register-bank-emitter"
 
 using namespace llvm;
@@ -37,11 +37,11 @@ private:
   RegisterClassesTy RCs;
 
   /// The register class with the largest register size.
-  const CodeGenRegisterClass *RCWithLargestRegsSize;
+  std::vector<const CodeGenRegisterClass *> RCsWithLargestRegSize;
 
 public:
-  RegisterBank(const Record &TheDef)
-      : TheDef(TheDef), RCWithLargestRegsSize(nullptr) {}
+  RegisterBank(const Record &TheDef, unsigned NumModeIds)
+      : TheDef(TheDef), RCsWithLargestRegSize(NumModeIds) {}
 
   /// Get the human-readable name for the bank.
   StringRef getName() const { return TheDef.getValueAsString("Name"); }
@@ -79,18 +79,21 @@ public:
     //        register size anywhere (we could sum the sizes of the subregisters
     //        but there may be additional bits too) and we can't derive it from
     //        the VT's reliably due to Untyped.
-    if (RCWithLargestRegsSize == nullptr)
-      RCWithLargestRegsSize = RC;
-    else if (RCWithLargestRegsSize->RSI.get(DefaultMode).SpillSize <
-             RC->RSI.get(DefaultMode).SpillSize)
-      RCWithLargestRegsSize = RC;
-    assert(RCWithLargestRegsSize && "RC was nullptr?");
+    unsigned NumModeIds = RCsWithLargestRegSize.size();
+    for (unsigned M = 0; M < NumModeIds; ++M) {
+      if (RCsWithLargestRegSize[M] == nullptr)
+        RCsWithLargestRegSize[M] = RC;
+      else if (RCsWithLargestRegSize[M]->RSI.get(M).SpillSize <
+               RC->RSI.get(M).SpillSize)
+        RCsWithLargestRegSize[M] = RC;
+      assert(RCsWithLargestRegSize[M] && "RC was nullptr?");
+    }
 
     RCs.emplace_back(RC);
   }
 
-  const CodeGenRegisterClass *getRCWithLargestRegsSize() const {
-    return RCWithLargestRegsSize;
+  const CodeGenRegisterClass *getRCWithLargestRegSize(unsigned HwMode) const {
+    return RCsWithLargestRegSize[HwMode];
   }
 
   iterator_range<typename RegisterClassesTy::const_iterator>
@@ -144,9 +147,10 @@ void RegisterBankEmitter::emitBaseClassDefinition(
     raw_ostream &OS, const StringRef TargetName,
     const std::vector<RegisterBank> &Banks) {
   OS << "private:\n"
-     << "  static RegisterBank *RegBanks[];\n\n"
+     << "  static const RegisterBank *RegBanks[];\n"
+     << "  static const unsigned Sizes[];\n\n"
      << "protected:\n"
-     << "  " << TargetName << "GenRegisterBankInfo();\n"
+     << "  " << TargetName << "GenRegisterBankInfo(unsigned HwMode = 0);\n"
      << "\n";
 }
 
@@ -211,6 +215,7 @@ void RegisterBankEmitter::emitBaseClassImplementation(
     raw_ostream &OS, StringRef TargetName,
     std::vector<RegisterBank> &Banks) {
   const CodeGenRegBank &RegisterClassHierarchy = Target.getRegBank();
+  const CodeGenHwModes &CGH = Target.getHwModes();
 
   OS << "namespace llvm {\n"
      << "namespace " << TargetName << " {\n";
@@ -241,11 +246,8 @@ void RegisterBankEmitter::emitBaseClassImplementation(
   for (const auto &Bank : Banks) {
     std::string QualifiedBankID =
         (TargetName + "::" + Bank.getEnumeratorName()).str();
-    const CodeGenRegisterClass &RC = *Bank.getRCWithLargestRegsSize();
-    unsigned Size = RC.RSI.get(DefaultMode).SpillSize;
-    OS << "RegisterBank " << Bank.getInstanceVarName() << "(/* ID */ "
-       << QualifiedBankID << ", /* Name */ \"" << Bank.getName()
-       << "\", /* Size */ " << Size << ", "
+    OS << "const RegisterBank " << Bank.getInstanceVarName() << "(/* ID */ "
+       << QualifiedBankID << ", /* Name */ \"" << Bank.getName() << "\", "
        << "/* CoveredRegClasses */ " << Bank.getCoverageArrayName()
        << ", /* NumRegClasses */ "
        << RegisterClassHierarchy.getRegClasses().size() << ");\n";
@@ -253,16 +255,33 @@ void RegisterBankEmitter::emitBaseClassImplementation(
   OS << "} // end namespace " << TargetName << "\n"
      << "\n";
 
-  OS << "RegisterBank *" << TargetName
+  OS << "const RegisterBank *" << TargetName
      << "GenRegisterBankInfo::RegBanks[] = {\n";
   for (const auto &Bank : Banks)
     OS << "    &" << TargetName << "::" << Bank.getInstanceVarName() << ",\n";
   OS << "};\n\n";
 
+  unsigned NumModeIds = CGH.getNumModeIds();
+  OS << "const unsigned " << TargetName << "GenRegisterBankInfo::Sizes[] = {\n";
+  for (unsigned M = 0; M < NumModeIds; ++M) {
+    OS << "    // Mode = " << M << " (";
+    if (M == DefaultMode)
+      OS << "Default";
+    else
+      OS << CGH.getMode(M).Name;
+    OS << ")\n";
+    for (const auto &Bank : Banks) {
+      const CodeGenRegisterClass &RC = *Bank.getRCWithLargestRegSize(M);
+      unsigned Size = RC.RSI.get(M).SpillSize;
+      OS << "    " << Size << ",\n";
+    }
+  }
+  OS << "};\n\n";
+
   OS << TargetName << "GenRegisterBankInfo::" << TargetName
-     << "GenRegisterBankInfo()\n"
+     << "GenRegisterBankInfo(unsigned HwMode)\n"
      << "    : RegisterBankInfo(RegBanks, " << TargetName
-     << "::NumRegisterBanks) {\n"
+     << "::NumRegisterBanks, Sizes, HwMode) {\n"
      << "  // Assert that RegBank indices match their ID's\n"
      << "#ifndef NDEBUG\n"
      << "  for (auto RB : enumerate(RegBanks))\n"
@@ -275,12 +294,13 @@ void RegisterBankEmitter::emitBaseClassImplementation(
 void RegisterBankEmitter::run(raw_ostream &OS) {
   StringRef TargetName = Target.getName();
   const CodeGenRegBank &RegisterClassHierarchy = Target.getRegBank();
+  const CodeGenHwModes &CGH = Target.getHwModes();
 
   Records.startTimer("Analyze records");
   std::vector<RegisterBank> Banks;
   for (const auto &V : Records.getAllDerivedDefinitions("RegisterBank")) {
     SmallPtrSet<const CodeGenRegisterClass *, 8> VisitedRCs;
-    RegisterBank Bank(*V);
+    RegisterBank Bank(*V, CGH.getNumModeIds());
 
     for (const CodeGenRegisterClass *RC :
          Bank.getExplicitlySpecifiedRegisterClasses(RegisterClassHierarchy)) {
@@ -327,10 +347,5 @@ void RegisterBankEmitter::run(raw_ostream &OS) {
   OS << "#endif // GET_TARGET_REGBANK_IMPL\n";
 }
 
-namespace llvm {
-
-void EmitRegisterBank(RecordKeeper &RK, raw_ostream &OS) {
-  RegisterBankEmitter(RK).run(OS);
-}
-
-} // end namespace llvm
+static TableGen::Emitter::OptClass<RegisterBankEmitter>
+    X("gen-register-bank", "Generate registers bank descriptions");
diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
index 113cebf8a08e..3101081114fb 100644
--- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp
@@ -12,8 +12,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CodeGenHwModes.h"
 #include "CodeGenRegisters.h"
 #include "CodeGenTarget.h"
+#include "InfoByHwMode.h"
 #include "SequenceToOffsetTable.h"
 #include "Types.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -23,10 +25,10 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/SparseBitVector.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Format.h"
-#include "llvm/Support/MachineValueType.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -633,17 +635,16 @@ static void printSubRegIndex(raw_ostream &OS, const CodeGenSubRegIndex *Idx) {
 // The initial value depends on the specific list. The list is terminated by a
 // 0 differential which means we can't encode repeated elements.
 
-typedef SmallVector<uint16_t, 4> DiffVec;
+typedef SmallVector<int16_t, 4> DiffVec;
 typedef SmallVector<LaneBitmask, 4> MaskVec;
 
-// Differentially encode a sequence of numbers into V. The starting value and
-// terminating 0 are not added to V, so it will have the same size as List.
-static
-DiffVec &diffEncode(DiffVec &V, unsigned InitVal, SparseBitVector<> List) {
+// Fills V with differentials between every two consecutive elements of List.
+static DiffVec &diffEncode(DiffVec &V, SparseBitVector<> List) {
   assert(V.empty() && "Clear DiffVec before diffEncode.");
-  uint16_t Val = uint16_t(InitVal);
-
-  for (uint16_t Cur : List) {
+  SparseBitVector<>::iterator I = List.begin(), E = List.end();
+  unsigned Val = *I;
+  while (++I != E) {
+    unsigned Cur = *I;
     V.push_back(Cur - Val);
     Val = Cur;
   }
@@ -654,18 +655,16 @@ template<typename Iter>
 static
 DiffVec &diffEncode(DiffVec &V, unsigned InitVal, Iter Begin, Iter End) {
   assert(V.empty() && "Clear DiffVec before diffEncode.");
-  uint16_t Val = uint16_t(InitVal);
+  unsigned Val = InitVal;
   for (Iter I = Begin; I != End; ++I) {
-    uint16_t Cur = (*I)->EnumValue;
+    unsigned Cur = (*I)->EnumValue;
     V.push_back(Cur - Val);
     Val = Cur;
   }
   return V;
 }
 
-static void printDiff16(raw_ostream &OS, uint16_t Val) {
-  OS << Val;
-}
+static void printDiff16(raw_ostream &OS, int16_t Val) { OS << Val; }
 
 static void printMask(raw_ostream &OS, LaneBitmask Val) {
   OS << "LaneBitmask(0x" << PrintLaneMask(Val) << ')';
@@ -889,7 +888,6 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target,
   SmallVector<DiffVec, 4> SubRegLists(Regs.size());
   SmallVector<DiffVec, 4> SuperRegLists(Regs.size());
   SmallVector<DiffVec, 4> RegUnitLists(Regs.size());
-  SmallVector<unsigned, 4> RegUnitInitScale(Regs.size());
 
   // List of lane masks accompanying register unit sequences.
   SequenceToOffsetTable<MaskVec> LaneMaskSeqs;
@@ -927,31 +925,8 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target,
                SuperRegList.end());
     DiffSeqs.add(SuperRegLists[i]);
 
-    // Differentially encode the register unit list, seeded by register number.
-    // First compute a scale factor that allows more diff-lists to be reused:
-    //
-    //   D0 -> (S0, S1)
-    //   D1 -> (S2, S3)
-    //
-    // A scale factor of 2 allows D0 and D1 to share a diff-list. The initial
-    // value for the differential decoder is the register number multiplied by
-    // the scale.
-    //
-    // Check the neighboring registers for arithmetic progressions.
-    unsigned ScaleA = ~0u, ScaleB = ~0u;
-    SparseBitVector<> RUs = Reg.getNativeRegUnits();
-    if (I != Regs.begin() &&
-        std::prev(I)->getNativeRegUnits().count() == RUs.count())
-      ScaleB = *RUs.begin() - *std::prev(I)->getNativeRegUnits().begin();
-    if (std::next(I) != Regs.end() &&
-        std::next(I)->getNativeRegUnits().count() == RUs.count())
-      ScaleA = *std::next(I)->getNativeRegUnits().begin() - *RUs.begin();
-    unsigned Scale = std::min(ScaleB, ScaleA);
-    // Default the scale to 0 if it can't be encoded in 4 bits.
-    if (Scale >= 16)
-      Scale = 0;
-    RegUnitInitScale[i] = Scale;
-    DiffSeqs.add(diffEncode(RegUnitLists[i], Scale * Reg.EnumValue, RUs));
+    const SparseBitVector<> &RUs = Reg.getNativeRegUnits();
+    DiffSeqs.add(diffEncode(RegUnitLists[i], RUs));
 
     const auto &RUMasks = Reg.getRegUnitLaneMasks();
     MaskVec &LaneMaskVec = RegUnitLaneMasks[i];
@@ -976,7 +951,7 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target,
   const std::string &TargetName = std::string(Target.getName());
 
   // Emit the shared table of differential lists.
-  OS << "extern const MCPhysReg " << TargetName << "RegDiffLists[] = {\n";
+  OS << "extern const int16_t " << TargetName << "RegDiffLists[] = {\n";
   DiffSeqs.emit(OS, printDiff16);
   OS << "};\n\n";
 
@@ -1012,10 +987,16 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target,
   // Emit the register descriptors now.
   i = 0;
   for (const auto &Reg : Regs) {
+    unsigned FirstRU = Reg.getNativeRegUnits().find_first();
+    unsigned Offset = DiffSeqs.get(RegUnitLists[i]);
+    // The value must be kept in sync with MCRegisterInfo.h.
+    constexpr unsigned RegUnitBits = 12;
+    assert(isUInt<RegUnitBits>(FirstRU) && "Too many regunits");
+    assert(isUInt<32 - RegUnitBits>(Offset) && "Offset is too big");
     OS << "  { " << RegStrings.get(std::string(Reg.getName())) << ", "
        << DiffSeqs.get(SubRegLists[i]) << ", " << DiffSeqs.get(SuperRegLists[i])
        << ", " << SubRegIdxSeqs.get(SubRegIdxLists[i]) << ", "
-       << (DiffSeqs.get(RegUnitLists[i]) * 16 + RegUnitInitScale[i]) << ", "
+       << (Offset << RegUnitBits | FirstRU) << ", "
        << LaneMaskSeqs.get(RegUnitLaneMasks[i]) << " },\n";
     ++i;
   }
@@ -1261,7 +1242,8 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
     for (const auto &RC : RegisterClasses) {
       std::vector<MVT::SimpleValueType> S;
       for (const ValueTypeByHwMode &VVT : RC.VTs)
-        S.push_back(VVT.get(M).SimpleTy);
+        if (VVT.hasDefault() || VVT.hasMode(M))
+          S.push_back(VVT.get(M).SimpleTy);
       VTSeqs.add(S);
     }
   }
@@ -1311,7 +1293,8 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
            << RI.SpillAlignment;
         std::vector<MVT::SimpleValueType> VTs;
         for (const ValueTypeByHwMode &VVT : RC.VTs)
-          VTs.push_back(VVT.get(M).SimpleTy);
+          if (VVT.hasDefault() || VVT.hasMode(M))
+            VTs.push_back(VVT.get(M).SimpleTy);
         OS << ", VTLists+" << VTSeqs.get(VTs) << " },    // "
            << RC.getName() << '\n';
       }
@@ -1649,7 +1632,7 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
 
   // Emit the constructor of the class...
   OS << "extern const MCRegisterDesc " << TargetName << "RegDesc[];\n";
-  OS << "extern const MCPhysReg " << TargetName << "RegDiffLists[];\n";
+  OS << "extern const int16_t " << TargetName << "RegDiffLists[];\n";
   OS << "extern const LaneBitmask " << TargetName << "LaneMaskLists[];\n";
   OS << "extern const char " << TargetName << "RegStrings[];\n";
   OS << "extern const char " << TargetName << "RegClassStrings[];\n";
@@ -1906,10 +1889,5 @@ void RegisterInfoEmitter::debugDump(raw_ostream &OS) {
   }
 }
 
-namespace llvm {
-
-void EmitRegisterInfo(RecordKeeper &RK, raw_ostream &OS) {
-  RegisterInfoEmitter(RK).run(OS);
-}
-
-} // end namespace llvm
+static TableGen::Emitter::OptClass<RegisterInfoEmitter>
+    X("gen-register-info", "Generate registers and register classes info");
diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp
index c88a2db55502..b6af02c28a80 100644
--- a/llvm/utils/TableGen/SearchableTableEmitter.cpp
+++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <algorithm>
 #include <set>
 #include <string>
@@ -173,6 +174,8 @@ private:
                                      "' lookup method '" + Index.Name +
                                      "', key field '" + Field.Name +
                                      "' of type bits is too large");
+    } else if (isa<BitRecTy>(Field.RecType)) {
+      return "bool";
     } else if (Field.Enum || Field.IsIntrinsic || Field.IsInstruction)
       return "unsigned";
     PrintFatalError(Index.Loc,
@@ -822,10 +825,5 @@ void SearchableTableEmitter::run(raw_ostream &OS) {
     OS << "#undef " << Guard << "\n";
 }
 
-namespace llvm {
-
-void EmitSearchableTables(RecordKeeper &RK, raw_ostream &OS) {
-  SearchableTableEmitter(RK).run(OS);
-}
-
-} // End llvm namespace.
+static TableGen::Emitter::OptClass<SearchableTableEmitter>
+    X("gen-searchable-tables", "Generate generic binary-searchable table");
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 8afe6d37d0e0..e4eb23649e96 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -10,22 +10,23 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CodeGenHwModes.h"
 #include "CodeGenSchedule.h"
 #include "CodeGenTarget.h"
 #include "PredicateExpander.h"
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/MC/MCSchedule.h"
-#include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
+#include "llvm/TargetParser/SubtargetFeature.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -40,6 +41,15 @@ using namespace llvm;
 
 namespace {
 
+/// Sorting predicate to sort record pointers by their
+/// FieldName field.
+struct LessRecordFieldFieldName {
+  bool operator()(const Record *Rec1, const Record *Rec2) const {
+    return Rec1->getValueAsString("FieldName") <
+           Rec2->getValueAsString("FieldName");
+  }
+};
+
 class SubtargetEmitter {
   // Each processor has a SchedClassDesc table with an entry for each SchedClass.
   // The SchedClassDesc table indexes into a global write resource table, write
@@ -68,7 +78,7 @@ class SubtargetEmitter {
     }
   };
 
-  const CodeGenTarget &TGT;
+  CodeGenTarget TGT;
   RecordKeeper &Records;
   CodeGenSchedModels &SchedModels;
   std::string Target;
@@ -110,6 +120,7 @@ class SubtargetEmitter {
   Record *FindReadAdvance(const CodeGenSchedRW &SchedRead,
                           const CodeGenProcModel &ProcModel);
   void ExpandProcResources(RecVec &PRVec, std::vector<int64_t> &Cycles,
+                           std::vector<int64_t> &StartAtCycles,
                            const CodeGenProcModel &ProcModel);
   void GenSchedClassTables(const CodeGenProcModel &ProcModel,
                            SchedClassTables &SchedTables);
@@ -126,8 +137,8 @@ class SubtargetEmitter {
   void ParseFeaturesFunction(raw_ostream &OS);
 
 public:
-  SubtargetEmitter(RecordKeeper &R, CodeGenTarget &TGT)
-      : TGT(TGT), Records(R), SchedModels(TGT.getSchedModels()),
+  SubtargetEmitter(RecordKeeper &R)
+      : TGT(R), Records(R), SchedModels(TGT.getSchedModels()),
         Target(TGT.getName()) {}
 
   void run(raw_ostream &o);
@@ -200,15 +211,15 @@ void SubtargetEmitter::EmitSubtargetInfoMacroCalls(raw_ostream &OS) {
 
   std::vector<Record *> FeatureList =
       Records.getAllDerivedDefinitions("SubtargetFeature");
-  llvm::sort(FeatureList, LessRecordFieldName());
+  llvm::sort(FeatureList, LessRecordFieldFieldName());
 
   for (const Record *Feature : FeatureList) {
-    const StringRef Attribute = Feature->getValueAsString("Attribute");
+    const StringRef FieldName = Feature->getValueAsString("FieldName");
     const StringRef Value = Feature->getValueAsString("Value");
 
     // Only handle boolean features for now, excluding BitVectors and enums.
     const bool IsBool = (Value == "false" || Value == "true") &&
-                        !StringRef(Attribute).contains('[');
+                        !StringRef(FieldName).contains('[');
     if (!IsBool)
       continue;
 
@@ -217,9 +228,9 @@ void SubtargetEmitter::EmitSubtargetInfoMacroCalls(raw_ostream &OS) {
 
     // Define the getter with lowercased first char: xxxYyy() { return XxxYyy; }
     const std::string Getter =
-        Attribute.substr(0, 1).lower() + Attribute.substr(1).str();
+        FieldName.substr(0, 1).lower() + FieldName.substr(1).str();
 
-    OS << "GET_SUBTARGETINFO_MACRO(" << Attribute << ", " << Default << ", "
+    OS << "GET_SUBTARGETINFO_MACRO(" << FieldName << ", " << Default << ", "
        << Getter << ")\n";
   }
   OS << "#undef GET_SUBTARGETINFO_MACRO\n";
@@ -967,6 +978,7 @@ Record *SubtargetEmitter::FindReadAdvance(const CodeGenSchedRW &SchedRead,
 // resource groups and super resources that cover them.
 void SubtargetEmitter::ExpandProcResources(RecVec &PRVec,
                                            std::vector<int64_t> &Cycles,
+                                           std::vector<int64_t> &StartAtCycles,
                                            const CodeGenProcModel &PM) {
   assert(PRVec.size() == Cycles.size() && "failed precondition");
   for (unsigned i = 0, e = PRVec.size(); i != e; ++i) {
@@ -989,6 +1001,7 @@ void SubtargetEmitter::ExpandProcResources(RecVec &PRVec,
                                          SubDef->getLoc());
         PRVec.push_back(SuperDef);
         Cycles.push_back(Cycles[i]);
+        StartAtCycles.push_back(StartAtCycles[i]);
         SubDef = SuperDef;
       }
     }
@@ -1005,6 +1018,7 @@ void SubtargetEmitter::ExpandProcResources(RecVec &PRVec,
       if (SubI == SubE) {
         PRVec.push_back(PR);
         Cycles.push_back(Cycles[i]);
+        StartAtCycles.push_back(StartAtCycles[i]);
       }
     }
   }
@@ -1139,22 +1153,48 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
         std::vector<int64_t> Cycles =
           WriteRes->getValueAsListOfInts("ResourceCycles");
 
-        if (Cycles.empty()) {
-          // If ResourceCycles is not provided, default to one cycle per
-          // resource.
-          Cycles.resize(PRVec.size(), 1);
-        } else if (Cycles.size() != PRVec.size()) {
+        std::vector<int64_t> StartAtCycles =
+            WriteRes->getValueAsListOfInts("StartAtCycles");
+
+        // Check consistency of the two vectors carrying the start and
+        // stop cycles of the resources.
+        if (!Cycles.empty() && Cycles.size() != PRVec.size()) {
           // If ResourceCycles is provided, check consistency.
           PrintFatalError(
               WriteRes->getLoc(),
-              Twine("Inconsistent resource cycles: !size(ResourceCycles) != "
-                    "!size(ProcResources): ")
+              Twine("Inconsistent resource cycles: size(ResourceCycles) != "
+                    "size(ProcResources): ")
                   .concat(Twine(PRVec.size()))
                   .concat(" vs ")
                   .concat(Twine(Cycles.size())));
         }
 
-        ExpandProcResources(PRVec, Cycles, ProcModel);
+        if (!StartAtCycles.empty() && StartAtCycles.size() != PRVec.size()) {
+          PrintFatalError(
+              WriteRes->getLoc(),
+              Twine("Inconsistent resource cycles: size(StartAtCycles) != "
+                    "size(ProcResources): ")
+                  .concat(Twine(StartAtCycles.size()))
+                  .concat(" vs ")
+                  .concat(Twine(PRVec.size())));
+        }
+
+        if (Cycles.empty()) {
+          // If ResourceCycles is not provided, default to one cycle
+          // per resource.
+          Cycles.resize(PRVec.size(), 1);
+        }
+
+        if (StartAtCycles.empty()) {
+          // If StartAtCycles is not provided, reserve the resource
+          // starting from cycle 0.
+          StartAtCycles.resize(PRVec.size(), 0);
+        }
+
+        assert(StartAtCycles.size() == Cycles.size());
+
+        ExpandProcResources(PRVec, Cycles, StartAtCycles, ProcModel);
+        assert(StartAtCycles.size() == Cycles.size());
 
         for (unsigned PRIdx = 0, PREnd = PRVec.size();
              PRIdx != PREnd; ++PRIdx) {
@@ -1162,6 +1202,17 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
           WPREntry.ProcResourceIdx = ProcModel.getProcResourceIdx(PRVec[PRIdx]);
           assert(WPREntry.ProcResourceIdx && "Bad ProcResourceIdx");
           WPREntry.Cycles = Cycles[PRIdx];
+          WPREntry.StartAtCycle = StartAtCycles[PRIdx];
+          if (StartAtCycles[PRIdx] > Cycles[PRIdx]) {
+            PrintFatalError(WriteRes->getLoc(),
+                            Twine("Inconsistent resource cycles: StartAtCycles "
+                                  "< Cycles must hold."));
+          }
+          if (StartAtCycles[PRIdx] < 0) {
+            PrintFatalError(WriteRes->getLoc(),
+                            Twine("Invalid value: StartAtCycle "
+                                  "must be a non-negative value."));
+          }
           // If this resource is already used in this sequence, add the current
           // entry's cycles so that the same resource appears to be used
           // serially, rather than multiple parallel uses. This is important for
@@ -1170,6 +1221,15 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
           for( ; WPRIdx != WPREnd; ++WPRIdx) {
             if (WriteProcResources[WPRIdx].ProcResourceIdx
                 == WPREntry.ProcResourceIdx) {
+              // TODO: multiple use of the same resources would
+              // require either 1. thinking of how to handle multiple
+              // intervals for the same resource in
+              // `<Target>WriteProcResTable` (see
+              // `SubtargetEmitter::EmitSchedClassTables`), or
+              // 2. thinking how to merge multiple intervals into a
+              // single interval.
+              assert(WPREntry.StartAtCycle == 0 &&
+                     "multiple use ofthe same resource is not yet handled");
               WriteProcResources[WPRIdx].Cycles += WPREntry.Cycles;
               break;
             }
@@ -1274,15 +1334,16 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
 void SubtargetEmitter::EmitSchedClassTables(SchedClassTables &SchedTables,
                                             raw_ostream &OS) {
   // Emit global WriteProcResTable.
-  OS << "\n// {ProcResourceIdx, Cycles}\n"
-     << "extern const llvm::MCWriteProcResEntry "
-     << Target << "WriteProcResTable[] = {\n"
-     << "  { 0,  0}, // Invalid\n";
+  OS << "\n// {ProcResourceIdx, Cycles, StartAtCycle}\n"
+     << "extern const llvm::MCWriteProcResEntry " << Target
+     << "WriteProcResTable[] = {\n"
+     << "  { 0,  0,  0 }, // Invalid\n";
   for (unsigned WPRIdx = 1, WPREnd = SchedTables.WriteProcResources.size();
        WPRIdx != WPREnd; ++WPRIdx) {
     MCWriteProcResEntry &WPREntry = SchedTables.WriteProcResources[WPRIdx];
     OS << "  {" << format("%2d", WPREntry.ProcResourceIdx) << ", "
-       << format("%2d", WPREntry.Cycles) << "}";
+       << format("%2d", WPREntry.Cycles) << ",  "
+       << format("%2d", WPREntry.StartAtCycle) << "}";
     if (WPRIdx + 1 < WPREnd)
       OS << ',';
     OS << " // #" << WPRIdx << '\n';
@@ -1401,6 +1462,12 @@ void SubtargetEmitter::EmitProcessorModels(raw_ostream &OS) {
     OS << "  " << (CompleteModel ? "true" : "false") << ", // "
        << "CompleteModel\n";
 
+    bool EnableIntervals =
+        (PM.ModelDef ? PM.ModelDef->getValueAsBit("EnableIntervals") : false);
+
+    OS << "  " << (EnableIntervals ? "true" : "false") << ", // "
+       << "EnableIntervals\n";
+
     OS << "  " << PM.Index << ", // Processor ID\n";
     if (PM.hasInstrSchedModel())
       OS << "  " << PM.ModelName << "ProcResources" << ",\n"
@@ -1746,17 +1813,17 @@ void SubtargetEmitter::ParseFeaturesFunction(raw_ostream &OS) {
     // Next record
     StringRef Instance = R->getName();
     StringRef Value = R->getValueAsString("Value");
-    StringRef Attribute = R->getValueAsString("Attribute");
+    StringRef FieldName = R->getValueAsString("FieldName");
 
     if (Value=="true" || Value=="false")
       OS << "  if (Bits[" << Target << "::"
          << Instance << "]) "
-         << Attribute << " = " << Value << ";\n";
+         << FieldName << " = " << Value << ";\n";
     else
       OS << "  if (Bits[" << Target << "::"
          << Instance << "] && "
-         << Attribute << " < " << Value << ") "
-         << Attribute << " = " << Value << ";\n";
+         << FieldName << " < " << Value << ") "
+         << FieldName << " = " << Value << ";\n";
   }
 
   OS << "}\n";
@@ -1983,11 +2050,5 @@ void SubtargetEmitter::run(raw_ostream &OS) {
   EmitMCInstrAnalysisPredicateFunctions(OS);
 }
 
-namespace llvm {
-
-void EmitSubtarget(RecordKeeper &RK, raw_ostream &OS) {
-  CodeGenTarget CGTarget(RK);
-  SubtargetEmitter(RK, CGTarget).run(OS);
-}
-
-} // end namespace llvm
+static TableGen::Emitter::OptClass<SubtargetEmitter>
+    X("gen-subtarget", "Generate subtarget enumerations");
diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/llvm/utils/TableGen/SubtargetFeatureInfo.cpp
index 2a63fc490380..1db8c0bf430a 100644
--- a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp
+++ b/llvm/utils/TableGen/SubtargetFeatureInfo.cpp
@@ -11,7 +11,6 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
-#include <map>
 
 using namespace llvm;
 
@@ -90,7 +89,7 @@ void SubtargetFeatureInfo::emitComputeAvailableFeatures(
     StringRef TargetName, StringRef ClassName, StringRef FuncName,
     SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS,
     StringRef ExtraParams) {
-  OS << "PredicateBitset " << TargetName << ClassName << "::\n"
+  OS << "PredicateBitset " << ClassName << "::\n"
      << FuncName << "(const " << TargetName << "Subtarget *Subtarget";
   if (!ExtraParams.empty())
     OS << ", " << ExtraParams;
@@ -118,16 +117,19 @@ static bool emitFeaturesAux(StringRef TargetName, const Init &Val,
     return false;
   }
   if (auto *D = dyn_cast<DagInit>(&Val)) {
-    std::string Op = D->getOperator()->getAsString();
-    if (Op == "not" && D->getNumArgs() == 1) {
+    auto *Op = dyn_cast<DefInit>(D->getOperator());
+    if (!Op)
+      return true;
+    StringRef OpName = Op->getDef()->getName();
+    if (OpName == "not" && D->getNumArgs() == 1) {
       OS << '!';
       return emitFeaturesAux(TargetName, *D->getArg(0), true, OS);
     }
-    if ((Op == "any_of" || Op == "all_of") && D->getNumArgs() > 0) {
+    if ((OpName == "any_of" || OpName == "all_of") && D->getNumArgs() > 0) {
       bool Paren = D->getNumArgs() > 1 && std::exchange(ParenIfBinOp, true);
       if (Paren)
         OS << '(';
-      ListSeparator LS(Op == "any_of" ? " || " : " && ");
+      ListSeparator LS(OpName == "any_of" ? " || " : " && ");
       for (auto *Arg : D->getArgs()) {
         OS << LS;
         if (emitFeaturesAux(TargetName, *Arg, ParenIfBinOp, OS))
diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.h b/llvm/utils/TableGen/SubtargetFeatureInfo.h
index 8c8a4487934c..77703e8a87f8 100644
--- a/llvm/utils/TableGen/SubtargetFeatureInfo.h
+++ b/llvm/utils/TableGen/SubtargetFeatureInfo.h
@@ -9,9 +9,11 @@
 #ifndef LLVM_UTIL_TABLEGEN_SUBTARGETFEATUREINFO_H
 #define LLVM_UTIL_TABLEGEN_SUBTARGETFEATUREINFO_H
 
+#include "llvm/ADT/StringRef.h"
 #include "llvm/TableGen/Record.h"
 #include <map>
 #include <string>
+#include <utility>
 #include <vector>
 
 namespace llvm {
@@ -67,8 +69,8 @@ struct SubtargetFeatureInfo {
   ///
   /// \param TargetName The name of the target as used in class prefixes (e.g.
   ///                   <TargetName>Subtarget)
-  /// \param ClassName  The name of the class (without the <Target> prefix)
-  ///                   that will contain the generated functions.
+  /// \param ClassName  The name of the class that will contain the generated
+  ///                   functions (including the target prefix.)
   /// \param FuncName   The name of the function to emit.
   /// \param SubtargetFeatures A map of TableGen records to the
   ///                          SubtargetFeatureInfo equivalent.
diff --git a/llvm/utils/TableGen/TableGen.cpp b/llvm/utils/TableGen/TableGen.cpp
index 746e2dd1db16..b2ed48cffe6b 100644
--- a/llvm/utils/TableGen/TableGen.cpp
+++ b/llvm/utils/TableGen/TableGen.cpp
@@ -10,57 +10,20 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "TableGenBackends.h" // Declares all backends.
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Main.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/SetTheory.h"
+#include "llvm/TableGen/TableGenBackend.h"
+#include <cassert>
+#include <string>
+#include <vector>
 
 using namespace llvm;
 
-enum ActionType {
-  PrintRecords,
-  PrintDetailedRecords,
-  NullBackend,
-  DumpJSON,
-  GenEmitter,
-  GenRegisterInfo,
-  GenInstrInfo,
-  GenInstrDocs,
-  GenAsmWriter,
-  GenAsmMatcher,
-  GenDisassembler,
-  GenPseudoLowering,
-  GenCompressInst,
-  GenCallingConv,
-  GenDAGISel,
-  GenDFAPacketizer,
-  GenFastISel,
-  GenSubtarget,
-  GenIntrinsicEnums,
-  GenIntrinsicImpl,
-  PrintEnums,
-  PrintSets,
-  GenOptParserDefs,
-  GenOptRST,
-  GenCTags,
-  GenAttributes,
-  GenSearchableTables,
-  GenGlobalISel,
-  GenGICombiner,
-  GenX86EVEX2VEXTables,
-  GenX86FoldTables,
-  GenX86MnemonicTables,
-  GenRegisterBank,
-  GenExegesis,
-  GenAutomata,
-  GenDirectivesEnumDecl,
-  GenDirectivesEnumImpl,
-  GenDXILOperation,
-  GenRISCVTargetDef,
-};
-
 namespace llvm {
 cl::opt<bool> EmitLongStrLiterals(
     "long-string-literals",
@@ -71,229 +34,54 @@ cl::opt<bool> EmitLongStrLiterals(
 } // end namespace llvm
 
 namespace {
-cl::opt<ActionType> Action(
-    cl::desc("Action to perform:"),
-    cl::values(
-        clEnumValN(PrintRecords, "print-records",
-                   "Print all records to stdout (default)"),
-        clEnumValN(PrintDetailedRecords, "print-detailed-records",
-                   "Print full details of all records to stdout"),
-        clEnumValN(NullBackend, "null-backend",
-                   "Do nothing after parsing (useful for timing)"),
-        clEnumValN(DumpJSON, "dump-json",
-                   "Dump all records as machine-readable JSON"),
-        clEnumValN(GenEmitter, "gen-emitter", "Generate machine code emitter"),
-        clEnumValN(GenRegisterInfo, "gen-register-info",
-                   "Generate registers and register classes info"),
-        clEnumValN(GenInstrInfo, "gen-instr-info",
-                   "Generate instruction descriptions"),
-        clEnumValN(GenInstrDocs, "gen-instr-docs",
-                   "Generate instruction documentation"),
-        clEnumValN(GenCallingConv, "gen-callingconv",
-                   "Generate calling convention descriptions"),
-        clEnumValN(GenAsmWriter, "gen-asm-writer", "Generate assembly writer"),
-        clEnumValN(GenDisassembler, "gen-disassembler",
-                   "Generate disassembler"),
-        clEnumValN(GenPseudoLowering, "gen-pseudo-lowering",
-                   "Generate pseudo instruction lowering"),
-        clEnumValN(GenCompressInst, "gen-compress-inst-emitter",
-                   "Generate RISCV compressed instructions."),
-        clEnumValN(GenAsmMatcher, "gen-asm-matcher",
-                   "Generate assembly instruction matcher"),
-        clEnumValN(GenDAGISel, "gen-dag-isel",
-                   "Generate a DAG instruction selector"),
-        clEnumValN(GenDFAPacketizer, "gen-dfa-packetizer",
-                   "Generate DFA Packetizer for VLIW targets"),
-        clEnumValN(GenFastISel, "gen-fast-isel",
-                   "Generate a \"fast\" instruction selector"),
-        clEnumValN(GenSubtarget, "gen-subtarget",
-                   "Generate subtarget enumerations"),
-        clEnumValN(GenIntrinsicEnums, "gen-intrinsic-enums",
-                   "Generate intrinsic enums"),
-        clEnumValN(GenIntrinsicImpl, "gen-intrinsic-impl",
-                   "Generate intrinsic information"),
-        clEnumValN(PrintEnums, "print-enums", "Print enum values for a class"),
-        clEnumValN(PrintSets, "print-sets",
-                   "Print expanded sets for testing DAG exprs"),
-        clEnumValN(GenOptParserDefs, "gen-opt-parser-defs",
-                   "Generate option definitions"),
-        clEnumValN(GenOptRST, "gen-opt-rst", "Generate option RST"),
-        clEnumValN(GenCTags, "gen-ctags", "Generate ctags-compatible index"),
-        clEnumValN(GenAttributes, "gen-attrs", "Generate attributes"),
-        clEnumValN(GenSearchableTables, "gen-searchable-tables",
-                   "Generate generic binary-searchable table"),
-        clEnumValN(GenGlobalISel, "gen-global-isel",
-                   "Generate GlobalISel selector"),
-        clEnumValN(GenGICombiner, "gen-global-isel-combiner",
-                   "Generate GlobalISel combiner"),
-        clEnumValN(GenX86EVEX2VEXTables, "gen-x86-EVEX2VEX-tables",
-                   "Generate X86 EVEX to VEX compress tables"),
-        clEnumValN(GenX86FoldTables, "gen-x86-fold-tables",
-                   "Generate X86 fold tables"),
-        clEnumValN(GenX86MnemonicTables, "gen-x86-mnemonic-tables",
-                   "Generate X86 mnemonic tables"),
-        clEnumValN(GenRegisterBank, "gen-register-bank",
-                   "Generate registers bank descriptions"),
-        clEnumValN(GenExegesis, "gen-exegesis",
-                   "Generate llvm-exegesis tables"),
-        clEnumValN(GenAutomata, "gen-automata", "Generate generic automata"),
-        clEnumValN(GenDirectivesEnumDecl, "gen-directive-decl",
-                   "Generate directive related declaration code (header file)"),
-        clEnumValN(GenDirectivesEnumImpl, "gen-directive-impl",
-                   "Generate directive related implementation code"),
-        clEnumValN(GenDXILOperation, "gen-dxil-operation",
-                   "Generate DXIL operation information"),
-        clEnumValN(GenRISCVTargetDef, "gen-riscv-target-def",
-                   "Generate the list of CPU for RISCV")));
+
 cl::OptionCategory PrintEnumsCat("Options for -print-enums");
 cl::opt<std::string> Class("class", cl::desc("Print Enum list for this class"),
                            cl::value_desc("class name"),
                            cl::cat(PrintEnumsCat));
 
-bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
-  switch (Action) {
-  case PrintRecords:
-    OS << Records;              // No argument, dump all contents
-    break;
-  case PrintDetailedRecords:
-    EmitDetailedRecords(Records, OS);
-    break;
-  case NullBackend:             // No backend at all.
-    break;
-  case DumpJSON:
-    EmitJSON(Records, OS);
-    break;
-  case GenEmitter:
-    EmitCodeEmitter(Records, OS);
-    break;
-  case GenRegisterInfo:
-    EmitRegisterInfo(Records, OS);
-    break;
-  case GenInstrInfo:
-    EmitInstrInfo(Records, OS);
-    break;
-  case GenInstrDocs:
-    EmitInstrDocs(Records, OS);
-    break;
-  case GenCallingConv:
-    EmitCallingConv(Records, OS);
-    break;
-  case GenAsmWriter:
-    EmitAsmWriter(Records, OS);
-    break;
-  case GenAsmMatcher:
-    EmitAsmMatcher(Records, OS);
-    break;
-  case GenDisassembler:
-    EmitDisassembler(Records, OS);
-    break;
-  case GenPseudoLowering:
-    EmitPseudoLowering(Records, OS);
-    break;
-  case GenCompressInst:
-    EmitCompressInst(Records, OS);
-    break;
-  case GenDAGISel:
-    EmitDAGISel(Records, OS);
-    break;
-  case GenDFAPacketizer:
-    EmitDFAPacketizer(Records, OS);
-    break;
-  case GenFastISel:
-    EmitFastISel(Records, OS);
-    break;
-  case GenSubtarget:
-    EmitSubtarget(Records, OS);
-    break;
-  case GenIntrinsicEnums:
-    EmitIntrinsicEnums(Records, OS);
-    break;
-  case GenIntrinsicImpl:
-    EmitIntrinsicImpl(Records, OS);
-    break;
-  case GenOptParserDefs:
-    EmitOptParser(Records, OS);
-    break;
-  case GenOptRST:
-    EmitOptRST(Records, OS);
-    break;
-  case PrintEnums:
-  {
-    for (Record *Rec : Records.getAllDerivedDefinitions(Class))
-      OS << Rec->getName() << ", ";
-    OS << "\n";
-    break;
-  }
-  case PrintSets:
-  {
-    SetTheory Sets;
-    Sets.addFieldExpander("Set", "Elements");
-    for (Record *Rec : Records.getAllDerivedDefinitions("Set")) {
-      OS << Rec->getName() << " = [";
-      const std::vector<Record*> *Elts = Sets.expand(Rec);
-      assert(Elts && "Couldn't expand Set instance");
-      for (Record *Elt : *Elts)
-        OS << ' ' << Elt->getName();
-      OS << " ]\n";
-    }
-    break;
-  }
-  case GenCTags:
-    EmitCTags(Records, OS);
-    break;
-  case GenAttributes:
-    EmitAttributes(Records, OS);
-    break;
-  case GenSearchableTables:
-    EmitSearchableTables(Records, OS);
-    break;
-  case GenGlobalISel:
-    EmitGlobalISel(Records, OS);
-    break;
-  case GenGICombiner:
-    EmitGICombiner(Records, OS);
-    break;
-  case GenRegisterBank:
-    EmitRegisterBank(Records, OS);
-    break;
-  case GenX86EVEX2VEXTables:
-    EmitX86EVEX2VEXTables(Records, OS);
-    break;
-  case GenX86MnemonicTables:
-    EmitX86MnemonicTables(Records, OS);
-    break;
-  case GenX86FoldTables:
-    EmitX86FoldTables(Records, OS);
-    break;
-  case GenExegesis:
-    EmitExegesis(Records, OS);
-    break;
-  case GenAutomata:
-    EmitAutomata(Records, OS);
-    break;
-  case GenDirectivesEnumDecl:
-    EmitDirectivesDecl(Records, OS);
-    break;
-  case GenDirectivesEnumImpl:
-    EmitDirectivesImpl(Records, OS);
-    break;
-  case GenDXILOperation:
-    EmitDXILOperation(Records, OS);
-    break;
-  case GenRISCVTargetDef:
-    EmitRISCVTargetDef(Records, OS);
-    break;
-  }
+void PrintRecords(RecordKeeper &Records, raw_ostream &OS) {
+  OS << Records; // No argument, dump all contents
+}
 
-  return false;
+void PrintEnums(RecordKeeper &Records, raw_ostream &OS) {
+  for (Record *Rec : Records.getAllDerivedDefinitions(Class))
+    OS << Rec->getName() << ", ";
+  OS << "\n";
 }
+
+void PrintSets(RecordKeeper &Records, raw_ostream &OS) {
+  SetTheory Sets;
+  Sets.addFieldExpander("Set", "Elements");
+  for (Record *Rec : Records.getAllDerivedDefinitions("Set")) {
+    OS << Rec->getName() << " = [";
+    const std::vector<Record *> *Elts = Sets.expand(Rec);
+    assert(Elts && "Couldn't expand Set instance");
+    for (Record *Elt : *Elts)
+      OS << ' ' << Elt->getName();
+    OS << " ]\n";
+  }
 }
 
+TableGen::Emitter::Opt X[] = {
+    {"print-records", PrintRecords, "Print all records to stdout (default)",
+     true},
+    {"print-detailed-records", EmitDetailedRecords,
+     "Print full details of all records to stdout"},
+    {"null-backend", [](RecordKeeper &Records, raw_ostream &OS) {},
+     "Do nothing after parsing (useful for timing)"},
+    {"dump-json", EmitJSON, "Dump all records as machine-readable JSON"},
+    {"print-enums", PrintEnums, "Print enum values for a class"},
+    {"print-sets", PrintSets, "Print expanded sets for testing DAG exprs"},
+};
+
+} // namespace
+
 int main(int argc, char **argv) {
   InitLLVM X(argc, argv);
   cl::ParseCommandLineOptions(argc, argv);
 
-  return TableGenMain(argv[0], &LLVMTableGenMain);
+  return TableGenMain(argv[0]);
 }
 
 #ifndef __has_feature
diff --git a/llvm/utils/TableGen/TableGenBackends.h b/llvm/utils/TableGen/TableGenBackends.h
index ac44babb1261..3afe6b01467b 100644
--- a/llvm/utils/TableGen/TableGenBackends.h
+++ b/llvm/utils/TableGen/TableGenBackends.h
@@ -15,6 +15,8 @@
 #ifndef LLVM_UTILS_TABLEGEN_TABLEGENBACKENDS_H
 #define LLVM_UTILS_TABLEGEN_TABLEGENBACKENDS_H
 
+#include <string>
+
 // A TableGen backend is a function that looks like
 //
 //    EmitFoo(RecordKeeper &RK, raw_ostream &OS /*, anything else you need */ )
@@ -61,41 +63,12 @@ namespace llvm {
 class raw_ostream;
 class RecordKeeper;
 
-void EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS);
-void EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS);
-void EmitAsmMatcher(RecordKeeper &RK, raw_ostream &OS);
-void EmitAsmWriter(RecordKeeper &RK, raw_ostream &OS);
-void EmitCallingConv(RecordKeeper &RK, raw_ostream &OS);
-void EmitCodeEmitter(RecordKeeper &RK, raw_ostream &OS);
-void EmitDAGISel(RecordKeeper &RK, raw_ostream &OS);
-void EmitDFAPacketizer(RecordKeeper &RK, raw_ostream &OS);
-void EmitDisassembler(RecordKeeper &RK, raw_ostream &OS);
-void EmitFastISel(RecordKeeper &RK, raw_ostream &OS);
-void EmitInstrInfo(RecordKeeper &RK, raw_ostream &OS);
-void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS);
-void EmitPseudoLowering(RecordKeeper &RK, raw_ostream &OS);
-void EmitCompressInst(RecordKeeper &RK, raw_ostream &OS);
-void EmitRegisterInfo(RecordKeeper &RK, raw_ostream &OS);
-void EmitSubtarget(RecordKeeper &RK, raw_ostream &OS);
 void EmitMapTable(RecordKeeper &RK, raw_ostream &OS);
-void EmitOptParser(RecordKeeper &RK, raw_ostream &OS);
-void EmitOptRST(RecordKeeper &RK, raw_ostream &OS);
-void EmitCTags(RecordKeeper &RK, raw_ostream &OS);
-void EmitAttributes(RecordKeeper &RK, raw_ostream &OS);
-void EmitSearchableTables(RecordKeeper &RK, raw_ostream &OS);
-void EmitGlobalISel(RecordKeeper &RK, raw_ostream &OS);
-void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS);
-void EmitX86EVEX2VEXTables(RecordKeeper &RK, raw_ostream &OS);
-void EmitX86FoldTables(RecordKeeper &RK, raw_ostream &OS);
-void EmitX86MnemonicTables(RecordKeeper &RK, raw_ostream &OS);
-void EmitRegisterBank(RecordKeeper &RK, raw_ostream &OS);
-void EmitExegesis(RecordKeeper &RK, raw_ostream &OS);
-void EmitAutomata(RecordKeeper &RK, raw_ostream &OS);
-void EmitDirectivesDecl(RecordKeeper &RK, raw_ostream &OS);
-void EmitDirectivesImpl(RecordKeeper &RK, raw_ostream &OS);
-void EmitDXILOperation(RecordKeeper &RK, raw_ostream &OS);
-void EmitRISCVTargetDef(const RecordKeeper &RK, raw_ostream &OS);
 
-} // End llvm namespace
+// Defined in DecoderEmitter.cpp
+void EmitDecoder(RecordKeeper &RK, raw_ostream &OS,
+                 const std::string &PredicateNamespace);
+
+} // namespace llvm
 
 #endif
diff --git a/llvm/utils/TableGen/Types.cpp b/llvm/utils/TableGen/Types.cpp
index a6682da90e6b..aca8e36b683d 100644
--- a/llvm/utils/TableGen/Types.cpp
+++ b/llvm/utils/TableGen/Types.cpp
@@ -34,11 +34,3 @@ const char *llvm::getMinimalTypeForRange(uint64_t Range, unsigned MaxSize LLVM_A
     return "uint16_t";
   return "uint8_t";
 }
-
-const char *llvm::getMinimalTypeForEnumBitfield(uint64_t Size) {
-  uint64_t MaxIndex = Size;
-  if (MaxIndex > 0)
-    MaxIndex--;
-  assert(MaxIndex <= 64 && "Too many bits");
-  return getMinimalTypeForRange(1ULL << MaxIndex);
-}
diff --git a/llvm/utils/TableGen/Types.h b/llvm/utils/TableGen/Types.h
index 17c7742ccaac..f369d61785c4 100644
--- a/llvm/utils/TableGen/Types.h
+++ b/llvm/utils/TableGen/Types.h
@@ -16,9 +16,6 @@ namespace llvm {
 /// MaxSize indicates the largest size of integer to consider (in bits) and only
 /// supports values of at least 32.
 const char *getMinimalTypeForRange(uint64_t Range, unsigned MaxSize = 64);
-
-/// Returns the smallest unsigned integer type that can hold the given bitfield.
-const char *getMinimalTypeForEnumBitfield(uint64_t Size);
 }
 
 #endif
diff --git a/llvm/utils/TableGen/VTEmitter.cpp b/llvm/utils/TableGen/VTEmitter.cpp
new file mode 100644
index 000000000000..d398a7e7b58f
--- /dev/null
+++ b/llvm/utils/TableGen/VTEmitter.cpp
@@ -0,0 +1,130 @@
+//===- VTEmitter.cpp - Generate properties from ValueTypes.td -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
+#include <array>
+#include <cassert>
+#include <map>
+using namespace llvm;
+
+namespace {
+
+class VTEmitter {
+private:
+  RecordKeeper &Records;
+
+public:
+  VTEmitter(RecordKeeper &R) : Records(R) {}
+
+  void run(raw_ostream &OS);
+};
+
+} // End anonymous namespace.
+
+void VTEmitter::run(raw_ostream &OS) {
+  emitSourceFileHeader("ValueTypes Source Fragment", OS);
+
+  std::array<const Record *, 256> VTsByNumber = {};
+  auto ValueTypes = Records.getAllDerivedDefinitions("ValueType");
+  for (auto *VT : ValueTypes) {
+    auto Number = VT->getValueAsInt("Value");
+    assert(0 <= Number && Number < (int)VTsByNumber.size() &&
+           "ValueType should be uint8_t");
+    assert(!VTsByNumber[Number] && "Duplicate ValueType");
+    VTsByNumber[Number] = VT;
+  }
+
+  struct VTRange {
+    StringRef First;
+    StringRef Last;
+    bool Closed;
+  };
+
+  std::map<StringRef, VTRange> VTRanges;
+
+  auto UpdateVTRange = [&VTRanges](const char *Key, StringRef Name,
+                                   bool Valid) {
+    if (Valid) {
+      if (!VTRanges.count(Key))
+        VTRanges[Key].First = Name;
+      assert(!VTRanges[Key].Closed && "Gap detected!");
+      VTRanges[Key].Last = Name;
+    } else if (VTRanges.count(Key)) {
+      VTRanges[Key].Closed = true;
+    }
+  };
+
+  OS << "#ifdef GET_VT_ATTR // (Ty, n, sz, Any, Int, FP, Vec, Sc)\n";
+  for (const auto *VT : VTsByNumber) {
+    if (!VT)
+      continue;
+    auto Name = VT->getValueAsString("LLVMName");
+    auto Value = VT->getValueAsInt("Value");
+    bool IsInteger = VT->getValueAsInt("isInteger");
+    bool IsFP = VT->getValueAsInt("isFP");
+    bool IsVector = VT->getValueAsInt("isVector");
+    bool IsScalable = VT->getValueAsInt("isScalable");
+
+    UpdateVTRange("INTEGER_FIXEDLEN_VECTOR_VALUETYPE", Name,
+                  IsInteger && IsVector && !IsScalable);
+    UpdateVTRange("INTEGER_SCALABLE_VECTOR_VALUETYPE", Name,
+                  IsInteger && IsScalable);
+    UpdateVTRange("FP_FIXEDLEN_VECTOR_VALUETYPE", Name,
+                  IsFP && IsVector && !IsScalable);
+    UpdateVTRange("FP_SCALABLE_VECTOR_VALUETYPE", Name, IsFP && IsScalable);
+    UpdateVTRange("FIXEDLEN_VECTOR_VALUETYPE", Name, IsVector && !IsScalable);
+    UpdateVTRange("SCALABLE_VECTOR_VALUETYPE", Name, IsScalable);
+    UpdateVTRange("VECTOR_VALUETYPE", Name, IsVector);
+    UpdateVTRange("INTEGER_VALUETYPE", Name, IsInteger && !IsVector);
+    UpdateVTRange("FP_VALUETYPE", Name, IsFP && !IsVector);
+    UpdateVTRange("VALUETYPE", Name, Value < 224);
+
+    // clang-format off
+    OS << "  GET_VT_ATTR("
+       << Name << ", "
+       << Value << ", "
+       << VT->getValueAsInt("Size") << ", "
+       << VT->getValueAsInt("isOverloaded") << ", "
+       << (IsInteger ? Name[0] == 'i' ? 3 : 1 : 0) << ", "
+       << (IsFP ? Name[0] == 'f' ? 3 : 1 : 0) << ", "
+       << IsVector << ", "
+       << IsScalable << ")\n";
+    // clang-format on
+  }
+  OS << "#endif\n\n";
+
+  OS << "#ifdef GET_VT_RANGES\n";
+  for (const auto &KV : VTRanges) {
+    assert(KV.second.Closed);
+    OS << "  FIRST_" << KV.first << " = " << KV.second.First << ",\n"
+       << "  LAST_" << KV.first << " = " << KV.second.Last << ",\n";
+  }
+  OS << "#endif\n\n";
+
+  OS << "#ifdef GET_VT_VECATTR // (Ty, Sc, nElem, ElTy, ElSz)\n";
+  for (const auto *VT : VTsByNumber) {
+    if (!VT || !VT->getValueAsInt("isVector"))
+      continue;
+    const auto *ElTy = VT->getValueAsDef("ElementType");
+    assert(ElTy);
+    // clang-format off
+    OS << "  GET_VT_VECATTR("
+       << VT->getValueAsString("LLVMName") << ", "
+       << VT->getValueAsInt("isScalable") << ", "
+       << VT->getValueAsInt("nElem") << ", "
+       << ElTy->getName() << ", "
+       << ElTy->getValueAsInt("Size") << ")\n";
+    // clang-format on
+  }
+  OS << "#endif\n\n";
+}
+
+static TableGen::Emitter::OptClass<VTEmitter> X("gen-vt", "Generate ValueType");
diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp
index 2c1acd8d910c..85da547d04c1 100644
--- a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp
@@ -58,6 +58,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 
 using namespace llvm;
 
diff --git a/llvm/utils/TableGen/X86DisassemblerTables.cpp b/llvm/utils/TableGen/X86DisassemblerTables.cpp
index 601591d9f53d..708c92aecfc8 100644
--- a/llvm/utils/TableGen/X86DisassemblerTables.cpp
+++ b/llvm/utils/TableGen/X86DisassemblerTables.cpp
@@ -76,7 +76,7 @@ static inline const char* stringForOperandEncoding(OperandEncoding encoding) {
 /// @return       - True if child is a subset of parent, false otherwise.
 static inline bool inheritsFrom(InstructionContext child,
                                 InstructionContext parent, bool noPrefix = true,
-                                bool VEX_LIG = false, bool VEX_WIG = false,
+                                bool VEX_LIG = false, bool WIG = false,
                                 bool AdSize64 = false) {
   if (child == parent)
     return true;
@@ -144,20 +144,20 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_64BIT_REXW_ADSIZE:
     return false;
   case IC_VEX:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_VEX_L_W)) ||
-           (VEX_WIG && inheritsFrom(child, IC_VEX_W)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_VEX_L_W)) ||
+           (WIG && inheritsFrom(child, IC_VEX_W)) ||
            (VEX_LIG && inheritsFrom(child, IC_VEX_L));
   case IC_VEX_XS:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_VEX_L_W_XS)) ||
-           (VEX_WIG && inheritsFrom(child, IC_VEX_W_XS)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_VEX_L_W_XS)) ||
+           (WIG && inheritsFrom(child, IC_VEX_W_XS)) ||
            (VEX_LIG && inheritsFrom(child, IC_VEX_L_XS));
   case IC_VEX_XD:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_VEX_L_W_XD)) ||
-           (VEX_WIG && inheritsFrom(child, IC_VEX_W_XD)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_VEX_L_W_XD)) ||
+           (WIG && inheritsFrom(child, IC_VEX_W_XD)) ||
            (VEX_LIG && inheritsFrom(child, IC_VEX_L_XD));
   case IC_VEX_OPSIZE:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_VEX_L_W_OPSIZE)) ||
-           (VEX_WIG && inheritsFrom(child, IC_VEX_W_OPSIZE)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_VEX_L_W_OPSIZE)) ||
+           (WIG && inheritsFrom(child, IC_VEX_W_OPSIZE)) ||
            (VEX_LIG && inheritsFrom(child, IC_VEX_L_OPSIZE));
   case IC_VEX_W:
     return VEX_LIG && inheritsFrom(child, IC_VEX_L_W);
@@ -168,88 +168,88 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_VEX_W_OPSIZE:
     return VEX_LIG && inheritsFrom(child, IC_VEX_L_W_OPSIZE);
   case IC_VEX_L:
-    return VEX_WIG && inheritsFrom(child, IC_VEX_L_W);
+    return WIG && inheritsFrom(child, IC_VEX_L_W);
   case IC_VEX_L_XS:
-    return VEX_WIG && inheritsFrom(child, IC_VEX_L_W_XS);
+    return WIG && inheritsFrom(child, IC_VEX_L_W_XS);
   case IC_VEX_L_XD:
-    return VEX_WIG && inheritsFrom(child, IC_VEX_L_W_XD);
+    return WIG && inheritsFrom(child, IC_VEX_L_W_XD);
   case IC_VEX_L_OPSIZE:
-    return VEX_WIG && inheritsFrom(child, IC_VEX_L_W_OPSIZE);
+    return WIG && inheritsFrom(child, IC_VEX_L_W_OPSIZE);
   case IC_VEX_L_W:
   case IC_VEX_L_W_XS:
   case IC_VEX_L_W_XD:
   case IC_VEX_L_W_OPSIZE:
     return false;
   case IC_EVEX:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2));
   case IC_EVEX_XS:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XS)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XS)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XS)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS));
   case IC_EVEX_XD:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XD)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XD)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XD)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD));
   case IC_EVEX_OPSIZE:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE));
   case IC_EVEX_K:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_K)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_K)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_K)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_K)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_K)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_K)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_K)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_K));
   case IC_EVEX_XS_K:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS_K)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XS_K)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS_K)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS_K));
   case IC_EVEX_XD_K:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD_K)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XD_K)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_K)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_K));
   case IC_EVEX_OPSIZE_K:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_K)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_K)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_K)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_K));
   case IC_EVEX_KZ:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_KZ)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_KZ)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_KZ)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_KZ)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_KZ)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_KZ));
   case IC_EVEX_XS_KZ:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS_KZ)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XS_KZ)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS_KZ)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS_KZ));
   case IC_EVEX_XD_KZ:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD_KZ)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XD_KZ)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_KZ)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_KZ));
   case IC_EVEX_OPSIZE_KZ:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_KZ)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_KZ)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_KZ)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_KZ));
   case IC_EVEX_W:
@@ -289,29 +289,29 @@ static inline bool inheritsFrom(InstructionContext child,
     return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ));
   case IC_EVEX_L:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W);
   case IC_EVEX_L_XS:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XS);
   case IC_EVEX_L_XD:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XD);
   case IC_EVEX_L_OPSIZE:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE);
   case IC_EVEX_L_K:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_K);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_K);
   case IC_EVEX_L_XS_K:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K);
   case IC_EVEX_L_XD_K:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K);
   case IC_EVEX_L_OPSIZE_K:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K);
   case IC_EVEX_L_KZ:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_KZ);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_KZ);
   case IC_EVEX_L_XS_KZ:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ);
   case IC_EVEX_L_XD_KZ:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ);
   case IC_EVEX_L_OPSIZE_KZ:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ);
   case IC_EVEX_L_W:
   case IC_EVEX_L_W_XS:
   case IC_EVEX_L_W_XD:
@@ -328,29 +328,29 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_EVEX_L_W_OPSIZE_KZ:
     return false;
   case IC_EVEX_L2:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W);
   case IC_EVEX_L2_XS:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XS);
   case IC_EVEX_L2_XD:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XD);
   case IC_EVEX_L2_OPSIZE:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE);
   case IC_EVEX_L2_K:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_K);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_K);
   case IC_EVEX_L2_XS_K:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K);
   case IC_EVEX_L2_XD_K:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K);
   case IC_EVEX_L2_OPSIZE_K:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K);
   case IC_EVEX_L2_KZ:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ);
   case IC_EVEX_L2_XS_KZ:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ);
   case IC_EVEX_L2_XD_KZ:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ);
   case IC_EVEX_L2_OPSIZE_KZ:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ);
   case IC_EVEX_L2_W:
   case IC_EVEX_L2_W_XS:
   case IC_EVEX_L2_W_XD:
@@ -367,79 +367,79 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_EVEX_L2_W_OPSIZE_KZ:
     return false;
   case IC_EVEX_B:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_B)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_B)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_B)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_B));
   case IC_EVEX_XS_B:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_B)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS_B)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XS_B)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XS_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS_B));
   case IC_EVEX_XD_B:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_B)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD_B)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XD_B)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XD_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_B));
   case IC_EVEX_OPSIZE_B:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_B)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_B)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_B)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_B));
   case IC_EVEX_K_B:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_K_B)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_K_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_K_B)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_K_B)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_K_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_K_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_K_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_K_B));
   case IC_EVEX_XS_K_B:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K_B)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS_K_B)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K_B)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XS_K_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS_K_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS_K_B));
   case IC_EVEX_XD_K_B:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K_B)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD_K_B)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K_B)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XD_K_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_K_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_K_B));
   case IC_EVEX_OPSIZE_K_B:
-    return (VEX_LIG && VEX_WIG &&
+    return (VEX_LIG && WIG &&
             inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B)) ||
-           (VEX_LIG && VEX_WIG &&
+           (VEX_LIG && WIG &&
             inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_K_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_K_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_K_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_K_B));
   case IC_EVEX_KZ_B:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_KZ_B)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_KZ_B)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_KZ_B)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_KZ_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_KZ_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_KZ_B));
   case IC_EVEX_XS_KZ_B:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ_B)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS_KZ_B)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ_B)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XS_KZ_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS_KZ_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS_KZ_B));
   case IC_EVEX_XD_KZ_B:
-    return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ_B)) ||
-           (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD_KZ_B)) ||
+    return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ_B)) ||
+           (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_XD_KZ_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_KZ_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_KZ_B));
   case IC_EVEX_OPSIZE_KZ_B:
-    return (VEX_LIG && VEX_WIG &&
+    return (VEX_LIG && WIG &&
             inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B)) ||
-           (VEX_LIG && VEX_WIG &&
+           (VEX_LIG && WIG &&
             inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B)) ||
-           (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_KZ_B)) ||
+           (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_KZ_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_KZ_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_KZ_B));
   case IC_EVEX_W_B:
@@ -479,29 +479,29 @@ static inline bool inheritsFrom(InstructionContext child,
     return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B)) ||
            (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B));
   case IC_EVEX_L_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_B);
   case IC_EVEX_L_XS_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XS_B);
   case IC_EVEX_L_XD_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XD_B);
   case IC_EVEX_L_OPSIZE_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_B);
   case IC_EVEX_L_K_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_K_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_K_B);
   case IC_EVEX_L_XS_K_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K_B);
   case IC_EVEX_L_XD_K_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K_B);
   case IC_EVEX_L_OPSIZE_K_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B);
   case IC_EVEX_L_KZ_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_KZ_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_KZ_B);
   case IC_EVEX_L_XS_KZ_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ_B);
   case IC_EVEX_L_XD_KZ_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ_B);
   case IC_EVEX_L_OPSIZE_KZ_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B);
   case IC_EVEX_L_W_B:
   case IC_EVEX_L_W_XS_B:
   case IC_EVEX_L_W_XD_B:
@@ -518,29 +518,29 @@ static inline bool inheritsFrom(InstructionContext child,
   case IC_EVEX_L_W_OPSIZE_KZ_B:
     return false;
   case IC_EVEX_L2_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_B);
   case IC_EVEX_L2_XS_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_B);
   case IC_EVEX_L2_XD_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_B);
   case IC_EVEX_L2_OPSIZE_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_B);
   case IC_EVEX_L2_K_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_K_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_K_B);
   case IC_EVEX_L2_XS_K_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K_B);
   case IC_EVEX_L2_XD_K_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K_B);
   case IC_EVEX_L2_OPSIZE_K_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B);
   case IC_EVEX_L2_KZ_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ_B);
   case IC_EVEX_L2_XS_KZ_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ_B);
   case IC_EVEX_L2_XD_KZ_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ_B);
   case IC_EVEX_L2_OPSIZE_KZ_B:
-    return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B);
+    return WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B);
   case IC_EVEX_L2_W_B:
   case IC_EVEX_L2_W_XS_B:
   case IC_EVEX_L2_W_XD_B:
@@ -1068,7 +1068,7 @@ void DisassemblerTables::setTableFields(OpcodeType          type,
                                         bool                is32bit,
                                         bool                noPrefix,
                                         bool                ignoresVEX_L,
-                                        bool                ignoresVEX_W,
+                                        bool                ignoresW,
                                         unsigned            addressSize) {
   ContextDecision &decision = *Tables[type];
 
@@ -1080,7 +1080,7 @@ void DisassemblerTables::setTableFields(OpcodeType          type,
     bool adSize64 = addressSize == 64;
     if (inheritsFrom((InstructionContext)index,
                      InstructionSpecifiers[uid].insnContext, noPrefix,
-                     ignoresVEX_L, ignoresVEX_W, adSize64))
+                     ignoresVEX_L, ignoresW, adSize64))
       setTableFields(decision.opcodeDecisions[index].modRMDecisions[opcode],
                      filter,
                      uid,
diff --git a/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp b/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
index 1384330ee8a1..35792ab67a4f 100644
--- a/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
@@ -15,6 +15,7 @@
 #include "CodeGenTarget.h"
 #include "X86RecognizableInstr.h"
 #include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 
 using namespace llvm;
@@ -113,10 +114,10 @@ public:
   bool operator()(const CodeGenInstruction *VEXInst) {
     RecognizableInstrBase VEXRI(*VEXInst);
     RecognizableInstrBase EVEXRI(*EVEXInst);
-    bool VEX_W = VEXRI.HasVEX_W;
-    bool EVEX_W = EVEXRI.HasVEX_W;
-    bool VEX_WIG  = VEXRI.IgnoresVEX_W;
-    bool EVEX_WIG  = EVEXRI.IgnoresVEX_W;
+    bool VEX_W = VEXRI.HasREX_W;
+    bool EVEX_W = EVEXRI.HasREX_W;
+    bool VEX_WIG  = VEXRI.IgnoresW;
+    bool EVEX_WIG  = EVEXRI.IgnoresW;
     bool EVEX_W1_VEX_W0 = EVEXInst->TheDef->getValueAsBit("EVEX_W1_VEX_W0");
 
     if (VEXRI.IsCodeGenOnly != EVEXRI.IsCodeGenOnly ||
@@ -237,10 +238,7 @@ void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) {
   // Print CheckVEXInstPredicate function.
   printCheckPredicate(EVEX2VEXPredicates, OS);
 }
-}
+} // namespace
 
-namespace llvm {
-void EmitX86EVEX2VEXTables(RecordKeeper &RK, raw_ostream &OS) {
-  X86EVEX2VEXTablesEmitter(RK).run(OS);
-}
-}
+static TableGen::Emitter::OptClass<X86EVEX2VEXTablesEmitter>
+    X("gen-x86-EVEX2VEX-tables", "Generate X86 EVEX to VEX compress tables");
diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
index 5b3f11848de6..89d93e4d3cbc 100644
--- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
@@ -11,34 +11,24 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CodeGenInstruction.h"
 #include "CodeGenTarget.h"
 #include "X86RecognizableInstr.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/FormattedStream.h"
-#include "llvm/TableGen/Error.h"
+#include "llvm/Support/X86FoldTablesUtils.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 
 using namespace llvm;
 using namespace X86Disassembler;
 
 namespace {
-
-// 3 possible strategies for the unfolding flag (TB_NO_REVERSE) of the
-// manual added entries.
-enum UnfoldStrategy {
-  UNFOLD,     // Allow unfolding
-  NO_UNFOLD,  // Prevent unfolding
-  NO_STRATEGY // Make decision according to operands' sizes
-};
-
 // Represents an entry in the manual mapped instructions set.
 struct ManualMapEntry {
   const char *RegInstStr;
   const char *MemInstStr;
-  UnfoldStrategy Strategy;
-
-  ManualMapEntry(const char *RegInstStr, const char *MemInstStr,
-                 UnfoldStrategy Strategy = NO_STRATEGY)
-      : RegInstStr(RegInstStr), MemInstStr(MemInstStr), Strategy(Strategy) {}
+  uint16_t Strategy;
 };
 
 // List of instructions requiring explicitly aligned memory.
@@ -50,36 +40,15 @@ const char *ExplicitUnalign[] = {"MOVDQU", "MOVUPS", "MOVUPD",
                                  "PCMPESTRM", "PCMPESTRI",
                                  "PCMPISTRM", "PCMPISTRI" };
 
-// For manually mapping instructions that do not match by their encoding.
 const ManualMapEntry ManualMapSet[] = {
-    { "ADD16ri_DB",         "ADD16mi",         NO_UNFOLD  },
-    { "ADD16ri8_DB",        "ADD16mi8",        NO_UNFOLD  },
-    { "ADD16rr_DB",         "ADD16mr",         NO_UNFOLD  },
-    { "ADD32ri_DB",         "ADD32mi",         NO_UNFOLD  },
-    { "ADD32ri8_DB",        "ADD32mi8",        NO_UNFOLD  },
-    { "ADD32rr_DB",         "ADD32mr",         NO_UNFOLD  },
-    { "ADD64ri32_DB",       "ADD64mi32",       NO_UNFOLD  },
-    { "ADD64ri8_DB",        "ADD64mi8",        NO_UNFOLD  },
-    { "ADD64rr_DB",         "ADD64mr",         NO_UNFOLD  },
-    { "ADD8ri_DB",          "ADD8mi",          NO_UNFOLD  },
-    { "ADD8rr_DB",          "ADD8mr",          NO_UNFOLD  },
-    { "ADD16rr_DB",         "ADD16rm",         NO_UNFOLD  },
-    { "ADD32rr_DB",         "ADD32rm",         NO_UNFOLD  },
-    { "ADD64rr_DB",         "ADD64rm",         NO_UNFOLD  },
-    { "ADD8rr_DB",          "ADD8rm",          NO_UNFOLD  },
-    { "MMX_MOVD64from64rr", "MMX_MOVQ64mr",    UNFOLD },
-    { "MMX_MOVD64grr",      "MMX_MOVD64mr",    UNFOLD },
-    { "MOVLHPSrr",          "MOVHPSrm",        NO_UNFOLD  },
-    { "PUSH16r",            "PUSH16rmm",       UNFOLD },
-    { "PUSH32r",            "PUSH32rmm",       UNFOLD },
-    { "PUSH64r",            "PUSH64rmm",       UNFOLD },
-    { "TAILJMPr",           "TAILJMPm",        UNFOLD },
-    { "TAILJMPr64",         "TAILJMPm64",      UNFOLD },
-    { "TAILJMPr64_REX",     "TAILJMPm64_REX",  UNFOLD },
-    { "VMOVLHPSZrr",        "VMOVHPSZ128rm",   NO_UNFOLD  },
-    { "VMOVLHPSrr",         "VMOVHPSrm",       NO_UNFOLD  },
+#define ENTRY(REG, MEM, FLAGS) {#REG, #MEM, FLAGS},
+#include "X86ManualFoldTables.def"
 };
 
+const std::set<StringRef> NoFoldSet= {
+#define NOFOLD(INSN) #INSN,
+#include "X86ManualFoldTables.def"
+};
 
 static bool isExplicitAlign(const CodeGenInstruction *Inst) {
   return any_of(ExplicitAlign, [Inst](const char *InstStr) {
@@ -103,51 +72,76 @@ class X86FoldTablesEmitter {
     const CodeGenInstruction *MemInst;
 
   public:
-    bool CannotUnfold = false;
-    bool IsLoad = false;
-    bool IsStore = false;
-    bool IsAligned = false;
-    unsigned int Alignment = 0;
+    bool NoReverse = false;
+    bool NoForward = false;
+    bool FoldLoad = false;
+    bool FoldStore = false;
+    Align Alignment;
 
+    X86FoldTableEntry() = default;
     X86FoldTableEntry(const CodeGenInstruction *RegInst,
                       const CodeGenInstruction *MemInst)
         : RegInst(RegInst), MemInst(MemInst) {}
 
     void print(formatted_raw_ostream &OS) const {
       OS.indent(2);
-      OS << "{ X86::" << RegInst->TheDef->getName() << ",";
-      OS.PadToColumn(40);
-      OS  << "X86::" << MemInst->TheDef->getName() << ",";
-      OS.PadToColumn(75);
+      OS << "{X86::" << RegInst->TheDef->getName() << ", ";
+      OS  << "X86::" << MemInst->TheDef->getName() << ", ";
 
       std::string Attrs;
-      if (IsLoad)
-        Attrs += "TB_FOLDED_LOAD | ";
-      if (IsStore)
-        Attrs += "TB_FOLDED_STORE | ";
-      if (CannotUnfold)
-        Attrs += "TB_NO_REVERSE | ";
-      if (IsAligned)
-        Attrs += "TB_ALIGN_" + std::to_string(Alignment) + " | ";
+      if (FoldLoad)
+        Attrs += "TB_FOLDED_LOAD|";
+      if (FoldStore)
+        Attrs += "TB_FOLDED_STORE|";
+      if (NoReverse)
+        Attrs += "TB_NO_REVERSE|";
+      if (NoForward)
+        Attrs += "TB_NO_FORWARD|";
+      if (Alignment != Align(1))
+        Attrs += "TB_ALIGN_" + std::to_string(Alignment.value()) + "|";
 
-      StringRef SimplifiedAttrs = StringRef(Attrs).rtrim("| ");
+      StringRef SimplifiedAttrs = StringRef(Attrs).rtrim("|");
       if (SimplifiedAttrs.empty())
         SimplifiedAttrs = "0";
 
-      OS << SimplifiedAttrs << " },\n";
+      OS << SimplifiedAttrs << "},\n";
     }
 
-    bool operator<(const X86FoldTableEntry &RHS) const {
-      bool LHSpseudo = RegInst->TheDef->getValueAsBit("isPseudo");
-      bool RHSpseudo = RHS.RegInst->TheDef->getValueAsBit("isPseudo");
-      if (LHSpseudo != RHSpseudo)
-        return LHSpseudo;
+#ifndef NDEBUG
+    // Check that Uses and Defs are same after memory fold.
+    void checkCorrectness() const {
+      auto &RegInstRec = *RegInst->TheDef;
+      auto &MemInstRec = *MemInst->TheDef;
+      auto ListOfUsesReg = RegInstRec.getValueAsListOfDefs("Uses");
+      auto ListOfUsesMem = MemInstRec.getValueAsListOfDefs("Uses");
+      auto ListOfDefsReg = RegInstRec.getValueAsListOfDefs("Defs");
+      auto ListOfDefsMem = MemInstRec.getValueAsListOfDefs("Defs");
+      if (ListOfUsesReg != ListOfUsesMem || ListOfDefsReg != ListOfDefsMem)
+        report_fatal_error("Uses/Defs couldn't be changed after folding " +
+                           RegInstRec.getName() + " to " +
+                           MemInstRec.getName());
+    }
+#endif
+  };
 
-      return RegInst->TheDef->getName() < RHS.RegInst->TheDef->getName();
+  // NOTE: We check the fold tables are sorted in X86InstrFoldTables.cpp by the enum of the
+  //       instruction, which is computed in CodeGenTarget::ComputeInstrsByEnum. So we should
+  //       use the same comparator here.
+  // FIXME: Could we share the code with CodeGenTarget::ComputeInstrsByEnum?
+  struct CompareInstrsByEnum {
+    bool operator()(const CodeGenInstruction *LHS,
+                    const CodeGenInstruction *RHS) const {
+      assert(LHS && RHS && "LHS and RHS shouldn't be nullptr");
+      const auto &D1 = *LHS->TheDef;
+      const auto &D2 = *RHS->TheDef;
+      return std::make_tuple(!D1.getValueAsBit("isPseudo"), D1.getName()) <
+             std::make_tuple(!D2.getValueAsBit("isPseudo"), D2.getName());
     }
   };
 
-  typedef std::vector<X86FoldTableEntry> FoldTable;
+  typedef std::map<const CodeGenInstruction *, X86FoldTableEntry,
+                   CompareInstrsByEnum>
+      FoldTable;
   // std::vector for each folding table.
   // Table2Addr - Holds instructions which their memory form performs load+store
   // Table#i - Holds instructions which the their memory form perform a load OR
@@ -163,20 +157,20 @@ public:
   X86FoldTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {}
 
   // run - Generate the 6 X86 memory fold tables.
-  void run(formatted_raw_ostream &OS);
+  void run(raw_ostream &OS);
 
 private:
   // Decides to which table to add the entry with the given instructions.
   // S sets the strategy of adding the TB_NO_REVERSE flag.
   void updateTables(const CodeGenInstruction *RegInstr,
-                    const CodeGenInstruction *MemInstr,
-                    const UnfoldStrategy S = NO_STRATEGY);
+                    const CodeGenInstruction *MemInstr, uint16_t S = 0,
+                    bool IsManual = false);
 
   // Generates X86FoldTableEntry with the given instructions and fill it with
   // the appropriate flags - then adds it to Table.
   void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInstr,
-                         const CodeGenInstruction *MemInstr,
-                         const UnfoldStrategy S, const unsigned int FoldedInd);
+                         const CodeGenInstruction *MemInstr, uint16_t S,
+                         unsigned FoldedIdx, bool isManual);
 
   // Print the given table as a static const C++ array of type
   // X86MemoryFoldTableEntry.
@@ -185,8 +179,8 @@ private:
     OS << "static const X86MemoryFoldTableEntry MemoryFold" << TableName
        << "[] = {\n";
 
-    for (const X86FoldTableEntry &E : Table)
-      E.print(OS);
+    for (auto &E : Table)
+      E.second.print(OS);
 
     OS << "};\n\n";
   }
@@ -206,76 +200,110 @@ static bool hasPtrTailcallRegClass(const CodeGenInstruction *Inst) {
   });
 }
 
-// Calculates the integer value representing the BitsInit object
-static inline uint64_t getValueFromBitsInit(const BitsInit *B) {
-  assert(B->getNumBits() <= sizeof(uint64_t) * 8 && "BitInits' too long!");
+static uint8_t byteFromBitsInit(const BitsInit *B) {
+  unsigned N = B->getNumBits();
+  assert(N <= 8 && "Field is too large for uint8_t!");
 
-  uint64_t Value = 0;
-  for (unsigned i = 0, e = B->getNumBits(); i != e; ++i) {
-    BitInit *Bit = cast<BitInit>(B->getBit(i));
-    Value |= uint64_t(Bit->getValue()) << i;
+  uint8_t Value = 0;
+  for (unsigned I = 0; I != N; ++I) {
+    BitInit *Bit = cast<BitInit>(B->getBit(I));
+    Value |= Bit->getValue() << I;
   }
   return Value;
 }
 
-// Return true if the instruction defined as a register flavor.
-static inline bool hasRegisterFormat(const Record *Inst) {
-  const BitsInit *FormBits = Inst->getValueAsBitsInit("FormBits");
-  uint64_t FormBitsNum = getValueFromBitsInit(FormBits);
-
-  // Values from X86Local namespace defined in X86RecognizableInstr.cpp
-  return FormBitsNum >= X86Local::MRMDestReg && FormBitsNum <= X86Local::MRM7r;
+static bool mayFoldFromForm(uint8_t Form) {
+  switch (Form) {
+  default:
+    return Form >= X86Local::MRM0r && Form <= X86Local::MRM7r;
+  case X86Local::MRMXr:
+  case X86Local::MRMXrCC:
+  case X86Local::MRMDestReg:
+  case X86Local::MRMSrcReg:
+  case X86Local::MRMSrcReg4VOp3:
+  case X86Local::MRMSrcRegOp4:
+  case X86Local::MRMSrcRegCC:
+    return true;
+  }
 }
 
-// Return true if the instruction defined as a memory flavor.
-static inline bool hasMemoryFormat(const Record *Inst) {
-  const BitsInit *FormBits = Inst->getValueAsBitsInit("FormBits");
-  uint64_t FormBitsNum = getValueFromBitsInit(FormBits);
-
-  // Values from X86Local namespace defined in X86RecognizableInstr.cpp
-  return FormBitsNum >= X86Local::MRMDestMem && FormBitsNum <= X86Local::MRM7m;
+static bool mayFoldToForm(uint8_t Form) {
+  switch (Form) {
+  default:
+    return Form >= X86Local::MRM0m && Form <= X86Local::MRM7m;
+  case X86Local::MRMXm:
+  case X86Local::MRMXmCC:
+  case X86Local::MRMDestMem:
+  case X86Local::MRMSrcMem:
+  case X86Local::MRMSrcMem4VOp3:
+  case X86Local::MRMSrcMemOp4:
+  case X86Local::MRMSrcMemCC:
+    return true;
+  }
 }
 
-static inline bool isNOREXRegClass(const Record *Op) {
-  return Op->getName().contains("_NOREX");
+static bool mayFoldFromLeftToRight(uint8_t LHS, uint8_t RHS) {
+  switch (LHS) {
+  default:
+    llvm_unreachable("Unexpected Form!");
+  case X86Local::MRM0r:
+    return RHS == X86Local::MRM0m;
+  case X86Local::MRM1r:
+    return RHS == X86Local::MRM1m;
+  case X86Local::MRM2r:
+    return RHS == X86Local::MRM2m;
+  case X86Local::MRM3r:
+    return RHS == X86Local::MRM3m;
+  case X86Local::MRM4r:
+    return RHS == X86Local::MRM4m;
+  case X86Local::MRM5r:
+    return RHS == X86Local::MRM5m;
+  case X86Local::MRM6r:
+    return RHS == X86Local::MRM6m;
+  case X86Local::MRM7r:
+    return RHS == X86Local::MRM7m;
+  case X86Local::MRMXr:
+    return RHS == X86Local::MRMXm;
+  case X86Local::MRMXrCC:
+    return RHS == X86Local::MRMXmCC;
+  case X86Local::MRMDestReg:
+    return RHS == X86Local::MRMDestMem;
+  case X86Local::MRMSrcReg:
+    return RHS == X86Local::MRMSrcMem;
+  case X86Local::MRMSrcReg4VOp3:
+    return RHS == X86Local::MRMSrcMem4VOp3;
+  case X86Local::MRMSrcRegOp4:
+    return RHS == X86Local::MRMSrcMemOp4;
+  case X86Local::MRMSrcRegCC:
+    return RHS == X86Local::MRMSrcMemCC;
+  }
 }
 
-// Get the alternative instruction pointed by "FoldGenRegForm" field.
-static inline const CodeGenInstruction *
-getAltRegInst(const CodeGenInstruction *I, const RecordKeeper &Records,
-              const CodeGenTarget &Target) {
-
-  StringRef AltRegInstStr = I->TheDef->getValueAsString("FoldGenRegForm");
-  Record *AltRegInstRec = Records.getDef(AltRegInstStr);
-  assert(AltRegInstRec &&
-         "Alternative register form instruction def not found");
-  CodeGenInstruction &AltRegInst = Target.getInstruction(AltRegInstRec);
-  return &AltRegInst;
+static bool isNOREXRegClass(const Record *Op) {
+  return Op->getName().contains("_NOREX");
 }
 
-// Function object - Operator() returns true if the given VEX instruction
-// matches the EVEX instruction of this object.
+// Function object - Operator() returns true if the given Reg instruction
+// matches the Mem instruction of this object.
 class IsMatch {
   const CodeGenInstruction *MemInst;
-  unsigned Variant;
+  const X86Disassembler::RecognizableInstrBase MemRI;
+  const unsigned Variant;
 
 public:
   IsMatch(const CodeGenInstruction *Inst, unsigned V)
-      : MemInst(Inst), Variant(V) {}
+      : MemInst(Inst), MemRI(*MemInst), Variant(V) {}
 
   bool operator()(const CodeGenInstruction *RegInst) {
     X86Disassembler::RecognizableInstrBase RegRI(*RegInst);
-    X86Disassembler::RecognizableInstrBase MemRI(*MemInst);
     const Record *RegRec = RegInst->TheDef;
     const Record *MemRec = MemInst->TheDef;
 
     // EVEX_B means different things for memory and register forms.
-    if (RegRI.HasEVEX_B != 0 || MemRI.HasEVEX_B != 0)
+    if (RegRI.HasEVEX_B || MemRI.HasEVEX_B)
       return false;
 
-    // Instruction's format - The register form's "Form" field should be
-    // the opposite of the memory form's "Form" field.
-    if (!areOppositeForms(RegRI.Form, MemRI.Form))
+    if (!mayFoldFromLeftToRight(RegRI.Form, MemRI.Form))
       return false;
 
     // X86 encoding is crazy, e.g
@@ -288,38 +316,32 @@ public:
         X86Disassembler::getMnemonic(RegInst, Variant))
       return false;
 
-    // Return false if one (at least) of the encoding fields of both
-    // instructions do not match.
-    if (RegRI.Encoding != MemRI.Encoding || RegRI.Opcode != MemRI.Opcode ||
-        RegRI.OpPrefix != MemRI.OpPrefix || RegRI.OpMap != MemRI.OpMap ||
-        RegRI.OpSize != MemRI.OpSize || RegRI.AdSize != MemRI.AdSize ||
-        RegRI.HasREX_W != MemRI.HasREX_W ||
-        RegRI.HasVEX_4V != MemRI.HasVEX_4V ||
-        RegRI.HasVEX_L != MemRI.HasVEX_L ||
-        RegRI.HasVEX_W != MemRI.HasVEX_W ||
-        RegRI.IgnoresVEX_L != MemRI.IgnoresVEX_L ||
-        RegRI.IgnoresVEX_W != MemRI.IgnoresVEX_W ||
-        RegRI.HasEVEX_K != MemRI.HasEVEX_K ||
-        RegRI.HasEVEX_KZ != MemRI.HasEVEX_KZ ||
-        RegRI.HasEVEX_L2 != MemRI.HasEVEX_L2 ||
-        RegRec->getValueAsBit("hasEVEX_RC") !=
-            MemRec->getValueAsBit("hasEVEX_RC") ||
-        RegRec->getValueAsBit("hasLockPrefix") !=
-            MemRec->getValueAsBit("hasLockPrefix") ||
-        RegRec->getValueAsBit("hasNoTrackPrefix") !=
-            MemRec->getValueAsBit("hasNoTrackPrefix") ||
-        RegRec->getValueAsBit("EVEX_W1_VEX_W0") !=
-            MemRec->getValueAsBit("EVEX_W1_VEX_W0"))
+    // Return false if any of the following fields of does not match.
+    if (std::make_tuple(RegRI.Encoding, RegRI.Opcode, RegRI.OpPrefix,
+                        RegRI.OpMap, RegRI.OpSize, RegRI.AdSize, RegRI.HasREX_W,
+                        RegRI.HasVEX_4V, RegRI.HasVEX_L, RegRI.IgnoresVEX_L,
+                        RegRI.IgnoresW, RegRI.HasEVEX_K, RegRI.HasEVEX_KZ,
+                        RegRI.HasEVEX_L2, RegRec->getValueAsBit("hasEVEX_RC"),
+                        RegRec->getValueAsBit("hasLockPrefix"),
+                        RegRec->getValueAsBit("hasNoTrackPrefix"),
+                        RegRec->getValueAsBit("EVEX_W1_VEX_W0")) !=
+        std::make_tuple(MemRI.Encoding, MemRI.Opcode, MemRI.OpPrefix,
+                        MemRI.OpMap, MemRI.OpSize, MemRI.AdSize, MemRI.HasREX_W,
+                        MemRI.HasVEX_4V, MemRI.HasVEX_L, MemRI.IgnoresVEX_L,
+                        MemRI.IgnoresW, MemRI.HasEVEX_K, MemRI.HasEVEX_KZ,
+                        MemRI.HasEVEX_L2, MemRec->getValueAsBit("hasEVEX_RC"),
+                        MemRec->getValueAsBit("hasLockPrefix"),
+                        MemRec->getValueAsBit("hasNoTrackPrefix"),
+                        MemRec->getValueAsBit("EVEX_W1_VEX_W0")))
       return false;
 
     // Make sure the sizes of the operands of both instructions suit each other.
     // This is needed for instructions with intrinsic version (_Int).
     // Where the only difference is the size of the operands.
-    // For example: VUCOMISDZrm and Int_VUCOMISDrm
+    // For example: VUCOMISDZrm and VUCOMISDrm_Int
     // Also for instructions that their EVEX version was upgraded to work with
     // k-registers. For example VPCMPEQBrm (xmm output register) and
     // VPCMPEQBZ128rm (k register output register).
-    bool ArgFolded = false;
     unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs();
     unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs();
     unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs();
@@ -330,59 +352,36 @@ public:
     unsigned RegStartIdx =
         (MemOutSize + 1 == RegOutSize) && (MemInSize == RegInSize) ? 1 : 0;
 
-    for (unsigned i = 0, e = MemInst->Operands.size(); i < e; i++) {
-      Record *MemOpRec = MemInst->Operands[i].Rec;
-      Record *RegOpRec = RegInst->Operands[i + RegStartIdx].Rec;
+    bool FoundFoldedOp = false;
+    for (unsigned I = 0, E = MemInst->Operands.size(); I != E; I++) {
+      Record *MemOpRec = MemInst->Operands[I].Rec;
+      Record *RegOpRec = RegInst->Operands[I + RegStartIdx].Rec;
 
       if (MemOpRec == RegOpRec)
         continue;
 
-      if (isRegisterOperand(MemOpRec) && isRegisterOperand(RegOpRec)) {
-        if (getRegOperandSize(MemOpRec) != getRegOperandSize(RegOpRec) ||
-            isNOREXRegClass(MemOpRec) != isNOREXRegClass(RegOpRec))
-          return false;
-      } else if (isMemoryOperand(MemOpRec) && isMemoryOperand(RegOpRec)) {
-        if (getMemOperandSize(MemOpRec) != getMemOperandSize(RegOpRec))
-          return false;
-      } else if (isImmediateOperand(MemOpRec) && isImmediateOperand(RegOpRec)) {
-        if (MemOpRec->getValueAsDef("Type") != RegOpRec->getValueAsDef("Type"))
-          return false;
-      } else {
-        // Only one operand can be folded.
-        if (ArgFolded)
-          return false;
+      if (isRegisterOperand(MemOpRec) && isRegisterOperand(RegOpRec) &&
+          ((getRegOperandSize(MemOpRec) != getRegOperandSize(RegOpRec)) ||
+           (isNOREXRegClass(MemOpRec) != isNOREXRegClass(RegOpRec))))
+        return false;
 
-        assert(isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec));
-        ArgFolded = true;
-      }
-    }
+      if (isMemoryOperand(MemOpRec) && isMemoryOperand(RegOpRec) &&
+          (getMemOperandSize(MemOpRec) != getMemOperandSize(RegOpRec)))
+        return false;
 
-    return true;
-  }
+      if (isImmediateOperand(MemOpRec) && isImmediateOperand(RegOpRec) &&
+          (MemOpRec->getValueAsDef("Type") != RegOpRec->getValueAsDef("Type")))
+        return false;
 
-private:
-  // Return true of the 2 given forms are the opposite of each other.
-  bool areOppositeForms(unsigned RegForm, unsigned MemForm) {
-    if ((MemForm == X86Local::MRM0m && RegForm == X86Local::MRM0r) ||
-        (MemForm == X86Local::MRM1m && RegForm == X86Local::MRM1r) ||
-        (MemForm == X86Local::MRM2m && RegForm == X86Local::MRM2r) ||
-        (MemForm == X86Local::MRM3m && RegForm == X86Local::MRM3r) ||
-        (MemForm == X86Local::MRM4m && RegForm == X86Local::MRM4r) ||
-        (MemForm == X86Local::MRM5m && RegForm == X86Local::MRM5r) ||
-        (MemForm == X86Local::MRM6m && RegForm == X86Local::MRM6r) ||
-        (MemForm == X86Local::MRM7m && RegForm == X86Local::MRM7r) ||
-        (MemForm == X86Local::MRMXm && RegForm == X86Local::MRMXr) ||
-        (MemForm == X86Local::MRMXmCC && RegForm == X86Local::MRMXrCC) ||
-        (MemForm == X86Local::MRMDestMem && RegForm == X86Local::MRMDestReg) ||
-        (MemForm == X86Local::MRMSrcMem && RegForm == X86Local::MRMSrcReg) ||
-        (MemForm == X86Local::MRMSrcMem4VOp3 &&
-         RegForm == X86Local::MRMSrcReg4VOp3) ||
-        (MemForm == X86Local::MRMSrcMemOp4 &&
-         RegForm == X86Local::MRMSrcRegOp4) ||
-        (MemForm == X86Local::MRMSrcMemCC && RegForm == X86Local::MRMSrcRegCC))
-      return true;
+      // Only one operand can be folded.
+      if (FoundFoldedOp)
+        return false;
 
-    return false;
+      assert(isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec));
+      FoundFoldedOp = true;
+    }
+
+    return FoundFoldedOp;
   }
 };
 
@@ -391,13 +390,23 @@ private:
 void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table,
                                              const CodeGenInstruction *RegInstr,
                                              const CodeGenInstruction *MemInstr,
-                                             const UnfoldStrategy S,
-                                             const unsigned int FoldedInd) {
+                                             uint16_t S, unsigned FoldedIdx,
+                                             bool isManual) {
 
   X86FoldTableEntry Result = X86FoldTableEntry(RegInstr, MemInstr);
   Record *RegRec = RegInstr->TheDef;
   Record *MemRec = MemInstr->TheDef;
 
+  if (isManual) {
+    Result.NoReverse = S & TB_NO_REVERSE;
+    Result.NoForward = S & TB_NO_FORWARD;
+    Result.FoldLoad = S & TB_FOLDED_LOAD;
+    Result.FoldStore = S & TB_FOLDED_STORE;
+    Result.Alignment = Align(1ULL << ((S & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT));
+    Table[RegInstr] = Result;
+    return;
+  }
+
   // Only table0 entries should explicitly specify a load or store flag.
   if (&Table == &Table0) {
     unsigned MemInOpsNum = MemRec->getValueAsDag("InOperandList")->getNumArgs();
@@ -408,48 +417,62 @@ void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table,
     // If the instruction reads from the folded operand, it well appear as in
     // input in both forms.
     if (MemInOpsNum == RegInOpsNum)
-      Result.IsLoad = true;
+      Result.FoldLoad = true;
     else
-      Result.IsStore = true;
+      Result.FoldStore = true;
   }
 
-  Record *RegOpRec = RegInstr->Operands[FoldedInd].Rec;
-  Record *MemOpRec = MemInstr->Operands[FoldedInd].Rec;
+  Record *RegOpRec = RegInstr->Operands[FoldedIdx].Rec;
+  Record *MemOpRec = MemInstr->Operands[FoldedIdx].Rec;
 
   // Unfolding code generates a load/store instruction according to the size of
   // the register in the register form instruction.
   // If the register's size is greater than the memory's operand size, do not
   // allow unfolding.
-  if (S == UNFOLD)
-    Result.CannotUnfold = false;
-  else if (S == NO_UNFOLD)
-    Result.CannotUnfold = true;
-  else if (getRegOperandSize(RegOpRec) > getMemOperandSize(MemOpRec))
-    Result.CannotUnfold = true; // S == NO_STRATEGY
 
-  uint64_t Enc = getValueFromBitsInit(RegRec->getValueAsBitsInit("OpEncBits"));
+  // the unfolded load size will be based on the register size. If that’s bigger
+  // than the memory operand size, the unfolded load will load more memory and
+  // potentially cause a memory fault.
+  if (getRegOperandSize(RegOpRec) > getMemOperandSize(MemOpRec))
+    Result.NoReverse = true;
+
+  // Check no-kz version's isMoveReg
+  StringRef RegInstName = RegRec->getName();
+  unsigned DropLen =
+      RegInstName.endswith("rkz") ? 2 : (RegInstName.endswith("rk") ? 1 : 0);
+  Record *BaseDef =
+      DropLen ? Records.getDef(RegInstName.drop_back(DropLen)) : nullptr;
+  bool IsMoveReg =
+      BaseDef ? Target.getInstruction(BaseDef).isMoveReg : RegInstr->isMoveReg;
+  // A masked load can not be unfolded to a full load, otherwise it would access
+  // unexpected memory. A simple store can not be unfolded.
+  if (IsMoveReg && (BaseDef || Result.FoldStore))
+    Result.NoReverse = true;
+
+  uint8_t Enc = byteFromBitsInit(RegRec->getValueAsBitsInit("OpEncBits"));
   if (isExplicitAlign(RegInstr)) {
     // The instruction require explicitly aligned memory.
     BitsInit *VectSize = RegRec->getValueAsBitsInit("VectSize");
-    uint64_t Value = getValueFromBitsInit(VectSize);
-    Result.IsAligned = true;
-    Result.Alignment = Value;
-  } else if (Enc != X86Local::XOP && Enc != X86Local::VEX &&
-             Enc != X86Local::EVEX) {
-    // Instructions with VEX encoding do not require alignment.
-    if (!isExplicitUnalign(RegInstr) && getMemOperandSize(MemOpRec) > 64) {
-      // SSE packed vector instructions require a 16 byte alignment.
-      Result.IsAligned = true;
-      Result.Alignment = 16;
-    }
+    Result.Alignment = Align(byteFromBitsInit(VectSize));
+  } else if (!Enc && !isExplicitUnalign(RegInstr) &&
+             getMemOperandSize(MemOpRec) > 64) {
+    // Instructions with XOP/VEX/EVEX encoding do not require alignment while
+    // SSE packed vector instructions require a 16 byte alignment.
+    Result.Alignment = Align(16);
   }
+  // Expand is only ever created as a masked instruction. It is not safe to
+  // unfold a masked expand because we don't know if it came from an expand load
+  // intrinsic or folding a plain load. If it is from a expand load intrinsic,
+  // Unfolding to plain load would read more elements and could trigger a fault.
+  if (RegRec->getName().contains("EXPAND"))
+    Result.NoReverse = true;
 
-  Table.push_back(Result);
+  Table[RegInstr] = Result;
 }
 
 void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr,
                                         const CodeGenInstruction *MemInstr,
-                                        const UnfoldStrategy S) {
+                                        uint16_t S, bool IsManual) {
 
   Record *RegRec = RegInstr->TheDef;
   Record *MemRec = MemInstr->TheDef;
@@ -459,8 +482,8 @@ void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr,
   unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs();
 
   // Instructions which Read-Modify-Write should be added to Table2Addr.
-  if (MemOutSize != RegOutSize && MemInSize == RegInSize) {
-    addEntryWithFlags(Table2Addr, RegInstr, MemInstr, S, 0);
+  if (!MemOutSize && RegOutSize == 1 && MemInSize == RegInSize) {
+    addEntryWithFlags(Table2Addr, RegInstr, MemInstr, S, 0, IsManual);
     return;
   }
 
@@ -477,19 +500,19 @@ void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr,
           isMemoryOperand(MemOpRec)) {
         switch (i) {
         case 0:
-          addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0);
+          addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0, IsManual);
           return;
         case 1:
-          addEntryWithFlags(Table1, RegInstr, MemInstr, S, 1);
+          addEntryWithFlags(Table1, RegInstr, MemInstr, S, 1, IsManual);
           return;
         case 2:
-          addEntryWithFlags(Table2, RegInstr, MemInstr, S, 2);
+          addEntryWithFlags(Table2, RegInstr, MemInstr, S, 2, IsManual);
           return;
         case 3:
-          addEntryWithFlags(Table3, RegInstr, MemInstr, S, 3);
+          addEntryWithFlags(Table3, RegInstr, MemInstr, S, 3, IsManual);
           return;
         case 4:
-          addEntryWithFlags(Table4, RegInstr, MemInstr, S, 4);
+          addEntryWithFlags(Table4, RegInstr, MemInstr, S, 4, IsManual);
           return;
         }
       }
@@ -506,12 +529,12 @@ void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr,
     Record *MemOpRec = MemInstr->Operands[RegOutSize - 1].Rec;
     if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec) &&
         getRegOperandSize(RegOpRec) == getMemOperandSize(MemOpRec))
-      addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0);
+      addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0, IsManual);
   }
 }
 
-void X86FoldTablesEmitter::run(formatted_raw_ostream &OS) {
-  emitSourceFileHeader("X86 fold tables", OS);
+void X86FoldTablesEmitter::run(raw_ostream &o) {
+  formatted_raw_ostream OS(o);
 
   // Holds all memory instructions
   std::vector<const CodeGenInstruction *> MemInsts;
@@ -526,7 +549,9 @@ void X86FoldTablesEmitter::run(formatted_raw_ostream &OS) {
     if (!Rec->isSubClassOf("X86Inst") || Rec->getValueAsBit("isAsmParserOnly"))
       continue;
 
-    // - Do not proceed if the instruction is marked as notMemoryFoldable.
+    if (NoFoldSet.find(Rec->getName()) != NoFoldSet.end())
+      continue;
+
     // - Instructions including RST register class operands are not relevant
     //   for memory folding (for further details check the explanation in
     //   lib/Target/X86/X86InstrFPStack.td file).
@@ -534,17 +559,18 @@ void X86FoldTablesEmitter::run(formatted_raw_ostream &OS) {
     //   class ptr_rc_tailcall, which can be of a size 32 or 64, to ensure
     //   safe mapping of these instruction we manually map them and exclude
     //   them from the automation.
-    if (Rec->getValueAsBit("isMemoryFoldable") == false ||
-        hasRSTRegClass(Inst) || hasPtrTailcallRegClass(Inst))
+    if (hasRSTRegClass(Inst) || hasPtrTailcallRegClass(Inst))
       continue;
 
     // Add all the memory form instructions to MemInsts, and all the register
-    // form instructions to RegInsts[Opc], where Opc in the opcode of each
+    // form instructions to RegInsts[Opc], where Opc is the opcode of each
     // instructions. this helps reducing the runtime of the backend.
-    if (hasMemoryFormat(Rec))
+    const BitsInit *FormBits = Rec->getValueAsBitsInit("FormBits");
+    uint8_t Form = byteFromBitsInit(FormBits);
+    if (mayFoldToForm(Form))
       MemInsts.push_back(Inst);
-    else if (hasRegisterFormat(Rec)) {
-      uint8_t Opc = getValueFromBitsInit(Rec->getValueAsBitsInit("Opcode"));
+    else if (mayFoldFromForm(Form)) {
+      uint8_t Opc = byteFromBitsInit(Rec->getValueAsBitsInit("Opcode"));
       RegInsts[Opc].push_back(Inst);
     }
   }
@@ -555,7 +581,7 @@ void X86FoldTablesEmitter::run(formatted_raw_ostream &OS) {
   // instruction.
   for (const CodeGenInstruction *MemInst : MemInsts) {
     uint8_t Opc =
-        getValueFromBitsInit(MemInst->TheDef->getValueAsBitsInit("Opcode"));
+        byteFromBitsInit(MemInst->TheDef->getValueAsBitsInit("Opcode"));
 
     auto RegInstsIt = RegInsts.find(Opc);
     if (RegInstsIt == RegInsts.end())
@@ -569,16 +595,13 @@ void X86FoldTablesEmitter::run(formatted_raw_ostream &OS) {
     auto Match = find_if(OpcRegInsts, IsMatch(MemInst, Variant));
     if (Match != OpcRegInsts.end()) {
       const CodeGenInstruction *RegInst = *Match;
-      // If the matched instruction has it's "FoldGenRegForm" set, map the
-      // memory form instruction to the register form instruction pointed by
-      // this field
-      if (RegInst->TheDef->isValueUnset("FoldGenRegForm")) {
-        updateTables(RegInst, MemInst);
-      } else {
-        const CodeGenInstruction *AltRegInst =
-            getAltRegInst(RegInst, Records, Target);
-        updateTables(AltRegInst, MemInst);
+      StringRef RegInstName = RegInst->TheDef->getName();
+      if (RegInstName.endswith("_REV") || RegInstName.endswith("_alt")) {
+        if (auto *RegAltRec = Records.getDef(RegInstName.drop_back(4))) {
+          RegInst = &Target.getInstruction(RegAltRec);
+        }
       }
+      updateTables(RegInst, MemInst);
       OpcRegInsts.erase(Match);
     }
   }
@@ -589,17 +612,23 @@ void X86FoldTablesEmitter::run(formatted_raw_ostream &OS) {
     Record *MemInstIter = Records.getDef(Entry.MemInstStr);
 
     updateTables(&(Target.getInstruction(RegInstIter)),
-                 &(Target.getInstruction(MemInstIter)), Entry.Strategy);
+                 &(Target.getInstruction(MemInstIter)), Entry.Strategy, true);
   }
 
-  // Sort the tables before printing.
-  llvm::sort(Table2Addr);
-  llvm::sort(Table0);
-  llvm::sort(Table1);
-  llvm::sort(Table2);
-  llvm::sort(Table3);
-  llvm::sort(Table4);
-
+#ifndef NDEBUG
+  auto CheckMemFoldTable = [](const FoldTable &Table) -> void {
+    for (const auto &Record : Table) {
+      auto &FoldEntry = Record.second;
+      FoldEntry.checkCorrectness();
+    }
+  };
+  CheckMemFoldTable(Table2Addr);
+  CheckMemFoldTable(Table0);
+  CheckMemFoldTable(Table1);
+  CheckMemFoldTable(Table2);
+  CheckMemFoldTable(Table3);
+  CheckMemFoldTable(Table4);
+#endif
   // Print all tables.
   printTable(Table2Addr, "Table2Addr", OS);
   printTable(Table0, "Table0", OS);
@@ -609,10 +638,5 @@ void X86FoldTablesEmitter::run(formatted_raw_ostream &OS) {
   printTable(Table4, "Table4", OS);
 }
 
-namespace llvm {
-
-void EmitX86FoldTables(RecordKeeper &RK, raw_ostream &o) {
-  formatted_raw_ostream OS(o);
-  X86FoldTablesEmitter(RK).run(OS);
-}
-} // namespace llvm
+static TableGen::Emitter::OptClass<X86FoldTablesEmitter>
+    X("gen-x86-fold-tables", "Generate X86 fold tables");
diff --git a/llvm/utils/TableGen/X86ManualFoldTables.def b/llvm/utils/TableGen/X86ManualFoldTables.def
new file mode 100644
index 000000000000..d949830b0988
--- /dev/null
+++ b/llvm/utils/TableGen/X86ManualFoldTables.def
@@ -0,0 +1,288 @@
+//===- X86ManualFoldTables.def ----------------------------*- C++ -*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// \file
+// This file defines all the entries in X86 memory folding tables that need
+// special handling.
+//===----------------------------------------------------------------------===//
+
+#ifndef NOFOLD
+#define NOFOLD(INSN)
+#endif
+NOFOLD(BTC16rr)
+NOFOLD(BTC32rr)
+NOFOLD(BTC64rr)
+NOFOLD(BTR16rr)
+NOFOLD(BTR32rr)
+NOFOLD(BTR64rr)
+NOFOLD(BTS16rr)
+NOFOLD(BTS32rr)
+NOFOLD(BTS64rr)
+NOFOLD(VCOMPRESSPDZ128rrk)
+NOFOLD(VCOMPRESSPDZ256rrk)
+NOFOLD(VCOMPRESSPDZrrk)
+NOFOLD(VCOMPRESSPSZ128rrk)
+NOFOLD(VCOMPRESSPSZ256rrk)
+NOFOLD(VCOMPRESSPSZrrk)
+NOFOLD(VCVTPS2PHZ128rrk)
+NOFOLD(VCVTPS2PHZ256rrk)
+NOFOLD(VCVTPS2PHZrrk)
+NOFOLD(VEXTRACTF32x4Z256rrk)
+NOFOLD(VEXTRACTF32x4Zrrk)
+NOFOLD(VEXTRACTF32x8Zrrk)
+NOFOLD(VEXTRACTF64x2Z256rrk)
+NOFOLD(VEXTRACTF64x2Zrrk)
+NOFOLD(VEXTRACTF64x4Zrrk)
+NOFOLD(VEXTRACTI32x4Z256rrk)
+NOFOLD(VEXTRACTI32x4Zrrk)
+NOFOLD(VEXTRACTI32x8Zrrk)
+NOFOLD(VEXTRACTI64x2Z256rrk)
+NOFOLD(VEXTRACTI64x2Zrrk)
+NOFOLD(VEXTRACTI64x4Zrrk)
+NOFOLD(VMOVAPDZ128mrk)
+NOFOLD(VMOVAPDZ256mrk)
+NOFOLD(VMOVAPDZmrk)
+NOFOLD(VMOVAPSZ128mrk)
+NOFOLD(VMOVAPSZ256mrk)
+NOFOLD(VMOVAPSZmrk)
+NOFOLD(VMOVDQA32Z128mrk)
+NOFOLD(VMOVDQA32Z256mrk)
+NOFOLD(VMOVDQA32Zmrk)
+NOFOLD(VMOVDQA64Z128mrk)
+NOFOLD(VMOVDQA64Z256mrk)
+NOFOLD(VMOVDQA64Zmrk)
+NOFOLD(VMOVDQU16Z128mrk)
+NOFOLD(VMOVDQU16Z256mrk)
+NOFOLD(VMOVDQU16Zmrk)
+NOFOLD(VMOVDQU32Z128mrk)
+NOFOLD(VMOVDQU32Z256mrk)
+NOFOLD(VMOVDQU32Zmrk)
+NOFOLD(VMOVDQU64Z128mrk)
+NOFOLD(VMOVDQU64Z256mrk)
+NOFOLD(VMOVDQU64Zmrk)
+NOFOLD(VMOVDQU8Z128mrk)
+NOFOLD(VMOVDQU8Z256mrk)
+NOFOLD(VMOVDQU8Zmrk)
+NOFOLD(VMOVUPDZ128mrk)
+NOFOLD(VMOVUPDZ256mrk)
+NOFOLD(VMOVUPDZmrk)
+NOFOLD(VMOVUPSZ128mrk)
+NOFOLD(VMOVUPSZ256mrk)
+NOFOLD(VMOVUPSZmrk)
+NOFOLD(VPCOMPRESSBZ128rrk)
+NOFOLD(VPCOMPRESSBZ256rrk)
+NOFOLD(VPCOMPRESSBZrrk)
+NOFOLD(VPCOMPRESSDZ128rrk)
+NOFOLD(VPCOMPRESSDZ256rrk)
+NOFOLD(VPCOMPRESSDZrrk)
+NOFOLD(VPCOMPRESSQZ128rrk)
+NOFOLD(VPCOMPRESSQZ256rrk)
+NOFOLD(VPCOMPRESSQZrrk)
+NOFOLD(VPCOMPRESSWZ128rrk)
+NOFOLD(VPCOMPRESSWZ256rrk)
+NOFOLD(VPCOMPRESSWZrrk)
+NOFOLD(VPMOVDBZ128rrk)
+NOFOLD(VPMOVDBZ256rrk)
+NOFOLD(VPMOVDBZrrk)
+NOFOLD(VPMOVDWZ128rrk)
+NOFOLD(VPMOVDWZ256rrk)
+NOFOLD(VPMOVDWZrrk)
+NOFOLD(VPMOVQBZ128rrk)
+NOFOLD(VPMOVQBZ256rrk)
+NOFOLD(VPMOVQBZrrk)
+NOFOLD(VPMOVQDZ128rrk)
+NOFOLD(VPMOVQDZ256rrk)
+NOFOLD(VPMOVQDZrrk)
+NOFOLD(VPMOVQWZ128rrk)
+NOFOLD(VPMOVQWZ256rrk)
+NOFOLD(VPMOVQWZrrk)
+NOFOLD(VPMOVSDBZ128rrk)
+NOFOLD(VPMOVSDBZ256rrk)
+NOFOLD(VPMOVSDBZrrk)
+NOFOLD(VPMOVSDWZ128rrk)
+NOFOLD(VPMOVSDWZ256rrk)
+NOFOLD(VPMOVSDWZrrk)
+NOFOLD(VPMOVSQBZ128rrk)
+NOFOLD(VPMOVSQBZ256rrk)
+NOFOLD(VPMOVSQBZrrk)
+NOFOLD(VPMOVSQDZ128rrk)
+NOFOLD(VPMOVSQDZ256rrk)
+NOFOLD(VPMOVSQDZrrk)
+NOFOLD(VPMOVSQWZ128rrk)
+NOFOLD(VPMOVSQWZ256rrk)
+NOFOLD(VPMOVSQWZrrk)
+NOFOLD(VPMOVSWBZ128rrk)
+NOFOLD(VPMOVSWBZ256rrk)
+NOFOLD(VPMOVSWBZrrk)
+NOFOLD(VPMOVUSDBZ128rrk)
+NOFOLD(VPMOVUSDBZ256rrk)
+NOFOLD(VPMOVUSDBZrrk)
+NOFOLD(VPMOVUSDWZ128rrk)
+NOFOLD(VPMOVUSDWZ256rrk)
+NOFOLD(VPMOVUSDWZrrk)
+NOFOLD(VPMOVUSQBZ128rrk)
+NOFOLD(VPMOVUSQBZ256rrk)
+NOFOLD(VPMOVUSQBZrrk)
+NOFOLD(VPMOVUSQDZ128rrk)
+NOFOLD(VPMOVUSQDZ256rrk)
+NOFOLD(VPMOVUSQDZrrk)
+NOFOLD(VPMOVUSQWZ128rrk)
+NOFOLD(VPMOVUSQWZ256rrk)
+NOFOLD(VPMOVUSQWZrrk)
+NOFOLD(VPMOVUSWBZ128rrk)
+NOFOLD(VPMOVUSWBZ256rrk)
+NOFOLD(VPMOVUSWBZrrk)
+NOFOLD(VPMOVWBZ128rrk)
+NOFOLD(VPMOVWBZ256rrk)
+NOFOLD(VPMOVWBZrrk)
+NOFOLD(ARPL16rr)
+NOFOLD(BT16rr)
+NOFOLD(BT32rr)
+NOFOLD(BT64rr)
+NOFOLD(CMPXCHG16rr)
+NOFOLD(CMPXCHG32rr)
+NOFOLD(CMPXCHG64rr)
+NOFOLD(CMPXCHG8rr)
+NOFOLD(LLDT16r)
+NOFOLD(LMSW16r)
+NOFOLD(LTRr)
+NOFOLD(NOOPLr)
+NOFOLD(NOOPQr)
+NOFOLD(NOOPWr)
+NOFOLD(POP16rmr)
+NOFOLD(POP32rmr)
+NOFOLD(POP64rmr)
+NOFOLD(PUSH16rmr)
+NOFOLD(PUSH32rmr)
+NOFOLD(PUSH64rmr)
+NOFOLD(VCOMPRESSPDZ128rr)
+NOFOLD(VCOMPRESSPDZ256rr)
+NOFOLD(VCOMPRESSPDZrr)
+NOFOLD(VCOMPRESSPSZ128rr)
+NOFOLD(VCOMPRESSPSZ256rr)
+NOFOLD(VCOMPRESSPSZrr)
+NOFOLD(VERRr)
+NOFOLD(VERWr)
+NOFOLD(VMREAD32rr)
+NOFOLD(VMREAD64rr)
+NOFOLD(VPCOMPRESSBZ128rr)
+NOFOLD(VPCOMPRESSBZ256rr)
+NOFOLD(VPCOMPRESSBZrr)
+NOFOLD(VPCOMPRESSDZ128rr)
+NOFOLD(VPCOMPRESSDZ256rr)
+NOFOLD(VPCOMPRESSDZrr)
+NOFOLD(VPCOMPRESSQZ128rr)
+NOFOLD(VPCOMPRESSQZ256rr)
+NOFOLD(VPCOMPRESSQZrr)
+NOFOLD(VPCOMPRESSWZ128rr)
+NOFOLD(VPCOMPRESSWZ256rr)
+NOFOLD(VPCOMPRESSWZrr)
+NOFOLD(LAR16rr)
+NOFOLD(LAR32rr)
+NOFOLD(LAR64rr)
+NOFOLD(LSL16rr)
+NOFOLD(LSL32rr)
+NOFOLD(LSL64rr)
+NOFOLD(MOVSX16rr16)
+NOFOLD(MOVZX16rr16)
+NOFOLD(VMWRITE32rr)
+NOFOLD(VMWRITE64rr)
+NOFOLD(VBLENDMPDZ128rrkz)
+NOFOLD(VBLENDMPDZ256rrkz)
+NOFOLD(VBLENDMPDZrrkz)
+NOFOLD(VBLENDMPSZ128rrkz)
+NOFOLD(VBLENDMPSZ256rrkz)
+NOFOLD(VBLENDMPSZrrkz)
+NOFOLD(VPBLENDMBZ128rrkz)
+NOFOLD(VPBLENDMBZ256rrkz)
+NOFOLD(VPBLENDMBZrrkz)
+NOFOLD(VPBLENDMDZ128rrkz)
+NOFOLD(VPBLENDMDZ256rrkz)
+NOFOLD(VPBLENDMDZrrkz)
+NOFOLD(VPBLENDMQZ128rrkz)
+NOFOLD(VPBLENDMQZ256rrkz)
+NOFOLD(VPBLENDMQZrrkz)
+NOFOLD(VPBLENDMWZ128rrkz)
+NOFOLD(VPBLENDMWZ256rrkz)
+NOFOLD(VPBLENDMWZrrkz)
+NOFOLD(UD1Lr)
+NOFOLD(UD1Qr)
+NOFOLD(UD1Wr)
+// Exclude these two b/c they would conflict with {MMX_MOVD64from64rr, MMX_MOVQ64mr} in unfolding table
+NOFOLD(MMX_MOVQ64rr)
+NOFOLD(MMX_MOVQ64rr_REV)
+// INSERTPSrm has no count_s while INSERTPSrr has count_s.
+// count_s is to indicate which element in dst vector is inserted.
+// if count_s!=0, we can not fold INSERTPSrr into INSERTPSrm
+//
+// the following folding can happen when count_s==0
+// load xmm0, m32
+// insertpsrr xmm1, xmm0, imm
+// =>
+// insertpsrm xmm1, m32, imm
+NOFOLD(INSERTPSrr)
+#undef NOFOLD
+
+#ifndef ENTRY
+#define ENTRY(REG, MEM, FLAGS)
+#endif
+// The following entries are added manually b/c the encodings of reg form does not match the
+// encoding of memory form
+ENTRY(ADD16ri_DB, ADD16mi, TB_NO_REVERSE)
+ENTRY(ADD16rr_DB, ADD16mr, TB_NO_REVERSE)
+ENTRY(ADD32ri_DB, ADD32mi, TB_NO_REVERSE)
+ENTRY(ADD32rr_DB, ADD32mr, TB_NO_REVERSE)
+ENTRY(ADD64ri32_DB, ADD64mi32, TB_NO_REVERSE)
+ENTRY(ADD64rr_DB, ADD64mr, TB_NO_REVERSE)
+ENTRY(ADD8ri_DB, ADD8mi, TB_NO_REVERSE)
+ENTRY(ADD8rr_DB, ADD8mr, TB_NO_REVERSE)
+ENTRY(ADD16rr_DB, ADD16rm, TB_NO_REVERSE)
+ENTRY(ADD32rr_DB, ADD32rm, TB_NO_REVERSE)
+ENTRY(ADD64rr_DB, ADD64rm, TB_NO_REVERSE)
+ENTRY(ADD8rr_DB, ADD8rm, TB_NO_REVERSE)
+ENTRY(MMX_MOVD64from64rr, MMX_MOVQ64mr, TB_FOLDED_STORE)
+ENTRY(MMX_MOVD64grr, MMX_MOVD64mr, TB_FOLDED_STORE)
+ENTRY(MOV64toSDrr, MOV64mr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(MOVDI2SSrr, MOV32mr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(MOVPQIto64rr, MOVPQI2QImr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(MOVSDto64rr, MOVSDmr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(MOVSS2DIrr, MOVSSmr, TB_FOLDED_STORE)
+ENTRY(MOVLHPSrr, MOVHPSrm, TB_NO_REVERSE)
+ENTRY(PUSH16r, PUSH16rmm, TB_FOLDED_LOAD)
+ENTRY(PUSH32r, PUSH32rmm, TB_FOLDED_LOAD)
+ENTRY(PUSH64r, PUSH64rmm, TB_FOLDED_LOAD)
+ENTRY(TAILJMPr, TAILJMPm, TB_FOLDED_LOAD)
+ENTRY(TAILJMPr64, TAILJMPm64, TB_FOLDED_LOAD)
+ENTRY(TAILJMPr64_REX, TAILJMPm64_REX, TB_FOLDED_LOAD)
+ENTRY(TCRETURNri, TCRETURNmi, TB_FOLDED_LOAD | TB_NO_FORWARD)
+ENTRY(TCRETURNri64, TCRETURNmi64, TB_FOLDED_LOAD | TB_NO_FORWARD)
+ENTRY(VMOVLHPSZrr, VMOVHPSZ128rm, TB_NO_REVERSE)
+ENTRY(VMOVLHPSrr, VMOVHPSrm, TB_NO_REVERSE)
+ENTRY(VMOV64toSDZrr, MOV64mr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(VMOV64toSDrr, MOV64mr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(VMOVDI2SSZrr, MOV32mr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(VMOVDI2SSrr, MOV32mr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(VMOVPQIto64Zrr, VMOVPQI2QIZmr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(VMOVPQIto64rr, VMOVPQI2QImr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(VMOVSDto64Zrr, VMOVSDZmr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(VMOVSDto64rr, VMOVSDmr, TB_FOLDED_STORE | TB_NO_REVERSE)
+ENTRY(VMOVSS2DIZrr, VMOVSSZmr, TB_FOLDED_STORE)
+ENTRY(VMOVSS2DIrr, VMOVSSmr, TB_FOLDED_STORE)
+ENTRY(MMX_MOVD64to64rr, MMX_MOVQ64rm, 0)
+ENTRY(MOV64toPQIrr, MOVQI2PQIrm, TB_NO_REVERSE)
+ENTRY(MOV64toSDrr, MOVSDrm_alt, TB_NO_REVERSE)
+ENTRY(MOVDI2SSrr, MOVSSrm_alt, 0)
+ENTRY(VMOV64toPQIZrr, VMOVQI2PQIZrm, TB_NO_REVERSE)
+ENTRY(VMOV64toPQIrr, VMOVQI2PQIrm, TB_NO_REVERSE)
+ENTRY(VMOV64toSDZrr, VMOVSDZrm_alt, TB_NO_REVERSE)
+ENTRY(VMOV64toSDrr, VMOVSDrm_alt, TB_NO_REVERSE)
+ENTRY(VMOVDI2SSZrr, VMOVSSZrm_alt, 0)
+ENTRY(VMOVDI2SSrr, VMOVSSrm_alt, 0)
+ENTRY(MOVSDrr, MOVLPDrm, TB_NO_REVERSE)
+ENTRY(VMOVSDZrr, VMOVLPDZ128rm, TB_NO_REVERSE)
+ENTRY(VMOVSDrr, VMOVLPDrm, TB_NO_REVERSE)
+#undef ENTRY
diff --git a/llvm/utils/TableGen/X86MnemonicTables.cpp b/llvm/utils/TableGen/X86MnemonicTables.cpp
index f405e051e355..aeafee157462 100644
--- a/llvm/utils/TableGen/X86MnemonicTables.cpp
+++ b/llvm/utils/TableGen/X86MnemonicTables.cpp
@@ -14,7 +14,7 @@
 #include "CodeGenInstruction.h"
 #include "CodeGenTarget.h"
 #include "X86RecognizableInstr.h"
-#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 
 using namespace llvm;
@@ -87,8 +87,5 @@ void X86MnemonicTablesEmitter::run(raw_ostream &OS) {
 
 } // namespace
 
-namespace llvm {
-void EmitX86MnemonicTables(RecordKeeper &RK, raw_ostream &OS) {
-  X86MnemonicTablesEmitter(RK).run(OS);
-}
-} // namespace llvm
+static TableGen::Emitter::OptClass<X86MnemonicTablesEmitter>
+    X("gen-x86-mnemonic-tables", "Generate X86 mnemonic tables");
diff --git a/llvm/utils/TableGen/X86ModRMFilters.h b/llvm/utils/TableGen/X86ModRMFilters.h
index e2d0907b4f8b..d2169a8e879b 100644
--- a/llvm/utils/TableGen/X86ModRMFilters.h
+++ b/llvm/utils/TableGen/X86ModRMFilters.h
@@ -17,7 +17,7 @@
 #ifndef LLVM_UTILS_TABLEGEN_X86MODRMFILTERS_H
 #define LLVM_UTILS_TABLEGEN_X86MODRMFILTERS_H
 
-#include "llvm/Support/DataTypes.h"
+#include <cstdint>
 
 namespace llvm {
 
diff --git a/llvm/utils/TableGen/X86RecognizableInstr.cpp b/llvm/utils/TableGen/X86RecognizableInstr.cpp
index e5c1e53936f6..b2f51ba01689 100644
--- a/llvm/utils/TableGen/X86RecognizableInstr.cpp
+++ b/llvm/utils/TableGen/X86RecognizableInstr.cpp
@@ -118,8 +118,7 @@ RecognizableInstrBase::RecognizableInstrBase(const CodeGenInstruction &insn) {
   AdSize = byteFromRec(Rec, "AdSizeBits");
   HasREX_W = Rec->getValueAsBit("hasREX_W");
   HasVEX_4V = Rec->getValueAsBit("hasVEX_4V");
-  HasVEX_W = Rec->getValueAsBit("HasVEX_W");
-  IgnoresVEX_W = Rec->getValueAsBit("IgnoresVEX_W");
+  IgnoresW = Rec->getValueAsBit("IgnoresW");
   IgnoresVEX_L = Rec->getValueAsBit("ignoresVEX_L");
   HasEVEX_L2 = Rec->getValueAsBit("hasEVEX_L2");
   HasEVEX_K = Rec->getValueAsBit("hasEVEX_K");
@@ -189,7 +188,7 @@ InstructionContext RecognizableInstr::insnContext() const {
       llvm_unreachable("Don't support VEX.L if EVEX_L2 is enabled");
     }
     // VEX_L & VEX_W
-    if (!EncodeRC && HasVEX_L && HasVEX_W) {
+    if (!EncodeRC && HasVEX_L && HasREX_W) {
       if (OpPrefix == X86Local::PD)
         insnContext = EVEX_KB(IC_EVEX_L_W_OPSIZE);
       else if (OpPrefix == X86Local::XS)
@@ -216,7 +215,7 @@ InstructionContext RecognizableInstr::insnContext() const {
         errs() << "Instruction does not use a prefix: " << Name << "\n";
         llvm_unreachable("Invalid prefix");
       }
-    } else if (!EncodeRC && HasEVEX_L2 && HasVEX_W) {
+    } else if (!EncodeRC && HasEVEX_L2 && HasREX_W) {
       // EVEX_L2 & VEX_W
       if (OpPrefix == X86Local::PD)
         insnContext = EVEX_KB(IC_EVEX_L2_W_OPSIZE);
@@ -245,7 +244,7 @@ InstructionContext RecognizableInstr::insnContext() const {
         llvm_unreachable("Invalid prefix");
       }
     }
-    else if (HasVEX_W) {
+    else if (HasREX_W) {
       // VEX_W
       if (OpPrefix == X86Local::PD)
         insnContext = EVEX_KB(IC_EVEX_W_OPSIZE);
@@ -275,7 +274,7 @@ InstructionContext RecognizableInstr::insnContext() const {
     }
     /// eof EVEX
   } else if (Encoding == X86Local::VEX || Encoding == X86Local::XOP) {
-    if (HasVEX_L && HasVEX_W) {
+    if (HasVEX_L && HasREX_W) {
       if (OpPrefix == X86Local::PD)
         insnContext = IC_VEX_L_W_OPSIZE;
       else if (OpPrefix == X86Local::XS)
@@ -290,7 +289,7 @@ InstructionContext RecognizableInstr::insnContext() const {
       }
     } else if (OpPrefix == X86Local::PD && HasVEX_L)
       insnContext = IC_VEX_L_OPSIZE;
-    else if (OpPrefix == X86Local::PD && HasVEX_W)
+    else if (OpPrefix == X86Local::PD && HasREX_W)
       insnContext = IC_VEX_W_OPSIZE;
     else if (OpPrefix == X86Local::PD)
       insnContext = IC_VEX_OPSIZE;
@@ -298,11 +297,11 @@ InstructionContext RecognizableInstr::insnContext() const {
       insnContext = IC_VEX_L_XS;
     else if (HasVEX_L && OpPrefix == X86Local::XD)
       insnContext = IC_VEX_L_XD;
-    else if (HasVEX_W && OpPrefix == X86Local::XS)
+    else if (HasREX_W && OpPrefix == X86Local::XS)
       insnContext = IC_VEX_W_XS;
-    else if (HasVEX_W && OpPrefix == X86Local::XD)
+    else if (HasREX_W && OpPrefix == X86Local::XD)
       insnContext = IC_VEX_W_XD;
-    else if (HasVEX_W && OpPrefix == X86Local::PS)
+    else if (HasREX_W && OpPrefix == X86Local::PS)
       insnContext = IC_VEX_W;
     else if (HasVEX_L && OpPrefix == X86Local::PS)
       insnContext = IC_VEX_L;
@@ -532,7 +531,7 @@ void RecognizableInstr::emitInstructionSpecifier() {
     // Operand 3 (optional) is an immediate.
     assert(numPhysicalOperands >= 2 + additionalOperands &&
            numPhysicalOperands <= 3 + additionalOperands &&
-           "Unexpected number of operands for MRMDestRegFrm");
+           "Unexpected number of operands for MRMDestReg");
 
     HANDLE_OPERAND(rmRegister)
     if (HasEVEX_K)
@@ -883,11 +882,11 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const {
       tables.setTableFields(*opcodeType, insnContext(), currentOpcode, *filter,
                             UID, Is32Bit, OpPrefix == 0,
                             IgnoresVEX_L || EncodeRC,
-                            IgnoresVEX_W, AddressSize);
+                            IgnoresW, AddressSize);
   } else {
     tables.setTableFields(*opcodeType, insnContext(), opcodeToSet, *filter, UID,
                           Is32Bit, OpPrefix == 0, IgnoresVEX_L || EncodeRC,
-                          IgnoresVEX_W, AddressSize);
+                          IgnoresW, AddressSize);
   }
 
 #undef MAP
@@ -955,6 +954,9 @@ OperandType RecognizableInstr::typeFromString(const std::string &s,
   TYPE("i128mem",             TYPE_M)
   TYPE("i256mem",             TYPE_M)
   TYPE("i512mem",             TYPE_M)
+  TYPE("i512mem_GR16",        TYPE_M)
+  TYPE("i512mem_GR32",        TYPE_M)
+  TYPE("i512mem_GR64",        TYPE_M)
   TYPE("i64i32imm_brtarget",  TYPE_REL)
   TYPE("i16imm_brtarget",     TYPE_REL)
   TYPE("i32imm_brtarget",     TYPE_REL)
@@ -1221,6 +1223,9 @@ RecognizableInstr::memoryEncodingFromString(const std::string &s,
   ENCODING("i128mem",         ENCODING_RM)
   ENCODING("i256mem",         ENCODING_RM)
   ENCODING("i512mem",         ENCODING_RM)
+  ENCODING("i512mem_GR16",    ENCODING_RM)
+  ENCODING("i512mem_GR32",    ENCODING_RM)
+  ENCODING("i512mem_GR64",    ENCODING_RM)
   ENCODING("f80mem",          ENCODING_RM)
   ENCODING("lea64_32mem",     ENCODING_RM)
   ENCODING("lea64mem",        ENCODING_RM)
diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h
index ea56a9d7d994..5efacdb27465 100644
--- a/llvm/utils/TableGen/X86RecognizableInstr.h
+++ b/llvm/utils/TableGen/X86RecognizableInstr.h
@@ -17,8 +17,10 @@
 #define LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H
 
 #include "CodeGenInstruction.h"
-#include "llvm/Support/DataTypes.h"
 #include "llvm/Support/X86DisassemblerDecoderCommon.h"
+#include <cstdint>
+#include <string>
+#include <vector>
 
 struct InstructionSpecifier;
 
@@ -180,10 +182,8 @@ struct RecognizableInstrBase {
   bool HasREX_W;
   /// The hasVEX_4V field from the record
   bool HasVEX_4V;
-  /// The HasVEX_WPrefix field from the record
-  bool HasVEX_W;
-  /// The IgnoresVEX_W field from the record
-  bool IgnoresVEX_W;
+  /// The IgnoresW field from the record
+  bool IgnoresW;
   /// The hasVEX_L field from the record
   bool HasVEX_L;
   /// The ignoreVEX_L field from the record