8 files changed, 314 insertions, 63 deletions

diff --git a/utils/TableGen/CodeGenDAGPatterns.cpp b/utils/TableGen/CodeGenDAGPatterns.cpp
index 03914ef98952..e48ba3845326 100644
--- a/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -3837,11 +3837,11 @@ void CodeGenDAGPatterns::GenerateVariants() {
       if (AlreadyExists) continue;
 
       // Otherwise, add it to the list of patterns we have.
-      PatternsToMatch.emplace_back(
+      PatternsToMatch.push_back(PatternToMatch(
           PatternsToMatch[i].getSrcRecord(), PatternsToMatch[i].getPredicates(),
           Variant, PatternsToMatch[i].getDstPattern(),
           PatternsToMatch[i].getDstRegs(),
-          PatternsToMatch[i].getAddedComplexity(), Record::getNewUID());
+          PatternsToMatch[i].getAddedComplexity(), Record::getNewUID()));
     }
 
     DEBUG(errs() << "\n");
diff --git a/utils/TableGen/CodeGenRegisters.cpp b/utils/TableGen/CodeGenRegisters.cpp
index 3907336221a4..d4a21a986c58 100644
--- a/utils/TableGen/CodeGenRegisters.cpp
+++ b/utils/TableGen/CodeGenRegisters.cpp
@@ -915,6 +915,84 @@ void CodeGenRegisterClass::computeSubClasses(CodeGenRegBank &RegBank) {
       RC.inheritProperties(RegBank);
 }
 
+Optional<std::pair<CodeGenRegisterClass *, CodeGenRegisterClass *>>
+CodeGenRegisterClass::getMatchingSubClassWithSubRegs(
+    CodeGenRegBank &RegBank, const CodeGenSubRegIndex *SubIdx) const {
+  auto SizeOrder = [](const CodeGenRegisterClass *A,
+                      const CodeGenRegisterClass *B) {
+    return A->getMembers().size() > B->getMembers().size();
+  };
+
+  auto &RegClasses = RegBank.getRegClasses();
+
+  // Find all the subclasses of this one that fully support the sub-register
+  // index and order them by size. BiggestSuperRC should always be first.
+  CodeGenRegisterClass *BiggestSuperRegRC = getSubClassWithSubReg(SubIdx);
+  if (!BiggestSuperRegRC)
+    return None;
+  BitVector SuperRegRCsBV = BiggestSuperRegRC->getSubClasses();
+  std::vector<CodeGenRegisterClass *> SuperRegRCs;
+  for (auto &RC : RegClasses)
+    if (SuperRegRCsBV[RC.EnumValue])
+      SuperRegRCs.emplace_back(&RC);
+  std::sort(SuperRegRCs.begin(), SuperRegRCs.end(), SizeOrder);
+  assert(SuperRegRCs.front() == BiggestSuperRegRC && "Biggest class wasn't first");
+
+  // Find all the subreg classes and order them by size too.
+  std::vector<std::pair<CodeGenRegisterClass *, BitVector>> SuperRegClasses;
+  for (auto &RC: RegClasses) {
+    BitVector SuperRegClassesBV(RegClasses.size());
+    RC.getSuperRegClasses(SubIdx, SuperRegClassesBV);
+    if (SuperRegClassesBV.any())
+      SuperRegClasses.push_back(std::make_pair(&RC, SuperRegClassesBV));
+  }
+  std::sort(SuperRegClasses.begin(), SuperRegClasses.end(),
+            [&](const std::pair<CodeGenRegisterClass *, BitVector> &A,
+                const std::pair<CodeGenRegisterClass *, BitVector> &B) {
+              return SizeOrder(A.first, B.first);
+            });
+
+  // Find the biggest subclass and subreg class such that R:subidx is in the
+  // subreg class for all R in subclass.
+  //
+  // For example:
+  // All registers in X86's GR64 have a sub_32bit subregister but no class
+  // exists that contains all the 32-bit subregisters because GR64 contains RIP
+  // but GR32 does not contain EIP. Instead, we constrain SuperRegRC to
+  // GR32_with_sub_8bit (which is identical to GR32_with_sub_32bit) and then,
+  // having excluded RIP, we are able to find a SubRegRC (GR32).
+  CodeGenRegisterClass *ChosenSuperRegClass = nullptr;
+  CodeGenRegisterClass *SubRegRC = nullptr;
+  for (auto *SuperRegRC : SuperRegRCs) {
+    for (const auto &SuperRegClassPair : SuperRegClasses) {
+      const BitVector &SuperRegClassBV = SuperRegClassPair.second;
+      if (SuperRegClassBV[SuperRegRC->EnumValue]) {
+        SubRegRC = SuperRegClassPair.first;
+        ChosenSuperRegClass = SuperRegRC;
+
+        // If SubRegRC is bigger than SuperRegRC then there are members of
+        // SubRegRC that don't have super registers via SubIdx. Keep looking to
+        // find a better fit and fall back on this one if there isn't one.
+        //
+        // This is intended to prevent X86 from making odd choices such as
+        // picking LOW32_ADDR_ACCESS_RBP instead of GR32 in the example above.
+        // LOW32_ADDR_ACCESS_RBP is a valid choice but contains registers that
+        // aren't subregisters of SuperRegRC whereas GR32 has a direct 1:1
+        // mapping.
+        if (SuperRegRC->getMembers().size() >= SubRegRC->getMembers().size())
+          return std::make_pair(ChosenSuperRegClass, SubRegRC);
+      }
+    }
+
+    // If we found a fit but it wasn't quite ideal because SubRegRC had excess
+    // registers, then we're done.
+    if (ChosenSuperRegClass)
+      return std::make_pair(ChosenSuperRegClass, SubRegRC);
+  }
+
+  return None;
+}
+
 void CodeGenRegisterClass::getSuperRegClasses(const CodeGenSubRegIndex *SubIdx,
                                               BitVector &Out) const {
   auto FindI = SuperRegClasses.find(SubIdx);
diff --git a/utils/TableGen/CodeGenRegisters.h b/utils/TableGen/CodeGenRegisters.h
index 1fcba8a135d1..d0f96a035ea1 100644
--- a/utils/TableGen/CodeGenRegisters.h
+++ b/utils/TableGen/CodeGenRegisters.h
@@ -329,6 +329,9 @@ namespace llvm {
     const std::string &getName() const { return Name; }
     std::string getQualifiedName() const;
     ArrayRef<MVT::SimpleValueType> getValueTypes() const {return VTs;}
+    bool hasValueType(MVT::SimpleValueType VT) const {
+      return std::find(VTs.begin(), VTs.end(), VT) != VTs.end();
+    }
     unsigned getNumValueTypes() const { return VTs.size(); }
 
     MVT::SimpleValueType getValueTypeNum(unsigned VTNum) const {
@@ -360,6 +363,18 @@ namespace llvm {
       return SubClassWithSubReg.lookup(SubIdx);
     }
 
+    /// Find largest subclass where all registers have SubIdx subregisters in
+    /// SubRegClass and the largest subregister class that contains those
+    /// subregisters without (as far as possible) also containing additional registers.
+    ///
+    /// This can be used to find a suitable pair of classes for subregister copies.
+    /// \return std::pair<SubClass, SubRegClass> where SubClass is a SubClass is
+    /// a class where every register has SubIdx and SubRegClass is a class where
+    /// every register is covered by the SubIdx subregister of SubClass.
+    Optional<std::pair<CodeGenRegisterClass *, CodeGenRegisterClass *>>
+    getMatchingSubClassWithSubRegs(CodeGenRegBank &RegBank,
+                                   const CodeGenSubRegIndex *SubIdx) const;
+
     void setSubClassWithSubReg(const CodeGenSubRegIndex *SubIdx,
                                CodeGenRegisterClass *SubRC) {
       SubClassWithSubReg[SubIdx] = SubRC;
@@ -370,7 +385,7 @@ namespace llvm {
     void getSuperRegClasses(const CodeGenSubRegIndex *SubIdx,
                             BitVector &Out) const;
 
-    // addSuperRegClass - Add a class containing only SudIdx super-registers.
+    // addSuperRegClass - Add a class containing only SubIdx super-registers.
     void addSuperRegClass(CodeGenSubRegIndex *SubIdx,
                           CodeGenRegisterClass *SuperRC) {
       SuperRegClasses[SubIdx].insert(SuperRC);
diff --git a/utils/TableGen/CodeGenSchedule.cpp b/utils/TableGen/CodeGenSchedule.cpp
index 20f6047052ff..50569b2ad989 100644
--- a/utils/TableGen/CodeGenSchedule.cpp
+++ b/utils/TableGen/CodeGenSchedule.cpp
@@ -140,6 +140,7 @@ CodeGenSchedModels::CodeGenSchedModels(RecordKeeper &RK,
 
   // Populate each CodeGenProcModel's WriteResDefs, ReadAdvanceDefs, and
   // ProcResourceDefs.
+  DEBUG(dbgs() << "\n+++ RESOURCE DEFINITIONS (collectProcResources) +++\n");
   collectProcResources();
 
   checkCompleteness();
@@ -160,6 +161,7 @@ void CodeGenSchedModels::collectProcModels() {
   ProcModelMap[NoModelDef] = 0;
 
   // For each processor, find a unique machine model.
+  DEBUG(dbgs() << "+++ PROCESSOR MODELs (addProcModel) +++\n");
   for (unsigned i = 0, N = ProcRecords.size(); i < N; ++i)
     addProcModel(ProcRecords[i]);
 }
@@ -315,6 +317,7 @@ void CodeGenSchedModels::collectSchedRW() {
     RW.Aliases.push_back(*AI);
   }
   DEBUG(
+    dbgs() << "\n+++ SCHED READS and WRITES (collectSchedRW) +++\n";
     for (unsigned WIdx = 0, WEnd = SchedWrites.size(); WIdx != WEnd; ++WIdx) {
       dbgs() << WIdx << ": ";
       SchedWrites[WIdx].dump();
@@ -531,6 +534,7 @@ void CodeGenSchedModels::collectSchedClasses() {
   // Create classes for InstRW defs.
   RecVec InstRWDefs = Records.getAllDerivedDefinitions("InstRW");
   std::sort(InstRWDefs.begin(), InstRWDefs.end(), LessRecord());
+  DEBUG(dbgs() << "\n+++ SCHED CLASSES (createInstRWClass) +++\n");
   for (RecIter OI = InstRWDefs.begin(), OE = InstRWDefs.end(); OI != OE; ++OI)
     createInstRWClass(*OI);
 
@@ -541,6 +545,7 @@ void CodeGenSchedModels::collectSchedClasses() {
   if (!EnableDump)
     return;
 
+  dbgs() << "\n+++ ITINERARIES and/or MACHINE MODELS (collectSchedClasses) +++\n";
   for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
     StringRef InstName = Inst->TheDef->getName();
     unsigned SCIdx = InstrClassMap.lookup(Inst->TheDef);
@@ -790,6 +795,7 @@ bool CodeGenSchedModels::hasItineraries() const {
 
 // Gather the processor itineraries.
 void CodeGenSchedModels::collectProcItins() {
+  DEBUG(dbgs() << "\n+++ PROBLEM ITINERARIES (collectProcItins) +++\n");
   for (CodeGenProcModel &ProcModel : ProcModels) {
     if (!ProcModel.hasItineraries())
       continue;
@@ -860,6 +866,7 @@ void CodeGenSchedModels::collectProcUnsupportedFeatures() {
 /// Infer new classes from existing classes. In the process, this may create new
 /// SchedWrites from sequences of existing SchedWrites.
 void CodeGenSchedModels::inferSchedClasses() {
+  DEBUG(dbgs() << "\n+++ INFERRING SCHED CLASSES (inferSchedClasses) +++\n");
   DEBUG(dbgs() << NumInstrSchedClasses << " instr sched classes.\n");
 
   // Visit all existing classes and newly created classes.
diff --git a/utils/TableGen/DAGISelMatcher.h b/utils/TableGen/DAGISelMatcher.h
index 6bda9ca5f96f..c672b0acac9f 100644
--- a/utils/TableGen/DAGISelMatcher.h
+++ b/utils/TableGen/DAGISelMatcher.h
@@ -208,7 +208,7 @@ public:
     Children.resize(NC);
   }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == Scope;
   }
 
@@ -233,7 +233,7 @@ public:
   const std::string &getWhatFor() const { return WhatFor; }
   unsigned getResultNo() const { return ResultNo; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == RecordNode;
   }
 
@@ -265,7 +265,7 @@ public:
   const std::string &getWhatFor() const { return WhatFor; }
   unsigned getResultNo() const { return ResultNo; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == RecordChild;
   }
 
@@ -281,7 +281,7 @@ class RecordMemRefMatcher : public Matcher {
 public:
   RecordMemRefMatcher() : Matcher(RecordMemRef) {}
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == RecordMemRef;
   }
 
@@ -297,7 +297,7 @@ class CaptureGlueInputMatcher : public Matcher {
 public:
   CaptureGlueInputMatcher() : Matcher(CaptureGlueInput) {}
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CaptureGlueInput;
   }
 
@@ -315,7 +315,7 @@ public:
 
   unsigned getChildNo() const { return ChildNo; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == MoveChild;
   }
 
@@ -332,7 +332,7 @@ class MoveParentMatcher : public Matcher {
 public:
   MoveParentMatcher() : Matcher(MoveParent) {}
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == MoveParent;
   }
 
@@ -352,7 +352,7 @@ public:
 
   unsigned getMatchNumber() const { return MatchNumber; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckSame;
   }
 
@@ -376,7 +376,7 @@ public:
   unsigned getChildNo() const { return ChildNo; }
   unsigned getMatchNumber() const { return MatchNumber; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckChildSame;
   }
 
@@ -399,7 +399,7 @@ public:
 
   StringRef getPredicate() const { return Predicate; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckPatternPredicate;
   }
 
@@ -419,7 +419,7 @@ public:
 
   TreePredicateFn getPredicate() const;
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckPredicate;
   }
 
@@ -441,7 +441,7 @@ public:
 
   const SDNodeInfo &getOpcode() const { return Opcode; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckOpcode;
   }
 
@@ -462,7 +462,7 @@ public:
     : Matcher(SwitchOpcode), Cases(cases.begin(), cases.end()) {}
   ~SwitchOpcodeMatcher() override;
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == SwitchOpcode;
   }
 
@@ -489,7 +489,7 @@ public:
   MVT::SimpleValueType getType() const { return Type; }
   unsigned getResNo() const { return ResNo; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckType;
   }
 
@@ -512,7 +512,7 @@ public:
   : Matcher(SwitchType), Cases(cases.begin(), cases.end()) {}
   ~SwitchTypeMatcher() override;
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == SwitchType;
   }
 
@@ -540,7 +540,7 @@ public:
   unsigned getChildNo() const { return ChildNo; }
   MVT::SimpleValueType getType() const { return Type; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckChildType;
   }
 
@@ -564,7 +564,7 @@ public:
 
   int64_t getValue() const { return Value; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckInteger;
   }
 
@@ -588,7 +588,7 @@ public:
   unsigned getChildNo() const { return ChildNo; }
   int64_t getValue() const { return Value; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckChildInteger;
   }
 
@@ -611,7 +611,7 @@ public:
 
   StringRef getCondCodeName() const { return CondCodeName; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckCondCode;
   }
 
@@ -632,7 +632,7 @@ public:
 
   StringRef getTypeName() const { return TypeName; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckValueType;
   }
 
@@ -673,7 +673,7 @@ public:
   const std::string getName() const { return Name; }
   unsigned getFirstResult() const { return FirstResult; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckComplexPat;
   }
 
@@ -695,7 +695,7 @@ public:
 
   int64_t getValue() const { return Value; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckAndImm;
   }
 
@@ -716,7 +716,7 @@ public:
 
   int64_t getValue() const { return Value; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckOrImm;
   }
 
@@ -734,7 +734,7 @@ public:
   CheckFoldableChainNodeMatcher()
     : Matcher(CheckFoldableChainNode) {}
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CheckFoldableChainNode;
   }
 
@@ -754,7 +754,7 @@ public:
   int64_t getValue() const { return Val; }
   MVT::SimpleValueType getVT() const { return VT; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == EmitInteger;
   }
 
@@ -778,7 +778,7 @@ public:
   const std::string &getValue() const { return Val; }
   MVT::SimpleValueType getVT() const { return VT; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == EmitStringInteger;
   }
 
@@ -803,7 +803,7 @@ public:
   const CodeGenRegister *getReg() const { return Reg; }
   MVT::SimpleValueType getVT() const { return VT; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == EmitRegister;
   }
 
@@ -826,7 +826,7 @@ public:
 
   unsigned getSlot() const { return Slot; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == EmitConvertToTarget;
   }
 
@@ -854,7 +854,7 @@ public:
     return ChainNodes[i];
   }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == EmitMergeInputChains;
   }
 
@@ -878,7 +878,7 @@ public:
   unsigned getSrcSlot() const { return SrcSlot; }
   Record *getDestPhysReg() const { return DestPhysReg; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == EmitCopyToReg;
   }
 
@@ -904,7 +904,7 @@ public:
   unsigned getSlot() const { return Slot; }
   Record *getNodeXForm() const { return NodeXForm; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == EmitNodeXForm;
   }
 
@@ -964,7 +964,7 @@ public:
   bool hasMemRefs() const { return HasMemRefs; }
   int getNumFixedArityOperands() const { return NumFixedArityOperands; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == EmitNode || N->getKind() == MorphNodeTo;
   }
 
@@ -991,7 +991,7 @@ public:
 
   unsigned getFirstResultSlot() const { return FirstResultSlot; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == EmitNode;
   }
 
@@ -1015,7 +1015,7 @@ public:
 
   const PatternToMatch &getPattern() const { return Pattern; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == MorphNodeTo;
   }
 };
@@ -1036,7 +1036,7 @@ public:
   unsigned getResult(unsigned R) const { return Results[R]; }
   const PatternToMatch &getPattern() const { return Pattern; }
 
-  static inline bool classof(const Matcher *N) {
+  static bool classof(const Matcher *N) {
     return N->getKind() == CompleteMatch;
   }
 
diff --git a/utils/TableGen/GlobalISelEmitter.cpp b/utils/TableGen/GlobalISelEmitter.cpp
index 03d231a153dc..924ed8f65c2c 100644
--- a/utils/TableGen/GlobalISelEmitter.cpp
+++ b/utils/TableGen/GlobalISelEmitter.cpp
@@ -80,8 +80,8 @@ public:
       return;
     }
     if (Ty.isVector()) {
-      OS << "LLT::vector(" << Ty.getNumElements() << ", " << Ty.getScalarSizeInBits()
-         << ")";
+      OS << "LLT::vector(" << Ty.getNumElements() << ", "
+         << Ty.getScalarSizeInBits() << ")";
       return;
     }
     llvm_unreachable("Unhandled LLT");
@@ -96,7 +96,8 @@ class InstructionMatcher;
 static Optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT) {
   MVT VT(SVT);
   if (VT.isVector() && VT.getVectorNumElements() != 1)
-    return LLTCodeGen(LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits()));
+    return LLTCodeGen(
+        LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits()));
   if (VT.isInteger() || VT.isFloatingPoint())
     return LLTCodeGen(LLT::scalar(VT.getSizeInBits()));
   return None;
@@ -241,12 +242,18 @@ public:
     return *static_cast<Kind *>(Predicates.back().get());
   }
 
-  typename PredicateVec::const_iterator predicates_begin() const { return Predicates.begin(); }
-  typename PredicateVec::const_iterator predicates_end() const { return Predicates.end(); }
+  typename PredicateVec::const_iterator predicates_begin() const {
+    return Predicates.begin();
+  }
+  typename PredicateVec::const_iterator predicates_end() const {
+    return Predicates.end();
+  }
   iterator_range<typename PredicateVec::const_iterator> predicates() const {
     return make_range(predicates_begin(), predicates_end());
   }
-  typename PredicateVec::size_type predicates_size() const { return Predicates.size(); }
+  typename PredicateVec::size_type predicates_size() const {
+    return Predicates.size();
+  }
 
   /// Emit a C++ expression that tests whether all the predicates are met.
   template <class... Args>
@@ -600,7 +607,8 @@ public:
   /// Compare the priority of this object and B.
   ///
   /// Returns true if this object is more important than B.
-  virtual bool isHigherPriorityThan(const InstructionPredicateMatcher &B) const {
+  virtual bool
+  isHigherPriorityThan(const InstructionPredicateMatcher &B) const {
     return Kind < B.Kind;
   };
 
@@ -631,7 +639,8 @@ public:
   /// Compare the priority of this object and B.
   ///
   /// Returns true if this object is more important than B.
-  bool isHigherPriorityThan(const InstructionPredicateMatcher &B) const override {
+  bool
+  isHigherPriorityThan(const InstructionPredicateMatcher &B) const override {
     if (InstructionPredicateMatcher::isHigherPriorityThan(B))
       return true;
     if (B.InstructionPredicateMatcher::isHigherPriorityThan(*this))
@@ -832,7 +841,13 @@ public:
 //===- Actions ------------------------------------------------------------===//
 class OperandRenderer {
 public:
-  enum RendererKind { OR_Copy, OR_Imm, OR_Register, OR_ComplexPattern };
+  enum RendererKind {
+    OR_Copy,
+    OR_CopySubReg,
+    OR_Imm,
+    OR_Register,
+    OR_ComplexPattern
+  };
 
 protected:
   RendererKind Kind;
@@ -877,6 +892,42 @@ public:
   }
 };
 
+/// A CopySubRegRenderer emits code to copy a single register operand from an
+/// existing instruction to the one being built and indicate that only a
+/// subregister should be copied.
+class CopySubRegRenderer : public OperandRenderer {
+protected:
+  /// The matcher for the instruction that this operand is copied from.
+  /// This provides the facility for looking up an a operand by it's name so
+  /// that it can be used as a source for the instruction being built.
+  const InstructionMatcher &Matched;
+  /// The name of the operand.
+  const StringRef SymbolicName;
+  /// The subregister to extract.
+  const CodeGenSubRegIndex *SubReg;
+
+public:
+  CopySubRegRenderer(const InstructionMatcher &Matched, StringRef SymbolicName,
+                     const CodeGenSubRegIndex *SubReg)
+      : OperandRenderer(OR_CopySubReg), Matched(Matched),
+        SymbolicName(SymbolicName), SubReg(SubReg) {}
+
+  static bool classof(const OperandRenderer *R) {
+    return R->getKind() == OR_CopySubReg;
+  }
+
+  const StringRef getSymbolicName() const { return SymbolicName; }
+
+  void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override {
+    const OperandMatcher &Operand = Matched.getOperand(SymbolicName);
+    StringRef InsnVarName =
+        Rule.getInsnVarName(Operand.getInstructionMatcher());
+    std::string OperandExpr = Operand.getOperandExpr(InsnVarName);
+    OS << "    MIB.addReg(" << OperandExpr << ".getReg() /*" << SymbolicName
+       << "*/, 0, " << SubReg->EnumValue << ");\n";
+  }
+};
+
 /// Adds a specific physical register to the instruction being built.
 /// This is typically useful for WZR/XZR on AArch64.
 class AddRegisterRenderer : public OperandRenderer {
@@ -1076,7 +1127,8 @@ public:
 
   void emitCxxActionStmts(raw_ostream &OS, RuleMatcher &Rule,
                           StringRef RecycleVarName) const override {
-    OS << "      constrainSelectedInstRegOperands(" << Name << ", TII, TRI, RBI);\n";
+    OS << "      constrainSelectedInstRegOperands(" << Name
+       << ", TII, TRI, RBI);\n";
   }
 };
 
@@ -1123,14 +1175,16 @@ std::string RuleMatcher::defineInsnVar(raw_ostream &OS,
   return InsnVarName;
 }
 
-StringRef RuleMatcher::getInsnVarName(const InstructionMatcher &InsnMatcher) const {
+StringRef
+RuleMatcher::getInsnVarName(const InstructionMatcher &InsnMatcher) const {
   const auto &I = InsnVariableNames.find(&InsnMatcher);
   if (I != InsnVariableNames.end())
     return I->second;
   llvm_unreachable("Matched Insn was not captured in a local variable");
 }
 
-/// Emit a C++ initializer_list containing references to every matched instruction.
+/// Emit a C++ initializer_list containing references to every matched
+/// instruction.
 void RuleMatcher::emitCxxCapturedInsnList(raw_ostream &OS) {
   SmallVector<StringRef, 2> Names;
   for (const auto &Pair : InsnVariableNames)
@@ -1292,6 +1346,7 @@ private:
   const RecordKeeper &RK;
   const CodeGenDAGPatterns CGP;
   const CodeGenTarget &Target;
+  CodeGenRegBank CGRegs;
 
   /// Keep track of the equivalence between SDNodes and Instruction.
   /// This is defined using 'GINodeEquiv' in the target description.
@@ -1315,9 +1370,9 @@ private:
   Error importChildMatcher(InstructionMatcher &InsnMatcher,
                            const TreePatternNode *SrcChild, unsigned OpIdx,
                            unsigned &TempOpIdx) const;
-  Expected<BuildMIAction &> createAndImportInstructionRenderer(
-      RuleMatcher &M, const TreePatternNode *Dst,
-      const InstructionMatcher &InsnMatcher) const;
+  Expected<BuildMIAction &>
+  createAndImportInstructionRenderer(RuleMatcher &M, const TreePatternNode *Dst,
+                                     const InstructionMatcher &InsnMatcher);
   Error importExplicitUseRenderer(BuildMIAction &DstMIBuilder,
                                   TreePatternNode *DstChild,
                                   const InstructionMatcher &InsnMatcher) const;
@@ -1354,7 +1409,7 @@ const CodeGenInstruction *GlobalISelEmitter::findNodeEquiv(Record *N) const {
 }
 
 GlobalISelEmitter::GlobalISelEmitter(RecordKeeper &RK)
-    : RK(RK), CGP(RK), Target(CGP.getTargetInfo()) {}
+    : RK(RK), CGP(RK), Target(CGP.getTargetInfo()), CGRegs(RK) {}
 
 //===- Emitter ------------------------------------------------------------===//
 
@@ -1382,7 +1437,8 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
       InsnMatcher.addPredicate<InstructionOpcodeMatcher>(
           &Target.getInstruction(RK.getDef("G_CONSTANT")));
     } else
-      return failedImport("Unable to deduce gMIR opcode to handle Src (which is a leaf)");
+      return failedImport(
+          "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
   } else {
     auto SrcGIOrNull = findNodeEquiv(Src->getOperator());
     if (!SrcGIOrNull)
@@ -1415,7 +1471,8 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
       OperandMatcher &OM = InsnMatcher.addOperand(OpIdx++, "", TempOpIdx);
       OM.addPredicate<LiteralIntOperandMatcher>(SrcIntInit->getValue());
     } else
-      return failedImport("Unable to deduce gMIR opcode to handle Src (which is a leaf)");
+      return failedImport(
+          "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
   } else {
     // Match the used operands (i.e. the children of the operator).
     for (unsigned i = 0, e = Src->getNumChildren(); i != e; ++i) {
@@ -1585,7 +1642,7 @@ Error GlobalISelEmitter::importExplicitUseRenderer(
 
 Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
     RuleMatcher &M, const TreePatternNode *Dst,
-    const InstructionMatcher &InsnMatcher) const {
+    const InstructionMatcher &InsnMatcher) {
   Record *DstOp = Dst->getOperator();
   if (!DstOp->isSubClassOf("Instruction")) {
     if (DstOp->isSubClassOf("ValueType"))
@@ -1597,13 +1654,17 @@ Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
 
   unsigned DstINumUses = DstI->Operands.size() - DstI->Operands.NumDefs;
   unsigned ExpectedDstINumUses = Dst->getNumChildren();
+  bool IsExtractSubReg = false;
 
   // COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction
-  // attached.
+  // attached. Similarly for EXTRACT_SUBREG except that's a subregister copy.
   if (DstI->TheDef->getName() == "COPY_TO_REGCLASS") {
     DstI = &Target.getInstruction(RK.getDef("COPY"));
     DstINumUses--; // Ignore the class constraint.
     ExpectedDstINumUses--;
+  } else if (DstI->TheDef->getName() == "EXTRACT_SUBREG") {
+    DstI = &Target.getInstruction(RK.getDef("COPY"));
+    IsExtractSubReg = true;
   }
 
   auto &DstMIBuilder = M.addAction<BuildMIAction>("NewI", DstI, InsnMatcher);
@@ -1614,6 +1675,33 @@ Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
     DstMIBuilder.addRenderer<CopyRenderer>(InsnMatcher, DstIOperand.Name);
   }
 
+  // EXTRACT_SUBREG needs to use a subregister COPY.
+  if (IsExtractSubReg) {
+    if (!Dst->getChild(0)->isLeaf())
+      return failedImport("EXTRACT_SUBREG child #1 is not a leaf");
+
+    if (DefInit *SubRegInit =
+            dyn_cast<DefInit>(Dst->getChild(1)->getLeafValue())) {
+      CodeGenRegisterClass *RC = CGRegs.getRegClass(
+          getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()));
+      CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
+
+      const auto &SrcRCDstRCPair =
+          RC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx);
+      if (SrcRCDstRCPair.hasValue()) {
+        assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
+        if (SrcRCDstRCPair->first != RC)
+          return failedImport("EXTRACT_SUBREG requires an additional COPY");
+      }
+
+      DstMIBuilder.addRenderer<CopySubRegRenderer>(
+          InsnMatcher, Dst->getChild(0)->getName(), SubIdx);
+      return DstMIBuilder;
+    }
+
+    return failedImport("EXTRACT_SUBREG child #1 is not a subreg index");
+  }
+
   // Render the explicit uses.
   unsigned Child = 0;
   unsigned NumDefaultOps = 0;
@@ -1740,10 +1828,22 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
       if (DstIOpRec == nullptr)
         return failedImport(
             "COPY_TO_REGCLASS operand #1 isn't a register class");
+    } else if (DstI.TheDef->getName() == "EXTRACT_SUBREG") {
+      if (!Dst->getChild(0)->isLeaf())
+        return failedImport("EXTRACT_SUBREG operand #0 isn't a leaf");
+
+      // We can assume that a subregister is in the same bank as it's super
+      // register.
+      DstIOpRec = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue());
+
+      if (DstIOpRec == nullptr)
+        return failedImport(
+            "EXTRACT_SUBREG operand #0 isn't a register class");
     } else if (DstIOpRec->isSubClassOf("RegisterOperand"))
       DstIOpRec = DstIOpRec->getValueAsDef("RegClass");
     else if (!DstIOpRec->isSubClassOf("RegisterClass"))
-      return failedImport("Dst MI def isn't a register class" + to_string(*Dst));
+      return failedImport("Dst MI def isn't a register class" +
+                          to_string(*Dst));
 
     OperandMatcher &OM = InsnMatcher.getOperand(OpIdx);
     OM.setSymbolicName(DstIOperand.Name);
@@ -1776,6 +1876,52 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
 
     M.addAction<ConstrainOperandToRegClassAction>(
         "NewI", 0, Target.getRegisterClass(DstIOpRec));
+
+    // We're done with this pattern!  It's eligible for GISel emission; return
+    // it.
+    ++NumPatternImported;
+    return std::move(M);
+  }
+
+  if (DstI.TheDef->getName() == "EXTRACT_SUBREG") {
+    // EXTRACT_SUBREG selects into a subregister COPY but unlike most
+    // instructions, the result register class is controlled by the
+    // subregisters of the operand. As a result, we must constrain the result
+    // class rather than check that it's already the right one.
+    if (!Dst->getChild(0)->isLeaf())
+      return failedImport("EXTRACT_SUBREG child #1 is not a leaf");
+
+    DefInit *SubRegInit = dyn_cast<DefInit>(Dst->getChild(1)->getLeafValue());
+    if (!SubRegInit)
+      return failedImport("EXTRACT_SUBREG child #1 is not a subreg index");
+
+    // Constrain the result to the same register bank as the operand.
+    Record *DstIOpRec =
+        getInitValueAsRegClass(Dst->getChild(0)->getLeafValue());
+
+    if (DstIOpRec == nullptr)
+      return failedImport("EXTRACT_SUBREG operand #1 isn't a register class");
+
+    CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
+    CodeGenRegisterClass *SrcRC = CGRegs.getRegClass(
+        getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()));
+
+    // It would be nice to leave this constraint implicit but we're required
+    // to pick a register class so constrain the result to a register class
+    // that can hold the correct MVT.
+    //
+    // FIXME: This may introduce an extra copy if the chosen class doesn't
+    //        actually contain the subregisters.
+    assert(Src->getExtTypes().size() == 1 &&
+             "Expected Src of EXTRACT_SUBREG to have one result type");
+
+    const auto &SrcRCDstRCPair =
+        SrcRC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx);
+    assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
+    M.addAction<ConstrainOperandToRegClassAction>("NewI", 0,
+                                                  *SrcRCDstRCPair->second);
+    M.addAction<ConstrainOperandToRegClassAction>("NewI", 1,
+                                                  *SrcRCDstRCPair->first);
   } else
     M.addAction<ConstrainOperandsToDefinitionAction>("NewI");
 
@@ -1874,8 +2020,10 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
      << "InstructionSelector::selectImpl(MachineInstr &I) const {\n"
      << "  MachineFunction &MF = *I.getParent()->getParent();\n"
      << "  const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
-     << "  // FIXME: This should be computed on a per-function basis rather than per-insn.\n"
-     << "  AvailableFunctionFeatures = computeAvailableFunctionFeatures(&STI, &MF);\n"
+     << "  // FIXME: This should be computed on a per-function basis rather "
+        "than per-insn.\n"
+     << "  AvailableFunctionFeatures = computeAvailableFunctionFeatures(&STI, "
+        "&MF);\n"
      << "  const PredicateBitset AvailableFeatures = getAvailableFeatures();\n";
 
   for (auto &Rule : Rules) {
diff --git a/utils/TableGen/RegisterInfoEmitter.cpp b/utils/TableGen/RegisterInfoEmitter.cpp
index 12cfb93a0c4f..bebb1a183fc7 100644
--- a/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/utils/TableGen/RegisterInfoEmitter.cpp
@@ -1195,7 +1195,8 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
   OS << "\" };\n\n";
 
   // Emit SubRegIndex lane masks, including 0.
-  OS << "\nstatic const LaneBitmask SubRegIndexLaneMaskTable[] = {\n  LaneBitmask::getAll(),\n";
+  OS << "\nstatic const LaneBitmask SubRegIndexLaneMaskTable[] = {\n  "
+        "LaneBitmask::getAll(),\n";
   for (const auto &Idx : SubRegIndices) {
     printMask(OS << "  ", Idx.LaneMask);
     OS << ", // " << Idx.getName() << '\n';
@@ -1234,7 +1235,8 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target,
     BitVector MaskBV(RegisterClasses.size());
 
     for (const auto &RC : RegisterClasses) {
-      OS << "static const uint32_t " << RC.getName() << "SubClassMask[] = {\n  ";
+      OS << "static const uint32_t " << RC.getName()
+         << "SubClassMask[] = {\n  ";
       printBitVectorAsHex(OS, RC.getSubClasses(), 32);
 
       // Emit super-reg class masks for any relevant SubRegIndices that can
diff --git a/utils/TableGen/SubtargetEmitter.cpp b/utils/TableGen/SubtargetEmitter.cpp
index 7e9f552eccc0..16d5740b79a3 100644
--- a/utils/TableGen/SubtargetEmitter.cpp
+++ b/utils/TableGen/SubtargetEmitter.cpp
@@ -805,6 +805,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
     return;
 
   std::vector<MCSchedClassDesc> &SCTab = SchedTables.ProcSchedClasses.back();
+  DEBUG(dbgs() << "\n+++ SCHED CLASSES (GenSchedClassTables) +++\n");
   for (const CodeGenSchedClass &SC : SchedModels.schedClasses()) {
     DEBUG(SC.dump(&SchedModels));