vendor/llvm/llvm-trunk-r321414

14 files changed, 325 insertions, 142 deletions

diff --git a/utils/TableGen/CMakeLists.txt b/utils/TableGen/CMakeLists.txt
index c84f4925aa781..0944d54a4273d 100644
--- a/utils/TableGen/CMakeLists.txt
+++ b/utils/TableGen/CMakeLists.txt
@@ -32,6 +32,7 @@ add_tablegen(llvm-tblgen LLVM
   PseudoLoweringEmitter.cpp
   RegisterBankEmitter.cpp
   RegisterInfoEmitter.cpp
+  SDNodeProperties.cpp
   SearchableTableEmitter.cpp
   SubtargetEmitter.cpp
   SubtargetFeatureInfo.cpp
diff --git a/utils/TableGen/CodeGenDAGPatterns.cpp b/utils/TableGen/CodeGenDAGPatterns.cpp
index 51473f06da79f..7755cd1be3556 100644
--- a/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -309,23 +309,23 @@ bool TypeSetByHwMode::intersect(SetType &Out, const SetType &In) {
   return Changed;
 }
 
-void TypeSetByHwMode::validate() const {
+bool TypeSetByHwMode::validate() const {
 #ifndef NDEBUG
   if (empty())
-    return;
+    return true;
   bool AllEmpty = true;
   for (const auto &I : *this)
     AllEmpty &= I.second.empty();
-  assert(!AllEmpty &&
-          "type set is empty for each HW mode: type contradiction?");
+  return !AllEmpty;
 #endif
+  return true;
 }
 
 // --- TypeInfer
 
 bool TypeInfer::MergeInTypeInfo(TypeSetByHwMode &Out,
                                 const TypeSetByHwMode &In) {
-  ValidateOnExit _1(Out);
+  ValidateOnExit _1(Out, *this);
   In.validate();
   if (In.empty() || Out == In || TP.hasError())
     return false;
@@ -342,7 +342,7 @@ bool TypeInfer::MergeInTypeInfo(TypeSetByHwMode &Out,
 }
 
 bool TypeInfer::forceArbitrary(TypeSetByHwMode &Out) {
-  ValidateOnExit _1(Out);
+  ValidateOnExit _1(Out, *this);
   if (TP.hasError())
     return false;
   assert(!Out.empty() && "cannot pick from an empty set");
@@ -361,7 +361,7 @@ bool TypeInfer::forceArbitrary(TypeSetByHwMode &Out) {
 }
 
 bool TypeInfer::EnforceInteger(TypeSetByHwMode &Out) {
-  ValidateOnExit _1(Out);
+  ValidateOnExit _1(Out, *this);
   if (TP.hasError())
     return false;
   if (!Out.empty())
@@ -371,7 +371,7 @@ bool TypeInfer::EnforceInteger(TypeSetByHwMode &Out) {
 }
 
 bool TypeInfer::EnforceFloatingPoint(TypeSetByHwMode &Out) {
-  ValidateOnExit _1(Out);
+  ValidateOnExit _1(Out, *this);
   if (TP.hasError())
     return false;
   if (!Out.empty())
@@ -381,7 +381,7 @@ bool TypeInfer::EnforceFloatingPoint(TypeSetByHwMode &Out) {
 }
 
 bool TypeInfer::EnforceScalar(TypeSetByHwMode &Out) {
-  ValidateOnExit _1(Out);
+  ValidateOnExit _1(Out, *this);
   if (TP.hasError())
     return false;
   if (!Out.empty())
@@ -391,7 +391,7 @@ bool TypeInfer::EnforceScalar(TypeSetByHwMode &Out) {
 }
 
 bool TypeInfer::EnforceVector(TypeSetByHwMode &Out) {
-  ValidateOnExit _1(Out);
+  ValidateOnExit _1(Out, *this);
   if (TP.hasError())
     return false;
   if (!Out.empty())
@@ -401,7 +401,7 @@ bool TypeInfer::EnforceVector(TypeSetByHwMode &Out) {
 }
 
 bool TypeInfer::EnforceAny(TypeSetByHwMode &Out) {
-  ValidateOnExit _1(Out);
+  ValidateOnExit _1(Out, *this);
   if (TP.hasError() || !Out.empty())
     return false;
 
@@ -440,7 +440,7 @@ static Iter max_if(Iter B, Iter E, Pred P, Less L) {
 /// Make sure that for each type in Small, there exists a larger type in Big.
 bool TypeInfer::EnforceSmallerThan(TypeSetByHwMode &Small,
                                    TypeSetByHwMode &Big) {
-  ValidateOnExit _1(Small), _2(Big);
+  ValidateOnExit _1(Small, *this), _2(Big, *this);
   if (TP.hasError())
     return false;
   bool Changed = false;
@@ -545,7 +545,7 @@ bool TypeInfer::EnforceSmallerThan(TypeSetByHwMode &Small,
 ///    type T in Vec, such that U is the element type of T.
 bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec,
                                        TypeSetByHwMode &Elem) {
-  ValidateOnExit _1(Vec), _2(Elem);
+  ValidateOnExit _1(Vec, *this), _2(Elem, *this);
   if (TP.hasError())
     return false;
   bool Changed = false;
@@ -586,7 +586,7 @@ bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec,
 bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec,
                                        const ValueTypeByHwMode &VVT) {
   TypeSetByHwMode Tmp(VVT);
-  ValidateOnExit _1(Vec), _2(Tmp);
+  ValidateOnExit _1(Vec, *this), _2(Tmp, *this);
   return EnforceVectorEltTypeIs(Vec, Tmp);
 }
 
@@ -595,7 +595,7 @@ bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec,
 /// element type as T and at least as many elements as T.
 bool TypeInfer::EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec,
                                              TypeSetByHwMode &Sub) {
-  ValidateOnExit _1(Vec), _2(Sub);
+  ValidateOnExit _1(Vec, *this), _2(Sub, *this);
   if (TP.hasError())
     return false;
 
@@ -661,7 +661,7 @@ bool TypeInfer::EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec,
 ///    type T in V, such that T and U have the same number of elements
 ///    (reverse of 2).
 bool TypeInfer::EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W) {
-  ValidateOnExit _1(V), _2(W);
+  ValidateOnExit _1(V, *this), _2(W, *this);
   if (TP.hasError())
     return false;
 
@@ -699,7 +699,7 @@ bool TypeInfer::EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W) {
 /// 2. Ensure that for each type U in B, there exists a type T in A
 ///    such that T and U have equal size in bits (reverse of 1).
 bool TypeInfer::EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B) {
-  ValidateOnExit _1(A), _2(B);
+  ValidateOnExit _1(A, *this), _2(B, *this);
   if (TP.hasError())
     return false;
   bool Changed = false;
@@ -730,7 +730,7 @@ bool TypeInfer::EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B) {
 }
 
 void TypeInfer::expandOverloads(TypeSetByHwMode &VTS) {
-  ValidateOnExit _1(VTS);
+  ValidateOnExit _1(VTS, *this);
   TypeSetByHwMode Legal = getLegalTypes();
   bool HaveLegalDef = Legal.hasDefault();
 
@@ -806,6 +806,19 @@ TypeSetByHwMode TypeInfer::getLegalTypes() {
   return VTS;
 }
 
+#ifndef NDEBUG
+TypeInfer::ValidateOnExit::~ValidateOnExit() {
+  if (!VTS.validate()) {
+    dbgs() << "Type set is empty for each HW mode:\n"
+              "possible type contradiction in the pattern below "
+              "(use -print-records with llvm-tblgen to see all "
+              "expanded records).\n";
+    Infer.TP.dump();
+    llvm_unreachable(nullptr);
+  }
+}
+#endif
+
 //===----------------------------------------------------------------------===//
 // TreePredicateFn Implementation
 //===----------------------------------------------------------------------===//
@@ -1591,37 +1604,7 @@ SDNodeInfo::SDNodeInfo(Record *R, const CodeGenHwModes &CGH) : Def(R) {
   NumOperands = TypeProfile->getValueAsInt("NumOperands");
 
   // Parse the properties.
-  Properties = 0;
-  for (Record *Property : R->getValueAsListOfDefs("Properties")) {
-    if (Property->getName() == "SDNPCommutative") {
-      Properties |= 1 << SDNPCommutative;
-    } else if (Property->getName() == "SDNPAssociative") {
-      Properties |= 1 << SDNPAssociative;
-    } else if (Property->getName() == "SDNPHasChain") {
-      Properties |= 1 << SDNPHasChain;
-    } else if (Property->getName() == "SDNPOutGlue") {
-      Properties |= 1 << SDNPOutGlue;
-    } else if (Property->getName() == "SDNPInGlue") {
-      Properties |= 1 << SDNPInGlue;
-    } else if (Property->getName() == "SDNPOptInGlue") {
-      Properties |= 1 << SDNPOptInGlue;
-    } else if (Property->getName() == "SDNPMayStore") {
-      Properties |= 1 << SDNPMayStore;
-    } else if (Property->getName() == "SDNPMayLoad") {
-      Properties |= 1 << SDNPMayLoad;
-    } else if (Property->getName() == "SDNPSideEffect") {
-      Properties |= 1 << SDNPSideEffect;
-    } else if (Property->getName() == "SDNPMemOperand") {
-      Properties |= 1 << SDNPMemOperand;
-    } else if (Property->getName() == "SDNPVariadic") {
-      Properties |= 1 << SDNPVariadic;
-    } else {
-      PrintFatalError("Unknown SD Node property '" +
-                      Property->getName() + "' on node '" +
-                      R->getName() + "'!");
-    }
-  }
-
+  Properties = parseSDPatternOperatorProperties(R);
 
   // Parse the type constraints.
   std::vector<Record*> ConstraintList =
@@ -2100,11 +2083,20 @@ bool TreePatternNode::NodeHasProperty(SDNP Property,
   if (isLeaf()) {
     if (const ComplexPattern *CP = getComplexPatternInfo(CGP))
       return CP->hasProperty(Property);
+
     return false;
   }
 
-  Record *Operator = getOperator();
-  if (!Operator->isSubClassOf("SDNode")) return false;
+  if (Property != SDNPHasChain) {
+    // The chain proprety is already present on the different intrinsic node
+    // types (intrinsic_w_chain, intrinsic_void), and is not explicitly listed
+    // on the intrinsic. Anything else is specific to the individual intrinsic.
+    if (const CodeGenIntrinsic *Int = getIntrinsicInfo(CGP))
+      return Int->hasProperty(Property);
+  }
+
+  if (!Operator->isSubClassOf("SDPatternOperator"))
+    return false;
 
   return CGP.getSDNodeInfo(Operator).hasProperty(Property);
 }
diff --git a/utils/TableGen/CodeGenDAGPatterns.h b/utils/TableGen/CodeGenDAGPatterns.h
index afbcb10a4b66b..8a8132c7f894e 100644
--- a/utils/TableGen/CodeGenDAGPatterns.h
+++ b/utils/TableGen/CodeGenDAGPatterns.h
@@ -18,6 +18,7 @@
 #include "CodeGenHwModes.h"
 #include "CodeGenIntrinsics.h"
 #include "CodeGenTarget.h"
+#include "SDNodeProperties.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
@@ -234,7 +235,7 @@ struct TypeSetByHwMode : public InfoByHwMode<MachineValueTypeSet> {
   bool operator!=(const TypeSetByHwMode &VTS) const { return !(*this == VTS); }
 
   void dump() const;
-  void validate() const;
+  bool validate() const;
 
 private:
   /// Intersect two sets. Return true if anything has changed.
@@ -319,8 +320,13 @@ struct TypeInfer {
                        const TypeSetByHwMode::SetType &Legal);
 
   struct ValidateOnExit {
-    ValidateOnExit(TypeSetByHwMode &T) : VTS(T) {}
-    ~ValidateOnExit() { VTS.validate(); }
+    ValidateOnExit(TypeSetByHwMode &T, TypeInfer &TI) : Infer(TI), VTS(T) {}
+  #ifndef NDEBUG
+    ~ValidateOnExit();
+  #else
+    ~ValidateOnExit() {}  // Empty destructor with NDEBUG.
+  #endif
+    TypeInfer &Infer;
     TypeSetByHwMode &VTS;
   };
 
@@ -1204,6 +1210,7 @@ inline bool SDNodeInfo::ApplyTypeConstraints(TreePatternNode *N,
       MadeChange |= TypeConstraints[i].ApplyTypeConstraint(N, *this, TP);
     return MadeChange;
   }
+
 } // end namespace llvm
 
 #endif
diff --git a/utils/TableGen/CodeGenIntrinsics.h b/utils/TableGen/CodeGenIntrinsics.h
index 24374127f536a..91305034dc243 100644
--- a/utils/TableGen/CodeGenIntrinsics.h
+++ b/utils/TableGen/CodeGenIntrinsics.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_UTILS_TABLEGEN_CODEGENINTRINSICS_H
 #define LLVM_UTILS_TABLEGEN_CODEGENINTRINSICS_H
 
+#include "SDNodeProperties.h"
 #include "llvm/CodeGen/MachineValueType.h"
 #include <string>
 #include <vector>
@@ -104,6 +105,9 @@ struct CodeGenIntrinsic {
   };
   ModRefBehavior ModRef;
 
+  /// SDPatternOperator Properties applied to the intrinsic.
+  unsigned Properties;
+
   /// This is set to true if the intrinsic is overloaded by its argument
   /// types.
   bool isOverloaded;
@@ -133,6 +137,10 @@ struct CodeGenIntrinsic {
   enum ArgAttribute { NoCapture, Returned, ReadOnly, WriteOnly, ReadNone };
   std::vector<std::pair<unsigned, ArgAttribute>> ArgumentAttributes;
 
+  bool hasProperty(enum SDNP Prop) const {
+    return Properties & (1 << Prop);
+  }
+
   CodeGenIntrinsic(Record *R);
 };
 
diff --git a/utils/TableGen/CodeGenTarget.cpp b/utils/TableGen/CodeGenTarget.cpp
index 827b6083c17fe..168bd690831f1 100644
--- a/utils/TableGen/CodeGenTarget.cpp
+++ b/utils/TableGen/CodeGenTarget.cpp
@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenTarget.h"
+#include "CodeGenDAGPatterns.h"
 #include "CodeGenIntrinsics.h"
 #include "CodeGenSchedule.h"
 #include "llvm/ADT/STLExtras.h"
@@ -450,6 +451,7 @@ ComplexPattern::ComplexPattern(Record *R) {
   else
     Complexity = RawComplexity;
 
+  // FIXME: Why is this different from parseSDPatternOperatorProperties?
   // Parse the properties.
   Properties = 0;
   std::vector<Record*> PropList = R->getValueAsListOfDefs("Properties");
@@ -512,6 +514,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
   TheDef = R;
   std::string DefName = R->getName();
   ModRef = ReadWriteMem;
+  Properties = 0;
   isOverloaded = false;
   isCommutative = false;
   canThrow = false;
@@ -681,6 +684,10 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
       llvm_unreachable("Unknown property!");
   }
 
+  // Also record the SDPatternOperator Properties.
+  Properties = parseSDPatternOperatorProperties(R);
+
   // Sort the argument attributes for later benefit.
   std::sort(ArgumentAttributes.begin(), ArgumentAttributes.end());
 }
+
diff --git a/utils/TableGen/CodeGenTarget.h b/utils/TableGen/CodeGenTarget.h
index 89aa81b5fc334..7280d707fba69 100644
--- a/utils/TableGen/CodeGenTarget.h
+++ b/utils/TableGen/CodeGenTarget.h
@@ -21,6 +21,7 @@
 #include "CodeGenInstruction.h"
 #include "CodeGenRegisters.h"
 #include "InfoByHwMode.h"
+#include "SDNodeProperties.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Record.h"
 #include <algorithm>
@@ -31,25 +32,6 @@ struct CodeGenRegister;
 class CodeGenSchedModels;
 class CodeGenTarget;
 
-// SelectionDAG node properties.
-//  SDNPMemOperand: indicates that a node touches memory and therefore must
-//                  have an associated memory operand that describes the access.
-enum SDNP {
-  SDNPCommutative,
-  SDNPAssociative,
-  SDNPHasChain,
-  SDNPOutGlue,
-  SDNPInGlue,
-  SDNPOptInGlue,
-  SDNPMayLoad,
-  SDNPMayStore,
-  SDNPSideEffect,
-  SDNPMemOperand,
-  SDNPVariadic,
-  SDNPWantRoot,
-  SDNPWantParent
-};
-
 /// getValueType - Return the MVT::SimpleValueType that the specified TableGen
 /// record corresponds to.
 MVT::SimpleValueType getValueType(Record *Rec);
diff --git a/utils/TableGen/GlobalISelEmitter.cpp b/utils/TableGen/GlobalISelEmitter.cpp
index b80f023550622..c7d662db5a2fe 100644
--- a/utils/TableGen/GlobalISelEmitter.cpp
+++ b/utils/TableGen/GlobalISelEmitter.cpp
@@ -574,6 +574,7 @@ class Matcher {
 public:
   virtual ~Matcher() = default;
   virtual void emit(MatchTable &Table) = 0;
+  virtual std::unique_ptr<PredicateMatcher> forgetFirstCondition() = 0;
 };
 
 class GroupMatcher : public Matcher {
@@ -595,6 +596,15 @@ public:
     Rules.clear();
   }
   void emit(MatchTable &Table) override;
+
+  std::unique_ptr<PredicateMatcher> forgetFirstCondition() override {
+    // We shouldn't need to mess up with groups, since we
+    // should have merged everything shareable upfront.
+    // If we start to look into reordering predicates,
+    // we may want to reconsider this.
+    assert(0 && "Groups should be formed maximal for now");
+    llvm_unreachable("No need for this for now");
+  }
 };
 
 /// Generates code to check that a match rule matches.
@@ -749,7 +759,7 @@ public:
   /// matcher.
   unsigned countRendererFns() const;
 
-  std::unique_ptr<PredicateMatcher> forgetFirstCondition();
+  std::unique_ptr<PredicateMatcher> forgetFirstCondition() override;
 
   // FIXME: Remove this as soon as possible
   InstructionMatcher &insnmatchers_front() const { return *Matchers.front(); }
@@ -921,7 +931,7 @@ class SameOperandMatcher : public OperandPredicateMatcher {
   std::string MatchingName;
 
 public:
-  SameOperandMatcher(StringRef MatchingName, unsigned InsnVarID, unsigned OpIdx)
+  SameOperandMatcher(unsigned InsnVarID, unsigned OpIdx, StringRef MatchingName)
       : OperandPredicateMatcher(OPM_SameOperand, InsnVarID, OpIdx),
         MatchingName(MatchingName) {}
 
@@ -941,7 +951,7 @@ protected:
 public:
   static std::set<LLTCodeGen> KnownTypes;
 
-  LLTOperandMatcher(const LLTCodeGen &Ty, unsigned InsnVarID, unsigned OpIdx)
+  LLTOperandMatcher(unsigned InsnVarID, unsigned OpIdx, const LLTCodeGen &Ty)
       : OperandPredicateMatcher(OPM_LLT, InsnVarID, OpIdx), Ty(Ty) {
     KnownTypes.insert(Ty);
   }
@@ -981,8 +991,8 @@ protected:
   unsigned SizeInBits;
 
 public:
-  PointerToAnyOperandMatcher(unsigned SizeInBits, unsigned InsnVarID,
-                             unsigned OpIdx)
+  PointerToAnyOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                             unsigned SizeInBits)
       : OperandPredicateMatcher(OPM_PointerToAny, InsnVarID, OpIdx),
         SizeInBits(SizeInBits) {}
 
@@ -1011,9 +1021,9 @@ protected:
 public:
   bool isIdentical(const PredicateMatcher &B) const override { return false; }
 
-  ComplexPatternOperandMatcher(const OperandMatcher &Operand,
-                               const Record &TheDef, unsigned InsnVarID,
-                               unsigned OpIdx)
+  ComplexPatternOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                               const OperandMatcher &Operand,
+                               const Record &TheDef)
       : OperandPredicateMatcher(OPM_ComplexPattern, InsnVarID, OpIdx),
         Operand(Operand), TheDef(TheDef) {}
 
@@ -1043,8 +1053,8 @@ protected:
   const CodeGenRegisterClass &RC;
 
 public:
-  RegisterBankOperandMatcher(const CodeGenRegisterClass &RC, unsigned InsnVarID,
-                             unsigned OpIdx)
+  RegisterBankOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                             const CodeGenRegisterClass &RC)
       : OperandPredicateMatcher(OPM_RegBank, InsnVarID, OpIdx), RC(RC) {}
 
   bool isIdentical(const PredicateMatcher &B) const override {
@@ -1092,7 +1102,7 @@ protected:
   int64_t Value;
 
 public:
-  ConstantIntOperandMatcher(int64_t Value, unsigned InsnVarID, unsigned OpIdx)
+  ConstantIntOperandMatcher(unsigned InsnVarID, unsigned OpIdx, int64_t Value)
       : OperandPredicateMatcher(OPM_Int, InsnVarID, OpIdx), Value(Value) {}
 
   bool isIdentical(const PredicateMatcher &B) const override {
@@ -1120,7 +1130,7 @@ protected:
   int64_t Value;
 
 public:
-  LiteralIntOperandMatcher(int64_t Value, unsigned InsnVarID, unsigned OpIdx)
+  LiteralIntOperandMatcher(unsigned InsnVarID, unsigned OpIdx, int64_t Value)
       : OperandPredicateMatcher(OPM_LiteralInt, InsnVarID, OpIdx),
         Value(Value) {}
 
@@ -1148,8 +1158,8 @@ protected:
   const CodeGenIntrinsic *II;
 
 public:
-  IntrinsicIDOperandMatcher(const CodeGenIntrinsic *II, unsigned InsnVarID,
-                            unsigned OpIdx)
+  IntrinsicIDOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                            const CodeGenIntrinsic *II)
       : OperandPredicateMatcher(OPM_IntrinsicID, InsnVarID, OpIdx), II(II) {}
 
   bool isIdentical(const PredicateMatcher &B) const override {
@@ -1284,9 +1294,9 @@ PredicateListMatcher<OperandPredicateMatcher>::addPredicate(Args &&... args) {
   auto *OpMatcher = static_cast<OperandMatcher *>(this);
   if (static_cast<OperandMatcher *>(this)->isSameAsAnotherOperand())
     return None;
-  Predicates.emplace_back(llvm::make_unique<Kind>(
-      std::forward<Args>(args)..., OpMatcher->getInsnVarID(),
-      OpMatcher->getOperandIndex()));
+  Predicates.emplace_back(llvm::make_unique<Kind>(OpMatcher->getInsnVarID(),
+                                                  OpMatcher->getOperandIndex(),
+                                                  std::forward<Args>(args)...));
   return static_cast<Kind *>(Predicates.back().get());
 }
 
@@ -1658,8 +1668,8 @@ protected:
   std::unique_ptr<InstructionMatcher> InsnMatcher;
 
 public:
-  InstructionOperandMatcher(RuleMatcher &Rule, StringRef SymbolicName,
-                            unsigned InsnVarID, unsigned OpIdx)
+  InstructionOperandMatcher(unsigned InsnVarID, unsigned OpIdx,
+                            RuleMatcher &Rule, StringRef SymbolicName)
       : OperandPredicateMatcher(OPM_Instruction, InsnVarID, OpIdx),
         InsnMatcher(new InstructionMatcher(Rule, SymbolicName)) {}
 
@@ -2336,6 +2346,10 @@ void RuleMatcher::emit(MatchTable &Table) {
   if (Matchers.empty())
     llvm_unreachable("Unexpected empty matcher!");
 
+  // Reset the ID generation so that the emitted IDs match the ones
+  // we set while building the InstructionMatcher and such.
+  clearImplicitMap();
+
   // The representation supports rules that require multiple roots such as:
   //    %ptr(p0) = ...
   //    %elt0(s32) = G_LOAD %ptr
@@ -2615,7 +2629,7 @@ private:
   ///   # predicate C
   /// \endverbatim
   std::vector<Matcher *> optimizeRules(
-      std::vector<RuleMatcher> &Rules,
+      const std::vector<Matcher *> &Rules,
       std::vector<std::unique_ptr<GroupMatcher>> &StorageGroupMatcher);
 };
 
@@ -3342,9 +3356,6 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
   unsigned TempOpIdx = 0;
   auto InsnMatcherOrError =
       createAndImportSelDAGMatcher(M, InsnMatcherTemp, Src, TempOpIdx);
-  // Reset the ID generation so that the emitted IDs match the ones
-  // in the InstructionMatcher and such.
-  M.clearImplicitMap();
   if (auto Error = InsnMatcherOrError.takeError())
     return std::move(Error);
   InstructionMatcher &InsnMatcher = InsnMatcherOrError.get();
@@ -3539,28 +3550,36 @@ void GlobalISelEmitter::emitImmPredicates(
     OS << "};\n";
   }
 
-  for (const auto *Record : MatchedRecords)
-    OS << "static bool Predicate_" << Record->getName() << "(" << Type
-       << " Imm) {" << Record->getValueAsString("ImmediateCode") << "}\n";
-
-  OS << "static InstructionSelector::" << TypeIdentifier
-     << "ImmediatePredicateFn " << TypeIdentifier << "ImmPredicateFns[] = {\n"
-     << "  nullptr,\n";
-  for (const auto *Record : MatchedRecords)
-    OS << "  Predicate_" << Record->getName() << ",\n";
-  OS << "};\n";
+  OS << "bool " << Target.getName() << "InstructionSelector::testImmPredicate_"
+     << TypeIdentifier << "(unsigned PredicateID, " << Type
+     << " Imm) const {\n";
+  if (!MatchedRecords.empty())
+    OS << "  switch (PredicateID) {\n";
+  for (const auto *Record : MatchedRecords) {
+    OS << "  case GIPFP_" << TypeIdentifier << "_Predicate_"
+       << Record->getName() << ": {\n"
+       << "    " << Record->getValueAsString("ImmediateCode") << "\n"
+       << "    llvm_unreachable(\"ImmediateCode should have returned\");\n"
+       << "    return false;\n"
+       << "  }\n";
+  }
+  if (!MatchedRecords.empty())
+    OS << "  }\n";
+  OS << "  llvm_unreachable(\"Unknown predicate\");\n"
+     << "  return false;\n"
+     << "}\n";
 }
 
 std::vector<Matcher *> GlobalISelEmitter::optimizeRules(
-    std::vector<RuleMatcher> &Rules,
+    const std::vector<Matcher *> &Rules,
     std::vector<std::unique_ptr<GroupMatcher>> &StorageGroupMatcher) {
   std::vector<Matcher *> OptRules;
   // Start with a stupid grouping for now.
   std::unique_ptr<GroupMatcher> CurrentGroup = make_unique<GroupMatcher>();
   assert(CurrentGroup->conditions_empty());
   unsigned NbGroup = 0;
-  for (RuleMatcher &Rule : Rules) {
-    std::unique_ptr<PredicateMatcher> Predicate = Rule.forgetFirstCondition();
+  for (Matcher *Rule : Rules) {
+    std::unique_ptr<PredicateMatcher> Predicate = Rule->forgetFirstCondition();
     if (!CurrentGroup->conditions_empty() &&
         !CurrentGroup->lastConditionMatches(*Predicate)) {
       // Start a new group.
@@ -3572,7 +3591,7 @@ std::vector<Matcher *> GlobalISelEmitter::optimizeRules(
     }
     if (CurrentGroup->conditions_empty())
       CurrentGroup->addCondition(std::move(Predicate));
-    CurrentGroup->addRule(Rule);
+    CurrentGroup->addRule(*Rule);
   }
   if (!CurrentGroup->conditions_empty()) {
     ++NbGroup;
@@ -3662,12 +3681,17 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
         "MatcherInfo;\n"
      << "  static " << Target.getName()
      << "InstructionSelector::ComplexMatcherMemFn ComplexPredicateFns[];\n"
+     << "bool testImmPredicate_I64(unsigned PredicateID, int64_t Imm) const "
+        "override;\n"
+     << "bool testImmPredicate_APInt(unsigned PredicateID, const APInt &Imm) "
+        "const override;\n"
+     << "bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat "
+        "&Imm) const override;\n"
      << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n\n";
 
   OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n"
      << ", State(" << MaxTemporaries << "),\n"
-     << "MatcherInfo({TypeObjects, FeatureBitsets, I64ImmPredicateFns, "
-        "APIntImmPredicateFns, APFloatImmPredicateFns, ComplexPredicateFns})\n"
+     << "MatcherInfo({TypeObjects, FeatureBitsets, ComplexPredicateFns})\n"
      << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n\n";
 
   OS << "#ifdef GET_GLOBALISEL_IMPL\n";
@@ -3823,12 +3847,13 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
   });
   std::vector<std::unique_ptr<GroupMatcher>> StorageGroupMatcher;
 
-  std::vector<Matcher *> OptRules;
-  if (OptimizeMatchTable)
-    OptRules = optimizeRules(Rules, StorageGroupMatcher);
-  else
-    for (Matcher &Rule : Rules)
-      OptRules.push_back(&Rule);
+  std::vector<Matcher *> InputRules;
+  for (Matcher &Rule : Rules)
+    InputRules.push_back(&Rule);
+
+  std::vector<Matcher *> OptRules =
+      OptimizeMatchTable ? optimizeRules(InputRules, StorageGroupMatcher)
+                         : InputRules;
 
   MatchTable Table(0);
   for (Matcher *Rule : OptRules) {
diff --git a/utils/TableGen/IntrinsicEmitter.cpp b/utils/TableGen/IntrinsicEmitter.cpp
index b4e61ec53c199..37e024b1665ed 100644
--- a/utils/TableGen/IntrinsicEmitter.cpp
+++ b/utils/TableGen/IntrinsicEmitter.cpp
@@ -717,6 +717,7 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
         if (addComma)
           OS << ",";
         OS << "Attribute::InaccessibleMemOrArgMemOnly";
+        break;
       case CodeGenIntrinsic::ReadWriteMem:
         break;
       }
diff --git a/utils/TableGen/SDNodeProperties.cpp b/utils/TableGen/SDNodeProperties.cpp
new file mode 100644
index 0000000000000..343febc99d1e6
--- /dev/null
+++ b/utils/TableGen/SDNodeProperties.cpp
@@ -0,0 +1,49 @@
+//===- SDNodeProperties.cpp -----------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SDNodeProperties.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+
+using namespace llvm;
+
+unsigned llvm::parseSDPatternOperatorProperties(Record *R) {
+  unsigned Properties = 0;
+  for (Record *Property : R->getValueAsListOfDefs("Properties")) {
+    if (Property->getName() == "SDNPCommutative") {
+      Properties |= 1 << SDNPCommutative;
+    } else if (Property->getName() == "SDNPAssociative") {
+      Properties |= 1 << SDNPAssociative;
+    } else if (Property->getName() == "SDNPHasChain") {
+      Properties |= 1 << SDNPHasChain;
+    } else if (Property->getName() == "SDNPOutGlue") {
+      Properties |= 1 << SDNPOutGlue;
+    } else if (Property->getName() == "SDNPInGlue") {
+      Properties |= 1 << SDNPInGlue;
+    } else if (Property->getName() == "SDNPOptInGlue") {
+      Properties |= 1 << SDNPOptInGlue;
+    } else if (Property->getName() == "SDNPMayStore") {
+      Properties |= 1 << SDNPMayStore;
+    } else if (Property->getName() == "SDNPMayLoad") {
+      Properties |= 1 << SDNPMayLoad;
+    } else if (Property->getName() == "SDNPSideEffect") {
+      Properties |= 1 << SDNPSideEffect;
+    } else if (Property->getName() == "SDNPMemOperand") {
+      Properties |= 1 << SDNPMemOperand;
+    } else if (Property->getName() == "SDNPVariadic") {
+      Properties |= 1 << SDNPVariadic;
+    } else {
+      PrintFatalError("Unknown SD Node property '" +
+                      Property->getName() + "' on node '" +
+                      R->getName() + "'!");
+    }
+  }
+
+  return Properties;
+}
diff --git a/utils/TableGen/SDNodeProperties.h b/utils/TableGen/SDNodeProperties.h
new file mode 100644
index 0000000000000..a8d4efb5dab04
--- /dev/null
+++ b/utils/TableGen/SDNodeProperties.h
@@ -0,0 +1,40 @@
+//===- SDNodeProperties.h ---------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UTILS_TABLEGEN_SDNODEPROPERTIES_H
+#define LLVM_UTILS_TABLEGEN_SDNODEPROPERTIES_H
+
+namespace llvm {
+
+class Record;
+
+// SelectionDAG node properties.
+//  SDNPMemOperand: indicates that a node touches memory and therefore must
+//                  have an associated memory operand that describes the access.
+enum SDNP {
+  SDNPCommutative,
+  SDNPAssociative,
+  SDNPHasChain,
+  SDNPOutGlue,
+  SDNPInGlue,
+  SDNPOptInGlue,
+  SDNPMayLoad,
+  SDNPMayStore,
+  SDNPSideEffect,
+  SDNPMemOperand,
+  SDNPVariadic,
+  SDNPWantRoot,
+  SDNPWantParent
+};
+
+unsigned parseSDPatternOperatorProperties(Record *R);
+
+}
+
+#endif
diff --git a/utils/docker/build_docker_image.sh b/utils/docker/build_docker_image.sh
index ad7831925f9a4..9b0ba46fe4b34 100755
--- a/utils/docker/build_docker_image.sh
+++ b/utils/docker/build_docker_image.sh
@@ -30,6 +30,10 @@ Available options:
                         'branches/release_40'
                         (default: 'trunk')
     -r|--revision       svn revision to checkout
+    -c|--cherrypick     revision to cherry-pick. Can be specified multiple times.
+                        Cherry-picks are performed in the sorted order using the
+                        following command:
+                        'svn patch <(svn diff -c \$rev)'.
     -p|--llvm-project   name of an svn project to checkout. Will also add the
                         project to a list LLVM_ENABLE_PROJECTS, passed to CMake.
                         For clang, please use 'clang', not 'cfe'.
@@ -92,7 +96,7 @@ while [[ $# -gt 0 ]]; do
       DOCKER_TAG="$1"
       shift
       ;;
-    -i|--install-target|-r|--revision|-b|--branch|-p|--llvm-project)
+    -i|--install-target|-r|--revision|-c|-cherrypick|-b|--branch|-p|--llvm-project)
       if [ "$1" == "-i" ] || [ "$1" == "--install-target" ]; then
         SEEN_INSTALL_TARGET=1
       fi
diff --git a/utils/docker/scripts/build_install_llvm.sh b/utils/docker/scripts/build_install_llvm.sh
index 79ce7e50efdc2..5141fdc9bb5eb 100755
--- a/utils/docker/scripts/build_install_llvm.sh
+++ b/utils/docker/scripts/build_install_llvm.sh
@@ -25,6 +25,10 @@ Available options:
                       'branches/release_40'
                       (default: 'trunk')
   -r|--revision       svn revision to checkout
+  -c|--cherrypick     revision to cherry-pick. Can be specified multiple times.
+                      Cherry-picks are performed in the sorted order using the
+                      following command:
+                      'svn patch <(svn diff -c \$rev)'.
   -p|--llvm-project   name of an svn project to checkout. Will also add the
                       project to a list LLVM_ENABLE_PROJECTS, passed to CMake.
                       For clang, please use 'clang', not 'cfe'.
@@ -40,6 +44,7 @@ EOF
 }
 
 LLVM_SVN_REV=""
+CHERRYPICKS=""
 LLVM_BRANCH=""
 CMAKE_ARGS=""
 CMAKE_INSTALL_TARGETS=""
@@ -77,6 +82,11 @@ while [[ $# -gt 0 ]]; do
       LLVM_SVN_REV="$1"
       shift
       ;;
+    -c|--cherrypick)
+      shift
+      CHERRYPICKS="$CHERRYPICKS $1"
+      shift
+      ;;
     -b|--branch)
       shift
       LLVM_BRANCH="$1"
@@ -153,6 +163,28 @@ else
   echo "Checking out latest svn revision."
 fi
 
+# Sort cherrypicks and remove duplicates.
+CHERRYPICKS="$(echo "$CHERRYPICKS" | xargs -n1 | sort | uniq | xargs)"
+
+function apply_cherrypicks() {
+  local CHECKOUT_DIR="$1"
+
+  [ "$CHERRYPICKS" == "" ] || echo "Applying cherrypicks"
+  pushd "$CHECKOUT_DIR"
+
+  # This function is always called on a sorted list of cherrypicks.
+  for CHERRY_REV in $CHERRYPICKS; do
+    echo "Cherry-picking r$CHERRY_REV into $CHECKOUT_DIR"
+
+    local PATCH_FILE="$(mktemp)"
+    svn diff -c $CHERRY_REV > "$PATCH_FILE"
+    svn patch "$PATCH_FILE"
+    rm "$PATCH_FILE"
+  done
+
+  popd
+}
+
 CLANG_BUILD_DIR=/tmp/clang-build
 CLANG_INSTALL_DIR=/tmp/clang-install
 
@@ -172,6 +204,11 @@ for LLVM_PROJECT in $LLVM_PROJECTS; do
   svn co -q $SVN_REV_ARG \
     "https://llvm.org/svn/llvm-project/$SVN_PROJECT/$LLVM_BRANCH" \
     "$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
+
+  # We apply cherrypicks to all repositories regardless of whether the revision
+  # changes this repository or not. For repositories not affected by the
+  # cherrypick, applying the cherrypick is a no-op.
+  apply_cherrypicks "$CLANG_BUILD_DIR/src/$LLVM_PROJECT"
 done
 
 if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
@@ -179,6 +216,8 @@ if [ $CLANG_TOOLS_EXTRA_ENABLED -ne 0 ]; then
   svn co -q $SVN_REV_ARG \
     "https://llvm.org/svn/llvm-project/clang-tools-extra/$LLVM_BRANCH" \
     "$CLANG_BUILD_DIR/src/clang/tools/extra"
+
+  apply_cherrypicks "$CLANG_BUILD_DIR/src/clang/tools/extra"
 fi
 
 CHECKSUMS_FILE="/tmp/checksums/checksums.txt"
diff --git a/utils/git-svn/git-llvm b/utils/git-svn/git-llvm
index 0d566dac430a4..5d9d4d291006e 100755
--- a/utils/git-svn/git-llvm
+++ b/utils/git-svn/git-llvm
@@ -178,7 +178,7 @@ def clean_and_update_svn(svn_repo):
 
     # Unfortunately it appears there's no svn equivalent for git clean, so we
     # have to do it ourselves.
-    for line in svn(svn_repo, 'status').split('\n'):
+    for line in svn(svn_repo, 'status', '--no-ignore').split('\n'):
         if not line.startswith('?'):
             continue
         filename = line[1:].strip()
@@ -252,7 +252,7 @@ def svn_push_one_rev(svn_repo, rev, dry_run):
     if not subrepos:
         raise RuntimeError('Empty diff for rev %s?' % rev)
 
-    status = svn(svn_repo, 'status')
+    status = svn(svn_repo, 'status', '--no-ignore')
     if status:
         die("Can't push git rev %s because svn status is not empty:\n%s" %
             (rev, status))
@@ -272,10 +272,11 @@ def svn_push_one_rev(svn_repo, rev, dry_run):
                    "first?")
             sys.exit(2)
 
-    status_lines = svn(svn_repo, 'status').split('\n')
+    status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n')
 
-    for l in (l for l in status_lines if l.startswith('?')):
-        svn(svn_repo, 'add', l[1:].strip())
+    for l in (l for l in status_lines if (l.startswith('?') or
+                                          l.startswith('I'))):
+        svn(svn_repo, 'add', '--no-ignore', l[1:].strip())
     for l in (l for l in status_lines if l.startswith('!')):
         svn(svn_repo, 'remove', l[1:].strip())
 
diff --git a/utils/update_mir_test_checks.py b/utils/update_mir_test_checks.py
index 2934f09f6b37e..3756af1b517d6 100755
--- a/utils/update_mir_test_checks.py
+++ b/utils/update_mir_test_checks.py
@@ -33,16 +33,20 @@ TRIPLE_IR_RE = re.compile(r'^\s*target\s+triple\s*=\s*"([^"]+)"$')
 CHECK_PREFIX_RE = re.compile('--?check-prefix(?:es)?[= ](\S+)')
 CHECK_RE = re.compile(r'^\s*[;#]\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:')
 
-FUNC_NAME_RE = re.compile(r' *name: *(?P<func>[A-Za-z0-9_.-]+)')
-BODY_BEGIN_RE = re.compile(r' *body: *\|')
-BASIC_BLOCK_RE = re.compile(r' *bb\.[0-9]+.*:$')
+MIR_FUNC_NAME_RE = re.compile(r' *name: *(?P<func>[A-Za-z0-9_.-]+)')
+MIR_BODY_BEGIN_RE = re.compile(r' *body: *\|')
+MIR_BASIC_BLOCK_RE = re.compile(r' *bb\.[0-9]+.*:$')
 VREG_RE = re.compile(r'(%[0-9]+)(?::[a-z0-9_]+)?(?:\([<>a-z0-9 ]+\))?')
 VREG_DEF_RE = re.compile(
     r'^ *(?P<vregs>{0}(?:, {0})*) '
     r'= (?P<opcode>[A-Zt][A-Za-z0-9_]+)'.format(VREG_RE.pattern))
-PREFIX_DATA_RE = re.compile(r'^ *(;|bb.[0-9].*: *$|[a-z]+:( |$)|$)')
+MIR_PREFIX_DATA_RE = re.compile(r'^ *(;|bb.[0-9].*: *$|[a-z]+:( |$)|$)')
 VREG_CLASS_RE = re.compile(r'^ *- *{ id: ([0-9]+), class: ([a-z0-9_]+)', re.M)
 
+IR_FUNC_NAME_RE = re.compile(
+    r'^\s*define\s+(?:internal\s+)?[^@]*@(?P<func>\w+)\s*\(')
+IR_PREFIX_DATA_RE = re.compile(r'^ *(;|$)')
+
 MIR_FUNC_RE = re.compile(
     r'^---$'
     r'\n'
@@ -164,13 +168,13 @@ def find_functions_with_one_bb(lines, verbose=False):
     cur_func = None
     bbs = 0
     for line in lines:
-        m = FUNC_NAME_RE.match(line)
+        m = MIR_FUNC_NAME_RE.match(line)
         if m:
             if bbs == 1:
                 result.append(cur_func)
             cur_func = m.group('func')
             bbs = 0
-        m = BASIC_BLOCK_RE.match(line)
+        m = MIR_BASIC_BLOCK_RE.match(line)
         if m:
             bbs += 1
     if bbs == 1:
@@ -340,8 +344,10 @@ def update_test_file(llc, test, remove_common_prefixes=False,
         warn('Ignoring common prefixes: {}'.format(common_prefixes),
              test_file=test)
 
-    autogenerated_note = ('# NOTE: Assertions have been autogenerated by '
-                          'utils/{}'.format(os.path.basename(__file__)))
+    comment_char = '#' if test.endswith('.mir') else ';'
+    autogenerated_note = ('{} NOTE: Assertions have been autogenerated by '
+                          'utils/{}'.format(comment_char,
+                                            os.path.basename(__file__)))
     output_lines = []
     output_lines.append(autogenerated_note)
 
@@ -350,48 +356,69 @@ def update_test_file(llc, test, remove_common_prefixes=False,
             continue
 
         if state == 'toplevel':
+            m = IR_FUNC_NAME_RE.match(input_line)
+            if m:
+                state = 'ir function prefix'
+                func_name = m.group('func')
             if input_line.strip() == '---':
                 state = 'document'
             output_lines.append(input_line)
         elif state == 'document':
-            m = FUNC_NAME_RE.match(input_line)
+            m = MIR_FUNC_NAME_RE.match(input_line)
             if m:
-                state = 'function metadata'
+                state = 'mir function metadata'
                 func_name = m.group('func')
             if input_line.strip() == '...':
                 state = 'toplevel'
                 func_name = None
             if should_add_line_to_output(input_line, prefix_set):
                 output_lines.append(input_line)
-        elif state == 'function metadata':
+        elif state == 'mir function metadata':
             if should_add_line_to_output(input_line, prefix_set):
                 output_lines.append(input_line)
-            m = BODY_BEGIN_RE.match(input_line)
+            m = MIR_BODY_BEGIN_RE.match(input_line)
             if m:
                 if func_name in simple_functions:
                     # If there's only one block, put the checks inside it
-                    state = 'function prefix'
+                    state = 'mir function prefix'
                     continue
-                state = 'function body'
+                state = 'mir function body'
                 add_checks_for_function(test, output_lines, run_list,
                                         func_dict, func_name, add_vreg_checks,
                                         single_bb=False, verbose=verbose)
-        elif state == 'function prefix':
-            m = PREFIX_DATA_RE.match(input_line)
+        elif state == 'mir function prefix':
+            m = MIR_PREFIX_DATA_RE.match(input_line)
             if not m:
-                state = 'function body'
+                state = 'mir function body'
                 add_checks_for_function(test, output_lines, run_list,
                                         func_dict, func_name, add_vreg_checks,
                                         single_bb=True, verbose=verbose)
 
             if should_add_line_to_output(input_line, prefix_set):
                 output_lines.append(input_line)
-        elif state == 'function body':
+        elif state == 'mir function body':
             if input_line.strip() == '...':
                 state = 'toplevel'
                 func_name = None
             if should_add_line_to_output(input_line, prefix_set):
                 output_lines.append(input_line)
+        elif state == 'ir function prefix':
+            m = IR_PREFIX_DATA_RE.match(input_line)
+            if not m:
+                state = 'ir function body'
+                add_checks_for_function(test, output_lines, run_list,
+                                        func_dict, func_name, add_vreg_checks,
+                                        single_bb=False, verbose=verbose)
+
+            if should_add_line_to_output(input_line, prefix_set):
+                output_lines.append(input_line)
+        elif state == 'ir function body':
+            if input_line.strip() == '}':
+                state = 'toplevel'
+                func_name = None
+            if should_add_line_to_output(input_line, prefix_set):
+                output_lines.append(input_line)
+
 
     log('Writing {} lines to {}...'.format(len(output_lines), test), verbose)