vendor/llvm/llvm-trunk-r301939

8 files changed, 128 insertions, 55 deletions

diff --git a/utils/TableGen/CodeGenDAGPatterns.cpp b/utils/TableGen/CodeGenDAGPatterns.cpp
index 972eb9cd3403..ef2cb4208eae 100644
--- a/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -2828,7 +2828,8 @@ public:
       if (IntInfo->ModRef & CodeGenIntrinsic::MR_Mod)
         mayStore = true;// Intrinsics that can write to memory are 'mayStore'.
 
-      if (IntInfo->ModRef >= CodeGenIntrinsic::ReadWriteMem)
+      if (IntInfo->ModRef >= CodeGenIntrinsic::ReadWriteMem ||
+          IntInfo->hasSideEffects)
         // ReadWriteMem intrinsics can have other strange effects.
         hasSideEffects = true;
     }
diff --git a/utils/TableGen/CodeGenIntrinsics.h b/utils/TableGen/CodeGenIntrinsics.h
index 6df0e6a62caf..24374127f536 100644
--- a/utils/TableGen/CodeGenIntrinsics.h
+++ b/utils/TableGen/CodeGenIntrinsics.h
@@ -123,6 +123,13 @@ struct CodeGenIntrinsic {
   /// True if the intrinsic is marked as convergent.
   bool isConvergent;
 
+  /// True if the intrinsic has side effects that aren't captured by any
+  /// of the other flags.
+  bool hasSideEffects;
+
+  // True if the intrinsic is marked as speculatable.
+  bool isSpeculatable;
+
   enum ArgAttribute { NoCapture, Returned, ReadOnly, WriteOnly, ReadNone };
   std::vector<std::pair<unsigned, ArgAttribute>> ArgumentAttributes;
 
diff --git a/utils/TableGen/CodeGenTarget.cpp b/utils/TableGen/CodeGenTarget.cpp
index 03c58ac09c2d..d1014a5668a5 100644
--- a/utils/TableGen/CodeGenTarget.cpp
+++ b/utils/TableGen/CodeGenTarget.cpp
@@ -515,6 +515,8 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
   isNoReturn = false;
   isNoDuplicate = false;
   isConvergent = false;
+  isSpeculatable = false;
+  hasSideEffects = false;
 
   if (DefName.size() <= 4 ||
       std::string(DefName.begin(), DefName.begin() + 4) != "int_")
@@ -653,6 +655,10 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
       isConvergent = true;
     else if (Property->getName() == "IntrNoReturn")
       isNoReturn = true;
+    else if (Property->getName() == "IntrSpeculatable")
+      isSpeculatable = true;
+    else if (Property->getName() == "IntrHasSideEffects")
+      hasSideEffects = true;
     else if (Property->isSubClassOf("NoCapture")) {
       unsigned ArgNo = Property->getValueAsInt("ArgNo");
       ArgumentAttributes.push_back(std::make_pair(ArgNo, NoCapture));
diff --git a/utils/TableGen/GlobalISelEmitter.cpp b/utils/TableGen/GlobalISelEmitter.cpp
index dcf10ab511da..65a1ea2f0f21 100644
--- a/utils/TableGen/GlobalISelEmitter.cpp
+++ b/utils/TableGen/GlobalISelEmitter.cpp
@@ -199,21 +199,19 @@ public:
   void emitCxxCapturedInsnList(raw_ostream &OS);
   void emitCxxCaptureStmts(raw_ostream &OS, StringRef Expr);
 
-  void emit(raw_ostream &OS,
-            std::map<Record *, SubtargetFeatureInfo, LessRecordByID>
-                SubtargetFeatures);
+void emit(raw_ostream &OS, SubtargetFeatureInfoMap SubtargetFeatures);
 
-  /// Compare the priority of this object and B.
-  ///
-  /// Returns true if this object is more important than B.
-  bool isHigherPriorityThan(const RuleMatcher &B) const;
+/// Compare the priority of this object and B.
+///
+/// Returns true if this object is more important than B.
+bool isHigherPriorityThan(const RuleMatcher &B) const;
 
-  /// Report the maximum number of temporary operands needed by the rule
-  /// matcher.
-  unsigned countRendererFns() const;
+/// Report the maximum number of temporary operands needed by the rule
+/// matcher.
+unsigned countRendererFns() const;
 
-  // FIXME: Remove this as soon as possible
-  InstructionMatcher &insnmatcher_front() const { return *Matchers.front(); }
+// FIXME: Remove this as soon as possible
+InstructionMatcher &insnmatcher_front() const { return *Matchers.front(); }
 };
 
 template <class PredicateTy> class PredicateListMatcher {
@@ -856,7 +854,9 @@ public:
   }
 
   void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override {
-    OS << "    MIB.addReg(" << RegisterDef->getValueAsString("Namespace")
+    OS << "    MIB.addReg(" << (RegisterDef->getValue("Namespace")
+                                    ? RegisterDef->getValueAsString("Namespace")
+                                    : "")
        << "::" << RegisterDef->getName() << ");\n";
   }
 };
@@ -951,6 +951,9 @@ private:
 
   /// True if the instruction can be built solely by mutating the opcode.
   bool canMutate() const {
+    if (OperandRenderers.size() != Matched.getNumOperands())
+      return false;
+
     for (const auto &Renderer : enumerate(OperandRenderers)) {
       if (const auto *Copy = dyn_cast<CopyRenderer>(&*Renderer.value())) {
         const OperandMatcher &OM = Matched.getOperand(Copy->getSymbolicName());
@@ -986,12 +989,16 @@ public:
            << ");\n";
 
         for (auto Def : I->ImplicitDefs) {
-          auto Namespace = Def->getValueAsString("Namespace");
+          auto Namespace = Def->getValue("Namespace")
+                               ? Def->getValueAsString("Namespace")
+                               : "";
           OS << "    MIB.addDef(" << Namespace << "::" << Def->getName()
              << ", RegState::Implicit);\n";
         }
         for (auto Use : I->ImplicitUses) {
-          auto Namespace = Use->getValueAsString("Namespace");
+          auto Namespace = Use->getValue("Namespace")
+                               ? Use->getValueAsString("Namespace")
+                               : "";
           OS << "    MIB.addUse(" << Namespace << "::" << Use->getName()
              << ", RegState::Implicit);\n";
         }
@@ -1072,8 +1079,7 @@ void RuleMatcher::emitCxxCaptureStmts(raw_ostream &OS, StringRef Expr) {
 }
 
 void RuleMatcher::emit(raw_ostream &OS,
-                       std::map<Record *, SubtargetFeatureInfo, LessRecordByID>
-                           SubtargetFeatures) {
+                       SubtargetFeatureInfoMap SubtargetFeatures) {
   if (Matchers.empty())
     llvm_unreachable("Unexpected empty matcher!");
 
@@ -1218,7 +1224,7 @@ private:
   DenseMap<const Record *, const Record *> ComplexPatternEquivs;
 
   // Map of predicates to their subtarget features.
-  std::map<Record *, SubtargetFeatureInfo, LessRecordByID> SubtargetFeatures;
+  SubtargetFeatureInfoMap SubtargetFeatures;
 
   void gatherNodeEquivs();
   const CodeGenInstruction *findNodeEquiv(Record *N) const;
@@ -1712,15 +1718,36 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
   OS << "#ifdef GET_GLOBALISEL_IMPL\n";
   SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(SubtargetFeatures,
                                                            OS);
-  SubtargetFeatureInfo::emitNameTable(SubtargetFeatures, OS);
+
+  // Separate subtarget features by how often they must be recomputed.
+  SubtargetFeatureInfoMap ModuleFeatures;
+  std::copy_if(SubtargetFeatures.begin(), SubtargetFeatures.end(),
+               std::inserter(ModuleFeatures, ModuleFeatures.end()),
+               [](const SubtargetFeatureInfoMap::value_type &X) {
+                 return !X.second.mustRecomputePerFunction();
+               });
+  SubtargetFeatureInfoMap FunctionFeatures;
+  std::copy_if(SubtargetFeatures.begin(), SubtargetFeatures.end(),
+               std::inserter(FunctionFeatures, FunctionFeatures.end()),
+               [](const SubtargetFeatureInfoMap::value_type &X) {
+                 return X.second.mustRecomputePerFunction();
+               });
+
   SubtargetFeatureInfo::emitComputeAvailableFeatures(
-      Target.getName(), "InstructionSelector", "computeAvailableFeatures",
-      SubtargetFeatures, OS);
+      Target.getName(), "InstructionSelector", "computeAvailableModuleFeatures",
+      ModuleFeatures, OS);
+  SubtargetFeatureInfo::emitComputeAvailableFeatures(
+      Target.getName(), "InstructionSelector",
+      "computeAvailableFunctionFeatures", FunctionFeatures, OS,
+      "const MachineFunction *MF");
 
   OS << "bool " << Target.getName()
      << "InstructionSelector::selectImpl(MachineInstr &I) const {\n"
      << "  MachineFunction &MF = *I.getParent()->getParent();\n"
-     << "  const MachineRegisterInfo &MRI = MF.getRegInfo();\n";
+     << "  const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
+     << "  // FIXME: This should be computed on a per-function basis rather than per-insn.\n"
+     << "  AvailableFunctionFeatures = computeAvailableFunctionFeatures(&STI, &MF);\n"
+     << "  const PredicateBitset AvailableFeatures = getAvailableFeatures();\n";
 
   for (auto &Rule : Rules) {
     Rule.emit(OS, SubtargetFeatures);
@@ -1730,6 +1757,26 @@ void GlobalISelEmitter::run(raw_ostream &OS) {
   OS << "  return false;\n"
      << "}\n"
      << "#endif // ifdef GET_GLOBALISEL_IMPL\n";
+
+  OS << "#ifdef GET_GLOBALISEL_PREDICATES_DECL\n"
+     << "PredicateBitset AvailableModuleFeatures;\n"
+     << "mutable PredicateBitset AvailableFunctionFeatures;\n"
+     << "PredicateBitset getAvailableFeatures() const {\n"
+     << "  return AvailableModuleFeatures | AvailableFunctionFeatures;\n"
+     << "}\n"
+     << "PredicateBitset\n"
+     << "computeAvailableModuleFeatures(const " << Target.getName()
+     << "Subtarget *Subtarget) const;\n"
+     << "PredicateBitset\n"
+     << "computeAvailableFunctionFeatures(const " << Target.getName()
+     << "Subtarget *Subtarget,\n"
+     << "                                 const MachineFunction *MF) const;\n"
+     << "#endif // ifdef GET_GLOBALISEL_PREDICATES_DECL\n";
+
+  OS << "#ifdef GET_GLOBALISEL_PREDICATES_INIT\n"
+     << "AvailableModuleFeatures(computeAvailableModuleFeatures(&STI)),\n"
+     << "AvailableFunctionFeatures()\n"
+     << "#endif // ifdef GET_GLOBALISEL_PREDICATES_INIT\n";
 }
 
 void GlobalISelEmitter::declareSubtargetFeature(Record *Predicate) {
diff --git a/utils/TableGen/IntrinsicEmitter.cpp b/utils/TableGen/IntrinsicEmitter.cpp
index e979b94e46d6..1fc18a5dd1d8 100644
--- a/utils/TableGen/IntrinsicEmitter.cpp
+++ b/utils/TableGen/IntrinsicEmitter.cpp
@@ -476,6 +476,12 @@ struct AttributeComparator {
     if (L->isConvergent != R->isConvergent)
       return R->isConvergent;
 
+    if (L->isSpeculatable != R->isSpeculatable)
+      return R->isSpeculatable;
+
+    if (L->hasSideEffects != R->hasSideEffects)
+      return R->hasSideEffects;
+
     // Try to order by readonly/readnone attribute.
     CodeGenIntrinsic::ModRefBehavior LK = L->ModRef;
     CodeGenIntrinsic::ModRefBehavior RK = R->ModRef;
@@ -600,7 +606,7 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
     if (!intrinsic.canThrow ||
         intrinsic.ModRef != CodeGenIntrinsic::ReadWriteMem ||
         intrinsic.isNoReturn || intrinsic.isNoDuplicate ||
-        intrinsic.isConvergent) {
+        intrinsic.isConvergent || intrinsic.isSpeculatable) {
       OS << "      const Attribute::AttrKind Atts[] = {";
       bool addComma = false;
       if (!intrinsic.canThrow) {
@@ -625,6 +631,12 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
         OS << "Attribute::Convergent";
         addComma = true;
       }
+      if (intrinsic.isSpeculatable) {
+        if (addComma)
+          OS << ",";
+        OS << "Attribute::Speculatable";
+        addComma = true;
+      }
 
       switch (intrinsic.ModRef) {
       case CodeGenIntrinsic::NoMem:
diff --git a/utils/TableGen/SubtargetFeatureInfo.cpp b/utils/TableGen/SubtargetFeatureInfo.cpp
index 96418dc77d55..5153c35b1261 100644
--- a/utils/TableGen/SubtargetFeatureInfo.cpp
+++ b/utils/TableGen/SubtargetFeatureInfo.cpp
@@ -45,8 +45,7 @@ SubtargetFeatureInfo::getAll(const RecordKeeper &Records) {
 }
 
 void SubtargetFeatureInfo::emitSubtargetFeatureFlagEnumeration(
-    std::map<Record *, SubtargetFeatureInfo, LessRecordByID> &SubtargetFeatures,
-    raw_ostream &OS) {
+    SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS) {
   OS << "// Flags for subtarget features that participate in "
      << "instruction matching.\n";
   OS << "enum SubtargetFeatureFlag : "
@@ -60,8 +59,7 @@ void SubtargetFeatureInfo::emitSubtargetFeatureFlagEnumeration(
 }
 
 void SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(
-    std::map<Record *, SubtargetFeatureInfo, LessRecordByID> &SubtargetFeatures,
-    raw_ostream &OS) {
+    SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS) {
   OS << "// Bits for subtarget features that participate in "
      << "instruction matching.\n";
   OS << "enum SubtargetFeatureBits : "
@@ -74,8 +72,7 @@ void SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(
 }
 
 void SubtargetFeatureInfo::emitNameTable(
-    std::map<Record *, SubtargetFeatureInfo, LessRecordByID> &SubtargetFeatures,
-    raw_ostream &OS) {
+    SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS) {
   // Need to sort the name table so that lookup by the log of the enum value
   // gives the proper name. More specifically, for a feature of value 1<<n,
   // SubtargetFeatureNames[n] should be the name of the feature.
@@ -102,11 +99,13 @@ void SubtargetFeatureInfo::emitNameTable(
 
 void SubtargetFeatureInfo::emitComputeAvailableFeatures(
     StringRef TargetName, StringRef ClassName, StringRef FuncName,
-    std::map<Record *, SubtargetFeatureInfo, LessRecordByID> &SubtargetFeatures,
-    raw_ostream &OS) {
+    SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS,
+    StringRef ExtraParams) {
   OS << "PredicateBitset " << TargetName << ClassName << "::\n"
-     << FuncName << "(const MachineFunction *MF, const " << TargetName
-     << "Subtarget *Subtarget) const {\n";
+     << FuncName << "(const " << TargetName << "Subtarget *Subtarget";
+  if (!ExtraParams.empty())
+    OS << ", " << ExtraParams;
+  OS << ") const {\n";
   OS << "  PredicateBitset Features;\n";
   for (const auto &SF : SubtargetFeatures) {
     const SubtargetFeatureInfo &SFI = SF.second;
@@ -120,8 +119,7 @@ void SubtargetFeatureInfo::emitComputeAvailableFeatures(
 
 void SubtargetFeatureInfo::emitComputeAssemblerAvailableFeatures(
     StringRef TargetName, StringRef ClassName, StringRef FuncName,
-    std::map<Record *, SubtargetFeatureInfo, LessRecordByID> &SubtargetFeatures,
-    raw_ostream &OS) {
+    SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS) {
   OS << "uint64_t " << TargetName << ClassName << "::\n"
      << FuncName << "(const FeatureBitset& FB) const {\n";
   OS << "  uint64_t Features = 0;\n";
diff --git a/utils/TableGen/SubtargetFeatureInfo.h b/utils/TableGen/SubtargetFeatureInfo.h
index bbaf45259606..c55c16a4031e 100644
--- a/utils/TableGen/SubtargetFeatureInfo.h
+++ b/utils/TableGen/SubtargetFeatureInfo.h
@@ -21,6 +21,9 @@ namespace llvm {
 class Record;
 class RecordKeeper;
 
+struct SubtargetFeatureInfo;
+using SubtargetFeatureInfoMap = std::map<Record *, SubtargetFeatureInfo, LessRecordByID>;
+
 /// Helper class for storing information on a subtarget feature which
 /// participates in instruction matching.
 struct SubtargetFeatureInfo {
@@ -43,6 +46,10 @@ struct SubtargetFeatureInfo {
     return "Feature_" + TheDef->getName().str() + "Bit";
   }
 
+  bool mustRecomputePerFunction() const {
+    return TheDef->getValueAsBit("RecomputePerFunction");
+  }
+
   void dump() const;
   static std::vector<std::pair<Record *, SubtargetFeatureInfo>>
   getAll(const RecordKeeper &Records);
@@ -52,21 +59,17 @@ struct SubtargetFeatureInfo {
   /// This version emits the bit value for the feature and is therefore limited
   /// to 64 feature bits.
   static void emitSubtargetFeatureFlagEnumeration(
-      std::map<Record *, SubtargetFeatureInfo, LessRecordByID>
-          &SubtargetFeatures,
-      raw_ostream &OS);
+      SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS);
 
   /// Emit the subtarget feature flag definitions.
   ///
   /// This version emits the bit index for the feature and can therefore support
   /// more than 64 feature bits.
-  static void emitSubtargetFeatureBitEnumeration(
-      std::map<Record *, SubtargetFeatureInfo, LessRecordByID>
-          &SubtargetFeatures,
-      raw_ostream &OS);
+  static void
+  emitSubtargetFeatureBitEnumeration(SubtargetFeatureInfoMap &SubtargetFeatures,
+                                     raw_ostream &OS);
 
-  static void emitNameTable(std::map<Record *, SubtargetFeatureInfo,
-                                     LessRecordByID> &SubtargetFeatures,
+  static void emitNameTable(SubtargetFeatureInfoMap &SubtargetFeatures,
                             raw_ostream &OS);
 
   /// Emit the function to compute the list of available features given a
@@ -82,11 +85,12 @@ struct SubtargetFeatureInfo {
   /// \param FuncName   The name of the function to emit.
   /// \param SubtargetFeatures A map of TableGen records to the
   ///                          SubtargetFeatureInfo equivalent.
-  static void emitComputeAvailableFeatures(
-      StringRef TargetName, StringRef ClassName, StringRef FuncName,
-      std::map<Record *, SubtargetFeatureInfo, LessRecordByID>
-          &SubtargetFeatures,
-      raw_ostream &OS);
+  /// \param ExtraParams Additional arguments to the generated function.
+  static void
+  emitComputeAvailableFeatures(StringRef TargetName, StringRef ClassName,
+                               StringRef FuncName,
+                               SubtargetFeatureInfoMap &SubtargetFeatures,
+                               raw_ostream &OS, StringRef ExtraParams = "");
 
   /// Emit the function to compute the list of available features given a
   /// subtarget.
@@ -103,9 +107,7 @@ struct SubtargetFeatureInfo {
   ///                          SubtargetFeatureInfo equivalent.
   static void emitComputeAssemblerAvailableFeatures(
       StringRef TargetName, StringRef ClassName, StringRef FuncName,
-      std::map<Record *, SubtargetFeatureInfo, LessRecordByID>
-          &SubtargetFeatures,
-      raw_ostream &OS);
+      SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS);
 };
 } // end namespace llvm
 
diff --git a/utils/TableGen/X86RecognizableInstr.cpp b/utils/TableGen/X86RecognizableInstr.cpp
index e703bbfc4496..4298bc5763b6 100644
--- a/utils/TableGen/X86RecognizableInstr.cpp
+++ b/utils/TableGen/X86RecognizableInstr.cpp
@@ -668,7 +668,7 @@ void RecognizableInstr::emitInstructionSpecifier() {
     break;
   case X86Local::MRMSrcReg4VOp3:
     assert(numPhysicalOperands == 3 &&
-           "Unexpected number of operands for MRMSrcRegFrm");
+           "Unexpected number of operands for MRMSrcReg4VOp3Frm");
     HANDLE_OPERAND(roRegister)
     HANDLE_OPERAND(rmRegister)
     HANDLE_OPERAND(vvvvRegister)
@@ -708,7 +708,7 @@ void RecognizableInstr::emitInstructionSpecifier() {
     break;
   case X86Local::MRMSrcMem4VOp3:
     assert(numPhysicalOperands == 3 &&
-           "Unexpected number of operands for MRMSrcMemFrm");
+           "Unexpected number of operands for MRMSrcMem4VOp3Frm");
     HANDLE_OPERAND(roRegister)
     HANDLE_OPERAND(memory)
     HANDLE_OPERAND(vvvvRegister)