diff options
Diffstat (limited to 'utils/TableGen')
25 files changed, 2693 insertions, 462 deletions
diff --git a/utils/TableGen/AsmMatcherEmitter.cpp b/utils/TableGen/AsmMatcherEmitter.cpp index 1272d2b9f2872..3947d0220ed5e 100644 --- a/utils/TableGen/AsmMatcherEmitter.cpp +++ b/utils/TableGen/AsmMatcherEmitter.cpp @@ -123,9 +123,12 @@ using namespace llvm; #define DEBUG_TYPE "asm-matcher-emitter" +cl::OptionCategory AsmMatcherEmitterCat("Options for -gen-asm-matcher"); + static cl::opt<std::string> -MatchPrefix("match-prefix", cl::init(""), - cl::desc("Only match instructions with the given prefix")); + MatchPrefix("match-prefix", cl::init(""), + cl::desc("Only match instructions with the given prefix"), + cl::cat(AsmMatcherEmitterCat)); namespace { class AsmMatcherInfo; @@ -2784,8 +2787,6 @@ void AsmMatcherEmitter::run(raw_ostream &OS) { } OS << " void convertToMapAndConstraints(unsigned Kind,\n "; OS << " const OperandVector &Operands) override;\n"; - if (HasMnemonicFirst) - OS << " bool mnemonicIsValid(StringRef Mnemonic, unsigned VariantID);\n"; OS << " unsigned MatchInstructionImpl(const OperandVector &Operands,\n" << " MCInst &Inst,\n" << " uint64_t &ErrorInfo," @@ -2883,7 +2884,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) { StringTable.EmitString(OS); OS << ";\n\n"; - // Emit the static match table; unused classes get initalized to 0 which is + // Emit the static match table; unused classes get initialized to 0 which is // guaranteed to be InvalidMatchClass. // // FIXME: We can reduce the size of this table very easily. First, we change @@ -2967,28 +2968,6 @@ void AsmMatcherEmitter::run(raw_ostream &OS) { OS << "};\n\n"; } - // A method to determine if a mnemonic is in the list. - if (HasMnemonicFirst) { - OS << "bool " << Target.getName() << ClassName << "::\n" - << "mnemonicIsValid(StringRef Mnemonic, unsigned VariantID) {\n"; - OS << " // Find the appropriate table for this asm variant.\n"; - OS << " const MatchEntry *Start, *End;\n"; - OS << " switch (VariantID) {\n"; - OS << " default: llvm_unreachable(\"invalid variant!\");\n"; - for (unsigned VC = 0; VC != VariantCount; ++VC) { - Record *AsmVariant = Target.getAsmParserVariant(VC); - int AsmVariantNo = AsmVariant->getValueAsInt("Variant"); - OS << " case " << AsmVariantNo << ": Start = std::begin(MatchTable" << VC - << "); End = std::end(MatchTable" << VC << "); break;\n"; - } - OS << " }\n"; - OS << " // Search the table.\n"; - OS << " auto MnemonicRange = "; - OS << "std::equal_range(Start, End, Mnemonic, LessOpcode());\n"; - OS << " return MnemonicRange.first != MnemonicRange.second;\n"; - OS << "}\n\n"; - } - // Finally, build the match function. OS << "unsigned " << Target.getName() << ClassName << "::\n" << "MatchInstructionImpl(const OperandVector &Operands,\n"; diff --git a/utils/TableGen/AsmWriterEmitter.cpp b/utils/TableGen/AsmWriterEmitter.cpp index a7c6104aaa214..40b7857ab994b 100644 --- a/utils/TableGen/AsmWriterEmitter.cpp +++ b/utils/TableGen/AsmWriterEmitter.cpp @@ -741,7 +741,7 @@ struct AliasPriorityComparator { if (LHS.second == RHS.second) { // We don't actually care about the order, but for consistency it // shouldn't depend on pointer comparisons. - return LHS.first.TheDef->getName() < RHS.first.TheDef->getName(); + return LessRecordByID()(LHS.first.TheDef, RHS.first.TheDef); } // Aliases with larger priorities should be considered first. @@ -813,10 +813,9 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { // We only consider ReqFeatures predicates if PassSubtarget std::vector<Record *> RF = CGA.TheDef->getValueAsListOfDefs("Predicates"); - std::copy_if(RF.begin(), RF.end(), std::back_inserter(ReqFeatures), - [](Record *R) { - return R->getValueAsBit("AssemblerMatcherPredicate"); - }); + copy_if(RF, std::back_inserter(ReqFeatures), [](Record *R) { + return R->getValueAsBit("AssemblerMatcherPredicate"); + }); } unsigned NumMIOps = 0; diff --git a/utils/TableGen/CMakeLists.txt b/utils/TableGen/CMakeLists.txt index 604e3f188eafd..b2913afae12a4 100644 --- a/utils/TableGen/CMakeLists.txt +++ b/utils/TableGen/CMakeLists.txt @@ -27,6 +27,7 @@ add_tablegen(llvm-tblgen LLVM IntrinsicEmitter.cpp OptParserEmitter.cpp PseudoLoweringEmitter.cpp + RegisterBankEmitter.cpp RegisterInfoEmitter.cpp SearchableTableEmitter.cpp SubtargetEmitter.cpp @@ -34,6 +35,7 @@ add_tablegen(llvm-tblgen LLVM TableGen.cpp Types.cpp X86DisassemblerTables.cpp + X86EVEX2VEXTablesEmitter.cpp X86ModRMFilters.cpp X86RecognizableInstr.cpp CTagsEmitter.cpp diff --git a/utils/TableGen/CallingConvEmitter.cpp b/utils/TableGen/CallingConvEmitter.cpp index a47662b28558c..013e960657523 100644 --- a/utils/TableGen/CallingConvEmitter.cpp +++ b/utils/TableGen/CallingConvEmitter.cpp @@ -96,7 +96,7 @@ void CallingConvEmitter::EmitAction(Record *Action, } else if (Action->isSubClassOf("CCIf")) { O << Action->getValueAsString("Predicate"); } else { - Action->dump(); + errs() << *Action; PrintFatalError("Unknown CCPredicateAction!"); } @@ -268,7 +268,7 @@ void CallingConvEmitter::EmitAction(Record *Action, << "LocVT, LocInfo, ArgFlags, State))\n"; O << IndentStr << IndentStr << "return false;\n"; } else { - Action->dump(); + errs() << *Action; PrintFatalError("Unknown CCAction!"); } } diff --git a/utils/TableGen/CodeGenDAGPatterns.cpp b/utils/TableGen/CodeGenDAGPatterns.cpp index b82a76bb035de..972eb9cd3403e 100644 --- a/utils/TableGen/CodeGenDAGPatterns.cpp +++ b/utils/TableGen/CodeGenDAGPatterns.cpp @@ -580,56 +580,74 @@ bool EEVT::TypeSet::EnforceVectorSubVectorTypeIs(EEVT::TypeSet &VTOperand, return MadeChange; } -/// EnforceVectorSameNumElts - 'this' is now constrained to -/// be a vector with same num elements as VTOperand. -bool EEVT::TypeSet::EnforceVectorSameNumElts(EEVT::TypeSet &VTOperand, - TreePattern &TP) { +/// EnforceameNumElts - If VTOperand is a scalar, then 'this' is a scalar. If +/// VTOperand is a vector, then 'this' must have the same number of elements. +bool EEVT::TypeSet::EnforceSameNumElts(EEVT::TypeSet &VTOperand, + TreePattern &TP) { if (TP.hasError()) return false; - // "This" must be a vector and "VTOperand" must be a vector. bool MadeChange = false; - MadeChange |= EnforceVector(TP); - MadeChange |= VTOperand.EnforceVector(TP); - // If we know one of the vector types, it forces the other type to agree. + if (isCompletelyUnknown()) + MadeChange = FillWithPossibleTypes(TP); + + if (VTOperand.isCompletelyUnknown()) + MadeChange = VTOperand.FillWithPossibleTypes(TP); + + // If one contains vectors but the other doesn't pull vectors out. + if (!hasVectorTypes()) + MadeChange |= VTOperand.EnforceScalar(TP); + else if (!hasScalarTypes()) + MadeChange |= VTOperand.EnforceVector(TP); + if (!VTOperand.hasVectorTypes()) + MadeChange |= EnforceScalar(TP); + else if (!VTOperand.hasScalarTypes()) + MadeChange |= EnforceVector(TP); + + // If one type is a vector, make sure the other has the same element count. + // If this a scalar, then we are already done with the above. if (isConcrete()) { MVT IVT = getConcrete(); - unsigned NumElems = IVT.getVectorNumElements(); + if (IVT.isVector()) { + unsigned NumElems = IVT.getVectorNumElements(); - // Only keep types that have same elements as 'this'. - TypeSet InputSet(VTOperand); + // Only keep types that have same elements as 'this'. + TypeSet InputSet(VTOperand); - auto I = remove_if(VTOperand.TypeVec, [NumElems](MVT VVT) { - return VVT.getVectorNumElements() != NumElems; - }); - MadeChange |= I != VTOperand.TypeVec.end(); - VTOperand.TypeVec.erase(I, VTOperand.TypeVec.end()); + auto I = remove_if(VTOperand.TypeVec, [NumElems](MVT VVT) { + return VVT.getVectorNumElements() != NumElems; + }); + MadeChange |= I != VTOperand.TypeVec.end(); + VTOperand.TypeVec.erase(I, VTOperand.TypeVec.end()); - if (VTOperand.TypeVec.empty()) { // FIXME: Really want an SMLoc here! - TP.error("Type inference contradiction found, forcing '" + - InputSet.getName() + "' to have same number elements as '" + - getName() + "'"); - return false; + if (VTOperand.TypeVec.empty()) { // FIXME: Really want an SMLoc here! + TP.error("Type inference contradiction found, forcing '" + + InputSet.getName() + "' to have same number elements as '" + + getName() + "'"); + return false; + } } } else if (VTOperand.isConcrete()) { MVT IVT = VTOperand.getConcrete(); - unsigned NumElems = IVT.getVectorNumElements(); + if (IVT.isVector()) { + unsigned NumElems = IVT.getVectorNumElements(); - // Only keep types that have same elements as VTOperand. - TypeSet InputSet(*this); + // Only keep types that have same elements as VTOperand. + TypeSet InputSet(*this); - auto I = remove_if(TypeVec, [NumElems](MVT VVT) { - return VVT.getVectorNumElements() != NumElems; - }); - MadeChange |= I != TypeVec.end(); - TypeVec.erase(I, TypeVec.end()); + auto I = remove_if(TypeVec, [NumElems](MVT VVT) { + return VVT.getVectorNumElements() != NumElems; + }); + MadeChange |= I != TypeVec.end(); + TypeVec.erase(I, TypeVec.end()); - if (TypeVec.empty()) { // FIXME: Really want an SMLoc here! - TP.error("Type inference contradiction found, forcing '" + - InputSet.getName() + "' to have same number elements than '" + - VTOperand.getName() + "'"); - return false; + if (TypeVec.empty()) { // FIXME: Really want an SMLoc here! + TP.error("Type inference contradiction found, forcing '" + + InputSet.getName() + "' to have same number elements than '" + + VTOperand.getName() + "'"); + return false; + } } } @@ -644,6 +662,12 @@ bool EEVT::TypeSet::EnforceSameSize(EEVT::TypeSet &VTOperand, bool MadeChange = false; + if (isCompletelyUnknown()) + MadeChange = FillWithPossibleTypes(TP); + + if (VTOperand.isCompletelyUnknown()) + MadeChange = VTOperand.FillWithPossibleTypes(TP); + // If we know one of the types, it forces the other type agree. if (isConcrete()) { MVT IVT = getConcrete(); @@ -1058,7 +1082,7 @@ bool SDTypeConstraint::ApplyTypeConstraint(TreePatternNode *N, getOperandNum(x.SDTCisSameNumEltsAs_Info.OtherOperandNum, N, NodeInfo, OResNo); return OtherNode->getExtType(OResNo). - EnforceVectorSameNumElts(NodeToApply->getExtType(ResNo), TP); + EnforceSameNumElts(NodeToApply->getExtType(ResNo), TP); } case SDTCisSameSizeAs: { unsigned OResNo = 0; @@ -1248,7 +1272,7 @@ static unsigned GetNumNodeResults(Record *Operator, CodeGenDAGPatterns &CDP) { if (Operator->isSubClassOf("ComplexPattern")) return 1; - Operator->dump(); + errs() << *Operator; PrintFatalError("Unhandled node in GetNumNodeResults"); } @@ -2114,7 +2138,7 @@ TreePatternNode *TreePattern::ParseTreePattern(Init *TheInit, StringRef OpName){ DagInit *Dag = dyn_cast<DagInit>(TheInit); if (!Dag) { - TheInit->dump(); + TheInit->print(errs()); error("Pattern has unexpected init kind!"); } DefInit *OpDef = dyn_cast<DefInit>(Dag->getOperator()); diff --git a/utils/TableGen/CodeGenDAGPatterns.h b/utils/TableGen/CodeGenDAGPatterns.h index 97401cd81713e..189d6e382ee7c 100644 --- a/utils/TableGen/CodeGenDAGPatterns.h +++ b/utils/TableGen/CodeGenDAGPatterns.h @@ -144,9 +144,10 @@ namespace EEVT { /// be a vector type VT. bool EnforceVectorSubVectorTypeIs(EEVT::TypeSet &VT, TreePattern &TP); - /// EnforceVectorSameNumElts - 'this' is now constrained to - /// be a vector with same num elements as VT. - bool EnforceVectorSameNumElts(EEVT::TypeSet &VT, TreePattern &TP); + /// EnforceSameNumElts - If VTOperand is a scalar, then 'this' is a scalar. + /// If VTOperand is a vector, then 'this' must have the same number of + /// elements. + bool EnforceSameNumElts(EEVT::TypeSet &VT, TreePattern &TP); /// EnforceSameSize - 'this' is now constrained to be the same size as VT. bool EnforceSameSize(EEVT::TypeSet &VT, TreePattern &TP); diff --git a/utils/TableGen/CodeGenMapTable.cpp b/utils/TableGen/CodeGenMapTable.cpp index 8032d7b3ee95a..60db6c267ad73 100644 --- a/utils/TableGen/CodeGenMapTable.cpp +++ b/utils/TableGen/CodeGenMapTable.cpp @@ -367,7 +367,7 @@ unsigned MapTableEmitter::emitBinSearchTable(raw_ostream &OS) { ArrayRef<const CodeGenInstruction*> NumberedInstructions = Target.getInstructionsByEnumValue(); - std::string TargetName = Target.getName(); + std::string Namespace = Target.getInstNamespace(); const std::vector<ListInit*> &ValueCols = InstrMapDesc.getValueCols(); unsigned NumCol = ValueCols.size(); unsigned TotalNumInstr = NumberedInstructions.size(); @@ -387,22 +387,22 @@ unsigned MapTableEmitter::emitBinSearchTable(raw_ostream &OS) { if (ColInstrs[j] != nullptr) { RelExists = 1; OutStr += ", "; - OutStr += TargetName; + OutStr += Namespace; OutStr += "::"; OutStr += ColInstrs[j]->getName(); } else { OutStr += ", (uint16_t)-1U";} } if (RelExists) { - OS << " { " << TargetName << "::" << CurInstr->getName(); + OS << " { " << Namespace << "::" << CurInstr->getName(); OS << OutStr <<" },\n"; TableSize++; } } } if (!TableSize) { - OS << " { " << TargetName << "::" << "INSTRUCTION_LIST_END, "; - OS << TargetName << "::" << "INSTRUCTION_LIST_END }"; + OS << " { " << Namespace << "::" << "INSTRUCTION_LIST_END, "; + OS << Namespace << "::" << "INSTRUCTION_LIST_END }"; } OS << "}; // End of " << InstrMapDesc.getName() << "Table\n\n"; return TableSize; @@ -567,7 +567,7 @@ namespace llvm { //===----------------------------------------------------------------------===// void EmitMapTable(RecordKeeper &Records, raw_ostream &OS) { CodeGenTarget Target(Records); - std::string TargetName = Target.getName(); + std::string NameSpace = Target.getInstNamespace(); std::vector<Record*> InstrMapVec; InstrMapVec = Records.getAllDerivedDefinitions("InstrMapping"); @@ -577,7 +577,7 @@ void EmitMapTable(RecordKeeper &Records, raw_ostream &OS) { OS << "#ifdef GET_INSTRMAP_INFO\n"; OS << "#undef GET_INSTRMAP_INFO\n"; OS << "namespace llvm {\n\n"; - OS << "namespace " << TargetName << " {\n\n"; + OS << "namespace " << NameSpace << " {\n\n"; // Emit coulumn field names and their values as enums. emitEnums(OS, Records); @@ -600,7 +600,7 @@ void EmitMapTable(RecordKeeper &Records, raw_ostream &OS) { // Emit map tables and the functions to query them. IMap.emitTablesWithFunc(OS); } - OS << "} // End " << TargetName << " namespace\n"; + OS << "} // End " << NameSpace << " namespace\n"; OS << "} // End llvm namespace\n"; OS << "#endif // GET_INSTRMAP_INFO\n\n"; } diff --git a/utils/TableGen/CodeGenRegisters.cpp b/utils/TableGen/CodeGenRegisters.cpp index c03e0d1fcf6b1..627614d991d52 100644 --- a/utils/TableGen/CodeGenRegisters.cpp +++ b/utils/TableGen/CodeGenRegisters.cpp @@ -1668,7 +1668,7 @@ void CodeGenRegBank::computeRegUnitSets() { dbgs() << "UnitSet " << USIdx << " " << RegUnitSets[USIdx].Name << ":"; for (auto &U : RegUnitSets[USIdx].Units) - dbgs() << " " << RegUnits[U].Roots[0]->getName(); + printRegUnitName(U); dbgs() << "\n"; }); @@ -1681,7 +1681,7 @@ void CodeGenRegBank::computeRegUnitSets() { dbgs() << "UnitSet " << USIdx << " " << RegUnitSets[USIdx].Name << ":"; for (auto &U : RegUnitSets[USIdx].Units) - dbgs() << " " << RegUnits[U].Roots[0]->getName(); + printRegUnitName(U); dbgs() << "\n"; } dbgs() << "\nUnion sets:\n"); @@ -1727,7 +1727,7 @@ void CodeGenRegBank::computeRegUnitSets() { DEBUG(dbgs() << "UnitSet " << RegUnitSets.size()-1 << " " << RegUnitSets.back().Name << ":"; for (auto &U : RegUnitSets.back().Units) - dbgs() << " " << RegUnits[U].Roots[0]->getName(); + printRegUnitName(U); dbgs() << "\n";); } } @@ -1742,7 +1742,7 @@ void CodeGenRegBank::computeRegUnitSets() { dbgs() << "UnitSet " << USIdx << " " << RegUnitSets[USIdx].Name << ":"; for (auto &U : RegUnitSets[USIdx].Units) - dbgs() << " " << RegUnits[U].Roots[0]->getName(); + printRegUnitName(U); dbgs() << "\n"; }); @@ -1763,8 +1763,8 @@ void CodeGenRegBank::computeRegUnitSets() { continue; DEBUG(dbgs() << "RC " << RC.getName() << " Units: \n"; - for (auto &U : RCRegUnits) - dbgs() << RegUnits[U].getRoots()[0]->getName() << " "; + for (auto U : RCRegUnits) + printRegUnitName(U); dbgs() << "\n UnitSetIDs:"); // Find all supersets. @@ -2170,3 +2170,10 @@ BitVector CodeGenRegBank::computeCoveredRegisters(ArrayRef<Record*> Regs) { BV.set(Set[i]->EnumValue); return BV; } + +void CodeGenRegBank::printRegUnitName(unsigned Unit) const { + if (Unit < NumNativeRegUnits) + dbgs() << ' ' << RegUnits[Unit].Roots[0]->getName(); + else + dbgs() << " #" << Unit; +} diff --git a/utils/TableGen/CodeGenRegisters.h b/utils/TableGen/CodeGenRegisters.h index 3ed26fa401a13..9366838c77cd0 100644 --- a/utils/TableGen/CodeGenRegisters.h +++ b/utils/TableGen/CodeGenRegisters.h @@ -735,6 +735,10 @@ namespace llvm { // LaneMask is contained in CoveringLanes will be completely covered by // another sub-register with the same or larger lane mask. LaneBitmask CoveringLanes; + + // Helper function for printing debug information. Handles artificial + // (non-native) reg units. + void printRegUnitName(unsigned Unit) const; }; } // end namespace llvm diff --git a/utils/TableGen/CodeGenTarget.cpp b/utils/TableGen/CodeGenTarget.cpp index 6503d5af2d48d..d93511b0d8733 100644 --- a/utils/TableGen/CodeGenTarget.cpp +++ b/utils/TableGen/CodeGenTarget.cpp @@ -25,13 +25,18 @@ #include <algorithm> using namespace llvm; +cl::OptionCategory AsmParserCat("Options for -gen-asm-parser"); +cl::OptionCategory AsmWriterCat("Options for -gen-asm-writer"); + static cl::opt<unsigned> -AsmParserNum("asmparsernum", cl::init(0), - cl::desc("Make -gen-asm-parser emit assembly parser #N")); + AsmParserNum("asmparsernum", cl::init(0), + cl::desc("Make -gen-asm-parser emit assembly parser #N"), + cl::cat(AsmParserCat)); static cl::opt<unsigned> -AsmWriterNum("asmwriternum", cl::init(0), - cl::desc("Make -gen-asm-writer emit assembly writer #N")); + AsmWriterNum("asmwriternum", cl::init(0), + cl::desc("Make -gen-asm-writer emit assembly writer #N"), + cl::cat(AsmWriterCat)); /// getValueType - Return the MVT::SimpleValueType that the specified TableGen /// record corresponds to. diff --git a/utils/TableGen/DAGISelMatcherEmitter.cpp b/utils/TableGen/DAGISelMatcherEmitter.cpp index d30fc5131cbaf..67e8f15b248e7 100644 --- a/utils/TableGen/DAGISelMatcherEmitter.cpp +++ b/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -11,14 +11,18 @@ // //===----------------------------------------------------------------------===// -#include "DAGISelMatcher.h" #include "CodeGenDAGPatterns.h" +#include "DAGISelMatcher.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/FormattedStream.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" using namespace llvm; @@ -26,10 +30,17 @@ enum { CommentIndent = 30 }; +cl::OptionCategory DAGISelCat("Options for -gen-dag-isel"); + // To reduce generated source code size. -static cl::opt<bool> -OmitComments("omit-comments", cl::desc("Do not generate comments"), - cl::init(false)); +static cl::opt<bool> OmitComments("omit-comments", + cl::desc("Do not generate comments"), + cl::init(false), cl::cat(DAGISelCat)); + +static cl::opt<bool> InstrumentCoverage( + "instrument-coverage", + cl::desc("Generates tables to help identify patterns matched"), + cl::init(false), cl::cat(DAGISelCat)); namespace { class MatcherTableEmitter { @@ -52,6 +63,19 @@ class MatcherTableEmitter { DenseMap<Record*, unsigned> NodeXFormMap; std::vector<Record*> NodeXForms; + std::vector<std::string> VecIncludeStrings; + MapVector<std::string, unsigned, StringMap<unsigned> > VecPatterns; + + unsigned getPatternIdxFromTable(std::string &&P, std::string &&include_loc) { + const auto It = VecPatterns.find(P); + if (It == VecPatterns.end()) { + VecPatterns.insert(make_pair(std::move(P), VecPatterns.size())); + VecIncludeStrings.push_back(std::move(include_loc)); + return VecIncludeStrings.size() - 1; + } + return It->second; + } + public: MatcherTableEmitter(const CodeGenDAGPatterns &cgp) : CGP(cgp) {} @@ -62,6 +86,9 @@ public: void EmitPredicateFunctions(formatted_raw_ostream &OS); void EmitHistogram(const Matcher *N, formatted_raw_ostream &OS); + + void EmitPatternMatchTable(raw_ostream &OS); + private: unsigned EmitMatcher(const Matcher *N, unsigned Indent, unsigned CurrentIdx, formatted_raw_ostream &OS); @@ -117,6 +144,14 @@ private: }; } // end anonymous namespace. +static std::string GetPatFromTreePatternNode(const TreePatternNode *N) { + std::string str; + raw_string_ostream Stream(str); + Stream << *N; + Stream.str(); + return str; +} + static unsigned GetVBRSize(unsigned Val) { if (Val <= 127) return 1; @@ -150,6 +185,56 @@ static uint64_t EmitVBRValue(uint64_t Val, raw_ostream &OS) { return NumBytes+1; } +// This is expensive and slow. +static std::string getIncludePath(const Record *R) { + std::string str; + raw_string_ostream Stream(str); + auto Locs = R->getLoc(); + SMLoc L; + if (Locs.size() > 1) { + // Get where the pattern prototype was instantiated + L = Locs[1]; + } else if (Locs.size() == 1) { + L = Locs[0]; + } + unsigned CurBuf = SrcMgr.FindBufferContainingLoc(L); + assert(CurBuf && "Invalid or unspecified location!"); + + Stream << SrcMgr.getBufferInfo(CurBuf).Buffer->getBufferIdentifier() << ":" + << SrcMgr.FindLineNumber(L, CurBuf); + Stream.str(); + return str; +} + +void MatcherTableEmitter::EmitPatternMatchTable(raw_ostream &OS) { + + assert(isUInt<16>(VecPatterns.size()) && + "Using only 16 bits to encode offset into Pattern Table"); + assert(VecPatterns.size() == VecIncludeStrings.size() && + "The sizes of Pattern and include vectors should be the same"); + OS << "StringRef getPatternForIndex(unsigned Index) override {\n"; + OS << "static const char * PATTERN_MATCH_TABLE[] = {\n"; + + for (const auto &It : VecPatterns) { + OS << "\"" << It.first << "\",\n"; + } + + OS << "\n};"; + OS << "\nreturn StringRef(PATTERN_MATCH_TABLE[Index]);"; + OS << "\n}"; + + OS << "\nStringRef getIncludePathForIndex(unsigned Index) override {\n"; + OS << "static const char * INCLUDE_PATH_TABLE[] = {\n"; + + for (const auto &It : VecIncludeStrings) { + OS << "\"" << It << "\",\n"; + } + + OS << "\n};"; + OS << "\nreturn StringRef(INCLUDE_PATH_TABLE[Index]);"; + OS << "\n}"; +} + /// EmitMatcher - Emit bytes for the specified matcher and return /// the number of bytes emitted. unsigned MatcherTableEmitter:: @@ -537,6 +622,23 @@ EmitMatcher(const Matcher *N, unsigned Indent, unsigned CurrentIdx, case Matcher::EmitNode: case Matcher::MorphNodeTo: { + auto NumCoveredBytes = 0; + if (InstrumentCoverage) { + if (const MorphNodeToMatcher *SNT = dyn_cast<MorphNodeToMatcher>(N)) { + NumCoveredBytes = 3; + OS << "OPC_Coverage, "; + std::string src = + GetPatFromTreePatternNode(SNT->getPattern().getSrcPattern()); + std::string dst = + GetPatFromTreePatternNode(SNT->getPattern().getDstPattern()); + Record *PatRecord = SNT->getPattern().getSrcRecord(); + std::string include_src = getIncludePath(PatRecord); + unsigned Offset = + getPatternIdxFromTable(src + " -> " + dst, std::move(include_src)); + OS << "TARGET_VAL(" << Offset << "),\n"; + OS.PadToColumn(Indent * 2); + } + } const EmitNodeMatcherCommon *EN = cast<EmitNodeMatcherCommon>(N); OS << (isa<EmitNodeMatcher>(EN) ? "OPC_EmitNode" : "OPC_MorphNodeTo"); bool CompressVTs = EN->getNumVTs() < 3; @@ -593,10 +695,26 @@ EmitMatcher(const Matcher *N, unsigned Indent, unsigned CurrentIdx, } else OS << '\n'; - return 5 + !CompressVTs + EN->getNumVTs() + NumOperandBytes; + return 5 + !CompressVTs + EN->getNumVTs() + NumOperandBytes + + NumCoveredBytes; } case Matcher::CompleteMatch: { const CompleteMatchMatcher *CM = cast<CompleteMatchMatcher>(N); + auto NumCoveredBytes = 0; + if (InstrumentCoverage) { + NumCoveredBytes = 3; + OS << "OPC_Coverage, "; + std::string src = + GetPatFromTreePatternNode(CM->getPattern().getSrcPattern()); + std::string dst = + GetPatFromTreePatternNode(CM->getPattern().getDstPattern()); + Record *PatRecord = CM->getPattern().getSrcRecord(); + std::string include_src = getIncludePath(PatRecord); + unsigned Offset = + getPatternIdxFromTable(src + " -> " + dst, std::move(include_src)); + OS << "TARGET_VAL(" << Offset << "),\n"; + OS.PadToColumn(Indent * 2); + } OS << "OPC_CompleteMatch, " << CM->getNumResults() << ", "; unsigned NumResultBytes = 0; for (unsigned i = 0, e = CM->getNumResults(); i != e; ++i) @@ -610,7 +728,7 @@ EmitMatcher(const Matcher *N, unsigned Indent, unsigned CurrentIdx, << *CM->getPattern().getDstPattern(); } OS << '\n'; - return 2 + NumResultBytes; + return 2 + NumResultBytes + NumCoveredBytes; } } llvm_unreachable("Unreachable"); @@ -686,8 +804,13 @@ void MatcherTableEmitter::EmitPredicateFunctions(formatted_raw_ostream &OS) { ++NumOps; // Get the chained node too. OS << " case " << i << ":\n"; + if (InstrumentCoverage) + OS << " {\n"; OS << " Result.resize(NextRes+" << NumOps << ");\n"; - OS << " return " << P.getSelectFunc(); + if (InstrumentCoverage) + OS << " bool Succeeded = " << P.getSelectFunc(); + else + OS << " return " << P.getSelectFunc(); OS << "("; // If the complex pattern wants the root of the match, pass it in as the @@ -704,6 +827,13 @@ void MatcherTableEmitter::EmitPredicateFunctions(formatted_raw_ostream &OS) { for (unsigned i = 0; i != NumOps; ++i) OS << ", Result[NextRes+" << i << "].first"; OS << ");\n"; + if (InstrumentCoverage) { + OS << " if (Succeeded)\n"; + OS << " dbgs() << \"\\nCOMPLEX_PATTERN: " << P.getSelectFunc() + << "\\n\" ;\n"; + OS << " return Succeeded;\n"; + OS << " }\n"; + } } OS << " }\n"; OS << "}\n\n"; @@ -827,7 +957,7 @@ void llvm::EmitMatcherTable(const Matcher *TheMatcher, formatted_raw_ostream OS(O); OS << "// The main instruction selector code.\n"; - OS << "SDNode *SelectCode(SDNode *N) {\n"; + OS << "void SelectCode(SDNode *N) {\n"; MatcherTableEmitter MatcherEmitter(CGP); @@ -842,9 +972,11 @@ void llvm::EmitMatcherTable(const Matcher *TheMatcher, OS << " #undef TARGET_VAL\n"; OS << " SelectCodeCommon(N, MatcherTable,sizeof(MatcherTable));\n"; - OS << " return nullptr;\n"; OS << "}\n"; // Next up, emit the function for node and pattern predicates: MatcherEmitter.EmitPredicateFunctions(OS); + + if (InstrumentCoverage) + MatcherEmitter.EmitPatternMatchTable(OS); } diff --git a/utils/TableGen/DAGISelMatcherOpt.cpp b/utils/TableGen/DAGISelMatcherOpt.cpp index 783b35e745f8f..0bb656826fbdf 100644 --- a/utils/TableGen/DAGISelMatcherOpt.cpp +++ b/utils/TableGen/DAGISelMatcherOpt.cpp @@ -181,15 +181,21 @@ static Matcher *FindNodeWithKind(Matcher *M, Matcher::KindTy Kind) { /// ABC /// XYZ /// -static void FactorNodes(std::unique_ptr<Matcher> &MatcherPtr) { - // If we reached the end of the chain, we're done. - Matcher *N = MatcherPtr.get(); - if (!N) return; - - // If this is not a push node, just scan for one. - ScopeMatcher *Scope = dyn_cast<ScopeMatcher>(N); - if (!Scope) - return FactorNodes(N->getNextPtr()); +static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) { + // Look for a push node. Iterates instead of recurses to reduce stack usage. + ScopeMatcher *Scope = nullptr; + std::unique_ptr<Matcher> *RebindableMatcherPtr = &InputMatcherPtr; + while (!Scope) { + // If we reached the end of the chain, we're done. + Matcher *N = RebindableMatcherPtr->get(); + if (!N) return; + + // If this is not a push node, just scan for one. + Scope = dyn_cast<ScopeMatcher>(N); + if (!Scope) + RebindableMatcherPtr = &(N->getNextPtr()); + } + std::unique_ptr<Matcher> &MatcherPtr = *RebindableMatcherPtr; // Okay, pull together the children of the scope node into a vector so we can // inspect it more easily. diff --git a/utils/TableGen/FastISelEmitter.cpp b/utils/TableGen/FastISelEmitter.cpp index 43c6a98263302..0e7b0dc09442d 100644 --- a/utils/TableGen/FastISelEmitter.cpp +++ b/utils/TableGen/FastISelEmitter.cpp @@ -1,4 +1,4 @@ -//===- FastISelEmitter.cpp - Generate an instruction selector -------------===// +///===- FastISelEmitter.cpp - Generate an instruction selector -------------===// // // The LLVM Compiler Infrastructure // @@ -640,12 +640,9 @@ void FastISelMap::emitInstructionCode(raw_ostream &OS, OneHadNoPredicate = true; } else { if (OneHadNoPredicate) { - // FIXME: This should be a PrintError once the x86 target - // fixes PR21575. - PrintWarning("Multiple instructions match and one with no " - "predicate came before one with a predicate! " - "name:" + Memo.Name + " predicate: " + - PredicateCheck); + PrintFatalError("Multiple instructions match and one with no " + "predicate came before one with a predicate! " + "name:" + Memo.Name + " predicate: " + PredicateCheck); } OS << " if (" + PredicateCheck + ") {\n"; OS << " "; diff --git a/utils/TableGen/GlobalISelEmitter.cpp b/utils/TableGen/GlobalISelEmitter.cpp index 2bc6181045c5d..7acc65e349ea6 100644 --- a/utils/TableGen/GlobalISelEmitter.cpp +++ b/utils/TableGen/GlobalISelEmitter.cpp @@ -32,197 +32,1279 @@ #include "CodeGenDAGPatterns.h" #include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineValueType.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/LowLevelTypeImpl.h" +#include "llvm/Support/ScopedPrinter.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" #include <string> +#include <numeric> using namespace llvm; #define DEBUG_TYPE "gisel-emitter" STATISTIC(NumPatternTotal, "Total number of patterns"); -STATISTIC(NumPatternSkipped, "Number of patterns skipped"); +STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG"); +STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped"); STATISTIC(NumPatternEmitted, "Number of patterns emitted"); +cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel"); + static cl::opt<bool> WarnOnSkippedPatterns( "warn-on-skipped-patterns", cl::desc("Explain why a pattern was skipped for inclusion " "in the GlobalISel selector"), - cl::init(false)); + cl::init(false), cl::cat(GlobalISelEmitterCat)); namespace { +//===- Helper functions ---------------------------------------------------===// + +/// This class stands in for LLT wherever we want to tablegen-erate an +/// equivalent at compiler run-time. +class LLTCodeGen { +private: + LLT Ty; -class GlobalISelEmitter { public: - explicit GlobalISelEmitter(RecordKeeper &RK); - void run(raw_ostream &OS); + LLTCodeGen(const LLT &Ty) : Ty(Ty) {} -private: - const RecordKeeper &RK; - const CodeGenDAGPatterns CGP; - const CodeGenTarget &Target; + void emitCxxConstructorCall(raw_ostream &OS) const { + if (Ty.isScalar()) { + OS << "LLT::scalar(" << Ty.getSizeInBits() << ")"; + return; + } + if (Ty.isVector()) { + OS << "LLT::vector(" << Ty.getNumElements() << ", " << Ty.getSizeInBits() + << ")"; + return; + } + llvm_unreachable("Unhandled LLT"); + } - /// Keep track of the equivalence between SDNodes and Instruction. - /// This is defined using 'GINodeEquiv' in the target description. - DenseMap<Record *, const CodeGenInstruction *> NodeEquivs; + const LLT &get() const { return Ty; } +}; - void gatherNodeEquivs(); - const CodeGenInstruction *findNodeEquiv(Record *N); +class InstructionMatcher; +class OperandPlaceholder { +private: + enum PlaceholderKind { + OP_MatchReference, + OP_Temporary, + } Kind; + + struct MatchReferenceData { + InstructionMatcher *InsnMatcher; + StringRef InsnVarName; + StringRef SymbolicName; + }; - struct SkipReason { - std::string Reason; + struct TemporaryData { + unsigned OpIdx; }; - /// Analyze pattern \p P, possibly emitting matching code for it to \p OS. - /// Otherwise, return a reason why this pattern was skipped for emission. - Optional<SkipReason> runOnPattern(const PatternToMatch &P, - raw_ostream &OS); -}; + union { + struct MatchReferenceData MatchReference; + struct TemporaryData Temporary; + }; -} // end anonymous namespace + OperandPlaceholder(PlaceholderKind Kind) : Kind(Kind) {} -//===- Helper functions ---------------------------------------------------===// +public: + ~OperandPlaceholder() {} + + static OperandPlaceholder + CreateMatchReference(InstructionMatcher *InsnMatcher, + StringRef InsnVarName, StringRef SymbolicName) { + OperandPlaceholder Result(OP_MatchReference); + Result.MatchReference.InsnMatcher = InsnMatcher; + Result.MatchReference.InsnVarName = InsnVarName; + Result.MatchReference.SymbolicName = SymbolicName; + return Result; + } + + static OperandPlaceholder CreateTemporary(unsigned OpIdx) { + OperandPlaceholder Result(OP_Temporary); + Result.Temporary.OpIdx = OpIdx; + return Result; + } + + void emitCxxValueExpr(raw_ostream &OS) const; +}; /// Convert an MVT to an equivalent LLT if possible, or the invalid LLT() for /// MVTs that don't map cleanly to an LLT (e.g., iPTR, *any, ...). -static Optional<std::string> MVTToLLT(MVT::SimpleValueType SVT) { - std::string TyStr; - raw_string_ostream OS(TyStr); +static Optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT) { MVT VT(SVT); - if (VT.isVector() && VT.getVectorNumElements() != 1) { - OS << "LLT::vector(" << VT.getVectorNumElements() << ", " - << VT.getScalarSizeInBits() << ")"; - } else if (VT.isInteger() || VT.isFloatingPoint()) { - OS << "LLT::scalar(" << VT.getSizeInBits() << ")"; - } else { - return None; + if (VT.isVector() && VT.getVectorNumElements() != 1) + return LLTCodeGen(LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits())); + if (VT.isInteger() || VT.isFloatingPoint()) + return LLTCodeGen(LLT::scalar(VT.getSizeInBits())); + return None; +} + +static std::string explainPredicates(const TreePatternNode *N) { + std::string Explanation = ""; + StringRef Separator = ""; + for (const auto &P : N->getPredicateFns()) { + Explanation += + (Separator + P.getOrigPatFragRecord()->getRecord()->getName()).str(); + if (P.isAlwaysTrue()) + Explanation += " always-true"; + if (P.isImmediatePattern()) + Explanation += " immediate"; } - OS.flush(); - return TyStr; + return Explanation; +} + +static std::string explainRulePredicates(const ArrayRef<Init *> Predicates) { + std::string Explanation = ""; + StringRef Separator = ""; + for (const auto *P : Predicates) { + Explanation += Separator; + + if (const DefInit *PDef = dyn_cast<DefInit>(P)) { + Explanation += PDef->getDef()->getName(); + } else + Explanation += "<unknown>"; + } + return Explanation; +} + +std::string explainOperator(Record *Operator) { + if (Operator->isSubClassOf("SDNode")) + return " (" + Operator->getValueAsString("Opcode") + ")"; + + if (Operator->isSubClassOf("Intrinsic")) + return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str(); + + return " (Operator not understood)"; +} + +/// Helper function to let the emitter report skip reason error messages. +static Error failedImport(const Twine &Reason) { + return make_error<StringError>(Reason, inconvertibleErrorCode()); } -static bool isTrivialOperatorNode(const TreePatternNode *N) { - return !N->isLeaf() && !N->hasAnyPredicate() && !N->getTransformFn(); +static Error isTrivialOperatorNode(const TreePatternNode *N) { + std::string Explanation = ""; + std::string Separator = ""; + if (N->isLeaf()) { + Explanation = "Is a leaf"; + Separator = ", "; + } + + if (N->hasAnyPredicate()) { + Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")"; + Separator = ", "; + } + + if (N->getTransformFn()) { + Explanation += Separator + "Has a transform function"; + Separator = ", "; + } + + if (!N->isLeaf() && !N->hasAnyPredicate() && !N->getTransformFn()) + return Error::success(); + + return failedImport(Explanation); } //===- Matchers -----------------------------------------------------------===// -struct Matcher { - virtual ~Matcher() {} - virtual void emit(raw_ostream &OS) const = 0; +class OperandMatcher; +class MatchAction; + +/// Generates code to check that a match rule matches. +class RuleMatcher { + /// A list of matchers that all need to succeed for the current rule to match. + /// FIXME: This currently supports a single match position but could be + /// extended to support multiple positions to support div/rem fusion or + /// load-multiple instructions. + std::vector<std::unique_ptr<InstructionMatcher>> Matchers; + + /// A list of actions that need to be taken when all predicates in this rule + /// have succeeded. + std::vector<std::unique_ptr<MatchAction>> Actions; + + /// A map of instruction matchers to the local variables created by + /// emitCxxCaptureStmts(). + std::map<const InstructionMatcher *, std::string> InsnVariableNames; + + /// ID for the next instruction variable defined with defineInsnVar() + unsigned NextInsnVarID; + +public: + RuleMatcher() + : Matchers(), Actions(), InsnVariableNames(), NextInsnVarID(0) {} + RuleMatcher(RuleMatcher &&Other) = default; + RuleMatcher &operator=(RuleMatcher &&Other) = default; + + InstructionMatcher &addInstructionMatcher(); + + template <class Kind, class... Args> Kind &addAction(Args &&... args); + + std::string defineInsnVar(raw_ostream &OS, const InstructionMatcher &Matcher, + StringRef Value); + StringRef getInsnVarName(const InstructionMatcher &InsnMatcher) const; + + void emitCxxCapturedInsnList(raw_ostream &OS); + void emitCxxCaptureStmts(raw_ostream &OS, StringRef Expr); + + void emit(raw_ostream &OS); + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + bool isHigherPriorityThan(const RuleMatcher &B) const; + + /// Report the maximum number of temporary operands needed by the rule + /// matcher. + unsigned countTemporaryOperands() const; }; -raw_ostream &operator<<(raw_ostream &S, const Matcher &M) { - M.emit(S); - return S; -} +template <class PredicateTy> class PredicateListMatcher { +private: + typedef std::vector<std::unique_ptr<PredicateTy>> PredicateVec; + PredicateVec Predicates; -struct MatchAction { - virtual ~MatchAction() {} - virtual void emit(raw_ostream &OS) const = 0; +public: + /// Construct a new operand predicate and add it to the matcher. + template <class Kind, class... Args> + Kind &addPredicate(Args&&... args) { + Predicates.emplace_back( + llvm::make_unique<Kind>(std::forward<Args>(args)...)); + return *static_cast<Kind *>(Predicates.back().get()); + } + + typename PredicateVec::const_iterator predicates_begin() const { return Predicates.begin(); } + typename PredicateVec::const_iterator predicates_end() const { return Predicates.end(); } + iterator_range<typename PredicateVec::const_iterator> predicates() const { + return make_range(predicates_begin(), predicates_end()); + } + typename PredicateVec::size_type predicates_size() const { return Predicates.size(); } + + /// Emit a C++ expression that tests whether all the predicates are met. + template <class... Args> + void emitCxxPredicateListExpr(raw_ostream &OS, Args &&... args) const { + if (Predicates.empty()) { + OS << "true"; + return; + } + + StringRef Separator = ""; + for (const auto &Predicate : predicates()) { + OS << Separator << "("; + Predicate->emitCxxPredicateExpr(OS, std::forward<Args>(args)...); + OS << ")"; + Separator = " &&\n"; + } + } }; -raw_ostream &operator<<(raw_ostream &S, const MatchAction &A) { - A.emit(S); - return S; -} +/// Generates code to check a predicate of an operand. +/// +/// Typical predicates include: +/// * Operand is a particular register. +/// * Operand is assigned a particular register bank. +/// * Operand is an MBB. +class OperandPredicateMatcher { +public: + /// This enum is used for RTTI and also defines the priority that is given to + /// the predicate when generating the matcher code. Kinds with higher priority + /// must be tested first. + /// + /// The relative priority of OPM_LLT, OPM_RegBank, and OPM_MBB do not matter + /// but OPM_Int must have priority over OPM_RegBank since constant integers + /// are represented by a virtual register defined by a G_CONSTANT instruction. + enum PredicateKind { + OPM_ComplexPattern, + OPM_Instruction, + OPM_Int, + OPM_LLT, + OPM_RegBank, + OPM_MBB, + }; -struct MatchOpcode : public Matcher { - MatchOpcode(const CodeGenInstruction *I) : I(I) {} - const CodeGenInstruction *I; +protected: + PredicateKind Kind; + +public: + OperandPredicateMatcher(PredicateKind Kind) : Kind(Kind) {} + virtual ~OperandPredicateMatcher() {} + + PredicateKind getKind() const { return Kind; } + + /// Return the OperandMatcher for the specified operand or nullptr if there + /// isn't one by that name in this operand predicate matcher. + /// + /// InstructionOperandMatcher is the only subclass that can return non-null + /// for this. + virtual Optional<const OperandMatcher *> + getOptionalOperand(StringRef SymbolicName) const { + assert(!SymbolicName.empty() && "Cannot lookup unnamed operand"); + return None; + } + + /// Emit C++ statements to capture instructions into local variables. + /// + /// Only InstructionOperandMatcher needs to do anything for this method. + virtual void emitCxxCaptureStmts(raw_ostream &OS, RuleMatcher &Rule, + StringRef Expr) const {} + + /// Emit a C++ expression that checks the predicate for the given operand. + virtual void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef OperandExpr) const = 0; + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + virtual bool isHigherPriorityThan(const OperandPredicateMatcher &B) const { + return Kind < B.Kind; + }; + + /// Report the maximum number of temporary operands needed by the predicate + /// matcher. + virtual unsigned countTemporaryOperands() const { return 0; } +}; + +/// Generates code to check that an operand is a particular LLT. +class LLTOperandMatcher : public OperandPredicateMatcher { +protected: + LLTCodeGen Ty; - virtual void emit(raw_ostream &OS) const { - OS << "I.getOpcode() == " << I->Namespace << "::" << I->TheDef->getName(); +public: + LLTOperandMatcher(const LLTCodeGen &Ty) + : OperandPredicateMatcher(OPM_LLT), Ty(Ty) {} + + static bool classof(const OperandPredicateMatcher *P) { + return P->getKind() == OPM_LLT; + } + + void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef OperandExpr) const override { + OS << "MRI.getType(" << OperandExpr << ".getReg()) == ("; + Ty.emitCxxConstructorCall(OS); + OS << ")"; } }; -struct MatchRegOpType : public Matcher { - MatchRegOpType(unsigned OpIdx, std::string Ty) - : OpIdx(OpIdx), Ty(Ty) {} - unsigned OpIdx; - std::string Ty; +/// Generates code to check that an operand is a particular target constant. +class ComplexPatternOperandMatcher : public OperandPredicateMatcher { +protected: + const OperandMatcher &Operand; + const Record &TheDef; + + unsigned getNumOperands() const { + return TheDef.getValueAsDag("Operands")->getNumArgs(); + } - virtual void emit(raw_ostream &OS) const { - OS << "MRI.getType(I.getOperand(" << OpIdx << ").getReg()) == (" << Ty - << ")"; + unsigned getAllocatedTemporariesBaseID() const; + +public: + ComplexPatternOperandMatcher(const OperandMatcher &Operand, + const Record &TheDef) + : OperandPredicateMatcher(OPM_ComplexPattern), Operand(Operand), + TheDef(TheDef) {} + + static bool classof(const OperandPredicateMatcher *P) { + return P->getKind() == OPM_ComplexPattern; + } + + void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef OperandExpr) const override { + OS << TheDef.getValueAsString("MatcherFn") << "(" << OperandExpr; + for (unsigned I = 0; I < getNumOperands(); ++I) { + OS << ", "; + OperandPlaceholder::CreateTemporary(getAllocatedTemporariesBaseID() + I) + .emitCxxValueExpr(OS); + } + OS << ")"; + } + + unsigned countTemporaryOperands() const override { + return getNumOperands(); } }; -struct MatchRegOpBank : public Matcher { - MatchRegOpBank(unsigned OpIdx, const CodeGenRegisterClass &RC) - : OpIdx(OpIdx), RC(RC) {} - unsigned OpIdx; +/// Generates code to check that an operand is in a particular register bank. +class RegisterBankOperandMatcher : public OperandPredicateMatcher { +protected: const CodeGenRegisterClass &RC; - virtual void emit(raw_ostream &OS) const { +public: + RegisterBankOperandMatcher(const CodeGenRegisterClass &RC) + : OperandPredicateMatcher(OPM_RegBank), RC(RC) {} + + static bool classof(const OperandPredicateMatcher *P) { + return P->getKind() == OPM_RegBank; + } + + void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef OperandExpr) const override { OS << "(&RBI.getRegBankFromRegClass(" << RC.getQualifiedName() - << "RegClass) == RBI.getRegBank(I.getOperand(" << OpIdx - << ").getReg(), MRI, TRI))"; + << "RegClass) == RBI.getRegBank(" << OperandExpr + << ".getReg(), MRI, TRI))"; } }; -struct MatchMBBOp : public Matcher { - MatchMBBOp(unsigned OpIdx) : OpIdx(OpIdx) {} +/// Generates code to check that an operand is a basic block. +class MBBOperandMatcher : public OperandPredicateMatcher { +public: + MBBOperandMatcher() : OperandPredicateMatcher(OPM_MBB) {} + + static bool classof(const OperandPredicateMatcher *P) { + return P->getKind() == OPM_MBB; + } + + void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef OperandExpr) const override { + OS << OperandExpr << ".isMBB()"; + } +}; + +/// Generates code to check that an operand is a particular int. +class IntOperandMatcher : public OperandPredicateMatcher { +protected: + int64_t Value; + +public: + IntOperandMatcher(int64_t Value) + : OperandPredicateMatcher(OPM_Int), Value(Value) {} + + static bool classof(const OperandPredicateMatcher *P) { + return P->getKind() == OPM_Int; + } + + void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef OperandExpr) const override { + OS << "isOperandImmEqual(" << OperandExpr << ", " << Value << ", MRI)"; + } +}; + +/// Generates code to check that a set of predicates match for a particular +/// operand. +class OperandMatcher : public PredicateListMatcher<OperandPredicateMatcher> { +protected: + InstructionMatcher &Insn; unsigned OpIdx; + std::string SymbolicName; + + /// The index of the first temporary variable allocated to this operand. The + /// number of allocated temporaries can be found with + /// countTemporaryOperands(). + unsigned AllocatedTemporariesBaseID; + +public: + OperandMatcher(InstructionMatcher &Insn, unsigned OpIdx, + const std::string &SymbolicName, + unsigned AllocatedTemporariesBaseID) + : Insn(Insn), OpIdx(OpIdx), SymbolicName(SymbolicName), + AllocatedTemporariesBaseID(AllocatedTemporariesBaseID) {} + + bool hasSymbolicName() const { return !SymbolicName.empty(); } + const StringRef getSymbolicName() const { return SymbolicName; } + void setSymbolicName(StringRef Name) { + assert(SymbolicName.empty() && "Operand already has a symbolic name"); + SymbolicName = Name; + } + unsigned getOperandIndex() const { return OpIdx; } + + std::string getOperandExpr(StringRef InsnVarName) const { + return (InsnVarName + ".getOperand(" + llvm::to_string(OpIdx) + ")").str(); + } + + Optional<const OperandMatcher *> + getOptionalOperand(StringRef DesiredSymbolicName) const { + assert(!DesiredSymbolicName.empty() && "Cannot lookup unnamed operand"); + if (DesiredSymbolicName == SymbolicName) + return this; + for (const auto &OP : predicates()) { + const auto &MaybeOperand = OP->getOptionalOperand(DesiredSymbolicName); + if (MaybeOperand.hasValue()) + return MaybeOperand.getValue(); + } + return None; + } + + InstructionMatcher &getInstructionMatcher() const { return Insn; } + + /// Emit C++ statements to capture instructions into local variables. + void emitCxxCaptureStmts(raw_ostream &OS, RuleMatcher &Rule, + StringRef OperandExpr) const { + for (const auto &Predicate : predicates()) + Predicate->emitCxxCaptureStmts(OS, Rule, OperandExpr); + } + + /// Emit a C++ expression that tests whether the instruction named in + /// InsnVarName matches all the predicate and all the operands. + void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef InsnVarName) const { + OS << "(/* "; + if (SymbolicName.empty()) + OS << "Operand " << OpIdx; + else + OS << SymbolicName; + OS << " */ "; + emitCxxPredicateListExpr(OS, Rule, getOperandExpr(InsnVarName)); + OS << ")"; + } + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + bool isHigherPriorityThan(const OperandMatcher &B) const { + // Operand matchers involving more predicates have higher priority. + if (predicates_size() > B.predicates_size()) + return true; + if (predicates_size() < B.predicates_size()) + return false; + + // This assumes that predicates are added in a consistent order. + for (const auto &Predicate : zip(predicates(), B.predicates())) { + if (std::get<0>(Predicate)->isHigherPriorityThan(*std::get<1>(Predicate))) + return true; + if (std::get<1>(Predicate)->isHigherPriorityThan(*std::get<0>(Predicate))) + return false; + } + + return false; + }; + + /// Report the maximum number of temporary operands needed by the operand + /// matcher. + unsigned countTemporaryOperands() const { + return std::accumulate( + predicates().begin(), predicates().end(), 0, + [](unsigned A, + const std::unique_ptr<OperandPredicateMatcher> &Predicate) { + return A + Predicate->countTemporaryOperands(); + }); + } - virtual void emit(raw_ostream &OS) const { - OS << "I.getOperand(" << OpIdx << ").isMBB()"; + unsigned getAllocatedTemporariesBaseID() const { + return AllocatedTemporariesBaseID; } }; -struct MutateOpcode : public MatchAction { - MutateOpcode(const CodeGenInstruction *I) : I(I) {} +unsigned ComplexPatternOperandMatcher::getAllocatedTemporariesBaseID() const { + return Operand.getAllocatedTemporariesBaseID(); +} + +/// Generates code to check a predicate on an instruction. +/// +/// Typical predicates include: +/// * The opcode of the instruction is a particular value. +/// * The nsw/nuw flag is/isn't set. +class InstructionPredicateMatcher { +protected: + /// This enum is used for RTTI and also defines the priority that is given to + /// the predicate when generating the matcher code. Kinds with higher priority + /// must be tested first. + enum PredicateKind { + IPM_Opcode, + }; + + PredicateKind Kind; + +public: + InstructionPredicateMatcher(PredicateKind Kind) : Kind(Kind) {} + virtual ~InstructionPredicateMatcher() {} + + PredicateKind getKind() const { return Kind; } + + /// Emit a C++ expression that tests whether the instruction named in + /// InsnVarName matches the predicate. + virtual void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef InsnVarName) const = 0; + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + virtual bool isHigherPriorityThan(const InstructionPredicateMatcher &B) const { + return Kind < B.Kind; + }; + + /// Report the maximum number of temporary operands needed by the predicate + /// matcher. + virtual unsigned countTemporaryOperands() const { return 0; } +}; + +/// Generates code to check the opcode of an instruction. +class InstructionOpcodeMatcher : public InstructionPredicateMatcher { +protected: const CodeGenInstruction *I; - virtual void emit(raw_ostream &OS) const { - OS << "I.setDesc(TII.get(" << I->Namespace << "::" << I->TheDef->getName() - << "));"; +public: + InstructionOpcodeMatcher(const CodeGenInstruction *I) + : InstructionPredicateMatcher(IPM_Opcode), I(I) {} + + static bool classof(const InstructionPredicateMatcher *P) { + return P->getKind() == IPM_Opcode; + } + + void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef InsnVarName) const override { + OS << InsnVarName << ".getOpcode() == " << I->Namespace + << "::" << I->TheDef->getName(); } + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + bool isHigherPriorityThan(const InstructionPredicateMatcher &B) const override { + if (InstructionPredicateMatcher::isHigherPriorityThan(B)) + return true; + if (B.InstructionPredicateMatcher::isHigherPriorityThan(*this)) + return false; + + // Prioritize opcodes for cosmetic reasons in the generated source. Although + // this is cosmetic at the moment, we may want to drive a similar ordering + // using instruction frequency information to improve compile time. + if (const InstructionOpcodeMatcher *BO = + dyn_cast<InstructionOpcodeMatcher>(&B)) + return I->TheDef->getName() < BO->I->TheDef->getName(); + + return false; + }; +}; + +/// Generates code to check that a set of predicates and operands match for a +/// particular instruction. +/// +/// Typical predicates include: +/// * Has a specific opcode. +/// * Has an nsw/nuw flag or doesn't. +class InstructionMatcher + : public PredicateListMatcher<InstructionPredicateMatcher> { +protected: + typedef std::vector<std::unique_ptr<OperandMatcher>> OperandVec; + + /// The operands to match. All rendered operands must be present even if the + /// condition is always true. + OperandVec Operands; + +public: + /// Add an operand to the matcher. + OperandMatcher &addOperand(unsigned OpIdx, const std::string &SymbolicName, + unsigned AllocatedTemporariesBaseID) { + Operands.emplace_back(new OperandMatcher(*this, OpIdx, SymbolicName, + AllocatedTemporariesBaseID)); + return *Operands.back(); + } + + OperandMatcher &getOperand(unsigned OpIdx) { + auto I = std::find_if(Operands.begin(), Operands.end(), + [&OpIdx](const std::unique_ptr<OperandMatcher> &X) { + return X->getOperandIndex() == OpIdx; + }); + if (I != Operands.end()) + return **I; + llvm_unreachable("Failed to lookup operand"); + } + + Optional<const OperandMatcher *> + getOptionalOperand(StringRef SymbolicName) const { + assert(!SymbolicName.empty() && "Cannot lookup unnamed operand"); + for (const auto &Operand : Operands) { + const auto &OM = Operand->getOptionalOperand(SymbolicName); + if (OM.hasValue()) + return OM.getValue(); + } + return None; + } + + const OperandMatcher &getOperand(StringRef SymbolicName) const { + Optional<const OperandMatcher *>OM = getOptionalOperand(SymbolicName); + if (OM.hasValue()) + return *OM.getValue(); + llvm_unreachable("Failed to lookup operand"); + } + + unsigned getNumOperands() const { return Operands.size(); } + OperandVec::iterator operands_begin() { return Operands.begin(); } + OperandVec::iterator operands_end() { return Operands.end(); } + iterator_range<OperandVec::iterator> operands() { + return make_range(operands_begin(), operands_end()); + } + OperandVec::const_iterator operands_begin() const { return Operands.begin(); } + OperandVec::const_iterator operands_end() const { return Operands.end(); } + iterator_range<OperandVec::const_iterator> operands() const { + return make_range(operands_begin(), operands_end()); + } + + /// Emit C++ statements to check the shape of the match and capture + /// instructions into local variables. + void emitCxxCaptureStmts(raw_ostream &OS, RuleMatcher &Rule, StringRef Expr) { + OS << "if (" << Expr << ".getNumOperands() < " << getNumOperands() << ")\n" + << " return false;\n"; + for (const auto &Operand : Operands) { + Operand->emitCxxCaptureStmts(OS, Rule, Operand->getOperandExpr(Expr)); + } + } + + /// Emit a C++ expression that tests whether the instruction named in + /// InsnVarName matches all the predicates and all the operands. + void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef InsnVarName) const { + emitCxxPredicateListExpr(OS, Rule, InsnVarName); + for (const auto &Operand : Operands) { + OS << " &&\n("; + Operand->emitCxxPredicateExpr(OS, Rule, InsnVarName); + OS << ")"; + } + } + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + bool isHigherPriorityThan(const InstructionMatcher &B) const { + // Instruction matchers involving more operands have higher priority. + if (Operands.size() > B.Operands.size()) + return true; + if (Operands.size() < B.Operands.size()) + return false; + + for (const auto &Predicate : zip(predicates(), B.predicates())) { + if (std::get<0>(Predicate)->isHigherPriorityThan(*std::get<1>(Predicate))) + return true; + if (std::get<1>(Predicate)->isHigherPriorityThan(*std::get<0>(Predicate))) + return false; + } + + for (const auto &Operand : zip(Operands, B.Operands)) { + if (std::get<0>(Operand)->isHigherPriorityThan(*std::get<1>(Operand))) + return true; + if (std::get<1>(Operand)->isHigherPriorityThan(*std::get<0>(Operand))) + return false; + } + + return false; + }; + + /// Report the maximum number of temporary operands needed by the instruction + /// matcher. + unsigned countTemporaryOperands() const { + return std::accumulate(predicates().begin(), predicates().end(), 0, + [](unsigned A, + const std::unique_ptr<InstructionPredicateMatcher> + &Predicate) { + return A + Predicate->countTemporaryOperands(); + }) + + std::accumulate( + Operands.begin(), Operands.end(), 0, + [](unsigned A, const std::unique_ptr<OperandMatcher> &Operand) { + return A + Operand->countTemporaryOperands(); + }); + } +}; + +/// Generates code to check that the operand is a register defined by an +/// instruction that matches the given instruction matcher. +/// +/// For example, the pattern: +/// (set $dst, (G_MUL (G_ADD $src1, $src2), $src3)) +/// would use an InstructionOperandMatcher for operand 1 of the G_MUL to match +/// the: +/// (G_ADD $src1, $src2) +/// subpattern. +class InstructionOperandMatcher : public OperandPredicateMatcher { +protected: + std::unique_ptr<InstructionMatcher> InsnMatcher; + +public: + InstructionOperandMatcher() + : OperandPredicateMatcher(OPM_Instruction), + InsnMatcher(new InstructionMatcher()) {} + + static bool classof(const OperandPredicateMatcher *P) { + return P->getKind() == OPM_Instruction; + } + + InstructionMatcher &getInsnMatcher() const { return *InsnMatcher; } + + Optional<const OperandMatcher *> + getOptionalOperand(StringRef SymbolicName) const override { + assert(!SymbolicName.empty() && "Cannot lookup unnamed operand"); + return InsnMatcher->getOptionalOperand(SymbolicName); + } + + void emitCxxCaptureStmts(raw_ostream &OS, RuleMatcher &Rule, + StringRef OperandExpr) const override { + OS << "if (!" << OperandExpr + ".isReg())\n" + << " return false;\n"; + std::string InsnVarName = Rule.defineInsnVar( + OS, *InsnMatcher, + ("*MRI.getVRegDef(" + OperandExpr + ".getReg())").str()); + InsnMatcher->emitCxxCaptureStmts(OS, Rule, InsnVarName); + } + + void emitCxxPredicateExpr(raw_ostream &OS, RuleMatcher &Rule, + StringRef OperandExpr) const override { + OperandExpr = Rule.getInsnVarName(*InsnMatcher); + OS << "("; + InsnMatcher->emitCxxPredicateExpr(OS, Rule, OperandExpr); + OS << ")\n"; + } +}; + +//===- Actions ------------------------------------------------------------===// +void OperandPlaceholder::emitCxxValueExpr(raw_ostream &OS) const { + switch (Kind) { + case OP_MatchReference: + OS << MatchReference.InsnMatcher->getOperand(MatchReference.SymbolicName) + .getOperandExpr(MatchReference.InsnVarName); + break; + case OP_Temporary: + OS << "TempOp" << Temporary.OpIdx; + break; + } +} + +class OperandRenderer { +public: + enum RendererKind { OR_Copy, OR_Imm, OR_Register, OR_ComplexPattern }; + +protected: + RendererKind Kind; + +public: + OperandRenderer(RendererKind Kind) : Kind(Kind) {} + virtual ~OperandRenderer() {} + + RendererKind getKind() const { return Kind; } + + virtual void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const = 0; }; -class MatcherEmitter { +/// A CopyRenderer emits code to copy a single operand from an existing +/// instruction to the one being built. +class CopyRenderer : public OperandRenderer { +protected: + /// The matcher for the instruction that this operand is copied from. + /// This provides the facility for looking up an a operand by it's name so + /// that it can be used as a source for the instruction being built. + const InstructionMatcher &Matched; + /// The name of the operand. + const StringRef SymbolicName; + +public: + CopyRenderer(const InstructionMatcher &Matched, StringRef SymbolicName) + : OperandRenderer(OR_Copy), Matched(Matched), SymbolicName(SymbolicName) { + } + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_Copy; + } + + const StringRef getSymbolicName() const { return SymbolicName; } + + void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override { + const OperandMatcher &Operand = Matched.getOperand(SymbolicName); + StringRef InsnVarName = + Rule.getInsnVarName(Operand.getInstructionMatcher()); + std::string OperandExpr = Operand.getOperandExpr(InsnVarName); + OS << " MIB.add(" << OperandExpr << "/*" << SymbolicName << "*/);\n"; + } +}; + +/// Adds a specific physical register to the instruction being built. +/// This is typically useful for WZR/XZR on AArch64. +class AddRegisterRenderer : public OperandRenderer { +protected: + const Record *RegisterDef; + +public: + AddRegisterRenderer(const Record *RegisterDef) + : OperandRenderer(OR_Register), RegisterDef(RegisterDef) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_Register; + } + + void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override { + OS << " MIB.addReg(" << RegisterDef->getValueAsString("Namespace") + << "::" << RegisterDef->getName() << ");\n"; + } +}; + +/// Adds a specific immediate to the instruction being built. +class ImmRenderer : public OperandRenderer { +protected: + int64_t Imm; + +public: + ImmRenderer(int64_t Imm) + : OperandRenderer(OR_Imm), Imm(Imm) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_Imm; + } + + void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override { + OS << " MIB.addImm(" << Imm << ");\n"; + } +}; + +class RenderComplexPatternOperand : public OperandRenderer { +private: + const Record &TheDef; + std::vector<OperandPlaceholder> Sources; + + unsigned getNumOperands() const { + return TheDef.getValueAsDag("Operands")->getNumArgs(); + } + +public: + RenderComplexPatternOperand(const Record &TheDef, + const ArrayRef<OperandPlaceholder> Sources) + : OperandRenderer(OR_ComplexPattern), TheDef(TheDef), Sources(Sources) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_ComplexPattern; + } + + void emitCxxRenderStmts(raw_ostream &OS, RuleMatcher &Rule) const override { + assert(Sources.size() == getNumOperands() && "Inconsistent number of operands"); + for (const auto &Source : Sources) { + OS << "MIB.add("; + Source.emitCxxValueExpr(OS); + OS << ");\n"; + } + } +}; + +/// An action taken when all Matcher predicates succeeded for a parent rule. +/// +/// Typical actions include: +/// * Changing the opcode of an instruction. +/// * Adding an operand to an instruction. +class MatchAction { +public: + virtual ~MatchAction() {} + + /// Emit the C++ statements to implement the action. + /// + /// \param RecycleVarName If given, it's an instruction to recycle. The + /// requirements on the instruction vary from action to + /// action. + virtual void emitCxxActionStmts(raw_ostream &OS, RuleMatcher &Rule, + StringRef RecycleVarName) const = 0; +}; + +/// Generates a comment describing the matched rule being acted upon. +class DebugCommentAction : public MatchAction { +private: const PatternToMatch &P; public: - std::vector<std::unique_ptr<Matcher>> Matchers; - std::vector<std::unique_ptr<MatchAction>> Actions; + DebugCommentAction(const PatternToMatch &P) : P(P) {} + + void emitCxxActionStmts(raw_ostream &OS, RuleMatcher &Rule, + StringRef RecycleVarName) const override { + OS << "// " << *P.getSrcPattern() << " => " << *P.getDstPattern() << "\n"; + } +}; - MatcherEmitter(const PatternToMatch &P) : P(P) {} +/// Generates code to build an instruction or mutate an existing instruction +/// into the desired instruction when this is possible. +class BuildMIAction : public MatchAction { +private: + const CodeGenInstruction *I; + const InstructionMatcher &Matched; + std::vector<std::unique_ptr<OperandRenderer>> OperandRenderers; + + /// True if the instruction can be built solely by mutating the opcode. + bool canMutate() const { + for (const auto &Renderer : enumerate(OperandRenderers)) { + if (const auto *Copy = dyn_cast<CopyRenderer>(&*Renderer.value())) { + if (Matched.getOperand(Copy->getSymbolicName()).getOperandIndex() != + Renderer.index()) + return false; + } else + return false; + } - void emit(raw_ostream &OS) { - if (Matchers.empty()) - llvm_unreachable("Unexpected empty matcher!"); + return true; + } - OS << " // Src: " << *P.getSrcPattern() << "\n" - << " // Dst: " << *P.getDstPattern() << "\n"; +public: + BuildMIAction(const CodeGenInstruction *I, const InstructionMatcher &Matched) + : I(I), Matched(Matched) {} + + template <class Kind, class... Args> + Kind &addRenderer(Args&&... args) { + OperandRenderers.emplace_back( + llvm::make_unique<Kind>(std::forward<Args>(args)...)); + return *static_cast<Kind *>(OperandRenderers.back().get()); + } - OS << " if ((" << *Matchers.front() << ")"; - for (auto &MA : makeArrayRef(Matchers).drop_front()) - OS << " &&\n (" << *MA << ")"; - OS << ") {\n"; + void emitCxxActionStmts(raw_ostream &OS, RuleMatcher &Rule, + StringRef RecycleVarName) const override { + if (canMutate()) { + OS << " " << RecycleVarName << ".setDesc(TII.get(" << I->Namespace + << "::" << I->TheDef->getName() << "));\n"; - for (auto &MA : Actions) - OS << " " << *MA << "\n"; + if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) { + OS << " auto MIB = MachineInstrBuilder(MF, &" << RecycleVarName + << ");\n"; - OS << " constrainSelectedInstRegOperands(I, TII, TRI, RBI);\n"; - OS << " return true;\n"; - OS << " }\n"; + for (auto Def : I->ImplicitDefs) { + auto Namespace = Def->getValueAsString("Namespace"); + OS << " MIB.addDef(" << Namespace << "::" << Def->getName() + << ", RegState::Implicit);\n"; + } + for (auto Use : I->ImplicitUses) { + auto Namespace = Use->getValueAsString("Namespace"); + OS << " MIB.addUse(" << Namespace << "::" << Use->getName() + << ", RegState::Implicit);\n"; + } + } + + OS << " MachineInstr &NewI = " << RecycleVarName << ";\n"; + return; + } + + // TODO: Simple permutation looks like it could be almost as common as + // mutation due to commutative operations. + + OS << "MachineInstrBuilder MIB = BuildMI(*I.getParent(), I, " + "I.getDebugLoc(), TII.get(" + << I->Namespace << "::" << I->TheDef->getName() << "));\n"; + for (const auto &Renderer : OperandRenderers) + Renderer->emitCxxRenderStmts(OS, Rule); + OS << " for (const auto *FromMI : "; + Rule.emitCxxCapturedInsnList(OS); + OS << ")\n"; + OS << " for (const auto &MMO : FromMI->memoperands())\n"; + OS << " MIB.addMemOperand(MMO);\n"; + OS << " " << RecycleVarName << ".eraseFromParent();\n"; + OS << " MachineInstr &NewI = *MIB;\n"; } }; +InstructionMatcher &RuleMatcher::addInstructionMatcher() { + Matchers.emplace_back(new InstructionMatcher()); + return *Matchers.back(); +} + +template <class Kind, class... Args> +Kind &RuleMatcher::addAction(Args &&... args) { + Actions.emplace_back(llvm::make_unique<Kind>(std::forward<Args>(args)...)); + return *static_cast<Kind *>(Actions.back().get()); +} + +std::string RuleMatcher::defineInsnVar(raw_ostream &OS, + const InstructionMatcher &Matcher, + StringRef Value) { + std::string InsnVarName = "MI" + llvm::to_string(NextInsnVarID++); + OS << "MachineInstr &" << InsnVarName << " = " << Value << ";\n"; + InsnVariableNames[&Matcher] = InsnVarName; + return InsnVarName; +} + +StringRef RuleMatcher::getInsnVarName(const InstructionMatcher &InsnMatcher) const { + const auto &I = InsnVariableNames.find(&InsnMatcher); + if (I != InsnVariableNames.end()) + return I->second; + llvm_unreachable("Matched Insn was not captured in a local variable"); +} + +/// Emit a C++ initializer_list containing references to every matched instruction. +void RuleMatcher::emitCxxCapturedInsnList(raw_ostream &OS) { + SmallVector<StringRef, 2> Names; + for (const auto &Pair : InsnVariableNames) + Names.push_back(Pair.second); + std::sort(Names.begin(), Names.end()); + + OS << "{"; + for (const auto &Name : Names) + OS << "&" << Name << ", "; + OS << "}"; +} + +/// Emit C++ statements to check the shape of the match and capture +/// instructions into local variables. +void RuleMatcher::emitCxxCaptureStmts(raw_ostream &OS, StringRef Expr) { + assert(Matchers.size() == 1 && "Cannot handle multi-root matchers yet"); + std::string InsnVarName = defineInsnVar(OS, *Matchers.front(), Expr); + Matchers.front()->emitCxxCaptureStmts(OS, *this, InsnVarName); +} + +void RuleMatcher::emit(raw_ostream &OS) { + if (Matchers.empty()) + llvm_unreachable("Unexpected empty matcher!"); + + // The representation supports rules that require multiple roots such as: + // %ptr(p0) = ... + // %elt0(s32) = G_LOAD %ptr + // %1(p0) = G_ADD %ptr, 4 + // %elt1(s32) = G_LOAD p0 %1 + // which could be usefully folded into: + // %ptr(p0) = ... + // %elt0(s32), %elt1(s32) = TGT_LOAD_PAIR %ptr + // on some targets but we don't need to make use of that yet. + assert(Matchers.size() == 1 && "Cannot handle multi-root matchers yet"); + OS << "if ([&]() {\n"; + + emitCxxCaptureStmts(OS, "I"); + + OS << " if ("; + Matchers.front()->emitCxxPredicateExpr(OS, *this, + getInsnVarName(*Matchers.front())); + OS << ") {\n"; + + // We must also check if it's safe to fold the matched instructions. + if (InsnVariableNames.size() >= 2) { + for (const auto &Pair : InsnVariableNames) { + // Skip the root node since it isn't moving anywhere. Everything else is + // sinking to meet it. + if (Pair.first == Matchers.front().get()) + continue; + + // Reject the difficult cases until we have a more accurate check. + OS << " if (!isObviouslySafeToFold(" << Pair.second + << ")) return false;\n"; + + // FIXME: Emit checks to determine it's _actually_ safe to fold and/or + // account for unsafe cases. + // + // Example: + // MI1--> %0 = ... + // %1 = ... %0 + // MI0--> %2 = ... %0 + // It's not safe to erase MI1. We currently handle this by not + // erasing %0 (even when it's dead). + // + // Example: + // MI1--> %0 = load volatile @a + // %1 = load volatile @a + // MI0--> %2 = ... %0 + // It's not safe to sink %0's def past %1. We currently handle + // this by rejecting all loads. + // + // Example: + // MI1--> %0 = load @a + // %1 = store @a + // MI0--> %2 = ... %0 + // It's not safe to sink %0's def past %1. We currently handle + // this by rejecting all loads. + // + // Example: + // G_CONDBR %cond, @BB1 + // BB0: + // MI1--> %0 = load @a + // G_BR @BB1 + // BB1: + // MI0--> %2 = ... %0 + // It's not always safe to sink %0 across control flow. In this + // case it may introduce a memory fault. We currentl handle this + // by rejecting all loads. + } + } + + for (const auto &MA : Actions) { + MA->emitCxxActionStmts(OS, *this, "I"); + } + + OS << " constrainSelectedInstRegOperands(NewI, TII, TRI, RBI);\n"; + OS << " return true;\n"; + OS << " }\n"; + OS << " return false;\n"; + OS << " }()) { return true; }\n\n"; +} + +bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const { + // Rules involving more match roots have higher priority. + if (Matchers.size() > B.Matchers.size()) + return true; + if (Matchers.size() < B.Matchers.size()) + return false; + + for (const auto &Matcher : zip(Matchers, B.Matchers)) { + if (std::get<0>(Matcher)->isHigherPriorityThan(*std::get<1>(Matcher))) + return true; + if (std::get<1>(Matcher)->isHigherPriorityThan(*std::get<0>(Matcher))) + return false; + } + + return false; +} + +unsigned RuleMatcher::countTemporaryOperands() const { + return std::accumulate( + Matchers.begin(), Matchers.end(), 0, + [](unsigned A, const std::unique_ptr<InstructionMatcher> &Matcher) { + return A + Matcher->countTemporaryOperands(); + }); +} + //===- GlobalISelEmitter class --------------------------------------------===// +class GlobalISelEmitter { +public: + explicit GlobalISelEmitter(RecordKeeper &RK); + void run(raw_ostream &OS); + +private: + const RecordKeeper &RK; + const CodeGenDAGPatterns CGP; + const CodeGenTarget &Target; + + /// Keep track of the equivalence between SDNodes and Instruction. + /// This is defined using 'GINodeEquiv' in the target description. + DenseMap<Record *, const CodeGenInstruction *> NodeEquivs; + + /// Keep track of the equivalence between ComplexPattern's and + /// GIComplexOperandMatcher. Map entries are specified by subclassing + /// GIComplexPatternEquiv. + DenseMap<const Record *, const Record *> ComplexPatternEquivs; + + void gatherNodeEquivs(); + const CodeGenInstruction *findNodeEquiv(Record *N) const; + + Error importRulePredicates(RuleMatcher &M, ArrayRef<Init *> Predicates) const; + Expected<InstructionMatcher &> + createAndImportSelDAGMatcher(InstructionMatcher &InsnMatcher, + const TreePatternNode *Src) const; + Error importChildMatcher(InstructionMatcher &InsnMatcher, + TreePatternNode *SrcChild, unsigned OpIdx, + unsigned &TempOpIdx) const; + Expected<BuildMIAction &> createAndImportInstructionRenderer( + RuleMatcher &M, const TreePatternNode *Dst, + const InstructionMatcher &InsnMatcher) const; + Error importExplicitUseRenderer(BuildMIAction &DstMIBuilder, + TreePatternNode *DstChild, + const InstructionMatcher &InsnMatcher) const; + Error + importImplicitDefRenderers(BuildMIAction &DstMIBuilder, + const std::vector<Record *> &ImplicitDefs) const; + + /// Analyze pattern \p P, returning a matcher for it if possible. + /// Otherwise, return an Error explaining why we don't support it. + Expected<RuleMatcher> runOnPattern(const PatternToMatch &P); +}; + void GlobalISelEmitter::gatherNodeEquivs() { assert(NodeEquivs.empty()); for (Record *Equiv : RK.getAllDerivedDefinitions("GINodeEquiv")) NodeEquivs[Equiv->getValueAsDef("Node")] = &Target.getInstruction(Equiv->getValueAsDef("I")); + + assert(ComplexPatternEquivs.empty()); + for (Record *Equiv : RK.getAllDerivedDefinitions("GIComplexPatternEquiv")) { + Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent"); + if (!SelDAGEquiv) + continue; + ComplexPatternEquivs[SelDAGEquiv] = Equiv; + } } -const CodeGenInstruction *GlobalISelEmitter::findNodeEquiv(Record *N) { +const CodeGenInstruction *GlobalISelEmitter::findNodeEquiv(Record *N) const { return NodeEquivs.lookup(N); } @@ -231,126 +1313,373 @@ GlobalISelEmitter::GlobalISelEmitter(RecordKeeper &RK) //===- Emitter ------------------------------------------------------------===// -Optional<GlobalISelEmitter::SkipReason> -GlobalISelEmitter::runOnPattern(const PatternToMatch &P, raw_ostream &OS) { +Error +GlobalISelEmitter::importRulePredicates(RuleMatcher &M, + ArrayRef<Init *> Predicates) const { + if (!Predicates.empty()) + return failedImport("Pattern has a rule predicate (" + + explainRulePredicates(Predicates) + ")"); + return Error::success(); +} - // Keep track of the matchers and actions to emit. - MatcherEmitter M(P); +Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( + InstructionMatcher &InsnMatcher, const TreePatternNode *Src) const { + // Start with the defined operands (i.e., the results of the root operator). + if (Src->getExtTypes().size() > 1) + return failedImport("Src pattern has multiple results"); + + auto SrcGIOrNull = findNodeEquiv(Src->getOperator()); + if (!SrcGIOrNull) + return failedImport("Pattern operator lacks an equivalent Instruction" + + explainOperator(Src->getOperator())); + auto &SrcGI = *SrcGIOrNull; - // First, analyze the whole pattern. - // If the entire pattern has a predicate (e.g., target features), ignore it. - if (!P.getPredicates()->getValues().empty()) - return SkipReason{"Pattern has a predicate"}; + // The operators look good: match the opcode and mutate it to the new one. + InsnMatcher.addPredicate<InstructionOpcodeMatcher>(&SrcGI); - // Physreg imp-defs require additional logic. Ignore the pattern. - if (!P.getDstRegs().empty()) - return SkipReason{"Pattern defines a physical register"}; + unsigned OpIdx = 0; + unsigned TempOpIdx = 0; + for (const EEVT::TypeSet &Ty : Src->getExtTypes()) { + auto OpTyOrNone = MVTToLLT(Ty.getConcrete()); - // Next, analyze the pattern operators. - TreePatternNode *Src = P.getSrcPattern(); - TreePatternNode *Dst = P.getDstPattern(); + if (!OpTyOrNone) + return failedImport( + "Result of Src pattern operator has an unsupported type"); - // If the root of either pattern isn't a simple operator, ignore it. - if (!isTrivialOperatorNode(Dst)) - return SkipReason{"Dst pattern root isn't a trivial operator"}; - if (!isTrivialOperatorNode(Src)) - return SkipReason{"Src pattern root isn't a trivial operator"}; + // Results don't have a name unless they are the root node. The caller will + // set the name if appropriate. + OperandMatcher &OM = InsnMatcher.addOperand(OpIdx++, "", TempOpIdx); + OM.addPredicate<LLTOperandMatcher>(*OpTyOrNone); + } - Record *DstOp = Dst->getOperator(); - if (!DstOp->isSubClassOf("Instruction")) - return SkipReason{"Pattern operator isn't an instruction"}; + // Match the used operands (i.e. the children of the operator). + for (unsigned i = 0, e = Src->getNumChildren(); i != e; ++i) { + if (auto Error = importChildMatcher(InsnMatcher, Src->getChild(i), OpIdx++, + TempOpIdx)) + return std::move(Error); + } - auto &DstI = Target.getInstruction(DstOp); + return InsnMatcher; +} - auto SrcGIOrNull = findNodeEquiv(Src->getOperator()); - if (!SrcGIOrNull) - return SkipReason{"Pattern operator lacks an equivalent Instruction"}; - auto &SrcGI = *SrcGIOrNull; +Error GlobalISelEmitter::importChildMatcher(InstructionMatcher &InsnMatcher, + TreePatternNode *SrcChild, + unsigned OpIdx, + unsigned &TempOpIdx) const { + OperandMatcher &OM = + InsnMatcher.addOperand(OpIdx, SrcChild->getName(), TempOpIdx); + + if (SrcChild->hasAnyPredicate()) + return failedImport("Src pattern child has predicate (" + + explainPredicates(SrcChild) + ")"); + + ArrayRef<EEVT::TypeSet> ChildTypes = SrcChild->getExtTypes(); + if (ChildTypes.size() != 1) + return failedImport("Src pattern child has multiple results"); + + // Check MBB's before the type check since they are not a known type. + if (!SrcChild->isLeaf()) { + if (SrcChild->getOperator()->isSubClassOf("SDNode")) { + auto &ChildSDNI = CGP.getSDNodeInfo(SrcChild->getOperator()); + if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") { + OM.addPredicate<MBBOperandMatcher>(); + return Error::success(); + } + } + } - // The operators look good: match the opcode and mutate it to the new one. - M.Matchers.emplace_back(new MatchOpcode(&SrcGI)); - M.Actions.emplace_back(new MutateOpcode(&DstI)); + auto OpTyOrNone = MVTToLLT(ChildTypes.front().getConcrete()); + if (!OpTyOrNone) + return failedImport("Src operand has an unsupported type"); + OM.addPredicate<LLTOperandMatcher>(*OpTyOrNone); + + // Check for nested instructions. + if (!SrcChild->isLeaf()) { + // Map the node to a gMIR instruction. + InstructionOperandMatcher &InsnOperand = + OM.addPredicate<InstructionOperandMatcher>(); + auto InsnMatcherOrError = + createAndImportSelDAGMatcher(InsnOperand.getInsnMatcher(), SrcChild); + if (auto Error = InsnMatcherOrError.takeError()) + return Error; + + return Error::success(); + } - // Next, analyze the children, only accepting patterns that don't require - // any change to operands. - if (Src->getNumChildren() != Dst->getNumChildren()) - return SkipReason{"Src/dst patterns have a different # of children"}; + // Check for constant immediates. + if (auto *ChildInt = dyn_cast<IntInit>(SrcChild->getLeafValue())) { + OM.addPredicate<IntOperandMatcher>(ChildInt->getValue()); + return Error::success(); + } - unsigned OpIdx = 0; + // Check for def's like register classes or ComplexPattern's. + if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild->getLeafValue())) { + auto *ChildRec = ChildDefInit->getDef(); - // Start with the defined operands (i.e., the results of the root operator). - if (DstI.Operands.NumDefs != Src->getExtTypes().size()) - return SkipReason{"Src pattern results and dst MI defs are different"}; + // Check for register classes. + if (ChildRec->isSubClassOf("RegisterClass")) { + OM.addPredicate<RegisterBankOperandMatcher>( + Target.getRegisterClass(ChildRec)); + return Error::success(); + } - for (const EEVT::TypeSet &Ty : Src->getExtTypes()) { - Record *DstIOpRec = DstI.Operands[OpIdx].Rec; - if (!DstIOpRec->isSubClassOf("RegisterClass")) - return SkipReason{"Dst MI def isn't a register class"}; + // Check for ComplexPattern's. + if (ChildRec->isSubClassOf("ComplexPattern")) { + const auto &ComplexPattern = ComplexPatternEquivs.find(ChildRec); + if (ComplexPattern == ComplexPatternEquivs.end()) + return failedImport("SelectionDAG ComplexPattern (" + + ChildRec->getName() + ") not mapped to GlobalISel"); + + const auto &Predicate = OM.addPredicate<ComplexPatternOperandMatcher>( + OM, *ComplexPattern->second); + TempOpIdx += Predicate.countTemporaryOperands(); + return Error::success(); + } - auto OpTyOrNone = MVTToLLT(Ty.getConcrete()); - if (!OpTyOrNone) - return SkipReason{"Dst operand has an unsupported type"}; + if (ChildRec->isSubClassOf("ImmLeaf")) { + return failedImport( + "Src pattern child def is an unsupported tablegen class (ImmLeaf)"); + } - M.Matchers.emplace_back(new MatchRegOpType(OpIdx, *OpTyOrNone)); - M.Matchers.emplace_back( - new MatchRegOpBank(OpIdx, Target.getRegisterClass(DstIOpRec))); - ++OpIdx; + return failedImport( + "Src pattern child def is an unsupported tablegen class"); } - // Finally match the used operands (i.e., the children of the root operator). - for (unsigned i = 0, e = Src->getNumChildren(); i != e; ++i) { - auto *SrcChild = Src->getChild(i); - auto *DstChild = Dst->getChild(i); - - // Patterns can reorder operands. Ignore those for now. - if (SrcChild->getName() != DstChild->getName()) - return SkipReason{"Src/dst pattern children not in same order"}; - - // The only non-leaf child we accept is 'bb': it's an operator because - // BasicBlockSDNode isn't inline, but in MI it's just another operand. - if (!SrcChild->isLeaf()) { - if (DstChild->isLeaf() || - SrcChild->getOperator() != DstChild->getOperator()) - return SkipReason{"Src/dst pattern child operator mismatch"}; - - if (SrcChild->getOperator()->isSubClassOf("SDNode")) { - auto &ChildSDNI = CGP.getSDNodeInfo(SrcChild->getOperator()); - if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") { - M.Matchers.emplace_back(new MatchMBBOp(OpIdx++)); - continue; - } + return failedImport("Src pattern child is an unsupported kind"); +} + +Error GlobalISelEmitter::importExplicitUseRenderer( + BuildMIAction &DstMIBuilder, TreePatternNode *DstChild, + const InstructionMatcher &InsnMatcher) const { + // The only non-leaf child we accept is 'bb': it's an operator because + // BasicBlockSDNode isn't inline, but in MI it's just another operand. + if (!DstChild->isLeaf()) { + if (DstChild->getOperator()->isSubClassOf("SDNode")) { + auto &ChildSDNI = CGP.getSDNodeInfo(DstChild->getOperator()); + if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") { + DstMIBuilder.addRenderer<CopyRenderer>(InsnMatcher, + DstChild->getName()); + return Error::success(); } - return SkipReason{"Src pattern child isn't a leaf node"}; } + return failedImport("Dst pattern child isn't a leaf node or an MBB"); + } - if (SrcChild->getLeafValue() != DstChild->getLeafValue()) - return SkipReason{"Src/dst pattern child leaf mismatch"}; + // Otherwise, we're looking for a bog-standard RegisterClass operand. + if (DstChild->hasAnyPredicate()) + return failedImport("Dst pattern child has predicate (" + + explainPredicates(DstChild) + ")"); - // Otherwise, we're looking for a bog-standard RegisterClass operand. - if (SrcChild->hasAnyPredicate()) - return SkipReason{"Src pattern child has predicate"}; - auto *ChildRec = cast<DefInit>(SrcChild->getLeafValue())->getDef(); - if (!ChildRec->isSubClassOf("RegisterClass")) - return SkipReason{"Src pattern child isn't a RegisterClass"}; + if (auto *ChildDefInit = dyn_cast<DefInit>(DstChild->getLeafValue())) { + auto *ChildRec = ChildDefInit->getDef(); - ArrayRef<EEVT::TypeSet> ChildTypes = SrcChild->getExtTypes(); + ArrayRef<EEVT::TypeSet> ChildTypes = DstChild->getExtTypes(); if (ChildTypes.size() != 1) - return SkipReason{"Src pattern child has multiple results"}; + return failedImport("Dst pattern child has multiple results"); auto OpTyOrNone = MVTToLLT(ChildTypes.front().getConcrete()); if (!OpTyOrNone) - return SkipReason{"Src operand has an unsupported type"}; + return failedImport("Dst operand has an unsupported type"); + + if (ChildRec->isSubClassOf("Register")) { + DstMIBuilder.addRenderer<AddRegisterRenderer>(ChildRec); + return Error::success(); + } + + if (ChildRec->isSubClassOf("RegisterClass")) { + DstMIBuilder.addRenderer<CopyRenderer>(InsnMatcher, DstChild->getName()); + return Error::success(); + } + + if (ChildRec->isSubClassOf("ComplexPattern")) { + const auto &ComplexPattern = ComplexPatternEquivs.find(ChildRec); + if (ComplexPattern == ComplexPatternEquivs.end()) + return failedImport( + "SelectionDAG ComplexPattern not mapped to GlobalISel"); + + SmallVector<OperandPlaceholder, 2> RenderedOperands; + const OperandMatcher &OM = InsnMatcher.getOperand(DstChild->getName()); + for (unsigned I = 0; I < OM.countTemporaryOperands(); ++I) + RenderedOperands.push_back(OperandPlaceholder::CreateTemporary( + OM.getAllocatedTemporariesBaseID() + I)); + DstMIBuilder.addRenderer<RenderComplexPatternOperand>( + *ComplexPattern->second, RenderedOperands); + return Error::success(); + } - M.Matchers.emplace_back(new MatchRegOpType(OpIdx, *OpTyOrNone)); - M.Matchers.emplace_back( - new MatchRegOpBank(OpIdx, Target.getRegisterClass(ChildRec))); + if (ChildRec->isSubClassOf("SDNodeXForm")) + return failedImport("Dst pattern child def is an unsupported tablegen " + "class (SDNodeXForm)"); + + return failedImport( + "Dst pattern child def is an unsupported tablegen class"); + } + + return failedImport("Dst pattern child is an unsupported kind"); +} + +Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer( + RuleMatcher &M, const TreePatternNode *Dst, + const InstructionMatcher &InsnMatcher) const { + Record *DstOp = Dst->getOperator(); + if (!DstOp->isSubClassOf("Instruction")) { + if (DstOp->isSubClassOf("ValueType")) + return failedImport( + "Pattern operator isn't an instruction (it's a ValueType)"); + return failedImport("Pattern operator isn't an instruction"); + } + auto &DstI = Target.getInstruction(DstOp); + + auto &DstMIBuilder = M.addAction<BuildMIAction>(&DstI, InsnMatcher); + + // Render the explicit defs. + for (unsigned I = 0; I < DstI.Operands.NumDefs; ++I) { + const auto &DstIOperand = DstI.Operands[I]; + DstMIBuilder.addRenderer<CopyRenderer>(InsnMatcher, DstIOperand.Name); + } + + // Figure out which operands need defaults inserted. Operands that subclass + // OperandWithDefaultOps are considered from left to right until we have + // enough operands to render the instruction. + SmallSet<unsigned, 2> DefaultOperands; + unsigned DstINumUses = DstI.Operands.size() - DstI.Operands.NumDefs; + unsigned NumDefaultOperands = 0; + for (unsigned I = 0; I < DstINumUses && + DstINumUses > Dst->getNumChildren() + NumDefaultOperands; + ++I) { + const auto &DstIOperand = DstI.Operands[DstI.Operands.NumDefs + I]; + if (DstIOperand.Rec->isSubClassOf("OperandWithDefaultOps")) { + DefaultOperands.insert(I); + NumDefaultOperands += + DstIOperand.Rec->getValueAsDag("DefaultOps")->getNumArgs(); + } + } + if (DstINumUses > Dst->getNumChildren() + DefaultOperands.size()) + return failedImport("Insufficient operands supplied and default ops " + "couldn't make up the shortfall"); + if (DstINumUses < Dst->getNumChildren() + DefaultOperands.size()) + return failedImport("Too many operands supplied"); + + // Render the explicit uses. + unsigned Child = 0; + for (unsigned I = 0; I != DstINumUses; ++I) { + // If we need to insert default ops here, then do so. + if (DefaultOperands.count(I)) { + const auto &DstIOperand = DstI.Operands[DstI.Operands.NumDefs + I]; + + DagInit *DefaultOps = DstIOperand.Rec->getValueAsDag("DefaultOps"); + for (const auto *DefaultOp : DefaultOps->args()) { + // Look through ValueType operators. + if (const DagInit *DefaultDagOp = dyn_cast<DagInit>(DefaultOp)) { + if (const DefInit *DefaultDagOperator = + dyn_cast<DefInit>(DefaultDagOp->getOperator())) { + if (DefaultDagOperator->getDef()->isSubClassOf("ValueType")) + DefaultOp = DefaultDagOp->getArg(0); + } + } + + if (const DefInit *DefaultDefOp = dyn_cast<DefInit>(DefaultOp)) { + DstMIBuilder.addRenderer<AddRegisterRenderer>(DefaultDefOp->getDef()); + continue; + } + + if (const IntInit *DefaultIntOp = dyn_cast<IntInit>(DefaultOp)) { + DstMIBuilder.addRenderer<ImmRenderer>(DefaultIntOp->getValue()); + continue; + } + + return failedImport("Could not add default op"); + } + + continue; + } + + if (auto Error = importExplicitUseRenderer( + DstMIBuilder, Dst->getChild(Child), InsnMatcher)) + return std::move(Error); + ++Child; + } + + return DstMIBuilder; +} + +Error GlobalISelEmitter::importImplicitDefRenderers( + BuildMIAction &DstMIBuilder, + const std::vector<Record *> &ImplicitDefs) const { + if (!ImplicitDefs.empty()) + return failedImport("Pattern defines a physical register"); + return Error::success(); +} + +Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { + // Keep track of the matchers and actions to emit. + RuleMatcher M; + M.addAction<DebugCommentAction>(P); + + if (auto Error = importRulePredicates(M, P.getPredicates()->getValues())) + return std::move(Error); + + // Next, analyze the pattern operators. + TreePatternNode *Src = P.getSrcPattern(); + TreePatternNode *Dst = P.getDstPattern(); + + // If the root of either pattern isn't a simple operator, ignore it. + if (auto Err = isTrivialOperatorNode(Dst)) + return failedImport("Dst pattern root isn't a trivial operator (" + + toString(std::move(Err)) + ")"); + if (auto Err = isTrivialOperatorNode(Src)) + return failedImport("Src pattern root isn't a trivial operator (" + + toString(std::move(Err)) + ")"); + + // Start with the defined operands (i.e., the results of the root operator). + Record *DstOp = Dst->getOperator(); + if (!DstOp->isSubClassOf("Instruction")) + return failedImport("Pattern operator isn't an instruction"); + + auto &DstI = Target.getInstruction(DstOp); + if (DstI.Operands.NumDefs != Src->getExtTypes().size()) + return failedImport("Src pattern results and dst MI defs are different (" + + to_string(Src->getExtTypes().size()) + " def(s) vs " + + to_string(DstI.Operands.NumDefs) + " def(s))"); + + InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(); + auto InsnMatcherOrError = createAndImportSelDAGMatcher(InsnMatcherTemp, Src); + if (auto Error = InsnMatcherOrError.takeError()) + return std::move(Error); + InstructionMatcher &InsnMatcher = InsnMatcherOrError.get(); + + // The root of the match also has constraints on the register bank so that it + // matches the result instruction. + unsigned OpIdx = 0; + for (const EEVT::TypeSet &Ty : Src->getExtTypes()) { + (void)Ty; + + const auto &DstIOperand = DstI.Operands[OpIdx]; + Record *DstIOpRec = DstIOperand.Rec; + if (!DstIOpRec->isSubClassOf("RegisterClass")) + return failedImport("Dst MI def isn't a register class"); + + OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); + OM.setSymbolicName(DstIOperand.Name); + OM.addPredicate<RegisterBankOperandMatcher>( + Target.getRegisterClass(DstIOpRec)); ++OpIdx; } - // We're done with this pattern! Emit the processed result. - M.emit(OS); - ++NumPatternEmitted; - return None; + auto DstMIBuilderOrError = + createAndImportInstructionRenderer(M, Dst, InsnMatcher); + if (auto Error = DstMIBuilderOrError.takeError()) + return std::move(Error); + BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get(); + + // Render the implicit defs. + // These are only added to the root of the result. + if (auto Error = importImplicitDefRenderers(DstMIBuilder, P.getDstRegs())) + return std::move(Error); + + // We're done with this pattern! It's eligible for GISel emission; return it. + ++NumPatternImported; + return std::move(M); } void GlobalISelEmitter::run(raw_ostream &OS) { @@ -359,26 +1688,71 @@ void GlobalISelEmitter::run(raw_ostream &OS) { emitSourceFileHeader(("Global Instruction Selector for the " + Target.getName() + " target").str(), OS); - OS << "bool " << Target.getName() - << "InstructionSelector::selectImpl" - "(MachineInstr &I) const {\n const MachineRegisterInfo &MRI = " - "I.getParent()->getParent()->getRegInfo();\n"; - + std::vector<RuleMatcher> Rules; // Look through the SelectionDAG patterns we found, possibly emitting some. for (const PatternToMatch &Pat : CGP.ptms()) { ++NumPatternTotal; - if (auto SkipReason = runOnPattern(Pat, OS)) { + auto MatcherOrErr = runOnPattern(Pat); + + // The pattern analysis can fail, indicating an unsupported pattern. + // Report that if we've been asked to do so. + if (auto Err = MatcherOrErr.takeError()) { if (WarnOnSkippedPatterns) { PrintWarning(Pat.getSrcRecord()->getLoc(), - "Skipped pattern: " + SkipReason->Reason); + "Skipped pattern: " + toString(std::move(Err))); + } else { + consumeError(std::move(Err)); } - ++NumPatternSkipped; + ++NumPatternImportsSkipped; + continue; } + + Rules.push_back(std::move(MatcherOrErr.get())); + } + + std::stable_sort(Rules.begin(), Rules.end(), + [&](const RuleMatcher &A, const RuleMatcher &B) { + if (A.isHigherPriorityThan(B)) { + assert(!B.isHigherPriorityThan(A) && "Cannot be more important " + "and less important at " + "the same time"); + return true; + } + return false; + }); + + unsigned MaxTemporaries = 0; + for (const auto &Rule : Rules) + MaxTemporaries = std::max(MaxTemporaries, Rule.countTemporaryOperands()); + + OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n"; + for (unsigned I = 0; I < MaxTemporaries; ++I) + OS << " mutable MachineOperand TempOp" << I << ";\n"; + OS << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n\n"; + + OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n"; + for (unsigned I = 0; I < MaxTemporaries; ++I) + OS << ", TempOp" << I << "(MachineOperand::CreatePlaceholder())\n"; + OS << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n\n"; + + OS << "#ifdef GET_GLOBALISEL_IMPL\n" + << "bool " << Target.getName() + << "InstructionSelector::selectImpl(MachineInstr &I) const {\n" + << " MachineFunction &MF = *I.getParent()->getParent();\n" + << " const MachineRegisterInfo &MRI = MF.getRegInfo();\n"; + + for (auto &Rule : Rules) { + Rule.emit(OS); + ++NumPatternEmitted; } - OS << " return false;\n}\n"; + OS << " return false;\n" + << "}\n" + << "#endif // ifdef GET_GLOBALISEL_IMPL\n"; } +} // end anonymous namespace + //===----------------------------------------------------------------------===// namespace llvm { diff --git a/utils/TableGen/IntrinsicEmitter.cpp b/utils/TableGen/IntrinsicEmitter.cpp index 33256ccba46c2..e9dd2fa0aca00 100644 --- a/utils/TableGen/IntrinsicEmitter.cpp +++ b/utils/TableGen/IntrinsicEmitter.cpp @@ -133,14 +133,14 @@ void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints, OS << "// Target mapping\n"; OS << "#ifdef GET_INTRINSIC_TARGET_DATA\n"; OS << "struct IntrinsicTargetInfo {\n" - << " StringRef Name;\n" + << " llvm::StringLiteral Name;\n" << " size_t Offset;\n" << " size_t Count;\n" << "};\n"; - OS << "static const IntrinsicTargetInfo TargetInfos[] = {\n"; + OS << "static constexpr IntrinsicTargetInfo TargetInfos[] = {\n"; for (auto Target : Ints.Targets) - OS << " {\"" << Target.Name << "\", " << Target.Offset << ", " - << Target.Count << "},\n"; + OS << " {llvm::StringLiteral(\"" << Target.Name << "\"), " << Target.Offset + << ", " << Target.Count << "},\n"; OS << "};\n"; OS << "#endif\n\n"; } @@ -497,10 +497,10 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, OS << "// Add parameter attributes that are not common to all intrinsics.\n"; OS << "#ifdef GET_INTRINSIC_ATTRIBUTES\n"; if (TargetOnly) - OS << "static AttributeSet getAttributes(LLVMContext &C, " << TargetPrefix + OS << "static AttributeList getAttributes(LLVMContext &C, " << TargetPrefix << "Intrinsic::ID id) {\n"; else - OS << "AttributeSet Intrinsic::getAttributes(LLVMContext &C, ID id) {\n"; + OS << "AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {\n"; // Compute the maximum number of attribute arguments and the map typedef std::map<const CodeGenIntrinsic*, unsigned, @@ -518,7 +518,7 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, N = ++AttrNum; } - // Emit an array of AttributeSet. Most intrinsics will have at least one + // Emit an array of AttributeList. Most intrinsics will have at least one // entry, for the function itself (index ~1), which is usually nounwind. OS << " static const uint8_t IntrinsicsToAttributesMap[] = {\n"; @@ -530,7 +530,7 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, } OS << " };\n\n"; - OS << " AttributeSet AS[" << maxArgAttrs+1 << "];\n"; + OS << " AttributeList AS[" << maxArgAttrs + 1 << "];\n"; OS << " unsigned NumAttrs = 0;\n"; OS << " if (id != 0) {\n"; OS << " switch(IntrinsicsToAttributesMap[id - "; @@ -595,8 +595,8 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, ++ai; } while (ai != ae && intrinsic.ArgumentAttributes[ai].first == argNo); OS << "};\n"; - OS << " AS[" << numAttrs++ << "] = AttributeSet::get(C, " - << argNo+1 << ", AttrParam" << argNo +1 << ");\n"; + OS << " AS[" << numAttrs++ << "] = AttributeList::get(C, " + << argNo + 1 << ", AttrParam" << argNo + 1 << ");\n"; } } @@ -699,8 +699,8 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, break; } OS << "};\n"; - OS << " AS[" << numAttrs++ << "] = AttributeSet::get(C, " - << "AttributeSet::FunctionIndex, Atts);\n"; + OS << " AS[" << numAttrs++ << "] = AttributeList::get(C, " + << "AttributeList::FunctionIndex, Atts);\n"; } if (numAttrs) { @@ -708,14 +708,14 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, OS << " break;\n"; OS << " }\n"; } else { - OS << " return AttributeSet();\n"; + OS << " return AttributeList();\n"; OS << " }\n"; } } OS << " }\n"; OS << " }\n"; - OS << " return AttributeSet::get(C, makeArrayRef(AS, NumAttrs));\n"; + OS << " return AttributeList::get(C, makeArrayRef(AS, NumAttrs));\n"; OS << "}\n"; OS << "#endif // GET_INTRINSIC_ATTRIBUTES\n\n"; } diff --git a/utils/TableGen/RegisterBankEmitter.cpp b/utils/TableGen/RegisterBankEmitter.cpp new file mode 100644 index 0000000000000..bf066412b2860 --- /dev/null +++ b/utils/TableGen/RegisterBankEmitter.cpp @@ -0,0 +1,320 @@ +//===- RegisterBankEmitter.cpp - Generate a Register Bank Desc. -*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend is responsible for emitting a description of a target +// register bank for a code generator. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/BitVector.h" +#include "llvm/Support/Debug.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" + +#include "CodeGenRegisters.h" + +#define DEBUG_TYPE "register-bank-emitter" + +using namespace llvm; + +namespace { +class RegisterBank { + + /// A vector of register classes that are included in the register bank. + typedef std::vector<const CodeGenRegisterClass *> RegisterClassesTy; + +private: + const Record &TheDef; + + /// The register classes that are covered by the register bank. + RegisterClassesTy RCs; + + /// The register class with the largest register size. + const CodeGenRegisterClass *RCWithLargestRegsSize; + +public: + RegisterBank(const Record &TheDef) + : TheDef(TheDef), RCs(), RCWithLargestRegsSize(nullptr) {} + + /// Get the human-readable name for the bank. + std::string getName() const { return TheDef.getValueAsString("Name"); } + /// Get the name of the enumerator in the ID enumeration. + std::string getEnumeratorName() const { return (TheDef.getName() + "ID").str(); } + + /// Get the name of the array holding the register class coverage data; + std::string getCoverageArrayName() const { + return (TheDef.getName() + "CoverageData").str(); + } + + /// Get the name of the global instance variable. + StringRef getInstanceVarName() const { return TheDef.getName(); } + + const Record &getDef() const { return TheDef; } + + /// Get the register classes listed in the RegisterBank.RegisterClasses field. + std::vector<const CodeGenRegisterClass *> + getExplictlySpecifiedRegisterClasses( + CodeGenRegBank &RegisterClassHierarchy) const { + std::vector<const CodeGenRegisterClass *> RCs; + for (const auto &RCDef : getDef().getValueAsListOfDefs("RegisterClasses")) + RCs.push_back(RegisterClassHierarchy.getRegClass(RCDef)); + return RCs; + } + + /// Add a register class to the bank without duplicates. + void addRegisterClass(const CodeGenRegisterClass *RC) { + if (std::find_if(RCs.begin(), RCs.end(), + [&RC](const CodeGenRegisterClass *X) { + return X == RC; + }) != RCs.end()) + return; + + // FIXME? We really want the register size rather than the spill size + // since the spill size may be bigger on some targets with + // limited load/store instructions. However, we don't store the + // register size anywhere (we could sum the sizes of the subregisters + // but there may be additional bits too) and we can't derive it from + // the VT's reliably due to Untyped. + if (RCWithLargestRegsSize == nullptr) + RCWithLargestRegsSize = RC; + else if (RCWithLargestRegsSize->SpillSize < RC->SpillSize) + RCWithLargestRegsSize = RC; + assert(RCWithLargestRegsSize && "RC was nullptr?"); + + RCs.emplace_back(RC); + } + + const CodeGenRegisterClass *getRCWithLargestRegsSize() const { + return RCWithLargestRegsSize; + } + + iterator_range<typename RegisterClassesTy::const_iterator> + register_classes() const { + return llvm::make_range(RCs.begin(), RCs.end()); + } +}; + +class RegisterBankEmitter { +private: + RecordKeeper &Records; + CodeGenRegBank RegisterClassHierarchy; + + void emitHeader(raw_ostream &OS, const StringRef TargetName, + const std::vector<RegisterBank> &Banks); + void emitBaseClassDefinition(raw_ostream &OS, const StringRef TargetName, + const std::vector<RegisterBank> &Banks); + void emitBaseClassImplementation(raw_ostream &OS, const StringRef TargetName, + std::vector<RegisterBank> &Banks); + +public: + RegisterBankEmitter(RecordKeeper &R) + : Records(R), RegisterClassHierarchy(Records) {} + + void run(raw_ostream &OS); +}; + +} // end anonymous namespace + +/// Emit code to declare the ID enumeration and external global instance +/// variables. +void RegisterBankEmitter::emitHeader(raw_ostream &OS, + const StringRef TargetName, + const std::vector<RegisterBank> &Banks) { + // <Target>RegisterBankInfo.h + OS << "namespace llvm {\n" + << "namespace " << TargetName << " {\n" + << "enum {\n"; + for (const auto &Bank : Banks) + OS << " " << Bank.getEnumeratorName() << ",\n"; + OS << " NumRegisterBanks,\n" + << "};\n" + << "} // end namespace " << TargetName << "\n" + << "} // end namespace llvm\n"; +} + +/// Emit declarations of the <Target>GenRegisterBankInfo class. +void RegisterBankEmitter::emitBaseClassDefinition( + raw_ostream &OS, const StringRef TargetName, + const std::vector<RegisterBank> &Banks) { + OS << "private:\n" + << " static RegisterBank *RegBanks[];\n\n" + << "protected:\n" + << " " << TargetName << "GenRegisterBankInfo();\n" + << "\n"; +} + +/// Visit each register class belonging to the given register bank. +/// +/// A class belongs to the bank iff any of these apply: +/// * It is explicitly specified +/// * It is a subclass of a class that is a member. +/// * It is a class containing subregisters of the registers of a class that +/// is a member. This is known as a subreg-class. +/// +/// This function must be called for each explicitly specified register class. +/// +/// \param RC The register class to search. +/// \param Kind A debug string containing the path the visitor took to reach RC. +/// \param VisitFn The action to take for each class visited. It may be called +/// multiple times for a given class if there are multiple paths +/// to the class. +static void visitRegisterBankClasses( + CodeGenRegBank &RegisterClassHierarchy, const CodeGenRegisterClass *RC, + const Twine Kind, + std::function<void(const CodeGenRegisterClass *, StringRef)> VisitFn, + SmallPtrSetImpl<const CodeGenRegisterClass *> &VisitedRCs) { + + // Make sure we only visit each class once to avoid infinite loops. + if (VisitedRCs.count(RC)) + return; + VisitedRCs.insert(RC); + + // Visit each explicitly named class. + VisitFn(RC, Kind.str()); + + for (const auto &PossibleSubclass : RegisterClassHierarchy.getRegClasses()) { + std::string TmpKind = + (Twine(Kind) + " (" + PossibleSubclass.getName() + ")").str(); + + // Visit each subclass of an explicitly named class. + if (RC != &PossibleSubclass && RC->hasSubClass(&PossibleSubclass)) + visitRegisterBankClasses(RegisterClassHierarchy, &PossibleSubclass, + TmpKind + " " + RC->getName() + " subclass", + VisitFn, VisitedRCs); + + // Visit each class that contains only subregisters of RC with a common + // subregister-index. + // + // More precisely, PossibleSubclass is a subreg-class iff Reg:SubIdx is in + // PossibleSubclass for all registers Reg from RC using any + // subregister-index SubReg + for (const auto &SubIdx : RegisterClassHierarchy.getSubRegIndices()) { + BitVector BV(RegisterClassHierarchy.getRegClasses().size()); + PossibleSubclass.getSuperRegClasses(&SubIdx, BV); + if (BV.test(RC->EnumValue)) { + std::string TmpKind2 = (Twine(TmpKind) + " " + RC->getName() + + " class-with-subregs: " + RC->getName()) + .str(); + VisitFn(&PossibleSubclass, TmpKind2); + } + } + } +} + +void RegisterBankEmitter::emitBaseClassImplementation( + raw_ostream &OS, StringRef TargetName, + std::vector<RegisterBank> &Banks) { + + OS << "namespace llvm {\n" + << "namespace " << TargetName << " {\n"; + for (const auto &Bank : Banks) { + std::vector<std::vector<const CodeGenRegisterClass *>> RCsGroupedByWord( + (RegisterClassHierarchy.getRegClasses().size() + 31) / 32); + + for (const auto &RC : Bank.register_classes()) + RCsGroupedByWord[RC->EnumValue / 32].push_back(RC); + + OS << "const uint32_t " << Bank.getCoverageArrayName() << "[] = {\n"; + unsigned LowestIdxInWord = 0; + for (const auto &RCs : RCsGroupedByWord) { + OS << " // " << LowestIdxInWord << "-" << (LowestIdxInWord + 31) << "\n"; + for (const auto &RC : RCs) { + std::string QualifiedRegClassID = + (Twine(TargetName) + "::" + RC->getName() + "RegClassID").str(); + OS << " (1u << (" << QualifiedRegClassID << " - " + << LowestIdxInWord << ")) |\n"; + } + OS << " 0,\n"; + LowestIdxInWord += 32; + } + OS << "};\n"; + } + OS << "\n"; + + for (const auto &Bank : Banks) { + std::string QualifiedBankID = + (TargetName + "::" + Bank.getEnumeratorName()).str(); + unsigned Size = Bank.getRCWithLargestRegsSize()->SpillSize; + OS << "RegisterBank " << Bank.getInstanceVarName() << "(/* ID */ " + << QualifiedBankID << ", /* Name */ \"" << Bank.getName() + << "\", /* Size */ " << Size << ", " + << "/* CoveredRegClasses */ " << Bank.getCoverageArrayName() + << ", /* NumRegClasses */ " + << RegisterClassHierarchy.getRegClasses().size() << ");\n"; + } + OS << "} // end namespace " << TargetName << "\n" + << "\n"; + + OS << "RegisterBank *" << TargetName + << "GenRegisterBankInfo::RegBanks[] = {\n"; + for (const auto &Bank : Banks) + OS << " &" << TargetName << "::" << Bank.getInstanceVarName() << ",\n"; + OS << "};\n\n"; + + OS << TargetName << "GenRegisterBankInfo::" << TargetName + << "GenRegisterBankInfo()\n" + << " : RegisterBankInfo(RegBanks, " << TargetName + << "::NumRegisterBanks) {\n" + << " // Assert that RegBank indices match their ID's\n" + << "#ifndef NDEBUG\n" + << " unsigned Index = 0;\n" + << " for (const auto &RB : RegBanks)\n" + << " assert(Index++ == RB->getID() && \"Index != ID\");\n" + << "#endif // NDEBUG\n" + << "}\n" + << "} // end namespace llvm\n"; +} + +void RegisterBankEmitter::run(raw_ostream &OS) { + std::vector<Record*> Targets = Records.getAllDerivedDefinitions("Target"); + if (Targets.size() != 1) + PrintFatalError("ERROR: Too many or too few subclasses of Target defined!"); + StringRef TargetName = Targets[0]->getName(); + + std::vector<RegisterBank> Banks; + for (const auto &V : Records.getAllDerivedDefinitions("RegisterBank")) { + SmallPtrSet<const CodeGenRegisterClass *, 8> VisitedRCs; + RegisterBank Bank(*V); + + for (const CodeGenRegisterClass *RC : + Bank.getExplictlySpecifiedRegisterClasses(RegisterClassHierarchy)) { + visitRegisterBankClasses( + RegisterClassHierarchy, RC, "explicit", + [&Bank](const CodeGenRegisterClass *RC, StringRef Kind) { + DEBUG(dbgs() << "Added " << RC->getName() << "(" << Kind << ")\n"); + Bank.addRegisterClass(RC); + }, VisitedRCs); + } + + Banks.push_back(Bank); + } + + emitSourceFileHeader("Register Bank Source Fragments", OS); + OS << "#ifdef GET_REGBANK_DECLARATIONS\n" + << "#undef GET_REGBANK_DECLARATIONS\n"; + emitHeader(OS, TargetName, Banks); + OS << "#endif // GET_REGBANK_DECLARATIONS\n\n" + << "#ifdef GET_TARGET_REGBANK_CLASS\n" + << "#undef GET_TARGET_REGBANK_CLASS\n"; + emitBaseClassDefinition(OS, TargetName, Banks); + OS << "#endif // GET_TARGET_REGBANK_CLASS\n\n" + << "#ifdef GET_TARGET_REGBANK_IMPL\n" + << "#undef GET_TARGET_REGBANK_IMPL\n"; + emitBaseClassImplementation(OS, TargetName, Banks); + OS << "#endif // GET_TARGET_REGBANK_IMPL\n"; +} + +namespace llvm { + +void EmitRegisterBank(RecordKeeper &RK, raw_ostream &OS) { + RegisterBankEmitter(RK).run(OS); +} + +} // end namespace llvm diff --git a/utils/TableGen/RegisterInfoEmitter.cpp b/utils/TableGen/RegisterInfoEmitter.cpp index b75be13c0480e..5b56578a64b3b 100644 --- a/utils/TableGen/RegisterInfoEmitter.cpp +++ b/utils/TableGen/RegisterInfoEmitter.cpp @@ -1023,18 +1023,14 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, << "MCRegisterClasses[] = {\n"; for (const auto &RC : RegisterClasses) { - // Asserts to make sure values will fit in table assuming types from - // MCRegisterInfo.h - assert((RC.SpillSize/8) <= 0xffff && "SpillSize too large."); - assert((RC.SpillAlignment/8) <= 0xffff && "SpillAlignment too large."); - assert(RC.CopyCost >= -128 && RC.CopyCost <= 127 && "Copy cost too large."); - + assert(isInt<8>(RC.CopyCost) && "Copy cost too large."); + // Register size and spill size will become independent, but are not at + // the moment. For now use SpillSize as the register size. OS << " { " << RC.getName() << ", " << RC.getName() << "Bits, " << RegClassStrings.get(RC.getName()) << ", " << RC.getOrder().size() << ", sizeof(" << RC.getName() << "Bits), " << RC.getQualifiedName() + "RegClassID" << ", " << RC.SpillSize/8 << ", " - << RC.SpillAlignment/8 << ", " << RC.CopyCost << ", " << ( RC.Allocatable ? "true" : "false" ) << " },\n"; } @@ -1316,9 +1312,13 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, << " { // Register class instances\n"; for (const auto &RC : RegisterClasses) { + assert(isUInt<16>(RC.SpillSize/8) && "SpillSize too large."); + assert(isUInt<16>(RC.SpillAlignment/8) && "SpillAlignment too large."); OS << " extern const TargetRegisterClass " << RC.getName() << "RegClass = {\n " << '&' << Target.getName() << "MCRegisterClasses[" << RC.getName() << "RegClassID],\n " + << RC.SpillSize/8 << ", /* SpillSize */\n " + << RC.SpillAlignment/8 << ", /* SpillAlignment */\n " << "VTLists + " << VTSeqs.get(RC.VTs) << ",\n " << RC.getName() << "SubClassMask,\n SuperRegIdxSeqs + " << SuperRegIdxSeqs.get(SuperRegIdxLists[RC.EnumValue]) << ",\n "; diff --git a/utils/TableGen/SubtargetEmitter.cpp b/utils/TableGen/SubtargetEmitter.cpp index bf7b392b15e58..30516ef5d10de 100644 --- a/utils/TableGen/SubtargetEmitter.cpp +++ b/utils/TableGen/SubtargetEmitter.cpp @@ -917,6 +917,8 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel, SCDesc.NumMicroOps += WriteRes->getValueAsInt("NumMicroOps"); SCDesc.BeginGroup |= WriteRes->getValueAsBit("BeginGroup"); SCDesc.EndGroup |= WriteRes->getValueAsBit("EndGroup"); + SCDesc.BeginGroup |= WriteRes->getValueAsBit("SingleIssue"); + SCDesc.EndGroup |= WriteRes->getValueAsBit("SingleIssue"); // Create an entry for each ProcResource listed in WriteRes. RecVec PRVec = WriteRes->getValueAsListOfDefs("ProcResources"); diff --git a/utils/TableGen/SubtargetFeatureInfo.cpp b/utils/TableGen/SubtargetFeatureInfo.cpp index 7db8813050fe6..72a556182b1dc 100644 --- a/utils/TableGen/SubtargetFeatureInfo.cpp +++ b/utils/TableGen/SubtargetFeatureInfo.cpp @@ -16,10 +16,11 @@ using namespace llvm; -void SubtargetFeatureInfo::dump() const { - errs() << getEnumName() << " " << Index << "\n"; - TheDef->dump(); +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +LLVM_DUMP_METHOD void SubtargetFeatureInfo::dump() const { + errs() << getEnumName() << " " << Index << "\n" << *TheDef; } +#endif std::vector<std::pair<Record *, SubtargetFeatureInfo>> SubtargetFeatureInfo::getAll(const RecordKeeper &Records) { @@ -61,11 +62,24 @@ void SubtargetFeatureInfo::emitSubtargetFeatureFlagEnumeration( void SubtargetFeatureInfo::emitNameTable( std::map<Record *, SubtargetFeatureInfo, LessRecordByID> &SubtargetFeatures, raw_ostream &OS) { + // Need to sort the name table so that lookup by the log of the enum value + // gives the proper name. More specifically, for a feature of value 1<<n, + // SubtargetFeatureNames[n] should be the name of the feature. + uint64_t IndexUB = 0; + for (const auto &SF : SubtargetFeatures) + if (IndexUB <= SF.second.Index) + IndexUB = SF.second.Index+1; + + std::vector<std::string> Names; + if (IndexUB > 0) + Names.resize(IndexUB); + for (const auto &SF : SubtargetFeatures) + Names[SF.second.Index] = SF.second.getEnumName(); + OS << "static const char *SubtargetFeatureNames[] = {\n"; - for (const auto &SF : SubtargetFeatures) { - const SubtargetFeatureInfo &SFI = SF.second; - OS << " \"" << SFI.getEnumName() << "\",\n"; - } + for (uint64_t I = 0; I < IndexUB; ++I) + OS << " \"" << Names[I] << "\",\n"; + // A small number of targets have no predicates. Null terminate the array to // avoid a zero-length array. OS << " nullptr\n" diff --git a/utils/TableGen/TableGen.cpp b/utils/TableGen/TableGen.cpp index 79a773161e4b4..00d20f1df6c27 100644 --- a/utils/TableGen/TableGen.cpp +++ b/utils/TableGen/TableGen.cpp @@ -46,6 +46,8 @@ enum ActionType { GenAttributes, GenSearchableTables, GenGlobalISel, + GenX86EVEX2VEXTables, + GenRegisterBank, }; namespace { @@ -94,11 +96,16 @@ namespace { clEnumValN(GenSearchableTables, "gen-searchable-tables", "Generate generic binary-searchable table"), clEnumValN(GenGlobalISel, "gen-global-isel", - "Generate GlobalISel selector"))); + "Generate GlobalISel selector"), + clEnumValN(GenX86EVEX2VEXTables, "gen-x86-EVEX2VEX-tables", + "Generate X86 EVEX to VEX compress tables"), + clEnumValN(GenRegisterBank, "gen-register-bank", + "Generate registers bank descriptions"))); + cl::OptionCategory PrintEnumsCat("Options for -print-enums"); cl::opt<std::string> Class("class", cl::desc("Print Enum list for this class"), - cl::value_desc("class name")); + cl::value_desc("class name"), cl::cat(PrintEnumsCat)); bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) { switch (Action) { @@ -183,6 +190,12 @@ bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) { case GenGlobalISel: EmitGlobalISel(Records, OS); break; + case GenRegisterBank: + EmitRegisterBank(Records, OS); + break; + case GenX86EVEX2VEXTables: + EmitX86EVEX2VEXTables(Records, OS); + break; } return false; diff --git a/utils/TableGen/TableGenBackends.h b/utils/TableGen/TableGenBackends.h index eb306d28180c5..2512997e27f93 100644 --- a/utils/TableGen/TableGenBackends.h +++ b/utils/TableGen/TableGenBackends.h @@ -81,6 +81,8 @@ void EmitCTags(RecordKeeper &RK, raw_ostream &OS); void EmitAttributes(RecordKeeper &RK, raw_ostream &OS); void EmitSearchableTables(RecordKeeper &RK, raw_ostream &OS); void EmitGlobalISel(RecordKeeper &RK, raw_ostream &OS); +void EmitX86EVEX2VEXTables(RecordKeeper &RK, raw_ostream &OS); +void EmitRegisterBank(RecordKeeper &RK, raw_ostream &OS); } // End llvm namespace diff --git a/utils/TableGen/X86DisassemblerTables.cpp b/utils/TableGen/X86DisassemblerTables.cpp index 5b710e4461507..c9e36f96736ad 100644 --- a/utils/TableGen/X86DisassemblerTables.cpp +++ b/utils/TableGen/X86DisassemblerTables.cpp @@ -879,6 +879,10 @@ void DisassemblerTables::setTableFields(ModRMDecision &decision, newInfo.name == "XCHG64ar")) continue; // special case for XCHG*ar and NOOP + if (previousInfo.name == "DATA16_PREFIX" && + newInfo.name == "DATA32_PREFIX") + continue; // special case for data16 and data32 + if (outranks(previousInfo.insnContext, newInfo.insnContext)) continue; diff --git a/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp b/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp new file mode 100644 index 0000000000000..07b96b03b01cc --- /dev/null +++ b/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp @@ -0,0 +1,339 @@ +//===- utils/TableGen/X86EVEX2VEXTablesEmitter.cpp - X86 backend-*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// This tablegen backend is responsible for emitting the X86 backend EVEX2VEX +/// compression tables. +/// +//===----------------------------------------------------------------------===// + +#include "CodeGenDAGPatterns.h" +#include "CodeGenTarget.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/TableGenBackend.h" + +using namespace llvm; + +namespace { + +class X86EVEX2VEXTablesEmitter { + CodeGenTarget Target; + + // Hold all non-masked & non-broadcasted EVEX encoded instructions + std::vector<const CodeGenInstruction *> EVEXInsts; + // Hold all VEX encoded instructions. Divided into groups with same opcodes + // to make the search more efficient + std::map<uint64_t, std::vector<const CodeGenInstruction *>> VEXInsts; + + typedef std::pair<const CodeGenInstruction *, const CodeGenInstruction *> Entry; + + // Represent both compress tables + std::vector<Entry> EVEX2VEX128; + std::vector<Entry> EVEX2VEX256; + + // Represents a manually added entry to the tables + struct ManualEntry { + const char *EVEXInstStr; + const char *VEXInstStr; + bool Is128Bit; + }; + +public: + X86EVEX2VEXTablesEmitter(RecordKeeper &R) : Target(R) {} + + // run - Output X86 EVEX2VEX tables. + void run(raw_ostream &OS); + +private: + // Prints the given table as a C++ array of type + // X86EvexToVexCompressTableEntry + void printTable(const std::vector<Entry> &Table, raw_ostream &OS); + + bool inExceptionList(const CodeGenInstruction *Inst) { + // List of EVEX instructions that match VEX instructions by the encoding + // but do not perform the same operation. + static constexpr const char *ExceptionList[] = { + "VCVTQQ2PD", + "VCVTQQ2PS", + "VPMAXSQ", + "VPMAXUQ", + "VPMINSQ", + "VPMINUQ", + "VPMULLQ", + "VPSRAQ", + "VDBPSADBW", + "VRNDSCALE", + "VSCALEFPS" + }; + // Instruction's name starts with one of the entries in the exception list + for (StringRef InstStr : ExceptionList) { + if (Inst->TheDef->getName().startswith(InstStr)) + return true; + } + return false; + } + +}; + +void X86EVEX2VEXTablesEmitter::printTable(const std::vector<Entry> &Table, + raw_ostream &OS) { + std::string Size = (Table == EVEX2VEX128) ? "128" : "256"; + + OS << "// X86 EVEX encoded instructions that have a VEX " << Size + << " encoding\n" + << "// (table format: <EVEX opcode, VEX-" << Size << " opcode>).\n" + << "static const X86EvexToVexCompressTableEntry X86EvexToVex" << Size + << "CompressTable[] = {\n" + << " // EVEX scalar with corresponding VEX.\n"; + + // Print all entries added to the table + for (auto Pair : Table) { + OS << " { X86::" << Pair.first->TheDef->getName() + << ", X86::" << Pair.second->TheDef->getName() << " },\n"; + } + + // Some VEX instructions were duplicated to multiple EVEX versions due the + // introduction of mask variants, and thus some of the EVEX versions have + // different encoding than the VEX instruction. In order to maximize the + // compression we add these entries manually. + static constexpr ManualEntry ManuallyAddedEntries[] = { + // EVEX-Inst VEX-Inst Is128-bit + {"VMOVDQU8Z128mr", "VMOVDQUmr", true}, + {"VMOVDQU8Z128rm", "VMOVDQUrm", true}, + {"VMOVDQU8Z128rr", "VMOVDQUrr", true}, + {"VMOVDQU8Z128rr_REV", "VMOVDQUrr_REV", true}, + {"VMOVDQU16Z128mr", "VMOVDQUmr", true}, + {"VMOVDQU16Z128rm", "VMOVDQUrm", true}, + {"VMOVDQU16Z128rr", "VMOVDQUrr", true}, + {"VMOVDQU16Z128rr_REV", "VMOVDQUrr_REV", true}, + {"VMOVDQU8Z256mr", "VMOVDQUYmr", false}, + {"VMOVDQU8Z256rm", "VMOVDQUYrm", false}, + {"VMOVDQU8Z256rr", "VMOVDQUYrr", false}, + {"VMOVDQU8Z256rr_REV", "VMOVDQUYrr_REV", false}, + {"VMOVDQU16Z256mr", "VMOVDQUYmr", false}, + {"VMOVDQU16Z256rm", "VMOVDQUYrm", false}, + {"VMOVDQU16Z256rr", "VMOVDQUYrr", false}, + {"VMOVDQU16Z256rr_REV", "VMOVDQUYrr_REV", false}, + + {"VPERMILPDZ128mi", "VPERMILPDmi", true}, + {"VPERMILPDZ128ri", "VPERMILPDri", true}, + {"VPERMILPDZ128rm", "VPERMILPDrm", true}, + {"VPERMILPDZ128rr", "VPERMILPDrr", true}, + {"VPERMILPDZ256mi", "VPERMILPDYmi", false}, + {"VPERMILPDZ256ri", "VPERMILPDYri", false}, + {"VPERMILPDZ256rm", "VPERMILPDYrm", false}, + {"VPERMILPDZ256rr", "VPERMILPDYrr", false}, + + {"VPBROADCASTQZ128m", "VPBROADCASTQrm", true}, + {"VPBROADCASTQZ128r", "VPBROADCASTQrr", true}, + {"VPBROADCASTQZ256m", "VPBROADCASTQYrm", false}, + {"VPBROADCASTQZ256r", "VPBROADCASTQYrr", false}, + + {"VBROADCASTSDZ256m", "VBROADCASTSDYrm", false}, + {"VBROADCASTSDZ256r", "VBROADCASTSDYrr", false}, + + {"VEXTRACTF64x2Z256mr", "VEXTRACTF128mr", false}, + {"VEXTRACTF64x2Z256rr", "VEXTRACTF128rr", false}, + {"VEXTRACTI64x2Z256mr", "VEXTRACTI128mr", false}, + {"VEXTRACTI64x2Z256rr", "VEXTRACTI128rr", false}, + + {"VINSERTF64x2Z256rm", "VINSERTF128rm", false}, + {"VINSERTF64x2Z256rr", "VINSERTF128rr", false}, + {"VINSERTI64x2Z256rm", "VINSERTI128rm", false}, + {"VINSERTI64x2Z256rr", "VINSERTI128rr", false} + }; + + // Print the manually added entries + for (const ManualEntry &Entry : ManuallyAddedEntries) { + if ((Table == EVEX2VEX128 && Entry.Is128Bit) || + (Table == EVEX2VEX256 && !Entry.Is128Bit)) { + OS << " { X86::" << Entry.EVEXInstStr << ", X86::" << Entry.VEXInstStr + << " },\n"; + } + } + + OS << "};\n\n"; +} + +// Return true if the 2 BitsInits are equal +static inline bool equalBitsInits(const BitsInit *B1, const BitsInit *B2) { + if (B1->getNumBits() != B2->getNumBits()) + PrintFatalError("Comparing two BitsInits with different sizes!"); + + for (unsigned i = 0, e = B1->getNumBits(); i != e; ++i) { + if (BitInit *Bit1 = dyn_cast<BitInit>(B1->getBit(i))) { + if (BitInit *Bit2 = dyn_cast<BitInit>(B2->getBit(i))) { + if (Bit1->getValue() != Bit2->getValue()) + return false; + } else + PrintFatalError("Invalid BitsInit bit"); + } else + PrintFatalError("Invalid BitsInit bit"); + } + return true; +} + +// Calculates the integer value residing BitsInit object +static inline uint64_t getValueFromBitsInit(const BitsInit *B) { + uint64_t Value = 0; + for (unsigned i = 0, e = B->getNumBits(); i != e; ++i) { + if (BitInit *Bit = dyn_cast<BitInit>(B->getBit(i))) + Value |= uint64_t(Bit->getValue()) << i; + else + PrintFatalError("Invalid VectSize bit"); + } + return Value; +} + +// Function object - Operator() returns true if the given VEX instruction +// matches the EVEX instruction of this object. +class IsMatch { + const CodeGenInstruction *Inst; + +public: + IsMatch(const CodeGenInstruction *Inst) : Inst(Inst) {} + + bool operator()(const CodeGenInstruction *Inst2) { + Record *Rec1 = Inst->TheDef; + Record *Rec2 = Inst2->TheDef; + uint64_t Rec1WVEX = + getValueFromBitsInit(Rec1->getValueAsBitsInit("VEX_WPrefix")); + uint64_t Rec2WVEX = + getValueFromBitsInit(Rec2->getValueAsBitsInit("VEX_WPrefix")); + + if (Rec2->getValueAsDef("OpEnc")->getName().str() != "EncVEX" || + // VEX/EVEX fields + Rec2->getValueAsDef("OpPrefix") != Rec1->getValueAsDef("OpPrefix") || + Rec2->getValueAsDef("OpMap") != Rec1->getValueAsDef("OpMap") || + Rec2->getValueAsBit("hasVEX_4V") != Rec1->getValueAsBit("hasVEX_4V") || + !equalBitsInits(Rec2->getValueAsBitsInit("EVEX_LL"), + Rec1->getValueAsBitsInit("EVEX_LL")) || + (Rec1WVEX != 2 && Rec2WVEX != 2 && Rec1WVEX != Rec2WVEX) || + // Instruction's format + Rec2->getValueAsDef("Form") != Rec1->getValueAsDef("Form") || + Rec2->getValueAsBit("isAsmParserOnly") != + Rec1->getValueAsBit("isAsmParserOnly")) + return false; + + // This is needed for instructions with intrinsic version (_Int). + // Where the only difference is the size of the operands. + // For example: VUCOMISDZrm and Int_VUCOMISDrm + // Also for instructions that their EVEX version was upgraded to work with + // k-registers. For example VPCMPEQBrm (xmm output register) and + // VPCMPEQBZ128rm (k register output register). + for (unsigned i = 0; i < Inst->Operands.size(); i++) { + Record *OpRec1 = Inst->Operands[i].Rec; + Record *OpRec2 = Inst2->Operands[i].Rec; + + if (OpRec1 == OpRec2) + continue; + + if (isRegisterOperand(OpRec1) && isRegisterOperand(OpRec2)) { + if (getRegOperandSize(OpRec1) != getRegOperandSize(OpRec2)) + return false; + } else if (isMemoryOperand(OpRec1) && isMemoryOperand(OpRec2)) { + return false; + } else if (isImmediateOperand(OpRec1) && isImmediateOperand(OpRec2)) { + if (OpRec1->getValueAsDef("Type") != OpRec2->getValueAsDef("Type")) + return false; + } else + return false; + } + + return true; + } + +private: + static inline bool isRegisterOperand(const Record *Rec) { + return Rec->isSubClassOf("RegisterClass") || + Rec->isSubClassOf("RegisterOperand"); + } + + static inline bool isMemoryOperand(const Record *Rec) { + return Rec->isSubClassOf("Operand") && + Rec->getValueAsString("OperandType") == "OPERAND_MEMORY"; + } + + static inline bool isImmediateOperand(const Record *Rec) { + return Rec->isSubClassOf("Operand") && + Rec->getValueAsString("OperandType") == "OPERAND_IMMEDIATE"; + } + + static inline unsigned int getRegOperandSize(const Record *RegRec) { + if (RegRec->isSubClassOf("RegisterClass")) + return RegRec->getValueAsInt("Alignment"); + if (RegRec->isSubClassOf("RegisterOperand")) + return RegRec->getValueAsDef("RegClass")->getValueAsInt("Alignment"); + + llvm_unreachable("Register operand's size not known!"); + } +}; + +void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) { + emitSourceFileHeader("X86 EVEX2VEX tables", OS); + + ArrayRef<const CodeGenInstruction *> NumberedInstructions = + Target.getInstructionsByEnumValue(); + + for (const CodeGenInstruction *Inst : NumberedInstructions) { + // Filter non-X86 instructions. + if (!Inst->TheDef->isSubClassOf("X86Inst")) + continue; + + // Add VEX encoded instructions to one of VEXInsts vectors according to + // it's opcode. + if (Inst->TheDef->getValueAsDef("OpEnc")->getName() == "EncVEX") { + uint64_t Opcode = getValueFromBitsInit(Inst->TheDef-> + getValueAsBitsInit("Opcode")); + VEXInsts[Opcode].push_back(Inst); + } + // Add relevant EVEX encoded instructions to EVEXInsts + else if (Inst->TheDef->getValueAsDef("OpEnc")->getName() == "EncEVEX" && + !Inst->TheDef->getValueAsBit("hasEVEX_K") && + !Inst->TheDef->getValueAsBit("hasEVEX_B") && + getValueFromBitsInit(Inst->TheDef-> + getValueAsBitsInit("EVEX_LL")) != 2 && + !inExceptionList(Inst)) + EVEXInsts.push_back(Inst); + } + + for (const CodeGenInstruction *EVEXInst : EVEXInsts) { + uint64_t Opcode = getValueFromBitsInit(EVEXInst->TheDef-> + getValueAsBitsInit("Opcode")); + // For each EVEX instruction look for a VEX match in the appropriate vector + // (instructions with the same opcode) using function object IsMatch. + auto Match = llvm::find_if(VEXInsts[Opcode], IsMatch(EVEXInst)); + if (Match != VEXInsts[Opcode].end()) { + const CodeGenInstruction *VEXInst = *Match; + + // In case a match is found add new entry to the appropriate table + switch (getValueFromBitsInit( + EVEXInst->TheDef->getValueAsBitsInit("EVEX_LL"))) { + case 0: + EVEX2VEX128.push_back(std::make_pair(EVEXInst, VEXInst)); // {0,0} + break; + case 1: + EVEX2VEX256.push_back(std::make_pair(EVEXInst, VEXInst)); // {0,1} + break; + default: + llvm_unreachable("Instruction's size not fit for the mapping!"); + } + } + } + + // Print both tables + printTable(EVEX2VEX128, OS); + printTable(EVEX2VEX256, OS); +} +} + +namespace llvm { +void EmitX86EVEX2VEXTables(RecordKeeper &RK, raw_ostream &OS) { + X86EVEX2VEXTablesEmitter(RK).run(OS); +} +} diff --git a/utils/TableGen/X86RecognizableInstr.cpp b/utils/TableGen/X86RecognizableInstr.cpp index 4736c4e510d12..e703bbfc4496f 100644 --- a/utils/TableGen/X86RecognizableInstr.cpp +++ b/utils/TableGen/X86RecognizableInstr.cpp @@ -138,6 +138,10 @@ namespace X86Local { enum { AdSize16 = 1, AdSize32 = 2, AdSize64 = 3 }; + + enum { + VEX_W0 = 0, VEX_W1 = 1, VEX_WIG = 2 + }; } using namespace X86Disassembler; @@ -203,7 +207,7 @@ RecognizableInstr::RecognizableInstr(DisassemblerTables &tables, AdSize = byteFromRec(Rec, "AdSizeBits"); HasREX_WPrefix = Rec->getValueAsBit("hasREX_WPrefix"); HasVEX_4V = Rec->getValueAsBit("hasVEX_4V"); - HasVEX_WPrefix = Rec->getValueAsBit("hasVEX_WPrefix"); + VEX_WPrefix = byteFromRec(Rec,"VEX_WPrefix"); IgnoresVEX_L = Rec->getValueAsBit("ignoresVEX_L"); HasEVEX_L2Prefix = Rec->getValueAsBit("hasEVEX_L2"); HasEVEX_K = Rec->getValueAsBit("hasEVEX_K"); @@ -280,7 +284,7 @@ InstructionContext RecognizableInstr::insnContext() const { llvm_unreachable("Don't support VEX.L if EVEX_L2 is enabled"); } // VEX_L & VEX_W - if (HasVEX_LPrefix && HasVEX_WPrefix) { + if (HasVEX_LPrefix && VEX_WPrefix == X86Local::VEX_W1) { if (OpPrefix == X86Local::PD) insnContext = EVEX_KB(IC_EVEX_L_W_OPSIZE); else if (OpPrefix == X86Local::XS) @@ -308,7 +312,7 @@ InstructionContext RecognizableInstr::insnContext() const { llvm_unreachable("Invalid prefix"); } } - else if (HasEVEX_L2Prefix && HasVEX_WPrefix) { + else if (HasEVEX_L2Prefix && VEX_WPrefix == X86Local::VEX_W1) { // EVEX_L2 & VEX_W if (OpPrefix == X86Local::PD) insnContext = EVEX_KB(IC_EVEX_L2_W_OPSIZE); @@ -337,7 +341,7 @@ InstructionContext RecognizableInstr::insnContext() const { llvm_unreachable("Invalid prefix"); } } - else if (HasVEX_WPrefix) { + else if (VEX_WPrefix == X86Local::VEX_W1) { // VEX_W if (OpPrefix == X86Local::PD) insnContext = EVEX_KB(IC_EVEX_W_OPSIZE); @@ -363,7 +367,7 @@ InstructionContext RecognizableInstr::insnContext() const { insnContext = EVEX_KB(IC_EVEX); /// eof EVEX } else if (Encoding == X86Local::VEX || Encoding == X86Local::XOP) { - if (HasVEX_LPrefix && HasVEX_WPrefix) { + if (HasVEX_LPrefix && VEX_WPrefix == X86Local::VEX_W1) { if (OpPrefix == X86Local::PD) insnContext = IC_VEX_L_W_OPSIZE; else if (OpPrefix == X86Local::XS) @@ -378,7 +382,7 @@ InstructionContext RecognizableInstr::insnContext() const { } } else if (OpPrefix == X86Local::PD && HasVEX_LPrefix) insnContext = IC_VEX_L_OPSIZE; - else if (OpPrefix == X86Local::PD && HasVEX_WPrefix) + else if (OpPrefix == X86Local::PD && VEX_WPrefix == X86Local::VEX_W1) insnContext = IC_VEX_W_OPSIZE; else if (OpPrefix == X86Local::PD) insnContext = IC_VEX_OPSIZE; @@ -386,11 +390,11 @@ InstructionContext RecognizableInstr::insnContext() const { insnContext = IC_VEX_L_XS; else if (HasVEX_LPrefix && OpPrefix == X86Local::XD) insnContext = IC_VEX_L_XD; - else if (HasVEX_WPrefix && OpPrefix == X86Local::XS) + else if (VEX_WPrefix == X86Local::VEX_W1 && OpPrefix == X86Local::XS) insnContext = IC_VEX_W_XS; - else if (HasVEX_WPrefix && OpPrefix == X86Local::XD) + else if (VEX_WPrefix == X86Local::VEX_W1 && OpPrefix == X86Local::XD) insnContext = IC_VEX_W_XD; - else if (HasVEX_WPrefix && OpPrefix == X86Local::PS) + else if (VEX_WPrefix == X86Local::VEX_W1 && OpPrefix == X86Local::PS) insnContext = IC_VEX_W; else if (HasVEX_LPrefix && OpPrefix == X86Local::PS) insnContext = IC_VEX_L; @@ -457,10 +461,12 @@ void RecognizableInstr::adjustOperandEncoding(OperandEncoding &encoding) { // The scaling factor for AVX512 compressed displacement encoding is an // instruction attribute. Adjust the ModRM encoding type to include the // scale for compressed displacement. - if (encoding != ENCODING_RM || CD8_Scale == 0) + if ((encoding != ENCODING_RM && encoding != ENCODING_VSIB) ||CD8_Scale == 0) return; encoding = (OperandEncoding)(encoding + Log2_32(CD8_Scale)); - assert(encoding <= ENCODING_RM_CD64 && "Invalid CDisp scaling"); + assert(((encoding >= ENCODING_RM && encoding <= ENCODING_RM_CD64) || + (encoding >= ENCODING_VSIB && encoding <= ENCODING_VSIB_CD64)) && + "Invalid CDisp scaling"); } void RecognizableInstr::handleOperand(bool optional, unsigned &operandIndex, @@ -944,121 +950,121 @@ OperandType RecognizableInstr::typeFromString(const std::string &s, // For OpSize16 instructions, a declared 16-bit register or // immediate encoding is special. TYPE("GR16", TYPE_Rv) - TYPE("i16imm", TYPE_IMMv) } else if(OpSize == X86Local::OpSize32) { // For OpSize32 instructions, a declared 32-bit register or // immediate encoding is special. TYPE("GR32", TYPE_Rv) } - TYPE("i16mem", TYPE_Mv) - TYPE("i16imm", TYPE_IMM16) - TYPE("i16i8imm", TYPE_IMMv) + TYPE("i16mem", TYPE_M) + TYPE("i16imm", TYPE_IMM) + TYPE("i16i8imm", TYPE_IMM) TYPE("GR16", TYPE_R16) - TYPE("i32mem", TYPE_Mv) - TYPE("i32imm", TYPE_IMMv) - TYPE("i32i8imm", TYPE_IMM32) + TYPE("i32mem", TYPE_M) + TYPE("i32imm", TYPE_IMM) + TYPE("i32i8imm", TYPE_IMM) TYPE("GR32", TYPE_R32) TYPE("GR32orGR64", TYPE_R32) - TYPE("i64mem", TYPE_Mv) - TYPE("i64i32imm", TYPE_IMM64) - TYPE("i64i8imm", TYPE_IMM64) + TYPE("i64mem", TYPE_M) + TYPE("i64i32imm", TYPE_IMM) + TYPE("i64i8imm", TYPE_IMM) TYPE("GR64", TYPE_R64) - TYPE("i8mem", TYPE_M8) - TYPE("i8imm", TYPE_IMM8) + TYPE("i8mem", TYPE_M) + TYPE("i8imm", TYPE_IMM) TYPE("u8imm", TYPE_UIMM8) TYPE("i32u8imm", TYPE_UIMM8) TYPE("GR8", TYPE_R8) - TYPE("VR128", TYPE_XMM128) - TYPE("VR128X", TYPE_XMM128) - TYPE("f128mem", TYPE_M128) - TYPE("f256mem", TYPE_M256) - TYPE("f512mem", TYPE_M512) - TYPE("FR128", TYPE_XMM128) - TYPE("FR64", TYPE_XMM64) - TYPE("FR64X", TYPE_XMM64) - TYPE("f64mem", TYPE_M64FP) - TYPE("sdmem", TYPE_M64FP) - TYPE("FR32", TYPE_XMM32) - TYPE("FR32X", TYPE_XMM32) - TYPE("f32mem", TYPE_M32FP) - TYPE("ssmem", TYPE_M32FP) + TYPE("VR128", TYPE_XMM) + TYPE("VR128X", TYPE_XMM) + TYPE("f128mem", TYPE_M) + TYPE("f256mem", TYPE_M) + TYPE("f512mem", TYPE_M) + TYPE("FR128", TYPE_XMM) + TYPE("FR64", TYPE_XMM) + TYPE("FR64X", TYPE_XMM) + TYPE("f64mem", TYPE_M) + TYPE("sdmem", TYPE_M) + TYPE("FR32", TYPE_XMM) + TYPE("FR32X", TYPE_XMM) + TYPE("f32mem", TYPE_M) + TYPE("ssmem", TYPE_M) TYPE("RST", TYPE_ST) - TYPE("i128mem", TYPE_M128) - TYPE("i256mem", TYPE_M256) - TYPE("i512mem", TYPE_M512) - TYPE("i64i32imm_pcrel", TYPE_REL64) - TYPE("i16imm_pcrel", TYPE_REL16) - TYPE("i32imm_pcrel", TYPE_REL32) + TYPE("i128mem", TYPE_M) + TYPE("i256mem", TYPE_M) + TYPE("i512mem", TYPE_M) + TYPE("i64i32imm_pcrel", TYPE_REL) + TYPE("i16imm_pcrel", TYPE_REL) + TYPE("i32imm_pcrel", TYPE_REL) TYPE("SSECC", TYPE_IMM3) TYPE("XOPCC", TYPE_IMM3) TYPE("AVXCC", TYPE_IMM5) TYPE("AVX512ICC", TYPE_AVX512ICC) - TYPE("AVX512RC", TYPE_IMM32) - TYPE("brtarget32", TYPE_RELv) - TYPE("brtarget16", TYPE_RELv) - TYPE("brtarget8", TYPE_REL8) - TYPE("f80mem", TYPE_M80FP) - TYPE("lea64_32mem", TYPE_LEA) - TYPE("lea64mem", TYPE_LEA) + TYPE("AVX512RC", TYPE_IMM) + TYPE("brtarget32", TYPE_REL) + TYPE("brtarget16", TYPE_REL) + TYPE("brtarget8", TYPE_REL) + TYPE("f80mem", TYPE_M) + TYPE("lea64_32mem", TYPE_M) + TYPE("lea64mem", TYPE_M) TYPE("VR64", TYPE_MM64) - TYPE("i64imm", TYPE_IMMv) + TYPE("i64imm", TYPE_IMM) TYPE("anymem", TYPE_M) - TYPE("opaque32mem", TYPE_M1616) - TYPE("opaque48mem", TYPE_M1632) - TYPE("opaque80mem", TYPE_M1664) - TYPE("opaque512mem", TYPE_M512) + TYPE("opaque32mem", TYPE_M) + TYPE("opaque48mem", TYPE_M) + TYPE("opaque80mem", TYPE_M) + TYPE("opaque512mem", TYPE_M) TYPE("SEGMENT_REG", TYPE_SEGMENTREG) TYPE("DEBUG_REG", TYPE_DEBUGREG) TYPE("CONTROL_REG", TYPE_CONTROLREG) - TYPE("srcidx8", TYPE_SRCIDX8) - TYPE("srcidx16", TYPE_SRCIDX16) - TYPE("srcidx32", TYPE_SRCIDX32) - TYPE("srcidx64", TYPE_SRCIDX64) - TYPE("dstidx8", TYPE_DSTIDX8) - TYPE("dstidx16", TYPE_DSTIDX16) - TYPE("dstidx32", TYPE_DSTIDX32) - TYPE("dstidx64", TYPE_DSTIDX64) - TYPE("offset16_8", TYPE_MOFFS8) - TYPE("offset16_16", TYPE_MOFFS16) - TYPE("offset16_32", TYPE_MOFFS32) - TYPE("offset32_8", TYPE_MOFFS8) - TYPE("offset32_16", TYPE_MOFFS16) - TYPE("offset32_32", TYPE_MOFFS32) - TYPE("offset32_64", TYPE_MOFFS64) - TYPE("offset64_8", TYPE_MOFFS8) - TYPE("offset64_16", TYPE_MOFFS16) - TYPE("offset64_32", TYPE_MOFFS32) - TYPE("offset64_64", TYPE_MOFFS64) - TYPE("VR256", TYPE_XMM256) - TYPE("VR256X", TYPE_XMM256) - TYPE("VR512", TYPE_XMM512) - TYPE("VK1", TYPE_VK1) - TYPE("VK1WM", TYPE_VK1) - TYPE("VK2", TYPE_VK2) - TYPE("VK2WM", TYPE_VK2) - TYPE("VK4", TYPE_VK4) - TYPE("VK4WM", TYPE_VK4) - TYPE("VK8", TYPE_VK8) - TYPE("VK8WM", TYPE_VK8) - TYPE("VK16", TYPE_VK16) - TYPE("VK16WM", TYPE_VK16) - TYPE("VK32", TYPE_VK32) - TYPE("VK32WM", TYPE_VK32) - TYPE("VK64", TYPE_VK64) - TYPE("VK64WM", TYPE_VK64) + TYPE("srcidx8", TYPE_SRCIDX) + TYPE("srcidx16", TYPE_SRCIDX) + TYPE("srcidx32", TYPE_SRCIDX) + TYPE("srcidx64", TYPE_SRCIDX) + TYPE("dstidx8", TYPE_DSTIDX) + TYPE("dstidx16", TYPE_DSTIDX) + TYPE("dstidx32", TYPE_DSTIDX) + TYPE("dstidx64", TYPE_DSTIDX) + TYPE("offset16_8", TYPE_MOFFS) + TYPE("offset16_16", TYPE_MOFFS) + TYPE("offset16_32", TYPE_MOFFS) + TYPE("offset32_8", TYPE_MOFFS) + TYPE("offset32_16", TYPE_MOFFS) + TYPE("offset32_32", TYPE_MOFFS) + TYPE("offset32_64", TYPE_MOFFS) + TYPE("offset64_8", TYPE_MOFFS) + TYPE("offset64_16", TYPE_MOFFS) + TYPE("offset64_32", TYPE_MOFFS) + TYPE("offset64_64", TYPE_MOFFS) + TYPE("VR256", TYPE_YMM) + TYPE("VR256X", TYPE_YMM) + TYPE("VR512", TYPE_ZMM) + TYPE("VK1", TYPE_VK) + TYPE("VK1WM", TYPE_VK) + TYPE("VK2", TYPE_VK) + TYPE("VK2WM", TYPE_VK) + TYPE("VK4", TYPE_VK) + TYPE("VK4WM", TYPE_VK) + TYPE("VK8", TYPE_VK) + TYPE("VK8WM", TYPE_VK) + TYPE("VK16", TYPE_VK) + TYPE("VK16WM", TYPE_VK) + TYPE("VK32", TYPE_VK) + TYPE("VK32WM", TYPE_VK) + TYPE("VK64", TYPE_VK) + TYPE("VK64WM", TYPE_VK) TYPE("GR32_NOAX", TYPE_Rv) - TYPE("vx64mem", TYPE_M64) - TYPE("vx128mem", TYPE_M128) - TYPE("vx256mem", TYPE_M256) - TYPE("vy128mem", TYPE_M128) - TYPE("vy256mem", TYPE_M256) - TYPE("vx64xmem", TYPE_M64) - TYPE("vx128xmem", TYPE_M128) - TYPE("vx256xmem", TYPE_M256) - TYPE("vy128xmem", TYPE_M128) - TYPE("vy256xmem", TYPE_M256) - TYPE("vy512mem", TYPE_M512) - TYPE("vz512mem", TYPE_M512) + TYPE("vx64mem", TYPE_M) + TYPE("vx128mem", TYPE_M) + TYPE("vx256mem", TYPE_M) + TYPE("vy128mem", TYPE_M) + TYPE("vy256mem", TYPE_M) + TYPE("vx64xmem", TYPE_M) + TYPE("vx128xmem", TYPE_M) + TYPE("vx256xmem", TYPE_M) + TYPE("vy128xmem", TYPE_M) + TYPE("vy256xmem", TYPE_M) + TYPE("vy512mem", TYPE_M) + TYPE("vz256xmem", TYPE_M) + TYPE("vz512mem", TYPE_M) TYPE("BNDR", TYPE_BNDR) errs() << "Unhandled type string " << s << "\n"; llvm_unreachable("Unhandled type string"); @@ -1242,18 +1248,19 @@ RecognizableInstr::memoryEncodingFromString(const std::string &s, ENCODING("opaque48mem", ENCODING_RM) ENCODING("opaque80mem", ENCODING_RM) ENCODING("opaque512mem", ENCODING_RM) - ENCODING("vx64mem", ENCODING_RM) - ENCODING("vx128mem", ENCODING_RM) - ENCODING("vx256mem", ENCODING_RM) - ENCODING("vy128mem", ENCODING_RM) - ENCODING("vy256mem", ENCODING_RM) - ENCODING("vx64xmem", ENCODING_RM) - ENCODING("vx128xmem", ENCODING_RM) - ENCODING("vx256xmem", ENCODING_RM) - ENCODING("vy128xmem", ENCODING_RM) - ENCODING("vy256xmem", ENCODING_RM) - ENCODING("vy512mem", ENCODING_RM) - ENCODING("vz512mem", ENCODING_RM) + ENCODING("vx64mem", ENCODING_VSIB) + ENCODING("vx128mem", ENCODING_VSIB) + ENCODING("vx256mem", ENCODING_VSIB) + ENCODING("vy128mem", ENCODING_VSIB) + ENCODING("vy256mem", ENCODING_VSIB) + ENCODING("vx64xmem", ENCODING_VSIB) + ENCODING("vx128xmem", ENCODING_VSIB) + ENCODING("vx256xmem", ENCODING_VSIB) + ENCODING("vy128xmem", ENCODING_VSIB) + ENCODING("vy256xmem", ENCODING_VSIB) + ENCODING("vy512mem", ENCODING_VSIB) + ENCODING("vz256xmem", ENCODING_VSIB) + ENCODING("vz512mem", ENCODING_VSIB) errs() << "Unhandled memory encoding " << s << "\n"; llvm_unreachable("Unhandled memory encoding"); } diff --git a/utils/TableGen/X86RecognizableInstr.h b/utils/TableGen/X86RecognizableInstr.h index 2e611587cc316..91ed928540c36 100644 --- a/utils/TableGen/X86RecognizableInstr.h +++ b/utils/TableGen/X86RecognizableInstr.h @@ -55,8 +55,8 @@ private: bool HasREX_WPrefix; /// The hasVEX_4V field from the record bool HasVEX_4V; - /// The hasVEX_WPrefix field from the record - bool HasVEX_WPrefix; + /// The VEX_WPrefix field from the record + uint8_t VEX_WPrefix; /// Inferred from the operands; indicates whether the L bit in the VEX prefix is set bool HasVEX_LPrefix; /// The ignoreVEX_L field from the record |