diff options
Diffstat (limited to 'contrib/llvm-project/llvm/utils')
67 files changed, 6979 insertions, 1726 deletions
diff --git a/contrib/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 146d10835b8d..ccf0959389ba 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -1111,6 +1111,7 @@ static std::string getEnumNameForToken(StringRef Str) { case '<': Res += "_LT_"; break; case '>': Res += "_GT_"; break; case '-': Res += "_MINUS_"; break; + case '#': Res += "_HASH_"; break; default: if ((*it >= 'A' && *it <= 'Z') || (*it >= 'a' && *it <= 'z') || @@ -1439,7 +1440,7 @@ void AsmMatcherInfo::buildOperandMatchInfo() { /// Map containing a mask with all operands indices that can be found for /// that class inside a instruction. - typedef std::map<ClassInfo *, unsigned, less_ptr<ClassInfo>> OpClassMaskTy; + typedef std::map<ClassInfo *, unsigned, deref<std::less<>>> OpClassMaskTy; OpClassMaskTy OpClassMask; for (const auto &MI : Matchables) { @@ -1515,7 +1516,7 @@ void AsmMatcherInfo::buildInfo() { if (!V.empty() && V != Variant.Name) continue; - auto II = llvm::make_unique<MatchableInfo>(*CGI); + auto II = std::make_unique<MatchableInfo>(*CGI); II->initialize(*this, SingletonRegisters, Variant, HasMnemonicFirst); @@ -1532,7 +1533,7 @@ void AsmMatcherInfo::buildInfo() { std::vector<Record*> AllInstAliases = Records.getAllDerivedDefinitions("InstAlias"); for (unsigned i = 0, e = AllInstAliases.size(); i != e; ++i) { - auto Alias = llvm::make_unique<CodeGenInstAlias>(AllInstAliases[i], + auto Alias = std::make_unique<CodeGenInstAlias>(AllInstAliases[i], Target); // If the tblgen -match-prefix option is specified (for tblgen hackers), @@ -1546,7 +1547,7 @@ void AsmMatcherInfo::buildInfo() { if (!V.empty() && V != Variant.Name) continue; - auto II = llvm::make_unique<MatchableInfo>(std::move(Alias)); + auto II = std::make_unique<MatchableInfo>(std::move(Alias)); II->initialize(*this, SingletonRegisters, Variant, HasMnemonicFirst); @@ -1615,7 +1616,7 @@ void AsmMatcherInfo::buildInfo() { II->TheDef->getValueAsString("TwoOperandAliasConstraint"); if (Constraint != "") { // Start by making a copy of the original matchable. - auto AliasII = llvm::make_unique<MatchableInfo>(*II); + auto AliasII = std::make_unique<MatchableInfo>(*II); // Adjust it to be a two-operand alias. AliasII->formTwoOperandAlias(Constraint); @@ -2381,7 +2382,7 @@ static void emitMatchClassEnumeration(CodeGenTarget &Target, OS << " NumMatchClassKinds\n"; OS << "};\n\n"; - OS << "}\n\n"; + OS << "} // end anonymous namespace\n\n"; } /// emitMatchClassDiagStrings - Emit a function to get the diagnostic text to be @@ -2866,7 +2867,7 @@ static void emitCustomOperandParsing(raw_ostream &OS, CodeGenTarget &Target, OS << " }\n"; OS << " };\n"; - OS << "} // end anonymous namespace.\n\n"; + OS << "} // end anonymous namespace\n\n"; OS << "static const OperandMatchEntry OperandMatchTable[" << Info.OperandMatchInfo.size() << "] = {\n"; @@ -3346,7 +3347,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) { return true; if (A.size() > B.size()) return false; - for (const auto &Pair : zip(A, B)) { + for (auto Pair : zip(A, B)) { if (std::get<0>(Pair)->getName() < std::get<1>(Pair)->getName()) return true; if (std::get<0>(Pair)->getName() > std::get<1>(Pair)->getName()) @@ -3366,7 +3367,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) { OS << " " << getNameForFeatureBitset(FeatureBitset) << ",\n"; } OS << "};\n\n" - << "const static FeatureBitset FeatureBitsets[] {\n" + << "static constexpr FeatureBitset FeatureBitsets[] = {\n" << " {}, // AMFBS_None\n"; for (const auto &FeatureBitset : FeatureBitsets) { if (FeatureBitset.empty()) @@ -3422,7 +3423,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) { OS << " }\n"; OS << " };\n"; - OS << "} // end anonymous namespace.\n\n"; + OS << "} // end anonymous namespace\n\n"; unsigned VariantCount = Target.getAsmParserVariantCount(); for (unsigned VC = 0; VC != VariantCount; ++VC) { diff --git a/contrib/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp index 05d81f133505..58c0d32d44eb 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp @@ -29,6 +29,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" +#include "llvm/Support/FormatVariadic.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Error.h" @@ -274,13 +275,14 @@ void AsmWriterEmitter::EmitPrintInstruction(raw_ostream &O) { StringRef ClassName = AsmWriter->getValueAsString("AsmWriterClassName"); bool PassSubtarget = AsmWriter->getValueAsInt("PassSubtarget"); - O << - "/// printInstruction - This method is automatically generated by tablegen\n" - "/// from the instruction set description.\n" - "void " << Target.getName() << ClassName - << "::printInstruction(const MCInst *MI, " - << (PassSubtarget ? "const MCSubtargetInfo &STI, " : "") - << "raw_ostream &O) {\n"; + O << "/// printInstruction - This method is automatically generated by " + "tablegen\n" + "/// from the instruction set description.\n" + "void " + << Target.getName() << ClassName + << "::printInstruction(const MCInst *MI, uint64_t Address, " + << (PassSubtarget ? "const MCSubtargetInfo &STI, " : "") + << "raw_ostream &O) {\n"; // Build an aggregate string, and build a table of offsets into it. SequenceToOffsetTable<std::string> StringTable; @@ -616,17 +618,22 @@ namespace { // they both have the same conditionals. In which case, we cannot print out the // alias for that pattern. class IAPrinter { - std::vector<std::string> Conds; std::map<StringRef, std::pair<int, int>> OpMap; + std::vector<std::string> Conds; + std::string Result; std::string AsmString; + unsigned NumMIOps; + public: - IAPrinter(std::string R, std::string AS) - : Result(std::move(R)), AsmString(std::move(AS)) {} + IAPrinter(std::string R, std::string AS, unsigned NumMIOps) + : Result(std::move(R)), AsmString(std::move(AS)), NumMIOps(NumMIOps) {} - void addCond(const std::string &C) { Conds.push_back(C); } + void addCond(std::string C) { Conds.push_back(std::move(C)); } + ArrayRef<std::string> getConds() const { return Conds; } + size_t getCondCount() const { return Conds.size(); } void addOperand(StringRef Op, int OpIdx, int PrintMethodIdx = -1) { assert(OpIdx >= 0 && OpIdx < 0xFE && "Idx out of range"); @@ -635,6 +642,10 @@ public: OpMap[Op] = std::make_pair(OpIdx, PrintMethodIdx); } + unsigned getNumMIOps() { return NumMIOps; } + + StringRef getResult() { return Result; } + bool isOpMapped(StringRef Op) { return OpMap.find(Op) != OpMap.end(); } int getOpIndex(StringRef Op) { return OpMap[Op].first; } std::pair<int, int> &getOpData(StringRef Op) { return OpMap[Op]; } @@ -664,35 +675,16 @@ public: return std::make_pair(StringRef(Start, I - Start), Next); } - void print(raw_ostream &O) { - if (Conds.empty()) { - O.indent(6) << "return true;\n"; - return; - } - - O << "if ("; - - for (std::vector<std::string>::iterator - I = Conds.begin(), E = Conds.end(); I != E; ++I) { - if (I != Conds.begin()) { - O << " &&\n"; - O.indent(8); - } - - O << *I; - } - - O << ") {\n"; - O.indent(6) << "// " << Result << "\n"; - + std::string formatAliasString(uint32_t &UnescapedSize) { // Directly mangle mapped operands into the string. Each operand is // identified by a '$' sign followed by a byte identifying the number of the // operand. We add one to the index to avoid zero bytes. StringRef ASM(AsmString); - SmallString<128> OutString; - raw_svector_ostream OS(OutString); + std::string OutString; + raw_string_ostream OS(OutString); for (StringRef::iterator I = ASM.begin(), E = ASM.end(); I != E;) { OS << *I; + ++UnescapedSize; if (*I == '$') { StringRef Name; std::tie(Name, I) = parseName(++I, E); @@ -703,23 +695,25 @@ public: if (PrintIndex == -1) { // Can use the default printOperand route. OS << format("\\x%02X", (unsigned char)OpIndex + 1); - } else + ++UnescapedSize; + } else { // 3 bytes if a PrintMethod is needed: 0xFF, the MCInst operand // number, and which of our pre-detected Methods to call. OS << format("\\xFF\\x%02X\\x%02X", OpIndex + 1, PrintIndex + 1); + UnescapedSize += 3; + } } else { ++I; } } - // Emit the string. - O.indent(6) << "AsmString = \"" << OutString << "\";\n"; - - O.indent(6) << "break;\n"; - O.indent(4) << '}'; + OS.flush(); + return OutString; } bool operator==(const IAPrinter &RHS) const { + if (NumMIOps != RHS.NumMIOps) + return false; if (Conds.size() != RHS.Conds.size()) return false; @@ -784,8 +778,7 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { continue; // Aliases with priority 0 are never emitted. const DagInit *DI = R->getValueAsDag("ResultInst"); - const DefInit *Op = cast<DefInit>(DI->getOperator()); - AliasMap[getQualifiedName(Op->getDef())].insert( + AliasMap[getQualifiedName(DI->getOperatorAsDef(R->getLoc()))].insert( std::make_pair(CodeGenInstAlias(R, Target), Priority)); } @@ -800,6 +793,9 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { DenseMap<const Record*, unsigned> MCOpPredicateMap; for (auto &Aliases : AliasMap) { + // Collection of instruction alias rules. May contain ambiguous rules. + std::vector<IAPrinter> IAPs; + for (auto &Alias : Aliases.second) { const CodeGenInstAlias &CGA = Alias.first; unsigned LastOpNo = CGA.ResultInstOperandIndex.size(); @@ -809,33 +805,18 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { unsigned NumResultOps = CountNumOperands(FlatInstAsmString, Variant); std::string FlatAliasAsmString = - CodeGenInstruction::FlattenAsmStringVariants(CGA.AsmString, - Variant); + CodeGenInstruction::FlattenAsmStringVariants(CGA.AsmString, Variant); // Don't emit the alias if it has more operands than what it's aliasing. if (NumResultOps < CountNumOperands(FlatAliasAsmString, Variant)) continue; - IAPrinter IAP(CGA.Result->getAsString(), FlatAliasAsmString); - StringRef Namespace = Target.getName(); - std::vector<Record *> ReqFeatures; - if (PassSubtarget) { - // We only consider ReqFeatures predicates if PassSubtarget - std::vector<Record *> RF = - CGA.TheDef->getValueAsListOfDefs("Predicates"); - copy_if(RF, std::back_inserter(ReqFeatures), [](Record *R) { - return R->getValueAsBit("AssemblerMatcherPredicate"); - }); - } - unsigned NumMIOps = 0; for (auto &ResultInstOpnd : CGA.ResultInst->Operands) NumMIOps += ResultInstOpnd.MINumOperands; - std::string Cond; - Cond = std::string("MI->getNumOperands() == ") + utostr(NumMIOps); - IAP.addCond(Cond); + IAPrinter IAP(CGA.Result->getAsString(), FlatAliasAsmString, NumMIOps); bool CantHandle = false; @@ -859,7 +840,9 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { break; } - std::string Op = "MI->getOperand(" + utostr(MIOpNum) + ")"; + // Ignore unchecked result operands. + while (IAP.getCondCount() < MIOpNum) + IAP.addCond("AliasPatternCond::K_Ignore, 0"); const CodeGenInstAlias::ResultOperand &RO = CGA.ResultOperands[i]; @@ -886,19 +869,17 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { if (Rec->isSubClassOf("RegisterOperand")) Rec = Rec->getValueAsDef("RegClass"); if (Rec->isSubClassOf("RegisterClass")) { - IAP.addCond(Op + ".isReg()"); - if (!IAP.isOpMapped(ROName)) { IAP.addOperand(ROName, MIOpNum, PrintMethodIdx); Record *R = CGA.ResultOperands[i].getRecord(); if (R->isSubClassOf("RegisterOperand")) R = R->getValueAsDef("RegClass"); - Cond = std::string("MRI.getRegClass(") + Target.getName().str() + - "::" + R->getName().str() + "RegClassID).contains(" + Op + - ".getReg())"; + IAP.addCond(formatv( + "AliasPatternCond::K_RegClass, {0}::{1}RegClassID", Namespace, + R->getName())); } else { - Cond = Op + ".getReg() == MI->getOperand(" + - utostr(IAP.getOpIndex(ROName)) + ").getReg()"; + IAP.addCond(formatv("AliasPatternCond::K_TiedReg, {0}", + IAP.getOpIndex(ROName))); } } else { // Assume all printable operands are desired for now. This can be @@ -915,21 +896,19 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { } else break; // No conditions on this operand at all } - Cond = (Target.getName() + ClassName + "ValidateMCOperand(" + Op + - ", STI, " + utostr(Entry) + ")") - .str(); + IAP.addCond(formatv("AliasPatternCond::K_Custom, {0}", Entry)); } - // for all subcases of ResultOperand::K_Record: - IAP.addCond(Cond); break; } case CodeGenInstAlias::ResultOperand::K_Imm: { // Just because the alias has an immediate result, doesn't mean the // MCInst will. An MCExpr could be present, for example. - IAP.addCond(Op + ".isImm()"); - - Cond = Op + ".getImm() == " + itostr(CGA.ResultOperands[i].getImm()); - IAP.addCond(Cond); + auto Imm = CGA.ResultOperands[i].getImm(); + int32_t Imm32 = int32_t(Imm); + if (Imm != Imm32) + PrintFatalError("Matching an alias with an immediate out of the " + "range of int32_t is not supported"); + IAP.addCond(formatv("AliasPatternCond::K_Imm, uint32_t({0})", Imm32)); break; } case CodeGenInstAlias::ResultOperand::K_Reg: @@ -940,9 +919,9 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { break; } - Cond = Op + ".getReg() == " + Target.getName().str() + "::" + - CGA.ResultOperands[i].getRegister()->getName().str(); - IAP.addCond(Cond); + StringRef Reg = CGA.ResultOperands[i].getRegister()->getName(); + IAP.addCond( + formatv("AliasPatternCond::K_Reg, {0}::{1}", Namespace, Reg)); break; } @@ -951,6 +930,16 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { if (CantHandle) continue; + std::vector<Record *> ReqFeatures; + if (PassSubtarget) { + // We only consider ReqFeatures predicates if PassSubtarget + std::vector<Record *> RF = + CGA.TheDef->getValueAsListOfDefs("Predicates"); + copy_if(RF, std::back_inserter(ReqFeatures), [](Record *R) { + return R->getValueAsBit("AssemblerMatcherPredicate"); + }); + } + for (auto I = ReqFeatures.cbegin(); I != ReqFeatures.cend(); I++) { Record *R = *I; StringRef AsmCondString = R->getValueAsString("AssemblerCondString"); @@ -960,16 +949,12 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { SplitString(AsmCondString, Ops, ","); assert(!Ops.empty() && "AssemblerCondString cannot be empty"); - for (auto &Op : Ops) { + for (StringRef Op : Ops) { assert(!Op.empty() && "Empty operator"); - if (Op[0] == '!') - Cond = ("!STI.getFeatureBits()[" + Namespace + "::" + Op.substr(1) + - "]") - .str(); - else - Cond = - ("STI.getFeatureBits()[" + Namespace + "::" + Op + "]").str(); - IAP.addCond(Cond); + bool IsNeg = Op[0] == '!'; + StringRef Feature = Op.drop_front(IsNeg ? 1 : 0); + IAP.addCond(formatv("AliasPatternCond::K_{0}Feature, {1}::{2}", + IsNeg ? "Neg" : "", Namespace, Feature)); } } @@ -989,13 +974,32 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { << " *MI, " << (PassSubtarget ? "const MCSubtargetInfo &STI, " : "") << "raw_ostream &OS) {\n"; - std::string Cases; - raw_string_ostream CasesO(Cases); - - for (auto &Entry : IAPrinterMap) { - std::vector<IAPrinter> &IAPs = Entry.second; + std::string PatternsForOpcode; + raw_string_ostream OpcodeO(PatternsForOpcode); + + unsigned PatternCount = 0; + std::string Patterns; + raw_string_ostream PatternO(Patterns); + + unsigned CondCount = 0; + std::string Conds; + raw_string_ostream CondO(Conds); + + // All flattened alias strings. + std::map<std::string, uint32_t> AsmStringOffsets; + std::vector<std::pair<uint32_t, std::string>> AsmStrings; + size_t AsmStringsSize = 0; + + // Iterate over the opcodes in enum order so they are sorted by opcode for + // binary search. + for (const CodeGenInstruction *Inst : NumberedInstructions) { + auto It = IAPrinterMap.find(getQualifiedName(Inst->TheDef)); + if (It == IAPrinterMap.end()) + continue; + std::vector<IAPrinter> &IAPs = It->second; std::vector<IAPrinter*> UniqueIAPs; + // Remove any ambiguous alias rules. for (auto &LHS : IAPs) { bool IsDup = false; for (const auto &RHS : IAPs) { @@ -1011,18 +1015,43 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { if (UniqueIAPs.empty()) continue; - CasesO.indent(2) << "case " << Entry.first << ":\n"; + unsigned PatternStart = PatternCount; + + // Insert the pattern start and opcode in the pattern list for debugging. + PatternO << formatv(" // {0} - {1}\n", It->first, PatternStart); for (IAPrinter *IAP : UniqueIAPs) { - CasesO.indent(4); - IAP->print(CasesO); - CasesO << '\n'; + // Start each condition list with a comment of the resulting pattern that + // we're trying to match. + unsigned CondStart = CondCount; + CondO << formatv(" // {0} - {1}\n", IAP->getResult(), CondStart); + for (const auto &Cond : IAP->getConds()) + CondO << " {" << Cond << "},\n"; + CondCount += IAP->getCondCount(); + + // After operands have been examined, re-encode the alias string with + // escapes indicating how operands should be printed. + uint32_t UnescapedSize = 0; + std::string EncodedAsmString = IAP->formatAliasString(UnescapedSize); + auto Insertion = + AsmStringOffsets.insert({EncodedAsmString, AsmStringsSize}); + if (Insertion.second) { + // If the string is new, add it to the vector. + AsmStrings.push_back({AsmStringsSize, EncodedAsmString}); + AsmStringsSize += UnescapedSize + 1; + } + unsigned AsmStrOffset = Insertion.first->second; + + PatternO << formatv(" {{{0}, {1}, {2}, {3} },\n", AsmStrOffset, + CondStart, IAP->getNumMIOps(), IAP->getCondCount()); + ++PatternCount; } - CasesO.indent(4) << "return false;\n"; + OpcodeO << formatv(" {{{0}, {1}, {2} },\n", It->first, PatternStart, + PatternCount - PatternStart); } - if (CasesO.str().empty()) { + if (OpcodeO.str().empty()) { O << HeaderO.str(); O << " return false;\n"; O << "}\n\n"; @@ -1030,6 +1059,7 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { return; } + // Forward declare the validation method if needed. if (!MCOpPredicates.empty()) O << "static bool " << Target.getName() << ClassName << "ValidateMCOperand(const MCOperand &MCOp,\n" @@ -1037,11 +1067,52 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { << " unsigned PredicateIndex);\n"; O << HeaderO.str(); - O.indent(2) << "const char *AsmString;\n"; - O.indent(2) << "switch (MI->getOpcode()) {\n"; - O.indent(2) << "default: return false;\n"; - O << CasesO.str(); - O.indent(2) << "}\n\n"; + O.indent(2) << "static const PatternsForOpcode OpToPatterns[] = {\n"; + O << OpcodeO.str(); + O.indent(2) << "};\n\n"; + O.indent(2) << "static const AliasPattern Patterns[] = {\n"; + O << PatternO.str(); + O.indent(2) << "};\n\n"; + O.indent(2) << "static const AliasPatternCond Conds[] = {\n"; + O << CondO.str(); + O.indent(2) << "};\n\n"; + O.indent(2) << "static const char AsmStrings[] =\n"; + for (const auto &P : AsmStrings) { + O.indent(4) << "/* " << P.first << " */ \"" << P.second << "\\0\"\n"; + } + + O.indent(2) << ";\n\n"; + + // Assert that the opcode table is sorted. Use a static local constructor to + // ensure that the check only happens once on first run. + O << "#ifndef NDEBUG\n"; + O.indent(2) << "static struct SortCheck {\n"; + O.indent(2) << " SortCheck(ArrayRef<PatternsForOpcode> OpToPatterns) {\n"; + O.indent(2) << " assert(std::is_sorted(\n"; + O.indent(2) << " OpToPatterns.begin(), OpToPatterns.end(),\n"; + O.indent(2) << " [](const PatternsForOpcode &L, const " + "PatternsForOpcode &R) {\n"; + O.indent(2) << " return L.Opcode < R.Opcode;\n"; + O.indent(2) << " }) &&\n"; + O.indent(2) << " \"tablegen failed to sort opcode patterns\");\n"; + O.indent(2) << " }\n"; + O.indent(2) << "} sortCheckVar(OpToPatterns);\n"; + O << "#endif\n\n"; + + O.indent(2) << "AliasMatchingData M {\n"; + O.indent(2) << " makeArrayRef(OpToPatterns),\n"; + O.indent(2) << " makeArrayRef(Patterns),\n"; + O.indent(2) << " makeArrayRef(Conds),\n"; + O.indent(2) << " StringRef(AsmStrings, array_lengthof(AsmStrings)),\n"; + if (MCOpPredicates.empty()) + O.indent(2) << " nullptr,\n"; + else + O.indent(2) << " &" << Target.getName() << ClassName << "ValidateMCOperand,\n"; + O.indent(2) << "};\n"; + + O.indent(2) << "const char *AsmString = matchAliasPatterns(MI, " + << (PassSubtarget ? "&STI" : "nullptr") << ", M);\n"; + O.indent(2) << "if (!AsmString) return false;\n\n"; // Code that prints the alias, replacing the operands with the ones from the // MCInst. diff --git a/contrib/llvm-project/llvm/utils/TableGen/AsmWriterInst.h b/contrib/llvm-project/llvm/utils/TableGen/AsmWriterInst.h index 7d88e5a9d037..a59112efea44 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/AsmWriterInst.h +++ b/contrib/llvm-project/llvm/utils/TableGen/AsmWriterInst.h @@ -36,7 +36,7 @@ namespace llvm { /// MiOpNo - For isMachineInstrOperand, this is the operand number of the /// machine instruction. - unsigned MIOpNo; + unsigned MIOpNo = 0; /// Str - For isLiteralTextOperand, this IS the literal text. For /// isMachineInstrOperand, this is the PrinterMethodName for the operand.. diff --git a/contrib/llvm-project/llvm/utils/TableGen/CallingConvEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/CallingConvEmitter.cpp index de5044e24d49..9eabb44d9004 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CallingConvEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/CallingConvEmitter.cpp @@ -264,6 +264,10 @@ void CallingConvEmitter::EmitAction(Record *Action, Record *DestTy = Action->getValueAsDef("DestTy"); O << IndentStr << "LocVT = " << getEnumName(getValueType(DestTy)) <<";\n"; O << IndentStr << "LocInfo = CCValAssign::BCvt;\n"; + } else if (Action->isSubClassOf("CCTruncToType")) { + Record *DestTy = Action->getValueAsDef("DestTy"); + O << IndentStr << "LocVT = " << getEnumName(getValueType(DestTy)) <<";\n"; + O << IndentStr << "LocInfo = CCValAssign::Trunc;\n"; } else if (Action->isSubClassOf("CCPassIndirect")) { Record *DestTy = Action->getValueAsDef("DestTy"); O << IndentStr << "LocVT = " << getEnumName(getValueType(DestTy)) <<";\n"; diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeEmitterGen.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeEmitterGen.cpp index da65763905a8..68cb8f181e62 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeEmitterGen.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeEmitterGen.cpp @@ -16,6 +16,7 @@ #include "CodeGenTarget.h" #include "SubtargetFeatureInfo.h" #include "Types.h" +#include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/Casting.h" @@ -45,12 +46,19 @@ public: private: int getVariableBit(const std::string &VarName, BitsInit *BI, int bit); std::string getInstructionCase(Record *R, CodeGenTarget &Target); + std::string getInstructionCaseForEncoding(Record *R, Record *EncodingDef, + CodeGenTarget &Target); void AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName, unsigned &NumberedOp, std::set<unsigned> &NamedOpIndices, std::string &Case, CodeGenTarget &Target); + void emitInstructionBaseValues( + raw_ostream &o, ArrayRef<const CodeGenInstruction *> NumberedInstructions, + CodeGenTarget &Target, int HwMode = -1); + unsigned BitWidth; + bool UseAPInt; }; // If the VarBitInit at position 'bit' matches the specified variable then @@ -126,7 +134,10 @@ AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName, std::pair<unsigned, unsigned> SO = CGI.Operands.getSubOperandNumber(OpIdx); std::string &EncoderMethodName = CGI.Operands[SO.first].EncoderMethodName; - + + if (UseAPInt) + Case += " op.clearAllBits();\n"; + // If the source operand has a custom encoder, use it. This will // get the encoding for all of the suboperands. if (!EncoderMethodName.empty()) { @@ -134,18 +145,54 @@ AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName, // sub-operands, if there are more than one, so only // query the encoder once per source operand. if (SO.second == 0) { - Case += " // op: " + VarName + "\n" + - " op = " + EncoderMethodName + "(MI, " + utostr(OpIdx); - Case += ", Fixups, STI"; - Case += ");\n"; + Case += " // op: " + VarName + "\n"; + if (UseAPInt) { + Case += " " + EncoderMethodName + "(MI, " + utostr(OpIdx); + Case += ", op"; + } else { + Case += " op = " + EncoderMethodName + "(MI, " + utostr(OpIdx); + } + Case += ", Fixups, STI);\n"; } } else { - Case += " // op: " + VarName + "\n" + - " op = getMachineOpValue(MI, MI.getOperand(" + utostr(OpIdx) + ")"; - Case += ", Fixups, STI"; + Case += " // op: " + VarName + "\n"; + if (UseAPInt) { + Case += " getMachineOpValue(MI, MI.getOperand(" + utostr(OpIdx) + ")"; + Case += ", op, Fixups, STI"; + } else { + Case += " op = getMachineOpValue(MI, MI.getOperand(" + utostr(OpIdx) + ")"; + Case += ", Fixups, STI"; + } Case += ");\n"; } - + + // Precalculate the number of lits this variable contributes to in the + // operand. If there is a single lit (consecutive range of bits) we can use a + // destructive sequence on APInt that reduces memory allocations. + int numOperandLits = 0; + for (int tmpBit = bit; tmpBit >= 0;) { + int varBit = getVariableBit(VarName, BI, tmpBit); + + // If this bit isn't from a variable, skip it. + if (varBit == -1) { + --tmpBit; + continue; + } + + // Figure out the consecutive range of bits covered by this operand, in + // order to generate better encoding code. + int beginVarBit = varBit; + int N = 1; + for (--tmpBit; tmpBit >= 0;) { + varBit = getVariableBit(VarName, BI, tmpBit); + if (varBit == -1 || varBit != (beginVarBit - N)) + break; + ++N; + --tmpBit; + } + ++numOperandLits; + } + for (; bit >= 0; ) { int varBit = getVariableBit(VarName, BI, bit); @@ -166,20 +213,52 @@ AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName, ++N; --bit; } - - uint64_t opMask = ~(uint64_t)0 >> (64-N); - int opShift = beginVarBit - N + 1; - opMask <<= opShift; - opShift = beginInstBit - beginVarBit; - - if (opShift > 0) { - Case += " Value |= (op & UINT64_C(" + utostr(opMask) + ")) << " + - itostr(opShift) + ";\n"; - } else if (opShift < 0) { - Case += " Value |= (op & UINT64_C(" + utostr(opMask) + ")) >> " + - itostr(-opShift) + ";\n"; + + std::string maskStr; + int opShift; + + unsigned loBit = beginVarBit - N + 1; + unsigned hiBit = loBit + N; + unsigned loInstBit = beginInstBit - N + 1; + if (UseAPInt) { + std::string extractStr; + if (N >= 64) { + extractStr = "op.extractBits(" + itostr(hiBit - loBit) + ", " + + itostr(loBit) + ")"; + Case += " Value.insertBits(" + extractStr + ", " + + itostr(loInstBit) + ");\n"; + } else { + extractStr = "op.extractBitsAsZExtValue(" + itostr(hiBit - loBit) + + ", " + itostr(loBit) + ")"; + Case += " Value.insertBits(" + extractStr + ", " + + itostr(loInstBit) + ", " + itostr(hiBit - loBit) + ");\n"; + } } else { - Case += " Value |= op & UINT64_C(" + utostr(opMask) + ");\n"; + uint64_t opMask = ~(uint64_t)0 >> (64 - N); + opShift = beginVarBit - N + 1; + opMask <<= opShift; + maskStr = "UINT64_C(" + utostr(opMask) + ")"; + opShift = beginInstBit - beginVarBit; + + if (numOperandLits == 1) { + Case += " op &= " + maskStr + ";\n"; + if (opShift > 0) { + Case += " op <<= " + itostr(opShift) + ";\n"; + } else if (opShift < 0) { + Case += " op >>= " + itostr(-opShift) + ";\n"; + } + Case += " Value |= op;\n"; + } else { + if (opShift > 0) { + Case += " Value |= (op & " + maskStr + ") << " + + itostr(opShift) + ";\n"; + } else if (opShift < 0) { + Case += " Value |= (op & " + maskStr + ") >> " + + itostr(-opShift) + ";\n"; + } else { + Case += " Value |= (op & " + maskStr + ");\n"; + } + } } } } @@ -187,7 +266,29 @@ AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName, std::string CodeEmitterGen::getInstructionCase(Record *R, CodeGenTarget &Target) { std::string Case; - BitsInit *BI = R->getValueAsBitsInit("Inst"); + if (const RecordVal *RV = R->getValue("EncodingInfos")) { + if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + const CodeGenHwModes &HWM = Target.getHwModes(); + EncodingInfoByHwMode EBM(DI->getDef(), HWM); + Case += " switch (HwMode) {\n"; + Case += " default: llvm_unreachable(\"Unhandled HwMode\");\n"; + for (auto &KV : EBM.Map) { + Case += " case " + itostr(KV.first) + ": {\n"; + Case += getInstructionCaseForEncoding(R, KV.second, Target); + Case += " break;\n"; + Case += " }\n"; + } + Case += " }\n"; + return Case; + } + } + return getInstructionCaseForEncoding(R, R, Target); +} + +std::string CodeEmitterGen::getInstructionCaseForEncoding(Record *R, Record *EncodingDef, + CodeGenTarget &Target) { + std::string Case; + BitsInit *BI = EncodingDef->getValueAsBitsInit("Inst"); unsigned NumberedOp = 0; std::set<unsigned> NamedOpIndices; @@ -207,7 +308,7 @@ std::string CodeEmitterGen::getInstructionCase(Record *R, // Loop over all of the fields in the instruction, determining which are the // operands to the instruction. - for (const RecordVal &RV : R->getValues()) { + for (const RecordVal &RV : EncodingDef->getValues()) { // Ignore fixed fields in the record, we're looking for values like: // bits<5> RST = { ?, ?, ?, ?, ? }; if (RV.getPrefix() || RV.getValue()->isComplete()) @@ -237,6 +338,54 @@ getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset) { return Name; } +static void emitInstBits(raw_ostream &OS, const APInt &Bits) { + for (unsigned I = 0; I < Bits.getNumWords(); ++I) + OS << ((I > 0) ? ", " : "") << "UINT64_C(" << utostr(Bits.getRawData()[I]) + << ")"; +} + +void CodeEmitterGen::emitInstructionBaseValues( + raw_ostream &o, ArrayRef<const CodeGenInstruction *> NumberedInstructions, + CodeGenTarget &Target, int HwMode) { + const CodeGenHwModes &HWM = Target.getHwModes(); + if (HwMode == -1) + o << " static const uint64_t InstBits[] = {\n"; + else + o << " static const uint64_t InstBits_" << HWM.getMode(HwMode).Name + << "[] = {\n"; + + for (const CodeGenInstruction *CGI : NumberedInstructions) { + Record *R = CGI->TheDef; + + if (R->getValueAsString("Namespace") == "TargetOpcode" || + R->getValueAsBit("isPseudo")) { + o << " "; emitInstBits(o, APInt(BitWidth, 0)); o << ",\n"; + continue; + } + + Record *EncodingDef = R; + if (const RecordVal *RV = R->getValue("EncodingInfos")) { + if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + EncodingInfoByHwMode EBM(DI->getDef(), HWM); + if (EBM.hasMode(HwMode)) + EncodingDef = EBM.get(HwMode); + } + } + BitsInit *BI = EncodingDef->getValueAsBitsInit("Inst"); + + // Start by filling in fixed values. + APInt Value(BitWidth, 0); + for (unsigned i = 0, e = BI->getNumBits(); i != e; ++i) { + if (BitInit *B = dyn_cast<BitInit>(BI->getBit(e - i - 1))) + Value |= APInt(BitWidth, (uint64_t)B->getValue()) << (e - i - 1); + } + o << " "; + emitInstBits(o, Value); + o << "," << '\t' << "// " << R->getName() << "\n"; + } + o << " UINT64_C(0)\n };\n"; +} + void CodeEmitterGen::run(raw_ostream &o) { CodeGenTarget Target(Records); std::vector<Record*> Insts = Records.getAllDerivedDefinitions("Instruction"); @@ -247,34 +396,66 @@ void CodeEmitterGen::run(raw_ostream &o) { ArrayRef<const CodeGenInstruction*> NumberedInstructions = Target.getInstructionsByEnumValue(); - // Emit function declaration - o << "uint64_t " << Target.getName(); - o << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n" - << " SmallVectorImpl<MCFixup> &Fixups,\n" - << " const MCSubtargetInfo &STI) const {\n"; - - // Emit instruction base values - o << " static const uint64_t InstBits[] = {\n"; + const CodeGenHwModes &HWM = Target.getHwModes(); + // The set of HwModes used by instruction encodings. + std::set<unsigned> HwModes; + BitWidth = 0; for (const CodeGenInstruction *CGI : NumberedInstructions) { Record *R = CGI->TheDef; - if (R->getValueAsString("Namespace") == "TargetOpcode" || - R->getValueAsBit("isPseudo")) { - o << " UINT64_C(0),\n"; + R->getValueAsBit("isPseudo")) continue; - } + if (const RecordVal *RV = R->getValue("EncodingInfos")) { + if (DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + EncodingInfoByHwMode EBM(DI->getDef(), HWM); + for (auto &KV : EBM.Map) { + BitsInit *BI = KV.second->getValueAsBitsInit("Inst"); + BitWidth = std::max(BitWidth, BI->getNumBits()); + HwModes.insert(KV.first); + } + continue; + } + } BitsInit *BI = R->getValueAsBitsInit("Inst"); + BitWidth = std::max(BitWidth, BI->getNumBits()); + } + UseAPInt = BitWidth > 64; + + // Emit function declaration + if (UseAPInt) { + o << "void " << Target.getName() + << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n" + << " SmallVectorImpl<MCFixup> &Fixups,\n" + << " APInt &Inst,\n" + << " APInt &Scratch,\n" + << " const MCSubtargetInfo &STI) const {\n"; + } else { + o << "uint64_t " << Target.getName(); + o << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n" + << " SmallVectorImpl<MCFixup> &Fixups,\n" + << " const MCSubtargetInfo &STI) const {\n"; + } + + // Emit instruction base values + if (HwModes.empty()) { + emitInstructionBaseValues(o, NumberedInstructions, Target, -1); + } else { + for (unsigned HwMode : HwModes) + emitInstructionBaseValues(o, NumberedInstructions, Target, (int)HwMode); + } - // Start by filling in fixed values. - uint64_t Value = 0; - for (unsigned i = 0, e = BI->getNumBits(); i != e; ++i) { - if (BitInit *B = dyn_cast<BitInit>(BI->getBit(e-i-1))) - Value |= (uint64_t)B->getValue() << (e-i-1); + if (!HwModes.empty()) { + o << " const uint64_t *InstBits;\n"; + o << " unsigned HwMode = STI.getHwMode();\n"; + o << " switch (HwMode) {\n"; + o << " default: llvm_unreachable(\"Unknown hardware mode!\"); break;\n"; + for (unsigned I : HwModes) { + o << " case " << I << ": InstBits = InstBits_" << HWM.getMode(I).Name + << "; break;\n"; } - o << " UINT64_C(" << Value << ")," << '\t' << "// " << R->getName() << "\n"; + o << " };\n"; } - o << " UINT64_C(0)\n };\n"; // Map to accumulate all the cases. std::map<std::string, std::vector<std::string>> CaseMap; @@ -294,11 +475,26 @@ void CodeEmitterGen::run(raw_ostream &o) { } // Emit initial function code - o << " const unsigned opcode = MI.getOpcode();\n" - << " uint64_t Value = InstBits[opcode];\n" - << " uint64_t op = 0;\n" - << " (void)op; // suppress warning\n" - << " switch (opcode) {\n"; + if (UseAPInt) { + int NumWords = APInt::getNumWords(BitWidth); + int NumBytes = (BitWidth + 7) / 8; + o << " const unsigned opcode = MI.getOpcode();\n" + << " if (Inst.getBitWidth() != " << BitWidth << ")\n" + << " Inst = Inst.zext(" << BitWidth << ");\n" + << " if (Scratch.getBitWidth() != " << BitWidth << ")\n" + << " Scratch = Scratch.zext(" << BitWidth << ");\n" + << " LoadIntFromMemory(Inst, (uint8_t*)&InstBits[opcode * " << NumWords + << "], " << NumBytes << ");\n" + << " APInt &Value = Inst;\n" + << " APInt &op = Scratch;\n" + << " switch (opcode) {\n"; + } else { + o << " const unsigned opcode = MI.getOpcode();\n" + << " uint64_t Value = InstBits[opcode];\n" + << " uint64_t op = 0;\n" + << " (void)op; // suppress warning\n" + << " switch (opcode) {\n"; + } // Emit each case statement std::map<std::string, std::vector<std::string>>::iterator IE, EE; @@ -322,9 +518,12 @@ void CodeEmitterGen::run(raw_ostream &o) { << " raw_string_ostream Msg(msg);\n" << " Msg << \"Not supported instr: \" << MI;\n" << " report_fatal_error(Msg.str());\n" - << " }\n" - << " return Value;\n" - << "}\n\n"; + << " }\n"; + if (UseAPInt) + o << " Inst = Value;\n"; + else + o << " return Value;\n"; + o << "}\n\n"; const auto &All = SubtargetFeatureInfo::getAll(Records); std::map<Record *, SubtargetFeatureInfo, LessRecordByID> SubtargetFeatures; @@ -364,7 +563,7 @@ void CodeEmitterGen::run(raw_ostream &o) { return true; if (A.size() > B.size()) return false; - for (const auto &Pair : zip(A, B)) { + for (auto Pair : zip(A, B)) { if (std::get<0>(Pair)->getName() < std::get<1>(Pair)->getName()) return true; if (std::get<0>(Pair)->getName() > std::get<1>(Pair)->getName()) @@ -385,8 +584,8 @@ void CodeEmitterGen::run(raw_ostream &o) { o << " " << getNameForFeatureBitset(FeatureBitset) << ",\n"; } o << "};\n\n" - << "const static FeatureBitset FeatureBitsets[] {\n" - << " {}, // CEFBS_None\n"; + << "static constexpr FeatureBitset FeatureBitsets[] = {\n" + << " {}, // CEFBS_None\n"; for (const auto &FeatureBitset : FeatureBitsets) { if (FeatureBitset.empty()) continue; diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.cpp index c8f710d66a03..7e0ba98da94a 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.cpp @@ -23,6 +23,7 @@ #include "llvm/ADT/Twine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/TypeSize.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" #include <algorithm> @@ -481,12 +482,12 @@ bool TypeInfer::EnforceSmallerThan(TypeSetByHwMode &Small, if (any_of(S, isIntegerOrPtr) && any_of(S, isIntegerOrPtr)) { auto NotInt = [](MVT VT) { return !isIntegerOrPtr(VT); }; - Changed |= berase_if(S, NotInt) | - berase_if(B, NotInt); + Changed |= berase_if(S, NotInt); + Changed |= berase_if(B, NotInt); } else if (any_of(S, isFloatingPoint) && any_of(B, isFloatingPoint)) { auto NotFP = [](MVT VT) { return !isFloatingPoint(VT); }; - Changed |= berase_if(S, NotFP) | - berase_if(B, NotFP); + Changed |= berase_if(S, NotFP); + Changed |= berase_if(B, NotFP); } else if (S.empty() || B.empty()) { Changed = !S.empty() || !B.empty(); S.clear(); @@ -497,24 +498,30 @@ bool TypeInfer::EnforceSmallerThan(TypeSetByHwMode &Small, } if (none_of(S, isVector) || none_of(B, isVector)) { - Changed |= berase_if(S, isVector) | - berase_if(B, isVector); + Changed |= berase_if(S, isVector); + Changed |= berase_if(B, isVector); } } auto LT = [](MVT A, MVT B) -> bool { - return A.getScalarSizeInBits() < B.getScalarSizeInBits() || - (A.getScalarSizeInBits() == B.getScalarSizeInBits() && - A.getSizeInBits() < B.getSizeInBits()); + // Always treat non-scalable MVTs as smaller than scalable MVTs for the + // purposes of ordering. + auto ASize = std::make_tuple(A.isScalableVector(), A.getScalarSizeInBits(), + A.getSizeInBits()); + auto BSize = std::make_tuple(B.isScalableVector(), B.getScalarSizeInBits(), + B.getSizeInBits()); + return ASize < BSize; }; - auto LE = [<](MVT A, MVT B) -> bool { + auto SameKindLE = [](MVT A, MVT B) -> bool { // This function is used when removing elements: when a vector is compared - // to a non-vector, it should return false (to avoid removal). - if (A.isVector() != B.isVector()) + // to a non-vector or a scalable vector to any non-scalable MVT, it should + // return false (to avoid removal). + if (std::make_tuple(A.isVector(), A.isScalableVector()) != + std::make_tuple(B.isVector(), B.isScalableVector())) return false; - return LT(A, B) || (A.getScalarSizeInBits() == B.getScalarSizeInBits() && - A.getSizeInBits() == B.getSizeInBits()); + return std::make_tuple(A.getScalarSizeInBits(), A.getSizeInBits()) <= + std::make_tuple(B.getScalarSizeInBits(), B.getSizeInBits()); }; for (unsigned M : Modes) { @@ -524,25 +531,29 @@ bool TypeInfer::EnforceSmallerThan(TypeSetByHwMode &Small, // smaller-or-equal than MinS. auto MinS = min_if(S.begin(), S.end(), isScalar, LT); if (MinS != S.end()) - Changed |= berase_if(B, std::bind(LE, std::placeholders::_1, *MinS)); + Changed |= berase_if(B, std::bind(SameKindLE, + std::placeholders::_1, *MinS)); // MaxS = max scalar in Big, remove all scalars from Small that are // larger than MaxS. auto MaxS = max_if(B.begin(), B.end(), isScalar, LT); if (MaxS != B.end()) - Changed |= berase_if(S, std::bind(LE, *MaxS, std::placeholders::_1)); + Changed |= berase_if(S, std::bind(SameKindLE, + *MaxS, std::placeholders::_1)); // MinV = min vector in Small, remove all vectors from Big that are // smaller-or-equal than MinV. auto MinV = min_if(S.begin(), S.end(), isVector, LT); if (MinV != S.end()) - Changed |= berase_if(B, std::bind(LE, std::placeholders::_1, *MinV)); + Changed |= berase_if(B, std::bind(SameKindLE, + std::placeholders::_1, *MinV)); // MaxV = max vector in Big, remove all vectors from Small that are // larger than MaxV. auto MaxV = max_if(B.begin(), B.end(), isVector, LT); if (MaxV != B.end()) - Changed |= berase_if(S, std::bind(LE, *MaxV, std::placeholders::_1)); + Changed |= berase_if(S, std::bind(SameKindLE, + *MaxV, std::placeholders::_1)); } return Changed; @@ -625,7 +636,7 @@ bool TypeInfer::EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec, /// Return true if S has no element (vector type) that T is a sub-vector of, /// i.e. has the same element type as T and more elements. auto NoSubV = [&IsSubVec](const TypeSetByHwMode::SetType &S, MVT T) -> bool { - for (const auto &I : S) + for (auto I : S) if (IsSubVec(T, I)) return false; return true; @@ -634,7 +645,7 @@ bool TypeInfer::EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec, /// Return true if S has no element (vector type) that T is a super-vector /// of, i.e. has the same element type as T and fewer elements. auto NoSupV = [&IsSubVec](const TypeSetByHwMode::SetType &S, MVT T) -> bool { - for (const auto &I : S) + for (auto I : S) if (IsSubVec(I, T)) return false; return true; @@ -769,7 +780,10 @@ void TypeInfer::expandOverloads(TypeSetByHwMode::SetType &Out, for (MVT T : MVT::integer_valuetypes()) if (Legal.count(T)) Out.insert(T); - for (MVT T : MVT::integer_vector_valuetypes()) + for (MVT T : MVT::integer_fixedlen_vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + for (MVT T : MVT::integer_scalable_vector_valuetypes()) if (Legal.count(T)) Out.insert(T); return; @@ -777,7 +791,10 @@ void TypeInfer::expandOverloads(TypeSetByHwMode::SetType &Out, for (MVT T : MVT::fp_valuetypes()) if (Legal.count(T)) Out.insert(T); - for (MVT T : MVT::fp_vector_valuetypes()) + for (MVT T : MVT::fp_fixedlen_vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + for (MVT T : MVT::fp_scalable_vector_valuetypes()) if (Legal.count(T)) Out.insert(T); return; @@ -883,7 +900,8 @@ std::string TreePredicateFn::getPredCode() const { if (isLoad()) { if (!isUnindexed() && !isNonExtLoad() && !isAnyExtLoad() && !isSignExtLoad() && !isZeroExtLoad() && getMemoryVT() == nullptr && - getScalarMemoryVT() == nullptr) + getScalarMemoryVT() == nullptr && getAddressSpaces() == nullptr && + getMinAlignment() < 1) PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), "IsLoad cannot be used by itself"); } else { @@ -903,7 +921,8 @@ std::string TreePredicateFn::getPredCode() const { if (isStore()) { if (!isUnindexed() && !isTruncStore() && !isNonTruncStore() && - getMemoryVT() == nullptr && getScalarMemoryVT() == nullptr) + getMemoryVT() == nullptr && getScalarMemoryVT() == nullptr && + getAddressSpaces() == nullptr && getMinAlignment() < 1) PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), "IsStore cannot be used by itself"); } else { @@ -917,6 +936,7 @@ std::string TreePredicateFn::getPredCode() const { if (isAtomic()) { if (getMemoryVT() == nullptr && !isAtomicOrderingMonotonic() && + getAddressSpaces() == nullptr && !isAtomicOrderingAcquire() && !isAtomicOrderingRelease() && !isAtomicOrderingAcquireRelease() && !isAtomicOrderingSequentiallyConsistent() && @@ -977,6 +997,13 @@ std::string TreePredicateFn::getPredCode() const { Code += ")\nreturn false;\n"; } + int64_t MinAlign = getMinAlignment(); + if (MinAlign > 0) { + Code += "if (cast<MemSDNode>(N)->getAlignment() < "; + Code += utostr(MinAlign); + Code += ")\nreturn false;\n"; + } + Record *MemoryVT = getMemoryVT(); if (MemoryVT) @@ -1177,6 +1204,13 @@ ListInit *TreePredicateFn::getAddressSpaces() const { return R->getValueAsListInit("AddressSpaces"); } +int64_t TreePredicateFn::getMinAlignment() const { + Record *R = getOrigPatFragRecord()->getRecord(); + if (R->isValueUnset("MinAlignment")) + return 0; + return R->getValueAsInt("MinAlignment"); +} + Record *TreePredicateFn::getScalarMemoryVT() const { Record *R = getOrigPatFragRecord()->getRecord(); if (R->isValueUnset("ScalarMemoryVT")) @@ -1281,13 +1315,29 @@ std::string TreePredicateFn::getCodeToRunOnSDNode() const { // Handle arbitrary node predicates. assert(hasPredCode() && "Don't have any predicate code!"); + + // If this is using PatFrags, there are multiple trees to search. They should + // all have the same class. FIXME: Is there a way to find a common + // superclass? StringRef ClassName; - if (PatFragRec->getOnlyTree()->isLeaf()) - ClassName = "SDNode"; - else { - Record *Op = PatFragRec->getOnlyTree()->getOperator(); - ClassName = PatFragRec->getDAGPatterns().getSDNodeInfo(Op).getSDClassName(); + for (const auto &Tree : PatFragRec->getTrees()) { + StringRef TreeClassName; + if (Tree->isLeaf()) + TreeClassName = "SDNode"; + else { + Record *Op = Tree->getOperator(); + const SDNodeInfo &Info = PatFragRec->getDAGPatterns().getSDNodeInfo(Op); + TreeClassName = Info.getSDClassName(); + } + + if (ClassName.empty()) + ClassName = TreeClassName; + else if (ClassName != TreeClassName) { + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "PatFrags trees do not have consistent class"); + } } + std::string Result; if (ClassName == "SDNode") Result = " SDNode *N = Node;\n"; @@ -1373,9 +1423,11 @@ getPatternComplexity(const CodeGenDAGPatterns &CGP) const { /// std::string PatternToMatch::getPredicateCheck() const { SmallVector<const Predicate*,4> PredList; - for (const Predicate &P : Predicates) - PredList.push_back(&P); - llvm::sort(PredList, deref<llvm::less>()); + for (const Predicate &P : Predicates) { + if (!P.getCondString().empty()) + PredList.push_back(&P); + } + llvm::sort(PredList, deref<std::less<>>()); std::string Check; for (unsigned i = 0, e = PredList.size(); i != e; ++i) { @@ -2772,6 +2824,7 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit, if (Operator->isSubClassOf("SDNode") && Operator->getName() != "imm" && + Operator->getName() != "timm" && Operator->getName() != "fpimm" && Operator->getName() != "tglobaltlsaddr" && Operator->getName() != "tconstpool" && @@ -2991,8 +3044,7 @@ CodeGenDAGPatterns::CodeGenDAGPatterns(RecordKeeper &R, : Records(R), Target(R), LegalVTS(Target.getLegalValueTypes()), PatternRewriter(PatternRewriter) { - Intrinsics = CodeGenIntrinsicTable(Records, false); - TgtIntrinsics = CodeGenIntrinsicTable(Records, true); + Intrinsics = CodeGenIntrinsicTable(Records); ParseNodeInfo(); ParseNodeTransforms(); ParseComplexPatterns(); @@ -3083,7 +3135,7 @@ void CodeGenDAGPatterns::ParsePatternFragments(bool OutFrags) { ListInit *LI = Frag->getValueAsListInit("Fragments"); TreePattern *P = - (PatternFragments[Frag] = llvm::make_unique<TreePattern>( + (PatternFragments[Frag] = std::make_unique<TreePattern>( Frag, LI, !Frag->isSubClassOf("OutPatFrag"), *this)).get(); diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.h index 2b49a64c3f1d..2c081b670609 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.h +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.h @@ -194,6 +194,7 @@ struct TypeSetByHwMode : public InfoByHwMode<MachineValueTypeSet> { TypeSetByHwMode() = default; TypeSetByHwMode(const TypeSetByHwMode &VTS) = default; + TypeSetByHwMode &operator=(const TypeSetByHwMode &) = default; TypeSetByHwMode(MVT::SimpleValueType VT) : TypeSetByHwMode(ValueTypeByHwMode(VT)) {} TypeSetByHwMode(ValueTypeByHwMode VT) @@ -594,6 +595,7 @@ public: Record *getScalarMemoryVT() const; ListInit *getAddressSpaces() const; + int64_t getMinAlignment() const; // If true, indicates that GlobalISel-based C++ code was supplied. bool hasGISelPredicateCode() const; @@ -1075,8 +1077,11 @@ public: std::string C = IsHwMode ? std::string("MF->getSubtarget().checkFeatures(\"" + Features + "\")") : std::string(Def->getValueAsString("CondString")); + if (C.empty()) + return ""; return IfCond ? C : "!("+C+')'; } + bool operator==(const Predicate &P) const { return IfCond == P.IfCond && IsHwMode == P.IsHwMode && Def == P.Def; } @@ -1140,7 +1145,6 @@ class CodeGenDAGPatterns { RecordKeeper &Records; CodeGenTarget Target; CodeGenIntrinsicTable Intrinsics; - CodeGenIntrinsicTable TgtIntrinsics; std::map<Record*, SDNodeInfo, LessRecordByID> SDNodes; std::map<Record*, std::pair<Record*, std::string>, LessRecordByID> @@ -1191,12 +1195,6 @@ public: return F->second; } - typedef std::map<Record*, NodeXForm, LessRecordByID>::const_iterator - nx_iterator; - nx_iterator nx_begin() const { return SDNodeXForms.begin(); } - nx_iterator nx_end() const { return SDNodeXForms.end(); } - - const ComplexPattern &getComplexPattern(Record *R) const { auto F = ComplexPatterns.find(R); assert(F != ComplexPatterns.end() && "Unknown addressing mode!"); @@ -1206,24 +1204,18 @@ public: const CodeGenIntrinsic &getIntrinsic(Record *R) const { for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i) if (Intrinsics[i].TheDef == R) return Intrinsics[i]; - for (unsigned i = 0, e = TgtIntrinsics.size(); i != e; ++i) - if (TgtIntrinsics[i].TheDef == R) return TgtIntrinsics[i]; llvm_unreachable("Unknown intrinsic!"); } const CodeGenIntrinsic &getIntrinsicInfo(unsigned IID) const { if (IID-1 < Intrinsics.size()) return Intrinsics[IID-1]; - if (IID-Intrinsics.size()-1 < TgtIntrinsics.size()) - return TgtIntrinsics[IID-Intrinsics.size()-1]; llvm_unreachable("Bad intrinsic ID!"); } unsigned getIntrinsicID(Record *R) const { for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i) if (Intrinsics[i].TheDef == R) return i; - for (unsigned i = 0, e = TgtIntrinsics.size(); i != e; ++i) - if (TgtIntrinsics[i].TheDef == R) return i + Intrinsics.size(); llvm_unreachable("Unknown intrinsic!"); } @@ -1280,8 +1272,6 @@ public: return intrinsic_wo_chain_sdnode; } - bool hasTargetIntrinsics() { return !TgtIntrinsics.empty(); } - unsigned allocateScope() { return ++NumScopes; } bool operandHasDefault(Record *Op) const { diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.cpp index 2463824469ab..6bb4dbb511b6 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.cpp @@ -363,6 +363,7 @@ CodeGenInstruction::CodeGenInstruction(Record *R) Namespace = R->getValueAsString("Namespace"); AsmString = R->getValueAsString("AsmString"); + isPreISelOpcode = R->getValueAsBit("isPreISelOpcode"); isReturn = R->getValueAsBit("isReturn"); isEHScopeReturn = R->getValueAsBit("isEHScopeReturn"); isBranch = R->getValueAsBit("isBranch"); @@ -395,6 +396,7 @@ CodeGenInstruction::CodeGenInstruction(Record *R) hasNoSchedulingInfo = R->getValueAsBit("hasNoSchedulingInfo"); FastISelShouldIgnore = R->getValueAsBit("FastISelShouldIgnore"); variadicOpsAreDefs = R->getValueAsBit("variadicOpsAreDefs"); + isAuthenticated = R->getValueAsBit("isAuthenticated"); bool Unset; mayLoad = R->getValueAsBitOrUnset("mayLoad", Unset); @@ -503,16 +505,18 @@ FlattenAsmStringVariants(StringRef Cur, unsigned Variant) { return Res; } -bool CodeGenInstruction::isOperandAPointer(unsigned i) const { - if (DagInit *ConstraintList = TheDef->getValueAsDag("InOperandList")) { - if (i < ConstraintList->getNumArgs()) { - if (DefInit *Constraint = dyn_cast<DefInit>(ConstraintList->getArg(i))) { - return Constraint->getDef()->isSubClassOf("TypedOperand") && - Constraint->getDef()->getValueAsBit("IsPointer"); - } - } - } - return false; +bool CodeGenInstruction::isOperandImpl(unsigned i, + StringRef PropertyName) const { + DagInit *ConstraintList = TheDef->getValueAsDag("InOperandList"); + if (!ConstraintList || i >= ConstraintList->getNumArgs()) + return false; + + DefInit *Constraint = dyn_cast<DefInit>(ConstraintList->getArg(i)); + if (!Constraint) + return false; + + return Constraint->getDef()->isSubClassOf("TypedOperand") && + Constraint->getDef()->getValueAsBit(PropertyName); } //===----------------------------------------------------------------------===// diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.h index bb5b1369649f..1f08ce481a89 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.h +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.h @@ -29,10 +29,11 @@ template <typename T> class ArrayRef; class CGIOperandList { public: class ConstraintInfo { - enum { None, EarlyClobber, Tied } Kind; - unsigned OtherTiedOperand; + enum { None, EarlyClobber, Tied } Kind = None; + unsigned OtherTiedOperand = 0; + public: - ConstraintInfo() : Kind(None) {} + ConstraintInfo() = default; static ConstraintInfo getEarlyClobber() { ConstraintInfo I; @@ -231,6 +232,7 @@ template <typename T> class ArrayRef; std::vector<Record*> ImplicitDefs, ImplicitUses; // Various boolean values we track for the instruction. + bool isPreISelOpcode : 1; bool isReturn : 1; bool isEHScopeReturn : 1; bool isBranch : 1; @@ -276,6 +278,7 @@ template <typename T> class ArrayRef; bool hasChain : 1; bool hasChain_Inferred : 1; bool variadicOpsAreDefs : 1; + bool isAuthenticated : 1; std::string DeprecatedReason; bool HasComplexDeprecationPredicate; @@ -307,7 +310,17 @@ template <typename T> class ArrayRef; // This can be used on intructions that use typeN or ptypeN to identify // operands that should be considered as pointers even though SelectionDAG // didn't make a distinction between integer and pointers. - bool isOperandAPointer(unsigned i) const; + bool isOperandAPointer(unsigned i) const { + return isOperandImpl(i, "IsPointer"); + } + + /// Check if the operand is required to be an immediate. + bool isOperandImmArg(unsigned i) const { + return isOperandImpl(i, "IsImmediate"); + } + + private: + bool isOperandImpl(unsigned i, StringRef PropertyName) const; }; @@ -331,9 +344,9 @@ template <typename T> class ArrayRef; struct ResultOperand { private: std::string Name; - Record *R; + Record *R = nullptr; + int64_t Imm = 0; - int64_t Imm; public: enum { K_Record, diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenIntrinsics.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenIntrinsics.h index 7b74bb07d6e0..723bbe0cc23d 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenIntrinsics.h +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenIntrinsics.h @@ -141,6 +141,7 @@ struct CodeGenIntrinsic { enum ArgAttribute { NoCapture, + NoAlias, Returned, ReadOnly, WriteOnly, @@ -154,6 +155,15 @@ struct CodeGenIntrinsic { return Properties & (1 << Prop); } + /// Returns true if the parameter at \p ParamIdx is a pointer type. Returns + /// false if the parameter is not a pointer, or \p ParamIdx is greater than + /// the size of \p IS.ParamVTs. + /// + /// Note that this requires that \p IS.ParamVTs is available. + bool isParamAPointer(unsigned ParamIdx) const; + + bool isParamImmArg(unsigned ParamIdx) const; + CodeGenIntrinsic(Record *R); }; @@ -168,7 +178,7 @@ public: }; std::vector<TargetSet> Targets; - explicit CodeGenIntrinsicTable(const RecordKeeper &RC, bool TargetOnly); + explicit CodeGenIntrinsicTable(const RecordKeeper &RC); CodeGenIntrinsicTable() = default; bool empty() const { return Intrinsics.empty(); } diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenMapTable.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenMapTable.cpp index b1774b01ba8c..793bb61481e7 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenMapTable.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenMapTable.cpp @@ -132,7 +132,7 @@ public: MapRec->getName() + "' has empty " + "`ValueCols' field!"); for (Init *I : ColValList->getValues()) { - ListInit *ColI = dyn_cast<ListInit>(I); + auto *ColI = cast<ListInit>(I); // Make sure that all the sub-lists in 'ValueCols' have same number of // elements as the fields in 'ColFields'. @@ -168,7 +168,7 @@ public: return ValueCols; } }; -} // End anonymous namespace. +} // end anonymous namespace //===----------------------------------------------------------------------===// @@ -226,7 +226,7 @@ public: void emitMapFuncBody(raw_ostream &OS, unsigned TableSize); }; -} // End anonymous namespace. +} // end anonymous namespace //===----------------------------------------------------------------------===// @@ -521,7 +521,7 @@ static void emitEnums(raw_ostream &OS, RecordKeeper &Records) { unsigned ListSize = List->size(); for (unsigned j = 0; j < ListSize; j++) { - ListInit *ListJ = dyn_cast<ListInit>(List->getElement(j)); + auto *ListJ = cast<ListInit>(List->getElement(j)); if (ListJ->size() != ColFields->size()) PrintFatalError("Record `" + CurMap->getName() + "', field " @@ -604,8 +604,8 @@ void EmitMapTable(RecordKeeper &Records, raw_ostream &OS) { // Emit map tables and the functions to query them. IMap.emitTablesWithFunc(OS); } - OS << "} // End " << NameSpace << " namespace\n"; - OS << "} // End llvm namespace\n"; + OS << "} // end namespace " << NameSpace << "\n"; + OS << "} // end namespace llvm\n"; OS << "#endif // GET_INSTRMAP_INFO\n\n"; } diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.cpp index f87c6d6c945a..6153c759b123 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.cpp @@ -639,7 +639,8 @@ struct TupleExpander : SetTheory::Expander { // Precompute some types. Record *RegisterCl = Def->getRecords().getClass("Register"); RecTy *RegisterRecTy = RecordRecTy::get(RegisterCl); - StringInit *BlankName = StringInit::get(""); + std::vector<StringRef> RegNames = + Def->getValueAsListOfStrings("RegAsmNames"); // Zip them up. for (unsigned n = 0; n != Length; ++n) { @@ -656,11 +657,20 @@ struct TupleExpander : SetTheory::Expander { unsigned(Reg->getValueAsInt("CostPerUse"))); } + StringInit *AsmName = StringInit::get(""); + if (!RegNames.empty()) { + if (RegNames.size() <= n) + PrintFatalError(Def->getLoc(), + "Register tuple definition missing name for '" + + Name + "'."); + AsmName = StringInit::get(RegNames[n]); + } + // Create a new Record representing the synthesized register. This record // is only for consumption by CodeGenRegister, it is not added to the // RecordKeeper. SynthDefs.emplace_back( - llvm::make_unique<Record>(Name, Def->getLoc(), Def->getRecords())); + std::make_unique<Record>(Name, Def->getLoc(), Def->getRecords())); Record *NewReg = SynthDefs.back().get(); Elts.insert(NewReg); @@ -683,9 +693,8 @@ struct TupleExpander : SetTheory::Expander { if (Field == "SubRegs") RV.setValue(ListInit::get(Tuple, RegisterRecTy)); - // Provide a blank AsmName. MC hacks are required anyway. if (Field == "AsmName") - RV.setValue(BlankName); + RV.setValue(AsmName); // CostPerUse is aggregated from all Tuple members. if (Field == "CostPerUse") @@ -725,8 +734,8 @@ struct TupleExpander : SetTheory::Expander { //===----------------------------------------------------------------------===// static void sortAndUniqueRegisters(CodeGenRegister::Vec &M) { - llvm::sort(M, deref<llvm::less>()); - M.erase(std::unique(M.begin(), M.end(), deref<llvm::equal>()), M.end()); + llvm::sort(M, deref<std::less<>>()); + M.erase(std::unique(M.begin(), M.end(), deref<std::equal_to<>>()), M.end()); } CodeGenRegisterClass::CodeGenRegisterClass(CodeGenRegBank &RegBank, Record *R) @@ -851,7 +860,7 @@ void CodeGenRegisterClass::inheritProperties(CodeGenRegBank &RegBank) { bool CodeGenRegisterClass::contains(const CodeGenRegister *Reg) const { return std::binary_search(Members.begin(), Members.end(), Reg, - deref<llvm::less>()); + deref<std::less<>>()); } namespace llvm { @@ -887,7 +896,7 @@ static bool testSubClass(const CodeGenRegisterClass *A, return A->RSI.isSubClassOf(B->RSI) && std::includes(A->getMembers().begin(), A->getMembers().end(), B->getMembers().begin(), B->getMembers().end(), - deref<llvm::less>()); + deref<std::less<>>()); } /// Sorting predicate for register classes. This provides a topological @@ -1089,7 +1098,7 @@ CodeGenRegBank::CodeGenRegBank(RecordKeeper &Records, Sets.addFieldExpander("RegisterClass", "MemberList"); Sets.addFieldExpander("CalleeSavedRegs", "SaveList"); Sets.addExpander("RegisterTuples", - llvm::make_unique<TupleExpander>(SynthDefs)); + std::make_unique<TupleExpander>(SynthDefs)); // Read in the user-defined (named) sub-register indices. // More indices will be synthesized later. @@ -2131,9 +2140,10 @@ void CodeGenRegBank::inferCommonSubClass(CodeGenRegisterClass *RC) { const CodeGenRegister::Vec &Memb1 = RC1->getMembers(); const CodeGenRegister::Vec &Memb2 = RC2->getMembers(); CodeGenRegister::Vec Intersection; - std::set_intersection( - Memb1.begin(), Memb1.end(), Memb2.begin(), Memb2.end(), - std::inserter(Intersection, Intersection.begin()), deref<llvm::less>()); + std::set_intersection(Memb1.begin(), Memb1.end(), Memb2.begin(), + Memb2.end(), + std::inserter(Intersection, Intersection.begin()), + deref<std::less<>>()); // Skip disjoint class pairs. if (Intersection.empty()) @@ -2158,7 +2168,8 @@ void CodeGenRegBank::inferCommonSubClass(CodeGenRegisterClass *RC) { void CodeGenRegBank::inferSubClassWithSubReg(CodeGenRegisterClass *RC) { // Map SubRegIndex to set of registers in RC supporting that SubRegIndex. typedef std::map<const CodeGenSubRegIndex *, CodeGenRegister::Vec, - deref<llvm::less>> SubReg2SetMap; + deref<std::less<>>> + SubReg2SetMap; // Compute the set of registers supporting each SubRegIndex. SubReg2SetMap SRSets; @@ -2357,6 +2368,21 @@ CodeGenRegBank::getRegClassForRegister(Record *R) { return FoundRC; } +const CodeGenRegisterClass * +CodeGenRegBank::getMinimalPhysRegClass(Record *RegRecord, + ValueTypeByHwMode *VT) { + const CodeGenRegister *Reg = getReg(RegRecord); + const CodeGenRegisterClass *BestRC = nullptr; + for (const auto &RC : getRegClasses()) { + if ((!VT || RC.hasType(*VT)) && + RC.contains(Reg) && (!BestRC || BestRC->hasSubClass(&RC))) + BestRC = &RC; + } + + assert(BestRC && "Couldn't find the register class"); + return BestRC; +} + BitVector CodeGenRegBank::computeCoveredRegisters(ArrayRef<Record*> Regs) { SetVector<const CodeGenRegister*> Set; diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.h index f04a90f8fde5..a8e9e0fbccbe 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.h +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.h @@ -93,7 +93,8 @@ namespace llvm { // Map of composite subreg indices. typedef std::map<CodeGenSubRegIndex *, CodeGenSubRegIndex *, - deref<llvm::less>> CompMap; + deref<std::less<>>> + CompMap; // Returns the subreg index that results from composing this with Idx. // Returns NULL if this and Idx don't compose. @@ -137,15 +138,14 @@ namespace llvm { /// list of subregisters they are composed of (if any). Do this recursively. void computeConcatTransitiveClosure(); + bool operator<(const CodeGenSubRegIndex &RHS) const { + return this->EnumValue < RHS.EnumValue; + } + private: CompMap Composed; }; - inline bool operator<(const CodeGenSubRegIndex &A, - const CodeGenSubRegIndex &B) { - return A.EnumValue < B.EnumValue; - } - /// CodeGenRegister - Represents a register definition. struct CodeGenRegister { Record *TheDef; @@ -156,7 +156,8 @@ namespace llvm { bool Artificial; // Map SubRegIndex -> Register. - typedef std::map<CodeGenSubRegIndex *, CodeGenRegister *, deref<llvm::less>> + typedef std::map<CodeGenSubRegIndex *, CodeGenRegister *, + deref<std::less<>>> SubRegMap; CodeGenRegister(Record *R, unsigned Enum); @@ -347,6 +348,10 @@ namespace llvm { ArrayRef<ValueTypeByHwMode> getValueTypes() const { return VTs; } unsigned getNumValueTypes() const { return VTs.size(); } + bool hasType(const ValueTypeByHwMode &VT) const { + return std::find(VTs.begin(), VTs.end(), VT) != VTs.end(); + } + const ValueTypeByHwMode &getValueTypeNum(unsigned VTNum) const { if (VTNum < VTs.size()) return VTs[VTNum]; @@ -630,9 +635,11 @@ namespace llvm { CodeGenSubRegIndex * getConcatSubRegIndex(const SmallVector<CodeGenSubRegIndex *, 8>&); - const std::deque<CodeGenRegister> &getRegisters() { return Registers; } + const std::deque<CodeGenRegister> &getRegisters() const { + return Registers; + } - const StringMap<CodeGenRegister*> &getRegistersByName() { + const StringMap<CodeGenRegister *> &getRegistersByName() const { return RegistersByName; } @@ -681,7 +688,7 @@ namespace llvm { // Native units are the singular unit of a leaf register. Register aliasing // is completely characterized by native units. Adopted units exist to give // register additional weight but don't affect aliasing. - bool isNativeUnit(unsigned RUID) { + bool isNativeUnit(unsigned RUID) const { return RUID < NumNativeRegUnits; } @@ -708,6 +715,13 @@ namespace llvm { /// return the superclass. Otherwise return null. const CodeGenRegisterClass* getRegClassForRegister(Record *R); + // Analog of TargetRegisterInfo::getMinimalPhysRegClass. Unlike + // getRegClassForRegister, this tries to find the smallest class containing + // the physical register. If \p VT is specified, it will only find classes + // with a matching type + const CodeGenRegisterClass * + getMinimalPhysRegClass(Record *RegRecord, ValueTypeByHwMode *VT = nullptr); + // Get the sum of unit weights. unsigned getRegUnitSetWeight(const std::vector<unsigned> &Units) const { unsigned Weight = 0; diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenSchedule.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenSchedule.cpp index fd007044a16e..f12d7d484a8e 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenSchedule.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenSchedule.cpp @@ -172,8 +172,8 @@ CodeGenSchedModels::CodeGenSchedModels(RecordKeeper &RK, // Allow Set evaluation to recognize the dags used in InstRW records: // (instrs Op1, Op1...) - Sets.addOperator("instrs", llvm::make_unique<InstrsOp>()); - Sets.addOperator("instregex", llvm::make_unique<InstRegexOp>(Target)); + Sets.addOperator("instrs", std::make_unique<InstrsOp>()); + Sets.addOperator("instregex", std::make_unique<InstRegexOp>(Target)); // Instantiate a CodeGenProcModel for each SchedMachineModel with the values // that are explicitly referenced in tablegen records. Resources associated @@ -1083,9 +1083,13 @@ void CodeGenSchedModels::createInstRWClass(Record *InstRWDef) { if (RWD->getValueAsDef("SchedModel") == RWModelDef && RWModelDef->getValueAsBit("FullInstRWOverlapCheck")) { for (Record *Inst : InstDefs) { - PrintFatalError(InstRWDef->getLoc(), "Overlapping InstRW def " + - Inst->getName() + " also matches " + - RWD->getValue("Instrs")->getValue()->getAsString()); + PrintFatalError + (InstRWDef->getLoc(), + "Overlapping InstRW definition for \"" + + Inst->getName() + + "\" also matches previous \"" + + RWD->getValue("Instrs")->getValue()->getAsString() + + "\"."); } } } @@ -1115,9 +1119,13 @@ void CodeGenSchedModels::createInstRWClass(Record *InstRWDef) { for (Record *OldRWDef : SchedClasses[OldSCIdx].InstRWs) { if (OldRWDef->getValueAsDef("SchedModel") == RWModelDef) { for (Record *InstDef : InstDefs) { - PrintFatalError(OldRWDef->getLoc(), "Overlapping InstRW def " + - InstDef->getName() + " also matches " + - OldRWDef->getValue("Instrs")->getValue()->getAsString()); + PrintFatalError + (InstRWDef->getLoc(), + "Overlapping InstRW definition for \"" + + InstDef->getName() + + "\" also matches previous \"" + + OldRWDef->getValue("Instrs")->getValue()->getAsString() + + "\"."); } } assert(OldRWDef != InstRWDef && diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.cpp index b65e1b6af791..acfb143120af 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.cpp @@ -98,6 +98,7 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) { case MVT::v256i8: return "MVT::v256i8"; case MVT::v1i16: return "MVT::v1i16"; case MVT::v2i16: return "MVT::v2i16"; + case MVT::v3i16: return "MVT::v3i16"; case MVT::v4i16: return "MVT::v4i16"; case MVT::v8i16: return "MVT::v8i16"; case MVT::v16i16: return "MVT::v16i16"; @@ -126,8 +127,11 @@ StringRef llvm::getEnumName(MVT::SimpleValueType T) { case MVT::v32i64: return "MVT::v32i64"; case MVT::v1i128: return "MVT::v1i128"; case MVT::v2f16: return "MVT::v2f16"; + case MVT::v3f16: return "MVT::v3f16"; case MVT::v4f16: return "MVT::v4f16"; case MVT::v8f16: return "MVT::v8f16"; + case MVT::v16f16: return "MVT::v16f16"; + case MVT::v32f16: return "MVT::v32f16"; case MVT::v1f32: return "MVT::v1f32"; case MVT::v2f32: return "MVT::v2f32"; case MVT::v3f32: return "MVT::v3f32"; @@ -256,7 +260,7 @@ Record *CodeGenTarget::getAsmParser() const { return LI[AsmParserNum]; } -/// getAsmParserVariant - Return the AssmblyParserVariant definition for +/// getAsmParserVariant - Return the AssemblyParserVariant definition for /// this target. /// Record *CodeGenTarget::getAsmParserVariant(unsigned i) const { @@ -268,7 +272,7 @@ Record *CodeGenTarget::getAsmParserVariant(unsigned i) const { return LI[i]; } -/// getAsmParserVariantCount - Return the AssmblyParserVariant definition +/// getAsmParserVariantCount - Return the AssemblyParserVariant definition /// available for this target. /// unsigned CodeGenTarget::getAsmParserVariantCount() const { @@ -289,10 +293,57 @@ Record *CodeGenTarget::getAsmWriter() const { CodeGenRegBank &CodeGenTarget::getRegBank() const { if (!RegBank) - RegBank = llvm::make_unique<CodeGenRegBank>(Records, getHwModes()); + RegBank = std::make_unique<CodeGenRegBank>(Records, getHwModes()); return *RegBank; } +Optional<CodeGenRegisterClass *> +CodeGenTarget::getSuperRegForSubReg(const ValueTypeByHwMode &ValueTy, + CodeGenRegBank &RegBank, + const CodeGenSubRegIndex *SubIdx) const { + std::vector<CodeGenRegisterClass *> Candidates; + auto &RegClasses = RegBank.getRegClasses(); + + // Try to find a register class which supports ValueTy, and also contains + // SubIdx. + for (CodeGenRegisterClass &RC : RegClasses) { + // Is there a subclass of this class which contains this subregister index? + CodeGenRegisterClass *SubClassWithSubReg = RC.getSubClassWithSubReg(SubIdx); + if (!SubClassWithSubReg) + continue; + + // We have a class. Check if it supports this value type. + if (llvm::none_of(SubClassWithSubReg->VTs, + [&ValueTy](const ValueTypeByHwMode &ClassVT) { + return ClassVT == ValueTy; + })) + continue; + + // We have a register class which supports both the value type and + // subregister index. Remember it. + Candidates.push_back(SubClassWithSubReg); + } + + // If we didn't find anything, we're done. + if (Candidates.empty()) + return None; + + // Find and return the largest of our candidate classes. + llvm::stable_sort(Candidates, [&](const CodeGenRegisterClass *A, + const CodeGenRegisterClass *B) { + if (A->getMembers().size() > B->getMembers().size()) + return true; + + if (A->getMembers().size() < B->getMembers().size()) + return false; + + // Order by name as a tie-breaker. + return StringRef(A->getName()) < B->getName(); + }); + + return Candidates[0]; +} + void CodeGenTarget::ReadRegAltNameIndices() const { RegAltNameIndices = Records.getAllDerivedDefinitions("RegAltNameIndex"); llvm::sort(RegAltNameIndices, LessRecord()); @@ -339,7 +390,7 @@ void CodeGenTarget::ReadLegalValueTypes() const { CodeGenSchedModels &CodeGenTarget::getSchedModels() const { if (!SchedModels) - SchedModels = llvm::make_unique<CodeGenSchedModels>(Records, *this); + SchedModels = std::make_unique<CodeGenSchedModels>(Records, *this); return *SchedModels; } @@ -352,7 +403,7 @@ void CodeGenTarget::ReadInstructions() const { // Parse the instructions defined in the .td file. for (unsigned i = 0, e = Insts.size(); i != e; ++i) - Instructions[Insts[i]] = llvm::make_unique<CodeGenInstruction>(Insts[i]); + Instructions[Insts[i]] = std::make_unique<CodeGenInstruction>(Insts[i]); } static const CodeGenInstruction * @@ -427,7 +478,8 @@ void CodeGenTarget::reverseBitsForLittleEndianEncoding() { if (!isLittleEndianEncoding()) return; - std::vector<Record*> Insts = Records.getAllDerivedDefinitions("Instruction"); + std::vector<Record *> Insts = + Records.getAllDerivedDefinitions("InstructionEncoding"); for (Record *R : Insts) { if (R->getValueAsString("Namespace") == "TargetOpcode" || R->getValueAsBit("isPseudo")) @@ -522,17 +574,14 @@ ComplexPattern::ComplexPattern(Record *R) { // CodeGenIntrinsic Implementation //===----------------------------------------------------------------------===// -CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC, - bool TargetOnly) { +CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) { std::vector<Record*> Defs = RC.getAllDerivedDefinitions("Intrinsic"); Intrinsics.reserve(Defs.size()); - for (unsigned I = 0, e = Defs.size(); I != e; ++I) { - bool isTarget = Defs[I]->getValueAsBit("isTarget"); - if (isTarget == TargetOnly) - Intrinsics.push_back(CodeGenIntrinsic(Defs[I])); - } + for (unsigned I = 0, e = Defs.size(); I != e; ++I) + Intrinsics.push_back(CodeGenIntrinsic(Defs[I])); + llvm::sort(Intrinsics, [](const CodeGenIntrinsic &LHS, const CodeGenIntrinsic &RHS) { return std::tie(LHS.TargetPrefix, LHS.Name) < @@ -733,6 +782,9 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) { else if (Property->isSubClassOf("NoCapture")) { unsigned ArgNo = Property->getValueAsInt("ArgNo"); ArgumentAttributes.push_back(std::make_pair(ArgNo, NoCapture)); + } else if (Property->isSubClassOf("NoAlias")) { + unsigned ArgNo = Property->getValueAsInt("ArgNo"); + ArgumentAttributes.push_back(std::make_pair(ArgNo, NoAlias)); } else if (Property->isSubClassOf("Returned")) { unsigned ArgNo = Property->getValueAsInt("ArgNo"); ArgumentAttributes.push_back(std::make_pair(ArgNo, Returned)); @@ -758,3 +810,16 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) { // Sort the argument attributes for later benefit. llvm::sort(ArgumentAttributes); } + +bool CodeGenIntrinsic::isParamAPointer(unsigned ParamIdx) const { + if (ParamIdx >= IS.ParamVTs.size()) + return false; + MVT ParamType = MVT(IS.ParamVTs[ParamIdx]); + return ParamType == MVT::iPTR || ParamType == MVT::iPTRAny; +} + +bool CodeGenIntrinsic::isParamImmArg(unsigned ParamIdx) const { + std::pair<unsigned, ArgAttribute> Val = {ParamIdx, ImmArg}; + return std::binary_search(ArgumentAttributes.begin(), + ArgumentAttributes.end(), Val); +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.h index 1ab2de269c76..6c89f34c50ec 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.h +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.h @@ -86,12 +86,12 @@ public: /// Record *getAsmParser() const; - /// getAsmParserVariant - Return the AssmblyParserVariant definition for + /// getAsmParserVariant - Return the AssemblyParserVariant definition for /// this target. /// Record *getAsmParserVariant(unsigned i) const; - /// getAsmParserVariantCount - Return the AssmblyParserVariant definition + /// getAsmParserVariantCount - Return the AssemblyParserVariant definition /// available for this target. /// unsigned getAsmParserVariantCount() const; @@ -103,6 +103,12 @@ public: /// getRegBank - Return the register bank description. CodeGenRegBank &getRegBank() const; + /// Return the largest register class on \p RegBank which supports \p Ty and + /// covers \p SubIdx if it exists. + Optional<CodeGenRegisterClass *> + getSuperRegForSubReg(const ValueTypeByHwMode &Ty, CodeGenRegBank &RegBank, + const CodeGenSubRegIndex *SubIdx) const; + /// getRegisterByName - If there is a register with the specific AsmName, /// return it. const CodeGenRegister *getRegisterByName(StringRef Name) const; diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DAGISelEmitter.cpp index fb0c6faa5295..d8e78ce55c7b 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/DAGISelEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelEmitter.cpp @@ -173,7 +173,7 @@ void DAGISelEmitter::run(raw_ostream &OS) { } std::unique_ptr<Matcher> TheMatcher = - llvm::make_unique<ScopeMatcher>(PatternMatchers); + std::make_unique<ScopeMatcher>(PatternMatchers); OptimizeMatcher(TheMatcher, CGP); //Matcher->dump(); diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcher.h b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcher.h index 0a782e84a372..223513fc8d38 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcher.h +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcher.h @@ -932,13 +932,15 @@ private: /// class EmitCopyToRegMatcher : public Matcher { unsigned SrcSlot; // Value to copy into the physreg. - Record *DestPhysReg; + const CodeGenRegister *DestPhysReg; + public: - EmitCopyToRegMatcher(unsigned srcSlot, Record *destPhysReg) + EmitCopyToRegMatcher(unsigned srcSlot, + const CodeGenRegister *destPhysReg) : Matcher(EmitCopyToReg), SrcSlot(srcSlot), DestPhysReg(destPhysReg) {} unsigned getSrcSlot() const { return SrcSlot; } - Record *getDestPhysReg() const { return DestPhysReg; } + const CodeGenRegister *getDestPhysReg() const { return DestPhysReg; } static bool classof(const Matcher *N) { return N->getKind() == EmitCopyToReg; diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp index cecbc6cccdff..e9f1fb93d516 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -670,12 +670,22 @@ EmitMatcher(const Matcher *N, unsigned Indent, unsigned CurrentIdx, OS << '\n'; return 2+MN->getNumNodes(); } - case Matcher::EmitCopyToReg: - OS << "OPC_EmitCopyToReg, " - << cast<EmitCopyToRegMatcher>(N)->getSrcSlot() << ", " - << getQualifiedName(cast<EmitCopyToRegMatcher>(N)->getDestPhysReg()) - << ",\n"; - return 3; + case Matcher::EmitCopyToReg: { + const auto *C2RMatcher = cast<EmitCopyToRegMatcher>(N); + int Bytes = 3; + const CodeGenRegister *Reg = C2RMatcher->getDestPhysReg(); + if (Reg->EnumValue > 255) { + assert(isUInt<16>(Reg->EnumValue) && "not handled"); + OS << "OPC_EmitCopyToReg2, " << C2RMatcher->getSrcSlot() << ", " + << "TARGET_VAL(" << getQualifiedName(Reg->TheDef) << "),\n"; + ++Bytes; + } else { + OS << "OPC_EmitCopyToReg, " << C2RMatcher->getSrcSlot() << ", " + << getQualifiedName(Reg->TheDef) << ",\n"; + } + + return Bytes; + } case Matcher::EmitNodeXForm: { const EmitNodeXFormMatcher *XF = cast<EmitNodeXFormMatcher>(N); OS << "OPC_EmitNodeXForm, " << getNodeXFormID(XF->getNodeXForm()) << ", " diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp index 8f54beeba65b..6a86868a9bcd 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp @@ -141,7 +141,7 @@ namespace { SmallVectorImpl<unsigned> &ResultOps); }; -} // end anon namespace. +} // end anonymous namespace MatcherGen::MatcherGen(const PatternToMatch &pattern, const CodeGenDAGPatterns &cgp) @@ -867,9 +867,13 @@ EmitResultInstructionAsOperand(const TreePatternNode *N, if (isRoot && !PhysRegInputs.empty()) { // Emit all of the CopyToReg nodes for the input physical registers. These // occur in patterns like (mul:i8 AL:i8, GR8:i8:$src). - for (unsigned i = 0, e = PhysRegInputs.size(); i != e; ++i) + for (unsigned i = 0, e = PhysRegInputs.size(); i != e; ++i) { + const CodeGenRegister *Reg = + CGP.getTargetInfo().getRegBank().getReg(PhysRegInputs[i].first); AddMatcher(new EmitCopyToRegMatcher(PhysRegInputs[i].second, - PhysRegInputs[i].first)); + Reg)); + } + // Even if the node has no other glue inputs, the resultant node must be // glued to the CopyFromReg nodes we just generated. TreeHasInGlue = true; @@ -1043,7 +1047,6 @@ void MatcherGen::EmitResultCode() { } } - assert(Ops.size() >= NumSrcResults && "Didn't provide enough results"); SmallVector<unsigned, 8> Results(Ops); // Apply result permutation. diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherOpt.cpp index 7d51b0769372..6746fdd676a7 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherOpt.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherOpt.cpp @@ -409,13 +409,14 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) { DenseMap<unsigned, unsigned> TypeEntry; SmallVector<std::pair<MVT::SimpleValueType, Matcher*>, 8> Cases; for (unsigned i = 0, e = NewOptionsToMatch.size(); i != e; ++i) { - CheckTypeMatcher *CTM = - cast_or_null<CheckTypeMatcher>(FindNodeWithKind(NewOptionsToMatch[i], - Matcher::CheckType)); + Matcher* M = FindNodeWithKind(NewOptionsToMatch[i], Matcher::CheckType); + assert(M && isa<CheckTypeMatcher>(M) && "Unknown Matcher type"); + + auto *CTM = cast<CheckTypeMatcher>(M); Matcher *MatcherWithoutCTM = NewOptionsToMatch[i]->unlinkNode(CTM); MVT::SimpleValueType CTMTy = CTM->getType(); delete CTM; - + unsigned &Entry = TypeEntry[CTMTy]; if (Entry != 0) { // If we have unfactored duplicate types, then we should factor them. diff --git a/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.cpp new file mode 100644 index 000000000000..dd3db7c150ba --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.cpp @@ -0,0 +1,394 @@ +//===- DFAEmitter.cpp - Finite state automaton emitter --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This class can produce a generic deterministic finite state automaton (DFA), +// given a set of possible states and transitions. +// +// The input transitions can be nondeterministic - this class will produce the +// deterministic equivalent state machine. +// +// The generated code can run the DFA and produce an accepted / not accepted +// state and also produce, given a sequence of transitions that results in an +// accepted state, the sequence of intermediate states. This is useful if the +// initial automaton was nondeterministic - it allows mapping back from the DFA +// to the NFA. +// +//===----------------------------------------------------------------------===// +#define DEBUG_TYPE "dfa-emitter" + +#include "DFAEmitter.h" +#include "CodeGenTarget.h" +#include "SequenceToOffsetTable.h" +#include "TableGenBackends.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/UniqueVector.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cassert> +#include <cstdint> +#include <map> +#include <set> +#include <string> +#include <vector> + +using namespace llvm; + +//===----------------------------------------------------------------------===// +// DfaEmitter implementation. This is independent of the GenAutomaton backend. +//===----------------------------------------------------------------------===// + +void DfaEmitter::addTransition(state_type From, state_type To, action_type A) { + Actions.insert(A); + NfaStates.insert(From); + NfaStates.insert(To); + NfaTransitions[{From, A}].push_back(To); + ++NumNfaTransitions; +} + +void DfaEmitter::visitDfaState(DfaState DS) { + // For every possible action... + auto FromId = DfaStates.idFor(DS); + for (action_type A : Actions) { + DfaState NewStates; + DfaTransitionInfo TI; + // For every represented state, word pair in the original NFA... + for (state_type &FromState : DS) { + // If this action is possible from this state add the transitioned-to + // states to NewStates. + auto I = NfaTransitions.find({FromState, A}); + if (I == NfaTransitions.end()) + continue; + for (state_type &ToState : I->second) { + NewStates.push_back(ToState); + TI.emplace_back(FromState, ToState); + } + } + if (NewStates.empty()) + continue; + // Sort and unique. + sort(NewStates); + NewStates.erase(std::unique(NewStates.begin(), NewStates.end()), + NewStates.end()); + sort(TI); + TI.erase(std::unique(TI.begin(), TI.end()), TI.end()); + unsigned ToId = DfaStates.insert(NewStates); + DfaTransitions.emplace(std::make_pair(FromId, A), std::make_pair(ToId, TI)); + } +} + +void DfaEmitter::constructDfa() { + DfaState Initial(1, /*NFA initial state=*/0); + DfaStates.insert(Initial); + + // Note that UniqueVector starts indices at 1, not zero. + unsigned DfaStateId = 1; + while (DfaStateId <= DfaStates.size()) + visitDfaState(DfaStates[DfaStateId++]); +} + +void DfaEmitter::emit(StringRef Name, raw_ostream &OS) { + constructDfa(); + + OS << "// Input NFA has " << NfaStates.size() << " states with " + << NumNfaTransitions << " transitions.\n"; + OS << "// Generated DFA has " << DfaStates.size() << " states with " + << DfaTransitions.size() << " transitions.\n\n"; + + // Implementation note: We don't bake a simple std::pair<> here as it requires + // significantly more effort to parse. A simple test with a large array of + // struct-pairs (N=100000) took clang-10 6s to parse. The same array of + // std::pair<uint64_t, uint64_t> took 242s. Instead we allow the user to + // define the pair type. + // + // FIXME: It may make sense to emit these as ULEB sequences instead of + // pairs of uint64_t. + OS << "// A zero-terminated sequence of NFA state transitions. Every DFA\n"; + OS << "// transition implies a set of NFA transitions. These are referred\n"; + OS << "// to by index in " << Name << "Transitions[].\n"; + + SequenceToOffsetTable<DfaTransitionInfo> Table; + std::map<DfaTransitionInfo, unsigned> EmittedIndices; + for (auto &T : DfaTransitions) + Table.add(T.second.second); + Table.layout(); + OS << "std::array<NfaStatePair, " << Table.size() << "> " << Name + << "TransitionInfo = {{\n"; + Table.emit( + OS, + [](raw_ostream &OS, std::pair<uint64_t, uint64_t> P) { + OS << "{" << P.first << ", " << P.second << "}"; + }, + "{0ULL, 0ULL}"); + + OS << "}};\n\n"; + + OS << "// A transition in the generated " << Name << " DFA.\n"; + OS << "struct " << Name << "Transition {\n"; + OS << " unsigned FromDfaState; // The transitioned-from DFA state.\n"; + OS << " "; + printActionType(OS); + OS << " Action; // The input symbol that causes this transition.\n"; + OS << " unsigned ToDfaState; // The transitioned-to DFA state.\n"; + OS << " unsigned InfoIdx; // Start index into " << Name + << "TransitionInfo.\n"; + OS << "};\n\n"; + + OS << "// A table of DFA transitions, ordered by {FromDfaState, Action}.\n"; + OS << "// The initial state is 1, not zero.\n"; + OS << "std::array<" << Name << "Transition, " << DfaTransitions.size() << "> " + << Name << "Transitions = {{\n"; + for (auto &KV : DfaTransitions) { + dfa_state_type From = KV.first.first; + dfa_state_type To = KV.second.first; + action_type A = KV.first.second; + unsigned InfoIdx = Table.get(KV.second.second); + OS << " {" << From << ", "; + printActionValue(A, OS); + OS << ", " << To << ", " << InfoIdx << "},\n"; + } + OS << "\n}};\n\n"; +} + +void DfaEmitter::printActionType(raw_ostream &OS) { OS << "uint64_t"; } + +void DfaEmitter::printActionValue(action_type A, raw_ostream &OS) { OS << A; } + +//===----------------------------------------------------------------------===// +// AutomatonEmitter implementation +//===----------------------------------------------------------------------===// + +namespace { +// FIXME: This entire discriminated union could be removed with c++17: +// using Action = std::variant<Record *, unsigned, std::string>; +struct Action { + Record *R = nullptr; + unsigned I = 0; + std::string S = nullptr; + + Action() = default; + Action(Record *R, unsigned I, std::string S) : R(R), I(I), S(S) {} + + void print(raw_ostream &OS) const { + if (R) + OS << R->getName(); + else if (!S.empty()) + OS << '"' << S << '"'; + else + OS << I; + } + bool operator<(const Action &Other) const { + return std::make_tuple(R, I, S) < + std::make_tuple(Other.R, Other.I, Other.S); + } +}; + +using ActionTuple = std::vector<Action>; +class Automaton; + +class Transition { + uint64_t NewState; + // The tuple of actions that causes this transition. + ActionTuple Actions; + // The types of the actions; this is the same across all transitions. + SmallVector<std::string, 4> Types; + +public: + Transition(Record *R, Automaton *Parent); + const ActionTuple &getActions() { return Actions; } + SmallVector<std::string, 4> getTypes() { return Types; } + + bool canTransitionFrom(uint64_t State); + uint64_t transitionFrom(uint64_t State); +}; + +class Automaton { + RecordKeeper &Records; + Record *R; + std::vector<Transition> Transitions; + /// All possible action tuples, uniqued. + UniqueVector<ActionTuple> Actions; + /// The fields within each Transition object to find the action symbols. + std::vector<StringRef> ActionSymbolFields; + +public: + Automaton(RecordKeeper &Records, Record *R); + void emit(raw_ostream &OS); + + ArrayRef<StringRef> getActionSymbolFields() { return ActionSymbolFields; } + /// If the type of action A has been overridden (there exists a field + /// "TypeOf_A") return that, otherwise return the empty string. + StringRef getActionSymbolType(StringRef A); +}; + +class AutomatonEmitter { + RecordKeeper &Records; + +public: + AutomatonEmitter(RecordKeeper &R) : Records(R) {} + void run(raw_ostream &OS); +}; + +/// A DfaEmitter implementation that can print our variant action type. +class CustomDfaEmitter : public DfaEmitter { + const UniqueVector<ActionTuple> &Actions; + std::string TypeName; + +public: + CustomDfaEmitter(const UniqueVector<ActionTuple> &Actions, StringRef TypeName) + : Actions(Actions), TypeName(TypeName) {} + + void printActionType(raw_ostream &OS) override; + void printActionValue(action_type A, raw_ostream &OS) override; +}; +} // namespace + +void AutomatonEmitter::run(raw_ostream &OS) { + for (Record *R : Records.getAllDerivedDefinitions("GenericAutomaton")) { + Automaton A(Records, R); + OS << "#ifdef GET_" << R->getName() << "_DECL\n"; + A.emit(OS); + OS << "#endif // GET_" << R->getName() << "_DECL\n"; + } +} + +Automaton::Automaton(RecordKeeper &Records, Record *R) + : Records(Records), R(R) { + LLVM_DEBUG(dbgs() << "Emitting automaton for " << R->getName() << "\n"); + ActionSymbolFields = R->getValueAsListOfStrings("SymbolFields"); +} + +void Automaton::emit(raw_ostream &OS) { + StringRef TransitionClass = R->getValueAsString("TransitionClass"); + for (Record *T : Records.getAllDerivedDefinitions(TransitionClass)) { + assert(T->isSubClassOf("Transition")); + Transitions.emplace_back(T, this); + Actions.insert(Transitions.back().getActions()); + } + + LLVM_DEBUG(dbgs() << " Action alphabet cardinality: " << Actions.size() + << "\n"); + LLVM_DEBUG(dbgs() << " Each state has " << Transitions.size() + << " potential transitions.\n"); + + StringRef Name = R->getName(); + + CustomDfaEmitter Emitter(Actions, std::string(Name) + "Action"); + // Starting from the initial state, build up a list of possible states and + // transitions. + std::deque<uint64_t> Worklist(1, 0); + std::set<uint64_t> SeenStates; + unsigned NumTransitions = 0; + SeenStates.insert(Worklist.front()); + while (!Worklist.empty()) { + uint64_t State = Worklist.front(); + Worklist.pop_front(); + for (Transition &T : Transitions) { + if (!T.canTransitionFrom(State)) + continue; + uint64_t NewState = T.transitionFrom(State); + if (SeenStates.emplace(NewState).second) + Worklist.emplace_back(NewState); + ++NumTransitions; + Emitter.addTransition(State, NewState, Actions.idFor(T.getActions())); + } + } + LLVM_DEBUG(dbgs() << " NFA automaton has " << SeenStates.size() + << " states with " << NumTransitions << " transitions.\n"); + + const auto &ActionTypes = Transitions.back().getTypes(); + OS << "// The type of an action in the " << Name << " automaton.\n"; + if (ActionTypes.size() == 1) { + OS << "using " << Name << "Action = " << ActionTypes[0] << ";\n"; + } else { + OS << "using " << Name << "Action = std::tuple<" << join(ActionTypes, ", ") + << ">;\n"; + } + OS << "\n"; + + Emitter.emit(Name, OS); +} + +StringRef Automaton::getActionSymbolType(StringRef A) { + Twine Ty = "TypeOf_" + A; + if (!R->getValue(Ty.str())) + return ""; + return R->getValueAsString(Ty.str()); +} + +Transition::Transition(Record *R, Automaton *Parent) { + BitsInit *NewStateInit = R->getValueAsBitsInit("NewState"); + NewState = 0; + assert(NewStateInit->getNumBits() <= sizeof(uint64_t) * 8 && + "State cannot be represented in 64 bits!"); + for (unsigned I = 0; I < NewStateInit->getNumBits(); ++I) { + if (auto *Bit = dyn_cast<BitInit>(NewStateInit->getBit(I))) { + if (Bit->getValue()) + NewState |= 1ULL << I; + } + } + + for (StringRef A : Parent->getActionSymbolFields()) { + RecordVal *SymbolV = R->getValue(A); + if (auto *Ty = dyn_cast<RecordRecTy>(SymbolV->getType())) { + Actions.emplace_back(R->getValueAsDef(A), 0, ""); + Types.emplace_back(Ty->getAsString()); + } else if (isa<IntRecTy>(SymbolV->getType())) { + Actions.emplace_back(nullptr, R->getValueAsInt(A), ""); + Types.emplace_back("unsigned"); + } else if (isa<StringRecTy>(SymbolV->getType()) || + isa<CodeRecTy>(SymbolV->getType())) { + Actions.emplace_back(nullptr, 0, R->getValueAsString(A)); + Types.emplace_back("std::string"); + } else { + report_fatal_error("Unhandled symbol type!"); + } + + StringRef TypeOverride = Parent->getActionSymbolType(A); + if (!TypeOverride.empty()) + Types.back() = TypeOverride; + } +} + +bool Transition::canTransitionFrom(uint64_t State) { + if ((State & NewState) == 0) + // The bits we want to set are not set; + return true; + return false; +} + +uint64_t Transition::transitionFrom(uint64_t State) { + return State | NewState; +} + +void CustomDfaEmitter::printActionType(raw_ostream &OS) { OS << TypeName; } + +void CustomDfaEmitter::printActionValue(action_type A, raw_ostream &OS) { + const ActionTuple &AT = Actions[A]; + if (AT.size() > 1) + OS << "std::make_tuple("; + bool First = true; + for (const auto &SingleAction : AT) { + if (!First) + OS << ", "; + First = false; + SingleAction.print(OS); + } + if (AT.size() > 1) + OS << ")"; +} + +namespace llvm { + +void EmitAutomata(RecordKeeper &RK, raw_ostream &OS) { + AutomatonEmitter(RK).run(OS); +} + +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.h b/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.h new file mode 100644 index 000000000000..76de8f72cd88 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.h @@ -0,0 +1,107 @@ +//===--------------------- DfaEmitter.h -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Defines a generic automaton builder. This takes a set of transitions and +// states that represent a nondeterministic finite state automaton (NFA) and +// emits a determinized DFA in a form that include/llvm/Support/Automaton.h can +// drive. +// +// See file llvm/TableGen/Automaton.td for the TableGen API definition. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_DFAEMITTER_H +#define LLVM_UTILS_TABLEGEN_DFAEMITTER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/UniqueVector.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Record.h" +#include <set> +#include <unordered_map> + +namespace llvm { + +class raw_ostream; +/// Construct a deterministic finite state automaton from possible +/// nondeterministic state and transition data. +/// +/// The state type is a 64-bit unsigned integer. The generated automaton is +/// invariant to the sparsity of the state representation - its size is only +/// a function of the cardinality of the set of states. +/// +/// The inputs to this emitter are considered to define a nondeterministic +/// finite state automaton (NFA). This is then converted to a DFA during +/// emission. The emitted tables can be used to by +/// include/llvm/Support/Automaton.h. +class DfaEmitter { +public: + // The type of an NFA state. The initial state is always zero. + using state_type = uint64_t; + // The type of an action. + using action_type = uint64_t; + + DfaEmitter() = default; + virtual ~DfaEmitter() = default; + + void addTransition(state_type From, state_type To, action_type A); + void emit(StringRef Name, raw_ostream &OS); + +protected: + /// Emit the C++ type of an action to OS. + virtual void printActionType(raw_ostream &OS); + /// Emit the C++ value of an action A to OS. + virtual void printActionValue(action_type A, raw_ostream &OS); + +private: + /// The state type of deterministic states. These are only used internally to + /// this class. This is an ID into the DfaStates UniqueVector. + using dfa_state_type = unsigned; + + /// The actual representation of a DFA state, which is a union of one or more + /// NFA states. + using DfaState = SmallVector<state_type, 4>; + + /// A DFA transition consists of a set of NFA states transitioning to a + /// new set of NFA states. The DfaTransitionInfo tracks, for every + /// transitioned-from NFA state, a set of valid transitioned-to states. + /// + /// Emission of this transition relation allows algorithmic determination of + /// the possible candidate NFA paths taken under a given input sequence to + /// reach a given DFA state. + using DfaTransitionInfo = SmallVector<std::pair<state_type, state_type>, 4>; + + /// The set of all possible actions. + std::set<action_type> Actions; + + /// The set of nondeterministic transitions. A state-action pair can + /// transition to multiple target states. + std::map<std::pair<state_type, action_type>, std::vector<state_type>> + NfaTransitions; + std::set<state_type> NfaStates; + unsigned NumNfaTransitions = 0; + + /// The set of deterministic states. DfaStates.getId(DfaState) returns an ID, + /// which is dfa_state_type. Note that because UniqueVector reserves state + /// zero, the initial DFA state is always 1. + UniqueVector<DfaState> DfaStates; + /// The set of deterministic transitions. A state-action pair has only a + /// single target state. + std::map<std::pair<dfa_state_type, action_type>, + std::pair<dfa_state_type, DfaTransitionInfo>> + DfaTransitions; + + /// Visit all NFA states and construct the DFA. + void constructDfa(); + /// Visit a single DFA state and construct all possible transitions to new DFA + /// states. + void visitDfaState(DfaState DS); +}; + +} // namespace llvm + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DFAPacketizerEmitter.cpp index dabcc8f8ed55..018bda1b6090 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/DFAPacketizerEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/DFAPacketizerEmitter.cpp @@ -16,683 +16,121 @@ #define DEBUG_TYPE "dfa-emitter" +#include "CodeGenSchedule.h" #include "CodeGenTarget.h" +#include "DFAEmitter.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" -#include "llvm/TableGen/Record.h" -#include "llvm/TableGen/TableGenBackend.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" #include <cassert> #include <cstdint> #include <map> #include <set> #include <string> +#include <unordered_map> #include <vector> using namespace llvm; -// -------------------------------------------------------------------- -// Definitions shared between DFAPacketizer.cpp and DFAPacketizerEmitter.cpp - -// DFA_MAX_RESTERMS * DFA_MAX_RESOURCES must fit within sizeof DFAInput. -// This is verified in DFAPacketizer.cpp:DFAPacketizer::DFAPacketizer. -// -// e.g. terms x resource bit combinations that fit in uint32_t: -// 4 terms x 8 bits = 32 bits -// 3 terms x 10 bits = 30 bits -// 2 terms x 16 bits = 32 bits -// -// e.g. terms x resource bit combinations that fit in uint64_t: -// 8 terms x 8 bits = 64 bits -// 7 terms x 9 bits = 63 bits -// 6 terms x 10 bits = 60 bits -// 5 terms x 12 bits = 60 bits -// 4 terms x 16 bits = 64 bits <--- current -// 3 terms x 21 bits = 63 bits -// 2 terms x 32 bits = 64 bits -// -#define DFA_MAX_RESTERMS 4 // The max # of AND'ed resource terms. -#define DFA_MAX_RESOURCES 16 // The max # of resource bits in one term. - -typedef uint64_t DFAInput; -typedef int64_t DFAStateInput; -#define DFA_TBLTYPE "int64_t" // For generating DFAStateInputTable. +// We use a uint64_t to represent a resource bitmask. +#define DFA_MAX_RESOURCES 64 namespace { +using ResourceVector = SmallVector<uint64_t, 4>; - DFAInput addDFAFuncUnits(DFAInput Inp, unsigned FuncUnits) { - return (Inp << DFA_MAX_RESOURCES) | FuncUnits; - } +struct ScheduleClass { + /// The parent itinerary index (processor model ID). + unsigned ItineraryID; - /// Return the DFAInput for an instruction class input vector. - /// This function is used in both DFAPacketizer.cpp and in - /// DFAPacketizerEmitter.cpp. - DFAInput getDFAInsnInput(const std::vector<unsigned> &InsnClass) { - DFAInput InsnInput = 0; - assert((InsnClass.size() <= DFA_MAX_RESTERMS) && - "Exceeded maximum number of DFA terms"); - for (auto U : InsnClass) - InsnInput = addDFAFuncUnits(InsnInput, U); - return InsnInput; - } + /// Index within this itinerary of the schedule class. + unsigned Idx; -} // end anonymous namespace - -// -------------------------------------------------------------------- - -#ifndef NDEBUG -// To enable debugging, run llvm-tblgen with: "-debug-only dfa-emitter". -// -// dbgsInsnClass - When debugging, print instruction class stages. -// -void dbgsInsnClass(const std::vector<unsigned> &InsnClass); -// -// dbgsStateInfo - When debugging, print the set of state info. -// -void dbgsStateInfo(const std::set<unsigned> &stateInfo); -// -// dbgsIndent - When debugging, indent by the specified amount. -// -void dbgsIndent(unsigned indent); -#endif + /// The index within the uniqued set of required resources of Resources. + unsigned ResourcesIdx; -// -// class DFAPacketizerEmitter: class that generates and prints out the DFA -// for resource tracking. -// -namespace { + /// Conjunctive list of resource requirements: + /// {a|b, b|c} => (a OR b) AND (b or c). + /// Resources are unique across all itineraries. + ResourceVector Resources; +}; +// Generates and prints out the DFA for resource tracking. class DFAPacketizerEmitter { private: std::string TargetName; - // - // allInsnClasses is the set of all possible resources consumed by an - // InstrStage. - // - std::vector<std::vector<unsigned>> allInsnClasses; RecordKeeper &Records; + UniqueVector<ResourceVector> UniqueResources; + std::vector<ScheduleClass> ScheduleClasses; + std::map<std::string, uint64_t> FUNameToBitsMap; + std::map<unsigned, uint64_t> ComboBitToBitsMap; + public: DFAPacketizerEmitter(RecordKeeper &R); - // - // collectAllFuncUnits - Construct a map of function unit names to bits. - // - int collectAllFuncUnits(std::vector<Record*> &ProcItinList, - std::map<std::string, unsigned> &FUNameToBitsMap, - int &maxResources, - raw_ostream &OS); - - // - // collectAllComboFuncs - Construct a map from a combo function unit bit to - // the bits of all included functional units. - // - int collectAllComboFuncs(std::vector<Record*> &ComboFuncList, - std::map<std::string, unsigned> &FUNameToBitsMap, - std::map<unsigned, unsigned> &ComboBitToBitsMap, - raw_ostream &OS); - - // - // collectOneInsnClass - Populate allInsnClasses with one instruction class. - // - int collectOneInsnClass(const std::string &ProcName, - std::vector<Record*> &ProcItinList, - std::map<std::string, unsigned> &FUNameToBitsMap, - Record *ItinData, - raw_ostream &OS); - - // - // collectAllInsnClasses - Populate allInsnClasses which is a set of units - // used in each stage. - // - int collectAllInsnClasses(const std::string &ProcName, - std::vector<Record*> &ProcItinList, - std::map<std::string, unsigned> &FUNameToBitsMap, - std::vector<Record*> &ItinDataList, - int &maxStages, - raw_ostream &OS); + // Construct a map of function unit names to bits. + int collectAllFuncUnits( + ArrayRef<const CodeGenProcModel *> ProcModels); - void run(raw_ostream &OS); -}; + // Construct a map from a combo function unit bit to the bits of all included + // functional units. + int collectAllComboFuncs(ArrayRef<Record *> ComboFuncList); -// -// State represents the usage of machine resources if the packet contains -// a set of instruction classes. -// -// Specifically, currentState is a set of bit-masks. -// The nth bit in a bit-mask indicates whether the nth resource is being used -// by this state. The set of bit-masks in a state represent the different -// possible outcomes of transitioning to this state. -// For example: consider a two resource architecture: resource L and resource M -// with three instruction classes: L, M, and L_or_M. -// From the initial state (currentState = 0x00), if we add instruction class -// L_or_M we will transition to a state with currentState = [0x01, 0x10]. This -// represents the possible resource states that can result from adding a L_or_M -// instruction -// -// Another way of thinking about this transition is we are mapping a NDFA with -// two states [0x01] and [0x10] into a DFA with a single state [0x01, 0x10]. -// -// A State instance also contains a collection of transitions from that state: -// a map from inputs to new states. -// -class State { - public: - static int currentStateNum; - // stateNum is the only member used for equality/ordering, all other members - // can be mutated even in const State objects. - const int stateNum; - mutable bool isInitial; - mutable std::set<unsigned> stateInfo; - typedef std::map<std::vector<unsigned>, const State *> TransitionMap; - mutable TransitionMap Transitions; - - State(); - - bool operator<(const State &s) const { - return stateNum < s.stateNum; - } - - // - // canMaybeAddInsnClass - Quickly verifies if an instruction of type InsnClass - // may be a valid transition from this state i.e., can an instruction of type - // InsnClass be added to the packet represented by this state. - // - // Note that for multiple stages, this quick check does not take into account - // any possible resource competition between the stages themselves. That is - // enforced in AddInsnClassStages which checks the cross product of all - // stages for resource availability (which is a more involved check). - // - bool canMaybeAddInsnClass(std::vector<unsigned> &InsnClass, - std::map<unsigned, unsigned> &ComboBitToBitsMap) const; - - // - // AddInsnClass - Return all combinations of resource reservation - // which are possible from this state (PossibleStates). - // - // PossibleStates is the set of valid resource states that ensue from valid - // transitions. - // - void AddInsnClass(std::vector<unsigned> &InsnClass, - std::map<unsigned, unsigned> &ComboBitToBitsMap, - std::set<unsigned> &PossibleStates) const; - - // - // AddInsnClassStages - Return all combinations of resource reservation - // resulting from the cross product of all stages for this InsnClass - // which are possible from this state (PossibleStates). - // - void AddInsnClassStages(std::vector<unsigned> &InsnClass, - std::map<unsigned, unsigned> &ComboBitToBitsMap, - unsigned chkstage, unsigned numstages, - unsigned prevState, unsigned origState, - DenseSet<unsigned> &VisitedResourceStates, - std::set<unsigned> &PossibleStates) const; - - // - // addTransition - Add a transition from this state given the input InsnClass - // - void addTransition(std::vector<unsigned> InsnClass, const State *To) const; - - // - // hasTransition - Returns true if there is a transition from this state - // given the input InsnClass - // - bool hasTransition(std::vector<unsigned> InsnClass) const; -}; - -// -// class DFA: deterministic finite automaton for processor resource tracking. -// -class DFA { -public: - DFA() = default; - - // Set of states. Need to keep this sorted to emit the transition table. - typedef std::set<State> StateSet; - StateSet states; + ResourceVector getResourcesForItinerary(Record *Itinerary); + void createScheduleClasses(unsigned ItineraryIdx, const RecVec &Itineraries); - State *currentState = nullptr; + // Emit code for a subset of itineraries. + void emitForItineraries(raw_ostream &OS, + std::vector<const CodeGenProcModel *> &ProcItinList, + std::string DFAName); - // - // Modify the DFA. - // - const State &newState(); - - // - // writeTable: Print out a table representing the DFA. - // - void writeTableAndAPI(raw_ostream &OS, const std::string &ClassName, - int numInsnClasses = 0, - int maxResources = 0, int numCombos = 0, int maxStages = 0); + void run(raw_ostream &OS); }; - } // end anonymous namespace -#ifndef NDEBUG -// To enable debugging, run llvm-tblgen with: "-debug-only dfa-emitter". -// -// dbgsInsnClass - When debugging, print instruction class stages. -// -void dbgsInsnClass(const std::vector<unsigned> &InsnClass) { - LLVM_DEBUG(dbgs() << "InsnClass: "); - for (unsigned i = 0; i < InsnClass.size(); ++i) { - if (i > 0) { - LLVM_DEBUG(dbgs() << ", "); - } - LLVM_DEBUG(dbgs() << "0x" << Twine::utohexstr(InsnClass[i])); - } - DFAInput InsnInput = getDFAInsnInput(InsnClass); - LLVM_DEBUG(dbgs() << " (input: 0x" << Twine::utohexstr(InsnInput) << ")"); -} +DFAPacketizerEmitter::DFAPacketizerEmitter(RecordKeeper &R) + : TargetName(CodeGenTarget(R).getName()), Records(R) {} -// -// dbgsStateInfo - When debugging, print the set of state info. -// -void dbgsStateInfo(const std::set<unsigned> &stateInfo) { - LLVM_DEBUG(dbgs() << "StateInfo: "); - unsigned i = 0; - for (std::set<unsigned>::iterator SI = stateInfo.begin(); - SI != stateInfo.end(); ++SI, ++i) { - unsigned thisState = *SI; - if (i > 0) { - LLVM_DEBUG(dbgs() << ", "); - } - LLVM_DEBUG(dbgs() << "0x" << Twine::utohexstr(thisState)); - } -} - -// -// dbgsIndent - When debugging, indent by the specified amount. -// -void dbgsIndent(unsigned indent) { - for (unsigned i = 0; i < indent; ++i) { - LLVM_DEBUG(dbgs() << " "); - } -} -#endif // NDEBUG - -// -// Constructors and destructors for State and DFA -// -State::State() : - stateNum(currentStateNum++), isInitial(false) {} - -// -// addTransition - Add a transition from this state given the input InsnClass -// -void State::addTransition(std::vector<unsigned> InsnClass, const State *To) - const { - assert(!Transitions.count(InsnClass) && - "Cannot have multiple transitions for the same input"); - Transitions[InsnClass] = To; -} - -// -// hasTransition - Returns true if there is a transition from this state -// given the input InsnClass -// -bool State::hasTransition(std::vector<unsigned> InsnClass) const { - return Transitions.count(InsnClass) > 0; -} - -// -// AddInsnClass - Return all combinations of resource reservation -// which are possible from this state (PossibleStates). -// -// PossibleStates is the set of valid resource states that ensue from valid -// transitions. -// -void State::AddInsnClass(std::vector<unsigned> &InsnClass, - std::map<unsigned, unsigned> &ComboBitToBitsMap, - std::set<unsigned> &PossibleStates) const { - // - // Iterate over all resource states in currentState. - // - unsigned numstages = InsnClass.size(); - assert((numstages > 0) && "InsnClass has no stages"); - - for (std::set<unsigned>::iterator SI = stateInfo.begin(); - SI != stateInfo.end(); ++SI) { - unsigned thisState = *SI; - - DenseSet<unsigned> VisitedResourceStates; - - LLVM_DEBUG(dbgs() << " thisState: 0x" << Twine::utohexstr(thisState) - << "\n"); - AddInsnClassStages(InsnClass, ComboBitToBitsMap, - numstages - 1, numstages, - thisState, thisState, - VisitedResourceStates, PossibleStates); - } -} - -void State::AddInsnClassStages(std::vector<unsigned> &InsnClass, - std::map<unsigned, unsigned> &ComboBitToBitsMap, - unsigned chkstage, unsigned numstages, - unsigned prevState, unsigned origState, - DenseSet<unsigned> &VisitedResourceStates, - std::set<unsigned> &PossibleStates) const { - assert((chkstage < numstages) && "AddInsnClassStages: stage out of range"); - unsigned thisStage = InsnClass[chkstage]; - - LLVM_DEBUG({ - dbgsIndent((1 + numstages - chkstage) << 1); - dbgs() << "AddInsnClassStages " << chkstage << " (0x" - << Twine::utohexstr(thisStage) << ") from "; - dbgsInsnClass(InsnClass); - dbgs() << "\n"; - }); - - // - // Iterate over all possible resources used in thisStage. - // For ex: for thisStage = 0x11, all resources = {0x01, 0x10}. - // - for (unsigned int j = 0; j < DFA_MAX_RESOURCES; ++j) { - unsigned resourceMask = (0x1 << j); - if (resourceMask & thisStage) { - unsigned combo = ComboBitToBitsMap[resourceMask]; - if (combo && ((~prevState & combo) != combo)) { - LLVM_DEBUG(dbgs() << "\tSkipped Add 0x" << Twine::utohexstr(prevState) - << " - combo op 0x" << Twine::utohexstr(resourceMask) - << " (0x" << Twine::utohexstr(combo) - << ") cannot be scheduled\n"); - continue; - } - // - // For each possible resource used in thisStage, generate the - // resource state if that resource was used. - // - unsigned ResultingResourceState = prevState | resourceMask | combo; - LLVM_DEBUG({ - dbgsIndent((2 + numstages - chkstage) << 1); - dbgs() << "0x" << Twine::utohexstr(prevState) << " | 0x" - << Twine::utohexstr(resourceMask); - if (combo) - dbgs() << " | 0x" << Twine::utohexstr(combo); - dbgs() << " = 0x" << Twine::utohexstr(ResultingResourceState) << " "; - }); - - // - // If this is the final stage for this class - // - if (chkstage == 0) { - // - // Check if the resulting resource state can be accommodated in this - // packet. - // We compute resource OR prevState (originally started as origState). - // If the result of the OR is different than origState, it implies - // that there is at least one resource that can be used to schedule - // thisStage in the current packet. - // Insert ResultingResourceState into PossibleStates only if we haven't - // processed ResultingResourceState before. - // - if (ResultingResourceState != prevState) { - if (VisitedResourceStates.count(ResultingResourceState) == 0) { - VisitedResourceStates.insert(ResultingResourceState); - PossibleStates.insert(ResultingResourceState); - LLVM_DEBUG(dbgs() - << "\tResultingResourceState: 0x" - << Twine::utohexstr(ResultingResourceState) << "\n"); - } else { - LLVM_DEBUG(dbgs() << "\tSkipped Add - state already seen\n"); - } - } else { - LLVM_DEBUG(dbgs() - << "\tSkipped Add - no final resources available\n"); - } - } else { - // - // If the current resource can be accommodated, check the next - // stage in InsnClass for available resources. - // - if (ResultingResourceState != prevState) { - LLVM_DEBUG(dbgs() << "\n"); - AddInsnClassStages(InsnClass, ComboBitToBitsMap, - chkstage - 1, numstages, - ResultingResourceState, origState, - VisitedResourceStates, PossibleStates); - } else { - LLVM_DEBUG(dbgs() << "\tSkipped Add - no resources available\n"); - } - } - } - } -} - -// -// canMaybeAddInsnClass - Quickly verifies if an instruction of type InsnClass -// may be a valid transition from this state i.e., can an instruction of type -// InsnClass be added to the packet represented by this state. -// -// Note that this routine is performing conservative checks that can be -// quickly executed acting as a filter before calling AddInsnClassStages. -// Any cases allowed through here will be caught later in AddInsnClassStages -// which performs the more expensive exact check. -// -bool State::canMaybeAddInsnClass(std::vector<unsigned> &InsnClass, - std::map<unsigned, unsigned> &ComboBitToBitsMap) const { - for (std::set<unsigned>::const_iterator SI = stateInfo.begin(); - SI != stateInfo.end(); ++SI) { - // Check to see if all required resources are available. - bool available = true; - - // Inspect each stage independently. - // note: This is a conservative check as we aren't checking for - // possible resource competition between the stages themselves - // The full cross product is examined later in AddInsnClass. - for (unsigned i = 0; i < InsnClass.size(); ++i) { - unsigned resources = *SI; - if ((~resources & InsnClass[i]) == 0) { - available = false; - break; - } - // Make sure _all_ resources for a combo function are available. - // note: This is a quick conservative check as it won't catch an - // unscheduleable combo if this stage is an OR expression - // containing a combo. - // These cases are caught later in AddInsnClass. - unsigned combo = ComboBitToBitsMap[InsnClass[i]]; - if (combo && ((~resources & combo) != combo)) { - LLVM_DEBUG(dbgs() << "\tSkipped canMaybeAdd 0x" - << Twine::utohexstr(resources) << " - combo op 0x" - << Twine::utohexstr(InsnClass[i]) << " (0x" - << Twine::utohexstr(combo) - << ") cannot be scheduled\n"); - available = false; - break; - } - } - - if (available) { - return true; - } - } - return false; -} - -const State &DFA::newState() { - auto IterPair = states.insert(State()); - assert(IterPair.second && "State already exists"); - return *IterPair.first; -} - -int State::currentStateNum = 0; - -DFAPacketizerEmitter::DFAPacketizerEmitter(RecordKeeper &R): - TargetName(CodeGenTarget(R).getName()), Records(R) {} - -// -// writeTableAndAPI - Print out a table representing the DFA and the -// associated API to create a DFA packetizer. -// -// Format: -// DFAStateInputTable[][2] = pairs of <Input, Transition> for all valid -// transitions. -// DFAStateEntryTable[i] = Index of the first entry in DFAStateInputTable for -// the ith state. -// -// -void DFA::writeTableAndAPI(raw_ostream &OS, const std::string &TargetName, - int numInsnClasses, - int maxResources, int numCombos, int maxStages) { - unsigned numStates = states.size(); - - LLVM_DEBUG(dbgs() << "-------------------------------------------------------" - "----------------------\n"); - LLVM_DEBUG(dbgs() << "writeTableAndAPI\n"); - LLVM_DEBUG(dbgs() << "Total states: " << numStates << "\n"); - - OS << "namespace llvm {\n"; - - OS << "\n// Input format:\n"; - OS << "#define DFA_MAX_RESTERMS " << DFA_MAX_RESTERMS - << "\t// maximum AND'ed resource terms\n"; - OS << "#define DFA_MAX_RESOURCES " << DFA_MAX_RESOURCES - << "\t// maximum resource bits in one term\n"; - - OS << "\n// " << TargetName << "DFAStateInputTable[][2] = " - << "pairs of <Input, NextState> for all valid\n"; - OS << "// transitions.\n"; - OS << "// " << numStates << "\tstates\n"; - OS << "// " << numInsnClasses << "\tinstruction classes\n"; - OS << "// " << maxResources << "\tresources max\n"; - OS << "// " << numCombos << "\tcombo resources\n"; - OS << "// " << maxStages << "\tstages max\n"; - OS << "const " << DFA_TBLTYPE << " " - << TargetName << "DFAStateInputTable[][2] = {\n"; - - // This table provides a map to the beginning of the transitions for State s - // in DFAStateInputTable. - std::vector<int> StateEntry(numStates+1); - static const std::string SentinelEntry = "{-1, -1}"; - - // Tracks the total valid transitions encountered so far. It is used - // to construct the StateEntry table. - int ValidTransitions = 0; - DFA::StateSet::iterator SI = states.begin(); - for (unsigned i = 0; i < numStates; ++i, ++SI) { - assert ((SI->stateNum == (int) i) && "Mismatch in state numbers"); - StateEntry[i] = ValidTransitions; - for (State::TransitionMap::iterator - II = SI->Transitions.begin(), IE = SI->Transitions.end(); - II != IE; ++II) { - OS << "{0x" << Twine::utohexstr(getDFAInsnInput(II->first)) << ", " - << II->second->stateNum << "},\t"; - } - ValidTransitions += SI->Transitions.size(); - - // If there are no valid transitions from this stage, we need a sentinel - // transition. - if (ValidTransitions == StateEntry[i]) { - OS << SentinelEntry << ",\t"; - ++ValidTransitions; - } - - OS << " // state " << i << ": " << StateEntry[i]; - if (StateEntry[i] != (ValidTransitions-1)) { // More than one transition. - OS << "-" << (ValidTransitions-1); - } - OS << "\n"; - } - - // Print out a sentinel entry at the end of the StateInputTable. This is - // needed to iterate over StateInputTable in DFAPacketizer::ReadTable() - OS << SentinelEntry << "\t"; - OS << " // state " << numStates << ": " << ValidTransitions; - OS << "\n"; - - OS << "};\n\n"; - OS << "// " << TargetName << "DFAStateEntryTable[i] = " - << "Index of the first entry in DFAStateInputTable for\n"; - OS << "// " - << "the ith state.\n"; - OS << "// " << numStates << " states\n"; - OS << "const unsigned int " << TargetName << "DFAStateEntryTable[] = {\n"; - - // Multiply i by 2 since each entry in DFAStateInputTable is a set of - // two numbers. - unsigned lastState = 0; - for (unsigned i = 0; i < numStates; ++i) { - if (i && ((i % 10) == 0)) { - lastState = i-1; - OS << " // states " << (i-10) << ":" << lastState << "\n"; - } - OS << StateEntry[i] << ", "; - } - - // Print out the index to the sentinel entry in StateInputTable - OS << ValidTransitions << ", "; - OS << " // states " << (lastState+1) << ":" << numStates << "\n"; - - OS << "};\n"; - OS << "} // namespace\n"; - - // - // Emit DFA Packetizer tables if the target is a VLIW machine. - // - std::string SubTargetClassName = TargetName + "GenSubtargetInfo"; - OS << "\n" << "#include \"llvm/CodeGen/DFAPacketizer.h\"\n"; - OS << "namespace llvm {\n"; - OS << "DFAPacketizer *" << SubTargetClassName << "::" - << "createDFAPacketizer(const InstrItineraryData *IID) const {\n" - << " return new DFAPacketizer(IID, " << TargetName - << "DFAStateInputTable, " << TargetName << "DFAStateEntryTable);\n}\n\n"; - OS << "} // End llvm namespace \n"; -} - -// -// collectAllFuncUnits - Construct a map of function unit names to bits. -// int DFAPacketizerEmitter::collectAllFuncUnits( - std::vector<Record*> &ProcItinList, - std::map<std::string, unsigned> &FUNameToBitsMap, - int &maxFUs, - raw_ostream &OS) { + ArrayRef<const CodeGenProcModel *> ProcModels) { LLVM_DEBUG(dbgs() << "-------------------------------------------------------" "----------------------\n"); LLVM_DEBUG(dbgs() << "collectAllFuncUnits"); - LLVM_DEBUG(dbgs() << " (" << ProcItinList.size() << " itineraries)\n"); + LLVM_DEBUG(dbgs() << " (" << ProcModels.size() << " itineraries)\n"); + + std::set<Record *> ProcItinList; + for (const CodeGenProcModel *Model : ProcModels) + ProcItinList.insert(Model->ItinsDef); int totalFUs = 0; // Parse functional units for all the itineraries. - for (unsigned i = 0, N = ProcItinList.size(); i < N; ++i) { - Record *Proc = ProcItinList[i]; - std::vector<Record*> FUs = Proc->getValueAsListOfDefs("FU"); + for (Record *Proc : ProcItinList) { + std::vector<Record *> FUs = Proc->getValueAsListOfDefs("FU"); - LLVM_DEBUG(dbgs() << " FU:" << i << " (" << FUs.size() << " FUs) " - << Proc->getName()); + LLVM_DEBUG(dbgs() << " FU:" + << " (" << FUs.size() << " FUs) " << Proc->getName()); // Convert macros to bits for each stage. unsigned numFUs = FUs.size(); for (unsigned j = 0; j < numFUs; ++j) { - assert ((j < DFA_MAX_RESOURCES) && - "Exceeded maximum number of representable resources"); - unsigned FuncResources = (unsigned) (1U << j); + assert((j < DFA_MAX_RESOURCES) && + "Exceeded maximum number of representable resources"); + uint64_t FuncResources = 1ULL << j; FUNameToBitsMap[FUs[j]->getName()] = FuncResources; LLVM_DEBUG(dbgs() << " " << FUs[j]->getName() << ":0x" << Twine::utohexstr(FuncResources)); } - if (((int) numFUs) > maxFUs) { - maxFUs = numFUs; - } totalFUs += numFUs; LLVM_DEBUG(dbgs() << "\n"); } return totalFUs; } -// -// collectAllComboFuncs - Construct a map from a combo function unit bit to -// the bits of all included functional units. -// -int DFAPacketizerEmitter::collectAllComboFuncs( - std::vector<Record*> &ComboFuncList, - std::map<std::string, unsigned> &FUNameToBitsMap, - std::map<unsigned, unsigned> &ComboBitToBitsMap, - raw_ostream &OS) { +int DFAPacketizerEmitter::collectAllComboFuncs(ArrayRef<Record *> ComboFuncList) { LLVM_DEBUG(dbgs() << "-------------------------------------------------------" "----------------------\n"); LLVM_DEBUG(dbgs() << "collectAllComboFuncs"); @@ -701,27 +139,27 @@ int DFAPacketizerEmitter::collectAllComboFuncs( int numCombos = 0; for (unsigned i = 0, N = ComboFuncList.size(); i < N; ++i) { Record *Func = ComboFuncList[i]; - std::vector<Record*> FUs = Func->getValueAsListOfDefs("CFD"); + std::vector<Record *> FUs = Func->getValueAsListOfDefs("CFD"); LLVM_DEBUG(dbgs() << " CFD:" << i << " (" << FUs.size() << " combo FUs) " << Func->getName() << "\n"); // Convert macros to bits for each stage. for (unsigned j = 0, N = FUs.size(); j < N; ++j) { - assert ((j < DFA_MAX_RESOURCES) && - "Exceeded maximum number of DFA resources"); + assert((j < DFA_MAX_RESOURCES) && + "Exceeded maximum number of DFA resources"); Record *FuncData = FUs[j]; Record *ComboFunc = FuncData->getValueAsDef("TheComboFunc"); - const std::vector<Record*> &FuncList = - FuncData->getValueAsListOfDefs("FuncList"); + const std::vector<Record *> &FuncList = + FuncData->getValueAsListOfDefs("FuncList"); const std::string &ComboFuncName = ComboFunc->getName(); - unsigned ComboBit = FUNameToBitsMap[ComboFuncName]; - unsigned ComboResources = ComboBit; + uint64_t ComboBit = FUNameToBitsMap[ComboFuncName]; + uint64_t ComboResources = ComboBit; LLVM_DEBUG(dbgs() << " combo: " << ComboFuncName << ":0x" << Twine::utohexstr(ComboResources) << "\n"); for (unsigned k = 0, M = FuncList.size(); k < M; ++k) { std::string FuncName = FuncList[k]->getName(); - unsigned FuncResources = FUNameToBitsMap[FuncName]; + uint64_t FuncResources = FUNameToBitsMap[FuncName]; LLVM_DEBUG(dbgs() << " " << FuncName << ":0x" << Twine::utohexstr(FuncResources) << "\n"); ComboResources |= FuncResources; @@ -736,254 +174,183 @@ int DFAPacketizerEmitter::collectAllComboFuncs( return numCombos; } -// -// collectOneInsnClass - Populate allInsnClasses with one instruction class -// -int DFAPacketizerEmitter::collectOneInsnClass(const std::string &ProcName, - std::vector<Record*> &ProcItinList, - std::map<std::string, unsigned> &FUNameToBitsMap, - Record *ItinData, - raw_ostream &OS) { - const std::vector<Record*> &StageList = - ItinData->getValueAsListOfDefs("Stages"); - - // The number of stages. - unsigned NStages = StageList.size(); - - LLVM_DEBUG(dbgs() << " " << ItinData->getValueAsDef("TheClass")->getName() - << "\n"); - - std::vector<unsigned> UnitBits; - - // Compute the bitwise or of each unit used in this stage. - for (unsigned i = 0; i < NStages; ++i) { - const Record *Stage = StageList[i]; - - // Get unit list. - const std::vector<Record*> &UnitList = - Stage->getValueAsListOfDefs("Units"); - - LLVM_DEBUG(dbgs() << " stage:" << i << " [" << UnitList.size() - << " units]:"); - unsigned dbglen = 26; // cursor after stage dbgs - - // Compute the bitwise or of each unit used in this stage. - unsigned UnitBitValue = 0; - for (unsigned j = 0, M = UnitList.size(); j < M; ++j) { - // Conduct bitwise or. - std::string UnitName = UnitList[j]->getName(); - LLVM_DEBUG(dbgs() << " " << j << ":" << UnitName); - dbglen += 3 + UnitName.length(); - assert(FUNameToBitsMap.count(UnitName)); - UnitBitValue |= FUNameToBitsMap[UnitName]; +ResourceVector +DFAPacketizerEmitter::getResourcesForItinerary(Record *Itinerary) { + ResourceVector Resources; + assert(Itinerary); + for (Record *StageDef : Itinerary->getValueAsListOfDefs("Stages")) { + uint64_t StageResources = 0; + for (Record *Unit : StageDef->getValueAsListOfDefs("Units")) { + StageResources |= FUNameToBitsMap[Unit->getName()]; } - - if (UnitBitValue != 0) - UnitBits.push_back(UnitBitValue); - - while (dbglen <= 64) { // line up bits dbgs - dbglen += 8; - LLVM_DEBUG(dbgs() << "\t"); - } - LLVM_DEBUG(dbgs() << " (bits: 0x" << Twine::utohexstr(UnitBitValue) - << ")\n"); + if (StageResources != 0) + Resources.push_back(StageResources); } - - if (!UnitBits.empty()) - allInsnClasses.push_back(UnitBits); - - LLVM_DEBUG({ - dbgs() << " "; - dbgsInsnClass(UnitBits); - dbgs() << "\n"; - }); - - return NStages; + return Resources; } -// -// collectAllInsnClasses - Populate allInsnClasses which is a set of units -// used in each stage. -// -int DFAPacketizerEmitter::collectAllInsnClasses(const std::string &ProcName, - std::vector<Record*> &ProcItinList, - std::map<std::string, unsigned> &FUNameToBitsMap, - std::vector<Record*> &ItinDataList, - int &maxStages, - raw_ostream &OS) { - // Collect all instruction classes. - unsigned M = ItinDataList.size(); - - int numInsnClasses = 0; - LLVM_DEBUG(dbgs() << "-------------------------------------------------------" - "----------------------\n" - << "collectAllInsnClasses " << ProcName << " (" << M - << " classes)\n"); - - // Collect stages for each instruction class for all itinerary data - for (unsigned j = 0; j < M; j++) { - Record *ItinData = ItinDataList[j]; - int NStages = collectOneInsnClass(ProcName, ProcItinList, - FUNameToBitsMap, ItinData, OS); - if (NStages > maxStages) { - maxStages = NStages; +void DFAPacketizerEmitter::createScheduleClasses(unsigned ItineraryIdx, + const RecVec &Itineraries) { + unsigned Idx = 0; + for (Record *Itinerary : Itineraries) { + if (!Itinerary) { + ScheduleClasses.push_back({ItineraryIdx, Idx++, 0, ResourceVector{}}); + continue; } - numInsnClasses++; + ResourceVector Resources = getResourcesForItinerary(Itinerary); + ScheduleClasses.push_back( + {ItineraryIdx, Idx++, UniqueResources.insert(Resources), Resources}); } - return numInsnClasses; } // // Run the worklist algorithm to generate the DFA. // void DFAPacketizerEmitter::run(raw_ostream &OS) { - // Collect processor iteraries. - std::vector<Record*> ProcItinList = - Records.getAllDerivedDefinitions("ProcessorItineraries"); - - // - // Collect the Functional units. - // - std::map<std::string, unsigned> FUNameToBitsMap; - int maxResources = 0; - collectAllFuncUnits(ProcItinList, - FUNameToBitsMap, maxResources, OS); - - // - // Collect the Combo Functional units. - // - std::map<unsigned, unsigned> ComboBitToBitsMap; - std::vector<Record*> ComboFuncList = - Records.getAllDerivedDefinitions("ComboFuncUnits"); - int numCombos = collectAllComboFuncs(ComboFuncList, - FUNameToBitsMap, ComboBitToBitsMap, OS); - - // - // Collect the itineraries. - // - int maxStages = 0; - int numInsnClasses = 0; - for (unsigned i = 0, N = ProcItinList.size(); i < N; i++) { - Record *Proc = ProcItinList[i]; + OS << "\n" + << "#include \"llvm/CodeGen/DFAPacketizer.h\"\n"; + OS << "namespace llvm {\n"; - // Get processor itinerary name. - const std::string &ProcName = Proc->getName(); + CodeGenTarget CGT(Records); + CodeGenSchedModels CGS(Records, CGT); - // Skip default. - if (ProcName == "NoItineraries") - continue; + std::unordered_map<std::string, std::vector<const CodeGenProcModel *>> + ItinsByNamespace; + for (const CodeGenProcModel &ProcModel : CGS.procModels()) { + if (ProcModel.hasItineraries()) { + auto NS = ProcModel.ItinsDef->getValueAsString("PacketizerNamespace"); + ItinsByNamespace[NS].push_back(&ProcModel); + } + } - // Sanity check for at least one instruction itinerary class. - unsigned NItinClasses = - Records.getAllDerivedDefinitions("InstrItinClass").size(); - if (NItinClasses == 0) - return; + for (auto &KV : ItinsByNamespace) + emitForItineraries(OS, KV.second, KV.first); + OS << "} // end namespace llvm\n"; +} - // Get itinerary data list. - std::vector<Record*> ItinDataList = Proc->getValueAsListOfDefs("IID"); +void DFAPacketizerEmitter::emitForItineraries( + raw_ostream &OS, std::vector<const CodeGenProcModel *> &ProcModels, + std::string DFAName) { + OS << "} // end namespace llvm\n\n"; + OS << "namespace {\n"; + collectAllFuncUnits(ProcModels); + collectAllComboFuncs(Records.getAllDerivedDefinitions("ComboFuncUnits")); - // Collect all instruction classes - numInsnClasses += collectAllInsnClasses(ProcName, ProcItinList, - FUNameToBitsMap, ItinDataList, maxStages, OS); + // Collect the itineraries. + DenseMap<const CodeGenProcModel *, unsigned> ProcModelStartIdx; + for (const CodeGenProcModel *Model : ProcModels) { + assert(Model->hasItineraries()); + ProcModelStartIdx[Model] = ScheduleClasses.size(); + createScheduleClasses(Model->Index, Model->ItinDefList); } - // - // Run a worklist algorithm to generate the DFA. - // - DFA D; - const State *Initial = &D.newState(); - Initial->isInitial = true; - Initial->stateInfo.insert(0x0); - SmallVector<const State*, 32> WorkList; - std::map<std::set<unsigned>, const State*> Visited; - - WorkList.push_back(Initial); - - // - // Worklist algorithm to create a DFA for processor resource tracking. - // C = {set of InsnClasses} - // Begin with initial node in worklist. Initial node does not have - // any consumed resources, - // ResourceState = 0x0 - // Visited = {} - // While worklist != empty - // S = first element of worklist - // For every instruction class C - // if we can accommodate C in S: - // S' = state with resource states = {S Union C} - // Add a new transition: S x C -> S' - // If S' is not in Visited: - // Add S' to worklist - // Add S' to Visited - // - while (!WorkList.empty()) { - const State *current = WorkList.pop_back_val(); - LLVM_DEBUG({ - dbgs() << "---------------------\n"; - dbgs() << "Processing state: " << current->stateNum << " - "; - dbgsStateInfo(current->stateInfo); - dbgs() << "\n"; - }); - for (unsigned i = 0; i < allInsnClasses.size(); i++) { - std::vector<unsigned> InsnClass = allInsnClasses[i]; - LLVM_DEBUG({ - dbgs() << i << " "; - dbgsInsnClass(InsnClass); - dbgs() << "\n"; - }); - - std::set<unsigned> NewStateResources; - // - // If we haven't already created a transition for this input - // and the state can accommodate this InsnClass, create a transition. - // - if (!current->hasTransition(InsnClass) && - current->canMaybeAddInsnClass(InsnClass, ComboBitToBitsMap)) { - const State *NewState = nullptr; - current->AddInsnClass(InsnClass, ComboBitToBitsMap, NewStateResources); - if (NewStateResources.empty()) { - LLVM_DEBUG(dbgs() << " Skipped - no new states generated\n"); - continue; - } - - LLVM_DEBUG({ - dbgs() << "\t"; - dbgsStateInfo(NewStateResources); - dbgs() << "\n"; - }); - - // - // If we have seen this state before, then do not create a new state. - // - auto VI = Visited.find(NewStateResources); - if (VI != Visited.end()) { - NewState = VI->second; - LLVM_DEBUG({ - dbgs() << "\tFound existing state: " << NewState->stateNum - << " - "; - dbgsStateInfo(NewState->stateInfo); - dbgs() << "\n"; - }); - } else { - NewState = &D.newState(); - NewState->stateInfo = NewStateResources; - Visited[NewStateResources] = NewState; - WorkList.push_back(NewState); - LLVM_DEBUG({ - dbgs() << "\tAccepted new state: " << NewState->stateNum << " - "; - dbgsStateInfo(NewState->stateInfo); - dbgs() << "\n"; - }); + // Output the mapping from ScheduleClass to ResourcesIdx. + unsigned Idx = 0; + OS << "unsigned " << TargetName << DFAName << "ResourceIndices[] = {"; + for (const ScheduleClass &SC : ScheduleClasses) { + if (Idx++ % 32 == 0) + OS << "\n "; + OS << SC.ResourcesIdx << ", "; + } + OS << "\n};\n\n"; + + // And the mapping from Itinerary index into the previous table. + OS << "unsigned " << TargetName << DFAName + << "ProcResourceIndexStart[] = {\n"; + OS << " 0, // NoSchedModel\n"; + for (const CodeGenProcModel *Model : ProcModels) { + OS << " " << ProcModelStartIdx[Model] << ", // " << Model->ModelName + << "\n"; + } + OS << ScheduleClasses.size() << "\n};\n\n"; + + // The type of a state in the nondeterministic automaton we're defining. + using NfaStateTy = uint64_t; + + // Given a resource state, return all resource states by applying + // InsnClass. + auto applyInsnClass = [&](const ResourceVector &InsnClass, + NfaStateTy State) -> std::deque<NfaStateTy> { + std::deque<NfaStateTy> V(1, State); + // Apply every stage in the class individually. + for (NfaStateTy Stage : InsnClass) { + // Apply this stage to every existing member of V in turn. + size_t Sz = V.size(); + for (unsigned I = 0; I < Sz; ++I) { + NfaStateTy S = V.front(); + V.pop_front(); + + // For this stage, state combination, try all possible resources. + for (unsigned J = 0; J < DFA_MAX_RESOURCES; ++J) { + NfaStateTy ResourceMask = 1ULL << J; + if ((ResourceMask & Stage) == 0) + // This resource isn't required by this stage. + continue; + NfaStateTy Combo = ComboBitToBitsMap[ResourceMask]; + if (Combo && ((~S & Combo) != Combo)) + // This combo units bits are not available. + continue; + NfaStateTy ResultingResourceState = S | ResourceMask | Combo; + if (ResultingResourceState == S) + continue; + V.push_back(ResultingResourceState); } - - current->addTransition(InsnClass, NewState); + } + } + return V; + }; + + // Given a resource state, return a quick (conservative) guess as to whether + // InsnClass can be applied. This is a filter for the more heavyweight + // applyInsnClass. + auto canApplyInsnClass = [](const ResourceVector &InsnClass, + NfaStateTy State) -> bool { + for (NfaStateTy Resources : InsnClass) { + if ((State | Resources) == State) + return false; + } + return true; + }; + + DfaEmitter Emitter; + std::deque<NfaStateTy> Worklist(1, 0); + std::set<NfaStateTy> SeenStates; + SeenStates.insert(Worklist.front()); + while (!Worklist.empty()) { + NfaStateTy State = Worklist.front(); + Worklist.pop_front(); + for (const ResourceVector &Resources : UniqueResources) { + if (!canApplyInsnClass(Resources, State)) + continue; + unsigned ResourcesID = UniqueResources.idFor(Resources); + for (uint64_t NewState : applyInsnClass(Resources, State)) { + if (SeenStates.emplace(NewState).second) + Worklist.emplace_back(NewState); + Emitter.addTransition(State, NewState, ResourcesID); } } } - // Print out the table. - D.writeTableAndAPI(OS, TargetName, - numInsnClasses, maxResources, numCombos, maxStages); + std::string TargetAndDFAName = TargetName + DFAName; + Emitter.emit(TargetAndDFAName, OS); + OS << "} // end anonymous namespace\n\n"; + + std::string SubTargetClassName = TargetName + "GenSubtargetInfo"; + OS << "namespace llvm {\n"; + OS << "DFAPacketizer *" << SubTargetClassName << "::" + << "create" << DFAName + << "DFAPacketizer(const InstrItineraryData *IID) const {\n" + << " static Automaton<uint64_t> A(ArrayRef<" << TargetAndDFAName + << "Transition>(" << TargetAndDFAName << "Transitions), " + << TargetAndDFAName << "TransitionInfo);\n" + << " unsigned ProcResIdxStart = " << TargetAndDFAName + << "ProcResourceIndexStart[IID->SchedModel.ProcID];\n" + << " unsigned ProcResIdxNum = " << TargetAndDFAName + << "ProcResourceIndexStart[IID->SchedModel.ProcID + 1] - " + "ProcResIdxStart;\n" + << " return new DFAPacketizer(IID, A, {&" << TargetAndDFAName + << "ResourceIndices[ProcResIdxStart], ProcResIdxNum});\n" + << "\n}\n\n"; } namespace llvm { diff --git a/contrib/llvm-project/llvm/utils/TableGen/DisassemblerEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DisassemblerEmitter.cpp index 9e75c7fba77b..0002b0e14db6 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/DisassemblerEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/DisassemblerEmitter.cpp @@ -153,4 +153,4 @@ void EmitDisassembler(RecordKeeper &Records, raw_ostream &OS) { "MCDisassembler::Success", "MCDisassembler::Fail", ""); } -} // End llvm namespace +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp index f5e975d2e5ae..21ec5897ea50 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp @@ -13,6 +13,7 @@ #include "CodeGenInstruction.h" #include "CodeGenTarget.h" +#include "InfoByHwMode.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/CachedHashString.h" @@ -64,9 +65,10 @@ struct OperandInfo { std::vector<EncodingField> Fields; std::string Decoder; bool HasCompleteDecoder; + uint64_t InitValue; OperandInfo(std::string D, bool HCD) - : Decoder(std::move(D)), HasCompleteDecoder(HCD) {} + : Decoder(std::move(D)), HasCompleteDecoder(HCD), InitValue(0) {} void addField(unsigned Base, unsigned Width, unsigned Offset) { Fields.push_back(EncodingField(Base, Width, Offset)); @@ -96,9 +98,11 @@ struct DecoderTableInfo { struct EncodingAndInst { const Record *EncodingDef; const CodeGenInstruction *Inst; + StringRef HwModeName; - EncodingAndInst(const Record *EncodingDef, const CodeGenInstruction *Inst) - : EncodingDef(EncodingDef), Inst(Inst) {} + EncodingAndInst(const Record *EncodingDef, const CodeGenInstruction *Inst, + StringRef HwModeName = "") + : EncodingDef(EncodingDef), Inst(Inst), HwModeName(HwModeName) {} }; struct EncodingIDAndOpcode { @@ -599,7 +603,7 @@ void Filter::recurse() { // Delegates to an inferior filter chooser for further processing on this // group of instructions whose segment values are variable. FilterChooserMap.insert( - std::make_pair(-1U, llvm::make_unique<FilterChooser>( + std::make_pair(-1U, std::make_unique<FilterChooser>( Owner->AllInstructions, VariableInstructions, Owner->Operands, BitValueArray, *Owner))); } @@ -625,7 +629,7 @@ void Filter::recurse() { // Delegates to an inferior filter chooser for further processing on this // category of instructions. FilterChooserMap.insert(std::make_pair( - Inst.first, llvm::make_unique<FilterChooser>( + Inst.first, std::make_unique<FilterChooser>( Owner->AllInstructions, Inst.second, Owner->Operands, BitValueArray, *Owner))); } @@ -1054,10 +1058,9 @@ unsigned FilterChooser::getIslands(std::vector<unsigned> &StartBits, // 1: Water (the bit value does not affect decoding) // 2: Island (well-known bit value needed for decoding) int State = 0; - int64_t Val = -1; for (unsigned i = 0; i < BitWidth; ++i) { - Val = Value(Insn[i]); + int64_t Val = Value(Insn[i]); bool Filtered = PositionFiltered(i); switch (State) { default: llvm_unreachable("Unreachable code!"); @@ -1103,12 +1106,15 @@ void FilterChooser::emitBinaryParser(raw_ostream &o, unsigned &Indentation, bool &OpHasCompleteDecoder) const { const std::string &Decoder = OpInfo.Decoder; - if (OpInfo.numFields() != 1) - o.indent(Indentation) << "tmp = 0;\n"; + if (OpInfo.numFields() != 1 || OpInfo.InitValue != 0) { + o.indent(Indentation) << "tmp = 0x"; + o.write_hex(OpInfo.InitValue); + o << ";\n"; + } for (const EncodingField &EF : OpInfo) { o.indent(Indentation) << "tmp "; - if (OpInfo.numFields() != 1) o << '|'; + if (OpInfo.numFields() != 1 || OpInfo.InitValue != 0) o << '|'; o << "= fieldFromInstruction" << "(insn, " << EF.Base << ", " << EF.Width << ')'; if (OpInfo.numFields() != 1 || EF.Offset != 0) @@ -2026,6 +2032,16 @@ populateInstruction(CodeGenTarget &Target, const Record &EncodingDef, HasCompleteDecoderBit->getValue() : true; OperandInfo OpInfo(Decoder, HasCompleteDecoder); + + // Some bits of the operand may be required to be 1 depending on the + // instruction's encoding. Collect those bits. + if (const RecordVal *EncodedValue = EncodingDef.getValue(Op.second)) + if (const BitsInit *OpBits = dyn_cast<BitsInit>(EncodedValue->getValue())) + for (unsigned I = 0; I < OpBits->getNumBits(); ++I) + if (const BitInit *OpBit = dyn_cast<BitInit>(OpBits->getBit(I))) + if (OpBit->getValue()) + OpInfo.InitValue |= 1ULL << I; + unsigned Base = ~0U; unsigned Width = 0; unsigned Offset = 0; @@ -2368,12 +2384,50 @@ void FixedLenDecoderEmitter::run(raw_ostream &o) { Target.reverseBitsForLittleEndianEncoding(); // Parameterize the decoders based on namespace and instruction width. + std::set<StringRef> HwModeNames; const auto &NumberedInstructions = Target.getInstructionsByEnumValue(); NumberedEncodings.reserve(NumberedInstructions.size()); DenseMap<Record *, unsigned> IndexOfInstruction; + // First, collect all HwModes referenced by the target. for (const auto &NumberedInstruction : NumberedInstructions) { IndexOfInstruction[NumberedInstruction->TheDef] = NumberedEncodings.size(); - NumberedEncodings.emplace_back(NumberedInstruction->TheDef, NumberedInstruction); + + if (const RecordVal *RV = + NumberedInstruction->TheDef->getValue("EncodingInfos")) { + if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + const CodeGenHwModes &HWM = Target.getHwModes(); + EncodingInfoByHwMode EBM(DI->getDef(), HWM); + for (auto &KV : EBM.Map) + HwModeNames.insert(HWM.getMode(KV.first).Name); + } + } + } + + // If HwModeNames is empty, add the empty string so we always have one HwMode. + if (HwModeNames.empty()) + HwModeNames.insert(""); + + for (const auto &NumberedInstruction : NumberedInstructions) { + IndexOfInstruction[NumberedInstruction->TheDef] = NumberedEncodings.size(); + + if (const RecordVal *RV = + NumberedInstruction->TheDef->getValue("EncodingInfos")) { + if (DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + const CodeGenHwModes &HWM = Target.getHwModes(); + EncodingInfoByHwMode EBM(DI->getDef(), HWM); + for (auto &KV : EBM.Map) { + NumberedEncodings.emplace_back(KV.second, NumberedInstruction, + HWM.getMode(KV.first).Name); + HwModeNames.insert(HWM.getMode(KV.first).Name); + } + continue; + } + } + // This instruction is encoded the same on all HwModes. Emit it for all + // HwModes. + for (StringRef HwModeName : HwModeNames) + NumberedEncodings.emplace_back(NumberedInstruction->TheDef, + NumberedInstruction, HwModeName); } for (const auto &NumberedAlias : RK.getAllDerivedDefinitions("AdditionalEncoding")) NumberedEncodings.emplace_back( @@ -2401,13 +2455,19 @@ void FixedLenDecoderEmitter::run(raw_ostream &o) { NumInstructions++; NumEncodings++; - StringRef DecoderNamespace = EncodingDef->getValueAsString("DecoderNamespace"); + if (!Size) + continue; - if (Size) { - if (populateInstruction(Target, *EncodingDef, *Inst, i, Operands)) { - OpcMap[std::make_pair(DecoderNamespace, Size)].emplace_back(i, IndexOfInstruction.find(Def)->second); - } else - NumEncodingsOmitted++; + if (populateInstruction(Target, *EncodingDef, *Inst, i, Operands)) { + std::string DecoderNamespace = + EncodingDef->getValueAsString("DecoderNamespace"); + if (!NumberedEncodings[i].HwModeName.empty()) + DecoderNamespace += + std::string("_") + NumberedEncodings[i].HwModeName.str(); + OpcMap[std::make_pair(DecoderNamespace, Size)].emplace_back( + i, IndexOfInstruction.find(Def)->second); + } else { + NumEncodingsOmitted++; } } @@ -2451,7 +2511,7 @@ void FixedLenDecoderEmitter::run(raw_ostream &o) { // Emit the main entry point for the decoder, decodeInstruction(). emitDecodeInstruction(OS); - OS << "\n} // End llvm namespace\n"; + OS << "\n} // end namespace llvm\n"; } namespace llvm { diff --git a/contrib/llvm-project/llvm/utils/TableGen/GICombinerEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/GICombinerEmitter.cpp new file mode 100644 index 000000000000..34eb4edac8de --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GICombinerEmitter.cpp @@ -0,0 +1,1001 @@ +//===- GlobalCombinerEmitter.cpp - Generate a combiner --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Generate a combiner implementation for GlobalISel from a declarative +/// syntax +/// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/Timer.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/StringMatcher.h" +#include "llvm/TableGen/TableGenBackend.h" +#include "CodeGenTarget.h" +#include "GlobalISel/CodeExpander.h" +#include "GlobalISel/CodeExpansions.h" +#include "GlobalISel/GIMatchDag.h" +#include "GlobalISel/GIMatchTree.h" +#include <cstdint> + +using namespace llvm; + +#define DEBUG_TYPE "gicombiner-emitter" + +// FIXME: Use ALWAYS_ENABLED_STATISTIC once it's available. +unsigned NumPatternTotal = 0; +STATISTIC(NumPatternTotalStatistic, "Total number of patterns"); + +cl::OptionCategory + GICombinerEmitterCat("Options for -gen-global-isel-combiner"); +static cl::list<std::string> + SelectedCombiners("combiners", cl::desc("Emit the specified combiners"), + cl::cat(GICombinerEmitterCat), cl::CommaSeparated); +static cl::opt<bool> ShowExpansions( + "gicombiner-show-expansions", + cl::desc("Use C++ comments to indicate occurence of code expansion"), + cl::cat(GICombinerEmitterCat)); +static cl::opt<bool> StopAfterParse( + "gicombiner-stop-after-parse", + cl::desc("Stop processing after parsing rules and dump state"), + cl::cat(GICombinerEmitterCat)); +static cl::opt<bool> StopAfterBuild( + "gicombiner-stop-after-build", + cl::desc("Stop processing after building the match tree"), + cl::cat(GICombinerEmitterCat)); + +namespace { +typedef uint64_t RuleID; + +// We're going to be referencing the same small strings quite a lot for operand +// names and the like. Make their lifetime management simple with a global +// string table. +StringSet<> StrTab; + +StringRef insertStrTab(StringRef S) { + if (S.empty()) + return S; + return StrTab.insert(S).first->first(); +} + +class format_partition_name { + const GIMatchTree &Tree; + unsigned Idx; + +public: + format_partition_name(const GIMatchTree &Tree, unsigned Idx) + : Tree(Tree), Idx(Idx) {} + void print(raw_ostream &OS) const { + Tree.getPartitioner()->emitPartitionName(OS, Idx); + } +}; +raw_ostream &operator<<(raw_ostream &OS, const format_partition_name &Fmt) { + Fmt.print(OS); + return OS; +} + +/// Declares data that is passed from the match stage to the apply stage. +class MatchDataInfo { + /// The symbol used in the tablegen patterns + StringRef PatternSymbol; + /// The data type for the variable + StringRef Type; + /// The name of the variable as declared in the generated matcher. + std::string VariableName; + +public: + MatchDataInfo(StringRef PatternSymbol, StringRef Type, StringRef VariableName) + : PatternSymbol(PatternSymbol), Type(Type), VariableName(VariableName) {} + + StringRef getPatternSymbol() const { return PatternSymbol; }; + StringRef getType() const { return Type; }; + StringRef getVariableName() const { return VariableName; }; +}; + +class RootInfo { + StringRef PatternSymbol; + +public: + RootInfo(StringRef PatternSymbol) : PatternSymbol(PatternSymbol) {} + + StringRef getPatternSymbol() const { return PatternSymbol; } +}; + +class CombineRule { +public: + + using const_matchdata_iterator = std::vector<MatchDataInfo>::const_iterator; + + struct VarInfo { + const GIMatchDagInstr *N; + const GIMatchDagOperand *Op; + const DagInit *Matcher; + + public: + VarInfo(const GIMatchDagInstr *N, const GIMatchDagOperand *Op, + const DagInit *Matcher) + : N(N), Op(Op), Matcher(Matcher) {} + }; + +protected: + /// A unique ID for this rule + /// ID's are used for debugging and run-time disabling of rules among other + /// things. + RuleID ID; + + /// A unique ID that can be used for anonymous objects belonging to this rule. + /// Used to create unique names in makeNameForAnon*() without making tests + /// overly fragile. + unsigned UID = 0; + + /// The record defining this rule. + const Record &TheDef; + + /// The roots of a match. These are the leaves of the DAG that are closest to + /// the end of the function. I.e. the nodes that are encountered without + /// following any edges of the DAG described by the pattern as we work our way + /// from the bottom of the function to the top. + std::vector<RootInfo> Roots; + + GIMatchDag MatchDag; + + /// A block of arbitrary C++ to finish testing the match. + /// FIXME: This is a temporary measure until we have actual pattern matching + const CodeInit *MatchingFixupCode = nullptr; + + /// The MatchData defined by the match stage and required by the apply stage. + /// This allows the plumbing of arbitrary data from C++ predicates between the + /// stages. + /// + /// For example, suppose you have: + /// %A = <some-constant-expr> + /// %0 = G_ADD %1, %A + /// you could define a GIMatchPredicate that walks %A, constant folds as much + /// as possible and returns an APInt containing the discovered constant. You + /// could then declare: + /// def apint : GIDefMatchData<"APInt">; + /// add it to the rule with: + /// (defs root:$root, apint:$constant) + /// evaluate it in the pattern with a C++ function that takes a + /// MachineOperand& and an APInt& with: + /// (match [{MIR %root = G_ADD %0, %A }], + /// (constantfold operand:$A, apint:$constant)) + /// and finally use it in the apply stage with: + /// (apply (create_operand + /// [{ MachineOperand::CreateImm(${constant}.getZExtValue()); + /// ]}, apint:$constant), + /// [{MIR %root = FOO %0, %constant }]) + std::vector<MatchDataInfo> MatchDataDecls; + + void declareMatchData(StringRef PatternSymbol, StringRef Type, + StringRef VarName); + + bool parseInstructionMatcher(const CodeGenTarget &Target, StringInit *ArgName, + const Init &Arg, + StringMap<std::vector<VarInfo>> &NamedEdgeDefs, + StringMap<std::vector<VarInfo>> &NamedEdgeUses); + bool parseWipMatchOpcodeMatcher(const CodeGenTarget &Target, + StringInit *ArgName, const Init &Arg); + +public: + CombineRule(const CodeGenTarget &Target, GIMatchDagContext &Ctx, RuleID ID, + const Record &R) + : ID(ID), TheDef(R), MatchDag(Ctx) {} + CombineRule(const CombineRule &) = delete; + + bool parseDefs(); + bool parseMatcher(const CodeGenTarget &Target); + + RuleID getID() const { return ID; } + unsigned allocUID() { return UID++; } + StringRef getName() const { return TheDef.getName(); } + const Record &getDef() const { return TheDef; } + const CodeInit *getMatchingFixupCode() const { return MatchingFixupCode; } + size_t getNumRoots() const { return Roots.size(); } + + GIMatchDag &getMatchDag() { return MatchDag; } + const GIMatchDag &getMatchDag() const { return MatchDag; } + + using const_root_iterator = std::vector<RootInfo>::const_iterator; + const_root_iterator roots_begin() const { return Roots.begin(); } + const_root_iterator roots_end() const { return Roots.end(); } + iterator_range<const_root_iterator> roots() const { + return llvm::make_range(Roots.begin(), Roots.end()); + } + + iterator_range<const_matchdata_iterator> matchdata_decls() const { + return make_range(MatchDataDecls.begin(), MatchDataDecls.end()); + } + + /// Export expansions for this rule + void declareExpansions(CodeExpansions &Expansions) const { + for (const auto &I : matchdata_decls()) + Expansions.declare(I.getPatternSymbol(), I.getVariableName()); + } + + /// The matcher will begin from the roots and will perform the match by + /// traversing the edges to cover the whole DAG. This function reverses DAG + /// edges such that everything is reachable from a root. This is part of the + /// preparation work for flattening the DAG into a tree. + void reorientToRoots() { + SmallSet<const GIMatchDagInstr *, 5> Roots; + SmallSet<const GIMatchDagInstr *, 5> Visited; + SmallSet<GIMatchDagEdge *, 20> EdgesRemaining; + + for (auto &I : MatchDag.roots()) { + Roots.insert(I); + Visited.insert(I); + } + for (auto &I : MatchDag.edges()) + EdgesRemaining.insert(I); + + bool Progressed = false; + SmallSet<GIMatchDagEdge *, 20> EdgesToRemove; + while (!EdgesRemaining.empty()) { + for (auto EI = EdgesRemaining.begin(), EE = EdgesRemaining.end(); + EI != EE; ++EI) { + if (Visited.count((*EI)->getFromMI())) { + if (Roots.count((*EI)->getToMI())) + PrintError(TheDef.getLoc(), "One or more roots are unnecessary"); + Visited.insert((*EI)->getToMI()); + EdgesToRemove.insert(*EI); + Progressed = true; + } + } + for (GIMatchDagEdge *ToRemove : EdgesToRemove) + EdgesRemaining.erase(ToRemove); + EdgesToRemove.clear(); + + for (auto EI = EdgesRemaining.begin(), EE = EdgesRemaining.end(); + EI != EE; ++EI) { + if (Visited.count((*EI)->getToMI())) { + (*EI)->reverse(); + Visited.insert((*EI)->getToMI()); + EdgesToRemove.insert(*EI); + Progressed = true; + } + for (GIMatchDagEdge *ToRemove : EdgesToRemove) + EdgesRemaining.erase(ToRemove); + EdgesToRemove.clear(); + } + + if (!Progressed) { + LLVM_DEBUG(dbgs() << "No progress\n"); + return; + } + Progressed = false; + } + } +}; + +/// A convenience function to check that an Init refers to a specific def. This +/// is primarily useful for testing for defs and similar in DagInit's since +/// DagInit's support any type inside them. +static bool isSpecificDef(const Init &N, StringRef Def) { + if (const DefInit *OpI = dyn_cast<DefInit>(&N)) + if (OpI->getDef()->getName() == Def) + return true; + return false; +} + +/// A convenience function to check that an Init refers to a def that is a +/// subclass of the given class and coerce it to a def if it is. This is +/// primarily useful for testing for subclasses of GIMatchKind and similar in +/// DagInit's since DagInit's support any type inside them. +static Record *getDefOfSubClass(const Init &N, StringRef Cls) { + if (const DefInit *OpI = dyn_cast<DefInit>(&N)) + if (OpI->getDef()->isSubClassOf(Cls)) + return OpI->getDef(); + return nullptr; +} + +/// A convenience function to check that an Init refers to a dag whose operator +/// is a specific def and coerce it to a dag if it is. This is primarily useful +/// for testing for subclasses of GIMatchKind and similar in DagInit's since +/// DagInit's support any type inside them. +static const DagInit *getDagWithSpecificOperator(const Init &N, + StringRef Name) { + if (const DagInit *I = dyn_cast<DagInit>(&N)) + if (I->getNumArgs() > 0) + if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator())) + if (OpI->getDef()->getName() == Name) + return I; + return nullptr; +} + +/// A convenience function to check that an Init refers to a dag whose operator +/// is a def that is a subclass of the given class and coerce it to a dag if it +/// is. This is primarily useful for testing for subclasses of GIMatchKind and +/// similar in DagInit's since DagInit's support any type inside them. +static const DagInit *getDagWithOperatorOfSubClass(const Init &N, + StringRef Cls) { + if (const DagInit *I = dyn_cast<DagInit>(&N)) + if (I->getNumArgs() > 0) + if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator())) + if (OpI->getDef()->isSubClassOf(Cls)) + return I; + return nullptr; +} + +StringRef makeNameForAnonInstr(CombineRule &Rule) { + return insertStrTab(to_string( + format("__anon%" PRIu64 "_%u", Rule.getID(), Rule.allocUID()))); +} + +StringRef makeDebugName(CombineRule &Rule, StringRef Name) { + return insertStrTab(Name.empty() ? makeNameForAnonInstr(Rule) : StringRef(Name)); +} + +StringRef makeNameForAnonPredicate(CombineRule &Rule) { + return insertStrTab(to_string( + format("__anonpred%" PRIu64 "_%u", Rule.getID(), Rule.allocUID()))); +} + +void CombineRule::declareMatchData(StringRef PatternSymbol, StringRef Type, + StringRef VarName) { + MatchDataDecls.emplace_back(PatternSymbol, Type, VarName); +} + +bool CombineRule::parseDefs() { + NamedRegionTimer T("parseDefs", "Time spent parsing the defs", "Rule Parsing", + "Time spent on rule parsing", TimeRegions); + DagInit *Defs = TheDef.getValueAsDag("Defs"); + + if (Defs->getOperatorAsDef(TheDef.getLoc())->getName() != "defs") { + PrintError(TheDef.getLoc(), "Expected defs operator"); + return false; + } + + for (unsigned I = 0, E = Defs->getNumArgs(); I < E; ++I) { + // Roots should be collected into Roots + if (isSpecificDef(*Defs->getArg(I), "root")) { + Roots.emplace_back(Defs->getArgNameStr(I)); + continue; + } + + // Subclasses of GIDefMatchData should declare that this rule needs to pass + // data from the match stage to the apply stage, and ensure that the + // generated matcher has a suitable variable for it to do so. + if (Record *MatchDataRec = + getDefOfSubClass(*Defs->getArg(I), "GIDefMatchData")) { + declareMatchData(Defs->getArgNameStr(I), + MatchDataRec->getValueAsString("Type"), + llvm::to_string(llvm::format("MatchData%" PRIu64, ID))); + continue; + } + + // Otherwise emit an appropriate error message. + if (getDefOfSubClass(*Defs->getArg(I), "GIDefKind")) + PrintError(TheDef.getLoc(), + "This GIDefKind not implemented in tablegen"); + else if (getDefOfSubClass(*Defs->getArg(I), "GIDefKindWithArgs")) + PrintError(TheDef.getLoc(), + "This GIDefKindWithArgs not implemented in tablegen"); + else + PrintError(TheDef.getLoc(), + "Expected a subclass of GIDefKind or a sub-dag whose " + "operator is of type GIDefKindWithArgs"); + return false; + } + + if (Roots.empty()) { + PrintError(TheDef.getLoc(), "Combine rules must have at least one root"); + return false; + } + return true; +} + +// Parse an (Instruction $a:Arg1, $b:Arg2, ...) matcher. Edges are formed +// between matching operand names between different matchers. +bool CombineRule::parseInstructionMatcher( + const CodeGenTarget &Target, StringInit *ArgName, const Init &Arg, + StringMap<std::vector<VarInfo>> &NamedEdgeDefs, + StringMap<std::vector<VarInfo>> &NamedEdgeUses) { + if (const DagInit *Matcher = + getDagWithOperatorOfSubClass(Arg, "Instruction")) { + auto &Instr = + Target.getInstruction(Matcher->getOperatorAsDef(TheDef.getLoc())); + + StringRef Name = ArgName ? ArgName->getValue() : ""; + + GIMatchDagInstr *N = + MatchDag.addInstrNode(makeDebugName(*this, Name), insertStrTab(Name), + MatchDag.getContext().makeOperandList(Instr)); + + N->setOpcodeAnnotation(&Instr); + const auto &P = MatchDag.addPredicateNode<GIMatchDagOpcodePredicate>( + makeNameForAnonPredicate(*this), Instr); + MatchDag.addPredicateDependency(N, nullptr, P, &P->getOperandInfo()["mi"]); + unsigned OpIdx = 0; + for (const auto &NameInit : Matcher->getArgNames()) { + StringRef Name = insertStrTab(NameInit->getAsUnquotedString()); + if (Name.empty()) + continue; + N->assignNameToOperand(OpIdx, Name); + + // Record the endpoints of any named edges. We'll add the cartesian + // product of edges later. + const auto &InstrOperand = N->getOperandInfo()[OpIdx]; + if (InstrOperand.isDef()) { + NamedEdgeDefs.try_emplace(Name); + NamedEdgeDefs[Name].emplace_back(N, &InstrOperand, Matcher); + } else { + NamedEdgeUses.try_emplace(Name); + NamedEdgeUses[Name].emplace_back(N, &InstrOperand, Matcher); + } + + if (InstrOperand.isDef()) { + if (find_if(Roots, [&](const RootInfo &X) { + return X.getPatternSymbol() == Name; + }) != Roots.end()) { + N->setMatchRoot(); + } + } + + OpIdx++; + } + + return true; + } + return false; +} + +// Parse the wip_match_opcode placeholder that's temporarily present in lieu of +// implementing macros or choices between two matchers. +bool CombineRule::parseWipMatchOpcodeMatcher(const CodeGenTarget &Target, + StringInit *ArgName, + const Init &Arg) { + if (const DagInit *Matcher = + getDagWithSpecificOperator(Arg, "wip_match_opcode")) { + StringRef Name = ArgName ? ArgName->getValue() : ""; + + GIMatchDagInstr *N = + MatchDag.addInstrNode(makeDebugName(*this, Name), insertStrTab(Name), + MatchDag.getContext().makeEmptyOperandList()); + + if (find_if(Roots, [&](const RootInfo &X) { + return ArgName && X.getPatternSymbol() == ArgName->getValue(); + }) != Roots.end()) { + N->setMatchRoot(); + } + + const auto &P = MatchDag.addPredicateNode<GIMatchDagOneOfOpcodesPredicate>( + makeNameForAnonPredicate(*this)); + MatchDag.addPredicateDependency(N, nullptr, P, &P->getOperandInfo()["mi"]); + // Each argument is an opcode that will pass this predicate. Add them all to + // the predicate implementation + for (const auto &Arg : Matcher->getArgs()) { + Record *OpcodeDef = getDefOfSubClass(*Arg, "Instruction"); + if (OpcodeDef) { + P->addOpcode(&Target.getInstruction(OpcodeDef)); + continue; + } + PrintError(TheDef.getLoc(), + "Arguments to wip_match_opcode must be instructions"); + return false; + } + return true; + } + return false; +} +bool CombineRule::parseMatcher(const CodeGenTarget &Target) { + NamedRegionTimer T("parseMatcher", "Time spent parsing the matcher", + "Rule Parsing", "Time spent on rule parsing", TimeRegions); + StringMap<std::vector<VarInfo>> NamedEdgeDefs; + StringMap<std::vector<VarInfo>> NamedEdgeUses; + DagInit *Matchers = TheDef.getValueAsDag("Match"); + + if (Matchers->getOperatorAsDef(TheDef.getLoc())->getName() != "match") { + PrintError(TheDef.getLoc(), "Expected match operator"); + return false; + } + + if (Matchers->getNumArgs() == 0) { + PrintError(TheDef.getLoc(), "Matcher is empty"); + return false; + } + + // The match section consists of a list of matchers and predicates. Parse each + // one and add the equivalent GIMatchDag nodes, predicates, and edges. + for (unsigned I = 0; I < Matchers->getNumArgs(); ++I) { + if (parseInstructionMatcher(Target, Matchers->getArgName(I), + *Matchers->getArg(I), NamedEdgeDefs, + NamedEdgeUses)) + continue; + + if (parseWipMatchOpcodeMatcher(Target, Matchers->getArgName(I), + *Matchers->getArg(I))) + continue; + + + // Parse arbitrary C++ code we have in lieu of supporting MIR matching + if (const CodeInit *CodeI = dyn_cast<CodeInit>(Matchers->getArg(I))) { + assert(!MatchingFixupCode && + "Only one block of arbitrary code is currently permitted"); + MatchingFixupCode = CodeI; + MatchDag.setHasPostMatchPredicate(true); + continue; + } + + PrintError(TheDef.getLoc(), + "Expected a subclass of GIMatchKind or a sub-dag whose " + "operator is either of a GIMatchKindWithArgs or Instruction"); + PrintNote("Pattern was `" + Matchers->getArg(I)->getAsString() + "'"); + return false; + } + + // Add the cartesian product of use -> def edges. + bool FailedToAddEdges = false; + for (const auto &NameAndDefs : NamedEdgeDefs) { + if (NameAndDefs.getValue().size() > 1) { + PrintError(TheDef.getLoc(), + "Two different MachineInstrs cannot def the same vreg"); + for (const auto &NameAndDefOp : NameAndDefs.getValue()) + PrintNote("in " + to_string(*NameAndDefOp.N) + " created from " + + to_string(*NameAndDefOp.Matcher) + ""); + FailedToAddEdges = true; + } + const auto &Uses = NamedEdgeUses[NameAndDefs.getKey()]; + for (const VarInfo &DefVar : NameAndDefs.getValue()) { + for (const VarInfo &UseVar : Uses) { + MatchDag.addEdge(insertStrTab(NameAndDefs.getKey()), UseVar.N, UseVar.Op, + DefVar.N, DefVar.Op); + } + } + } + if (FailedToAddEdges) + return false; + + // If a variable is referenced in multiple use contexts then we need a + // predicate to confirm they are the same operand. We can elide this if it's + // also referenced in a def context and we're traversing the def-use chain + // from the def to the uses but we can't know which direction we're going + // until after reorientToRoots(). + for (const auto &NameAndUses : NamedEdgeUses) { + const auto &Uses = NameAndUses.getValue(); + if (Uses.size() > 1) { + const auto &LeadingVar = Uses.front(); + for (const auto &Var : ArrayRef<VarInfo>(Uses).drop_front()) { + // Add a predicate for each pair until we've covered the whole + // equivalence set. We could test the whole set in a single predicate + // but that means we can't test any equivalence until all the MO's are + // available which can lead to wasted work matching the DAG when this + // predicate can already be seen to have failed. + // + // We have a similar problem due to the need to wait for a particular MO + // before being able to test any of them. However, that is mitigated by + // the order in which we build the DAG. We build from the roots outwards + // so by using the first recorded use in all the predicates, we are + // making the dependency on one of the earliest visited references in + // the DAG. It's not guaranteed once the generated matcher is optimized + // (because the factoring the common portions of rules might change the + // visit order) but this should mean that these predicates depend on the + // first MO to become available. + const auto &P = MatchDag.addPredicateNode<GIMatchDagSameMOPredicate>( + makeNameForAnonPredicate(*this)); + MatchDag.addPredicateDependency(LeadingVar.N, LeadingVar.Op, P, + &P->getOperandInfo()["mi0"]); + MatchDag.addPredicateDependency(Var.N, Var.Op, P, + &P->getOperandInfo()["mi1"]); + } + } + } + return true; +} + +class GICombinerEmitter { + StringRef Name; + const CodeGenTarget &Target; + Record *Combiner; + std::vector<std::unique_ptr<CombineRule>> Rules; + GIMatchDagContext MatchDagCtx; + + std::unique_ptr<CombineRule> makeCombineRule(const Record &R); + + void gatherRules(std::vector<std::unique_ptr<CombineRule>> &ActiveRules, + const std::vector<Record *> &&RulesAndGroups); + +public: + explicit GICombinerEmitter(RecordKeeper &RK, const CodeGenTarget &Target, + StringRef Name, Record *Combiner); + ~GICombinerEmitter() {} + + StringRef getClassName() const { + return Combiner->getValueAsString("Classname"); + } + void run(raw_ostream &OS); + + /// Emit the name matcher (guarded by #ifndef NDEBUG) used to disable rules in + /// response to the generated cl::opt. + void emitNameMatcher(raw_ostream &OS) const; + + void generateDeclarationsCodeForTree(raw_ostream &OS, const GIMatchTree &Tree) const; + void generateCodeForTree(raw_ostream &OS, const GIMatchTree &Tree, + StringRef Indent) const; +}; + +GICombinerEmitter::GICombinerEmitter(RecordKeeper &RK, + const CodeGenTarget &Target, + StringRef Name, Record *Combiner) + : Name(Name), Target(Target), Combiner(Combiner) {} + +void GICombinerEmitter::emitNameMatcher(raw_ostream &OS) const { + std::vector<std::pair<std::string, std::string>> Cases; + Cases.reserve(Rules.size()); + + for (const CombineRule &EnumeratedRule : make_pointee_range(Rules)) { + std::string Code; + raw_string_ostream SS(Code); + SS << "return " << EnumeratedRule.getID() << ";\n"; + Cases.push_back(std::make_pair(EnumeratedRule.getName(), SS.str())); + } + + OS << "static Optional<uint64_t> getRuleIdxForIdentifier(StringRef " + "RuleIdentifier) {\n" + << " uint64_t I;\n" + << " // getAtInteger(...) returns false on success\n" + << " bool Parsed = !RuleIdentifier.getAsInteger(0, I);\n" + << " if (Parsed)\n" + << " return I;\n\n" + << "#ifndef NDEBUG\n"; + StringMatcher Matcher("RuleIdentifier", Cases, OS); + Matcher.Emit(); + OS << "#endif // ifndef NDEBUG\n\n" + << " return None;\n" + << "}\n"; +} + +std::unique_ptr<CombineRule> +GICombinerEmitter::makeCombineRule(const Record &TheDef) { + std::unique_ptr<CombineRule> Rule = + std::make_unique<CombineRule>(Target, MatchDagCtx, NumPatternTotal, TheDef); + + if (!Rule->parseDefs()) + return nullptr; + if (!Rule->parseMatcher(Target)) + return nullptr; + + Rule->reorientToRoots(); + + LLVM_DEBUG({ + dbgs() << "Parsed rule defs/match for '" << Rule->getName() << "'\n"; + Rule->getMatchDag().dump(); + Rule->getMatchDag().writeDOTGraph(dbgs(), Rule->getName()); + }); + if (StopAfterParse) + return Rule; + + // For now, don't support traversing from def to use. We'll come back to + // this later once we have the algorithm changes to support it. + bool EmittedDefToUseError = false; + for (const auto &E : Rule->getMatchDag().edges()) { + if (E->isDefToUse()) { + if (!EmittedDefToUseError) { + PrintError( + TheDef.getLoc(), + "Generated state machine cannot lookup uses from a def (yet)"); + EmittedDefToUseError = true; + } + PrintNote("Node " + to_string(*E->getFromMI())); + PrintNote("Node " + to_string(*E->getToMI())); + PrintNote("Edge " + to_string(*E)); + } + } + if (EmittedDefToUseError) + return nullptr; + + // For now, don't support multi-root rules. We'll come back to this later + // once we have the algorithm changes to support it. + if (Rule->getNumRoots() > 1) { + PrintError(TheDef.getLoc(), "Multi-root matches are not supported (yet)"); + return nullptr; + } + return Rule; +} + +/// Recurse into GICombineGroup's and flatten the ruleset into a simple list. +void GICombinerEmitter::gatherRules( + std::vector<std::unique_ptr<CombineRule>> &ActiveRules, + const std::vector<Record *> &&RulesAndGroups) { + for (Record *R : RulesAndGroups) { + if (R->isValueUnset("Rules")) { + std::unique_ptr<CombineRule> Rule = makeCombineRule(*R); + if (Rule == nullptr) { + PrintError(R->getLoc(), "Failed to parse rule"); + continue; + } + ActiveRules.emplace_back(std::move(Rule)); + ++NumPatternTotal; + } else + gatherRules(ActiveRules, R->getValueAsListOfDefs("Rules")); + } +} + +void GICombinerEmitter::generateCodeForTree(raw_ostream &OS, + const GIMatchTree &Tree, + StringRef Indent) const { + if (Tree.getPartitioner() != nullptr) { + Tree.getPartitioner()->generatePartitionSelectorCode(OS, Indent); + for (const auto &EnumChildren : enumerate(Tree.children())) { + OS << Indent << "if (Partition == " << EnumChildren.index() << " /* " + << format_partition_name(Tree, EnumChildren.index()) << " */) {\n"; + generateCodeForTree(OS, EnumChildren.value(), (Indent + " ").str()); + OS << Indent << "}\n"; + } + return; + } + + bool AnyFullyTested = false; + for (const auto &Leaf : Tree.possible_leaves()) { + OS << Indent << "// Leaf name: " << Leaf.getName() << "\n"; + + const CombineRule *Rule = Leaf.getTargetData<CombineRule>(); + const Record &RuleDef = Rule->getDef(); + + OS << Indent << "// Rule: " << RuleDef.getName() << "\n" + << Indent << "if (!isRuleDisabled(" << Rule->getID() << ")) {\n"; + + CodeExpansions Expansions; + for (const auto &VarBinding : Leaf.var_bindings()) { + if (VarBinding.isInstr()) + Expansions.declare(VarBinding.getName(), + "MIs[" + to_string(VarBinding.getInstrID()) + "]"); + else + Expansions.declare(VarBinding.getName(), + "MIs[" + to_string(VarBinding.getInstrID()) + + "]->getOperand(" + + to_string(VarBinding.getOpIdx()) + ")"); + } + Rule->declareExpansions(Expansions); + + DagInit *Applyer = RuleDef.getValueAsDag("Apply"); + if (Applyer->getOperatorAsDef(RuleDef.getLoc())->getName() != + "apply") { + PrintError(RuleDef.getLoc(), "Expected apply operator"); + return; + } + + OS << Indent << " if (1\n"; + + // Attempt to emit code for any untested predicates left over. Note that + // isFullyTested() will remain false even if we succeed here and therefore + // combine rule elision will not be performed. This is because we do not + // know if there's any connection between the predicates for each leaf and + // therefore can't tell if one makes another unreachable. Ideally, the + // partitioner(s) would be sufficiently complete to prevent us from having + // untested predicates left over. + for (const GIMatchDagPredicate *Predicate : Leaf.untested_predicates()) { + if (Predicate->generateCheckCode(OS, (Indent + " ").str(), + Expansions)) + continue; + PrintError(RuleDef.getLoc(), + "Unable to test predicate used in rule"); + PrintNote(SMLoc(), + "This indicates an incomplete implementation in tablegen"); + Predicate->print(errs()); + errs() << "\n"; + OS << Indent + << "llvm_unreachable(\"TableGen did not emit complete code for this " + "path\");\n"; + break; + } + + if (Rule->getMatchingFixupCode() && + !Rule->getMatchingFixupCode()->getValue().empty()) { + // FIXME: Single-use lambda's like this are a serious compile-time + // performance and memory issue. It's convenient for this early stage to + // defer some work to successive patches but we need to eliminate this + // before the ruleset grows to small-moderate size. Last time, it became + // a big problem for low-mem systems around the 500 rule mark but by the + // time we grow that large we should have merged the ISel match table + // mechanism with the Combiner. + OS << Indent << " && [&]() {\n" + << Indent << " " + << CodeExpander(Rule->getMatchingFixupCode()->getValue(), Expansions, + Rule->getMatchingFixupCode()->getLoc(), ShowExpansions) + << "\n" + << Indent << " return true;\n" + << Indent << " }()"; + } + OS << ") {\n" << Indent << " "; + + if (const CodeInit *Code = dyn_cast<CodeInit>(Applyer->getArg(0))) { + OS << CodeExpander(Code->getAsUnquotedString(), Expansions, + Code->getLoc(), ShowExpansions) + << "\n" + << Indent << " return true;\n" + << Indent << " }\n"; + } else { + PrintError(RuleDef.getLoc(), "Expected apply code block"); + return; + } + + OS << Indent << "}\n"; + + assert(Leaf.isFullyTraversed()); + + // If we didn't have any predicates left over and we're not using the + // trap-door we have to support arbitrary C++ code while we're migrating to + // the declarative style then we know that subsequent leaves are + // unreachable. + if (Leaf.isFullyTested() && + (!Rule->getMatchingFixupCode() || + Rule->getMatchingFixupCode()->getValue().empty())) { + AnyFullyTested = true; + OS << Indent + << "llvm_unreachable(\"Combine rule elision was incorrect\");\n" + << Indent << "return false;\n"; + } + } + if (!AnyFullyTested) + OS << Indent << "return false;\n"; +} + +void GICombinerEmitter::run(raw_ostream &OS) { + gatherRules(Rules, Combiner->getValueAsListOfDefs("Rules")); + if (StopAfterParse) { + MatchDagCtx.print(errs()); + PrintNote(Combiner->getLoc(), + "Terminating due to -gicombiner-stop-after-parse"); + return; + } + if (ErrorsPrinted) + PrintFatalError(Combiner->getLoc(), "Failed to parse one or more rules"); + LLVM_DEBUG(dbgs() << "Optimizing tree for " << Rules.size() << " rules\n"); + std::unique_ptr<GIMatchTree> Tree; + { + NamedRegionTimer T("Optimize", "Time spent optimizing the combiner", + "Code Generation", "Time spent generating code", + TimeRegions); + + GIMatchTreeBuilder TreeBuilder(0); + for (const auto &Rule : Rules) { + bool HadARoot = false; + for (const auto &Root : enumerate(Rule->getMatchDag().roots())) { + TreeBuilder.addLeaf(Rule->getName(), Root.index(), Rule->getMatchDag(), + Rule.get()); + HadARoot = true; + } + if (!HadARoot) + PrintFatalError(Rule->getDef().getLoc(), "All rules must have a root"); + } + + Tree = TreeBuilder.run(); + } + if (StopAfterBuild) { + Tree->writeDOTGraph(outs()); + PrintNote(Combiner->getLoc(), + "Terminating due to -gicombiner-stop-after-build"); + return; + } + + NamedRegionTimer T("Emit", "Time spent emitting the combiner", + "Code Generation", "Time spent generating code", + TimeRegions); + OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_DEPS\n" + << "#include \"llvm/ADT/SparseBitVector.h\"\n" + << "namespace llvm {\n" + << "extern cl::OptionCategory GICombinerOptionCategory;\n" + << "} // end namespace llvm\n" + << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_DEPS\n\n"; + + OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_H\n" + << "class " << getClassName() << " {\n" + << " SparseBitVector<> DisabledRules;\n" + << "\n" + << "public:\n" + << " bool parseCommandLineOption();\n" + << " bool isRuleDisabled(unsigned ID) const;\n" + << " bool setRuleDisabled(StringRef RuleIdentifier);\n" + << "\n" + << " bool tryCombineAll(\n" + << " GISelChangeObserver &Observer,\n" + << " MachineInstr &MI,\n" + << " MachineIRBuilder &B,\n" + << " CombinerHelper &Helper) const;\n" + << "};\n\n"; + + emitNameMatcher(OS); + + OS << "bool " << getClassName() + << "::setRuleDisabled(StringRef RuleIdentifier) {\n" + << " std::pair<StringRef, StringRef> RangePair = " + "RuleIdentifier.split('-');\n" + << " if (!RangePair.second.empty()) {\n" + << " const auto First = getRuleIdxForIdentifier(RangePair.first);\n" + << " const auto Last = getRuleIdxForIdentifier(RangePair.second);\n" + << " if (!First.hasValue() || !Last.hasValue())\n" + << " return false;\n" + << " if (First >= Last)\n" + << " report_fatal_error(\"Beginning of range should be before end of " + "range\");\n" + << " for (auto I = First.getValue(); I < Last.getValue(); ++I)\n" + << " DisabledRules.set(I);\n" + << " return true;\n" + << " } else {\n" + << " const auto I = getRuleIdxForIdentifier(RangePair.first);\n" + << " if (!I.hasValue())\n" + << " return false;\n" + << " DisabledRules.set(I.getValue());\n" + << " return true;\n" + << " }\n" + << " return false;\n" + << "}\n"; + + OS << "bool " << getClassName() + << "::isRuleDisabled(unsigned RuleID) const {\n" + << " return DisabledRules.test(RuleID);\n" + << "}\n"; + OS << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_H\n\n"; + + OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_CPP\n" + << "\n" + << "cl::list<std::string> " << Name << "Option(\n" + << " \"" << Name.lower() << "-disable-rule\",\n" + << " cl::desc(\"Disable one or more combiner rules temporarily in " + << "the " << Name << " pass\"),\n" + << " cl::CommaSeparated,\n" + << " cl::Hidden,\n" + << " cl::cat(GICombinerOptionCategory));\n" + << "\n" + << "bool " << getClassName() << "::parseCommandLineOption() {\n" + << " for (const auto &Identifier : " << Name << "Option)\n" + << " if (!setRuleDisabled(Identifier))\n" + << " return false;\n" + << " return true;\n" + << "}\n\n"; + + OS << "bool " << getClassName() << "::tryCombineAll(\n" + << " GISelChangeObserver &Observer,\n" + << " MachineInstr &MI,\n" + << " MachineIRBuilder &B,\n" + << " CombinerHelper &Helper) const {\n" + << " MachineBasicBlock *MBB = MI.getParent();\n" + << " MachineFunction *MF = MBB->getParent();\n" + << " MachineRegisterInfo &MRI = MF->getRegInfo();\n" + << " SmallVector<MachineInstr *, 8> MIs = { &MI };\n\n" + << " (void)MBB; (void)MF; (void)MRI;\n\n"; + + OS << " // Match data\n"; + for (const auto &Rule : Rules) + for (const auto &I : Rule->matchdata_decls()) + OS << " " << I.getType() << " " << I.getVariableName() << ";\n"; + OS << "\n"; + + OS << " int Partition = -1;\n"; + generateCodeForTree(OS, *Tree, " "); + OS << "\n return false;\n" + << "}\n" + << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_CPP\n"; +} + +} // end anonymous namespace + +//===----------------------------------------------------------------------===// + +namespace llvm { +void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS) { + CodeGenTarget Target(RK); + emitSourceFileHeader("Global Combiner", OS); + + if (SelectedCombiners.empty()) + PrintFatalError("No combiners selected with -combiners"); + for (const auto &Combiner : SelectedCombiners) { + Record *CombinerDef = RK.getDef(Combiner); + if (!CombinerDef) + PrintFatalError("Could not find " + Combiner); + GICombinerEmitter(RK, Target, Combiner, CombinerDef).run(OS); + } + NumPatternTotalStatistic = NumPatternTotal; +} + +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp new file mode 100644 index 000000000000..d59a9b8e3b65 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp @@ -0,0 +1,93 @@ +//===- CodeExpander.cpp - Expand variables in a string --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Expand the variables in a string. +// +//===----------------------------------------------------------------------===// + +#include "CodeExpander.h" +#include "CodeExpansions.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" + +using namespace llvm; + +void CodeExpander::emit(raw_ostream &OS) const { + StringRef Current = Code; + + while (!Current.empty()) { + size_t Pos = Current.find_first_of("$\n\\"); + if (Pos == StringRef::npos) { + OS << Current; + Current = ""; + continue; + } + + OS << Current.substr(0, Pos); + Current = Current.substr(Pos); + + if (Current.startswith("\n")) { + OS << "\n" << Indent; + Current = Current.drop_front(1); + continue; + } + + if (Current.startswith("\\$") || Current.startswith("\\\\")) { + OS << Current[1]; + Current = Current.drop_front(2); + continue; + } + + if (Current.startswith("\\")) { + Current = Current.drop_front(1); + continue; + } + + if (Current.startswith("${")) { + StringRef StartVar = Current; + Current = Current.drop_front(2); + StringRef Var; + std::tie(Var, Current) = Current.split("}"); + + // Warn if we split because no terminator was found. + StringRef EndVar = StartVar.drop_front(2 /* ${ */ + Var.size()); + if (EndVar.empty()) { + size_t LocOffset = StartVar.data() - Code.data(); + PrintWarning( + Loc.size() > 0 && Loc[0].isValid() + ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset) + : SMLoc(), + "Unterminated expansion"); + } + + auto ValueI = Expansions.find(Var); + if (ValueI == Expansions.end()) { + size_t LocOffset = StartVar.data() - Code.data(); + PrintError(Loc.size() > 0 && Loc[0].isValid() + ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset) + : SMLoc(), + "Attempting to expand an undeclared variable " + Var); + } + if (ShowExpansions) + OS << "/*$" << Var << "{*/"; + OS << Expansions.lookup(Var); + if (ShowExpansions) + OS << "/*}*/"; + continue; + } + + size_t LocOffset = Current.data() - Code.data(); + PrintWarning(Loc.size() > 0 && Loc[0].isValid() + ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset) + : SMLoc(), + "Assuming missing escape character"); + OS << "$"; + Current = Current.drop_front(1); + } +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.h new file mode 100644 index 000000000000..bd6946de5925 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.h @@ -0,0 +1,55 @@ +//===- CodeExpander.h - Expand variables in a string ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Expand the variables in a string. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H +#define LLVM_UTILS_TABLEGEN_CODEEXPANDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/SMLoc.h" + +namespace llvm { +class CodeExpansions; +class raw_ostream; + +/// Emit the given code with all '${foo}' placeholders expanded to their +/// replacements. +/// +/// It's an error to use an undefined expansion and expansion-like output that +/// needs to be emitted verbatim can be escaped as '\${foo}' +/// +/// The emitted code can be given a custom indent to enable both indentation by +/// an arbitrary amount of whitespace and emission of the code as a comment. +class CodeExpander { + StringRef Code; + const CodeExpansions &Expansions; + const ArrayRef<SMLoc> &Loc; + bool ShowExpansions; + StringRef Indent; + +public: + CodeExpander(StringRef Code, const CodeExpansions &Expansions, + const ArrayRef<SMLoc> &Loc, bool ShowExpansions, + StringRef Indent = " ") + : Code(Code), Expansions(Expansions), Loc(Loc), + ShowExpansions(ShowExpansions), Indent(Indent) {} + + void emit(raw_ostream &OS) const; +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const CodeExpander &Expander) { + Expander.emit(OS); + return OS; +} +} // end namespace llvm + +#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpansions.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpansions.h new file mode 100644 index 000000000000..bb890ec8f57e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpansions.h @@ -0,0 +1,43 @@ +//===- CodeExpansions.h - Record expansions for CodeExpander --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Record the expansions to use in a CodeExpander. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringMap.h" + +#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H +#define LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H +namespace llvm { +class CodeExpansions { +public: + using const_iterator = StringMap<std::string>::const_iterator; + +protected: + StringMap<std::string> Expansions; + +public: + void declare(StringRef Name, StringRef Expansion) { + bool Inserted = Expansions.try_emplace(Name, Expansion).second; + assert(Inserted && "Declared variable twice"); + (void)Inserted; + } + + std::string lookup(StringRef Variable) const { + return Expansions.lookup(Variable); + } + + const_iterator begin() const { return Expansions.begin(); } + const_iterator end() const { return Expansions.end(); } + const_iterator find(StringRef Variable) const { + return Expansions.find(Variable); + } +}; +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.cpp new file mode 100644 index 000000000000..a3a9b7d8b037 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.cpp @@ -0,0 +1,138 @@ +//===- GIMatchDag.cpp - A DAG representation of a pattern to be matched ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDag.h" + +#include "llvm/Support/Format.h" +#include "llvm/TableGen/Record.h" +#include "../CodeGenInstruction.h" + +using namespace llvm; + +void GIMatchDag::writeDOTGraph(raw_ostream &OS, StringRef ID) const { + const auto writePorts = [&](StringRef Prefix, + const GIMatchDagOperandList &Operands) { + StringRef Separator = ""; + OS << "{"; + for (const auto &Op : enumerate(Operands)) { + OS << Separator << "<" << Prefix << format("%d", Op.index()) << ">" + << "#" << Op.index() << " $" << Op.value().getName(); + Separator = "|"; + } + OS << "}"; + }; + + OS << "digraph \"" << ID << "\" {\n" + << " rankdir=\"BT\"\n"; + for (const auto &N : InstrNodes) { + OS << " " << format("Node%p", &*N) << " [shape=record,label=\"{"; + writePorts("s", N->getOperandInfo()); + OS << "|" << N->getName(); + if (N->getOpcodeAnnotation()) + OS << "|" << N->getOpcodeAnnotation()->TheDef->getName(); + if (N->isMatchRoot()) + OS << "|Match starts here"; + OS << "|"; + SmallVector<std::pair<unsigned, StringRef>, 8> ToPrint; + for (const auto &Assignment : N->user_assigned_operand_names()) + ToPrint.emplace_back(Assignment.first, Assignment.second); + llvm::sort(ToPrint.begin(), ToPrint.end()); + StringRef Separator = ""; + for (const auto &Assignment : ToPrint) { + OS << Separator << "$" << Assignment.second << "=getOperand(" + << Assignment.first << ")"; + Separator = ", "; + } + OS << format("|%p|", &N); + writePorts("d", N->getOperandInfo()); + OS << "}\""; + if (N->isMatchRoot()) + OS << ",color=red"; + OS << "]\n"; + } + + for (const auto &E : Edges) { + const char *FromFmt = "Node%p:s%d:n"; + const char *ToFmt = "Node%p:d%d:s"; + if (E->getFromMO()->isDef() && !E->getToMO()->isDef()) + std::swap(FromFmt, ToFmt); + auto From = format(FromFmt, E->getFromMI(), E->getFromMO()->getIdx()); + auto To = format(ToFmt, E->getToMI(), E->getToMO()->getIdx()); + if (E->getFromMO()->isDef() && !E->getToMO()->isDef()) + std::swap(From, To); + + OS << " " << From << " -> " << To << " [label=\"$" << E->getName(); + if (E->getFromMO()->isDef() == E->getToMO()->isDef()) + OS << " INVALID EDGE!"; + OS << "\""; + if (E->getFromMO()->isDef() == E->getToMO()->isDef()) + OS << ",color=red"; + else if (E->getFromMO()->isDef() && !E->getToMO()->isDef()) + OS << ",dir=back,arrowtail=crow"; + OS << "]\n"; + } + + for (const auto &N : PredicateNodes) { + OS << " " << format("Pred%p", &*N) << " [shape=record,label=\"{"; + writePorts("s", N->getOperandInfo()); + OS << "|" << N->getName() << "|"; + N->printDescription(OS); + OS << format("|%p|", &N); + writePorts("d", N->getOperandInfo()); + OS << "}\",style=dotted]\n"; + } + + for (const auto &E : PredicateDependencies) { + const char *FromMIFmt = "Node%p:e"; + const char *FromMOFmt = "Node%p:s%d:n"; + const char *ToFmt = "Pred%p:d%d:s"; + auto To = format(ToFmt, E->getPredicate(), E->getPredicateOp()->getIdx()); + auto Style = "[style=dotted]"; + if (E->getRequiredMO()) { + auto From = + format(FromMOFmt, E->getRequiredMI(), E->getRequiredMO()->getIdx()); + OS << " " << From << " -> " << To << " " << Style << "\n"; + continue; + } + auto From = format(FromMIFmt, E->getRequiredMI()); + OS << " " << From << " -> " << To << " " << Style << "\n"; + } + + OS << "}\n"; +} + +LLVM_DUMP_METHOD void GIMatchDag::print(raw_ostream &OS) const { + OS << "matchdag {\n"; + for (const auto &N : InstrNodes) { + OS << " "; + N->print(OS); + OS << "\n"; + } + for (const auto &E : Edges) { + OS << " "; + E->print(OS); + OS << "\n"; + } + + for (const auto &P : PredicateNodes) { + OS << " "; + P->print(OS); + OS << "\n"; + } + for (const auto &D : PredicateDependencies) { + OS << " "; + D->print(OS); + OS << "\n"; + } + OS << "}\n"; +} + +raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDag &G) { + G.print(OS); + return OS; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.h new file mode 100644 index 000000000000..567580540877 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.h @@ -0,0 +1,243 @@ +//===- GIMatchDag.h - Represent a DAG to be matched -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAG_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAG_H + +#include "GIMatchDagEdge.h" +#include "GIMatchDagInstr.h" +#include "GIMatchDagOperands.h" +#include "GIMatchDagPredicate.h" +#include "GIMatchDagPredicateDependencyEdge.h" + +namespace llvm { +class GIMatchDag; + +/// This class manages lifetimes for data associated with the GIMatchDag object. +class GIMatchDagContext { + GIMatchDagOperandListContext OperandListCtx; + +public: + const GIMatchDagOperandList &makeEmptyOperandList() { + return OperandListCtx.makeEmptyOperandList(); + } + + const GIMatchDagOperandList &makeOperandList(const CodeGenInstruction &I) { + return OperandListCtx.makeOperandList(I); + } + + const GIMatchDagOperandList &makeMIPredicateOperandList() { + return OperandListCtx.makeMIPredicateOperandList(); + } + + + const GIMatchDagOperandList &makeTwoMOPredicateOperandList() { + return OperandListCtx.makeTwoMOPredicateOperandList(); + } + + void print(raw_ostream &OS) const { + OperandListCtx.print(OS); + } + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +class GIMatchDag { +public: + using InstrNodesVec = std::vector<std::unique_ptr<GIMatchDagInstr>>; + using instr_node_iterator = raw_pointer_iterator<InstrNodesVec::iterator>; + using const_instr_node_iterator = + raw_pointer_iterator<InstrNodesVec::const_iterator>; + + using EdgesVec = std::vector<std::unique_ptr<GIMatchDagEdge>>; + using edge_iterator = raw_pointer_iterator<EdgesVec::iterator>; + using const_edge_iterator = raw_pointer_iterator<EdgesVec::const_iterator>; + + using PredicateNodesVec = std::vector<std::unique_ptr<GIMatchDagPredicate>>; + using predicate_iterator = raw_pointer_iterator<PredicateNodesVec::iterator>; + using const_predicate_iterator = + raw_pointer_iterator<PredicateNodesVec::const_iterator>; + + using PredicateDependencyEdgesVec = + std::vector<std::unique_ptr<GIMatchDagPredicateDependencyEdge>>; + using predicate_edge_iterator = + raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>; + using const_predicate_edge_iterator = + raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>; + +protected: + GIMatchDagContext &Ctx; + InstrNodesVec InstrNodes; + PredicateNodesVec PredicateNodes; + EdgesVec Edges; + PredicateDependencyEdgesVec PredicateDependencies; + std::vector<GIMatchDagInstr *> MatchRoots; + // FIXME: This is a temporary measure while we still accept arbitrary code + // blocks to fix up the matcher while it's being developed. + bool HasPostMatchPredicate = false; + +public: + GIMatchDag(GIMatchDagContext &Ctx) + : Ctx(Ctx), InstrNodes(), PredicateNodes(), Edges(), + PredicateDependencies() {} + GIMatchDag(const GIMatchDag &) = delete; + + GIMatchDagContext &getContext() const { return Ctx; } + edge_iterator edges_begin() { + return raw_pointer_iterator<EdgesVec::iterator>(Edges.begin()); + } + edge_iterator edges_end() { + return raw_pointer_iterator<EdgesVec::iterator>(Edges.end()); + } + const_edge_iterator edges_begin() const { + return raw_pointer_iterator<EdgesVec::const_iterator>(Edges.begin()); + } + const_edge_iterator edges_end() const { + return raw_pointer_iterator<EdgesVec::const_iterator>(Edges.end()); + } + iterator_range<edge_iterator> edges() { + return make_range(edges_begin(), edges_end()); + } + iterator_range<const_edge_iterator> edges() const { + return make_range(edges_begin(), edges_end()); + } + iterator_range<std::vector<GIMatchDagInstr *>::iterator> roots() { + return make_range(MatchRoots.begin(), MatchRoots.end()); + } + iterator_range<std::vector<GIMatchDagInstr *>::const_iterator> roots() const { + return make_range(MatchRoots.begin(), MatchRoots.end()); + } + + instr_node_iterator instr_nodes_begin() { + return raw_pointer_iterator<InstrNodesVec::iterator>(InstrNodes.begin()); + } + instr_node_iterator instr_nodes_end() { + return raw_pointer_iterator<InstrNodesVec::iterator>(InstrNodes.end()); + } + const_instr_node_iterator instr_nodes_begin() const { + return raw_pointer_iterator<InstrNodesVec::const_iterator>( + InstrNodes.begin()); + } + const_instr_node_iterator instr_nodes_end() const { + return raw_pointer_iterator<InstrNodesVec::const_iterator>( + InstrNodes.end()); + } + iterator_range<instr_node_iterator> instr_nodes() { + return make_range(instr_nodes_begin(), instr_nodes_end()); + } + iterator_range<const_instr_node_iterator> instr_nodes() const { + return make_range(instr_nodes_begin(), instr_nodes_end()); + } + predicate_edge_iterator predicate_edges_begin() { + return raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>( + PredicateDependencies.begin()); + } + predicate_edge_iterator predicate_edges_end() { + return raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>( + PredicateDependencies.end()); + } + const_predicate_edge_iterator predicate_edges_begin() const { + return raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>( + PredicateDependencies.begin()); + } + const_predicate_edge_iterator predicate_edges_end() const { + return raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>( + PredicateDependencies.end()); + } + iterator_range<predicate_edge_iterator> predicate_edges() { + return make_range(predicate_edges_begin(), predicate_edges_end()); + } + iterator_range<const_predicate_edge_iterator> predicate_edges() const { + return make_range(predicate_edges_begin(), predicate_edges_end()); + } + predicate_iterator predicates_begin() { + return raw_pointer_iterator<PredicateNodesVec::iterator>( + PredicateNodes.begin()); + } + predicate_iterator predicates_end() { + return raw_pointer_iterator<PredicateNodesVec::iterator>( + PredicateNodes.end()); + } + const_predicate_iterator predicates_begin() const { + return raw_pointer_iterator<PredicateNodesVec::const_iterator>( + PredicateNodes.begin()); + } + const_predicate_iterator predicates_end() const { + return raw_pointer_iterator<PredicateNodesVec::const_iterator>( + PredicateNodes.end()); + } + iterator_range<predicate_iterator> predicates() { + return make_range(predicates_begin(), predicates_end()); + } + iterator_range<const_predicate_iterator> predicates() const { + return make_range(predicates_begin(), predicates_end()); + } + + template <class... Args> GIMatchDagInstr *addInstrNode(Args &&... args) { + auto Obj = + std::make_unique<GIMatchDagInstr>(*this, std::forward<Args>(args)...); + auto ObjRaw = Obj.get(); + InstrNodes.push_back(std::move(Obj)); + return ObjRaw; + } + + template <class T, class... Args> + T *addPredicateNode(Args &&... args) { + auto Obj = std::make_unique<T>(getContext(), std::forward<Args>(args)...); + auto ObjRaw = Obj.get(); + PredicateNodes.push_back(std::move(Obj)); + return ObjRaw; + } + + template <class... Args> GIMatchDagEdge *addEdge(Args &&... args) { + auto Obj = std::make_unique<GIMatchDagEdge>(std::forward<Args>(args)...); + auto ObjRaw = Obj.get(); + Edges.push_back(std::move(Obj)); + return ObjRaw; + } + + template <class... Args> + GIMatchDagPredicateDependencyEdge *addPredicateDependency(Args &&... args) { + auto Obj = std::make_unique<GIMatchDagPredicateDependencyEdge>( + std::forward<Args>(args)...); + auto ObjRaw = Obj.get(); + PredicateDependencies.push_back(std::move(Obj)); + return ObjRaw; + } + + size_t getInstrNodeIdx(instr_node_iterator I) { + return std::distance(instr_nodes_begin(), I); + } + size_t getInstrNodeIdx(const_instr_node_iterator I) const { + return std::distance(instr_nodes_begin(), I); + } + size_t getNumInstrNodes() const { return InstrNodes.size(); } + size_t getNumEdges() const { return Edges.size(); } + size_t getNumPredicates() const { return PredicateNodes.size(); } + + void setHasPostMatchPredicate(bool V) { HasPostMatchPredicate = V; } + bool hasPostMatchPredicate() const { return HasPostMatchPredicate; } + + void addMatchRoot(GIMatchDagInstr *N) { MatchRoots.push_back(N); } + + LLVM_DUMP_METHOD void print(raw_ostream &OS) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + + void writeDOTGraph(raw_ostream &OS, StringRef ID) const; +}; + +raw_ostream &operator<<(raw_ostream &OS, const GIMatchDag &G); + +} // end namespace llvm + +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAG_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp new file mode 100644 index 000000000000..e59cb3aae49a --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp @@ -0,0 +1,38 @@ +//===- GIMatchDagEdge.cpp - An edge describing a def/use lookup -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDagEdge.h" +#include "GIMatchDagInstr.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +LLVM_DUMP_METHOD void GIMatchDagEdge::print(raw_ostream &OS) const { + OS << getFromMI()->getName() << "[" << getFromMO()->getName() << "] --[" + << Name << "]--> " << getToMI()->getName() << "[" << getToMO()->getName() + << "]"; +} + +bool GIMatchDagEdge::isDefToUse() const { + // Def -> Def is invalid so we only need to check FromMO. + return FromMO->isDef(); +} + +void GIMatchDagEdge::reverse() { + std::swap(FromMI, ToMI); + std::swap(FromMO, ToMO); +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +LLVM_DUMP_METHOD void GIMatchDagEdge::dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + +raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDagEdge &E) { + E.print(OS); + return OS; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h new file mode 100644 index 000000000000..8e845ff0a51e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h @@ -0,0 +1,70 @@ +//===- GIMatchDagEdge.h - Represent a shared operand list for nodes -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGEDGE_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAGEDGE_H + +#include "llvm/ADT/StringRef.h" + +namespace llvm { +class raw_ostream; +class GIMatchDagInstr; +class GIMatchDagOperand; + +/// Represents an edge that connects two instructions together via a pair of +/// operands. For example: +/// %a = FOO ... +/// %0 = BAR %a +/// %1 = BAZ %a +/// would have two edges for %a like so: +/// BAR:Op#1 --[a]----> Op#0:FOO +/// ^ +/// BAZ:Op#1 --[a]------/ +/// Ideally, all edges in the DAG are from a use to a def as this is a many +/// to one edge but edges from defs to uses are supported too. +class GIMatchDagEdge { + /// The name of the edge. For example, + /// (FOO $a, $b, $c) + /// (BAR $d, $e, $a) + /// will create an edge named 'a' to connect FOO to BAR. Although the name + /// refers to the edge, the canonical value of 'a' is the operand that defines + /// it. + StringRef Name; + const GIMatchDagInstr *FromMI; + const GIMatchDagOperand *FromMO; + const GIMatchDagInstr *ToMI; + const GIMatchDagOperand *ToMO; + +public: + GIMatchDagEdge(StringRef Name, const GIMatchDagInstr *FromMI, const GIMatchDagOperand *FromMO, + const GIMatchDagInstr *ToMI, const GIMatchDagOperand *ToMO) + : Name(Name), FromMI(FromMI), FromMO(FromMO), ToMI(ToMI), ToMO(ToMO) {} + + StringRef getName() const { return Name; } + const GIMatchDagInstr *getFromMI() const { return FromMI; } + const GIMatchDagOperand *getFromMO() const { return FromMO; } + const GIMatchDagInstr *getToMI() const { return ToMI; } + const GIMatchDagOperand *getToMO() const { return ToMO; } + + /// Flip the direction of the edge. + void reverse(); + + /// Does this edge run from a def to (one of many) uses? + bool isDefToUse() const; + + LLVM_DUMP_METHOD void print(raw_ostream &OS) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const; +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagEdge &E); + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGEDGE_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.cpp new file mode 100644 index 000000000000..218b741be20c --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.cpp @@ -0,0 +1,48 @@ +//===- GIMatchDagInstr.cpp - A shared operand list for nodes --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDagInstr.h" +#include "../CodeGenInstruction.h" +#include "GIMatchDag.h" +#include "llvm/TableGen/Record.h" + +using namespace llvm; + +void GIMatchDagInstr::print(raw_ostream &OS) const { + OS << "("; + if (const auto *Annotation = getOpcodeAnnotation()) + OS << Annotation->TheDef->getName(); + else + OS << "<unknown>"; + OS << " "; + OperandInfo.print(OS); + OS << "):$" << Name; + if (!UserAssignedNamesForOperands.empty()) { + OS << " // "; + SmallVector<std::pair<unsigned, StringRef>, 8> ToPrint; + for (const auto &Assignment : UserAssignedNamesForOperands) + ToPrint.emplace_back(Assignment.first, Assignment.second); + llvm::sort(ToPrint.begin(), ToPrint.end()); + StringRef Separator = ""; + for (const auto &Assignment : ToPrint) { + OS << Separator << "$" << Assignment.second << "=getOperand(" + << Assignment.first << ")"; + Separator = ", "; + } + } +} + +void GIMatchDagInstr::setMatchRoot() { + IsMatchRoot = true; + Dag.addMatchRoot(this); +} + +raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDagInstr &N) { + N.print(OS); + return OS; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h new file mode 100644 index 000000000000..4a07767a2e19 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h @@ -0,0 +1,115 @@ +//===- GIMatchDagInstr.h - Represent a instruction to be matched ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGINSTR_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAGINSTR_H + +#include "GIMatchDagOperands.h" +#include "llvm/ADT/DenseMap.h" + +namespace llvm { +class GIMatchDag; + +/// Represents an instruction in the match DAG. This object knows very little +/// about the actual instruction to be matched as the bulk of that is in +/// predicates that are associated with the match DAG. It merely knows the names +/// and indices of any operands that need to be matched in order to allow edges +/// to link to them. +/// +/// Instances of this class objects are owned by the GIMatchDag and are not +/// shareable between instances of GIMatchDag. This is because the Name, +/// IsMatchRoot, and OpcodeAnnotation are likely to differ between GIMatchDag +/// instances. +class GIMatchDagInstr { +public: + using const_user_assigned_operand_names_iterator = + DenseMap<unsigned, StringRef>::const_iterator; + +protected: + /// The match DAG this instruction belongs to. + GIMatchDag &Dag; + + /// The name of the instruction in the pattern. For example: + /// (FOO $a, $b, $c):$name + /// will cause name to be assigned to this member. Anonymous instructions will + /// have a name assigned for debugging purposes. + StringRef Name; + + /// The name of the instruction in the pattern as assigned by the user. For + /// example: + /// (FOO $a, $b, $c):$name + /// will cause name to be assigned to this member. If a name is not provided, + /// this will be empty. This name is used to bind variables from rules to the + /// matched instruction. + StringRef UserAssignedName; + + /// The name of each operand (if any) that was assigned by the user. For + /// example: + /// (FOO $a, $b, $c):$name + /// will cause {0, "a"}, {1, "b"}, {2, "c} to be inserted into this map. + DenseMap<unsigned, StringRef> UserAssignedNamesForOperands; + + /// The operand list for this instruction. This object may be shared with + /// other instructions of a similar 'shape'. + const GIMatchDagOperandList &OperandInfo; + + /// For debugging purposes, it's helpful to have access to a description of + /// the Opcode. However, this object shouldn't use it for more than debugging + /// output since predicates are expected to be handled outside the DAG. + CodeGenInstruction *OpcodeAnnotation = 0; + + /// When true, this instruction will be a starting point for a match attempt. + bool IsMatchRoot = false; + +public: + GIMatchDagInstr(GIMatchDag &Dag, StringRef Name, StringRef UserAssignedName, + const GIMatchDagOperandList &OperandInfo) + : Dag(Dag), Name(Name), UserAssignedName(UserAssignedName), + OperandInfo(OperandInfo) {} + + const GIMatchDagOperandList &getOperandInfo() const { return OperandInfo; } + StringRef getName() const { return Name; } + StringRef getUserAssignedName() const { return UserAssignedName; } + void assignNameToOperand(unsigned Idx, StringRef Name) { + assert(UserAssignedNamesForOperands[Idx].empty() && "Cannot assign twice"); + UserAssignedNamesForOperands[Idx] = Name; + } + + const_user_assigned_operand_names_iterator + user_assigned_operand_names_begin() const { + return UserAssignedNamesForOperands.begin(); + } + const_user_assigned_operand_names_iterator + user_assigned_operand_names_end() const { + return UserAssignedNamesForOperands.end(); + } + iterator_range<const_user_assigned_operand_names_iterator> + user_assigned_operand_names() const { + return make_range(user_assigned_operand_names_begin(), + user_assigned_operand_names_end()); + } + + /// Mark this instruction as being a root of the match. This means that the + /// matcher will start from this node when attempting to match MIR. + void setMatchRoot(); + bool isMatchRoot() const { return IsMatchRoot; } + + void setOpcodeAnnotation(CodeGenInstruction *I) { OpcodeAnnotation = I; } + CodeGenInstruction *getOpcodeAnnotation() const { return OpcodeAnnotation; } + + void print(raw_ostream &OS) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagInstr &N); + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGINSTR_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp new file mode 100644 index 000000000000..e79e4686b91e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp @@ -0,0 +1,153 @@ +//===- GIMatchDagOperands.cpp - A shared operand list for nodes -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDagOperands.h" + +#include "../CodeGenInstruction.h" + +using namespace llvm; + +void GIMatchDagOperand::Profile(FoldingSetNodeID &ID) const { + Profile(ID, Idx, Name, IsDef); +} + +void GIMatchDagOperand::Profile(FoldingSetNodeID &ID, size_t Idx, + StringRef Name, bool IsDef) { + ID.AddInteger(Idx); + ID.AddString(Name); + ID.AddBoolean(IsDef); +} + +void GIMatchDagOperandList::add(StringRef Name, unsigned Idx, bool IsDef) { + assert(Idx == Operands.size() && "Operands added in wrong order"); + Operands.emplace_back(Operands.size(), Name, IsDef); + OperandsByName.try_emplace(Operands.back().getName(), Operands.size() - 1); +} + +void GIMatchDagOperandList::Profile(FoldingSetNodeID &ID) const { + for (const auto &I : enumerate(Operands)) + GIMatchDagOperand::Profile(ID, I.index(), I.value().getName(), + I.value().isDef()); +} + +void GIMatchDagOperandList::print(raw_ostream &OS) const { + if (Operands.empty()) { + OS << "<empty>"; + return; + } + StringRef Separator = ""; + for (const auto &I : Operands) { + OS << Separator << I.getIdx() << ":" << I.getName(); + if (I.isDef()) + OS << "<def>"; + Separator = ", "; + } +} + +const GIMatchDagOperandList::value_type &GIMatchDagOperandList:: +operator[](StringRef K) const { + const auto &I = OperandsByName.find(K); + assert(I != OperandsByName.end() && "Operand not found by name"); + return Operands[I->second]; +} + +const GIMatchDagOperandList & +GIMatchDagOperandListContext::makeEmptyOperandList() { + FoldingSetNodeID ID; + + void *InsertPoint; + GIMatchDagOperandList *Value = + OperandLists.FindNodeOrInsertPos(ID, InsertPoint); + if (Value) + return *Value; + + std::unique_ptr<GIMatchDagOperandList> NewValue = + std::make_unique<GIMatchDagOperandList>(); + OperandLists.InsertNode(NewValue.get(), InsertPoint); + OperandListsOwner.push_back(std::move(NewValue)); + return *OperandListsOwner.back().get(); +} + +const GIMatchDagOperandList & +GIMatchDagOperandListContext::makeOperandList(const CodeGenInstruction &I) { + FoldingSetNodeID ID; + for (unsigned i = 0; i < I.Operands.size(); ++i) + GIMatchDagOperand::Profile(ID, i, I.Operands[i].Name, + i < I.Operands.NumDefs); + + void *InsertPoint; + GIMatchDagOperandList *Value = + OperandLists.FindNodeOrInsertPos(ID, InsertPoint); + if (Value) + return *Value; + + std::unique_ptr<GIMatchDagOperandList> NewValue = + std::make_unique<GIMatchDagOperandList>(); + for (unsigned i = 0; i < I.Operands.size(); ++i) + NewValue->add(I.Operands[i].Name, i, i < I.Operands.NumDefs); + OperandLists.InsertNode(NewValue.get(), InsertPoint); + OperandListsOwner.push_back(std::move(NewValue)); + return *OperandListsOwner.back().get(); +} + +const GIMatchDagOperandList & +GIMatchDagOperandListContext::makeMIPredicateOperandList() { + FoldingSetNodeID ID; + GIMatchDagOperand::Profile(ID, 0, "$", true); + GIMatchDagOperand::Profile(ID, 1, "mi", false); + + void *InsertPoint; + GIMatchDagOperandList *Value = + OperandLists.FindNodeOrInsertPos(ID, InsertPoint); + if (Value) + return *Value; + + std::unique_ptr<GIMatchDagOperandList> NewValue = + std::make_unique<GIMatchDagOperandList>(); + NewValue->add("$", 0, true); + NewValue->add("mi", 1, false); + OperandLists.InsertNode(NewValue.get(), InsertPoint); + OperandListsOwner.push_back(std::move(NewValue)); + return *OperandListsOwner.back().get(); +} + + +const GIMatchDagOperandList & +GIMatchDagOperandListContext::makeTwoMOPredicateOperandList() { + FoldingSetNodeID ID; + GIMatchDagOperand::Profile(ID, 0, "$", true); + GIMatchDagOperand::Profile(ID, 1, "mi0", false); + GIMatchDagOperand::Profile(ID, 2, "mi1", false); + + void *InsertPoint; + GIMatchDagOperandList *Value = + OperandLists.FindNodeOrInsertPos(ID, InsertPoint); + if (Value) + return *Value; + + std::unique_ptr<GIMatchDagOperandList> NewValue = + std::make_unique<GIMatchDagOperandList>(); + NewValue->add("$", 0, true); + NewValue->add("mi0", 1, false); + NewValue->add("mi1", 2, false); + OperandLists.InsertNode(NewValue.get(), InsertPoint); + OperandListsOwner.push_back(std::move(NewValue)); + return *OperandListsOwner.back().get(); +} + +void GIMatchDagOperandListContext::print(raw_ostream &OS) const { + OS << "GIMatchDagOperandListContext {\n" + << " OperandLists {\n"; + for (const auto &I : OperandListsOwner) { + OS << " "; + I->print(OS); + OS << "\n"; + } + OS << " }\n" + << "}\n"; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h new file mode 100644 index 000000000000..c2d30574231d --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h @@ -0,0 +1,133 @@ +//===- GIMatchDagOperands.h - Represent a shared operand list for nodes ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGOPERANDS_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAGOPERANDS_H + +#include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" + +#include <vector> + +namespace llvm { +class CodeGenInstruction; +/// Describes an operand of a MachineInstr w.r.t the DAG Matching. This +/// information is derived from CodeGenInstruction::Operands but is more +/// readily available for context-less access as we don't need to know which +/// instruction it's used with or know how many defs that instruction had. +/// +/// There may be multiple GIMatchDagOperand's with the same contents. However, +/// they are uniqued within the set of instructions that have the same overall +/// operand list. For example, given: +/// Inst1 operands ($dst:<def>, $src1, $src2) +/// Inst2 operands ($dst:<def>, $src1, $src2) +/// Inst3 operands ($dst:<def>, $src) +/// $src1 will have a single instance of GIMatchDagOperand shared by Inst1 and +/// Inst2, as will $src2. $dst however, will have two instances one shared +/// between Inst1 and Inst2 and one unique to Inst3. We could potentially +/// fully de-dupe the GIMatchDagOperand instances but the saving is not expected +/// to be worth the overhead. +/// +/// The result of this is that the address of the object can be relied upon to +/// trivially identify commonality between two instructions which will be useful +/// when generating the matcher. When the pointers differ, the contents can be +/// inspected instead. +class GIMatchDagOperand { + unsigned Idx; + StringRef Name; + bool IsDef; + +public: + GIMatchDagOperand(unsigned Idx, StringRef Name, bool IsDef) + : Idx(Idx), Name(Name), IsDef(IsDef) {} + + unsigned getIdx() const { return Idx; } + StringRef getName() const { return Name; } + bool isDef() const { return IsDef; } + + /// This object isn't a FoldingSetNode but it's part of one. See FoldingSet + /// for details on the Profile function. + void Profile(FoldingSetNodeID &ID) const; + + /// A helper that behaves like Profile() but is also usable without the object. + /// We use size_t here to match enumerate<...>::index(). If we don't match + /// that the hashes won't be equal. + static void Profile(FoldingSetNodeID &ID, size_t Idx, StringRef Name, + bool IsDef); +}; + +/// A list of GIMatchDagOperands for an instruction without any association with +/// a particular instruction. +/// +/// An important detail to be aware of with this class is that they are shared +/// with other instructions of a similar 'shape'. For example, all the binary +/// instructions are likely to share a single GIMatchDagOperandList. This is +/// primarily a memory optimization as it's fairly common to have a large number +/// of instructions but only a few 'shapes'. +/// +/// See GIMatchDagOperandList::Profile() for the details on how they are folded. +class GIMatchDagOperandList : public FoldingSetNode { +public: + using value_type = GIMatchDagOperand; + +protected: + using vector_type = SmallVector<GIMatchDagOperand, 3>; + +public: + using iterator = vector_type::iterator; + using const_iterator = vector_type::const_iterator; + +protected: + vector_type Operands; + StringMap<unsigned> OperandsByName; + +public: + void add(StringRef Name, unsigned Idx, bool IsDef); + + /// See FoldingSet for details. + void Profile(FoldingSetNodeID &ID) const; + + iterator begin() { return Operands.begin(); } + const_iterator begin() const { return Operands.begin(); } + iterator end() { return Operands.end(); } + const_iterator end() const { return Operands.end(); } + + const value_type &operator[](unsigned I) const { return Operands[I]; } + const value_type &operator[](StringRef K) const; + + void print(raw_ostream &OS) const; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +/// This is the portion of GIMatchDagContext that directly relates to +/// GIMatchDagOperandList and GIMatchDagOperandList. +class GIMatchDagOperandListContext { + FoldingSet<GIMatchDagOperandList> OperandLists; + std::vector<std::unique_ptr<GIMatchDagOperandList>> OperandListsOwner; + +public: + const GIMatchDagOperandList &makeEmptyOperandList(); + const GIMatchDagOperandList &makeOperandList(const CodeGenInstruction &I); + const GIMatchDagOperandList &makeMIPredicateOperandList(); + const GIMatchDagOperandList &makeTwoMOPredicateOperandList(); + + void print(raw_ostream &OS) const; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGOPERANDS_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp new file mode 100644 index 000000000000..1aca2f9dc135 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp @@ -0,0 +1,69 @@ +//===- GIMatchDagPredicate.cpp - Represent a predicate to check -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDagPredicate.h" + +#include "llvm/TableGen/Record.h" + +#include "GIMatchDagOperands.h" +#include "../CodeGenInstruction.h" + +using namespace llvm; + +void GIMatchDagPredicate::print(raw_ostream &OS) const { + OS << "<<"; + printDescription(OS); + OS << ">>:$" << Name; +} + +void GIMatchDagPredicate::printDescription(raw_ostream &OS) const { OS << ""; } + +GIMatchDagOpcodePredicate::GIMatchDagOpcodePredicate( + GIMatchDagContext &Ctx, StringRef Name, const CodeGenInstruction &Instr) + : GIMatchDagPredicate(GIMatchDagPredicateKind_Opcode, Name, + Ctx.makeMIPredicateOperandList()), + Instr(Instr) {} + +void GIMatchDagOpcodePredicate::printDescription(raw_ostream &OS) const { + OS << "$mi.getOpcode() == " << Instr.TheDef->getName(); +} + +GIMatchDagOneOfOpcodesPredicate::GIMatchDagOneOfOpcodesPredicate( + GIMatchDagContext &Ctx, StringRef Name) + : GIMatchDagPredicate(GIMatchDagPredicateKind_OneOfOpcodes, Name, + Ctx.makeMIPredicateOperandList()) {} + +void GIMatchDagOneOfOpcodesPredicate::printDescription(raw_ostream &OS) const { + OS << "$mi.getOpcode() == oneof("; + StringRef Separator = ""; + for (const CodeGenInstruction *Instr : Instrs) { + OS << Separator << Instr->TheDef->getName(); + Separator = ","; + } + OS << ")"; +} + +GIMatchDagSameMOPredicate::GIMatchDagSameMOPredicate(GIMatchDagContext &Ctx, + StringRef Name) + : GIMatchDagPredicate(GIMatchDagPredicateKind_SameMO, Name, + Ctx.makeTwoMOPredicateOperandList()) {} + +void GIMatchDagSameMOPredicate::printDescription(raw_ostream &OS) const { + OS << "$mi0 == $mi1"; +} + +raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDagPredicate &N) { + N.print(OS); + return OS; +} + +raw_ostream &llvm::operator<<(raw_ostream &OS, + const GIMatchDagOpcodePredicate &N) { + N.print(OS); + return OS; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h new file mode 100644 index 000000000000..9b030d6edb13 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h @@ -0,0 +1,141 @@ +//===- GIMatchDagPredicate - Represent a predicate to check ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATE_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATE_H + +#include "llvm/ADT/StringRef.h" +#include "GIMatchDag.h" + +namespace llvm { +class CodeExpansions; +class CodeGenInstruction; +class GIMatchDagOperandList; +class GIMatchDagContext; +class raw_ostream; + +/// Represents a predicate on the match DAG. This records the details of the +/// predicate. The dependencies are stored in the GIMatchDag as edges. +/// +/// Instances of this class objects are owned by the GIMatchDag and are not +/// shareable between instances of GIMatchDag. +class GIMatchDagPredicate { +public: + enum GIMatchDagPredicateKind { + GIMatchDagPredicateKind_Opcode, + GIMatchDagPredicateKind_OneOfOpcodes, + GIMatchDagPredicateKind_SameMO, + }; + +protected: + const GIMatchDagPredicateKind Kind; + + /// The name of the predicate. For example: + /// (FOO $a:s32, $b, $c) + /// will cause 's32' to be assigned to this member for the $a predicate. + /// Similarly, the opcode predicate will cause 'FOO' to be assigned to this + /// member. Anonymous instructions will have a name assigned for debugging + /// purposes. + StringRef Name; + + /// The operand list for this predicate. This object may be shared with + /// other predicates of a similar 'shape'. + const GIMatchDagOperandList &OperandInfo; + +public: + GIMatchDagPredicate(GIMatchDagPredicateKind Kind, StringRef Name, + const GIMatchDagOperandList &OperandInfo) + : Kind(Kind), Name(Name), OperandInfo(OperandInfo) {} + virtual ~GIMatchDagPredicate() {} + + GIMatchDagPredicateKind getKind() const { return Kind; } + + StringRef getName() const { return Name; } + const GIMatchDagOperandList &getOperandInfo() const { return OperandInfo; } + + // Generate C++ code to check this predicate. If a partitioner has already + // tested this predicate then this function won't be called. If this function + // is called, it must emit code and return true to indicate that it did so. If + // it ever returns false, then the caller will abort due to an untested + // predicate. + virtual bool generateCheckCode(raw_ostream &OS, StringRef Indent, + const CodeExpansions &Expansions) const { + return false; + } + + virtual void print(raw_ostream &OS) const; + virtual void printDescription(raw_ostream &OS) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + virtual LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +class GIMatchDagOpcodePredicate : public GIMatchDagPredicate { + const CodeGenInstruction &Instr; + +public: + GIMatchDagOpcodePredicate(GIMatchDagContext &Ctx, StringRef Name, + const CodeGenInstruction &Instr); + + static bool classof(const GIMatchDagPredicate *P) { + return P->getKind() == GIMatchDagPredicateKind_Opcode; + } + + const CodeGenInstruction *getInstr() const { return &Instr; } + + void printDescription(raw_ostream &OS) const override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + virtual LLVM_DUMP_METHOD void dump() const override { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +class GIMatchDagOneOfOpcodesPredicate : public GIMatchDagPredicate { + SmallVector<const CodeGenInstruction *, 4> Instrs; + +public: + GIMatchDagOneOfOpcodesPredicate(GIMatchDagContext &Ctx, StringRef Name); + + void addOpcode(const CodeGenInstruction *Instr) { Instrs.push_back(Instr); } + + static bool classof(const GIMatchDagPredicate *P) { + return P->getKind() == GIMatchDagPredicateKind_OneOfOpcodes; + } + + const SmallVectorImpl<const CodeGenInstruction *> &getInstrs() const { + return Instrs; + } + + void printDescription(raw_ostream &OS) const override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + virtual LLVM_DUMP_METHOD void dump() const override { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +class GIMatchDagSameMOPredicate : public GIMatchDagPredicate { +public: + GIMatchDagSameMOPredicate(GIMatchDagContext &Ctx, StringRef Name); + + static bool classof(const GIMatchDagPredicate *P) { + return P->getKind() == GIMatchDagPredicateKind_SameMO; + } + + void printDescription(raw_ostream &OS) const override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + virtual LLVM_DUMP_METHOD void dump() const override { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagPredicate &N); +raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagOpcodePredicate &N); + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATE_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp new file mode 100644 index 000000000000..2e804de1cd4e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp @@ -0,0 +1,37 @@ +//===- GIMatchDagPredicateDependencyEdge.cpp - Have inputs before check ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDagPredicateDependencyEdge.h" + +#include "GIMatchDagInstr.h" +#include "GIMatchDagPredicate.h" + +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +LLVM_DUMP_METHOD void +GIMatchDagPredicateDependencyEdge::print(raw_ostream &OS) const { + OS << getRequiredMI()->getName(); + if (getRequiredMO()) + OS << "[" << getRequiredMO()->getName() << "]"; + OS << " ==> " << getPredicate()->getName() << "[" + << getPredicateOp()->getName() << "]"; +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +LLVM_DUMP_METHOD void GIMatchDagPredicateDependencyEdge::dump() const { + print(errs()); +} +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + +raw_ostream &llvm::operator<<(raw_ostream &OS, + const GIMatchDagPredicateDependencyEdge &E) { + E.print(OS); + return OS; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.h new file mode 100644 index 000000000000..865455fe4e4d --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.h @@ -0,0 +1,60 @@ +//===- GIMatchDagPredicateDependencyEdge - Ensure predicates have inputs --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATEEDGE_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATEEDGE_H + +#include "GIMatchDagOperands.h" + +namespace llvm { +class GIMatchDag; +class GIMatchDagInstr; +class GIMatchDagEdge; +class GIMatchDagPredicate; + +/// Represents a dependency that must be met to evaluate a predicate. +/// +/// Instances of this class objects are owned by the GIMatchDag and are not +/// shareable between instances of GIMatchDag. +class GIMatchDagPredicateDependencyEdge { + /// The MI that must be available in order to test the predicate. + const GIMatchDagInstr *RequiredMI; + /// The MO that must be available in order to test the predicate. May be + /// nullptr when only the MI is required. + const GIMatchDagOperand *RequiredMO; + /// The Predicate that requires information from RequiredMI/RequiredMO. + const GIMatchDagPredicate *Predicate; + /// The Predicate operand that requires information from + /// RequiredMI/RequiredMO. + const GIMatchDagOperand *PredicateOp; + +public: + GIMatchDagPredicateDependencyEdge(const GIMatchDagInstr *RequiredMI, + const GIMatchDagOperand *RequiredMO, + const GIMatchDagPredicate *Predicate, + const GIMatchDagOperand *PredicateOp) + : RequiredMI(RequiredMI), RequiredMO(RequiredMO), Predicate(Predicate), + PredicateOp(PredicateOp) {} + + const GIMatchDagInstr *getRequiredMI() const { return RequiredMI; } + const GIMatchDagOperand *getRequiredMO() const { return RequiredMO; } + const GIMatchDagPredicate *getPredicate() const { return Predicate; } + const GIMatchDagOperand *getPredicateOp() const { return PredicateOp; } + + void print(raw_ostream &OS) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const; +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +raw_ostream &operator<<(raw_ostream &OS, + const GIMatchDagPredicateDependencyEdge &N); + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATEEDGE_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp new file mode 100644 index 000000000000..4884bdadea91 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp @@ -0,0 +1,777 @@ +//===- GIMatchTree.cpp - A decision tree to match GIMatchDag's ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchTree.h" + +#include "../CodeGenInstruction.h" + +#include "llvm/Support/Format.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" + +#define DEBUG_TYPE "gimatchtree" + +using namespace llvm; + +void GIMatchTree::writeDOTGraph(raw_ostream &OS) const { + OS << "digraph \"matchtree\" {\n"; + writeDOTGraphNode(OS); + OS << "}\n"; +} + +void GIMatchTree::writeDOTGraphNode(raw_ostream &OS) const { + OS << format(" Node%p", this) << " [shape=record,label=\"{"; + if (Partitioner) { + Partitioner->emitDescription(OS); + OS << "|" << Partitioner->getNumPartitions() << " partitions|"; + } else + OS << "No partitioner|"; + bool IsFullyTraversed = true; + bool IsFullyTested = true; + StringRef Separator = ""; + for (const auto &Leaf : PossibleLeaves) { + OS << Separator << Leaf.getName(); + Separator = ","; + if (!Leaf.isFullyTraversed()) + IsFullyTraversed = false; + if (!Leaf.isFullyTested()) + IsFullyTested = false; + } + if (!Partitioner && !IsFullyTraversed) + OS << "|Not fully traversed"; + if (!Partitioner && !IsFullyTested) { + OS << "|Not fully tested"; + if (IsFullyTraversed) { + for (const GIMatchTreeLeafInfo &Leaf : PossibleLeaves) { + if (Leaf.isFullyTested()) + continue; + OS << "\\n" << Leaf.getName() << ": " << &Leaf; + for (const GIMatchDagPredicate *P : Leaf.untested_predicates()) + OS << *P; + } + } + } + OS << "}\""; + if (!Partitioner && + (!IsFullyTraversed || !IsFullyTested || PossibleLeaves.size() > 1)) + OS << ",color=red"; + OS << "]\n"; + for (const auto &C : Children) + C.writeDOTGraphNode(OS); + writeDOTGraphEdges(OS); +} + +void GIMatchTree::writeDOTGraphEdges(raw_ostream &OS) const { + for (const auto &Child : enumerate(Children)) { + OS << format(" Node%p", this) << " -> " << format("Node%p", &Child.value()) + << " [label=\"#" << Child.index() << " "; + Partitioner->emitPartitionName(OS, Child.index()); + OS << "\"]\n"; + } +} + +GIMatchTreeBuilderLeafInfo::GIMatchTreeBuilderLeafInfo( + GIMatchTreeBuilder &Builder, StringRef Name, unsigned RootIdx, + const GIMatchDag &MatchDag, void *Data) + : Builder(Builder), Info(Name, RootIdx, Data), MatchDag(MatchDag), + InstrNodeToInfo(), + RemainingInstrNodes(BitVector(MatchDag.getNumInstrNodes(), true)), + RemainingEdges(BitVector(MatchDag.getNumEdges(), true)), + RemainingPredicates(BitVector(MatchDag.getNumPredicates(), true)), + TraversableEdges(MatchDag.getNumEdges()), + TestablePredicates(MatchDag.getNumPredicates()) { + // Number all the predicates in this DAG + for (auto &P : enumerate(MatchDag.predicates())) { + PredicateIDs.insert(std::make_pair(P.value(), P.index())); + } + + // Number all the predicate dependencies in this DAG and set up a bitvector + // for each predicate indicating the unsatisfied dependencies. + for (auto &Dep : enumerate(MatchDag.predicate_edges())) { + PredicateDepIDs.insert(std::make_pair(Dep.value(), Dep.index())); + } + UnsatisfiedPredDepsForPred.resize(MatchDag.getNumPredicates(), + BitVector(PredicateDepIDs.size())); + for (auto &Dep : enumerate(MatchDag.predicate_edges())) { + unsigned ID = PredicateIDs.lookup(Dep.value()->getPredicate()); + UnsatisfiedPredDepsForPred[ID].set(Dep.index()); + } +} + +void GIMatchTreeBuilderLeafInfo::declareInstr(const GIMatchDagInstr *Instr, unsigned ID) { + // Record the assignment of this instr to the given ID. + auto InfoI = InstrNodeToInfo.insert(std::make_pair( + Instr, GIMatchTreeInstrInfo(ID, Instr))); + InstrIDToInfo.insert(std::make_pair(ID, &InfoI.first->second)); + + if (Instr == nullptr) + return; + + if (!Instr->getUserAssignedName().empty()) + Info.bindInstrVariable(Instr->getUserAssignedName(), ID); + for (const auto &VarBinding : Instr->user_assigned_operand_names()) + Info.bindOperandVariable(VarBinding.second, ID, VarBinding.first); + + // Clear the bit indicating we haven't visited this instr. + const auto &NodeI = std::find(MatchDag.instr_nodes_begin(), + MatchDag.instr_nodes_end(), Instr); + assert(NodeI != MatchDag.instr_nodes_end() && "Instr isn't in this DAG"); + unsigned InstrIdx = MatchDag.getInstrNodeIdx(NodeI); + RemainingInstrNodes.reset(InstrIdx); + + // When we declare an instruction, we don't expose any traversable edges just + // yet. A partitioner has to check they exist and are registers before they + // are traversable. + + // When we declare an instruction, we potentially activate some predicates. + // Mark the dependencies that are now satisfied as a result of this + // instruction and mark any predicates whose dependencies are fully + // satisfied. + for (auto &Dep : enumerate(MatchDag.predicate_edges())) { + if (Dep.value()->getRequiredMI() == Instr && + Dep.value()->getRequiredMO() == nullptr) { + for (auto &DepsFor : enumerate(UnsatisfiedPredDepsForPred)) { + DepsFor.value().reset(Dep.index()); + if (DepsFor.value().none()) + TestablePredicates.set(DepsFor.index()); + } + } + } +} + +void GIMatchTreeBuilderLeafInfo::declareOperand(unsigned InstrID, + unsigned OpIdx) { + const GIMatchDagInstr *Instr = InstrIDToInfo.lookup(InstrID)->getInstrNode(); + + OperandIDToInfo.insert(std::make_pair( + std::make_pair(InstrID, OpIdx), + GIMatchTreeOperandInfo(Instr, OpIdx))); + + // When an operand becomes reachable, we potentially activate some traversals. + // Record the edges that can now be followed as a result of this + // instruction. + for (auto &E : enumerate(MatchDag.edges())) { + if (E.value()->getFromMI() == Instr && + E.value()->getFromMO()->getIdx() == OpIdx) { + TraversableEdges.set(E.index()); + } + } + + // When an operand becomes reachable, we potentially activate some predicates. + // Clear the dependencies that are now satisfied as a result of this + // operand and activate any predicates whose dependencies are fully + // satisfied. + for (auto &Dep : enumerate(MatchDag.predicate_edges())) { + if (Dep.value()->getRequiredMI() == Instr && Dep.value()->getRequiredMO() && + Dep.value()->getRequiredMO()->getIdx() == OpIdx) { + for (auto &DepsFor : enumerate(UnsatisfiedPredDepsForPred)) { + DepsFor.value().reset(Dep.index()); + if (DepsFor.value().none()) + TestablePredicates.set(DepsFor.index()); + } + } + } +} + +void GIMatchTreeBuilder::addPartitionersForInstr(unsigned InstrIdx) { + // Find the partitioners that can be used now that this node is + // uncovered. Our choices are: + // - Test the opcode + addPartitioner(std::make_unique<GIMatchTreeOpcodePartitioner>(InstrIdx)); +} + +void GIMatchTreeBuilder::addPartitionersForOperand(unsigned InstrID, + unsigned OpIdx) { + LLVM_DEBUG(dbgs() << "Add partitioners for Instrs[" << InstrID + << "].getOperand(" << OpIdx << ")\n"); + addPartitioner( + std::make_unique<GIMatchTreeVRegDefPartitioner>(InstrID, OpIdx)); +} + +void GIMatchTreeBuilder::filterRedundantPartitioners() { + // TODO: Filter partitioners for facts that are already known + // - If we know the opcode, we can elide the num operand check so long as + // the instruction has a fixed number of operands. + // - If we know an exact number of operands then we can elide further number + // of operand checks. + // - If the current min number of operands exceeds the one we want to check + // then we can elide it. +} + +void GIMatchTreeBuilder::evaluatePartitioners() { + // Determine the partitioning the partitioner would produce + for (auto &Partitioner : Partitioners) { + LLVM_DEBUG(dbgs() << " Weighing up "; + Partitioner->emitDescription(dbgs()); dbgs() << "\n"); + Partitioner->repartition(Leaves); + LLVM_DEBUG(Partitioner->emitPartitionResults(dbgs())); + } +} + +void GIMatchTreeBuilder::runStep() { + LLVM_DEBUG(dbgs() << "Building match tree node for " << TreeNode << "\n"); + LLVM_DEBUG(dbgs() << " Rules reachable at this node:\n"); + for (const auto &Leaf : Leaves) { + LLVM_DEBUG(dbgs() << " " << Leaf.getName() << " (" << &Leaf.getInfo() << "\n"); + TreeNode->addPossibleLeaf(Leaf.getInfo(), Leaf.isFullyTraversed(), + Leaf.isFullyTested()); + } + + LLVM_DEBUG(dbgs() << " Partitioners available at this node:\n"); +#ifndef NDEBUG + for (const auto &Partitioner : Partitioners) + LLVM_DEBUG(dbgs() << " "; Partitioner->emitDescription(dbgs()); + dbgs() << "\n"); +#endif // ifndef NDEBUG + + // Check for unreachable rules. Rules are unreachable if they are preceeded by + // a fully tested rule. + // Note: This is only true for the current algorithm, if we allow the + // algorithm to compare equally valid rules then they will become + // reachable. + { + auto FullyTestedLeafI = Leaves.end(); + for (auto LeafI = Leaves.begin(), LeafE = Leaves.end(); + LeafI != LeafE; ++LeafI) { + if (LeafI->isFullyTraversed() && LeafI->isFullyTested()) + FullyTestedLeafI = LeafI; + else if (FullyTestedLeafI != Leaves.end()) { + PrintError("Leaf " + LeafI->getName() + " is unreachable"); + PrintNote("Leaf " + FullyTestedLeafI->getName() + + " will have already matched"); + } + } + } + + LLVM_DEBUG(dbgs() << " Eliminating redundant partitioners:\n"); + filterRedundantPartitioners(); + LLVM_DEBUG(dbgs() << " Partitioners remaining:\n"); +#ifndef NDEBUG + for (const auto &Partitioner : Partitioners) + LLVM_DEBUG(dbgs() << " "; Partitioner->emitDescription(dbgs()); + dbgs() << "\n"); +#endif // ifndef NDEBUG + + if (Partitioners.empty()) { + // Nothing left to do but check we really did identify a single rule. + if (Leaves.size() > 1) { + LLVM_DEBUG(dbgs() << "Leaf contains multiple rules, drop after the first " + "fully tested rule\n"); + auto FirstFullyTested = + std::find_if(Leaves.begin(), Leaves.end(), + [](const GIMatchTreeBuilderLeafInfo &X) { + return X.isFullyTraversed() && X.isFullyTested() && + !X.getMatchDag().hasPostMatchPredicate(); + }); + if (FirstFullyTested != Leaves.end()) + FirstFullyTested++; + +#ifndef NDEBUG + for (auto &Leaf : make_range(Leaves.begin(), FirstFullyTested)) + LLVM_DEBUG(dbgs() << " Kept " << Leaf.getName() << "\n"); + for (const auto &Leaf : make_range(FirstFullyTested, Leaves.end())) + LLVM_DEBUG(dbgs() << " Dropped " << Leaf.getName() << "\n"); +#endif // ifndef NDEBUG + TreeNode->dropLeavesAfter( + std::distance(Leaves.begin(), FirstFullyTested)); + } + for (const auto &Leaf : Leaves) { + if (!Leaf.isFullyTraversed()) { + PrintError("Leaf " + Leaf.getName() + " is not fully traversed"); + PrintNote("This indicates a missing partitioner within tblgen"); + Leaf.dump(errs()); + for (unsigned InstrIdx : Leaf.untested_instrs()) + PrintNote("Instr " + llvm::to_string(*Leaf.getInstr(InstrIdx))); + for (unsigned EdgeIdx : Leaf.untested_edges()) + PrintNote("Edge " + llvm::to_string(*Leaf.getEdge(EdgeIdx))); + } + } + + // Copy out information about untested predicates so the user of the tree + // can deal with them. + for (auto LeafPair : zip(Leaves, TreeNode->possible_leaves())) { + const GIMatchTreeBuilderLeafInfo &BuilderLeaf = std::get<0>(LeafPair); + GIMatchTreeLeafInfo &TreeLeaf = std::get<1>(LeafPair); + if (!BuilderLeaf.isFullyTested()) + for (unsigned PredicateIdx : BuilderLeaf.untested_predicates()) + TreeLeaf.addUntestedPredicate(BuilderLeaf.getPredicate(PredicateIdx)); + } + return; + } + + LLVM_DEBUG(dbgs() << " Weighing up partitioners:\n"); + evaluatePartitioners(); + + // Select the best partitioner by its ability to partition + // - Prefer partitioners that don't distinguish between partitions. This + // is to fail early on decisions that must go a single way. + auto PartitionerI = std::max_element( + Partitioners.begin(), Partitioners.end(), + [](const std::unique_ptr<GIMatchTreePartitioner> &A, + const std::unique_ptr<GIMatchTreePartitioner> &B) { + // We generally want partitioners that subdivide the + // ruleset as much as possible since these take fewer + // checks to converge on a particular rule. However, + // it's important to note that one leaf can end up in + // multiple partitions if the check isn't mutually + // exclusive (e.g. getVRegDef() vs isReg()). + // We therefore minimize average leaves per partition. + return (double)A->getNumLeavesWithDupes() / A->getNumPartitions() > + (double)B->getNumLeavesWithDupes() / B->getNumPartitions(); + }); + + // Select a partitioner and partition the ruleset + // Note that it's possible for a single rule to end up in multiple + // partitions. For example, an opcode test on a rule without an opcode + // predicate will result in it being passed to all partitions. + std::unique_ptr<GIMatchTreePartitioner> Partitioner = std::move(*PartitionerI); + Partitioners.erase(PartitionerI); + LLVM_DEBUG(dbgs() << " Selected partitioner: "; + Partitioner->emitDescription(dbgs()); dbgs() << "\n"); + + assert(Partitioner->getNumPartitions() > 0 && + "Must always partition into at least one partition"); + + TreeNode->setNumChildren(Partitioner->getNumPartitions()); + for (auto &C : enumerate(TreeNode->children())) { + SubtreeBuilders.emplace_back(&C.value(), NextInstrID); + Partitioner->applyForPartition(C.index(), *this, SubtreeBuilders.back()); + } + + TreeNode->setPartitioner(std::move(Partitioner)); + + // Recurse into the subtree builders. Each one must get a copy of the + // remaining partitioners as each path has to check everything. + for (auto &SubtreeBuilder : SubtreeBuilders) { + for (const auto &Partitioner : Partitioners) + SubtreeBuilder.addPartitioner(Partitioner->clone()); + SubtreeBuilder.runStep(); + } +} + +std::unique_ptr<GIMatchTree> GIMatchTreeBuilder::run() { + unsigned NewInstrID = allocInstrID(); + // Start by recording the root instruction as instr #0 and set up the initial + // partitioners. + for (auto &Leaf : Leaves) { + LLVM_DEBUG(Leaf.getMatchDag().writeDOTGraph(dbgs(), Leaf.getName())); + GIMatchDagInstr *Root = + *(Leaf.getMatchDag().roots().begin() + Leaf.getRootIdx()); + Leaf.declareInstr(Root, NewInstrID); + } + + addPartitionersForInstr(NewInstrID); + + std::unique_ptr<GIMatchTree> TreeRoot = std::make_unique<GIMatchTree>(); + TreeNode = TreeRoot.get(); + runStep(); + + return TreeRoot; +} + +void GIMatchTreeOpcodePartitioner::emitPartitionName(raw_ostream &OS, unsigned Idx) const { + if (PartitionToInstr[Idx] == nullptr) { + OS << "* or nullptr"; + return; + } + OS << PartitionToInstr[Idx]->Namespace + << "::" << PartitionToInstr[Idx]->TheDef->getName(); +} + +void GIMatchTreeOpcodePartitioner::repartition( + GIMatchTreeBuilder::LeafVec &Leaves) { + Partitions.clear(); + InstrToPartition.clear(); + PartitionToInstr.clear(); + TestedPredicates.clear(); + + for (const auto &Leaf : enumerate(Leaves)) { + bool AllOpcodes = true; + GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); + BitVector TestedPredicatesForLeaf( + Leaf.value().getMatchDag().getNumPredicates()); + + // If the instruction isn't declared then we don't care about it. Ignore + // it for now and add it to all partitions later once we know what + // partitions we have. + if (!InstrInfo) { + LLVM_DEBUG(dbgs() << " " << Leaf.value().getName() + << " doesn't care about Instr[" << InstrID << "]\n"); + assert(TestedPredicatesForLeaf.size() == Leaf.value().getMatchDag().getNumPredicates()); + TestedPredicates.push_back(TestedPredicatesForLeaf); + continue; + } + + // If the opcode is available to test then any opcode predicates will have + // been enabled too. + for (unsigned PIdx : Leaf.value().TestablePredicates.set_bits()) { + const auto &P = Leaf.value().getPredicate(PIdx); + SmallVector<const CodeGenInstruction *, 1> OpcodesForThisPredicate; + if (const auto *OpcodeP = dyn_cast<const GIMatchDagOpcodePredicate>(P)) { + // We've found _an_ opcode predicate, but we don't know if it's + // checking this instruction yet. + bool IsThisPredicate = false; + for (const auto &PDep : Leaf.value().getMatchDag().predicate_edges()) { + if (PDep->getRequiredMI() == InstrInfo->getInstrNode() && + PDep->getRequiredMO() == nullptr && PDep->getPredicate() == P) { + IsThisPredicate = true; + break; + } + } + if (!IsThisPredicate) + continue; + + // If we get here twice then we've somehow ended up with two opcode + // predicates for one instruction in the same DAG. That should be + // impossible. + assert(AllOpcodes && "Conflicting opcode predicates"); + const CodeGenInstruction *Expected = OpcodeP->getInstr(); + OpcodesForThisPredicate.push_back(Expected); + } + + if (const auto *OpcodeP = + dyn_cast<const GIMatchDagOneOfOpcodesPredicate>(P)) { + // We've found _an_ oneof(opcodes) predicate, but we don't know if it's + // checking this instruction yet. + bool IsThisPredicate = false; + for (const auto &PDep : Leaf.value().getMatchDag().predicate_edges()) { + if (PDep->getRequiredMI() == InstrInfo->getInstrNode() && + PDep->getRequiredMO() == nullptr && PDep->getPredicate() == P) { + IsThisPredicate = true; + break; + } + } + if (!IsThisPredicate) + continue; + + // If we get here twice then we've somehow ended up with two opcode + // predicates for one instruction in the same DAG. That should be + // impossible. + assert(AllOpcodes && "Conflicting opcode predicates"); + for (const CodeGenInstruction *Expected : OpcodeP->getInstrs()) + OpcodesForThisPredicate.push_back(Expected); + } + + for (const CodeGenInstruction *Expected : OpcodesForThisPredicate) { + // Mark this predicate as one we're testing. + TestedPredicatesForLeaf.set(PIdx); + + // Partitions must be numbered 0, 1, .., N but instructions don't meet + // that requirement. Assign a partition number to each opcode if we + // lack one ... + auto Partition = InstrToPartition.find(Expected); + if (Partition == InstrToPartition.end()) { + BitVector Contents(Leaves.size()); + Partition = InstrToPartition + .insert(std::make_pair(Expected, Partitions.size())) + .first; + PartitionToInstr.push_back(Expected); + Partitions.insert(std::make_pair(Partitions.size(), Contents)); + } + // ... and mark this leaf as being in that partition. + Partitions.find(Partition->second)->second.set(Leaf.index()); + AllOpcodes = false; + LLVM_DEBUG(dbgs() << " " << Leaf.value().getName() + << " is in partition " << Partition->second << "\n"); + } + + // TODO: This is where we would handle multiple choices of opcode + // the end result will be that this leaf ends up in multiple + // partitions similarly to AllOpcodes. + } + + // If we never check the opcode, add it to every partition. + if (AllOpcodes) { + // Add a partition for the default case if we don't already have one. + if (InstrToPartition.insert(std::make_pair(nullptr, 0)).second) { + PartitionToInstr.push_back(nullptr); + BitVector Contents(Leaves.size()); + Partitions.insert(std::make_pair(Partitions.size(), Contents)); + } + LLVM_DEBUG(dbgs() << " " << Leaf.value().getName() + << " is in all partitions (opcode not checked)\n"); + for (auto &Partition : Partitions) + Partition.second.set(Leaf.index()); + } + + assert(TestedPredicatesForLeaf.size() == Leaf.value().getMatchDag().getNumPredicates()); + TestedPredicates.push_back(TestedPredicatesForLeaf); + } + + if (Partitions.size() == 0) { + // Add a partition for the default case if we don't already have one. + if (InstrToPartition.insert(std::make_pair(nullptr, 0)).second) { + PartitionToInstr.push_back(nullptr); + BitVector Contents(Leaves.size()); + Partitions.insert(std::make_pair(Partitions.size(), Contents)); + } + } + + // Add any leaves that don't care about this instruction to all partitions. + for (const auto &Leaf : enumerate(Leaves)) { + GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); + if (!InstrInfo) { + // Add a partition for the default case if we don't already have one. + if (InstrToPartition.insert(std::make_pair(nullptr, 0)).second) { + PartitionToInstr.push_back(nullptr); + BitVector Contents(Leaves.size()); + Partitions.insert(std::make_pair(Partitions.size(), Contents)); + } + for (auto &Partition : Partitions) + Partition.second.set(Leaf.index()); + } + } + +} + +void GIMatchTreeOpcodePartitioner::applyForPartition( + unsigned PartitionIdx, GIMatchTreeBuilder &Builder, GIMatchTreeBuilder &SubBuilder) { + LLVM_DEBUG(dbgs() << " Making partition " << PartitionIdx << "\n"); + const CodeGenInstruction *CGI = PartitionToInstr[PartitionIdx]; + + BitVector PossibleLeaves = getPossibleLeavesForPartition(PartitionIdx); + // Consume any predicates we handled. + for (auto &EnumeratedLeaf : enumerate(Builder.getPossibleLeaves())) { + if (!PossibleLeaves[EnumeratedLeaf.index()]) + continue; + + auto &Leaf = EnumeratedLeaf.value(); + const auto &TestedPredicatesForLeaf = + TestedPredicates[EnumeratedLeaf.index()]; + + for (unsigned PredIdx : TestedPredicatesForLeaf.set_bits()) { + LLVM_DEBUG(dbgs() << " " << Leaf.getName() << " tested predicate #" + << PredIdx << " of " << TestedPredicatesForLeaf.size() + << " " << *Leaf.getPredicate(PredIdx) << "\n"); + Leaf.RemainingPredicates.reset(PredIdx); + Leaf.TestablePredicates.reset(PredIdx); + } + SubBuilder.addLeaf(Leaf); + } + + // Nothing to do, we don't know anything about this instruction as a result + // of this partitioner. + if (CGI == nullptr) + return; + + GIMatchTreeBuilder::LeafVec &NewLeaves = SubBuilder.getPossibleLeaves(); + // Find all the operands we know to exist and are referenced. This will + // usually be all the referenced operands but there are some cases where + // instructions are variadic. Such operands must be handled by partitioners + // that check the number of operands. + BitVector ReferencedOperands(1); + for (auto &Leaf : NewLeaves) { + GIMatchTreeInstrInfo *InstrInfo = Leaf.getInstrInfo(InstrID); + // Skip any leaves that don't care about this instruction. + if (!InstrInfo) + continue; + const GIMatchDagInstr *Instr = InstrInfo->getInstrNode(); + for (auto &E : enumerate(Leaf.getMatchDag().edges())) { + if (E.value()->getFromMI() == Instr && + E.value()->getFromMO()->getIdx() < CGI->Operands.size()) { + ReferencedOperands.resize(E.value()->getFromMO()->getIdx() + 1); + ReferencedOperands.set(E.value()->getFromMO()->getIdx()); + } + } + } + for (auto &Leaf : NewLeaves) { + for (unsigned OpIdx : ReferencedOperands.set_bits()) { + Leaf.declareOperand(InstrID, OpIdx); + } + } + for (unsigned OpIdx : ReferencedOperands.set_bits()) { + SubBuilder.addPartitionersForOperand(InstrID, OpIdx); + } +} + +void GIMatchTreeOpcodePartitioner::emitPartitionResults( + raw_ostream &OS) const { + OS << "Partitioning by opcode would produce " << Partitions.size() + << " partitions\n"; + for (const auto &Partition : InstrToPartition) { + if (Partition.first == nullptr) + OS << "Default: "; + else + OS << Partition.first->TheDef->getName() << ": "; + StringRef Separator = ""; + for (unsigned I : Partitions.find(Partition.second)->second.set_bits()) { + OS << Separator << I; + Separator = ", "; + } + OS << "\n"; + } +} + +void GIMatchTreeOpcodePartitioner::generatePartitionSelectorCode( + raw_ostream &OS, StringRef Indent) const { + OS << Indent << "Partition = -1;\n" + << Indent << "switch (MIs[" << InstrID << "]->getOpcode()) {\n"; + for (const auto &EnumInstr : enumerate(PartitionToInstr)) { + if (EnumInstr.value() == nullptr) + OS << Indent << "default:"; + else + OS << Indent << "case " << EnumInstr.value()->Namespace + << "::" << EnumInstr.value()->TheDef->getName() << ":"; + OS << " Partition = " << EnumInstr.index() << "; break;\n"; + } + OS << Indent << "}\n" + << Indent + << "// Default case but without conflicting with potential default case " + "in selection.\n" + << Indent << "if (Partition == -1) return false;\n"; +} + +void GIMatchTreeVRegDefPartitioner::addToPartition(bool Result, + unsigned LeafIdx) { + auto I = ResultToPartition.find(Result); + if (I == ResultToPartition.end()) { + ResultToPartition.insert(std::make_pair(Result, PartitionToResult.size())); + PartitionToResult.push_back(Result); + } + I = ResultToPartition.find(Result); + auto P = Partitions.find(I->second); + if (P == Partitions.end()) + P = Partitions.insert(std::make_pair(I->second, BitVector())).first; + P->second.resize(LeafIdx + 1); + P->second.set(LeafIdx); +} + +void GIMatchTreeVRegDefPartitioner::repartition( + GIMatchTreeBuilder::LeafVec &Leaves) { + Partitions.clear(); + + for (const auto &Leaf : enumerate(Leaves)) { + GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); + BitVector TraversedEdgesForLeaf(Leaf.value().getMatchDag().getNumEdges()); + + // If the instruction isn't declared then we don't care about it. Ignore + // it for now and add it to all partitions later once we know what + // partitions we have. + if (!InstrInfo) { + TraversedEdges.push_back(TraversedEdgesForLeaf); + continue; + } + + // If this node has an use -> def edge from this operand then this + // instruction must be in partition 1 (isVRegDef()). + bool WantsEdge = false; + for (unsigned EIdx : Leaf.value().TraversableEdges.set_bits()) { + const auto &E = Leaf.value().getEdge(EIdx); + if (E->getFromMI() != InstrInfo->getInstrNode() || + E->getFromMO()->getIdx() != OpIdx || E->isDefToUse()) + continue; + + // We're looking at the right edge. This leaf wants a vreg def so we'll + // put it in partition 1. + addToPartition(true, Leaf.index()); + TraversedEdgesForLeaf.set(EIdx); + WantsEdge = true; + } + + bool isNotReg = false; + if (!WantsEdge && isNotReg) { + // If this leaf doesn't have an edge and we _don't_ want a register, + // then add it to partition 0. + addToPartition(false, Leaf.index()); + } else if (!WantsEdge) { + // If this leaf doesn't have an edge and we don't know what we want, + // then add it to partition 0 and 1. + addToPartition(false, Leaf.index()); + addToPartition(true, Leaf.index()); + } + + TraversedEdges.push_back(TraversedEdgesForLeaf); + } + + // Add any leaves that don't care about this instruction to all partitions. + for (const auto &Leaf : enumerate(Leaves)) { + GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); + if (!InstrInfo) + for (auto &Partition : Partitions) + Partition.second.set(Leaf.index()); + } +} + +void GIMatchTreeVRegDefPartitioner::applyForPartition( + unsigned PartitionIdx, GIMatchTreeBuilder &Builder, + GIMatchTreeBuilder &SubBuilder) { + BitVector PossibleLeaves = getPossibleLeavesForPartition(PartitionIdx); + + std::vector<BitVector> TraversedEdgesByNewLeaves; + // Consume any edges we handled. + for (auto &EnumeratedLeaf : enumerate(Builder.getPossibleLeaves())) { + if (!PossibleLeaves[EnumeratedLeaf.index()]) + continue; + + auto &Leaf = EnumeratedLeaf.value(); + const auto &TraversedEdgesForLeaf = TraversedEdges[EnumeratedLeaf.index()]; + TraversedEdgesByNewLeaves.push_back(TraversedEdgesForLeaf); + Leaf.RemainingEdges.reset(TraversedEdgesForLeaf); + Leaf.TraversableEdges.reset(TraversedEdgesForLeaf); + SubBuilder.addLeaf(Leaf); + } + + // Nothing to do. The only thing we know is that it isn't a vreg-def. + if (PartitionToResult[PartitionIdx] == false) + return; + + NewInstrID = SubBuilder.allocInstrID(); + + GIMatchTreeBuilder::LeafVec &NewLeaves = SubBuilder.getPossibleLeaves(); + for (const auto I : zip(NewLeaves, TraversedEdgesByNewLeaves)) { + auto &Leaf = std::get<0>(I); + auto &TraversedEdgesForLeaf = std::get<1>(I); + GIMatchTreeInstrInfo *InstrInfo = Leaf.getInstrInfo(InstrID); + // Skip any leaves that don't care about this instruction. + if (!InstrInfo) + continue; + for (unsigned EIdx : TraversedEdgesForLeaf.set_bits()) { + const GIMatchDagEdge *E = Leaf.getEdge(EIdx); + Leaf.declareInstr(E->getToMI(), NewInstrID); + } + } + SubBuilder.addPartitionersForInstr(NewInstrID); +} + +void GIMatchTreeVRegDefPartitioner::emitPartitionResults( + raw_ostream &OS) const { + OS << "Partitioning by vreg-def would produce " << Partitions.size() + << " partitions\n"; + for (const auto &Partition : Partitions) { + OS << Partition.first << " ("; + emitPartitionName(OS, Partition.first); + OS << "): "; + StringRef Separator = ""; + for (unsigned I : Partition.second.set_bits()) { + OS << Separator << I; + Separator = ", "; + } + OS << "\n"; + } +} + +void GIMatchTreeVRegDefPartitioner::generatePartitionSelectorCode( + raw_ostream &OS, StringRef Indent) const { + OS << Indent << "Partition = -1\n" + << Indent << "if (MIs.size() <= NewInstrID) MIs.resize(NewInstrID + 1);\n" + << Indent << "MIs[" << NewInstrID << "] = nullptr;\n" + << Indent << "if (MIs[" << InstrID << "].getOperand(" << OpIdx + << ").isReg()))\n" + << Indent << " MIs[" << NewInstrID << "] = MRI.getVRegDef(MIs[" << InstrID + << "].getOperand(" << OpIdx << ").getReg()));\n"; + + for (const auto &Pair : ResultToPartition) + OS << Indent << "if (MIs[" << NewInstrID << "] " + << (Pair.first ? "==" : "!=") + << " nullptr) Partition = " << Pair.second << ";\n"; + + OS << Indent << "if (Partition == -1) return false;\n"; +} + diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.h new file mode 100644 index 000000000000..b86f6454589c --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.h @@ -0,0 +1,629 @@ +//===- GIMatchTree.h - A decision tree to match GIMatchDag's --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHTREE_H +#define LLVM_UTILS_TABLEGEN_GIMATCHTREE_H + +#include "GIMatchDag.h" +#include "llvm/ADT/BitVector.h" + +namespace llvm { +class raw_ostream; + +class GIMatchTreeBuilder; +class GIMatchTreePartitioner; + +/// Describes the binding of a variable to the matched MIR +class GIMatchTreeVariableBinding { + /// The name of the variable described by this binding. + StringRef Name; + // The matched instruction it is bound to. + unsigned InstrID; + // The matched operand (if appropriate) it is bound to. + Optional<unsigned> OpIdx; + +public: + GIMatchTreeVariableBinding(StringRef Name, unsigned InstrID, + Optional<unsigned> OpIdx = None) + : Name(Name), InstrID(InstrID), OpIdx(OpIdx) {} + + bool isInstr() const { return !OpIdx.hasValue(); } + StringRef getName() const { return Name; } + unsigned getInstrID() const { return InstrID; } + unsigned getOpIdx() const { + assert(OpIdx.hasValue() && "Is not an operand binding"); + return *OpIdx; + } +}; + +/// Associates a matchable with a leaf of the decision tree. +class GIMatchTreeLeafInfo { +public: + using const_var_binding_iterator = + std::vector<GIMatchTreeVariableBinding>::const_iterator; + using UntestedPredicatesTy = SmallVector<const GIMatchDagPredicate *, 1>; + using const_untested_predicates_iterator = UntestedPredicatesTy::const_iterator; + +protected: + /// A name for the matchable. This is primarily for debugging. + StringRef Name; + /// Where rules have multiple roots, this is which root we're starting from. + unsigned RootIdx; + /// Opaque data the caller of the tree building code understands. + void *Data; + /// Has the decision tree covered every edge traversal? If it hasn't then this + /// is an unrecoverable error indicating there's something wrong with the + /// partitioners. + bool IsFullyTraversed; + /// Has the decision tree covered every predicate test? If it has, then + /// subsequent matchables on the same leaf are unreachable. If it hasn't, the + /// code that requested the GIMatchTree is responsible for finishing off any + /// remaining predicates. + bool IsFullyTested; + /// The variable bindings associated with this leaf so far. + std::vector<GIMatchTreeVariableBinding> VarBindings; + /// Any predicates left untested by the time we reach this leaf. + UntestedPredicatesTy UntestedPredicates; + +public: + GIMatchTreeLeafInfo() { llvm_unreachable("Cannot default-construct"); } + GIMatchTreeLeafInfo(StringRef Name, unsigned RootIdx, void *Data) + : Name(Name), RootIdx(RootIdx), Data(Data), IsFullyTraversed(false), + IsFullyTested(false) {} + + StringRef getName() const { return Name; } + unsigned getRootIdx() const { return RootIdx; } + template <class Ty> Ty *getTargetData() const { + return static_cast<Ty *>(Data); + } + bool isFullyTraversed() const { return IsFullyTraversed; } + void setIsFullyTraversed(bool V) { IsFullyTraversed = V; } + bool isFullyTested() const { return IsFullyTested; } + void setIsFullyTested(bool V) { IsFullyTested = V; } + + void bindInstrVariable(StringRef Name, unsigned InstrID) { + VarBindings.emplace_back(Name, InstrID); + } + void bindOperandVariable(StringRef Name, unsigned InstrID, unsigned OpIdx) { + VarBindings.emplace_back(Name, InstrID, OpIdx); + } + + const_var_binding_iterator var_bindings_begin() const { + return VarBindings.begin(); + } + const_var_binding_iterator var_bindings_end() const { + return VarBindings.end(); + } + iterator_range<const_var_binding_iterator> var_bindings() const { + return make_range(VarBindings.begin(), VarBindings.end()); + } + iterator_range<const_untested_predicates_iterator> untested_predicates() const { + return make_range(UntestedPredicates.begin(), UntestedPredicates.end()); + } + void addUntestedPredicate(const GIMatchDagPredicate *P) { + UntestedPredicates.push_back(P); + } +}; + +/// The nodes of a decision tree used to perform the match. +/// This will be used to generate the C++ code or state machine equivalent. +/// +/// It should be noted that some nodes of this tree (most notably nodes handling +/// def -> use edges) will need to iterate over several possible matches. As +/// such, code generated from this will sometimes need to support backtracking. +class GIMatchTree { + using LeafVector = std::vector<GIMatchTreeLeafInfo>; + + /// The partitioner that has been chosen for this node. This may be nullptr if + /// a partitioner hasn't been chosen yet or if the node is a leaf. + std::unique_ptr<GIMatchTreePartitioner> Partitioner; + /// All the leaves that are possible for this node of the tree. + /// Note: This should be emptied after the tree is built when there are + /// children but this currently isn't done to aid debuggability of the DOT + /// graph for the decision tree. + LeafVector PossibleLeaves; + /// The children of this node. The index into this array must match the index + /// chosen by the partitioner. + std::vector<GIMatchTree> Children; + + void writeDOTGraphNode(raw_ostream &OS) const; + void writeDOTGraphEdges(raw_ostream &OS) const; + +public: + void writeDOTGraph(raw_ostream &OS) const; + + void setNumChildren(unsigned Num) { Children.resize(Num); } + void addPossibleLeaf(const GIMatchTreeLeafInfo &V, bool IsFullyTraversed, + bool IsFullyTested) { + PossibleLeaves.push_back(V); + PossibleLeaves.back().setIsFullyTraversed(IsFullyTraversed); + PossibleLeaves.back().setIsFullyTested(IsFullyTested); + } + void dropLeavesAfter(size_t Length) { + if (PossibleLeaves.size() > Length) + PossibleLeaves.resize(Length); + } + void setPartitioner(std::unique_ptr<GIMatchTreePartitioner> &&V) { + Partitioner = std::move(V); + } + GIMatchTreePartitioner *getPartitioner() const { return Partitioner.get(); } + + std::vector<GIMatchTree>::iterator children_begin() { + return Children.begin(); + } + std::vector<GIMatchTree>::iterator children_end() { return Children.end(); } + iterator_range<std::vector<GIMatchTree>::iterator> children() { + return make_range(children_begin(), children_end()); + } + std::vector<GIMatchTree>::const_iterator children_begin() const { + return Children.begin(); + } + std::vector<GIMatchTree>::const_iterator children_end() const { + return Children.end(); + } + iterator_range<std::vector<GIMatchTree>::const_iterator> children() const { + return make_range(children_begin(), children_end()); + } + + LeafVector::const_iterator possible_leaves_begin() const { + return PossibleLeaves.begin(); + } + LeafVector::const_iterator possible_leaves_end() const { + return PossibleLeaves.end(); + } + iterator_range<LeafVector::const_iterator> + possible_leaves() const { + return make_range(possible_leaves_begin(), possible_leaves_end()); + } + LeafVector::iterator possible_leaves_begin() { + return PossibleLeaves.begin(); + } + LeafVector::iterator possible_leaves_end() { + return PossibleLeaves.end(); + } + iterator_range<LeafVector::iterator> possible_leaves() { + return make_range(possible_leaves_begin(), possible_leaves_end()); + } +}; + +/// Record information that is known about the instruction bound to this ID and +/// GIMatchDagInstrNode. Every rule gets its own set of +/// GIMatchTreeInstrInfo to bind the shared IDs to an instr node in its +/// DAG. +/// +/// For example, if we know that there are 3 operands. We can record it here to +/// elide duplicate checks. +class GIMatchTreeInstrInfo { + /// The instruction ID for the matched instruction. + unsigned ID; + /// The corresponding instruction node in the MatchDAG. + const GIMatchDagInstr *InstrNode; + +public: + GIMatchTreeInstrInfo(unsigned ID, const GIMatchDagInstr *InstrNode) + : ID(ID), InstrNode(InstrNode) {} + + unsigned getID() const { return ID; } + const GIMatchDagInstr *getInstrNode() const { return InstrNode; } +}; + +/// Record information that is known about the operand bound to this ID, OpIdx, +/// and GIMatchDagInstrNode. Every rule gets its own set of +/// GIMatchTreeOperandInfo to bind the shared IDs to an operand of an +/// instr node from its DAG. +/// +/// For example, if we know that there the operand is a register. We can record +/// it here to elide duplicate checks. +class GIMatchTreeOperandInfo { + /// The corresponding instruction node in the MatchDAG that the operand + /// belongs to. + const GIMatchDagInstr *InstrNode; + unsigned OpIdx; + +public: + GIMatchTreeOperandInfo(const GIMatchDagInstr *InstrNode, unsigned OpIdx) + : InstrNode(InstrNode), OpIdx(OpIdx) {} + + const GIMatchDagInstr *getInstrNode() const { return InstrNode; } + unsigned getOpIdx() const { return OpIdx; } +}; + +/// Represent a leaf of the match tree and any working data we need to build the +/// tree. +/// +/// It's important to note that each rule can have multiple +/// GIMatchTreeBuilderLeafInfo's since the partitioners do not always partition +/// into mutually-exclusive partitions. For example: +/// R1: (FOO ..., ...) +/// R2: (oneof(FOO, BAR) ..., ...) +/// will partition by opcode into two partitions FOO=>[R1, R2], and BAR=>[R2] +/// +/// As an optimization, all instructions, edges, and predicates in the DAGs are +/// numbered and tracked in BitVectors. As such, the GIMatchDAG must not be +/// modified once construction of the tree has begun. +class GIMatchTreeBuilderLeafInfo { +protected: + GIMatchTreeBuilder &Builder; + GIMatchTreeLeafInfo Info; + const GIMatchDag &MatchDag; + /// The association between GIMatchDagInstr* and GIMatchTreeInstrInfo. + /// The primary reason for this members existence is to allow the use of + /// InstrIDToInfo.lookup() since that requires that the value is + /// default-constructible. + DenseMap<const GIMatchDagInstr *, GIMatchTreeInstrInfo> InstrNodeToInfo; + /// The instruction information for a given ID in the context of this + /// particular leaf. + DenseMap<unsigned, GIMatchTreeInstrInfo *> InstrIDToInfo; + /// The operand information for a given ID and OpIdx in the context of this + /// particular leaf. + DenseMap<std::pair<unsigned, unsigned>, GIMatchTreeOperandInfo> + OperandIDToInfo; + +public: + /// The remaining instrs/edges/predicates to visit + BitVector RemainingInstrNodes; + BitVector RemainingEdges; + BitVector RemainingPredicates; + + // The remaining predicate dependencies for each predicate + std::vector<BitVector> UnsatisfiedPredDepsForPred; + + /// The edges/predicates we can visit as a result of the declare*() calls we + /// have already made. We don't need an instrs version since edges imply the + /// instr. + BitVector TraversableEdges; + BitVector TestablePredicates; + + /// Map predicates from the DAG to their position in the DAG predicate + /// iterators. + DenseMap<GIMatchDagPredicate *, unsigned> PredicateIDs; + /// Map predicate dependency edges from the DAG to their position in the DAG + /// predicate dependency iterators. + DenseMap<GIMatchDagPredicateDependencyEdge *, unsigned> PredicateDepIDs; + +public: + GIMatchTreeBuilderLeafInfo(GIMatchTreeBuilder &Builder, StringRef Name, + unsigned RootIdx, const GIMatchDag &MatchDag, + void *Data); + + StringRef getName() const { return Info.getName(); } + GIMatchTreeLeafInfo &getInfo() { return Info; } + const GIMatchTreeLeafInfo &getInfo() const { return Info; } + const GIMatchDag &getMatchDag() const { return MatchDag; } + unsigned getRootIdx() const { return Info.getRootIdx(); } + + /// Has this DAG been fully traversed. This must be true by the time the tree + /// builder finishes. + bool isFullyTraversed() const { + // We don't need UnsatisfiedPredDepsForPred because RemainingPredicates + // can't be all-zero without satisfying all the dependencies. The same is + // almost true for Edges and Instrs but it's possible to have Instrs without + // Edges. + return RemainingInstrNodes.none() && RemainingEdges.none(); + } + + /// Has this DAG been fully tested. This hould be true by the time the tree + /// builder finishes but clients can finish any untested predicates left over + /// if it's not true. + bool isFullyTested() const { + // We don't need UnsatisfiedPredDepsForPred because RemainingPredicates + // can't be all-zero without satisfying all the dependencies. The same is + // almost true for Edges and Instrs but it's possible to have Instrs without + // Edges. + return RemainingInstrNodes.none() && RemainingEdges.none() && + RemainingPredicates.none(); + } + + const GIMatchDagInstr *getInstr(unsigned Idx) const { + return *(MatchDag.instr_nodes_begin() + Idx); + } + const GIMatchDagEdge *getEdge(unsigned Idx) const { + return *(MatchDag.edges_begin() + Idx); + } + GIMatchDagEdge *getEdge(unsigned Idx) { + return *(MatchDag.edges_begin() + Idx); + } + const GIMatchDagPredicate *getPredicate(unsigned Idx) const { + return *(MatchDag.predicates_begin() + Idx); + } + iterator_range<llvm::BitVector::const_set_bits_iterator> + untested_instrs() const { + return RemainingInstrNodes.set_bits(); + } + iterator_range<llvm::BitVector::const_set_bits_iterator> + untested_edges() const { + return RemainingEdges.set_bits(); + } + iterator_range<llvm::BitVector::const_set_bits_iterator> + untested_predicates() const { + return RemainingPredicates.set_bits(); + } + + /// Bind an instr node to the given ID and clear any blocking dependencies + /// that were waiting for it. + void declareInstr(const GIMatchDagInstr *Instr, unsigned ID); + + /// Bind an operand to the given ID and OpIdx and clear any blocking + /// dependencies that were waiting for it. + void declareOperand(unsigned InstrID, unsigned OpIdx); + + GIMatchTreeInstrInfo *getInstrInfo(unsigned ID) const { + auto I = InstrIDToInfo.find(ID); + if (I != InstrIDToInfo.end()) + return I->second; + return nullptr; + } + + void dump(raw_ostream &OS) const { + OS << "Leaf " << getName() << " for root #" << getRootIdx() << "\n"; + MatchDag.print(OS); + for (const auto &I : InstrIDToInfo) + OS << "Declared Instr #" << I.first << "\n"; + for (const auto &I : OperandIDToInfo) + OS << "Declared Instr #" << I.first.first << ", Op #" << I.first.second + << "\n"; + OS << RemainingInstrNodes.count() << " untested instrs of " + << RemainingInstrNodes.size() << "\n"; + OS << RemainingEdges.count() << " untested edges of " + << RemainingEdges.size() << "\n"; + OS << RemainingPredicates.count() << " untested predicates of " + << RemainingPredicates.size() << "\n"; + + OS << TraversableEdges.count() << " edges could be traversed\n"; + OS << TestablePredicates.count() << " predicates could be tested\n"; + } +}; + +/// The tree builder has a fairly tough job. It's purpose is to merge all the +/// DAGs from the ruleset into a decision tree that walks all of them +/// simultaneously and identifies the rule that was matched. In addition to +/// that, it also needs to find the most efficient order to make decisions +/// without violating any dependencies and ensure that every DAG covers every +/// instr/edge/predicate. +class GIMatchTreeBuilder { +public: + using LeafVec = std::vector<GIMatchTreeBuilderLeafInfo>; + +protected: + /// The leaves that the resulting decision tree will distinguish. + LeafVec Leaves; + /// The tree node being constructed. + GIMatchTree *TreeNode; + /// The builders for each subtree resulting from the current decision. + std::vector<GIMatchTreeBuilder> SubtreeBuilders; + /// The possible partitioners we could apply right now. + std::vector<std::unique_ptr<GIMatchTreePartitioner>> Partitioners; + /// The next instruction ID to allocate when requested by the chosen + /// Partitioner. + unsigned NextInstrID; + + /// Use any context we have stored to cull partitioners that only test things + /// we already know. At the time of writing, there's no need to do anything + /// here but it will become important once, for example, there is a + /// num-operands and an opcode partitioner. This is because applying an opcode + /// partitioner (usually) makes the number of operands known which makes + /// additional checking pointless. + void filterRedundantPartitioners(); + + /// Evaluate the available partioners and select the best one at the moment. + void evaluatePartitioners(); + + /// Construct the current tree node. + void runStep(); + +public: + GIMatchTreeBuilder(unsigned NextInstrID) : NextInstrID(NextInstrID) {} + GIMatchTreeBuilder(GIMatchTree *TreeNode, unsigned NextInstrID) + : TreeNode(TreeNode), NextInstrID(NextInstrID) {} + + void addLeaf(StringRef Name, unsigned RootIdx, const GIMatchDag &MatchDag, + void *Data) { + Leaves.emplace_back(*this, Name, RootIdx, MatchDag, Data); + } + void addLeaf(const GIMatchTreeBuilderLeafInfo &L) { Leaves.push_back(L); } + void addPartitioner(std::unique_ptr<GIMatchTreePartitioner> P) { + Partitioners.push_back(std::move(P)); + } + void addPartitionersForInstr(unsigned InstrIdx); + void addPartitionersForOperand(unsigned InstrID, unsigned OpIdx); + + LeafVec &getPossibleLeaves() { return Leaves; } + + unsigned allocInstrID() { return NextInstrID++; } + + /// Construct the decision tree. + std::unique_ptr<GIMatchTree> run(); +}; + +/// Partitioners are the core of the tree builder and are unfortunately rather +/// tricky to write. +class GIMatchTreePartitioner { +protected: + /// The partitions resulting from applying the partitioner to the possible + /// leaves. The keys must be consecutive integers starting from 0. This can + /// lead to some unfortunate situations where partitioners test a predicate + /// and use 0 for success and 1 for failure if the ruleset encounters a + /// success case first but is necessary to assign the partition to one of the + /// tree nodes children. As a result, you usually need some kind of + /// indirection to map the natural keys (e.g. ptrs/bools) to this linear + /// sequence. The values are a bitvector indicating which leaves belong to + /// this partition. + DenseMap<unsigned, BitVector> Partitions; + +public: + virtual ~GIMatchTreePartitioner() {} + virtual std::unique_ptr<GIMatchTreePartitioner> clone() const = 0; + + /// Determines which partitions the given leaves belong to. A leaf may belong + /// to multiple partitions in which case it will be duplicated during + /// applyForPartition(). + /// + /// This function can be rather complicated. A few particular things to be + /// aware of include: + /// * One leaf can be assigned to multiple partitions when there's some + /// ambiguity. + /// * Not all DAG's for the leaves may be able to perform the test. For + /// example, the opcode partitiioner must account for one DAG being a + /// superset of another such as [(ADD ..., ..., ...)], and [(MUL t, ..., + /// ...), (ADD ..., t, ...)] + /// * Attaching meaning to a particular partition index will generally not + /// work due to the '0, 1, ..., n' requirement. You might encounter cases + /// where only partition 1 is seen, leaving a missing 0. + /// * Finding a specific predicate such as the opcode predicate for a specific + /// instruction is non-trivial. It's often O(NumPredicates), leading to + /// O(NumPredicates*NumRules) when applied to the whole ruleset. The good + /// news there is that n is typically small thanks to predicate dependencies + /// limiting how many are testable at once. Also, with opcode and type + /// predicates being so frequent the value of m drops very fast too. It + /// wouldn't be terribly surprising to see a 10k ruleset drop down to an + /// average of 100 leaves per partition after a single opcode partitioner. + /// * The same goes for finding specific edges. The need to traverse them in + /// dependency order dramatically limits the search space at any given + /// moment. + /// * If you need to add a leaf to all partitions, make sure you don't forget + /// them when adding partitions later. + virtual void repartition(GIMatchTreeBuilder::LeafVec &Leaves) = 0; + + /// Delegate the leaves for a given partition to the corresponding subbuilder, + /// update any recorded context for this partition (e.g. allocate instr id's + /// for instrs recorder by the current node), and clear any blocking + /// dependencies this partitioner resolved. + virtual void applyForPartition(unsigned PartitionIdx, + GIMatchTreeBuilder &Builder, + GIMatchTreeBuilder &SubBuilder) = 0; + + /// Return a BitVector indicating which leaves should be transferred to the + /// specified partition. Note that the same leaf can be indicated for multiple + /// partitions. + BitVector getPossibleLeavesForPartition(unsigned Idx) { + const auto &I = Partitions.find(Idx); + assert(I != Partitions.end() && "Requested non-existant partition"); + return I->second; + } + + size_t getNumPartitions() const { return Partitions.size(); } + size_t getNumLeavesWithDupes() const { + size_t S = 0; + for (const auto &P : Partitions) + S += P.second.size(); + return S; + } + + /// Emit a brief description of the partitioner suitable for debug printing or + /// use in a DOT graph. + virtual void emitDescription(raw_ostream &OS) const = 0; + /// Emit a label for the given partition suitable for debug printing or use in + /// a DOT graph. + virtual void emitPartitionName(raw_ostream &OS, unsigned Idx) const = 0; + + /// Emit a long description of how the partitioner partitions the leaves. + virtual void emitPartitionResults(raw_ostream &OS) const = 0; + + /// Generate code to select between partitions based on the MIR being matched. + /// This is typically a switch statement that picks a partition index. + virtual void generatePartitionSelectorCode(raw_ostream &OS, + StringRef Indent) const = 0; +}; + +/// Partition according to the opcode of the instruction. +/// +/// Numbers CodeGenInstr ptrs for use as partition ID's. One special partition, +/// nullptr, represents the case where the instruction isn't known. +/// +/// * If the opcode can be tested and is a single opcode, create the partition +/// for that opcode and assign the leaf to it. This partition no longer needs +/// to test the opcode, and many details about the instruction will usually +/// become known (e.g. number of operands for non-variadic instrs) via the +/// CodeGenInstr ptr. +/// * (not implemented yet) If the opcode can be tested and is a choice of +/// opcodes, then the leaf can be treated like the single-opcode case but must +/// be added to all relevant partitions and not quite as much becomes known as +/// a result. That said, multiple-choice opcodes are likely similar enough +/// (because if they aren't then handling them together makes little sense) +/// that plenty still becomes known. The main implementation issue with this +/// is having a description to represent the commonality between instructions. +/// * If the opcode is not tested, the leaf must be added to all partitions +/// including the wildcard nullptr partition. What becomes known as a result +/// varies between partitions. +/// * If the instruction to be tested is not declared then add the leaf to all +/// partitions. This occurs when we encounter one rule that is a superset of +/// the other and we are still matching the remainder of the superset. The +/// result is that the cases that don't match the superset will match the +/// subset rule, while the ones that do match the superset will match either +/// (which one is algorithm dependent but will usually be the superset). +class GIMatchTreeOpcodePartitioner : public GIMatchTreePartitioner { + unsigned InstrID; + DenseMap<const CodeGenInstruction *, unsigned> InstrToPartition; + std::vector<const CodeGenInstruction *> PartitionToInstr; + std::vector<BitVector> TestedPredicates; + +public: + GIMatchTreeOpcodePartitioner(unsigned InstrID) : InstrID(InstrID) {} + + std::unique_ptr<GIMatchTreePartitioner> clone() const override { + return std::make_unique<GIMatchTreeOpcodePartitioner>(*this); + } + + void emitDescription(raw_ostream &OS) const override { + OS << "MI[" << InstrID << "].getOpcode()"; + } + + void emitPartitionName(raw_ostream &OS, unsigned Idx) const override; + + void repartition(GIMatchTreeBuilder::LeafVec &Leaves) override; + void applyForPartition(unsigned Idx, GIMatchTreeBuilder &SubBuilder, + GIMatchTreeBuilder &Builder) override; + + void emitPartitionResults(raw_ostream &OS) const override; + + void generatePartitionSelectorCode(raw_ostream &OS, + StringRef Indent) const override; +}; + +class GIMatchTreeVRegDefPartitioner : public GIMatchTreePartitioner { + unsigned NewInstrID = -1; + unsigned InstrID; + unsigned OpIdx; + std::vector<BitVector> TraversedEdges; + DenseMap<unsigned, unsigned> ResultToPartition; + std::vector<bool> PartitionToResult; + + void addToPartition(bool Result, unsigned LeafIdx); + +public: + GIMatchTreeVRegDefPartitioner(unsigned InstrID, unsigned OpIdx) + : InstrID(InstrID), OpIdx(OpIdx) {} + + std::unique_ptr<GIMatchTreePartitioner> clone() const override { + return std::make_unique<GIMatchTreeVRegDefPartitioner>(*this); + } + + void emitDescription(raw_ostream &OS) const override { + OS << "MI[" << NewInstrID << "] = getVRegDef(MI[" << InstrID + << "].getOperand(" << OpIdx << "))"; + } + + void emitPartitionName(raw_ostream &OS, unsigned Idx) const override { + bool Result = PartitionToResult[Idx]; + if (Result) + OS << "true"; + else + OS << "false"; + } + + void repartition(GIMatchTreeBuilder::LeafVec &Leaves) override; + void applyForPartition(unsigned PartitionIdx, GIMatchTreeBuilder &Builder, + GIMatchTreeBuilder &SubBuilder) override; + void emitPartitionResults(raw_ostream &OS) const override; + + void generatePartitionSelectorCode(raw_ostream &OS, + StringRef Indent) const override; +}; + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHTREE_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp index f1c02134198b..c14294951cc1 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -249,6 +249,10 @@ static std::string explainPredicates(const TreePatternNode *N) { OS << ']'; } + int64_t MinAlign = P.getMinAlignment(); + if (MinAlign > 0) + Explanation += " MinAlign=" + utostr(MinAlign); + if (P.isAtomicOrderingMonotonic()) Explanation += " monotonic"; if (P.isAtomicOrderingAcquire()) @@ -329,6 +333,9 @@ static Error isTrivialOperatorNode(const TreePatternNode *N) { const ListInit *AddrSpaces = Predicate.getAddressSpaces(); if (AddrSpaces && !AddrSpaces->empty()) continue; + + if (Predicate.getMinAlignment() > 0) + continue; } if (Predicate.isAtomic() && Predicate.getMemoryVT()) @@ -602,7 +609,7 @@ MatchTableRecord MatchTable::LineBreak = { void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis, const MatchTable &Table) const { bool UseLineComment = - LineBreakIsNextAfterThis | (Flags & MTRF_LineBreakFollows); + LineBreakIsNextAfterThis || (Flags & MTRF_LineBreakFollows); if (Flags & (MTRF_JumpTarget | MTRF_CommaFollows)) UseLineComment = false; @@ -613,7 +620,7 @@ void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis, if (Flags & MTRF_Label) OS << ": @" << Table.getLabelIndex(LabelID); - if (Flags & MTRF_Comment && !UseLineComment) + if ((Flags & MTRF_Comment) && !UseLineComment) OS << "*/"; if (Flags & MTRF_JumpTarget) { @@ -822,6 +829,10 @@ protected: /// the renderers. StringMap<OperandMatcher *> DefinedOperands; + /// A map of anonymous physical register operands defined by the matchers that + /// may be referenced by the renderers. + DenseMap<Record *, OperandMatcher *> PhysRegOperands; + /// ID for the next instruction variable defined with implicitlyDefineInsnVar() unsigned NextInsnVarID; @@ -904,6 +915,8 @@ public: void defineOperand(StringRef SymbolicName, OperandMatcher &OM); + void definePhysRegOperand(Record *Reg, OperandMatcher &OM); + Error defineComplexSubOperand(StringRef SymbolicName, Record *ComplexPattern, unsigned RendererID, unsigned SubOperandID) { if (ComplexSubOperands.count(SymbolicName)) @@ -927,6 +940,7 @@ public: InstructionMatcher &getInstructionMatcher(StringRef SymbolicName) const; const OperandMatcher &getOperandMatcher(StringRef Name) const; + const OperandMatcher &getPhysRegOperandMatcher(Record *) const; void optimize() override; void emit(MatchTable &Table) override; @@ -1048,14 +1062,17 @@ public: IPM_Opcode, IPM_NumOperands, IPM_ImmPredicate, + IPM_Imm, IPM_AtomicOrderingMMO, IPM_MemoryLLTSize, IPM_MemoryVsLLTSize, IPM_MemoryAddressSpace, + IPM_MemoryAlignment, IPM_GenericPredicate, OPM_SameOperand, OPM_ComplexPattern, OPM_IntrinsicID, + OPM_CmpPredicate, OPM_Instruction, OPM_Int, OPM_LiteralInt, @@ -1164,7 +1181,7 @@ public: TypeIDValues.clear(); unsigned ID = 0; - for (const LLTCodeGen LLTy : KnownTypes) + for (const LLTCodeGen &LLTy : KnownTypes) TypeIDValues[LLTy] = ID++; } @@ -1324,6 +1341,23 @@ public: } }; +class ImmOperandMatcher : public OperandPredicateMatcher { +public: + ImmOperandMatcher(unsigned InsnVarID, unsigned OpIdx) + : OperandPredicateMatcher(IPM_Imm, InsnVarID, OpIdx) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_Imm; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckIsImm") << MatchTable::Comment("MI") + << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op") + << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak; + } +}; + /// Generates code to check that an operand is a G_CONSTANT with a particular /// int. class ConstantIntOperandMatcher : public OperandPredicateMatcher { @@ -1381,6 +1415,36 @@ public: } }; +/// Generates code to check that an operand is an CmpInst predicate +class CmpPredicateOperandMatcher : public OperandPredicateMatcher { +protected: + std::string PredName; + +public: + CmpPredicateOperandMatcher(unsigned InsnVarID, unsigned OpIdx, + std::string P) + : OperandPredicateMatcher(OPM_CmpPredicate, InsnVarID, OpIdx), PredName(P) {} + + bool isIdentical(const PredicateMatcher &B) const override { + return OperandPredicateMatcher::isIdentical(B) && + PredName == cast<CmpPredicateOperandMatcher>(&B)->PredName; + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_CmpPredicate; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckCmpPredicate") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) + << MatchTable::Comment("Predicate") + << MatchTable::NamedValue("CmpInst", PredName) + << MatchTable::LineBreak; + } +}; + /// Generates code to check that an operand is an intrinsic ID. class IntrinsicIDOperandMatcher : public OperandPredicateMatcher { protected: @@ -1442,7 +1506,7 @@ public: Optional<Kind *> addPredicate(Args &&... args) { if (isSameAsAnotherOperand()) return None; - Predicates.emplace_back(llvm::make_unique<Kind>( + Predicates.emplace_back(std::make_unique<Kind>( getInsnVarID(), getOpIdx(), std::forward<Args>(args)...)); return static_cast<Kind *>(Predicates.back().get()); } @@ -1643,7 +1707,7 @@ public: } StringRef getOpcode() const { return I->TheDef->getName(); } - unsigned getNumOperands() const { return I->Operands.size(); } + bool isVariadicNumOperands() const { return I->Operands.isVariadic; } StringRef getOperandType(unsigned OpIdx) const { return I->Operands[OpIdx].OperandType; @@ -1849,6 +1913,40 @@ public: } }; +class MemoryAlignmentPredicateMatcher : public InstructionPredicateMatcher { +protected: + unsigned MMOIdx; + int MinAlign; + +public: + MemoryAlignmentPredicateMatcher(unsigned InsnVarID, unsigned MMOIdx, + int MinAlign) + : InstructionPredicateMatcher(IPM_MemoryAlignment, InsnVarID), + MMOIdx(MMOIdx), MinAlign(MinAlign) { + assert(MinAlign > 0); + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_MemoryAlignment; + } + + bool isIdentical(const PredicateMatcher &B) const override { + if (!InstructionPredicateMatcher::isIdentical(B)) + return false; + auto *Other = cast<MemoryAlignmentPredicateMatcher>(&B); + return MMOIdx == Other->MMOIdx && MinAlign == Other->MinAlign; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckMemoryAlignment") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx) + << MatchTable::Comment("MinAlign") << MatchTable::IntValue(MinAlign) + << MatchTable::LineBreak; + } +}; + /// Generates code to check that the size of an MMO is less-than, equal-to, or /// greater than a given LLT. class MemoryVsLLTSizePredicateMatcher : public InstructionPredicateMatcher { @@ -1945,6 +2043,11 @@ protected: std::string SymbolicName; unsigned InsnVarID; + /// PhysRegInputs - List list has an entry for each explicitly specified + /// physreg input to the pattern. The first elt is the Register node, the + /// second is the recorded slot number the input pattern match saved it in. + SmallVector<std::pair<Record *, unsigned>, 2> PhysRegInputs; + public: InstructionMatcher(RuleMatcher &Rule, StringRef SymbolicName) : Rule(Rule), SymbolicName(SymbolicName) { @@ -1957,7 +2060,7 @@ public: template <class Kind, class... Args> Optional<Kind *> addPredicate(Args &&... args) { Predicates.emplace_back( - llvm::make_unique<Kind>(getInsnVarID(), std::forward<Args>(args)...)); + std::make_unique<Kind>(getInsnVarID(), std::forward<Args>(args)...)); return static_cast<Kind *>(Predicates.back().get()); } @@ -1986,6 +2089,20 @@ public: llvm_unreachable("Failed to lookup operand"); } + OperandMatcher &addPhysRegInput(Record *Reg, unsigned OpIdx, + unsigned TempOpIdx) { + assert(SymbolicName.empty()); + OperandMatcher *OM = new OperandMatcher(*this, OpIdx, "", TempOpIdx); + Operands.emplace_back(OM); + Rule.definePhysRegOperand(Reg, *OM); + PhysRegInputs.emplace_back(Reg, OpIdx); + return *OM; + } + + ArrayRef<std::pair<Record *, unsigned>> getPhysRegInputs() const { + return PhysRegInputs; + } + StringRef getSymbolicName() const { return SymbolicName; } unsigned getNumOperands() const { return Operands.size(); } OperandVec::iterator operands_begin() { return Operands.begin(); } @@ -2036,7 +2153,7 @@ public: return false; } - for (const auto &Operand : zip(Operands, B.Operands)) { + for (auto Operand : zip(Operands, B.Operands)) { if (std::get<0>(Operand)->isHigherPriorityThan(*std::get<1>(Operand))) return true; if (std::get<1>(Operand)->isHigherPriorityThan(*std::get<0>(Operand))) @@ -2156,7 +2273,7 @@ void InstructionMatcher::optimize() { Stash.push_back(predicates_pop_front()); if (Stash.back().get() == &OpcMatcher) { - if (NumOperandsCheck && OpcMatcher.getNumOperands() < getNumOperands()) + if (NumOperandsCheck && OpcMatcher.isVariadicNumOperands()) Stash.emplace_back( new InstructionNumOperandsMatcher(InsnVarID, getNumOperands())); NumOperandsCheck = false; @@ -2193,13 +2310,16 @@ public: OR_Copy, OR_CopyOrAddZeroReg, OR_CopySubReg, + OR_CopyPhysReg, OR_CopyConstantAsImm, OR_CopyFConstantAsFPImm, OR_Imm, + OR_SubRegIndex, OR_Register, OR_TempRegister, OR_ComplexPattern, - OR_Custom + OR_Custom, + OR_CustomOperand }; protected: @@ -2247,6 +2367,38 @@ public: } }; +/// A CopyRenderer emits code to copy a virtual register to a specific physical +/// register. +class CopyPhysRegRenderer : public OperandRenderer { +protected: + unsigned NewInsnID; + Record *PhysReg; + +public: + CopyPhysRegRenderer(unsigned NewInsnID, Record *Reg) + : OperandRenderer(OR_CopyPhysReg), NewInsnID(NewInsnID), + PhysReg(Reg) { + assert(PhysReg); + } + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_CopyPhysReg; + } + + Record *getPhysReg() const { return PhysReg; } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + const OperandMatcher &Operand = Rule.getPhysRegOperandMatcher(PhysReg); + unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher()); + Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID") + << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx") + << MatchTable::IntValue(Operand.getOpIdx()) + << MatchTable::Comment(PhysReg->getName()) + << MatchTable::LineBreak; + } +}; + /// A CopyOrAddZeroRegRenderer emits code to copy a single operand from an /// existing instruction to the one being built. If the operand turns out to be /// a 'G_CONSTANT 0' then it replaces the operand with a zero register. @@ -2393,11 +2545,13 @@ class AddRegisterRenderer : public OperandRenderer { protected: unsigned InsnID; const Record *RegisterDef; + bool IsDef; public: - AddRegisterRenderer(unsigned InsnID, const Record *RegisterDef) - : OperandRenderer(OR_Register), InsnID(InsnID), RegisterDef(RegisterDef) { - } + AddRegisterRenderer(unsigned InsnID, const Record *RegisterDef, + bool IsDef = false) + : OperandRenderer(OR_Register), InsnID(InsnID), RegisterDef(RegisterDef), + IsDef(IsDef) {} static bool classof(const OperandRenderer *R) { return R->getKind() == OR_Register; @@ -2411,7 +2565,16 @@ public: ? RegisterDef->getValueAsString("Namespace") : ""), RegisterDef->getName()) - << MatchTable::LineBreak; + << MatchTable::Comment("AddRegisterRegFlags"); + + // TODO: This is encoded as a 64-bit element, but only 16 or 32-bits are + // really needed for a physical register reference. We can pack the + // register and flags in a single field. + if (IsDef) + Table << MatchTable::NamedValue("RegState::Define"); + else + Table << MatchTable::IntValue(0); + Table << MatchTable::LineBreak; } }; @@ -2467,6 +2630,28 @@ public: } }; +/// Adds an enum value for a subreg index to the instruction being built. +class SubRegIndexRenderer : public OperandRenderer { +protected: + unsigned InsnID; + const CodeGenSubRegIndex *SubRegIdx; + +public: + SubRegIndexRenderer(unsigned InsnID, const CodeGenSubRegIndex *SRI) + : OperandRenderer(OR_SubRegIndex), InsnID(InsnID), SubRegIdx(SRI) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_SubRegIndex; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID") + << MatchTable::IntValue(InsnID) << MatchTable::Comment("SubRegIndex") + << MatchTable::IntValue(SubRegIdx->EnumValue) + << MatchTable::LineBreak; + } +}; + /// Adds operands by calling a renderer function supplied by the ComplexPattern /// matcher function. class RenderComplexPatternOperand : public OperandRenderer { @@ -2542,6 +2727,38 @@ public: } }; +class CustomOperandRenderer : public OperandRenderer { +protected: + unsigned InsnID; + const Record &Renderer; + /// The name of the operand. + const std::string SymbolicName; + +public: + CustomOperandRenderer(unsigned InsnID, const Record &Renderer, + StringRef SymbolicName) + : OperandRenderer(OR_CustomOperand), InsnID(InsnID), Renderer(Renderer), + SymbolicName(SymbolicName) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_CustomOperand; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + const OperandMatcher &OpdMatcher = Rule.getOperandMatcher(SymbolicName); + Table << MatchTable::Opcode("GIR_CustomOperandRenderer") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OpdMatcher.getInsnVarID()) + << MatchTable::Comment("OpIdx") + << MatchTable::IntValue(OpdMatcher.getOpIdx()) + << MatchTable::Comment("OperandRenderer") + << MatchTable::NamedValue( + "GICR_" + Renderer.getValueAsString("RendererFn").str()) + << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; + } +}; + /// An action taken when all Matcher predicates succeeded for a parent rule. /// /// Typical actions include: @@ -2620,7 +2837,7 @@ public: template <class Kind, class... Args> Kind &addRenderer(Args&&... args) { OperandRenderers.emplace_back( - llvm::make_unique<Kind>(InsnID, std::forward<Args>(args)...)); + std::make_unique<Kind>(InsnID, std::forward<Args>(args)...)); return *static_cast<Kind *>(OperandRenderers.back().get()); } @@ -2747,7 +2964,9 @@ private: public: MakeTempRegisterAction(const LLTCodeGen &Ty, unsigned TempRegID) - : Ty(Ty), TempRegID(TempRegID) {} + : Ty(Ty), TempRegID(TempRegID) { + KnownTypes.insert(Ty); + } void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { Table << MatchTable::Opcode("GIR_MakeTempReg") @@ -2781,7 +3000,7 @@ const std::vector<Record *> &RuleMatcher::getRequiredFeatures() const { // iterator. template <class Kind, class... Args> Kind &RuleMatcher::addAction(Args &&... args) { - Actions.emplace_back(llvm::make_unique<Kind>(std::forward<Args>(args)...)); + Actions.emplace_back(std::make_unique<Kind>(std::forward<Args>(args)...)); return *static_cast<Kind *>(Actions.back().get()); } @@ -2796,7 +3015,7 @@ template <class Kind, class... Args> action_iterator RuleMatcher::insertAction(action_iterator InsertPt, Args &&... args) { return Actions.emplace(InsertPt, - llvm::make_unique<Kind>(std::forward<Args>(args)...)); + std::make_unique<Kind>(std::forward<Args>(args)...)); } unsigned RuleMatcher::implicitlyDefineInsnVar(InstructionMatcher &Matcher) { @@ -2823,6 +3042,13 @@ void RuleMatcher::defineOperand(StringRef SymbolicName, OperandMatcher &OM) { OM.addPredicate<SameOperandMatcher>(OM.getSymbolicName()); } +void RuleMatcher::definePhysRegOperand(Record *Reg, OperandMatcher &OM) { + if (PhysRegOperands.find(Reg) == PhysRegOperands.end()) { + PhysRegOperands[Reg] = &OM; + return; + } +} + InstructionMatcher & RuleMatcher::getInstructionMatcher(StringRef SymbolicName) const { for (const auto &I : InsnVariableIDs) @@ -2833,6 +3059,18 @@ RuleMatcher::getInstructionMatcher(StringRef SymbolicName) const { } const OperandMatcher & +RuleMatcher::getPhysRegOperandMatcher(Record *Reg) const { + const auto &I = PhysRegOperands.find(Reg); + + if (I == PhysRegOperands.end()) { + PrintFatalError(SrcLoc, "Register " + Reg->getName() + + " was not declared in matcher"); + } + + return *I->second; +} + +const OperandMatcher & RuleMatcher::getOperandMatcher(StringRef Name) const { const auto &I = DefinedOperands.find(Name); @@ -2954,7 +3192,7 @@ bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const { if (Matchers.size() < B.Matchers.size()) return false; - for (const auto &Matcher : zip(Matchers, B.Matchers)) { + for (auto Matcher : zip(Matchers, B.Matchers)) { if (std::get<0>(Matcher)->isHigherPriorityThan(*std::get<1>(Matcher))) return true; if (std::get<1>(Matcher)->isHigherPriorityThan(*std::get<0>(Matcher))) @@ -3076,12 +3314,12 @@ private: unsigned &TempOpIdx) const; Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher, const TreePatternNode *SrcChild, - bool OperandIsAPointer, unsigned OpIdx, - unsigned &TempOpIdx); + bool OperandIsAPointer, bool OperandIsImmArg, + unsigned OpIdx, unsigned &TempOpIdx); - Expected<BuildMIAction &> - createAndImportInstructionRenderer(RuleMatcher &M, - const TreePatternNode *Dst); + Expected<BuildMIAction &> createAndImportInstructionRenderer( + RuleMatcher &M, InstructionMatcher &InsnMatcher, + const TreePatternNode *Src, const TreePatternNode *Dst); Expected<action_iterator> createAndImportSubInstructionRenderer( action_iterator InsertPt, RuleMatcher &M, const TreePatternNode *Dst, unsigned TempReg); @@ -3089,6 +3327,7 @@ private: createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M, const TreePatternNode *Dst); void importExplicitDefRenderers(BuildMIAction &DstMIBuilder); + Expected<action_iterator> importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, @@ -3122,6 +3361,32 @@ private: MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize, bool WithCoverage); + /// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned + /// CodeGenRegisterClass will support the CodeGenRegisterClass of + /// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode. + /// If no register class is found, return None. + Optional<const CodeGenRegisterClass *> + inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty, + TreePatternNode *SuperRegNode, + TreePatternNode *SubRegIdxNode); + Optional<CodeGenSubRegIndex *> + inferSubRegIndexForNode(TreePatternNode *SubRegIdxNode); + + /// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode. + /// Return None if no such class exists. + Optional<const CodeGenRegisterClass *> + inferSuperRegisterClass(const TypeSetByHwMode &Ty, + TreePatternNode *SubRegIdxNode); + + /// Return the CodeGenRegisterClass associated with \p Leaf if it has one. + Optional<const CodeGenRegisterClass *> + getRegClassFromLeaf(TreePatternNode *Leaf); + + /// Return a CodeGenRegisterClass for \p N if one can be found. Return None + /// otherwise. + Optional<const CodeGenRegisterClass *> + inferRegClassFromPattern(TreePatternNode *N); + public: /// Takes a sequence of \p Rules and group them based on the predicates /// they share. \p MatcherStorage is used as a memory container @@ -3190,6 +3455,13 @@ Record *GlobalISelEmitter::findNodeEquiv(Record *N) const { const CodeGenInstruction * GlobalISelEmitter::getEquivNode(Record &Equiv, const TreePatternNode *N) const { + if (N->getNumChildren() >= 1) { + // setcc operation maps to two different G_* instructions based on the type. + if (!Equiv.isValueUnset("IfFloatingPoint") && + MVT(N->getChild(0)->getSimpleType(0)).isFloatingPoint()) + return &Target.getInstruction(Equiv.getValueAsDef("IfFloatingPoint")); + } + for (const TreePredicateCall &Call : N->getPredicateCalls()) { const TreePredicateFn &Predicate = Call.Fn; if (!Equiv.isValueUnset("IfSignExtend") && Predicate.isLoad() && @@ -3199,6 +3471,7 @@ GlobalISelEmitter::getEquivNode(Record &Equiv, const TreePatternNode *N) const { Predicate.isZeroExtLoad()) return &Target.getInstruction(Equiv.getValueAsDef("IfZeroExtend")); } + return &Target.getInstruction(Equiv.getValueAsDef("I")); } @@ -3212,7 +3485,7 @@ Error GlobalISelEmitter::importRulePredicates(RuleMatcher &M, ArrayRef<Predicate> Predicates) { for (const Predicate &P : Predicates) { - if (!P.Def) + if (!P.Def || P.getCondString().empty()) continue; declareSubtargetFeature(P.Def); M.addRequiredFeature(P.Def); @@ -3287,6 +3560,10 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( 0, ParsedAddrSpaces); } } + + int64_t MinAlign = Predicate.getMinAlignment(); + if (MinAlign > 0) + InsnMatcher.addPredicate<MemoryAlignmentPredicateMatcher>(0, MinAlign); } // G_LOAD is used for both non-extending and any-extending loads. @@ -3301,11 +3578,19 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( continue; } - if (Predicate.isStore() && Predicate.isTruncStore()) { - // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size. - InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( - 0, MemoryVsLLTSizePredicateMatcher::LessThan, 0); - continue; + if (Predicate.isStore()) { + if (Predicate.isTruncStore()) { + // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size. + InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( + 0, MemoryVsLLTSizePredicateMatcher::LessThan, 0); + continue; + } + if (Predicate.isNonTruncStore()) { + // We need to check the sizes match here otherwise we could incorrectly + // match truncating stores with non-truncating ones. + InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( + 0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0); + } } // No check required. We already did it by swapping the opcode. @@ -3405,6 +3690,10 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( } if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("NotAtomic"); + else if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( + "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); + } if (Src->isLeaf()) { Init *SrcInit = Src->getLeafValue(); @@ -3427,33 +3716,81 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( return InsnMatcher; } + // Special case because the operand order is changed from setcc. The + // predicate operand needs to be swapped from the last operand to the first + // source. + + unsigned NumChildren = Src->getNumChildren(); + bool IsFCmp = SrcGIOrNull->TheDef->getName() == "G_FCMP"; + + if (IsFCmp || SrcGIOrNull->TheDef->getName() == "G_ICMP") { + TreePatternNode *SrcChild = Src->getChild(NumChildren - 1); + if (SrcChild->isLeaf()) { + DefInit *DI = dyn_cast<DefInit>(SrcChild->getLeafValue()); + Record *CCDef = DI ? DI->getDef() : nullptr; + if (!CCDef || !CCDef->isSubClassOf("CondCode")) + return failedImport("Unable to handle CondCode"); + + OperandMatcher &OM = + InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx); + StringRef PredType = IsFCmp ? CCDef->getValueAsString("FCmpPredicate") : + CCDef->getValueAsString("ICmpPredicate"); + + if (!PredType.empty()) { + OM.addPredicate<CmpPredicateOperandMatcher>(PredType); + // Process the other 2 operands normally. + --NumChildren; + } + } + } + // Match the used operands (i.e. the children of the operator). - for (unsigned i = 0, e = Src->getNumChildren(); i != e; ++i) { + bool IsIntrinsic = + SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" || + SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_W_SIDE_EFFECTS"; + const CodeGenIntrinsic *II = Src->getIntrinsicInfo(CGP); + if (IsIntrinsic && !II) + return failedImport("Expected IntInit containing intrinsic ID)"); + + for (unsigned i = 0; i != NumChildren; ++i) { TreePatternNode *SrcChild = Src->getChild(i); + // We need to determine the meaning of a literal integer based on the + // context. If this is a field required to be an immediate (such as an + // immarg intrinsic argument), the required predicates are different than + // a constant which may be materialized in a register. If we have an + // argument that is required to be an immediate, we should not emit an LLT + // type check, and should not be looking for a G_CONSTANT defined + // register. + bool OperandIsImmArg = SrcGIOrNull->isOperandImmArg(i); + // SelectionDAG allows pointers to be represented with iN since it doesn't // distinguish between pointers and integers but they are different types in GlobalISel. // Coerce integers to pointers to address space 0 if the context indicates a pointer. + // bool OperandIsAPointer = SrcGIOrNull->isOperandAPointer(i); - // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately - // following the defs is an intrinsic ID. - if ((SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" || - SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_W_SIDE_EFFECTS") && - i == 0) { - if (const CodeGenIntrinsic *II = Src->getIntrinsicInfo(CGP)) { + if (IsIntrinsic) { + // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately + // following the defs is an intrinsic ID. + if (i == 0) { OperandMatcher &OM = InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx); OM.addPredicate<IntrinsicIDOperandMatcher>(II); continue; } - return failedImport("Expected IntInit containing instrinsic ID)"); + // We have to check intrinsics for llvm_anyptr_ty and immarg parameters. + // + // Note that we have to look at the i-1th parameter, because we don't + // have the intrinsic ID in the intrinsic's parameter list. + OperandIsAPointer |= II->isParamAPointer(i - 1); + OperandIsImmArg |= II->isParamImmArg(i - 1); } if (auto Error = importChildMatcher(Rule, InsnMatcher, SrcChild, OperandIsAPointer, - OpIdx++, TempOpIdx)) + OperandIsImmArg, OpIdx++, TempOpIdx)) return std::move(Error); } } @@ -3473,14 +3810,35 @@ Error GlobalISelEmitter::importComplexPatternOperandMatcher( return Error::success(); } -Error GlobalISelEmitter::importChildMatcher(RuleMatcher &Rule, - InstructionMatcher &InsnMatcher, - const TreePatternNode *SrcChild, - bool OperandIsAPointer, - unsigned OpIdx, - unsigned &TempOpIdx) { - OperandMatcher &OM = - InsnMatcher.addOperand(OpIdx, SrcChild->getName(), TempOpIdx); +// Get the name to use for a pattern operand. For an anonymous physical register +// input, this should use the register name. +static StringRef getSrcChildName(const TreePatternNode *SrcChild, + Record *&PhysReg) { + StringRef SrcChildName = SrcChild->getName(); + if (SrcChildName.empty() && SrcChild->isLeaf()) { + if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild->getLeafValue())) { + auto *ChildRec = ChildDefInit->getDef(); + if (ChildRec->isSubClassOf("Register")) { + SrcChildName = ChildRec->getName(); + PhysReg = ChildRec; + } + } + } + + return SrcChildName; +} + +Error GlobalISelEmitter::importChildMatcher( + RuleMatcher &Rule, InstructionMatcher &InsnMatcher, + const TreePatternNode *SrcChild, bool OperandIsAPointer, + bool OperandIsImmArg, unsigned OpIdx, unsigned &TempOpIdx) { + + Record *PhysReg = nullptr; + StringRef SrcChildName = getSrcChildName(SrcChild, PhysReg); + + OperandMatcher &OM = PhysReg ? + InsnMatcher.addPhysRegInput(PhysReg, OpIdx, TempOpIdx) : + InsnMatcher.addOperand(OpIdx, SrcChildName, TempOpIdx); if (OM.isSameAsAnotherOperand()) return Error::success(); @@ -3496,13 +3854,21 @@ Error GlobalISelEmitter::importChildMatcher(RuleMatcher &Rule, OM.addPredicate<MBBOperandMatcher>(); return Error::success(); } + if (SrcChild->getOperator()->getName() == "timm") { + OM.addPredicate<ImmOperandMatcher>(); + return Error::success(); + } } } - if (auto Error = - OM.addTypeCheckPredicate(ChildTypes.front(), OperandIsAPointer)) - return failedImport(toString(std::move(Error)) + " for Src operand (" + - to_string(*SrcChild) + ")"); + // Immediate arguments have no meaningful type to check as they don't have + // registers. + if (!OperandIsImmArg) { + if (auto Error = + OM.addTypeCheckPredicate(ChildTypes.front(), OperandIsAPointer)) + return failedImport(toString(std::move(Error)) + " for Src operand (" + + to_string(*SrcChild) + ")"); + } // Check for nested instructions. if (!SrcChild->isLeaf()) { @@ -3553,7 +3919,13 @@ Error GlobalISelEmitter::importChildMatcher(RuleMatcher &Rule, // Check for constant immediates. if (auto *ChildInt = dyn_cast<IntInit>(SrcChild->getLeafValue())) { - OM.addPredicate<ConstantIntOperandMatcher>(ChildInt->getValue()); + if (OperandIsImmArg) { + // Checks for argument directly in operand list + OM.addPredicate<LiteralIntOperandMatcher>(ChildInt->getValue()); + } else { + // Checks for materialized constant + OM.addPredicate<ConstantIntOperandMatcher>(ChildInt->getValue()); + } return Error::success(); } @@ -3569,6 +3941,20 @@ Error GlobalISelEmitter::importChildMatcher(RuleMatcher &Rule, return Error::success(); } + if (ChildRec->isSubClassOf("Register")) { + // This just be emitted as a copy to the specific register. + ValueTypeByHwMode VT = ChildTypes.front().getValueTypeByHwMode(); + const CodeGenRegisterClass *RC + = CGRegs.getMinimalPhysRegClass(ChildRec, &VT); + if (!RC) { + return failedImport( + "Could not determine physical register class of pattern source"); + } + + OM.addPredicate<RegisterBankOperandMatcher>(*RC); + return Error::success(); + } + // Check for ValueType. if (ChildRec->isSubClassOf("ValueType")) { // We already added a type check as standard practice so this doesn't need @@ -3605,12 +3991,22 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer( } if (!DstChild->isLeaf()) { - if (DstChild->getOperator()->isSubClassOf("SDNodeXForm")) { auto Child = DstChild->getChild(0); auto I = SDNodeXFormEquivs.find(DstChild->getOperator()); if (I != SDNodeXFormEquivs.end()) { - DstMIBuilder.addRenderer<CustomRenderer>(*I->second, Child->getName()); + Record *XFormOpc = DstChild->getOperator()->getValueAsDef("Opcode"); + if (XFormOpc->getName() == "timm") { + // If this is a TargetConstant, there won't be a corresponding + // instruction to transform. Instead, this will refer directly to an + // operand in an instruction's operand list. + DstMIBuilder.addRenderer<CustomOperandRenderer>(*I->second, + Child->getName()); + } else { + DstMIBuilder.addRenderer<CustomRenderer>(*I->second, + Child->getName()); + } + return InsertPt; } return failedImport("SDNodeXForm " + Child->getName() + @@ -3631,7 +4027,10 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer( // rendered as operands. // FIXME: The target should be able to choose sign-extended when appropriate // (e.g. on Mips). - if (DstChild->getOperator()->getName() == "imm") { + if (DstChild->getOperator()->getName() == "timm") { + DstMIBuilder.addRenderer<CopyRenderer>(DstChild->getName()); + return InsertPt; + } else if (DstChild->getOperator()->getName() == "imm") { DstMIBuilder.addRenderer<CopyConstantAsImmRenderer>(DstChild->getName()); return InsertPt; } else if (DstChild->getOperator()->getName() == "fpimm") { @@ -3708,6 +4107,12 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer( return InsertPt; } + if (ChildRec->isSubClassOf("SubRegIndex")) { + CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(ChildRec); + DstMIBuilder.addRenderer<ImmRenderer>(SubIdx->EnumValue); + return InsertPt; + } + if (ChildRec->isSubClassOf("ComplexPattern")) { const auto &ComplexPattern = ComplexPatternEquivs.find(ChildRec); if (ComplexPattern == ComplexPatternEquivs.end()) @@ -3729,7 +4134,8 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer( } Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer( - RuleMatcher &M, const TreePatternNode *Dst) { + RuleMatcher &M, InstructionMatcher &InsnMatcher, const TreePatternNode *Src, + const TreePatternNode *Dst) { auto InsertPtOrError = createInstructionRenderer(M.actions_end(), M, Dst); if (auto Error = InsertPtOrError.takeError()) return std::move(Error); @@ -3737,6 +4143,17 @@ Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer( action_iterator InsertPt = InsertPtOrError.get(); BuildMIAction &DstMIBuilder = *static_cast<BuildMIAction *>(InsertPt->get()); + for (auto PhysInput : InsnMatcher.getPhysRegInputs()) { + InsertPt = M.insertAction<BuildMIAction>( + InsertPt, M.allocateOutputInsnID(), + &Target.getInstruction(RK.getDef("COPY"))); + BuildMIAction &CopyToPhysRegMIBuilder = + *static_cast<BuildMIAction *>(InsertPt->get()); + CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(PhysInput.first, + true); + CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(PhysInput.first); + } + importExplicitDefRenderers(DstMIBuilder); if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst) @@ -3768,6 +4185,78 @@ GlobalISelEmitter::createAndImportSubInstructionRenderer( if (auto Error = InsertPtOrError.takeError()) return std::move(Error); + // We need to make sure that when we import an INSERT_SUBREG as a + // subinstruction that it ends up being constrained to the correct super + // register and subregister classes. + auto OpName = Target.getInstruction(Dst->getOperator()).TheDef->getName(); + if (OpName == "INSERT_SUBREG") { + auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); + if (!SubClass) + return failedImport( + "Cannot infer register class from INSERT_SUBREG operand #1"); + Optional<const CodeGenRegisterClass *> SuperClass = + inferSuperRegisterClassForNode(Dst->getExtType(0), Dst->getChild(0), + Dst->getChild(2)); + if (!SuperClass) + return failedImport( + "Cannot infer register class for INSERT_SUBREG operand #0"); + // The destination and the super register source of an INSERT_SUBREG must + // be the same register class. + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 1, **SuperClass); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 2, **SubClass); + return InsertPtOrError.get(); + } + + if (OpName == "EXTRACT_SUBREG") { + // EXTRACT_SUBREG selects into a subregister COPY but unlike most + // instructions, the result register class is controlled by the + // subregisters of the operand. As a result, we must constrain the result + // class rather than check that it's already the right one. + auto SuperClass = inferRegClassFromPattern(Dst->getChild(0)); + if (!SuperClass) + return failedImport( + "Cannot infer register class from EXTRACT_SUBREG operand #0"); + + auto SubIdx = inferSubRegIndexForNode(Dst->getChild(1)); + if (!SubIdx) + return failedImport("EXTRACT_SUBREG child #1 is not a subreg index"); + + const auto &SrcRCDstRCPair = + (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx); + assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 0, *SrcRCDstRCPair->second); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 1, *SrcRCDstRCPair->first); + + // We're done with this pattern! It's eligible for GISel emission; return + // it. + return InsertPtOrError.get(); + } + + // Similar to INSERT_SUBREG, we also have to handle SUBREG_TO_REG as a + // subinstruction. + if (OpName == "SUBREG_TO_REG") { + auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); + if (!SubClass) + return failedImport( + "Cannot infer register class from SUBREG_TO_REG child #1"); + auto SuperClass = inferSuperRegisterClass(Dst->getExtType(0), + Dst->getChild(2)); + if (!SuperClass) + return failedImport( + "Cannot infer register class for SUBREG_TO_REG operand #0"); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 2, **SubClass); + return InsertPtOrError.get(); + } + M.insertAction<ConstrainOperandsToDefinitionAction>(InsertPt, DstMIBuilder.getInsnID()); return InsertPtOrError.get(); @@ -3786,12 +4275,9 @@ Expected<action_iterator> GlobalISelEmitter::createInstructionRenderer( // COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction // attached. Similarly for EXTRACT_SUBREG except that's a subregister copy. - if (DstI->TheDef->getName() == "COPY_TO_REGCLASS") + StringRef Name = DstI->TheDef->getName(); + if (Name == "COPY_TO_REGCLASS" || Name == "EXTRACT_SUBREG") DstI = &Target.getInstruction(RK.getDef("COPY")); - else if (DstI->TheDef->getName() == "EXTRACT_SUBREG") - DstI = &Target.getInstruction(RK.getDef("COPY")); - else if (DstI->TheDef->getName() == "REG_SEQUENCE") - return failedImport("Unable to emit REG_SEQUENCE"); return M.insertAction<BuildMIAction>(InsertPt, M.allocateOutputInsnID(), DstI); @@ -3812,8 +4298,11 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers( const CodeGenInstruction *DstI = DstMIBuilder.getCGI(); CodeGenInstruction *OrigDstI = &Target.getInstruction(Dst->getOperator()); + StringRef Name = OrigDstI->TheDef->getName(); + unsigned ExpectedDstINumUses = Dst->getNumChildren(); + // EXTRACT_SUBREG needs to use a subregister COPY. - if (OrigDstI->TheDef->getName() == "EXTRACT_SUBREG") { + if (Name == "EXTRACT_SUBREG") { if (!Dst->getChild(0)->isLeaf()) return failedImport("EXTRACT_SUBREG child #1 is not a leaf"); @@ -3843,26 +4332,87 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers( return failedImport("EXTRACT_SUBREG child #1 is not a subreg index"); } + if (Name == "REG_SEQUENCE") { + if (!Dst->getChild(0)->isLeaf()) + return failedImport("REG_SEQUENCE child #0 is not a leaf"); + + Record *RCDef = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); + if (!RCDef) + return failedImport("REG_SEQUENCE child #0 could not " + "be coerced to a register class"); + + if ((ExpectedDstINumUses - 1) % 2 != 0) + return failedImport("Malformed REG_SEQUENCE"); + + for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) { + TreePatternNode *ValChild = Dst->getChild(I); + TreePatternNode *SubRegChild = Dst->getChild(I + 1); + + if (DefInit *SubRegInit = + dyn_cast<DefInit>(SubRegChild->getLeafValue())) { + CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef()); + + auto InsertPtOrError = + importExplicitUseRenderer(InsertPt, M, DstMIBuilder, ValChild); + if (auto Error = InsertPtOrError.takeError()) + return std::move(Error); + InsertPt = InsertPtOrError.get(); + DstMIBuilder.addRenderer<SubRegIndexRenderer>(SubIdx); + } + } + + return InsertPt; + } + // Render the explicit uses. unsigned DstINumUses = OrigDstI->Operands.size() - OrigDstI->Operands.NumDefs; - unsigned ExpectedDstINumUses = Dst->getNumChildren(); - if (OrigDstI->TheDef->getName() == "COPY_TO_REGCLASS") { + if (Name == "COPY_TO_REGCLASS") { DstINumUses--; // Ignore the class constraint. ExpectedDstINumUses--; } + // NumResults - This is the number of results produced by the instruction in + // the "outs" list. + unsigned NumResults = OrigDstI->Operands.NumDefs; + + // Number of operands we know the output instruction must have. If it is + // variadic, we could have more operands. + unsigned NumFixedOperands = DstI->Operands.size(); + + // Loop over all of the fixed operands of the instruction pattern, emitting + // code to fill them all in. The node 'N' usually has number children equal to + // the number of input operands of the instruction. However, in cases where + // there are predicate operands for an instruction, we need to fill in the + // 'execute always' values. Match up the node operands to the instruction + // operands to do this. unsigned Child = 0; + + // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the + // number of operands at the end of the list which have default values. + // Those can come from the pattern if it provides enough arguments, or be + // filled in with the default if the pattern hasn't provided them. But any + // operand with a default value _before_ the last mandatory one will be + // filled in with their defaults unconditionally. + unsigned NonOverridableOperands = NumFixedOperands; + while (NonOverridableOperands > NumResults && + CGP.operandHasDefault(DstI->Operands[NonOverridableOperands - 1].Rec)) + --NonOverridableOperands; + unsigned NumDefaultOps = 0; for (unsigned I = 0; I != DstINumUses; ++I) { - const CGIOperandList::OperandInfo &DstIOperand = - DstI->Operands[DstI->Operands.NumDefs + I]; + unsigned InstOpNo = DstI->Operands.NumDefs + I; + + // Determine what to emit for this operand. + Record *OperandNode = DstI->Operands[InstOpNo].Rec; // If the operand has default values, introduce them now. - // FIXME: Until we have a decent test case that dictates we should do - // otherwise, we're going to assume that operands with default values cannot - // be specified in the patterns. Therefore, adding them will not cause us to - // end up with too many rendered operands. - if (DstIOperand.Rec->isSubClassOf("OperandWithDefaultOps")) { + if (CGP.operandHasDefault(OperandNode) && + (InstOpNo < NonOverridableOperands || Child >= Dst->getNumChildren())) { + // This is a predicate or optional def operand which the pattern has not + // overridden, or which we aren't letting it override; emit the 'default + // ops' operands. + + const CGIOperandList::OperandInfo &DstIOperand = DstI->Operands[InstOpNo]; DagInit *DefaultOps = DstIOperand.Rec->getValueAsDag("DefaultOps"); if (auto Error = importDefaultOperandRenderers( InsertPt, M, DstMIBuilder, DefaultOps)) @@ -3945,6 +4495,126 @@ Error GlobalISelEmitter::importImplicitDefRenderers( return Error::success(); } +Optional<const CodeGenRegisterClass *> +GlobalISelEmitter::getRegClassFromLeaf(TreePatternNode *Leaf) { + assert(Leaf && "Expected node?"); + assert(Leaf->isLeaf() && "Expected leaf?"); + Record *RCRec = getInitValueAsRegClass(Leaf->getLeafValue()); + if (!RCRec) + return None; + CodeGenRegisterClass *RC = CGRegs.getRegClass(RCRec); + if (!RC) + return None; + return RC; +} + +Optional<const CodeGenRegisterClass *> +GlobalISelEmitter::inferRegClassFromPattern(TreePatternNode *N) { + if (!N) + return None; + + if (N->isLeaf()) + return getRegClassFromLeaf(N); + + // We don't have a leaf node, so we have to try and infer something. Check + // that we have an instruction that we an infer something from. + + // Only handle things that produce a single type. + if (N->getNumTypes() != 1) + return None; + Record *OpRec = N->getOperator(); + + // We only want instructions. + if (!OpRec->isSubClassOf("Instruction")) + return None; + + // Don't want to try and infer things when there could potentially be more + // than one candidate register class. + auto &Inst = Target.getInstruction(OpRec); + if (Inst.Operands.NumDefs > 1) + return None; + + // Handle any special-case instructions which we can safely infer register + // classes from. + StringRef InstName = Inst.TheDef->getName(); + bool IsRegSequence = InstName == "REG_SEQUENCE"; + if (IsRegSequence || InstName == "COPY_TO_REGCLASS") { + // If we have a COPY_TO_REGCLASS, then we need to handle it specially. It + // has the desired register class as the first child. + TreePatternNode *RCChild = N->getChild(IsRegSequence ? 0 : 1); + if (!RCChild->isLeaf()) + return None; + return getRegClassFromLeaf(RCChild); + } + + // Handle destination record types that we can safely infer a register class + // from. + const auto &DstIOperand = Inst.Operands[0]; + Record *DstIOpRec = DstIOperand.Rec; + if (DstIOpRec->isSubClassOf("RegisterOperand")) { + DstIOpRec = DstIOpRec->getValueAsDef("RegClass"); + const CodeGenRegisterClass &RC = Target.getRegisterClass(DstIOpRec); + return &RC; + } + + if (DstIOpRec->isSubClassOf("RegisterClass")) { + const CodeGenRegisterClass &RC = Target.getRegisterClass(DstIOpRec); + return &RC; + } + + return None; +} + +Optional<const CodeGenRegisterClass *> +GlobalISelEmitter::inferSuperRegisterClass(const TypeSetByHwMode &Ty, + TreePatternNode *SubRegIdxNode) { + assert(SubRegIdxNode && "Expected subregister index node!"); + // We need a ValueTypeByHwMode for getSuperRegForSubReg. + if (!Ty.isValueTypeByHwMode(false)) + return None; + if (!SubRegIdxNode->isLeaf()) + return None; + DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode->getLeafValue()); + if (!SubRegInit) + return None; + CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef()); + + // Use the information we found above to find a minimal register class which + // supports the subregister and type we want. + auto RC = + Target.getSuperRegForSubReg(Ty.getValueTypeByHwMode(), CGRegs, SubIdx); + if (!RC) + return None; + return *RC; +} + +Optional<const CodeGenRegisterClass *> +GlobalISelEmitter::inferSuperRegisterClassForNode( + const TypeSetByHwMode &Ty, TreePatternNode *SuperRegNode, + TreePatternNode *SubRegIdxNode) { + assert(SuperRegNode && "Expected super register node!"); + // Check if we already have a defined register class for the super register + // node. If we do, then we should preserve that rather than inferring anything + // from the subregister index node. We can assume that whoever wrote the + // pattern in the first place made sure that the super register and + // subregister are compatible. + if (Optional<const CodeGenRegisterClass *> SuperRegisterClass = + inferRegClassFromPattern(SuperRegNode)) + return *SuperRegisterClass; + return inferSuperRegisterClass(Ty, SubRegIdxNode); +} + +Optional<CodeGenSubRegIndex *> +GlobalISelEmitter::inferSubRegIndexForNode(TreePatternNode *SubRegIdxNode) { + if (!SubRegIdxNode->isLeaf()) + return None; + + DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode->getLeafValue()); + if (!SubRegInit) + return None; + return CGRegs.getSubRegIdx(SubRegInit->getDef()); +} + Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { // Keep track of the matchers and actions to emit. int Score = P.getPatternComplexity(CGP); @@ -4035,6 +4705,8 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { return failedImport("Pattern operator isn't an instruction"); auto &DstI = Target.getInstruction(DstOp); + StringRef DstIName = DstI.TheDef->getName(); + if (DstI.Operands.NumDefs != Src->getExtTypes().size()) return failedImport("Src pattern results and dst MI defs are different (" + to_string(Src->getExtTypes().size()) + " def(s) vs " + @@ -4048,13 +4720,17 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { const auto &DstIOperand = DstI.Operands[OpIdx]; Record *DstIOpRec = DstIOperand.Rec; - if (DstI.TheDef->getName() == "COPY_TO_REGCLASS") { + if (DstIName == "COPY_TO_REGCLASS") { DstIOpRec = getInitValueAsRegClass(Dst->getChild(1)->getLeafValue()); if (DstIOpRec == nullptr) return failedImport( "COPY_TO_REGCLASS operand #1 isn't a register class"); - } else if (DstI.TheDef->getName() == "EXTRACT_SUBREG") { + } else if (DstIName == "REG_SEQUENCE") { + DstIOpRec = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); + if (DstIOpRec == nullptr) + return failedImport("REG_SEQUENCE operand #0 isn't a register class"); + } else if (DstIName == "EXTRACT_SUBREG") { if (!Dst->getChild(0)->isLeaf()) return failedImport("EXTRACT_SUBREG operand #0 isn't a leaf"); @@ -4063,8 +4739,33 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { DstIOpRec = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); if (DstIOpRec == nullptr) + return failedImport("EXTRACT_SUBREG operand #0 isn't a register class"); + } else if (DstIName == "INSERT_SUBREG") { + auto MaybeSuperClass = inferSuperRegisterClassForNode( + VTy, Dst->getChild(0), Dst->getChild(2)); + if (!MaybeSuperClass) return failedImport( - "EXTRACT_SUBREG operand #0 isn't a register class"); + "Cannot infer register class for INSERT_SUBREG operand #0"); + // Move to the next pattern here, because the register class we found + // doesn't necessarily have a record associated with it. So, we can't + // set DstIOpRec using this. + OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); + OM.setSymbolicName(DstIOperand.Name); + M.defineOperand(OM.getSymbolicName(), OM); + OM.addPredicate<RegisterBankOperandMatcher>(**MaybeSuperClass); + ++OpIdx; + continue; + } else if (DstIName == "SUBREG_TO_REG") { + auto MaybeRegClass = inferSuperRegisterClass(VTy, Dst->getChild(2)); + if (!MaybeRegClass) + return failedImport( + "Cannot infer register class for SUBREG_TO_REG operand #0"); + OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); + OM.setSymbolicName(DstIOperand.Name); + M.defineOperand(OM.getSymbolicName(), OM); + OM.addPredicate<RegisterBankOperandMatcher>(**MaybeRegClass); + ++OpIdx; + continue; } else if (DstIOpRec->isSubClassOf("RegisterOperand")) DstIOpRec = DstIOpRec->getValueAsDef("RegClass"); else if (!DstIOpRec->isSubClassOf("RegisterClass")) @@ -4079,7 +4780,8 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { ++OpIdx; } - auto DstMIBuilderOrError = createAndImportInstructionRenderer(M, Dst); + auto DstMIBuilderOrError = + createAndImportInstructionRenderer(M, InsnMatcher, Src, Dst); if (auto Error = DstMIBuilderOrError.takeError()) return std::move(Error); BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get(); @@ -4093,7 +4795,7 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { // Constrain the registers to classes. This is normally derived from the // emitted instruction but a few instructions require special handling. - if (DstI.TheDef->getName() == "COPY_TO_REGCLASS") { + if (DstIName == "COPY_TO_REGCLASS") { // COPY_TO_REGCLASS does not provide operand constraints itself but the // result is constrained to the class given by the second child. Record *DstIOpRec = @@ -4111,28 +4813,16 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { return std::move(M); } - if (DstI.TheDef->getName() == "EXTRACT_SUBREG") { - // EXTRACT_SUBREG selects into a subregister COPY but unlike most - // instructions, the result register class is controlled by the - // subregisters of the operand. As a result, we must constrain the result - // class rather than check that it's already the right one. - if (!Dst->getChild(0)->isLeaf()) - return failedImport("EXTRACT_SUBREG child #1 is not a leaf"); + if (DstIName == "EXTRACT_SUBREG") { + auto SuperClass = inferRegClassFromPattern(Dst->getChild(0)); + if (!SuperClass) + return failedImport( + "Cannot infer register class from EXTRACT_SUBREG operand #0"); - DefInit *SubRegInit = dyn_cast<DefInit>(Dst->getChild(1)->getLeafValue()); - if (!SubRegInit) + auto SubIdx = inferSubRegIndexForNode(Dst->getChild(1)); + if (!SubIdx) return failedImport("EXTRACT_SUBREG child #1 is not a subreg index"); - // Constrain the result to the same register bank as the operand. - Record *DstIOpRec = - getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); - - if (DstIOpRec == nullptr) - return failedImport("EXTRACT_SUBREG operand #1 isn't a register class"); - - CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef()); - CodeGenRegisterClass *SrcRC = CGRegs.getRegClass(DstIOpRec); - // It would be nice to leave this constraint implicit but we're required // to pick a register class so constrain the result to a register class // that can hold the correct MVT. @@ -4143,7 +4833,7 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { "Expected Src of EXTRACT_SUBREG to have one result type"); const auto &SrcRCDstRCPair = - SrcRC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx); + (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx); assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); M.addAction<ConstrainOperandToRegClassAction>(0, 0, *SrcRCDstRCPair->second); M.addAction<ConstrainOperandToRegClassAction>(0, 1, *SrcRCDstRCPair->first); @@ -4154,6 +4844,51 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { return std::move(M); } + if (DstIName == "INSERT_SUBREG") { + assert(Src->getExtTypes().size() == 1 && + "Expected Src of INSERT_SUBREG to have one result type"); + // We need to constrain the destination, a super regsister source, and a + // subregister source. + auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); + if (!SubClass) + return failedImport( + "Cannot infer register class from INSERT_SUBREG operand #1"); + auto SuperClass = inferSuperRegisterClassForNode( + Src->getExtType(0), Dst->getChild(0), Dst->getChild(2)); + if (!SuperClass) + return failedImport( + "Cannot infer register class for INSERT_SUBREG operand #0"); + M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass); + M.addAction<ConstrainOperandToRegClassAction>(0, 1, **SuperClass); + M.addAction<ConstrainOperandToRegClassAction>(0, 2, **SubClass); + ++NumPatternImported; + return std::move(M); + } + + if (DstIName == "SUBREG_TO_REG") { + // We need to constrain the destination and subregister source. + assert(Src->getExtTypes().size() == 1 && + "Expected Src of SUBREG_TO_REG to have one result type"); + + // Attempt to infer the subregister source from the first child. If it has + // an explicitly given register class, we'll use that. Otherwise, we will + // fail. + auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); + if (!SubClass) + return failedImport( + "Cannot infer register class from SUBREG_TO_REG child #1"); + // We don't have a child to look at that might have a super register node. + auto SuperClass = + inferSuperRegisterClass(Src->getExtType(0), Dst->getChild(2)); + if (!SuperClass) + return failedImport( + "Cannot infer register class for SUBREG_TO_REG operand #0"); + M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass); + M.addAction<ConstrainOperandToRegClassAction>(0, 2, **SubClass); + ++NumPatternImported; + return std::move(M); + } + M.addAction<ConstrainOperandsToDefinitionAction>(0); // We're done with this pattern! It's eligible for GISel emission; return it. @@ -4235,7 +4970,7 @@ std::vector<Matcher *> GlobalISelEmitter::optimizeRules( std::vector<std::unique_ptr<Matcher>> &MatcherStorage) { std::vector<Matcher *> OptRules; - std::unique_ptr<GroupT> CurrentGroup = make_unique<GroupT>(); + std::unique_ptr<GroupT> CurrentGroup = std::make_unique<GroupT>(); assert(CurrentGroup->empty() && "Newly created group isn't empty!"); unsigned NumGroups = 0; @@ -4256,7 +4991,7 @@ std::vector<Matcher *> GlobalISelEmitter::optimizeRules( MatcherStorage.emplace_back(std::move(CurrentGroup)); ++NumGroups; } - CurrentGroup = make_unique<GroupT>(); + CurrentGroup = std::make_unique<GroupT>(); }; for (Matcher *Rule : Rules) { // Greedily add as many matchers as possible to the current group: @@ -4439,7 +5174,7 @@ void GlobalISelEmitter::run(raw_ostream &OS) { << " typedef void(" << Target.getName() << "InstructionSelector::*CustomRendererFn)(MachineInstrBuilder &, const " - "MachineInstr&) " + "MachineInstr&, int) " "const;\n" << " const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, " "CustomRendererFn> " @@ -4484,8 +5219,23 @@ void GlobalISelEmitter::run(raw_ostream &OS) { }); SubtargetFeatureInfo::emitComputeAvailableFeatures( - Target.getName(), "InstructionSelector", "computeAvailableModuleFeatures", + Target.getName(), "InstructionSelector", "computeAvailableModuleFeatures", ModuleFeatures, OS); + + + OS << "void " << Target.getName() << "InstructionSelector" + "::setupGeneratedPerFunctionState(MachineFunction &MF) {\n" + " AvailableFunctionFeatures = computeAvailableFunctionFeatures(" + "(const " << Target.getName() << "Subtarget*)&MF.getSubtarget(), &MF);\n" + "}\n"; + + if (Target.getName() == "X86" || Target.getName() == "AArch64") { + // TODO: Implement PGSO. + OS << "static bool shouldOptForSize(const MachineFunction *MF) {\n"; + OS << " return MF->getFunction().hasOptSize();\n"; + OS << "}\n\n"; + } + SubtargetFeatureInfo::emitComputeAvailableFeatures( Target.getName(), "InstructionSelector", "computeAvailableFunctionFeatures", FunctionFeatures, OS, @@ -4525,7 +5275,7 @@ void GlobalISelEmitter::run(raw_ostream &OS) { return true; if (A.size() > B.size()) return false; - for (const auto &Pair : zip(A, B)) { + for (auto Pair : zip(A, B)) { if (std::get<0>(Pair)->getName() < std::get<1>(Pair)->getName()) return true; if (std::get<0>(Pair)->getName() > std::get<1>(Pair)->getName()) @@ -4602,7 +5352,7 @@ void GlobalISelEmitter::run(raw_ostream &OS) { OS << Target.getName() << "InstructionSelector::CustomRendererFn\n" << Target.getName() << "InstructionSelector::CustomRenderers[] = {\n" - << " nullptr, // GICP_Invalid\n"; + << " nullptr, // GICR_Invalid\n"; for (const auto &Record : CustomRendererFns) OS << " &" << Target.getName() << "InstructionSelector::" << Record->getValueAsString("RendererFn") @@ -4630,10 +5380,6 @@ void GlobalISelEmitter::run(raw_ostream &OS) { "&CoverageInfo) const {\n" << " MachineFunction &MF = *I.getParent()->getParent();\n" << " MachineRegisterInfo &MRI = MF.getRegInfo();\n" - << " // FIXME: This should be computed on a per-function basis rather " - "than per-insn.\n" - << " AvailableFunctionFeatures = computeAvailableFunctionFeatures(&STI, " - "&MF);\n" << " const PredicateBitset AvailableFeatures = getAvailableFeatures();\n" << " NewMIVector OutMIs;\n" << " State.MIs.clear();\n" @@ -4669,6 +5415,7 @@ void GlobalISelEmitter::run(raw_ostream &OS) { << "computeAvailableFunctionFeatures(const " << Target.getName() << "Subtarget *Subtarget,\n" << " const MachineFunction *MF) const;\n" + << "void setupGeneratedPerFunctionState(MachineFunction &MF) override;\n" << "#endif // ifdef GET_GLOBALISEL_PREDICATES_DECL\n"; OS << "#ifdef GET_GLOBALISEL_PREDICATES_INIT\n" diff --git a/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.cpp b/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.cpp index d9662889a5db..7cd1b0f08132 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.cpp @@ -192,6 +192,17 @@ void RegSizeInfoByHwMode::writeToStream(raw_ostream &OS) const { OS << '}'; } +EncodingInfoByHwMode::EncodingInfoByHwMode(Record *R, const CodeGenHwModes &CGH) { + const HwModeSelect &MS = CGH.getHwModeSelect(R); + for (const HwModeSelect::PairType &P : MS.Items) { + assert(P.second && P.second->isSubClassOf("InstructionEncoding") && + "Encoding must subclass InstructionEncoding"); + auto I = Map.insert({P.first, P.second}); + assert(I.second && "Duplicate entry?"); + (void)I; + } +} + namespace llvm { raw_ostream &operator<<(raw_ostream &OS, const ValueTypeByHwMode &T) { T.writeToStream(OS); diff --git a/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.h b/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.h index 9e5cc3d5f2a4..d92e5901a7f3 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.h +++ b/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.h @@ -184,6 +184,11 @@ raw_ostream &operator<<(raw_ostream &OS, const ValueTypeByHwMode &T); raw_ostream &operator<<(raw_ostream &OS, const RegSizeInfo &T); raw_ostream &operator<<(raw_ostream &OS, const RegSizeInfoByHwMode &T); +struct EncodingInfoByHwMode : public InfoByHwMode<Record*> { + EncodingInfoByHwMode(Record *R, const CodeGenHwModes &CGH); + EncodingInfoByHwMode() = default; +}; + } // namespace llvm #endif // LLVM_UTILS_TABLEGEN_INFOBYHWMODE_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/InstrDocsEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/InstrDocsEmitter.cpp index 91c457ba08fd..07efa1885409 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/InstrDocsEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/InstrDocsEmitter.cpp @@ -138,6 +138,7 @@ void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS) { FLAG(isConvergent) FLAG(hasNoSchedulingInfo) FLAG(variadicOpsAreDefs) + FLAG(isAuthenticated) if (!FlagStrings.empty()) { OS << "Flags: "; bool IsFirst = true; @@ -231,4 +232,4 @@ void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS) { } } -} // end llvm namespace +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp index 2d367f538b71..6ab58bd26a2c 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp @@ -164,6 +164,11 @@ InstrInfoEmitter::GetOperandInfo(const CodeGenInstruction &Inst) { if (Op.Rec->isSubClassOf("OptionalDefOperand")) Res += "|(1<<MCOI::OptionalDef)"; + // Branch target operands. Check to see if the original unexpanded + // operand was of type BranchTargetOperand. + if (Op.Rec->isSubClassOf("BranchTargetOperand")) + Res += "|(1<<MCOI::BranchTarget)"; + // Fill in operand type. Res += ", "; assert(!Op.OperandType.empty() && "Invalid operand type."); @@ -332,6 +337,10 @@ void InstrInfoEmitter::emitOperandTypeMappings( StringRef Namespace = Target.getInstNamespace(); std::vector<Record *> Operands = Records.getAllDerivedDefinitions("Operand"); + std::vector<Record *> RegisterOperands = + Records.getAllDerivedDefinitions("RegisterOperand"); + std::vector<Record *> RegisterClasses = + Records.getAllDerivedDefinitions("RegisterClass"); OS << "#ifdef GET_INSTRINFO_OPERAND_TYPES_ENUM\n"; OS << "#undef GET_INSTRINFO_OPERAND_TYPES_ENUM\n"; @@ -341,10 +350,13 @@ void InstrInfoEmitter::emitOperandTypeMappings( OS << "enum OperandType {\n"; unsigned EnumVal = 0; - for (const Record *Op : Operands) { - if (!Op->isAnonymous()) - OS << " " << Op->getName() << " = " << EnumVal << ",\n"; - ++EnumVal; + for (const std::vector<Record *> *RecordsToAdd : + {&Operands, &RegisterOperands, &RegisterClasses}) { + for (const Record *Op : *RecordsToAdd) { + if (!Op->isAnonymous()) + OS << " " << Op->getName() << " = " << EnumVal << ",\n"; + ++EnumVal; + } } OS << " OPERAND_TYPE_LIST_END" << "\n};\n"; @@ -358,7 +370,8 @@ void InstrInfoEmitter::emitOperandTypeMappings( OS << "namespace llvm {\n"; OS << "namespace " << Namespace << " {\n"; OS << "LLVM_READONLY\n"; - OS << "int getOperandType(uint16_t Opcode, uint16_t OpIdx) {\n"; + OS << "static int getOperandType(uint16_t Opcode, uint16_t OpIdx) {\n"; + // TODO: Factor out instructions with same operands to compress the tables. if (!NumberedInstructions.empty()) { std::vector<int> OperandOffsets; std::vector<Record *> OperandRecords; @@ -399,7 +412,10 @@ void InstrInfoEmitter::emitOperandTypeMappings( OS << "/**/\n "; } Record *OpR = OperandRecords[I]; - if (OpR->isSubClassOf("Operand") && !OpR->isAnonymous()) + if ((OpR->isSubClassOf("Operand") || + OpR->isSubClassOf("RegisterOperand") || + OpR->isSubClassOf("RegisterClass")) && + !OpR->isAnonymous()) OS << "OpTypes::" << OpR->getName(); else OS << -1; @@ -414,7 +430,7 @@ void InstrInfoEmitter::emitOperandTypeMappings( OS << "}\n"; OS << "} // end namespace " << Namespace << "\n"; OS << "} // end namespace llvm\n"; - OS << "#endif //GET_INSTRINFO_OPERAND_TYPE\n\n"; + OS << "#endif // GET_INSTRINFO_OPERAND_TYPE\n\n"; } void InstrInfoEmitter::emitMCIIHelperMethods(raw_ostream &OS, @@ -436,8 +452,8 @@ void InstrInfoEmitter::emitMCIIHelperMethods(raw_ostream &OS, << "(const MCInst &MI);\n"; } - OS << "\n} // end " << TargetName << "_MC namespace\n"; - OS << "} // end llvm namespace\n\n"; + OS << "\n} // end namespace " << TargetName << "_MC\n"; + OS << "} // end namespace llvm\n\n"; OS << "#endif // GET_INSTRINFO_MC_HELPER_DECLS\n\n"; @@ -459,8 +475,8 @@ void InstrInfoEmitter::emitMCIIHelperMethods(raw_ostream &OS, OS << "\n}\n\n"; } - OS << "} // end " << TargetName << "_MC namespace\n"; - OS << "} // end llvm namespace\n\n"; + OS << "} // end namespace " << TargetName << "_MC\n"; + OS << "} // end namespace llvm\n\n"; OS << "#endif // GET_GENISTRINFO_MC_HELPERS\n"; } @@ -576,7 +592,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) { << TargetName << "InstrNameIndices, " << TargetName << "InstrNameData, " << NumberedInstructions.size() << ");\n}\n\n"; - OS << "} // end llvm namespace\n"; + OS << "} // end namespace llvm\n"; OS << "#endif // GET_INSTRINFO_MC_DESC\n\n"; @@ -592,7 +608,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) { << " ~" << ClassName << "() override = default;\n"; - OS << "\n};\n} // end llvm namespace\n"; + OS << "\n};\n} // end namespace llvm\n"; OS << "#endif // GET_INSTRINFO_HEADER\n\n"; @@ -620,7 +636,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) { << " InitMCInstrInfo(" << TargetName << "Insts, " << TargetName << "InstrNameIndices, " << TargetName << "InstrNameData, " << NumberedInstructions.size() << ");\n}\n"; - OS << "} // end llvm namespace\n"; + OS << "} // end namespace llvm\n"; OS << "#endif // GET_INSTRINFO_CTOR_DTOR\n\n"; @@ -651,6 +667,7 @@ void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num, CodeGenTarget &Target = CDP.getTargetInfo(); // Emit all of the target independent flags... + if (Inst.isPreISelOpcode) OS << "|(1ULL<<MCID::PreISelOpcode)"; if (Inst.isPseudo) OS << "|(1ULL<<MCID::Pseudo)"; if (Inst.isReturn) OS << "|(1ULL<<MCID::Return)"; if (Inst.isEHScopeReturn) OS << "|(1ULL<<MCID::EHScopeReturn)"; @@ -691,6 +708,7 @@ void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num, if (Inst.isInsertSubreg) OS << "|(1ULL<<MCID::InsertSubreg)"; if (Inst.isConvergent) OS << "|(1ULL<<MCID::Convergent)"; if (Inst.variadicOpsAreDefs) OS << "|(1ULL<<MCID::VariadicOpsAreDefs)"; + if (Inst.isAuthenticated) OS << "|(1ULL<<MCID::Authenticated)"; // Emit all of the target-specific flags... BitsInit *TSF = Inst.TheDef->getValueAsBitsInit("TSFlags"); @@ -765,8 +783,8 @@ void InstrInfoEmitter::emitEnums(raw_ostream &OS) { OS << " " << Inst->TheDef->getName() << "\t= " << Num++ << ",\n"; OS << " INSTRUCTION_LIST_END = " << Num << "\n"; OS << " };\n\n"; - OS << "} // end " << Namespace << " namespace\n"; - OS << "} // end llvm namespace\n"; + OS << "} // end namespace " << Namespace << "\n"; + OS << "} // end namespace llvm\n"; OS << "#endif // GET_INSTRINFO_ENUM\n\n"; OS << "#ifdef GET_INSTRINFO_SCHED_ENUM\n"; @@ -780,9 +798,9 @@ void InstrInfoEmitter::emitEnums(raw_ostream &OS) { OS << " " << Class.Name << "\t= " << Num++ << ",\n"; OS << " SCHED_LIST_END = " << Num << "\n"; OS << " };\n"; - OS << "} // end Sched namespace\n"; - OS << "} // end " << Namespace << " namespace\n"; - OS << "} // end llvm namespace\n"; + OS << "} // end namespace Sched\n"; + OS << "} // end namespace " << Namespace << "\n"; + OS << "} // end namespace llvm\n"; OS << "#endif // GET_INSTRINFO_SCHED_ENUM\n\n"; } @@ -794,4 +812,4 @@ void EmitInstrInfo(RecordKeeper &RK, raw_ostream &OS) { EmitMapTable(RK, OS); } -} // end llvm namespace +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp index 979af98f6768..9a12571ac6bc 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -15,28 +15,30 @@ #include "SequenceToOffsetTable.h" #include "TableGenBackends.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/Support/CommandLine.h" #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/StringMatcher.h" -#include "llvm/TableGen/TableGenBackend.h" #include "llvm/TableGen/StringToOffsetTable.h" +#include "llvm/TableGen/TableGenBackend.h" #include <algorithm> using namespace llvm; +cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums"); +cl::opt<std::string> + IntrinsicPrefix("intrinsic-prefix", + cl::desc("Generate intrinsics with this target prefix"), + cl::value_desc("target prefix"), cl::cat(GenIntrinsicCat)); + namespace { class IntrinsicEmitter { RecordKeeper &Records; - bool TargetOnly; - std::string TargetPrefix; public: - IntrinsicEmitter(RecordKeeper &R, bool T) - : Records(R), TargetOnly(T) {} + IntrinsicEmitter(RecordKeeper &R) : Records(R) {} void run(raw_ostream &OS, bool Enums); - void EmitPrefix(raw_ostream &OS); - void EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); void EmitTargetInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); void EmitIntrinsicToNameTable(const CodeGenIntrinsicTable &Ints, @@ -47,7 +49,6 @@ public: void EmitAttributes(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); void EmitIntrinsicToBuiltinMap(const CodeGenIntrinsicTable &Ints, bool IsGCC, raw_ostream &OS); - void EmitSuffix(raw_ostream &OS); }; } // End anonymous namespace @@ -58,12 +59,7 @@ public: void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) { emitSourceFileHeader("Intrinsic Function Source Fragment", OS); - CodeGenIntrinsicTable Ints(Records, TargetOnly); - - if (TargetOnly && !Ints.empty()) - TargetPrefix = Ints[0].TargetPrefix; - - EmitPrefix(OS); + CodeGenIntrinsicTable Ints(Records); if (Enums) { // Emit the enum information. @@ -90,40 +86,64 @@ void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) { // Emit code to translate MS builtins into LLVM intrinsics. EmitIntrinsicToBuiltinMap(Ints, false, OS); } - - EmitSuffix(OS); -} - -void IntrinsicEmitter::EmitPrefix(raw_ostream &OS) { - OS << "// VisualStudio defines setjmp as _setjmp\n" - "#if defined(_MSC_VER) && defined(setjmp) && \\\n" - " !defined(setjmp_undefined_for_msvc)\n" - "# pragma push_macro(\"setjmp\")\n" - "# undef setjmp\n" - "# define setjmp_undefined_for_msvc\n" - "#endif\n\n"; -} - -void IntrinsicEmitter::EmitSuffix(raw_ostream &OS) { - OS << "#if defined(_MSC_VER) && defined(setjmp_undefined_for_msvc)\n" - "// let's return it to _setjmp state\n" - "# pragma pop_macro(\"setjmp\")\n" - "# undef setjmp_undefined_for_msvc\n" - "#endif\n\n"; } void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS) { - OS << "// Enum values for Intrinsics.h\n"; - OS << "#ifdef GET_INTRINSIC_ENUM_VALUES\n"; - for (unsigned i = 0, e = Ints.size(); i != e; ++i) { + // Find the TargetSet for which to generate enums. There will be an initial + // set with an empty target prefix which will include target independent + // intrinsics like dbg.value. + const CodeGenIntrinsicTable::TargetSet *Set = nullptr; + for (const auto &Target : Ints.Targets) { + if (Target.Name == IntrinsicPrefix) { + Set = &Target; + break; + } + } + if (!Set) { + std::vector<std::string> KnownTargets; + for (const auto &Target : Ints.Targets) + if (!Target.Name.empty()) + KnownTargets.push_back(Target.Name); + PrintFatalError("tried to generate intrinsics for unknown target " + + IntrinsicPrefix + + "\nKnown targets are: " + join(KnownTargets, ", ") + "\n"); + } + + // Generate a complete header for target specific intrinsics. + if (!IntrinsicPrefix.empty()) { + std::string UpperPrefix = StringRef(IntrinsicPrefix).upper(); + OS << "#ifndef LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n"; + OS << "#define LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n\n"; + OS << "namespace llvm {\n"; + OS << "namespace Intrinsic {\n"; + OS << "enum " << UpperPrefix << "Intrinsics : unsigned {\n"; + } + + OS << "// Enum values for intrinsics\n"; + for (unsigned i = Set->Offset, e = Set->Offset + Set->Count; i != e; ++i) { OS << " " << Ints[i].EnumName; - OS << ((i != e-1) ? ", " : " "); + + // Assign a value to the first intrinsic in this target set so that all + // intrinsic ids are distinct. + if (i == Set->Offset) + OS << " = " << (Set->Offset + 1); + + OS << ", "; if (Ints[i].EnumName.size() < 40) - OS << std::string(40-Ints[i].EnumName.size(), ' '); + OS.indent(40 - Ints[i].EnumName.size()); OS << " // " << Ints[i].Name << "\n"; } - OS << "#endif\n\n"; + + // Emit num_intrinsics into the target neutral enum. + if (IntrinsicPrefix.empty()) { + OS << " num_intrinsics = " << (Ints.size() + 1) << "\n"; + } else { + OS << "}; // enum\n"; + OS << "} // namespace Intrinsic\n"; + OS << "} // namespace llvm\n\n"; + OS << "#endif\n"; + } } void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints, @@ -220,7 +240,11 @@ enum IIT_Info { IIT_STRUCT7 = 39, IIT_STRUCT8 = 40, IIT_F128 = 41, - IIT_VEC_ELEMENT = 42 + IIT_VEC_ELEMENT = 42, + IIT_SCALABLE_VEC = 43, + IIT_SUBDIVIDE2_ARG = 44, + IIT_SUBDIVIDE4_ARG = 45, + IIT_VEC_OF_BITCASTS_TO_INT = 46 }; static void EncodeFixedValueType(MVT::SimpleValueType VT, @@ -292,6 +316,12 @@ static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes, Sig.push_back(IIT_PTR_TO_ELT); else if (R->isSubClassOf("LLVMVectorElementType")) Sig.push_back(IIT_VEC_ELEMENT); + else if (R->isSubClassOf("LLVMSubdivide2VectorType")) + Sig.push_back(IIT_SUBDIVIDE2_ARG); + else if (R->isSubClassOf("LLVMSubdivide4VectorType")) + Sig.push_back(IIT_SUBDIVIDE4_ARG); + else if (R->isSubClassOf("LLVMVectorOfBitcastsToInt")) + Sig.push_back(IIT_VEC_OF_BITCASTS_TO_INT); else Sig.push_back(IIT_ARG); return Sig.push_back((Number << 3) | 7 /*IITDescriptor::AK_MatchType*/); @@ -339,6 +369,8 @@ static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes, if (MVT(VT).isVector()) { MVT VVT = VT; + if (VVT.isScalableVector()) + Sig.push_back(IIT_SCALABLE_VEC); switch (VVT.getVectorNumElements()) { default: PrintFatalError("unhandled vector type width in intrinsic!"); case 1: Sig.push_back(IIT_V1); break; @@ -576,11 +608,7 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, raw_ostream &OS) { OS << "// Add parameter attributes that are not common to all intrinsics.\n"; OS << "#ifdef GET_INTRINSIC_ATTRIBUTES\n"; - if (TargetOnly) - OS << "static AttributeList getAttributes(LLVMContext &C, " << TargetPrefix - << "Intrinsic::ID id) {\n"; - else - OS << "AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {\n"; + OS << "AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {\n"; // Compute the maximum number of attribute arguments and the map typedef std::map<const CodeGenIntrinsic*, unsigned, @@ -613,12 +641,7 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, OS << " AttributeList AS[" << maxArgAttrs + 1 << "];\n"; OS << " unsigned NumAttrs = 0;\n"; OS << " if (id != 0) {\n"; - OS << " switch(IntrinsicsToAttributesMap[id - "; - if (TargetOnly) - OS << "Intrinsic::num_intrinsics"; - else - OS << "1"; - OS << "]) {\n"; + OS << " switch(IntrinsicsToAttributesMap[id - 1]) {\n"; OS << " default: llvm_unreachable(\"Invalid attribute number\");\n"; for (UniqAttrMapTy::const_iterator I = UniqAttributes.begin(), E = UniqAttributes.end(); I != E; ++I) { @@ -647,6 +670,12 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, OS << "Attribute::NoCapture"; addComma = true; break; + case CodeGenIntrinsic::NoAlias: + if (addComma) + OS << ","; + OS << "Attribute::NoAlias"; + addComma = true; + break; case CodeGenIntrinsic::Returned: if (addComma) OS << ","; @@ -857,21 +886,12 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap( OS << "// in as TargetPrefix. The result is assigned to 'IntrinsicID'.\n"; OS << "#ifdef GET_LLVM_INTRINSIC_FOR_" << CompilerName << "_BUILTIN\n"; - if (TargetOnly) { - OS << "static " << TargetPrefix << "Intrinsic::ID " - << "getIntrinsicFor" << CompilerName << "Builtin(const char " - << "*TargetPrefixStr, StringRef BuiltinNameStr) {\n"; - } else { - OS << "Intrinsic::ID Intrinsic::getIntrinsicFor" << CompilerName - << "Builtin(const char " - << "*TargetPrefixStr, StringRef BuiltinNameStr) {\n"; - } + OS << "Intrinsic::ID Intrinsic::getIntrinsicFor" << CompilerName + << "Builtin(const char " + << "*TargetPrefixStr, StringRef BuiltinNameStr) {\n"; if (Table.Empty()) { - OS << " return "; - if (!TargetPrefix.empty()) - OS << "(" << TargetPrefix << "Intrinsic::ID)"; - OS << "Intrinsic::not_intrinsic;\n"; + OS << " return Intrinsic::not_intrinsic;\n"; OS << "}\n"; OS << "#endif\n\n"; return; @@ -919,19 +939,15 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap( OS << " }\n"; } OS << " return "; - if (!TargetPrefix.empty()) - OS << "(" << TargetPrefix << "Intrinsic::ID)"; OS << "Intrinsic::not_intrinsic;\n"; OS << "}\n"; OS << "#endif\n\n"; } -void llvm::EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS, - bool TargetOnly) { - IntrinsicEmitter(RK, TargetOnly).run(OS, /*Enums=*/true); +void llvm::EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS) { + IntrinsicEmitter(RK).run(OS, /*Enums=*/true); } -void llvm::EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS, - bool TargetOnly) { - IntrinsicEmitter(RK, TargetOnly).run(OS, /*Enums=*/false); +void llvm::EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS) { + IntrinsicEmitter(RK).run(OS, /*Enums=*/false); } diff --git a/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.cpp new file mode 100644 index 000000000000..7fcf3074e093 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.cpp @@ -0,0 +1,84 @@ +//===- OptEmitter.cpp - Helper for emitting options.----------- -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OptEmitter.h" +#include "llvm/ADT/Twine.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include <cctype> +#include <cstring> + +namespace llvm { + +// Ordering on Info. The logic should match with the consumer-side function in +// llvm/Option/OptTable.h. +// FIXME: Make this take StringRefs instead of null terminated strings to +// simplify callers. +static int StrCmpOptionName(const char *A, const char *B) { + const char *X = A, *Y = B; + char a = tolower(*A), b = tolower(*B); + while (a == b) { + if (a == '\0') + return strcmp(A, B); + + a = tolower(*++X); + b = tolower(*++Y); + } + + if (a == '\0') // A is a prefix of B. + return 1; + if (b == '\0') // B is a prefix of A. + return -1; + + // Otherwise lexicographic. + return (a < b) ? -1 : 1; +} + +int CompareOptionRecords(Record *const *Av, Record *const *Bv) { + const Record *A = *Av; + const Record *B = *Bv; + + // Sentinel options precede all others and are only ordered by precedence. + bool ASent = A->getValueAsDef("Kind")->getValueAsBit("Sentinel"); + bool BSent = B->getValueAsDef("Kind")->getValueAsBit("Sentinel"); + if (ASent != BSent) + return ASent ? -1 : 1; + + // Compare options by name, unless they are sentinels. + if (!ASent) + if (int Cmp = StrCmpOptionName(A->getValueAsString("Name").str().c_str(), + B->getValueAsString("Name").str().c_str())) + return Cmp; + + if (!ASent) { + std::vector<StringRef> APrefixes = A->getValueAsListOfStrings("Prefixes"); + std::vector<StringRef> BPrefixes = B->getValueAsListOfStrings("Prefixes"); + + for (std::vector<StringRef>::const_iterator APre = APrefixes.begin(), + AEPre = APrefixes.end(), + BPre = BPrefixes.begin(), + BEPre = BPrefixes.end(); + APre != AEPre && BPre != BEPre; ++APre, ++BPre) { + if (int Cmp = StrCmpOptionName(APre->str().c_str(), BPre->str().c_str())) + return Cmp; + } + } + + // Then by the kind precedence; + int APrec = A->getValueAsDef("Kind")->getValueAsInt("Precedence"); + int BPrec = B->getValueAsDef("Kind")->getValueAsInt("Precedence"); + if (APrec == BPrec && A->getValueAsListOfStrings("Prefixes") == + B->getValueAsListOfStrings("Prefixes")) { + PrintError(A->getLoc(), Twine("Option is equivalent to")); + PrintError(B->getLoc(), Twine("Other defined here")); + PrintFatalError("Equivalent Options found."); + } + return APrec < BPrec ? -1 : 1; +} + +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.h b/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.h new file mode 100644 index 000000000000..c8f9246ef1e6 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.h @@ -0,0 +1,16 @@ +//===- OptEmitter.h - Helper for emitting options. --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_OPTEMITTER_H +#define LLVM_UTILS_TABLEGEN_OPTEMITTER_H + +namespace llvm { +class Record; +int CompareOptionRecords(Record *const *Av, Record *const *Bv); +} // namespace llvm +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/OptParserEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/OptParserEmitter.cpp index 51b1cb093b21..c1978ac7ac66 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/OptParserEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/OptParserEmitter.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "llvm/TableGen/Error.h" +#include "OptEmitter.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Twine.h" @@ -18,75 +18,6 @@ using namespace llvm; -// Ordering on Info. The logic should match with the consumer-side function in -// llvm/Option/OptTable.h. -// FIXME: Mmake this take StringRefs instead of null terminated strings to -// simplify callers. -static int StrCmpOptionName(const char *A, const char *B) { - const char *X = A, *Y = B; - char a = tolower(*A), b = tolower(*B); - while (a == b) { - if (a == '\0') - return strcmp(A, B); - - a = tolower(*++X); - b = tolower(*++Y); - } - - if (a == '\0') // A is a prefix of B. - return 1; - if (b == '\0') // B is a prefix of A. - return -1; - - // Otherwise lexicographic. - return (a < b) ? -1 : 1; -} - -static int CompareOptionRecords(Record *const *Av, Record *const *Bv) { - const Record *A = *Av; - const Record *B = *Bv; - - // Sentinel options precede all others and are only ordered by precedence. - bool ASent = A->getValueAsDef("Kind")->getValueAsBit("Sentinel"); - bool BSent = B->getValueAsDef("Kind")->getValueAsBit("Sentinel"); - if (ASent != BSent) - return ASent ? -1 : 1; - - // Compare options by name, unless they are sentinels. - if (!ASent) - if (int Cmp = StrCmpOptionName(A->getValueAsString("Name").str().c_str(), - B->getValueAsString("Name").str().c_str())) - return Cmp; - - if (!ASent) { - std::vector<StringRef> APrefixes = A->getValueAsListOfStrings("Prefixes"); - std::vector<StringRef> BPrefixes = B->getValueAsListOfStrings("Prefixes"); - - for (std::vector<StringRef>::const_iterator APre = APrefixes.begin(), - AEPre = APrefixes.end(), - BPre = BPrefixes.begin(), - BEPre = BPrefixes.end(); - APre != AEPre && - BPre != BEPre; - ++APre, ++BPre) { - if (int Cmp = StrCmpOptionName(APre->str().c_str(), BPre->str().c_str())) - return Cmp; - } - } - - // Then by the kind precedence; - int APrec = A->getValueAsDef("Kind")->getValueAsInt("Precedence"); - int BPrec = B->getValueAsDef("Kind")->getValueAsInt("Precedence"); - if (APrec == BPrec && - A->getValueAsListOfStrings("Prefixes") == - B->getValueAsListOfStrings("Prefixes")) { - PrintError(A->getLoc(), Twine("Option is equivalent to")); - PrintError(B->getLoc(), Twine("Other defined here")); - PrintFatalError("Equivalent Options found."); - } - return APrec < BPrec ? -1 : 1; -} - static const std::string getOptionName(const Record &R) { // Use the record name unless EnumName is defined. if (isa<UnsetInit>(R.getValueInit("EnumName"))) @@ -310,9 +241,9 @@ void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) { OS << "bool ValuesWereAdded;\n"; OS << R.getValueAsString("ValuesCode"); OS << "\n"; - for (const std::string &Pref : R.getValueAsListOfStrings("Prefixes")) { + for (std::string S : R.getValueAsListOfStrings("Prefixes")) { OS << "ValuesWereAdded = Opt.addValues("; - std::string S = (Pref + R.getValueAsString("Name")).str(); + S += R.getValueAsString("Name"); write_cstring(OS, S); OS << ", Values);\n"; OS << "(void)ValuesWereAdded;\n"; diff --git a/contrib/llvm-project/llvm/utils/TableGen/OptRSTEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/OptRSTEmitter.cpp new file mode 100644 index 000000000000..3102f378bc1e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/OptRSTEmitter.cpp @@ -0,0 +1,86 @@ +//===- OptParserEmitter.cpp - Table Driven Command Line Parsing -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OptEmitter.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/Twine.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cctype> +#include <cstring> +#include <map> + +using namespace llvm; + +/// OptParserEmitter - This tablegen backend takes an input .td file +/// describing a list of options and emits a RST man page. +namespace llvm { +void EmitOptRST(RecordKeeper &Records, raw_ostream &OS) { + llvm::StringMap<std::vector<Record *>> OptionsByGroup; + std::vector<Record *> OptionsWithoutGroup; + + // Get the options. + std::vector<Record *> Opts = Records.getAllDerivedDefinitions("Option"); + array_pod_sort(Opts.begin(), Opts.end(), CompareOptionRecords); + + // Get the option groups. + const std::vector<Record *> &Groups = + Records.getAllDerivedDefinitions("OptionGroup"); + for (unsigned i = 0, e = Groups.size(); i != e; ++i) { + const Record &R = *Groups[i]; + OptionsByGroup.try_emplace(R.getValueAsString("Name")); + } + + // Map options to their group. + for (unsigned i = 0, e = Opts.size(); i != e; ++i) { + const Record &R = *Opts[i]; + if (const DefInit *DI = dyn_cast<DefInit>(R.getValueInit("Group"))) { + OptionsByGroup[DI->getDef()->getValueAsString("Name")].push_back(Opts[i]); + } else { + OptionsByGroup["options"].push_back(Opts[i]); + } + } + + // Print options under their group. + for (const auto &KV : OptionsByGroup) { + std::string GroupName = KV.getKey().upper(); + OS << GroupName << '\n'; + OS << std::string(GroupName.size(), '-') << '\n'; + OS << '\n'; + + for (Record *R : KV.getValue()) { + OS << ".. option:: "; + + // Print the prefix. + std::vector<StringRef> Prefixes = R->getValueAsListOfStrings("Prefixes"); + if (!Prefixes.empty()) + OS << Prefixes[0]; + + // Print the option name. + OS << R->getValueAsString("Name"); + + // Print the meta-variable. + if (!isa<UnsetInit>(R->getValueInit("MetaVarName"))) { + OS << '='; + OS.write_escaped(R->getValueAsString("MetaVarName")); + } + + OS << "\n\n"; + + // The option help text. + if (!isa<UnsetInit>(R->getValueInit("HelpText"))) { + OS << ' '; + OS.write_escaped(R->getValueAsString("HelpText")); + OS << "\n\n"; + } + } + } +} +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp index e62f528ebc2e..96e4f95937b2 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp @@ -45,6 +45,14 @@ // const MCRegisterInfo &MRI, // const MCSubtargetInfo &STI); // +// In addition, it exports a function for checking whether +// an instruction is compressable: +// +// bool isCompressibleInst(const MachineInstr& MI, +// const RISCVSubtarget *Subtarget, +// const MCRegisterInfo &MRI, +// const MCSubtargetInfo &STI); +// // The clients that include this auto-generated header file and // invoke these functions can compress an instruction before emitting // it in the target-specific ASM or ELF streamer or can uncompress @@ -99,7 +107,7 @@ class RISCVCompressInstEmitter { : Source(S), Dest(D), PatReqFeatures(RF), SourceOperandMap(SourceMap), DestOperandMap(DestMap) {} }; - + enum EmitterType { Compress, Uncompress, CheckCompress }; RecordKeeper &Records; CodeGenTarget Target; SmallVector<CompressPat, 4> CompressPatterns; @@ -107,7 +115,7 @@ class RISCVCompressInstEmitter { void addDagOperandMapping(Record *Rec, DagInit *Dag, CodeGenInstruction &Inst, IndexedMap<OpData> &OperandMap, bool IsSourceInst); void evaluateCompressPat(Record *Compress); - void emitCompressInstEmitter(raw_ostream &o, bool Compress); + void emitCompressInstEmitter(raw_ostream &o, EmitterType EType); bool validateTypes(Record *SubType, Record *Type, bool IsSourceInst); bool validateRegister(Record *Reg, Record *RegClass); void createDagOperandMapping(Record *Rec, StringMap<unsigned> &SourceOperands, @@ -411,12 +419,8 @@ void RISCVCompressInstEmitter::evaluateCompressPat(Record *Rec) { assert(SourceDag && "Missing 'Input' in compress pattern!"); LLVM_DEBUG(dbgs() << "Input: " << *SourceDag << "\n"); - DefInit *OpDef = dyn_cast<DefInit>(SourceDag->getOperator()); - if (!OpDef) - PrintFatalError(Rec->getLoc(), - Rec->getName() + " has unexpected operator type!"); // Checking we are transforming from compressed to uncompressed instructions. - Record *Operator = OpDef->getDef(); + Record *Operator = SourceDag->getOperatorAsDef(Rec->getLoc()); if (!Operator->isSubClassOf("RVInst")) PrintFatalError(Rec->getLoc(), "Input instruction '" + Operator->getName() + "' is not a 32 bit wide instruction!"); @@ -428,12 +432,7 @@ void RISCVCompressInstEmitter::evaluateCompressPat(Record *Rec) { assert(DestDag && "Missing 'Output' in compress pattern!"); LLVM_DEBUG(dbgs() << "Output: " << *DestDag << "\n"); - DefInit *DestOpDef = dyn_cast<DefInit>(DestDag->getOperator()); - if (!DestOpDef) - PrintFatalError(Rec->getLoc(), - Rec->getName() + " has unexpected operator type!"); - - Record *DestOperator = DestOpDef->getDef(); + Record *DestOperator = DestDag->getOperatorAsDef(Rec->getLoc()); if (!DestOperator->isSubClassOf("RVInst16")) PrintFatalError(Rec->getLoc(), "Output instruction '" + DestOperator->getName() + @@ -491,26 +490,39 @@ static void getReqFeatures(std::set<StringRef> &FeaturesSet, } } -unsigned getMCOpPredicate(DenseMap<const Record *, unsigned> &MCOpPredicateMap, - std::vector<const Record *> &MCOpPredicates, - Record *Rec) { - unsigned Entry = MCOpPredicateMap[Rec]; +static unsigned getPredicates(DenseMap<const Record *, unsigned> &PredicateMap, + std::vector<const Record *> &Predicates, + Record *Rec, StringRef Name) { + unsigned Entry = PredicateMap[Rec]; if (Entry) return Entry; - if (!Rec->isValueUnset("MCOperandPredicate")) { - MCOpPredicates.push_back(Rec); - Entry = MCOpPredicates.size(); - MCOpPredicateMap[Rec] = Entry; + if (!Rec->isValueUnset(Name)) { + Predicates.push_back(Rec); + Entry = Predicates.size(); + PredicateMap[Rec] = Entry; return Entry; } - PrintFatalError(Rec->getLoc(), - "No MCOperandPredicate on this operand at all: " + - Rec->getName().str() + "'"); + PrintFatalError(Rec->getLoc(), "No " + Name + + " predicate on this operand at all: '" + Rec->getName().str() + "'"); return 0; } +static void printPredicates(std::vector<const Record *> &Predicates, + StringRef Name, raw_ostream &o) { + for (unsigned i = 0; i < Predicates.size(); ++i) { + Init *Pred = Predicates[i]->getValueInit(Name); + if (CodeInit *SI = dyn_cast<CodeInit>(Pred)) + o << " case " << i + 1 << ": {\n" + << " // " << Predicates[i]->getName().str() << "\n" + << " " << SI->getValue() << "\n" + << " }\n"; + else + llvm_unreachable("Unexpected predicate field!"); + } +} + static std::string mergeCondAndCode(raw_string_ostream &CondStream, raw_string_ostream &CodeStream) { std::string S; @@ -528,7 +540,7 @@ static std::string mergeCondAndCode(raw_string_ostream &CondStream, } void RISCVCompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, - bool Compress) { + EmitterType EType) { Record *AsmWriter = Target.getAsmWriter(); if (!AsmWriter->getValueAsInt("PassSubtarget")) PrintFatalError(AsmWriter->getLoc(), @@ -543,8 +555,9 @@ void RISCVCompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, // source and destination are flipped and the sort key needs to change // accordingly. llvm::stable_sort(CompressPatterns, - [Compress](const CompressPat &LHS, const CompressPat &RHS) { - if (Compress) + [EType](const CompressPat &LHS, const CompressPat &RHS) { + if (EType == EmitterType::Compress || + EType == EmitterType::CheckCompress) return (LHS.Source.TheDef->getName().str() < RHS.Source.TheDef->getName().str()); else @@ -555,6 +568,9 @@ void RISCVCompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, // A list of MCOperandPredicates for all operands in use, and the reverse map. std::vector<const Record *> MCOpPredicates; DenseMap<const Record *, unsigned> MCOpPredicateMap; + // A list of ImmLeaf Predicates for all operands in use, and the reverse map. + std::vector<const Record *> ImmLeafPredicates; + DenseMap<const Record *, unsigned> ImmLeafPredicateMap; std::string F; std::string FH; @@ -562,32 +578,42 @@ void RISCVCompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, raw_string_ostream FuncH(FH); bool NeedMRI = false; - if (Compress) + if (EType == EmitterType::Compress) o << "\n#ifdef GEN_COMPRESS_INSTR\n" << "#undef GEN_COMPRESS_INSTR\n\n"; - else + else if (EType == EmitterType::Uncompress) o << "\n#ifdef GEN_UNCOMPRESS_INSTR\n" << "#undef GEN_UNCOMPRESS_INSTR\n\n"; + else if (EType == EmitterType::CheckCompress) + o << "\n#ifdef GEN_CHECK_COMPRESS_INSTR\n" + << "#undef GEN_CHECK_COMPRESS_INSTR\n\n"; - if (Compress) { + if (EType == EmitterType::Compress) { FuncH << "static bool compressInst(MCInst& OutInst,\n"; FuncH.indent(25) << "const MCInst &MI,\n"; FuncH.indent(25) << "const MCSubtargetInfo &STI,\n"; FuncH.indent(25) << "MCContext &Context) {\n"; - } else { + } else if (EType == EmitterType::Uncompress){ FuncH << "static bool uncompressInst(MCInst& OutInst,\n"; FuncH.indent(27) << "const MCInst &MI,\n"; FuncH.indent(27) << "const MCRegisterInfo &MRI,\n"; FuncH.indent(27) << "const MCSubtargetInfo &STI) {\n"; + } else if (EType == EmitterType::CheckCompress) { + FuncH << "static bool isCompressibleInst(const MachineInstr& MI,\n"; + FuncH.indent(27) << "const RISCVSubtarget *Subtarget,\n"; + FuncH.indent(27) << "const MCRegisterInfo &MRI,\n"; + FuncH.indent(27) << "const MCSubtargetInfo &STI) {\n"; } if (CompressPatterns.empty()) { o << FuncH.str(); o.indent(2) << "return false;\n}\n"; - if (Compress) + if (EType == EmitterType::Compress) o << "\n#endif //GEN_COMPRESS_INSTR\n"; - else + else if (EType == EmitterType::Uncompress) o << "\n#endif //GEN_UNCOMPRESS_INSTR\n\n"; + else if (EType == EmitterType::CheckCompress) + o << "\n#endif //GEN_CHECK_COMPRESS_INSTR\n\n"; return; } @@ -598,18 +624,24 @@ void RISCVCompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, CaseStream << " switch (MI.getOpcode()) {\n"; CaseStream << " default: return false;\n"; + bool CompressOrCheck = + EType == EmitterType::Compress || EType == EmitterType::CheckCompress; + bool CompressOrUncompress = + EType == EmitterType::Compress || EType == EmitterType::Uncompress; + for (auto &CompressPat : CompressPatterns) { std::string CondString; std::string CodeString; raw_string_ostream CondStream(CondString); raw_string_ostream CodeStream(CodeString); CodeGenInstruction &Source = - Compress ? CompressPat.Source : CompressPat.Dest; - CodeGenInstruction &Dest = Compress ? CompressPat.Dest : CompressPat.Source; - IndexedMap<OpData> SourceOperandMap = - Compress ? CompressPat.SourceOperandMap : CompressPat.DestOperandMap; - IndexedMap<OpData> &DestOperandMap = - Compress ? CompressPat.DestOperandMap : CompressPat.SourceOperandMap; + CompressOrCheck ? CompressPat.Source : CompressPat.Dest; + CodeGenInstruction &Dest = + CompressOrCheck ? CompressPat.Dest : CompressPat.Source; + IndexedMap<OpData> SourceOperandMap = CompressOrCheck ? + CompressPat.SourceOperandMap : CompressPat.DestOperandMap; + IndexedMap<OpData> &DestOperandMap = CompressOrCheck ? + CompressPat.DestOperandMap : CompressPat.SourceOperandMap; CurOp = Source.TheDef->getName().str(); // Check current and previous opcode to decide to continue or end a case. @@ -679,7 +711,8 @@ void RISCVCompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, } } CodeStream.indent(6) << "// " + Dest.AsmString + "\n"; - CodeStream.indent(6) << "OutInst.setOpcode(" + Namespace + + if (CompressOrUncompress) + CodeStream.indent(6) << "OutInst.setOpcode(" + Namespace + "::" + Dest.TheDef->getName().str() + ");\n"; OpNo = 0; for (const auto &DestOperand : Dest.Operands) { @@ -701,42 +734,69 @@ void RISCVCompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, "RegClassID).contains(" + "MI.getOperand(" + std::to_string(OpIdx) + ").getReg())) &&\n"; - CodeStream.indent(6) << "OutInst.addOperand(MI.getOperand(" + - std::to_string(OpIdx) + "));\n"; + if (CompressOrUncompress) + CodeStream.indent(6) << "OutInst.addOperand(MI.getOperand(" + + std::to_string(OpIdx) + "));\n"; } else { // Handling immediate operands. - unsigned Entry = getMCOpPredicate(MCOpPredicateMap, MCOpPredicates, - DestOperand.Rec); - CondStream.indent(6) << Namespace + "ValidateMCOperand(" + - "MI.getOperand(" + std::to_string(OpIdx) + - "), STI, " + std::to_string(Entry) + - ") &&\n"; - CodeStream.indent(6) << "OutInst.addOperand(MI.getOperand(" + - std::to_string(OpIdx) + "));\n"; + if (CompressOrUncompress) { + unsigned Entry = getPredicates(MCOpPredicateMap, MCOpPredicates, + DestOperand.Rec, StringRef("MCOperandPredicate")); + CondStream.indent(6) << Namespace + "ValidateMCOperand(" + + "MI.getOperand(" + std::to_string(OpIdx) + + "), STI, " + std::to_string(Entry) + + ") &&\n"; + } else { + unsigned Entry = getPredicates(ImmLeafPredicateMap, ImmLeafPredicates, + DestOperand.Rec, StringRef("ImmediateCode")); + CondStream.indent(6) << "MI.getOperand(" + std::to_string(OpIdx) + + ").isImm() && \n"; + CondStream.indent(6) << Namespace + "ValidateMachineOperand(" + + "MI.getOperand(" + std::to_string(OpIdx) + + "), Subtarget, " + std::to_string(Entry) + + ") &&\n"; + } + if (CompressOrUncompress) + CodeStream.indent(6) << "OutInst.addOperand(MI.getOperand(" + + std::to_string(OpIdx) + "));\n"; } break; } case OpData::Imm: { - unsigned Entry = - getMCOpPredicate(MCOpPredicateMap, MCOpPredicates, DestOperand.Rec); - CondStream.indent(6) - << Namespace + "ValidateMCOperand(" + "MCOperand::createImm(" + - std::to_string(DestOperandMap[OpNo].Data.Imm) + "), STI, " + - std::to_string(Entry) + ") &&\n"; - CodeStream.indent(6) - << "OutInst.addOperand(MCOperand::createImm(" + - std::to_string(DestOperandMap[OpNo].Data.Imm) + "));\n"; + if (CompressOrUncompress) { + unsigned Entry = getPredicates(MCOpPredicateMap, MCOpPredicates, + DestOperand.Rec, StringRef("MCOperandPredicate")); + CondStream.indent(6) + << Namespace + "ValidateMCOperand(" + "MCOperand::createImm(" + + std::to_string(DestOperandMap[OpNo].Data.Imm) + "), STI, " + + std::to_string(Entry) + ") &&\n"; + } else { + unsigned Entry = getPredicates(ImmLeafPredicateMap, ImmLeafPredicates, + DestOperand.Rec, StringRef("ImmediateCode")); + CondStream.indent(6) + << Namespace + "ValidateMachineOperand(" + "MachineOperand::CreateImm(" + + std::to_string(DestOperandMap[OpNo].Data.Imm) + "), SubTarget, " + + std::to_string(Entry) + ") &&\n"; + } + if (CompressOrUncompress) + CodeStream.indent(6) + << "OutInst.addOperand(MCOperand::createImm(" + + std::to_string(DestOperandMap[OpNo].Data.Imm) + "));\n"; } break; case OpData::Reg: { - // Fixed register has been validated at pattern validation time. - Record *Reg = DestOperandMap[OpNo].Data.Reg; - CodeStream.indent(6) << "OutInst.addOperand(MCOperand::createReg(" + - Namespace + "::" + Reg->getName().str() + - "));\n"; + if (CompressOrUncompress) { + // Fixed register has been validated at pattern validation time. + Record *Reg = DestOperandMap[OpNo].Data.Reg; + CodeStream.indent(6) << "OutInst.addOperand(MCOperand::createReg(" + + Namespace + "::" + Reg->getName().str() + + "));\n"; + } } break; } ++OpNo; } + if (CompressOrUncompress) + CodeStream.indent(6) << "OutInst.setLoc(MI.getLoc());\n"; CaseStream << mergeCondAndCode(CondStream, CodeStream); PrevOp = CurOp; } @@ -756,29 +816,40 @@ void RISCVCompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, << " llvm_unreachable(\"Unknown MCOperandPredicate kind\");\n" << " break;\n"; - for (unsigned i = 0; i < MCOpPredicates.size(); ++i) { - Init *MCOpPred = MCOpPredicates[i]->getValueInit("MCOperandPredicate"); - if (CodeInit *SI = dyn_cast<CodeInit>(MCOpPred)) - o << " case " << i + 1 << ": {\n" - << " // " << MCOpPredicates[i]->getName().str() << SI->getValue() - << "\n" - << " }\n"; - else - llvm_unreachable("Unexpected MCOperandPredicate field!"); - } + printPredicates(MCOpPredicates, "MCOperandPredicate", o); + + o << " }\n" + << "}\n\n"; + } + + if (!ImmLeafPredicates.empty()) { + o << "static bool " << Namespace + << "ValidateMachineOperand(const MachineOperand &MO,\n" + << " const RISCVSubtarget *Subtarget,\n" + << " unsigned PredicateIndex) {\n" + << " int64_t Imm = MO.getImm(); \n" + << " switch (PredicateIndex) {\n" + << " default:\n" + << " llvm_unreachable(\"Unknown ImmLeaf Predicate kind\");\n" + << " break;\n"; + + printPredicates(ImmLeafPredicates, "ImmediateCode", o); + o << " }\n" << "}\n\n"; } o << FuncH.str(); - if (NeedMRI && Compress) + if (NeedMRI && EType == EmitterType::Compress) o.indent(2) << "const MCRegisterInfo &MRI = *Context.getRegisterInfo();\n"; o << Func.str(); - if (Compress) + if (EType == EmitterType::Compress) o << "\n#endif //GEN_COMPRESS_INSTR\n"; - else + else if (EType == EmitterType::Uncompress) o << "\n#endif //GEN_UNCOMPRESS_INSTR\n\n"; + else if (EType == EmitterType::CheckCompress) + o << "\n#endif //GEN_CHECK_COMPRESS_INSTR\n\n"; } void RISCVCompressInstEmitter::run(raw_ostream &o) { @@ -797,9 +868,11 @@ void RISCVCompressInstEmitter::run(raw_ostream &o) { // Emit file header. emitSourceFileHeader("Compress instruction Source Fragment", o); // Generate compressInst() function. - emitCompressInstEmitter(o, true); + emitCompressInstEmitter(o, EmitterType::Compress); // Generate uncompressInst() function. - emitCompressInstEmitter(o, false); + emitCompressInstEmitter(o, EmitterType::Uncompress); + // Generate isCompressibleInst() function. + emitCompressInstEmitter(o, EmitterType::CheckCompress); } namespace llvm { diff --git a/contrib/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp index 1b619072c814..2586ec671b2a 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -742,7 +742,7 @@ RegisterInfoEmitter::emitComposeSubRegIndices(raw_ostream &OS, OS << " { "; for (unsigned i = 0, e = SubRegIndicesSize; i != e; ++i) if (Rows[r][i]) - OS << Rows[r][i]->EnumValue << ", "; + OS << Rows[r][i]->getQualifiedName() << ", "; else OS << "0, "; OS << "},\n"; @@ -888,7 +888,7 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, // Keep track of sub-register names as well. These are not differentially // encoded. typedef SmallVector<const CodeGenSubRegIndex*, 4> SubRegIdxVec; - SequenceToOffsetTable<SubRegIdxVec, deref<llvm::less>> SubRegIdxSeqs; + SequenceToOffsetTable<SubRegIdxVec, deref<std::less<>>> SubRegIdxSeqs; SmallVector<SubRegIdxVec, 4> SubRegIdxLists(Regs.size()); SequenceToOffsetTable<std::string> RegStrings; @@ -1315,7 +1315,7 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, // Compress the sub-reg index lists. typedef std::vector<const CodeGenSubRegIndex*> IdxList; SmallVector<IdxList, 8> SuperRegIdxLists(RegisterClasses.size()); - SequenceToOffsetTable<IdxList, deref<llvm::less>> SuperRegIdxSeqs; + SequenceToOffsetTable<IdxList, deref<std::less<>>> SuperRegIdxSeqs; BitVector MaskBV(RegisterClasses.size()); for (const auto &RC : RegisterClasses) { diff --git a/contrib/llvm-project/llvm/utils/TableGen/SearchableTableEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/SearchableTableEmitter.cpp index 954b63e7253c..cfe48eb1949d 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/SearchableTableEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/SearchableTableEmitter.cpp @@ -44,7 +44,7 @@ struct GenericEnum { using Entry = std::pair<StringRef, int64_t>; std::string Name; - Record *Class; + Record *Class = nullptr; std::string PreprocessorGuard; std::vector<std::unique_ptr<Entry>> Entries; DenseMap<Record *, Entry *> EntryMap; @@ -63,7 +63,7 @@ struct GenericField { struct SearchIndex { std::string Name; SmallVector<GenericField, 1> Fields; - bool EarlyOut; + bool EarlyOut = false; }; struct GenericTable { @@ -134,7 +134,7 @@ private: CodeGenIntrinsic &getIntrinsic(Init *I) { std::unique_ptr<CodeGenIntrinsic> &Intr = Intrinsics[I]; if (!Intr) - Intr = make_unique<CodeGenIntrinsic>(cast<DefInit>(I)->getDef()); + Intr = std::make_unique<CodeGenIntrinsic>(cast<DefInit>(I)->getDef()); return *Intr; } @@ -496,7 +496,7 @@ void SearchableTableEmitter::emitGenericTable(const GenericTable &Table, emitIfdef((Twine("GET_") + Table.PreprocessorGuard + "_IMPL").str(), OS); // The primary data table contains all the fields defined for this map. - OS << "const " << Table.CppTypeName << " " << Table.Name << "[] = {\n"; + OS << "constexpr " << Table.CppTypeName << " " << Table.Name << "[] = {\n"; for (unsigned i = 0; i < Table.Entries.size(); ++i) { Record *Entry = Table.Entries[i]; OS << " { "; @@ -541,7 +541,7 @@ std::unique_ptr<SearchIndex> SearchableTableEmitter::parseSearchIndex(GenericTable &Table, StringRef Name, const std::vector<StringRef> &Key, bool EarlyOut) { - auto Index = llvm::make_unique<SearchIndex>(); + auto Index = std::make_unique<SearchIndex>(); Index->Name = Name; Index->EarlyOut = EarlyOut; @@ -577,7 +577,7 @@ void SearchableTableEmitter::collectEnumEntries( if (!ValueField.empty()) Value = getInt(EntryRec, ValueField); - Enum.Entries.push_back(llvm::make_unique<GenericEnum::Entry>(Name, Value)); + Enum.Entries.push_back(std::make_unique<GenericEnum::Entry>(Name, Value)); Enum.EntryMap.insert(std::make_pair(EntryRec, Enum.Entries.back().get())); } @@ -647,7 +647,7 @@ void SearchableTableEmitter::run(raw_ostream &OS) { if (!EnumRec->isValueUnset("ValueField")) ValueField = EnumRec->getValueAsString("ValueField"); - auto Enum = llvm::make_unique<GenericEnum>(); + auto Enum = std::make_unique<GenericEnum>(); Enum->Name = EnumRec->getName(); Enum->PreprocessorGuard = EnumRec->getName(); @@ -664,7 +664,7 @@ void SearchableTableEmitter::run(raw_ostream &OS) { } for (auto TableRec : Records.getAllDerivedDefinitions("GenericTable")) { - auto Table = llvm::make_unique<GenericTable>(); + auto Table = std::make_unique<GenericTable>(); Table->Name = TableRec->getName(); Table->PreprocessorGuard = TableRec->getName(); Table->CppTypeName = TableRec->getValueAsString("CppTypeName"); @@ -733,7 +733,7 @@ void SearchableTableEmitter::run(raw_ostream &OS) { if (!Class->isValueUnset("EnumValueField")) ValueField = Class->getValueAsString("EnumValueField"); - auto Enum = llvm::make_unique<GenericEnum>(); + auto Enum = std::make_unique<GenericEnum>(); Enum->Name = (Twine(Class->getName()) + "Values").str(); Enum->PreprocessorGuard = Class->getName().upper(); Enum->Class = Class; @@ -743,7 +743,7 @@ void SearchableTableEmitter::run(raw_ostream &OS) { Enums.emplace_back(std::move(Enum)); } - auto Table = llvm::make_unique<GenericTable>(); + auto Table = std::make_unique<GenericTable>(); Table->Name = (Twine(Class->getName()) + "sList").str(); Table->PreprocessorGuard = Class->getName().upper(); Table->CppTypeName = Class->getName(); diff --git a/contrib/llvm-project/llvm/utils/TableGen/SequenceToOffsetTable.h b/contrib/llvm-project/llvm/utils/TableGen/SequenceToOffsetTable.h index 8a826eff311d..327da39f4774 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/SequenceToOffsetTable.h +++ b/contrib/llvm-project/llvm/utils/TableGen/SequenceToOffsetTable.h @@ -83,7 +83,7 @@ public: bool empty() const { return Seqs.empty(); } unsigned size() const { - assert(Entries && "Call layout() before size()"); + assert((empty() || Entries) && "Call layout() before size()"); return Entries; } @@ -113,7 +113,7 @@ public: void emit(raw_ostream &OS, void (*Print)(raw_ostream&, ElemT), const char *Term = "0") const { - assert(Entries && "Call layout() before emit()"); + assert((empty() || Entries) && "Call layout() before emit()"); for (typename SeqMap::const_iterator I = Seqs.begin(), E = Seqs.end(); I != E; ++I) { OS << " /* " << I->second << " */ "; diff --git a/contrib/llvm-project/llvm/utils/TableGen/SubtargetEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/SubtargetEmitter.cpp index 9ce2b3b275c8..9b094adb7d5c 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/SubtargetEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/SubtargetEmitter.cpp @@ -1057,6 +1057,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel, LLVM_DEBUG(dbgs() << ProcModel.ModelName << " does not have resources for class " << SC.Name << '\n'); + SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps; } } // Sum resources across all operand writes. @@ -1728,7 +1729,7 @@ void SubtargetEmitter::emitGenMCSubtargetInfo(raw_ostream &OS) { << " const MCInst *MI, unsigned CPUID) {\n"; emitSchedModelHelpersImpl(OS, /* OnlyExpandMCPredicates */ true); OS << "}\n"; - OS << "} // end of namespace " << Target << "_MC\n\n"; + OS << "} // end namespace " << Target << "_MC\n\n"; OS << "struct " << Target << "GenMCSubtargetInfo : public MCSubtargetInfo {\n"; @@ -1746,7 +1747,10 @@ void SubtargetEmitter::emitGenMCSubtargetInfo(raw_ostream &OS) { << " return " << Target << "_MC" << "::resolveVariantSchedClassImpl(SchedClass, MI, CPUID); \n"; OS << " }\n"; + if (TGT.getHwModes().getNumModeIds() > 1) + OS << " unsigned getHwMode() const override;\n"; OS << "};\n"; + EmitHwModeCheck(Target + "GenMCSubtargetInfo", OS); } void SubtargetEmitter::EmitMCInstrAnalysisPredicateFunctions(raw_ostream &OS) { @@ -1858,7 +1862,7 @@ void SubtargetEmitter::run(raw_ostream &OS) { OS << "namespace " << Target << "_MC {\n" << "unsigned resolveVariantSchedClassImpl(unsigned SchedClass," << " const MCInst *MI, unsigned CPUID);\n" - << "}\n\n"; + << "} // end namespace " << Target << "_MC\n\n"; OS << "struct " << ClassName << " : public TargetSubtargetInfo {\n" << " explicit " << ClassName << "(const Triple &TT, StringRef CPU, " << "StringRef FS);\n" diff --git a/contrib/llvm-project/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/contrib/llvm-project/llvm/utils/TableGen/SubtargetFeatureInfo.cpp index edf0b4a01c6d..5430f73d5e09 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/SubtargetFeatureInfo.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/SubtargetFeatureInfo.cpp @@ -38,6 +38,10 @@ SubtargetFeatureInfo::getAll(const RecordKeeper &Records) { if (Pred->getName().empty()) PrintFatalError(Pred->getLoc(), "Predicate has no name!"); + // Ignore always true predicates. + if (Pred->getValueAsString("CondString").empty()) + continue; + SubtargetFeatures.emplace_back( Pred, SubtargetFeatureInfo(Pred, SubtargetFeatures.size())); } @@ -95,9 +99,11 @@ void SubtargetFeatureInfo::emitComputeAvailableFeatures( OS << " PredicateBitset Features;\n"; for (const auto &SF : SubtargetFeatures) { const SubtargetFeatureInfo &SFI = SF.second; + StringRef CondStr = SFI.TheDef->getValueAsString("CondString"); + assert(!CondStr.empty() && "true predicate should have been filtered"); - OS << " if (" << SFI.TheDef->getValueAsString("CondString") << ")\n"; - OS << " Features[" << SFI.getEnumBitName() << "] = 1;\n"; + OS << " if (" << CondStr << ")\n"; + OS << " Features.set(" << SFI.getEnumBitName() << ");\n"; } OS << " return Features;\n"; OS << "}\n\n"; @@ -142,7 +148,7 @@ void SubtargetFeatureInfo::emitComputeAssemblerAvailableFeatures( } while (true); OS << ")\n"; - OS << " Features[" << SFI.getEnumBitName() << "] = 1;\n"; + OS << " Features.set(" << SFI.getEnumBitName() << ");\n"; } OS << " return Features;\n"; OS << "}\n\n"; diff --git a/contrib/llvm-project/llvm/utils/TableGen/TableGen.cpp b/contrib/llvm-project/llvm/utils/TableGen/TableGen.cpp index c485ed2feb7a..bdb963c15d32 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/TableGen.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/TableGen.cpp @@ -40,19 +40,20 @@ enum ActionType { GenSubtarget, GenIntrinsicEnums, GenIntrinsicImpl, - GenTgtIntrinsicEnums, - GenTgtIntrinsicImpl, PrintEnums, PrintSets, GenOptParserDefs, + GenOptRST, GenCTags, GenAttributes, GenSearchableTables, GenGlobalISel, + GenGICombiner, GenX86EVEX2VEXTables, GenX86FoldTables, GenRegisterBank, GenExegesis, + GenAutomata, }; namespace llvm { @@ -62,75 +63,71 @@ bool TimeRegions = false; } // end namespace llvm namespace { - cl::opt<ActionType> - Action(cl::desc("Action to perform:"), - cl::values(clEnumValN(PrintRecords, "print-records", - "Print all records to stdout (default)"), - clEnumValN(DumpJSON, "dump-json", - "Dump all records as machine-readable JSON"), - clEnumValN(GenEmitter, "gen-emitter", - "Generate machine code emitter"), - clEnumValN(GenRegisterInfo, "gen-register-info", - "Generate registers and register classes info"), - clEnumValN(GenInstrInfo, "gen-instr-info", - "Generate instruction descriptions"), - clEnumValN(GenInstrDocs, "gen-instr-docs", - "Generate instruction documentation"), - clEnumValN(GenCallingConv, "gen-callingconv", - "Generate calling convention descriptions"), - clEnumValN(GenAsmWriter, "gen-asm-writer", - "Generate assembly writer"), - clEnumValN(GenDisassembler, "gen-disassembler", - "Generate disassembler"), - clEnumValN(GenPseudoLowering, "gen-pseudo-lowering", - "Generate pseudo instruction lowering"), - clEnumValN(GenCompressInst, "gen-compress-inst-emitter", - "Generate RISCV compressed instructions."), - clEnumValN(GenAsmMatcher, "gen-asm-matcher", - "Generate assembly instruction matcher"), - clEnumValN(GenDAGISel, "gen-dag-isel", - "Generate a DAG instruction selector"), - clEnumValN(GenDFAPacketizer, "gen-dfa-packetizer", - "Generate DFA Packetizer for VLIW targets"), - clEnumValN(GenFastISel, "gen-fast-isel", - "Generate a \"fast\" instruction selector"), - clEnumValN(GenSubtarget, "gen-subtarget", - "Generate subtarget enumerations"), - clEnumValN(GenIntrinsicEnums, "gen-intrinsic-enums", - "Generate intrinsic enums"), - clEnumValN(GenIntrinsicImpl, "gen-intrinsic-impl", - "Generate intrinsic information"), - clEnumValN(GenTgtIntrinsicEnums, "gen-tgt-intrinsic-enums", - "Generate target intrinsic enums"), - clEnumValN(GenTgtIntrinsicImpl, "gen-tgt-intrinsic-impl", - "Generate target intrinsic information"), - clEnumValN(PrintEnums, "print-enums", - "Print enum values for a class"), - clEnumValN(PrintSets, "print-sets", - "Print expanded sets for testing DAG exprs"), - clEnumValN(GenOptParserDefs, "gen-opt-parser-defs", - "Generate option definitions"), - clEnumValN(GenCTags, "gen-ctags", - "Generate ctags-compatible index"), - clEnumValN(GenAttributes, "gen-attrs", - "Generate attributes"), - clEnumValN(GenSearchableTables, "gen-searchable-tables", - "Generate generic binary-searchable table"), - clEnumValN(GenGlobalISel, "gen-global-isel", - "Generate GlobalISel selector"), - clEnumValN(GenX86EVEX2VEXTables, "gen-x86-EVEX2VEX-tables", - "Generate X86 EVEX to VEX compress tables"), - clEnumValN(GenX86FoldTables, "gen-x86-fold-tables", - "Generate X86 fold tables"), - clEnumValN(GenRegisterBank, "gen-register-bank", - "Generate registers bank descriptions"), - clEnumValN(GenExegesis, "gen-exegesis", - "Generate llvm-exegesis tables"))); +cl::opt<ActionType> Action( + cl::desc("Action to perform:"), + cl::values( + clEnumValN(PrintRecords, "print-records", + "Print all records to stdout (default)"), + clEnumValN(DumpJSON, "dump-json", + "Dump all records as machine-readable JSON"), + clEnumValN(GenEmitter, "gen-emitter", "Generate machine code emitter"), + clEnumValN(GenRegisterInfo, "gen-register-info", + "Generate registers and register classes info"), + clEnumValN(GenInstrInfo, "gen-instr-info", + "Generate instruction descriptions"), + clEnumValN(GenInstrDocs, "gen-instr-docs", + "Generate instruction documentation"), + clEnumValN(GenCallingConv, "gen-callingconv", + "Generate calling convention descriptions"), + clEnumValN(GenAsmWriter, "gen-asm-writer", "Generate assembly writer"), + clEnumValN(GenDisassembler, "gen-disassembler", + "Generate disassembler"), + clEnumValN(GenPseudoLowering, "gen-pseudo-lowering", + "Generate pseudo instruction lowering"), + clEnumValN(GenCompressInst, "gen-compress-inst-emitter", + "Generate RISCV compressed instructions."), + clEnumValN(GenAsmMatcher, "gen-asm-matcher", + "Generate assembly instruction matcher"), + clEnumValN(GenDAGISel, "gen-dag-isel", + "Generate a DAG instruction selector"), + clEnumValN(GenDFAPacketizer, "gen-dfa-packetizer", + "Generate DFA Packetizer for VLIW targets"), + clEnumValN(GenFastISel, "gen-fast-isel", + "Generate a \"fast\" instruction selector"), + clEnumValN(GenSubtarget, "gen-subtarget", + "Generate subtarget enumerations"), + clEnumValN(GenIntrinsicEnums, "gen-intrinsic-enums", + "Generate intrinsic enums"), + clEnumValN(GenIntrinsicImpl, "gen-intrinsic-impl", + "Generate intrinsic information"), + clEnumValN(PrintEnums, "print-enums", "Print enum values for a class"), + clEnumValN(PrintSets, "print-sets", + "Print expanded sets for testing DAG exprs"), + clEnumValN(GenOptParserDefs, "gen-opt-parser-defs", + "Generate option definitions"), + clEnumValN(GenOptRST, "gen-opt-rst", "Generate option RST"), + clEnumValN(GenCTags, "gen-ctags", "Generate ctags-compatible index"), + clEnumValN(GenAttributes, "gen-attrs", "Generate attributes"), + clEnumValN(GenSearchableTables, "gen-searchable-tables", + "Generate generic binary-searchable table"), + clEnumValN(GenGlobalISel, "gen-global-isel", + "Generate GlobalISel selector"), + clEnumValN(GenGICombiner, "gen-global-isel-combiner", + "Generate GlobalISel combiner"), + clEnumValN(GenX86EVEX2VEXTables, "gen-x86-EVEX2VEX-tables", + "Generate X86 EVEX to VEX compress tables"), + clEnumValN(GenX86FoldTables, "gen-x86-fold-tables", + "Generate X86 fold tables"), + clEnumValN(GenRegisterBank, "gen-register-bank", + "Generate registers bank descriptions"), + clEnumValN(GenExegesis, "gen-exegesis", + "Generate llvm-exegesis tables"), + clEnumValN(GenAutomata, "gen-automata", "Generate generic automata"))); - cl::OptionCategory PrintEnumsCat("Options for -print-enums"); - cl::opt<std::string> - Class("class", cl::desc("Print Enum list for this class"), - cl::value_desc("class name"), cl::cat(PrintEnumsCat)); +cl::OptionCategory PrintEnumsCat("Options for -print-enums"); +cl::opt<std::string> Class("class", cl::desc("Print Enum list for this class"), + cl::value_desc("class name"), + cl::cat(PrintEnumsCat)); cl::opt<bool, true> TimeRegionsOpt("time-regions", @@ -193,15 +190,12 @@ bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) { case GenIntrinsicImpl: EmitIntrinsicImpl(Records, OS); break; - case GenTgtIntrinsicEnums: - EmitIntrinsicEnums(Records, OS, true); - break; - case GenTgtIntrinsicImpl: - EmitIntrinsicImpl(Records, OS, true); - break; case GenOptParserDefs: EmitOptParser(Records, OS); break; + case GenOptRST: + EmitOptRST(Records, OS); + break; case PrintEnums: { for (Record *Rec : Records.getAllDerivedDefinitions(Class)) @@ -235,6 +229,9 @@ bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) { case GenGlobalISel: EmitGlobalISel(Records, OS); break; + case GenGICombiner: + EmitGICombiner(Records, OS); + break; case GenRegisterBank: EmitRegisterBank(Records, OS); break; @@ -247,6 +244,9 @@ bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) { case GenExegesis: EmitExegesis(Records, OS); break; + case GenAutomata: + EmitAutomata(Records, OS); + break; } return false; @@ -263,11 +263,16 @@ int main(int argc, char **argv) { return TableGenMain(argv[0], &LLVMTableGenMain); } -#ifdef __has_feature -#if __has_feature(address_sanitizer) +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) || \ + __has_feature(leak_sanitizer) + #include <sanitizer/lsan_interface.h> // Disable LeakSanitizer for this binary as it has too many leaks that are not // very interesting to fix. See compiler-rt/include/sanitizer/lsan_interface.h . LLVM_ATTRIBUTE_USED int __lsan_is_turned_off() { return 1; } -#endif // __has_feature(address_sanitizer) -#endif // defined(__has_feature) + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/TableGenBackends.h b/contrib/llvm-project/llvm/utils/TableGen/TableGenBackends.h index 135ec65c0f95..9eef77a4577f 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/TableGenBackends.h +++ b/contrib/llvm-project/llvm/utils/TableGen/TableGenBackends.h @@ -61,10 +61,8 @@ namespace llvm { class raw_ostream; class RecordKeeper; -void EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS, - bool TargetOnly = false); -void EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS, - bool TargetOnly = false); +void EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS); +void EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS); void EmitAsmMatcher(RecordKeeper &RK, raw_ostream &OS); void EmitAsmWriter(RecordKeeper &RK, raw_ostream &OS); void EmitCallingConv(RecordKeeper &RK, raw_ostream &OS); @@ -81,14 +79,17 @@ void EmitRegisterInfo(RecordKeeper &RK, raw_ostream &OS); void EmitSubtarget(RecordKeeper &RK, raw_ostream &OS); void EmitMapTable(RecordKeeper &RK, raw_ostream &OS); void EmitOptParser(RecordKeeper &RK, raw_ostream &OS); +void EmitOptRST(RecordKeeper &RK, raw_ostream &OS); void EmitCTags(RecordKeeper &RK, raw_ostream &OS); void EmitAttributes(RecordKeeper &RK, raw_ostream &OS); void EmitSearchableTables(RecordKeeper &RK, raw_ostream &OS); void EmitGlobalISel(RecordKeeper &RK, raw_ostream &OS); +void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS); void EmitX86EVEX2VEXTables(RecordKeeper &RK, raw_ostream &OS); void EmitX86FoldTables(RecordKeeper &RK, raw_ostream &OS); void EmitRegisterBank(RecordKeeper &RK, raw_ostream &OS); void EmitExegesis(RecordKeeper &RK, raw_ostream &OS); +void EmitAutomata(RecordKeeper &RK, raw_ostream &OS); } // End llvm namespace diff --git a/contrib/llvm-project/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp index 365cba5a60ca..54aa5a8164f2 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp @@ -167,7 +167,7 @@ void emitWebAssemblyDisassemblerTables( OS << " },\n"; } OS << " { 0, nullptr }\n};\n\n"; - OS << "} // End llvm namespace\n"; + OS << "} // end namespace llvm\n"; } } // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp b/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp index 8036aecc4f4b..5dc653ac3806 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp @@ -651,7 +651,7 @@ static const char* stringForDecisionType(ModRMDecisionType dt) { DisassemblerTables::DisassemblerTables() { for (unsigned i = 0; i < array_lengthof(Tables); i++) - Tables[i] = llvm::make_unique<ContextDecision>(); + Tables[i] = std::make_unique<ContextDecision>(); HasConflicts = false; } @@ -667,16 +667,9 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, static uint32_t sEntryNumber = 1; ModRMDecisionType dt = getDecisionType(decision); - if (dt == MODRM_ONEENTRY && decision.instructionIDs[0] == 0) - { - o2.indent(i2) << "{ /* ModRMDecision */" << "\n"; - i2++; - - o2.indent(i2) << stringForDecisionType(dt) << "," << "\n"; - o2.indent(i2) << 0 << " /* EmptyTable */\n"; - - i2--; - o2.indent(i2) << "}"; + if (dt == MODRM_ONEENTRY && decision.instructionIDs[0] == 0) { + // Empty table. + o2 << "{ " << stringForDecisionType(dt) << ", 0 }"; return; } @@ -725,14 +718,8 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, i1--; } - o2.indent(i2) << "{ /* struct ModRMDecision */" << "\n"; - i2++; - - o2.indent(i2) << stringForDecisionType(dt) << "," << "\n"; - o2.indent(i2) << EntryNumber << " /* Table" << EntryNumber << " */\n"; - - i2--; - o2.indent(i2) << "}"; + o2 << "{ " << stringForDecisionType(dt) << ", " << EntryNumber << " /* Table" + << EntryNumber << " */ }"; switch (dt) { default: @@ -764,30 +751,43 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, void DisassemblerTables::emitOpcodeDecision(raw_ostream &o1, raw_ostream &o2, unsigned &i1, unsigned &i2, unsigned &ModRMTableNum, - OpcodeDecision &decision) const { - o2.indent(i2) << "{ /* struct OpcodeDecision */" << "\n"; - i2++; - o2.indent(i2) << "{" << "\n"; - i2++; - - for (unsigned index = 0; index < 256; ++index) { - o2.indent(i2); - - o2 << "/* 0x" << format("%02hhx", index) << " */" << "\n"; - - emitModRMDecision(o1, o2, i1, i2, ModRMTableNum, - decision.modRMDecisions[index]); - - if (index < 255) - o2 << ","; - - o2 << "\n"; + OpcodeDecision &opDecision) const { + o2 << "{"; + ++i2; + + unsigned index; + for (index = 0; index < 256; ++index) { + auto &decision = opDecision.modRMDecisions[index]; + ModRMDecisionType dt = getDecisionType(decision); + if (!(dt == MODRM_ONEENTRY && decision.instructionIDs[0] == 0)) + break; + } + if (index == 256) { + // If all 256 entries are MODRM_ONEENTRY, omit output. + assert(MODRM_ONEENTRY == 0); + --i2; + o2 << "},\n"; + } else { + o2 << " /* struct OpcodeDecision */ {\n"; + ++i2; + for (index = 0; index < 256; ++index) { + o2.indent(i2); + + o2 << "/* 0x" << format("%02hhx", index) << " */ "; + + emitModRMDecision(o1, o2, i1, i2, ModRMTableNum, + opDecision.modRMDecisions[index]); + + if (index < 255) + o2 << ","; + + o2 << "\n"; + } + --i2; + o2.indent(i2) << "}\n"; + --i2; + o2.indent(i2) << "},\n"; } - - i2--; - o2.indent(i2) << "}" << "\n"; - i2--; - o2.indent(i2) << "}" << "\n"; } void DisassemblerTables::emitContextDecision(raw_ostream &o1, raw_ostream &o2, @@ -803,14 +803,10 @@ void DisassemblerTables::emitContextDecision(raw_ostream &o1, raw_ostream &o2, for (unsigned index = 0; index < IC_max; ++index) { o2.indent(i2) << "/* "; o2 << stringForContext((InstructionContext)index); - o2 << " */"; - o2 << "\n"; + o2 << " */ "; emitOpcodeDecision(o1, o2, i1, i2, ModRMTableNum, decision.opcodeDecisions[index]); - - if (index + 1 < IC_max) - o2 << ", "; } i2--; diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp index 3df14f40e4a9..6dc7e31e0dab 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp @@ -98,6 +98,7 @@ public: bool EVEX_W1_VEX_W0 = RecE->getValueAsBit("EVEX_W1_VEX_W0"); if (RecV->getValueAsDef("OpEnc")->getName().str() != "EncVEX" || + RecV->getValueAsBit("isCodeGenOnly") != RecE->getValueAsBit("isCodeGenOnly") || // VEX/EVEX fields RecV->getValueAsDef("OpPrefix") != RecE->getValueAsDef("OpPrefix") || RecV->getValueAsDef("OpMap") != RecE->getValueAsDef("OpMap") || diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/X86FoldTablesEmitter.cpp index 2c15e35f234d..8026c324cd40 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/X86FoldTablesEmitter.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/X86FoldTablesEmitter.cpp @@ -618,14 +618,14 @@ void X86FoldTablesEmitter::run(formatted_raw_ostream &OS) { uint8_t Opc = getValueFromBitsInit(MemInst->TheDef->getValueAsBitsInit("Opcode")); - if (RegInsts.count(Opc) == 0) + auto RegInstsIt = RegInsts.find(Opc); + if (RegInstsIt == RegInsts.end()) continue; // Two forms (memory & register) of the same instruction must have the same // opcode. try matching only with register form instructions with the same // opcode. - std::vector<const CodeGenInstruction *> &OpcRegInsts = - RegInsts.find(Opc)->second; + std::vector<const CodeGenInstruction *> &OpcRegInsts = RegInstsIt->second; auto Match = find_if(OpcRegInsts, IsMatch(MemInst, Records)); if (Match != OpcRegInsts.end()) { diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp b/contrib/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp index ab8a8855c478..1048ef81a378 100644 --- a/contrib/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp +++ b/contrib/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp @@ -749,7 +749,7 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const { case X86Local::RawFrmImm8: case X86Local::RawFrmImm16: case X86Local::AddCCFrm: - filter = llvm::make_unique<DumbFilter>(); + filter = std::make_unique<DumbFilter>(); break; case X86Local::MRMDestReg: case X86Local::MRMSrcReg: @@ -758,7 +758,7 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const { case X86Local::MRMSrcRegCC: case X86Local::MRMXrCC: case X86Local::MRMXr: - filter = llvm::make_unique<ModFilter>(true); + filter = std::make_unique<ModFilter>(true); break; case X86Local::MRMDestMem: case X86Local::MRMSrcMem: @@ -767,22 +767,22 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const { case X86Local::MRMSrcMemCC: case X86Local::MRMXmCC: case X86Local::MRMXm: - filter = llvm::make_unique<ModFilter>(false); + filter = std::make_unique<ModFilter>(false); break; case X86Local::MRM0r: case X86Local::MRM1r: case X86Local::MRM2r: case X86Local::MRM3r: case X86Local::MRM4r: case X86Local::MRM5r: case X86Local::MRM6r: case X86Local::MRM7r: - filter = llvm::make_unique<ExtendedFilter>(true, Form - X86Local::MRM0r); + filter = std::make_unique<ExtendedFilter>(true, Form - X86Local::MRM0r); break; case X86Local::MRM0m: case X86Local::MRM1m: case X86Local::MRM2m: case X86Local::MRM3m: case X86Local::MRM4m: case X86Local::MRM5m: case X86Local::MRM6m: case X86Local::MRM7m: - filter = llvm::make_unique<ExtendedFilter>(false, Form - X86Local::MRM0m); + filter = std::make_unique<ExtendedFilter>(false, Form - X86Local::MRM0m); break; X86_INSTR_MRM_MAPPING - filter = llvm::make_unique<ExactFilter>(0xC0 + Form - X86Local::MRM_C0); + filter = std::make_unique<ExactFilter>(0xC0 + Form - X86Local::MRM_C0); break; } // switch (Form) @@ -854,6 +854,7 @@ OperandType RecognizableInstr::typeFromString(const std::string &s, TYPE("GR64", TYPE_R64) TYPE("i8mem", TYPE_M) TYPE("i8imm", TYPE_IMM) + TYPE("u4imm", TYPE_UIMM8) TYPE("u8imm", TYPE_UIMM8) TYPE("i16u8imm", TYPE_UIMM8) TYPE("i32u8imm", TYPE_UIMM8) @@ -878,9 +879,9 @@ OperandType RecognizableInstr::typeFromString(const std::string &s, TYPE("i128mem", TYPE_M) TYPE("i256mem", TYPE_M) TYPE("i512mem", TYPE_M) - TYPE("i64i32imm_pcrel", TYPE_REL) - TYPE("i16imm_pcrel", TYPE_REL) - TYPE("i32imm_pcrel", TYPE_REL) + TYPE("i64i32imm_brtarget", TYPE_REL) + TYPE("i16imm_brtarget", TYPE_REL) + TYPE("i32imm_brtarget", TYPE_REL) TYPE("ccode", TYPE_IMM) TYPE("AVX512RC", TYPE_IMM) TYPE("brtarget32", TYPE_REL) @@ -973,6 +974,7 @@ RecognizableInstr::immediateEncodingFromString(const std::string &s, ENCODING("i64i32imm", ENCODING_ID) ENCODING("i64i8imm", ENCODING_IB) ENCODING("i8imm", ENCODING_IB) + ENCODING("u4imm", ENCODING_IB) ENCODING("u8imm", ENCODING_IB) ENCODING("i16u8imm", ENCODING_IB) ENCODING("i32u8imm", ENCODING_IB) @@ -1167,45 +1169,45 @@ RecognizableInstr::relocationEncodingFromString(const std::string &s, if(OpSize != X86Local::OpSize16) { // For instructions without an OpSize prefix, a declared 16-bit register or // immediate encoding is special. - ENCODING("i16imm", ENCODING_IW) + ENCODING("i16imm", ENCODING_IW) } - ENCODING("i16imm", ENCODING_Iv) - ENCODING("i16i8imm", ENCODING_IB) - ENCODING("i32imm", ENCODING_Iv) - ENCODING("i32i8imm", ENCODING_IB) - ENCODING("i64i32imm", ENCODING_ID) - ENCODING("i64i8imm", ENCODING_IB) - ENCODING("i8imm", ENCODING_IB) - ENCODING("u8imm", ENCODING_IB) - ENCODING("i16u8imm", ENCODING_IB) - ENCODING("i32u8imm", ENCODING_IB) - ENCODING("i64u8imm", ENCODING_IB) - ENCODING("i64i32imm_pcrel", ENCODING_ID) - ENCODING("i16imm_pcrel", ENCODING_IW) - ENCODING("i32imm_pcrel", ENCODING_ID) - ENCODING("brtarget32", ENCODING_ID) - ENCODING("brtarget16", ENCODING_IW) - ENCODING("brtarget8", ENCODING_IB) - ENCODING("i64imm", ENCODING_IO) - ENCODING("offset16_8", ENCODING_Ia) - ENCODING("offset16_16", ENCODING_Ia) - ENCODING("offset16_32", ENCODING_Ia) - ENCODING("offset32_8", ENCODING_Ia) - ENCODING("offset32_16", ENCODING_Ia) - ENCODING("offset32_32", ENCODING_Ia) - ENCODING("offset32_64", ENCODING_Ia) - ENCODING("offset64_8", ENCODING_Ia) - ENCODING("offset64_16", ENCODING_Ia) - ENCODING("offset64_32", ENCODING_Ia) - ENCODING("offset64_64", ENCODING_Ia) - ENCODING("srcidx8", ENCODING_SI) - ENCODING("srcidx16", ENCODING_SI) - ENCODING("srcidx32", ENCODING_SI) - ENCODING("srcidx64", ENCODING_SI) - ENCODING("dstidx8", ENCODING_DI) - ENCODING("dstidx16", ENCODING_DI) - ENCODING("dstidx32", ENCODING_DI) - ENCODING("dstidx64", ENCODING_DI) + ENCODING("i16imm", ENCODING_Iv) + ENCODING("i16i8imm", ENCODING_IB) + ENCODING("i32imm", ENCODING_Iv) + ENCODING("i32i8imm", ENCODING_IB) + ENCODING("i64i32imm", ENCODING_ID) + ENCODING("i64i8imm", ENCODING_IB) + ENCODING("i8imm", ENCODING_IB) + ENCODING("u8imm", ENCODING_IB) + ENCODING("i16u8imm", ENCODING_IB) + ENCODING("i32u8imm", ENCODING_IB) + ENCODING("i64u8imm", ENCODING_IB) + ENCODING("i64i32imm_brtarget", ENCODING_ID) + ENCODING("i16imm_brtarget", ENCODING_IW) + ENCODING("i32imm_brtarget", ENCODING_ID) + ENCODING("brtarget32", ENCODING_ID) + ENCODING("brtarget16", ENCODING_IW) + ENCODING("brtarget8", ENCODING_IB) + ENCODING("i64imm", ENCODING_IO) + ENCODING("offset16_8", ENCODING_Ia) + ENCODING("offset16_16", ENCODING_Ia) + ENCODING("offset16_32", ENCODING_Ia) + ENCODING("offset32_8", ENCODING_Ia) + ENCODING("offset32_16", ENCODING_Ia) + ENCODING("offset32_32", ENCODING_Ia) + ENCODING("offset32_64", ENCODING_Ia) + ENCODING("offset64_8", ENCODING_Ia) + ENCODING("offset64_16", ENCODING_Ia) + ENCODING("offset64_32", ENCODING_Ia) + ENCODING("offset64_64", ENCODING_Ia) + ENCODING("srcidx8", ENCODING_SI) + ENCODING("srcidx16", ENCODING_SI) + ENCODING("srcidx32", ENCODING_SI) + ENCODING("srcidx64", ENCODING_SI) + ENCODING("dstidx8", ENCODING_DI) + ENCODING("dstidx16", ENCODING_DI) + ENCODING("dstidx32", ENCODING_DI) + ENCODING("dstidx64", ENCODING_DI) errs() << "Unhandled relocation encoding " << s << "\n"; llvm_unreachable("Unhandled relocation encoding"); } |