diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2023-12-09 13:28:42 +0000 |
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2023-12-09 13:28:42 +0000 |
| commit | b1c73532ee8997fe5dfbeb7d223027bdf99758a0 (patch) | |
| tree | 7d6e51c294ab6719475d660217aa0c0ad0526292 /llvm/utils | |
| parent | 7fa27ce4a07f19b07799a767fc29416f3b625afb (diff) | |
Diffstat (limited to 'llvm/utils')
67 files changed, 7090 insertions, 6661 deletions
diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp index 1c195200a888..f774f0c1018b 100644 --- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp +++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -910,7 +910,7 @@ extractSingletonRegisterForAsmOperand(MatchableInfo::AsmOperand &Op, return; } - if (!Tok.startswith(RegisterPrefix)) + if (!Tok.starts_with(RegisterPrefix)) return; StringRef RegName = Tok.substr(RegisterPrefix.size()); @@ -1520,7 +1520,7 @@ void AsmMatcherInfo::buildInfo() { // If the tblgen -match-prefix option is specified (for tblgen hackers), // filter the set of instructions we consider. - if (!StringRef(CGI->TheDef->getName()).startswith(MatchPrefix)) + if (!StringRef(CGI->TheDef->getName()).starts_with(MatchPrefix)) continue; // Ignore "codegen only" instructions. @@ -1555,7 +1555,7 @@ void AsmMatcherInfo::buildInfo() { // filter the set of instruction aliases we consider, based on the target // instruction. if (!StringRef(Alias->ResultInst->TheDef->getName()) - .startswith( MatchPrefix)) + .starts_with(MatchPrefix)) continue; StringRef V = Alias->TheDef->getValueAsString("AsmVariantName"); @@ -3204,7 +3204,7 @@ void AsmMatcherEmitter::run(raw_ostream &OS) { Record *AsmParser = Target.getAsmParser(); StringRef ClassName = AsmParser->getValueAsString("AsmParserClassName"); - emitSourceFileHeader("Assembly Matcher Source Fragment", OS); + emitSourceFileHeader("Assembly Matcher Source Fragment", OS, Records); // Compute the information on the instructions to match. AsmMatcherInfo Info(AsmParser, Target, Records); diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp index 92e71910a800..0220927295cf 100644 --- a/llvm/utils/TableGen/AsmWriterEmitter.cpp +++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp @@ -1302,7 +1302,7 @@ void AsmWriterEmitter::run(raw_ostream &O) { std::vector<std::vector<std::string>> TableDrivenOperandPrinters; unsigned BitsLeft = 0; unsigned AsmStrBits = 0; - emitSourceFileHeader("Assembly Writer Source Fragment", O); + emitSourceFileHeader("Assembly Writer Source Fragment", O, Records); EmitGetMnemonic(O, TableDrivenOperandPrinters, BitsLeft, AsmStrBits); EmitPrintInstruction(O, TableDrivenOperandPrinters, BitsLeft, AsmStrBits); EmitGetRegisterName(O); diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp index de3810b2e227..02e7000130cd 100644 --- a/llvm/utils/TableGen/CallingConvEmitter.cpp +++ b/llvm/utils/TableGen/CallingConvEmitter.cpp @@ -16,6 +16,7 @@ #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" #include <deque> +#include <set> using namespace llvm; @@ -106,12 +107,12 @@ void CallingConvEmitter::EmitCallingConv(Record *CC, raw_ostream &O) { // Emit all of the actions, in order. for (unsigned i = 0, e = CCActions->size(); i != e; ++i) { Record *Action = CCActions->getElementAsRecord(i); - SwiftAction = llvm::any_of(Action->getSuperClasses(), - [](const std::pair<Record *, SMRange> &Class) { - std::string Name = - Class.first->getNameInitAsString(); - return StringRef(Name).startswith("CCIfSwift"); - }); + SwiftAction = + llvm::any_of(Action->getSuperClasses(), + [](const std::pair<Record *, SMRange> &Class) { + std::string Name = Class.first->getNameInitAsString(); + return StringRef(Name).starts_with("CCIfSwift"); + }); O << "\n"; EmitAction(Action, 2, O); diff --git a/llvm/utils/TableGen/CodeGenRegisters.cpp b/llvm/utils/TableGen/CodeGenRegisters.cpp index 5c45290a0657..d1abdb74ea4a 100644 --- a/llvm/utils/TableGen/CodeGenRegisters.cpp +++ b/llvm/utils/TableGen/CodeGenRegisters.cpp @@ -965,11 +965,20 @@ static bool TopoOrderRC(const CodeGenRegisterClass &PA, return StringRef(A->getName()) < B->getName(); } +std::string CodeGenRegisterClass::getNamespaceQualification() const { + return Namespace.empty() ? "" : (Namespace + "::").str(); +} + std::string CodeGenRegisterClass::getQualifiedName() const { - if (Namespace.empty()) - return getName(); - else - return (Namespace + "::" + getName()).str(); + return getNamespaceQualification() + getName(); +} + +std::string CodeGenRegisterClass::getIdName() const { + return getName() + "RegClassID"; +} + +std::string CodeGenRegisterClass::getQualifiedIdName() const { + return getNamespaceQualification() + getIdName(); } // Compute sub-classes of all register classes. @@ -1027,8 +1036,8 @@ void CodeGenRegisterClass::computeSubClasses(CodeGenRegBank &RegBank) { std::optional<std::pair<CodeGenRegisterClass *, CodeGenRegisterClass *>> CodeGenRegisterClass::getMatchingSubClassWithSubRegs( CodeGenRegBank &RegBank, const CodeGenSubRegIndex *SubIdx) const { - auto SizeOrder = [this](const CodeGenRegisterClass *A, - const CodeGenRegisterClass *B) { + auto WeakSizeOrder = [this](const CodeGenRegisterClass *A, + const CodeGenRegisterClass *B) { // If there are multiple, identical register classes, prefer the original // register class. if (A == B) @@ -1050,7 +1059,7 @@ CodeGenRegisterClass::getMatchingSubClassWithSubRegs( for (auto &RC : RegClasses) if (SuperRegRCsBV[RC.EnumValue]) SuperRegRCs.emplace_back(&RC); - llvm::stable_sort(SuperRegRCs, SizeOrder); + llvm::stable_sort(SuperRegRCs, WeakSizeOrder); assert(SuperRegRCs.front() == BiggestSuperRegRC && "Biggest class wasn't first"); @@ -1063,11 +1072,11 @@ CodeGenRegisterClass::getMatchingSubClassWithSubRegs( if (SuperRegClassesBV.any()) SuperRegClasses.push_back(std::make_pair(&RC, SuperRegClassesBV)); } - llvm::sort(SuperRegClasses, - [&](const std::pair<CodeGenRegisterClass *, BitVector> &A, - const std::pair<CodeGenRegisterClass *, BitVector> &B) { - return SizeOrder(A.first, B.first); - }); + llvm::stable_sort(SuperRegClasses, + [&](const std::pair<CodeGenRegisterClass *, BitVector> &A, + const std::pair<CodeGenRegisterClass *, BitVector> &B) { + return WeakSizeOrder(A.first, B.first); + }); // Find the biggest subclass and subreg class such that R:subidx is in the // subreg class for all R in subclass. @@ -1166,22 +1175,42 @@ CodeGenRegBank::CodeGenRegBank(RecordKeeper &Records, for (auto &Idx : SubRegIndices) Idx.updateComponents(*this); - // Read in the register definitions. - std::vector<Record*> Regs = Records.getAllDerivedDefinitions("Register"); - llvm::sort(Regs, LessRecordRegister()); - // Assign the enumeration values. - for (unsigned i = 0, e = Regs.size(); i != e; ++i) - getReg(Regs[i]); + // Read in the register and register tuple definitions. + std::vector<Record *> Regs = Records.getAllDerivedDefinitions("Register"); + if (!Regs.empty() && Regs[0]->isSubClassOf("X86Reg")) { + // For X86, we need to sort Registers and RegisterTuples together to list + // new registers and register tuples at a later position. So that we can + // reduce unnecessary iterations on unsupported registers in LiveVariables. + // TODO: Remove this logic when migrate from LiveVariables to LiveIntervals + // completely. + std::vector<Record *> Tups = + Records.getAllDerivedDefinitions("RegisterTuples"); + for (Record *R : Tups) { + // Expand tuples and merge the vectors + std::vector<Record *> TupRegs = *Sets.expand(R); + Regs.insert(Regs.end(), TupRegs.begin(), TupRegs.end()); + } + + llvm::sort(Regs, LessRecordRegister()); + // Assign the enumeration values. + for (unsigned i = 0, e = Regs.size(); i != e; ++i) + getReg(Regs[i]); + } else { + llvm::sort(Regs, LessRecordRegister()); + // Assign the enumeration values. + for (unsigned i = 0, e = Regs.size(); i != e; ++i) + getReg(Regs[i]); - // Expand tuples and number the new registers. - std::vector<Record*> Tups = - Records.getAllDerivedDefinitions("RegisterTuples"); + // Expand tuples and number the new registers. + std::vector<Record *> Tups = + Records.getAllDerivedDefinitions("RegisterTuples"); - for (Record *R : Tups) { - std::vector<Record *> TupRegs = *Sets.expand(R); - llvm::sort(TupRegs, LessRecordRegister()); - for (Record *RC : TupRegs) - getReg(RC); + for (Record *R : Tups) { + std::vector<Record *> TupRegs = *Sets.expand(R); + llvm::sort(TupRegs, LessRecordRegister()); + for (Record *RC : TupRegs) + getReg(RC); + } } // Now all the registers are known. Build the object graph of explicit @@ -2114,8 +2143,8 @@ void CodeGenRegBank::computeRegUnitLaneMasks() { for (auto &Register : Registers) { // Create an initial lane mask for all register units. const auto &RegUnits = Register.getRegUnits(); - CodeGenRegister::RegUnitLaneMaskList - RegUnitLaneMasks(RegUnits.count(), LaneBitmask::getNone()); + CodeGenRegister::RegUnitLaneMaskList RegUnitLaneMasks( + RegUnits.count(), LaneBitmask::getAll()); // Iterate through SubRegisters. typedef CodeGenRegister::SubRegMap SubRegMap; const SubRegMap &SubRegs = Register.getSubRegs(); @@ -2134,7 +2163,7 @@ void CodeGenRegBank::computeRegUnitLaneMasks() { unsigned u = 0; for (unsigned RU : RegUnits) { if (SUI == RU) { - RegUnitLaneMasks[u] |= LaneMask; + RegUnitLaneMasks[u] &= LaneMask; assert(!Found); Found = true; } @@ -2288,8 +2317,8 @@ void CodeGenRegBank::inferSubClassWithSubReg(CodeGenRegisterClass *RC) { void CodeGenRegBank::inferMatchingSuperRegClass(CodeGenRegisterClass *RC, std::list<CodeGenRegisterClass>::iterator FirstSubRegRC) { - SmallVector<std::pair<const CodeGenRegister*, - const CodeGenRegister*>, 16> SSPairs; + DenseMap<const CodeGenRegister *, std::vector<const CodeGenRegister *>> + SubToSuperRegs; BitVector TopoSigs(getNumTopoSigs()); // Iterate in SubRegIndex numerical order to visit synthetic indices last. @@ -2301,12 +2330,12 @@ void CodeGenRegBank::inferMatchingSuperRegClass(CodeGenRegisterClass *RC, continue; // Build list of (Super, Sub) pairs for this SubIdx. - SSPairs.clear(); + SubToSuperRegs.clear(); TopoSigs.reset(); for (const auto Super : RC->getMembers()) { const CodeGenRegister *Sub = Super->getSubRegs().find(&SubIdx)->second; assert(Sub && "Missing sub-register"); - SSPairs.push_back(std::make_pair(Super, Sub)); + SubToSuperRegs[Sub].push_back(Super); TopoSigs.set(Sub->getTopoSig()); } @@ -2325,16 +2354,20 @@ void CodeGenRegBank::inferMatchingSuperRegClass(CodeGenRegisterClass *RC, continue; // Compute the subset of RC that maps into SubRC. CodeGenRegister::Vec SubSetVec; - for (unsigned i = 0, e = SSPairs.size(); i != e; ++i) - if (SubRC.contains(SSPairs[i].second)) - SubSetVec.push_back(SSPairs[i].first); + for (const CodeGenRegister *R : SubRC.getMembers()) { + auto It = SubToSuperRegs.find(R); + if (It != SubToSuperRegs.end()) { + const std::vector<const CodeGenRegister *> &SuperRegs = It->second; + SubSetVec.insert(SubSetVec.end(), SuperRegs.begin(), SuperRegs.end()); + } + } if (SubSetVec.empty()) continue; // RC injects completely into SubRC. sortAndUniqueRegisters(SubSetVec); - if (SubSetVec.size() == SSPairs.size()) { + if (SubSetVec.size() == RC->getMembers().size()) { SubRC.addSuperRegClass(&SubIdx, RC); continue; } diff --git a/llvm/utils/TableGen/CodeGenRegisters.h b/llvm/utils/TableGen/CodeGenRegisters.h index 15f08d1431f9..97f60811a7d8 100644 --- a/llvm/utils/TableGen/CodeGenRegisters.h +++ b/llvm/utils/TableGen/CodeGenRegisters.h @@ -353,8 +353,11 @@ namespace llvm { // created by TableGen. Record *getDef() const { return TheDef; } + std::string getNamespaceQualification() const; const std::string &getName() const { return Name; } std::string getQualifiedName() const; + std::string getIdName() const; + std::string getQualifiedIdName() const; ArrayRef<ValueTypeByHwMode> getValueTypes() const { return VTs; } unsigned getNumValueTypes() const { return VTs.size(); } bool hasType(const ValueTypeByHwMode &VT) const; @@ -365,7 +368,7 @@ namespace llvm { llvm_unreachable("VTNum greater than number of ValueTypes in RegClass!"); } - // Return true if this this class contains the register. + // Return true if this class contains the register. bool contains(const CodeGenRegister*) const; // Returns true if RC is a subclass. diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/CodeGenSchedule.cpp index 04219a6e54d9..54463da19821 100644 --- a/llvm/utils/TableGen/CodeGenSchedule.cpp +++ b/llvm/utils/TableGen/CodeGenSchedule.cpp @@ -91,10 +91,25 @@ struct InstRegexOp : public SetTheory::Operator { PrintFatalError(Loc, "instregex requires pattern string: " + Expr->getAsString()); StringRef Original = SI->getValue(); + // Drop an explicit ^ anchor to not interfere with prefix search. + bool HadAnchor = Original.consume_front("^"); // Extract a prefix that we can binary search on. static const char RegexMetachars[] = "()^$|*+?.[]\\{}"; auto FirstMeta = Original.find_first_of(RegexMetachars); + if (FirstMeta != StringRef::npos && FirstMeta > 0) { + // If we have a regex like ABC* we can only use AB as the prefix, as + // the * acts on C. + switch (Original[FirstMeta]) { + case '+': + case '*': + case '?': + --FirstMeta; + break; + default: + break; + } + } // Look for top-level | or ?. We cannot optimize them to binary search. if (removeParens(Original).find_first_of("|?") != std::string::npos) @@ -106,7 +121,10 @@ struct InstRegexOp : public SetTheory::Operator { if (!PatStr.empty()) { // For the rest use a python-style prefix match. std::string pat = std::string(PatStr); - if (pat[0] != '^') { + // Add ^ anchor. If we had one originally, don't need the group. + if (HadAnchor) { + pat.insert(0, "^"); + } else { pat.insert(0, "^("); pat.insert(pat.end(), ')'); } @@ -118,7 +136,7 @@ struct InstRegexOp : public SetTheory::Operator { // The generic opcodes are unsorted, handle them manually. for (auto *Inst : Generics) { StringRef InstName = Inst->TheDef->getName(); - if (InstName.startswith(Prefix) && + if (InstName.starts_with(Prefix) && (!Regexpr || Regexpr->match(InstName.substr(Prefix.size())))) { Elts.insert(Inst->TheDef); NumMatches++; @@ -134,7 +152,7 @@ struct InstRegexOp : public SetTheory::Operator { } bool operator()(StringRef LHS, const CodeGenInstruction *RHS) { return LHS < RHS->TheDef->getName() && - !RHS->TheDef->getName().startswith(LHS); + !RHS->TheDef->getName().starts_with(LHS); } }; auto Range1 = @@ -369,19 +387,20 @@ processSTIPredicate(STIPredicateFunction &Fn, const std::pair<APInt, APInt> &LhsMasks = OpcodeMasks[LhsIdx]; const std::pair<APInt, APInt> &RhsMasks = OpcodeMasks[RhsIdx]; - auto LessThan = [](const APInt &Lhs, const APInt &Rhs) { - unsigned LhsCountPopulation = Lhs.popcount(); - unsigned RhsCountPopulation = Rhs.popcount(); - return ((LhsCountPopulation < RhsCountPopulation) || - ((LhsCountPopulation == RhsCountPopulation) && - (Lhs.countl_zero() > Rhs.countl_zero()))); + auto PopulationCountAndLeftBit = + [](const APInt &Other) -> std::pair<int, int> { + return std::pair<int, int>(Other.popcount(), + -Other.countl_zero()); }; + auto lhsmask_first = PopulationCountAndLeftBit(LhsMasks.first); + auto rhsmask_first = PopulationCountAndLeftBit(RhsMasks.first); + if (lhsmask_first != rhsmask_first) + return lhsmask_first < rhsmask_first; - if (LhsMasks.first != RhsMasks.first) - return LessThan(LhsMasks.first, RhsMasks.first); - - if (LhsMasks.second != RhsMasks.second) - return LessThan(LhsMasks.second, RhsMasks.second); + auto lhsmask_second = PopulationCountAndLeftBit(LhsMasks.second); + auto rhsmask_second = PopulationCountAndLeftBit(RhsMasks.second); + if (lhsmask_second != rhsmask_second) + return lhsmask_second < rhsmask_second; return LhsIdx < RhsIdx; }); diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/CodeGenTarget.cpp index fbdc0499a8cf..53efa66f9dfc 100644 --- a/llvm/utils/TableGen/CodeGenTarget.cpp +++ b/llvm/utils/TableGen/CodeGenTarget.cpp @@ -43,7 +43,7 @@ static cl::opt<unsigned> /// getValueType - Return the MVT::SimpleValueType that the specified TableGen /// record corresponds to. -MVT::SimpleValueType llvm::getValueType(Record *Rec) { +MVT::SimpleValueType llvm::getValueType(const Record *Rec) { return (MVT::SimpleValueType)Rec->getValueAsInt("Value"); } diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/CodeGenTarget.h index 2ba3af724d36..a2b559d53b19 100644 --- a/llvm/utils/TableGen/CodeGenTarget.h +++ b/llvm/utils/TableGen/CodeGenTarget.h @@ -43,7 +43,7 @@ class CodeGenSubRegIndex; /// getValueType - Return the MVT::SimpleValueType that the specified TableGen /// record corresponds to. -MVT::SimpleValueType getValueType(Record *Rec); +MVT::SimpleValueType getValueType(const Record *Rec); StringRef getName(MVT::SimpleValueType T); StringRef getEnumName(MVT::SimpleValueType T); diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp index 9d9b69f4cfbd..f703fff0ef3e 100644 --- a/llvm/utils/TableGen/CompressInstEmitter.cpp +++ b/llvm/utils/TableGen/CompressInstEmitter.cpp @@ -16,7 +16,7 @@ // This tablegen backend processes CompressPat declarations in a // td file and generates all the required checks to validate the pattern // declarations; validate the input and output operands to generate the correct -// compressed instructions. The checks include validating different types of +// compressed instructions. The checks include validating different types of // operands; register operands, immediate operands, fixed register and fixed // immediate inputs. // @@ -26,7 +26,7 @@ // /// CompressInstEmitter backend. // class CompressPat<dag input, dag output, list<Predicate> predicates = []> { // /// Uncompressed instruction description. -// dag Input = input; +// dag Input = input; // /// Compressed instruction description. // dag Output = output; // /// Predicates that must be true for this to match. @@ -105,8 +105,8 @@ class CompressInstEmitter { // Required target features to enable pattern. std::vector<Record *> PatReqFeatures; // Maps operands in the Source Instruction to - IndexedMap<OpData> SourceOperandMap; // the corresponding Dest instruction operand. + IndexedMap<OpData> SourceOperandMap; // Maps operands in the Dest Instruction // to the corresponding Source instruction operand. IndexedMap<OpData> DestOperandMap; @@ -126,7 +126,7 @@ class CompressInstEmitter { void addDagOperandMapping(Record *Rec, DagInit *Dag, CodeGenInstruction &Inst, IndexedMap<OpData> &OperandMap, bool IsSourceInst); void evaluateCompressPat(Record *Compress); - void emitCompressInstEmitter(raw_ostream &o, EmitterType EType); + void emitCompressInstEmitter(raw_ostream &OS, EmitterType EType); bool validateTypes(Record *SubType, Record *Type, bool IsSourceInst); bool validateRegister(Record *Reg, Record *RegClass); void createDagOperandMapping(Record *Rec, StringMap<unsigned> &SourceOperands, @@ -144,7 +144,7 @@ class CompressInstEmitter { public: CompressInstEmitter(RecordKeeper &R) : Records(R), Target(R) {} - void run(raw_ostream &o); + void run(raw_ostream &OS); }; } // End anonymous namespace. @@ -206,53 +206,53 @@ void CompressInstEmitter::addDagOperandMapping(Record *Rec, DagInit *Dag, // than number of operands in the Dag due to how tied operands // are represented. unsigned TiedCount = 0; - for (unsigned i = 0, e = Inst.Operands.size(); i != e; ++i) { - int TiedOpIdx = Inst.Operands[i].getTiedRegister(); + for (unsigned I = 0, E = Inst.Operands.size(); I != E; ++I) { + int TiedOpIdx = Inst.Operands[I].getTiedRegister(); if (-1 != TiedOpIdx) { // Set the entry in OperandMap for the tied operand we're skipping. - OperandMap[i].Kind = OperandMap[TiedOpIdx].Kind; - OperandMap[i].Data = OperandMap[TiedOpIdx].Data; + OperandMap[I].Kind = OperandMap[TiedOpIdx].Kind; + OperandMap[I].Data = OperandMap[TiedOpIdx].Data; TiedCount++; continue; } - if (DefInit *DI = dyn_cast<DefInit>(Dag->getArg(i - TiedCount))) { + if (DefInit *DI = dyn_cast<DefInit>(Dag->getArg(I - TiedCount))) { if (DI->getDef()->isSubClassOf("Register")) { // Check if the fixed register belongs to the Register class. - if (!validateRegister(DI->getDef(), Inst.Operands[i].Rec)) + if (!validateRegister(DI->getDef(), Inst.Operands[I].Rec)) PrintFatalError(Rec->getLoc(), "Error in Dag '" + Dag->getAsString() + "'Register: '" + DI->getDef()->getName() + "' is not in register class '" + - Inst.Operands[i].Rec->getName() + "'"); - OperandMap[i].Kind = OpData::Reg; - OperandMap[i].Data.Reg = DI->getDef(); + Inst.Operands[I].Rec->getName() + "'"); + OperandMap[I].Kind = OpData::Reg; + OperandMap[I].Data.Reg = DI->getDef(); continue; } // Validate that Dag operand type matches the type defined in the // corresponding instruction. Operands in the input Dag pattern are // allowed to be a subclass of the type specified in corresponding // instruction operand instead of being an exact match. - if (!validateTypes(DI->getDef(), Inst.Operands[i].Rec, IsSourceInst)) + if (!validateTypes(DI->getDef(), Inst.Operands[I].Rec, IsSourceInst)) PrintFatalError(Rec->getLoc(), "Error in Dag '" + Dag->getAsString() + "'. Operand '" + - Dag->getArgNameStr(i - TiedCount) + "' has type '" + + Dag->getArgNameStr(I - TiedCount) + "' has type '" + DI->getDef()->getName() + "' which does not match the type '" + - Inst.Operands[i].Rec->getName() + + Inst.Operands[I].Rec->getName() + "' in the corresponding instruction operand!"); - OperandMap[i].Kind = OpData::Operand; - } else if (IntInit *II = dyn_cast<IntInit>(Dag->getArg(i - TiedCount))) { + OperandMap[I].Kind = OpData::Operand; + } else if (IntInit *II = dyn_cast<IntInit>(Dag->getArg(I - TiedCount))) { // Validate that corresponding instruction operand expects an immediate. - if (Inst.Operands[i].Rec->isSubClassOf("RegisterClass")) + if (Inst.Operands[I].Rec->isSubClassOf("RegisterClass")) PrintFatalError( Rec->getLoc(), "Error in Dag '" + Dag->getAsString() + "' Found immediate: '" + II->getAsString() + "' but corresponding instruction operand expected a register!"); // No pattern validation check possible for values of fixed immediate. - OperandMap[i].Kind = OpData::Imm; - OperandMap[i].Data.Imm = II->getValue(); + OperandMap[I].Kind = OpData::Imm; + OperandMap[I].Data.Imm = II->getValue(); LLVM_DEBUG( dbgs() << " Found immediate '" << II->getValue() << "' at " << (IsSourceInst ? "input " : "output ") @@ -281,7 +281,7 @@ static bool verifyDagOpCount(CodeGenInstruction &Inst, DagInit *Dag, "' and Dag operand count mismatch"); // The Instruction might have tied operands so the Dag might have - // a fewer operand count. + // a fewer operand count. unsigned RealCount = Inst.Operands.size(); for (const auto &Operand : Inst.Operands) if (Operand.getTiedRegister() != -1) @@ -307,43 +307,43 @@ void CompressInstEmitter::createDagOperandMapping( Record *Rec, StringMap<unsigned> &SourceOperands, StringMap<unsigned> &DestOperands, DagInit *SourceDag, DagInit *DestDag, IndexedMap<OpData> &SourceOperandMap) { - for (unsigned i = 0; i < DestDag->getNumArgs(); ++i) { + for (unsigned I = 0; I < DestDag->getNumArgs(); ++I) { // Skip fixed immediates and registers, they were handled in // addDagOperandMapping. - if ("" == DestDag->getArgNameStr(i)) + if ("" == DestDag->getArgNameStr(I)) continue; - DestOperands[DestDag->getArgNameStr(i)] = i; + DestOperands[DestDag->getArgNameStr(I)] = I; } - for (unsigned i = 0; i < SourceDag->getNumArgs(); ++i) { + for (unsigned I = 0; I < SourceDag->getNumArgs(); ++I) { // Skip fixed immediates and registers, they were handled in // addDagOperandMapping. - if ("" == SourceDag->getArgNameStr(i)) + if ("" == SourceDag->getArgNameStr(I)) continue; - StringMap<unsigned>::iterator it = - SourceOperands.find(SourceDag->getArgNameStr(i)); - if (it != SourceOperands.end()) { + StringMap<unsigned>::iterator It = + SourceOperands.find(SourceDag->getArgNameStr(I)); + if (It != SourceOperands.end()) { // Operand sharing the same name in the Dag should be mapped as tied. - SourceOperandMap[i].TiedOpIdx = it->getValue(); - if (!validateArgsTypes(SourceDag->getArg(it->getValue()), - SourceDag->getArg(i))) + SourceOperandMap[I].TiedOpIdx = It->getValue(); + if (!validateArgsTypes(SourceDag->getArg(It->getValue()), + SourceDag->getArg(I))) PrintFatalError(Rec->getLoc(), - "Input Operand '" + SourceDag->getArgNameStr(i) + + "Input Operand '" + SourceDag->getArgNameStr(I) + "' has a mismatched tied operand!\n"); } - it = DestOperands.find(SourceDag->getArgNameStr(i)); - if (it == DestOperands.end()) - PrintFatalError(Rec->getLoc(), "Operand " + SourceDag->getArgNameStr(i) + + It = DestOperands.find(SourceDag->getArgNameStr(I)); + if (It == DestOperands.end()) + PrintFatalError(Rec->getLoc(), "Operand " + SourceDag->getArgNameStr(I) + " defined in Input Dag but not used in" " Output Dag!\n"); // Input Dag operand types must match output Dag operand type. - if (!validateArgsTypes(DestDag->getArg(it->getValue()), - SourceDag->getArg(i))) + if (!validateArgsTypes(DestDag->getArg(It->getValue()), + SourceDag->getArg(I))) PrintFatalError(Rec->getLoc(), "Type mismatch between Input and " "Output Dag operand '" + - SourceDag->getArgNameStr(i) + "'!"); - SourceOperands[SourceDag->getArgNameStr(i)] = i; + SourceDag->getArgNameStr(I) + "'!"); + SourceOperands[SourceDag->getArgNameStr(I)] = I; } } @@ -358,27 +358,27 @@ void CompressInstEmitter::createInstOperandMapping( // operands list to get to the corresponding Dag operand. unsigned TiedCount = 0; LLVM_DEBUG(dbgs() << " Operand mapping:\n Source Dest\n"); - for (unsigned i = 0, e = DestInst.Operands.size(); i != e; ++i) { - int TiedInstOpIdx = DestInst.Operands[i].getTiedRegister(); + for (unsigned I = 0, E = DestInst.Operands.size(); I != E; ++I) { + int TiedInstOpIdx = DestInst.Operands[I].getTiedRegister(); if (TiedInstOpIdx != -1) { ++TiedCount; - DestOperandMap[i].Data = DestOperandMap[TiedInstOpIdx].Data; - DestOperandMap[i].Kind = DestOperandMap[TiedInstOpIdx].Kind; - if (DestOperandMap[i].Kind == OpData::Operand) + DestOperandMap[I].Data = DestOperandMap[TiedInstOpIdx].Data; + DestOperandMap[I].Kind = DestOperandMap[TiedInstOpIdx].Kind; + if (DestOperandMap[I].Kind == OpData::Operand) // No need to fill the SourceOperandMap here since it was mapped to // destination operand 'TiedInstOpIdx' in a previous iteration. - LLVM_DEBUG(dbgs() << " " << DestOperandMap[i].Data.Operand - << " ====> " << i + LLVM_DEBUG(dbgs() << " " << DestOperandMap[I].Data.Operand + << " ====> " << I << " Dest operand tied with operand '" << TiedInstOpIdx << "'\n"); continue; } // Skip fixed immediates and registers, they were handled in // addDagOperandMapping. - if (DestOperandMap[i].Kind != OpData::Operand) + if (DestOperandMap[I].Kind != OpData::Operand) continue; - unsigned DagArgIdx = i - TiedCount; + unsigned DagArgIdx = I - TiedCount; StringMap<unsigned>::iterator SourceOp = SourceOperands.find(DestDag->getArgNameStr(DagArgIdx)); if (SourceOp == SourceOperands.end()) @@ -390,9 +390,9 @@ void CompressInstEmitter::createInstOperandMapping( assert(DestDag->getArgNameStr(DagArgIdx) == SourceDag->getArgNameStr(SourceOp->getValue()) && "Incorrect operand mapping detected!\n"); - DestOperandMap[i].Data.Operand = SourceOp->getValue(); - SourceOperandMap[SourceOp->getValue()].Data.Operand = i; - LLVM_DEBUG(dbgs() << " " << SourceOp->getValue() << " ====> " << i + DestOperandMap[I].Data.Operand = SourceOp->getValue(); + SourceOperandMap[SourceOp->getValue()].Data.Operand = I; + LLVM_DEBUG(dbgs() << " " << SourceOp->getValue() << " ====> " << I << "\n"); } } @@ -403,10 +403,10 @@ void CompressInstEmitter::createInstOperandMapping( /// - Dag Input opcode is an expanded instruction and Dag Output opcode is a /// compressed instruction. /// - Operands in Dag Input must be all used in Dag Output. -/// Register Operand type in Dag Input Type must be contained in the +/// Register Operand type in Dag Input Type must be contained in the /// corresponding Source Instruction type. -/// - Register Operand type in Dag Input must be the same as in Dag Ouput. -/// - Register Operand type in Dag Output must be the same as the +/// - Register Operand type in Dag Input must be the same as in Dag Ouput. +/// - Register Operand type in Dag Output must be the same as the /// corresponding Destination Inst type. /// - Immediate Operand type in Dag Input must be the same as in Dag Ouput. /// - Immediate Operand type in Dag Ouput must be the same as the corresponding @@ -414,11 +414,12 @@ void CompressInstEmitter::createInstOperandMapping( /// - Fixed register must be contained in the corresponding Source Instruction /// type. /// - Fixed register must be contained in the corresponding Destination -/// Instruction type. Warning message printed under these conditions: +/// Instruction type. +/// Warning message printed under these conditions: /// - Fixed immediate in Dag Input or Dag Ouput cannot be checked at this time /// and generate warning. /// - Immediate operand type in Dag Input differs from the corresponding Source -/// Instruction type and generate a warning. +/// Instruction type and generate a warning. void CompressInstEmitter::evaluateCompressPat(Record *Rec) { // Validate input Dag operands. DagInit *SourceDag = Rec->getValueAsDag("Input"); @@ -426,8 +427,8 @@ void CompressInstEmitter::evaluateCompressPat(Record *Rec) { LLVM_DEBUG(dbgs() << "Input: " << *SourceDag << "\n"); // Checking we are transforming from compressed to uncompressed instructions. - Record *Operator = SourceDag->getOperatorAsDef(Rec->getLoc()); - CodeGenInstruction SourceInst(Operator); + Record *SourceOperator = SourceDag->getOperatorAsDef(Rec->getLoc()); + CodeGenInstruction SourceInst(SourceOperator); verifyDagOpCount(SourceInst, SourceDag, true); // Validate output Dag operands. @@ -439,12 +440,13 @@ void CompressInstEmitter::evaluateCompressPat(Record *Rec) { CodeGenInstruction DestInst(DestOperator); verifyDagOpCount(DestInst, DestDag, false); - if (Operator->getValueAsInt("Size") <= DestOperator->getValueAsInt("Size")) + if (SourceOperator->getValueAsInt("Size") <= + DestOperator->getValueAsInt("Size")) PrintFatalError( Rec->getLoc(), "Compressed instruction '" + DestOperator->getName() + "'is not strictly smaller than the uncompressed instruction '" + - Operator->getName() + "' !"); + SourceOperator->getName() + "' !"); // Fill the mapping from the source to destination instructions. @@ -538,13 +540,13 @@ static unsigned getPredicates(DenseMap<const Record *, unsigned> &PredicateMap, } static void printPredicates(const std::vector<const Record *> &Predicates, - StringRef Name, raw_ostream &o) { - for (unsigned i = 0; i < Predicates.size(); ++i) { - StringRef Pred = Predicates[i]->getValueAsString(Name); - o << " case " << i + 1 << ": {\n" - << " // " << Predicates[i]->getName() << "\n" - << " " << Pred << "\n" - << " }\n"; + StringRef Name, raw_ostream &OS) { + for (unsigned I = 0; I < Predicates.size(); ++I) { + StringRef Pred = Predicates[I]->getValueAsString(Name); + OS << " case " << I + 1 << ": {\n" + << " // " << Predicates[I]->getName() << "\n" + << " " << Pred << "\n" + << " }\n"; } } @@ -558,7 +560,7 @@ static void mergeCondAndCode(raw_ostream &CombinedStream, StringRef CondStr, CombinedStream.indent(4) << "} // if\n"; } -void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, +void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &OS, EmitterType EType) { Record *AsmWriter = Target.getAsmWriter(); if (!AsmWriter->getValueAsInt("PassSubtarget")) @@ -577,8 +579,7 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, const CompressPat &RHS) { if (EType == EmitterType::Compress || EType == EmitterType::CheckCompress) return (LHS.Source.TheDef->getName() < RHS.Source.TheDef->getName()); - else - return (LHS.Dest.TheDef->getName() < RHS.Dest.TheDef->getName()); + return (LHS.Dest.TheDef->getName() < RHS.Dest.TheDef->getName()); }); // A list of MCOperandPredicates for all operands in use, and the reverse map. @@ -594,14 +595,14 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, raw_string_ostream FuncH(FH); if (EType == EmitterType::Compress) - o << "\n#ifdef GEN_COMPRESS_INSTR\n" - << "#undef GEN_COMPRESS_INSTR\n\n"; + OS << "\n#ifdef GEN_COMPRESS_INSTR\n" + << "#undef GEN_COMPRESS_INSTR\n\n"; else if (EType == EmitterType::Uncompress) - o << "\n#ifdef GEN_UNCOMPRESS_INSTR\n" - << "#undef GEN_UNCOMPRESS_INSTR\n\n"; + OS << "\n#ifdef GEN_UNCOMPRESS_INSTR\n" + << "#undef GEN_UNCOMPRESS_INSTR\n\n"; else if (EType == EmitterType::CheckCompress) - o << "\n#ifdef GEN_CHECK_COMPRESS_INSTR\n" - << "#undef GEN_CHECK_COMPRESS_INSTR\n\n"; + OS << "\n#ifdef GEN_CHECK_COMPRESS_INSTR\n" + << "#undef GEN_CHECK_COMPRESS_INSTR\n\n"; if (EType == EmitterType::Compress) { FuncH << "static bool compressInst(MCInst &OutInst,\n"; @@ -617,14 +618,14 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, } if (CompressPatterns.empty()) { - o << FuncH.str(); - o.indent(2) << "return false;\n}\n"; + OS << FuncH.str(); + OS.indent(2) << "return false;\n}\n"; if (EType == EmitterType::Compress) - o << "\n#endif //GEN_COMPRESS_INSTR\n"; + OS << "\n#endif //GEN_COMPRESS_INSTR\n"; else if (EType == EmitterType::Uncompress) - o << "\n#endif //GEN_UNCOMPRESS_INSTR\n\n"; + OS << "\n#endif //GEN_UNCOMPRESS_INSTR\n\n"; else if (EType == EmitterType::CheckCompress) - o << "\n#endif //GEN_CHECK_COMPRESS_INSTR\n\n"; + OS << "\n#endif //GEN_CHECK_COMPRESS_INSTR\n\n"; return; } @@ -698,11 +699,11 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, for (auto &Set : AnyOfFeatureSets) { CondStream.indent(6) << "("; for (auto &Op : Set) { - bool isLast = &Op == &*Set.rbegin(); + bool IsLast = &Op == &*Set.rbegin(); StringRef Not = Op.first ? "!" : ""; CondStream << Not << "STI.getFeatureBits()[" << TargetName << "::" << Op.second << "]"; - if (!isLast) + if (!IsLast) CondStream << " || "; } CondStream << ") &&\n"; @@ -766,8 +767,8 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, if (DestOperand.getTiedRegister() == -1) CondStream.indent(6) << "(MI.getOperand(" << OpIdx << ").isReg()) &&\n" - << " (" << TargetName << "MCRegisterClasses[" - << TargetName << "::" << ClassRec->getName() + << " (" << TargetName << "MCRegisterClasses[" << TargetName + << "::" << ClassRec->getName() << "RegClassID].contains(MI.getOperand(" << OpIdx << ").getReg())) &&\n"; @@ -790,8 +791,8 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, CondStream.indent(6) << "MI.getOperand(" << OpIdx << ").isImm() &&\n"; CondStream.indent(6) << TargetName << "ValidateMachineOperand(" - << "MI.getOperand(" << OpIdx - << "), &STI, " << Entry << ") &&\n"; + << "MI.getOperand(" << OpIdx << "), &STI, " + << Entry << ") &&\n"; } if (CompressOrUncompress) CodeStream.indent(6) @@ -844,63 +845,63 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, Func.indent(2) << "return false;\n}\n"; if (!MCOpPredicates.empty()) { - o << "static bool " << ValidatorName << "(const MCOperand &MCOp,\n" - << " const MCSubtargetInfo &STI,\n" - << " unsigned PredicateIndex) {\n" - << " switch (PredicateIndex) {\n" - << " default:\n" - << " llvm_unreachable(\"Unknown MCOperandPredicate kind\");\n" - << " break;\n"; + OS << "static bool " << ValidatorName << "(const MCOperand &MCOp,\n" + << " const MCSubtargetInfo &STI,\n" + << " unsigned PredicateIndex) {\n" + << " switch (PredicateIndex) {\n" + << " default:\n" + << " llvm_unreachable(\"Unknown MCOperandPredicate kind\");\n" + << " break;\n"; - printPredicates(MCOpPredicates, "MCOperandPredicate", o); + printPredicates(MCOpPredicates, "MCOperandPredicate", OS); - o << " }\n" - << "}\n\n"; + OS << " }\n" + << "}\n\n"; } if (!ImmLeafPredicates.empty()) { - o << "static bool " << TargetName - << "ValidateMachineOperand(const MachineOperand &MO,\n" - << " const " << TargetName << "Subtarget *Subtarget,\n" - << " unsigned PredicateIndex) {\n" - << " int64_t Imm = MO.getImm();\n" - << " switch (PredicateIndex) {\n" - << " default:\n" - << " llvm_unreachable(\"Unknown ImmLeaf Predicate kind\");\n" - << " break;\n"; + OS << "static bool " << TargetName + << "ValidateMachineOperand(const MachineOperand &MO,\n" + << " const " << TargetName << "Subtarget *Subtarget,\n" + << " unsigned PredicateIndex) {\n" + << " int64_t Imm = MO.getImm();\n" + << " switch (PredicateIndex) {\n" + << " default:\n" + << " llvm_unreachable(\"Unknown ImmLeaf Predicate kind\");\n" + << " break;\n"; - printPredicates(ImmLeafPredicates, "ImmediateCode", o); + printPredicates(ImmLeafPredicates, "ImmediateCode", OS); - o << " }\n" - << "}\n\n"; + OS << " }\n" + << "}\n\n"; } - o << FuncH.str(); - o << Func.str(); + OS << FuncH.str(); + OS << Func.str(); if (EType == EmitterType::Compress) - o << "\n#endif //GEN_COMPRESS_INSTR\n"; + OS << "\n#endif //GEN_COMPRESS_INSTR\n"; else if (EType == EmitterType::Uncompress) - o << "\n#endif //GEN_UNCOMPRESS_INSTR\n\n"; + OS << "\n#endif //GEN_UNCOMPRESS_INSTR\n\n"; else if (EType == EmitterType::CheckCompress) - o << "\n#endif //GEN_CHECK_COMPRESS_INSTR\n\n"; + OS << "\n#endif //GEN_CHECK_COMPRESS_INSTR\n\n"; } -void CompressInstEmitter::run(raw_ostream &o) { +void CompressInstEmitter::run(raw_ostream &OS) { std::vector<Record *> Insts = Records.getAllDerivedDefinitions("CompressPat"); // Process the CompressPat definitions, validating them as we do so. - for (unsigned i = 0, e = Insts.size(); i != e; ++i) - evaluateCompressPat(Insts[i]); + for (unsigned I = 0, E = Insts.size(); I != E; ++I) + evaluateCompressPat(Insts[I]); // Emit file header. - emitSourceFileHeader("Compress instruction Source Fragment", o); + emitSourceFileHeader("Compress instruction Source Fragment", OS, Records); // Generate compressInst() function. - emitCompressInstEmitter(o, EmitterType::Compress); + emitCompressInstEmitter(OS, EmitterType::Compress); // Generate uncompressInst() function. - emitCompressInstEmitter(o, EmitterType::Uncompress); + emitCompressInstEmitter(OS, EmitterType::Uncompress); // Generate isCompressibleInst() function. - emitCompressInstEmitter(o, EmitterType::CheckCompress); + emitCompressInstEmitter(OS, EmitterType::CheckCompress); } static TableGen::Emitter::OptClass<CompressInstEmitter> diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp index 28d4d585f3dd..ed4be74ad0d4 100644 --- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -472,12 +472,16 @@ EmitMatcher(const Matcher *N, const unsigned Indent, unsigned CurrentIdx, return 2; case Matcher::CheckPatternPredicate: { - StringRef Pred =cast<CheckPatternPredicateMatcher>(N)->getPredicate(); - OS << "OPC_CheckPatternPredicate, " << getPatternPredicate(Pred) << ','; + StringRef Pred = cast<CheckPatternPredicateMatcher>(N)->getPredicate(); + unsigned PredNo = getPatternPredicate(Pred); + if (PredNo > 255) + OS << "OPC_CheckPatternPredicate2, TARGET_VAL(" << PredNo << "),"; + else + OS << "OPC_CheckPatternPredicate, " << PredNo << ','; if (!OmitComments) OS << " // " << Pred; OS << '\n'; - return 2; + return 2 + (PredNo > 255); } case Matcher::CheckPredicate: { TreePredicateFn Pred = cast<CheckPredicateMatcher>(N)->getPredicate(); @@ -665,19 +669,42 @@ EmitMatcher(const Matcher *N, const unsigned Indent, unsigned CurrentIdx, case Matcher::EmitInteger: { int64_t Val = cast<EmitIntegerMatcher>(N)->getValue(); - OS << "OPC_EmitInteger, " - << getEnumName(cast<EmitIntegerMatcher>(N)->getVT()) << ", "; - unsigned Bytes = 2 + EmitSignedVBRValue(Val, OS); + MVT::SimpleValueType VT = cast<EmitIntegerMatcher>(N)->getVT(); + unsigned OpBytes; + switch (VT) { + case MVT::i8: + case MVT::i16: + case MVT::i32: + case MVT::i64: + OpBytes = 1; + OS << "OPC_EmitInteger" << MVT(VT).getScalarSizeInBits() << ", "; + break; + default: + OpBytes = 2; + OS << "OPC_EmitInteger, " << getEnumName(VT) << ", "; + break; + } + unsigned Bytes = OpBytes + EmitSignedVBRValue(Val, OS); OS << '\n'; return Bytes; } case Matcher::EmitStringInteger: { const std::string &Val = cast<EmitStringIntegerMatcher>(N)->getValue(); + MVT::SimpleValueType VT = cast<EmitStringIntegerMatcher>(N)->getVT(); // These should always fit into 7 bits. - OS << "OPC_EmitStringInteger, " - << getEnumName(cast<EmitStringIntegerMatcher>(N)->getVT()) << ", " << Val - << ",\n"; - return 3; + unsigned OpBytes; + switch (VT) { + case MVT::i32: + OpBytes = 1; + OS << "OPC_EmitStringInteger" << MVT(VT).getScalarSizeInBits() << ", "; + break; + default: + OpBytes = 2; + OS << "OPC_EmitStringInteger, " << getEnumName(VT) << ", "; + break; + } + OS << Val << ",\n"; + return OpBytes + 1; } case Matcher::EmitRegister: { @@ -1033,45 +1060,80 @@ void MatcherTableEmitter::EmitPredicateFunctions(raw_ostream &OS) { static StringRef getOpcodeString(Matcher::KindTy Kind) { switch (Kind) { - case Matcher::Scope: return "OPC_Scope"; break; - case Matcher::RecordNode: return "OPC_RecordNode"; break; - case Matcher::RecordChild: return "OPC_RecordChild"; break; - case Matcher::RecordMemRef: return "OPC_RecordMemRef"; break; - case Matcher::CaptureGlueInput: return "OPC_CaptureGlueInput"; break; - case Matcher::MoveChild: return "OPC_MoveChild"; break; - case Matcher::MoveParent: return "OPC_MoveParent"; break; - case Matcher::CheckSame: return "OPC_CheckSame"; break; - case Matcher::CheckChildSame: return "OPC_CheckChildSame"; break; + case Matcher::Scope: + return "OPC_Scope"; + case Matcher::RecordNode: + return "OPC_RecordNode"; + case Matcher::RecordChild: + return "OPC_RecordChild"; + case Matcher::RecordMemRef: + return "OPC_RecordMemRef"; + case Matcher::CaptureGlueInput: + return "OPC_CaptureGlueInput"; + case Matcher::MoveChild: + return "OPC_MoveChild"; + case Matcher::MoveParent: + return "OPC_MoveParent"; + case Matcher::CheckSame: + return "OPC_CheckSame"; + case Matcher::CheckChildSame: + return "OPC_CheckChildSame"; case Matcher::CheckPatternPredicate: - return "OPC_CheckPatternPredicate"; break; - case Matcher::CheckPredicate: return "OPC_CheckPredicate"; break; - case Matcher::CheckOpcode: return "OPC_CheckOpcode"; break; - case Matcher::SwitchOpcode: return "OPC_SwitchOpcode"; break; - case Matcher::CheckType: return "OPC_CheckType"; break; - case Matcher::SwitchType: return "OPC_SwitchType"; break; - case Matcher::CheckChildType: return "OPC_CheckChildType"; break; - case Matcher::CheckInteger: return "OPC_CheckInteger"; break; - case Matcher::CheckChildInteger: return "OPC_CheckChildInteger"; break; - case Matcher::CheckCondCode: return "OPC_CheckCondCode"; break; - case Matcher::CheckChild2CondCode: return "OPC_CheckChild2CondCode"; break; - case Matcher::CheckValueType: return "OPC_CheckValueType"; break; - case Matcher::CheckComplexPat: return "OPC_CheckComplexPat"; break; - case Matcher::CheckAndImm: return "OPC_CheckAndImm"; break; - case Matcher::CheckOrImm: return "OPC_CheckOrImm"; break; + return "OPC_CheckPatternPredicate"; + case Matcher::CheckPredicate: + return "OPC_CheckPredicate"; + case Matcher::CheckOpcode: + return "OPC_CheckOpcode"; + case Matcher::SwitchOpcode: + return "OPC_SwitchOpcode"; + case Matcher::CheckType: + return "OPC_CheckType"; + case Matcher::SwitchType: + return "OPC_SwitchType"; + case Matcher::CheckChildType: + return "OPC_CheckChildType"; + case Matcher::CheckInteger: + return "OPC_CheckInteger"; + case Matcher::CheckChildInteger: + return "OPC_CheckChildInteger"; + case Matcher::CheckCondCode: + return "OPC_CheckCondCode"; + case Matcher::CheckChild2CondCode: + return "OPC_CheckChild2CondCode"; + case Matcher::CheckValueType: + return "OPC_CheckValueType"; + case Matcher::CheckComplexPat: + return "OPC_CheckComplexPat"; + case Matcher::CheckAndImm: + return "OPC_CheckAndImm"; + case Matcher::CheckOrImm: + return "OPC_CheckOrImm"; case Matcher::CheckFoldableChainNode: - return "OPC_CheckFoldableChainNode"; break; - case Matcher::CheckImmAllOnesV: return "OPC_CheckImmAllOnesV"; break; - case Matcher::CheckImmAllZerosV: return "OPC_CheckImmAllZerosV"; break; - case Matcher::EmitInteger: return "OPC_EmitInteger"; break; - case Matcher::EmitStringInteger: return "OPC_EmitStringInteger"; break; - case Matcher::EmitRegister: return "OPC_EmitRegister"; break; - case Matcher::EmitConvertToTarget: return "OPC_EmitConvertToTarget"; break; - case Matcher::EmitMergeInputChains: return "OPC_EmitMergeInputChains"; break; - case Matcher::EmitCopyToReg: return "OPC_EmitCopyToReg"; break; - case Matcher::EmitNode: return "OPC_EmitNode"; break; - case Matcher::MorphNodeTo: return "OPC_MorphNodeTo"; break; - case Matcher::EmitNodeXForm: return "OPC_EmitNodeXForm"; break; - case Matcher::CompleteMatch: return "OPC_CompleteMatch"; break; + return "OPC_CheckFoldableChainNode"; + case Matcher::CheckImmAllOnesV: + return "OPC_CheckImmAllOnesV"; + case Matcher::CheckImmAllZerosV: + return "OPC_CheckImmAllZerosV"; + case Matcher::EmitInteger: + return "OPC_EmitInteger"; + case Matcher::EmitStringInteger: + return "OPC_EmitStringInteger"; + case Matcher::EmitRegister: + return "OPC_EmitRegister"; + case Matcher::EmitConvertToTarget: + return "OPC_EmitConvertToTarget"; + case Matcher::EmitMergeInputChains: + return "OPC_EmitMergeInputChains"; + case Matcher::EmitCopyToReg: + return "OPC_EmitCopyToReg"; + case Matcher::EmitNode: + return "OPC_EmitNode"; + case Matcher::MorphNodeTo: + return "OPC_MorphNodeTo"; + case Matcher::EmitNodeXForm: + return "OPC_EmitNodeXForm"; + case Matcher::CompleteMatch: + return "OPC_CompleteMatch"; } llvm_unreachable("Unhandled opcode?"); diff --git a/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/llvm/utils/TableGen/DAGISelMatcherGen.cpp index f773f7c77a77..d08f57b84b95 100644 --- a/llvm/utils/TableGen/DAGISelMatcherGen.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherGen.cpp @@ -710,7 +710,7 @@ void MatcherGen::EmitResultLeafAsOperand(const TreePatternNode *N, const CodeGenRegisterClass &RC = CGP.getTargetInfo().getRegisterClass(Def); if (RC.EnumValue <= 127) { - std::string Value = getQualifiedName(Def) + "RegClassID"; + std::string Value = RC.getQualifiedIdName(); AddMatcher(new EmitStringIntegerMatcher(Value, MVT::i32)); ResultOps.push_back(NextRecordedOperandNo++); } else { diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp index b294c66007f8..a199463961be 100644 --- a/llvm/utils/TableGen/DXILEmitter.cpp +++ b/llvm/utils/TableGen/DXILEmitter.cpp @@ -81,7 +81,7 @@ struct DXILOperationData { if (R->getValue("llvm_intrinsic")) { auto *IntrinsicDef = R->getValueAsDef("llvm_intrinsic"); auto DefName = IntrinsicDef->getName(); - assert(DefName.startswith("int_") && "invalid intrinsic name"); + assert(DefName.starts_with("int_") && "invalid intrinsic name"); // Remove the int_ from intrinsic name. Intrinsic = DefName.substr(4); } diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp index 67033c6290ca..b6aee665f8ee 100644 --- a/llvm/utils/TableGen/DirectiveEmitter.cpp +++ b/llvm/utils/TableGen/DirectiveEmitter.cpp @@ -736,6 +736,7 @@ static void GenerateFlangClausesParser(const DirectiveLanguage &DirLang, .Case("Name", "name") .Case("ScalarIntConstantExpr", "scalarIntConstantExpr") .Case("ScalarIntExpr", "scalarIntExpr") + .Case("ScalarExpr", "scalarExpr") .Case("ScalarLogicalExpr", "scalarLogicalExpr") .Default(("Parser<" + Clause.getFlangClass() + ">{}") .toStringRef(Scratch)); diff --git a/llvm/utils/TableGen/FastISelEmitter.cpp b/llvm/utils/TableGen/FastISelEmitter.cpp index 3f3a63de0c0c..b773a6b91ee2 100644 --- a/llvm/utils/TableGen/FastISelEmitter.cpp +++ b/llvm/utils/TableGen/FastISelEmitter.cpp @@ -26,6 +26,7 @@ #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" +#include <set> #include <utility> using namespace llvm; diff --git a/llvm/utils/TableGen/GICombinerEmitter.cpp b/llvm/utils/TableGen/GICombinerEmitter.cpp deleted file mode 100644 index ec26024b6518..000000000000 --- a/llvm/utils/TableGen/GICombinerEmitter.cpp +++ /dev/null @@ -1,1044 +0,0 @@ -//===- GlobalCombinerEmitter.cpp - Generate a combiner --------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -/// \file Generate a combiner implementation for GlobalISel from a declarative -/// syntax -/// -//===----------------------------------------------------------------------===// - -#include "CodeGenTarget.h" -#include "GlobalISel/CodeExpander.h" -#include "GlobalISel/CodeExpansions.h" -#include "GlobalISel/CombinerUtils.h" -#include "GlobalISel/GIMatchDag.h" -#include "GlobalISel/GIMatchDagEdge.h" -#include "GlobalISel/GIMatchDagInstr.h" -#include "GlobalISel/GIMatchDagOperands.h" -#include "GlobalISel/GIMatchDagPredicate.h" -#include "GlobalISel/GIMatchTree.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/StringSet.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ScopedPrinter.h" -#include "llvm/TableGen/Error.h" -#include "llvm/TableGen/Record.h" -#include "llvm/TableGen/StringMatcher.h" -#include "llvm/TableGen/TableGenBackend.h" -#include <cstdint> - -using namespace llvm; - -#define DEBUG_TYPE "gicombiner-emitter" - -// FIXME: Use ALWAYS_ENABLED_STATISTIC once it's available. -unsigned NumPatternTotal = 0; -STATISTIC(NumPatternTotalStatistic, "Total number of patterns"); - -cl::OptionCategory - GICombinerEmitterCat("Options for -gen-global-isel-combiner"); -cl::list<std::string> - SelectedCombiners("combiners", cl::desc("Emit the specified combiners"), - cl::cat(GICombinerEmitterCat), cl::CommaSeparated); -static cl::opt<bool> ShowExpansions( - "gicombiner-show-expansions", - cl::desc("Use C++ comments to indicate occurence of code expansion"), - cl::cat(GICombinerEmitterCat)); -cl::opt<bool> StopAfterParse( - "gicombiner-stop-after-parse", - cl::desc("Stop processing after parsing rules and dump state"), - cl::cat(GICombinerEmitterCat)); -static cl::opt<bool> StopAfterBuild( - "gicombiner-stop-after-build", - cl::desc("Stop processing after building the match tree"), - cl::cat(GICombinerEmitterCat)); - -namespace { -typedef uint64_t RuleID; - -// We're going to be referencing the same small strings quite a lot for operand -// names and the like. Make their lifetime management simple with a global -// string table. -StringSet<> StrTab; - -StringRef insertStrTab(StringRef S) { - if (S.empty()) - return S; - return StrTab.insert(S).first->first(); -} - -class format_partition_name { - const GIMatchTree &Tree; - unsigned Idx; - -public: - format_partition_name(const GIMatchTree &Tree, unsigned Idx) - : Tree(Tree), Idx(Idx) {} - void print(raw_ostream &OS) const { - Tree.getPartitioner()->emitPartitionName(OS, Idx); - } -}; -raw_ostream &operator<<(raw_ostream &OS, const format_partition_name &Fmt) { - Fmt.print(OS); - return OS; -} - -/// Declares data that is passed from the match stage to the apply stage. -class MatchDataInfo { - /// The symbol used in the tablegen patterns - StringRef PatternSymbol; - /// The data type for the variable - StringRef Type; - /// The name of the variable as declared in the generated matcher. - std::string VariableName; - -public: - MatchDataInfo(StringRef PatternSymbol, StringRef Type, StringRef VariableName) - : PatternSymbol(PatternSymbol), Type(Type), VariableName(VariableName) {} - - StringRef getPatternSymbol() const { return PatternSymbol; }; - StringRef getType() const { return Type; }; - StringRef getVariableName() const { return VariableName; }; -}; - -class RootInfo { - StringRef PatternSymbol; - -public: - RootInfo(StringRef PatternSymbol) : PatternSymbol(PatternSymbol) {} - - StringRef getPatternSymbol() const { return PatternSymbol; } -}; - -class CombineRule { -public: - - using const_matchdata_iterator = std::vector<MatchDataInfo>::const_iterator; - - struct VarInfo { - const GIMatchDagInstr *N; - const GIMatchDagOperand *Op; - const DagInit *Matcher; - - public: - VarInfo(const GIMatchDagInstr *N, const GIMatchDagOperand *Op, - const DagInit *Matcher) - : N(N), Op(Op), Matcher(Matcher) {} - }; - -protected: - /// A unique ID for this rule - /// ID's are used for debugging and run-time disabling of rules among other - /// things. - RuleID ID; - - /// A unique ID that can be used for anonymous objects belonging to this rule. - /// Used to create unique names in makeNameForAnon*() without making tests - /// overly fragile. - unsigned UID = 0; - - /// The record defining this rule. - const Record &TheDef; - - /// The roots of a match. These are the leaves of the DAG that are closest to - /// the end of the function. I.e. the nodes that are encountered without - /// following any edges of the DAG described by the pattern as we work our way - /// from the bottom of the function to the top. - std::vector<RootInfo> Roots; - - GIMatchDag MatchDag; - - /// A block of arbitrary C++ to finish testing the match. - /// FIXME: This is a temporary measure until we have actual pattern matching - const StringInit *MatchingFixupCode = nullptr; - - /// The MatchData defined by the match stage and required by the apply stage. - /// This allows the plumbing of arbitrary data from C++ predicates between the - /// stages. - /// - /// For example, suppose you have: - /// %A = <some-constant-expr> - /// %0 = G_ADD %1, %A - /// you could define a GIMatchPredicate that walks %A, constant folds as much - /// as possible and returns an APInt containing the discovered constant. You - /// could then declare: - /// def apint : GIDefMatchData<"APInt">; - /// add it to the rule with: - /// (defs root:$root, apint:$constant) - /// evaluate it in the pattern with a C++ function that takes a - /// MachineOperand& and an APInt& with: - /// (match [{MIR %root = G_ADD %0, %A }], - /// (constantfold operand:$A, apint:$constant)) - /// and finally use it in the apply stage with: - /// (apply (create_operand - /// [{ MachineOperand::CreateImm(${constant}.getZExtValue()); - /// ]}, apint:$constant), - /// [{MIR %root = FOO %0, %constant }]) - std::vector<MatchDataInfo> MatchDataDecls; - - void declareMatchData(StringRef PatternSymbol, StringRef Type, - StringRef VarName); - - bool parseInstructionMatcher(const CodeGenTarget &Target, StringInit *ArgName, - const Init &Arg, - StringMap<std::vector<VarInfo>> &NamedEdgeDefs, - StringMap<std::vector<VarInfo>> &NamedEdgeUses); - bool parseWipMatchOpcodeMatcher(const CodeGenTarget &Target, - StringInit *ArgName, const Init &Arg); - -public: - CombineRule(const CodeGenTarget &Target, GIMatchDagContext &Ctx, RuleID ID, - const Record &R) - : ID(ID), TheDef(R), MatchDag(Ctx) {} - CombineRule(const CombineRule &) = delete; - - bool parseDefs(); - bool parseMatcher(const CodeGenTarget &Target); - - RuleID getID() const { return ID; } - unsigned allocUID() { return UID++; } - StringRef getName() const { return TheDef.getName(); } - const Record &getDef() const { return TheDef; } - const StringInit *getMatchingFixupCode() const { return MatchingFixupCode; } - size_t getNumRoots() const { return Roots.size(); } - - GIMatchDag &getMatchDag() { return MatchDag; } - const GIMatchDag &getMatchDag() const { return MatchDag; } - - using const_root_iterator = std::vector<RootInfo>::const_iterator; - const_root_iterator roots_begin() const { return Roots.begin(); } - const_root_iterator roots_end() const { return Roots.end(); } - iterator_range<const_root_iterator> roots() const { - return llvm::make_range(Roots.begin(), Roots.end()); - } - - iterator_range<const_matchdata_iterator> matchdata_decls() const { - return make_range(MatchDataDecls.begin(), MatchDataDecls.end()); - } - - /// Export expansions for this rule - void declareExpansions(CodeExpansions &Expansions) const { - for (const auto &I : matchdata_decls()) - Expansions.declare(I.getPatternSymbol(), I.getVariableName()); - } - - /// The matcher will begin from the roots and will perform the match by - /// traversing the edges to cover the whole DAG. This function reverses DAG - /// edges such that everything is reachable from a root. This is part of the - /// preparation work for flattening the DAG into a tree. - void reorientToRoots() { - SmallSet<const GIMatchDagInstr *, 5> Roots; - SmallSet<const GIMatchDagInstr *, 5> Visited; - SmallSet<GIMatchDagEdge *, 20> EdgesRemaining; - - for (auto &I : MatchDag.roots()) { - Roots.insert(I); - Visited.insert(I); - } - for (auto &I : MatchDag.edges()) - EdgesRemaining.insert(I); - - bool Progressed = false; - SmallSet<GIMatchDagEdge *, 20> EdgesToRemove; - while (!EdgesRemaining.empty()) { - for (auto *EI : EdgesRemaining) { - if (Visited.count(EI->getFromMI())) { - if (Roots.count(EI->getToMI())) - PrintError(TheDef.getLoc(), "One or more roots are unnecessary"); - Visited.insert(EI->getToMI()); - EdgesToRemove.insert(EI); - Progressed = true; - } - } - for (GIMatchDagEdge *ToRemove : EdgesToRemove) - EdgesRemaining.erase(ToRemove); - EdgesToRemove.clear(); - - for (auto EI = EdgesRemaining.begin(), EE = EdgesRemaining.end(); - EI != EE; ++EI) { - if (Visited.count((*EI)->getToMI())) { - (*EI)->reverse(); - Visited.insert((*EI)->getToMI()); - EdgesToRemove.insert(*EI); - Progressed = true; - } - for (GIMatchDagEdge *ToRemove : EdgesToRemove) - EdgesRemaining.erase(ToRemove); - EdgesToRemove.clear(); - } - - if (!Progressed) { - LLVM_DEBUG(dbgs() << "No progress\n"); - return; - } - Progressed = false; - } - } -}; - -StringRef makeNameForAnonInstr(CombineRule &Rule) { - return insertStrTab(to_string( - format("__anon%" PRIu64 "_%u", Rule.getID(), Rule.allocUID()))); -} - -StringRef makeDebugName(CombineRule &Rule, StringRef Name) { - return insertStrTab(Name.empty() ? makeNameForAnonInstr(Rule) : StringRef(Name)); -} - -StringRef makeNameForAnonPredicate(CombineRule &Rule) { - return insertStrTab(to_string( - format("__anonpred%" PRIu64 "_%u", Rule.getID(), Rule.allocUID()))); -} - -void CombineRule::declareMatchData(StringRef PatternSymbol, StringRef Type, - StringRef VarName) { - MatchDataDecls.emplace_back(PatternSymbol, Type, VarName); -} - -bool CombineRule::parseDefs() { - DagInit *Defs = TheDef.getValueAsDag("Defs"); - - if (Defs->getOperatorAsDef(TheDef.getLoc())->getName() != "defs") { - PrintError(TheDef.getLoc(), "Expected defs operator"); - return false; - } - - for (unsigned I = 0, E = Defs->getNumArgs(); I < E; ++I) { - // Roots should be collected into Roots - if (isSpecificDef(*Defs->getArg(I), "root")) { - Roots.emplace_back(Defs->getArgNameStr(I)); - continue; - } - - // Subclasses of GIDefMatchData should declare that this rule needs to pass - // data from the match stage to the apply stage, and ensure that the - // generated matcher has a suitable variable for it to do so. - if (Record *MatchDataRec = - getDefOfSubClass(*Defs->getArg(I), "GIDefMatchData")) { - declareMatchData(Defs->getArgNameStr(I), - MatchDataRec->getValueAsString("Type"), - llvm::to_string(llvm::format("MatchData%" PRIu64, ID))); - continue; - } - - // Otherwise emit an appropriate error message. - if (getDefOfSubClass(*Defs->getArg(I), "GIDefKind")) - PrintError(TheDef.getLoc(), - "This GIDefKind not implemented in tablegen"); - else if (getDefOfSubClass(*Defs->getArg(I), "GIDefKindWithArgs")) - PrintError(TheDef.getLoc(), - "This GIDefKindWithArgs not implemented in tablegen"); - else - PrintError(TheDef.getLoc(), - "Expected a subclass of GIDefKind or a sub-dag whose " - "operator is of type GIDefKindWithArgs"); - return false; - } - - if (Roots.empty()) { - PrintError(TheDef.getLoc(), "Combine rules must have at least one root"); - return false; - } - return true; -} - -// Parse an (Instruction $a:Arg1, $b:Arg2, ...) matcher. Edges are formed -// between matching operand names between different matchers. -bool CombineRule::parseInstructionMatcher( - const CodeGenTarget &Target, StringInit *ArgName, const Init &Arg, - StringMap<std::vector<VarInfo>> &NamedEdgeDefs, - StringMap<std::vector<VarInfo>> &NamedEdgeUses) { - if (const DagInit *Matcher = - getDagWithOperatorOfSubClass(Arg, "Instruction")) { - auto &Instr = - Target.getInstruction(Matcher->getOperatorAsDef(TheDef.getLoc())); - - StringRef Name = ArgName ? ArgName->getValue() : ""; - - GIMatchDagInstr *N = - MatchDag.addInstrNode(makeDebugName(*this, Name), insertStrTab(Name), - MatchDag.getContext().makeOperandList(Instr)); - - N->setOpcodeAnnotation(&Instr); - const auto &P = MatchDag.addPredicateNode<GIMatchDagOpcodePredicate>( - makeNameForAnonPredicate(*this), Instr); - MatchDag.addPredicateDependency(N, nullptr, P, &P->getOperandInfo()["mi"]); - unsigned OpIdx = 0; - for (const auto &NameInit : Matcher->getArgNames()) { - StringRef Name = insertStrTab(NameInit->getAsUnquotedString()); - if (Name.empty()) - continue; - N->assignNameToOperand(OpIdx, Name); - - // Record the endpoints of any named edges. We'll add the cartesian - // product of edges later. - const auto &InstrOperand = N->getOperandInfo()[OpIdx]; - if (InstrOperand.isDef()) { - NamedEdgeDefs.try_emplace(Name); - NamedEdgeDefs[Name].emplace_back(N, &InstrOperand, Matcher); - } else { - NamedEdgeUses.try_emplace(Name); - NamedEdgeUses[Name].emplace_back(N, &InstrOperand, Matcher); - } - - if (InstrOperand.isDef()) { - if (any_of(Roots, [&](const RootInfo &X) { - return X.getPatternSymbol() == Name; - })) { - N->setMatchRoot(); - } - } - - OpIdx++; - } - - return true; - } - return false; -} - -// Parse the wip_match_opcode placeholder that's temporarily present in lieu of -// implementing macros or choices between two matchers. -bool CombineRule::parseWipMatchOpcodeMatcher(const CodeGenTarget &Target, - StringInit *ArgName, - const Init &Arg) { - if (const DagInit *Matcher = - getDagWithSpecificOperator(Arg, "wip_match_opcode")) { - StringRef Name = ArgName ? ArgName->getValue() : ""; - - GIMatchDagInstr *N = - MatchDag.addInstrNode(makeDebugName(*this, Name), insertStrTab(Name), - MatchDag.getContext().makeEmptyOperandList()); - - if (any_of(Roots, [&](const RootInfo &X) { - return ArgName && X.getPatternSymbol() == ArgName->getValue(); - })) { - N->setMatchRoot(); - } - - const auto &P = MatchDag.addPredicateNode<GIMatchDagOneOfOpcodesPredicate>( - makeNameForAnonPredicate(*this)); - MatchDag.addPredicateDependency(N, nullptr, P, &P->getOperandInfo()["mi"]); - // Each argument is an opcode that will pass this predicate. Add them all to - // the predicate implementation - for (const auto &Arg : Matcher->getArgs()) { - Record *OpcodeDef = getDefOfSubClass(*Arg, "Instruction"); - if (OpcodeDef) { - P->addOpcode(&Target.getInstruction(OpcodeDef)); - continue; - } - PrintError(TheDef.getLoc(), - "Arguments to wip_match_opcode must be instructions"); - return false; - } - return true; - } - return false; -} -bool CombineRule::parseMatcher(const CodeGenTarget &Target) { - StringMap<std::vector<VarInfo>> NamedEdgeDefs; - StringMap<std::vector<VarInfo>> NamedEdgeUses; - DagInit *Matchers = TheDef.getValueAsDag("Match"); - - if (Matchers->getOperatorAsDef(TheDef.getLoc())->getName() != "match") { - PrintError(TheDef.getLoc(), "Expected match operator"); - return false; - } - - if (Matchers->getNumArgs() == 0) { - PrintError(TheDef.getLoc(), "Matcher is empty"); - return false; - } - - // The match section consists of a list of matchers and predicates. Parse each - // one and add the equivalent GIMatchDag nodes, predicates, and edges. - for (unsigned I = 0; I < Matchers->getNumArgs(); ++I) { - if (parseInstructionMatcher(Target, Matchers->getArgName(I), - *Matchers->getArg(I), NamedEdgeDefs, - NamedEdgeUses)) - continue; - - if (parseWipMatchOpcodeMatcher(Target, Matchers->getArgName(I), - *Matchers->getArg(I))) - continue; - - - // Parse arbitrary C++ code we have in lieu of supporting MIR matching - if (const StringInit *StringI = dyn_cast<StringInit>(Matchers->getArg(I))) { - assert(!MatchingFixupCode && - "Only one block of arbitrary code is currently permitted"); - MatchingFixupCode = StringI; - MatchDag.setHasPostMatchPredicate(true); - continue; - } - - PrintError(TheDef.getLoc(), - "Expected a subclass of GIMatchKind or a sub-dag whose " - "operator is either of a GIMatchKindWithArgs or Instruction"); - PrintNote("Pattern was `" + Matchers->getArg(I)->getAsString() + "'"); - return false; - } - - // Add the cartesian product of use -> def edges. - bool FailedToAddEdges = false; - for (const auto &NameAndDefs : NamedEdgeDefs) { - if (NameAndDefs.getValue().size() > 1) { - PrintError(TheDef.getLoc(), - "Two different MachineInstrs cannot def the same vreg"); - for (const auto &NameAndDefOp : NameAndDefs.getValue()) - PrintNote("in " + to_string(*NameAndDefOp.N) + " created from " + - to_string(*NameAndDefOp.Matcher) + ""); - FailedToAddEdges = true; - } - const auto &Uses = NamedEdgeUses[NameAndDefs.getKey()]; - for (const VarInfo &DefVar : NameAndDefs.getValue()) { - for (const VarInfo &UseVar : Uses) { - MatchDag.addEdge(insertStrTab(NameAndDefs.getKey()), UseVar.N, UseVar.Op, - DefVar.N, DefVar.Op); - } - } - } - if (FailedToAddEdges) - return false; - - // If a variable is referenced in multiple use contexts then we need a - // predicate to confirm they are the same operand. We can elide this if it's - // also referenced in a def context and we're traversing the def-use chain - // from the def to the uses but we can't know which direction we're going - // until after reorientToRoots(). - for (const auto &NameAndUses : NamedEdgeUses) { - const auto &Uses = NameAndUses.getValue(); - if (Uses.size() > 1) { - const auto &LeadingVar = Uses.front(); - for (const auto &Var : ArrayRef<VarInfo>(Uses).drop_front()) { - // Add a predicate for each pair until we've covered the whole - // equivalence set. We could test the whole set in a single predicate - // but that means we can't test any equivalence until all the MO's are - // available which can lead to wasted work matching the DAG when this - // predicate can already be seen to have failed. - // - // We have a similar problem due to the need to wait for a particular MO - // before being able to test any of them. However, that is mitigated by - // the order in which we build the DAG. We build from the roots outwards - // so by using the first recorded use in all the predicates, we are - // making the dependency on one of the earliest visited references in - // the DAG. It's not guaranteed once the generated matcher is optimized - // (because the factoring the common portions of rules might change the - // visit order) but this should mean that these predicates depend on the - // first MO to become available. - const auto &P = MatchDag.addPredicateNode<GIMatchDagSameMOPredicate>( - makeNameForAnonPredicate(*this)); - MatchDag.addPredicateDependency(LeadingVar.N, LeadingVar.Op, P, - &P->getOperandInfo()["mi0"]); - MatchDag.addPredicateDependency(Var.N, Var.Op, P, - &P->getOperandInfo()["mi1"]); - } - } - } - return true; -} - -class GICombinerEmitter { - RecordKeeper &Records; - StringRef Name; - const CodeGenTarget &Target; - Record *Combiner; - std::vector<std::unique_ptr<CombineRule>> Rules; - GIMatchDagContext MatchDagCtx; - - std::unique_ptr<CombineRule> makeCombineRule(const Record &R); - - void gatherRules(std::vector<std::unique_ptr<CombineRule>> &ActiveRules, - const std::vector<Record *> &&RulesAndGroups); - -public: - explicit GICombinerEmitter(RecordKeeper &RK, const CodeGenTarget &Target, - StringRef Name, Record *Combiner); - ~GICombinerEmitter() {} - - StringRef getClassName() const { - return Combiner->getValueAsString("Classname"); - } - void run(raw_ostream &OS); - - /// Emit the name matcher (guarded by #ifndef NDEBUG) used to disable rules in - /// response to the generated cl::opt. - void emitNameMatcher(raw_ostream &OS) const; - - void generateCodeForTree(raw_ostream &OS, const GIMatchTree &Tree, - StringRef Indent) const; -}; - -GICombinerEmitter::GICombinerEmitter(RecordKeeper &RK, - const CodeGenTarget &Target, - StringRef Name, Record *Combiner) - : Records(RK), Name(Name), Target(Target), Combiner(Combiner) {} - -void GICombinerEmitter::emitNameMatcher(raw_ostream &OS) const { - std::vector<std::pair<std::string, std::string>> Cases; - Cases.reserve(Rules.size()); - - for (const CombineRule &EnumeratedRule : make_pointee_range(Rules)) { - std::string Code; - raw_string_ostream SS(Code); - SS << "return " << EnumeratedRule.getID() << ";\n"; - Cases.push_back( - std::make_pair(std::string(EnumeratedRule.getName()), Code)); - } - - OS << "static std::optional<uint64_t> getRuleIdxForIdentifier(StringRef " - "RuleIdentifier) {\n" - << " uint64_t I;\n" - << " // getAtInteger(...) returns false on success\n" - << " bool Parsed = !RuleIdentifier.getAsInteger(0, I);\n" - << " if (Parsed)\n" - << " return I;\n\n" - << "#ifndef NDEBUG\n"; - StringMatcher Matcher("RuleIdentifier", Cases, OS); - Matcher.Emit(); - OS << "#endif // ifndef NDEBUG\n\n" - << " return std::nullopt;\n" - << "}\n"; -} - -std::unique_ptr<CombineRule> -GICombinerEmitter::makeCombineRule(const Record &TheDef) { - std::unique_ptr<CombineRule> Rule = - std::make_unique<CombineRule>(Target, MatchDagCtx, NumPatternTotal, TheDef); - - if (!Rule->parseDefs()) - return nullptr; - if (!Rule->parseMatcher(Target)) - return nullptr; - - Rule->reorientToRoots(); - - LLVM_DEBUG({ - dbgs() << "Parsed rule defs/match for '" << Rule->getName() << "'\n"; - Rule->getMatchDag().dump(); - Rule->getMatchDag().writeDOTGraph(dbgs(), Rule->getName()); - }); - if (StopAfterParse) - return Rule; - - // For now, don't support traversing from def to use. We'll come back to - // this later once we have the algorithm changes to support it. - bool EmittedDefToUseError = false; - for (const auto &E : Rule->getMatchDag().edges()) { - if (E->isDefToUse()) { - if (!EmittedDefToUseError) { - PrintError( - TheDef.getLoc(), - "Generated state machine cannot lookup uses from a def (yet)"); - EmittedDefToUseError = true; - } - PrintNote("Node " + to_string(*E->getFromMI())); - PrintNote("Node " + to_string(*E->getToMI())); - PrintNote("Edge " + to_string(*E)); - } - } - if (EmittedDefToUseError) - return nullptr; - - // For now, don't support multi-root rules. We'll come back to this later - // once we have the algorithm changes to support it. - if (Rule->getNumRoots() > 1) { - PrintError(TheDef.getLoc(), "Multi-root matches are not supported (yet)"); - return nullptr; - } - return Rule; -} - -/// Recurse into GICombineGroup's and flatten the ruleset into a simple list. -void GICombinerEmitter::gatherRules( - std::vector<std::unique_ptr<CombineRule>> &ActiveRules, - const std::vector<Record *> &&RulesAndGroups) { - for (Record *R : RulesAndGroups) { - if (R->isValueUnset("Rules")) { - std::unique_ptr<CombineRule> Rule = makeCombineRule(*R); - if (Rule == nullptr) { - PrintError(R->getLoc(), "Failed to parse rule"); - continue; - } - ActiveRules.emplace_back(std::move(Rule)); - ++NumPatternTotal; - } else - gatherRules(ActiveRules, R->getValueAsListOfDefs("Rules")); - } -} - -void GICombinerEmitter::generateCodeForTree(raw_ostream &OS, - const GIMatchTree &Tree, - StringRef Indent) const { - if (Tree.getPartitioner() != nullptr) { - Tree.getPartitioner()->generatePartitionSelectorCode(OS, Indent); - for (const auto &EnumChildren : enumerate(Tree.children())) { - OS << Indent << "if (Partition == " << EnumChildren.index() << " /* " - << format_partition_name(Tree, EnumChildren.index()) << " */) {\n"; - generateCodeForTree(OS, EnumChildren.value(), (Indent + " ").str()); - OS << Indent << "}\n"; - } - return; - } - - bool AnyFullyTested = false; - for (const auto &Leaf : Tree.possible_leaves()) { - OS << Indent << "// Leaf name: " << Leaf.getName() << "\n"; - - const CombineRule *Rule = Leaf.getTargetData<CombineRule>(); - const Record &RuleDef = Rule->getDef(); - - OS << Indent << "// Rule: " << RuleDef.getName() << "\n" - << Indent << "if (!RuleConfig->isRuleDisabled(" << Rule->getID() - << ")) {\n"; - - CodeExpansions Expansions; - for (const auto &VarBinding : Leaf.var_bindings()) { - if (VarBinding.isInstr()) - Expansions.declare(VarBinding.getName(), - "MIs[" + to_string(VarBinding.getInstrID()) + "]"); - else - Expansions.declare(VarBinding.getName(), - "MIs[" + to_string(VarBinding.getInstrID()) + - "]->getOperand(" + - to_string(VarBinding.getOpIdx()) + ")"); - } - Rule->declareExpansions(Expansions); - - DagInit *Applyer = RuleDef.getValueAsDag("Apply"); - if (Applyer->getOperatorAsDef(RuleDef.getLoc())->getName() != - "apply") { - PrintError(RuleDef.getLoc(), "Expected 'apply' operator in Apply DAG"); - return; - } - - OS << Indent << " if (1\n"; - - // Emit code for C++ Predicates. - if (RuleDef.getValue("Predicates")) { - ListInit *Preds = RuleDef.getValueAsListInit("Predicates"); - for (Init *I : Preds->getValues()) { - if (DefInit *Pred = dyn_cast<DefInit>(I)) { - Record *Def = Pred->getDef(); - if (!Def->isSubClassOf("Predicate")) { - PrintError(Def->getLoc(), "Unknown 'Predicate' Type"); - return; - } - - StringRef CondString = Def->getValueAsString("CondString"); - if (CondString.empty()) - continue; - - OS << Indent << " && (\n" - << Indent << " // Predicate: " << Def->getName() << "\n" - << Indent << " " << CondString << "\n" - << Indent << " )\n"; - } - } - } - - // Attempt to emit code for any untested predicates left over. Note that - // isFullyTested() will remain false even if we succeed here and therefore - // combine rule elision will not be performed. This is because we do not - // know if there's any connection between the predicates for each leaf and - // therefore can't tell if one makes another unreachable. Ideally, the - // partitioner(s) would be sufficiently complete to prevent us from having - // untested predicates left over. - for (const GIMatchDagPredicate *Predicate : Leaf.untested_predicates()) { - if (Predicate->generateCheckCode(OS, (Indent + " ").str(), - Expansions)) - continue; - PrintError(RuleDef.getLoc(), - "Unable to test predicate used in rule"); - PrintNote(SMLoc(), - "This indicates an incomplete implementation in tablegen"); - Predicate->print(errs()); - errs() << "\n"; - OS << Indent - << "llvm_unreachable(\"TableGen did not emit complete code for this " - "path\");\n"; - break; - } - - if (Rule->getMatchingFixupCode() && - !Rule->getMatchingFixupCode()->getValue().empty()) { - // FIXME: Single-use lambda's like this are a serious compile-time - // performance and memory issue. It's convenient for this early stage to - // defer some work to successive patches but we need to eliminate this - // before the ruleset grows to small-moderate size. Last time, it became - // a big problem for low-mem systems around the 500 rule mark but by the - // time we grow that large we should have merged the ISel match table - // mechanism with the Combiner. - OS << Indent << " && [&]() {\n" - << Indent << " " - << CodeExpander(Rule->getMatchingFixupCode()->getValue(), Expansions, - RuleDef.getLoc(), ShowExpansions) - << '\n' - << Indent << " return true;\n" - << Indent << " }()"; - } - OS << Indent << " ) {\n" << Indent << " "; - - if (const StringInit *Code = dyn_cast<StringInit>(Applyer->getArg(0))) { - OS << " LLVM_DEBUG(dbgs() << \"Applying rule '" - << RuleDef.getName() - << "'\\n\");\n" - << CodeExpander(Code->getAsUnquotedString(), Expansions, - RuleDef.getLoc(), ShowExpansions) - << '\n' - << Indent << " return true;\n" - << Indent << " }\n"; - } else { - PrintError(RuleDef.getLoc(), "Expected apply code block"); - return; - } - - OS << Indent << "}\n"; - - assert(Leaf.isFullyTraversed()); - - // If we didn't have any predicates left over and we're not using the - // trap-door we have to support arbitrary C++ code while we're migrating to - // the declarative style then we know that subsequent leaves are - // unreachable. - if (Leaf.isFullyTested() && - (!Rule->getMatchingFixupCode() || - Rule->getMatchingFixupCode()->getValue().empty())) { - AnyFullyTested = true; - OS << Indent - << "llvm_unreachable(\"Combine rule elision was incorrect\");\n" - << Indent << "return false;\n"; - } - } - if (!AnyFullyTested) - OS << Indent << "return false;\n"; -} - -static void emitAdditionalHelperMethodArguments(raw_ostream &OS, - Record *Combiner) { - for (Record *Arg : Combiner->getValueAsListOfDefs("AdditionalArguments")) - OS << ",\n " << Arg->getValueAsString("Type") - << " " << Arg->getValueAsString("Name"); -} - -void GICombinerEmitter::run(raw_ostream &OS) { - Records.startTimer("Gather rules"); - gatherRules(Rules, Combiner->getValueAsListOfDefs("Rules")); - if (StopAfterParse) { - MatchDagCtx.print(errs()); - PrintNote(Combiner->getLoc(), - "Terminating due to -gicombiner-stop-after-parse"); - return; - } - if (ErrorsPrinted) - PrintFatalError(Combiner->getLoc(), "Failed to parse one or more rules"); - LLVM_DEBUG(dbgs() << "Optimizing tree for " << Rules.size() << " rules\n"); - std::unique_ptr<GIMatchTree> Tree; - Records.startTimer("Optimize combiner"); - { - GIMatchTreeBuilder TreeBuilder(0); - for (const auto &Rule : Rules) { - bool HadARoot = false; - for (const auto &Root : enumerate(Rule->getMatchDag().roots())) { - TreeBuilder.addLeaf(Rule->getName(), Root.index(), Rule->getMatchDag(), - Rule.get()); - HadARoot = true; - } - if (!HadARoot) - PrintFatalError(Rule->getDef().getLoc(), "All rules must have a root"); - } - - Tree = TreeBuilder.run(); - } - if (StopAfterBuild) { - Tree->writeDOTGraph(outs()); - PrintNote(Combiner->getLoc(), - "Terminating due to -gicombiner-stop-after-build"); - return; - } - - Records.startTimer("Emit combiner"); - OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_DEPS\n" - << "#include \"llvm/ADT/SparseBitVector.h\"\n" - << "namespace llvm {\n" - << "extern cl::OptionCategory GICombinerOptionCategory;\n" - << "} // end namespace llvm\n" - << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_DEPS\n\n"; - - OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_H\n" - << "class " << getClassName() << "RuleConfig {\n" - << " SparseBitVector<> DisabledRules;\n" - << "\n" - << "public:\n" - << " bool parseCommandLineOption();\n" - << " bool isRuleDisabled(unsigned ID) const;\n" - << " bool setRuleEnabled(StringRef RuleIdentifier);\n" - << " bool setRuleDisabled(StringRef RuleIdentifier);\n" - << "};\n" - << "\n" - << "class " << getClassName(); - StringRef StateClass = Combiner->getValueAsString("StateClass"); - if (!StateClass.empty()) - OS << " : public " << StateClass; - OS << " {\n" - << " const " << getClassName() << "RuleConfig *RuleConfig;\n" - << "\n" - << "public:\n" - << " template <typename... Args>" << getClassName() << "(const " - << getClassName() << "RuleConfig &RuleConfig, Args &&... args) : "; - if (!StateClass.empty()) - OS << StateClass << "(std::forward<Args>(args)...), "; - OS << "RuleConfig(&RuleConfig) {}\n" - << "\n" - << " bool tryCombineAll(\n" - << " GISelChangeObserver &Observer,\n" - << " MachineInstr &MI,\n" - << " MachineIRBuilder &B"; - emitAdditionalHelperMethodArguments(OS, Combiner); - OS << ") const;\n"; - OS << "};\n\n"; - - emitNameMatcher(OS); - - OS << "static std::optional<std::pair<uint64_t, uint64_t>> " - "getRuleRangeForIdentifier(StringRef RuleIdentifier) {\n" - << " std::pair<StringRef, StringRef> RangePair = " - "RuleIdentifier.split('-');\n" - << " if (!RangePair.second.empty()) {\n" - << " const auto First = " - "getRuleIdxForIdentifier(RangePair.first);\n" - << " const auto Last = " - "getRuleIdxForIdentifier(RangePair.second);\n" - << " if (!First || !Last)\n" - << " return std::nullopt;\n" - << " if (First >= Last)\n" - << " report_fatal_error(\"Beginning of range should be before " - "end of range\");\n" - << " return {{*First, *Last + 1}};\n" - << " }\n" - << " if (RangePair.first == \"*\") {\n" - << " return {{0, " << Rules.size() << "}};\n" - << " }\n" - << " const auto I = getRuleIdxForIdentifier(RangePair.first);\n" - << " if (!I)\n" - << " return std::nullopt;\n" - << " return {{*I, *I + 1}};\n" - << "}\n\n"; - - for (bool Enabled : {true, false}) { - OS << "bool " << getClassName() << "RuleConfig::setRule" - << (Enabled ? "Enabled" : "Disabled") << "(StringRef RuleIdentifier) {\n" - << " auto MaybeRange = getRuleRangeForIdentifier(RuleIdentifier);\n" - << " if (!MaybeRange)\n" - << " return false;\n" - << " for (auto I = MaybeRange->first; I < MaybeRange->second; ++I)\n" - << " DisabledRules." << (Enabled ? "reset" : "set") << "(I);\n" - << " return true;\n" - << "}\n\n"; - } - - OS << "bool " << getClassName() - << "RuleConfig::isRuleDisabled(unsigned RuleID) const {\n" - << " return DisabledRules.test(RuleID);\n" - << "}\n"; - OS << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_H\n\n"; - - OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_CPP\n" - << "\n" - << "std::vector<std::string> " << Name << "Option;\n" - << "cl::list<std::string> " << Name << "DisableOption(\n" - << " \"" << Name.lower() << "-disable-rule\",\n" - << " cl::desc(\"Disable one or more combiner rules temporarily in " - << "the " << Name << " pass\"),\n" - << " cl::CommaSeparated,\n" - << " cl::Hidden,\n" - << " cl::cat(GICombinerOptionCategory),\n" - << " cl::callback([](const std::string &Str) {\n" - << " " << Name << "Option.push_back(Str);\n" - << " }));\n" - << "cl::list<std::string> " << Name << "OnlyEnableOption(\n" - << " \"" << Name.lower() << "-only-enable-rule\",\n" - << " cl::desc(\"Disable all rules in the " << Name - << " pass then re-enable the specified ones\"),\n" - << " cl::Hidden,\n" - << " cl::cat(GICombinerOptionCategory),\n" - << " cl::callback([](const std::string &CommaSeparatedArg) {\n" - << " StringRef Str = CommaSeparatedArg;\n" - << " " << Name << "Option.push_back(\"*\");\n" - << " do {\n" - << " auto X = Str.split(\",\");\n" - << " " << Name << "Option.push_back((\"!\" + X.first).str());\n" - << " Str = X.second;\n" - << " } while (!Str.empty());\n" - << " }));\n" - << "\n" - << "bool " << getClassName() << "RuleConfig::parseCommandLineOption() {\n" - << " for (StringRef Identifier : " << Name << "Option) {\n" - << " bool Enabled = Identifier.consume_front(\"!\");\n" - << " if (Enabled && !setRuleEnabled(Identifier))\n" - << " return false;\n" - << " if (!Enabled && !setRuleDisabled(Identifier))\n" - << " return false;\n" - << " }\n" - << " return true;\n" - << "}\n\n"; - - OS << "bool " << getClassName() << "::tryCombineAll(\n" - << " GISelChangeObserver &Observer,\n" - << " MachineInstr &MI,\n" - << " MachineIRBuilder &B"; - emitAdditionalHelperMethodArguments(OS, Combiner); - OS << ") const {\n" - << " MachineBasicBlock *MBB = MI.getParent();\n" - << " MachineFunction *MF = MBB->getParent();\n" - << " MachineRegisterInfo &MRI = MF->getRegInfo();\n" - << " SmallVector<MachineInstr *, 8> MIs = {&MI};\n\n" - << " (void)MBB; (void)MF; (void)MRI; (void)RuleConfig;\n\n"; - - OS << " // Match data\n"; - for (const auto &Rule : Rules) - for (const auto &I : Rule->matchdata_decls()) - OS << " " << I.getType() << " " << I.getVariableName() << ";\n"; - OS << "\n"; - - OS << " int Partition = -1;\n"; - generateCodeForTree(OS, *Tree, " "); - OS << "\n return false;\n" - << "}\n" - << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_CPP\n"; -} - -} // end anonymous namespace - -//===----------------------------------------------------------------------===// - -static void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS) { - PrintWarning( - "'-gen-global-isel-combiner' is deprecated and will be removed soon; " - "please use '-gen-global-isel-combiner-match-table' instead"); - PrintNote( - "See " - "https://discourse.llvm.org/t/rfc-matchtable-based-globalisel-combiners"); - - CodeGenTarget Target(RK); - emitSourceFileHeader("Global Combiner", OS); - - if (SelectedCombiners.empty()) - PrintFatalError("No combiners selected with -combiners"); - for (const auto &Combiner : SelectedCombiners) { - Record *CombinerDef = RK.getDef(Combiner); - if (!CombinerDef) - PrintFatalError("Could not find " + Combiner); - GICombinerEmitter(RK, Target, Combiner, CombinerDef).run(OS); - } - NumPatternTotalStatistic = NumPatternTotal; -} - -static TableGen::Emitter::Opt X("gen-global-isel-combiner", EmitGICombiner, - "Generate GlobalISel combiner"); diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp b/llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp new file mode 100644 index 000000000000..e39293ebfe7a --- /dev/null +++ b/llvm/utils/TableGen/GlobalISel/CXXPredicates.cpp @@ -0,0 +1,51 @@ +//===- CXXPredicates.cpp ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// +//===----------------------------------------------------------------------===// + +#include "CXXPredicates.h" +#include "llvm/ADT/STLExtras.h" + +namespace llvm { +namespace gi { + +std::vector<const CXXPredicateCode *> +CXXPredicateCode::getSorted(const CXXPredicateCodePool &Pool) { + std::vector<const CXXPredicateCode *> Out; + std::transform(Pool.begin(), Pool.end(), std::back_inserter(Out), + [&](auto &Elt) { return Elt.second.get(); }); + sort(Out, [](const auto *A, const auto *B) { return A->ID < B->ID; }); + return Out; +} + +const CXXPredicateCode &CXXPredicateCode::get(CXXPredicateCodePool &Pool, + std::string Code) { + // Check if we already have an identical piece of code, if not, create an + // entry in the pool. + const auto CodeHash = hash_value(Code); + if (auto It = Pool.find(CodeHash); It != Pool.end()) + return *It->second; + + const auto ID = Pool.size(); + auto OwnedData = std::unique_ptr<CXXPredicateCode>( + new CXXPredicateCode(std::move(Code), ID)); + const auto &DataRef = *OwnedData; + Pool[CodeHash] = std::move(OwnedData); + return DataRef; +} + +// TODO: Make BaseEnumName prefix configurable. +CXXPredicateCode::CXXPredicateCode(std::string Code, unsigned ID) + : Code(Code), ID(ID), BaseEnumName("GICombiner" + std::to_string(ID)) {} + +CXXPredicateCode::CXXPredicateCodePool CXXPredicateCode::AllCXXMatchCode; +CXXPredicateCode::CXXPredicateCodePool CXXPredicateCode::AllCXXApplyCode; + +} // namespace gi +} // namespace llvm diff --git a/llvm/utils/TableGen/GlobalISel/CXXPredicates.h b/llvm/utils/TableGen/GlobalISel/CXXPredicates.h new file mode 100644 index 000000000000..01610a13110d --- /dev/null +++ b/llvm/utils/TableGen/GlobalISel/CXXPredicates.h @@ -0,0 +1,86 @@ +//===- CXXPredicates.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Contains utilities related to handling C++ code in MIR patterns for +/// GlobalISel. C++ predicates need to be expanded, and then stored in a +/// static pool until they can be emitted. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_MIRPATTERNS_CXXPREDICATES_H +#define LLVM_UTILS_MIRPATTERNS_CXXPREDICATES_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/StringRef.h" +#include <memory> +#include <string> +#include <vector> + +namespace llvm { +namespace gi { + +/// Entry into the static pool of all CXX Predicate code. This contains +/// fully expanded C++ code. +/// +/// The static pool is hidden inside the object and can be accessed through +/// getAllMatchCode/getAllApplyCode +/// +/// Note that CXXPattern trims C++ code, so the Code is already expected to be +/// free of leading/trailing whitespace. +class CXXPredicateCode { + using CXXPredicateCodePool = + DenseMap<hash_code, std::unique_ptr<CXXPredicateCode>>; + static CXXPredicateCodePool AllCXXMatchCode; + static CXXPredicateCodePool AllCXXApplyCode; + + /// Sorts a `CXXPredicateCodePool` by their IDs and returns it. + static std::vector<const CXXPredicateCode *> + getSorted(const CXXPredicateCodePool &Pool); + + /// Gets an instance of `CXXPredicateCode` for \p Code, or returns an already + /// existing one. + static const CXXPredicateCode &get(CXXPredicateCodePool &Pool, + std::string Code); + + CXXPredicateCode(std::string Code, unsigned ID); + +public: + static const CXXPredicateCode &getMatchCode(std::string Code) { + return get(AllCXXMatchCode, std::move(Code)); + } + + static const CXXPredicateCode &getApplyCode(std::string Code) { + return get(AllCXXApplyCode, std::move(Code)); + } + + static std::vector<const CXXPredicateCode *> getAllMatchCode() { + return getSorted(AllCXXMatchCode); + } + + static std::vector<const CXXPredicateCode *> getAllApplyCode() { + return getSorted(AllCXXApplyCode); + } + + const std::string Code; + const unsigned ID; + const std::string BaseEnumName; + + bool needsUnreachable() const { + return !StringRef(Code).starts_with("return"); + } + + std::string getEnumNameWithPrefix(StringRef Prefix) const { + return Prefix.str() + BaseEnumName; + } +}; + +} // namespace gi +} // end namespace llvm + +#endif // ifndef LLVM_UTILS_MIRPATTERNS_CXXPREDICATES_H diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp b/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp index 42b4aabf2755..20f98bef4887 100644 --- a/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp +++ b/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp @@ -31,24 +31,24 @@ void CodeExpander::emit(raw_ostream &OS) const { OS << Current.substr(0, Pos); Current = Current.substr(Pos); - if (Current.startswith("\n")) { + if (Current.starts_with("\n")) { OS << "\n" << Indent; Current = Current.drop_front(1); continue; } - if (Current.startswith("\\$") || Current.startswith("\\\\")) { + if (Current.starts_with("\\$") || Current.starts_with("\\\\")) { OS << Current[1]; Current = Current.drop_front(2); continue; } - if (Current.startswith("\\")) { + if (Current.starts_with("\\")) { Current = Current.drop_front(1); continue; } - if (Current.startswith("${")) { + if (Current.starts_with("${")) { StringRef StartVar = Current; Current = Current.drop_front(2); StringRef Var; diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpansions.h b/llvm/utils/TableGen/GlobalISel/CodeExpansions.h index f536e801b27f..b82c3257b321 100644 --- a/llvm/utils/TableGen/GlobalISel/CodeExpansions.h +++ b/llvm/utils/TableGen/GlobalISel/CodeExpansions.h @@ -29,6 +29,10 @@ public: Expansions.try_emplace(Name, Expansion); } + void redeclare(StringRef Name, StringRef Expansion) { + Expansions[Name] = Expansion; + } + std::string lookup(StringRef Variable) const { return Expansions.lookup(Variable); } diff --git a/llvm/utils/TableGen/GlobalISel/CombinerUtils.h b/llvm/utils/TableGen/GlobalISel/CombinerUtils.h index 394c43e3fa83..8cb2514a10e8 100644 --- a/llvm/utils/TableGen/GlobalISel/CombinerUtils.h +++ b/llvm/utils/TableGen/GlobalISel/CombinerUtils.h @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// // /// \file Utility functions used by both Combiner backends. -/// TODO: Can remove when MatchDAG-based backend is removed. // //===----------------------------------------------------------------------===// @@ -31,7 +30,7 @@ inline bool isSpecificDef(const Init &N, StringRef Def) { /// A convenience function to check that an Init refers to a def that is a /// subclass of the given class and coerce it to a def if it is. This is -/// primarily useful for testing for subclasses of GIMatchKind and similar in +/// primarily useful for testing for subclasses of GIDefKind and similar in /// DagInit's since DagInit's support any type inside them. inline Record *getDefOfSubClass(const Init &N, StringRef Cls) { if (const DefInit *OpI = dyn_cast<DefInit>(&N)) @@ -42,7 +41,7 @@ inline Record *getDefOfSubClass(const Init &N, StringRef Cls) { /// A convenience function to check that an Init refers to a dag whose operator /// is a specific def and coerce it to a dag if it is. This is primarily useful -/// for testing for subclasses of GIMatchKind and similar in DagInit's since +/// for testing for subclasses of GIDefKind and similar in DagInit's since /// DagInit's support any type inside them. inline const DagInit *getDagWithSpecificOperator(const Init &N, StringRef Name) { @@ -56,15 +55,14 @@ inline const DagInit *getDagWithSpecificOperator(const Init &N, /// A convenience function to check that an Init refers to a dag whose operator /// is a def that is a subclass of the given class and coerce it to a dag if it -/// is. This is primarily useful for testing for subclasses of GIMatchKind and +/// is. This is primarily useful for testing for subclasses of GIDefKind and /// similar in DagInit's since DagInit's support any type inside them. inline const DagInit *getDagWithOperatorOfSubClass(const Init &N, StringRef Cls) { if (const DagInit *I = dyn_cast<DagInit>(&N)) - if (I->getNumArgs() > 0) - if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator())) - if (OpI->getDef()->isSubClassOf(Cls)) - return I; + if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator())) + if (OpI->getDef()->isSubClassOf(Cls)) + return I; return nullptr; } } // namespace llvm diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDag.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchDag.cpp deleted file mode 100644 index 8be32d2effa6..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDag.cpp +++ /dev/null @@ -1,138 +0,0 @@ -//===- GIMatchDag.cpp - A DAG representation of a pattern to be matched ---===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "GIMatchDag.h" - -#include "llvm/Support/Format.h" -#include "llvm/TableGen/Record.h" -#include "../CodeGenInstruction.h" - -using namespace llvm; - -void GIMatchDag::writeDOTGraph(raw_ostream &OS, StringRef ID) const { - const auto writePorts = [&](StringRef Prefix, - const GIMatchDagOperandList &Operands) { - StringRef Separator = ""; - OS << "{"; - for (const auto &Op : enumerate(Operands)) { - OS << Separator << "<" << Prefix << format("%d", Op.index()) << ">" - << "#" << Op.index() << " $" << Op.value().getName(); - Separator = "|"; - } - OS << "}"; - }; - - OS << "digraph \"" << ID << "\" {\n" - << " rankdir=\"BT\"\n"; - for (const auto &N : InstrNodes) { - OS << " " << format("Node%p", &*N) << " [shape=record,label=\"{"; - writePorts("s", N->getOperandInfo()); - OS << "|" << N->getName(); - if (N->getOpcodeAnnotation()) - OS << "|" << N->getOpcodeAnnotation()->TheDef->getName(); - if (N->isMatchRoot()) - OS << "|Match starts here"; - OS << "|"; - SmallVector<std::pair<unsigned, StringRef>, 8> ToPrint; - for (const auto &Assignment : N->user_assigned_operand_names()) - ToPrint.emplace_back(Assignment.first, Assignment.second); - llvm::sort(ToPrint); - StringRef Separator = ""; - for (const auto &Assignment : ToPrint) { - OS << Separator << "$" << Assignment.second << "=getOperand(" - << Assignment.first << ")"; - Separator = ", "; - } - OS << llvm::format("|%p|", &N); - writePorts("d", N->getOperandInfo()); - OS << "}\""; - if (N->isMatchRoot()) - OS << ",color=red"; - OS << "]\n"; - } - - for (const auto &E : Edges) { - const char *FromFmt = "Node%p:s%d:n"; - const char *ToFmt = "Node%p:d%d:s"; - if (E->getFromMO()->isDef() && !E->getToMO()->isDef()) - std::swap(FromFmt, ToFmt); - auto From = format(FromFmt, E->getFromMI(), E->getFromMO()->getIdx()); - auto To = format(ToFmt, E->getToMI(), E->getToMO()->getIdx()); - if (E->getFromMO()->isDef() && !E->getToMO()->isDef()) - std::swap(From, To); - - OS << " " << From << " -> " << To << " [label=\"$" << E->getName(); - if (E->getFromMO()->isDef() == E->getToMO()->isDef()) - OS << " INVALID EDGE!"; - OS << "\""; - if (E->getFromMO()->isDef() == E->getToMO()->isDef()) - OS << ",color=red"; - else if (E->getFromMO()->isDef() && !E->getToMO()->isDef()) - OS << ",dir=back,arrowtail=crow"; - OS << "]\n"; - } - - for (const auto &N : PredicateNodes) { - OS << " " << format("Pred%p", &*N) << " [shape=record,label=\"{"; - writePorts("s", N->getOperandInfo()); - OS << "|" << N->getName() << "|"; - N->printDescription(OS); - OS << llvm::format("|%p|", &N); - writePorts("d", N->getOperandInfo()); - OS << "}\",style=dotted]\n"; - } - - for (const auto &E : PredicateDependencies) { - const char *FromMIFmt = "Node%p:e"; - const char *FromMOFmt = "Node%p:s%d:n"; - const char *ToFmt = "Pred%p:d%d:s"; - auto To = format(ToFmt, E->getPredicate(), E->getPredicateOp()->getIdx()); - auto Style = "[style=dotted]"; - if (E->getRequiredMO()) { - auto From = - format(FromMOFmt, E->getRequiredMI(), E->getRequiredMO()->getIdx()); - OS << " " << From << " -> " << To << " " << Style << "\n"; - continue; - } - auto From = format(FromMIFmt, E->getRequiredMI()); - OS << " " << From << " -> " << To << " " << Style << "\n"; - } - - OS << "}\n"; -} - -LLVM_DUMP_METHOD void GIMatchDag::print(raw_ostream &OS) const { - OS << "matchdag {\n"; - for (const auto &N : InstrNodes) { - OS << " "; - N->print(OS); - OS << "\n"; - } - for (const auto &E : Edges) { - OS << " "; - E->print(OS); - OS << "\n"; - } - - for (const auto &P : PredicateNodes) { - OS << " "; - P->print(OS); - OS << "\n"; - } - for (const auto &D : PredicateDependencies) { - OS << " "; - D->print(OS); - OS << "\n"; - } - OS << "}\n"; -} - -raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDag &G) { - G.print(OS); - return OS; -} diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDag.h b/llvm/utils/TableGen/GlobalISel/GIMatchDag.h deleted file mode 100644 index c566dd73f709..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDag.h +++ /dev/null @@ -1,240 +0,0 @@ -//===- GIMatchDag.h - Represent a DAG to be matched -------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAG_H -#define LLVM_UTILS_TABLEGEN_GIMATCHDAG_H - -#include "GIMatchDagEdge.h" -#include "GIMatchDagInstr.h" -#include "GIMatchDagOperands.h" -#include "GIMatchDagPredicate.h" -#include "GIMatchDagPredicateDependencyEdge.h" - -namespace llvm { - -/// This class manages lifetimes for data associated with the GIMatchDag object. -class GIMatchDagContext { - GIMatchDagOperandListContext OperandListCtx; - -public: - const GIMatchDagOperandList &makeEmptyOperandList() { - return OperandListCtx.makeEmptyOperandList(); - } - - const GIMatchDagOperandList &makeOperandList(const CodeGenInstruction &I) { - return OperandListCtx.makeOperandList(I); - } - - const GIMatchDagOperandList &makeMIPredicateOperandList() { - return OperandListCtx.makeMIPredicateOperandList(); - } - - - const GIMatchDagOperandList &makeTwoMOPredicateOperandList() { - return OperandListCtx.makeTwoMOPredicateOperandList(); - } - - void print(raw_ostream &OS) const { - OperandListCtx.print(OS); - } - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const { print(errs()); } -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -}; - -class GIMatchDag { -public: - using InstrNodesVec = std::vector<std::unique_ptr<GIMatchDagInstr>>; - using instr_node_iterator = raw_pointer_iterator<InstrNodesVec::iterator>; - using const_instr_node_iterator = - raw_pointer_iterator<InstrNodesVec::const_iterator>; - - using EdgesVec = std::vector<std::unique_ptr<GIMatchDagEdge>>; - using edge_iterator = raw_pointer_iterator<EdgesVec::iterator>; - using const_edge_iterator = raw_pointer_iterator<EdgesVec::const_iterator>; - - using PredicateNodesVec = std::vector<std::unique_ptr<GIMatchDagPredicate>>; - using predicate_iterator = raw_pointer_iterator<PredicateNodesVec::iterator>; - using const_predicate_iterator = - raw_pointer_iterator<PredicateNodesVec::const_iterator>; - - using PredicateDependencyEdgesVec = - std::vector<std::unique_ptr<GIMatchDagPredicateDependencyEdge>>; - using predicate_edge_iterator = - raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>; - using const_predicate_edge_iterator = - raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>; - -protected: - GIMatchDagContext &Ctx; - InstrNodesVec InstrNodes; - PredicateNodesVec PredicateNodes; - EdgesVec Edges; - PredicateDependencyEdgesVec PredicateDependencies; - std::vector<GIMatchDagInstr *> MatchRoots; - // FIXME: This is a temporary measure while we still accept arbitrary code - // blocks to fix up the matcher while it's being developed. - bool HasPostMatchPredicate = false; - -public: - GIMatchDag(GIMatchDagContext &Ctx) : Ctx(Ctx) {} - GIMatchDag(const GIMatchDag &) = delete; - - GIMatchDagContext &getContext() const { return Ctx; } - edge_iterator edges_begin() { - return raw_pointer_iterator<EdgesVec::iterator>(Edges.begin()); - } - edge_iterator edges_end() { - return raw_pointer_iterator<EdgesVec::iterator>(Edges.end()); - } - const_edge_iterator edges_begin() const { - return raw_pointer_iterator<EdgesVec::const_iterator>(Edges.begin()); - } - const_edge_iterator edges_end() const { - return raw_pointer_iterator<EdgesVec::const_iterator>(Edges.end()); - } - iterator_range<edge_iterator> edges() { - return make_range(edges_begin(), edges_end()); - } - iterator_range<const_edge_iterator> edges() const { - return make_range(edges_begin(), edges_end()); - } - iterator_range<std::vector<GIMatchDagInstr *>::iterator> roots() { - return make_range(MatchRoots.begin(), MatchRoots.end()); - } - iterator_range<std::vector<GIMatchDagInstr *>::const_iterator> roots() const { - return make_range(MatchRoots.begin(), MatchRoots.end()); - } - - instr_node_iterator instr_nodes_begin() { - return raw_pointer_iterator<InstrNodesVec::iterator>(InstrNodes.begin()); - } - instr_node_iterator instr_nodes_end() { - return raw_pointer_iterator<InstrNodesVec::iterator>(InstrNodes.end()); - } - const_instr_node_iterator instr_nodes_begin() const { - return raw_pointer_iterator<InstrNodesVec::const_iterator>( - InstrNodes.begin()); - } - const_instr_node_iterator instr_nodes_end() const { - return raw_pointer_iterator<InstrNodesVec::const_iterator>( - InstrNodes.end()); - } - iterator_range<instr_node_iterator> instr_nodes() { - return make_range(instr_nodes_begin(), instr_nodes_end()); - } - iterator_range<const_instr_node_iterator> instr_nodes() const { - return make_range(instr_nodes_begin(), instr_nodes_end()); - } - predicate_edge_iterator predicate_edges_begin() { - return raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>( - PredicateDependencies.begin()); - } - predicate_edge_iterator predicate_edges_end() { - return raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>( - PredicateDependencies.end()); - } - const_predicate_edge_iterator predicate_edges_begin() const { - return raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>( - PredicateDependencies.begin()); - } - const_predicate_edge_iterator predicate_edges_end() const { - return raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>( - PredicateDependencies.end()); - } - iterator_range<predicate_edge_iterator> predicate_edges() { - return make_range(predicate_edges_begin(), predicate_edges_end()); - } - iterator_range<const_predicate_edge_iterator> predicate_edges() const { - return make_range(predicate_edges_begin(), predicate_edges_end()); - } - predicate_iterator predicates_begin() { - return raw_pointer_iterator<PredicateNodesVec::iterator>( - PredicateNodes.begin()); - } - predicate_iterator predicates_end() { - return raw_pointer_iterator<PredicateNodesVec::iterator>( - PredicateNodes.end()); - } - const_predicate_iterator predicates_begin() const { - return raw_pointer_iterator<PredicateNodesVec::const_iterator>( - PredicateNodes.begin()); - } - const_predicate_iterator predicates_end() const { - return raw_pointer_iterator<PredicateNodesVec::const_iterator>( - PredicateNodes.end()); - } - iterator_range<predicate_iterator> predicates() { - return make_range(predicates_begin(), predicates_end()); - } - iterator_range<const_predicate_iterator> predicates() const { - return make_range(predicates_begin(), predicates_end()); - } - - template <class... Args> GIMatchDagInstr *addInstrNode(Args &&... args) { - auto Obj = - std::make_unique<GIMatchDagInstr>(*this, std::forward<Args>(args)...); - auto ObjRaw = Obj.get(); - InstrNodes.push_back(std::move(Obj)); - return ObjRaw; - } - - template <class T, class... Args> - T *addPredicateNode(Args &&... args) { - auto Obj = std::make_unique<T>(getContext(), std::forward<Args>(args)...); - auto ObjRaw = Obj.get(); - PredicateNodes.push_back(std::move(Obj)); - return ObjRaw; - } - - template <class... Args> GIMatchDagEdge *addEdge(Args &&... args) { - auto Obj = std::make_unique<GIMatchDagEdge>(std::forward<Args>(args)...); - auto ObjRaw = Obj.get(); - Edges.push_back(std::move(Obj)); - return ObjRaw; - } - - template <class... Args> - GIMatchDagPredicateDependencyEdge *addPredicateDependency(Args &&... args) { - auto Obj = std::make_unique<GIMatchDagPredicateDependencyEdge>( - std::forward<Args>(args)...); - auto ObjRaw = Obj.get(); - PredicateDependencies.push_back(std::move(Obj)); - return ObjRaw; - } - - size_t getInstrNodeIdx(instr_node_iterator I) { - return std::distance(instr_nodes_begin(), I); - } - size_t getInstrNodeIdx(const_instr_node_iterator I) const { - return std::distance(instr_nodes_begin(), I); - } - size_t getNumInstrNodes() const { return InstrNodes.size(); } - size_t getNumEdges() const { return Edges.size(); } - size_t getNumPredicates() const { return PredicateNodes.size(); } - - void setHasPostMatchPredicate(bool V) { HasPostMatchPredicate = V; } - bool hasPostMatchPredicate() const { return HasPostMatchPredicate; } - - void addMatchRoot(GIMatchDagInstr *N) { MatchRoots.push_back(N); } - - LLVM_DUMP_METHOD void print(raw_ostream &OS) const; - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const { print(errs()); } -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - - void writeDOTGraph(raw_ostream &OS, StringRef ID) const; -}; - -raw_ostream &operator<<(raw_ostream &OS, const GIMatchDag &G); - -} // end namespace llvm - -#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAG_H diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp deleted file mode 100644 index 796479467df7..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp +++ /dev/null @@ -1,39 +0,0 @@ -//===- GIMatchDagEdge.cpp - An edge describing a def/use lookup -----------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "GIMatchDagEdge.h" -#include "GIMatchDagInstr.h" -#include "GIMatchDagOperands.h" -#include "llvm/Support/raw_ostream.h" - -using namespace llvm; - -LLVM_DUMP_METHOD void GIMatchDagEdge::print(raw_ostream &OS) const { - OS << getFromMI()->getName() << "[" << getFromMO()->getName() << "] --[" - << Name << "]--> " << getToMI()->getName() << "[" << getToMO()->getName() - << "]"; -} - -bool GIMatchDagEdge::isDefToUse() const { - // Def -> Def is invalid so we only need to check FromMO. - return FromMO->isDef(); -} - -void GIMatchDagEdge::reverse() { - std::swap(FromMI, ToMI); - std::swap(FromMO, ToMO); -} - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -LLVM_DUMP_METHOD void GIMatchDagEdge::dump() const { print(errs()); } -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - -raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDagEdge &E) { - E.print(OS); - return OS; -} diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h deleted file mode 100644 index e76ef1b4a3aa..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h +++ /dev/null @@ -1,70 +0,0 @@ -//===- GIMatchDagEdge.h - Represent node shared operand lists ---*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGEDGE_H -#define LLVM_UTILS_TABLEGEN_GIMATCHDAGEDGE_H - -#include "llvm/ADT/StringRef.h" - -namespace llvm { -class raw_ostream; -class GIMatchDagInstr; -class GIMatchDagOperand; - -/// Represents an edge that connects two instructions together via a pair of -/// operands. For example: -/// %a = FOO ... -/// %0 = BAR %a -/// %1 = BAZ %a -/// would have two edges for %a like so: -/// BAR:Op#1 --[a]----> Op#0:FOO -/// ^ -/// BAZ:Op#1 --[a]------/ -/// Ideally, all edges in the DAG are from a use to a def as this is a many -/// to one edge but edges from defs to uses are supported too. -class GIMatchDagEdge { - /// The name of the edge. For example, - /// (FOO $a, $b, $c) - /// (BAR $d, $e, $a) - /// will create an edge named 'a' to connect FOO to BAR. Although the name - /// refers to the edge, the canonical value of 'a' is the operand that defines - /// it. - StringRef Name; - const GIMatchDagInstr *FromMI; - const GIMatchDagOperand *FromMO; - const GIMatchDagInstr *ToMI; - const GIMatchDagOperand *ToMO; - -public: - GIMatchDagEdge(StringRef Name, const GIMatchDagInstr *FromMI, const GIMatchDagOperand *FromMO, - const GIMatchDagInstr *ToMI, const GIMatchDagOperand *ToMO) - : Name(Name), FromMI(FromMI), FromMO(FromMO), ToMI(ToMI), ToMO(ToMO) {} - - StringRef getName() const { return Name; } - const GIMatchDagInstr *getFromMI() const { return FromMI; } - const GIMatchDagOperand *getFromMO() const { return FromMO; } - const GIMatchDagInstr *getToMI() const { return ToMI; } - const GIMatchDagOperand *getToMO() const { return ToMO; } - - /// Flip the direction of the edge. - void reverse(); - - /// Does this edge run from a def to (one of many) uses? - bool isDefToUse() const; - - LLVM_DUMP_METHOD void print(raw_ostream &OS) const; - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const; -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -}; - -raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagEdge &E); - -} // end namespace llvm -#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGEDGE_H diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.cpp deleted file mode 100644 index ad9fbea8f881..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.cpp +++ /dev/null @@ -1,48 +0,0 @@ -//===- GIMatchDagInstr.cpp - A shared operand list for nodes --------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "GIMatchDagInstr.h" -#include "../CodeGenInstruction.h" -#include "GIMatchDag.h" -#include "llvm/TableGen/Record.h" - -using namespace llvm; - -void GIMatchDagInstr::print(raw_ostream &OS) const { - OS << "("; - if (const auto *Annotation = getOpcodeAnnotation()) - OS << Annotation->TheDef->getName(); - else - OS << "<unknown>"; - OS << " "; - OperandInfo.print(OS); - OS << "):$" << Name; - if (!UserAssignedNamesForOperands.empty()) { - OS << " // "; - SmallVector<std::pair<unsigned, StringRef>, 8> ToPrint; - for (const auto &Assignment : UserAssignedNamesForOperands) - ToPrint.emplace_back(Assignment.first, Assignment.second); - llvm::sort(ToPrint); - StringRef Separator = ""; - for (const auto &Assignment : ToPrint) { - OS << Separator << "$" << Assignment.second << "=getOperand(" - << Assignment.first << ")"; - Separator = ", "; - } - } -} - -void GIMatchDagInstr::setMatchRoot() { - IsMatchRoot = true; - Dag.addMatchRoot(this); -} - -raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDagInstr &N) { - N.print(OS); - return OS; -} diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h deleted file mode 100644 index d2c746dda9e9..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h +++ /dev/null @@ -1,118 +0,0 @@ -//===- GIMatchDagInstr.h - Represent instruction to be matched --*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGINSTR_H -#define LLVM_UTILS_TABLEGEN_GIMATCHDAGINSTR_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/Support/raw_ostream.h" - -namespace llvm { -class CodeGenInstruction; -class GIMatchDag; -class GIMatchDagOperandList; - -/// Represents an instruction in the match DAG. This object knows very little -/// about the actual instruction to be matched as the bulk of that is in -/// predicates that are associated with the match DAG. It merely knows the names -/// and indices of any operands that need to be matched in order to allow edges -/// to link to them. -/// -/// Instances of this class objects are owned by the GIMatchDag and are not -/// shareable between instances of GIMatchDag. This is because the Name, -/// IsMatchRoot, and OpcodeAnnotation are likely to differ between GIMatchDag -/// instances. -class GIMatchDagInstr { -public: - using const_user_assigned_operand_names_iterator = - DenseMap<unsigned, StringRef>::const_iterator; - -protected: - /// The match DAG this instruction belongs to. - GIMatchDag &Dag; - - /// The name of the instruction in the pattern. For example: - /// (FOO $a, $b, $c):$name - /// will cause name to be assigned to this member. Anonymous instructions will - /// have a name assigned for debugging purposes. - StringRef Name; - - /// The name of the instruction in the pattern as assigned by the user. For - /// example: - /// (FOO $a, $b, $c):$name - /// will cause name to be assigned to this member. If a name is not provided, - /// this will be empty. This name is used to bind variables from rules to the - /// matched instruction. - StringRef UserAssignedName; - - /// The name of each operand (if any) that was assigned by the user. For - /// example: - /// (FOO $a, $b, $c):$name - /// will cause {0, "a"}, {1, "b"}, {2, "c} to be inserted into this map. - DenseMap<unsigned, StringRef> UserAssignedNamesForOperands; - - /// The operand list for this instruction. This object may be shared with - /// other instructions of a similar 'shape'. - const GIMatchDagOperandList &OperandInfo; - - /// For debugging purposes, it's helpful to have access to a description of - /// the Opcode. However, this object shouldn't use it for more than debugging - /// output since predicates are expected to be handled outside the DAG. - CodeGenInstruction *OpcodeAnnotation = nullptr; - - /// When true, this instruction will be a starting point for a match attempt. - bool IsMatchRoot = false; - -public: - GIMatchDagInstr(GIMatchDag &Dag, StringRef Name, StringRef UserAssignedName, - const GIMatchDagOperandList &OperandInfo) - : Dag(Dag), Name(Name), UserAssignedName(UserAssignedName), - OperandInfo(OperandInfo) {} - - const GIMatchDagOperandList &getOperandInfo() const { return OperandInfo; } - StringRef getName() const { return Name; } - StringRef getUserAssignedName() const { return UserAssignedName; } - void assignNameToOperand(unsigned Idx, StringRef Name) { - assert(UserAssignedNamesForOperands[Idx].empty() && "Cannot assign twice"); - UserAssignedNamesForOperands[Idx] = Name; - } - - const_user_assigned_operand_names_iterator - user_assigned_operand_names_begin() const { - return UserAssignedNamesForOperands.begin(); - } - const_user_assigned_operand_names_iterator - user_assigned_operand_names_end() const { - return UserAssignedNamesForOperands.end(); - } - iterator_range<const_user_assigned_operand_names_iterator> - user_assigned_operand_names() const { - return make_range(user_assigned_operand_names_begin(), - user_assigned_operand_names_end()); - } - - /// Mark this instruction as being a root of the match. This means that the - /// matcher will start from this node when attempting to match MIR. - void setMatchRoot(); - bool isMatchRoot() const { return IsMatchRoot; } - - void setOpcodeAnnotation(CodeGenInstruction *I) { OpcodeAnnotation = I; } - CodeGenInstruction *getOpcodeAnnotation() const { return OpcodeAnnotation; } - - void print(raw_ostream &OS) const; - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const { print(errs()); } -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -}; - -raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagInstr &N); - -} // end namespace llvm -#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGINSTR_H diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp deleted file mode 100644 index e79e4686b91e..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp +++ /dev/null @@ -1,153 +0,0 @@ -//===- GIMatchDagOperands.cpp - A shared operand list for nodes -----------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "GIMatchDagOperands.h" - -#include "../CodeGenInstruction.h" - -using namespace llvm; - -void GIMatchDagOperand::Profile(FoldingSetNodeID &ID) const { - Profile(ID, Idx, Name, IsDef); -} - -void GIMatchDagOperand::Profile(FoldingSetNodeID &ID, size_t Idx, - StringRef Name, bool IsDef) { - ID.AddInteger(Idx); - ID.AddString(Name); - ID.AddBoolean(IsDef); -} - -void GIMatchDagOperandList::add(StringRef Name, unsigned Idx, bool IsDef) { - assert(Idx == Operands.size() && "Operands added in wrong order"); - Operands.emplace_back(Operands.size(), Name, IsDef); - OperandsByName.try_emplace(Operands.back().getName(), Operands.size() - 1); -} - -void GIMatchDagOperandList::Profile(FoldingSetNodeID &ID) const { - for (const auto &I : enumerate(Operands)) - GIMatchDagOperand::Profile(ID, I.index(), I.value().getName(), - I.value().isDef()); -} - -void GIMatchDagOperandList::print(raw_ostream &OS) const { - if (Operands.empty()) { - OS << "<empty>"; - return; - } - StringRef Separator = ""; - for (const auto &I : Operands) { - OS << Separator << I.getIdx() << ":" << I.getName(); - if (I.isDef()) - OS << "<def>"; - Separator = ", "; - } -} - -const GIMatchDagOperandList::value_type &GIMatchDagOperandList:: -operator[](StringRef K) const { - const auto &I = OperandsByName.find(K); - assert(I != OperandsByName.end() && "Operand not found by name"); - return Operands[I->second]; -} - -const GIMatchDagOperandList & -GIMatchDagOperandListContext::makeEmptyOperandList() { - FoldingSetNodeID ID; - - void *InsertPoint; - GIMatchDagOperandList *Value = - OperandLists.FindNodeOrInsertPos(ID, InsertPoint); - if (Value) - return *Value; - - std::unique_ptr<GIMatchDagOperandList> NewValue = - std::make_unique<GIMatchDagOperandList>(); - OperandLists.InsertNode(NewValue.get(), InsertPoint); - OperandListsOwner.push_back(std::move(NewValue)); - return *OperandListsOwner.back().get(); -} - -const GIMatchDagOperandList & -GIMatchDagOperandListContext::makeOperandList(const CodeGenInstruction &I) { - FoldingSetNodeID ID; - for (unsigned i = 0; i < I.Operands.size(); ++i) - GIMatchDagOperand::Profile(ID, i, I.Operands[i].Name, - i < I.Operands.NumDefs); - - void *InsertPoint; - GIMatchDagOperandList *Value = - OperandLists.FindNodeOrInsertPos(ID, InsertPoint); - if (Value) - return *Value; - - std::unique_ptr<GIMatchDagOperandList> NewValue = - std::make_unique<GIMatchDagOperandList>(); - for (unsigned i = 0; i < I.Operands.size(); ++i) - NewValue->add(I.Operands[i].Name, i, i < I.Operands.NumDefs); - OperandLists.InsertNode(NewValue.get(), InsertPoint); - OperandListsOwner.push_back(std::move(NewValue)); - return *OperandListsOwner.back().get(); -} - -const GIMatchDagOperandList & -GIMatchDagOperandListContext::makeMIPredicateOperandList() { - FoldingSetNodeID ID; - GIMatchDagOperand::Profile(ID, 0, "$", true); - GIMatchDagOperand::Profile(ID, 1, "mi", false); - - void *InsertPoint; - GIMatchDagOperandList *Value = - OperandLists.FindNodeOrInsertPos(ID, InsertPoint); - if (Value) - return *Value; - - std::unique_ptr<GIMatchDagOperandList> NewValue = - std::make_unique<GIMatchDagOperandList>(); - NewValue->add("$", 0, true); - NewValue->add("mi", 1, false); - OperandLists.InsertNode(NewValue.get(), InsertPoint); - OperandListsOwner.push_back(std::move(NewValue)); - return *OperandListsOwner.back().get(); -} - - -const GIMatchDagOperandList & -GIMatchDagOperandListContext::makeTwoMOPredicateOperandList() { - FoldingSetNodeID ID; - GIMatchDagOperand::Profile(ID, 0, "$", true); - GIMatchDagOperand::Profile(ID, 1, "mi0", false); - GIMatchDagOperand::Profile(ID, 2, "mi1", false); - - void *InsertPoint; - GIMatchDagOperandList *Value = - OperandLists.FindNodeOrInsertPos(ID, InsertPoint); - if (Value) - return *Value; - - std::unique_ptr<GIMatchDagOperandList> NewValue = - std::make_unique<GIMatchDagOperandList>(); - NewValue->add("$", 0, true); - NewValue->add("mi0", 1, false); - NewValue->add("mi1", 2, false); - OperandLists.InsertNode(NewValue.get(), InsertPoint); - OperandListsOwner.push_back(std::move(NewValue)); - return *OperandListsOwner.back().get(); -} - -void GIMatchDagOperandListContext::print(raw_ostream &OS) const { - OS << "GIMatchDagOperandListContext {\n" - << " OperandLists {\n"; - for (const auto &I : OperandListsOwner) { - OS << " "; - I->print(OS); - OS << "\n"; - } - OS << " }\n" - << "}\n"; -} diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h deleted file mode 100644 index ae7190cb7296..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h +++ /dev/null @@ -1,133 +0,0 @@ -//===- GIMatchDagOperands.h - Represent operand lists for nodes -*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGOPERANDS_H -#define LLVM_UTILS_TABLEGEN_GIMATCHDAGOPERANDS_H - -#include "llvm/ADT/FoldingSet.h" -#include "llvm/ADT/StringMap.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/Support/raw_ostream.h" - -#include <vector> - -namespace llvm { -class CodeGenInstruction; -/// Describes an operand of a MachineInstr w.r.t the DAG Matching. This -/// information is derived from CodeGenInstruction::Operands but is more -/// readily available for context-less access as we don't need to know which -/// instruction it's used with or know how many defs that instruction had. -/// -/// There may be multiple GIMatchDagOperand's with the same contents. However, -/// they are uniqued within the set of instructions that have the same overall -/// operand list. For example, given: -/// Inst1 operands ($dst:<def>, $src1, $src2) -/// Inst2 operands ($dst:<def>, $src1, $src2) -/// Inst3 operands ($dst:<def>, $src) -/// $src1 will have a single instance of GIMatchDagOperand shared by Inst1 and -/// Inst2, as will $src2. $dst however, will have two instances one shared -/// between Inst1 and Inst2 and one unique to Inst3. We could potentially -/// fully de-dupe the GIMatchDagOperand instances but the saving is not expected -/// to be worth the overhead. -/// -/// The result of this is that the address of the object can be relied upon to -/// trivially identify commonality between two instructions which will be useful -/// when generating the matcher. When the pointers differ, the contents can be -/// inspected instead. -class GIMatchDagOperand { - unsigned Idx; - StringRef Name; - bool IsDef; - -public: - GIMatchDagOperand(unsigned Idx, StringRef Name, bool IsDef) - : Idx(Idx), Name(Name), IsDef(IsDef) {} - - unsigned getIdx() const { return Idx; } - StringRef getName() const { return Name; } - bool isDef() const { return IsDef; } - - /// This object isn't a FoldingSetNode but it's part of one. See FoldingSet - /// for details on the Profile function. - void Profile(FoldingSetNodeID &ID) const; - - /// A helper that behaves like Profile() but is also usable without the object. - /// We use size_t here to match enumerate<...>::index(). If we don't match - /// that the hashes won't be equal. - static void Profile(FoldingSetNodeID &ID, size_t Idx, StringRef Name, - bool IsDef); -}; - -/// A list of GIMatchDagOperands for an instruction without any association with -/// a particular instruction. -/// -/// An important detail to be aware of with this class is that they are shared -/// with other instructions of a similar 'shape'. For example, all the binary -/// instructions are likely to share a single GIMatchDagOperandList. This is -/// primarily a memory optimization as it's fairly common to have a large number -/// of instructions but only a few 'shapes'. -/// -/// See GIMatchDagOperandList::Profile() for the details on how they are folded. -class GIMatchDagOperandList : public FoldingSetNode { -public: - using value_type = GIMatchDagOperand; - -protected: - using vector_type = SmallVector<GIMatchDagOperand, 3>; - -public: - using iterator = vector_type::iterator; - using const_iterator = vector_type::const_iterator; - -protected: - vector_type Operands; - StringMap<unsigned> OperandsByName; - -public: - void add(StringRef Name, unsigned Idx, bool IsDef); - - /// See FoldingSet for details. - void Profile(FoldingSetNodeID &ID) const; - - iterator begin() { return Operands.begin(); } - const_iterator begin() const { return Operands.begin(); } - iterator end() { return Operands.end(); } - const_iterator end() const { return Operands.end(); } - - const value_type &operator[](unsigned I) const { return Operands[I]; } - const value_type &operator[](StringRef K) const; - - void print(raw_ostream &OS) const; -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const { print(errs()); } -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -}; - -/// This is the portion of GIMatchDagContext that directly relates to -/// GIMatchDagOperandList and GIMatchDagOperandList. -class GIMatchDagOperandListContext { - FoldingSet<GIMatchDagOperandList> OperandLists; - std::vector<std::unique_ptr<GIMatchDagOperandList>> OperandListsOwner; - -public: - const GIMatchDagOperandList &makeEmptyOperandList(); - const GIMatchDagOperandList &makeOperandList(const CodeGenInstruction &I); - const GIMatchDagOperandList &makeMIPredicateOperandList(); - const GIMatchDagOperandList &makeTwoMOPredicateOperandList(); - - void print(raw_ostream &OS) const; -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const { print(errs()); } -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -}; - -} // end namespace llvm -#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGOPERANDS_H diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp deleted file mode 100644 index 6a9e33ac515e..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp +++ /dev/null @@ -1,69 +0,0 @@ -//===- GIMatchDagPredicate.cpp - Represent a predicate to check -----------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "GIMatchDagPredicate.h" - -#include "llvm/TableGen/Record.h" - -#include "../CodeGenInstruction.h" -#include "GIMatchDag.h" - -using namespace llvm; - -void GIMatchDagPredicate::print(raw_ostream &OS) const { - OS << "<<"; - printDescription(OS); - OS << ">>:$" << Name; -} - -void GIMatchDagPredicate::printDescription(raw_ostream &OS) const { OS << ""; } - -GIMatchDagOpcodePredicate::GIMatchDagOpcodePredicate( - GIMatchDagContext &Ctx, StringRef Name, const CodeGenInstruction &Instr) - : GIMatchDagPredicate(GIMatchDagPredicateKind_Opcode, Name, - Ctx.makeMIPredicateOperandList()), - Instr(Instr) {} - -void GIMatchDagOpcodePredicate::printDescription(raw_ostream &OS) const { - OS << "$mi.getOpcode() == " << Instr.TheDef->getName(); -} - -GIMatchDagOneOfOpcodesPredicate::GIMatchDagOneOfOpcodesPredicate( - GIMatchDagContext &Ctx, StringRef Name) - : GIMatchDagPredicate(GIMatchDagPredicateKind_OneOfOpcodes, Name, - Ctx.makeMIPredicateOperandList()) {} - -void GIMatchDagOneOfOpcodesPredicate::printDescription(raw_ostream &OS) const { - OS << "$mi.getOpcode() == oneof("; - StringRef Separator = ""; - for (const CodeGenInstruction *Instr : Instrs) { - OS << Separator << Instr->TheDef->getName(); - Separator = ","; - } - OS << ")"; -} - -GIMatchDagSameMOPredicate::GIMatchDagSameMOPredicate(GIMatchDagContext &Ctx, - StringRef Name) - : GIMatchDagPredicate(GIMatchDagPredicateKind_SameMO, Name, - Ctx.makeTwoMOPredicateOperandList()) {} - -void GIMatchDagSameMOPredicate::printDescription(raw_ostream &OS) const { - OS << "$mi0 == $mi1"; -} - -raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDagPredicate &N) { - N.print(OS); - return OS; -} - -raw_ostream &llvm::operator<<(raw_ostream &OS, - const GIMatchDagOpcodePredicate &N) { - N.print(OS); - return OS; -} diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h deleted file mode 100644 index 952cbdb24f54..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h +++ /dev/null @@ -1,145 +0,0 @@ -//===- GIMatchDagPredicate - Represent a predicate to check -----*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATE_H -#define LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATE_H - -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -#include "llvm/Support/raw_ostream.h" -#endif - -namespace llvm { -class CodeExpansions; -class CodeGenInstruction; -class GIMatchDagOperandList; -class GIMatchDagContext; -class raw_ostream; - -/// Represents a predicate on the match DAG. This records the details of the -/// predicate. The dependencies are stored in the GIMatchDag as edges. -/// -/// Instances of this class objects are owned by the GIMatchDag and are not -/// shareable between instances of GIMatchDag. -class GIMatchDagPredicate { -public: - enum GIMatchDagPredicateKind { - GIMatchDagPredicateKind_Opcode, - GIMatchDagPredicateKind_OneOfOpcodes, - GIMatchDagPredicateKind_SameMO, - }; - -protected: - const GIMatchDagPredicateKind Kind; - - /// The name of the predicate. For example: - /// (FOO $a:s32, $b, $c) - /// will cause 's32' to be assigned to this member for the $a predicate. - /// Similarly, the opcode predicate will cause 'FOO' to be assigned to this - /// member. Anonymous instructions will have a name assigned for debugging - /// purposes. - StringRef Name; - - /// The operand list for this predicate. This object may be shared with - /// other predicates of a similar 'shape'. - const GIMatchDagOperandList &OperandInfo; - -public: - GIMatchDagPredicate(GIMatchDagPredicateKind Kind, StringRef Name, - const GIMatchDagOperandList &OperandInfo) - : Kind(Kind), Name(Name), OperandInfo(OperandInfo) {} - virtual ~GIMatchDagPredicate() {} - - GIMatchDagPredicateKind getKind() const { return Kind; } - - StringRef getName() const { return Name; } - const GIMatchDagOperandList &getOperandInfo() const { return OperandInfo; } - - // Generate C++ code to check this predicate. If a partitioner has already - // tested this predicate then this function won't be called. If this function - // is called, it must emit code and return true to indicate that it did so. If - // it ever returns false, then the caller will abort due to an untested - // predicate. - virtual bool generateCheckCode(raw_ostream &OS, StringRef Indent, - const CodeExpansions &Expansions) const { - return false; - } - - virtual void print(raw_ostream &OS) const; - virtual void printDescription(raw_ostream &OS) const; - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - virtual LLVM_DUMP_METHOD void dump() const { print(errs()); } -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -}; - -class GIMatchDagOpcodePredicate : public GIMatchDagPredicate { - const CodeGenInstruction &Instr; - -public: - GIMatchDagOpcodePredicate(GIMatchDagContext &Ctx, StringRef Name, - const CodeGenInstruction &Instr); - - static bool classof(const GIMatchDagPredicate *P) { - return P->getKind() == GIMatchDagPredicateKind_Opcode; - } - - const CodeGenInstruction *getInstr() const { return &Instr; } - - void printDescription(raw_ostream &OS) const override; - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const override { print(errs()); } -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -}; - -class GIMatchDagOneOfOpcodesPredicate : public GIMatchDagPredicate { - SmallVector<const CodeGenInstruction *, 4> Instrs; - -public: - GIMatchDagOneOfOpcodesPredicate(GIMatchDagContext &Ctx, StringRef Name); - - void addOpcode(const CodeGenInstruction *Instr) { Instrs.push_back(Instr); } - - static bool classof(const GIMatchDagPredicate *P) { - return P->getKind() == GIMatchDagPredicateKind_OneOfOpcodes; - } - - const SmallVectorImpl<const CodeGenInstruction *> &getInstrs() const { - return Instrs; - } - - void printDescription(raw_ostream &OS) const override; - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const override { print(errs()); } -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -}; - -class GIMatchDagSameMOPredicate : public GIMatchDagPredicate { -public: - GIMatchDagSameMOPredicate(GIMatchDagContext &Ctx, StringRef Name); - - static bool classof(const GIMatchDagPredicate *P) { - return P->getKind() == GIMatchDagPredicateKind_SameMO; - } - - void printDescription(raw_ostream &OS) const override; - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const override { print(errs()); } -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -}; - -raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagPredicate &N); -raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagOpcodePredicate &N); - -} // end namespace llvm -#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATE_H diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp deleted file mode 100644 index 921cbaf9c408..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//===- GIMatchDagPredicateDependencyEdge.cpp - Have inputs before check ---===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "GIMatchDagPredicateDependencyEdge.h" - -#include "GIMatchDagInstr.h" -#include "GIMatchDagOperands.h" -#include "GIMatchDagPredicate.h" - -#include "llvm/Support/raw_ostream.h" - -using namespace llvm; - -LLVM_DUMP_METHOD void -GIMatchDagPredicateDependencyEdge::print(raw_ostream &OS) const { - OS << getRequiredMI()->getName(); - if (getRequiredMO()) - OS << "[" << getRequiredMO()->getName() << "]"; - OS << " ==> " << getPredicate()->getName() << "[" - << getPredicateOp()->getName() << "]"; -} - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -LLVM_DUMP_METHOD void GIMatchDagPredicateDependencyEdge::dump() const { - print(errs()); -} -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - -raw_ostream &llvm::operator<<(raw_ostream &OS, - const GIMatchDagPredicateDependencyEdge &E) { - E.print(OS); - return OS; -} diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.h b/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.h deleted file mode 100644 index af91afc6073d..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.h +++ /dev/null @@ -1,61 +0,0 @@ -//===- GIMatchDagPredicateDependencyEdge - Ensure predicates have inputs --===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATEEDGE_H -#define LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATEEDGE_H - -#include "llvm/Support/Compiler.h" - -namespace llvm { -class GIMatchDagInstr; -class GIMatchDagPredicate; -class GIMatchDagOperand; - -class raw_ostream; - -/// Represents a dependency that must be met to evaluate a predicate. -/// -/// Instances of this class objects are owned by the GIMatchDag and are not -/// shareable between instances of GIMatchDag. -class GIMatchDagPredicateDependencyEdge { - /// The MI that must be available in order to test the predicate. - const GIMatchDagInstr *RequiredMI; - /// The MO that must be available in order to test the predicate. May be - /// nullptr when only the MI is required. - const GIMatchDagOperand *RequiredMO; - /// The Predicate that requires information from RequiredMI/RequiredMO. - const GIMatchDagPredicate *Predicate; - /// The Predicate operand that requires information from - /// RequiredMI/RequiredMO. - const GIMatchDagOperand *PredicateOp; - -public: - GIMatchDagPredicateDependencyEdge(const GIMatchDagInstr *RequiredMI, - const GIMatchDagOperand *RequiredMO, - const GIMatchDagPredicate *Predicate, - const GIMatchDagOperand *PredicateOp) - : RequiredMI(RequiredMI), RequiredMO(RequiredMO), Predicate(Predicate), - PredicateOp(PredicateOp) {} - - const GIMatchDagInstr *getRequiredMI() const { return RequiredMI; } - const GIMatchDagOperand *getRequiredMO() const { return RequiredMO; } - const GIMatchDagPredicate *getPredicate() const { return Predicate; } - const GIMatchDagOperand *getPredicateOp() const { return PredicateOp; } - - void print(raw_ostream &OS) const; - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const; -#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -}; - -raw_ostream &operator<<(raw_ostream &OS, - const GIMatchDagPredicateDependencyEdge &N); - -} // end namespace llvm -#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATEEDGE_H diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp b/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp deleted file mode 100644 index 23697fd9e2e2..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp +++ /dev/null @@ -1,761 +0,0 @@ -//===- GIMatchTree.cpp - A decision tree to match GIMatchDag's ------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "GIMatchTree.h" -#include "GIMatchDagPredicate.h" - -#include "../CodeGenInstruction.h" - -#include "llvm/Support/Debug.h" -#include "llvm/Support/Format.h" -#include "llvm/Support/ScopedPrinter.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/TableGen/Error.h" -#include "llvm/TableGen/Record.h" - -#define DEBUG_TYPE "gimatchtree" - -using namespace llvm; - -void GIMatchTree::writeDOTGraph(raw_ostream &OS) const { - OS << "digraph \"matchtree\" {\n"; - writeDOTGraphNode(OS); - OS << "}\n"; -} - -void GIMatchTree::writeDOTGraphNode(raw_ostream &OS) const { - OS << format(" Node%p", this) << " [shape=record,label=\"{"; - if (Partitioner) { - Partitioner->emitDescription(OS); - OS << "|" << Partitioner->getNumPartitions() << " partitions|"; - } else - OS << "No partitioner|"; - bool IsFullyTraversed = true; - bool IsFullyTested = true; - StringRef Separator = ""; - for (const auto &Leaf : PossibleLeaves) { - OS << Separator << Leaf.getName(); - Separator = ","; - if (!Leaf.isFullyTraversed()) - IsFullyTraversed = false; - if (!Leaf.isFullyTested()) - IsFullyTested = false; - } - if (!Partitioner && !IsFullyTraversed) - OS << "|Not fully traversed"; - if (!Partitioner && !IsFullyTested) { - OS << "|Not fully tested"; - if (IsFullyTraversed) { - for (const GIMatchTreeLeafInfo &Leaf : PossibleLeaves) { - if (Leaf.isFullyTested()) - continue; - OS << "\\n" << Leaf.getName() << ": " << &Leaf; - for (const GIMatchDagPredicate *P : Leaf.untested_predicates()) - OS << *P; - } - } - } - OS << "}\""; - if (!Partitioner && - (!IsFullyTraversed || !IsFullyTested || PossibleLeaves.size() > 1)) - OS << ",color=red"; - OS << "]\n"; - for (const auto &C : Children) - C.writeDOTGraphNode(OS); - writeDOTGraphEdges(OS); -} - -void GIMatchTree::writeDOTGraphEdges(raw_ostream &OS) const { - for (const auto &Child : enumerate(Children)) { - OS << format(" Node%p", this) << " -> " << format("Node%p", &Child.value()) - << " [label=\"#" << Child.index() << " "; - Partitioner->emitPartitionName(OS, Child.index()); - OS << "\"]\n"; - } -} - -GIMatchTreeBuilderLeafInfo::GIMatchTreeBuilderLeafInfo( - GIMatchTreeBuilder &Builder, StringRef Name, unsigned RootIdx, - const GIMatchDag &MatchDag, void *Data) - : Builder(Builder), Info(Name, RootIdx, Data), MatchDag(MatchDag), - RemainingInstrNodes(BitVector(MatchDag.getNumInstrNodes(), true)), - RemainingEdges(BitVector(MatchDag.getNumEdges(), true)), - RemainingPredicates(BitVector(MatchDag.getNumPredicates(), true)), - TraversableEdges(MatchDag.getNumEdges()), - TestablePredicates(MatchDag.getNumPredicates()) { - // Number all the predicates in this DAG - for (const auto &[Idx, P] : enumerate(MatchDag.predicates())) { - PredicateIDs.insert(std::make_pair(P, Idx)); - } - - // Number all the predicate dependencies in this DAG and set up a bitvector - // for each predicate indicating the unsatisfied dependencies. - for (const auto &[Idx, Dep] : enumerate(MatchDag.predicate_edges())) { - PredicateDepIDs.insert(std::make_pair(Dep, Idx)); - } - UnsatisfiedPredDepsForPred.resize(MatchDag.getNumPredicates(), - BitVector(PredicateDepIDs.size())); - for (const auto &[Idx, Dep] : enumerate(MatchDag.predicate_edges())) { - unsigned ID = PredicateIDs.lookup(Dep->getPredicate()); - UnsatisfiedPredDepsForPred[ID].set(Idx); - } -} - -void GIMatchTreeBuilderLeafInfo::declareInstr(const GIMatchDagInstr *Instr, unsigned ID) { - // Record the assignment of this instr to the given ID. - auto InfoI = InstrNodeToInfo.insert(std::make_pair( - Instr, GIMatchTreeInstrInfo(ID, Instr))); - InstrIDToInfo.insert(std::make_pair(ID, &InfoI.first->second)); - - if (Instr == nullptr) - return; - - if (!Instr->getUserAssignedName().empty()) - Info.bindInstrVariable(Instr->getUserAssignedName(), ID); - for (const auto &VarBinding : Instr->user_assigned_operand_names()) - Info.bindOperandVariable(VarBinding.second, ID, VarBinding.first); - - // Clear the bit indicating we haven't visited this instr. - const auto &NodeI = find(MatchDag.instr_nodes(), Instr); - assert(NodeI != MatchDag.instr_nodes_end() && "Instr isn't in this DAG"); - unsigned InstrIdx = MatchDag.getInstrNodeIdx(NodeI); - RemainingInstrNodes.reset(InstrIdx); - - // When we declare an instruction, we don't expose any traversable edges just - // yet. A partitioner has to check they exist and are registers before they - // are traversable. - - // When we declare an instruction, we potentially activate some predicates. - // Mark the dependencies that are now satisfied as a result of this - // instruction and mark any predicates whose dependencies are fully - // satisfied. - for (const auto &Dep : enumerate(MatchDag.predicate_edges())) { - if (Dep.value()->getRequiredMI() == Instr && - Dep.value()->getRequiredMO() == nullptr) { - for (const auto &DepsFor : enumerate(UnsatisfiedPredDepsForPred)) { - DepsFor.value().reset(Dep.index()); - if (DepsFor.value().none()) - TestablePredicates.set(DepsFor.index()); - } - } - } -} - -void GIMatchTreeBuilderLeafInfo::declareOperand(unsigned InstrID, - unsigned OpIdx) { - const GIMatchDagInstr *Instr = InstrIDToInfo.lookup(InstrID)->getInstrNode(); - - OperandIDToInfo.insert(std::make_pair( - std::make_pair(InstrID, OpIdx), - GIMatchTreeOperandInfo(Instr, OpIdx))); - - // When an operand becomes reachable, we potentially activate some traversals. - // Record the edges that can now be followed as a result of this - // instruction. - for (const auto &[Idx, E] : enumerate(MatchDag.edges())) { - if (E->getFromMI() == Instr && E->getFromMO()->getIdx() == OpIdx) { - TraversableEdges.set(Idx); - } - } - - // When an operand becomes reachable, we potentially activate some predicates. - // Clear the dependencies that are now satisfied as a result of this - // operand and activate any predicates whose dependencies are fully - // satisfied. - for (const auto &Dep : enumerate(MatchDag.predicate_edges())) { - if (Dep.value()->getRequiredMI() == Instr && Dep.value()->getRequiredMO() && - Dep.value()->getRequiredMO()->getIdx() == OpIdx) { - for (const auto &DepsFor : enumerate(UnsatisfiedPredDepsForPred)) { - DepsFor.value().reset(Dep.index()); - if (DepsFor.value().none()) - TestablePredicates.set(DepsFor.index()); - } - } - } -} - -void GIMatchTreeBuilder::addPartitionersForInstr(unsigned InstrIdx) { - // Find the partitioners that can be used now that this node is - // uncovered. Our choices are: - // - Test the opcode - addPartitioner(std::make_unique<GIMatchTreeOpcodePartitioner>(InstrIdx)); -} - -void GIMatchTreeBuilder::addPartitionersForOperand(unsigned InstrID, - unsigned OpIdx) { - LLVM_DEBUG(dbgs() << "Add partitioners for Instrs[" << InstrID - << "].getOperand(" << OpIdx << ")\n"); - addPartitioner( - std::make_unique<GIMatchTreeVRegDefPartitioner>(InstrID, OpIdx)); -} - -void GIMatchTreeBuilder::filterRedundantPartitioners() { - // TODO: Filter partitioners for facts that are already known - // - If we know the opcode, we can elide the num operand check so long as - // the instruction has a fixed number of operands. - // - If we know an exact number of operands then we can elide further number - // of operand checks. - // - If the current min number of operands exceeds the one we want to check - // then we can elide it. -} - -void GIMatchTreeBuilder::evaluatePartitioners() { - // Determine the partitioning the partitioner would produce - for (auto &Partitioner : Partitioners) { - LLVM_DEBUG(dbgs() << " Weighing up "; - Partitioner->emitDescription(dbgs()); dbgs() << "\n"); - Partitioner->repartition(Leaves); - LLVM_DEBUG(Partitioner->emitPartitionResults(dbgs())); - } -} - -void GIMatchTreeBuilder::runStep() { - LLVM_DEBUG(dbgs() << "Building match tree node for " << TreeNode << "\n"); - LLVM_DEBUG(dbgs() << " Rules reachable at this node:\n"); - for (const auto &Leaf : Leaves) { - LLVM_DEBUG(dbgs() << " " << Leaf.getName() << " (" << &Leaf.getInfo() << "\n"); - TreeNode->addPossibleLeaf(Leaf.getInfo(), Leaf.isFullyTraversed(), - Leaf.isFullyTested()); - } - - LLVM_DEBUG(dbgs() << " Partitioners available at this node:\n"); -#ifndef NDEBUG - for (const auto &Partitioner : Partitioners) - LLVM_DEBUG(dbgs() << " "; Partitioner->emitDescription(dbgs()); - dbgs() << "\n"); -#endif // ifndef NDEBUG - - LLVM_DEBUG(dbgs() << " Eliminating redundant partitioners:\n"); - filterRedundantPartitioners(); - LLVM_DEBUG(dbgs() << " Partitioners remaining:\n"); -#ifndef NDEBUG - for (const auto &Partitioner : Partitioners) - LLVM_DEBUG(dbgs() << " "; Partitioner->emitDescription(dbgs()); - dbgs() << "\n"); -#endif // ifndef NDEBUG - - if (Partitioners.empty()) { - // Nothing left to do but check we really did identify a single rule. - if (Leaves.size() > 1) { - LLVM_DEBUG(dbgs() << "Leaf contains multiple rules, drop after the first " - "fully tested rule\n"); - auto FirstFullyTested = - llvm::find_if(Leaves, [](const GIMatchTreeBuilderLeafInfo &X) { - return X.isFullyTraversed() && X.isFullyTested() && - !X.getMatchDag().hasPostMatchPredicate(); - }); - if (FirstFullyTested != Leaves.end()) - FirstFullyTested++; - -#ifndef NDEBUG - for (auto &Leaf : make_range(Leaves.begin(), FirstFullyTested)) - LLVM_DEBUG(dbgs() << " Kept " << Leaf.getName() << "\n"); - for (const auto &Leaf : make_range(FirstFullyTested, Leaves.end())) - LLVM_DEBUG(dbgs() << " Dropped " << Leaf.getName() << "\n"); -#endif // ifndef NDEBUG - TreeNode->dropLeavesAfter( - std::distance(Leaves.begin(), FirstFullyTested)); - } - for (const auto &Leaf : Leaves) { - if (!Leaf.isFullyTraversed()) { - PrintError("Leaf " + Leaf.getName() + " is not fully traversed"); - PrintNote("This indicates a missing partitioner within tblgen"); - Leaf.dump(errs()); - for (unsigned InstrIdx : Leaf.untested_instrs()) - PrintNote("Instr " + llvm::to_string(*Leaf.getInstr(InstrIdx))); - for (unsigned EdgeIdx : Leaf.untested_edges()) - PrintNote("Edge " + llvm::to_string(*Leaf.getEdge(EdgeIdx))); - } - } - - // Copy out information about untested predicates so the user of the tree - // can deal with them. - for (auto LeafPair : zip(Leaves, TreeNode->possible_leaves())) { - const GIMatchTreeBuilderLeafInfo &BuilderLeaf = std::get<0>(LeafPair); - GIMatchTreeLeafInfo &TreeLeaf = std::get<1>(LeafPair); - if (!BuilderLeaf.isFullyTested()) - for (unsigned PredicateIdx : BuilderLeaf.untested_predicates()) - TreeLeaf.addUntestedPredicate(BuilderLeaf.getPredicate(PredicateIdx)); - } - return; - } - - LLVM_DEBUG(dbgs() << " Weighing up partitioners:\n"); - evaluatePartitioners(); - - // Select the best partitioner by its ability to partition - // - Prefer partitioners that don't distinguish between partitions. This - // is to fail early on decisions that must go a single way. - auto PartitionerI = std::max_element( - Partitioners.begin(), Partitioners.end(), - [](const std::unique_ptr<GIMatchTreePartitioner> &A, - const std::unique_ptr<GIMatchTreePartitioner> &B) { - // We generally want partitioners that subdivide the - // ruleset as much as possible since these take fewer - // checks to converge on a particular rule. However, - // it's important to note that one leaf can end up in - // multiple partitions if the check isn't mutually - // exclusive (e.g. getVRegDef() vs isReg()). - // We therefore minimize average leaves per partition. - return (double)A->getNumLeavesWithDupes() / A->getNumPartitions() > - (double)B->getNumLeavesWithDupes() / B->getNumPartitions(); - }); - - // Select a partitioner and partition the ruleset - // Note that it's possible for a single rule to end up in multiple - // partitions. For example, an opcode test on a rule without an opcode - // predicate will result in it being passed to all partitions. - std::unique_ptr<GIMatchTreePartitioner> Partitioner = std::move(*PartitionerI); - Partitioners.erase(PartitionerI); - LLVM_DEBUG(dbgs() << " Selected partitioner: "; - Partitioner->emitDescription(dbgs()); dbgs() << "\n"); - - assert(Partitioner->getNumPartitions() > 0 && - "Must always partition into at least one partition"); - - TreeNode->setNumChildren(Partitioner->getNumPartitions()); - for (const auto &[Idx, Child] : enumerate(TreeNode->children())) { - SubtreeBuilders.emplace_back(&Child, NextInstrID); - Partitioner->applyForPartition(Idx, *this, SubtreeBuilders.back()); - } - - TreeNode->setPartitioner(std::move(Partitioner)); - - // Recurse into the subtree builders. Each one must get a copy of the - // remaining partitioners as each path has to check everything. - for (auto &SubtreeBuilder : SubtreeBuilders) { - for (const auto &Partitioner : Partitioners) - SubtreeBuilder.addPartitioner(Partitioner->clone()); - SubtreeBuilder.runStep(); - } -} - -std::unique_ptr<GIMatchTree> GIMatchTreeBuilder::run() { - unsigned NewInstrID = allocInstrID(); - // Start by recording the root instruction as instr #0 and set up the initial - // partitioners. - for (auto &Leaf : Leaves) { - LLVM_DEBUG(Leaf.getMatchDag().writeDOTGraph(dbgs(), Leaf.getName())); - GIMatchDagInstr *Root = - *(Leaf.getMatchDag().roots().begin() + Leaf.getRootIdx()); - Leaf.declareInstr(Root, NewInstrID); - } - - addPartitionersForInstr(NewInstrID); - - std::unique_ptr<GIMatchTree> TreeRoot = std::make_unique<GIMatchTree>(); - TreeNode = TreeRoot.get(); - runStep(); - - return TreeRoot; -} - -void GIMatchTreeOpcodePartitioner::emitPartitionName(raw_ostream &OS, unsigned Idx) const { - if (PartitionToInstr[Idx] == nullptr) { - OS << "* or nullptr"; - return; - } - OS << PartitionToInstr[Idx]->Namespace - << "::" << PartitionToInstr[Idx]->TheDef->getName(); -} - -void GIMatchTreeOpcodePartitioner::repartition( - GIMatchTreeBuilder::LeafVec &Leaves) { - Partitions.clear(); - InstrToPartition.clear(); - PartitionToInstr.clear(); - TestedPredicates.clear(); - - for (const auto &Leaf : enumerate(Leaves)) { - bool AllOpcodes = true; - GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); - BitVector TestedPredicatesForLeaf( - Leaf.value().getMatchDag().getNumPredicates()); - - // If the instruction isn't declared then we don't care about it. Ignore - // it for now and add it to all partitions later once we know what - // partitions we have. - if (!InstrInfo) { - LLVM_DEBUG(dbgs() << " " << Leaf.value().getName() - << " doesn't care about Instr[" << InstrID << "]\n"); - assert(TestedPredicatesForLeaf.size() == Leaf.value().getMatchDag().getNumPredicates()); - TestedPredicates.push_back(TestedPredicatesForLeaf); - continue; - } - - // If the opcode is available to test then any opcode predicates will have - // been enabled too. - for (unsigned PIdx : Leaf.value().TestablePredicates.set_bits()) { - const auto &P = Leaf.value().getPredicate(PIdx); - SmallVector<const CodeGenInstruction *, 1> OpcodesForThisPredicate; - if (const auto *OpcodeP = dyn_cast<const GIMatchDagOpcodePredicate>(P)) { - // We've found _an_ opcode predicate, but we don't know if it's - // checking this instruction yet. - bool IsThisPredicate = false; - for (const auto &PDep : Leaf.value().getMatchDag().predicate_edges()) { - if (PDep->getRequiredMI() == InstrInfo->getInstrNode() && - PDep->getRequiredMO() == nullptr && PDep->getPredicate() == P) { - IsThisPredicate = true; - break; - } - } - if (!IsThisPredicate) - continue; - - // If we get here twice then we've somehow ended up with two opcode - // predicates for one instruction in the same DAG. That should be - // impossible. - assert(AllOpcodes && "Conflicting opcode predicates"); - const CodeGenInstruction *Expected = OpcodeP->getInstr(); - OpcodesForThisPredicate.push_back(Expected); - } - - if (const auto *OpcodeP = - dyn_cast<const GIMatchDagOneOfOpcodesPredicate>(P)) { - // We've found _an_ oneof(opcodes) predicate, but we don't know if it's - // checking this instruction yet. - bool IsThisPredicate = false; - for (const auto &PDep : Leaf.value().getMatchDag().predicate_edges()) { - if (PDep->getRequiredMI() == InstrInfo->getInstrNode() && - PDep->getRequiredMO() == nullptr && PDep->getPredicate() == P) { - IsThisPredicate = true; - break; - } - } - if (!IsThisPredicate) - continue; - - // If we get here twice then we've somehow ended up with two opcode - // predicates for one instruction in the same DAG. That should be - // impossible. - assert(AllOpcodes && "Conflicting opcode predicates"); - append_range(OpcodesForThisPredicate, OpcodeP->getInstrs()); - } - - for (const CodeGenInstruction *Expected : OpcodesForThisPredicate) { - // Mark this predicate as one we're testing. - TestedPredicatesForLeaf.set(PIdx); - - // Partitions must be numbered 0, 1, .., N but instructions don't meet - // that requirement. Assign a partition number to each opcode if we - // lack one ... - auto Partition = InstrToPartition.find(Expected); - if (Partition == InstrToPartition.end()) { - BitVector Contents(Leaves.size()); - Partition = InstrToPartition - .insert(std::make_pair(Expected, Partitions.size())) - .first; - PartitionToInstr.push_back(Expected); - Partitions.insert(std::make_pair(Partitions.size(), Contents)); - } - // ... and mark this leaf as being in that partition. - Partitions.find(Partition->second)->second.set(Leaf.index()); - AllOpcodes = false; - LLVM_DEBUG(dbgs() << " " << Leaf.value().getName() - << " is in partition " << Partition->second << "\n"); - } - - // TODO: This is where we would handle multiple choices of opcode - // the end result will be that this leaf ends up in multiple - // partitions similarly to AllOpcodes. - } - - // If we never check the opcode, add it to every partition. - if (AllOpcodes) { - // Add a partition for the default case if we don't already have one. - if (InstrToPartition.insert(std::make_pair(nullptr, 0)).second) { - PartitionToInstr.push_back(nullptr); - BitVector Contents(Leaves.size()); - Partitions.insert(std::make_pair(Partitions.size(), Contents)); - } - LLVM_DEBUG(dbgs() << " " << Leaf.value().getName() - << " is in all partitions (opcode not checked)\n"); - for (auto &Partition : Partitions) - Partition.second.set(Leaf.index()); - } - - assert(TestedPredicatesForLeaf.size() == Leaf.value().getMatchDag().getNumPredicates()); - TestedPredicates.push_back(TestedPredicatesForLeaf); - } - - if (Partitions.size() == 0) { - // Add a partition for the default case if we don't already have one. - if (InstrToPartition.insert(std::make_pair(nullptr, 0)).second) { - PartitionToInstr.push_back(nullptr); - BitVector Contents(Leaves.size()); - Partitions.insert(std::make_pair(Partitions.size(), Contents)); - } - } - - // Add any leaves that don't care about this instruction to all partitions. - for (const auto &Leaf : enumerate(Leaves)) { - GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); - if (!InstrInfo) { - // Add a partition for the default case if we don't already have one. - if (InstrToPartition.insert(std::make_pair(nullptr, 0)).second) { - PartitionToInstr.push_back(nullptr); - BitVector Contents(Leaves.size()); - Partitions.insert(std::make_pair(Partitions.size(), Contents)); - } - for (auto &Partition : Partitions) - Partition.second.set(Leaf.index()); - } - } - -} - -void GIMatchTreeOpcodePartitioner::applyForPartition( - unsigned PartitionIdx, GIMatchTreeBuilder &Builder, GIMatchTreeBuilder &SubBuilder) { - LLVM_DEBUG(dbgs() << " Making partition " << PartitionIdx << "\n"); - const CodeGenInstruction *CGI = PartitionToInstr[PartitionIdx]; - - BitVector PossibleLeaves = getPossibleLeavesForPartition(PartitionIdx); - // Consume any predicates we handled. - for (const auto &[Index, EnumeratedLeaf] : - enumerate(Builder.getPossibleLeaves())) { - if (!PossibleLeaves[Index]) - continue; - - const auto &TestedPredicatesForLeaf = TestedPredicates[Index]; - - for (unsigned PredIdx : TestedPredicatesForLeaf.set_bits()) { - LLVM_DEBUG(dbgs() << " " << EnumeratedLeaf.getName() - << " tested predicate #" << PredIdx << " of " - << TestedPredicatesForLeaf.size() << " " - << *EnumeratedLeaf.getPredicate(PredIdx) << "\n"); - EnumeratedLeaf.RemainingPredicates.reset(PredIdx); - EnumeratedLeaf.TestablePredicates.reset(PredIdx); - } - SubBuilder.addLeaf(EnumeratedLeaf); - } - - // Nothing to do, we don't know anything about this instruction as a result - // of this partitioner. - if (CGI == nullptr) - return; - - GIMatchTreeBuilder::LeafVec &NewLeaves = SubBuilder.getPossibleLeaves(); - // Find all the operands we know to exist and are referenced. This will - // usually be all the referenced operands but there are some cases where - // instructions are variadic. Such operands must be handled by partitioners - // that check the number of operands. - BitVector ReferencedOperands(1); - for (auto &Leaf : NewLeaves) { - GIMatchTreeInstrInfo *InstrInfo = Leaf.getInstrInfo(InstrID); - // Skip any leaves that don't care about this instruction. - if (!InstrInfo) - continue; - const GIMatchDagInstr *Instr = InstrInfo->getInstrNode(); - for (const auto &E : Leaf.getMatchDag().edges()) { - if (E->getFromMI() == Instr && - E->getFromMO()->getIdx() < CGI->Operands.size()) { - ReferencedOperands.resize(E->getFromMO()->getIdx() + 1); - ReferencedOperands.set(E->getFromMO()->getIdx()); - } - } - } - for (auto &Leaf : NewLeaves) { - // Skip any leaves that don't care about this instruction. - if (!Leaf.getInstrInfo(InstrID)) - continue; - - for (unsigned OpIdx : ReferencedOperands.set_bits()) { - Leaf.declareOperand(InstrID, OpIdx); - } - } - for (unsigned OpIdx : ReferencedOperands.set_bits()) { - SubBuilder.addPartitionersForOperand(InstrID, OpIdx); - } -} - -void GIMatchTreeOpcodePartitioner::emitPartitionResults( - raw_ostream &OS) const { - OS << "Partitioning by opcode would produce " << Partitions.size() - << " partitions\n"; - for (const auto &Partition : InstrToPartition) { - if (Partition.first == nullptr) - OS << "Default: "; - else - OS << Partition.first->TheDef->getName() << ": "; - StringRef Separator = ""; - for (unsigned I : Partitions.find(Partition.second)->second.set_bits()) { - OS << Separator << I; - Separator = ", "; - } - OS << "\n"; - } -} - -void GIMatchTreeOpcodePartitioner::generatePartitionSelectorCode( - raw_ostream &OS, StringRef Indent) const { - // Make sure not to emit empty switch or switch with just default - if (PartitionToInstr.size() == 1 && PartitionToInstr[0] == nullptr) { - OS << Indent << "Partition = 0;\n"; - } else if (PartitionToInstr.size()) { - OS << Indent << "Partition = -1;\n" - << Indent << "switch (MIs[" << InstrID << "]->getOpcode()) {\n"; - for (const auto &EnumInstr : enumerate(PartitionToInstr)) { - if (EnumInstr.value() == nullptr) - OS << Indent << "default:"; - else - OS << Indent << "case " << EnumInstr.value()->Namespace - << "::" << EnumInstr.value()->TheDef->getName() << ":"; - OS << " Partition = " << EnumInstr.index() << "; break;\n"; - } - OS << Indent << "}\n"; - } - OS << Indent - << "// Default case but without conflicting with potential default case " - "in selection.\n" - << Indent << "if (Partition == -1) return false;\n"; -} - -void GIMatchTreeVRegDefPartitioner::addToPartition(bool Result, - unsigned LeafIdx) { - auto I = ResultToPartition.find(Result); - if (I == ResultToPartition.end()) { - ResultToPartition.insert(std::make_pair(Result, PartitionToResult.size())); - PartitionToResult.push_back(Result); - } - I = ResultToPartition.find(Result); - auto P = Partitions.find(I->second); - if (P == Partitions.end()) - P = Partitions.insert(std::make_pair(I->second, BitVector())).first; - P->second.resize(LeafIdx + 1); - P->second.set(LeafIdx); -} - -void GIMatchTreeVRegDefPartitioner::repartition( - GIMatchTreeBuilder::LeafVec &Leaves) { - Partitions.clear(); - - for (const auto &Leaf : enumerate(Leaves)) { - GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); - BitVector TraversedEdgesForLeaf(Leaf.value().getMatchDag().getNumEdges()); - - // If the instruction isn't declared then we don't care about it. Ignore - // it for now and add it to all partitions later once we know what - // partitions we have. - if (!InstrInfo) { - TraversedEdges.push_back(TraversedEdgesForLeaf); - continue; - } - - // If this node has an use -> def edge from this operand then this - // instruction must be in partition 1 (isVRegDef()). - bool WantsEdge = false; - for (unsigned EIdx : Leaf.value().TraversableEdges.set_bits()) { - const auto &E = Leaf.value().getEdge(EIdx); - if (E->getFromMI() != InstrInfo->getInstrNode() || - E->getFromMO()->getIdx() != OpIdx || E->isDefToUse()) - continue; - - // We're looking at the right edge. This leaf wants a vreg def so we'll - // put it in partition 1. - addToPartition(true, Leaf.index()); - TraversedEdgesForLeaf.set(EIdx); - WantsEdge = true; - } - - if (!WantsEdge) { - // If this leaf doesn't have an edge and we don't know what we want, - // then add it to partition 0 and 1. - addToPartition(false, Leaf.index()); - addToPartition(true, Leaf.index()); - } - - TraversedEdges.push_back(TraversedEdgesForLeaf); - } - - // Add any leaves that don't care about this instruction to all partitions. - for (const auto &Leaf : enumerate(Leaves)) { - GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); - if (!InstrInfo) - for (auto &Partition : Partitions) { - Partition.second.resize(Leaf.index() + 1); - Partition.second.set(Leaf.index()); - } - } -} - -void GIMatchTreeVRegDefPartitioner::applyForPartition( - unsigned PartitionIdx, GIMatchTreeBuilder &Builder, - GIMatchTreeBuilder &SubBuilder) { - BitVector PossibleLeaves = getPossibleLeavesForPartition(PartitionIdx); - - std::vector<BitVector> TraversedEdgesByNewLeaves; - // Consume any edges we handled. - for (const auto &[Index, EnumeratedLeaf] : - enumerate(Builder.getPossibleLeaves())) { - if (!PossibleLeaves[Index]) - continue; - - const auto &TraversedEdgesForLeaf = TraversedEdges[Index]; - TraversedEdgesByNewLeaves.push_back(TraversedEdgesForLeaf); - EnumeratedLeaf.RemainingEdges.reset(TraversedEdgesForLeaf); - EnumeratedLeaf.TraversableEdges.reset(TraversedEdgesForLeaf); - SubBuilder.addLeaf(EnumeratedLeaf); - } - - // Nothing to do. The only thing we know is that it isn't a vreg-def. - if (PartitionToResult[PartitionIdx] == false) - return; - - NewInstrID = SubBuilder.allocInstrID(); - - GIMatchTreeBuilder::LeafVec &NewLeaves = SubBuilder.getPossibleLeaves(); - for (const auto &I : zip(NewLeaves, TraversedEdgesByNewLeaves)) { - auto &Leaf = std::get<0>(I); - auto &TraversedEdgesForLeaf = std::get<1>(I); - GIMatchTreeInstrInfo *InstrInfo = Leaf.getInstrInfo(InstrID); - // Skip any leaves that don't care about this instruction. - if (!InstrInfo) - continue; - for (unsigned EIdx : TraversedEdgesForLeaf.set_bits()) { - const GIMatchDagEdge *E = Leaf.getEdge(EIdx); - Leaf.declareInstr(E->getToMI(), NewInstrID); - } - } - SubBuilder.addPartitionersForInstr(NewInstrID); -} - -void GIMatchTreeVRegDefPartitioner::emitPartitionResults( - raw_ostream &OS) const { - OS << "Partitioning by vreg-def would produce " << Partitions.size() - << " partitions\n"; - for (const auto &Partition : Partitions) { - OS << Partition.first << " ("; - emitPartitionName(OS, Partition.first); - OS << "): "; - StringRef Separator = ""; - for (unsigned I : Partition.second.set_bits()) { - OS << Separator << I; - Separator = ", "; - } - OS << "\n"; - } -} - -void GIMatchTreeVRegDefPartitioner::generatePartitionSelectorCode( - raw_ostream &OS, StringRef Indent) const { - OS << Indent << "Partition = -1;\n" - << Indent << "if (MIs.size() <= " << NewInstrID << ") MIs.resize(" - << (NewInstrID + 1) << ");\n" - << Indent << "MIs[" << NewInstrID << "] = nullptr;\n" - << Indent << "if (MIs[" << InstrID << "]->getOperand(" << OpIdx - << ").isReg())\n" - << Indent << " MIs[" << NewInstrID << "] = MRI.getVRegDef(MIs[" << InstrID - << "]->getOperand(" << OpIdx << ").getReg());\n"; - - for (const auto &Pair : ResultToPartition) - OS << Indent << "if (MIs[" << NewInstrID << "] " - << (Pair.first ? "!=" : "==") - << " nullptr) Partition = " << Pair.second << ";\n"; - - OS << Indent << "if (Partition == -1) return false;\n"; -} diff --git a/llvm/utils/TableGen/GlobalISel/GIMatchTree.h b/llvm/utils/TableGen/GlobalISel/GIMatchTree.h deleted file mode 100644 index c65423ddacdb..000000000000 --- a/llvm/utils/TableGen/GlobalISel/GIMatchTree.h +++ /dev/null @@ -1,626 +0,0 @@ -//===- GIMatchTree.h - A decision tree to match GIMatchDag's --------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_UTILS_TABLEGEN_GIMATCHTREE_H -#define LLVM_UTILS_TABLEGEN_GIMATCHTREE_H - -#include "GIMatchDag.h" -#include "llvm/ADT/BitVector.h" - -namespace llvm { -class raw_ostream; - -class GIMatchTreeBuilder; -class GIMatchTreePartitioner; - -/// Describes the binding of a variable to the matched MIR -class GIMatchTreeVariableBinding { - /// The name of the variable described by this binding. - StringRef Name; - // The matched instruction it is bound to. - unsigned InstrID; - // The matched operand (if appropriate) it is bound to. - std::optional<unsigned> OpIdx; - -public: - GIMatchTreeVariableBinding(StringRef Name, unsigned InstrID, - std::optional<unsigned> OpIdx = std::nullopt) - : Name(Name), InstrID(InstrID), OpIdx(OpIdx) {} - - bool isInstr() const { return !OpIdx; } - StringRef getName() const { return Name; } - unsigned getInstrID() const { return InstrID; } - unsigned getOpIdx() const { - assert(OpIdx && "Is not an operand binding"); - return *OpIdx; - } -}; - -/// Associates a matchable with a leaf of the decision tree. -class GIMatchTreeLeafInfo { -public: - using const_var_binding_iterator = - std::vector<GIMatchTreeVariableBinding>::const_iterator; - using UntestedPredicatesTy = SmallVector<const GIMatchDagPredicate *, 1>; - using const_untested_predicates_iterator = UntestedPredicatesTy::const_iterator; - -protected: - /// A name for the matchable. This is primarily for debugging. - StringRef Name; - /// Where rules have multiple roots, this is which root we're starting from. - unsigned RootIdx; - /// Opaque data the caller of the tree building code understands. - void *Data; - /// Has the decision tree covered every edge traversal? If it hasn't then this - /// is an unrecoverable error indicating there's something wrong with the - /// partitioners. - bool IsFullyTraversed; - /// Has the decision tree covered every predicate test? If it has, then - /// subsequent matchables on the same leaf are unreachable. If it hasn't, the - /// code that requested the GIMatchTree is responsible for finishing off any - /// remaining predicates. - bool IsFullyTested; - /// The variable bindings associated with this leaf so far. - std::vector<GIMatchTreeVariableBinding> VarBindings; - /// Any predicates left untested by the time we reach this leaf. - UntestedPredicatesTy UntestedPredicates; - -public: - GIMatchTreeLeafInfo() { llvm_unreachable("Cannot default-construct"); } - GIMatchTreeLeafInfo(StringRef Name, unsigned RootIdx, void *Data) - : Name(Name), RootIdx(RootIdx), Data(Data), IsFullyTraversed(false), - IsFullyTested(false) {} - - StringRef getName() const { return Name; } - unsigned getRootIdx() const { return RootIdx; } - template <class Ty> Ty *getTargetData() const { - return static_cast<Ty *>(Data); - } - bool isFullyTraversed() const { return IsFullyTraversed; } - void setIsFullyTraversed(bool V) { IsFullyTraversed = V; } - bool isFullyTested() const { return IsFullyTested; } - void setIsFullyTested(bool V) { IsFullyTested = V; } - - void bindInstrVariable(StringRef Name, unsigned InstrID) { - VarBindings.emplace_back(Name, InstrID); - } - void bindOperandVariable(StringRef Name, unsigned InstrID, unsigned OpIdx) { - VarBindings.emplace_back(Name, InstrID, OpIdx); - } - - const_var_binding_iterator var_bindings_begin() const { - return VarBindings.begin(); - } - const_var_binding_iterator var_bindings_end() const { - return VarBindings.end(); - } - iterator_range<const_var_binding_iterator> var_bindings() const { - return make_range(VarBindings.begin(), VarBindings.end()); - } - iterator_range<const_untested_predicates_iterator> untested_predicates() const { - return make_range(UntestedPredicates.begin(), UntestedPredicates.end()); - } - void addUntestedPredicate(const GIMatchDagPredicate *P) { - UntestedPredicates.push_back(P); - } -}; - -/// The nodes of a decision tree used to perform the match. -/// This will be used to generate the C++ code or state machine equivalent. -/// -/// It should be noted that some nodes of this tree (most notably nodes handling -/// def -> use edges) will need to iterate over several possible matches. As -/// such, code generated from this will sometimes need to support backtracking. -class GIMatchTree { - using LeafVector = std::vector<GIMatchTreeLeafInfo>; - - /// The partitioner that has been chosen for this node. This may be nullptr if - /// a partitioner hasn't been chosen yet or if the node is a leaf. - std::unique_ptr<GIMatchTreePartitioner> Partitioner; - /// All the leaves that are possible for this node of the tree. - /// Note: This should be emptied after the tree is built when there are - /// children but this currently isn't done to aid debuggability of the DOT - /// graph for the decision tree. - LeafVector PossibleLeaves; - /// The children of this node. The index into this array must match the index - /// chosen by the partitioner. - std::vector<GIMatchTree> Children; - - void writeDOTGraphNode(raw_ostream &OS) const; - void writeDOTGraphEdges(raw_ostream &OS) const; - -public: - void writeDOTGraph(raw_ostream &OS) const; - - void setNumChildren(unsigned Num) { Children.resize(Num); } - void addPossibleLeaf(const GIMatchTreeLeafInfo &V, bool IsFullyTraversed, - bool IsFullyTested) { - PossibleLeaves.push_back(V); - PossibleLeaves.back().setIsFullyTraversed(IsFullyTraversed); - PossibleLeaves.back().setIsFullyTested(IsFullyTested); - } - void dropLeavesAfter(size_t Length) { - if (PossibleLeaves.size() > Length) - PossibleLeaves.resize(Length); - } - void setPartitioner(std::unique_ptr<GIMatchTreePartitioner> &&V) { - Partitioner = std::move(V); - } - GIMatchTreePartitioner *getPartitioner() const { return Partitioner.get(); } - - std::vector<GIMatchTree>::iterator children_begin() { - return Children.begin(); - } - std::vector<GIMatchTree>::iterator children_end() { return Children.end(); } - iterator_range<std::vector<GIMatchTree>::iterator> children() { - return make_range(children_begin(), children_end()); - } - std::vector<GIMatchTree>::const_iterator children_begin() const { - return Children.begin(); - } - std::vector<GIMatchTree>::const_iterator children_end() const { - return Children.end(); - } - iterator_range<std::vector<GIMatchTree>::const_iterator> children() const { - return make_range(children_begin(), children_end()); - } - - LeafVector::const_iterator possible_leaves_begin() const { - return PossibleLeaves.begin(); - } - LeafVector::const_iterator possible_leaves_end() const { - return PossibleLeaves.end(); - } - iterator_range<LeafVector::const_iterator> - possible_leaves() const { - return make_range(possible_leaves_begin(), possible_leaves_end()); - } - LeafVector::iterator possible_leaves_begin() { - return PossibleLeaves.begin(); - } - LeafVector::iterator possible_leaves_end() { - return PossibleLeaves.end(); - } - iterator_range<LeafVector::iterator> possible_leaves() { - return make_range(possible_leaves_begin(), possible_leaves_end()); - } -}; - -/// Record information that is known about the instruction bound to this ID and -/// GIMatchDagInstrNode. Every rule gets its own set of -/// GIMatchTreeInstrInfo to bind the shared IDs to an instr node in its -/// DAG. -/// -/// For example, if we know that there are 3 operands. We can record it here to -/// elide duplicate checks. -class GIMatchTreeInstrInfo { - /// The instruction ID for the matched instruction. - unsigned ID; - /// The corresponding instruction node in the MatchDAG. - const GIMatchDagInstr *InstrNode; - -public: - GIMatchTreeInstrInfo(unsigned ID, const GIMatchDagInstr *InstrNode) - : ID(ID), InstrNode(InstrNode) {} - - unsigned getID() const { return ID; } - const GIMatchDagInstr *getInstrNode() const { return InstrNode; } -}; - -/// Record information that is known about the operand bound to this ID, OpIdx, -/// and GIMatchDagInstrNode. Every rule gets its own set of -/// GIMatchTreeOperandInfo to bind the shared IDs to an operand of an -/// instr node from its DAG. -/// -/// For example, if we know that there the operand is a register. We can record -/// it here to elide duplicate checks. -class GIMatchTreeOperandInfo { - /// The corresponding instruction node in the MatchDAG that the operand - /// belongs to. - const GIMatchDagInstr *InstrNode; - unsigned OpIdx; - -public: - GIMatchTreeOperandInfo(const GIMatchDagInstr *InstrNode, unsigned OpIdx) - : InstrNode(InstrNode), OpIdx(OpIdx) {} - - const GIMatchDagInstr *getInstrNode() const { return InstrNode; } - unsigned getOpIdx() const { return OpIdx; } -}; - -/// Represent a leaf of the match tree and any working data we need to build the -/// tree. -/// -/// It's important to note that each rule can have multiple -/// GIMatchTreeBuilderLeafInfo's since the partitioners do not always partition -/// into mutually-exclusive partitions. For example: -/// R1: (FOO ..., ...) -/// R2: (oneof(FOO, BAR) ..., ...) -/// will partition by opcode into two partitions FOO=>[R1, R2], and BAR=>[R2] -/// -/// As an optimization, all instructions, edges, and predicates in the DAGs are -/// numbered and tracked in BitVectors. As such, the GIMatchDAG must not be -/// modified once construction of the tree has begun. -class GIMatchTreeBuilderLeafInfo { -protected: - GIMatchTreeBuilder &Builder; - GIMatchTreeLeafInfo Info; - const GIMatchDag &MatchDag; - /// The association between GIMatchDagInstr* and GIMatchTreeInstrInfo. - /// The primary reason for this members existence is to allow the use of - /// InstrIDToInfo.lookup() since that requires that the value is - /// default-constructible. - DenseMap<const GIMatchDagInstr *, GIMatchTreeInstrInfo> InstrNodeToInfo; - /// The instruction information for a given ID in the context of this - /// particular leaf. - DenseMap<unsigned, GIMatchTreeInstrInfo *> InstrIDToInfo; - /// The operand information for a given ID and OpIdx in the context of this - /// particular leaf. - DenseMap<std::pair<unsigned, unsigned>, GIMatchTreeOperandInfo> - OperandIDToInfo; - -public: - /// The remaining instrs/edges/predicates to visit - BitVector RemainingInstrNodes; - BitVector RemainingEdges; - BitVector RemainingPredicates; - - // The remaining predicate dependencies for each predicate - std::vector<BitVector> UnsatisfiedPredDepsForPred; - - /// The edges/predicates we can visit as a result of the declare*() calls we - /// have already made. We don't need an instrs version since edges imply the - /// instr. - BitVector TraversableEdges; - BitVector TestablePredicates; - - /// Map predicates from the DAG to their position in the DAG predicate - /// iterators. - DenseMap<GIMatchDagPredicate *, unsigned> PredicateIDs; - /// Map predicate dependency edges from the DAG to their position in the DAG - /// predicate dependency iterators. - DenseMap<GIMatchDagPredicateDependencyEdge *, unsigned> PredicateDepIDs; - -public: - GIMatchTreeBuilderLeafInfo(GIMatchTreeBuilder &Builder, StringRef Name, - unsigned RootIdx, const GIMatchDag &MatchDag, - void *Data); - - StringRef getName() const { return Info.getName(); } - GIMatchTreeLeafInfo &getInfo() { return Info; } - const GIMatchTreeLeafInfo &getInfo() const { return Info; } - const GIMatchDag &getMatchDag() const { return MatchDag; } - unsigned getRootIdx() const { return Info.getRootIdx(); } - - /// Has this DAG been fully traversed. This must be true by the time the tree - /// builder finishes. - bool isFullyTraversed() const { - // We don't need UnsatisfiedPredDepsForPred because RemainingPredicates - // can't be all-zero without satisfying all the dependencies. The same is - // almost true for Edges and Instrs but it's possible to have Instrs without - // Edges. - return RemainingInstrNodes.none() && RemainingEdges.none(); - } - - /// Has this DAG been fully tested. This hould be true by the time the tree - /// builder finishes but clients can finish any untested predicates left over - /// if it's not true. - bool isFullyTested() const { - // We don't need UnsatisfiedPredDepsForPred because RemainingPredicates - // can't be all-zero without satisfying all the dependencies. The same is - // almost true for Edges and Instrs but it's possible to have Instrs without - // Edges. - return RemainingInstrNodes.none() && RemainingEdges.none() && - RemainingPredicates.none(); - } - - const GIMatchDagInstr *getInstr(unsigned Idx) const { - return *(MatchDag.instr_nodes_begin() + Idx); - } - const GIMatchDagEdge *getEdge(unsigned Idx) const { - return *(MatchDag.edges_begin() + Idx); - } - GIMatchDagEdge *getEdge(unsigned Idx) { - return *(MatchDag.edges_begin() + Idx); - } - const GIMatchDagPredicate *getPredicate(unsigned Idx) const { - return *(MatchDag.predicates_begin() + Idx); - } - iterator_range<llvm::BitVector::const_set_bits_iterator> - untested_instrs() const { - return RemainingInstrNodes.set_bits(); - } - iterator_range<llvm::BitVector::const_set_bits_iterator> - untested_edges() const { - return RemainingEdges.set_bits(); - } - iterator_range<llvm::BitVector::const_set_bits_iterator> - untested_predicates() const { - return RemainingPredicates.set_bits(); - } - - /// Bind an instr node to the given ID and clear any blocking dependencies - /// that were waiting for it. - void declareInstr(const GIMatchDagInstr *Instr, unsigned ID); - - /// Bind an operand to the given ID and OpIdx and clear any blocking - /// dependencies that were waiting for it. - void declareOperand(unsigned InstrID, unsigned OpIdx); - - GIMatchTreeInstrInfo *getInstrInfo(unsigned ID) const { - return InstrIDToInfo.lookup(ID); - } - - void dump(raw_ostream &OS) const { - OS << "Leaf " << getName() << " for root #" << getRootIdx() << "\n"; - MatchDag.print(OS); - for (const auto &I : InstrIDToInfo) - OS << "Declared Instr #" << I.first << "\n"; - for (const auto &I : OperandIDToInfo) - OS << "Declared Instr #" << I.first.first << ", Op #" << I.first.second - << "\n"; - OS << RemainingInstrNodes.count() << " untested instrs of " - << RemainingInstrNodes.size() << "\n"; - OS << RemainingEdges.count() << " untested edges of " - << RemainingEdges.size() << "\n"; - OS << RemainingPredicates.count() << " untested predicates of " - << RemainingPredicates.size() << "\n"; - - OS << TraversableEdges.count() << " edges could be traversed\n"; - OS << TestablePredicates.count() << " predicates could be tested\n"; - } -}; - -/// The tree builder has a fairly tough job. It's purpose is to merge all the -/// DAGs from the ruleset into a decision tree that walks all of them -/// simultaneously and identifies the rule that was matched. In addition to -/// that, it also needs to find the most efficient order to make decisions -/// without violating any dependencies and ensure that every DAG covers every -/// instr/edge/predicate. -class GIMatchTreeBuilder { -public: - using LeafVec = std::vector<GIMatchTreeBuilderLeafInfo>; - -protected: - /// The leaves that the resulting decision tree will distinguish. - LeafVec Leaves; - /// The tree node being constructed. - GIMatchTree *TreeNode = nullptr; - /// The builders for each subtree resulting from the current decision. - std::vector<GIMatchTreeBuilder> SubtreeBuilders; - /// The possible partitioners we could apply right now. - std::vector<std::unique_ptr<GIMatchTreePartitioner>> Partitioners; - /// The next instruction ID to allocate when requested by the chosen - /// Partitioner. - unsigned NextInstrID; - - /// Use any context we have stored to cull partitioners that only test things - /// we already know. At the time of writing, there's no need to do anything - /// here but it will become important once, for example, there is a - /// num-operands and an opcode partitioner. This is because applying an opcode - /// partitioner (usually) makes the number of operands known which makes - /// additional checking pointless. - void filterRedundantPartitioners(); - - /// Evaluate the available partioners and select the best one at the moment. - void evaluatePartitioners(); - - /// Construct the current tree node. - void runStep(); - -public: - GIMatchTreeBuilder(unsigned NextInstrID) : NextInstrID(NextInstrID) {} - GIMatchTreeBuilder(GIMatchTree *TreeNode, unsigned NextInstrID) - : TreeNode(TreeNode), NextInstrID(NextInstrID) {} - - void addLeaf(StringRef Name, unsigned RootIdx, const GIMatchDag &MatchDag, - void *Data) { - Leaves.emplace_back(*this, Name, RootIdx, MatchDag, Data); - } - void addLeaf(const GIMatchTreeBuilderLeafInfo &L) { Leaves.push_back(L); } - void addPartitioner(std::unique_ptr<GIMatchTreePartitioner> P) { - Partitioners.push_back(std::move(P)); - } - void addPartitionersForInstr(unsigned InstrIdx); - void addPartitionersForOperand(unsigned InstrID, unsigned OpIdx); - - LeafVec &getPossibleLeaves() { return Leaves; } - - unsigned allocInstrID() { return NextInstrID++; } - - /// Construct the decision tree. - std::unique_ptr<GIMatchTree> run(); -}; - -/// Partitioners are the core of the tree builder and are unfortunately rather -/// tricky to write. -class GIMatchTreePartitioner { -protected: - /// The partitions resulting from applying the partitioner to the possible - /// leaves. The keys must be consecutive integers starting from 0. This can - /// lead to some unfortunate situations where partitioners test a predicate - /// and use 0 for success and 1 for failure if the ruleset encounters a - /// success case first but is necessary to assign the partition to one of the - /// tree nodes children. As a result, you usually need some kind of - /// indirection to map the natural keys (e.g. ptrs/bools) to this linear - /// sequence. The values are a bitvector indicating which leaves belong to - /// this partition. - DenseMap<unsigned, BitVector> Partitions; - -public: - virtual ~GIMatchTreePartitioner() {} - virtual std::unique_ptr<GIMatchTreePartitioner> clone() const = 0; - - /// Determines which partitions the given leaves belong to. A leaf may belong - /// to multiple partitions in which case it will be duplicated during - /// applyForPartition(). - /// - /// This function can be rather complicated. A few particular things to be - /// aware of include: - /// * One leaf can be assigned to multiple partitions when there's some - /// ambiguity. - /// * Not all DAG's for the leaves may be able to perform the test. For - /// example, the opcode partitiioner must account for one DAG being a - /// superset of another such as [(ADD ..., ..., ...)], and [(MUL t, ..., - /// ...), (ADD ..., t, ...)] - /// * Attaching meaning to a particular partition index will generally not - /// work due to the '0, 1, ..., n' requirement. You might encounter cases - /// where only partition 1 is seen, leaving a missing 0. - /// * Finding a specific predicate such as the opcode predicate for a specific - /// instruction is non-trivial. It's often O(NumPredicates), leading to - /// O(NumPredicates*NumRules) when applied to the whole ruleset. The good - /// news there is that n is typically small thanks to predicate dependencies - /// limiting how many are testable at once. Also, with opcode and type - /// predicates being so frequent the value of m drops very fast too. It - /// wouldn't be terribly surprising to see a 10k ruleset drop down to an - /// average of 100 leaves per partition after a single opcode partitioner. - /// * The same goes for finding specific edges. The need to traverse them in - /// dependency order dramatically limits the search space at any given - /// moment. - /// * If you need to add a leaf to all partitions, make sure you don't forget - /// them when adding partitions later. - virtual void repartition(GIMatchTreeBuilder::LeafVec &Leaves) = 0; - - /// Delegate the leaves for a given partition to the corresponding subbuilder, - /// update any recorded context for this partition (e.g. allocate instr id's - /// for instrs recorder by the current node), and clear any blocking - /// dependencies this partitioner resolved. - virtual void applyForPartition(unsigned PartitionIdx, - GIMatchTreeBuilder &Builder, - GIMatchTreeBuilder &SubBuilder) = 0; - - /// Return a BitVector indicating which leaves should be transferred to the - /// specified partition. Note that the same leaf can be indicated for multiple - /// partitions. - BitVector getPossibleLeavesForPartition(unsigned Idx) { - const auto &I = Partitions.find(Idx); - assert(I != Partitions.end() && "Requested non-existant partition"); - return I->second; - } - - size_t getNumPartitions() const { return Partitions.size(); } - size_t getNumLeavesWithDupes() const { - size_t S = 0; - for (const auto &P : Partitions) - S += P.second.size(); - return S; - } - - /// Emit a brief description of the partitioner suitable for debug printing or - /// use in a DOT graph. - virtual void emitDescription(raw_ostream &OS) const = 0; - /// Emit a label for the given partition suitable for debug printing or use in - /// a DOT graph. - virtual void emitPartitionName(raw_ostream &OS, unsigned Idx) const = 0; - - /// Emit a long description of how the partitioner partitions the leaves. - virtual void emitPartitionResults(raw_ostream &OS) const = 0; - - /// Generate code to select between partitions based on the MIR being matched. - /// This is typically a switch statement that picks a partition index. - virtual void generatePartitionSelectorCode(raw_ostream &OS, - StringRef Indent) const = 0; -}; - -/// Partition according to the opcode of the instruction. -/// -/// Numbers CodeGenInstr ptrs for use as partition ID's. One special partition, -/// nullptr, represents the case where the instruction isn't known. -/// -/// * If the opcode can be tested and is a single opcode, create the partition -/// for that opcode and assign the leaf to it. This partition no longer needs -/// to test the opcode, and many details about the instruction will usually -/// become known (e.g. number of operands for non-variadic instrs) via the -/// CodeGenInstr ptr. -/// * (not implemented yet) If the opcode can be tested and is a choice of -/// opcodes, then the leaf can be treated like the single-opcode case but must -/// be added to all relevant partitions and not quite as much becomes known as -/// a result. That said, multiple-choice opcodes are likely similar enough -/// (because if they aren't then handling them together makes little sense) -/// that plenty still becomes known. The main implementation issue with this -/// is having a description to represent the commonality between instructions. -/// * If the opcode is not tested, the leaf must be added to all partitions -/// including the wildcard nullptr partition. What becomes known as a result -/// varies between partitions. -/// * If the instruction to be tested is not declared then add the leaf to all -/// partitions. This occurs when we encounter one rule that is a superset of -/// the other and we are still matching the remainder of the superset. The -/// result is that the cases that don't match the superset will match the -/// subset rule, while the ones that do match the superset will match either -/// (which one is algorithm dependent but will usually be the superset). -class GIMatchTreeOpcodePartitioner : public GIMatchTreePartitioner { - unsigned InstrID; - DenseMap<const CodeGenInstruction *, unsigned> InstrToPartition; - std::vector<const CodeGenInstruction *> PartitionToInstr; - std::vector<BitVector> TestedPredicates; - -public: - GIMatchTreeOpcodePartitioner(unsigned InstrID) : InstrID(InstrID) {} - - std::unique_ptr<GIMatchTreePartitioner> clone() const override { - return std::make_unique<GIMatchTreeOpcodePartitioner>(*this); - } - - void emitDescription(raw_ostream &OS) const override { - OS << "MI[" << InstrID << "].getOpcode()"; - } - - void emitPartitionName(raw_ostream &OS, unsigned Idx) const override; - - void repartition(GIMatchTreeBuilder::LeafVec &Leaves) override; - void applyForPartition(unsigned Idx, GIMatchTreeBuilder &SubBuilder, - GIMatchTreeBuilder &Builder) override; - - void emitPartitionResults(raw_ostream &OS) const override; - - void generatePartitionSelectorCode(raw_ostream &OS, - StringRef Indent) const override; -}; - -class GIMatchTreeVRegDefPartitioner : public GIMatchTreePartitioner { - unsigned NewInstrID = -1; - unsigned InstrID; - unsigned OpIdx; - std::vector<BitVector> TraversedEdges; - DenseMap<unsigned, unsigned> ResultToPartition; - BitVector PartitionToResult; - - void addToPartition(bool Result, unsigned LeafIdx); - -public: - GIMatchTreeVRegDefPartitioner(unsigned InstrID, unsigned OpIdx) - : InstrID(InstrID), OpIdx(OpIdx) {} - - std::unique_ptr<GIMatchTreePartitioner> clone() const override { - return std::make_unique<GIMatchTreeVRegDefPartitioner>(*this); - } - - void emitDescription(raw_ostream &OS) const override { - OS << "MI[" << NewInstrID << "] = getVRegDef(MI[" << InstrID - << "].getOperand(" << OpIdx << "))"; - } - - void emitPartitionName(raw_ostream &OS, unsigned Idx) const override { - bool Result = PartitionToResult[Idx]; - if (Result) - OS << "true"; - else - OS << "false"; - } - - void repartition(GIMatchTreeBuilder::LeafVec &Leaves) override; - void applyForPartition(unsigned PartitionIdx, GIMatchTreeBuilder &Builder, - GIMatchTreeBuilder &SubBuilder) override; - void emitPartitionResults(raw_ostream &OS) const override; - - void generatePartitionSelectorCode(raw_ostream &OS, - StringRef Indent) const override; -}; - -} // end namespace llvm -#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHTREE_H diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp b/llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp new file mode 100644 index 000000000000..b5c9e4f8c248 --- /dev/null +++ b/llvm/utils/TableGen/GlobalISel/MatchDataInfo.cpp @@ -0,0 +1,49 @@ +//===- MatchDataInfo.cpp ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// +//===----------------------------------------------------------------------===// + +#include "MatchDataInfo.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace gi { + +StringMap<std::vector<std::string>> AllMatchDataVars; + +StringRef MatchDataInfo::getVariableName() const { + assert(hasVariableName()); + return VarName; +} + +void MatchDataInfo::print(raw_ostream &OS) const { + OS << "(MatchDataInfo pattern_symbol:" << PatternSymbol << " type:'" << Type + << "' var_name:" << (VarName.empty() ? "<unassigned>" : VarName) << ")"; +} + +void MatchDataInfo::dump() const { print(dbgs()); } + +void AssignMatchDataVariables(MutableArrayRef<MatchDataInfo> Infos) { + static unsigned NextVarID = 0; + + StringMap<unsigned> SeenTypes; + for (auto &Info : Infos) { + unsigned &NumSeen = SeenTypes[Info.getType()]; + auto &ExistingVars = AllMatchDataVars[Info.getType()]; + + if (NumSeen == ExistingVars.size()) + ExistingVars.push_back("MDInfo" + std::to_string(NextVarID++)); + + Info.setVariableName(ExistingVars[NumSeen++]); + } +} + +} // namespace gi +} // namespace llvm diff --git a/llvm/utils/TableGen/GlobalISel/MatchDataInfo.h b/llvm/utils/TableGen/GlobalISel/MatchDataInfo.h new file mode 100644 index 000000000000..abe1245bc67d --- /dev/null +++ b/llvm/utils/TableGen/GlobalISel/MatchDataInfo.h @@ -0,0 +1,90 @@ +//===- MatchDataInfo.h ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Contains utilities related to handling "match data" for GlobalISel +/// Combiners. Match data allows for setting some arbitrary data in the "match" +/// phase and pass it down to the "apply" phase. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_MIRPATTERNS_MATCHDATAINFO_H +#define LLVM_UTILS_MIRPATTERNS_MATCHDATAINFO_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include <string> +#include <vector> + +namespace llvm { + +class raw_ostream; + +namespace gi { + +/// Represents MatchData defined by the match stage and required by the apply +/// stage. +/// +/// This allows the plumbing of arbitrary data from C++ predicates between the +/// stages. +/// +/// When this class is initially created, it only has a pattern symbol and a +/// type. When all of the MatchDatas declarations of a given pattern have been +/// parsed, `AssignVariables` must be called to assign storage variable names to +/// each MatchDataInfo. +class MatchDataInfo { + StringRef PatternSymbol; + StringRef Type; + std::string VarName; + +public: + static constexpr StringLiteral StructTypeName = "MatchInfosTy"; + static constexpr StringLiteral StructName = "MatchInfos"; + + MatchDataInfo(StringRef PatternSymbol, StringRef Type) + : PatternSymbol(PatternSymbol), Type(Type.trim()) {} + + StringRef getPatternSymbol() const { return PatternSymbol; }; + StringRef getType() const { return Type; }; + + bool hasVariableName() const { return !VarName.empty(); } + void setVariableName(StringRef Name) { VarName = Name; } + StringRef getVariableName() const; + + std::string getQualifiedVariableName() const { + return StructName.str() + "." + getVariableName().str(); + } + + void print(raw_ostream &OS) const; + void dump() const; +}; + +/// Pool of type -> variables used to emit MatchData variables declarations. +/// +/// e.g. if the map contains "int64_t" -> ["MD0", "MD1"], then two variable +/// declarations must be emitted: `int64_t MD0` and `int64_t MD1`. +/// +/// This has a static lifetime and will outlive all the `MatchDataInfo` objects +/// by design. It needs a static lifetime so the backends can emit variable +/// declarations after processing all the inputs. +extern StringMap<std::vector<std::string>> AllMatchDataVars; + +/// Assign variable names to all MatchDatas used by a pattern. This must be +/// called after all MatchData decls have been parsed for a given processing +/// unit (e.g. a combine rule) +/// +/// Requires an array of MatchDataInfo so we can handle cases where a pattern +/// uses multiple instances of the same MatchData type. +/// +/// Writes to \ref AllMatchDataVars. +void AssignMatchDataVariables(MutableArrayRef<MatchDataInfo> Infos); + +} // namespace gi +} // end namespace llvm + +#endif // ifndef LLVM_UTILS_MIRPATTERNS_MATCHDATAINFO_H diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.cpp b/llvm/utils/TableGen/GlobalISel/Patterns.cpp new file mode 100644 index 000000000000..0a6d05e06dca --- /dev/null +++ b/llvm/utils/TableGen/GlobalISel/Patterns.cpp @@ -0,0 +1,842 @@ +//===- Patterns.cpp --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Patterns.h" +#include "../CodeGenInstruction.h" +#include "CXXPredicates.h" +#include "CodeExpander.h" +#include "CodeExpansions.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" + +namespace llvm { +namespace gi { + +//===- PatternType --------------------------------------------------------===// + +std::optional<PatternType> PatternType::get(ArrayRef<SMLoc> DiagLoc, + const Record *R, Twine DiagCtx) { + assert(R); + if (R->isSubClassOf("ValueType")) { + PatternType PT(PT_ValueType); + PT.Data.Def = R; + return PT; + } + + if (R->isSubClassOf(TypeOfClassName)) { + auto RawOpName = R->getValueAsString("OpName"); + if (!RawOpName.starts_with("$")) { + PrintError(DiagLoc, DiagCtx + ": invalid operand name format '" + + RawOpName + "' in " + TypeOfClassName + + ": expected '$' followed by an operand name"); + return std::nullopt; + } + + PatternType PT(PT_TypeOf); + PT.Data.Str = RawOpName.drop_front(1); + return PT; + } + + PrintError(DiagLoc, DiagCtx + ": unknown type '" + R->getName() + "'"); + return std::nullopt; +} + +PatternType PatternType::getTypeOf(StringRef OpName) { + PatternType PT(PT_TypeOf); + PT.Data.Str = OpName; + return PT; +} + +StringRef PatternType::getTypeOfOpName() const { + assert(isTypeOf()); + return Data.Str; +} + +const Record *PatternType::getLLTRecord() const { + assert(isLLT()); + return Data.Def; +} + +bool PatternType::operator==(const PatternType &Other) const { + if (Kind != Other.Kind) + return false; + + switch (Kind) { + case PT_None: + return true; + case PT_ValueType: + return Data.Def == Other.Data.Def; + case PT_TypeOf: + return Data.Str == Other.Data.Str; + } + + llvm_unreachable("Unknown Type Kind"); +} + +std::string PatternType::str() const { + switch (Kind) { + case PT_None: + return ""; + case PT_ValueType: + return Data.Def->getName().str(); + case PT_TypeOf: + return (TypeOfClassName + "<$" + getTypeOfOpName() + ">").str(); + } + + llvm_unreachable("Unknown type!"); +} + +//===- Pattern ------------------------------------------------------------===// + +void Pattern::dump() const { return print(dbgs()); } + +const char *Pattern::getKindName() const { + switch (Kind) { + case K_AnyOpcode: + return "AnyOpcodePattern"; + case K_CXX: + return "CXXPattern"; + case K_CodeGenInstruction: + return "CodeGenInstructionPattern"; + case K_PatFrag: + return "PatFragPattern"; + case K_Builtin: + return "BuiltinPattern"; + } + + llvm_unreachable("unknown pattern kind!"); +} + +void Pattern::printImpl(raw_ostream &OS, bool PrintName, + function_ref<void()> ContentPrinter) const { + OS << "(" << getKindName() << " "; + if (PrintName) + OS << "name:" << getName() << " "; + ContentPrinter(); + OS << ")"; +} + +//===- AnyOpcodePattern ---------------------------------------------------===// + +void AnyOpcodePattern::print(raw_ostream &OS, bool PrintName) const { + printImpl(OS, PrintName, [&OS, this]() { + OS << "[" + << join(map_range(Insts, + [](const auto *I) { return I->TheDef->getName(); }), + ", ") + << "]"; + }); +} + +//===- CXXPattern ---------------------------------------------------------===// + +CXXPattern::CXXPattern(const StringInit &Code, StringRef Name) + : CXXPattern(Code.getAsUnquotedString(), Name) {} + +const CXXPredicateCode & +CXXPattern::expandCode(const CodeExpansions &CE, ArrayRef<SMLoc> Locs, + function_ref<void(raw_ostream &)> AddComment) const { + std::string Result; + raw_string_ostream OS(Result); + + if (AddComment) + AddComment(OS); + + CodeExpander Expander(RawCode, CE, Locs, /*ShowExpansions*/ false); + Expander.emit(OS); + if (IsApply) + return CXXPredicateCode::getApplyCode(std::move(Result)); + return CXXPredicateCode::getMatchCode(std::move(Result)); +} + +void CXXPattern::print(raw_ostream &OS, bool PrintName) const { + printImpl(OS, PrintName, [&OS, this] { + OS << (IsApply ? "apply" : "match") << " code:\""; + printEscapedString(getRawCode(), OS); + OS << "\""; + }); +} + +//===- InstructionOperand -------------------------------------------------===// + +std::string InstructionOperand::describe() const { + if (!hasImmValue()) + return "MachineOperand $" + getOperandName().str() + ""; + std::string Str = "imm " + std::to_string(getImmValue()); + if (isNamedImmediate()) + Str += ":$" + getOperandName().str() + ""; + return Str; +} + +void InstructionOperand::print(raw_ostream &OS) const { + if (isDef()) + OS << "<def>"; + + bool NeedsColon = true; + if (Type) { + if (hasImmValue()) + OS << "(" << Type.str() << " " << getImmValue() << ")"; + else + OS << Type.str(); + } else if (hasImmValue()) + OS << getImmValue(); + else + NeedsColon = false; + + if (isNamedOperand()) + OS << (NeedsColon ? ":" : "") << "$" << getOperandName(); +} + +void InstructionOperand::dump() const { return print(dbgs()); } + +//===- InstructionPattern -------------------------------------------------===// + +bool InstructionPattern::diagnoseAllSpecialTypes(ArrayRef<SMLoc> Loc, + Twine Msg) const { + bool HasDiag = false; + for (const auto &[Idx, Op] : enumerate(operands())) { + if (Op.getType().isSpecial()) { + PrintError(Loc, Msg); + PrintNote(Loc, "operand " + Twine(Idx) + " of '" + getName() + + "' has type '" + Op.getType().str() + "'"); + HasDiag = true; + } + } + return HasDiag; +} + +void InstructionPattern::reportUnreachable(ArrayRef<SMLoc> Locs) const { + PrintError(Locs, "pattern '" + getName() + "' ('" + getInstName() + + "') is unreachable from the pattern root!"); +} + +bool InstructionPattern::checkSemantics(ArrayRef<SMLoc> Loc) { + unsigned NumExpectedOperands = getNumInstOperands(); + + if (isVariadic()) { + if (Operands.size() < NumExpectedOperands) { + PrintError(Loc, +"'" + getInstName() + "' expected at least " + + Twine(NumExpectedOperands) + " operands, got " + + Twine(Operands.size())); + return false; + } + } else if (NumExpectedOperands != Operands.size()) { + PrintError(Loc, +"'" + getInstName() + "' expected " + + Twine(NumExpectedOperands) + " operands, got " + + Twine(Operands.size())); + return false; + } + + unsigned OpIdx = 0; + unsigned NumDefs = getNumInstDefs(); + for (auto &Op : Operands) + Op.setIsDef(OpIdx++ < NumDefs); + + return true; +} + +void InstructionPattern::print(raw_ostream &OS, bool PrintName) const { + printImpl(OS, PrintName, [&OS, this] { + OS << getInstName() << " operands:["; + StringRef Sep; + for (const auto &Op : Operands) { + OS << Sep; + Op.print(OS); + Sep = ", "; + } + OS << "]"; + + printExtras(OS); + }); +} + +//===- OperandTable -------------------------------------------------------===// + +bool OperandTable::addPattern(InstructionPattern *P, + function_ref<void(StringRef)> DiagnoseRedef) { + for (const auto &Op : P->named_operands()) { + StringRef OpName = Op.getOperandName(); + + // We always create an entry in the OperandTable, even for uses. + // Uses of operands that don't have a def (= live-ins) will remain with a + // nullptr as the Def. + // + // This allows us tell whether an operand exists in a pattern or not. If + // there is no entry for it, it doesn't exist, if there is an entry, it's + // used/def'd at least once. + auto &Def = Table[OpName]; + + if (!Op.isDef()) + continue; + + if (Def) { + DiagnoseRedef(OpName); + return false; + } + + Def = P; + } + + return true; +} + +void OperandTable::print(raw_ostream &OS, StringRef Name, + StringRef Indent) const { + OS << Indent << "(OperandTable "; + if (!Name.empty()) + OS << Name << " "; + if (Table.empty()) { + OS << "<empty>)\n"; + return; + } + + SmallVector<StringRef, 0> Keys(Table.keys()); + sort(Keys); + + OS << '\n'; + for (const auto &Key : Keys) { + const auto *Def = Table.at(Key); + OS << Indent << " " << Key << " -> " + << (Def ? Def->getName() : "<live-in>") << '\n'; + } + OS << Indent << ")\n"; +} + +void OperandTable::dump() const { print(dbgs()); } + +//===- MIFlagsInfo --------------------------------------------------------===// + +void MIFlagsInfo::addSetFlag(const Record *R) { + SetF.insert(R->getValueAsString("EnumName")); +} + +void MIFlagsInfo::addUnsetFlag(const Record *R) { + UnsetF.insert(R->getValueAsString("EnumName")); +} + +void MIFlagsInfo::addCopyFlag(StringRef InstName) { CopyF.insert(InstName); } + +//===- CodeGenInstructionPattern ------------------------------------------===// + +bool CodeGenInstructionPattern::is(StringRef OpcodeName) const { + return I.TheDef->getName() == OpcodeName; +} + +bool CodeGenInstructionPattern::isVariadic() const { + return I.Operands.isVariadic; +} + +bool CodeGenInstructionPattern::hasVariadicDefs() const { + // Note: we cannot use variadicOpsAreDefs, it's not set for + // GenericInstructions. + if (!isVariadic()) + return false; + + if (I.variadicOpsAreDefs) + return true; + + DagInit *OutOps = I.TheDef->getValueAsDag("OutOperandList"); + if (OutOps->arg_empty()) + return false; + + auto *LastArgTy = dyn_cast<DefInit>(OutOps->getArg(OutOps->arg_size() - 1)); + return LastArgTy && LastArgTy->getDef()->getName() == "variable_ops"; +} + +unsigned CodeGenInstructionPattern::getNumInstDefs() const { + if (!isVariadic() || !hasVariadicDefs()) + return I.Operands.NumDefs; + unsigned NumOuts = I.Operands.size() - I.Operands.NumDefs; + assert(Operands.size() > NumOuts); + return std::max<unsigned>(I.Operands.NumDefs, Operands.size() - NumOuts); +} + +unsigned CodeGenInstructionPattern::getNumInstOperands() const { + unsigned NumCGIOps = I.Operands.size(); + return isVariadic() ? std::max<unsigned>(NumCGIOps, Operands.size()) + : NumCGIOps; +} + +MIFlagsInfo &CodeGenInstructionPattern::getOrCreateMIFlagsInfo() { + if (!FI) + FI = std::make_unique<MIFlagsInfo>(); + return *FI; +} + +StringRef CodeGenInstructionPattern::getInstName() const { + return I.TheDef->getName(); +} + +void CodeGenInstructionPattern::printExtras(raw_ostream &OS) const { + if (!FI) + return; + + OS << " (MIFlags"; + if (!FI->set_flags().empty()) + OS << " (set " << join(FI->set_flags(), ", ") << ")"; + if (!FI->unset_flags().empty()) + OS << " (unset " << join(FI->unset_flags(), ", ") << ")"; + if (!FI->copy_flags().empty()) + OS << " (copy " << join(FI->copy_flags(), ", ") << ")"; + OS << ')'; +} + +//===- OperandTypeChecker -------------------------------------------------===// + +bool OperandTypeChecker::check( + InstructionPattern &P, + std::function<bool(const PatternType &)> VerifyTypeOfOperand) { + Pats.push_back(&P); + + for (auto &Op : P.operands()) { + const auto Ty = Op.getType(); + if (!Ty) + continue; + + if (Ty.isTypeOf() && !VerifyTypeOfOperand(Ty)) + return false; + + if (!Op.isNamedOperand()) + continue; + + StringRef OpName = Op.getOperandName(); + auto &Info = Types[OpName]; + if (!Info.Type) { + Info.Type = Ty; + Info.PrintTypeSrcNote = [this, OpName, Ty, &P]() { + PrintSeenWithTypeIn(P, OpName, Ty); + }; + continue; + } + + if (Info.Type != Ty) { + PrintError(DiagLoc, "conflicting types for operand '" + + Op.getOperandName() + "': '" + Info.Type.str() + + "' vs '" + Ty.str() + "'"); + PrintSeenWithTypeIn(P, OpName, Ty); + Info.PrintTypeSrcNote(); + return false; + } + } + + return true; +} + +void OperandTypeChecker::propagateTypes() { + for (auto *Pat : Pats) { + for (auto &Op : Pat->named_operands()) { + if (auto &Info = Types[Op.getOperandName()]; Info.Type) + Op.setType(Info.Type); + } + } +} + +void OperandTypeChecker::PrintSeenWithTypeIn(InstructionPattern &P, + StringRef OpName, + PatternType Ty) const { + PrintNote(DiagLoc, "'" + OpName + "' seen with type '" + Ty.str() + "' in '" + + P.getName() + "'"); +} + +StringRef PatFrag::getParamKindStr(ParamKind OK) { + switch (OK) { + case PK_Root: + return "root"; + case PK_MachineOperand: + return "machine_operand"; + case PK_Imm: + return "imm"; + } + + llvm_unreachable("Unknown operand kind!"); +} + +//===- PatFrag -----------------------------------------------------------===// + +PatFrag::PatFrag(const Record &Def) : Def(Def) { + assert(Def.isSubClassOf(ClassName)); +} + +StringRef PatFrag::getName() const { return Def.getName(); } + +ArrayRef<SMLoc> PatFrag::getLoc() const { return Def.getLoc(); } + +void PatFrag::addInParam(StringRef Name, ParamKind Kind) { + Params.emplace_back(Param{Name, Kind}); +} + +iterator_range<PatFrag::ParamIt> PatFrag::in_params() const { + return {Params.begin() + NumOutParams, Params.end()}; +} + +void PatFrag::addOutParam(StringRef Name, ParamKind Kind) { + assert(NumOutParams == Params.size() && + "Adding out-param after an in-param!"); + Params.emplace_back(Param{Name, Kind}); + ++NumOutParams; +} + +iterator_range<PatFrag::ParamIt> PatFrag::out_params() const { + return {Params.begin(), Params.begin() + NumOutParams}; +} + +unsigned PatFrag::num_roots() const { + return count_if(out_params(), + [&](const auto &P) { return P.Kind == PK_Root; }); +} + +unsigned PatFrag::getParamIdx(StringRef Name) const { + for (const auto &[Idx, Op] : enumerate(Params)) { + if (Op.Name == Name) + return Idx; + } + + return -1; +} + +bool PatFrag::checkSemantics() { + for (const auto &Alt : Alts) { + for (const auto &Pat : Alt.Pats) { + switch (Pat->getKind()) { + case Pattern::K_AnyOpcode: + PrintError("wip_match_opcode cannot be used in " + ClassName); + return false; + case Pattern::K_Builtin: + PrintError("Builtin instructions cannot be used in " + ClassName); + return false; + case Pattern::K_CXX: + continue; + case Pattern::K_CodeGenInstruction: + if (cast<CodeGenInstructionPattern>(Pat.get())->diagnoseAllSpecialTypes( + Def.getLoc(), PatternType::SpecialTyClassName + + " is not supported in " + ClassName)) + return false; + continue; + case Pattern::K_PatFrag: + // TODO: It's just that the emitter doesn't handle it but technically + // there is no reason why we can't. We just have to be careful with + // operand mappings, it could get complex. + PrintError("nested " + ClassName + " are not supported"); + return false; + } + } + } + + StringSet<> SeenOps; + for (const auto &Op : in_params()) { + if (SeenOps.count(Op.Name)) { + PrintError("duplicate parameter '" + Op.Name + "'"); + return false; + } + + // Check this operand is NOT defined in any alternative's patterns. + for (const auto &Alt : Alts) { + if (Alt.OpTable.lookup(Op.Name).Def) { + PrintError("input parameter '" + Op.Name + "' cannot be redefined!"); + return false; + } + } + + if (Op.Kind == PK_Root) { + PrintError("input parameterr '" + Op.Name + "' cannot be a root!"); + return false; + } + + SeenOps.insert(Op.Name); + } + + for (const auto &Op : out_params()) { + if (Op.Kind != PK_Root && Op.Kind != PK_MachineOperand) { + PrintError("output parameter '" + Op.Name + + "' must be 'root' or 'gi_mo'"); + return false; + } + + if (SeenOps.count(Op.Name)) { + PrintError("duplicate parameter '" + Op.Name + "'"); + return false; + } + + // Check this operand is defined in all alternative's patterns. + for (const auto &Alt : Alts) { + const auto *OpDef = Alt.OpTable.getDef(Op.Name); + if (!OpDef) { + PrintError("output parameter '" + Op.Name + + "' must be defined by all alternative patterns in '" + + Def.getName() + "'"); + return false; + } + + if (Op.Kind == PK_Root && OpDef->getNumInstDefs() != 1) { + // The instruction that defines the root must have a single def. + // Otherwise we'd need to support multiple roots and it gets messy. + // + // e.g. this is not supported: + // (pattern (G_UNMERGE_VALUES $x, $root, $vec)) + PrintError("all instructions that define root '" + Op.Name + "' in '" + + Def.getName() + "' can only have a single output operand"); + return false; + } + } + + SeenOps.insert(Op.Name); + } + + if (num_out_params() != 0 && num_roots() == 0) { + PrintError(ClassName + " must have one root in its 'out' operands"); + return false; + } + + if (num_roots() > 1) { + PrintError(ClassName + " can only have one root"); + return false; + } + + // TODO: find unused params + + const auto CheckTypeOf = [&](const PatternType &) -> bool { + llvm_unreachable("GITypeOf should have been rejected earlier!"); + }; + + // Now, typecheck all alternatives. + for (auto &Alt : Alts) { + OperandTypeChecker OTC(Def.getLoc()); + for (auto &Pat : Alt.Pats) { + if (auto *IP = dyn_cast<InstructionPattern>(Pat.get())) { + if (!OTC.check(*IP, CheckTypeOf)) + return false; + } + } + OTC.propagateTypes(); + } + + return true; +} + +bool PatFrag::handleUnboundInParam(StringRef ParamName, StringRef ArgName, + ArrayRef<SMLoc> DiagLoc) const { + // The parameter must be a live-in of all alternatives for this to work. + // Otherwise, we risk having unbound parameters being used (= crashes). + // + // Examples: + // + // in (ins $y), (patterns (G_FNEG $dst, $y), "return matchFnegOp(${y})") + // even if $y is unbound, we'll lazily bind it when emitting the G_FNEG. + // + // in (ins $y), (patterns "return matchFnegOp(${y})") + // if $y is unbound when this fragment is emitted, C++ code expansion will + // fail. + for (const auto &Alt : Alts) { + auto &OT = Alt.OpTable; + if (!OT.lookup(ParamName).Found) { + llvm::PrintError(DiagLoc, "operand '" + ArgName + "' (for parameter '" + + ParamName + "' of '" + getName() + + "') cannot be unbound"); + PrintNote( + DiagLoc, + "one or more alternatives of '" + getName() + "' do not bind '" + + ParamName + + "' to an instruction operand; either use a bound operand or " + "ensure '" + + Def.getName() + "' binds '" + ParamName + + "' in all alternatives"); + return false; + } + } + + return true; +} + +bool PatFrag::buildOperandsTables() { + // enumerate(...) doesn't seem to allow lvalues so we need to count the old + // way. + unsigned Idx = 0; + + const auto DiagnoseRedef = [this, &Idx](StringRef OpName) { + PrintError("Operand '" + OpName + + "' is defined multiple times in patterns of alternative #" + + std::to_string(Idx)); + }; + + for (auto &Alt : Alts) { + for (auto &Pat : Alt.Pats) { + auto *IP = dyn_cast<InstructionPattern>(Pat.get()); + if (!IP) + continue; + + if (!Alt.OpTable.addPattern(IP, DiagnoseRedef)) + return false; + } + + ++Idx; + } + + return true; +} + +void PatFrag::print(raw_ostream &OS, StringRef Indent) const { + OS << Indent << "(PatFrag name:" << getName() << '\n'; + if (!in_params().empty()) { + OS << Indent << " (ins "; + printParamsList(OS, in_params()); + OS << ")\n"; + } + + if (!out_params().empty()) { + OS << Indent << " (outs "; + printParamsList(OS, out_params()); + OS << ")\n"; + } + + // TODO: Dump OperandTable as well. + OS << Indent << " (alternatives [\n"; + for (const auto &Alt : Alts) { + OS << Indent << " [\n"; + for (const auto &Pat : Alt.Pats) { + OS << Indent << " "; + Pat->print(OS, /*PrintName=*/true); + OS << ",\n"; + } + OS << Indent << " ],\n"; + } + OS << Indent << " ])\n"; + + OS << Indent << ')'; +} + +void PatFrag::dump() const { print(dbgs()); } + +void PatFrag::printParamsList(raw_ostream &OS, iterator_range<ParamIt> Params) { + OS << '[' + << join(map_range(Params, + [](auto &O) { + return (O.Name + ":" + getParamKindStr(O.Kind)).str(); + }), + ", ") + << ']'; +} + +void PatFrag::PrintError(Twine Msg) const { llvm::PrintError(&Def, Msg); } + +ArrayRef<InstructionOperand> PatFragPattern::getApplyDefsNeeded() const { + assert(PF.num_roots() == 1); + // Only roots need to be redef. + for (auto [Idx, Param] : enumerate(PF.out_params())) { + if (Param.Kind == PatFrag::PK_Root) + return getOperand(Idx); + } + llvm_unreachable("root not found!"); +} + +//===- PatFragPattern -----------------------------------------------------===// + +bool PatFragPattern::checkSemantics(ArrayRef<SMLoc> DiagLoc) { + if (!InstructionPattern::checkSemantics(DiagLoc)) + return false; + + for (const auto &[Idx, Op] : enumerate(Operands)) { + switch (PF.getParam(Idx).Kind) { + case PatFrag::PK_Imm: + if (!Op.hasImmValue()) { + PrintError(DiagLoc, "expected operand " + std::to_string(Idx) + + " of '" + getInstName() + + "' to be an immediate; got " + Op.describe()); + return false; + } + if (Op.isNamedImmediate()) { + PrintError(DiagLoc, "operand " + std::to_string(Idx) + " of '" + + getInstName() + + "' cannot be a named immediate"); + return false; + } + break; + case PatFrag::PK_Root: + case PatFrag::PK_MachineOperand: + if (!Op.isNamedOperand() || Op.isNamedImmediate()) { + PrintError(DiagLoc, "expected operand " + std::to_string(Idx) + + " of '" + getInstName() + + "' to be a MachineOperand; got " + + Op.describe()); + return false; + } + break; + } + } + + return true; +} + +bool PatFragPattern::mapInputCodeExpansions(const CodeExpansions &ParentCEs, + CodeExpansions &PatFragCEs, + ArrayRef<SMLoc> DiagLoc) const { + for (const auto &[Idx, Op] : enumerate(operands())) { + StringRef ParamName = PF.getParam(Idx).Name; + + // Operands to a PFP can only be named, or be an immediate, but not a named + // immediate. + assert(!Op.isNamedImmediate()); + + if (Op.isNamedOperand()) { + StringRef ArgName = Op.getOperandName(); + // Map it only if it's been defined. + auto It = ParentCEs.find(ArgName); + if (It == ParentCEs.end()) { + if (!PF.handleUnboundInParam(ParamName, ArgName, DiagLoc)) + return false; + } else + PatFragCEs.declare(ParamName, It->second); + continue; + } + + if (Op.hasImmValue()) { + PatFragCEs.declare(ParamName, std::to_string(Op.getImmValue())); + continue; + } + + llvm_unreachable("Unknown Operand Type!"); + } + + return true; +} + +//===- BuiltinPattern -----------------------------------------------------===// + +BuiltinPattern::BuiltinInfo BuiltinPattern::getBuiltinInfo(const Record &Def) { + assert(Def.isSubClassOf(ClassName)); + + StringRef Name = Def.getName(); + for (const auto &KBI : KnownBuiltins) { + if (KBI.DefName == Name) + return KBI; + } + + PrintFatalError(Def.getLoc(), + "Unimplemented " + ClassName + " def '" + Name + "'"); +} + +bool BuiltinPattern::checkSemantics(ArrayRef<SMLoc> Loc) { + if (!InstructionPattern::checkSemantics(Loc)) + return false; + + // For now all builtins just take names, no immediates. + for (const auto &[Idx, Op] : enumerate(operands())) { + if (!Op.isNamedOperand() || Op.isNamedImmediate()) { + PrintError(Loc, "expected operand " + std::to_string(Idx) + " of '" + + getInstName() + "' to be a name"); + return false; + } + } + + return true; +} + +} // namespace gi +} // namespace llvm diff --git a/llvm/utils/TableGen/GlobalISel/Patterns.h b/llvm/utils/TableGen/GlobalISel/Patterns.h new file mode 100644 index 000000000000..b3160552a21f --- /dev/null +++ b/llvm/utils/TableGen/GlobalISel/Patterns.h @@ -0,0 +1,690 @@ +//===- Patterns.h ----------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Contains the Pattern hierarchy alongside helper classes such as +/// PatFrag, MIFlagsInfo, PatternType, etc. +/// +/// These classes are used by the GlobalISel Combiner backend to help parse, +/// process and emit MIR patterns. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_GLOBALISEL_PATTERNS_H +#define LLVM_UTILS_GLOBALISEL_PATTERNS_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include <memory> +#include <optional> +#include <string> + +namespace llvm { + +class Record; +class SMLoc; +class StringInit; +class CodeExpansions; +class CodeGenInstruction; + +namespace gi { + +class CXXPredicateCode; +class LLTCodeGen; +class LLTCodeGenOrTempType; +class RuleMatcher; + +//===- PatternType --------------------------------------------------------===// + +/// Represent the type of a Pattern Operand. +/// +/// Types have two form: +/// - LLTs, which are straightforward. +/// - Special types, e.g. GITypeOf +class PatternType { +public: + static constexpr StringLiteral SpecialTyClassName = "GISpecialType"; + static constexpr StringLiteral TypeOfClassName = "GITypeOf"; + + enum PTKind : uint8_t { + PT_None, + + PT_ValueType, + PT_TypeOf, + }; + + PatternType() : Kind(PT_None), Data() {} + + static std::optional<PatternType> get(ArrayRef<SMLoc> DiagLoc, + const Record *R, Twine DiagCtx); + static PatternType getTypeOf(StringRef OpName); + + bool isNone() const { return Kind == PT_None; } + bool isLLT() const { return Kind == PT_ValueType; } + bool isSpecial() const { return isTypeOf(); } + bool isTypeOf() const { return Kind == PT_TypeOf; } + + StringRef getTypeOfOpName() const; + const Record *getLLTRecord() const; + + explicit operator bool() const { return !isNone(); } + + bool operator==(const PatternType &Other) const; + bool operator!=(const PatternType &Other) const { return !operator==(Other); } + + std::string str() const; + +private: + PatternType(PTKind Kind) : Kind(Kind), Data() {} + + PTKind Kind; + union DataT { + DataT() : Str() {} + + /// PT_ValueType -> ValueType Def. + const Record *Def; + + /// PT_TypeOf -> Operand name (without the '$') + StringRef Str; + } Data; +}; + +//===- Pattern Base Class -------------------------------------------------===// + +/// Base class for all patterns that can be written in an `apply`, `match` or +/// `pattern` DAG operator. +/// +/// For example: +/// +/// (apply (G_ZEXT $x, $y), (G_ZEXT $y, $z), "return isFoo(${z})") +/// +/// Creates 3 Pattern objects: +/// - Two CodeGenInstruction Patterns +/// - A CXXPattern +class Pattern { +public: + enum { + K_AnyOpcode, + K_CXX, + + K_CodeGenInstruction, + K_PatFrag, + K_Builtin, + }; + + virtual ~Pattern() = default; + + unsigned getKind() const { return Kind; } + const char *getKindName() const; + + bool hasName() const { return !Name.empty(); } + StringRef getName() const { return Name; } + + virtual void print(raw_ostream &OS, bool PrintName = true) const = 0; + void dump() const; + +protected: + Pattern(unsigned Kind, StringRef Name) : Kind(Kind), Name(Name) { + assert(!Name.empty() && "unnamed pattern!"); + } + + void printImpl(raw_ostream &OS, bool PrintName, + function_ref<void()> ContentPrinter) const; + +private: + unsigned Kind; + StringRef Name; +}; + +//===- AnyOpcodePattern ---------------------------------------------------===// + +/// `wip_match_opcode` patterns. +/// This matches one or more opcodes, and does not check any operands +/// whatsoever. +/// +/// TODO: Long-term, this needs to be removed. It's a hack around MIR +/// pattern matching limitations. +class AnyOpcodePattern : public Pattern { +public: + AnyOpcodePattern(StringRef Name) : Pattern(K_AnyOpcode, Name) {} + + static bool classof(const Pattern *P) { return P->getKind() == K_AnyOpcode; } + + void addOpcode(const CodeGenInstruction *I) { Insts.push_back(I); } + const auto &insts() const { return Insts; } + + void print(raw_ostream &OS, bool PrintName = true) const override; + +private: + SmallVector<const CodeGenInstruction *, 4> Insts; +}; + +//===- CXXPattern ---------------------------------------------------------===// + +/// Represents raw C++ code which may need some expansions. +/// +/// e.g. [{ return isFooBux(${src}.getReg()); }] +/// +/// For the expanded code, \see CXXPredicateCode. CXXPredicateCode objects are +/// created through `expandCode`. +/// +/// \see CodeExpander and \see CodeExpansions for more information on code +/// expansions. +/// +/// This object has two purposes: +/// - Represent C++ code as a pattern entry. +/// - Be a factory for expanded C++ code. +/// - It's immutable and only holds the raw code so we can expand the same +/// CXX pattern multiple times if we need to. +/// +/// Note that the code is always trimmed in the constructor, so leading and +/// trailing whitespaces are removed. This removes bloat in the output, avoids +/// formatting issues, but also allows us to check things like +/// `.startswith("return")` trivially without worrying about spaces. +class CXXPattern : public Pattern { +public: + CXXPattern(const StringInit &Code, StringRef Name); + + CXXPattern(StringRef Code, StringRef Name) + : Pattern(K_CXX, Name), RawCode(Code.trim().str()) {} + + static bool classof(const Pattern *P) { return P->getKind() == K_CXX; } + + void setIsApply(bool Value = true) { IsApply = Value; } + StringRef getRawCode() const { return RawCode; } + + /// Expands raw code, replacing things such as `${foo}` with their + /// substitution in \p CE. + /// + /// \param CE Map of Code Expansions + /// \param Locs SMLocs for the Code Expander, in case it needs to emit + /// diagnostics. + /// \param AddComment Optionally called to emit a comment before the expanded + /// code. + /// + /// \return A CXXPredicateCode object that contains the expanded code. Note + /// that this may or may not insert a new object. All CXXPredicateCode objects + /// are held in a set to avoid emitting duplicate C++ code. + const CXXPredicateCode & + expandCode(const CodeExpansions &CE, ArrayRef<SMLoc> Locs, + function_ref<void(raw_ostream &)> AddComment = {}) const; + + void print(raw_ostream &OS, bool PrintName = true) const override; + +private: + bool IsApply = false; + std::string RawCode; +}; + +//===- InstructionPattern ---------------------------------------------===// + +/// An operand for an InstructionPattern. +/// +/// Operands are composed of three elements: +/// - (Optional) Value +/// - (Optional) Name +/// - (Optional) Type +/// +/// Some examples: +/// (i32 0):$x -> V=int(0), Name='x', Type=i32 +/// 0:$x -> V=int(0), Name='x' +/// $x -> Name='x' +/// i32:$x -> Name='x', Type = i32 +class InstructionOperand { +public: + using IntImmTy = int64_t; + + InstructionOperand(IntImmTy Imm, StringRef Name, PatternType Type) + : Value(Imm), Name(Name), Type(Type) {} + + InstructionOperand(StringRef Name, PatternType Type) + : Name(Name), Type(Type) {} + + bool isNamedImmediate() const { return hasImmValue() && isNamedOperand(); } + + bool hasImmValue() const { return Value.has_value(); } + IntImmTy getImmValue() const { return *Value; } + + bool isNamedOperand() const { return !Name.empty(); } + StringRef getOperandName() const { + assert(isNamedOperand() && "Operand is unnamed"); + return Name; + } + + InstructionOperand withNewName(StringRef NewName) const { + InstructionOperand Result = *this; + Result.Name = NewName; + return Result; + } + + void setIsDef(bool Value = true) { Def = Value; } + bool isDef() const { return Def; } + + void setType(PatternType NewType) { + assert((!Type || (Type == NewType)) && "Overwriting type!"); + Type = NewType; + } + PatternType getType() const { return Type; } + + std::string describe() const; + void print(raw_ostream &OS) const; + + void dump() const; + +private: + std::optional<int64_t> Value; + StringRef Name; + PatternType Type; + bool Def = false; +}; + +/// Base class for CodeGenInstructionPattern & PatFragPattern, which handles all +/// the boilerplate for patterns that have a list of operands for some (pseudo) +/// instruction. +class InstructionPattern : public Pattern { +public: + virtual ~InstructionPattern() = default; + + static bool classof(const Pattern *P) { + return P->getKind() == K_CodeGenInstruction || P->getKind() == K_PatFrag || + P->getKind() == K_Builtin; + } + + template <typename... Ty> void addOperand(Ty &&...Init) { + Operands.emplace_back(std::forward<Ty>(Init)...); + } + + auto &operands() { return Operands; } + const auto &operands() const { return Operands; } + unsigned operands_size() const { return Operands.size(); } + InstructionOperand &getOperand(unsigned K) { return Operands[K]; } + const InstructionOperand &getOperand(unsigned K) const { return Operands[K]; } + + /// When this InstructionPattern is used as the match root, returns the + /// operands that must be redefined in the 'apply' pattern for the rule to be + /// valid. + /// + /// For most patterns, this just returns the defs. + /// For PatFrag this only returns the root of the PF. + /// + /// Returns an empty array on error. + virtual ArrayRef<InstructionOperand> getApplyDefsNeeded() const { + return {operands().begin(), getNumInstDefs()}; + } + + auto named_operands() { + return make_filter_range(Operands, + [&](auto &O) { return O.isNamedOperand(); }); + } + + auto named_operands() const { + return make_filter_range(Operands, + [&](auto &O) { return O.isNamedOperand(); }); + } + + virtual bool isVariadic() const { return false; } + virtual unsigned getNumInstOperands() const = 0; + virtual unsigned getNumInstDefs() const = 0; + + bool hasAllDefs() const { return operands_size() >= getNumInstDefs(); } + + virtual StringRef getInstName() const = 0; + + /// Diagnoses all uses of special types in this Pattern and returns true if at + /// least one diagnostic was emitted. + bool diagnoseAllSpecialTypes(ArrayRef<SMLoc> Loc, Twine Msg) const; + + void reportUnreachable(ArrayRef<SMLoc> Locs) const; + virtual bool checkSemantics(ArrayRef<SMLoc> Loc); + + void print(raw_ostream &OS, bool PrintName = true) const override; + +protected: + InstructionPattern(unsigned K, StringRef Name) : Pattern(K, Name) {} + + virtual void printExtras(raw_ostream &OS) const {} + + SmallVector<InstructionOperand, 4> Operands; +}; + +//===- OperandTable -------------------------------------------------------===// + +/// Maps InstructionPattern operands to their definitions. This allows us to tie +/// different patterns of a (apply), (match) or (patterns) set of patterns +/// together. +class OperandTable { +public: + bool addPattern(InstructionPattern *P, + function_ref<void(StringRef)> DiagnoseRedef); + + struct LookupResult { + LookupResult() = default; + LookupResult(InstructionPattern *Def) : Found(true), Def(Def) {} + + bool Found = false; + InstructionPattern *Def = nullptr; + + bool isLiveIn() const { return Found && !Def; } + }; + + LookupResult lookup(StringRef OpName) const { + if (auto It = Table.find(OpName); It != Table.end()) + return LookupResult(It->second); + return LookupResult(); + } + + InstructionPattern *getDef(StringRef OpName) const { + return lookup(OpName).Def; + } + + void print(raw_ostream &OS, StringRef Name = "", StringRef Indent = "") const; + + auto begin() const { return Table.begin(); } + auto end() const { return Table.end(); } + + void dump() const; + +private: + StringMap<InstructionPattern *> Table; +}; + +//===- CodeGenInstructionPattern ------------------------------------------===// + +/// Helper class to contain data associated with a MIFlags operand. +class MIFlagsInfo { +public: + void addSetFlag(const Record *R); + void addUnsetFlag(const Record *R); + void addCopyFlag(StringRef InstName); + + const auto &set_flags() const { return SetF; } + const auto &unset_flags() const { return UnsetF; } + const auto ©_flags() const { return CopyF; } + +private: + SetVector<StringRef> SetF, UnsetF, CopyF; +}; + +/// Matches an instruction, e.g. `G_ADD $x, $y, $z`. +class CodeGenInstructionPattern : public InstructionPattern { +public: + CodeGenInstructionPattern(const CodeGenInstruction &I, StringRef Name) + : InstructionPattern(K_CodeGenInstruction, Name), I(I) {} + + static bool classof(const Pattern *P) { + return P->getKind() == K_CodeGenInstruction; + } + + bool is(StringRef OpcodeName) const; + + bool hasVariadicDefs() const; + bool isVariadic() const override; + unsigned getNumInstDefs() const override; + unsigned getNumInstOperands() const override; + + MIFlagsInfo &getOrCreateMIFlagsInfo(); + const MIFlagsInfo *getMIFlagsInfo() const { return FI.get(); } + + const CodeGenInstruction &getInst() const { return I; } + StringRef getInstName() const override; + +private: + void printExtras(raw_ostream &OS) const override; + + const CodeGenInstruction &I; + std::unique_ptr<MIFlagsInfo> FI; +}; + +//===- OperandTypeChecker -------------------------------------------------===// + +/// This is a trivial type checker for all operands in a set of +/// InstructionPatterns. +/// +/// It infers the type of each operand, check it's consistent with the known +/// type of the operand, and then sets all of the types in all operands in +/// propagateTypes. +/// +/// It also handles verifying correctness of special types. +class OperandTypeChecker { +public: + OperandTypeChecker(ArrayRef<SMLoc> DiagLoc) : DiagLoc(DiagLoc) {} + + /// Step 1: Check each pattern one by one. All patterns that pass through here + /// are added to a common worklist so propagateTypes can access them. + bool check(InstructionPattern &P, + std::function<bool(const PatternType &)> VerifyTypeOfOperand); + + /// Step 2: Propagate all types. e.g. if one use of "$a" has type i32, make + /// all uses of "$a" have type i32. + void propagateTypes(); + +protected: + ArrayRef<SMLoc> DiagLoc; + +private: + using InconsistentTypeDiagFn = std::function<void()>; + + void PrintSeenWithTypeIn(InstructionPattern &P, StringRef OpName, + PatternType Ty) const; + + struct OpTypeInfo { + PatternType Type; + InconsistentTypeDiagFn PrintTypeSrcNote = []() {}; + }; + + StringMap<OpTypeInfo> Types; + + SmallVector<InstructionPattern *, 16> Pats; +}; + +//===- PatFrag ------------------------------------------------------------===// + +/// Represents a parsed GICombinePatFrag. This can be thought of as the +/// equivalent of a CodeGenInstruction, but for PatFragPatterns. +/// +/// PatFrags are made of 3 things: +/// - Out parameters (defs) +/// - In parameters +/// - A set of pattern lists (alternatives). +/// +/// If the PatFrag uses instruction patterns, the root must be one of the defs. +/// +/// Note that this DOES NOT represent the use of the PatFrag, only its +/// definition. The use of the PatFrag in a Pattern is represented by +/// PatFragPattern. +/// +/// PatFrags use the term "parameter" instead of operand because they're +/// essentially macros, and using that name avoids confusion. Other than that, +/// they're structured similarly to a MachineInstruction - all parameters +/// (operands) are in the same list, with defs at the start. This helps mapping +/// parameters to values, because, param N of a PatFrag is always operand N of a +/// PatFragPattern. +class PatFrag { +public: + static constexpr StringLiteral ClassName = "GICombinePatFrag"; + + enum ParamKind { + PK_Root, + PK_MachineOperand, + PK_Imm, + }; + + struct Param { + StringRef Name; + ParamKind Kind; + }; + + using ParamVec = SmallVector<Param, 4>; + using ParamIt = ParamVec::const_iterator; + + /// Represents an alternative of the PatFrag. When parsing a GICombinePatFrag, + /// this is created from its "Alternatives" list. Each alternative is a list + /// of patterns written wrapped in a `(pattern ...)` dag init. + /// + /// Each argument to the `pattern` DAG operator is parsed into a Pattern + /// instance. + struct Alternative { + OperandTable OpTable; + SmallVector<std::unique_ptr<Pattern>, 4> Pats; + }; + + explicit PatFrag(const Record &Def); + + static StringRef getParamKindStr(ParamKind OK); + + StringRef getName() const; + + const Record &getDef() const { return Def; } + ArrayRef<SMLoc> getLoc() const; + + Alternative &addAlternative() { return Alts.emplace_back(); } + const Alternative &getAlternative(unsigned K) const { return Alts[K]; } + unsigned num_alternatives() const { return Alts.size(); } + + void addInParam(StringRef Name, ParamKind Kind); + iterator_range<ParamIt> in_params() const; + unsigned num_in_params() const { return Params.size() - NumOutParams; } + + void addOutParam(StringRef Name, ParamKind Kind); + iterator_range<ParamIt> out_params() const; + unsigned num_out_params() const { return NumOutParams; } + + unsigned num_roots() const; + unsigned num_params() const { return num_in_params() + num_out_params(); } + + /// Finds the operand \p Name and returns its index or -1 if not found. + /// Remember that all params are part of the same list, with out params at the + /// start. This means that the index returned can be used to access operands + /// of InstructionPatterns. + unsigned getParamIdx(StringRef Name) const; + const Param &getParam(unsigned K) const { return Params[K]; } + + bool canBeMatchRoot() const { return num_roots() == 1; } + + void print(raw_ostream &OS, StringRef Indent = "") const; + void dump() const; + + /// Checks if the in-param \p ParamName can be unbound or not. + /// \p ArgName is the name of the argument passed to the PatFrag. + /// + /// An argument can be unbound only if, for all alternatives: + /// - There is no CXX pattern, OR: + /// - There is an InstructionPattern that binds the parameter. + /// + /// e.g. in (MyPatFrag $foo), if $foo has never been seen before (= it's + /// unbound), this checks if MyPatFrag supports it or not. + bool handleUnboundInParam(StringRef ParamName, StringRef ArgName, + ArrayRef<SMLoc> DiagLoc) const; + + bool checkSemantics(); + bool buildOperandsTables(); + +private: + static void printParamsList(raw_ostream &OS, iterator_range<ParamIt> Params); + + void PrintError(Twine Msg) const; + + const Record &Def; + unsigned NumOutParams = 0; + ParamVec Params; + SmallVector<Alternative, 2> Alts; +}; + +//===- PatFragPattern -----------------------------------------------------===// + +/// Represents a use of a GICombinePatFrag. +class PatFragPattern : public InstructionPattern { +public: + PatFragPattern(const PatFrag &PF, StringRef Name) + : InstructionPattern(K_PatFrag, Name), PF(PF) {} + + static bool classof(const Pattern *P) { return P->getKind() == K_PatFrag; } + + const PatFrag &getPatFrag() const { return PF; } + StringRef getInstName() const override { return PF.getName(); } + + unsigned getNumInstDefs() const override { return PF.num_out_params(); } + unsigned getNumInstOperands() const override { return PF.num_params(); } + + ArrayRef<InstructionOperand> getApplyDefsNeeded() const override; + + bool checkSemantics(ArrayRef<SMLoc> DiagLoc) override; + + /// Before emitting the patterns inside the PatFrag, add all necessary code + /// expansions to \p PatFragCEs imported from \p ParentCEs. + /// + /// For a MachineOperand PatFrag parameter, this will fetch the expansion for + /// that operand from \p ParentCEs and add it to \p PatFragCEs. Errors can be + /// emitted if the MachineOperand reference is unbound. + /// + /// For an Immediate PatFrag parameter this simply adds the integer value to + /// \p PatFragCEs as an expansion. + /// + /// \param ParentCEs Contains all of the code expansions declared by the other + /// patterns emitted so far in the pattern list containing + /// this PatFragPattern. + /// \param PatFragCEs Output Code Expansions (usually empty) + /// \param DiagLoc Diagnostic loc in case an error occurs. + /// \return `true` on success, `false` on failure. + bool mapInputCodeExpansions(const CodeExpansions &ParentCEs, + CodeExpansions &PatFragCEs, + ArrayRef<SMLoc> DiagLoc) const; + +private: + const PatFrag &PF; +}; + +//===- BuiltinPattern -----------------------------------------------------===// + +/// Represents builtin instructions such as "GIReplaceReg" and "GIEraseRoot". +enum BuiltinKind { + BI_ReplaceReg, + BI_EraseRoot, +}; + +class BuiltinPattern : public InstructionPattern { + struct BuiltinInfo { + StringLiteral DefName; + BuiltinKind Kind; + unsigned NumOps; + unsigned NumDefs; + }; + + static constexpr std::array<BuiltinInfo, 2> KnownBuiltins = {{ + {"GIReplaceReg", BI_ReplaceReg, 2, 1}, + {"GIEraseRoot", BI_EraseRoot, 0, 0}, + }}; + +public: + static constexpr StringLiteral ClassName = "GIBuiltinInst"; + + BuiltinPattern(const Record &Def, StringRef Name) + : InstructionPattern(K_Builtin, Name), I(getBuiltinInfo(Def)) {} + + static bool classof(const Pattern *P) { return P->getKind() == K_Builtin; } + + unsigned getNumInstOperands() const override { return I.NumOps; } + unsigned getNumInstDefs() const override { return I.NumDefs; } + StringRef getInstName() const override { return I.DefName; } + BuiltinKind getBuiltinKind() const { return I.Kind; } + + bool checkSemantics(ArrayRef<SMLoc> Loc) override; + +private: + static BuiltinInfo getBuiltinInfo(const Record &Def); + + BuiltinInfo I; +}; + +} // namespace gi +} // end namespace llvm + +#endif // ifndef LLVM_UTILS_GLOBALISEL_PATTERNS_H diff --git a/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp new file mode 100644 index 000000000000..89aca87a28ec --- /dev/null +++ b/llvm/utils/TableGen/GlobalISelCombinerEmitter.cpp @@ -0,0 +1,3041 @@ +//===- GlobalISelCombinerMatchTableEmitter.cpp - --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Generate a combiner implementation for GlobalISel from a declarative +/// syntax using GlobalISelMatchTable. +/// +/// Usually, TableGen backends use "assert is an error" as a means to report +/// invalid input. They try to diagnose common case but don't try very hard and +/// crashes can be common. This backend aims to behave closer to how a language +/// compiler frontend would behave: we try extra hard to diagnose invalid inputs +/// early, and any crash should be considered a bug (= a feature or diagnostic +/// is missing). +/// +/// While this can make the backend a bit more complex than it needs to be, it +/// pays off because MIR patterns can get complicated. Giving useful error +/// messages to combine writers can help boost their productivity. +/// +/// As with anything, a good balance has to be found. We also don't want to +/// write hundreds of lines of code to detect edge cases. In practice, crashing +/// very occasionally, or giving poor errors in some rare instances, is fine. +/// +//===----------------------------------------------------------------------===// + +#include "CodeGenInstruction.h" +#include "CodeGenTarget.h" +#include "GlobalISel/CXXPredicates.h" +#include "GlobalISel/CodeExpander.h" +#include "GlobalISel/CodeExpansions.h" +#include "GlobalISel/CombinerUtils.h" +#include "GlobalISel/MatchDataInfo.h" +#include "GlobalISel/Patterns.h" +#include "GlobalISelMatchTable.h" +#include "GlobalISelMatchTableExecutorEmitter.h" +#include "SubtargetFeatureInfo.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/EquivalenceClasses.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/StringMatcher.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cstdint> + +using namespace llvm; +using namespace llvm::gi; + +#define DEBUG_TYPE "gicombiner-emitter" + +namespace { +cl::OptionCategory + GICombinerEmitterCat("Options for -gen-global-isel-combiner"); +cl::opt<bool> StopAfterParse( + "gicombiner-stop-after-parse", + cl::desc("Stop processing after parsing rules and dump state"), + cl::cat(GICombinerEmitterCat)); +cl::list<std::string> + SelectedCombiners("combiners", cl::desc("Emit the specified combiners"), + cl::cat(GICombinerEmitterCat), cl::CommaSeparated); +cl::opt<bool> DebugCXXPreds( + "gicombiner-debug-cxxpreds", + cl::desc("Add Contextual/Debug comments to all C++ predicates"), + cl::cat(GICombinerEmitterCat)); +cl::opt<bool> DebugTypeInfer("gicombiner-debug-typeinfer", + cl::desc("Print type inference debug logs"), + cl::cat(GICombinerEmitterCat)); + +constexpr StringLiteral CXXApplyPrefix = "GICXXCustomAction_CombineApply"; +constexpr StringLiteral CXXPredPrefix = "GICXXPred_MI_Predicate_"; +constexpr StringLiteral MIFlagsEnumClassName = "MIFlagEnum"; + +//===- CodeExpansions Helpers --------------------------------------------===// + +void declareInstExpansion(CodeExpansions &CE, const InstructionMatcher &IM, + StringRef Name) { + CE.declare(Name, "State.MIs[" + to_string(IM.getInsnVarID()) + "]"); +} + +void declareInstExpansion(CodeExpansions &CE, const BuildMIAction &A, + StringRef Name) { + // Note: we use redeclare here because this may overwrite a matcher inst + // expansion. + CE.redeclare(Name, "OutMIs[" + to_string(A.getInsnID()) + "]"); +} + +void declareOperandExpansion(CodeExpansions &CE, const OperandMatcher &OM, + StringRef Name) { + CE.declare(Name, "State.MIs[" + to_string(OM.getInsnVarID()) + + "]->getOperand(" + to_string(OM.getOpIdx()) + ")"); +} + +void declareTempRegExpansion(CodeExpansions &CE, unsigned TempRegID, + StringRef Name) { + CE.declare(Name, "State.TempRegisters[" + to_string(TempRegID) + "]"); +} + +//===- Misc. Helpers -----------------------------------------------------===// + +/// Copies a StringRef into a static pool to preserve it. +/// Most Pattern classes use StringRef so we need this. +StringRef insertStrRef(StringRef S) { + if (S.empty()) + return {}; + + static StringSet<> Pool; + auto [It, Inserted] = Pool.insert(S); + return It->getKey(); +} + +template <typename Container> auto keys(Container &&C) { + return map_range(C, [](auto &Entry) -> auto & { return Entry.first; }); +} + +template <typename Container> auto values(Container &&C) { + return map_range(C, [](auto &Entry) -> auto & { return Entry.second; }); +} + +std::string getIsEnabledPredicateEnumName(unsigned CombinerRuleID) { + return "GICXXPred_Simple_IsRule" + to_string(CombinerRuleID) + "Enabled"; +} + +//===- MatchTable Helpers ------------------------------------------------===// + +LLTCodeGen getLLTCodeGen(const PatternType &PT) { + return *MVTToLLT(getValueType(PT.getLLTRecord())); +} + +LLTCodeGenOrTempType getLLTCodeGenOrTempType(const PatternType &PT, + RuleMatcher &RM) { + assert(!PT.isNone()); + + if (PT.isLLT()) + return getLLTCodeGen(PT); + + assert(PT.isTypeOf()); + auto &OM = RM.getOperandMatcher(PT.getTypeOfOpName()); + return OM.getTempTypeIdx(RM); +} + +//===- PrettyStackTrace Helpers ------------------------------------------===// + +class PrettyStackTraceParse : public PrettyStackTraceEntry { + const Record &Def; + +public: + PrettyStackTraceParse(const Record &Def) : Def(Def) {} + + void print(raw_ostream &OS) const override { + if (Def.isSubClassOf("GICombineRule")) + OS << "Parsing GICombineRule '" << Def.getName() << "'"; + else if (Def.isSubClassOf(PatFrag::ClassName)) + OS << "Parsing " << PatFrag::ClassName << " '" << Def.getName() << "'"; + else + OS << "Parsing '" << Def.getName() << "'"; + OS << '\n'; + } +}; + +class PrettyStackTraceEmit : public PrettyStackTraceEntry { + const Record &Def; + const Pattern *Pat = nullptr; + +public: + PrettyStackTraceEmit(const Record &Def, const Pattern *Pat = nullptr) + : Def(Def), Pat(Pat) {} + + void print(raw_ostream &OS) const override { + if (Def.isSubClassOf("GICombineRule")) + OS << "Emitting GICombineRule '" << Def.getName() << "'"; + else if (Def.isSubClassOf(PatFrag::ClassName)) + OS << "Emitting " << PatFrag::ClassName << " '" << Def.getName() << "'"; + else + OS << "Emitting '" << Def.getName() << "'"; + + if (Pat) + OS << " [" << Pat->getKindName() << " '" << Pat->getName() << "']"; + OS << '\n'; + } +}; + +//===- CombineRuleOperandTypeChecker --------------------------------------===// + +/// This is a wrapper around OperandTypeChecker specialized for Combiner Rules. +/// On top of doing the same things as OperandTypeChecker, this also attempts to +/// infer as many types as possible for temporary register defs & immediates in +/// apply patterns. +/// +/// The inference is trivial and leverages the MCOI OperandTypes encoded in +/// CodeGenInstructions to infer types across patterns in a CombineRule. It's +/// thus very limited and only supports CodeGenInstructions (but that's the main +/// use case so it's fine). +/// +/// We only try to infer untyped operands in apply patterns when they're temp +/// reg defs, or immediates. Inference always outputs a `TypeOf<$x>` where $x is +/// a named operand from a match pattern. +class CombineRuleOperandTypeChecker : private OperandTypeChecker { +public: + CombineRuleOperandTypeChecker(const Record &RuleDef, + const OperandTable &MatchOpTable) + : OperandTypeChecker(RuleDef.getLoc()), RuleDef(RuleDef), + MatchOpTable(MatchOpTable) {} + + /// Records and checks a 'match' pattern. + bool processMatchPattern(InstructionPattern &P); + + /// Records and checks an 'apply' pattern. + bool processApplyPattern(InstructionPattern &P); + + /// Propagates types, then perform type inference and do a second round of + /// propagation in the apply patterns only if any types were inferred. + void propagateAndInferTypes(); + +private: + /// TypeEquivalenceClasses are groups of operands of an instruction that share + /// a common type. + /// + /// e.g. [[a, b], [c, d]] means a and b have the same type, and c and + /// d have the same type too. b/c and a/d don't have to have the same type, + /// though. + using TypeEquivalenceClasses = EquivalenceClasses<StringRef>; + + /// \returns true for `OPERAND_GENERIC_` 0 through 5. + /// These are the MCOI types that can be registers. The other MCOI types are + /// either immediates, or fancier operands used only post-ISel, so we don't + /// care about them for combiners. + static bool canMCOIOperandTypeBeARegister(StringRef MCOIType) { + // Assume OPERAND_GENERIC_0 through 5 can be registers. The other MCOI + // OperandTypes are either never used in gMIR, or not relevant (e.g. + // OPERAND_GENERIC_IMM, which is definitely never a register). + return MCOIType.drop_back(1).ends_with("OPERAND_GENERIC_"); + } + + /// Finds the "MCOI::"" operand types for each operand of \p CGP. + /// + /// This is a bit trickier than it looks because we need to handle variadic + /// in/outs. + /// + /// e.g. for + /// (G_BUILD_VECTOR $vec, $x, $y) -> + /// [MCOI::OPERAND_GENERIC_0, MCOI::OPERAND_GENERIC_1, + /// MCOI::OPERAND_GENERIC_1] + /// + /// For unknown types (which can happen in variadics where varargs types are + /// inconsistent), a unique name is given, e.g. "unknown_type_0". + static std::vector<std::string> + getMCOIOperandTypes(const CodeGenInstructionPattern &CGP); + + /// Adds the TypeEquivalenceClasses for \p P in \p OutTECs. + void getInstEqClasses(const InstructionPattern &P, + TypeEquivalenceClasses &OutTECs) const; + + /// Calls `getInstEqClasses` on all patterns of the rule to produce the whole + /// rule's TypeEquivalenceClasses. + TypeEquivalenceClasses getRuleEqClasses() const; + + /// Tries to infer the type of the \p ImmOpIdx -th operand of \p IP using \p + /// TECs. + /// + /// This is achieved by trying to find a named operand in \p IP that shares + /// the same type as \p ImmOpIdx, and using \ref inferNamedOperandType on that + /// operand instead. + /// + /// \returns the inferred type or an empty PatternType if inference didn't + /// succeed. + PatternType inferImmediateType(const InstructionPattern &IP, + unsigned ImmOpIdx, + const TypeEquivalenceClasses &TECs) const; + + /// Looks inside \p TECs to infer \p OpName's type. + /// + /// \returns the inferred type or an empty PatternType if inference didn't + /// succeed. + PatternType inferNamedOperandType(const InstructionPattern &IP, + StringRef OpName, + const TypeEquivalenceClasses &TECs) const; + + const Record &RuleDef; + SmallVector<InstructionPattern *, 8> MatchPats; + SmallVector<InstructionPattern *, 8> ApplyPats; + + const OperandTable &MatchOpTable; +}; + +bool CombineRuleOperandTypeChecker::processMatchPattern(InstructionPattern &P) { + MatchPats.push_back(&P); + return check(P, /*CheckTypeOf*/ [](const auto &) { + // GITypeOf in 'match' is currently always rejected by the + // CombineRuleBuilder after inference is done. + return true; + }); +} + +bool CombineRuleOperandTypeChecker::processApplyPattern(InstructionPattern &P) { + ApplyPats.push_back(&P); + return check(P, /*CheckTypeOf*/ [&](const PatternType &Ty) { + // GITypeOf<"$x"> can only be used if "$x" is a matched operand. + const auto OpName = Ty.getTypeOfOpName(); + if (MatchOpTable.lookup(OpName).Found) + return true; + + PrintError(RuleDef.getLoc(), "'" + OpName + "' ('" + Ty.str() + + "') does not refer to a matched operand!"); + return false; + }); +} + +void CombineRuleOperandTypeChecker::propagateAndInferTypes() { + /// First step here is to propagate types using the OperandTypeChecker. That + /// way we ensure all uses of a given register have consistent types. + propagateTypes(); + + /// Build the TypeEquivalenceClasses for the whole rule. + const TypeEquivalenceClasses TECs = getRuleEqClasses(); + + /// Look at the apply patterns and find operands that need to be + /// inferred. We then try to find an equivalence class that they're a part of + /// and select the best operand to use for the `GITypeOf` type. We prioritize + /// defs of matched instructions because those are guaranteed to be registers. + bool InferredAny = false; + for (auto *Pat : ApplyPats) { + for (unsigned K = 0; K < Pat->operands_size(); ++K) { + auto &Op = Pat->getOperand(K); + + // We only want to take a look at untyped defs or immediates. + if ((!Op.isDef() && !Op.hasImmValue()) || Op.getType()) + continue; + + // Infer defs & named immediates. + if (Op.isDef() || Op.isNamedImmediate()) { + // Check it's not a redefinition of a matched operand. + // In such cases, inference is not necessary because we just copy + // operands and don't create temporary registers. + if (MatchOpTable.lookup(Op.getOperandName()).Found) + continue; + + // Inference is needed here, so try to do it. + if (PatternType Ty = + inferNamedOperandType(*Pat, Op.getOperandName(), TECs)) { + if (DebugTypeInfer) + errs() << "INFER: " << Op.describe() << " -> " << Ty.str() << '\n'; + Op.setType(Ty); + InferredAny = true; + } + + continue; + } + + // Infer immediates + if (Op.hasImmValue()) { + if (PatternType Ty = inferImmediateType(*Pat, K, TECs)) { + if (DebugTypeInfer) + errs() << "INFER: " << Op.describe() << " -> " << Ty.str() << '\n'; + Op.setType(Ty); + InferredAny = true; + } + continue; + } + } + } + + // If we've inferred any types, we want to propagate them across the apply + // patterns. Type inference only adds GITypeOf types that point to Matched + // operands, so we definitely don't want to propagate types into the match + // patterns as well, otherwise bad things happen. + if (InferredAny) { + OperandTypeChecker OTC(RuleDef.getLoc()); + for (auto *Pat : ApplyPats) { + if (!OTC.check(*Pat, [&](const auto &) { return true; })) + PrintFatalError(RuleDef.getLoc(), + "OperandTypeChecker unexpectedly failed on '" + + Pat->getName() + "' during Type Inference"); + } + OTC.propagateTypes(); + + if (DebugTypeInfer) { + errs() << "Apply patterns for rule " << RuleDef.getName() + << " after inference:\n"; + for (auto *Pat : ApplyPats) { + errs() << " "; + Pat->print(errs(), /*PrintName*/ true); + errs() << '\n'; + } + errs() << '\n'; + } + } +} + +PatternType CombineRuleOperandTypeChecker::inferImmediateType( + const InstructionPattern &IP, unsigned ImmOpIdx, + const TypeEquivalenceClasses &TECs) const { + // We can only infer CGPs. + const auto *CGP = dyn_cast<CodeGenInstructionPattern>(&IP); + if (!CGP) + return {}; + + // For CGPs, we try to infer immediates by trying to infer another named + // operand that shares its type. + // + // e.g. + // Pattern: G_BUILD_VECTOR $x, $y, 0 + // MCOIs: [MCOI::OPERAND_GENERIC_0, MCOI::OPERAND_GENERIC_1, + // MCOI::OPERAND_GENERIC_1] + // $y has the same type as 0, so we can infer $y and get the type 0 should + // have. + + // We infer immediates by looking for a named operand that shares the same + // MCOI type. + const auto MCOITypes = getMCOIOperandTypes(*CGP); + StringRef ImmOpTy = MCOITypes[ImmOpIdx]; + + for (const auto &[Idx, Ty] : enumerate(MCOITypes)) { + if (Idx != ImmOpIdx && Ty == ImmOpTy) { + const auto &Op = IP.getOperand(Idx); + if (!Op.isNamedOperand()) + continue; + + // Named operand with the same name, try to infer that. + if (PatternType InferTy = + inferNamedOperandType(IP, Op.getOperandName(), TECs)) + return InferTy; + } + } + + return {}; +} + +PatternType CombineRuleOperandTypeChecker::inferNamedOperandType( + const InstructionPattern &IP, StringRef OpName, + const TypeEquivalenceClasses &TECs) const { + // This is the simplest possible case, we just need to find a TEC that + // contains OpName. Look at all other operands in equivalence class and try to + // find a suitable one. + + // Check for a def of a matched pattern. This is guaranteed to always + // be a register so we can blindly use that. + StringRef GoodOpName; + for (auto It = TECs.findLeader(OpName); It != TECs.member_end(); ++It) { + if (*It == OpName) + continue; + + const auto LookupRes = MatchOpTable.lookup(*It); + if (LookupRes.Def) // Favor defs + return PatternType::getTypeOf(*It); + + // Otherwise just save this in case we don't find any def. + if (GoodOpName.empty() && LookupRes.Found) + GoodOpName = *It; + } + + if (!GoodOpName.empty()) + return PatternType::getTypeOf(GoodOpName); + + // No good operand found, give up. + return {}; +} + +std::vector<std::string> CombineRuleOperandTypeChecker::getMCOIOperandTypes( + const CodeGenInstructionPattern &CGP) { + // FIXME?: Should we cache this? We call it twice when inferring immediates. + + static unsigned UnknownTypeIdx = 0; + + std::vector<std::string> OpTypes; + auto &CGI = CGP.getInst(); + Record *VarArgsTy = CGI.TheDef->isSubClassOf("GenericInstruction") + ? CGI.TheDef->getValueAsOptionalDef("variadicOpsType") + : nullptr; + std::string VarArgsTyName = + VarArgsTy ? ("MCOI::" + VarArgsTy->getValueAsString("OperandType")).str() + : ("unknown_type_" + Twine(UnknownTypeIdx++)).str(); + + // First, handle defs. + for (unsigned K = 0; K < CGI.Operands.NumDefs; ++K) + OpTypes.push_back(CGI.Operands[K].OperandType); + + // Then, handle variadic defs if there are any. + if (CGP.hasVariadicDefs()) { + for (unsigned K = CGI.Operands.NumDefs; K < CGP.getNumInstDefs(); ++K) + OpTypes.push_back(VarArgsTyName); + } + + // If we had variadic defs, the op idx in the pattern won't match the op idx + // in the CGI anymore. + int CGIOpOffset = int(CGI.Operands.NumDefs) - CGP.getNumInstDefs(); + assert(CGP.hasVariadicDefs() ? (CGIOpOffset <= 0) : (CGIOpOffset == 0)); + + // Handle all remaining use operands, including variadic ones. + for (unsigned K = CGP.getNumInstDefs(); K < CGP.getNumInstOperands(); ++K) { + unsigned CGIOpIdx = K + CGIOpOffset; + if (CGIOpIdx >= CGI.Operands.size()) { + assert(CGP.isVariadic()); + OpTypes.push_back(VarArgsTyName); + } else { + OpTypes.push_back(CGI.Operands[CGIOpIdx].OperandType); + } + } + + assert(OpTypes.size() == CGP.operands_size()); + return OpTypes; +} + +void CombineRuleOperandTypeChecker::getInstEqClasses( + const InstructionPattern &P, TypeEquivalenceClasses &OutTECs) const { + // Determine the TypeEquivalenceClasses by: + // - Getting the MCOI Operand Types. + // - Creating a Map of MCOI Type -> [Operand Indexes] + // - Iterating over the map, filtering types we don't like, and just adding + // the array of Operand Indexes to \p OutTECs. + + // We can only do this on CodeGenInstructions. Other InstructionPatterns have + // no type inference information associated with them. + // TODO: Could we add some inference information to builtins at least? e.g. + // ReplaceReg should always replace with a reg of the same type, for instance. + // Though, those patterns are often used alone so it might not be worth the + // trouble to infer their types. + auto *CGP = dyn_cast<CodeGenInstructionPattern>(&P); + if (!CGP) + return; + + const auto MCOITypes = getMCOIOperandTypes(*CGP); + assert(MCOITypes.size() == P.operands_size()); + + DenseMap<StringRef, std::vector<unsigned>> TyToOpIdx; + for (const auto &[Idx, Ty] : enumerate(MCOITypes)) + TyToOpIdx[Ty].push_back(Idx); + + if (DebugTypeInfer) + errs() << "\tGroups for " << P.getName() << ":\t"; + + for (const auto &[Ty, Idxs] : TyToOpIdx) { + if (!canMCOIOperandTypeBeARegister(Ty)) + continue; + + if (DebugTypeInfer) + errs() << '['; + StringRef Sep = ""; + + // We only collect named operands. + StringRef Leader; + for (unsigned Idx : Idxs) { + const auto &Op = P.getOperand(Idx); + if (!Op.isNamedOperand()) + continue; + + const auto OpName = Op.getOperandName(); + if (DebugTypeInfer) { + errs() << Sep << OpName; + Sep = ", "; + } + + if (Leader.empty()) + OutTECs.insert((Leader = OpName)); + else + OutTECs.unionSets(Leader, OpName); + } + + if (DebugTypeInfer) + errs() << "] "; + } + + if (DebugTypeInfer) + errs() << '\n'; +} + +CombineRuleOperandTypeChecker::TypeEquivalenceClasses +CombineRuleOperandTypeChecker::getRuleEqClasses() const { + StringMap<unsigned> OpNameToEqClassIdx; + TypeEquivalenceClasses TECs; + + if (DebugTypeInfer) + errs() << "Rule Operand Type Equivalence Classes for " << RuleDef.getName() + << ":\n"; + + for (const auto *Pat : MatchPats) + getInstEqClasses(*Pat, TECs); + for (const auto *Pat : ApplyPats) + getInstEqClasses(*Pat, TECs); + + if (DebugTypeInfer) { + errs() << "Final Type Equivalence Classes: "; + for (auto ClassIt = TECs.begin(); ClassIt != TECs.end(); ++ClassIt) { + // only print non-empty classes. + if (auto MembIt = TECs.member_begin(ClassIt); + MembIt != TECs.member_end()) { + errs() << '['; + StringRef Sep = ""; + for (; MembIt != TECs.member_end(); ++MembIt) { + errs() << Sep << *MembIt; + Sep = ", "; + } + errs() << "] "; + } + } + errs() << '\n'; + } + + return TECs; +} + +//===- CombineRuleBuilder -------------------------------------------------===// + +/// Parses combine rule and builds a small intermediate representation to tie +/// patterns together and emit RuleMatchers to match them. This may emit more +/// than one RuleMatcher, e.g. for `wip_match_opcode`. +/// +/// Memory management for `Pattern` objects is done through `std::unique_ptr`. +/// In most cases, there are two stages to a pattern's lifetime: +/// - Creation in a `parse` function +/// - The unique_ptr is stored in a variable, and may be destroyed if the +/// pattern is found to be semantically invalid. +/// - Ownership transfer into a `PatternMap` +/// - Once a pattern is moved into either the map of Match or Apply +/// patterns, it is known to be valid and it never moves back. +class CombineRuleBuilder { +public: + using PatternMap = MapVector<StringRef, std::unique_ptr<Pattern>>; + using PatternAlternatives = DenseMap<const Pattern *, unsigned>; + + CombineRuleBuilder(const CodeGenTarget &CGT, + SubtargetFeatureInfoMap &SubtargetFeatures, + Record &RuleDef, unsigned ID, + std::vector<RuleMatcher> &OutRMs) + : CGT(CGT), SubtargetFeatures(SubtargetFeatures), RuleDef(RuleDef), + RuleID(ID), OutRMs(OutRMs) {} + + /// Parses all fields in the RuleDef record. + bool parseAll(); + + /// Emits all RuleMatchers into the vector of RuleMatchers passed in the + /// constructor. + bool emitRuleMatchers(); + + void print(raw_ostream &OS) const; + void dump() const { print(dbgs()); } + + /// Debug-only verification of invariants. +#ifndef NDEBUG + void verify() const; +#endif + +private: + const CodeGenInstruction &getGConstant() const { + return CGT.getInstruction(RuleDef.getRecords().getDef("G_CONSTANT")); + } + + void PrintError(Twine Msg) const { ::PrintError(&RuleDef, Msg); } + void PrintWarning(Twine Msg) const { ::PrintWarning(RuleDef.getLoc(), Msg); } + void PrintNote(Twine Msg) const { ::PrintNote(RuleDef.getLoc(), Msg); } + + void print(raw_ostream &OS, const PatternAlternatives &Alts) const; + + bool addApplyPattern(std::unique_ptr<Pattern> Pat); + bool addMatchPattern(std::unique_ptr<Pattern> Pat); + + /// Adds the expansions from \see MatchDatas to \p CE. + void declareAllMatchDatasExpansions(CodeExpansions &CE) const; + + /// Adds a matcher \p P to \p IM, expanding its code using \p CE. + /// Note that the predicate is added on the last InstructionMatcher. + /// + /// \p Alts is only used if DebugCXXPreds is enabled. + void addCXXPredicate(RuleMatcher &M, const CodeExpansions &CE, + const CXXPattern &P, const PatternAlternatives &Alts); + + /// Adds an apply \p P to \p IM, expanding its code using \p CE. + void addCXXAction(RuleMatcher &M, const CodeExpansions &CE, + const CXXPattern &P); + + bool hasOnlyCXXApplyPatterns() const; + bool hasEraseRoot() const; + + // Infer machine operand types and check their consistency. + bool typecheckPatterns(); + + /// For all PatFragPatterns, add a new entry in PatternAlternatives for each + /// PatternList it contains. This is multiplicative, so if we have 2 + /// PatFrags with 3 alternatives each, we get 2*3 permutations added to + /// PermutationsToEmit. The "MaxPermutations" field controls how many + /// permutations are allowed before an error is emitted and this function + /// returns false. This is a simple safeguard to prevent combination of + /// PatFrags from generating enormous amounts of rules. + bool buildPermutationsToEmit(); + + /// Checks additional semantics of the Patterns. + bool checkSemantics(); + + /// Creates a new RuleMatcher with some boilerplate + /// settings/actions/predicates, and and adds it to \p OutRMs. + /// \see addFeaturePredicates too. + /// + /// \param Alts Current set of alternatives, for debug comment. + /// \param AdditionalComment Comment string to be added to the + /// `DebugCommentAction`. + RuleMatcher &addRuleMatcher(const PatternAlternatives &Alts, + Twine AdditionalComment = ""); + bool addFeaturePredicates(RuleMatcher &M); + + bool findRoots(); + bool buildRuleOperandsTable(); + + bool parseDefs(const DagInit &Def); + bool + parsePatternList(const DagInit &List, + function_ref<bool(std::unique_ptr<Pattern>)> ParseAction, + StringRef Operator, ArrayRef<SMLoc> DiagLoc, + StringRef AnonPatNamePrefix) const; + + std::unique_ptr<Pattern> parseInstructionPattern(const Init &Arg, + StringRef PatName) const; + std::unique_ptr<Pattern> parseWipMatchOpcodeMatcher(const Init &Arg, + StringRef PatName) const; + bool parseInstructionPatternOperand(InstructionPattern &IP, + const Init *OpInit, + const StringInit *OpName) const; + bool parseInstructionPatternMIFlags(InstructionPattern &IP, + const DagInit *Op) const; + std::unique_ptr<PatFrag> parsePatFragImpl(const Record *Def) const; + bool parsePatFragParamList( + ArrayRef<SMLoc> DiagLoc, const DagInit &OpsList, + function_ref<bool(StringRef, PatFrag::ParamKind)> ParseAction) const; + const PatFrag *parsePatFrag(const Record *Def) const; + + bool emitMatchPattern(CodeExpansions &CE, const PatternAlternatives &Alts, + const InstructionPattern &IP); + bool emitMatchPattern(CodeExpansions &CE, const PatternAlternatives &Alts, + const AnyOpcodePattern &AOP); + + bool emitPatFragMatchPattern(CodeExpansions &CE, + const PatternAlternatives &Alts, RuleMatcher &RM, + InstructionMatcher *IM, + const PatFragPattern &PFP, + DenseSet<const Pattern *> &SeenPats); + + bool emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M); + + // Recursively visits InstructionPatterns from P to build up the + // RuleMatcher actions. + bool emitInstructionApplyPattern(CodeExpansions &CE, RuleMatcher &M, + const InstructionPattern &P, + DenseSet<const Pattern *> &SeenPats, + StringMap<unsigned> &OperandToTempRegID); + + bool emitCodeGenInstructionApplyImmOperand(RuleMatcher &M, + BuildMIAction &DstMI, + const CodeGenInstructionPattern &P, + const InstructionOperand &O); + + bool emitBuiltinApplyPattern(CodeExpansions &CE, RuleMatcher &M, + const BuiltinPattern &P, + StringMap<unsigned> &OperandToTempRegID); + + // Recursively visits CodeGenInstructionPattern from P to build up the + // RuleMatcher/InstructionMatcher. May create new InstructionMatchers as + // needed. + using OperandMapperFnRef = + function_ref<InstructionOperand(const InstructionOperand &)>; + using OperandDefLookupFn = + function_ref<const InstructionPattern *(StringRef)>; + bool emitCodeGenInstructionMatchPattern( + CodeExpansions &CE, const PatternAlternatives &Alts, RuleMatcher &M, + InstructionMatcher &IM, const CodeGenInstructionPattern &P, + DenseSet<const Pattern *> &SeenPats, OperandDefLookupFn LookupOperandDef, + OperandMapperFnRef OperandMapper = [](const auto &O) { return O; }); + + const CodeGenTarget &CGT; + SubtargetFeatureInfoMap &SubtargetFeatures; + Record &RuleDef; + const unsigned RuleID; + std::vector<RuleMatcher> &OutRMs; + + // For InstructionMatcher::addOperand + unsigned AllocatedTemporariesBaseID = 0; + + /// The root of the pattern. + StringRef RootName; + + /// These maps have ownership of the actual Pattern objects. + /// They both map a Pattern's name to the Pattern instance. + PatternMap MatchPats; + PatternMap ApplyPats; + + /// Operand tables to tie match/apply patterns together. + OperandTable MatchOpTable; + OperandTable ApplyOpTable; + + /// Set by findRoots. + Pattern *MatchRoot = nullptr; + SmallDenseSet<InstructionPattern *, 2> ApplyRoots; + + SmallVector<MatchDataInfo, 2> MatchDatas; + SmallVector<PatternAlternatives, 1> PermutationsToEmit; + + // print()/debug-only members. + mutable SmallPtrSet<const PatFrag *, 2> SeenPatFrags; +}; + +bool CombineRuleBuilder::parseAll() { + auto StackTrace = PrettyStackTraceParse(RuleDef); + + if (!parseDefs(*RuleDef.getValueAsDag("Defs"))) + return false; + + if (!parsePatternList( + *RuleDef.getValueAsDag("Match"), + [this](auto Pat) { return addMatchPattern(std::move(Pat)); }, "match", + RuleDef.getLoc(), (RuleDef.getName() + "_match").str())) + return false; + + if (!parsePatternList( + *RuleDef.getValueAsDag("Apply"), + [this](auto Pat) { return addApplyPattern(std::move(Pat)); }, "apply", + RuleDef.getLoc(), (RuleDef.getName() + "_apply").str())) + return false; + + if (!buildRuleOperandsTable() || !typecheckPatterns() || !findRoots() || + !checkSemantics() || !buildPermutationsToEmit()) + return false; + LLVM_DEBUG(verify()); + return true; +} + +bool CombineRuleBuilder::emitRuleMatchers() { + auto StackTrace = PrettyStackTraceEmit(RuleDef); + + assert(MatchRoot); + CodeExpansions CE; + declareAllMatchDatasExpansions(CE); + + assert(!PermutationsToEmit.empty()); + for (const auto &Alts : PermutationsToEmit) { + switch (MatchRoot->getKind()) { + case Pattern::K_AnyOpcode: { + if (!emitMatchPattern(CE, Alts, *cast<AnyOpcodePattern>(MatchRoot))) + return false; + break; + } + case Pattern::K_PatFrag: + case Pattern::K_Builtin: + case Pattern::K_CodeGenInstruction: + if (!emitMatchPattern(CE, Alts, *cast<InstructionPattern>(MatchRoot))) + return false; + break; + case Pattern::K_CXX: + PrintError("C++ code cannot be the root of a rule!"); + return false; + default: + llvm_unreachable("unknown pattern kind!"); + } + } + + return true; +} + +void CombineRuleBuilder::print(raw_ostream &OS) const { + OS << "(CombineRule name:" << RuleDef.getName() << " id:" << RuleID + << " root:" << RootName << '\n'; + + if (!MatchDatas.empty()) { + OS << " (MatchDatas\n"; + for (const auto &MD : MatchDatas) { + OS << " "; + MD.print(OS); + OS << '\n'; + } + OS << " )\n"; + } + + if (!SeenPatFrags.empty()) { + OS << " (PatFrags\n"; + for (const auto *PF : SeenPatFrags) { + PF->print(OS, /*Indent=*/" "); + OS << '\n'; + } + OS << " )\n"; + } + + const auto DumpPats = [&](StringRef Name, const PatternMap &Pats) { + OS << " (" << Name << " "; + if (Pats.empty()) { + OS << "<empty>)\n"; + return; + } + + OS << '\n'; + for (const auto &[Name, Pat] : Pats) { + OS << " "; + if (Pat.get() == MatchRoot) + OS << "<match_root>"; + if (isa<InstructionPattern>(Pat.get()) && + ApplyRoots.contains(cast<InstructionPattern>(Pat.get()))) + OS << "<apply_root>"; + OS << Name << ":"; + Pat->print(OS, /*PrintName=*/false); + OS << '\n'; + } + OS << " )\n"; + }; + + DumpPats("MatchPats", MatchPats); + DumpPats("ApplyPats", ApplyPats); + + MatchOpTable.print(OS, "MatchPats", /*Indent*/ " "); + ApplyOpTable.print(OS, "ApplyPats", /*Indent*/ " "); + + if (PermutationsToEmit.size() > 1) { + OS << " (PermutationsToEmit\n"; + for (const auto &Perm : PermutationsToEmit) { + OS << " "; + print(OS, Perm); + OS << ",\n"; + } + OS << " )\n"; + } + + OS << ")\n"; +} + +#ifndef NDEBUG +void CombineRuleBuilder::verify() const { + const auto VerifyPats = [&](const PatternMap &Pats) { + for (const auto &[Name, Pat] : Pats) { + if (!Pat) + PrintFatalError("null pattern in pattern map!"); + + if (Name != Pat->getName()) { + Pat->dump(); + PrintFatalError("Pattern name mismatch! Map name: " + Name + + ", Pat name: " + Pat->getName()); + } + + // Sanity check: the map should point to the same data as the Pattern. + // Both strings are allocated in the pool using insertStrRef. + if (Name.data() != Pat->getName().data()) { + dbgs() << "Map StringRef: '" << Name << "' @ " + << (const void *)Name.data() << '\n'; + dbgs() << "Pat String: '" << Pat->getName() << "' @ " + << (const void *)Pat->getName().data() << '\n'; + PrintFatalError("StringRef stored in the PatternMap is not referencing " + "the same string as its Pattern!"); + } + } + }; + + VerifyPats(MatchPats); + VerifyPats(ApplyPats); + + // Check there are no wip_match_opcode patterns in the "apply" patterns. + if (any_of(ApplyPats, + [&](auto &E) { return isa<AnyOpcodePattern>(E.second.get()); })) { + dump(); + PrintFatalError( + "illegal wip_match_opcode pattern in the 'apply' patterns!"); + } + + // Check there are no nullptrs in ApplyRoots. + if (ApplyRoots.contains(nullptr)) { + PrintFatalError( + "CombineRuleBuilder's ApplyRoots set contains a null pointer!"); + } +} +#endif + +void CombineRuleBuilder::print(raw_ostream &OS, + const PatternAlternatives &Alts) const { + SmallVector<std::string, 1> Strings( + map_range(Alts, [](const auto &PatAndPerm) { + return PatAndPerm.first->getName().str() + "[" + + to_string(PatAndPerm.second) + "]"; + })); + // Sort so output is deterministic for tests. Otherwise it's sorted by pointer + // values. + sort(Strings); + OS << "[" << join(Strings, ", ") << "]"; +} + +bool CombineRuleBuilder::addApplyPattern(std::unique_ptr<Pattern> Pat) { + StringRef Name = Pat->getName(); + if (ApplyPats.contains(Name)) { + PrintError("'" + Name + "' apply pattern defined more than once!"); + return false; + } + + if (isa<AnyOpcodePattern>(Pat.get())) { + PrintError("'" + Name + + "': wip_match_opcode is not supported in apply patterns"); + return false; + } + + if (isa<PatFragPattern>(Pat.get())) { + PrintError("'" + Name + "': using " + PatFrag::ClassName + + " is not supported in apply patterns"); + return false; + } + + if (auto *CXXPat = dyn_cast<CXXPattern>(Pat.get())) + CXXPat->setIsApply(); + + ApplyPats[Name] = std::move(Pat); + return true; +} + +bool CombineRuleBuilder::addMatchPattern(std::unique_ptr<Pattern> Pat) { + StringRef Name = Pat->getName(); + if (MatchPats.contains(Name)) { + PrintError("'" + Name + "' match pattern defined more than once!"); + return false; + } + + // For now, none of the builtins can appear in 'match'. + if (const auto *BP = dyn_cast<BuiltinPattern>(Pat.get())) { + PrintError("'" + BP->getInstName() + + "' cannot be used in a 'match' pattern"); + return false; + } + + MatchPats[Name] = std::move(Pat); + return true; +} + +void CombineRuleBuilder::declareAllMatchDatasExpansions( + CodeExpansions &CE) const { + for (const auto &MD : MatchDatas) + CE.declare(MD.getPatternSymbol(), MD.getQualifiedVariableName()); +} + +void CombineRuleBuilder::addCXXPredicate(RuleMatcher &M, + const CodeExpansions &CE, + const CXXPattern &P, + const PatternAlternatives &Alts) { + // FIXME: Hack so C++ code is executed last. May not work for more complex + // patterns. + auto &IM = *std::prev(M.insnmatchers().end()); + auto Loc = RuleDef.getLoc(); + const auto AddComment = [&](raw_ostream &OS) { + OS << "// Pattern Alternatives: "; + print(OS, Alts); + OS << '\n'; + }; + const auto &ExpandedCode = + DebugCXXPreds ? P.expandCode(CE, Loc, AddComment) : P.expandCode(CE, Loc); + IM->addPredicate<GenericInstructionPredicateMatcher>( + ExpandedCode.getEnumNameWithPrefix(CXXPredPrefix)); +} + +void CombineRuleBuilder::addCXXAction(RuleMatcher &M, const CodeExpansions &CE, + const CXXPattern &P) { + const auto &ExpandedCode = P.expandCode(CE, RuleDef.getLoc()); + M.addAction<CustomCXXAction>( + ExpandedCode.getEnumNameWithPrefix(CXXApplyPrefix)); +} + +bool CombineRuleBuilder::hasOnlyCXXApplyPatterns() const { + return all_of(ApplyPats, [&](auto &Entry) { + return isa<CXXPattern>(Entry.second.get()); + }); +} + +bool CombineRuleBuilder::hasEraseRoot() const { + return any_of(ApplyPats, [&](auto &Entry) { + if (const auto *BP = dyn_cast<BuiltinPattern>(Entry.second.get())) + return BP->getBuiltinKind() == BI_EraseRoot; + return false; + }); +} + +bool CombineRuleBuilder::typecheckPatterns() { + CombineRuleOperandTypeChecker OTC(RuleDef, MatchOpTable); + + for (auto &Pat : values(MatchPats)) { + if (auto *IP = dyn_cast<InstructionPattern>(Pat.get())) { + if (!OTC.processMatchPattern(*IP)) + return false; + } + } + + for (auto &Pat : values(ApplyPats)) { + if (auto *IP = dyn_cast<InstructionPattern>(Pat.get())) { + if (!OTC.processApplyPattern(*IP)) + return false; + } + } + + OTC.propagateAndInferTypes(); + + // Always check this after in case inference adds some special types to the + // match patterns. + for (auto &Pat : values(MatchPats)) { + if (auto *IP = dyn_cast<InstructionPattern>(Pat.get())) { + if (IP->diagnoseAllSpecialTypes( + RuleDef.getLoc(), PatternType::SpecialTyClassName + + " is not supported in 'match' patterns")) { + return false; + } + } + } + return true; +} + +bool CombineRuleBuilder::buildPermutationsToEmit() { + PermutationsToEmit.clear(); + + // Start with one empty set of alternatives. + PermutationsToEmit.emplace_back(); + for (const auto &Pat : values(MatchPats)) { + unsigned NumAlts = 0; + // Note: technically, AnyOpcodePattern also needs permutations, but: + // - We only allow a single one of them in the root. + // - They cannot be mixed with any other pattern other than C++ code. + // So we don't really need to take them into account here. We could, but + // that pattern is a hack anyway and the less it's involved, the better. + if (const auto *PFP = dyn_cast<PatFragPattern>(Pat.get())) + NumAlts = PFP->getPatFrag().num_alternatives(); + else + continue; + + // For each pattern that needs permutations, multiply the current set of + // alternatives. + auto CurPerms = PermutationsToEmit; + PermutationsToEmit.clear(); + + for (const auto &Perm : CurPerms) { + assert(!Perm.count(Pat.get()) && "Pattern already emitted?"); + for (unsigned K = 0; K < NumAlts; ++K) { + PatternAlternatives NewPerm = Perm; + NewPerm[Pat.get()] = K; + PermutationsToEmit.emplace_back(std::move(NewPerm)); + } + } + } + + if (int64_t MaxPerms = RuleDef.getValueAsInt("MaxPermutations"); + MaxPerms > 0) { + if ((int64_t)PermutationsToEmit.size() > MaxPerms) { + PrintError("cannot emit rule '" + RuleDef.getName() + "'; " + + Twine(PermutationsToEmit.size()) + + " permutations would be emitted, but the max is " + + Twine(MaxPerms)); + return false; + } + } + + // Ensure we always have a single empty entry, it simplifies the emission + // logic so it doesn't need to handle the case where there are no perms. + if (PermutationsToEmit.empty()) { + PermutationsToEmit.emplace_back(); + return true; + } + + return true; +} + +bool CombineRuleBuilder::checkSemantics() { + assert(MatchRoot && "Cannot call this before findRoots()"); + + bool UsesWipMatchOpcode = false; + for (const auto &Match : MatchPats) { + const auto *Pat = Match.second.get(); + + if (const auto *CXXPat = dyn_cast<CXXPattern>(Pat)) { + if (!CXXPat->getRawCode().contains("return ")) + PrintWarning("'match' C++ code does not seem to return!"); + continue; + } + + // MIFlags in match cannot use the following syntax: (MIFlags $mi) + if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Pat)) { + if (auto *FI = CGP->getMIFlagsInfo()) { + if (!FI->copy_flags().empty()) { + PrintError( + "'match' patterns cannot refer to flags from other instructions"); + PrintNote("MIFlags in '" + CGP->getName() + + "' refer to: " + join(FI->copy_flags(), ", ")); + return false; + } + } + } + + const auto *AOP = dyn_cast<AnyOpcodePattern>(Pat); + if (!AOP) + continue; + + if (UsesWipMatchOpcode) { + PrintError("wip_opcode_match can only be present once"); + return false; + } + + UsesWipMatchOpcode = true; + } + + for (const auto &Apply : ApplyPats) { + assert(Apply.second.get()); + const auto *IP = dyn_cast<InstructionPattern>(Apply.second.get()); + if (!IP) + continue; + + if (UsesWipMatchOpcode) { + PrintError("cannot use wip_match_opcode in combination with apply " + "instruction patterns!"); + return false; + } + + // Check that the insts mentioned in copy_flags exist. + if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(IP)) { + if (auto *FI = CGP->getMIFlagsInfo()) { + for (auto InstName : FI->copy_flags()) { + auto It = MatchPats.find(InstName); + if (It == MatchPats.end()) { + PrintError("unknown instruction '$" + InstName + + "' referenced in MIFlags of '" + CGP->getName() + "'"); + return false; + } + + if (!isa<CodeGenInstructionPattern>(It->second.get())) { + PrintError( + "'$" + InstName + + "' does not refer to a CodeGenInstruction in MIFlags of '" + + CGP->getName() + "'"); + return false; + } + } + } + } + + const auto *BIP = dyn_cast<BuiltinPattern>(IP); + if (!BIP) + continue; + StringRef Name = BIP->getInstName(); + + // (GIEraseInst) has to be the only apply pattern, or it can not be used at + // all. The root cannot have any defs either. + switch (BIP->getBuiltinKind()) { + case BI_EraseRoot: { + if (ApplyPats.size() > 1) { + PrintError(Name + " must be the only 'apply' pattern"); + return false; + } + + const auto *IRoot = dyn_cast<CodeGenInstructionPattern>(MatchRoot); + if (!IRoot) { + PrintError(Name + + " can only be used if the root is a CodeGenInstruction"); + return false; + } + + if (IRoot->getNumInstDefs() != 0) { + PrintError(Name + " can only be used if on roots that do " + "not have any output operand"); + PrintNote("'" + IRoot->getInstName() + "' has " + + Twine(IRoot->getNumInstDefs()) + " output operands"); + return false; + } + break; + } + case BI_ReplaceReg: { + // (GIReplaceReg can only be used on the root instruction) + // TODO: When we allow rewriting non-root instructions, also allow this. + StringRef OldRegName = BIP->getOperand(0).getOperandName(); + auto *Def = MatchOpTable.getDef(OldRegName); + if (!Def) { + PrintError(Name + " cannot find a matched pattern that defines '" + + OldRegName + "'"); + return false; + } + if (MatchOpTable.getDef(OldRegName) != MatchRoot) { + PrintError(Name + " cannot replace '" + OldRegName + + "': this builtin can only replace a register defined by the " + "match root"); + return false; + } + break; + } + } + } + + return true; +} + +RuleMatcher &CombineRuleBuilder::addRuleMatcher(const PatternAlternatives &Alts, + Twine AdditionalComment) { + auto &RM = OutRMs.emplace_back(RuleDef.getLoc()); + addFeaturePredicates(RM); + RM.setPermanentGISelFlags(GISF_IgnoreCopies); + RM.addRequiredSimplePredicate(getIsEnabledPredicateEnumName(RuleID)); + + std::string Comment; + raw_string_ostream CommentOS(Comment); + CommentOS << "Combiner Rule #" << RuleID << ": " << RuleDef.getName(); + if (!Alts.empty()) { + CommentOS << " @ "; + print(CommentOS, Alts); + } + if (!AdditionalComment.isTriviallyEmpty()) + CommentOS << "; " << AdditionalComment; + RM.addAction<DebugCommentAction>(Comment); + return RM; +} + +bool CombineRuleBuilder::addFeaturePredicates(RuleMatcher &M) { + if (!RuleDef.getValue("Predicates")) + return true; + + ListInit *Preds = RuleDef.getValueAsListInit("Predicates"); + for (Init *PI : Preds->getValues()) { + DefInit *Pred = dyn_cast<DefInit>(PI); + if (!Pred) + continue; + + Record *Def = Pred->getDef(); + if (!Def->isSubClassOf("Predicate")) { + ::PrintError(Def, "Unknown 'Predicate' Type"); + return false; + } + + if (Def->getValueAsString("CondString").empty()) + continue; + + if (SubtargetFeatures.count(Def) == 0) { + SubtargetFeatures.emplace( + Def, SubtargetFeatureInfo(Def, SubtargetFeatures.size())); + } + + M.addRequiredFeature(Def); + } + + return true; +} + +bool CombineRuleBuilder::findRoots() { + const auto Finish = [&]() { + assert(MatchRoot); + + if (hasOnlyCXXApplyPatterns() || hasEraseRoot()) + return true; + + auto *IPRoot = dyn_cast<InstructionPattern>(MatchRoot); + if (!IPRoot) + return true; + + if (IPRoot->getNumInstDefs() == 0) { + // No defs to work with -> find the root using the pattern name. + auto It = ApplyPats.find(RootName); + if (It == ApplyPats.end()) { + PrintError("Cannot find root '" + RootName + "' in apply patterns!"); + return false; + } + + auto *ApplyRoot = dyn_cast<InstructionPattern>(It->second.get()); + if (!ApplyRoot) { + PrintError("apply pattern root '" + RootName + + "' must be an instruction pattern"); + return false; + } + + ApplyRoots.insert(ApplyRoot); + return true; + } + + // Collect all redefinitions of the MatchRoot's defs and put them in + // ApplyRoots. + const auto DefsNeeded = IPRoot->getApplyDefsNeeded(); + for (auto &Op : DefsNeeded) { + assert(Op.isDef() && Op.isNamedOperand()); + StringRef Name = Op.getOperandName(); + + auto *ApplyRedef = ApplyOpTable.getDef(Name); + if (!ApplyRedef) { + PrintError("'" + Name + "' must be redefined in the 'apply' pattern"); + return false; + } + + ApplyRoots.insert((InstructionPattern *)ApplyRedef); + } + + if (auto It = ApplyPats.find(RootName); It != ApplyPats.end()) { + if (find(ApplyRoots, It->second.get()) == ApplyRoots.end()) { + PrintError("apply pattern '" + RootName + + "' is supposed to be a root but it does not redefine any of " + "the defs of the match root"); + return false; + } + } + + return true; + }; + + // Look by pattern name, e.g. + // (G_FNEG $x, $y):$root + if (auto MatchPatIt = MatchPats.find(RootName); + MatchPatIt != MatchPats.end()) { + MatchRoot = MatchPatIt->second.get(); + return Finish(); + } + + // Look by def: + // (G_FNEG $root, $y) + auto LookupRes = MatchOpTable.lookup(RootName); + if (!LookupRes.Found) { + PrintError("Cannot find root '" + RootName + "' in match patterns!"); + return false; + } + + MatchRoot = LookupRes.Def; + if (!MatchRoot) { + PrintError("Cannot use live-in operand '" + RootName + + "' as match pattern root!"); + return false; + } + + return Finish(); +} + +bool CombineRuleBuilder::buildRuleOperandsTable() { + const auto DiagnoseRedefMatch = [&](StringRef OpName) { + PrintError("Operand '" + OpName + + "' is defined multiple times in the 'match' patterns"); + }; + + const auto DiagnoseRedefApply = [&](StringRef OpName) { + PrintError("Operand '" + OpName + + "' is defined multiple times in the 'apply' patterns"); + }; + + for (auto &Pat : values(MatchPats)) { + auto *IP = dyn_cast<InstructionPattern>(Pat.get()); + if (IP && !MatchOpTable.addPattern(IP, DiagnoseRedefMatch)) + return false; + } + + for (auto &Pat : values(ApplyPats)) { + auto *IP = dyn_cast<InstructionPattern>(Pat.get()); + if (IP && !ApplyOpTable.addPattern(IP, DiagnoseRedefApply)) + return false; + } + + return true; +} + +bool CombineRuleBuilder::parseDefs(const DagInit &Def) { + if (Def.getOperatorAsDef(RuleDef.getLoc())->getName() != "defs") { + PrintError("Expected defs operator"); + return false; + } + + SmallVector<StringRef> Roots; + for (unsigned I = 0, E = Def.getNumArgs(); I < E; ++I) { + if (isSpecificDef(*Def.getArg(I), "root")) { + Roots.emplace_back(Def.getArgNameStr(I)); + continue; + } + + // Subclasses of GIDefMatchData should declare that this rule needs to pass + // data from the match stage to the apply stage, and ensure that the + // generated matcher has a suitable variable for it to do so. + if (Record *MatchDataRec = + getDefOfSubClass(*Def.getArg(I), "GIDefMatchData")) { + MatchDatas.emplace_back(Def.getArgNameStr(I), + MatchDataRec->getValueAsString("Type")); + continue; + } + + // Otherwise emit an appropriate error message. + if (getDefOfSubClass(*Def.getArg(I), "GIDefKind")) + PrintError("This GIDefKind not implemented in tablegen"); + else if (getDefOfSubClass(*Def.getArg(I), "GIDefKindWithArgs")) + PrintError("This GIDefKindWithArgs not implemented in tablegen"); + else + PrintError("Expected a subclass of GIDefKind or a sub-dag whose " + "operator is of type GIDefKindWithArgs"); + return false; + } + + if (Roots.size() != 1) { + PrintError("Combine rules must have exactly one root"); + return false; + } + + RootName = Roots.front(); + + // Assign variables to all MatchDatas. + AssignMatchDataVariables(MatchDatas); + return true; +} + +bool CombineRuleBuilder::parsePatternList( + const DagInit &List, + function_ref<bool(std::unique_ptr<Pattern>)> ParseAction, + StringRef Operator, ArrayRef<SMLoc> DiagLoc, + StringRef AnonPatNamePrefix) const { + if (List.getOperatorAsDef(RuleDef.getLoc())->getName() != Operator) { + ::PrintError(DiagLoc, "Expected " + Operator + " operator"); + return false; + } + + if (List.getNumArgs() == 0) { + ::PrintError(DiagLoc, Operator + " pattern list is empty"); + return false; + } + + // The match section consists of a list of matchers and predicates. Parse each + // one and add the equivalent GIMatchDag nodes, predicates, and edges. + for (unsigned I = 0; I < List.getNumArgs(); ++I) { + Init *Arg = List.getArg(I); + std::string Name = List.getArgName(I) + ? List.getArgName(I)->getValue().str() + : ("__" + AnonPatNamePrefix + "_" + Twine(I)).str(); + + if (auto Pat = parseInstructionPattern(*Arg, Name)) { + if (!ParseAction(std::move(Pat))) + return false; + continue; + } + + if (auto Pat = parseWipMatchOpcodeMatcher(*Arg, Name)) { + if (!ParseAction(std::move(Pat))) + return false; + continue; + } + + // Parse arbitrary C++ code + if (const auto *StringI = dyn_cast<StringInit>(Arg)) { + auto CXXPat = std::make_unique<CXXPattern>(*StringI, insertStrRef(Name)); + if (!ParseAction(std::move(CXXPat))) + return false; + continue; + } + + ::PrintError(DiagLoc, + "Failed to parse pattern: '" + Arg->getAsString() + "'"); + return false; + } + + return true; +} + +std::unique_ptr<Pattern> +CombineRuleBuilder::parseInstructionPattern(const Init &Arg, + StringRef Name) const { + const DagInit *DagPat = dyn_cast<DagInit>(&Arg); + if (!DagPat) + return nullptr; + + std::unique_ptr<InstructionPattern> Pat; + if (const DagInit *IP = getDagWithOperatorOfSubClass(Arg, "Instruction")) { + auto &Instr = CGT.getInstruction(IP->getOperatorAsDef(RuleDef.getLoc())); + Pat = + std::make_unique<CodeGenInstructionPattern>(Instr, insertStrRef(Name)); + } else if (const DagInit *PFP = + getDagWithOperatorOfSubClass(Arg, PatFrag::ClassName)) { + const Record *Def = PFP->getOperatorAsDef(RuleDef.getLoc()); + const PatFrag *PF = parsePatFrag(Def); + if (!PF) + return nullptr; // Already diagnosed by parsePatFrag + Pat = std::make_unique<PatFragPattern>(*PF, insertStrRef(Name)); + } else if (const DagInit *BP = + getDagWithOperatorOfSubClass(Arg, BuiltinPattern::ClassName)) { + Pat = std::make_unique<BuiltinPattern>( + *BP->getOperatorAsDef(RuleDef.getLoc()), insertStrRef(Name)); + } else { + return nullptr; + } + + for (unsigned K = 0; K < DagPat->getNumArgs(); ++K) { + Init *Arg = DagPat->getArg(K); + if (auto *DagArg = getDagWithSpecificOperator(*Arg, "MIFlags")) { + if (!parseInstructionPatternMIFlags(*Pat, DagArg)) + return nullptr; + continue; + } + + if (!parseInstructionPatternOperand(*Pat, Arg, DagPat->getArgName(K))) + return nullptr; + } + + if (!Pat->checkSemantics(RuleDef.getLoc())) + return nullptr; + + return std::move(Pat); +} + +std::unique_ptr<Pattern> +CombineRuleBuilder::parseWipMatchOpcodeMatcher(const Init &Arg, + StringRef Name) const { + const DagInit *Matcher = getDagWithSpecificOperator(Arg, "wip_match_opcode"); + if (!Matcher) + return nullptr; + + if (Matcher->getNumArgs() == 0) { + PrintError("Empty wip_match_opcode"); + return nullptr; + } + + // Each argument is an opcode that can match. + auto Result = std::make_unique<AnyOpcodePattern>(insertStrRef(Name)); + for (const auto &Arg : Matcher->getArgs()) { + Record *OpcodeDef = getDefOfSubClass(*Arg, "Instruction"); + if (OpcodeDef) { + Result->addOpcode(&CGT.getInstruction(OpcodeDef)); + continue; + } + + PrintError("Arguments to wip_match_opcode must be instructions"); + return nullptr; + } + + return std::move(Result); +} + +bool CombineRuleBuilder::parseInstructionPatternOperand( + InstructionPattern &IP, const Init *OpInit, + const StringInit *OpName) const { + const auto ParseErr = [&]() { + PrintError("cannot parse operand '" + OpInit->getAsUnquotedString() + "' "); + if (OpName) + PrintNote("operand name is '" + OpName->getAsUnquotedString() + "'"); + return false; + }; + + // untyped immediate, e.g. 0 + if (const auto *IntImm = dyn_cast<IntInit>(OpInit)) { + std::string Name = OpName ? OpName->getAsUnquotedString() : ""; + IP.addOperand(IntImm->getValue(), insertStrRef(Name), PatternType()); + return true; + } + + // typed immediate, e.g. (i32 0) + if (const auto *DagOp = dyn_cast<DagInit>(OpInit)) { + if (DagOp->getNumArgs() != 1) + return ParseErr(); + + const Record *TyDef = DagOp->getOperatorAsDef(RuleDef.getLoc()); + auto ImmTy = PatternType::get(RuleDef.getLoc(), TyDef, + "cannot parse immediate '" + + DagOp->getAsUnquotedString() + "'"); + if (!ImmTy) + return false; + + if (!IP.hasAllDefs()) { + PrintError("out operand of '" + IP.getInstName() + + "' cannot be an immediate"); + return false; + } + + const auto *Val = dyn_cast<IntInit>(DagOp->getArg(0)); + if (!Val) + return ParseErr(); + + std::string Name = OpName ? OpName->getAsUnquotedString() : ""; + IP.addOperand(Val->getValue(), insertStrRef(Name), *ImmTy); + return true; + } + + // Typed operand e.g. $x/$z in (G_FNEG $x, $z) + if (auto *DefI = dyn_cast<DefInit>(OpInit)) { + if (!OpName) { + PrintError("expected an operand name after '" + OpInit->getAsString() + + "'"); + return false; + } + const Record *Def = DefI->getDef(); + auto Ty = + PatternType::get(RuleDef.getLoc(), Def, "cannot parse operand type"); + if (!Ty) + return false; + IP.addOperand(insertStrRef(OpName->getAsUnquotedString()), *Ty); + return true; + } + + // Untyped operand e.g. $x/$z in (G_FNEG $x, $z) + if (isa<UnsetInit>(OpInit)) { + assert(OpName && "Unset w/ no OpName?"); + IP.addOperand(insertStrRef(OpName->getAsUnquotedString()), PatternType()); + return true; + } + + return ParseErr(); +} + +bool CombineRuleBuilder::parseInstructionPatternMIFlags( + InstructionPattern &IP, const DagInit *Op) const { + auto *CGIP = dyn_cast<CodeGenInstructionPattern>(&IP); + if (!CGIP) { + PrintError("matching/writing MIFlags is only allowed on CodeGenInstruction " + "patterns"); + return false; + } + + const auto CheckFlagEnum = [&](const Record *R) { + if (!R->isSubClassOf(MIFlagsEnumClassName)) { + PrintError("'" + R->getName() + "' is not a subclass of '" + + MIFlagsEnumClassName + "'"); + return false; + } + + return true; + }; + + if (CGIP->getMIFlagsInfo()) { + PrintError("MIFlags can only be present once on an instruction"); + return false; + } + + auto &FI = CGIP->getOrCreateMIFlagsInfo(); + for (unsigned K = 0; K < Op->getNumArgs(); ++K) { + const Init *Arg = Op->getArg(K); + + // Match/set a flag: (MIFlags FmNoNans) + if (const auto *Def = dyn_cast<DefInit>(Arg)) { + const Record *R = Def->getDef(); + if (!CheckFlagEnum(R)) + return false; + + FI.addSetFlag(R); + continue; + } + + // Do not match a flag/unset a flag: (MIFlags (not FmNoNans)) + if (const DagInit *NotDag = getDagWithSpecificOperator(*Arg, "not")) { + for (const Init *NotArg : NotDag->getArgs()) { + const DefInit *DefArg = dyn_cast<DefInit>(NotArg); + if (!DefArg) { + PrintError("cannot parse '" + NotArg->getAsUnquotedString() + + "': expected a '" + MIFlagsEnumClassName + "'"); + return false; + } + + const Record *R = DefArg->getDef(); + if (!CheckFlagEnum(R)) + return false; + + FI.addUnsetFlag(R); + continue; + } + + continue; + } + + // Copy flags from a matched instruction: (MIFlags $mi) + if (isa<UnsetInit>(Arg)) { + FI.addCopyFlag(insertStrRef(Op->getArgName(K)->getAsUnquotedString())); + continue; + } + } + + return true; +} + +std::unique_ptr<PatFrag> +CombineRuleBuilder::parsePatFragImpl(const Record *Def) const { + auto StackTrace = PrettyStackTraceParse(*Def); + if (!Def->isSubClassOf(PatFrag::ClassName)) + return nullptr; + + const DagInit *Ins = Def->getValueAsDag("InOperands"); + if (Ins->getOperatorAsDef(Def->getLoc())->getName() != "ins") { + ::PrintError(Def, "expected 'ins' operator for " + PatFrag::ClassName + + " in operands list"); + return nullptr; + } + + const DagInit *Outs = Def->getValueAsDag("OutOperands"); + if (Outs->getOperatorAsDef(Def->getLoc())->getName() != "outs") { + ::PrintError(Def, "expected 'outs' operator for " + PatFrag::ClassName + + " out operands list"); + return nullptr; + } + + auto Result = std::make_unique<PatFrag>(*Def); + if (!parsePatFragParamList(Def->getLoc(), *Outs, + [&](StringRef Name, PatFrag::ParamKind Kind) { + Result->addOutParam(insertStrRef(Name), Kind); + return true; + })) + return nullptr; + + if (!parsePatFragParamList(Def->getLoc(), *Ins, + [&](StringRef Name, PatFrag::ParamKind Kind) { + Result->addInParam(insertStrRef(Name), Kind); + return true; + })) + return nullptr; + + const ListInit *Alts = Def->getValueAsListInit("Alternatives"); + unsigned AltIdx = 0; + for (const Init *Alt : *Alts) { + const auto *PatDag = dyn_cast<DagInit>(Alt); + if (!PatDag) { + ::PrintError(Def, "expected dag init for PatFrag pattern alternative"); + return nullptr; + } + + PatFrag::Alternative &A = Result->addAlternative(); + const auto AddPat = [&](std::unique_ptr<Pattern> Pat) { + A.Pats.push_back(std::move(Pat)); + return true; + }; + + if (!parsePatternList( + *PatDag, AddPat, "pattern", Def->getLoc(), + /*AnonPatPrefix*/ + (Def->getName() + "_alt" + Twine(AltIdx++) + "_pattern").str())) + return nullptr; + } + + if (!Result->buildOperandsTables() || !Result->checkSemantics()) + return nullptr; + + return Result; +} + +bool CombineRuleBuilder::parsePatFragParamList( + ArrayRef<SMLoc> DiagLoc, const DagInit &OpsList, + function_ref<bool(StringRef, PatFrag::ParamKind)> ParseAction) const { + for (unsigned K = 0; K < OpsList.getNumArgs(); ++K) { + const StringInit *Name = OpsList.getArgName(K); + const Init *Ty = OpsList.getArg(K); + + if (!Name) { + ::PrintError(DiagLoc, "all operands must be named'"); + return false; + } + const std::string NameStr = Name->getAsUnquotedString(); + + PatFrag::ParamKind OpKind; + if (isSpecificDef(*Ty, "gi_imm")) + OpKind = PatFrag::PK_Imm; + else if (isSpecificDef(*Ty, "root")) + OpKind = PatFrag::PK_Root; + else if (isa<UnsetInit>(Ty) || + isSpecificDef(*Ty, "gi_mo")) // no type = gi_mo. + OpKind = PatFrag::PK_MachineOperand; + else { + ::PrintError( + DiagLoc, + "'" + NameStr + + "' operand type was expected to be 'root', 'gi_imm' or 'gi_mo'"); + return false; + } + + if (!ParseAction(NameStr, OpKind)) + return false; + } + + return true; +} + +const PatFrag *CombineRuleBuilder::parsePatFrag(const Record *Def) const { + // Cache already parsed PatFrags to avoid doing extra work. + static DenseMap<const Record *, std::unique_ptr<PatFrag>> ParsedPatFrags; + + auto It = ParsedPatFrags.find(Def); + if (It != ParsedPatFrags.end()) { + SeenPatFrags.insert(It->second.get()); + return It->second.get(); + } + + std::unique_ptr<PatFrag> NewPatFrag = parsePatFragImpl(Def); + if (!NewPatFrag) { + ::PrintError(Def, "Could not parse " + PatFrag::ClassName + " '" + + Def->getName() + "'"); + // Put a nullptr in the map so we don't attempt parsing this again. + ParsedPatFrags[Def] = nullptr; + return nullptr; + } + + const auto *Res = NewPatFrag.get(); + ParsedPatFrags[Def] = std::move(NewPatFrag); + SeenPatFrags.insert(Res); + return Res; +} + +bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE, + const PatternAlternatives &Alts, + const InstructionPattern &IP) { + auto StackTrace = PrettyStackTraceEmit(RuleDef, &IP); + + auto &M = addRuleMatcher(Alts); + InstructionMatcher &IM = M.addInstructionMatcher(IP.getName()); + declareInstExpansion(CE, IM, IP.getName()); + + DenseSet<const Pattern *> SeenPats; + + const auto FindOperandDef = [&](StringRef Op) -> InstructionPattern * { + return MatchOpTable.getDef(Op); + }; + + if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(&IP)) { + if (!emitCodeGenInstructionMatchPattern(CE, Alts, M, IM, *CGP, SeenPats, + FindOperandDef)) + return false; + } else if (const auto *PFP = dyn_cast<PatFragPattern>(&IP)) { + if (!PFP->getPatFrag().canBeMatchRoot()) { + PrintError("cannot use '" + PFP->getInstName() + " as match root"); + return false; + } + + if (!emitPatFragMatchPattern(CE, Alts, M, &IM, *PFP, SeenPats)) + return false; + } else if (isa<BuiltinPattern>(&IP)) { + llvm_unreachable("No match builtins known!"); + } else + llvm_unreachable("Unknown kind of InstructionPattern!"); + + // Emit remaining patterns + for (auto &Pat : values(MatchPats)) { + if (SeenPats.contains(Pat.get())) + continue; + + switch (Pat->getKind()) { + case Pattern::K_AnyOpcode: + PrintError("wip_match_opcode can not be used with instruction patterns!"); + return false; + case Pattern::K_PatFrag: { + if (!emitPatFragMatchPattern(CE, Alts, M, /*IM*/ nullptr, + *cast<PatFragPattern>(Pat.get()), SeenPats)) + return false; + continue; + } + case Pattern::K_Builtin: + PrintError("No known match builtins"); + return false; + case Pattern::K_CodeGenInstruction: + cast<InstructionPattern>(Pat.get())->reportUnreachable(RuleDef.getLoc()); + return false; + case Pattern::K_CXX: { + addCXXPredicate(M, CE, *cast<CXXPattern>(Pat.get()), Alts); + continue; + } + default: + llvm_unreachable("unknown pattern kind!"); + } + } + + return emitApplyPatterns(CE, M); +} + +bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE, + const PatternAlternatives &Alts, + const AnyOpcodePattern &AOP) { + auto StackTrace = PrettyStackTraceEmit(RuleDef, &AOP); + + for (const CodeGenInstruction *CGI : AOP.insts()) { + auto &M = addRuleMatcher(Alts, "wip_match_opcode '" + + CGI->TheDef->getName() + "'"); + + InstructionMatcher &IM = M.addInstructionMatcher(AOP.getName()); + declareInstExpansion(CE, IM, AOP.getName()); + // declareInstExpansion needs to be identical, otherwise we need to create a + // CodeExpansions object here instead. + assert(IM.getInsnVarID() == 0); + + IM.addPredicate<InstructionOpcodeMatcher>(CGI); + + // Emit remaining patterns. + for (auto &Pat : values(MatchPats)) { + if (Pat.get() == &AOP) + continue; + + switch (Pat->getKind()) { + case Pattern::K_AnyOpcode: + PrintError("wip_match_opcode can only be present once!"); + return false; + case Pattern::K_PatFrag: { + DenseSet<const Pattern *> SeenPats; + if (!emitPatFragMatchPattern(CE, Alts, M, /*IM*/ nullptr, + *cast<PatFragPattern>(Pat.get()), + SeenPats)) + return false; + continue; + } + case Pattern::K_Builtin: + PrintError("No known match builtins"); + return false; + case Pattern::K_CodeGenInstruction: + cast<InstructionPattern>(Pat.get())->reportUnreachable( + RuleDef.getLoc()); + return false; + case Pattern::K_CXX: { + addCXXPredicate(M, CE, *cast<CXXPattern>(Pat.get()), Alts); + break; + } + default: + llvm_unreachable("unknown pattern kind!"); + } + } + + if (!emitApplyPatterns(CE, M)) + return false; + } + + return true; +} + +bool CombineRuleBuilder::emitPatFragMatchPattern( + CodeExpansions &CE, const PatternAlternatives &Alts, RuleMatcher &RM, + InstructionMatcher *IM, const PatFragPattern &PFP, + DenseSet<const Pattern *> &SeenPats) { + auto StackTrace = PrettyStackTraceEmit(RuleDef, &PFP); + + if (SeenPats.contains(&PFP)) + return true; + SeenPats.insert(&PFP); + + const auto &PF = PFP.getPatFrag(); + + if (!IM) { + // When we don't have an IM, this means this PatFrag isn't reachable from + // the root. This is only acceptable if it doesn't define anything (e.g. a + // pure C++ PatFrag). + if (PF.num_out_params() != 0) { + PFP.reportUnreachable(RuleDef.getLoc()); + return false; + } + } else { + // When an IM is provided, this is reachable from the root, and we're + // expecting to have output operands. + // TODO: If we want to allow for multiple roots we'll need a map of IMs + // then, and emission becomes a bit more complicated. + assert(PF.num_roots() == 1); + } + + CodeExpansions PatFragCEs; + if (!PFP.mapInputCodeExpansions(CE, PatFragCEs, RuleDef.getLoc())) + return false; + + // List of {ParamName, ArgName}. + // When all patterns have been emitted, find expansions in PatFragCEs named + // ArgName and add their expansion to CE using ParamName as the key. + SmallVector<std::pair<std::string, std::string>, 4> CEsToImport; + + // Map parameter names to the actual argument. + const auto OperandMapper = + [&](const InstructionOperand &O) -> InstructionOperand { + if (!O.isNamedOperand()) + return O; + + StringRef ParamName = O.getOperandName(); + + // Not sure what to do with those tbh. They should probably never be here. + assert(!O.isNamedImmediate() && "TODO: handle named imms"); + unsigned PIdx = PF.getParamIdx(ParamName); + + // Map parameters to the argument values. + if (PIdx == (unsigned)-1) { + // This is a temp of the PatFragPattern, prefix the name to avoid + // conflicts. + return O.withNewName( + insertStrRef((PFP.getName() + "." + ParamName).str())); + } + + // The operand will be added to PatFragCEs's code expansions using the + // parameter's name. If it's bound to some operand during emission of the + // patterns, we'll want to add it to CE. + auto ArgOp = PFP.getOperand(PIdx); + if (ArgOp.isNamedOperand()) + CEsToImport.emplace_back(ArgOp.getOperandName().str(), ParamName); + + if (ArgOp.getType() && O.getType() && ArgOp.getType() != O.getType()) { + StringRef PFName = PF.getName(); + PrintWarning("impossible type constraints: operand " + Twine(PIdx) + + " of '" + PFP.getName() + "' has type '" + + ArgOp.getType().str() + "', but '" + PFName + + "' constrains it to '" + O.getType().str() + "'"); + if (ArgOp.isNamedOperand()) + PrintNote("operand " + Twine(PIdx) + " of '" + PFP.getName() + + "' is '" + ArgOp.getOperandName() + "'"); + if (O.isNamedOperand()) + PrintNote("argument " + Twine(PIdx) + " of '" + PFName + "' is '" + + ParamName + "'"); + } + + return ArgOp; + }; + + // PatFragPatterns are only made of InstructionPatterns or CXXPatterns. + // Emit instructions from the root. + const auto &FragAlt = PF.getAlternative(Alts.lookup(&PFP)); + const auto &FragAltOT = FragAlt.OpTable; + const auto LookupOperandDef = + [&](StringRef Op) -> const InstructionPattern * { + return FragAltOT.getDef(Op); + }; + + DenseSet<const Pattern *> PatFragSeenPats; + for (const auto &[Idx, InOp] : enumerate(PF.out_params())) { + if (InOp.Kind != PatFrag::PK_Root) + continue; + + StringRef ParamName = InOp.Name; + const auto *Def = FragAltOT.getDef(ParamName); + assert(Def && "PatFrag::checkSemantics should have emitted an error if " + "an out operand isn't defined!"); + assert(isa<CodeGenInstructionPattern>(Def) && + "Nested PatFrags not supported yet"); + + if (!emitCodeGenInstructionMatchPattern( + PatFragCEs, Alts, RM, *IM, *cast<CodeGenInstructionPattern>(Def), + PatFragSeenPats, LookupOperandDef, OperandMapper)) + return false; + } + + // Emit leftovers. + for (const auto &Pat : FragAlt.Pats) { + if (PatFragSeenPats.contains(Pat.get())) + continue; + + if (const auto *CXXPat = dyn_cast<CXXPattern>(Pat.get())) { + addCXXPredicate(RM, PatFragCEs, *CXXPat, Alts); + continue; + } + + if (const auto *IP = dyn_cast<InstructionPattern>(Pat.get())) { + IP->reportUnreachable(PF.getLoc()); + return false; + } + + llvm_unreachable("Unexpected pattern kind in PatFrag"); + } + + for (const auto &[ParamName, ArgName] : CEsToImport) { + // Note: we're find if ParamName already exists. It just means it's been + // bound before, so we prefer to keep the first binding. + CE.declare(ParamName, PatFragCEs.lookup(ArgName)); + } + + return true; +} + +bool CombineRuleBuilder::emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M) { + if (hasOnlyCXXApplyPatterns()) { + for (auto &Pat : values(ApplyPats)) + addCXXAction(M, CE, *cast<CXXPattern>(Pat.get())); + return true; + } + + DenseSet<const Pattern *> SeenPats; + StringMap<unsigned> OperandToTempRegID; + + for (auto *ApplyRoot : ApplyRoots) { + assert(isa<InstructionPattern>(ApplyRoot) && + "Root can only be a InstructionPattern!"); + if (!emitInstructionApplyPattern(CE, M, + cast<InstructionPattern>(*ApplyRoot), + SeenPats, OperandToTempRegID)) + return false; + } + + for (auto &Pat : values(ApplyPats)) { + if (SeenPats.contains(Pat.get())) + continue; + + switch (Pat->getKind()) { + case Pattern::K_AnyOpcode: + llvm_unreachable("Unexpected pattern in apply!"); + case Pattern::K_PatFrag: + // TODO: We could support pure C++ PatFrags as a temporary thing. + llvm_unreachable("Unexpected pattern in apply!"); + case Pattern::K_Builtin: + if (!emitInstructionApplyPattern(CE, M, cast<BuiltinPattern>(*Pat), + SeenPats, OperandToTempRegID)) + return false; + break; + case Pattern::K_CodeGenInstruction: + cast<CodeGenInstructionPattern>(*Pat).reportUnreachable(RuleDef.getLoc()); + return false; + case Pattern::K_CXX: { + addCXXAction(M, CE, *cast<CXXPattern>(Pat.get())); + continue; + } + default: + llvm_unreachable("unknown pattern kind!"); + } + } + + return true; +} + +bool CombineRuleBuilder::emitInstructionApplyPattern( + CodeExpansions &CE, RuleMatcher &M, const InstructionPattern &P, + DenseSet<const Pattern *> &SeenPats, + StringMap<unsigned> &OperandToTempRegID) { + auto StackTrace = PrettyStackTraceEmit(RuleDef, &P); + + if (SeenPats.contains(&P)) + return true; + + SeenPats.insert(&P); + + // First, render the uses. + for (auto &Op : P.named_operands()) { + if (Op.isDef()) + continue; + + StringRef OpName = Op.getOperandName(); + if (const auto *DefPat = ApplyOpTable.getDef(OpName)) { + if (!emitInstructionApplyPattern(CE, M, *DefPat, SeenPats, + OperandToTempRegID)) + return false; + } else { + // If we have no def, check this exists in the MatchRoot. + if (!Op.isNamedImmediate() && !MatchOpTable.lookup(OpName).Found) { + PrintError("invalid output operand '" + OpName + + "': operand is not a live-in of the match pattern, and it " + "has no definition"); + return false; + } + } + } + + if (const auto *BP = dyn_cast<BuiltinPattern>(&P)) + return emitBuiltinApplyPattern(CE, M, *BP, OperandToTempRegID); + + if (isa<PatFragPattern>(&P)) + llvm_unreachable("PatFragPatterns is not supported in 'apply'!"); + + auto &CGIP = cast<CodeGenInstructionPattern>(P); + + // Now render this inst. + auto &DstMI = + M.addAction<BuildMIAction>(M.allocateOutputInsnID(), &CGIP.getInst()); + + for (auto &Op : P.operands()) { + if (Op.isNamedImmediate()) { + PrintError("invalid output operand '" + Op.getOperandName() + + "': output immediates cannot be named"); + PrintNote("while emitting pattern '" + P.getName() + "' (" + + P.getInstName() + ")"); + return false; + } + + if (Op.hasImmValue()) { + if (!emitCodeGenInstructionApplyImmOperand(M, DstMI, CGIP, Op)) + return false; + continue; + } + + StringRef OpName = Op.getOperandName(); + + // Uses of operand. + if (!Op.isDef()) { + if (auto It = OperandToTempRegID.find(OpName); + It != OperandToTempRegID.end()) { + assert(!MatchOpTable.lookup(OpName).Found && + "Temp reg is also from match pattern?"); + DstMI.addRenderer<TempRegRenderer>(It->second); + } else { + // This should be a match live in or a redef of a matched instr. + // If it's a use of a temporary register, then we messed up somewhere - + // the previous condition should have passed. + assert(MatchOpTable.lookup(OpName).Found && + !ApplyOpTable.getDef(OpName) && "Temp reg not emitted yet!"); + DstMI.addRenderer<CopyRenderer>(OpName); + } + continue; + } + + // Determine what we're dealing with. Are we replace a matched instruction? + // Creating a new one? + auto OpLookupRes = MatchOpTable.lookup(OpName); + if (OpLookupRes.Found) { + if (OpLookupRes.isLiveIn()) { + // live-in of the match pattern. + PrintError("Cannot define live-in operand '" + OpName + + "' in the 'apply' pattern"); + return false; + } + assert(OpLookupRes.Def); + + // TODO: Handle this. We need to mutate the instr, or delete the old + // one. + // Likewise, we also need to ensure we redef everything, if the + // instr has more than one def, we need to redef all or nothing. + if (OpLookupRes.Def != MatchRoot) { + PrintError("redefining an instruction other than the root is not " + "supported (operand '" + + OpName + "')"); + return false; + } + // redef of a match + DstMI.addRenderer<CopyRenderer>(OpName); + continue; + } + + // Define a new register unique to the apply patterns (AKA a "temp" + // register). + unsigned TempRegID; + if (auto It = OperandToTempRegID.find(OpName); + It != OperandToTempRegID.end()) { + TempRegID = It->second; + } else { + // This is a brand new register. + TempRegID = M.allocateTempRegID(); + OperandToTempRegID[OpName] = TempRegID; + const auto Ty = Op.getType(); + if (!Ty) { + PrintError("def of a new register '" + OpName + + "' in the apply patterns must have a type"); + return false; + } + + declareTempRegExpansion(CE, TempRegID, OpName); + // Always insert the action at the beginning, otherwise we may end up + // using the temp reg before it's available. + M.insertAction<MakeTempRegisterAction>( + M.actions_begin(), getLLTCodeGenOrTempType(Ty, M), TempRegID); + } + + DstMI.addRenderer<TempRegRenderer>(TempRegID); + } + + // Render MIFlags + if (const auto *FI = CGIP.getMIFlagsInfo()) { + for (StringRef InstName : FI->copy_flags()) + DstMI.addCopiedMIFlags(M.getInstructionMatcher(InstName)); + for (StringRef F : FI->set_flags()) + DstMI.addSetMIFlags(F); + for (StringRef F : FI->unset_flags()) + DstMI.addUnsetMIFlags(F); + } + + // Don't allow mutating opcodes for GISel combiners. We want a more precise + // handling of MIFlags so we require them to be explicitly preserved. + // + // TODO: We don't mutate very often, if at all in combiners, but it'd be nice + // to re-enable this. We'd then need to always clear MIFlags when mutating + // opcodes, and never mutate an inst that we copy flags from. + // DstMI.chooseInsnToMutate(M); + declareInstExpansion(CE, DstMI, P.getName()); + + return true; +} + +bool CombineRuleBuilder::emitCodeGenInstructionApplyImmOperand( + RuleMatcher &M, BuildMIAction &DstMI, const CodeGenInstructionPattern &P, + const InstructionOperand &O) { + // If we have a type, we implicitly emit a G_CONSTANT, except for G_CONSTANT + // itself where we emit a CImm. + // + // No type means we emit a simple imm. + // G_CONSTANT is a special case and needs a CImm though so this is likely a + // mistake. + const bool isGConstant = P.is("G_CONSTANT"); + const auto Ty = O.getType(); + if (!Ty) { + if (isGConstant) { + PrintError("'G_CONSTANT' immediate must be typed!"); + PrintNote("while emitting pattern '" + P.getName() + "' (" + + P.getInstName() + ")"); + return false; + } + + DstMI.addRenderer<ImmRenderer>(O.getImmValue()); + return true; + } + + auto ImmTy = getLLTCodeGenOrTempType(Ty, M); + + if (isGConstant) { + DstMI.addRenderer<ImmRenderer>(O.getImmValue(), ImmTy); + return true; + } + + unsigned TempRegID = M.allocateTempRegID(); + // Ensure MakeTempReg & the BuildConstantAction occur at the beginning. + auto InsertIt = M.insertAction<MakeTempRegisterAction>(M.actions_begin(), + ImmTy, TempRegID); + M.insertAction<BuildConstantAction>(++InsertIt, TempRegID, O.getImmValue()); + DstMI.addRenderer<TempRegRenderer>(TempRegID); + return true; +} + +bool CombineRuleBuilder::emitBuiltinApplyPattern( + CodeExpansions &CE, RuleMatcher &M, const BuiltinPattern &P, + StringMap<unsigned> &OperandToTempRegID) { + const auto Error = [&](Twine Reason) { + PrintError("cannot emit '" + P.getInstName() + "' builtin: " + Reason); + return false; + }; + + switch (P.getBuiltinKind()) { + case BI_EraseRoot: { + // Root is always inst 0. + M.addAction<EraseInstAction>(/*InsnID*/ 0); + return true; + } + case BI_ReplaceReg: { + StringRef Old = P.getOperand(0).getOperandName(); + StringRef New = P.getOperand(1).getOperandName(); + + if (!ApplyOpTable.lookup(New).Found && !MatchOpTable.lookup(New).Found) + return Error("unknown operand '" + Old + "'"); + + auto &OldOM = M.getOperandMatcher(Old); + if (auto It = OperandToTempRegID.find(New); + It != OperandToTempRegID.end()) { + // Replace with temp reg. + M.addAction<ReplaceRegAction>(OldOM.getInsnVarID(), OldOM.getOpIdx(), + It->second); + } else { + // Replace with matched reg. + auto &NewOM = M.getOperandMatcher(New); + M.addAction<ReplaceRegAction>(OldOM.getInsnVarID(), OldOM.getOpIdx(), + NewOM.getInsnVarID(), NewOM.getOpIdx()); + } + // checkSemantics should have ensured that we can only rewrite the root. + // Ensure we're deleting it. + assert(MatchOpTable.getDef(Old) == MatchRoot); + // TODO: We could avoid adding the action again if it's already in. The + // MatchTable is smart enough to only emit one opcode even if + // EraseInstAction is present multiple times. I think searching for a copy + // is more expensive than just blindly adding it though. + M.addAction<EraseInstAction>(/*InsnID*/ 0); + + return true; + } + } + + llvm_unreachable("Unknown BuiltinKind!"); +} + +bool isLiteralImm(const InstructionPattern &P, unsigned OpIdx) { + if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(&P)) { + StringRef InstName = CGP->getInst().TheDef->getName(); + return (InstName == "G_CONSTANT" || InstName == "G_FCONSTANT") && + OpIdx == 1; + } + + llvm_unreachable("TODO"); +} + +bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern( + CodeExpansions &CE, const PatternAlternatives &Alts, RuleMatcher &M, + InstructionMatcher &IM, const CodeGenInstructionPattern &P, + DenseSet<const Pattern *> &SeenPats, OperandDefLookupFn LookupOperandDef, + OperandMapperFnRef OperandMapper) { + auto StackTrace = PrettyStackTraceEmit(RuleDef, &P); + + if (SeenPats.contains(&P)) + return true; + + SeenPats.insert(&P); + + IM.addPredicate<InstructionOpcodeMatcher>(&P.getInst()); + declareInstExpansion(CE, IM, P.getName()); + + // Check flags if needed. + if (const auto *FI = P.getMIFlagsInfo()) { + assert(FI->copy_flags().empty()); + + if (const auto &SetF = FI->set_flags(); !SetF.empty()) + IM.addPredicate<MIFlagsInstructionPredicateMatcher>(SetF.getArrayRef()); + if (const auto &UnsetF = FI->unset_flags(); !UnsetF.empty()) + IM.addPredicate<MIFlagsInstructionPredicateMatcher>(UnsetF.getArrayRef(), + /*CheckNot=*/true); + } + + for (const auto &[Idx, OriginalO] : enumerate(P.operands())) { + // Remap the operand. This is used when emitting InstructionPatterns inside + // PatFrags, so it can remap them to the arguments passed to the pattern. + // + // We use the remapped operand to emit immediates, and for the symbolic + // operand names (in IM.addOperand). CodeExpansions and OperandTable lookups + // still use the original name. + // + // The "def" flag on the remapped operand is always ignored. + auto RemappedO = OperandMapper(OriginalO); + assert(RemappedO.isNamedOperand() == OriginalO.isNamedOperand() && + "Cannot remap an unnamed operand to a named one!"); + + const auto OpName = + RemappedO.isNamedOperand() ? RemappedO.getOperandName().str() : ""; + OperandMatcher &OM = + IM.addOperand(Idx, OpName, AllocatedTemporariesBaseID++); + if (!OpName.empty()) + declareOperandExpansion(CE, OM, OriginalO.getOperandName()); + + // Handle immediates. + if (RemappedO.hasImmValue()) { + if (isLiteralImm(P, Idx)) + OM.addPredicate<LiteralIntOperandMatcher>(RemappedO.getImmValue()); + else + OM.addPredicate<ConstantIntOperandMatcher>(RemappedO.getImmValue()); + } + + // Handle typed operands, but only bother to check if it hasn't been done + // before. + // + // getOperandMatcher will always return the first OM to have been created + // for that Operand. "OM" here is always a new OperandMatcher. + // + // Always emit a check for unnamed operands. + if (OpName.empty() || + !M.getOperandMatcher(OpName).contains<LLTOperandMatcher>()) { + if (const auto Ty = RemappedO.getType()) { + // TODO: We could support GITypeOf here on the condition that the + // OperandMatcher exists already. Though it's clunky to make this work + // and isn't all that useful so it's just rejected in typecheckPatterns + // at this time. + assert(Ty.isLLT() && "Only LLTs are supported in match patterns!"); + OM.addPredicate<LLTOperandMatcher>(getLLTCodeGen(Ty)); + } + } + + // Stop here if the operand is a def, or if it had no name. + if (OriginalO.isDef() || !OriginalO.isNamedOperand()) + continue; + + const auto *DefPat = LookupOperandDef(OriginalO.getOperandName()); + if (!DefPat) + continue; + + if (OriginalO.hasImmValue()) { + assert(!OpName.empty()); + // This is a named immediate that also has a def, that's not okay. + // e.g. + // (G_SEXT $y, (i32 0)) + // (COPY $x, 42:$y) + PrintError("'" + OpName + + "' is a named immediate, it cannot be defined by another " + "instruction"); + PrintNote("'" + OpName + "' is defined by '" + DefPat->getName() + "'"); + return false; + } + + // From here we know that the operand defines an instruction, and we need to + // emit it. + auto InstOpM = + OM.addPredicate<InstructionOperandMatcher>(M, DefPat->getName()); + if (!InstOpM) { + // TODO: copy-pasted from GlobalISelEmitter.cpp. Is it still relevant + // here? + PrintError("Nested instruction '" + DefPat->getName() + + "' cannot be the same as another operand '" + + OriginalO.getOperandName() + "'"); + return false; + } + + auto &IM = (*InstOpM)->getInsnMatcher(); + if (const auto *CGIDef = dyn_cast<CodeGenInstructionPattern>(DefPat)) { + if (!emitCodeGenInstructionMatchPattern(CE, Alts, M, IM, *CGIDef, + SeenPats, LookupOperandDef, + OperandMapper)) + return false; + continue; + } + + if (const auto *PFPDef = dyn_cast<PatFragPattern>(DefPat)) { + if (!emitPatFragMatchPattern(CE, Alts, M, &IM, *PFPDef, SeenPats)) + return false; + continue; + } + + llvm_unreachable("unknown type of InstructionPattern"); + } + + return true; +} + +//===- GICombinerEmitter --------------------------------------------------===// + +/// Main implementation class. This emits the tablegenerated output. +/// +/// It collects rules, uses `CombineRuleBuilder` to parse them and accumulate +/// RuleMatchers, then takes all the necessary state/data from the various +/// static storage pools and wires them together to emit the match table & +/// associated function/data structures. +class GICombinerEmitter final : public GlobalISelMatchTableExecutorEmitter { + RecordKeeper &Records; + StringRef Name; + const CodeGenTarget &Target; + Record *Combiner; + unsigned NextRuleID = 0; + + // List all combine rules (ID, name) imported. + // Note that the combiner rule ID is different from the RuleMatcher ID. The + // latter is internal to the MatchTable, the former is the canonical ID of the + // combine rule used to disable/enable it. + std::vector<std::pair<unsigned, std::string>> AllCombineRules; + + // Keep track of all rules we've seen so far to ensure we don't process + // the same rule twice. + StringSet<> RulesSeen; + + MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules); + + void emitRuleConfigImpl(raw_ostream &OS); + + void emitAdditionalImpl(raw_ostream &OS) override; + + void emitMIPredicateFns(raw_ostream &OS) override; + void emitI64ImmPredicateFns(raw_ostream &OS) override; + void emitAPFloatImmPredicateFns(raw_ostream &OS) override; + void emitAPIntImmPredicateFns(raw_ostream &OS) override; + void emitTestSimplePredicate(raw_ostream &OS) override; + void emitRunCustomAction(raw_ostream &OS) override; + + void emitAdditionalTemporariesDecl(raw_ostream &OS, + StringRef Indent) override; + + const CodeGenTarget &getTarget() const override { return Target; } + StringRef getClassName() const override { + return Combiner->getValueAsString("Classname"); + } + + StringRef getCombineAllMethodName() const { + return Combiner->getValueAsString("CombineAllMethodName"); + } + + std::string getRuleConfigClassName() const { + return getClassName().str() + "RuleConfig"; + } + + void gatherRules(std::vector<RuleMatcher> &Rules, + const std::vector<Record *> &&RulesAndGroups); + +public: + explicit GICombinerEmitter(RecordKeeper &RK, const CodeGenTarget &Target, + StringRef Name, Record *Combiner); + ~GICombinerEmitter() {} + + void run(raw_ostream &OS); +}; + +void GICombinerEmitter::emitRuleConfigImpl(raw_ostream &OS) { + OS << "struct " << getRuleConfigClassName() << " {\n" + << " SparseBitVector<> DisabledRules;\n\n" + << " bool isRuleEnabled(unsigned RuleID) const;\n" + << " bool parseCommandLineOption();\n" + << " bool setRuleEnabled(StringRef RuleIdentifier);\n" + << " bool setRuleDisabled(StringRef RuleIdentifier);\n" + << "};\n\n"; + + std::vector<std::pair<std::string, std::string>> Cases; + Cases.reserve(AllCombineRules.size()); + + for (const auto &[ID, Name] : AllCombineRules) + Cases.emplace_back(Name, "return " + to_string(ID) + ";\n"); + + OS << "static std::optional<uint64_t> getRuleIdxForIdentifier(StringRef " + "RuleIdentifier) {\n" + << " uint64_t I;\n" + << " // getAtInteger(...) returns false on success\n" + << " bool Parsed = !RuleIdentifier.getAsInteger(0, I);\n" + << " if (Parsed)\n" + << " return I;\n\n" + << "#ifndef NDEBUG\n"; + StringMatcher Matcher("RuleIdentifier", Cases, OS); + Matcher.Emit(); + OS << "#endif // ifndef NDEBUG\n\n" + << " return std::nullopt;\n" + << "}\n"; + + OS << "static std::optional<std::pair<uint64_t, uint64_t>> " + "getRuleRangeForIdentifier(StringRef RuleIdentifier) {\n" + << " std::pair<StringRef, StringRef> RangePair = " + "RuleIdentifier.split('-');\n" + << " if (!RangePair.second.empty()) {\n" + << " const auto First = " + "getRuleIdxForIdentifier(RangePair.first);\n" + << " const auto Last = " + "getRuleIdxForIdentifier(RangePair.second);\n" + << " if (!First || !Last)\n" + << " return std::nullopt;\n" + << " if (First >= Last)\n" + << " report_fatal_error(\"Beginning of range should be before " + "end of range\");\n" + << " return {{*First, *Last + 1}};\n" + << " }\n" + << " if (RangePair.first == \"*\") {\n" + << " return {{0, " << AllCombineRules.size() << "}};\n" + << " }\n" + << " const auto I = getRuleIdxForIdentifier(RangePair.first);\n" + << " if (!I)\n" + << " return std::nullopt;\n" + << " return {{*I, *I + 1}};\n" + << "}\n\n"; + + for (bool Enabled : {true, false}) { + OS << "bool " << getRuleConfigClassName() << "::setRule" + << (Enabled ? "Enabled" : "Disabled") << "(StringRef RuleIdentifier) {\n" + << " auto MaybeRange = getRuleRangeForIdentifier(RuleIdentifier);\n" + << " if (!MaybeRange)\n" + << " return false;\n" + << " for (auto I = MaybeRange->first; I < MaybeRange->second; ++I)\n" + << " DisabledRules." << (Enabled ? "reset" : "set") << "(I);\n" + << " return true;\n" + << "}\n\n"; + } + + OS << "static std::vector<std::string> " << Name << "Option;\n" + << "static cl::list<std::string> " << Name << "DisableOption(\n" + << " \"" << Name.lower() << "-disable-rule\",\n" + << " cl::desc(\"Disable one or more combiner rules temporarily in " + << "the " << Name << " pass\"),\n" + << " cl::CommaSeparated,\n" + << " cl::Hidden,\n" + << " cl::cat(GICombinerOptionCategory),\n" + << " cl::callback([](const std::string &Str) {\n" + << " " << Name << "Option.push_back(Str);\n" + << " }));\n" + << "static cl::list<std::string> " << Name << "OnlyEnableOption(\n" + << " \"" << Name.lower() << "-only-enable-rule\",\n" + << " cl::desc(\"Disable all rules in the " << Name + << " pass then re-enable the specified ones\"),\n" + << " cl::Hidden,\n" + << " cl::cat(GICombinerOptionCategory),\n" + << " cl::callback([](const std::string &CommaSeparatedArg) {\n" + << " StringRef Str = CommaSeparatedArg;\n" + << " " << Name << "Option.push_back(\"*\");\n" + << " do {\n" + << " auto X = Str.split(\",\");\n" + << " " << Name << "Option.push_back((\"!\" + X.first).str());\n" + << " Str = X.second;\n" + << " } while (!Str.empty());\n" + << " }));\n" + << "\n\n" + << "bool " << getRuleConfigClassName() + << "::isRuleEnabled(unsigned RuleID) const {\n" + << " return !DisabledRules.test(RuleID);\n" + << "}\n" + << "bool " << getRuleConfigClassName() << "::parseCommandLineOption() {\n" + << " for (StringRef Identifier : " << Name << "Option) {\n" + << " bool Enabled = Identifier.consume_front(\"!\");\n" + << " if (Enabled && !setRuleEnabled(Identifier))\n" + << " return false;\n" + << " if (!Enabled && !setRuleDisabled(Identifier))\n" + << " return false;\n" + << " }\n" + << " return true;\n" + << "}\n\n"; +} + +void GICombinerEmitter::emitAdditionalImpl(raw_ostream &OS) { + OS << "bool " << getClassName() << "::" << getCombineAllMethodName() + << "(MachineInstr &I) const {\n" + << " const TargetSubtargetInfo &ST = MF.getSubtarget();\n" + << " const PredicateBitset AvailableFeatures = " + "getAvailableFeatures();\n" + << " B.setInstrAndDebugLoc(I);\n" + << " State.MIs.clear();\n" + << " State.MIs.push_back(&I);\n" + << " " << MatchDataInfo::StructName << " = " + << MatchDataInfo::StructTypeName << "();\n\n" + << " if (executeMatchTable(*this, State, ExecInfo, B" + << ", getMatchTable(), *ST.getInstrInfo(), MRI, " + "*MRI.getTargetRegisterInfo(), *ST.getRegBankInfo(), AvailableFeatures" + << ", /*CoverageInfo*/ nullptr)) {\n" + << " return true;\n" + << " }\n\n" + << " return false;\n" + << "}\n\n"; +} + +void GICombinerEmitter::emitMIPredicateFns(raw_ostream &OS) { + auto MatchCode = CXXPredicateCode::getAllMatchCode(); + emitMIPredicateFnsImpl<const CXXPredicateCode *>( + OS, "", ArrayRef<const CXXPredicateCode *>(MatchCode), + [](const CXXPredicateCode *C) -> StringRef { return C->BaseEnumName; }, + [](const CXXPredicateCode *C) -> StringRef { return C->Code; }); +} + +void GICombinerEmitter::emitI64ImmPredicateFns(raw_ostream &OS) { + // Unused, but still needs to be called. + emitImmPredicateFnsImpl<unsigned>( + OS, "I64", "int64_t", {}, [](unsigned) { return ""; }, + [](unsigned) { return ""; }); +} + +void GICombinerEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) { + // Unused, but still needs to be called. + emitImmPredicateFnsImpl<unsigned>( + OS, "APFloat", "const APFloat &", {}, [](unsigned) { return ""; }, + [](unsigned) { return ""; }); +} + +void GICombinerEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) { + // Unused, but still needs to be called. + emitImmPredicateFnsImpl<unsigned>( + OS, "APInt", "const APInt &", {}, [](unsigned) { return ""; }, + [](unsigned) { return ""; }); +} + +void GICombinerEmitter::emitTestSimplePredicate(raw_ostream &OS) { + if (!AllCombineRules.empty()) { + OS << "enum {\n"; + std::string EnumeratorSeparator = " = GICXXPred_Invalid + 1,\n"; + // To avoid emitting a switch, we expect that all those rules are in order. + // That way we can just get the RuleID from the enum by subtracting + // (GICXXPred_Invalid + 1). + unsigned ExpectedID = 0; + (void)ExpectedID; + for (const auto &ID : keys(AllCombineRules)) { + assert(ExpectedID++ == ID && "combine rules are not ordered!"); + OS << " " << getIsEnabledPredicateEnumName(ID) << EnumeratorSeparator; + EnumeratorSeparator = ",\n"; + } + OS << "};\n\n"; + } + + OS << "bool " << getClassName() + << "::testSimplePredicate(unsigned Predicate) const {\n" + << " return RuleConfig.isRuleEnabled(Predicate - " + "GICXXPred_Invalid - " + "1);\n" + << "}\n"; +} + +void GICombinerEmitter::emitRunCustomAction(raw_ostream &OS) { + const auto ApplyCode = CXXPredicateCode::getAllApplyCode(); + + if (!ApplyCode.empty()) { + OS << "enum {\n"; + std::string EnumeratorSeparator = " = GICXXCustomAction_Invalid + 1,\n"; + for (const auto &Apply : ApplyCode) { + OS << " " << Apply->getEnumNameWithPrefix(CXXApplyPrefix) + << EnumeratorSeparator; + EnumeratorSeparator = ",\n"; + } + OS << "};\n"; + } + + OS << "void " << getClassName() + << "::runCustomAction(unsigned ApplyID, const MatcherState &State, " + "NewMIVector &OutMIs) const " + "{\n"; + if (!ApplyCode.empty()) { + OS << " switch(ApplyID) {\n"; + for (const auto &Apply : ApplyCode) { + OS << " case " << Apply->getEnumNameWithPrefix(CXXApplyPrefix) << ":{\n" + << " " << join(split(Apply->Code, '\n'), "\n ") << '\n' + << " return;\n"; + OS << " }\n"; + } + OS << "}\n"; + } + OS << " llvm_unreachable(\"Unknown Apply Action\");\n" + << "}\n"; +} + +void GICombinerEmitter::emitAdditionalTemporariesDecl(raw_ostream &OS, + StringRef Indent) { + OS << Indent << "struct " << MatchDataInfo::StructTypeName << " {\n"; + for (const auto &[Type, VarNames] : AllMatchDataVars) { + assert(!VarNames.empty() && "Cannot have no vars for this type!"); + OS << Indent << " " << Type << " " << join(VarNames, ", ") << ";\n"; + } + OS << Indent << "};\n" + << Indent << "mutable " << MatchDataInfo::StructTypeName << " " + << MatchDataInfo::StructName << ";\n\n"; +} + +GICombinerEmitter::GICombinerEmitter(RecordKeeper &RK, + const CodeGenTarget &Target, + StringRef Name, Record *Combiner) + : Records(RK), Name(Name), Target(Target), Combiner(Combiner) {} + +MatchTable +GICombinerEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules) { + std::vector<Matcher *> InputRules; + for (Matcher &Rule : Rules) + InputRules.push_back(&Rule); + + unsigned CurrentOrdering = 0; + StringMap<unsigned> OpcodeOrder; + for (RuleMatcher &Rule : Rules) { + const StringRef Opcode = Rule.getOpcode(); + assert(!Opcode.empty() && "Didn't expect an undefined opcode"); + if (OpcodeOrder.count(Opcode) == 0) + OpcodeOrder[Opcode] = CurrentOrdering++; + } + + llvm::stable_sort(InputRules, [&OpcodeOrder](const Matcher *A, + const Matcher *B) { + auto *L = static_cast<const RuleMatcher *>(A); + auto *R = static_cast<const RuleMatcher *>(B); + return std::make_tuple(OpcodeOrder[L->getOpcode()], L->getNumOperands()) < + std::make_tuple(OpcodeOrder[R->getOpcode()], R->getNumOperands()); + }); + + for (Matcher *Rule : InputRules) + Rule->optimize(); + + std::vector<std::unique_ptr<Matcher>> MatcherStorage; + std::vector<Matcher *> OptRules = + optimizeRules<GroupMatcher>(InputRules, MatcherStorage); + + for (Matcher *Rule : OptRules) + Rule->optimize(); + + OptRules = optimizeRules<SwitchMatcher>(OptRules, MatcherStorage); + + return MatchTable::buildTable(OptRules, /*WithCoverage*/ false, + /*IsCombiner*/ true); +} + +/// Recurse into GICombineGroup's and flatten the ruleset into a simple list. +void GICombinerEmitter::gatherRules( + std::vector<RuleMatcher> &ActiveRules, + const std::vector<Record *> &&RulesAndGroups) { + for (Record *Rec : RulesAndGroups) { + if (!Rec->isValueUnset("Rules")) { + gatherRules(ActiveRules, Rec->getValueAsListOfDefs("Rules")); + continue; + } + + StringRef RuleName = Rec->getName(); + if (!RulesSeen.insert(RuleName).second) { + PrintWarning(Rec->getLoc(), + "skipping rule '" + Rec->getName() + + "' because it has already been processed"); + continue; + } + + AllCombineRules.emplace_back(NextRuleID, Rec->getName().str()); + CombineRuleBuilder CRB(Target, SubtargetFeatures, *Rec, NextRuleID++, + ActiveRules); + + if (!CRB.parseAll()) { + assert(ErrorsPrinted && "Parsing failed without errors!"); + continue; + } + + if (StopAfterParse) { + CRB.print(outs()); + continue; + } + + if (!CRB.emitRuleMatchers()) { + assert(ErrorsPrinted && "Emission failed without errors!"); + continue; + } + } +} + +void GICombinerEmitter::run(raw_ostream &OS) { + InstructionOpcodeMatcher::initOpcodeValuesMap(Target); + LLTOperandMatcher::initTypeIDValuesMap(); + + Records.startTimer("Gather rules"); + std::vector<RuleMatcher> Rules; + gatherRules(Rules, Combiner->getValueAsListOfDefs("Rules")); + if (ErrorsPrinted) + PrintFatalError(Combiner->getLoc(), "Failed to parse one or more rules"); + + if (StopAfterParse) + return; + + Records.startTimer("Creating Match Table"); + unsigned MaxTemporaries = 0; + for (const auto &Rule : Rules) + MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns()); + + llvm::stable_sort(Rules, [&](const RuleMatcher &A, const RuleMatcher &B) { + if (A.isHigherPriorityThan(B)) { + assert(!B.isHigherPriorityThan(A) && "Cannot be more important " + "and less important at " + "the same time"); + return true; + } + return false; + }); + + const MatchTable Table = buildMatchTable(Rules); + + Records.startTimer("Emit combiner"); + + emitSourceFileHeader(getClassName().str() + " Combiner Match Table", OS); + + // Unused + std::vector<StringRef> CustomRendererFns; + // Unused + std::vector<Record *> ComplexPredicates; + + SmallVector<LLTCodeGen, 16> TypeObjects; + append_range(TypeObjects, KnownTypes); + llvm::sort(TypeObjects); + + // Hack: Avoid empty declarator. + if (TypeObjects.empty()) + TypeObjects.push_back(LLT::scalar(1)); + + // GET_GICOMBINER_DEPS, which pulls in extra dependencies. + OS << "#ifdef GET_GICOMBINER_DEPS\n" + << "#include \"llvm/ADT/SparseBitVector.h\"\n" + << "namespace llvm {\n" + << "extern cl::OptionCategory GICombinerOptionCategory;\n" + << "} // end namespace llvm\n" + << "#endif // ifdef GET_GICOMBINER_DEPS\n\n"; + + // GET_GICOMBINER_TYPES, which needs to be included before the declaration of + // the class. + OS << "#ifdef GET_GICOMBINER_TYPES\n"; + emitRuleConfigImpl(OS); + OS << "#endif // ifdef GET_GICOMBINER_TYPES\n\n"; + emitPredicateBitset(OS, "GET_GICOMBINER_TYPES"); + + // GET_GICOMBINER_CLASS_MEMBERS, which need to be included inside the class. + emitPredicatesDecl(OS, "GET_GICOMBINER_CLASS_MEMBERS"); + emitTemporariesDecl(OS, "GET_GICOMBINER_CLASS_MEMBERS"); + + // GET_GICOMBINER_IMPL, which needs to be included outside the class. + emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexPredicates, + CustomRendererFns, "GET_GICOMBINER_IMPL"); + + // GET_GICOMBINER_CONSTRUCTOR_INITS, which are in the constructor's + // initializer list. + emitPredicatesInit(OS, "GET_GICOMBINER_CONSTRUCTOR_INITS"); + emitTemporariesInit(OS, MaxTemporaries, "GET_GICOMBINER_CONSTRUCTOR_INITS"); +} + +} // end anonymous namespace + +//===----------------------------------------------------------------------===// + +static void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS) { + EnablePrettyStackTrace(); + CodeGenTarget Target(RK); + + if (SelectedCombiners.empty()) + PrintFatalError("No combiners selected with -combiners"); + for (const auto &Combiner : SelectedCombiners) { + Record *CombinerDef = RK.getDef(Combiner); + if (!CombinerDef) + PrintFatalError("Could not find " + Combiner); + GICombinerEmitter(RK, Target, Combiner, CombinerDef).run(OS); + } +} + +static TableGen::Emitter::Opt X("gen-global-isel-combiner", EmitGICombiner, + "Generate GlobalISel Combiner"); diff --git a/llvm/utils/TableGen/GlobalISelCombinerMatchTableEmitter.cpp b/llvm/utils/TableGen/GlobalISelCombinerMatchTableEmitter.cpp deleted file mode 100644 index 3ae66ed01b3a..000000000000 --- a/llvm/utils/TableGen/GlobalISelCombinerMatchTableEmitter.cpp +++ /dev/null @@ -1,1575 +0,0 @@ -//===- GlobalISelCombinerMatchTableEmitter.cpp - --------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -/// \file Generate a combiner implementation for GlobalISel from a declarative -/// syntax using GlobalISelMatchTable. -/// -//===----------------------------------------------------------------------===// - -#include "CodeGenInstruction.h" -#include "CodeGenTarget.h" -#include "GlobalISel/CodeExpander.h" -#include "GlobalISel/CodeExpansions.h" -#include "GlobalISel/CombinerUtils.h" -#include "GlobalISelMatchTable.h" -#include "GlobalISelMatchTableExecutorEmitter.h" -#include "SubtargetFeatureInfo.h" -#include "llvm/ADT/Hashing.h" -#include "llvm/ADT/MapVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/StringSet.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ScopedPrinter.h" -#include "llvm/TableGen/Error.h" -#include "llvm/TableGen/Record.h" -#include "llvm/TableGen/StringMatcher.h" -#include "llvm/TableGen/TableGenBackend.h" -#include <cstdint> - -using namespace llvm; -using namespace llvm::gi; - -#define DEBUG_TYPE "gicombiner-matchtable-emitter" - -extern cl::list<std::string> SelectedCombiners; -extern cl::opt<bool> StopAfterParse; - -namespace { -constexpr StringLiteral CXXApplyPrefix = "GICXXCustomAction_CombineApply"; -constexpr StringLiteral CXXPredPrefix = "GICXXPred_MI_Predicate_"; - -std::string getIsEnabledPredicateEnumName(unsigned CombinerRuleID) { - return "GICXXPred_Simple_IsRule" + to_string(CombinerRuleID) + "Enabled"; -} - -void declareInstExpansion(CodeExpansions &CE, const InstructionMatcher &IM, - StringRef Name) { - CE.declare(Name, "State.MIs[" + to_string(IM.getInsnVarID()) + "]"); -} - -void declareOperandExpansion(CodeExpansions &CE, const OperandMatcher &OM, - StringRef Name) { - CE.declare(Name, "State.MIs[" + to_string(OM.getInsnVarID()) + - "]->getOperand(" + to_string(OM.getOpIdx()) + ")"); -} - -//===- MatchData Handling -------------------------------------------------===// - -/// Represents MatchData defined by the match stage and required by the apply -/// stage. -/// -/// This allows the plumbing of arbitrary data from C++ predicates between the -/// stages. -/// -/// When this class is initially created, it only has a pattern symbol and a -/// type. When all of the MatchDatas declarations of a given pattern have been -/// parsed, `AssignVariables` must be called to assign storage variable names to -/// each MatchDataInfo. -class MatchDataInfo { - StringRef PatternSymbol; - StringRef Type; - std::string VarName; - -public: - static constexpr StringLiteral StructTypeName = "MatchInfosTy"; - static constexpr StringLiteral StructName = "MatchInfos"; - - MatchDataInfo(StringRef PatternSymbol, StringRef Type) - : PatternSymbol(PatternSymbol), Type(Type.trim()) {} - - StringRef getPatternSymbol() const { return PatternSymbol; }; - StringRef getType() const { return Type; }; - - bool hasVariableName() const { return !VarName.empty(); } - void setVariableName(StringRef Name) { VarName = Name; } - StringRef getVariableName() const; - - std::string getQualifiedVariableName() const { - return StructName.str() + "." + getVariableName().str(); - } - - void print(raw_ostream &OS) const; - void dump() const { print(dbgs()); } -}; - -StringRef MatchDataInfo::getVariableName() const { - assert(hasVariableName()); - return VarName; -} - -void MatchDataInfo::print(raw_ostream &OS) const { - OS << "(MatchDataInfo pattern_symbol:" << PatternSymbol << " type:'" << Type - << "' var_name:" << (VarName.empty() ? "<unassigned>" : VarName) << ")"; -} - -/// Pool of type -> variables used to emit MatchData variables declarations. -/// -/// e.g. if the map contains "int64_t" -> ["MD0", "MD1"], then two variable -/// declarations must be emitted: `int64_t MD0` and `int64_t MD1`. -/// -/// This has a static lifetime and will outlive all the `MatchDataInfo` objects -/// by design. It needs to persist after all `CombineRuleBuilder` objects died -/// so we can emit the variable declarations. -StringMap<std::vector<std::string>> AllMatchDataVars; - -// Assign variable names to all MatchDatas used by a pattern. This must be -// called after all MatchData decls have been parsed inside a rule. -// -// Requires an array of MatchDataInfo so we can handle cases where a pattern -// uses multiple instances of the same MatchData type. -void AssignMatchDataVariables(MutableArrayRef<MatchDataInfo> Infos) { - static unsigned NextVarID = 0; - - StringMap<unsigned> SeenTypes; - for (auto &I : Infos) { - unsigned &NumSeen = SeenTypes[I.getType()]; - auto &ExistingVars = AllMatchDataVars[I.getType()]; - - if (NumSeen == ExistingVars.size()) - ExistingVars.push_back("MDInfo" + to_string(NextVarID++)); - - I.setVariableName(ExistingVars[NumSeen++]); - } -} - -//===- C++ Predicates Handling --------------------------------------------===// - -/// Entry into the static pool of all CXX Predicate code. This contains the -/// fully expanded C++ code. -/// -/// Each CXXPattern creates a new entry in the pool to store its data, even -/// after the pattern is destroyed. -/// -/// Note that CXXPattern trims C++ code, so the Code is already expected to be -/// free of leading/trailing whitespace. -struct CXXPredicateCode { - CXXPredicateCode(std::string Code, unsigned ID) - : Code(Code), ID(ID), BaseEnumName("GICombiner" + to_string(ID)) { - assert(StringRef(Code).trim() == Code && - "Code was expected to be trimmed!"); - } - - const std::string Code; - const unsigned ID; - const std::string BaseEnumName; - - bool needsUnreachable() const { - return !StringRef(Code).starts_with("return"); - } - - std::string getEnumNameWithPrefix(StringRef Prefix) const { - return Prefix.str() + BaseEnumName; - } -}; - -using CXXPredicateCodePool = - DenseMap<hash_code, std::unique_ptr<CXXPredicateCode>>; -CXXPredicateCodePool AllCXXMatchCode; -CXXPredicateCodePool AllCXXApplyCode; - -/// Gets an instance of `CXXPredicateCode` for \p Code, or returns an already -/// existing one. -const CXXPredicateCode &getOrInsert(CXXPredicateCodePool &Pool, - std::string Code) { - // Check if we already have an identical piece of code, if not, create an - // entry in the pool. - const auto CodeHash = hash_value(Code); - if (auto It = Pool.find(CodeHash); It != Pool.end()) - return *It->second; - - const auto ID = Pool.size(); - auto OwnedData = std::make_unique<CXXPredicateCode>(std::move(Code), ID); - const auto &DataRef = *OwnedData; - Pool[CodeHash] = std::move(OwnedData); - return DataRef; -} - -/// Sorts a `CXXPredicateCodePool` by their IDs and returns it. -std::vector<const CXXPredicateCode *> -getSorted(const CXXPredicateCodePool &Pool) { - std::vector<const CXXPredicateCode *> Out; - std::transform(Pool.begin(), Pool.end(), std::back_inserter(Out), - [&](auto &Elt) { return Elt.second.get(); }); - sort(Out, [](const auto *A, const auto *B) { return A->ID < B->ID; }); - return Out; -} - -//===- Pattern Base Class -------------------------------------------------===// - -// An abstract pattern found in a combine rule. This can be an apply or match -// pattern. -class Pattern { -public: - enum { - K_AnyOpcode, - K_Inst, - K_CXX, - }; - - virtual ~Pattern() = default; - - unsigned getKind() const { return Kind; } - const char *getKindName() const; - - bool hasName() const { return !Name.empty(); } - StringRef getName() const { return Name; } - - virtual void print(raw_ostream &OS, bool PrintName = true) const = 0; - void dump() const { return print(dbgs()); } - -protected: - Pattern(unsigned Kind, StringRef Name) : Kind(Kind), Name(Name.str()) { - assert(!Name.empty() && "unnamed pattern!"); - } - - void printImpl(raw_ostream &OS, bool PrintName, - function_ref<void()> ContentPrinter) const; - -private: - unsigned Kind; - - // Note: if this ever changes to a StringRef (e.g. allocated in a pool or - // something), CombineRuleBuilder::verify() needs to be updated as well. - // It currently checks that the StringRef in the PatternMap references this. - std::string Name; -}; - -const char *Pattern::getKindName() const { - switch (Kind) { - case K_AnyOpcode: - return "AnyOpcodePattern"; - case K_Inst: - return "InstructionPattern"; - case K_CXX: - return "CXXPattern"; - } - - llvm_unreachable("unknown pattern kind!"); -} - -void Pattern::printImpl(raw_ostream &OS, bool PrintName, - function_ref<void()> ContentPrinter) const { - OS << "(" << getKindName() << " "; - if (PrintName) - OS << "name:" << getName() << " "; - ContentPrinter(); - OS << ")"; -} - -//===- AnyOpcodePattern ---------------------------------------------------===// - -/// `wip_match_opcode` patterns. -/// This matches one or more opcodes, and does not check any operands -/// whatsoever. -class AnyOpcodePattern : public Pattern { -public: - AnyOpcodePattern(StringRef Name) : Pattern(K_AnyOpcode, Name) {} - - static bool classof(const Pattern *P) { return P->getKind() == K_AnyOpcode; } - - void addOpcode(const CodeGenInstruction *I) { Insts.push_back(I); } - const auto &insts() const { return Insts; } - - void print(raw_ostream &OS, bool PrintName = true) const override; - -private: - SmallVector<const CodeGenInstruction *, 4> Insts; -}; - -void AnyOpcodePattern::print(raw_ostream &OS, bool PrintName) const { - printImpl(OS, PrintName, [&OS, this]() { - OS << "[" - << join(map_range(Insts, - [](const auto *I) { return I->TheDef->getName(); }), - ", ") - << "]"; - }); -} - -//===- InstructionPattern -------------------------------------------------===// - -/// Matches an instruction, e.g. `G_ADD $x, $y, $z`. -/// -/// This pattern is simply CodeGenInstruction + a list of operands. -class InstructionPattern : public Pattern { -public: - struct Operand { - std::string Name; - bool IsDef = false; - }; - - InstructionPattern(const CodeGenInstruction &I, StringRef Name) - : Pattern(K_Inst, Name), I(I) {} - - static bool classof(const Pattern *P) { return P->getKind() == K_Inst; } - - const auto &operands() const { return Operands; } - void addOperand(StringRef Name); - unsigned getNumDefs() const { return I.Operands.NumDefs; } - - const CodeGenInstruction &getInst() const { return I; } - StringRef getInstName() const { return I.TheDef->getName(); } - - void reportUnreachable(ArrayRef<SMLoc> Locs) const; - bool checkSemantics(ArrayRef<SMLoc> Loc) const; - - void print(raw_ostream &OS, bool PrintName = true) const override; - -private: - const CodeGenInstruction &I; - SmallVector<Operand, 4> Operands; -}; - -void InstructionPattern::addOperand(StringRef Name) { - const bool IsDef = Operands.size() < getNumDefs(); - Operands.emplace_back(Operand{Name.str(), IsDef}); -} - -void InstructionPattern::reportUnreachable(ArrayRef<SMLoc> Locs) const { - PrintError(Locs, "Instruction pattern '" + getName() + - "' is unreachable from the pattern root!"); -} - -bool InstructionPattern::checkSemantics(ArrayRef<SMLoc> Loc) const { - unsigned NumExpectedOperands = I.Operands.size(); - if (NumExpectedOperands != Operands.size()) { - - PrintError(Loc, "'" + getInstName() + "' expected " + - Twine(NumExpectedOperands) + " operands, got " + - Twine(Operands.size())); - return false; - } - return true; -} - -void InstructionPattern::print(raw_ostream &OS, bool PrintName) const { - printImpl(OS, PrintName, [&OS, this]() { - OS << "inst:" << I.TheDef->getName() << " operands:[" - << join(map_range(Operands, - [](const auto &O) { - return (O.IsDef ? "<def>" : "") + O.Name; - }), - ", ") - << "]"; - }); -} - -//===- CXXPattern ---------------------------------------------------------===// - -/// Raw C++ code which may need some expansions. -/// -/// e.g. [{ return isFooBux(${src}.getReg()); }] -/// -/// For the expanded code, \see CXXPredicateCode. CXXPredicateCode objects are -/// created through `expandCode`. -/// -/// \see CodeExpander and \see CodeExpansions for more information on code -/// expansions. -/// -/// This object has two purposes: -/// - Represent C++ code as a pattern entry. -/// - Be a factory for expanded C++ code. -/// - It's immutable and only holds the raw code so we can expand the same -/// CXX pattern multiple times if we need to. -/// -/// Note that the code is always trimmed in the constructor, so leading and -/// trailing whitespaces are removed. This removes bloat in the output, avoids -/// formatting issues, but also allows us to check things like -/// `.startswith("return")` trivially without worrying about spaces. -class CXXPattern : public Pattern { -public: - CXXPattern(const StringInit &Code, StringRef Name, bool IsApply) - : CXXPattern(Code.getAsUnquotedString(), Name, IsApply) {} - - CXXPattern(StringRef Code, StringRef Name, bool IsApply) - : Pattern(K_CXX, Name), IsApply(IsApply), RawCode(Code.trim().str()) {} - - static bool classof(const Pattern *P) { return P->getKind() == K_CXX; } - - bool isApply() const { return IsApply; } - StringRef getRawCode() const { return RawCode; } - - /// Expands raw code, replacing things such as `${foo}` with their - /// substitution in \p CE. - /// - /// \param CE Map of Code Expansions - /// \param Locs SMLocs for the Code Expander, in case it needs to emit - /// diagnostics. - /// \return A CXXPredicateCode object that contains the expanded code. Note - /// that this may or may not insert a new object. All CXXPredicateCode objects - /// are held in a set to avoid emitting duplicate C++ code. - const CXXPredicateCode &expandCode(const CodeExpansions &CE, - ArrayRef<SMLoc> Locs) const; - - void print(raw_ostream &OS, bool PrintName = true) const override; - -private: - bool IsApply; - std::string RawCode; -}; - -const CXXPredicateCode &CXXPattern::expandCode(const CodeExpansions &CE, - ArrayRef<SMLoc> Locs) const { - std::string Result; - raw_string_ostream OS(Result); - CodeExpander Expander(RawCode, CE, Locs, /*ShowExpansions*/ false); - Expander.emit(OS); - return getOrInsert(IsApply ? AllCXXApplyCode : AllCXXMatchCode, - std::move(Result)); -} - -void CXXPattern::print(raw_ostream &OS, bool PrintName) const { - printImpl(OS, PrintName, [&OS, this] { - OS << (IsApply ? "apply" : "match") << " code:\""; - printEscapedString(getRawCode(), OS); - OS << "\""; - }); -} - -//===- CombineRuleBuilder -------------------------------------------------===// - -/// Helper for CombineRuleBuilder. -/// -/// Represents information about an operand. -/// Operands with no MatchPat are considered live-in to the pattern. -struct OperandTableEntry { - // The matcher pattern that defines this operand. - // null for live-ins. - InstructionPattern *MatchPat = nullptr; - // The apply pattern that (re)defines this operand. - // This can only be non-null if MatchPat is. - InstructionPattern *ApplyPat = nullptr; - - bool isLiveIn() const { return !MatchPat; } -}; - -/// Parses combine rule and builds a small intermediate representation to tie -/// patterns together and emit RuleMatchers to match them. This may emit more -/// than one RuleMatcher, e.g. for `wip_match_opcode`. -/// -/// Memory management for `Pattern` objects is done through `std::unique_ptr`. -/// In most cases, there are two stages to a pattern's lifetime: -/// - Creation in a `parse` function -/// - The unique_ptr is stored in a variable, and may be destroyed if the -/// pattern is found to be semantically invalid. -/// - Ownership transfer into a `PatternMap` -/// - Once a pattern is moved into either the map of Match or Apply -/// patterns, it is known to be valid and it never moves back. -class CombineRuleBuilder { -public: - using PatternMap = MapVector<StringRef, std::unique_ptr<Pattern>>; - - CombineRuleBuilder(const CodeGenTarget &CGT, - SubtargetFeatureInfoMap &SubtargetFeatures, - Record &RuleDef, unsigned ID, - std::vector<RuleMatcher> &OutRMs) - : CGT(CGT), SubtargetFeatures(SubtargetFeatures), RuleDef(RuleDef), - RuleID(ID), OutRMs(OutRMs) {} - - /// Parses all fields in the RuleDef record. - bool parseAll(); - - /// Emits all RuleMatchers into the vector of RuleMatchers passed in the - /// constructor. - bool emitRuleMatchers(); - - void print(raw_ostream &OS) const; - void dump() const { print(dbgs()); } - - /// Debug-only verification of invariants. - void verify() const; - -private: - void PrintError(Twine Msg) const { ::PrintError(RuleDef.getLoc(), Msg); } - - /// Adds the expansions from \see MatchDatas to \p CE. - void declareAllMatchDatasExpansions(CodeExpansions &CE) const; - - /// Adds \p P to \p IM, expanding its code using \p CE. - void addCXXPredicate(InstructionMatcher &IM, const CodeExpansions &CE, - const CXXPattern &P); - - /// Generates a name for anonymous patterns. - /// - /// e.g. (G_ADD $x, $y, $z):$foo is a pattern named "foo", but if ":$foo" is - /// absent, then the pattern is anonymous and this is used to assign it a - /// name. - std::string makeAnonPatName(StringRef Prefix) const; - mutable unsigned AnonIDCnt = 0; - - /// Creates a new RuleMatcher with some boilerplate - /// settings/actions/predicates, and and adds it to \p OutRMs. - /// \see addFeaturePredicates too. - /// - /// \param AdditionalComment Comment string to be added to the - /// `DebugCommentAction`. - RuleMatcher &addRuleMatcher(Twine AdditionalComment = ""); - bool addFeaturePredicates(RuleMatcher &M); - - bool findRoots(); - bool buildOperandsTable(); - - bool parseDefs(DagInit &Def); - bool parseMatch(DagInit &Match); - bool parseApply(DagInit &Apply); - - std::unique_ptr<Pattern> parseInstructionMatcher(const Init &Arg, - StringRef PatName); - std::unique_ptr<Pattern> parseWipMatchOpcodeMatcher(const Init &Arg, - StringRef PatName); - - bool emitMatchPattern(CodeExpansions &CE, const InstructionPattern &IP); - bool emitMatchPattern(CodeExpansions &CE, const AnyOpcodePattern &AOP); - - bool emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M); - - // Recursively visits InstructionPattern from P to build up the - // RuleMatcher/InstructionMatcher. May create new InstructionMatchers as - // needed. - bool emitInstructionMatchPattern(CodeExpansions &CE, RuleMatcher &M, - InstructionMatcher &IM, - const InstructionPattern &P, - DenseSet<const Pattern *> &SeenPats); - - const CodeGenTarget &CGT; - SubtargetFeatureInfoMap &SubtargetFeatures; - Record &RuleDef; - const unsigned RuleID; - std::vector<RuleMatcher> &OutRMs; - - // For InstructionMatcher::addOperand - unsigned AllocatedTemporariesBaseID = 0; - - /// The root of the pattern. - StringRef RootName; - - /// These maps have ownership of the actual Pattern objects. - /// They both map a Pattern's name to the Pattern instance. - PatternMap MatchPats; - PatternMap ApplyPats; - - /// Set by findRoots. - Pattern *MatchRoot = nullptr; - - MapVector<StringRef, OperandTableEntry> OperandTable; - SmallVector<MatchDataInfo, 2> MatchDatas; -}; - -bool CombineRuleBuilder::parseAll() { - if (!parseDefs(*RuleDef.getValueAsDag("Defs"))) - return false; - if (!parseMatch(*RuleDef.getValueAsDag("Match"))) - return false; - if (!parseApply(*RuleDef.getValueAsDag("Apply"))) - return false; - if (!buildOperandsTable()) - return false; - if (!findRoots()) - return false; - LLVM_DEBUG(verify()); - return true; -} - -bool CombineRuleBuilder::emitRuleMatchers() { - assert(MatchRoot); - CodeExpansions CE; - declareAllMatchDatasExpansions(CE); - - switch (MatchRoot->getKind()) { - case Pattern::K_AnyOpcode: { - if (!emitMatchPattern(CE, *cast<AnyOpcodePattern>(MatchRoot))) - return false; - break; - } - case Pattern::K_Inst: - if (!emitMatchPattern(CE, *cast<InstructionPattern>(MatchRoot))) - return false; - break; - case Pattern::K_CXX: - PrintError("C++ code cannot be the root of a pattern!"); - return false; - default: - llvm_unreachable("unknown pattern kind!"); - } - - return true; -} - -void CombineRuleBuilder::print(raw_ostream &OS) const { - OS << "(CombineRule name:" << RuleDef.getName() << " id:" << RuleID - << " root:" << RootName << "\n"; - - OS << " (MatchDatas "; - if (MatchDatas.empty()) - OS << "<empty>)\n"; - else { - OS << "\n"; - for (const auto &MD : MatchDatas) { - OS << " "; - MD.print(OS); - OS << "\n"; - } - OS << " )\n"; - } - - const auto DumpPats = [&](StringRef Name, const PatternMap &Pats) { - OS << " (" << Name << " "; - if (Pats.empty()) { - OS << "<empty>)\n"; - return; - } - - OS << "\n"; - for (const auto &[Name, Pat] : Pats) { - OS << " "; - if (Pat.get() == MatchRoot) - OS << "<root>"; - OS << Name << ":"; - Pat->print(OS, /*PrintName=*/false); - OS << "\n"; - } - OS << " )\n"; - }; - - DumpPats("MatchPats", MatchPats); - DumpPats("ApplyPats", ApplyPats); - - OS << " (OperandTable "; - if (OperandTable.empty()) - OS << "<empty>)\n"; - else { - OS << "\n"; - for (const auto &[Key, Val] : OperandTable) { - OS << " [" << Key; - if (const auto *P = Val.MatchPat) - OS << " match_pat:" << P->getName(); - if (const auto *P = Val.ApplyPat) - OS << " apply_pat:" << P->getName(); - if (Val.isLiveIn()) - OS << " live-in"; - OS << "]\n"; - } - OS << " )\n"; - } - - OS << ")\n"; -} - -void CombineRuleBuilder::verify() const { - const auto VerifyPats = [&](const PatternMap &Pats) { - for (const auto &[Name, Pat] : Pats) { - if (!Pat) - PrintFatalError("null pattern in pattern map!"); - - if (Name != Pat->getName()) { - Pat->dump(); - PrintFatalError("Pattern name mismatch! Map name: " + Name + - ", Pat name: " + Pat->getName()); - } - - // As an optimization, the PatternMaps don't re-allocate the PatternName - // string. They simply reference the std::string inside Pattern. Ensure - // this is the case to avoid memory issues. - if (Name.data() != Pat->getName().data()) { - dbgs() << "Map StringRef: '" << Name << "' @ " - << (const void *)Name.data() << "\n"; - dbgs() << "Pat String: '" << Pat->getName() << "' @ " - << (const void *)Pat->getName().data() << "\n"; - PrintFatalError("StringRef stored in the PatternMap is not referencing " - "the same string as its Pattern!"); - } - } - }; - - VerifyPats(MatchPats); - VerifyPats(ApplyPats); - - for (const auto &[Name, Op] : OperandTable) { - if (Op.ApplyPat && !Op.MatchPat) { - dump(); - PrintFatalError("Operand " + Name + - " has an apply pattern, but no match pattern!"); - } - } -} - -bool CombineRuleBuilder::addFeaturePredicates(RuleMatcher &M) { - if (!RuleDef.getValue("Predicates")) - return true; - - ListInit *Preds = RuleDef.getValueAsListInit("Predicates"); - for (Init *I : Preds->getValues()) { - if (DefInit *Pred = dyn_cast<DefInit>(I)) { - Record *Def = Pred->getDef(); - if (!Def->isSubClassOf("Predicate")) { - ::PrintError(Def->getLoc(), "Unknown 'Predicate' Type"); - return false; - } - - if (Def->getValueAsString("CondString").empty()) - continue; - - if (SubtargetFeatures.count(Def) == 0) { - SubtargetFeatures.emplace( - Def, SubtargetFeatureInfo(Def, SubtargetFeatures.size())); - } - - M.addRequiredFeature(Def); - } - } - - return true; -} - -void CombineRuleBuilder::declareAllMatchDatasExpansions( - CodeExpansions &CE) const { - for (const auto &MD : MatchDatas) - CE.declare(MD.getPatternSymbol(), MD.getQualifiedVariableName()); -} - -void CombineRuleBuilder::addCXXPredicate(InstructionMatcher &IM, - const CodeExpansions &CE, - const CXXPattern &P) { - const auto &ExpandedCode = P.expandCode(CE, RuleDef.getLoc()); - IM.addPredicate<GenericInstructionPredicateMatcher>( - ExpandedCode.getEnumNameWithPrefix(CXXPredPrefix)); -} - -std::string CombineRuleBuilder::makeAnonPatName(StringRef Prefix) const { - return to_string("__anon_pat_" + Prefix + "_" + to_string(RuleID) + "_" + - to_string(AnonIDCnt++)); -} - -RuleMatcher &CombineRuleBuilder::addRuleMatcher(Twine AdditionalComment) { - auto &RM = OutRMs.emplace_back(RuleDef.getLoc()); - addFeaturePredicates(RM); - RM.addRequiredSimplePredicate(getIsEnabledPredicateEnumName(RuleID)); - const std::string AdditionalCommentStr = AdditionalComment.str(); - RM.addAction<DebugCommentAction>( - "Combiner Rule #" + to_string(RuleID) + ": " + RuleDef.getName().str() + - (AdditionalCommentStr.empty() ? "" : "; " + AdditionalCommentStr)); - return RM; -} - -bool CombineRuleBuilder::findRoots() { - // Look by pattern name, e.g. - // (G_FNEG $x, $y):$root - if (auto It = MatchPats.find(RootName); It != MatchPats.end()) { - MatchRoot = It->second.get(); - return true; - } - - // Look by def: - // (G_FNEG $root, $y) - auto It = OperandTable.find(RootName); - if (It == OperandTable.end()) { - PrintError("Cannot find root '" + RootName + "' in match patterns!"); - return false; - } - - if (!It->second.MatchPat) { - PrintError("Cannot use live-in operand '" + RootName + - "' as match pattern root!"); - return false; - } - - MatchRoot = It->second.MatchPat; - return true; -} - -bool CombineRuleBuilder::buildOperandsTable() { - // Walk each instruction pattern - for (auto &[_, P] : MatchPats) { - auto *IP = dyn_cast<InstructionPattern>(P.get()); - if (!IP) - continue; - for (const auto &Operand : IP->operands()) { - // Create an entry, no matter if it's a use or a def. - auto &Entry = OperandTable[Operand.Name]; - - // We only need to do additional checking on defs, though. - if (!Operand.IsDef) - continue; - - if (Entry.MatchPat) { - PrintError("Operand '" + Operand.Name + - "' is defined multiple times in the 'match' patterns"); - return false; - } - Entry.MatchPat = IP; - } - } - - for (auto &[_, P] : ApplyPats) { - auto *IP = dyn_cast<InstructionPattern>(P.get()); - if (!IP) - continue; - for (const auto &Operand : IP->operands()) { - // Create an entry, no matter if it's a use or a def. - auto &Entry = OperandTable[Operand.Name]; - - // We only need to do additional checking on defs, though. - if (!Operand.IsDef) - continue; - - if (!Entry.MatchPat) { - PrintError("Cannot define live-in operand '" + Operand.Name + - "' in the 'apply' pattern"); - return false; - } - if (Entry.ApplyPat) { - PrintError("Operand '" + Operand.Name + - "' is defined multiple times in the 'apply' patterns"); - return false; - } - Entry.ApplyPat = IP; - } - } - - return true; -} - -bool CombineRuleBuilder::parseDefs(DagInit &Def) { - if (Def.getOperatorAsDef(RuleDef.getLoc())->getName() != "defs") { - PrintError("Expected defs operator"); - return false; - } - - SmallVector<StringRef> Roots; - for (unsigned I = 0, E = Def.getNumArgs(); I < E; ++I) { - if (isSpecificDef(*Def.getArg(I), "root")) { - Roots.emplace_back(Def.getArgNameStr(I)); - continue; - } - - // Subclasses of GIDefMatchData should declare that this rule needs to pass - // data from the match stage to the apply stage, and ensure that the - // generated matcher has a suitable variable for it to do so. - if (Record *MatchDataRec = - getDefOfSubClass(*Def.getArg(I), "GIDefMatchData")) { - MatchDatas.emplace_back(Def.getArgNameStr(I), - MatchDataRec->getValueAsString("Type")); - continue; - } - - // Otherwise emit an appropriate error message. - if (getDefOfSubClass(*Def.getArg(I), "GIDefKind")) - PrintError("This GIDefKind not implemented in tablegen"); - else if (getDefOfSubClass(*Def.getArg(I), "GIDefKindWithArgs")) - PrintError("This GIDefKindWithArgs not implemented in tablegen"); - else - PrintError("Expected a subclass of GIDefKind or a sub-dag whose " - "operator is of type GIDefKindWithArgs"); - return false; - } - - if (Roots.size() != 1) { - PrintError("Combine rules must have exactly one root"); - return false; - } - - RootName = Roots.front(); - - // Assign variables to all MatchDatas. - AssignMatchDataVariables(MatchDatas); - return true; -} - -bool CombineRuleBuilder::parseMatch(DagInit &Match) { - if (Match.getOperatorAsDef(RuleDef.getLoc())->getName() != "match") { - PrintError("Expected match operator"); - return false; - } - - if (Match.getNumArgs() == 0) { - PrintError("Matcher is empty"); - return false; - } - - // The match section consists of a list of matchers and predicates. Parse each - // one and add the equivalent GIMatchDag nodes, predicates, and edges. - bool HasOpcodeMatcher = false; - for (unsigned I = 0; I < Match.getNumArgs(); ++I) { - Init *Arg = Match.getArg(I); - std::string Name = Match.getArgName(I) - ? Match.getArgName(I)->getValue().str() - : makeAnonPatName("match"); - - if (MatchPats.contains(Name)) { - PrintError("'" + Name + "' match pattern defined more than once!"); - return false; - } - - if (auto Pat = parseInstructionMatcher(*Arg, Name)) { - MatchPats[Pat->getName()] = std::move(Pat); - continue; - } - - if (auto Pat = parseWipMatchOpcodeMatcher(*Arg, Name)) { - if (HasOpcodeMatcher) { - PrintError("wip_opcode_match can only be present once"); - return false; - } - HasOpcodeMatcher = true; - MatchPats[Pat->getName()] = std::move(Pat); - continue; - } - - // Parse arbitrary C++ code - if (const auto *StringI = dyn_cast<StringInit>(Arg)) { - auto CXXPat = - std::make_unique<CXXPattern>(*StringI, Name, /*IsApply*/ false); - if (!CXXPat->getRawCode().contains("return ")) { - PrintWarning(RuleDef.getLoc(), - "'match' C++ code does not seem to return!"); - } - MatchPats[CXXPat->getName()] = std::move(CXXPat); - continue; - } - - // TODO: don't print this on, e.g. bad operand count in inst pat - PrintError("Expected a subclass of GIMatchKind or a sub-dag whose " - "operator is either of a GIMatchKindWithArgs or Instruction"); - PrintNote("Pattern was `" + Arg->getAsString() + "'"); - return false; - } - - return true; -} - -bool CombineRuleBuilder::parseApply(DagInit &Apply) { - // Currently we only support C++ :( - if (Apply.getOperatorAsDef(RuleDef.getLoc())->getName() != "apply") { - PrintError("Expected 'apply' operator in Apply DAG"); - return false; - } - - if (Apply.getNumArgs() != 1) { - PrintError("Expected exactly 1 argument in 'apply'"); - return false; - } - - const StringInit *Code = dyn_cast<StringInit>(Apply.getArg(0)); - auto Pat = std::make_unique<CXXPattern>(*Code, makeAnonPatName("apply"), - /*IsApply*/ true); - ApplyPats[Pat->getName()] = std::move(Pat); - return true; -} - -std::unique_ptr<Pattern> -CombineRuleBuilder::parseInstructionMatcher(const Init &Arg, StringRef Name) { - const DagInit *Matcher = getDagWithOperatorOfSubClass(Arg, "Instruction"); - if (!Matcher) - return nullptr; - - auto &Instr = CGT.getInstruction(Matcher->getOperatorAsDef(RuleDef.getLoc())); - auto Pat = std::make_unique<InstructionPattern>(Instr, Name); - - for (const auto &NameInit : Matcher->getArgNames()) - Pat->addOperand(NameInit->getAsUnquotedString()); - - if (!Pat->checkSemantics(RuleDef.getLoc())) - return nullptr; - - return std::move(Pat); -} - -std::unique_ptr<Pattern> -CombineRuleBuilder::parseWipMatchOpcodeMatcher(const Init &Arg, - StringRef Name) { - const DagInit *Matcher = getDagWithSpecificOperator(Arg, "wip_match_opcode"); - if (!Matcher) - return nullptr; - - if (Matcher->getNumArgs() == 0) { - PrintError("Empty wip_match_opcode"); - return nullptr; - } - - // Each argument is an opcode that can match. - auto Result = std::make_unique<AnyOpcodePattern>(Name); - for (const auto &Arg : Matcher->getArgs()) { - Record *OpcodeDef = getDefOfSubClass(*Arg, "Instruction"); - if (OpcodeDef) { - Result->addOpcode(&CGT.getInstruction(OpcodeDef)); - continue; - } - - PrintError("Arguments to wip_match_opcode must be instructions"); - return nullptr; - } - - return std::move(Result); -} - -bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE, - const InstructionPattern &IP) { - auto &M = addRuleMatcher(); - InstructionMatcher &IM = M.addInstructionMatcher("root"); - declareInstExpansion(CE, IM, IP.getName()); - - DenseSet<const Pattern *> SeenPats; - if (!emitInstructionMatchPattern(CE, M, IM, IP, SeenPats)) - return false; - - // Emit remaining patterns - for (auto &[_, Pat] : MatchPats) { - if (SeenPats.contains(Pat.get())) - continue; - - switch (Pat->getKind()) { - case Pattern::K_AnyOpcode: - PrintError("wip_match_opcode can not be used with instruction patterns!"); - return false; - case Pattern::K_Inst: - cast<InstructionPattern>(Pat.get())->reportUnreachable(RuleDef.getLoc()); - return false; - case Pattern::K_CXX: { - addCXXPredicate(IM, CE, *cast<CXXPattern>(Pat.get())); - continue; - } - default: - llvm_unreachable("unknown pattern kind!"); - } - } - - return emitApplyPatterns(CE, M); -} - -bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE, - const AnyOpcodePattern &AOP) { - - for (const CodeGenInstruction *CGI : AOP.insts()) { - auto &M = addRuleMatcher("wip_match_opcode alternative '" + - CGI->TheDef->getName() + "'"); - - InstructionMatcher &IM = M.addInstructionMatcher(AOP.getName()); - declareInstExpansion(CE, IM, AOP.getName()); - // declareInstExpansion needs to be identical, otherwise we need to create a - // CodeExpansions object here instead. - assert(IM.getInsnVarID() == 0); - - IM.addPredicate<InstructionOpcodeMatcher>(CGI); - - // Emit remaining patterns. - for (auto &[_, Pat] : MatchPats) { - if (Pat.get() == &AOP) - continue; - - switch (Pat->getKind()) { - case Pattern::K_AnyOpcode: - PrintError("wip_match_opcode can only be present once!"); - return false; - case Pattern::K_Inst: - cast<InstructionPattern>(Pat.get())->reportUnreachable( - RuleDef.getLoc()); - return false; - case Pattern::K_CXX: { - addCXXPredicate(IM, CE, *cast<CXXPattern>(Pat.get())); - break; - } - default: - llvm_unreachable("unknown pattern kind!"); - } - } - - if (!emitApplyPatterns(CE, M)) - return false; - } - - return true; -} - -bool CombineRuleBuilder::emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M) { - for (auto &[_, Pat] : ApplyPats) { - switch (Pat->getKind()) { - case Pattern::K_AnyOpcode: - case Pattern::K_Inst: - llvm_unreachable("Unsupported pattern kind in output pattern!"); - case Pattern::K_CXX: { - CXXPattern *CXXPat = cast<CXXPattern>(Pat.get()); - const auto &ExpandedCode = CXXPat->expandCode(CE, RuleDef.getLoc()); - M.addAction<CustomCXXAction>( - ExpandedCode.getEnumNameWithPrefix(CXXApplyPrefix)); - continue; - } - default: - llvm_unreachable("Unknown pattern kind!"); - } - } - - return true; -} - -bool CombineRuleBuilder::emitInstructionMatchPattern( - CodeExpansions &CE, RuleMatcher &M, InstructionMatcher &IM, - const InstructionPattern &P, DenseSet<const Pattern *> &SeenPats) { - if (SeenPats.contains(&P)) - return true; - - SeenPats.insert(&P); - - IM.addPredicate<InstructionOpcodeMatcher>(&P.getInst()); - declareInstExpansion(CE, IM, P.getName()); - - unsigned OpIdx = 0; - for (auto &O : P.operands()) { - auto &OpTableEntry = OperandTable.find(O.Name)->second; - - OperandMatcher &OM = - IM.addOperand(OpIdx++, O.Name, AllocatedTemporariesBaseID++); - declareOperandExpansion(CE, OM, O.Name); - - if (O.IsDef) - continue; - - if (InstructionPattern *DefPat = OpTableEntry.MatchPat) { - auto InstOpM = OM.addPredicate<InstructionOperandMatcher>(M, O.Name); - if (!InstOpM) { - // TODO: copy-pasted from GlobalISelEmitter.cpp. Is it still relevant - // here? - PrintError("Nested instruction '" + DefPat->getName() + - "' cannot be the same as another operand '" + O.Name + "'"); - return false; - } - - if (!emitInstructionMatchPattern(CE, M, (*InstOpM)->getInsnMatcher(), - *DefPat, SeenPats)) - return false; - } - } - - return true; -} - -//===- GICombinerEmitter --------------------------------------------------===// - -/// This class is essentially the driver. It fetches all TableGen records, calls -/// CombineRuleBuilder to build the MatchTable's RuleMatchers, then creates the -/// MatchTable & emits it. It also handles emitting all the supporting code such -/// as the list of LLTs, the CXXPredicates, etc. -class GICombinerEmitter final : public GlobalISelMatchTableExecutorEmitter { - RecordKeeper &Records; - StringRef Name; - const CodeGenTarget &Target; - Record *Combiner; - unsigned NextRuleID = 0; - - // List all combine rules (ID, name) imported. - // Note that the combiner rule ID is different from the RuleMatcher ID. The - // latter is internal to the MatchTable, the former is the canonical ID of the - // combine rule used to disable/enable it. - std::vector<std::pair<unsigned, std::string>> AllCombineRules; - - MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules); - - void emitRuleConfigImpl(raw_ostream &OS); - - void emitAdditionalImpl(raw_ostream &OS) override; - - void emitMIPredicateFns(raw_ostream &OS) override; - void emitI64ImmPredicateFns(raw_ostream &OS) override; - void emitAPFloatImmPredicateFns(raw_ostream &OS) override; - void emitAPIntImmPredicateFns(raw_ostream &OS) override; - void emitTestSimplePredicate(raw_ostream &OS) override; - void emitRunCustomAction(raw_ostream &OS) override; - - void emitAdditionalTemporariesDecl(raw_ostream &OS, - StringRef Indent) override; - - const CodeGenTarget &getTarget() const override { return Target; } - StringRef getClassName() const override { - return Combiner->getValueAsString("Classname"); - } - - std::string getRuleConfigClassName() const { - return getClassName().str() + "RuleConfig"; - } - - void gatherRules(std::vector<RuleMatcher> &Rules, - const std::vector<Record *> &&RulesAndGroups); - -public: - explicit GICombinerEmitter(RecordKeeper &RK, const CodeGenTarget &Target, - StringRef Name, Record *Combiner); - ~GICombinerEmitter() {} - - void run(raw_ostream &OS); -}; - -void GICombinerEmitter::emitRuleConfigImpl(raw_ostream &OS) { - OS << "struct " << getRuleConfigClassName() << " {\n" - << " SparseBitVector<> DisabledRules;\n\n" - << " bool isRuleEnabled(unsigned RuleID) const;\n" - << " bool parseCommandLineOption();\n" - << " bool setRuleEnabled(StringRef RuleIdentifier);\n" - << " bool setRuleDisabled(StringRef RuleIdentifier);\n" - << "};\n\n"; - - std::vector<std::pair<std::string, std::string>> Cases; - Cases.reserve(AllCombineRules.size()); - - for (const auto &[ID, Name] : AllCombineRules) - Cases.emplace_back(Name, "return " + to_string(ID) + ";\n"); - - OS << "static std::optional<uint64_t> getRuleIdxForIdentifier(StringRef " - "RuleIdentifier) {\n" - << " uint64_t I;\n" - << " // getAtInteger(...) returns false on success\n" - << " bool Parsed = !RuleIdentifier.getAsInteger(0, I);\n" - << " if (Parsed)\n" - << " return I;\n\n" - << "#ifndef NDEBUG\n"; - StringMatcher Matcher("RuleIdentifier", Cases, OS); - Matcher.Emit(); - OS << "#endif // ifndef NDEBUG\n\n" - << " return std::nullopt;\n" - << "}\n"; - - OS << "static std::optional<std::pair<uint64_t, uint64_t>> " - "getRuleRangeForIdentifier(StringRef RuleIdentifier) {\n" - << " std::pair<StringRef, StringRef> RangePair = " - "RuleIdentifier.split('-');\n" - << " if (!RangePair.second.empty()) {\n" - << " const auto First = " - "getRuleIdxForIdentifier(RangePair.first);\n" - << " const auto Last = " - "getRuleIdxForIdentifier(RangePair.second);\n" - << " if (!First || !Last)\n" - << " return std::nullopt;\n" - << " if (First >= Last)\n" - << " report_fatal_error(\"Beginning of range should be before " - "end of range\");\n" - << " return {{*First, *Last + 1}};\n" - << " }\n" - << " if (RangePair.first == \"*\") {\n" - << " return {{0, " << AllCombineRules.size() << "}};\n" - << " }\n" - << " const auto I = getRuleIdxForIdentifier(RangePair.first);\n" - << " if (!I)\n" - << " return std::nullopt;\n" - << " return {{*I, *I + 1}};\n" - << "}\n\n"; - - for (bool Enabled : {true, false}) { - OS << "bool " << getRuleConfigClassName() << "::setRule" - << (Enabled ? "Enabled" : "Disabled") << "(StringRef RuleIdentifier) {\n" - << " auto MaybeRange = getRuleRangeForIdentifier(RuleIdentifier);\n" - << " if (!MaybeRange)\n" - << " return false;\n" - << " for (auto I = MaybeRange->first; I < MaybeRange->second; ++I)\n" - << " DisabledRules." << (Enabled ? "reset" : "set") << "(I);\n" - << " return true;\n" - << "}\n\n"; - } - - OS << "static std::vector<std::string> " << Name << "Option;\n" - << "static cl::list<std::string> " << Name << "DisableOption(\n" - << " \"" << Name.lower() << "-disable-rule\",\n" - << " cl::desc(\"Disable one or more combiner rules temporarily in " - << "the " << Name << " pass\"),\n" - << " cl::CommaSeparated,\n" - << " cl::Hidden,\n" - << " cl::cat(GICombinerOptionCategory),\n" - << " cl::callback([](const std::string &Str) {\n" - << " " << Name << "Option.push_back(Str);\n" - << " }));\n" - << "static cl::list<std::string> " << Name << "OnlyEnableOption(\n" - << " \"" << Name.lower() << "-only-enable-rule\",\n" - << " cl::desc(\"Disable all rules in the " << Name - << " pass then re-enable the specified ones\"),\n" - << " cl::Hidden,\n" - << " cl::cat(GICombinerOptionCategory),\n" - << " cl::callback([](const std::string &CommaSeparatedArg) {\n" - << " StringRef Str = CommaSeparatedArg;\n" - << " " << Name << "Option.push_back(\"*\");\n" - << " do {\n" - << " auto X = Str.split(\",\");\n" - << " " << Name << "Option.push_back((\"!\" + X.first).str());\n" - << " Str = X.second;\n" - << " } while (!Str.empty());\n" - << " }));\n" - << "\n\n" - << "bool " << getRuleConfigClassName() - << "::isRuleEnabled(unsigned RuleID) const {\n" - << " return !DisabledRules.test(RuleID);\n" - << "}\n" - << "bool " << getRuleConfigClassName() << "::parseCommandLineOption() {\n" - << " for (StringRef Identifier : " << Name << "Option) {\n" - << " bool Enabled = Identifier.consume_front(\"!\");\n" - << " if (Enabled && !setRuleEnabled(Identifier))\n" - << " return false;\n" - << " if (!Enabled && !setRuleDisabled(Identifier))\n" - << " return false;\n" - << " }\n" - << " return true;\n" - << "}\n\n"; -} - -void GICombinerEmitter::emitAdditionalImpl(raw_ostream &OS) { - OS << "bool " << getClassName() - << "::tryCombineAll(MachineInstr &I) const {\n" - << " const TargetSubtargetInfo &ST = MF.getSubtarget();\n" - << " const PredicateBitset AvailableFeatures = " - "getAvailableFeatures();\n" - << " NewMIVector OutMIs;\n" - << " State.MIs.clear();\n" - << " State.MIs.push_back(&I);\n" - << " " << MatchDataInfo::StructName << " = " - << MatchDataInfo::StructTypeName << "();\n\n" - << " if (executeMatchTable(*this, OutMIs, State, ExecInfo" - << ", getMatchTable(), *ST.getInstrInfo(), MRI, " - "*MRI.getTargetRegisterInfo(), *ST.getRegBankInfo(), AvailableFeatures" - << ", /*CoverageInfo*/ nullptr)) {\n" - << " return true;\n" - << " }\n\n" - << " return false;\n" - << "}\n\n"; -} - -void GICombinerEmitter::emitMIPredicateFns(raw_ostream &OS) { - auto MatchCode = getSorted(AllCXXMatchCode); - emitMIPredicateFnsImpl<const CXXPredicateCode *>( - OS, "", ArrayRef<const CXXPredicateCode *>(MatchCode), - [](const CXXPredicateCode *C) -> StringRef { return C->BaseEnumName; }, - [](const CXXPredicateCode *C) -> StringRef { return C->Code; }); -} - -void GICombinerEmitter::emitI64ImmPredicateFns(raw_ostream &OS) { - // Unused, but still needs to be called. - emitImmPredicateFnsImpl<unsigned>( - OS, "I64", "int64_t", {}, [](unsigned) { return ""; }, - [](unsigned) { return ""; }); -} - -void GICombinerEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) { - // Unused, but still needs to be called. - emitImmPredicateFnsImpl<unsigned>( - OS, "APFloat", "const APFloat &", {}, [](unsigned) { return ""; }, - [](unsigned) { return ""; }); -} - -void GICombinerEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) { - // Unused, but still needs to be called. - emitImmPredicateFnsImpl<unsigned>( - OS, "APInt", "const APInt &", {}, [](unsigned) { return ""; }, - [](unsigned) { return ""; }); -} - -void GICombinerEmitter::emitTestSimplePredicate(raw_ostream &OS) { - if (!AllCombineRules.empty()) { - OS << "enum {\n"; - std::string EnumeratorSeparator = " = GICXXPred_Invalid + 1,\n"; - // To avoid emitting a switch, we expect that all those rules are in order. - // That way we can just get the RuleID from the enum by subtracting - // (GICXXPred_Invalid + 1). - unsigned ExpectedID = 0; - (void)ExpectedID; - for (const auto &[ID, _] : AllCombineRules) { - assert(ExpectedID++ == ID && "combine rules are not ordered!"); - OS << " " << getIsEnabledPredicateEnumName(ID) << EnumeratorSeparator; - EnumeratorSeparator = ",\n"; - } - OS << "};\n\n"; - } - - OS << "bool " << getClassName() - << "::testSimplePredicate(unsigned Predicate) const {\n" - << " return RuleConfig.isRuleEnabled(Predicate - " - "GICXXPred_Invalid - " - "1);\n" - << "}\n"; -} - -void GICombinerEmitter::emitRunCustomAction(raw_ostream &OS) { - const auto ApplyCode = getSorted(AllCXXApplyCode); - - if (!ApplyCode.empty()) { - OS << "enum {\n"; - std::string EnumeratorSeparator = " = GICXXCustomAction_Invalid + 1,\n"; - for (const auto &Apply : ApplyCode) { - OS << " " << Apply->getEnumNameWithPrefix(CXXApplyPrefix) - << EnumeratorSeparator; - EnumeratorSeparator = ",\n"; - } - OS << "};\n"; - } - - OS << "void " << getClassName() - << "::runCustomAction(unsigned ApplyID, const MatcherState &State) const " - "{\n"; - if (!ApplyCode.empty()) { - OS << " switch(ApplyID) {\n"; - for (const auto &Apply : ApplyCode) { - OS << " case " << Apply->getEnumNameWithPrefix(CXXApplyPrefix) << ":{\n" - << " " << Apply->Code << "\n" - << " return;\n"; - OS << " }\n"; - } - OS << "}\n"; - } - OS << " llvm_unreachable(\"Unknown Apply Action\");\n" - << "}\n"; -} - -void GICombinerEmitter::emitAdditionalTemporariesDecl(raw_ostream &OS, - StringRef Indent) { - OS << Indent << "struct " << MatchDataInfo::StructTypeName << " {\n"; - for (const auto &[Type, VarNames] : AllMatchDataVars) { - assert(!VarNames.empty() && "Cannot have no vars for this type!"); - OS << Indent << " " << Type << " " << join(VarNames, ", ") << ";\n"; - } - OS << Indent << "};\n" - << Indent << "mutable " << MatchDataInfo::StructTypeName << " " - << MatchDataInfo::StructName << ";\n\n"; -} - -GICombinerEmitter::GICombinerEmitter(RecordKeeper &RK, - const CodeGenTarget &Target, - StringRef Name, Record *Combiner) - : Records(RK), Name(Name), Target(Target), Combiner(Combiner) {} - -MatchTable -GICombinerEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules) { - std::vector<Matcher *> InputRules; - for (Matcher &Rule : Rules) - InputRules.push_back(&Rule); - - unsigned CurrentOrdering = 0; - StringMap<unsigned> OpcodeOrder; - for (RuleMatcher &Rule : Rules) { - const StringRef Opcode = Rule.getOpcode(); - assert(!Opcode.empty() && "Didn't expect an undefined opcode"); - if (OpcodeOrder.count(Opcode) == 0) - OpcodeOrder[Opcode] = CurrentOrdering++; - } - - llvm::stable_sort(InputRules, [&OpcodeOrder](const Matcher *A, - const Matcher *B) { - auto *L = static_cast<const RuleMatcher *>(A); - auto *R = static_cast<const RuleMatcher *>(B); - return std::make_tuple(OpcodeOrder[L->getOpcode()], L->getNumOperands()) < - std::make_tuple(OpcodeOrder[R->getOpcode()], R->getNumOperands()); - }); - - for (Matcher *Rule : InputRules) - Rule->optimize(); - - std::vector<std::unique_ptr<Matcher>> MatcherStorage; - std::vector<Matcher *> OptRules = - optimizeRules<GroupMatcher>(InputRules, MatcherStorage); - - for (Matcher *Rule : OptRules) - Rule->optimize(); - - OptRules = optimizeRules<SwitchMatcher>(OptRules, MatcherStorage); - - return MatchTable::buildTable(OptRules, /*WithCoverage*/ false, - /*IsCombiner*/ true); -} - -/// Recurse into GICombineGroup's and flatten the ruleset into a simple list. -void GICombinerEmitter::gatherRules( - std::vector<RuleMatcher> &ActiveRules, - const std::vector<Record *> &&RulesAndGroups) { - for (Record *R : RulesAndGroups) { - if (R->isValueUnset("Rules")) { - AllCombineRules.emplace_back(NextRuleID, R->getName().str()); - CombineRuleBuilder CRB(Target, SubtargetFeatures, *R, NextRuleID++, - ActiveRules); - - if (!CRB.parseAll()) - continue; - - if (StopAfterParse) { - CRB.print(outs()); - continue; - } - - if (!CRB.emitRuleMatchers()) - continue; - } else - gatherRules(ActiveRules, R->getValueAsListOfDefs("Rules")); - } -} - -void GICombinerEmitter::run(raw_ostream &OS) { - Records.startTimer("Gather rules"); - std::vector<RuleMatcher> Rules; - gatherRules(Rules, Combiner->getValueAsListOfDefs("Rules")); - if (ErrorsPrinted) - PrintFatalError(Combiner->getLoc(), "Failed to parse one or more rules"); - - Records.startTimer("Creating Match Table"); - unsigned MaxTemporaries = 0; - for (const auto &Rule : Rules) - MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns()); - - const MatchTable Table = buildMatchTable(Rules); - - Records.startTimer("Emit combiner"); - - emitSourceFileHeader(getClassName().str() + " Combiner Match Table", OS); - - // Unused - std::vector<StringRef> CustomRendererFns; - // Unused, but hack to avoid empty declarator - std::vector<LLTCodeGen> TypeObjects = {LLTCodeGen(LLT::scalar(1))}; - // Unused - std::vector<Record *> ComplexPredicates; - - // GET_GICOMBINER_DEPS, which pulls in extra dependencies. - OS << "#ifdef GET_GICOMBINER_DEPS\n" - << "#include \"llvm/ADT/SparseBitVector.h\"\n" - << "namespace llvm {\n" - << "extern cl::OptionCategory GICombinerOptionCategory;\n" - << "} // end namespace llvm\n" - << "#endif // ifdef GET_GICOMBINER_DEPS\n\n"; - - // GET_GICOMBINER_TYPES, which needs to be included before the declaration of - // the class. - OS << "#ifdef GET_GICOMBINER_TYPES\n"; - emitRuleConfigImpl(OS); - OS << "#endif // ifdef GET_GICOMBINER_TYPES\n\n"; - emitPredicateBitset(OS, "GET_GICOMBINER_TYPES"); - - // GET_GICOMBINER_CLASS_MEMBERS, which need to be included inside the class. - emitPredicatesDecl(OS, "GET_GICOMBINER_CLASS_MEMBERS"); - emitTemporariesDecl(OS, "GET_GICOMBINER_CLASS_MEMBERS"); - - // GET_GICOMBINER_IMPL, which needs to be included outside the class. - emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexPredicates, - CustomRendererFns, "GET_GICOMBINER_IMPL"); - - // GET_GICOMBINER_CONSTRUCTOR_INITS, which are in the constructor's - // initializer list. - emitPredicatesInit(OS, "GET_GICOMBINER_CONSTRUCTOR_INITS"); - emitTemporariesInit(OS, MaxTemporaries, "GET_GICOMBINER_CONSTRUCTOR_INITS"); -} - -} // end anonymous namespace - -//===----------------------------------------------------------------------===// - -static void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS) { - CodeGenTarget Target(RK); - - if (SelectedCombiners.empty()) - PrintFatalError("No combiners selected with -combiners"); - for (const auto &Combiner : SelectedCombiners) { - Record *CombinerDef = RK.getDef(Combiner); - if (!CombinerDef) - PrintFatalError("Could not find " + Combiner); - GICombinerEmitter(RK, Target, Combiner, CombinerDef).run(OS); - } -} - -static TableGen::Emitter::Opt X("gen-global-isel-combiner-matchtable", - EmitGICombiner, - "Generate GlobalISel combiner Match Table"); diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index 3bdcfec06e24..8d9ded1b2ac5 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -312,6 +312,8 @@ public: void emitTestSimplePredicate(raw_ostream &OS) override; void emitRunCustomAction(raw_ostream &OS) override; + void postProcessRule(RuleMatcher &M); + const CodeGenTarget &getTarget() const override { return Target; } StringRef getClassName() const override { return ClassName; } @@ -355,8 +357,8 @@ private: /// to the number of named operands that predicate expects. Store locations in /// StoreIdxForName correspond to the order in which operand names appear in /// predicate's argument list. - /// When we visit named leaf operand and WaitingForNamedOperands is not zero, - /// add matcher that will record operand and decrease counter. + /// When we visit named operand and WaitingForNamedOperands is not zero, add + /// matcher that will record operand and decrease counter. unsigned WaitingForNamedOperands = 0; StringMap<unsigned> StoreIdxForName; @@ -413,6 +415,8 @@ private: void declareSubtargetFeature(Record *Predicate); + unsigned declareHwModeCheck(StringRef HwModeFeatures); + MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize, bool WithCoverage); @@ -498,6 +502,10 @@ GlobalISelEmitter::getEquivNode(Record &Equiv, const TreePatternNode *N) const { return &Target.getInstruction(Equiv.getValueAsDef("IfFloatingPoint")); } + if (!Equiv.isValueUnset("IfConvergent") && + N->getIntrinsicInfo(CGP)->isConvergent) + return &Target.getInstruction(Equiv.getValueAsDef("IfConvergent")); + for (const TreePredicateCall &Call : N->getPredicateCalls()) { const TreePredicateFn &Predicate = Call.Fn; if (!Equiv.isValueUnset("IfSignExtend") && @@ -779,13 +787,11 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( } } - bool IsAtomic = false; if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("NotAtomic"); else if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) { - IsAtomic = true; InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); } @@ -839,31 +845,13 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( } } - // Hack around an unfortunate mistake in how atomic store (and really - // atomicrmw in general) operands were ordered. A ISD::STORE used the order - // <stored value>, <pointer> order. ISD::ATOMIC_STORE used the opposite, - // <pointer>, <stored value>. In GlobalISel there's just the one store - // opcode, so we need to swap the operands here to get the right type check. - if (IsAtomic && SrcGIOrNull->TheDef->getName() == "G_STORE") { - assert(NumChildren == 2 && "wrong operands for atomic store"); - - const TreePatternNode *PtrChild = Src->getChild(0); - const TreePatternNode *ValueChild = Src->getChild(1); - - if (auto Error = importChildMatcher(Rule, InsnMatcher, PtrChild, true, - false, 1, TempOpIdx)) - return std::move(Error); - - if (auto Error = importChildMatcher(Rule, InsnMatcher, ValueChild, false, - false, 0, TempOpIdx)) - return std::move(Error); - return InsnMatcher; - } - // Match the used operands (i.e. the children of the operator). bool IsIntrinsic = SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" || - SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_W_SIDE_EFFECTS"; + SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_W_SIDE_EFFECTS" || + SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_CONVERGENT" || + SrcGIOrNull->TheDef->getName() == + "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS"; const CodeGenIntrinsic *II = Src->getIntrinsicInfo(CGP); if (IsIntrinsic && !II) return failedImport("Expected IntInit containing intrinsic ID)"); @@ -1009,6 +997,17 @@ Error GlobalISelEmitter::importChildMatcher( to_string(*SrcChild) + ")"); } + // Try look up SrcChild for a (named) predicate operand if there is any. + if (WaitingForNamedOperands) { + auto &ScopedNames = SrcChild->getNamesAsPredicateArg(); + if (!ScopedNames.empty()) { + auto PA = ScopedNames.begin(); + std::string Name = getScopedName(PA->getScope(), PA->getIdentifier()); + OM.addPredicate<RecordNamedOperandMatcher>(StoreIdxForName[Name], Name); + --WaitingForNamedOperands; + } + } + // Check for nested instructions. if (!SrcChild->isLeaf()) { if (SrcChild->getOperator()->isSubClassOf("ComplexPattern")) { @@ -1073,13 +1072,6 @@ Error GlobalISelEmitter::importChildMatcher( if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild->getLeafValue())) { auto *ChildRec = ChildDefInit->getDef(); - if (WaitingForNamedOperands) { - auto PA = SrcChild->getNamesAsPredicateArg().begin(); - std::string Name = getScopedName(PA->getScope(), PA->getIdentifier()); - OM.addPredicate<RecordNamedOperandMatcher>(StoreIdxForName[Name], Name); - --WaitingForNamedOperands; - } - // Check for register classes. if (ChildRec->isSubClassOf("RegisterClass") || ChildRec->isSubClassOf("RegisterOperand")) { @@ -1908,6 +1900,9 @@ Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { if (auto Error = importRulePredicates(M, Predicates)) return std::move(Error); + if (!P.getHwModeFeatures().empty()) + M.addHwModeIdx(declareHwModeCheck(P.getHwModeFeatures())); + // Next, analyze the pattern operators. TreePatternNode *Src = P.getSrcPattern(); TreePatternNode *Dst = P.getDstPattern(); @@ -2272,10 +2267,10 @@ void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) { "&CoverageInfo) const {\n" << " const PredicateBitset AvailableFeatures = " "getAvailableFeatures();\n" - << " NewMIVector OutMIs;\n" + << " MachineIRBuilder B(I);\n" << " State.MIs.clear();\n" << " State.MIs.push_back(&I);\n\n" - << " if (executeMatchTable(*this, OutMIs, State, ExecInfo" + << " if (executeMatchTable(*this, State, ExecInfo, B" << ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures" << ", &CoverageInfo)) {\n" << " return true;\n" @@ -2357,12 +2352,38 @@ void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) { void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) { OS << "void " << getClassName() - << "::runCustomAction(unsigned, const MatcherState&) const {\n" + << "::runCustomAction(unsigned, const MatcherState&, NewMIVector &) const " + "{\n" << " llvm_unreachable(\"" + getClassName() + " does not support custom C++ actions!\");\n" << "}\n"; } +void GlobalISelEmitter::postProcessRule(RuleMatcher &M) { + SmallPtrSet<Record *, 16> UsedRegs; + + // TODO: deal with subregs? + for (auto &A : M.actions()) { + auto *MI = dyn_cast<BuildMIAction>(A.get()); + if (!MI) + continue; + + for (auto *Use : MI->getCGI()->ImplicitUses) + UsedRegs.insert(Use); + } + + for (auto &A : M.actions()) { + auto *MI = dyn_cast<BuildMIAction>(A.get()); + if (!MI) + continue; + + for (auto *Def : MI->getCGI()->ImplicitDefs) { + if (!UsedRegs.contains(Def)) + MI->setDeadImplicitDef(Def); + } + } +} + void GlobalISelEmitter::run(raw_ostream &OS) { if (!UseCoverageFile.empty()) { RuleCoverage = CodeGenCoverage(); @@ -2420,6 +2441,7 @@ void GlobalISelEmitter::run(raw_ostream &OS) { "Pattern is not covered by a test"); } Rules.push_back(std::move(MatcherOrErr.get())); + postProcessRule(Rules.back()); } // Comparison function to order records by name. @@ -2484,9 +2506,11 @@ void GlobalISelEmitter::run(raw_ostream &OS) { } void GlobalISelEmitter::declareSubtargetFeature(Record *Predicate) { - if (SubtargetFeatures.count(Predicate) == 0) - SubtargetFeatures.emplace( - Predicate, SubtargetFeatureInfo(Predicate, SubtargetFeatures.size())); + SubtargetFeatures.try_emplace(Predicate, Predicate, SubtargetFeatures.size()); +} + +unsigned GlobalISelEmitter::declareHwModeCheck(StringRef HwModeFeatures) { + return HwModes.emplace(HwModeFeatures.str(), HwModes.size()).first->second; } } // end anonymous namespace diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp index aab772f020a6..481f3f16e013 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp @@ -43,11 +43,13 @@ std::string getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) { //===- Helpers ------------------------------------------------------------===// -std::string -getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset) { +std::string getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset, + int HwModeIdx) { std::string Name = "GIFBS"; for (const auto &Feature : FeatureBitset) Name += ("_" + Feature->getName()).str(); + if (HwModeIdx >= 0) + Name += ("_HwMode" + std::to_string(HwModeIdx)); return Name; } @@ -241,7 +243,7 @@ void MatchTable::emitDeclaration(raw_ostream &OS) const { if (I->Flags & MatchTableRecord::MTRF_Outdent) Indentation -= 2; } - OS << "};\n"; + OS << "}; // Size: " << (CurrentSize * 8) << " bytes\n"; } MatchTable MatchTable::buildTable(ArrayRef<Matcher *> Rules, bool WithCoverage, @@ -820,6 +822,15 @@ const OperandMatcher &RuleMatcher::getPhysRegOperandMatcher(Record *Reg) const { return *I->second; } +OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) { + const auto &I = DefinedOperands.find(Name); + + if (I == DefinedOperands.end()) + PrintFatalError(SrcLoc, "Operand " + Name + " was not declared in matcher"); + + return *I->second; +} + const OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) const { const auto &I = DefinedOperands.find(Name); @@ -851,9 +862,10 @@ void RuleMatcher::emit(MatchTable &Table) { << MatchTable::Comment(("Rule ID " + Twine(RuleID) + " //").str()) << MatchTable::LineBreak; - if (!RequiredFeatures.empty()) { + if (!RequiredFeatures.empty() || HwModeIdx >= 0) { Table << MatchTable::Opcode("GIM_CheckFeatures") - << MatchTable::NamedValue(getNameForFeatureBitset(RequiredFeatures)) + << MatchTable::NamedValue( + getNameForFeatureBitset(RequiredFeatures, HwModeIdx)) << MatchTable::LineBreak; } @@ -866,6 +878,10 @@ void RuleMatcher::emit(MatchTable &Table) { Matchers.front()->emitPredicateOpcodes(Table, *this); + // Check if it's safe to replace registers. + for (const auto &MA : Actions) + MA->emitAdditionalPredicates(Table, *this); + // We must also check if it's safe to fold the matched instructions. if (InsnVariableIDs.size() >= 2) { // Invert the map to create stable ordering (by var names) @@ -1074,6 +1090,17 @@ void RecordNamedOperandMatcher::emitPredicateOpcodes(MatchTable &Table, << MatchTable::Comment("Name : " + Name) << MatchTable::LineBreak; } +//===- RecordRegisterType ------------------------------------------===// + +void RecordRegisterType::emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const { + assert(Idx < 0 && "Temp types always have negative indexes!"); + Table << MatchTable::Opcode("GIM_RecordRegType") << MatchTable::Comment("MI") + << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op") + << MatchTable::IntValue(OpIdx) << MatchTable::Comment("TempTypeIdx") + << MatchTable::IntValue(Idx) << MatchTable::LineBreak; +} + //===- ComplexPatternOperandMatcher ---------------------------------------===// void ComplexPatternOperandMatcher::emitPredicateOpcodes( @@ -1104,7 +1131,7 @@ void RegisterBankOperandMatcher::emitPredicateOpcodes(MatchTable &Table, << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) << MatchTable::Comment("RC") - << MatchTable::NamedValue(RC.getQualifiedName() + "RegClassID") + << MatchTable::NamedValue(RC.getQualifiedIdName()) << MatchTable::LineBreak; } @@ -1189,6 +1216,18 @@ std::string OperandMatcher::getOperandExpr(unsigned InsnVarID) const { unsigned OperandMatcher::getInsnVarID() const { return Insn.getInsnVarID(); } +TempTypeIdx OperandMatcher::getTempTypeIdx(RuleMatcher &Rule) { + if (TTIdx >= 0) { + // Temp type index not assigned yet, so assign one and add the necessary + // predicate. + TTIdx = Rule.getNextTempTypeIdx(); + assert(TTIdx < 0); + addPredicate<RecordRegisterType>(TTIdx); + return TTIdx; + } + return TTIdx; +} + void OperandMatcher::emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule) { if (!Optimized) { @@ -1502,6 +1541,24 @@ void GenericInstructionPredicateMatcher::emitPredicateOpcodes( << MatchTable::LineBreak; } +//===- MIFlagsInstructionPredicateMatcher ---------------------------------===// + +bool MIFlagsInstructionPredicateMatcher::isIdentical( + const PredicateMatcher &B) const { + if (!InstructionPredicateMatcher::isIdentical(B)) + return false; + const auto &Other = + static_cast<const MIFlagsInstructionPredicateMatcher &>(B); + return Flags == Other.Flags && CheckNot == Other.CheckNot; +} + +void MIFlagsInstructionPredicateMatcher::emitPredicateOpcodes( + MatchTable &Table, RuleMatcher &Rule) const { + Table << MatchTable::Opcode(CheckNot ? "GIM_MIFlagsNot" : "GIM_MIFlags") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::NamedValue(join(Flags, " | ")) << MatchTable::LineBreak; +} + //===- InstructionMatcher -------------------------------------------------===// OperandMatcher & @@ -1917,6 +1974,30 @@ void BuildMIAction::chooseInsnToMutate(RuleMatcher &Rule) { void BuildMIAction::emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const { + const auto AddMIFlags = [&]() { + for (const InstructionMatcher *IM : CopiedFlags) { + Table << MatchTable::Opcode("GIR_CopyMIFlags") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(IM->getInsnVarID()) + << MatchTable::LineBreak; + } + + if (!SetFlags.empty()) { + Table << MatchTable::Opcode("GIR_SetMIFlags") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::NamedValue(join(SetFlags, " | ")) + << MatchTable::LineBreak; + } + + if (!UnsetFlags.empty()) { + Table << MatchTable::Opcode("GIR_UnsetMIFlags") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::NamedValue(join(UnsetFlags, " | ")) + << MatchTable::LineBreak; + } + }; + if (Matched) { assert(canMutate(Rule, Matched) && "Arranged to mutate an insn that isn't mutatable"); @@ -1935,9 +2016,12 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table, auto Namespace = Def->getValue("Namespace") ? Def->getValueAsString("Namespace") : ""; + const bool IsDead = DeadImplicitDefs.contains(Def); Table << MatchTable::Opcode("GIR_AddImplicitDef") << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) << MatchTable::NamedValue(Namespace, Def->getName()) + << (IsDead ? MatchTable::NamedValue("RegState", "Dead") + : MatchTable::IntValue(0)) << MatchTable::LineBreak; } for (auto *Use : I->ImplicitUses) { @@ -1950,6 +2034,8 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table, << MatchTable::LineBreak; } } + + AddMIFlags(); return; } @@ -1963,6 +2049,19 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table, for (const auto &Renderer : OperandRenderers) Renderer->emitRenderOpcodes(Table, Rule); + for (auto [OpIdx, Def] : enumerate(I->ImplicitDefs)) { + auto Namespace = + Def->getValue("Namespace") ? Def->getValueAsString("Namespace") : ""; + if (DeadImplicitDefs.contains(Def)) { + Table + << MatchTable::Opcode("GIR_SetImplicitDefDead") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment( + ("OpIdx for " + Namespace + "::" + Def->getName() + "").str()) + << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak; + } + } + if (I->mayLoad || I->mayStore) { Table << MatchTable::Opcode("GIR_MergeMemOperands") << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) @@ -1984,13 +2083,74 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table, << MatchTable::LineBreak; } + AddMIFlags(); + // FIXME: This is a hack but it's sufficient for ISel. We'll need to do // better for combines. Particularly when there are multiple match // roots. if (InsnID == 0) - Table << MatchTable::Opcode("GIR_EraseFromParent") - << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + EraseInstAction::emitActionOpcodes(Table, Rule, /*InsnID*/ 0); +} + +//===- BuildConstantAction ------------------------------------------------===// + +void BuildConstantAction::emitActionOpcodes(MatchTable &Table, + RuleMatcher &Rule) const { + Table << MatchTable::Opcode("GIR_BuildConstant") + << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID) + << MatchTable::Comment("Val") << MatchTable::IntValue(Val) + << MatchTable::LineBreak; +} + +//===- EraseInstAction ----------------------------------------------------===// + +void EraseInstAction::emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule, + unsigned InsnID) { + // Avoid erasing the same inst twice. + if (!Rule.tryEraseInsnID(InsnID)) + return; + + Table << MatchTable::Opcode("GIR_EraseFromParent") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::LineBreak; +} + +void EraseInstAction::emitActionOpcodes(MatchTable &Table, + RuleMatcher &Rule) const { + emitActionOpcodes(Table, Rule, InsnID); +} + +//===- ReplaceRegAction ---------------------------------------------------===// + +void ReplaceRegAction::emitAdditionalPredicates(MatchTable &Table, + RuleMatcher &Rule) const { + if (TempRegID != (unsigned)-1) + return; + + Table << MatchTable::Opcode("GIM_CheckCanReplaceReg") + << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID) + << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx) + << MatchTable::Comment("NewInsnId") << MatchTable::IntValue(NewInsnId) + << MatchTable::Comment("NewOpIdx") << MatchTable::IntValue(NewOpIdx) + << MatchTable::LineBreak; +} + +void ReplaceRegAction::emitActionOpcodes(MatchTable &Table, + RuleMatcher &Rule) const { + if (TempRegID != (unsigned)-1) { + Table << MatchTable::Opcode("GIR_ReplaceRegWithTempReg") + << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID) + << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx) + << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID) << MatchTable::LineBreak; + } else { + Table << MatchTable::Opcode("GIR_ReplaceReg") + << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID) + << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx) + << MatchTable::Comment("NewInsnId") << MatchTable::IntValue(NewInsnId) + << MatchTable::Comment("NewOpIdx") << MatchTable::IntValue(NewOpIdx) + << MatchTable::LineBreak; + } } //===- ConstrainOperandToRegClassAction -----------------------------------===// @@ -2000,7 +2160,7 @@ void ConstrainOperandToRegClassAction::emitActionOpcodes( Table << MatchTable::Opcode("GIR_ConstrainOperandRC") << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) - << MatchTable::NamedValue(RC.getQualifiedName() + "RegClassID") + << MatchTable::NamedValue(RC.getQualifiedIdName()) << MatchTable::LineBreak; } @@ -2010,9 +2170,7 @@ void MakeTempRegisterAction::emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const { Table << MatchTable::Opcode("GIR_MakeTempReg") << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID) - << MatchTable::Comment("TypeID") - << MatchTable::NamedValue(Ty.getCxxEnumValue()) - << MatchTable::LineBreak; + << MatchTable::Comment("TypeID") << Ty << MatchTable::LineBreak; } } // namespace gi diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/GlobalISelMatchTable.h index fcb3392226c1..469390d73123 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.h +++ b/llvm/utils/TableGen/GlobalISelMatchTable.h @@ -59,7 +59,8 @@ using GISelFlags = std::uint16_t; //===- Helper functions ---------------------------------------------------===// -std::string getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset); +std::string getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset, + int HwModeIdx); /// Takes a sequence of \p Rules and group them based on the predicates /// they share. \p MatcherStorage is used as a memory container @@ -272,6 +273,40 @@ extern std::set<LLTCodeGen> KnownTypes; /// MVTs that don't map cleanly to an LLT (e.g., iPTR, *any, ...). std::optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT); +using TempTypeIdx = int64_t; +class LLTCodeGenOrTempType { +public: + LLTCodeGenOrTempType(const LLTCodeGen &LLT) : Data(LLT) {} + LLTCodeGenOrTempType(TempTypeIdx TempTy) : Data(TempTy) {} + + bool isLLTCodeGen() const { return std::holds_alternative<LLTCodeGen>(Data); } + bool isTempTypeIdx() const { + return std::holds_alternative<TempTypeIdx>(Data); + } + + const LLTCodeGen &getLLTCodeGen() const { + assert(isLLTCodeGen()); + return std::get<LLTCodeGen>(Data); + } + + TempTypeIdx getTempTypeIdx() const { + assert(isTempTypeIdx()); + return std::get<TempTypeIdx>(Data); + } + +private: + std::variant<LLTCodeGen, TempTypeIdx> Data; +}; + +inline MatchTable &operator<<(MatchTable &Table, + const LLTCodeGenOrTempType &Ty) { + if (Ty.isLLTCodeGen()) + Table << MatchTable::NamedValue(Ty.getLLTCodeGen().getCxxEnumValue()); + else + Table << MatchTable::IntValue(Ty.getTempTypeIdx()); + return Table; +} + //===- Matchers -----------------------------------------------------------===// class Matcher { public: @@ -458,6 +493,12 @@ protected: /// ID for the next temporary register ID allocated with allocateTempRegID() unsigned NextTempRegID; + /// ID for the next recorded type. Starts at -1 and counts down. + TempTypeIdx NextTempTypeIdx = -1; + + // HwMode predicate index for this rule. -1 if no HwMode. + int HwModeIdx = -1; + /// Current GISelFlags GISelFlags Flags = 0; @@ -465,6 +506,8 @@ protected: std::vector<Record *> RequiredFeatures; std::vector<std::unique_ptr<PredicateMatcher>> EpilogueMatchers; + DenseSet<unsigned> ErasedInsnIDs; + ArrayRef<SMLoc> SrcLoc; typedef std::tuple<Record *, unsigned, unsigned> @@ -492,15 +535,25 @@ public: RuleMatcher(RuleMatcher &&Other) = default; RuleMatcher &operator=(RuleMatcher &&Other) = default; + TempTypeIdx getNextTempTypeIdx() { return NextTempTypeIdx--; } + uint64_t getRuleID() const { return RuleID; } InstructionMatcher &addInstructionMatcher(StringRef SymbolicName); void addRequiredFeature(Record *Feature); const std::vector<Record *> &getRequiredFeatures() const; + void addHwModeIdx(unsigned Idx) { HwModeIdx = Idx; } + int getHwModeIdx() const { return HwModeIdx; } + void addRequiredSimplePredicate(StringRef PredName); const std::vector<std::string> &getRequiredSimplePredicates(); + /// Attempts to mark \p ID as erased (GIR_EraseFromParent called on it). + /// If \p ID has already been erased, returns false and GIR_EraseFromParent + /// should NOT be emitted. + bool tryEraseInsnID(unsigned ID) { return ErasedInsnIDs.insert(ID).second; } + // Emplaces an action of the specified Kind at the end of the action list. // // Returns a reference to the newly created action. @@ -526,6 +579,8 @@ public: std::make_unique<Kind>(std::forward<Args>(args)...)); } + void setPermanentGISelFlags(GISelFlags V) { Flags = V; } + // Update the active GISelFlags based on the GISelFlags Record R. // A SaveAndRestore object is returned so the old GISelFlags are restored // at the end of the scope. @@ -586,6 +641,7 @@ public: } InstructionMatcher &getInstructionMatcher(StringRef SymbolicName) const; + OperandMatcher &getOperandMatcher(StringRef Name); const OperandMatcher &getOperandMatcher(StringRef Name) const; const OperandMatcher &getPhysRegOperandMatcher(Record *) const; @@ -648,6 +704,10 @@ public: } bool predicates_empty() const { return Predicates.empty(); } + template <typename Ty> bool contains() const { + return any_of(Predicates, [&](auto &P) { return isa<Ty>(P.get()); }); + } + std::unique_ptr<PredicateTy> predicates_pop_front() { std::unique_ptr<PredicateTy> Front = std::move(Predicates.front()); Predicates.pop_front(); @@ -730,6 +790,7 @@ public: IPM_VectorSplatImm, IPM_NoUse, IPM_GenericPredicate, + IPM_MIFlags, OPM_SameOperand, OPM_ComplexPattern, OPM_IntrinsicID, @@ -742,6 +803,7 @@ public: OPM_RegBank, OPM_MBB, OPM_RecordNamedOperand, + OPM_RecordRegType, }; protected: @@ -943,6 +1005,30 @@ public: RuleMatcher &Rule) const override; }; +/// Generates code to store a register operand's type into the set of temporary +/// LLTs. +class RecordRegisterType : public OperandPredicateMatcher { +protected: + TempTypeIdx Idx; + +public: + RecordRegisterType(unsigned InsnVarID, unsigned OpIdx, TempTypeIdx Idx) + : OperandPredicateMatcher(OPM_RecordRegType, InsnVarID, OpIdx), Idx(Idx) { + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_RecordRegType; + } + + bool isIdentical(const PredicateMatcher &B) const override { + return OperandPredicateMatcher::isIdentical(B) && + Idx == cast<RecordRegisterType>(&B)->Idx; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override; +}; + /// Generates code to check that an operand is a particular target constant. class ComplexPatternOperandMatcher : public OperandPredicateMatcher { protected: @@ -1149,6 +1235,8 @@ protected: /// countRendererFns(). unsigned AllocatedTemporariesBaseID; + TempTypeIdx TTIdx = 0; + public: OperandMatcher(InstructionMatcher &Insn, unsigned OpIdx, const std::string &SymbolicName, @@ -1176,6 +1264,11 @@ public: unsigned getOpIdx() const { return OpIdx; } unsigned getInsnVarID() const; + /// If this OperandMatcher has not been assigned a TempTypeIdx yet, assigns it + /// one and adds a `RecordRegisterType` predicate to this matcher. If one has + /// already been assigned, simply returns it. + TempTypeIdx getTempTypeIdx(RuleMatcher &Rule); + std::string getOperandExpr(unsigned InsnVarID) const; InstructionMatcher &getInstructionMatcher() const { return Insn; } @@ -1536,6 +1629,28 @@ public: RuleMatcher &Rule) const override; }; +class MIFlagsInstructionPredicateMatcher : public InstructionPredicateMatcher { + SmallVector<StringRef, 2> Flags; + bool CheckNot; // false = GIM_MIFlags, true = GIM_MIFlagsNot + +public: + MIFlagsInstructionPredicateMatcher(unsigned InsnVarID, + ArrayRef<StringRef> FlagsToCheck, + bool CheckNot = false) + : InstructionPredicateMatcher(IPM_MIFlags, InsnVarID), + Flags(FlagsToCheck), CheckNot(CheckNot) { + sort(Flags); + } + + static bool classof(const InstructionPredicateMatcher *P) { + return P->getKind() == IPM_MIFlags; + } + + bool isIdentical(const PredicateMatcher &B) const override; + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override; +}; + /// Generates code to check for the absence of use of the result. // TODO? Generalize this to support checking for one use. class NoUsePredicateMatcher : public InstructionPredicateMatcher { @@ -1930,23 +2045,41 @@ public: }; /// Adds a specific immediate to the instruction being built. +/// If a LLT is passed, a ConstantInt immediate is created instead. class ImmRenderer : public OperandRenderer { protected: unsigned InsnID; int64_t Imm; + std::optional<LLTCodeGenOrTempType> CImmLLT; public: ImmRenderer(unsigned InsnID, int64_t Imm) : OperandRenderer(OR_Imm), InsnID(InsnID), Imm(Imm) {} + ImmRenderer(unsigned InsnID, int64_t Imm, const LLTCodeGenOrTempType &CImmLLT) + : OperandRenderer(OR_Imm), InsnID(InsnID), Imm(Imm), CImmLLT(CImmLLT) { + if (CImmLLT.isLLTCodeGen()) + KnownTypes.insert(CImmLLT.getLLTCodeGen()); + } + static bool classof(const OperandRenderer *R) { return R->getKind() == OR_Imm; } void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { - Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID") - << MatchTable::IntValue(InsnID) << MatchTable::Comment("Imm") - << MatchTable::IntValue(Imm) << MatchTable::LineBreak; + if (CImmLLT) { + assert(Table.isCombiner() && + "ConstantInt immediate are only for combiners!"); + Table << MatchTable::Opcode("GIR_AddCImm") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("Type") << *CImmLLT + << MatchTable::Comment("Imm") << MatchTable::IntValue(Imm) + << MatchTable::LineBreak; + } else { + Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID") + << MatchTable::IntValue(InsnID) << MatchTable::Comment("Imm") + << MatchTable::IntValue(Imm) << MatchTable::LineBreak; + } } }; @@ -2051,11 +2184,34 @@ public: /// * Adding an operand to an instruction. class MatchAction { public: + enum ActionKind { + AK_DebugComment, + AK_CustomCXX, + AK_BuildMI, + AK_BuildConstantMI, + AK_EraseInst, + AK_ReplaceReg, + AK_ConstraintOpsToDef, + AK_ConstraintOpsToRC, + AK_MakeTempReg, + }; + + MatchAction(ActionKind K) : Kind(K) {} + + ActionKind getKind() const { return Kind; } + virtual ~MatchAction() {} + // Some actions may need to add extra predicates to ensure they can run. + virtual void emitAdditionalPredicates(MatchTable &Table, + RuleMatcher &Rule) const {} + /// Emit the MatchTable opcodes to implement the action. virtual void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const = 0; + +private: + ActionKind Kind; }; /// Generates a comment describing the matched rule being acted upon. @@ -2064,7 +2220,12 @@ private: std::string S; public: - DebugCommentAction(StringRef S) : S(std::string(S)) {} + DebugCommentAction(StringRef S) + : MatchAction(AK_DebugComment), S(std::string(S)) {} + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_DebugComment; + } void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { Table << MatchTable::Comment(S) << MatchTable::LineBreak; @@ -2075,7 +2236,12 @@ class CustomCXXAction : public MatchAction { std::string FnEnumName; public: - CustomCXXAction(StringRef FnEnumName) : FnEnumName(FnEnumName.str()) {} + CustomCXXAction(StringRef FnEnumName) + : MatchAction(AK_CustomCXX), FnEnumName(FnEnumName.str()) {} + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_CustomCXX; + } void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; }; @@ -2088,19 +2254,36 @@ private: const CodeGenInstruction *I; InstructionMatcher *Matched; std::vector<std::unique_ptr<OperandRenderer>> OperandRenderers; + SmallPtrSet<Record *, 4> DeadImplicitDefs; + + std::vector<const InstructionMatcher *> CopiedFlags; + std::vector<StringRef> SetFlags; + std::vector<StringRef> UnsetFlags; /// True if the instruction can be built solely by mutating the opcode. bool canMutate(RuleMatcher &Rule, const InstructionMatcher *Insn) const; public: BuildMIAction(unsigned InsnID, const CodeGenInstruction *I) - : InsnID(InsnID), I(I), Matched(nullptr) {} + : MatchAction(AK_BuildMI), InsnID(InsnID), I(I), Matched(nullptr) {} + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_BuildMI; + } unsigned getInsnID() const { return InsnID; } const CodeGenInstruction *getCGI() const { return I; } + void addSetMIFlags(StringRef Flag) { SetFlags.push_back(Flag); } + void addUnsetMIFlags(StringRef Flag) { UnsetFlags.push_back(Flag); } + void addCopiedMIFlags(const InstructionMatcher &IM) { + CopiedFlags.push_back(&IM); + } + void chooseInsnToMutate(RuleMatcher &Rule); + void setDeadImplicitDef(Record *R) { DeadImplicitDefs.insert(R); } + template <class Kind, class... Args> Kind &addRenderer(Args &&...args) { OperandRenderers.emplace_back( std::make_unique<Kind>(InsnID, std::forward<Args>(args)...)); @@ -2110,13 +2293,76 @@ public: void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; }; +/// Generates code to create a constant that defines a TempReg. +/// The instruction created is usually a G_CONSTANT but it could also be a +/// G_BUILD_VECTOR for vector types. +class BuildConstantAction : public MatchAction { + unsigned TempRegID; + int64_t Val; + +public: + BuildConstantAction(unsigned TempRegID, int64_t Val) + : MatchAction(AK_BuildConstantMI), TempRegID(TempRegID), Val(Val) {} + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_BuildConstantMI; + } + + void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; +}; + +class EraseInstAction : public MatchAction { + unsigned InsnID; + +public: + EraseInstAction(unsigned InsnID) + : MatchAction(AK_EraseInst), InsnID(InsnID) {} + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_EraseInst; + } + + void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; + static void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule, + unsigned InsnID); +}; + +class ReplaceRegAction : public MatchAction { + unsigned OldInsnID, OldOpIdx; + unsigned NewInsnId = -1, NewOpIdx; + unsigned TempRegID = -1; + +public: + ReplaceRegAction(unsigned OldInsnID, unsigned OldOpIdx, unsigned NewInsnId, + unsigned NewOpIdx) + : MatchAction(AK_EraseInst), OldInsnID(OldInsnID), OldOpIdx(OldOpIdx), + NewInsnId(NewInsnId), NewOpIdx(NewOpIdx) {} + + ReplaceRegAction(unsigned OldInsnID, unsigned OldOpIdx, unsigned TempRegID) + : MatchAction(AK_EraseInst), OldInsnID(OldInsnID), OldOpIdx(OldOpIdx), + TempRegID(TempRegID) {} + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_ReplaceReg; + } + + void emitAdditionalPredicates(MatchTable &Table, + RuleMatcher &Rule) const override; + void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; +}; + /// Generates code to constrain the operands of an output instruction to the /// register classes specified by the definition of that instruction. class ConstrainOperandsToDefinitionAction : public MatchAction { unsigned InsnID; public: - ConstrainOperandsToDefinitionAction(unsigned InsnID) : InsnID(InsnID) {} + ConstrainOperandsToDefinitionAction(unsigned InsnID) + : MatchAction(AK_ConstraintOpsToDef), InsnID(InsnID) {} + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_ConstraintOpsToDef; + } void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { Table << MatchTable::Opcode("GIR_ConstrainSelectedInstOperands") @@ -2135,7 +2381,12 @@ class ConstrainOperandToRegClassAction : public MatchAction { public: ConstrainOperandToRegClassAction(unsigned InsnID, unsigned OpIdx, const CodeGenRegisterClass &RC) - : InsnID(InsnID), OpIdx(OpIdx), RC(RC) {} + : MatchAction(AK_ConstraintOpsToRC), InsnID(InsnID), OpIdx(OpIdx), + RC(RC) {} + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_ConstraintOpsToRC; + } void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; }; @@ -2144,13 +2395,18 @@ public: /// instructions together. class MakeTempRegisterAction : public MatchAction { private: - LLTCodeGen Ty; + LLTCodeGenOrTempType Ty; unsigned TempRegID; public: - MakeTempRegisterAction(const LLTCodeGen &Ty, unsigned TempRegID) - : Ty(Ty), TempRegID(TempRegID) { - KnownTypes.insert(Ty); + MakeTempRegisterAction(const LLTCodeGenOrTempType &Ty, unsigned TempRegID) + : MatchAction(AK_MakeTempReg), Ty(Ty), TempRegID(TempRegID) { + if (Ty.isLLTCodeGen()) + KnownTypes.insert(Ty.getLLTCodeGen()); + } + + static bool classof(const MatchAction *A) { + return A->getKind() == AK_MakeTempReg; } void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override; diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp index 8dc422b140a5..c6cd3240a94e 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp @@ -15,7 +15,7 @@ using namespace llvm::gi; void GlobalISelMatchTableExecutorEmitter::emitSubtargetFeatureBitsetImpl( raw_ostream &OS, ArrayRef<RuleMatcher> Rules) { SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(SubtargetFeatures, - OS); + OS, &HwModes); // Separate subtarget features by how often they must be recomputed. SubtargetFeatureInfoMap ModuleFeatures; @@ -33,7 +33,7 @@ void GlobalISelMatchTableExecutorEmitter::emitSubtargetFeatureBitsetImpl( SubtargetFeatureInfo::emitComputeAvailableFeatures( getTarget().getName(), getClassName(), "computeAvailableModuleFeatures", - ModuleFeatures, OS); + ModuleFeatures, OS, "", &HwModes); OS << "void " << getClassName() << "::setupGeneratedPerFunctionState(MachineFunction &MF) {\n" @@ -49,24 +49,27 @@ void GlobalISelMatchTableExecutorEmitter::emitSubtargetFeatureBitsetImpl( // Emit a table containing the PredicateBitsets objects needed by the matcher // and an enum for the matcher to reference them with. - std::vector<std::vector<Record *>> FeatureBitsets; + std::vector<std::pair<std::vector<Record *>, int>> FeatureBitsets; FeatureBitsets.reserve(Rules.size()); for (auto &Rule : Rules) - FeatureBitsets.push_back(Rule.getRequiredFeatures()); - llvm::sort(FeatureBitsets, [&](const std::vector<Record *> &A, - const std::vector<Record *> &B) { - if (A.size() < B.size()) - return true; - if (A.size() > B.size()) - return false; - for (auto [First, Second] : zip(A, B)) { - if (First->getName() < Second->getName()) - return true; - if (First->getName() > Second->getName()) - return false; - } - return false; - }); + FeatureBitsets.emplace_back(Rule.getRequiredFeatures(), + Rule.getHwModeIdx()); + llvm::sort(FeatureBitsets, + [&](const std::pair<std::vector<Record *>, int> &A, + const std::pair<std::vector<Record *>, int> &B) { + if (A.first.size() < B.first.size()) + return true; + if (A.first.size() > B.first.size()) + return false; + for (auto [First, Second] : zip(A.first, B.first)) { + if (First->getName() < Second->getName()) + return true; + if (First->getName() > Second->getName()) + return false; + } + + return (A.second < B.second); + }); FeatureBitsets.erase( std::unique(FeatureBitsets.begin(), FeatureBitsets.end()), FeatureBitsets.end()); @@ -74,22 +77,28 @@ void GlobalISelMatchTableExecutorEmitter::emitSubtargetFeatureBitsetImpl( << "enum {\n" << " GIFBS_Invalid,\n"; for (const auto &FeatureBitset : FeatureBitsets) { - if (FeatureBitset.empty()) + if (FeatureBitset.first.empty() && FeatureBitset.second < 0) continue; - OS << " " << getNameForFeatureBitset(FeatureBitset) << ",\n"; + OS << " " + << getNameForFeatureBitset(FeatureBitset.first, FeatureBitset.second) + << ",\n"; } OS << "};\n" - << "const static PredicateBitset FeatureBitsets[] {\n" + << "constexpr static PredicateBitset FeatureBitsets[] {\n" << " {}, // GIFBS_Invalid\n"; for (const auto &FeatureBitset : FeatureBitsets) { - if (FeatureBitset.empty()) + if (FeatureBitset.first.empty() && FeatureBitset.second < 0) continue; OS << " {"; - for (const auto &Feature : FeatureBitset) { + for (const auto &Feature : FeatureBitset.first) { const auto &I = SubtargetFeatures.find(Feature); assert(I != SubtargetFeatures.end() && "Didn't import predicate?"); OS << I->second.getEnumBitName() << ", "; } + // HwModeIdx + if (FeatureBitset.second >= 0) { + OS << "Feature_HwMode" << FeatureBitset.second << "Bit, "; + } OS << "},\n"; } OS << "};\n\n"; @@ -184,11 +193,11 @@ void GlobalISelMatchTableExecutorEmitter::emitExecutorImpl( void GlobalISelMatchTableExecutorEmitter::emitPredicateBitset( raw_ostream &OS, StringRef IfDefName) { + unsigned Size = SubtargetFeatures.size() + HwModes.size(); OS << "#ifdef " << IfDefName << "\n" - << "const unsigned MAX_SUBTARGET_PREDICATES = " << SubtargetFeatures.size() - << ";\n" + << "const unsigned MAX_SUBTARGET_PREDICATES = " << Size << ";\n" << "using PredicateBitset = " - "llvm::PredicateBitsetImpl<MAX_SUBTARGET_PREDICATES>;\n" + "llvm::Bitset<MAX_SUBTARGET_PREDICATES>;\n" << "#endif // ifdef " << IfDefName << "\n\n"; } @@ -222,7 +231,8 @@ void GlobalISelMatchTableExecutorEmitter::emitTemporariesDecl( ", const MatcherState &State) " "const override;\n" << " bool testSimplePredicate(unsigned PredicateID) const override;\n" - << " void runCustomAction(unsigned FnID, const MatcherState &State) " + << " void runCustomAction(unsigned FnID, const MatcherState &State, " + "NewMIVector &OutMIs) " "const override;\n"; emitAdditionalTemporariesDecl(OS, " "); OS << "#endif // ifdef " << IfDefName << "\n\n"; diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h index d526e08a96e3..7e952d6df309 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h +++ b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h @@ -20,7 +20,6 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include <functional> -#include <vector> namespace llvm { class CodeGenTarget; @@ -105,11 +104,12 @@ class GlobalISelMatchTableExecutorEmitter { if (!Predicates.empty()) { OS << " switch (PredicateID) {\n"; for (const auto &Pred : Predicates) { - const auto Code = GetPredCode(Pred); + // Ensure all code is indented. + const auto Code = join(split(GetPredCode(Pred).str(), "\n"), "\n "); OS << " case GICXXPred_" << TypeIdentifier << "_Predicate_" << GetPredEnumName(Pred) << ": {\n" << " " << Code << "\n"; - if (!StringRef(Code).ltrim().startswith("return")) { + if (!StringRef(Code).ltrim().starts_with("return")) { OS << " llvm_unreachable(\"" << GetPredEnumName(Pred) << " should have returned\");\n"; } @@ -222,6 +222,8 @@ public: // Map of predicates to their subtarget features. SubtargetFeatureInfoMap SubtargetFeatures; + + std::map<std::string, unsigned> HwModes; }; } // namespace llvm diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/InfoByHwMode.cpp index 4e9136e936af..7e4ab5346621 100644 --- a/llvm/utils/TableGen/InfoByHwMode.cpp +++ b/llvm/utils/TableGen/InfoByHwMode.cpp @@ -35,6 +35,8 @@ ValueTypeByHwMode::ValueTypeByHwMode(Record *R, const CodeGenHwModes &CGH) { assert(I.second && "Duplicate entry?"); (void)I; } + if (R->isSubClassOf("PtrValueType")) + PtrAddrSpace = R->getValueAsInt("AddrSpace"); } ValueTypeByHwMode::ValueTypeByHwMode(Record *R, MVT T) : ValueTypeByHwMode(T) { diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/InfoByHwMode.h index b8a6645baca5..a9295f6dc342 100644 --- a/llvm/utils/TableGen/InfoByHwMode.h +++ b/llvm/utils/TableGen/InfoByHwMode.h @@ -15,7 +15,6 @@ #define LLVM_UTILS_TABLEGEN_INFOBYHWMODE_H #include "CodeGenHwModes.h" -#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineValueType.h" diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp index cab9ecd4ea97..b2250c0cf989 100644 --- a/llvm/utils/TableGen/InstrInfoEmitter.cpp +++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp @@ -22,6 +22,7 @@ #include "Types.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Support/Casting.h" #include "llvm/Support/raw_ostream.h" @@ -474,7 +475,7 @@ void InstrInfoEmitter::emitOperandTypeMappings( OS << "LLVM_READONLY\n"; OS << "static int getMemOperandSize(int OpType) {\n"; OS << " switch (OpType) {\n"; - std::map<int, std::vector<StringRef>> SizeToOperandName; + std::map<int, SmallVector<StringRef, 0>> SizeToOperandName; for (const Record *Op : Operands) { if (!Op->isSubClassOf("X86MemOperand")) continue; @@ -482,7 +483,7 @@ void InstrInfoEmitter::emitOperandTypeMappings( SizeToOperandName[Size].push_back(Op->getName()); } OS << " default: return 0;\n"; - for (auto KV : SizeToOperandName) { + for (const auto &KV : SizeToOperandName) { for (const StringRef &OperandName : KV.second) OS << " case OpTypes::" << OperandName << ":\n"; OS << " return " << KV.first << ";\n\n"; @@ -733,7 +734,9 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS, std::map<Record *, SubtargetFeatureInfo, LessRecordByID> SubtargetFeatures; SubtargetFeatures.insert(All.begin(), All.end()); - OS << "#if defined(ENABLE_INSTR_PREDICATE_VERIFIER) && !defined(NDEBUG)\n" + OS << "#if (defined(ENABLE_INSTR_PREDICATE_VERIFIER) && !defined(NDEBUG)) " + << "||\\\n" + << " defined(GET_AVAILABLE_OPCODE_CHECKER)\n" << "#define GET_COMPUTE_FEATURES\n" << "#endif\n"; OS << "#ifdef GET_COMPUTE_FEATURES\n" @@ -799,7 +802,7 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS, OS << "},\n"; } OS << " };\n" - << " static " << getMinimalTypeForRange(FeatureBitsets.size()) + << " static constexpr " << getMinimalTypeForRange(FeatureBitsets.size()) << " RequiredFeaturesRefs[] = {\n"; unsigned InstIdx = 0; for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) { @@ -826,6 +829,25 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS, << "} // end namespace llvm\n" << "#endif // GET_COMPUTE_FEATURES\n\n"; + OS << "#ifdef GET_AVAILABLE_OPCODE_CHECKER\n" + << "#undef GET_AVAILABLE_OPCODE_CHECKER\n" + << "namespace llvm {\n" + << "namespace " << Target.getName() << "_MC {\n"; + OS << "bool isOpcodeAvailable(" + << "unsigned Opcode, const FeatureBitset &Features) {\n" + << " FeatureBitset AvailableFeatures = " + << "computeAvailableFeatures(Features);\n" + << " FeatureBitset RequiredFeatures = " + << "computeRequiredFeatures(Opcode);\n" + << " FeatureBitset MissingFeatures =\n" + << " (AvailableFeatures & RequiredFeatures) ^\n" + << " RequiredFeatures;\n" + << " return !MissingFeatures.any();\n" + << "}\n"; + OS << "} // end namespace " << Target.getName() << "_MC\n" + << "} // end namespace llvm\n" + << "#endif // GET_AVAILABLE_OPCODE_CHECKER\n\n"; + OS << "#ifdef ENABLE_INSTR_PREDICATE_VERIFIER\n" << "#undef ENABLE_INSTR_PREDICATE_VERIFIER\n" << "#include <sstream>\n\n"; diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp index 09aad78536fe..28604c5600bf 100644 --- a/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -12,7 +12,6 @@ #include "CodeGenIntrinsics.h" #include "SequenceToOffsetTable.h" -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" @@ -214,7 +213,7 @@ void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints, << " size_t Count;\n" << "};\n"; OS << "static constexpr IntrinsicTargetInfo TargetInfos[] = {\n"; - for (auto Target : Ints.Targets) + for (const auto &Target : Ints.Targets) OS << " {llvm::StringLiteral(\"" << Target.Name << "\"), " << Target.Offset << ", " << Target.Count << "},\n"; OS << "};\n"; diff --git a/llvm/utils/TableGen/OptParserEmitter.cpp b/llvm/utils/TableGen/OptParserEmitter.cpp index a04680b5d91e..257cd44d9e80 100644 --- a/llvm/utils/TableGen/OptParserEmitter.cpp +++ b/llvm/utils/TableGen/OptParserEmitter.cpp @@ -34,30 +34,14 @@ static raw_ostream &write_cstring(raw_ostream &OS, llvm::StringRef Str) { return OS; } -static std::string getOptionSpelling(const Record &R, size_t &PrefixLength) { +static std::string getOptionPrefixedName(const Record &R) { std::vector<StringRef> Prefixes = R.getValueAsListOfStrings("Prefixes"); StringRef Name = R.getValueAsString("Name"); - if (Prefixes.empty()) { - PrefixLength = 0; + if (Prefixes.empty()) return Name.str(); - } - - PrefixLength = Prefixes[0].size(); - return (Twine(Prefixes[0]) + Twine(Name)).str(); -} -static std::string getOptionSpelling(const Record &R) { - size_t PrefixLength; - return getOptionSpelling(R, PrefixLength); -} - -static void emitNameUsingSpelling(raw_ostream &OS, const Record &R) { - size_t PrefixLength; - OS << "llvm::StringLiteral("; - write_cstring( - OS, StringRef(getOptionSpelling(R, PrefixLength)).substr(PrefixLength)); - OS << ")"; + return (Prefixes[0] + Twine(Name)).str(); } class MarshallingInfo { @@ -105,8 +89,6 @@ struct SimpleEnumValueTable { } void emit(raw_ostream &OS) const { - write_cstring(OS, StringRef(getOptionSpelling(R))); - OS << ", "; OS << ShouldParse; OS << ", "; OS << ShouldAlwaysEmit; @@ -320,7 +302,7 @@ static void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) { OS << "INVALID"; // The other option arguments (unused for groups). - OS << ", INVALID, nullptr, 0, 0"; + OS << ", INVALID, nullptr, 0, 0, 0"; // The option help text. if (!isa<UnsetInit>(R.getValueInit("HelpText"))) { @@ -346,8 +328,8 @@ static void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) { std::vector<StringRef> RPrefixes = R.getValueAsListOfStrings("Prefixes"); OS << Prefixes[PrefixKeyT(RPrefixes.begin(), RPrefixes.end())] << ", "; - // The option string. - emitNameUsingSpelling(OS, R); + // The option prefixed name. + write_cstring(OS, getOptionPrefixedName(R)); // The option identifier name. OS << ", " << getOptionName(R); @@ -358,8 +340,10 @@ static void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) { // The containing option group (if any). OS << ", "; const ListInit *GroupFlags = nullptr; + const ListInit *GroupVis = nullptr; if (const DefInit *DI = dyn_cast<DefInit>(R.getValueInit("Group"))) { GroupFlags = DI->getDef()->getValueAsListInit("Flags"); + GroupVis = DI->getDef()->getValueAsListInit("Visibility"); OS << getOptionName(*DI->getDef()); } else OS << "INVALID"; @@ -386,7 +370,7 @@ static void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) { OS << "\""; } - // The option flags. + // "Flags" for the option, such as HelpHidden and Render* OS << ", "; int NumFlags = 0; const ListInit *LI = R.getValueAsListInit("Flags"); @@ -400,6 +384,21 @@ static void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) { if (NumFlags == 0) OS << '0'; + // Option visibility, for sharing options between drivers. + OS << ", "; + int NumVisFlags = 0; + LI = R.getValueAsListInit("Visibility"); + for (Init *I : *LI) + OS << (NumVisFlags++ ? " | " : "") + << cast<DefInit>(I)->getDef()->getName(); + if (GroupVis) { + for (Init *I : *GroupVis) + OS << (NumVisFlags++ ? " | " : "") + << cast<DefInit>(I)->getDef()->getName(); + } + if (NumVisFlags == 0) + OS << '0'; + // The option parameter field. OS << ", " << R.getValueAsInt("NumArgs"); diff --git a/llvm/utils/TableGen/OptRSTEmitter.cpp b/llvm/utils/TableGen/OptRSTEmitter.cpp index 87e755d943a1..5a7f079dc168 100644 --- a/llvm/utils/TableGen/OptRSTEmitter.cpp +++ b/llvm/utils/TableGen/OptRSTEmitter.cpp @@ -91,7 +91,7 @@ static void EmitOptRST(RecordKeeper &Records, raw_ostream &OS) { HelpText += join(Values.begin(), Values.end() - 1, "', '"); HelpText += "' or '"; } - HelpText += (Values.front() + "'.").str(); + HelpText += (Values.back() + "'.").str(); } if (!HelpText.empty()) { diff --git a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp index 12174fd83f56..7a6439cb9491 100644 --- a/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp +++ b/llvm/utils/TableGen/RISCVTargetDefEmitter.cpp @@ -49,7 +49,7 @@ static std::string getMArch(const Record &Rec) { static void EmitRISCVTargetDef(RecordKeeper &RK, raw_ostream &OS) { OS << "#ifndef PROC\n" - << "#define PROC(ENUM, NAME, DEFAULT_MARCH)\n" + << "#define PROC(ENUM, NAME, DEFAULT_MARCH, FAST_UNALIGNED_ACCESS)\n" << "#endif\n\n"; // Iterate on all definition records. @@ -60,9 +60,14 @@ static void EmitRISCVTargetDef(RecordKeeper &RK, raw_ostream &OS) { if (MArch.empty()) MArch = getMArch(*Rec); + const bool FastUnalignedAccess = + any_of(Rec->getValueAsListOfDefs("Features"), [&](auto &Feature) { + return Feature->getValueAsString("Name") == "fast-unaligned-access"; + }); + OS << "PROC(" << Rec->getName() << ", " << "{\"" << Rec->getValueAsString("Name") << "\"}, " - << "{\"" << MArch << "\"})\n"; + << "{\"" << MArch << "\"}, " << FastUnalignedAccess << ")\n"; } OS << "\n#undef PROC\n"; OS << "\n"; diff --git a/llvm/utils/TableGen/RegisterBankEmitter.cpp b/llvm/utils/TableGen/RegisterBankEmitter.cpp index 2d23bf86b6ad..f851d9a79870 100644 --- a/llvm/utils/TableGen/RegisterBankEmitter.cpp +++ b/llvm/utils/TableGen/RegisterBankEmitter.cpp @@ -231,9 +231,7 @@ void RegisterBankEmitter::emitBaseClassImplementation( for (const auto &RCs : RCsGroupedByWord) { OS << " // " << LowestIdxInWord << "-" << (LowestIdxInWord + 31) << "\n"; for (const auto &RC : RCs) { - std::string QualifiedRegClassID = - (Twine(RC->Namespace) + "::" + RC->getName() + "RegClassID").str(); - OS << " (1u << (" << QualifiedRegClassID << " - " + OS << " (1u << (" << RC->getQualifiedIdName() << " - " << LowestIdxInWord << ")) |\n"; } OS << " 0,\n"; @@ -246,7 +244,7 @@ void RegisterBankEmitter::emitBaseClassImplementation( for (const auto &Bank : Banks) { std::string QualifiedBankID = (TargetName + "::" + Bank.getEnumeratorName()).str(); - OS << "const RegisterBank " << Bank.getInstanceVarName() << "(/* ID */ " + OS << "constexpr RegisterBank " << Bank.getInstanceVarName() << "(/* ID */ " << QualifiedBankID << ", /* Name */ \"" << Bank.getName() << "\", " << "/* CoveredRegClasses */ " << Bank.getCoverageArrayName() << ", /* NumRegClasses */ " diff --git a/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/llvm/utils/TableGen/RegisterInfoEmitter.cpp index 3101081114fb..92ef9199cc47 100644 --- a/llvm/utils/TableGen/RegisterInfoEmitter.cpp +++ b/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -146,8 +146,7 @@ void RegisterInfoEmitter::runEnums(raw_ostream &OS, OS << "namespace " << Namespace << " {\n"; OS << "enum {\n"; for (const auto &RC : RegisterClasses) - OS << " " << RC.getName() << "RegClassID" - << " = " << RC.EnumValue << ",\n"; + OS << " " << RC.getIdName() << " = " << RC.EnumValue << ",\n"; OS << "\n};\n"; if (!Namespace.empty()) OS << "} // end namespace " << Namespace << "\n\n"; @@ -932,12 +931,6 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, MaskVec &LaneMaskVec = RegUnitLaneMasks[i]; assert(LaneMaskVec.empty()); llvm::append_range(LaneMaskVec, RUMasks); - // Terminator mask should not be used inside of the list. -#ifndef NDEBUG - for (LaneBitmask M : LaneMaskVec) { - assert(!M.all() && "terminator mask should not be part of the list"); - } -#endif LaneMaskSeqs.add(LaneMaskVec); } @@ -957,6 +950,8 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, // Emit the shared table of regunit lane mask sequences. OS << "extern const LaneBitmask " << TargetName << "LaneMaskLists[] = {\n"; + // TODO: Omit the terminator since it is never used. The length of this list + // is known implicitly from the corresponding reg unit list. LaneMaskSeqs.emit(OS, printMask, "LaneBitmask::getAll()"); OS << "};\n\n"; @@ -1072,8 +1067,8 @@ RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, RegSize = RC.RSI.getSimple().RegSize; OS << " { " << RCName << ", " << RCBitsName << ", " << RegClassStrings.get(RC.getName()) << ", " << RC.getOrder().size() - << ", " << RCBitsSize << ", " << RC.getQualifiedName() + "RegClassID" - << ", " << RegSize << ", " << RC.CopyCost << ", " + << ", " << RCBitsSize << ", " << RC.getQualifiedIdName() << ", " + << RegSize << ", " << RC.CopyCost << ", " << (RC.Allocatable ? "true" : "false") << " },\n"; } @@ -1295,7 +1290,7 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, for (const ValueTypeByHwMode &VVT : RC.VTs) if (VVT.hasDefault() || VVT.hasMode(M)) VTs.push_back(VVT.get(M).SimpleTy); - OS << ", VTLists+" << VTSeqs.get(VTs) << " }, // " + OS << ", /*VTLists+*/" << VTSeqs.get(VTs) << " }, // " << RC.getName() << '\n'; } } @@ -1591,8 +1586,8 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, BaseClasses.push_back(&RC); } if (!BaseClasses.empty()) { - // Represent class indexes with uint8_t and allocate one index for nullptr - assert(BaseClasses.size() <= UINT8_MAX && "Too many base register classes"); + assert(BaseClasses.size() < UINT16_MAX && + "Too many base register classes"); // Apply order struct BaseClassOrdering { @@ -1603,30 +1598,34 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, }; llvm::stable_sort(BaseClasses, BaseClassOrdering()); - // Build mapping for Regs (+1 for NoRegister) - std::vector<uint8_t> Mapping(Regs.size() + 1, 0); - for (int RCIdx = BaseClasses.size() - 1; RCIdx >= 0; --RCIdx) { - for (const auto Reg : BaseClasses[RCIdx]->getMembers()) - Mapping[Reg->EnumValue] = RCIdx + 1; - } - OS << "\n// Register to base register class mapping\n\n"; OS << "\n"; OS << "const TargetRegisterClass *" << ClassName << "::getPhysRegBaseClass(MCRegister Reg)" << " const {\n"; - OS << " static const TargetRegisterClass *BaseClasses[" << (BaseClasses.size() + 1) << "] = {\n"; - OS << " nullptr,\n"; - for (const auto RC : BaseClasses) - OS << " &" << RC->getQualifiedName() << "RegClass,\n"; - OS << " };\n"; - OS << " static const uint8_t Mapping[" << Mapping.size() << "] = {\n "; - for (const uint8_t Value : Mapping) - OS << (unsigned)Value << ","; - OS << " };\n\n"; - OS << " assert(Reg < sizeof(Mapping));\n"; - OS << " return BaseClasses[Mapping[Reg]];\n"; - OS << "}\n"; + OS << " static const uint16_t InvalidRegClassID = UINT16_MAX;\n\n"; + OS << " static const uint16_t Mapping[" << Regs.size() + 1 << "] = {\n"; + OS << " InvalidRegClassID, // NoRegister\n"; + for (const CodeGenRegister &Reg : Regs) { + const CodeGenRegisterClass *BaseRC = nullptr; + for (const CodeGenRegisterClass *RC : BaseClasses) { + if (is_contained(RC->getMembers(), &Reg)) { + BaseRC = RC; + break; + } + } + + OS << " " + << (BaseRC ? BaseRC->getQualifiedIdName() : "InvalidRegClassID") + << ", // " << Reg.getName() << "\n"; + } + OS << " };\n\n" + " assert(Reg < ArrayRef(Mapping).size());\n" + " unsigned RCID = Mapping[Reg];\n" + " if (RCID == InvalidRegClassID)\n" + " return nullptr;\n" + " return RegisterClasses[RCID];\n" + "}\n"; } } @@ -1653,7 +1652,7 @@ RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, << " SubRegIndexNameTable, SubRegIndexLaneMaskTable,\n" << " "; printMask(OS, RegBank.CoveringLanes); - OS << ", RegClassInfos, HwMode) {\n" + OS << ", RegClassInfos, VTLists, HwMode) {\n" << " InitMCRegisterInfo(" << TargetName << "RegDesc, " << Regs.size() + 1 << ", RA, PC,\n " << TargetName << "MCRegisterClasses, " << RegisterClasses.size() << ",\n" diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp index b6af02c28a80..9987d1ec73d9 100644 --- a/llvm/utils/TableGen/SearchableTableEmitter.cpp +++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp @@ -31,12 +31,12 @@ using namespace llvm; namespace { -int getAsInt(Init *B) { +int64_t getAsInt(Init *B) { return cast<IntInit>( B->convertInitializerTo(IntRecTy::get(B->getRecordKeeper()))) ->getValue(); } -int getInt(Record *R, StringRef Field) { +int64_t getInt(Record *R, StringRef Field) { return getAsInt(R->getValueInit(Field)); } @@ -720,7 +720,23 @@ void SearchableTableEmitter::run(raw_ostream &OS) { Twine("Table FilterClass '") + FilterClass + "' does not exist"); - collectTableEntries(*Table, Records.getAllDerivedDefinitions(FilterClass)); + RecordVal *FilterClassFieldVal = TableRec->getValue("FilterClassField"); + std::vector<Record *> Definitions = + Records.getAllDerivedDefinitions(FilterClass); + if (auto *FilterClassFieldInit = + dyn_cast<StringInit>(FilterClassFieldVal->getValue())) { + StringRef FilterClassField = FilterClassFieldInit->getValue(); + llvm::erase_if(Definitions, [&](const Record *R) { + const RecordVal *Filter = R->getValue(FilterClassField); + if (auto *BitV = dyn_cast<BitInit>(Filter->getValue())) + return !BitV->getValue(); + + PrintFatalError(Filter, Twine("FilterClassField '") + FilterClass + + "' should be a bit value"); + return true; + }); + } + collectTableEntries(*Table, Definitions); if (!TableRec->isValueUnset("PrimaryKey")) { Table->PrimaryKey = diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp index e4eb23649e96..f7a7172d61fc 100644 --- a/llvm/utils/TableGen/SubtargetEmitter.cpp +++ b/llvm/utils/TableGen/SubtargetEmitter.cpp @@ -119,8 +119,8 @@ class SubtargetEmitter { const CodeGenProcModel &ProcModel); Record *FindReadAdvance(const CodeGenSchedRW &SchedRead, const CodeGenProcModel &ProcModel); - void ExpandProcResources(RecVec &PRVec, std::vector<int64_t> &Cycles, - std::vector<int64_t> &StartAtCycles, + void ExpandProcResources(RecVec &PRVec, std::vector<int64_t> &ReleaseAtCycles, + std::vector<int64_t> &AcquireAtCycles, const CodeGenProcModel &ProcModel); void GenSchedClassTables(const CodeGenProcModel &ProcModel, SchedClassTables &SchedTables); @@ -976,11 +976,10 @@ Record *SubtargetEmitter::FindReadAdvance(const CodeGenSchedRW &SchedRead, // Expand an explicit list of processor resources into a full list of implied // resource groups and super resources that cover them. -void SubtargetEmitter::ExpandProcResources(RecVec &PRVec, - std::vector<int64_t> &Cycles, - std::vector<int64_t> &StartAtCycles, - const CodeGenProcModel &PM) { - assert(PRVec.size() == Cycles.size() && "failed precondition"); +void SubtargetEmitter::ExpandProcResources( + RecVec &PRVec, std::vector<int64_t> &ReleaseAtCycles, + std::vector<int64_t> &AcquireAtCycles, const CodeGenProcModel &PM) { + assert(PRVec.size() == ReleaseAtCycles.size() && "failed precondition"); for (unsigned i = 0, e = PRVec.size(); i != e; ++i) { Record *PRDef = PRVec[i]; RecVec SubResources; @@ -1000,8 +999,8 @@ void SubtargetEmitter::ExpandProcResources(RecVec &PRVec, SchedModels.findProcResUnits(SubDef->getValueAsDef("Super"), PM, SubDef->getLoc()); PRVec.push_back(SuperDef); - Cycles.push_back(Cycles[i]); - StartAtCycles.push_back(StartAtCycles[i]); + ReleaseAtCycles.push_back(ReleaseAtCycles[i]); + AcquireAtCycles.push_back(AcquireAtCycles[i]); SubDef = SuperDef; } } @@ -1017,8 +1016,8 @@ void SubtargetEmitter::ExpandProcResources(RecVec &PRVec, } if (SubI == SubE) { PRVec.push_back(PR); - Cycles.push_back(Cycles[i]); - StartAtCycles.push_back(StartAtCycles[i]); + ReleaseAtCycles.push_back(ReleaseAtCycles[i]); + AcquireAtCycles.push_back(AcquireAtCycles[i]); } } } @@ -1150,67 +1149,69 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel, // Create an entry for each ProcResource listed in WriteRes. RecVec PRVec = WriteRes->getValueAsListOfDefs("ProcResources"); - std::vector<int64_t> Cycles = - WriteRes->getValueAsListOfInts("ResourceCycles"); + std::vector<int64_t> ReleaseAtCycles = + WriteRes->getValueAsListOfInts("ReleaseAtCycles"); - std::vector<int64_t> StartAtCycles = - WriteRes->getValueAsListOfInts("StartAtCycles"); + std::vector<int64_t> AcquireAtCycles = + WriteRes->getValueAsListOfInts("AcquireAtCycles"); // Check consistency of the two vectors carrying the start and // stop cycles of the resources. - if (!Cycles.empty() && Cycles.size() != PRVec.size()) { - // If ResourceCycles is provided, check consistency. + if (!ReleaseAtCycles.empty() && + ReleaseAtCycles.size() != PRVec.size()) { + // If ReleaseAtCycles is provided, check consistency. PrintFatalError( WriteRes->getLoc(), - Twine("Inconsistent resource cycles: size(ResourceCycles) != " + Twine("Inconsistent release at cycles: size(ReleaseAtCycles) != " "size(ProcResources): ") .concat(Twine(PRVec.size())) .concat(" vs ") - .concat(Twine(Cycles.size()))); + .concat(Twine(ReleaseAtCycles.size()))); } - if (!StartAtCycles.empty() && StartAtCycles.size() != PRVec.size()) { + if (!AcquireAtCycles.empty() && AcquireAtCycles.size() != PRVec.size()) { PrintFatalError( WriteRes->getLoc(), - Twine("Inconsistent resource cycles: size(StartAtCycles) != " + Twine("Inconsistent resource cycles: size(AcquireAtCycles) != " "size(ProcResources): ") - .concat(Twine(StartAtCycles.size())) + .concat(Twine(AcquireAtCycles.size())) .concat(" vs ") .concat(Twine(PRVec.size()))); } - if (Cycles.empty()) { - // If ResourceCycles is not provided, default to one cycle + if (ReleaseAtCycles.empty()) { + // If ReleaseAtCycles is not provided, default to one cycle // per resource. - Cycles.resize(PRVec.size(), 1); + ReleaseAtCycles.resize(PRVec.size(), 1); } - if (StartAtCycles.empty()) { - // If StartAtCycles is not provided, reserve the resource + if (AcquireAtCycles.empty()) { + // If AcquireAtCycles is not provided, reserve the resource // starting from cycle 0. - StartAtCycles.resize(PRVec.size(), 0); + AcquireAtCycles.resize(PRVec.size(), 0); } - assert(StartAtCycles.size() == Cycles.size()); + assert(AcquireAtCycles.size() == ReleaseAtCycles.size()); - ExpandProcResources(PRVec, Cycles, StartAtCycles, ProcModel); - assert(StartAtCycles.size() == Cycles.size()); + ExpandProcResources(PRVec, ReleaseAtCycles, AcquireAtCycles, ProcModel); + assert(AcquireAtCycles.size() == ReleaseAtCycles.size()); for (unsigned PRIdx = 0, PREnd = PRVec.size(); PRIdx != PREnd; ++PRIdx) { MCWriteProcResEntry WPREntry; WPREntry.ProcResourceIdx = ProcModel.getProcResourceIdx(PRVec[PRIdx]); assert(WPREntry.ProcResourceIdx && "Bad ProcResourceIdx"); - WPREntry.Cycles = Cycles[PRIdx]; - WPREntry.StartAtCycle = StartAtCycles[PRIdx]; - if (StartAtCycles[PRIdx] > Cycles[PRIdx]) { - PrintFatalError(WriteRes->getLoc(), - Twine("Inconsistent resource cycles: StartAtCycles " - "< Cycles must hold.")); + WPREntry.ReleaseAtCycle = ReleaseAtCycles[PRIdx]; + WPREntry.AcquireAtCycle = AcquireAtCycles[PRIdx]; + if (AcquireAtCycles[PRIdx] > ReleaseAtCycles[PRIdx]) { + PrintFatalError( + WriteRes->getLoc(), + Twine("Inconsistent resource cycles: AcquireAtCycles " + "< ReleaseAtCycles must hold.")); } - if (StartAtCycles[PRIdx] < 0) { + if (AcquireAtCycles[PRIdx] < 0) { PrintFatalError(WriteRes->getLoc(), - Twine("Invalid value: StartAtCycle " + Twine("Invalid value: AcquireAtCycle " "must be a non-negative value.")); } // If this resource is already used in this sequence, add the current @@ -1228,9 +1229,10 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel, // `SubtargetEmitter::EmitSchedClassTables`), or // 2. thinking how to merge multiple intervals into a // single interval. - assert(WPREntry.StartAtCycle == 0 && + assert(WPREntry.AcquireAtCycle == 0 && "multiple use ofthe same resource is not yet handled"); - WriteProcResources[WPRIdx].Cycles += WPREntry.Cycles; + WriteProcResources[WPRIdx].ReleaseAtCycle += + WPREntry.ReleaseAtCycle; break; } } @@ -1334,7 +1336,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel, void SubtargetEmitter::EmitSchedClassTables(SchedClassTables &SchedTables, raw_ostream &OS) { // Emit global WriteProcResTable. - OS << "\n// {ProcResourceIdx, Cycles, StartAtCycle}\n" + OS << "\n// {ProcResourceIdx, ReleaseAtCycle, AcquireAtCycle}\n" << "extern const llvm::MCWriteProcResEntry " << Target << "WriteProcResTable[] = {\n" << " { 0, 0, 0 }, // Invalid\n"; @@ -1342,8 +1344,8 @@ void SubtargetEmitter::EmitSchedClassTables(SchedClassTables &SchedTables, WPRIdx != WPREnd; ++WPRIdx) { MCWriteProcResEntry &WPREntry = SchedTables.WriteProcResources[WPRIdx]; OS << " {" << format("%2d", WPREntry.ProcResourceIdx) << ", " - << format("%2d", WPREntry.Cycles) << ", " - << format("%2d", WPREntry.StartAtCycle) << "}"; + << format("%2d", WPREntry.ReleaseAtCycle) << ", " + << format("%2d", WPREntry.AcquireAtCycle) << "}"; if (WPRIdx + 1 < WPREnd) OS << ','; OS << " // #" << WPRIdx << '\n'; @@ -1933,7 +1935,7 @@ void SubtargetEmitter::run(raw_ostream &OS) { if (NumProcs) OS << Target << "SubTypeKV, "; else - OS << "None, "; + OS << "std::nullopt, "; OS << '\n'; OS.indent(22); OS << Target << "WriteProcResTable, " << Target << "WriteLatencyTable, " @@ -2026,7 +2028,7 @@ void SubtargetEmitter::run(raw_ostream &OS) { if (NumProcs) OS << "ArrayRef(" << Target << "SubTypeKV, " << NumProcs << "), "; else - OS << "None, "; + OS << "std::nullopt, "; OS << '\n'; OS.indent(24); OS << Target << "WriteProcResTable, " << Target << "WriteLatencyTable, " diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/llvm/utils/TableGen/SubtargetFeatureInfo.cpp index 1db8c0bf430a..52afb4d89162 100644 --- a/llvm/utils/TableGen/SubtargetFeatureInfo.cpp +++ b/llvm/utils/TableGen/SubtargetFeatureInfo.cpp @@ -47,15 +47,29 @@ SubtargetFeatureInfo::getAll(const RecordKeeper &Records) { } void SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration( - SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS) { + const SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS, + const std::map<std::string, unsigned> *HwModes) { OS << "// Bits for subtarget features that participate in " << "instruction matching.\n"; - OS << "enum SubtargetFeatureBits : " - << getMinimalTypeForRange(SubtargetFeatures.size()) << " {\n"; + unsigned Size = SubtargetFeatures.size(); + if (HwModes) + Size += HwModes->size(); + + OS << "enum SubtargetFeatureBits : " << getMinimalTypeForRange(Size) + << " {\n"; for (const auto &SF : SubtargetFeatures) { const SubtargetFeatureInfo &SFI = SF.second; OS << " " << SFI.getEnumBitName() << " = " << SFI.Index << ",\n"; } + + if (HwModes) { + unsigned Offset = SubtargetFeatures.size(); + for (const auto &M : *HwModes) { + OS << " Feature_HwMode" << M.second << "Bit = " << (M.second + Offset) + << ",\n"; + } + } + OS << "};\n\n"; } @@ -87,8 +101,8 @@ void SubtargetFeatureInfo::emitNameTable( void SubtargetFeatureInfo::emitComputeAvailableFeatures( StringRef TargetName, StringRef ClassName, StringRef FuncName, - SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS, - StringRef ExtraParams) { + const SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS, + StringRef ExtraParams, const std::map<std::string, unsigned> *HwModes) { OS << "PredicateBitset " << ClassName << "::\n" << FuncName << "(const " << TargetName << "Subtarget *Subtarget"; if (!ExtraParams.empty()) @@ -103,6 +117,14 @@ void SubtargetFeatureInfo::emitComputeAvailableFeatures( OS << " if (" << CondStr << ")\n"; OS << " Features.set(" << SFI.getEnumBitName() << ");\n"; } + + if (HwModes) { + for (const auto &M : *HwModes) { + OS << " if (" << M.first << ")\n"; + OS << " Features.set(Feature_HwMode" << M.second << "Bit);\n"; + } + } + OS << " return Features;\n"; OS << "}\n\n"; } diff --git a/llvm/utils/TableGen/SubtargetFeatureInfo.h b/llvm/utils/TableGen/SubtargetFeatureInfo.h index 77703e8a87f8..940100448435 100644 --- a/llvm/utils/TableGen/SubtargetFeatureInfo.h +++ b/llvm/utils/TableGen/SubtargetFeatureInfo.h @@ -54,9 +54,9 @@ struct SubtargetFeatureInfo { /// /// This version emits the bit index for the feature and can therefore support /// more than 64 feature bits. - static void - emitSubtargetFeatureBitEnumeration(SubtargetFeatureInfoMap &SubtargetFeatures, - raw_ostream &OS); + static void emitSubtargetFeatureBitEnumeration( + const SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS, + const std::map<std::string, unsigned> *HwModes = nullptr); static void emitNameTable(SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS); @@ -75,11 +75,12 @@ struct SubtargetFeatureInfo { /// \param SubtargetFeatures A map of TableGen records to the /// SubtargetFeatureInfo equivalent. /// \param ExtraParams Additional arguments to the generated function. - static void - emitComputeAvailableFeatures(StringRef TargetName, StringRef ClassName, - StringRef FuncName, - SubtargetFeatureInfoMap &SubtargetFeatures, - raw_ostream &OS, StringRef ExtraParams = ""); + /// \param HwModes Map of HwMode conditions to check. + static void emitComputeAvailableFeatures( + StringRef TargetName, StringRef ClassName, StringRef FuncName, + const SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS, + StringRef ExtraParams = "", + const std::map<std::string, unsigned> *HwModes = nullptr); /// Emit the function to compute the list of available features given a /// subtarget. diff --git a/llvm/utils/TableGen/VTEmitter.cpp b/llvm/utils/TableGen/VTEmitter.cpp index d398a7e7b58f..5ec1f59318f7 100644 --- a/llvm/utils/TableGen/VTEmitter.cpp +++ b/llvm/utils/TableGen/VTEmitter.cpp @@ -30,7 +30,7 @@ public: } // End anonymous namespace. void VTEmitter::run(raw_ostream &OS) { - emitSourceFileHeader("ValueTypes Source Fragment", OS); + emitSourceFileHeader("ValueTypes Source Fragment", OS, Records); std::array<const Record *, 256> VTsByNumber = {}; auto ValueTypes = Records.getAllDerivedDefinitions("ValueType"); diff --git a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp index 85da547d04c1..bfb7e5c33317 100644 --- a/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp +++ b/llvm/utils/TableGen/VarLenCodeEmitterGen.cpp @@ -60,6 +60,8 @@ #include "llvm/TableGen/Error.h" #include "llvm/TableGen/Record.h" +#include <algorithm> + using namespace llvm; namespace { @@ -67,17 +69,26 @@ namespace { class VarLenCodeEmitterGen { RecordKeeper &Records; - DenseMap<Record *, VarLenInst> VarLenInsts; + // Representaton of alternative encodings used for HwModes. + using AltEncodingTy = int; + // Mode identifier when only one encoding is defined. + const AltEncodingTy Universal = -1; + // The set of alternative instruction encodings with a descriptive + // name suffix to improve readability of the generated code. + std::map<AltEncodingTy, std::string> Modes; + + DenseMap<Record *, DenseMap<AltEncodingTy, VarLenInst>> VarLenInsts; // Emit based values (i.e. fixed bits in the encoded instructions) void emitInstructionBaseValues( raw_ostream &OS, ArrayRef<const CodeGenInstruction *> NumberedInstructions, - CodeGenTarget &Target, int HwMode = -1); + CodeGenTarget &Target, AltEncodingTy Mode); - std::string getInstructionCase(Record *R, CodeGenTarget &Target); - std::string getInstructionCaseForEncoding(Record *R, Record *EncodingDef, - CodeGenTarget &Target); + std::string getInstructionCases(Record *R, CodeGenTarget &Target); + std::string getInstructionCaseForEncoding(Record *R, AltEncodingTy Mode, + const VarLenInst &VLI, + CodeGenTarget &Target, int I); public: explicit VarLenCodeEmitterGen(RecordKeeper &R) : Records(R) {} @@ -114,7 +125,7 @@ static std::pair<StringRef, StringRef> getCustomCoders(ArrayRef<Init *> Args) { } VarLenInst::VarLenInst(const DagInit *DI, const RecordVal *TheDef) - : TheDef(TheDef), NumBits(0U) { + : TheDef(TheDef), NumBits(0U), HasDynamicSegment(false) { buildRec(DI); for (const auto &S : Segments) NumBits += S.BitWidth; @@ -214,36 +225,38 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) { auto Insts = Records.getAllDerivedDefinitions("Instruction"); auto NumberedInstructions = Target.getInstructionsByEnumValue(); - const CodeGenHwModes &HWM = Target.getHwModes(); - // The set of HwModes used by instruction encodings. - std::set<unsigned> HwModes; for (const CodeGenInstruction *CGI : NumberedInstructions) { Record *R = CGI->TheDef; - // Create the corresponding VarLenInst instance. if (R->getValueAsString("Namespace") == "TargetOpcode" || R->getValueAsBit("isPseudo")) continue; + // Setup alternative encodings according to HwModes if (const RecordVal *RV = R->getValue("EncodingInfos")) { if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + const CodeGenHwModes &HWM = Target.getHwModes(); EncodingInfoByHwMode EBM(DI->getDef(), HWM); for (auto &KV : EBM) { - HwModes.insert(KV.first); + AltEncodingTy Mode = KV.first; + Modes.insert({Mode, "_" + HWM.getMode(Mode).Name.str()}); Record *EncodingDef = KV.second; RecordVal *RV = EncodingDef->getValue("Inst"); DagInit *DI = cast<DagInit>(RV->getValue()); - VarLenInsts.insert({EncodingDef, VarLenInst(DI, RV)}); + VarLenInsts[R].insert({Mode, VarLenInst(DI, RV)}); } continue; } } RecordVal *RV = R->getValue("Inst"); DagInit *DI = cast<DagInit>(RV->getValue()); - VarLenInsts.insert({R, VarLenInst(DI, RV)}); + VarLenInsts[R].insert({Universal, VarLenInst(DI, RV)}); } + if (Modes.empty()) + Modes.insert({Universal, ""}); // Base case, skip suffix. + // Emit function declaration OS << "void " << Target.getName() << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n" @@ -253,36 +266,26 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) { << " const MCSubtargetInfo &STI) const {\n"; // Emit instruction base values - if (HwModes.empty()) { - emitInstructionBaseValues(OS, NumberedInstructions, Target); - } else { - for (unsigned HwMode : HwModes) - emitInstructionBaseValues(OS, NumberedInstructions, Target, (int)HwMode); - } + for (const auto &Mode : Modes) + emitInstructionBaseValues(OS, NumberedInstructions, Target, Mode.first); - if (!HwModes.empty()) { - OS << " const unsigned **Index;\n"; - OS << " const uint64_t *InstBits;\n"; - OS << " unsigned HwMode = STI.getHwMode();\n"; - OS << " switch (HwMode) {\n"; - OS << " default: llvm_unreachable(\"Unknown hardware mode!\"); break;\n"; - for (unsigned I : HwModes) { - OS << " case " << I << ": InstBits = InstBits_" << HWM.getMode(I).Name - << "; Index = Index_" << HWM.getMode(I).Name << "; break;\n"; - } - OS << " };\n"; + if (Modes.size() > 1) { + OS << " unsigned Mode = STI.getHwMode();\n"; } - // Emit helper function to retrieve base values. - OS << " auto getInstBits = [&](unsigned Opcode) -> APInt {\n" - << " unsigned NumBits = Index[Opcode][0];\n" - << " if (!NumBits)\n" - << " return APInt::getZeroWidth();\n" - << " unsigned Idx = Index[Opcode][1];\n" - << " ArrayRef<uint64_t> Data(&InstBits[Idx], " - << "APInt::getNumWords(NumBits));\n" - << " return APInt(NumBits, Data);\n" - << " };\n"; + for (const auto &Mode : Modes) { + // Emit helper function to retrieve base values. + OS << " auto getInstBits" << Mode.second + << " = [&](unsigned Opcode) -> APInt {\n" + << " unsigned NumBits = Index" << Mode.second << "[Opcode][0];\n" + << " if (!NumBits)\n" + << " return APInt::getZeroWidth();\n" + << " unsigned Idx = Index" << Mode.second << "[Opcode][1];\n" + << " ArrayRef<uint64_t> Data(&InstBits" << Mode.second << "[Idx], " + << "APInt::getNumWords(NumBits));\n" + << " return APInt(NumBits, Data);\n" + << " };\n"; + } // Map to accumulate all the cases. std::map<std::string, std::vector<std::string>> CaseMap; @@ -294,7 +297,7 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) { continue; std::string InstName = (R->getValueAsString("Namespace") + "::" + R->getName()).str(); - std::string Case = getInstructionCase(R, Target); + std::string Case = getInstructionCases(R, Target); CaseMap[Case].push_back(std::move(InstName)); } @@ -344,19 +347,12 @@ static void emitInstBits(raw_ostream &IS, raw_ostream &SS, const APInt &Bits, void VarLenCodeEmitterGen::emitInstructionBaseValues( raw_ostream &OS, ArrayRef<const CodeGenInstruction *> NumberedInstructions, - CodeGenTarget &Target, int HwMode) { + CodeGenTarget &Target, AltEncodingTy Mode) { std::string IndexArray, StorageArray; raw_string_ostream IS(IndexArray), SS(StorageArray); - const CodeGenHwModes &HWM = Target.getHwModes(); - if (HwMode == -1) { - IS << " static const unsigned Index[][2] = {\n"; - SS << " static const uint64_t InstBits[] = {\n"; - } else { - StringRef Name = HWM.getMode(HwMode).Name; - IS << " static const unsigned Index_" << Name << "[][2] = {\n"; - SS << " static const uint64_t InstBits_" << Name << "[] = {\n"; - } + IS << " static const unsigned Index" << Modes[Mode] << "[][2] = {\n"; + SS << " static const uint64_t InstBits" << Modes[Mode] << "[] = {\n"; unsigned NumFixedValueWords = 0U; for (const CodeGenInstruction *CGI : NumberedInstructions) { @@ -368,20 +364,18 @@ void VarLenCodeEmitterGen::emitInstructionBaseValues( continue; } - Record *EncodingDef = R; - if (const RecordVal *RV = R->getValue("EncodingInfos")) { - if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { - EncodingInfoByHwMode EBM(DI->getDef(), HWM); - if (EBM.hasMode(HwMode)) - EncodingDef = EBM.get(HwMode); - } + const auto InstIt = VarLenInsts.find(R); + if (InstIt == VarLenInsts.end()) + PrintFatalError(R, "VarLenInst not found for this record"); + auto ModeIt = InstIt->second.find(Mode); + if (ModeIt == InstIt->second.end()) + ModeIt = InstIt->second.find(Universal); + if (ModeIt == InstIt->second.end()) { + IS.indent(4) << "{/*NumBits*/0, /*Index*/0},\t" + << "// " << R->getName() << " no encoding\n"; + continue; } - - auto It = VarLenInsts.find(EncodingDef); - if (It == VarLenInsts.end()) - PrintFatalError(EncodingDef, "VarLenInst not found for this record"); - const VarLenInst &VLI = It->second; - + const VarLenInst &VLI = ModeIt->second; unsigned i = 0U, BitWidth = VLI.size(); // Start by filling in fixed values. @@ -414,48 +408,56 @@ void VarLenCodeEmitterGen::emitInstructionBaseValues( OS << IS.str() << SS.str(); } -std::string VarLenCodeEmitterGen::getInstructionCase(Record *R, - CodeGenTarget &Target) { +std::string VarLenCodeEmitterGen::getInstructionCases(Record *R, + CodeGenTarget &Target) { + auto It = VarLenInsts.find(R); + if (It == VarLenInsts.end()) + PrintFatalError(R, "Parsed encoding record not found"); + const auto &Map = It->second; + + // Is this instructions encoding universal (same for all modes)? + // Allways true if there is only one mode. + if (Map.size() == 1 && Map.begin()->first == Universal) { + // Universal, just pick the first mode. + AltEncodingTy Mode = Modes.begin()->first; + const auto &Encoding = Map.begin()->second; + return getInstructionCaseForEncoding(R, Mode, Encoding, Target, 6); + } + std::string Case; - if (const RecordVal *RV = R->getValue("EncodingInfos")) { - if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { - const CodeGenHwModes &HWM = Target.getHwModes(); - EncodingInfoByHwMode EBM(DI->getDef(), HWM); - Case += " switch (HwMode) {\n"; - Case += " default: llvm_unreachable(\"Unhandled HwMode\");\n"; - for (auto &KV : EBM) { - Case += " case " + itostr(KV.first) + ": {\n"; - Case += getInstructionCaseForEncoding(R, KV.second, Target); - Case += " break;\n"; - Case += " }\n"; - } - Case += " }\n"; - return Case; + Case += " switch (Mode) {\n"; + Case += " default: llvm_unreachable(\"Unhandled Mode\");\n"; + for (const auto &Mode : Modes) { + Case += " case " + itostr(Mode.first) + ": {\n"; + const auto &It = Map.find(Mode.first); + if (It == Map.end()) { + Case += + " llvm_unreachable(\"Undefined encoding in this mode\");\n"; + } else { + Case += + getInstructionCaseForEncoding(R, It->first, It->second, Target, 8); } + Case += " break;\n"; + Case += " }\n"; } - return getInstructionCaseForEncoding(R, R, Target); + Case += " }\n"; + return Case; } std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding( - Record *R, Record *EncodingDef, CodeGenTarget &Target) { - auto It = VarLenInsts.find(EncodingDef); - if (It == VarLenInsts.end()) - PrintFatalError(EncodingDef, "Parsed encoding record not found"); - const VarLenInst &VLI = It->second; - size_t BitWidth = VLI.size(); + Record *R, AltEncodingTy Mode, const VarLenInst &VLI, CodeGenTarget &Target, + int I) { CodeGenInstruction &CGI = Target.getInstruction(R); std::string Case; raw_string_ostream SS(Case); - // Resize the scratch buffer. - if (BitWidth && !VLI.isFixedValueOnly()) - SS.indent(6) << "Scratch = Scratch.zext(" << BitWidth << ");\n"; // Populate based value. - SS.indent(6) << "Inst = getInstBits(opcode);\n"; + SS.indent(I) << "Inst = getInstBits" << Modes[Mode] << "(opcode);\n"; // Process each segment in VLI. size_t Offset = 0U; + unsigned HighScratchAccess = 0U; for (const auto &ES : VLI) { unsigned NumBits = ES.BitWidth; const Init *Val = ES.Value; @@ -480,29 +482,40 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding( if (ES.CustomEncoder.size()) CustomEncoder = ES.CustomEncoder; - SS.indent(6) << "Scratch.clearAllBits();\n"; - SS.indent(6) << "// op: " << OperandName.drop_front(1) << "\n"; + SS.indent(I) << "Scratch.clearAllBits();\n"; + SS.indent(I) << "// op: " << OperandName.drop_front(1) << "\n"; if (CustomEncoder.empty()) - SS.indent(6) << "getMachineOpValue(MI, MI.getOperand(" + SS.indent(I) << "getMachineOpValue(MI, MI.getOperand(" << utostr(FlatOpIdx) << ")"; else - SS.indent(6) << CustomEncoder << "(MI, /*OpIdx=*/" << utostr(FlatOpIdx); + SS.indent(I) << CustomEncoder << "(MI, /*OpIdx=*/" << utostr(FlatOpIdx); SS << ", /*Pos=*/" << utostr(Offset) << ", Scratch, Fixups, STI);\n"; - SS.indent(6) << "Inst.insertBits(" + SS.indent(I) << "Inst.insertBits(" << "Scratch.extractBits(" << utostr(NumBits) << ", " << utostr(LoBit) << ")" << ", " << Offset << ");\n"; + + HighScratchAccess = std::max(HighScratchAccess, NumBits + LoBit); } Offset += NumBits; } StringRef PostEmitter = R->getValueAsString("PostEncoderMethod"); if (!PostEmitter.empty()) - SS.indent(6) << "Inst = " << PostEmitter << "(MI, Inst, STI);\n"; + SS.indent(I) << "Inst = " << PostEmitter << "(MI, Inst, STI);\n"; - return Case; + // Resize the scratch buffer if it's to small. + std::string ScratchResizeStr; + if (VLI.size() && !VLI.isFixedValueOnly()) { + raw_string_ostream RS(ScratchResizeStr); + RS.indent(I) << "if (Scratch.getBitWidth() < " << HighScratchAccess + << ") { Scratch = Scratch.zext(" << HighScratchAccess + << "); }\n"; + } + + return ScratchResizeStr + Case; } namespace llvm { diff --git a/llvm/utils/TableGen/X86DisassemblerTables.cpp b/llvm/utils/TableGen/X86DisassemblerTables.cpp index 708c92aecfc8..959e0fda50b8 100644 --- a/llvm/utils/TableGen/X86DisassemblerTables.cpp +++ b/llvm/utils/TableGen/X86DisassemblerTables.cpp @@ -31,39 +31,47 @@ using namespace X86Disassembler; /// @param insnContext - The instruction class to transform to a string. /// @return - A statically-allocated string constant that contains the /// name of the instruction class. -static inline const char* stringForContext(InstructionContext insnContext) { +static inline const char *stringForContext(InstructionContext insnContext) { switch (insnContext) { default: llvm_unreachable("Unhandled instruction class"); -#define ENUM_ENTRY(n, r, d) case n: return #n; break; -#define ENUM_ENTRY_K_B(n, r, d) ENUM_ENTRY(n, r, d) ENUM_ENTRY(n##_K_B, r, d)\ - ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) ENUM_ENTRY(n##_B, r, d)\ - ENUM_ENTRY(n##_KZ_B, r, d) - INSTRUCTION_CONTEXTS +#define ENUM_ENTRY(n, r, d) \ + case n: \ + return #n; \ + break; +#define ENUM_ENTRY_K_B(n, r, d) \ + ENUM_ENTRY(n, r, d) \ + ENUM_ENTRY(n##_K_B, r, d) ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) \ + ENUM_ENTRY(n##_B, r, d) ENUM_ENTRY(n##_KZ_B, r, d) + INSTRUCTION_CONTEXTS #undef ENUM_ENTRY #undef ENUM_ENTRY_K_B } } /// stringForOperandType - Like stringForContext, but for OperandTypes. -static inline const char* stringForOperandType(OperandType type) { +static inline const char *stringForOperandType(OperandType type) { switch (type) { default: llvm_unreachable("Unhandled type"); -#define ENUM_ENTRY(i, d) case i: return #i; - TYPES +#define ENUM_ENTRY(i, d) \ + case i: \ + return #i; + TYPES #undef ENUM_ENTRY } } /// stringForOperandEncoding - like stringForContext, but for /// OperandEncodings. -static inline const char* stringForOperandEncoding(OperandEncoding encoding) { +static inline const char *stringForOperandEncoding(OperandEncoding encoding) { switch (encoding) { default: llvm_unreachable("Unhandled encoding"); -#define ENUM_ENTRY(i, d) case i: return #i; - ENCODINGS +#define ENUM_ENTRY(i, d) \ + case i: \ + return #i; + ENCODINGS #undef ENUM_ENTRY } } @@ -83,17 +91,17 @@ static inline bool inheritsFrom(InstructionContext child, switch (parent) { case IC: - return(inheritsFrom(child, IC_64BIT, AdSize64) || - (noPrefix && inheritsFrom(child, IC_OPSIZE, noPrefix)) || - inheritsFrom(child, IC_ADSIZE) || - (noPrefix && inheritsFrom(child, IC_XD, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_XS, noPrefix))); + return (inheritsFrom(child, IC_64BIT, AdSize64) || + (noPrefix && inheritsFrom(child, IC_OPSIZE, noPrefix)) || + inheritsFrom(child, IC_ADSIZE) || + (noPrefix && inheritsFrom(child, IC_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_XS, noPrefix))); case IC_64BIT: - return(inheritsFrom(child, IC_64BIT_REXW) || - (noPrefix && inheritsFrom(child, IC_64BIT_OPSIZE, noPrefix)) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_ADSIZE)) || - (noPrefix && inheritsFrom(child, IC_64BIT_XD, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_64BIT_XS, noPrefix))); + return (inheritsFrom(child, IC_64BIT_REXW) || + (noPrefix && inheritsFrom(child, IC_64BIT_OPSIZE, noPrefix)) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_ADSIZE)) || + (noPrefix && inheritsFrom(child, IC_64BIT_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_XS, noPrefix))); case IC_OPSIZE: return inheritsFrom(child, IC_64BIT_OPSIZE) || inheritsFrom(child, IC_OPSIZE_ADSIZE); @@ -118,20 +126,20 @@ static inline bool inheritsFrom(InstructionContext child, case IC_XS_ADSIZE: return inheritsFrom(child, IC_64BIT_XS_ADSIZE); case IC_64BIT_REXW: - return((noPrefix && inheritsFrom(child, IC_64BIT_REXW_XS, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_64BIT_REXW_XD, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_64BIT_REXW_OPSIZE, noPrefix)) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE))); + return ((noPrefix && inheritsFrom(child, IC_64BIT_REXW_XS, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_REXW_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_REXW_OPSIZE, noPrefix)) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE))); case IC_64BIT_OPSIZE: return inheritsFrom(child, IC_64BIT_REXW_OPSIZE) || (!AdSize64 && inheritsFrom(child, IC_64BIT_OPSIZE_ADSIZE)) || (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE)); case IC_64BIT_XD: - return(inheritsFrom(child, IC_64BIT_REXW_XD) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_XD_ADSIZE))); + return (inheritsFrom(child, IC_64BIT_REXW_XD) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_XD_ADSIZE))); case IC_64BIT_XS: - return(inheritsFrom(child, IC_64BIT_REXW_XS) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_XS_ADSIZE))); + return (inheritsFrom(child, IC_64BIT_REXW_XS) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_XS_ADSIZE))); case IC_64BIT_XD_OPSIZE: case IC_64BIT_XS_OPSIZE: return false; @@ -142,6 +150,7 @@ static inline bool inheritsFrom(InstructionContext child, case IC_64BIT_REXW_XS: case IC_64BIT_REXW_OPSIZE: case IC_64BIT_REXW_ADSIZE: + case IC_64BIT_REX2: return false; case IC_VEX: return (VEX_LIG && WIG && inheritsFrom(child, IC_VEX_L_W)) || @@ -409,10 +418,8 @@ static inline bool inheritsFrom(InstructionContext child, (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_K_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_K_B)); case IC_EVEX_OPSIZE_K_B: - return (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B)) || - (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B)) || + return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B)) || + (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B)) || (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_K_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_K_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_K_B)); @@ -435,10 +442,8 @@ static inline bool inheritsFrom(InstructionContext child, (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_KZ_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_KZ_B)); case IC_EVEX_OPSIZE_KZ_B: - return (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B)) || - (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B)) || + return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B)) || + (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B)) || (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_KZ_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_KZ_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_KZ_B)); @@ -557,8 +562,8 @@ static inline bool inheritsFrom(InstructionContext child, case IC_EVEX_L2_W_OPSIZE_KZ_B: return false; default: - errs() << "Unknown instruction class: " << - stringForContext((InstructionContext)parent) << "\n"; + errs() << "Unknown instruction class: " + << stringForContext((InstructionContext)parent) << "\n"; llvm_unreachable("Unknown instruction class"); } } @@ -576,12 +581,12 @@ static inline bool outranks(InstructionContext upper, assert(lower < IC_max); #define ENUM_ENTRY(n, r, d) r, -#define ENUM_ENTRY_K_B(n, r, d) ENUM_ENTRY(n, r, d) \ - ENUM_ENTRY(n##_K_B, r, d) ENUM_ENTRY(n##_KZ_B, r, d) \ - ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) ENUM_ENTRY(n##_B, r, d) - static int ranks[IC_max] = { - INSTRUCTION_CONTEXTS - }; +#define ENUM_ENTRY_K_B(n, r, d) \ + ENUM_ENTRY(n, r, d) \ + ENUM_ENTRY(n##_K_B, r, d) \ + ENUM_ENTRY(n##_KZ_B, r, d) ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) \ + ENUM_ENTRY(n##_B, r, d) + static int ranks[IC_max] = {INSTRUCTION_CONTEXTS}; #undef ENUM_ENTRY #undef ENUM_ENTRY_K_B @@ -604,19 +609,19 @@ static ModRMDecisionType getDecisionType(ModRMDecision &decision) { satisfiesOneEntry = false; if (((index & 0xc0) == 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[0xc0])) + (decision.instructionIDs[index] != decision.instructionIDs[0xc0])) satisfiesSplitRM = false; if (((index & 0xc0) != 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[0x00])) + (decision.instructionIDs[index] != decision.instructionIDs[0x00])) satisfiesSplitRM = false; - if (((index & 0xc0) == 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[index&0xf8])) + if (((index & 0xc0) == 0xc0) && (decision.instructionIDs[index] != + decision.instructionIDs[index & 0xf8])) satisfiesSplitReg = false; - if (((index & 0xc0) != 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[index&0x38])) + if (((index & 0xc0) != 0xc0) && (decision.instructionIDs[index] != + decision.instructionIDs[index & 0x38])) satisfiesSplitMisc = false; } @@ -641,11 +646,13 @@ static ModRMDecisionType getDecisionType(ModRMDecision &decision) { /// @param dt - The decision type. /// @return - A pointer to the statically-allocated string (e.g., /// "MODRM_ONEENTRY" for MODRM_ONEENTRY). -static const char* stringForDecisionType(ModRMDecisionType dt) { -#define ENUM_ENTRY(n) case n: return #n; +static const char *stringForDecisionType(ModRMDecisionType dt) { +#define ENUM_ENTRY(n) \ + case n: \ + return #n; switch (dt) { - default: - llvm_unreachable("Unknown decision type"); + default: + llvm_unreachable("Unknown decision type"); MODRMTYPES }; #undef ENUM_ENTRY @@ -658,8 +665,7 @@ DisassemblerTables::DisassemblerTables() { HasConflicts = false; } -DisassemblerTables::~DisassemblerTables() { -} +DisassemblerTables::~DisassemblerTables() {} void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, unsigned &i1, unsigned &i2, @@ -677,31 +683,31 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, std::vector<unsigned> ModRMDecision; switch (dt) { - default: - llvm_unreachable("Unknown decision type"); - case MODRM_ONEENTRY: - ModRMDecision.push_back(decision.instructionIDs[0]); - break; - case MODRM_SPLITRM: - ModRMDecision.push_back(decision.instructionIDs[0x00]); - ModRMDecision.push_back(decision.instructionIDs[0xc0]); - break; - case MODRM_SPLITREG: - for (unsigned index = 0; index < 64; index += 8) - ModRMDecision.push_back(decision.instructionIDs[index]); - for (unsigned index = 0xc0; index < 256; index += 8) - ModRMDecision.push_back(decision.instructionIDs[index]); - break; - case MODRM_SPLITMISC: - for (unsigned index = 0; index < 64; index += 8) - ModRMDecision.push_back(decision.instructionIDs[index]); - for (unsigned index = 0xc0; index < 256; ++index) - ModRMDecision.push_back(decision.instructionIDs[index]); - break; - case MODRM_FULL: - for (unsigned short InstructionID : decision.instructionIDs) - ModRMDecision.push_back(InstructionID); - break; + default: + llvm_unreachable("Unknown decision type"); + case MODRM_ONEENTRY: + ModRMDecision.push_back(decision.instructionIDs[0]); + break; + case MODRM_SPLITRM: + ModRMDecision.push_back(decision.instructionIDs[0x00]); + ModRMDecision.push_back(decision.instructionIDs[0xc0]); + break; + case MODRM_SPLITREG: + for (unsigned index = 0; index < 64; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + for (unsigned index = 0xc0; index < 256; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + break; + case MODRM_SPLITMISC: + for (unsigned index = 0; index < 64; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + for (unsigned index = 0xc0; index < 256; ++index) + ModRMDecision.push_back(decision.instructionIDs[index]); + break; + case MODRM_FULL: + for (unsigned short InstructionID : decision.instructionIDs) + ModRMDecision.push_back(InstructionID); + break; } unsigned &EntryNumber = ModRMTable[ModRMDecision]; @@ -721,23 +727,23 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, o2 << "{" << stringForDecisionType(dt) << ", " << EntryNumber << "}"; switch (dt) { - default: - llvm_unreachable("Unknown decision type"); - case MODRM_ONEENTRY: - sEntryNumber += 1; - break; - case MODRM_SPLITRM: - sEntryNumber += 2; - break; - case MODRM_SPLITREG: - sEntryNumber += 16; - break; - case MODRM_SPLITMISC: - sEntryNumber += 8 + 64; - break; - case MODRM_FULL: - sEntryNumber += 256; - break; + default: + llvm_unreachable("Unknown decision type"); + case MODRM_ONEENTRY: + sEntryNumber += 1; + break; + case MODRM_SPLITRM: + sEntryNumber += 2; + break; + case MODRM_SPLITREG: + sEntryNumber += 16; + break; + case MODRM_SPLITMISC: + sEntryNumber += 8 + 64; + break; + case MODRM_FULL: + sEntryNumber += 256; + break; } // We assume that the index can fit into uint16_t. @@ -790,8 +796,9 @@ void DisassemblerTables::emitContextDecision(raw_ostream &o1, raw_ostream &o2, unsigned &i1, unsigned &i2, unsigned &ModRMTableNum, ContextDecision &decision, - const char* name) const { - o2.indent(i2) << "static const struct ContextDecision " << name << " = {{/* opcodeDecisions */\n"; + const char *name) const { + o2.indent(i2) << "static const struct ContextDecision " << name + << " = {{/* opcodeDecisions */\n"; i2++; for (unsigned index = 0; index < IC_max; ++index) { @@ -814,8 +821,8 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o, o << "static const struct OperandSpecifier x86OperandSets[][" << X86_MAX_OPERANDS << "] = {\n"; - typedef SmallVector<std::pair<OperandEncoding, OperandType>, - X86_MAX_OPERANDS> OperandListTy; + typedef SmallVector<std::pair<OperandEncoding, OperandType>, X86_MAX_OPERANDS> + OperandListTy; std::map<OperandListTy, unsigned> OperandSets; unsigned OperandSetNum = 0; @@ -828,14 +835,15 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o, OperandList.push_back(std::make_pair(Encoding, Type)); } unsigned &N = OperandSets[OperandList]; - if (N != 0) continue; + if (N != 0) + continue; N = ++OperandSetNum; o << " { /* " << (OperandSetNum - 1) << " */\n"; for (unsigned i = 0, e = OperandList.size(); i != e; ++i) { const char *Encoding = stringForOperandEncoding(OperandList[i].first); - const char *Type = stringForOperandType(OperandList[i].second); + const char *Type = stringForOperandType(OperandList[i].second); o << " { " << Encoding << ", " << Type << " },\n"; } o << " },\n"; @@ -870,8 +878,8 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o, } void DisassemblerTables::emitContextTable(raw_ostream &o, unsigned &i) const { - o.indent(i * 2) << "static const uint8_t " CONTEXTS_STR - "[" << ATTR_max << "] = {\n"; + o.indent(i * 2) << "static const uint8_t " CONTEXTS_STR "[" << ATTR_max + << "] = {\n"; i++; for (unsigned index = 0; index < ATTR_max; ++index) { @@ -907,7 +915,8 @@ void DisassemblerTables::emitContextTable(raw_ostream &o, unsigned &i) const { if (index & ATTR_EVEXB) o << "_B"; } - } + } else if ((index & ATTR_64BIT) && (index & ATTR_REX2)) + o << "IC_64BIT_REX2"; else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XS)) o << "IC_64BIT_REXW_XS"; else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XD)) @@ -974,14 +983,19 @@ void DisassemblerTables::emitContextDecisions(raw_ostream &o1, raw_ostream &o2, unsigned &ModRMTableNum) const { emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[0], ONEBYTE_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[1], TWOBYTE_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[2], THREEBYTE38_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[3], THREEBYTE3A_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[2], + THREEBYTE38_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[3], + THREEBYTE3A_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[4], XOP8_MAP_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[5], XOP9_MAP_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[6], XOPA_MAP_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[7], THREEDNOW_MAP_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[8], MAP5_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[9], MAP6_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[7], + THREEDNOW_MAP_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[8], MAP4_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[9], MAP5_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[10], MAP6_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[11], MAP7_STR); } void DisassemblerTables::emit(raw_ostream &o) const { @@ -1021,24 +1035,22 @@ void DisassemblerTables::emit(raw_ostream &o) const { o << "\n"; } -void DisassemblerTables::setTableFields(ModRMDecision &decision, - const ModRMFilter &filter, - InstrUID uid, - uint8_t opcode) { +void DisassemblerTables::setTableFields(ModRMDecision &decision, + const ModRMFilter &filter, InstrUID uid, + uint8_t opcode) { for (unsigned index = 0; index < 256; ++index) { if (filter.accepts(index)) { if (decision.instructionIDs[index] == uid) continue; if (decision.instructionIDs[index] != 0) { - InstructionSpecifier &newInfo = - InstructionSpecifiers[uid]; + InstructionSpecifier &newInfo = InstructionSpecifiers[uid]; InstructionSpecifier &previousInfo = - InstructionSpecifiers[decision.instructionIDs[index]]; + InstructionSpecifiers[decision.instructionIDs[index]]; - if(previousInfo.name == "NOOP" && (newInfo.name == "XCHG16ar" || - newInfo.name == "XCHG32ar" || - newInfo.name == "XCHG64ar")) + if (previousInfo.name == "NOOP" && + (newInfo.name == "XCHG16ar" || newInfo.name == "XCHG32ar" || + newInfo.name == "XCHG64ar")) continue; // special case for XCHG*ar and NOOP if (outranks(previousInfo.insnContext, newInfo.insnContext)) @@ -1060,16 +1072,10 @@ void DisassemblerTables::setTableFields(ModRMDecision &decision, } } -void DisassemblerTables::setTableFields(OpcodeType type, - InstructionContext insnContext, - uint8_t opcode, - const ModRMFilter &filter, - InstrUID uid, - bool is32bit, - bool noPrefix, - bool ignoresVEX_L, - bool ignoresW, - unsigned addressSize) { +void DisassemblerTables::setTableFields( + OpcodeType type, InstructionContext insnContext, uint8_t opcode, + const ModRMFilter &filter, InstrUID uid, bool is32bit, bool noPrefix, + bool ignoresVEX_L, bool ignoresW, unsigned addressSize) { ContextDecision &decision = *Tables[type]; for (unsigned index = 0; index < IC_max; ++index) { @@ -1082,8 +1088,6 @@ void DisassemblerTables::setTableFields(OpcodeType type, InstructionSpecifiers[uid].insnContext, noPrefix, ignoresVEX_L, ignoresW, adSize64)) setTableFields(decision.opcodeDecisions[index].modRMDecisions[opcode], - filter, - uid, - opcode); + filter, uid, opcode); } } diff --git a/llvm/utils/TableGen/X86DisassemblerTables.h b/llvm/utils/TableGen/X86DisassemblerTables.h index 966f7406efec..4fbc58bea338 100644 --- a/llvm/utils/TableGen/X86DisassemblerTables.h +++ b/llvm/utils/TableGen/X86DisassemblerTables.h @@ -44,9 +44,11 @@ private: /// [5] XOP9 map opcode /// [6] XOPA map opcode /// [7] 3dnow map opcode - /// [8] fixed length MAP5 opcode - /// [9] fixed length MAP6 opcode - std::unique_ptr<ContextDecision> Tables[10]; + /// [8] fixed length MAP4 opcode + /// [9] fixed length MAP5 opcode + /// [10] fixed length MAP6 opcode + /// [11] fixed length MAP7 opcode + std::unique_ptr<ContextDecision> Tables[12]; // Table of ModRM encodings. typedef std::map<std::vector<unsigned>, unsigned> ModRMMapTy; diff --git a/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp b/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp index 35792ab67a4f..c80d9a199fa3 100644 --- a/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp @@ -33,14 +33,12 @@ class X86EVEX2VEXTablesEmitter { // to make the search more efficient std::map<uint64_t, std::vector<const CodeGenInstruction *>> VEXInsts; - typedef std::pair<const CodeGenInstruction *, const CodeGenInstruction *> Entry; - typedef std::pair<StringRef, StringRef> Predicate; + typedef std::pair<const CodeGenInstruction *, const CodeGenInstruction *> + Entry; // Represent both compress tables std::vector<Entry> EVEX2VEX128; std::vector<Entry> EVEX2VEX256; - // Represent predicates of VEX instructions. - std::vector<Predicate> EVEX2VEXPredicates; public: X86EVEX2VEXTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} @@ -52,9 +50,6 @@ private: // Prints the given table as a C++ array of type // X86EvexToVexCompressTableEntry void printTable(const std::vector<Entry> &Table, raw_ostream &OS); - // Prints function which checks target feature specific predicate. - void printCheckPredicate(const std::vector<Predicate> &Predicates, - raw_ostream &OS); }; void X86EVEX2VEXTablesEmitter::printTable(const std::vector<Entry> &Table, @@ -77,19 +72,6 @@ void X86EVEX2VEXTablesEmitter::printTable(const std::vector<Entry> &Table, OS << "};\n\n"; } -void X86EVEX2VEXTablesEmitter::printCheckPredicate( - const std::vector<Predicate> &Predicates, raw_ostream &OS) { - OS << "static bool CheckVEXInstPredicate" - << "(MachineInstr &MI, const X86Subtarget *Subtarget) {\n" - << " unsigned Opc = MI.getOpcode();\n" - << " switch (Opc) {\n" - << " default: return true;\n"; - for (const auto &Pair : Predicates) - OS << " case X86::" << Pair.first << ": return " << Pair.second << ";\n"; - OS << " }\n" - << "}\n\n"; -} - // Return true if the 2 BitsInits are equal // Calculates the integer value residing BitsInit object static inline uint64_t getValueFromBitsInit(const BitsInit *B) { @@ -164,18 +146,6 @@ public: }; void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) { - auto getPredicates = [&](const CodeGenInstruction *Inst) { - std::vector<Record *> PredicatesRecords = - Inst->TheDef->getValueAsListOfDefs("Predicates"); - // Currently we only do AVX related checks and assume each instruction - // has one and only one AVX related predicates. - for (unsigned i = 0, e = PredicatesRecords.size(); i != e; ++i) - if (PredicatesRecords[i]->getName().startswith("HasAVX")) - return PredicatesRecords[i]->getValueAsString("CondString"); - llvm_unreachable( - "Instruction with checkPredicate set must have one predicate!"); - }; - emitSourceFileHeader("X86 EVEX2VEX tables", OS); ArrayRef<const CodeGenInstruction *> NumberedInstructions = @@ -186,6 +156,9 @@ void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) { // Filter non-X86 instructions. if (!Def->isSubClassOf("X86Inst")) continue; + // _REV instruction should not appear before encoding optimization + if (Def->getName().ends_with("_REV")) + continue; RecognizableInstrBase RI(*Inst); // Add VEX encoded instructions to one of VEXInsts vectors according to @@ -225,18 +198,11 @@ void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) { EVEX2VEX256.push_back(std::make_pair(EVEXInst, VEXInst)); // {0,1} else EVEX2VEX128.push_back(std::make_pair(EVEXInst, VEXInst)); // {0,0} - - // Adding predicate check to EVEX2VEXPredicates table when needed. - if (VEXInst->TheDef->getValueAsBit("checkVEXPredicate")) - EVEX2VEXPredicates.push_back( - std::make_pair(EVEXInst->TheDef->getName(), getPredicates(VEXInst))); } // Print both tables printTable(EVEX2VEX128, OS); printTable(EVEX2VEX256, OS); - // Print CheckVEXInstPredicate function. - printCheckPredicate(EVEX2VEXPredicates, OS); } } // namespace diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp index 89d93e4d3cbc..5fb6b048542b 100644 --- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp +++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp @@ -14,11 +14,11 @@ #include "CodeGenInstruction.h" #include "CodeGenTarget.h" #include "X86RecognizableInstr.h" -#include "llvm/ADT/DenseMap.h" #include "llvm/Support/FormattedStream.h" #include "llvm/Support/X86FoldTablesUtils.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" +#include <set> using namespace llvm; using namespace X86Disassembler; @@ -36,16 +36,16 @@ const char *ExplicitAlign[] = {"MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS", "MOVNTPD", "MOVNTDQ", "MOVNTDQA"}; // List of instructions NOT requiring explicit memory alignment. -const char *ExplicitUnalign[] = {"MOVDQU", "MOVUPS", "MOVUPD", - "PCMPESTRM", "PCMPESTRI", - "PCMPISTRM", "PCMPISTRI" }; +const char *ExplicitUnalign[] = {"MOVDQU", "MOVUPS", "MOVUPD", + "PCMPESTRM", "PCMPESTRI", "PCMPISTRM", + "PCMPISTRI"}; const ManualMapEntry ManualMapSet[] = { #define ENTRY(REG, MEM, FLAGS) {#REG, #MEM, FLAGS}, #include "X86ManualFoldTables.def" }; -const std::set<StringRef> NoFoldSet= { +const std::set<StringRef> NoFoldSet = { #define NOFOLD(INSN) #INSN, #include "X86ManualFoldTables.def" }; @@ -76,6 +76,16 @@ class X86FoldTablesEmitter { bool NoForward = false; bool FoldLoad = false; bool FoldStore = false; + enum BcastType { + BCAST_NONE, + BCAST_D, + BCAST_Q, + BCAST_SS, + BCAST_SD, + BCAST_SH, + }; + BcastType BroadcastKind = BCAST_NONE; + Align Alignment; X86FoldTableEntry() = default; @@ -86,7 +96,7 @@ class X86FoldTablesEmitter { void print(formatted_raw_ostream &OS) const { OS.indent(2); OS << "{X86::" << RegInst->TheDef->getName() << ", "; - OS << "X86::" << MemInst->TheDef->getName() << ", "; + OS << "X86::" << MemInst->TheDef->getName() << ", "; std::string Attrs; if (FoldLoad) @@ -99,6 +109,25 @@ class X86FoldTablesEmitter { Attrs += "TB_NO_FORWARD|"; if (Alignment != Align(1)) Attrs += "TB_ALIGN_" + std::to_string(Alignment.value()) + "|"; + switch (BroadcastKind) { + case BCAST_NONE: + break; + case BCAST_D: + Attrs += "TB_BCAST_D|"; + break; + case BCAST_Q: + Attrs += "TB_BCAST_Q|"; + break; + case BCAST_SS: + Attrs += "TB_BCAST_SS|"; + break; + case BCAST_SD: + Attrs += "TB_BCAST_SD|"; + break; + case BCAST_SH: + Attrs += "TB_BCAST_SH|"; + break; + } StringRef SimplifiedAttrs = StringRef(Attrs).rtrim("|"); if (SimplifiedAttrs.empty()) @@ -124,9 +153,10 @@ class X86FoldTablesEmitter { #endif }; - // NOTE: We check the fold tables are sorted in X86InstrFoldTables.cpp by the enum of the - // instruction, which is computed in CodeGenTarget::ComputeInstrsByEnum. So we should - // use the same comparator here. + // NOTE: We check the fold tables are sorted in X86InstrFoldTables.cpp by the + // enum of the instruction, which is computed in + // CodeGenTarget::ComputeInstrsByEnum. So we should use the same comparator + // here. // FIXME: Could we share the code with CodeGenTarget::ComputeInstrsByEnum? struct CompareInstrsByEnum { bool operator()(const CodeGenInstruction *LHS, @@ -142,16 +172,24 @@ class X86FoldTablesEmitter { typedef std::map<const CodeGenInstruction *, X86FoldTableEntry, CompareInstrsByEnum> FoldTable; - // std::vector for each folding table. - // Table2Addr - Holds instructions which their memory form performs load+store - // Table#i - Holds instructions which the their memory form perform a load OR - // a store, and their #i'th operand is folded. + // Table2Addr - Holds instructions which their memory form performs + // load+store. + // + // Table#i - Holds instructions which the their memory form + // performs a load OR a store, and their #i'th operand is folded. + // + // BroadcastTable#i - Holds instructions which the their memory form performs + // a broadcast load and their #i'th operand is folded. FoldTable Table2Addr; FoldTable Table0; FoldTable Table1; FoldTable Table2; FoldTable Table3; FoldTable Table4; + FoldTable BroadcastTable1; + FoldTable BroadcastTable2; + FoldTable BroadcastTable3; + FoldTable BroadcastTable4; public: X86FoldTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} @@ -162,22 +200,25 @@ public: private: // Decides to which table to add the entry with the given instructions. // S sets the strategy of adding the TB_NO_REVERSE flag. - void updateTables(const CodeGenInstruction *RegInstr, - const CodeGenInstruction *MemInstr, uint16_t S = 0, - bool IsManual = false); + void updateTables(const CodeGenInstruction *RegInst, + const CodeGenInstruction *MemInst, uint16_t S = 0, + bool IsManual = false, bool IsBroadcast = false); // Generates X86FoldTableEntry with the given instructions and fill it with - // the appropriate flags - then adds it to Table. - void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInstr, - const CodeGenInstruction *MemInstr, uint16_t S, - unsigned FoldedIdx, bool isManual); + // the appropriate flags, then adds it to a memory fold table. + void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInst, + const CodeGenInstruction *MemInst, uint16_t S, + unsigned FoldedIdx, bool IsManual); + // Generates X86FoldTableEntry with the given instructions and adds it to a + // broadcast table. + void addBroadcastEntry(FoldTable &Table, const CodeGenInstruction *RegInst, + const CodeGenInstruction *MemInst); // Print the given table as a static const C++ array of type - // X86MemoryFoldTableEntry. + // X86FoldTableEntry. void printTable(const FoldTable &Table, StringRef TableName, formatted_raw_ostream &OS) { - OS << "static const X86MemoryFoldTableEntry MemoryFold" << TableName - << "[] = {\n"; + OS << "static const X86FoldTableEntry " << TableName << "[] = {\n"; for (auto &E : Table) E.second.print(OS); @@ -288,11 +329,12 @@ static bool isNOREXRegClass(const Record *Op) { class IsMatch { const CodeGenInstruction *MemInst; const X86Disassembler::RecognizableInstrBase MemRI; + bool IsBroadcast; const unsigned Variant; public: - IsMatch(const CodeGenInstruction *Inst, unsigned V) - : MemInst(Inst), MemRI(*MemInst), Variant(V) {} + IsMatch(const CodeGenInstruction *Inst, bool IsBroadcast, unsigned V) + : MemInst(Inst), MemRI(*MemInst), IsBroadcast(IsBroadcast), Variant(V) {} bool operator()(const CodeGenInstruction *RegInst) { X86Disassembler::RecognizableInstrBase RegRI(*RegInst); @@ -300,7 +342,11 @@ public: const Record *MemRec = MemInst->TheDef; // EVEX_B means different things for memory and register forms. - if (RegRI.HasEVEX_B || MemRI.HasEVEX_B) + // register form: rounding control or SAE + // memory form: broadcast + if (IsBroadcast && (RegRI.HasEVEX_B || !MemRI.HasEVEX_B)) + return false; + if (!IsBroadcast && (RegRI.HasEVEX_B || MemRI.HasEVEX_B)) return false; if (!mayFoldFromLeftToRight(RegRI.Form, MemRI.Form)) @@ -388,22 +434,24 @@ public: } // end anonymous namespace void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table, - const CodeGenInstruction *RegInstr, - const CodeGenInstruction *MemInstr, + const CodeGenInstruction *RegInst, + const CodeGenInstruction *MemInst, uint16_t S, unsigned FoldedIdx, - bool isManual) { + bool IsManual) { - X86FoldTableEntry Result = X86FoldTableEntry(RegInstr, MemInstr); - Record *RegRec = RegInstr->TheDef; - Record *MemRec = MemInstr->TheDef; + assert((IsManual || Table.find(RegInst) == Table.end()) && + "Override entry unexpectedly"); + X86FoldTableEntry Result = X86FoldTableEntry(RegInst, MemInst); + Record *RegRec = RegInst->TheDef; + Record *MemRec = MemInst->TheDef; - if (isManual) { - Result.NoReverse = S & TB_NO_REVERSE; - Result.NoForward = S & TB_NO_FORWARD; - Result.FoldLoad = S & TB_FOLDED_LOAD; - Result.FoldStore = S & TB_FOLDED_STORE; - Result.Alignment = Align(1ULL << ((S & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT)); - Table[RegInstr] = Result; + Result.NoReverse = S & TB_NO_REVERSE; + Result.NoForward = S & TB_NO_FORWARD; + Result.FoldLoad = S & TB_FOLDED_LOAD; + Result.FoldStore = S & TB_FOLDED_STORE; + Result.Alignment = Align(1ULL << ((S & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT)); + if (IsManual) { + Table[RegInst] = Result; return; } @@ -422,8 +470,8 @@ void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table, Result.FoldStore = true; } - Record *RegOpRec = RegInstr->Operands[FoldedIdx].Rec; - Record *MemOpRec = MemInstr->Operands[FoldedIdx].Rec; + Record *RegOpRec = RegInst->Operands[FoldedIdx].Rec; + Record *MemOpRec = MemInst->Operands[FoldedIdx].Rec; // Unfolding code generates a load/store instruction according to the size of // the register in the register form instruction. @@ -439,22 +487,22 @@ void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table, // Check no-kz version's isMoveReg StringRef RegInstName = RegRec->getName(); unsigned DropLen = - RegInstName.endswith("rkz") ? 2 : (RegInstName.endswith("rk") ? 1 : 0); + RegInstName.ends_with("rkz") ? 2 : (RegInstName.ends_with("rk") ? 1 : 0); Record *BaseDef = DropLen ? Records.getDef(RegInstName.drop_back(DropLen)) : nullptr; bool IsMoveReg = - BaseDef ? Target.getInstruction(BaseDef).isMoveReg : RegInstr->isMoveReg; + BaseDef ? Target.getInstruction(BaseDef).isMoveReg : RegInst->isMoveReg; // A masked load can not be unfolded to a full load, otherwise it would access // unexpected memory. A simple store can not be unfolded. if (IsMoveReg && (BaseDef || Result.FoldStore)) Result.NoReverse = true; uint8_t Enc = byteFromBitsInit(RegRec->getValueAsBitsInit("OpEncBits")); - if (isExplicitAlign(RegInstr)) { + if (isExplicitAlign(RegInst)) { // The instruction require explicitly aligned memory. BitsInit *VectSize = RegRec->getValueAsBitsInit("VectSize"); Result.Alignment = Align(byteFromBitsInit(VectSize)); - } else if (!Enc && !isExplicitUnalign(RegInstr) && + } else if (!Enc && !isExplicitUnalign(RegInst) && getMemOperandSize(MemOpRec) > 64) { // Instructions with XOP/VEX/EVEX encoding do not require alignment while // SSE packed vector instructions require a 16 byte alignment. @@ -467,15 +515,75 @@ void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table, if (RegRec->getName().contains("EXPAND")) Result.NoReverse = true; - Table[RegInstr] = Result; + Table[RegInst] = Result; +} + +void X86FoldTablesEmitter::addBroadcastEntry( + FoldTable &Table, const CodeGenInstruction *RegInst, + const CodeGenInstruction *MemInst) { + + assert(Table.find(RegInst) == Table.end() && "Override entry unexpectedly"); + X86FoldTableEntry Result = X86FoldTableEntry(RegInst, MemInst); + + Record *RegRec = RegInst->TheDef; + StringRef RegInstName = RegRec->getName(); + StringRef MemInstName = MemInst->TheDef->getName(); + Record *Domain = RegRec->getValueAsDef("ExeDomain"); + bool IsSSEPackedInt = Domain->getName() == "SSEPackedInt"; + // TODO: Rename AVX512 instructions to simplify conditions, e.g. + // D128 -> DZ128 + // D256 -> DZ256 + // VPERMI2Drr -> VPERMI2DZrr + // VPERMI2Drmb -> VPERMI2DZrmb + if ((RegInstName.contains("DZ") || RegInstName.contains("DWZ") || + RegInstName.contains("D128") || RegInstName.contains("D256") || + RegInstName.contains("Dr") || RegInstName.contains("I32")) && + IsSSEPackedInt) { + assert((MemInstName.contains("DZ") || RegInstName.contains("DWZ") || + MemInstName.contains("D128") || MemInstName.contains("D256") || + MemInstName.contains("Dr") || MemInstName.contains("I32")) && + "Unmatched names for broadcast"); + Result.BroadcastKind = X86FoldTableEntry::BCAST_D; + } else if ((RegInstName.contains("QZ") || RegInstName.contains("QBZ") || + RegInstName.contains("Q128") || RegInstName.contains("Q256") || + RegInstName.contains("Qr") || RegInstName.contains("I64")) && + IsSSEPackedInt) { + assert((MemInstName.contains("QZ") || MemInstName.contains("QBZ") || + MemInstName.contains("Q128") || MemInstName.contains("Q256") || + MemInstName.contains("Qr") || MemInstName.contains("I64")) && + "Unmatched names for broadcast"); + Result.BroadcastKind = X86FoldTableEntry::BCAST_Q; + } else if ((RegInstName.contains("PS") || RegInstName.contains("F32") || + RegInstName.contains("CPH")) && + !RegInstName.contains("PH2PS")) { + assert((MemInstName.contains("PS") || MemInstName.contains("F32") || + MemInstName.contains("CPH")) && + "Unmatched names for broadcast"); + Result.BroadcastKind = X86FoldTableEntry::BCAST_SS; + } else if ((RegInstName.contains("PD") || RegInstName.contains("F64")) && + !RegInstName.contains("PH2PD")) { + assert((MemInstName.contains("PD") || MemInstName.contains("F64")) && + "Unmatched names for broadcast"); + Result.BroadcastKind = X86FoldTableEntry::BCAST_SD; + } else if (RegInstName.contains("PH")) { + assert(MemInstName.contains("PH") && "Unmatched names for broadcast"); + Result.BroadcastKind = X86FoldTableEntry::BCAST_SH; + } else { + errs() << RegInstName << ", " << MemInstName << "\n"; + llvm_unreachable("Name is not canoicalized for broadcast or " + "ExeDomain is incorrect"); + } + + Table[RegInst] = Result; } -void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr, - const CodeGenInstruction *MemInstr, - uint16_t S, bool IsManual) { +void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInst, + const CodeGenInstruction *MemInst, + uint16_t S, bool IsManual, + bool IsBroadcast) { - Record *RegRec = RegInstr->TheDef; - Record *MemRec = MemInstr->TheDef; + Record *RegRec = RegInst->TheDef; + Record *MemRec = MemInst->TheDef; unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); @@ -483,7 +591,10 @@ void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr, // Instructions which Read-Modify-Write should be added to Table2Addr. if (!MemOutSize && RegOutSize == 1 && MemInSize == RegInSize) { - addEntryWithFlags(Table2Addr, RegInstr, MemInstr, S, 0, IsManual); + assert(!IsBroadcast && "Read-Modify-Write can not be broadcast"); + // X86 would not unfold Read-Modify-Write instructions so add TB_NO_REVERSE. + addEntryWithFlags(Table2Addr, RegInst, MemInst, S | TB_NO_REVERSE, 0, + IsManual); return; } @@ -491,28 +602,38 @@ void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr, // Load-Folding cases. // If the i'th register form operand is a register and the i'th memory form // operand is a memory operand, add instructions to Table#i. - for (unsigned i = RegOutSize, e = RegInstr->Operands.size(); i < e; i++) { - Record *RegOpRec = RegInstr->Operands[i].Rec; - Record *MemOpRec = MemInstr->Operands[i].Rec; - // PointerLikeRegClass: For instructions like TAILJMPr, TAILJMPr64, TAILJMPr64_REX + for (unsigned I = RegOutSize, E = RegInst->Operands.size(); I < E; I++) { + Record *RegOpRec = RegInst->Operands[I].Rec; + Record *MemOpRec = MemInst->Operands[I].Rec; + // PointerLikeRegClass: For instructions like TAILJMPr, TAILJMPr64, + // TAILJMPr64_REX if ((isRegisterOperand(RegOpRec) || RegOpRec->isSubClassOf("PointerLikeRegClass")) && isMemoryOperand(MemOpRec)) { - switch (i) { + switch (I) { case 0: - addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0, IsManual); + assert(!IsBroadcast && "BroadcastTable0 needs to be added"); + addEntryWithFlags(Table0, RegInst, MemInst, S, 0, IsManual); return; case 1: - addEntryWithFlags(Table1, RegInstr, MemInstr, S, 1, IsManual); + IsBroadcast + ? addBroadcastEntry(BroadcastTable1, RegInst, MemInst) + : addEntryWithFlags(Table1, RegInst, MemInst, S, 1, IsManual); return; case 2: - addEntryWithFlags(Table2, RegInstr, MemInstr, S, 2, IsManual); + IsBroadcast + ? addBroadcastEntry(BroadcastTable2, RegInst, MemInst) + : addEntryWithFlags(Table2, RegInst, MemInst, S, 2, IsManual); return; case 3: - addEntryWithFlags(Table3, RegInstr, MemInstr, S, 3, IsManual); + IsBroadcast + ? addBroadcastEntry(BroadcastTable3, RegInst, MemInst) + : addEntryWithFlags(Table3, RegInst, MemInst, S, 3, IsManual); return; case 4: - addEntryWithFlags(Table4, RegInstr, MemInstr, S, 4, IsManual); + IsBroadcast + ? addBroadcastEntry(BroadcastTable4, RegInst, MemInst) + : addEntryWithFlags(Table4, RegInst, MemInst, S, 4, IsManual); return; } } @@ -525,16 +646,18 @@ void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr, // For example: // MOVAPSrr => (outs VR128:$dst), (ins VR128:$src) // MOVAPSmr => (outs), (ins f128mem:$dst, VR128:$src) - Record *RegOpRec = RegInstr->Operands[RegOutSize - 1].Rec; - Record *MemOpRec = MemInstr->Operands[RegOutSize - 1].Rec; + Record *RegOpRec = RegInst->Operands[RegOutSize - 1].Rec; + Record *MemOpRec = MemInst->Operands[RegOutSize - 1].Rec; if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec) && - getRegOperandSize(RegOpRec) == getMemOperandSize(MemOpRec)) - addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0, IsManual); + getRegOperandSize(RegOpRec) == getMemOperandSize(MemOpRec)) { + assert(!IsBroadcast && "Store can not be broadcast"); + addEntryWithFlags(Table0, RegInst, MemInst, S, 0, IsManual); + } } } -void X86FoldTablesEmitter::run(raw_ostream &o) { - formatted_raw_ostream OS(o); +void X86FoldTablesEmitter::run(raw_ostream &O) { + formatted_raw_ostream OS(O); // Holds all memory instructions std::vector<const CodeGenInstruction *> MemInsts; @@ -575,8 +698,19 @@ void X86FoldTablesEmitter::run(raw_ostream &o) { } } + // Create a copy b/c the register instruction will removed when a new entry is + // added into memory fold tables. + auto RegInstsForBroadcast = RegInsts; + Record *AsmWriter = Target.getAsmWriter(); unsigned Variant = AsmWriter->getValueAsInt("Variant"); + auto FixUp = [&](const CodeGenInstruction *RegInst) { + StringRef RegInstName = RegInst->TheDef->getName(); + if (RegInstName.ends_with("_REV") || RegInstName.ends_with("_alt")) + if (auto *RegAltRec = Records.getDef(RegInstName.drop_back(4))) + RegInst = &Target.getInstruction(RegAltRec); + return RegInst; + }; // For each memory form instruction, try to find its register form // instruction. for (const CodeGenInstruction *MemInst : MemInsts) { @@ -588,22 +722,33 @@ void X86FoldTablesEmitter::run(raw_ostream &o) { continue; // Two forms (memory & register) of the same instruction must have the same - // opcode. try matching only with register form instructions with the same // opcode. std::vector<const CodeGenInstruction *> &OpcRegInsts = RegInstsIt->second; - auto Match = find_if(OpcRegInsts, IsMatch(MemInst, Variant)); + // Memory fold tables + auto Match = + find_if(OpcRegInsts, IsMatch(MemInst, /*IsBroadcast=*/false, Variant)); if (Match != OpcRegInsts.end()) { - const CodeGenInstruction *RegInst = *Match; - StringRef RegInstName = RegInst->TheDef->getName(); - if (RegInstName.endswith("_REV") || RegInstName.endswith("_alt")) { - if (auto *RegAltRec = Records.getDef(RegInstName.drop_back(4))) { - RegInst = &Target.getInstruction(RegAltRec); - } - } - updateTables(RegInst, MemInst); + updateTables(FixUp(*Match), MemInst); OpcRegInsts.erase(Match); } + + // Broadcast tables + StringRef MemInstName = MemInst->TheDef->getName(); + if (!MemInstName.contains("mb") && !MemInstName.contains("mib")) + continue; + RegInstsIt = RegInstsForBroadcast.find(Opc); + assert(RegInstsIt != RegInstsForBroadcast.end() && + "Unexpected control flow"); + std::vector<const CodeGenInstruction *> &OpcRegInstsForBroadcast = + RegInstsIt->second; + Match = find_if(OpcRegInstsForBroadcast, + IsMatch(MemInst, /*IsBroadcast=*/true, Variant)); + if (Match != OpcRegInstsForBroadcast.end()) { + updateTables(FixUp(*Match), MemInst, 0, /*IsManual=*/false, + /*IsBroadcast=*/true); + OpcRegInstsForBroadcast.erase(Match); + } } // Add the manually mapped instructions listed above. @@ -628,14 +773,23 @@ void X86FoldTablesEmitter::run(raw_ostream &o) { CheckMemFoldTable(Table2); CheckMemFoldTable(Table3); CheckMemFoldTable(Table4); + CheckMemFoldTable(BroadcastTable1); + CheckMemFoldTable(BroadcastTable2); + CheckMemFoldTable(BroadcastTable3); + CheckMemFoldTable(BroadcastTable4); #endif +#define PRINT_TABLE(TABLE) printTable(TABLE, #TABLE, OS); // Print all tables. - printTable(Table2Addr, "Table2Addr", OS); - printTable(Table0, "Table0", OS); - printTable(Table1, "Table1", OS); - printTable(Table2, "Table2", OS); - printTable(Table3, "Table3", OS); - printTable(Table4, "Table4", OS); + PRINT_TABLE(Table2Addr) + PRINT_TABLE(Table0) + PRINT_TABLE(Table1) + PRINT_TABLE(Table2) + PRINT_TABLE(Table3) + PRINT_TABLE(Table4) + PRINT_TABLE(BroadcastTable1) + PRINT_TABLE(BroadcastTable2) + PRINT_TABLE(BroadcastTable3) + PRINT_TABLE(BroadcastTable4) } static TableGen::Emitter::OptClass<X86FoldTablesEmitter> diff --git a/llvm/utils/TableGen/X86ManualFoldTables.def b/llvm/utils/TableGen/X86ManualFoldTables.def index d949830b0988..8e6cb4a7bd87 100644 --- a/llvm/utils/TableGen/X86ManualFoldTables.def +++ b/llvm/utils/TableGen/X86ManualFoldTables.def @@ -225,6 +225,8 @@ NOFOLD(MMX_MOVQ64rr_REV) // => // insertpsrm xmm1, m32, imm NOFOLD(INSERTPSrr) +NOFOLD(VINSERTPSZrr) +NOFOLD(VINSERTPSrr) #undef NOFOLD #ifndef ENTRY diff --git a/llvm/utils/TableGen/X86RecognizableInstr.cpp b/llvm/utils/TableGen/X86RecognizableInstr.cpp index b2f51ba01689..6e03fc11d6d9 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.cpp +++ b/llvm/utils/TableGen/X86RecognizableInstr.cpp @@ -24,18 +24,19 @@ using namespace llvm; using namespace X86Disassembler; -std::string X86Disassembler::getMnemonic(const CodeGenInstruction *I, unsigned Variant) { - std::string AsmString = I->FlattenAsmStringVariants(I->AsmString, Variant); - StringRef Mnemonic(AsmString); - // Extract a mnemonic assuming it's separated by \t - Mnemonic = Mnemonic.take_until([](char C) { return C == '\t'; }); +std::string X86Disassembler::getMnemonic(const CodeGenInstruction *I, + unsigned Variant) { + std::string AsmString = I->FlattenAsmStringVariants(I->AsmString, Variant); + StringRef Mnemonic(AsmString); + // Extract a mnemonic assuming it's separated by \t + Mnemonic = Mnemonic.take_until([](char C) { return C == '\t'; }); - // Special case: CMOVCC, JCC, SETCC have "${cond}" in mnemonic. - // Replace it with "CC" in-place. - size_t CondPos = Mnemonic.find("${cond}"); - if (CondPos != StringRef::npos) - Mnemonic = AsmString.replace(CondPos, StringRef::npos, "CC"); - return Mnemonic.upper(); + // Special case: CMOVCC, JCC, SETCC have "${cond}" in mnemonic. + // Replace it with "CC" in-place. + size_t CondPos = Mnemonic.find("${cond}"); + if (CondPos != StringRef::npos) + Mnemonic = AsmString.replace(CondPos, StringRef::npos, "CC"); + return Mnemonic.upper(); } bool X86Disassembler::isRegisterOperand(const Record *Rec) { @@ -80,7 +81,7 @@ static uint8_t byteFromBitsInit(BitsInit &init) { assert(width <= 8 && "Field is too large for uint8_t!"); - int index; + int index; uint8_t mask = 0x01; uint8_t ret = 0; @@ -101,8 +102,8 @@ static uint8_t byteFromBitsInit(BitsInit &init) { /// @param rec - The record from which to extract the value. /// @param name - The name of the field in the record. /// @return - The field, as translated by byteFromBitsInit(). -static uint8_t byteFromRec(const Record* rec, StringRef name) { - BitsInit* bits = rec->getValueAsBitsInit(name); +static uint8_t byteFromRec(const Record *rec, StringRef name) { + BitsInit *bits = rec->getValueAsBitsInit(name); return byteFromBitsInit(*bits); } @@ -129,6 +130,8 @@ RecognizableInstrBase::RecognizableInstrBase(const CodeGenInstruction &insn) { ForceDisassemble = Rec->getValueAsBit("ForceDisassemble"); CD8_Scale = byteFromRec(Rec, "CD8_Scale"); HasVEX_L = Rec->getValueAsBit("hasVEX_L"); + ExplicitREX2Prefix = + byteFromRec(Rec, "explicitOpPrefixBits") == X86Local::ExplicitREX2; EncodeRC = HasEVEX_B && (Form == X86Local::MRMDestReg || Form == X86Local::MRMSrcReg); @@ -174,10 +177,13 @@ void RecognizableInstr::processInstr(DisassemblerTables &tables, recogInstr.emitDecodePath(tables); } -#define EVEX_KB(n) (HasEVEX_KZ && HasEVEX_B ? n##_KZ_B : \ - (HasEVEX_K && HasEVEX_B ? n##_K_B : \ - (HasEVEX_KZ ? n##_KZ : \ - (HasEVEX_K? n##_K : (HasEVEX_B ? n##_B : n))))) +#define EVEX_KB(n) \ + (HasEVEX_KZ && HasEVEX_B \ + ? n##_KZ_B \ + : (HasEVEX_K && HasEVEX_B \ + ? n##_K_B \ + : (HasEVEX_KZ ? n##_KZ \ + : (HasEVEX_K ? n##_K : (HasEVEX_B ? n##_B : n))))) InstructionContext RecognizableInstr::insnContext() const { InstructionContext insnContext; @@ -243,8 +249,7 @@ InstructionContext RecognizableInstr::insnContext() const { errs() << "Instruction does not use a prefix: " << Name << "\n"; llvm_unreachable("Invalid prefix"); } - } - else if (HasREX_W) { + } else if (HasREX_W) { // VEX_W if (OpPrefix == X86Local::PD) insnContext = EVEX_KB(IC_EVEX_W_OPSIZE); @@ -340,6 +345,8 @@ InstructionContext RecognizableInstr::insnContext() const { insnContext = IC_64BIT_XD; else if (OpPrefix == X86Local::XS) insnContext = IC_64BIT_XS; + else if (ExplicitREX2Prefix) + insnContext = IC_64BIT_REX2; else if (HasREX_W) insnContext = IC_64BIT_REXW; else @@ -376,9 +383,9 @@ void RecognizableInstr::adjustOperandEncoding(OperandEncoding &encoding) { // The scaling factor for AVX512 compressed displacement encoding is an // instruction attribute. Adjust the ModRM encoding type to include the // scale for compressed displacement. - if ((encoding != ENCODING_RM && - encoding != ENCODING_VSIB && - encoding != ENCODING_SIB) ||CD8_Scale == 0) + if ((encoding != ENCODING_RM && encoding != ENCODING_VSIB && + encoding != ENCODING_SIB) || + CD8_Scale == 0) return; encoding = (OperandEncoding)(encoding + Log2_32(CD8_Scale)); assert(((encoding >= ENCODING_RM && encoding <= ENCODING_RM_CD64) || @@ -387,13 +394,11 @@ void RecognizableInstr::adjustOperandEncoding(OperandEncoding &encoding) { "Invalid CDisp scaling"); } -void RecognizableInstr::handleOperand(bool optional, unsigned &operandIndex, - unsigned &physicalOperandIndex, - unsigned numPhysicalOperands, - const unsigned *operandMapping, - OperandEncoding (*encodingFromString) - (const std::string&, - uint8_t OpSize)) { +void RecognizableInstr::handleOperand( + bool optional, unsigned &operandIndex, unsigned &physicalOperandIndex, + unsigned numPhysicalOperands, const unsigned *operandMapping, + OperandEncoding (*encodingFromString)(const std::string &, + uint8_t OpSize)) { if (optional) { if (physicalOperandIndex >= numPhysicalOperands) return; @@ -404,7 +409,7 @@ void RecognizableInstr::handleOperand(bool optional, unsigned &operandIndex, while (operandMapping[operandIndex] != operandIndex) { Spec->operands[operandIndex].encoding = ENCODING_DUP; Spec->operands[operandIndex].type = - (OperandType)(TYPE_DUP0 + operandMapping[operandIndex]); + (OperandType)(TYPE_DUP0 + operandMapping[operandIndex]); ++operandIndex; } @@ -422,7 +427,7 @@ void RecognizableInstr::handleOperand(bool optional, unsigned &operandIndex, } void RecognizableInstr::emitInstructionSpecifier() { - Spec->name = Name; + Spec->name = Name; Spec->insnContext = insnContext(); @@ -434,12 +439,13 @@ void RecognizableInstr::emitInstructionSpecifier() { // operandMapping maps from operands in OperandList to their originals. // If operandMapping[i] != i, then the entry is a duplicate. unsigned operandMapping[X86_MAX_OPERANDS]; - assert(numOperands <= X86_MAX_OPERANDS && "X86_MAX_OPERANDS is not large enough"); + assert(numOperands <= X86_MAX_OPERANDS && + "X86_MAX_OPERANDS is not large enough"); for (unsigned operandIndex = 0; operandIndex < numOperands; ++operandIndex) { if (!OperandList[operandIndex].Constraints.empty()) { const CGIOperandList::ConstraintInfo &Constraint = - OperandList[operandIndex].Constraints[0]; + OperandList[operandIndex].Constraints[0]; if (Constraint.isTied()) { operandMapping[operandIndex] = operandIndex; operandMapping[Constraint.getTiedOperand()] = operandIndex; @@ -453,21 +459,14 @@ void RecognizableInstr::emitInstructionSpecifier() { } } -#define HANDLE_OPERAND(class) \ - handleOperand(false, \ - operandIndex, \ - physicalOperandIndex, \ - numPhysicalOperands, \ - operandMapping, \ +#define HANDLE_OPERAND(class) \ + handleOperand(false, operandIndex, physicalOperandIndex, \ + numPhysicalOperands, operandMapping, \ class##EncodingFromString); -#define HANDLE_OPTIONAL(class) \ - handleOperand(true, \ - operandIndex, \ - physicalOperandIndex, \ - numPhysicalOperands, \ - operandMapping, \ - class##EncodingFromString); +#define HANDLE_OPTIONAL(class) \ + handleOperand(true, operandIndex, physicalOperandIndex, numPhysicalOperands, \ + operandMapping, class##EncodingFromString); // operandIndex should always be < numOperands unsigned operandIndex = 0; @@ -485,7 +484,8 @@ void RecognizableInstr::emitInstructionSpecifier() { #endif switch (Form) { - default: llvm_unreachable("Unhandled form"); + default: + llvm_unreachable("Unhandled form"); case X86Local::PrefixByte: return; case X86Local::RawFrmSrc: @@ -762,7 +762,7 @@ void RecognizableInstr::emitInstructionSpecifier() { case X86Local::MRM6X: case X86Local::MRM7X: #define MAP(from, to) case X86Local::MRM_##from: - X86_INSTR_MRM_MAPPING + X86_INSTR_MRM_MAPPING #undef MAP HANDLE_OPTIONAL(relocation) break; @@ -775,28 +775,56 @@ void RecognizableInstr::emitInstructionSpecifier() { void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const { // Special cases where the LLVM tables are not complete -#define MAP(from, to) \ - case X86Local::MRM_##from: +#define MAP(from, to) case X86Local::MRM_##from: std::optional<OpcodeType> opcodeType; switch (OpMap) { - default: llvm_unreachable("Invalid map!"); - case X86Local::OB: opcodeType = ONEBYTE; break; - case X86Local::TB: opcodeType = TWOBYTE; break; - case X86Local::T8: opcodeType = THREEBYTE_38; break; - case X86Local::TA: opcodeType = THREEBYTE_3A; break; - case X86Local::XOP8: opcodeType = XOP8_MAP; break; - case X86Local::XOP9: opcodeType = XOP9_MAP; break; - case X86Local::XOPA: opcodeType = XOPA_MAP; break; - case X86Local::ThreeDNow: opcodeType = THREEDNOW_MAP; break; - case X86Local::T_MAP5: opcodeType = MAP5; break; - case X86Local::T_MAP6: opcodeType = MAP6; break; + default: + llvm_unreachable("Invalid map!"); + case X86Local::OB: + opcodeType = ONEBYTE; + break; + case X86Local::TB: + opcodeType = TWOBYTE; + break; + case X86Local::T8: + opcodeType = THREEBYTE_38; + break; + case X86Local::TA: + opcodeType = THREEBYTE_3A; + break; + case X86Local::XOP8: + opcodeType = XOP8_MAP; + break; + case X86Local::XOP9: + opcodeType = XOP9_MAP; + break; + case X86Local::XOPA: + opcodeType = XOPA_MAP; + break; + case X86Local::ThreeDNow: + opcodeType = THREEDNOW_MAP; + break; + case X86Local::T_MAP4: + opcodeType = MAP4; + break; + case X86Local::T_MAP5: + opcodeType = MAP5; + break; + case X86Local::T_MAP6: + opcodeType = MAP6; + break; + case X86Local::T_MAP7: + opcodeType = MAP7; + break; } std::unique_ptr<ModRMFilter> filter; switch (Form) { - default: llvm_unreachable("Invalid form!"); - case X86Local::Pseudo: llvm_unreachable("Pseudo should not be emitted!"); + default: + llvm_unreachable("Invalid form!"); + case X86Local::Pseudo: + llvm_unreachable("Pseudo should not be emitted!"); case X86Local::RawFrm: case X86Local::AddRegFrm: case X86Local::RawFrmMemOffs: @@ -830,28 +858,40 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const { case X86Local::MRMXm: filter = std::make_unique<ModFilter>(false); break; - case X86Local::MRM0r: case X86Local::MRM1r: - case X86Local::MRM2r: case X86Local::MRM3r: - case X86Local::MRM4r: case X86Local::MRM5r: - case X86Local::MRM6r: case X86Local::MRM7r: + case X86Local::MRM0r: + case X86Local::MRM1r: + case X86Local::MRM2r: + case X86Local::MRM3r: + case X86Local::MRM4r: + case X86Local::MRM5r: + case X86Local::MRM6r: + case X86Local::MRM7r: filter = std::make_unique<ExtendedFilter>(true, Form - X86Local::MRM0r); break; - case X86Local::MRM0X: case X86Local::MRM1X: - case X86Local::MRM2X: case X86Local::MRM3X: - case X86Local::MRM4X: case X86Local::MRM5X: - case X86Local::MRM6X: case X86Local::MRM7X: + case X86Local::MRM0X: + case X86Local::MRM1X: + case X86Local::MRM2X: + case X86Local::MRM3X: + case X86Local::MRM4X: + case X86Local::MRM5X: + case X86Local::MRM6X: + case X86Local::MRM7X: filter = std::make_unique<ExtendedFilter>(true, Form - X86Local::MRM0X); break; case X86Local::MRMr0: filter = std::make_unique<ExtendedRMFilter>(true, Form - X86Local::MRMr0); break; - case X86Local::MRM0m: case X86Local::MRM1m: - case X86Local::MRM2m: case X86Local::MRM3m: - case X86Local::MRM4m: case X86Local::MRM5m: - case X86Local::MRM6m: case X86Local::MRM7m: + case X86Local::MRM0m: + case X86Local::MRM1m: + case X86Local::MRM2m: + case X86Local::MRM3m: + case X86Local::MRM4m: + case X86Local::MRM5m: + case X86Local::MRM6m: + case X86Local::MRM7m: filter = std::make_unique<ExtendedFilter>(false, Form - X86Local::MRM0m); break; - X86_INSTR_MRM_MAPPING + X86_INSTR_MRM_MAPPING filter = std::make_unique<ExactFilter>(0xC0 + Form - X86Local::MRM_C0); break; } // switch (Form) @@ -860,9 +900,15 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const { unsigned AddressSize = 0; switch (AdSize) { - case X86Local::AdSize16: AddressSize = 16; break; - case X86Local::AdSize32: AddressSize = 32; break; - case X86Local::AdSize64: AddressSize = 64; break; + case X86Local::AdSize16: + AddressSize = 16; + break; + case X86Local::AdSize32: + AddressSize = 32; + break; + case X86Local::AdSize64: + AddressSize = 64; + break; } assert(opcodeType && "Opcode type not set"); @@ -881,8 +927,7 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const { currentOpcode < (uint8_t)(opcodeToSet + Count); ++currentOpcode) tables.setTableFields(*opcodeType, insnContext(), currentOpcode, *filter, UID, Is32Bit, OpPrefix == 0, - IgnoresVEX_L || EncodeRC, - IgnoresW, AddressSize); + IgnoresVEX_L || EncodeRC, IgnoresW, AddressSize); } else { tables.setTableFields(*opcodeType, insnContext(), opcodeToSet, *filter, UID, Is32Bit, OpPrefix == 0, IgnoresVEX_L || EncodeRC, @@ -892,187 +937,190 @@ void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const { #undef MAP } -#define TYPE(str, type) if (s == str) return type; +#define TYPE(str, type) \ + if (s == str) \ + return type; OperandType RecognizableInstr::typeFromString(const std::string &s, - bool hasREX_W, - uint8_t OpSize) { - if(hasREX_W) { + bool hasREX_W, uint8_t OpSize) { + if (hasREX_W) { // For instructions with a REX_W prefix, a declared 32-bit register encoding // is special. - TYPE("GR32", TYPE_R32) + TYPE("GR32", TYPE_R32) } - if(OpSize == X86Local::OpSize16) { + if (OpSize == X86Local::OpSize16) { // For OpSize16 instructions, a declared 16-bit register or // immediate encoding is special. - TYPE("GR16", TYPE_Rv) - } else if(OpSize == X86Local::OpSize32) { + TYPE("GR16", TYPE_Rv) + } else if (OpSize == X86Local::OpSize32) { // For OpSize32 instructions, a declared 32-bit register or // immediate encoding is special. - TYPE("GR32", TYPE_Rv) + TYPE("GR32", TYPE_Rv) } - TYPE("i16mem", TYPE_M) - TYPE("i16imm", TYPE_IMM) - TYPE("i16i8imm", TYPE_IMM) - TYPE("GR16", TYPE_R16) - TYPE("GR16orGR32orGR64", TYPE_R16) - TYPE("i32mem", TYPE_M) - TYPE("i32imm", TYPE_IMM) - TYPE("i32i8imm", TYPE_IMM) - TYPE("GR32", TYPE_R32) - TYPE("GR32orGR64", TYPE_R32) - TYPE("i64mem", TYPE_M) - TYPE("i64i32imm", TYPE_IMM) - TYPE("i64i8imm", TYPE_IMM) - TYPE("GR64", TYPE_R64) - TYPE("i8mem", TYPE_M) - TYPE("i8imm", TYPE_IMM) - TYPE("u4imm", TYPE_UIMM8) - TYPE("u8imm", TYPE_UIMM8) - TYPE("i16u8imm", TYPE_UIMM8) - TYPE("i32u8imm", TYPE_UIMM8) - TYPE("i64u8imm", TYPE_UIMM8) - TYPE("GR8", TYPE_R8) - TYPE("VR128", TYPE_XMM) - TYPE("VR128X", TYPE_XMM) - TYPE("f128mem", TYPE_M) - TYPE("f256mem", TYPE_M) - TYPE("f512mem", TYPE_M) - TYPE("FR128", TYPE_XMM) - TYPE("FR64", TYPE_XMM) - TYPE("FR64X", TYPE_XMM) - TYPE("f64mem", TYPE_M) - TYPE("sdmem", TYPE_M) - TYPE("FR16X", TYPE_XMM) - TYPE("FR32", TYPE_XMM) - TYPE("FR32X", TYPE_XMM) - TYPE("f32mem", TYPE_M) - TYPE("f16mem", TYPE_M) - TYPE("ssmem", TYPE_M) - TYPE("shmem", TYPE_M) - TYPE("RST", TYPE_ST) - TYPE("RSTi", TYPE_ST) - TYPE("i128mem", TYPE_M) - TYPE("i256mem", TYPE_M) - TYPE("i512mem", TYPE_M) - TYPE("i512mem_GR16", TYPE_M) - TYPE("i512mem_GR32", TYPE_M) - TYPE("i512mem_GR64", TYPE_M) - TYPE("i64i32imm_brtarget", TYPE_REL) - TYPE("i16imm_brtarget", TYPE_REL) - TYPE("i32imm_brtarget", TYPE_REL) - TYPE("ccode", TYPE_IMM) - TYPE("AVX512RC", TYPE_IMM) - TYPE("brtarget32", TYPE_REL) - TYPE("brtarget16", TYPE_REL) - TYPE("brtarget8", TYPE_REL) - TYPE("f80mem", TYPE_M) - TYPE("lea64_32mem", TYPE_M) - TYPE("lea64mem", TYPE_M) - TYPE("VR64", TYPE_MM64) - TYPE("i64imm", TYPE_IMM) - TYPE("anymem", TYPE_M) - TYPE("opaquemem", TYPE_M) - TYPE("sibmem", TYPE_MSIB) - TYPE("SEGMENT_REG", TYPE_SEGMENTREG) - TYPE("DEBUG_REG", TYPE_DEBUGREG) - TYPE("CONTROL_REG", TYPE_CONTROLREG) - TYPE("srcidx8", TYPE_SRCIDX) - TYPE("srcidx16", TYPE_SRCIDX) - TYPE("srcidx32", TYPE_SRCIDX) - TYPE("srcidx64", TYPE_SRCIDX) - TYPE("dstidx8", TYPE_DSTIDX) - TYPE("dstidx16", TYPE_DSTIDX) - TYPE("dstidx32", TYPE_DSTIDX) - TYPE("dstidx64", TYPE_DSTIDX) - TYPE("offset16_8", TYPE_MOFFS) - TYPE("offset16_16", TYPE_MOFFS) - TYPE("offset16_32", TYPE_MOFFS) - TYPE("offset32_8", TYPE_MOFFS) - TYPE("offset32_16", TYPE_MOFFS) - TYPE("offset32_32", TYPE_MOFFS) - TYPE("offset32_64", TYPE_MOFFS) - TYPE("offset64_8", TYPE_MOFFS) - TYPE("offset64_16", TYPE_MOFFS) - TYPE("offset64_32", TYPE_MOFFS) - TYPE("offset64_64", TYPE_MOFFS) - TYPE("VR256", TYPE_YMM) - TYPE("VR256X", TYPE_YMM) - TYPE("VR512", TYPE_ZMM) - TYPE("VK1", TYPE_VK) - TYPE("VK1WM", TYPE_VK) - TYPE("VK2", TYPE_VK) - TYPE("VK2WM", TYPE_VK) - TYPE("VK4", TYPE_VK) - TYPE("VK4WM", TYPE_VK) - TYPE("VK8", TYPE_VK) - TYPE("VK8WM", TYPE_VK) - TYPE("VK16", TYPE_VK) - TYPE("VK16WM", TYPE_VK) - TYPE("VK32", TYPE_VK) - TYPE("VK32WM", TYPE_VK) - TYPE("VK64", TYPE_VK) - TYPE("VK64WM", TYPE_VK) - TYPE("VK1Pair", TYPE_VK_PAIR) - TYPE("VK2Pair", TYPE_VK_PAIR) - TYPE("VK4Pair", TYPE_VK_PAIR) - TYPE("VK8Pair", TYPE_VK_PAIR) - TYPE("VK16Pair", TYPE_VK_PAIR) - TYPE("vx64mem", TYPE_MVSIBX) - TYPE("vx128mem", TYPE_MVSIBX) - TYPE("vx256mem", TYPE_MVSIBX) - TYPE("vy128mem", TYPE_MVSIBY) - TYPE("vy256mem", TYPE_MVSIBY) - TYPE("vx64xmem", TYPE_MVSIBX) - TYPE("vx128xmem", TYPE_MVSIBX) - TYPE("vx256xmem", TYPE_MVSIBX) - TYPE("vy128xmem", TYPE_MVSIBY) - TYPE("vy256xmem", TYPE_MVSIBY) - TYPE("vy512xmem", TYPE_MVSIBY) - TYPE("vz256mem", TYPE_MVSIBZ) - TYPE("vz512mem", TYPE_MVSIBZ) - TYPE("BNDR", TYPE_BNDR) - TYPE("TILE", TYPE_TMM) + TYPE("i16mem", TYPE_M) + TYPE("i16imm", TYPE_IMM) + TYPE("i16i8imm", TYPE_IMM) + TYPE("GR16", TYPE_R16) + TYPE("GR16orGR32orGR64", TYPE_R16) + TYPE("i32mem", TYPE_M) + TYPE("i32imm", TYPE_IMM) + TYPE("i32i8imm", TYPE_IMM) + TYPE("GR32", TYPE_R32) + TYPE("GR32orGR64", TYPE_R32) + TYPE("i64mem", TYPE_M) + TYPE("i64i32imm", TYPE_IMM) + TYPE("i64i8imm", TYPE_IMM) + TYPE("GR64", TYPE_R64) + TYPE("i8mem", TYPE_M) + TYPE("i8imm", TYPE_IMM) + TYPE("u4imm", TYPE_UIMM8) + TYPE("u8imm", TYPE_UIMM8) + TYPE("i16u8imm", TYPE_UIMM8) + TYPE("i32u8imm", TYPE_UIMM8) + TYPE("i64u8imm", TYPE_UIMM8) + TYPE("GR8", TYPE_R8) + TYPE("VR128", TYPE_XMM) + TYPE("VR128X", TYPE_XMM) + TYPE("f128mem", TYPE_M) + TYPE("f256mem", TYPE_M) + TYPE("f512mem", TYPE_M) + TYPE("FR128", TYPE_XMM) + TYPE("FR64", TYPE_XMM) + TYPE("FR64X", TYPE_XMM) + TYPE("f64mem", TYPE_M) + TYPE("sdmem", TYPE_M) + TYPE("FR16X", TYPE_XMM) + TYPE("FR32", TYPE_XMM) + TYPE("FR32X", TYPE_XMM) + TYPE("f32mem", TYPE_M) + TYPE("f16mem", TYPE_M) + TYPE("ssmem", TYPE_M) + TYPE("shmem", TYPE_M) + TYPE("RST", TYPE_ST) + TYPE("RSTi", TYPE_ST) + TYPE("i128mem", TYPE_M) + TYPE("i256mem", TYPE_M) + TYPE("i512mem", TYPE_M) + TYPE("i512mem_GR16", TYPE_M) + TYPE("i512mem_GR32", TYPE_M) + TYPE("i512mem_GR64", TYPE_M) + TYPE("i64i32imm_brtarget", TYPE_REL) + TYPE("i16imm_brtarget", TYPE_REL) + TYPE("i32imm_brtarget", TYPE_REL) + TYPE("ccode", TYPE_IMM) + TYPE("AVX512RC", TYPE_IMM) + TYPE("brtarget32", TYPE_REL) + TYPE("brtarget16", TYPE_REL) + TYPE("brtarget8", TYPE_REL) + TYPE("f80mem", TYPE_M) + TYPE("lea64_32mem", TYPE_M) + TYPE("lea64mem", TYPE_M) + TYPE("VR64", TYPE_MM64) + TYPE("i64imm", TYPE_IMM) + TYPE("anymem", TYPE_M) + TYPE("opaquemem", TYPE_M) + TYPE("sibmem", TYPE_MSIB) + TYPE("SEGMENT_REG", TYPE_SEGMENTREG) + TYPE("DEBUG_REG", TYPE_DEBUGREG) + TYPE("CONTROL_REG", TYPE_CONTROLREG) + TYPE("srcidx8", TYPE_SRCIDX) + TYPE("srcidx16", TYPE_SRCIDX) + TYPE("srcidx32", TYPE_SRCIDX) + TYPE("srcidx64", TYPE_SRCIDX) + TYPE("dstidx8", TYPE_DSTIDX) + TYPE("dstidx16", TYPE_DSTIDX) + TYPE("dstidx32", TYPE_DSTIDX) + TYPE("dstidx64", TYPE_DSTIDX) + TYPE("offset16_8", TYPE_MOFFS) + TYPE("offset16_16", TYPE_MOFFS) + TYPE("offset16_32", TYPE_MOFFS) + TYPE("offset32_8", TYPE_MOFFS) + TYPE("offset32_16", TYPE_MOFFS) + TYPE("offset32_32", TYPE_MOFFS) + TYPE("offset32_64", TYPE_MOFFS) + TYPE("offset64_8", TYPE_MOFFS) + TYPE("offset64_16", TYPE_MOFFS) + TYPE("offset64_32", TYPE_MOFFS) + TYPE("offset64_64", TYPE_MOFFS) + TYPE("VR256", TYPE_YMM) + TYPE("VR256X", TYPE_YMM) + TYPE("VR512", TYPE_ZMM) + TYPE("VK1", TYPE_VK) + TYPE("VK1WM", TYPE_VK) + TYPE("VK2", TYPE_VK) + TYPE("VK2WM", TYPE_VK) + TYPE("VK4", TYPE_VK) + TYPE("VK4WM", TYPE_VK) + TYPE("VK8", TYPE_VK) + TYPE("VK8WM", TYPE_VK) + TYPE("VK16", TYPE_VK) + TYPE("VK16WM", TYPE_VK) + TYPE("VK32", TYPE_VK) + TYPE("VK32WM", TYPE_VK) + TYPE("VK64", TYPE_VK) + TYPE("VK64WM", TYPE_VK) + TYPE("VK1Pair", TYPE_VK_PAIR) + TYPE("VK2Pair", TYPE_VK_PAIR) + TYPE("VK4Pair", TYPE_VK_PAIR) + TYPE("VK8Pair", TYPE_VK_PAIR) + TYPE("VK16Pair", TYPE_VK_PAIR) + TYPE("vx64mem", TYPE_MVSIBX) + TYPE("vx128mem", TYPE_MVSIBX) + TYPE("vx256mem", TYPE_MVSIBX) + TYPE("vy128mem", TYPE_MVSIBY) + TYPE("vy256mem", TYPE_MVSIBY) + TYPE("vx64xmem", TYPE_MVSIBX) + TYPE("vx128xmem", TYPE_MVSIBX) + TYPE("vx256xmem", TYPE_MVSIBX) + TYPE("vy128xmem", TYPE_MVSIBY) + TYPE("vy256xmem", TYPE_MVSIBY) + TYPE("vy512xmem", TYPE_MVSIBY) + TYPE("vz256mem", TYPE_MVSIBZ) + TYPE("vz512mem", TYPE_MVSIBZ) + TYPE("BNDR", TYPE_BNDR) + TYPE("TILE", TYPE_TMM) errs() << "Unhandled type string " << s << "\n"; llvm_unreachable("Unhandled type string"); } #undef TYPE -#define ENCODING(str, encoding) if (s == str) return encoding; +#define ENCODING(str, encoding) \ + if (s == str) \ + return encoding; OperandEncoding RecognizableInstr::immediateEncodingFromString(const std::string &s, uint8_t OpSize) { - if(OpSize != X86Local::OpSize16) { + if (OpSize != X86Local::OpSize16) { // For instructions without an OpSize prefix, a declared 16-bit register or // immediate encoding is special. - ENCODING("i16imm", ENCODING_IW) + ENCODING("i16imm", ENCODING_IW) } - ENCODING("i32i8imm", ENCODING_IB) - ENCODING("AVX512RC", ENCODING_IRC) - ENCODING("i16imm", ENCODING_Iv) - ENCODING("i16i8imm", ENCODING_IB) - ENCODING("i32imm", ENCODING_Iv) - ENCODING("i64i32imm", ENCODING_ID) - ENCODING("i64i8imm", ENCODING_IB) - ENCODING("i8imm", ENCODING_IB) - ENCODING("u4imm", ENCODING_IB) - ENCODING("u8imm", ENCODING_IB) - ENCODING("i16u8imm", ENCODING_IB) - ENCODING("i32u8imm", ENCODING_IB) - ENCODING("i64u8imm", ENCODING_IB) + ENCODING("i32i8imm", ENCODING_IB) + ENCODING("AVX512RC", ENCODING_IRC) + ENCODING("i16imm", ENCODING_Iv) + ENCODING("i16i8imm", ENCODING_IB) + ENCODING("i32imm", ENCODING_Iv) + ENCODING("i64i32imm", ENCODING_ID) + ENCODING("i64i8imm", ENCODING_IB) + ENCODING("i8imm", ENCODING_IB) + ENCODING("u4imm", ENCODING_IB) + ENCODING("u8imm", ENCODING_IB) + ENCODING("i16u8imm", ENCODING_IB) + ENCODING("i32u8imm", ENCODING_IB) + ENCODING("i64u8imm", ENCODING_IB) // This is not a typo. Instructions like BLENDVPD put // register IDs in 8-bit immediates nowadays. - ENCODING("FR32", ENCODING_IB) - ENCODING("FR64", ENCODING_IB) - ENCODING("FR128", ENCODING_IB) - ENCODING("VR128", ENCODING_IB) - ENCODING("VR256", ENCODING_IB) - ENCODING("FR16X", ENCODING_IB) - ENCODING("FR32X", ENCODING_IB) - ENCODING("FR64X", ENCODING_IB) - ENCODING("VR128X", ENCODING_IB) - ENCODING("VR256X", ENCODING_IB) - ENCODING("VR512", ENCODING_IB) - ENCODING("TILE", ENCODING_IB) + ENCODING("FR32", ENCODING_IB) + ENCODING("FR64", ENCODING_IB) + ENCODING("FR128", ENCODING_IB) + ENCODING("VR128", ENCODING_IB) + ENCODING("VR256", ENCODING_IB) + ENCODING("FR16X", ENCODING_IB) + ENCODING("FR32X", ENCODING_IB) + ENCODING("FR64X", ENCODING_IB) + ENCODING("VR128X", ENCODING_IB) + ENCODING("VR256X", ENCODING_IB) + ENCODING("VR512", ENCODING_IB) + ENCODING("TILE", ENCODING_IB) errs() << "Unhandled immediate encoding " << s << "\n"; llvm_unreachable("Unhandled immediate encoding"); } @@ -1080,35 +1128,35 @@ RecognizableInstr::immediateEncodingFromString(const std::string &s, OperandEncoding RecognizableInstr::rmRegisterEncodingFromString(const std::string &s, uint8_t OpSize) { - ENCODING("RST", ENCODING_FP) - ENCODING("RSTi", ENCODING_FP) - ENCODING("GR16", ENCODING_RM) - ENCODING("GR16orGR32orGR64",ENCODING_RM) - ENCODING("GR32", ENCODING_RM) - ENCODING("GR32orGR64", ENCODING_RM) - ENCODING("GR64", ENCODING_RM) - ENCODING("GR8", ENCODING_RM) - ENCODING("VR128", ENCODING_RM) - ENCODING("VR128X", ENCODING_RM) - ENCODING("FR128", ENCODING_RM) - ENCODING("FR64", ENCODING_RM) - ENCODING("FR32", ENCODING_RM) - ENCODING("FR64X", ENCODING_RM) - ENCODING("FR32X", ENCODING_RM) - ENCODING("FR16X", ENCODING_RM) - ENCODING("VR64", ENCODING_RM) - ENCODING("VR256", ENCODING_RM) - ENCODING("VR256X", ENCODING_RM) - ENCODING("VR512", ENCODING_RM) - ENCODING("VK1", ENCODING_RM) - ENCODING("VK2", ENCODING_RM) - ENCODING("VK4", ENCODING_RM) - ENCODING("VK8", ENCODING_RM) - ENCODING("VK16", ENCODING_RM) - ENCODING("VK32", ENCODING_RM) - ENCODING("VK64", ENCODING_RM) - ENCODING("BNDR", ENCODING_RM) - ENCODING("TILE", ENCODING_RM) + ENCODING("RST", ENCODING_FP) + ENCODING("RSTi", ENCODING_FP) + ENCODING("GR16", ENCODING_RM) + ENCODING("GR16orGR32orGR64", ENCODING_RM) + ENCODING("GR32", ENCODING_RM) + ENCODING("GR32orGR64", ENCODING_RM) + ENCODING("GR64", ENCODING_RM) + ENCODING("GR8", ENCODING_RM) + ENCODING("VR128", ENCODING_RM) + ENCODING("VR128X", ENCODING_RM) + ENCODING("FR128", ENCODING_RM) + ENCODING("FR64", ENCODING_RM) + ENCODING("FR32", ENCODING_RM) + ENCODING("FR64X", ENCODING_RM) + ENCODING("FR32X", ENCODING_RM) + ENCODING("FR16X", ENCODING_RM) + ENCODING("VR64", ENCODING_RM) + ENCODING("VR256", ENCODING_RM) + ENCODING("VR256X", ENCODING_RM) + ENCODING("VR512", ENCODING_RM) + ENCODING("VK1", ENCODING_RM) + ENCODING("VK2", ENCODING_RM) + ENCODING("VK4", ENCODING_RM) + ENCODING("VK8", ENCODING_RM) + ENCODING("VK16", ENCODING_RM) + ENCODING("VK32", ENCODING_RM) + ENCODING("VK64", ENCODING_RM) + ENCODING("BNDR", ENCODING_RM) + ENCODING("TILE", ENCODING_RM) errs() << "Unhandled R/M register encoding " << s << "\n"; llvm_unreachable("Unhandled R/M register encoding"); } @@ -1116,48 +1164,48 @@ RecognizableInstr::rmRegisterEncodingFromString(const std::string &s, OperandEncoding RecognizableInstr::roRegisterEncodingFromString(const std::string &s, uint8_t OpSize) { - ENCODING("GR16", ENCODING_REG) - ENCODING("GR16orGR32orGR64",ENCODING_REG) - ENCODING("GR32", ENCODING_REG) - ENCODING("GR32orGR64", ENCODING_REG) - ENCODING("GR64", ENCODING_REG) - ENCODING("GR8", ENCODING_REG) - ENCODING("VR128", ENCODING_REG) - ENCODING("FR128", ENCODING_REG) - ENCODING("FR64", ENCODING_REG) - ENCODING("FR32", ENCODING_REG) - ENCODING("VR64", ENCODING_REG) - ENCODING("SEGMENT_REG", ENCODING_REG) - ENCODING("DEBUG_REG", ENCODING_REG) - ENCODING("CONTROL_REG", ENCODING_REG) - ENCODING("VR256", ENCODING_REG) - ENCODING("VR256X", ENCODING_REG) - ENCODING("VR128X", ENCODING_REG) - ENCODING("FR64X", ENCODING_REG) - ENCODING("FR32X", ENCODING_REG) - ENCODING("FR16X", ENCODING_REG) - ENCODING("VR512", ENCODING_REG) - ENCODING("VK1", ENCODING_REG) - ENCODING("VK2", ENCODING_REG) - ENCODING("VK4", ENCODING_REG) - ENCODING("VK8", ENCODING_REG) - ENCODING("VK16", ENCODING_REG) - ENCODING("VK32", ENCODING_REG) - ENCODING("VK64", ENCODING_REG) - ENCODING("VK1Pair", ENCODING_REG) - ENCODING("VK2Pair", ENCODING_REG) - ENCODING("VK4Pair", ENCODING_REG) - ENCODING("VK8Pair", ENCODING_REG) - ENCODING("VK16Pair", ENCODING_REG) - ENCODING("VK1WM", ENCODING_REG) - ENCODING("VK2WM", ENCODING_REG) - ENCODING("VK4WM", ENCODING_REG) - ENCODING("VK8WM", ENCODING_REG) - ENCODING("VK16WM", ENCODING_REG) - ENCODING("VK32WM", ENCODING_REG) - ENCODING("VK64WM", ENCODING_REG) - ENCODING("BNDR", ENCODING_REG) - ENCODING("TILE", ENCODING_REG) + ENCODING("GR16", ENCODING_REG) + ENCODING("GR16orGR32orGR64", ENCODING_REG) + ENCODING("GR32", ENCODING_REG) + ENCODING("GR32orGR64", ENCODING_REG) + ENCODING("GR64", ENCODING_REG) + ENCODING("GR8", ENCODING_REG) + ENCODING("VR128", ENCODING_REG) + ENCODING("FR128", ENCODING_REG) + ENCODING("FR64", ENCODING_REG) + ENCODING("FR32", ENCODING_REG) + ENCODING("VR64", ENCODING_REG) + ENCODING("SEGMENT_REG", ENCODING_REG) + ENCODING("DEBUG_REG", ENCODING_REG) + ENCODING("CONTROL_REG", ENCODING_REG) + ENCODING("VR256", ENCODING_REG) + ENCODING("VR256X", ENCODING_REG) + ENCODING("VR128X", ENCODING_REG) + ENCODING("FR64X", ENCODING_REG) + ENCODING("FR32X", ENCODING_REG) + ENCODING("FR16X", ENCODING_REG) + ENCODING("VR512", ENCODING_REG) + ENCODING("VK1", ENCODING_REG) + ENCODING("VK2", ENCODING_REG) + ENCODING("VK4", ENCODING_REG) + ENCODING("VK8", ENCODING_REG) + ENCODING("VK16", ENCODING_REG) + ENCODING("VK32", ENCODING_REG) + ENCODING("VK64", ENCODING_REG) + ENCODING("VK1Pair", ENCODING_REG) + ENCODING("VK2Pair", ENCODING_REG) + ENCODING("VK4Pair", ENCODING_REG) + ENCODING("VK8Pair", ENCODING_REG) + ENCODING("VK16Pair", ENCODING_REG) + ENCODING("VK1WM", ENCODING_REG) + ENCODING("VK2WM", ENCODING_REG) + ENCODING("VK4WM", ENCODING_REG) + ENCODING("VK8WM", ENCODING_REG) + ENCODING("VK16WM", ENCODING_REG) + ENCODING("VK32WM", ENCODING_REG) + ENCODING("VK64WM", ENCODING_REG) + ENCODING("BNDR", ENCODING_REG) + ENCODING("TILE", ENCODING_REG) errs() << "Unhandled reg/opcode register encoding " << s << "\n"; llvm_unreachable("Unhandled reg/opcode register encoding"); } @@ -1165,27 +1213,27 @@ RecognizableInstr::roRegisterEncodingFromString(const std::string &s, OperandEncoding RecognizableInstr::vvvvRegisterEncodingFromString(const std::string &s, uint8_t OpSize) { - ENCODING("GR32", ENCODING_VVVV) - ENCODING("GR64", ENCODING_VVVV) - ENCODING("FR32", ENCODING_VVVV) - ENCODING("FR128", ENCODING_VVVV) - ENCODING("FR64", ENCODING_VVVV) - ENCODING("VR128", ENCODING_VVVV) - ENCODING("VR256", ENCODING_VVVV) - ENCODING("FR16X", ENCODING_VVVV) - ENCODING("FR32X", ENCODING_VVVV) - ENCODING("FR64X", ENCODING_VVVV) - ENCODING("VR128X", ENCODING_VVVV) - ENCODING("VR256X", ENCODING_VVVV) - ENCODING("VR512", ENCODING_VVVV) - ENCODING("VK1", ENCODING_VVVV) - ENCODING("VK2", ENCODING_VVVV) - ENCODING("VK4", ENCODING_VVVV) - ENCODING("VK8", ENCODING_VVVV) - ENCODING("VK16", ENCODING_VVVV) - ENCODING("VK32", ENCODING_VVVV) - ENCODING("VK64", ENCODING_VVVV) - ENCODING("TILE", ENCODING_VVVV) + ENCODING("GR32", ENCODING_VVVV) + ENCODING("GR64", ENCODING_VVVV) + ENCODING("FR32", ENCODING_VVVV) + ENCODING("FR128", ENCODING_VVVV) + ENCODING("FR64", ENCODING_VVVV) + ENCODING("VR128", ENCODING_VVVV) + ENCODING("VR256", ENCODING_VVVV) + ENCODING("FR16X", ENCODING_VVVV) + ENCODING("FR32X", ENCODING_VVVV) + ENCODING("FR64X", ENCODING_VVVV) + ENCODING("VR128X", ENCODING_VVVV) + ENCODING("VR256X", ENCODING_VVVV) + ENCODING("VR512", ENCODING_VVVV) + ENCODING("VK1", ENCODING_VVVV) + ENCODING("VK2", ENCODING_VVVV) + ENCODING("VK4", ENCODING_VVVV) + ENCODING("VK8", ENCODING_VVVV) + ENCODING("VK16", ENCODING_VVVV) + ENCODING("VK32", ENCODING_VVVV) + ENCODING("VK64", ENCODING_VVVV) + ENCODING("TILE", ENCODING_VVVV) errs() << "Unhandled VEX.vvvv register encoding " << s << "\n"; llvm_unreachable("Unhandled VEX.vvvv register encoding"); } @@ -1193,13 +1241,13 @@ RecognizableInstr::vvvvRegisterEncodingFromString(const std::string &s, OperandEncoding RecognizableInstr::writemaskRegisterEncodingFromString(const std::string &s, uint8_t OpSize) { - ENCODING("VK1WM", ENCODING_WRITEMASK) - ENCODING("VK2WM", ENCODING_WRITEMASK) - ENCODING("VK4WM", ENCODING_WRITEMASK) - ENCODING("VK8WM", ENCODING_WRITEMASK) - ENCODING("VK16WM", ENCODING_WRITEMASK) - ENCODING("VK32WM", ENCODING_WRITEMASK) - ENCODING("VK64WM", ENCODING_WRITEMASK) + ENCODING("VK1WM", ENCODING_WRITEMASK) + ENCODING("VK2WM", ENCODING_WRITEMASK) + ENCODING("VK4WM", ENCODING_WRITEMASK) + ENCODING("VK8WM", ENCODING_WRITEMASK) + ENCODING("VK16WM", ENCODING_WRITEMASK) + ENCODING("VK32WM", ENCODING_WRITEMASK) + ENCODING("VK64WM", ENCODING_WRITEMASK) errs() << "Unhandled mask register encoding " << s << "\n"; llvm_unreachable("Unhandled mask register encoding"); } @@ -1207,44 +1255,44 @@ RecognizableInstr::writemaskRegisterEncodingFromString(const std::string &s, OperandEncoding RecognizableInstr::memoryEncodingFromString(const std::string &s, uint8_t OpSize) { - ENCODING("i16mem", ENCODING_RM) - ENCODING("i32mem", ENCODING_RM) - ENCODING("i64mem", ENCODING_RM) - ENCODING("i8mem", ENCODING_RM) - ENCODING("shmem", ENCODING_RM) - ENCODING("ssmem", ENCODING_RM) - ENCODING("sdmem", ENCODING_RM) - ENCODING("f128mem", ENCODING_RM) - ENCODING("f256mem", ENCODING_RM) - ENCODING("f512mem", ENCODING_RM) - ENCODING("f64mem", ENCODING_RM) - ENCODING("f32mem", ENCODING_RM) - ENCODING("f16mem", ENCODING_RM) - ENCODING("i128mem", ENCODING_RM) - ENCODING("i256mem", ENCODING_RM) - ENCODING("i512mem", ENCODING_RM) - ENCODING("i512mem_GR16", ENCODING_RM) - ENCODING("i512mem_GR32", ENCODING_RM) - ENCODING("i512mem_GR64", ENCODING_RM) - ENCODING("f80mem", ENCODING_RM) - ENCODING("lea64_32mem", ENCODING_RM) - ENCODING("lea64mem", ENCODING_RM) - ENCODING("anymem", ENCODING_RM) - ENCODING("opaquemem", ENCODING_RM) - ENCODING("sibmem", ENCODING_SIB) - ENCODING("vx64mem", ENCODING_VSIB) - ENCODING("vx128mem", ENCODING_VSIB) - ENCODING("vx256mem", ENCODING_VSIB) - ENCODING("vy128mem", ENCODING_VSIB) - ENCODING("vy256mem", ENCODING_VSIB) - ENCODING("vx64xmem", ENCODING_VSIB) - ENCODING("vx128xmem", ENCODING_VSIB) - ENCODING("vx256xmem", ENCODING_VSIB) - ENCODING("vy128xmem", ENCODING_VSIB) - ENCODING("vy256xmem", ENCODING_VSIB) - ENCODING("vy512xmem", ENCODING_VSIB) - ENCODING("vz256mem", ENCODING_VSIB) - ENCODING("vz512mem", ENCODING_VSIB) + ENCODING("i16mem", ENCODING_RM) + ENCODING("i32mem", ENCODING_RM) + ENCODING("i64mem", ENCODING_RM) + ENCODING("i8mem", ENCODING_RM) + ENCODING("shmem", ENCODING_RM) + ENCODING("ssmem", ENCODING_RM) + ENCODING("sdmem", ENCODING_RM) + ENCODING("f128mem", ENCODING_RM) + ENCODING("f256mem", ENCODING_RM) + ENCODING("f512mem", ENCODING_RM) + ENCODING("f64mem", ENCODING_RM) + ENCODING("f32mem", ENCODING_RM) + ENCODING("f16mem", ENCODING_RM) + ENCODING("i128mem", ENCODING_RM) + ENCODING("i256mem", ENCODING_RM) + ENCODING("i512mem", ENCODING_RM) + ENCODING("i512mem_GR16", ENCODING_RM) + ENCODING("i512mem_GR32", ENCODING_RM) + ENCODING("i512mem_GR64", ENCODING_RM) + ENCODING("f80mem", ENCODING_RM) + ENCODING("lea64_32mem", ENCODING_RM) + ENCODING("lea64mem", ENCODING_RM) + ENCODING("anymem", ENCODING_RM) + ENCODING("opaquemem", ENCODING_RM) + ENCODING("sibmem", ENCODING_SIB) + ENCODING("vx64mem", ENCODING_VSIB) + ENCODING("vx128mem", ENCODING_VSIB) + ENCODING("vx256mem", ENCODING_VSIB) + ENCODING("vy128mem", ENCODING_VSIB) + ENCODING("vy256mem", ENCODING_VSIB) + ENCODING("vx64xmem", ENCODING_VSIB) + ENCODING("vx128xmem", ENCODING_VSIB) + ENCODING("vx256xmem", ENCODING_VSIB) + ENCODING("vy128xmem", ENCODING_VSIB) + ENCODING("vy256xmem", ENCODING_VSIB) + ENCODING("vy512xmem", ENCODING_VSIB) + ENCODING("vz256mem", ENCODING_VSIB) + ENCODING("vz512mem", ENCODING_VSIB) errs() << "Unhandled memory encoding " << s << "\n"; llvm_unreachable("Unhandled memory encoding"); } @@ -1252,48 +1300,48 @@ RecognizableInstr::memoryEncodingFromString(const std::string &s, OperandEncoding RecognizableInstr::relocationEncodingFromString(const std::string &s, uint8_t OpSize) { - if(OpSize != X86Local::OpSize16) { + if (OpSize != X86Local::OpSize16) { // For instructions without an OpSize prefix, a declared 16-bit register or // immediate encoding is special. - ENCODING("i16imm", ENCODING_IW) + ENCODING("i16imm", ENCODING_IW) } - ENCODING("i16imm", ENCODING_Iv) - ENCODING("i16i8imm", ENCODING_IB) - ENCODING("i32imm", ENCODING_Iv) - ENCODING("i32i8imm", ENCODING_IB) - ENCODING("i64i32imm", ENCODING_ID) - ENCODING("i64i8imm", ENCODING_IB) - ENCODING("i8imm", ENCODING_IB) - ENCODING("u8imm", ENCODING_IB) - ENCODING("i16u8imm", ENCODING_IB) - ENCODING("i32u8imm", ENCODING_IB) - ENCODING("i64u8imm", ENCODING_IB) + ENCODING("i16imm", ENCODING_Iv) + ENCODING("i16i8imm", ENCODING_IB) + ENCODING("i32imm", ENCODING_Iv) + ENCODING("i32i8imm", ENCODING_IB) + ENCODING("i64i32imm", ENCODING_ID) + ENCODING("i64i8imm", ENCODING_IB) + ENCODING("i8imm", ENCODING_IB) + ENCODING("u8imm", ENCODING_IB) + ENCODING("i16u8imm", ENCODING_IB) + ENCODING("i32u8imm", ENCODING_IB) + ENCODING("i64u8imm", ENCODING_IB) ENCODING("i64i32imm_brtarget", ENCODING_ID) - ENCODING("i16imm_brtarget", ENCODING_IW) - ENCODING("i32imm_brtarget", ENCODING_ID) - ENCODING("brtarget32", ENCODING_ID) - ENCODING("brtarget16", ENCODING_IW) - ENCODING("brtarget8", ENCODING_IB) - ENCODING("i64imm", ENCODING_IO) - ENCODING("offset16_8", ENCODING_Ia) - ENCODING("offset16_16", ENCODING_Ia) - ENCODING("offset16_32", ENCODING_Ia) - ENCODING("offset32_8", ENCODING_Ia) - ENCODING("offset32_16", ENCODING_Ia) - ENCODING("offset32_32", ENCODING_Ia) - ENCODING("offset32_64", ENCODING_Ia) - ENCODING("offset64_8", ENCODING_Ia) - ENCODING("offset64_16", ENCODING_Ia) - ENCODING("offset64_32", ENCODING_Ia) - ENCODING("offset64_64", ENCODING_Ia) - ENCODING("srcidx8", ENCODING_SI) - ENCODING("srcidx16", ENCODING_SI) - ENCODING("srcidx32", ENCODING_SI) - ENCODING("srcidx64", ENCODING_SI) - ENCODING("dstidx8", ENCODING_DI) - ENCODING("dstidx16", ENCODING_DI) - ENCODING("dstidx32", ENCODING_DI) - ENCODING("dstidx64", ENCODING_DI) + ENCODING("i16imm_brtarget", ENCODING_IW) + ENCODING("i32imm_brtarget", ENCODING_ID) + ENCODING("brtarget32", ENCODING_ID) + ENCODING("brtarget16", ENCODING_IW) + ENCODING("brtarget8", ENCODING_IB) + ENCODING("i64imm", ENCODING_IO) + ENCODING("offset16_8", ENCODING_Ia) + ENCODING("offset16_16", ENCODING_Ia) + ENCODING("offset16_32", ENCODING_Ia) + ENCODING("offset32_8", ENCODING_Ia) + ENCODING("offset32_16", ENCODING_Ia) + ENCODING("offset32_32", ENCODING_Ia) + ENCODING("offset32_64", ENCODING_Ia) + ENCODING("offset64_8", ENCODING_Ia) + ENCODING("offset64_16", ENCODING_Ia) + ENCODING("offset64_32", ENCODING_Ia) + ENCODING("offset64_64", ENCODING_Ia) + ENCODING("srcidx8", ENCODING_SI) + ENCODING("srcidx16", ENCODING_SI) + ENCODING("srcidx32", ENCODING_SI) + ENCODING("srcidx64", ENCODING_SI) + ENCODING("dstidx8", ENCODING_DI) + ENCODING("dstidx16", ENCODING_DI) + ENCODING("dstidx32", ENCODING_DI) + ENCODING("dstidx64", ENCODING_DI) errs() << "Unhandled relocation encoding " << s << "\n"; llvm_unreachable("Unhandled relocation encoding"); } @@ -1301,11 +1349,11 @@ RecognizableInstr::relocationEncodingFromString(const std::string &s, OperandEncoding RecognizableInstr::opcodeModifierEncodingFromString(const std::string &s, uint8_t OpSize) { - ENCODING("GR32", ENCODING_Rv) - ENCODING("GR64", ENCODING_RO) - ENCODING("GR16", ENCODING_Rv) - ENCODING("GR8", ENCODING_RB) - ENCODING("ccode", ENCODING_CC) + ENCODING("GR32", ENCODING_Rv) + ENCODING("GR64", ENCODING_RO) + ENCODING("GR16", ENCODING_Rv) + ENCODING("GR8", ENCODING_RB) + ENCODING("ccode", ENCODING_CC) errs() << "Unhandled opcode modifier encoding " << s << "\n"; llvm_unreachable("Unhandled opcode modifier encoding"); } diff --git a/llvm/utils/TableGen/X86RecognizableInstr.h b/llvm/utils/TableGen/X86RecognizableInstr.h index 5efacdb27465..61ad5e32b3fb 100644 --- a/llvm/utils/TableGen/X86RecognizableInstr.h +++ b/llvm/utils/TableGen/X86RecognizableInstr.h @@ -25,142 +25,158 @@ struct InstructionSpecifier; namespace llvm { - class Record; - -#define X86_INSTR_MRM_MAPPING \ - MAP(C0, 64) \ - MAP(C1, 65) \ - MAP(C2, 66) \ - MAP(C3, 67) \ - MAP(C4, 68) \ - MAP(C5, 69) \ - MAP(C6, 70) \ - MAP(C7, 71) \ - MAP(C8, 72) \ - MAP(C9, 73) \ - MAP(CA, 74) \ - MAP(CB, 75) \ - MAP(CC, 76) \ - MAP(CD, 77) \ - MAP(CE, 78) \ - MAP(CF, 79) \ - MAP(D0, 80) \ - MAP(D1, 81) \ - MAP(D2, 82) \ - MAP(D3, 83) \ - MAP(D4, 84) \ - MAP(D5, 85) \ - MAP(D6, 86) \ - MAP(D7, 87) \ - MAP(D8, 88) \ - MAP(D9, 89) \ - MAP(DA, 90) \ - MAP(DB, 91) \ - MAP(DC, 92) \ - MAP(DD, 93) \ - MAP(DE, 94) \ - MAP(DF, 95) \ - MAP(E0, 96) \ - MAP(E1, 97) \ - MAP(E2, 98) \ - MAP(E3, 99) \ - MAP(E4, 100) \ - MAP(E5, 101) \ - MAP(E6, 102) \ - MAP(E7, 103) \ - MAP(E8, 104) \ - MAP(E9, 105) \ - MAP(EA, 106) \ - MAP(EB, 107) \ - MAP(EC, 108) \ - MAP(ED, 109) \ - MAP(EE, 110) \ - MAP(EF, 111) \ - MAP(F0, 112) \ - MAP(F1, 113) \ - MAP(F2, 114) \ - MAP(F3, 115) \ - MAP(F4, 116) \ - MAP(F5, 117) \ - MAP(F6, 118) \ - MAP(F7, 119) \ - MAP(F8, 120) \ - MAP(F9, 121) \ - MAP(FA, 122) \ - MAP(FB, 123) \ - MAP(FC, 124) \ - MAP(FD, 125) \ - MAP(FE, 126) \ +#define X86_INSTR_MRM_MAPPING \ + MAP(C0, 64) \ + MAP(C1, 65) \ + MAP(C2, 66) \ + MAP(C3, 67) \ + MAP(C4, 68) \ + MAP(C5, 69) \ + MAP(C6, 70) \ + MAP(C7, 71) \ + MAP(C8, 72) \ + MAP(C9, 73) \ + MAP(CA, 74) \ + MAP(CB, 75) \ + MAP(CC, 76) \ + MAP(CD, 77) \ + MAP(CE, 78) \ + MAP(CF, 79) \ + MAP(D0, 80) \ + MAP(D1, 81) \ + MAP(D2, 82) \ + MAP(D3, 83) \ + MAP(D4, 84) \ + MAP(D5, 85) \ + MAP(D6, 86) \ + MAP(D7, 87) \ + MAP(D8, 88) \ + MAP(D9, 89) \ + MAP(DA, 90) \ + MAP(DB, 91) \ + MAP(DC, 92) \ + MAP(DD, 93) \ + MAP(DE, 94) \ + MAP(DF, 95) \ + MAP(E0, 96) \ + MAP(E1, 97) \ + MAP(E2, 98) \ + MAP(E3, 99) \ + MAP(E4, 100) \ + MAP(E5, 101) \ + MAP(E6, 102) \ + MAP(E7, 103) \ + MAP(E8, 104) \ + MAP(E9, 105) \ + MAP(EA, 106) \ + MAP(EB, 107) \ + MAP(EC, 108) \ + MAP(ED, 109) \ + MAP(EE, 110) \ + MAP(EF, 111) \ + MAP(F0, 112) \ + MAP(F1, 113) \ + MAP(F2, 114) \ + MAP(F3, 115) \ + MAP(F4, 116) \ + MAP(F5, 117) \ + MAP(F6, 118) \ + MAP(F7, 119) \ + MAP(F8, 120) \ + MAP(F9, 121) \ + MAP(FA, 122) \ + MAP(FB, 123) \ + MAP(FC, 124) \ + MAP(FD, 125) \ + MAP(FE, 126) \ MAP(FF, 127) // A clone of X86 since we can't depend on something that is generated. namespace X86Local { - enum { - Pseudo = 0, - RawFrm = 1, - AddRegFrm = 2, - RawFrmMemOffs = 3, - RawFrmSrc = 4, - RawFrmDst = 5, - RawFrmDstSrc = 6, - RawFrmImm8 = 7, - RawFrmImm16 = 8, - AddCCFrm = 9, - PrefixByte = 10, - MRMDestMem4VOp3CC = 20, - MRMr0 = 21, - MRMSrcMemFSIB = 22, - MRMDestMemFSIB = 23, - MRMDestMem = 24, - MRMSrcMem = 25, - MRMSrcMem4VOp3 = 26, - MRMSrcMemOp4 = 27, - MRMSrcMemCC = 28, - MRMXmCC = 30, MRMXm = 31, - MRM0m = 32, MRM1m = 33, MRM2m = 34, MRM3m = 35, - MRM4m = 36, MRM5m = 37, MRM6m = 38, MRM7m = 39, - MRMDestReg = 40, - MRMSrcReg = 41, - MRMSrcReg4VOp3 = 42, - MRMSrcRegOp4 = 43, - MRMSrcRegCC = 44, - MRMXrCC = 46, MRMXr = 47, - MRM0r = 48, MRM1r = 49, MRM2r = 50, MRM3r = 51, - MRM4r = 52, MRM5r = 53, MRM6r = 54, MRM7r = 55, - MRM0X = 56, MRM1X = 57, MRM2X = 58, MRM3X = 59, - MRM4X = 60, MRM5X = 61, MRM6X = 62, MRM7X = 63, +enum { + Pseudo = 0, + RawFrm = 1, + AddRegFrm = 2, + RawFrmMemOffs = 3, + RawFrmSrc = 4, + RawFrmDst = 5, + RawFrmDstSrc = 6, + RawFrmImm8 = 7, + RawFrmImm16 = 8, + AddCCFrm = 9, + PrefixByte = 10, + MRMDestMem4VOp3CC = 20, + MRMr0 = 21, + MRMSrcMemFSIB = 22, + MRMDestMemFSIB = 23, + MRMDestMem = 24, + MRMSrcMem = 25, + MRMSrcMem4VOp3 = 26, + MRMSrcMemOp4 = 27, + MRMSrcMemCC = 28, + MRMXmCC = 30, + MRMXm = 31, + MRM0m = 32, + MRM1m = 33, + MRM2m = 34, + MRM3m = 35, + MRM4m = 36, + MRM5m = 37, + MRM6m = 38, + MRM7m = 39, + MRMDestReg = 40, + MRMSrcReg = 41, + MRMSrcReg4VOp3 = 42, + MRMSrcRegOp4 = 43, + MRMSrcRegCC = 44, + MRMXrCC = 46, + MRMXr = 47, + MRM0r = 48, + MRM1r = 49, + MRM2r = 50, + MRM3r = 51, + MRM4r = 52, + MRM5r = 53, + MRM6r = 54, + MRM7r = 55, + MRM0X = 56, + MRM1X = 57, + MRM2X = 58, + MRM3X = 59, + MRM4X = 60, + MRM5X = 61, + MRM6X = 62, + MRM7X = 63, #define MAP(from, to) MRM_##from = to, - X86_INSTR_MRM_MAPPING + X86_INSTR_MRM_MAPPING #undef MAP - }; - - enum { - OB = 0, TB = 1, T8 = 2, TA = 3, XOP8 = 4, XOP9 = 5, XOPA = 6, ThreeDNow = 7, - T_MAP5 = 8, T_MAP6 = 9 - }; - - enum { - PD = 1, XS = 2, XD = 3, PS = 4 - }; - - enum { - VEX = 1, XOP = 2, EVEX = 3 - }; +}; - enum { - OpSize16 = 1, OpSize32 = 2 - }; +enum { + OB = 0, + TB = 1, + T8 = 2, + TA = 3, + XOP8 = 4, + XOP9 = 5, + XOPA = 6, + ThreeDNow = 7, + T_MAP4 = 8, + T_MAP5 = 9, + T_MAP6 = 10, + T_MAP7 = 11 +}; - enum { - AdSize16 = 1, AdSize32 = 2, AdSize64 = 3 - }; -} +enum { PD = 1, XS = 2, XD = 3, PS = 4 }; +enum { VEX = 1, XOP = 2, EVEX = 3 }; +enum { OpSize16 = 1, OpSize32 = 2 }; +enum { AdSize16 = 1, AdSize32 = 2, AdSize64 = 3 }; +enum { ExplicitREX2 = 1 }; +} // namespace X86Local namespace X86Disassembler { - class DisassemblerTables; - /// Extract common fields of a single X86 instruction from a CodeGenInstruction struct RecognizableInstrBase { /// The OpPrefix field from the record @@ -206,6 +222,8 @@ struct RecognizableInstrBase { bool ForceDisassemble; // The CD8_Scale field from the record uint8_t CD8_Scale; + /// If explicitOpPrefix field from the record equals ExplicitREX2 + bool ExplicitREX2Prefix; /// \param insn The CodeGenInstruction to extract information from. RecognizableInstrBase(const CodeGenInstruction &insn); /// \returns true if this instruction should be emitted @@ -219,7 +237,7 @@ struct RecognizableInstrBase { class RecognizableInstr : public RecognizableInstrBase { private: /// The record from the .td files corresponding to this instruction - const Record* Rec; + const Record *Rec; /// The instruction name as listed in the tables std::string Name; // Whether the instruction has the predicate "In32BitMode" @@ -229,13 +247,13 @@ private: /// The operands of the instruction, as listed in the CodeGenInstruction. /// They are not one-to-one with operands listed in the MCInst; for example, /// memory operands expand to 5 operands in the MCInst - const std::vector<CGIOperandList::OperandInfo>* Operands; + const std::vector<CGIOperandList::OperandInfo> *Operands; /// The opcode of the instruction, as used in an MCInst InstrUID UID; /// The description of the instruction that is emitted into the instruction /// info table - InstructionSpecifier* Spec; + InstructionSpecifier *Spec; /// insnContext - Returns the primary context in which the instruction is /// valid. @@ -255,8 +273,8 @@ private: /// If register size does not match OpSize, then /// register sizes keep their size. /// @return - The operand's type. - static OperandType typeFromString(const std::string& s, - bool hasREX_W, uint8_t OpSize); + static OperandType typeFromString(const std::string &s, bool hasREX_W, + uint8_t OpSize); /// immediateEncodingFromString - Translates an immediate encoding from the /// string provided in the LLVM tables to an OperandEncoding for use in @@ -286,8 +304,8 @@ private: uint8_t OpSize); static OperandEncoding vvvvRegisterEncodingFromString(const std::string &s, uint8_t OpSize); - static OperandEncoding writemaskRegisterEncodingFromString(const std::string &s, - uint8_t OpSize); + static OperandEncoding + writemaskRegisterEncodingFromString(const std::string &s, uint8_t OpSize); /// Adjust the encoding type for an operand based on the instruction. void adjustOperandEncoding(OperandEncoding &encoding); @@ -310,14 +328,12 @@ private: /// @param operandMapping - The operand mapping, which has an entry for /// each operand that indicates whether it is a /// duplicate, and of what. - void handleOperand(bool optional, - unsigned &operandIndex, + void handleOperand(bool optional, unsigned &operandIndex, unsigned &physicalOperandIndex, unsigned numPhysicalOperands, const unsigned *operandMapping, - OperandEncoding (*encodingFromString) - (const std::string&, - uint8_t OpSize)); + OperandEncoding (*encodingFromString)(const std::string &, + uint8_t OpSize)); /// emitInstructionSpecifier - Loads the instruction specifier for the current /// instruction into a DisassemblerTables. @@ -338,8 +354,7 @@ public: /// \param tables The DisassemblerTables that the specifier will be added to. /// \param insn The CodeGenInstruction to extract information from. /// \param uid The unique ID of the current instruction. - RecognizableInstr(DisassemblerTables &tables, - const CodeGenInstruction &insn, + RecognizableInstr(DisassemblerTables &tables, const CodeGenInstruction &insn, InstrUID uid); /// processInstr - Accepts a CodeGenInstruction and loads decode information /// for it into a DisassemblerTables if appropriate. @@ -350,8 +365,7 @@ public: /// information. /// \param uid The unique ID of the instruction. static void processInstr(DisassemblerTables &tables, - const CodeGenInstruction &insn, - InstrUID uid); + const CodeGenInstruction &insn, InstrUID uid); }; std::string getMnemonic(const CodeGenInstruction *I, unsigned Variant); @@ -361,7 +375,5 @@ bool isImmediateOperand(const Record *Rec); unsigned getRegOperandSize(const Record *RegRec); unsigned getMemOperandSize(const Record *MemRec); } // namespace X86Disassembler - } // namespace llvm - #endif |