diff options
Diffstat (limited to 'contrib/llvm-project/llvm/utils')
92 files changed, 56808 insertions, 0 deletions
diff --git a/contrib/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp new file mode 100644 index 000000000000..3d63059dcb8b --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/AsmMatcherEmitter.cpp @@ -0,0 +1,3920 @@ +//===- AsmMatcherEmitter.cpp - Generate an assembly matcher ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend emits a target specifier matcher for converting parsed +// assembly operands in the MCInst structures. It also emits a matcher for +// custom operand parsing. +// +// Converting assembly operands into MCInst structures +// --------------------------------------------------- +// +// The input to the target specific matcher is a list of literal tokens and +// operands. The target specific parser should generally eliminate any syntax +// which is not relevant for matching; for example, comma tokens should have +// already been consumed and eliminated by the parser. Most instructions will +// end up with a single literal token (the instruction name) and some number of +// operands. +// +// Some example inputs, for X86: +// 'addl' (immediate ...) (register ...) +// 'add' (immediate ...) (memory ...) +// 'call' '*' %epc +// +// The assembly matcher is responsible for converting this input into a precise +// machine instruction (i.e., an instruction with a well defined encoding). This +// mapping has several properties which complicate matching: +// +// - It may be ambiguous; many architectures can legally encode particular +// variants of an instruction in different ways (for example, using a smaller +// encoding for small immediates). Such ambiguities should never be +// arbitrarily resolved by the assembler, the assembler is always responsible +// for choosing the "best" available instruction. +// +// - It may depend on the subtarget or the assembler context. Instructions +// which are invalid for the current mode, but otherwise unambiguous (e.g., +// an SSE instruction in a file being assembled for i486) should be accepted +// and rejected by the assembler front end. However, if the proper encoding +// for an instruction is dependent on the assembler context then the matcher +// is responsible for selecting the correct machine instruction for the +// current mode. +// +// The core matching algorithm attempts to exploit the regularity in most +// instruction sets to quickly determine the set of possibly matching +// instructions, and the simplify the generated code. Additionally, this helps +// to ensure that the ambiguities are intentionally resolved by the user. +// +// The matching is divided into two distinct phases: +// +// 1. Classification: Each operand is mapped to the unique set which (a) +// contains it, and (b) is the largest such subset for which a single +// instruction could match all members. +// +// For register classes, we can generate these subgroups automatically. For +// arbitrary operands, we expect the user to define the classes and their +// relations to one another (for example, 8-bit signed immediates as a +// subset of 32-bit immediates). +// +// By partitioning the operands in this way, we guarantee that for any +// tuple of classes, any single instruction must match either all or none +// of the sets of operands which could classify to that tuple. +// +// In addition, the subset relation amongst classes induces a partial order +// on such tuples, which we use to resolve ambiguities. +// +// 2. The input can now be treated as a tuple of classes (static tokens are +// simple singleton sets). Each such tuple should generally map to a single +// instruction (we currently ignore cases where this isn't true, whee!!!), +// which we can emit a simple matcher for. +// +// Custom Operand Parsing +// ---------------------- +// +// Some targets need a custom way to parse operands, some specific instructions +// can contain arguments that can represent processor flags and other kinds of +// identifiers that need to be mapped to specific values in the final encoded +// instructions. The target specific custom operand parsing works in the +// following way: +// +// 1. A operand match table is built, each entry contains a mnemonic, an +// operand class, a mask for all operand positions for that same +// class/mnemonic and target features to be checked while trying to match. +// +// 2. The operand matcher will try every possible entry with the same +// mnemonic and will check if the target feature for this mnemonic also +// matches. After that, if the operand to be matched has its index +// present in the mask, a successful match occurs. Otherwise, fallback +// to the regular operand parsing. +// +// 3. For a match success, each operand class that has a 'ParserMethod' +// becomes part of a switch from where the custom method is called. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "SubtargetFeatureInfo.h" +#include "Types.h" +#include "llvm/ADT/CachedHashString.h" +#include "llvm/ADT/PointerUnion.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/StringMatcher.h" +#include "llvm/TableGen/StringToOffsetTable.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cassert> +#include <cctype> +#include <forward_list> +#include <map> +#include <set> + +using namespace llvm; + +#define DEBUG_TYPE "asm-matcher-emitter" + +cl::OptionCategory AsmMatcherEmitterCat("Options for -gen-asm-matcher"); + +static cl::opt<std::string> + MatchPrefix("match-prefix", cl::init(""), + cl::desc("Only match instructions with the given prefix"), + cl::cat(AsmMatcherEmitterCat)); + +namespace { +class AsmMatcherInfo; + +// Register sets are used as keys in some second-order sets TableGen creates +// when generating its data structures. This means that the order of two +// RegisterSets can be seen in the outputted AsmMatcher tables occasionally, and +// can even affect compiler output (at least seen in diagnostics produced when +// all matches fail). So we use a type that sorts them consistently. +typedef std::set<Record*, LessRecordByID> RegisterSet; + +class AsmMatcherEmitter { + RecordKeeper &Records; +public: + AsmMatcherEmitter(RecordKeeper &R) : Records(R) {} + + void run(raw_ostream &o); +}; + +/// ClassInfo - Helper class for storing the information about a particular +/// class of operands which can be matched. +struct ClassInfo { + enum ClassInfoKind { + /// Invalid kind, for use as a sentinel value. + Invalid = 0, + + /// The class for a particular token. + Token, + + /// The (first) register class, subsequent register classes are + /// RegisterClass0+1, and so on. + RegisterClass0, + + /// The (first) user defined class, subsequent user defined classes are + /// UserClass0+1, and so on. + UserClass0 = 1<<16 + }; + + /// Kind - The class kind, which is either a predefined kind, or (UserClass0 + + /// N) for the Nth user defined class. + unsigned Kind; + + /// SuperClasses - The super classes of this class. Note that for simplicities + /// sake user operands only record their immediate super class, while register + /// operands include all superclasses. + std::vector<ClassInfo*> SuperClasses; + + /// Name - The full class name, suitable for use in an enum. + std::string Name; + + /// ClassName - The unadorned generic name for this class (e.g., Token). + std::string ClassName; + + /// ValueName - The name of the value this class represents; for a token this + /// is the literal token string, for an operand it is the TableGen class (or + /// empty if this is a derived class). + std::string ValueName; + + /// PredicateMethod - The name of the operand method to test whether the + /// operand matches this class; this is not valid for Token or register kinds. + std::string PredicateMethod; + + /// RenderMethod - The name of the operand method to add this operand to an + /// MCInst; this is not valid for Token or register kinds. + std::string RenderMethod; + + /// ParserMethod - The name of the operand method to do a target specific + /// parsing on the operand. + std::string ParserMethod; + + /// For register classes: the records for all the registers in this class. + RegisterSet Registers; + + /// For custom match classes: the diagnostic kind for when the predicate fails. + std::string DiagnosticType; + + /// For custom match classes: the diagnostic string for when the predicate fails. + std::string DiagnosticString; + + /// Is this operand optional and not always required. + bool IsOptional; + + /// DefaultMethod - The name of the method that returns the default operand + /// for optional operand + std::string DefaultMethod; + +public: + /// isRegisterClass() - Check if this is a register class. + bool isRegisterClass() const { + return Kind >= RegisterClass0 && Kind < UserClass0; + } + + /// isUserClass() - Check if this is a user defined class. + bool isUserClass() const { + return Kind >= UserClass0; + } + + /// isRelatedTo - Check whether this class is "related" to \p RHS. Classes + /// are related if they are in the same class hierarchy. + bool isRelatedTo(const ClassInfo &RHS) const { + // Tokens are only related to tokens. + if (Kind == Token || RHS.Kind == Token) + return Kind == Token && RHS.Kind == Token; + + // Registers classes are only related to registers classes, and only if + // their intersection is non-empty. + if (isRegisterClass() || RHS.isRegisterClass()) { + if (!isRegisterClass() || !RHS.isRegisterClass()) + return false; + + RegisterSet Tmp; + std::insert_iterator<RegisterSet> II(Tmp, Tmp.begin()); + std::set_intersection(Registers.begin(), Registers.end(), + RHS.Registers.begin(), RHS.Registers.end(), + II, LessRecordByID()); + + return !Tmp.empty(); + } + + // Otherwise we have two users operands; they are related if they are in the + // same class hierarchy. + // + // FIXME: This is an oversimplification, they should only be related if they + // intersect, however we don't have that information. + assert(isUserClass() && RHS.isUserClass() && "Unexpected class!"); + const ClassInfo *Root = this; + while (!Root->SuperClasses.empty()) + Root = Root->SuperClasses.front(); + + const ClassInfo *RHSRoot = &RHS; + while (!RHSRoot->SuperClasses.empty()) + RHSRoot = RHSRoot->SuperClasses.front(); + + return Root == RHSRoot; + } + + /// isSubsetOf - Test whether this class is a subset of \p RHS. + bool isSubsetOf(const ClassInfo &RHS) const { + // This is a subset of RHS if it is the same class... + if (this == &RHS) + return true; + + // ... or if any of its super classes are a subset of RHS. + SmallVector<const ClassInfo *, 16> Worklist(SuperClasses.begin(), + SuperClasses.end()); + SmallPtrSet<const ClassInfo *, 16> Visited; + while (!Worklist.empty()) { + auto *CI = Worklist.pop_back_val(); + if (CI == &RHS) + return true; + for (auto *Super : CI->SuperClasses) + if (Visited.insert(Super).second) + Worklist.push_back(Super); + } + + return false; + } + + int getTreeDepth() const { + int Depth = 0; + const ClassInfo *Root = this; + while (!Root->SuperClasses.empty()) { + Depth++; + Root = Root->SuperClasses.front(); + } + return Depth; + } + + const ClassInfo *findRoot() const { + const ClassInfo *Root = this; + while (!Root->SuperClasses.empty()) + Root = Root->SuperClasses.front(); + return Root; + } + + /// Compare two classes. This does not produce a total ordering, but does + /// guarantee that subclasses are sorted before their parents, and that the + /// ordering is transitive. + bool operator<(const ClassInfo &RHS) const { + if (this == &RHS) + return false; + + // First, enforce the ordering between the three different types of class. + // Tokens sort before registers, which sort before user classes. + if (Kind == Token) { + if (RHS.Kind != Token) + return true; + assert(RHS.Kind == Token); + } else if (isRegisterClass()) { + if (RHS.Kind == Token) + return false; + else if (RHS.isUserClass()) + return true; + assert(RHS.isRegisterClass()); + } else if (isUserClass()) { + if (!RHS.isUserClass()) + return false; + assert(RHS.isUserClass()); + } else { + llvm_unreachable("Unknown ClassInfoKind"); + } + + if (Kind == Token || isUserClass()) { + // Related tokens and user classes get sorted by depth in the inheritence + // tree (so that subclasses are before their parents). + if (isRelatedTo(RHS)) { + if (getTreeDepth() > RHS.getTreeDepth()) + return true; + if (getTreeDepth() < RHS.getTreeDepth()) + return false; + } else { + // Unrelated tokens and user classes are ordered by the name of their + // root nodes, so that there is a consistent ordering between + // unconnected trees. + return findRoot()->ValueName < RHS.findRoot()->ValueName; + } + } else if (isRegisterClass()) { + // For register sets, sort by number of registers. This guarantees that + // a set will always sort before all of it's strict supersets. + if (Registers.size() != RHS.Registers.size()) + return Registers.size() < RHS.Registers.size(); + } else { + llvm_unreachable("Unknown ClassInfoKind"); + } + + // FIXME: We should be able to just return false here, as we only need a + // partial order (we use stable sorts, so this is deterministic) and the + // name of a class shouldn't be significant. However, some of the backends + // accidentally rely on this behaviour, so it will have to stay like this + // until they are fixed. + return ValueName < RHS.ValueName; + } +}; + +class AsmVariantInfo { +public: + StringRef RegisterPrefix; + StringRef TokenizingCharacters; + StringRef SeparatorCharacters; + StringRef BreakCharacters; + StringRef Name; + int AsmVariantNo; +}; + +/// MatchableInfo - Helper class for storing the necessary information for an +/// instruction or alias which is capable of being matched. +struct MatchableInfo { + struct AsmOperand { + /// Token - This is the token that the operand came from. + StringRef Token; + + /// The unique class instance this operand should match. + ClassInfo *Class; + + /// The operand name this is, if anything. + StringRef SrcOpName; + + /// The operand name this is, before renaming for tied operands. + StringRef OrigSrcOpName; + + /// The suboperand index within SrcOpName, or -1 for the entire operand. + int SubOpIdx; + + /// Whether the token is "isolated", i.e., it is preceded and followed + /// by separators. + bool IsIsolatedToken; + + /// Register record if this token is singleton register. + Record *SingletonReg; + + explicit AsmOperand(bool IsIsolatedToken, StringRef T) + : Token(T), Class(nullptr), SubOpIdx(-1), + IsIsolatedToken(IsIsolatedToken), SingletonReg(nullptr) {} + }; + + /// ResOperand - This represents a single operand in the result instruction + /// generated by the match. In cases (like addressing modes) where a single + /// assembler operand expands to multiple MCOperands, this represents the + /// single assembler operand, not the MCOperand. + struct ResOperand { + enum { + /// RenderAsmOperand - This represents an operand result that is + /// generated by calling the render method on the assembly operand. The + /// corresponding AsmOperand is specified by AsmOperandNum. + RenderAsmOperand, + + /// TiedOperand - This represents a result operand that is a duplicate of + /// a previous result operand. + TiedOperand, + + /// ImmOperand - This represents an immediate value that is dumped into + /// the operand. + ImmOperand, + + /// RegOperand - This represents a fixed register that is dumped in. + RegOperand + } Kind; + + /// Tuple containing the index of the (earlier) result operand that should + /// be copied from, as well as the indices of the corresponding (parsed) + /// operands in the asm string. + struct TiedOperandsTuple { + unsigned ResOpnd; + unsigned SrcOpnd1Idx; + unsigned SrcOpnd2Idx; + }; + + union { + /// This is the operand # in the AsmOperands list that this should be + /// copied from. + unsigned AsmOperandNum; + + /// Description of tied operands. + TiedOperandsTuple TiedOperands; + + /// ImmVal - This is the immediate value added to the instruction. + int64_t ImmVal; + + /// Register - This is the register record. + Record *Register; + }; + + /// MINumOperands - The number of MCInst operands populated by this + /// operand. + unsigned MINumOperands; + + static ResOperand getRenderedOp(unsigned AsmOpNum, unsigned NumOperands) { + ResOperand X; + X.Kind = RenderAsmOperand; + X.AsmOperandNum = AsmOpNum; + X.MINumOperands = NumOperands; + return X; + } + + static ResOperand getTiedOp(unsigned TiedOperandNum, unsigned SrcOperand1, + unsigned SrcOperand2) { + ResOperand X; + X.Kind = TiedOperand; + X.TiedOperands = { TiedOperandNum, SrcOperand1, SrcOperand2 }; + X.MINumOperands = 1; + return X; + } + + static ResOperand getImmOp(int64_t Val) { + ResOperand X; + X.Kind = ImmOperand; + X.ImmVal = Val; + X.MINumOperands = 1; + return X; + } + + static ResOperand getRegOp(Record *Reg) { + ResOperand X; + X.Kind = RegOperand; + X.Register = Reg; + X.MINumOperands = 1; + return X; + } + }; + + /// AsmVariantID - Target's assembly syntax variant no. + int AsmVariantID; + + /// AsmString - The assembly string for this instruction (with variants + /// removed), e.g. "movsx $src, $dst". + std::string AsmString; + + /// TheDef - This is the definition of the instruction or InstAlias that this + /// matchable came from. + Record *const TheDef; + + /// DefRec - This is the definition that it came from. + PointerUnion<const CodeGenInstruction*, const CodeGenInstAlias*> DefRec; + + const CodeGenInstruction *getResultInst() const { + if (DefRec.is<const CodeGenInstruction*>()) + return DefRec.get<const CodeGenInstruction*>(); + return DefRec.get<const CodeGenInstAlias*>()->ResultInst; + } + + /// ResOperands - This is the operand list that should be built for the result + /// MCInst. + SmallVector<ResOperand, 8> ResOperands; + + /// Mnemonic - This is the first token of the matched instruction, its + /// mnemonic. + StringRef Mnemonic; + + /// AsmOperands - The textual operands that this instruction matches, + /// annotated with a class and where in the OperandList they were defined. + /// This directly corresponds to the tokenized AsmString after the mnemonic is + /// removed. + SmallVector<AsmOperand, 8> AsmOperands; + + /// Predicates - The required subtarget features to match this instruction. + SmallVector<const SubtargetFeatureInfo *, 4> RequiredFeatures; + + /// ConversionFnKind - The enum value which is passed to the generated + /// convertToMCInst to convert parsed operands into an MCInst for this + /// function. + std::string ConversionFnKind; + + /// If this instruction is deprecated in some form. + bool HasDeprecation; + + /// If this is an alias, this is use to determine whether or not to using + /// the conversion function defined by the instruction's AsmMatchConverter + /// or to use the function generated by the alias. + bool UseInstAsmMatchConverter; + + MatchableInfo(const CodeGenInstruction &CGI) + : AsmVariantID(0), AsmString(CGI.AsmString), TheDef(CGI.TheDef), DefRec(&CGI), + UseInstAsmMatchConverter(true) { + } + + MatchableInfo(std::unique_ptr<const CodeGenInstAlias> Alias) + : AsmVariantID(0), AsmString(Alias->AsmString), TheDef(Alias->TheDef), + DefRec(Alias.release()), + UseInstAsmMatchConverter( + TheDef->getValueAsBit("UseInstAsmMatchConverter")) { + } + + // Could remove this and the dtor if PointerUnion supported unique_ptr + // elements with a dynamic failure/assertion (like the one below) in the case + // where it was copied while being in an owning state. + MatchableInfo(const MatchableInfo &RHS) + : AsmVariantID(RHS.AsmVariantID), AsmString(RHS.AsmString), + TheDef(RHS.TheDef), DefRec(RHS.DefRec), ResOperands(RHS.ResOperands), + Mnemonic(RHS.Mnemonic), AsmOperands(RHS.AsmOperands), + RequiredFeatures(RHS.RequiredFeatures), + ConversionFnKind(RHS.ConversionFnKind), + HasDeprecation(RHS.HasDeprecation), + UseInstAsmMatchConverter(RHS.UseInstAsmMatchConverter) { + assert(!DefRec.is<const CodeGenInstAlias *>()); + } + + ~MatchableInfo() { + delete DefRec.dyn_cast<const CodeGenInstAlias*>(); + } + + // Two-operand aliases clone from the main matchable, but mark the second + // operand as a tied operand of the first for purposes of the assembler. + void formTwoOperandAlias(StringRef Constraint); + + void initialize(const AsmMatcherInfo &Info, + SmallPtrSetImpl<Record*> &SingletonRegisters, + AsmVariantInfo const &Variant, + bool HasMnemonicFirst); + + /// validate - Return true if this matchable is a valid thing to match against + /// and perform a bunch of validity checking. + bool validate(StringRef CommentDelimiter, bool IsAlias) const; + + /// findAsmOperand - Find the AsmOperand with the specified name and + /// suboperand index. + int findAsmOperand(StringRef N, int SubOpIdx) const { + auto I = find_if(AsmOperands, [&](const AsmOperand &Op) { + return Op.SrcOpName == N && Op.SubOpIdx == SubOpIdx; + }); + return (I != AsmOperands.end()) ? I - AsmOperands.begin() : -1; + } + + /// findAsmOperandNamed - Find the first AsmOperand with the specified name. + /// This does not check the suboperand index. + int findAsmOperandNamed(StringRef N, int LastIdx = -1) const { + auto I = std::find_if(AsmOperands.begin() + LastIdx + 1, AsmOperands.end(), + [&](const AsmOperand &Op) { return Op.SrcOpName == N; }); + return (I != AsmOperands.end()) ? I - AsmOperands.begin() : -1; + } + + int findAsmOperandOriginallyNamed(StringRef N) const { + auto I = + find_if(AsmOperands, + [&](const AsmOperand &Op) { return Op.OrigSrcOpName == N; }); + return (I != AsmOperands.end()) ? I - AsmOperands.begin() : -1; + } + + void buildInstructionResultOperands(); + void buildAliasResultOperands(bool AliasConstraintsAreChecked); + + /// operator< - Compare two matchables. + bool operator<(const MatchableInfo &RHS) const { + // The primary comparator is the instruction mnemonic. + if (int Cmp = Mnemonic.compare(RHS.Mnemonic)) + return Cmp == -1; + + if (AsmOperands.size() != RHS.AsmOperands.size()) + return AsmOperands.size() < RHS.AsmOperands.size(); + + // Compare lexicographically by operand. The matcher validates that other + // orderings wouldn't be ambiguous using \see couldMatchAmbiguouslyWith(). + for (unsigned i = 0, e = AsmOperands.size(); i != e; ++i) { + if (*AsmOperands[i].Class < *RHS.AsmOperands[i].Class) + return true; + if (*RHS.AsmOperands[i].Class < *AsmOperands[i].Class) + return false; + } + + // Give matches that require more features higher precedence. This is useful + // because we cannot define AssemblerPredicates with the negation of + // processor features. For example, ARM v6 "nop" may be either a HINT or + // MOV. With v6, we want to match HINT. The assembler has no way to + // predicate MOV under "NoV6", but HINT will always match first because it + // requires V6 while MOV does not. + if (RequiredFeatures.size() != RHS.RequiredFeatures.size()) + return RequiredFeatures.size() > RHS.RequiredFeatures.size(); + + return false; + } + + /// couldMatchAmbiguouslyWith - Check whether this matchable could + /// ambiguously match the same set of operands as \p RHS (without being a + /// strictly superior match). + bool couldMatchAmbiguouslyWith(const MatchableInfo &RHS) const { + // The primary comparator is the instruction mnemonic. + if (Mnemonic != RHS.Mnemonic) + return false; + + // Different variants can't conflict. + if (AsmVariantID != RHS.AsmVariantID) + return false; + + // The number of operands is unambiguous. + if (AsmOperands.size() != RHS.AsmOperands.size()) + return false; + + // Otherwise, make sure the ordering of the two instructions is unambiguous + // by checking that either (a) a token or operand kind discriminates them, + // or (b) the ordering among equivalent kinds is consistent. + + // Tokens and operand kinds are unambiguous (assuming a correct target + // specific parser). + for (unsigned i = 0, e = AsmOperands.size(); i != e; ++i) + if (AsmOperands[i].Class->Kind != RHS.AsmOperands[i].Class->Kind || + AsmOperands[i].Class->Kind == ClassInfo::Token) + if (*AsmOperands[i].Class < *RHS.AsmOperands[i].Class || + *RHS.AsmOperands[i].Class < *AsmOperands[i].Class) + return false; + + // Otherwise, this operand could commute if all operands are equivalent, or + // there is a pair of operands that compare less than and a pair that + // compare greater than. + bool HasLT = false, HasGT = false; + for (unsigned i = 0, e = AsmOperands.size(); i != e; ++i) { + if (*AsmOperands[i].Class < *RHS.AsmOperands[i].Class) + HasLT = true; + if (*RHS.AsmOperands[i].Class < *AsmOperands[i].Class) + HasGT = true; + } + + return HasLT == HasGT; + } + + void dump() const; + +private: + void tokenizeAsmString(AsmMatcherInfo const &Info, + AsmVariantInfo const &Variant); + void addAsmOperand(StringRef Token, bool IsIsolatedToken = false); +}; + +struct OperandMatchEntry { + unsigned OperandMask; + const MatchableInfo* MI; + ClassInfo *CI; + + static OperandMatchEntry create(const MatchableInfo *mi, ClassInfo *ci, + unsigned opMask) { + OperandMatchEntry X; + X.OperandMask = opMask; + X.CI = ci; + X.MI = mi; + return X; + } +}; + +class AsmMatcherInfo { +public: + /// Tracked Records + RecordKeeper &Records; + + /// The tablegen AsmParser record. + Record *AsmParser; + + /// Target - The target information. + CodeGenTarget &Target; + + /// The classes which are needed for matching. + std::forward_list<ClassInfo> Classes; + + /// The information on the matchables to match. + std::vector<std::unique_ptr<MatchableInfo>> Matchables; + + /// Info for custom matching operands by user defined methods. + std::vector<OperandMatchEntry> OperandMatchInfo; + + /// Map of Register records to their class information. + typedef std::map<Record*, ClassInfo*, LessRecordByID> RegisterClassesTy; + RegisterClassesTy RegisterClasses; + + /// Map of Predicate records to their subtarget information. + std::map<Record *, SubtargetFeatureInfo, LessRecordByID> SubtargetFeatures; + + /// Map of AsmOperandClass records to their class information. + std::map<Record*, ClassInfo*> AsmOperandClasses; + + /// Map of RegisterClass records to their class information. + std::map<Record*, ClassInfo*> RegisterClassClasses; + +private: + /// Map of token to class information which has already been constructed. + std::map<std::string, ClassInfo*> TokenClasses; + +private: + /// getTokenClass - Lookup or create the class for the given token. + ClassInfo *getTokenClass(StringRef Token); + + /// getOperandClass - Lookup or create the class for the given operand. + ClassInfo *getOperandClass(const CGIOperandList::OperandInfo &OI, + int SubOpIdx); + ClassInfo *getOperandClass(Record *Rec, int SubOpIdx); + + /// buildRegisterClasses - Build the ClassInfo* instances for register + /// classes. + void buildRegisterClasses(SmallPtrSetImpl<Record*> &SingletonRegisters); + + /// buildOperandClasses - Build the ClassInfo* instances for user defined + /// operand classes. + void buildOperandClasses(); + + void buildInstructionOperandReference(MatchableInfo *II, StringRef OpName, + unsigned AsmOpIdx); + void buildAliasOperandReference(MatchableInfo *II, StringRef OpName, + MatchableInfo::AsmOperand &Op); + +public: + AsmMatcherInfo(Record *AsmParser, + CodeGenTarget &Target, + RecordKeeper &Records); + + /// Construct the various tables used during matching. + void buildInfo(); + + /// buildOperandMatchInfo - Build the necessary information to handle user + /// defined operand parsing methods. + void buildOperandMatchInfo(); + + /// getSubtargetFeature - Lookup or create the subtarget feature info for the + /// given operand. + const SubtargetFeatureInfo *getSubtargetFeature(Record *Def) const { + assert(Def->isSubClassOf("Predicate") && "Invalid predicate type!"); + const auto &I = SubtargetFeatures.find(Def); + return I == SubtargetFeatures.end() ? nullptr : &I->second; + } + + RecordKeeper &getRecords() const { + return Records; + } + + bool hasOptionalOperands() const { + return find_if(Classes, [](const ClassInfo &Class) { + return Class.IsOptional; + }) != Classes.end(); + } +}; + +} // end anonymous namespace + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +LLVM_DUMP_METHOD void MatchableInfo::dump() const { + errs() << TheDef->getName() << " -- " << "flattened:\"" << AsmString <<"\"\n"; + + errs() << " variant: " << AsmVariantID << "\n"; + + for (unsigned i = 0, e = AsmOperands.size(); i != e; ++i) { + const AsmOperand &Op = AsmOperands[i]; + errs() << " op[" << i << "] = " << Op.Class->ClassName << " - "; + errs() << '\"' << Op.Token << "\"\n"; + } +} +#endif + +static std::pair<StringRef, StringRef> +parseTwoOperandConstraint(StringRef S, ArrayRef<SMLoc> Loc) { + // Split via the '='. + std::pair<StringRef, StringRef> Ops = S.split('='); + if (Ops.second == "") + PrintFatalError(Loc, "missing '=' in two-operand alias constraint"); + // Trim whitespace and the leading '$' on the operand names. + size_t start = Ops.first.find_first_of('$'); + if (start == std::string::npos) + PrintFatalError(Loc, "expected '$' prefix on asm operand name"); + Ops.first = Ops.first.slice(start + 1, std::string::npos); + size_t end = Ops.first.find_last_of(" \t"); + Ops.first = Ops.first.slice(0, end); + // Now the second operand. + start = Ops.second.find_first_of('$'); + if (start == std::string::npos) + PrintFatalError(Loc, "expected '$' prefix on asm operand name"); + Ops.second = Ops.second.slice(start + 1, std::string::npos); + end = Ops.second.find_last_of(" \t"); + Ops.first = Ops.first.slice(0, end); + return Ops; +} + +void MatchableInfo::formTwoOperandAlias(StringRef Constraint) { + // Figure out which operands are aliased and mark them as tied. + std::pair<StringRef, StringRef> Ops = + parseTwoOperandConstraint(Constraint, TheDef->getLoc()); + + // Find the AsmOperands that refer to the operands we're aliasing. + int SrcAsmOperand = findAsmOperandNamed(Ops.first); + int DstAsmOperand = findAsmOperandNamed(Ops.second); + if (SrcAsmOperand == -1) + PrintFatalError(TheDef->getLoc(), + "unknown source two-operand alias operand '" + Ops.first + + "'."); + if (DstAsmOperand == -1) + PrintFatalError(TheDef->getLoc(), + "unknown destination two-operand alias operand '" + + Ops.second + "'."); + + // Find the ResOperand that refers to the operand we're aliasing away + // and update it to refer to the combined operand instead. + for (ResOperand &Op : ResOperands) { + if (Op.Kind == ResOperand::RenderAsmOperand && + Op.AsmOperandNum == (unsigned)SrcAsmOperand) { + Op.AsmOperandNum = DstAsmOperand; + break; + } + } + // Remove the AsmOperand for the alias operand. + AsmOperands.erase(AsmOperands.begin() + SrcAsmOperand); + // Adjust the ResOperand references to any AsmOperands that followed + // the one we just deleted. + for (ResOperand &Op : ResOperands) { + switch(Op.Kind) { + default: + // Nothing to do for operands that don't reference AsmOperands. + break; + case ResOperand::RenderAsmOperand: + if (Op.AsmOperandNum > (unsigned)SrcAsmOperand) + --Op.AsmOperandNum; + break; + } + } +} + +/// extractSingletonRegisterForAsmOperand - Extract singleton register, +/// if present, from specified token. +static void +extractSingletonRegisterForAsmOperand(MatchableInfo::AsmOperand &Op, + const AsmMatcherInfo &Info, + StringRef RegisterPrefix) { + StringRef Tok = Op.Token; + + // If this token is not an isolated token, i.e., it isn't separated from + // other tokens (e.g. with whitespace), don't interpret it as a register name. + if (!Op.IsIsolatedToken) + return; + + if (RegisterPrefix.empty()) { + std::string LoweredTok = Tok.lower(); + if (const CodeGenRegister *Reg = Info.Target.getRegisterByName(LoweredTok)) + Op.SingletonReg = Reg->TheDef; + return; + } + + if (!Tok.startswith(RegisterPrefix)) + return; + + StringRef RegName = Tok.substr(RegisterPrefix.size()); + if (const CodeGenRegister *Reg = Info.Target.getRegisterByName(RegName)) + Op.SingletonReg = Reg->TheDef; + + // If there is no register prefix (i.e. "%" in "%eax"), then this may + // be some random non-register token, just ignore it. +} + +void MatchableInfo::initialize(const AsmMatcherInfo &Info, + SmallPtrSetImpl<Record*> &SingletonRegisters, + AsmVariantInfo const &Variant, + bool HasMnemonicFirst) { + AsmVariantID = Variant.AsmVariantNo; + AsmString = + CodeGenInstruction::FlattenAsmStringVariants(AsmString, + Variant.AsmVariantNo); + + tokenizeAsmString(Info, Variant); + + // The first token of the instruction is the mnemonic, which must be a + // simple string, not a $foo variable or a singleton register. + if (AsmOperands.empty()) + PrintFatalError(TheDef->getLoc(), + "Instruction '" + TheDef->getName() + "' has no tokens"); + + assert(!AsmOperands[0].Token.empty()); + if (HasMnemonicFirst) { + Mnemonic = AsmOperands[0].Token; + if (Mnemonic[0] == '$') + PrintFatalError(TheDef->getLoc(), + "Invalid instruction mnemonic '" + Mnemonic + "'!"); + + // Remove the first operand, it is tracked in the mnemonic field. + AsmOperands.erase(AsmOperands.begin()); + } else if (AsmOperands[0].Token[0] != '$') + Mnemonic = AsmOperands[0].Token; + + // Compute the require features. + for (Record *Predicate : TheDef->getValueAsListOfDefs("Predicates")) + if (const SubtargetFeatureInfo *Feature = + Info.getSubtargetFeature(Predicate)) + RequiredFeatures.push_back(Feature); + + // Collect singleton registers, if used. + for (MatchableInfo::AsmOperand &Op : AsmOperands) { + extractSingletonRegisterForAsmOperand(Op, Info, Variant.RegisterPrefix); + if (Record *Reg = Op.SingletonReg) + SingletonRegisters.insert(Reg); + } + + const RecordVal *DepMask = TheDef->getValue("DeprecatedFeatureMask"); + if (!DepMask) + DepMask = TheDef->getValue("ComplexDeprecationPredicate"); + + HasDeprecation = + DepMask ? !DepMask->getValue()->getAsUnquotedString().empty() : false; +} + +/// Append an AsmOperand for the given substring of AsmString. +void MatchableInfo::addAsmOperand(StringRef Token, bool IsIsolatedToken) { + AsmOperands.push_back(AsmOperand(IsIsolatedToken, Token)); +} + +/// tokenizeAsmString - Tokenize a simplified assembly string. +void MatchableInfo::tokenizeAsmString(const AsmMatcherInfo &Info, + AsmVariantInfo const &Variant) { + StringRef String = AsmString; + size_t Prev = 0; + bool InTok = false; + bool IsIsolatedToken = true; + for (size_t i = 0, e = String.size(); i != e; ++i) { + char Char = String[i]; + if (Variant.BreakCharacters.find(Char) != std::string::npos) { + if (InTok) { + addAsmOperand(String.slice(Prev, i), false); + Prev = i; + IsIsolatedToken = false; + } + InTok = true; + continue; + } + if (Variant.TokenizingCharacters.find(Char) != std::string::npos) { + if (InTok) { + addAsmOperand(String.slice(Prev, i), IsIsolatedToken); + InTok = false; + IsIsolatedToken = false; + } + addAsmOperand(String.slice(i, i + 1), IsIsolatedToken); + Prev = i + 1; + IsIsolatedToken = true; + continue; + } + if (Variant.SeparatorCharacters.find(Char) != std::string::npos) { + if (InTok) { + addAsmOperand(String.slice(Prev, i), IsIsolatedToken); + InTok = false; + } + Prev = i + 1; + IsIsolatedToken = true; + continue; + } + + switch (Char) { + case '\\': + if (InTok) { + addAsmOperand(String.slice(Prev, i), false); + InTok = false; + IsIsolatedToken = false; + } + ++i; + assert(i != String.size() && "Invalid quoted character"); + addAsmOperand(String.slice(i, i + 1), IsIsolatedToken); + Prev = i + 1; + IsIsolatedToken = false; + break; + + case '$': { + if (InTok) { + addAsmOperand(String.slice(Prev, i), false); + InTok = false; + IsIsolatedToken = false; + } + + // If this isn't "${", start new identifier looking like "$xxx" + if (i + 1 == String.size() || String[i + 1] != '{') { + Prev = i; + break; + } + + size_t EndPos = String.find('}', i); + assert(EndPos != StringRef::npos && + "Missing brace in operand reference!"); + addAsmOperand(String.slice(i, EndPos+1), IsIsolatedToken); + Prev = EndPos + 1; + i = EndPos; + IsIsolatedToken = false; + break; + } + + default: + InTok = true; + break; + } + } + if (InTok && Prev != String.size()) + addAsmOperand(String.substr(Prev), IsIsolatedToken); +} + +bool MatchableInfo::validate(StringRef CommentDelimiter, bool IsAlias) const { + // Reject matchables with no .s string. + if (AsmString.empty()) + PrintFatalError(TheDef->getLoc(), "instruction with empty asm string"); + + // Reject any matchables with a newline in them, they should be marked + // isCodeGenOnly if they are pseudo instructions. + if (AsmString.find('\n') != std::string::npos) + PrintFatalError(TheDef->getLoc(), + "multiline instruction is not valid for the asmparser, " + "mark it isCodeGenOnly"); + + // Remove comments from the asm string. We know that the asmstring only + // has one line. + if (!CommentDelimiter.empty() && + StringRef(AsmString).find(CommentDelimiter) != StringRef::npos) + PrintFatalError(TheDef->getLoc(), + "asmstring for instruction has comment character in it, " + "mark it isCodeGenOnly"); + + // Reject matchables with operand modifiers, these aren't something we can + // handle, the target should be refactored to use operands instead of + // modifiers. + // + // Also, check for instructions which reference the operand multiple times, + // if they don't define a custom AsmMatcher: this implies a constraint that + // the built-in matching code would not honor. + std::set<std::string> OperandNames; + for (const AsmOperand &Op : AsmOperands) { + StringRef Tok = Op.Token; + if (Tok[0] == '$' && Tok.find(':') != StringRef::npos) + PrintFatalError(TheDef->getLoc(), + "matchable with operand modifier '" + Tok + + "' not supported by asm matcher. Mark isCodeGenOnly!"); + // Verify that any operand is only mentioned once. + // We reject aliases and ignore instructions for now. + if (!IsAlias && TheDef->getValueAsString("AsmMatchConverter").empty() && + Tok[0] == '$' && !OperandNames.insert(std::string(Tok)).second) { + LLVM_DEBUG({ + errs() << "warning: '" << TheDef->getName() << "': " + << "ignoring instruction with tied operand '" + << Tok << "'\n"; + }); + return false; + } + } + + return true; +} + +static std::string getEnumNameForToken(StringRef Str) { + std::string Res; + + for (StringRef::iterator it = Str.begin(), ie = Str.end(); it != ie; ++it) { + switch (*it) { + case '*': Res += "_STAR_"; break; + case '%': Res += "_PCT_"; break; + case ':': Res += "_COLON_"; break; + case '!': Res += "_EXCLAIM_"; break; + case '.': Res += "_DOT_"; break; + case '<': Res += "_LT_"; break; + case '>': Res += "_GT_"; break; + case '-': Res += "_MINUS_"; break; + case '#': Res += "_HASH_"; break; + default: + if ((*it >= 'A' && *it <= 'Z') || + (*it >= 'a' && *it <= 'z') || + (*it >= '0' && *it <= '9')) + Res += *it; + else + Res += "_" + utostr((unsigned) *it) + "_"; + } + } + + return Res; +} + +ClassInfo *AsmMatcherInfo::getTokenClass(StringRef Token) { + ClassInfo *&Entry = TokenClasses[std::string(Token)]; + + if (!Entry) { + Classes.emplace_front(); + Entry = &Classes.front(); + Entry->Kind = ClassInfo::Token; + Entry->ClassName = "Token"; + Entry->Name = "MCK_" + getEnumNameForToken(Token); + Entry->ValueName = std::string(Token); + Entry->PredicateMethod = "<invalid>"; + Entry->RenderMethod = "<invalid>"; + Entry->ParserMethod = ""; + Entry->DiagnosticType = ""; + Entry->IsOptional = false; + Entry->DefaultMethod = "<invalid>"; + } + + return Entry; +} + +ClassInfo * +AsmMatcherInfo::getOperandClass(const CGIOperandList::OperandInfo &OI, + int SubOpIdx) { + Record *Rec = OI.Rec; + if (SubOpIdx != -1) + Rec = cast<DefInit>(OI.MIOperandInfo->getArg(SubOpIdx))->getDef(); + return getOperandClass(Rec, SubOpIdx); +} + +ClassInfo * +AsmMatcherInfo::getOperandClass(Record *Rec, int SubOpIdx) { + if (Rec->isSubClassOf("RegisterOperand")) { + // RegisterOperand may have an associated ParserMatchClass. If it does, + // use it, else just fall back to the underlying register class. + const RecordVal *R = Rec->getValue("ParserMatchClass"); + if (!R || !R->getValue()) + PrintFatalError(Rec->getLoc(), + "Record `" + Rec->getName() + + "' does not have a ParserMatchClass!\n"); + + if (DefInit *DI= dyn_cast<DefInit>(R->getValue())) { + Record *MatchClass = DI->getDef(); + if (ClassInfo *CI = AsmOperandClasses[MatchClass]) + return CI; + } + + // No custom match class. Just use the register class. + Record *ClassRec = Rec->getValueAsDef("RegClass"); + if (!ClassRec) + PrintFatalError(Rec->getLoc(), "RegisterOperand `" + Rec->getName() + + "' has no associated register class!\n"); + if (ClassInfo *CI = RegisterClassClasses[ClassRec]) + return CI; + PrintFatalError(Rec->getLoc(), "register class has no class info!"); + } + + if (Rec->isSubClassOf("RegisterClass")) { + if (ClassInfo *CI = RegisterClassClasses[Rec]) + return CI; + PrintFatalError(Rec->getLoc(), "register class has no class info!"); + } + + if (!Rec->isSubClassOf("Operand")) + PrintFatalError(Rec->getLoc(), "Operand `" + Rec->getName() + + "' does not derive from class Operand!\n"); + Record *MatchClass = Rec->getValueAsDef("ParserMatchClass"); + if (ClassInfo *CI = AsmOperandClasses[MatchClass]) + return CI; + + PrintFatalError(Rec->getLoc(), "operand has no match class!"); +} + +struct LessRegisterSet { + bool operator() (const RegisterSet &LHS, const RegisterSet & RHS) const { + // std::set<T> defines its own compariso "operator<", but it + // performs a lexicographical comparison by T's innate comparison + // for some reason. We don't want non-deterministic pointer + // comparisons so use this instead. + return std::lexicographical_compare(LHS.begin(), LHS.end(), + RHS.begin(), RHS.end(), + LessRecordByID()); + } +}; + +void AsmMatcherInfo:: +buildRegisterClasses(SmallPtrSetImpl<Record*> &SingletonRegisters) { + const auto &Registers = Target.getRegBank().getRegisters(); + auto &RegClassList = Target.getRegBank().getRegClasses(); + + typedef std::set<RegisterSet, LessRegisterSet> RegisterSetSet; + + // The register sets used for matching. + RegisterSetSet RegisterSets; + + // Gather the defined sets. + for (const CodeGenRegisterClass &RC : RegClassList) + RegisterSets.insert( + RegisterSet(RC.getOrder().begin(), RC.getOrder().end())); + + // Add any required singleton sets. + for (Record *Rec : SingletonRegisters) { + RegisterSets.insert(RegisterSet(&Rec, &Rec + 1)); + } + + // Introduce derived sets where necessary (when a register does not determine + // a unique register set class), and build the mapping of registers to the set + // they should classify to. + std::map<Record*, RegisterSet> RegisterMap; + for (const CodeGenRegister &CGR : Registers) { + // Compute the intersection of all sets containing this register. + RegisterSet ContainingSet; + + for (const RegisterSet &RS : RegisterSets) { + if (!RS.count(CGR.TheDef)) + continue; + + if (ContainingSet.empty()) { + ContainingSet = RS; + continue; + } + + RegisterSet Tmp; + std::swap(Tmp, ContainingSet); + std::insert_iterator<RegisterSet> II(ContainingSet, + ContainingSet.begin()); + std::set_intersection(Tmp.begin(), Tmp.end(), RS.begin(), RS.end(), II, + LessRecordByID()); + } + + if (!ContainingSet.empty()) { + RegisterSets.insert(ContainingSet); + RegisterMap.insert(std::make_pair(CGR.TheDef, ContainingSet)); + } + } + + // Construct the register classes. + std::map<RegisterSet, ClassInfo*, LessRegisterSet> RegisterSetClasses; + unsigned Index = 0; + for (const RegisterSet &RS : RegisterSets) { + Classes.emplace_front(); + ClassInfo *CI = &Classes.front(); + CI->Kind = ClassInfo::RegisterClass0 + Index; + CI->ClassName = "Reg" + utostr(Index); + CI->Name = "MCK_Reg" + utostr(Index); + CI->ValueName = ""; + CI->PredicateMethod = ""; // unused + CI->RenderMethod = "addRegOperands"; + CI->Registers = RS; + // FIXME: diagnostic type. + CI->DiagnosticType = ""; + CI->IsOptional = false; + CI->DefaultMethod = ""; // unused + RegisterSetClasses.insert(std::make_pair(RS, CI)); + ++Index; + } + + // Find the superclasses; we could compute only the subgroup lattice edges, + // but there isn't really a point. + for (const RegisterSet &RS : RegisterSets) { + ClassInfo *CI = RegisterSetClasses[RS]; + for (const RegisterSet &RS2 : RegisterSets) + if (RS != RS2 && + std::includes(RS2.begin(), RS2.end(), RS.begin(), RS.end(), + LessRecordByID())) + CI->SuperClasses.push_back(RegisterSetClasses[RS2]); + } + + // Name the register classes which correspond to a user defined RegisterClass. + for (const CodeGenRegisterClass &RC : RegClassList) { + // Def will be NULL for non-user defined register classes. + Record *Def = RC.getDef(); + if (!Def) + continue; + ClassInfo *CI = RegisterSetClasses[RegisterSet(RC.getOrder().begin(), + RC.getOrder().end())]; + if (CI->ValueName.empty()) { + CI->ClassName = RC.getName(); + CI->Name = "MCK_" + RC.getName(); + CI->ValueName = RC.getName(); + } else + CI->ValueName = CI->ValueName + "," + RC.getName(); + + Init *DiagnosticType = Def->getValueInit("DiagnosticType"); + if (StringInit *SI = dyn_cast<StringInit>(DiagnosticType)) + CI->DiagnosticType = std::string(SI->getValue()); + + Init *DiagnosticString = Def->getValueInit("DiagnosticString"); + if (StringInit *SI = dyn_cast<StringInit>(DiagnosticString)) + CI->DiagnosticString = std::string(SI->getValue()); + + // If we have a diagnostic string but the diagnostic type is not specified + // explicitly, create an anonymous diagnostic type. + if (!CI->DiagnosticString.empty() && CI->DiagnosticType.empty()) + CI->DiagnosticType = RC.getName(); + + RegisterClassClasses.insert(std::make_pair(Def, CI)); + } + + // Populate the map for individual registers. + for (std::map<Record*, RegisterSet>::iterator it = RegisterMap.begin(), + ie = RegisterMap.end(); it != ie; ++it) + RegisterClasses[it->first] = RegisterSetClasses[it->second]; + + // Name the register classes which correspond to singleton registers. + for (Record *Rec : SingletonRegisters) { + ClassInfo *CI = RegisterClasses[Rec]; + assert(CI && "Missing singleton register class info!"); + + if (CI->ValueName.empty()) { + CI->ClassName = std::string(Rec->getName()); + CI->Name = "MCK_" + Rec->getName().str(); + CI->ValueName = std::string(Rec->getName()); + } else + CI->ValueName = CI->ValueName + "," + Rec->getName().str(); + } +} + +void AsmMatcherInfo::buildOperandClasses() { + std::vector<Record*> AsmOperands = + Records.getAllDerivedDefinitions("AsmOperandClass"); + + // Pre-populate AsmOperandClasses map. + for (Record *Rec : AsmOperands) { + Classes.emplace_front(); + AsmOperandClasses[Rec] = &Classes.front(); + } + + unsigned Index = 0; + for (Record *Rec : AsmOperands) { + ClassInfo *CI = AsmOperandClasses[Rec]; + CI->Kind = ClassInfo::UserClass0 + Index; + + ListInit *Supers = Rec->getValueAsListInit("SuperClasses"); + for (Init *I : Supers->getValues()) { + DefInit *DI = dyn_cast<DefInit>(I); + if (!DI) { + PrintError(Rec->getLoc(), "Invalid super class reference!"); + continue; + } + + ClassInfo *SC = AsmOperandClasses[DI->getDef()]; + if (!SC) + PrintError(Rec->getLoc(), "Invalid super class reference!"); + else + CI->SuperClasses.push_back(SC); + } + CI->ClassName = std::string(Rec->getValueAsString("Name")); + CI->Name = "MCK_" + CI->ClassName; + CI->ValueName = std::string(Rec->getName()); + + // Get or construct the predicate method name. + Init *PMName = Rec->getValueInit("PredicateMethod"); + if (StringInit *SI = dyn_cast<StringInit>(PMName)) { + CI->PredicateMethod = std::string(SI->getValue()); + } else { + assert(isa<UnsetInit>(PMName) && "Unexpected PredicateMethod field!"); + CI->PredicateMethod = "is" + CI->ClassName; + } + + // Get or construct the render method name. + Init *RMName = Rec->getValueInit("RenderMethod"); + if (StringInit *SI = dyn_cast<StringInit>(RMName)) { + CI->RenderMethod = std::string(SI->getValue()); + } else { + assert(isa<UnsetInit>(RMName) && "Unexpected RenderMethod field!"); + CI->RenderMethod = "add" + CI->ClassName + "Operands"; + } + + // Get the parse method name or leave it as empty. + Init *PRMName = Rec->getValueInit("ParserMethod"); + if (StringInit *SI = dyn_cast<StringInit>(PRMName)) + CI->ParserMethod = std::string(SI->getValue()); + + // Get the diagnostic type and string or leave them as empty. + Init *DiagnosticType = Rec->getValueInit("DiagnosticType"); + if (StringInit *SI = dyn_cast<StringInit>(DiagnosticType)) + CI->DiagnosticType = std::string(SI->getValue()); + Init *DiagnosticString = Rec->getValueInit("DiagnosticString"); + if (StringInit *SI = dyn_cast<StringInit>(DiagnosticString)) + CI->DiagnosticString = std::string(SI->getValue()); + // If we have a DiagnosticString, we need a DiagnosticType for use within + // the matcher. + if (!CI->DiagnosticString.empty() && CI->DiagnosticType.empty()) + CI->DiagnosticType = CI->ClassName; + + Init *IsOptional = Rec->getValueInit("IsOptional"); + if (BitInit *BI = dyn_cast<BitInit>(IsOptional)) + CI->IsOptional = BI->getValue(); + + // Get or construct the default method name. + Init *DMName = Rec->getValueInit("DefaultMethod"); + if (StringInit *SI = dyn_cast<StringInit>(DMName)) { + CI->DefaultMethod = std::string(SI->getValue()); + } else { + assert(isa<UnsetInit>(DMName) && "Unexpected DefaultMethod field!"); + CI->DefaultMethod = "default" + CI->ClassName + "Operands"; + } + + ++Index; + } +} + +AsmMatcherInfo::AsmMatcherInfo(Record *asmParser, + CodeGenTarget &target, + RecordKeeper &records) + : Records(records), AsmParser(asmParser), Target(target) { +} + +/// buildOperandMatchInfo - Build the necessary information to handle user +/// defined operand parsing methods. +void AsmMatcherInfo::buildOperandMatchInfo() { + + /// Map containing a mask with all operands indices that can be found for + /// that class inside a instruction. + typedef std::map<ClassInfo *, unsigned, deref<std::less<>>> OpClassMaskTy; + OpClassMaskTy OpClassMask; + + for (const auto &MI : Matchables) { + OpClassMask.clear(); + + // Keep track of all operands of this instructions which belong to the + // same class. + for (unsigned i = 0, e = MI->AsmOperands.size(); i != e; ++i) { + const MatchableInfo::AsmOperand &Op = MI->AsmOperands[i]; + if (Op.Class->ParserMethod.empty()) + continue; + unsigned &OperandMask = OpClassMask[Op.Class]; + OperandMask |= (1 << i); + } + + // Generate operand match info for each mnemonic/operand class pair. + for (const auto &OCM : OpClassMask) { + unsigned OpMask = OCM.second; + ClassInfo *CI = OCM.first; + OperandMatchInfo.push_back(OperandMatchEntry::create(MI.get(), CI, + OpMask)); + } + } +} + +void AsmMatcherInfo::buildInfo() { + // Build information about all of the AssemblerPredicates. + const std::vector<std::pair<Record *, SubtargetFeatureInfo>> + &SubtargetFeaturePairs = SubtargetFeatureInfo::getAll(Records); + SubtargetFeatures.insert(SubtargetFeaturePairs.begin(), + SubtargetFeaturePairs.end()); +#ifndef NDEBUG + for (const auto &Pair : SubtargetFeatures) + LLVM_DEBUG(Pair.second.dump()); +#endif // NDEBUG + + bool HasMnemonicFirst = AsmParser->getValueAsBit("HasMnemonicFirst"); + bool ReportMultipleNearMisses = + AsmParser->getValueAsBit("ReportMultipleNearMisses"); + + // Parse the instructions; we need to do this first so that we can gather the + // singleton register classes. + SmallPtrSet<Record*, 16> SingletonRegisters; + unsigned VariantCount = Target.getAsmParserVariantCount(); + for (unsigned VC = 0; VC != VariantCount; ++VC) { + Record *AsmVariant = Target.getAsmParserVariant(VC); + StringRef CommentDelimiter = + AsmVariant->getValueAsString("CommentDelimiter"); + AsmVariantInfo Variant; + Variant.RegisterPrefix = AsmVariant->getValueAsString("RegisterPrefix"); + Variant.TokenizingCharacters = + AsmVariant->getValueAsString("TokenizingCharacters"); + Variant.SeparatorCharacters = + AsmVariant->getValueAsString("SeparatorCharacters"); + Variant.BreakCharacters = + AsmVariant->getValueAsString("BreakCharacters"); + Variant.Name = AsmVariant->getValueAsString("Name"); + Variant.AsmVariantNo = AsmVariant->getValueAsInt("Variant"); + + for (const CodeGenInstruction *CGI : Target.getInstructionsByEnumValue()) { + + // If the tblgen -match-prefix option is specified (for tblgen hackers), + // filter the set of instructions we consider. + if (!StringRef(CGI->TheDef->getName()).startswith(MatchPrefix)) + continue; + + // Ignore "codegen only" instructions. + if (CGI->TheDef->getValueAsBit("isCodeGenOnly")) + continue; + + // Ignore instructions for different instructions + StringRef V = CGI->TheDef->getValueAsString("AsmVariantName"); + if (!V.empty() && V != Variant.Name) + continue; + + auto II = std::make_unique<MatchableInfo>(*CGI); + + II->initialize(*this, SingletonRegisters, Variant, HasMnemonicFirst); + + // Ignore instructions which shouldn't be matched and diagnose invalid + // instruction definitions with an error. + if (!II->validate(CommentDelimiter, false)) + continue; + + Matchables.push_back(std::move(II)); + } + + // Parse all of the InstAlias definitions and stick them in the list of + // matchables. + std::vector<Record*> AllInstAliases = + Records.getAllDerivedDefinitions("InstAlias"); + for (unsigned i = 0, e = AllInstAliases.size(); i != e; ++i) { + auto Alias = std::make_unique<CodeGenInstAlias>(AllInstAliases[i], + Target); + + // If the tblgen -match-prefix option is specified (for tblgen hackers), + // filter the set of instruction aliases we consider, based on the target + // instruction. + if (!StringRef(Alias->ResultInst->TheDef->getName()) + .startswith( MatchPrefix)) + continue; + + StringRef V = Alias->TheDef->getValueAsString("AsmVariantName"); + if (!V.empty() && V != Variant.Name) + continue; + + auto II = std::make_unique<MatchableInfo>(std::move(Alias)); + + II->initialize(*this, SingletonRegisters, Variant, HasMnemonicFirst); + + // Validate the alias definitions. + II->validate(CommentDelimiter, true); + + Matchables.push_back(std::move(II)); + } + } + + // Build info for the register classes. + buildRegisterClasses(SingletonRegisters); + + // Build info for the user defined assembly operand classes. + buildOperandClasses(); + + // Build the information about matchables, now that we have fully formed + // classes. + std::vector<std::unique_ptr<MatchableInfo>> NewMatchables; + for (auto &II : Matchables) { + // Parse the tokens after the mnemonic. + // Note: buildInstructionOperandReference may insert new AsmOperands, so + // don't precompute the loop bound. + for (unsigned i = 0; i != II->AsmOperands.size(); ++i) { + MatchableInfo::AsmOperand &Op = II->AsmOperands[i]; + StringRef Token = Op.Token; + + // Check for singleton registers. + if (Record *RegRecord = Op.SingletonReg) { + Op.Class = RegisterClasses[RegRecord]; + assert(Op.Class && Op.Class->Registers.size() == 1 && + "Unexpected class for singleton register"); + continue; + } + + // Check for simple tokens. + if (Token[0] != '$') { + Op.Class = getTokenClass(Token); + continue; + } + + if (Token.size() > 1 && isdigit(Token[1])) { + Op.Class = getTokenClass(Token); + continue; + } + + // Otherwise this is an operand reference. + StringRef OperandName; + if (Token[1] == '{') + OperandName = Token.substr(2, Token.size() - 3); + else + OperandName = Token.substr(1); + + if (II->DefRec.is<const CodeGenInstruction*>()) + buildInstructionOperandReference(II.get(), OperandName, i); + else + buildAliasOperandReference(II.get(), OperandName, Op); + } + + if (II->DefRec.is<const CodeGenInstruction*>()) { + II->buildInstructionResultOperands(); + // If the instruction has a two-operand alias, build up the + // matchable here. We'll add them in bulk at the end to avoid + // confusing this loop. + StringRef Constraint = + II->TheDef->getValueAsString("TwoOperandAliasConstraint"); + if (Constraint != "") { + // Start by making a copy of the original matchable. + auto AliasII = std::make_unique<MatchableInfo>(*II); + + // Adjust it to be a two-operand alias. + AliasII->formTwoOperandAlias(Constraint); + + // Add the alias to the matchables list. + NewMatchables.push_back(std::move(AliasII)); + } + } else + // FIXME: The tied operands checking is not yet integrated with the + // framework for reporting multiple near misses. To prevent invalid + // formats from being matched with an alias if a tied-operands check + // would otherwise have disallowed it, we just disallow such constructs + // in TableGen completely. + II->buildAliasResultOperands(!ReportMultipleNearMisses); + } + if (!NewMatchables.empty()) + Matchables.insert(Matchables.end(), + std::make_move_iterator(NewMatchables.begin()), + std::make_move_iterator(NewMatchables.end())); + + // Process token alias definitions and set up the associated superclass + // information. + std::vector<Record*> AllTokenAliases = + Records.getAllDerivedDefinitions("TokenAlias"); + for (Record *Rec : AllTokenAliases) { + ClassInfo *FromClass = getTokenClass(Rec->getValueAsString("FromToken")); + ClassInfo *ToClass = getTokenClass(Rec->getValueAsString("ToToken")); + if (FromClass == ToClass) + PrintFatalError(Rec->getLoc(), + "error: Destination value identical to source value."); + FromClass->SuperClasses.push_back(ToClass); + } + + // Reorder classes so that classes precede super classes. + Classes.sort(); + +#ifdef EXPENSIVE_CHECKS + // Verify that the table is sorted and operator < works transitively. + for (auto I = Classes.begin(), E = Classes.end(); I != E; ++I) { + for (auto J = I; J != E; ++J) { + assert(!(*J < *I)); + assert(I == J || !J->isSubsetOf(*I)); + } + } +#endif +} + +/// buildInstructionOperandReference - The specified operand is a reference to a +/// named operand such as $src. Resolve the Class and OperandInfo pointers. +void AsmMatcherInfo:: +buildInstructionOperandReference(MatchableInfo *II, + StringRef OperandName, + unsigned AsmOpIdx) { + const CodeGenInstruction &CGI = *II->DefRec.get<const CodeGenInstruction*>(); + const CGIOperandList &Operands = CGI.Operands; + MatchableInfo::AsmOperand *Op = &II->AsmOperands[AsmOpIdx]; + + // Map this token to an operand. + unsigned Idx; + if (!Operands.hasOperandNamed(OperandName, Idx)) + PrintFatalError(II->TheDef->getLoc(), + "error: unable to find operand: '" + OperandName + "'"); + + // If the instruction operand has multiple suboperands, but the parser + // match class for the asm operand is still the default "ImmAsmOperand", + // then handle each suboperand separately. + if (Op->SubOpIdx == -1 && Operands[Idx].MINumOperands > 1) { + Record *Rec = Operands[Idx].Rec; + assert(Rec->isSubClassOf("Operand") && "Unexpected operand!"); + Record *MatchClass = Rec->getValueAsDef("ParserMatchClass"); + if (MatchClass && MatchClass->getValueAsString("Name") == "Imm") { + // Insert remaining suboperands after AsmOpIdx in II->AsmOperands. + StringRef Token = Op->Token; // save this in case Op gets moved + for (unsigned SI = 1, SE = Operands[Idx].MINumOperands; SI != SE; ++SI) { + MatchableInfo::AsmOperand NewAsmOp(/*IsIsolatedToken=*/true, Token); + NewAsmOp.SubOpIdx = SI; + II->AsmOperands.insert(II->AsmOperands.begin()+AsmOpIdx+SI, NewAsmOp); + } + // Replace Op with first suboperand. + Op = &II->AsmOperands[AsmOpIdx]; // update the pointer in case it moved + Op->SubOpIdx = 0; + } + } + + // Set up the operand class. + Op->Class = getOperandClass(Operands[Idx], Op->SubOpIdx); + Op->OrigSrcOpName = OperandName; + + // If the named operand is tied, canonicalize it to the untied operand. + // For example, something like: + // (outs GPR:$dst), (ins GPR:$src) + // with an asmstring of + // "inc $src" + // we want to canonicalize to: + // "inc $dst" + // so that we know how to provide the $dst operand when filling in the result. + int OITied = -1; + if (Operands[Idx].MINumOperands == 1) + OITied = Operands[Idx].getTiedRegister(); + if (OITied != -1) { + // The tied operand index is an MIOperand index, find the operand that + // contains it. + std::pair<unsigned, unsigned> Idx = Operands.getSubOperandNumber(OITied); + OperandName = Operands[Idx.first].Name; + Op->SubOpIdx = Idx.second; + } + + Op->SrcOpName = OperandName; +} + +/// buildAliasOperandReference - When parsing an operand reference out of the +/// matching string (e.g. "movsx $src, $dst"), determine what the class of the +/// operand reference is by looking it up in the result pattern definition. +void AsmMatcherInfo::buildAliasOperandReference(MatchableInfo *II, + StringRef OperandName, + MatchableInfo::AsmOperand &Op) { + const CodeGenInstAlias &CGA = *II->DefRec.get<const CodeGenInstAlias*>(); + + // Set up the operand class. + for (unsigned i = 0, e = CGA.ResultOperands.size(); i != e; ++i) + if (CGA.ResultOperands[i].isRecord() && + CGA.ResultOperands[i].getName() == OperandName) { + // It's safe to go with the first one we find, because CodeGenInstAlias + // validates that all operands with the same name have the same record. + Op.SubOpIdx = CGA.ResultInstOperandIndex[i].second; + // Use the match class from the Alias definition, not the + // destination instruction, as we may have an immediate that's + // being munged by the match class. + Op.Class = getOperandClass(CGA.ResultOperands[i].getRecord(), + Op.SubOpIdx); + Op.SrcOpName = OperandName; + Op.OrigSrcOpName = OperandName; + return; + } + + PrintFatalError(II->TheDef->getLoc(), + "error: unable to find operand: '" + OperandName + "'"); +} + +void MatchableInfo::buildInstructionResultOperands() { + const CodeGenInstruction *ResultInst = getResultInst(); + + // Loop over all operands of the result instruction, determining how to + // populate them. + for (const CGIOperandList::OperandInfo &OpInfo : ResultInst->Operands) { + // If this is a tied operand, just copy from the previously handled operand. + int TiedOp = -1; + if (OpInfo.MINumOperands == 1) + TiedOp = OpInfo.getTiedRegister(); + if (TiedOp != -1) { + int TiedSrcOperand = findAsmOperandOriginallyNamed(OpInfo.Name); + if (TiedSrcOperand != -1 && + ResOperands[TiedOp].Kind == ResOperand::RenderAsmOperand) + ResOperands.push_back(ResOperand::getTiedOp( + TiedOp, ResOperands[TiedOp].AsmOperandNum, TiedSrcOperand)); + else + ResOperands.push_back(ResOperand::getTiedOp(TiedOp, 0, 0)); + continue; + } + + int SrcOperand = findAsmOperandNamed(OpInfo.Name); + if (OpInfo.Name.empty() || SrcOperand == -1) { + // This may happen for operands that are tied to a suboperand of a + // complex operand. Simply use a dummy value here; nobody should + // use this operand slot. + // FIXME: The long term goal is for the MCOperand list to not contain + // tied operands at all. + ResOperands.push_back(ResOperand::getImmOp(0)); + continue; + } + + // Check if the one AsmOperand populates the entire operand. + unsigned NumOperands = OpInfo.MINumOperands; + if (AsmOperands[SrcOperand].SubOpIdx == -1) { + ResOperands.push_back(ResOperand::getRenderedOp(SrcOperand, NumOperands)); + continue; + } + + // Add a separate ResOperand for each suboperand. + for (unsigned AI = 0; AI < NumOperands; ++AI) { + assert(AsmOperands[SrcOperand+AI].SubOpIdx == (int)AI && + AsmOperands[SrcOperand+AI].SrcOpName == OpInfo.Name && + "unexpected AsmOperands for suboperands"); + ResOperands.push_back(ResOperand::getRenderedOp(SrcOperand + AI, 1)); + } + } +} + +void MatchableInfo::buildAliasResultOperands(bool AliasConstraintsAreChecked) { + const CodeGenInstAlias &CGA = *DefRec.get<const CodeGenInstAlias*>(); + const CodeGenInstruction *ResultInst = getResultInst(); + + // Map of: $reg -> #lastref + // where $reg is the name of the operand in the asm string + // where #lastref is the last processed index where $reg was referenced in + // the asm string. + SmallDenseMap<StringRef, int> OperandRefs; + + // Loop over all operands of the result instruction, determining how to + // populate them. + unsigned AliasOpNo = 0; + unsigned LastOpNo = CGA.ResultInstOperandIndex.size(); + for (unsigned i = 0, e = ResultInst->Operands.size(); i != e; ++i) { + const CGIOperandList::OperandInfo *OpInfo = &ResultInst->Operands[i]; + + // If this is a tied operand, just copy from the previously handled operand. + int TiedOp = -1; + if (OpInfo->MINumOperands == 1) + TiedOp = OpInfo->getTiedRegister(); + if (TiedOp != -1) { + unsigned SrcOp1 = 0; + unsigned SrcOp2 = 0; + + // If an operand has been specified twice in the asm string, + // add the two source operand's indices to the TiedOp so that + // at runtime the 'tied' constraint is checked. + if (ResOperands[TiedOp].Kind == ResOperand::RenderAsmOperand) { + SrcOp1 = ResOperands[TiedOp].AsmOperandNum; + + // Find the next operand (similarly named operand) in the string. + StringRef Name = AsmOperands[SrcOp1].SrcOpName; + auto Insert = OperandRefs.try_emplace(Name, SrcOp1); + SrcOp2 = findAsmOperandNamed(Name, Insert.first->second); + + // Not updating the record in OperandRefs will cause TableGen + // to fail with an error at the end of this function. + if (AliasConstraintsAreChecked) + Insert.first->second = SrcOp2; + + // In case it only has one reference in the asm string, + // it doesn't need to be checked for tied constraints. + SrcOp2 = (SrcOp2 == (unsigned)-1) ? SrcOp1 : SrcOp2; + } + + // If the alias operand is of a different operand class, we only want + // to benefit from the tied-operands check and just match the operand + // as a normal, but not copy the original (TiedOp) to the result + // instruction. We do this by passing -1 as the tied operand to copy. + if (ResultInst->Operands[i].Rec->getName() != + ResultInst->Operands[TiedOp].Rec->getName()) { + SrcOp1 = ResOperands[TiedOp].AsmOperandNum; + int SubIdx = CGA.ResultInstOperandIndex[AliasOpNo].second; + StringRef Name = CGA.ResultOperands[AliasOpNo].getName(); + SrcOp2 = findAsmOperand(Name, SubIdx); + ResOperands.push_back( + ResOperand::getTiedOp((unsigned)-1, SrcOp1, SrcOp2)); + } else { + ResOperands.push_back(ResOperand::getTiedOp(TiedOp, SrcOp1, SrcOp2)); + continue; + } + } + + // Handle all the suboperands for this operand. + const std::string &OpName = OpInfo->Name; + for ( ; AliasOpNo < LastOpNo && + CGA.ResultInstOperandIndex[AliasOpNo].first == i; ++AliasOpNo) { + int SubIdx = CGA.ResultInstOperandIndex[AliasOpNo].second; + + // Find out what operand from the asmparser that this MCInst operand + // comes from. + switch (CGA.ResultOperands[AliasOpNo].Kind) { + case CodeGenInstAlias::ResultOperand::K_Record: { + StringRef Name = CGA.ResultOperands[AliasOpNo].getName(); + int SrcOperand = findAsmOperand(Name, SubIdx); + if (SrcOperand == -1) + PrintFatalError(TheDef->getLoc(), "Instruction '" + + TheDef->getName() + "' has operand '" + OpName + + "' that doesn't appear in asm string!"); + + // Add it to the operand references. If it is added a second time, the + // record won't be updated and it will fail later on. + OperandRefs.try_emplace(Name, SrcOperand); + + unsigned NumOperands = (SubIdx == -1 ? OpInfo->MINumOperands : 1); + ResOperands.push_back(ResOperand::getRenderedOp(SrcOperand, + NumOperands)); + break; + } + case CodeGenInstAlias::ResultOperand::K_Imm: { + int64_t ImmVal = CGA.ResultOperands[AliasOpNo].getImm(); + ResOperands.push_back(ResOperand::getImmOp(ImmVal)); + break; + } + case CodeGenInstAlias::ResultOperand::K_Reg: { + Record *Reg = CGA.ResultOperands[AliasOpNo].getRegister(); + ResOperands.push_back(ResOperand::getRegOp(Reg)); + break; + } + } + } + } + + // Check that operands are not repeated more times than is supported. + for (auto &T : OperandRefs) { + if (T.second != -1 && findAsmOperandNamed(T.first, T.second) != -1) + PrintFatalError(TheDef->getLoc(), + "Operand '" + T.first + "' can never be matched"); + } +} + +static unsigned +getConverterOperandID(const std::string &Name, + SmallSetVector<CachedHashString, 16> &Table, + bool &IsNew) { + IsNew = Table.insert(CachedHashString(Name)); + + unsigned ID = IsNew ? Table.size() - 1 : find(Table, Name) - Table.begin(); + + assert(ID < Table.size()); + + return ID; +} + +static unsigned +emitConvertFuncs(CodeGenTarget &Target, StringRef ClassName, + std::vector<std::unique_ptr<MatchableInfo>> &Infos, + bool HasMnemonicFirst, bool HasOptionalOperands, + raw_ostream &OS) { + SmallSetVector<CachedHashString, 16> OperandConversionKinds; + SmallSetVector<CachedHashString, 16> InstructionConversionKinds; + std::vector<std::vector<uint8_t> > ConversionTable; + size_t MaxRowLength = 2; // minimum is custom converter plus terminator. + + // TargetOperandClass - This is the target's operand class, like X86Operand. + std::string TargetOperandClass = Target.getName().str() + "Operand"; + + // Write the convert function to a separate stream, so we can drop it after + // the enum. We'll build up the conversion handlers for the individual + // operand types opportunistically as we encounter them. + std::string ConvertFnBody; + raw_string_ostream CvtOS(ConvertFnBody); + // Start the unified conversion function. + if (HasOptionalOperands) { + CvtOS << "void " << Target.getName() << ClassName << "::\n" + << "convertToMCInst(unsigned Kind, MCInst &Inst, " + << "unsigned Opcode,\n" + << " const OperandVector &Operands,\n" + << " const SmallBitVector &OptionalOperandsMask) {\n"; + } else { + CvtOS << "void " << Target.getName() << ClassName << "::\n" + << "convertToMCInst(unsigned Kind, MCInst &Inst, " + << "unsigned Opcode,\n" + << " const OperandVector &Operands) {\n"; + } + CvtOS << " assert(Kind < CVT_NUM_SIGNATURES && \"Invalid signature!\");\n"; + CvtOS << " const uint8_t *Converter = ConversionTable[Kind];\n"; + if (HasOptionalOperands) { + size_t MaxNumOperands = 0; + for (const auto &MI : Infos) { + MaxNumOperands = std::max(MaxNumOperands, MI->AsmOperands.size()); + } + CvtOS << " unsigned DefaultsOffset[" << (MaxNumOperands + 1) + << "] = { 0 };\n"; + CvtOS << " assert(OptionalOperandsMask.size() == " << (MaxNumOperands) + << ");\n"; + CvtOS << " for (unsigned i = 0, NumDefaults = 0; i < " << (MaxNumOperands) + << "; ++i) {\n"; + CvtOS << " DefaultsOffset[i + 1] = NumDefaults;\n"; + CvtOS << " NumDefaults += (OptionalOperandsMask[i] ? 1 : 0);\n"; + CvtOS << " }\n"; + } + CvtOS << " unsigned OpIdx;\n"; + CvtOS << " Inst.setOpcode(Opcode);\n"; + CvtOS << " for (const uint8_t *p = Converter; *p; p+= 2) {\n"; + if (HasOptionalOperands) { + CvtOS << " OpIdx = *(p + 1) - DefaultsOffset[*(p + 1)];\n"; + } else { + CvtOS << " OpIdx = *(p + 1);\n"; + } + CvtOS << " switch (*p) {\n"; + CvtOS << " default: llvm_unreachable(\"invalid conversion entry!\");\n"; + CvtOS << " case CVT_Reg:\n"; + CvtOS << " static_cast<" << TargetOperandClass + << "&>(*Operands[OpIdx]).addRegOperands(Inst, 1);\n"; + CvtOS << " break;\n"; + CvtOS << " case CVT_Tied: {\n"; + CvtOS << " assert(OpIdx < (size_t)(std::end(TiedAsmOperandTable) -\n"; + CvtOS << " std::begin(TiedAsmOperandTable)) &&\n"; + CvtOS << " \"Tied operand not found\");\n"; + CvtOS << " unsigned TiedResOpnd = TiedAsmOperandTable[OpIdx][0];\n"; + CvtOS << " if (TiedResOpnd != (uint8_t) -1)\n"; + CvtOS << " Inst.addOperand(Inst.getOperand(TiedResOpnd));\n"; + CvtOS << " break;\n"; + CvtOS << " }\n"; + + std::string OperandFnBody; + raw_string_ostream OpOS(OperandFnBody); + // Start the operand number lookup function. + OpOS << "void " << Target.getName() << ClassName << "::\n" + << "convertToMapAndConstraints(unsigned Kind,\n"; + OpOS.indent(27); + OpOS << "const OperandVector &Operands) {\n" + << " assert(Kind < CVT_NUM_SIGNATURES && \"Invalid signature!\");\n" + << " unsigned NumMCOperands = 0;\n" + << " const uint8_t *Converter = ConversionTable[Kind];\n" + << " for (const uint8_t *p = Converter; *p; p+= 2) {\n" + << " switch (*p) {\n" + << " default: llvm_unreachable(\"invalid conversion entry!\");\n" + << " case CVT_Reg:\n" + << " Operands[*(p + 1)]->setMCOperandNum(NumMCOperands);\n" + << " Operands[*(p + 1)]->setConstraint(\"r\");\n" + << " ++NumMCOperands;\n" + << " break;\n" + << " case CVT_Tied:\n" + << " ++NumMCOperands;\n" + << " break;\n"; + + // Pre-populate the operand conversion kinds with the standard always + // available entries. + OperandConversionKinds.insert(CachedHashString("CVT_Done")); + OperandConversionKinds.insert(CachedHashString("CVT_Reg")); + OperandConversionKinds.insert(CachedHashString("CVT_Tied")); + enum { CVT_Done, CVT_Reg, CVT_Tied }; + + // Map of e.g. <0, 2, 3> -> "Tie_0_2_3" enum label. + std::map<std::tuple<uint8_t, uint8_t, uint8_t>, std::string> + TiedOperandsEnumMap; + + for (auto &II : Infos) { + // Check if we have a custom match function. + StringRef AsmMatchConverter = + II->getResultInst()->TheDef->getValueAsString("AsmMatchConverter"); + if (!AsmMatchConverter.empty() && II->UseInstAsmMatchConverter) { + std::string Signature = ("ConvertCustom_" + AsmMatchConverter).str(); + II->ConversionFnKind = Signature; + + // Check if we have already generated this signature. + if (!InstructionConversionKinds.insert(CachedHashString(Signature))) + continue; + + // Remember this converter for the kind enum. + unsigned KindID = OperandConversionKinds.size(); + OperandConversionKinds.insert( + CachedHashString("CVT_" + getEnumNameForToken(AsmMatchConverter))); + + // Add the converter row for this instruction. + ConversionTable.emplace_back(); + ConversionTable.back().push_back(KindID); + ConversionTable.back().push_back(CVT_Done); + + // Add the handler to the conversion driver function. + CvtOS << " case CVT_" + << getEnumNameForToken(AsmMatchConverter) << ":\n" + << " " << AsmMatchConverter << "(Inst, Operands);\n" + << " break;\n"; + + // FIXME: Handle the operand number lookup for custom match functions. + continue; + } + + // Build the conversion function signature. + std::string Signature = "Convert"; + + std::vector<uint8_t> ConversionRow; + + // Compute the convert enum and the case body. + MaxRowLength = std::max(MaxRowLength, II->ResOperands.size()*2 + 1 ); + + for (unsigned i = 0, e = II->ResOperands.size(); i != e; ++i) { + const MatchableInfo::ResOperand &OpInfo = II->ResOperands[i]; + + // Generate code to populate each result operand. + switch (OpInfo.Kind) { + case MatchableInfo::ResOperand::RenderAsmOperand: { + // This comes from something we parsed. + const MatchableInfo::AsmOperand &Op = + II->AsmOperands[OpInfo.AsmOperandNum]; + + // Registers are always converted the same, don't duplicate the + // conversion function based on them. + Signature += "__"; + std::string Class; + Class = Op.Class->isRegisterClass() ? "Reg" : Op.Class->ClassName; + Signature += Class; + Signature += utostr(OpInfo.MINumOperands); + Signature += "_" + itostr(OpInfo.AsmOperandNum); + + // Add the conversion kind, if necessary, and get the associated ID + // the index of its entry in the vector). + std::string Name = "CVT_" + (Op.Class->isRegisterClass() ? "Reg" : + Op.Class->RenderMethod); + if (Op.Class->IsOptional) { + // For optional operands we must also care about DefaultMethod + assert(HasOptionalOperands); + Name += "_" + Op.Class->DefaultMethod; + } + Name = getEnumNameForToken(Name); + + bool IsNewConverter = false; + unsigned ID = getConverterOperandID(Name, OperandConversionKinds, + IsNewConverter); + + // Add the operand entry to the instruction kind conversion row. + ConversionRow.push_back(ID); + ConversionRow.push_back(OpInfo.AsmOperandNum + HasMnemonicFirst); + + if (!IsNewConverter) + break; + + // This is a new operand kind. Add a handler for it to the + // converter driver. + CvtOS << " case " << Name << ":\n"; + if (Op.Class->IsOptional) { + // If optional operand is not present in actual instruction then we + // should call its DefaultMethod before RenderMethod + assert(HasOptionalOperands); + CvtOS << " if (OptionalOperandsMask[*(p + 1) - 1]) {\n" + << " " << Op.Class->DefaultMethod << "()" + << "->" << Op.Class->RenderMethod << "(Inst, " + << OpInfo.MINumOperands << ");\n" + << " } else {\n" + << " static_cast<" << TargetOperandClass + << "&>(*Operands[OpIdx])." << Op.Class->RenderMethod + << "(Inst, " << OpInfo.MINumOperands << ");\n" + << " }\n"; + } else { + CvtOS << " static_cast<" << TargetOperandClass + << "&>(*Operands[OpIdx])." << Op.Class->RenderMethod + << "(Inst, " << OpInfo.MINumOperands << ");\n"; + } + CvtOS << " break;\n"; + + // Add a handler for the operand number lookup. + OpOS << " case " << Name << ":\n" + << " Operands[*(p + 1)]->setMCOperandNum(NumMCOperands);\n"; + + if (Op.Class->isRegisterClass()) + OpOS << " Operands[*(p + 1)]->setConstraint(\"r\");\n"; + else + OpOS << " Operands[*(p + 1)]->setConstraint(\"m\");\n"; + OpOS << " NumMCOperands += " << OpInfo.MINumOperands << ";\n" + << " break;\n"; + break; + } + case MatchableInfo::ResOperand::TiedOperand: { + // If this operand is tied to a previous one, just copy the MCInst + // operand from the earlier one.We can only tie single MCOperand values. + assert(OpInfo.MINumOperands == 1 && "Not a singular MCOperand"); + uint8_t TiedOp = OpInfo.TiedOperands.ResOpnd; + uint8_t SrcOp1 = + OpInfo.TiedOperands.SrcOpnd1Idx + HasMnemonicFirst; + uint8_t SrcOp2 = + OpInfo.TiedOperands.SrcOpnd2Idx + HasMnemonicFirst; + assert((i > TiedOp || TiedOp == (uint8_t)-1) && + "Tied operand precedes its target!"); + auto TiedTupleName = std::string("Tie") + utostr(TiedOp) + '_' + + utostr(SrcOp1) + '_' + utostr(SrcOp2); + Signature += "__" + TiedTupleName; + ConversionRow.push_back(CVT_Tied); + ConversionRow.push_back(TiedOp); + ConversionRow.push_back(SrcOp1); + ConversionRow.push_back(SrcOp2); + + // Also create an 'enum' for this combination of tied operands. + auto Key = std::make_tuple(TiedOp, SrcOp1, SrcOp2); + TiedOperandsEnumMap.emplace(Key, TiedTupleName); + break; + } + case MatchableInfo::ResOperand::ImmOperand: { + int64_t Val = OpInfo.ImmVal; + std::string Ty = "imm_" + itostr(Val); + Ty = getEnumNameForToken(Ty); + Signature += "__" + Ty; + + std::string Name = "CVT_" + Ty; + bool IsNewConverter = false; + unsigned ID = getConverterOperandID(Name, OperandConversionKinds, + IsNewConverter); + // Add the operand entry to the instruction kind conversion row. + ConversionRow.push_back(ID); + ConversionRow.push_back(0); + + if (!IsNewConverter) + break; + + CvtOS << " case " << Name << ":\n" + << " Inst.addOperand(MCOperand::createImm(" << Val << "));\n" + << " break;\n"; + + OpOS << " case " << Name << ":\n" + << " Operands[*(p + 1)]->setMCOperandNum(NumMCOperands);\n" + << " Operands[*(p + 1)]->setConstraint(\"\");\n" + << " ++NumMCOperands;\n" + << " break;\n"; + break; + } + case MatchableInfo::ResOperand::RegOperand: { + std::string Reg, Name; + if (!OpInfo.Register) { + Name = "reg0"; + Reg = "0"; + } else { + Reg = getQualifiedName(OpInfo.Register); + Name = "reg" + OpInfo.Register->getName().str(); + } + Signature += "__" + Name; + Name = "CVT_" + Name; + bool IsNewConverter = false; + unsigned ID = getConverterOperandID(Name, OperandConversionKinds, + IsNewConverter); + // Add the operand entry to the instruction kind conversion row. + ConversionRow.push_back(ID); + ConversionRow.push_back(0); + + if (!IsNewConverter) + break; + CvtOS << " case " << Name << ":\n" + << " Inst.addOperand(MCOperand::createReg(" << Reg << "));\n" + << " break;\n"; + + OpOS << " case " << Name << ":\n" + << " Operands[*(p + 1)]->setMCOperandNum(NumMCOperands);\n" + << " Operands[*(p + 1)]->setConstraint(\"m\");\n" + << " ++NumMCOperands;\n" + << " break;\n"; + } + } + } + + // If there were no operands, add to the signature to that effect + if (Signature == "Convert") + Signature += "_NoOperands"; + + II->ConversionFnKind = Signature; + + // Save the signature. If we already have it, don't add a new row + // to the table. + if (!InstructionConversionKinds.insert(CachedHashString(Signature))) + continue; + + // Add the row to the table. + ConversionTable.push_back(std::move(ConversionRow)); + } + + // Finish up the converter driver function. + CvtOS << " }\n }\n}\n\n"; + + // Finish up the operand number lookup function. + OpOS << " }\n }\n}\n\n"; + + // Output a static table for tied operands. + if (TiedOperandsEnumMap.size()) { + // The number of tied operand combinations will be small in practice, + // but just add the assert to be sure. + assert(TiedOperandsEnumMap.size() <= 254 && + "Too many tied-operand combinations to reference with " + "an 8bit offset from the conversion table, where index " + "'255' is reserved as operand not to be copied."); + + OS << "enum {\n"; + for (auto &KV : TiedOperandsEnumMap) { + OS << " " << KV.second << ",\n"; + } + OS << "};\n\n"; + + OS << "static const uint8_t TiedAsmOperandTable[][3] = {\n"; + for (auto &KV : TiedOperandsEnumMap) { + OS << " /* " << KV.second << " */ { " + << utostr(std::get<0>(KV.first)) << ", " + << utostr(std::get<1>(KV.first)) << ", " + << utostr(std::get<2>(KV.first)) << " },\n"; + } + OS << "};\n\n"; + } else + OS << "static const uint8_t TiedAsmOperandTable[][3] = " + "{ /* empty */ {0, 0, 0} };\n\n"; + + OS << "namespace {\n"; + + // Output the operand conversion kind enum. + OS << "enum OperatorConversionKind {\n"; + for (const auto &Converter : OperandConversionKinds) + OS << " " << Converter << ",\n"; + OS << " CVT_NUM_CONVERTERS\n"; + OS << "};\n\n"; + + // Output the instruction conversion kind enum. + OS << "enum InstructionConversionKind {\n"; + for (const auto &Signature : InstructionConversionKinds) + OS << " " << Signature << ",\n"; + OS << " CVT_NUM_SIGNATURES\n"; + OS << "};\n\n"; + + OS << "} // end anonymous namespace\n\n"; + + // Output the conversion table. + OS << "static const uint8_t ConversionTable[CVT_NUM_SIGNATURES][" + << MaxRowLength << "] = {\n"; + + for (unsigned Row = 0, ERow = ConversionTable.size(); Row != ERow; ++Row) { + assert(ConversionTable[Row].size() % 2 == 0 && "bad conversion row!"); + OS << " // " << InstructionConversionKinds[Row] << "\n"; + OS << " { "; + for (unsigned i = 0, e = ConversionTable[Row].size(); i != e; i += 2) { + OS << OperandConversionKinds[ConversionTable[Row][i]] << ", "; + if (OperandConversionKinds[ConversionTable[Row][i]] != + CachedHashString("CVT_Tied")) { + OS << (unsigned)(ConversionTable[Row][i + 1]) << ", "; + continue; + } + + // For a tied operand, emit a reference to the TiedAsmOperandTable + // that contains the operand to copy, and the parsed operands to + // check for their tied constraints. + auto Key = std::make_tuple((uint8_t)ConversionTable[Row][i + 1], + (uint8_t)ConversionTable[Row][i + 2], + (uint8_t)ConversionTable[Row][i + 3]); + auto TiedOpndEnum = TiedOperandsEnumMap.find(Key); + assert(TiedOpndEnum != TiedOperandsEnumMap.end() && + "No record for tied operand pair"); + OS << TiedOpndEnum->second << ", "; + i += 2; + } + OS << "CVT_Done },\n"; + } + + OS << "};\n\n"; + + // Spit out the conversion driver function. + OS << CvtOS.str(); + + // Spit out the operand number lookup function. + OS << OpOS.str(); + + return ConversionTable.size(); +} + +/// emitMatchClassEnumeration - Emit the enumeration for match class kinds. +static void emitMatchClassEnumeration(CodeGenTarget &Target, + std::forward_list<ClassInfo> &Infos, + raw_ostream &OS) { + OS << "namespace {\n\n"; + + OS << "/// MatchClassKind - The kinds of classes which participate in\n" + << "/// instruction matching.\n"; + OS << "enum MatchClassKind {\n"; + OS << " InvalidMatchClass = 0,\n"; + OS << " OptionalMatchClass = 1,\n"; + ClassInfo::ClassInfoKind LastKind = ClassInfo::Token; + StringRef LastName = "OptionalMatchClass"; + for (const auto &CI : Infos) { + if (LastKind == ClassInfo::Token && CI.Kind != ClassInfo::Token) { + OS << " MCK_LAST_TOKEN = " << LastName << ",\n"; + } else if (LastKind < ClassInfo::UserClass0 && + CI.Kind >= ClassInfo::UserClass0) { + OS << " MCK_LAST_REGISTER = " << LastName << ",\n"; + } + LastKind = (ClassInfo::ClassInfoKind)CI.Kind; + LastName = CI.Name; + + OS << " " << CI.Name << ", // "; + if (CI.Kind == ClassInfo::Token) { + OS << "'" << CI.ValueName << "'\n"; + } else if (CI.isRegisterClass()) { + if (!CI.ValueName.empty()) + OS << "register class '" << CI.ValueName << "'\n"; + else + OS << "derived register class\n"; + } else { + OS << "user defined class '" << CI.ValueName << "'\n"; + } + } + OS << " NumMatchClassKinds\n"; + OS << "};\n\n"; + + OS << "} // end anonymous namespace\n\n"; +} + +/// emitMatchClassDiagStrings - Emit a function to get the diagnostic text to be +/// used when an assembly operand does not match the expected operand class. +static void emitOperandMatchErrorDiagStrings(AsmMatcherInfo &Info, raw_ostream &OS) { + // If the target does not use DiagnosticString for any operands, don't emit + // an unused function. + if (std::all_of( + Info.Classes.begin(), Info.Classes.end(), + [](const ClassInfo &CI) { return CI.DiagnosticString.empty(); })) + return; + + OS << "static const char *getMatchKindDiag(" << Info.Target.getName() + << "AsmParser::" << Info.Target.getName() + << "MatchResultTy MatchResult) {\n"; + OS << " switch (MatchResult) {\n"; + + for (const auto &CI: Info.Classes) { + if (!CI.DiagnosticString.empty()) { + assert(!CI.DiagnosticType.empty() && + "DiagnosticString set without DiagnosticType"); + OS << " case " << Info.Target.getName() + << "AsmParser::Match_" << CI.DiagnosticType << ":\n"; + OS << " return \"" << CI.DiagnosticString << "\";\n"; + } + } + + OS << " default:\n"; + OS << " return nullptr;\n"; + + OS << " }\n"; + OS << "}\n\n"; +} + +static void emitRegisterMatchErrorFunc(AsmMatcherInfo &Info, raw_ostream &OS) { + OS << "static unsigned getDiagKindFromRegisterClass(MatchClassKind " + "RegisterClass) {\n"; + if (none_of(Info.Classes, [](const ClassInfo &CI) { + return CI.isRegisterClass() && !CI.DiagnosticType.empty(); + })) { + OS << " return MCTargetAsmParser::Match_InvalidOperand;\n"; + } else { + OS << " switch (RegisterClass) {\n"; + for (const auto &CI: Info.Classes) { + if (CI.isRegisterClass() && !CI.DiagnosticType.empty()) { + OS << " case " << CI.Name << ":\n"; + OS << " return " << Info.Target.getName() << "AsmParser::Match_" + << CI.DiagnosticType << ";\n"; + } + } + + OS << " default:\n"; + OS << " return MCTargetAsmParser::Match_InvalidOperand;\n"; + + OS << " }\n"; + } + OS << "}\n\n"; +} + +/// emitValidateOperandClass - Emit the function to validate an operand class. +static void emitValidateOperandClass(AsmMatcherInfo &Info, + raw_ostream &OS) { + OS << "static unsigned validateOperandClass(MCParsedAsmOperand &GOp, " + << "MatchClassKind Kind) {\n"; + OS << " " << Info.Target.getName() << "Operand &Operand = (" + << Info.Target.getName() << "Operand&)GOp;\n"; + + // The InvalidMatchClass is not to match any operand. + OS << " if (Kind == InvalidMatchClass)\n"; + OS << " return MCTargetAsmParser::Match_InvalidOperand;\n\n"; + + // Check for Token operands first. + // FIXME: Use a more specific diagnostic type. + OS << " if (Operand.isToken() && Kind <= MCK_LAST_TOKEN)\n"; + OS << " return isSubclass(matchTokenString(Operand.getToken()), Kind) ?\n" + << " MCTargetAsmParser::Match_Success :\n" + << " MCTargetAsmParser::Match_InvalidOperand;\n\n"; + + // Check the user classes. We don't care what order since we're only + // actually matching against one of them. + OS << " switch (Kind) {\n" + " default: break;\n"; + for (const auto &CI : Info.Classes) { + if (!CI.isUserClass()) + continue; + + OS << " // '" << CI.ClassName << "' class\n"; + OS << " case " << CI.Name << ": {\n"; + OS << " DiagnosticPredicate DP(Operand." << CI.PredicateMethod + << "());\n"; + OS << " if (DP.isMatch())\n"; + OS << " return MCTargetAsmParser::Match_Success;\n"; + if (!CI.DiagnosticType.empty()) { + OS << " if (DP.isNearMatch())\n"; + OS << " return " << Info.Target.getName() << "AsmParser::Match_" + << CI.DiagnosticType << ";\n"; + OS << " break;\n"; + } + else + OS << " break;\n"; + OS << " }\n"; + } + OS << " } // end switch (Kind)\n\n"; + + // Check for register operands, including sub-classes. + OS << " if (Operand.isReg()) {\n"; + OS << " MatchClassKind OpKind;\n"; + OS << " switch (Operand.getReg()) {\n"; + OS << " default: OpKind = InvalidMatchClass; break;\n"; + for (const auto &RC : Info.RegisterClasses) + OS << " case " << RC.first->getValueAsString("Namespace") << "::" + << RC.first->getName() << ": OpKind = " << RC.second->Name + << "; break;\n"; + OS << " }\n"; + OS << " return isSubclass(OpKind, Kind) ? " + << "(unsigned)MCTargetAsmParser::Match_Success :\n " + << " getDiagKindFromRegisterClass(Kind);\n }\n\n"; + + // Expected operand is a register, but actual is not. + OS << " if (Kind > MCK_LAST_TOKEN && Kind <= MCK_LAST_REGISTER)\n"; + OS << " return getDiagKindFromRegisterClass(Kind);\n\n"; + + // Generic fallthrough match failure case for operands that don't have + // specialized diagnostic types. + OS << " return MCTargetAsmParser::Match_InvalidOperand;\n"; + OS << "}\n\n"; +} + +/// emitIsSubclass - Emit the subclass predicate function. +static void emitIsSubclass(CodeGenTarget &Target, + std::forward_list<ClassInfo> &Infos, + raw_ostream &OS) { + OS << "/// isSubclass - Compute whether \\p A is a subclass of \\p B.\n"; + OS << "static bool isSubclass(MatchClassKind A, MatchClassKind B) {\n"; + OS << " if (A == B)\n"; + OS << " return true;\n\n"; + + bool EmittedSwitch = false; + for (const auto &A : Infos) { + std::vector<StringRef> SuperClasses; + if (A.IsOptional) + SuperClasses.push_back("OptionalMatchClass"); + for (const auto &B : Infos) { + if (&A != &B && A.isSubsetOf(B)) + SuperClasses.push_back(B.Name); + } + + if (SuperClasses.empty()) + continue; + + // If this is the first SuperClass, emit the switch header. + if (!EmittedSwitch) { + OS << " switch (A) {\n"; + OS << " default:\n"; + OS << " return false;\n"; + EmittedSwitch = true; + } + + OS << "\n case " << A.Name << ":\n"; + + if (SuperClasses.size() == 1) { + OS << " return B == " << SuperClasses.back() << ";\n"; + continue; + } + + if (!SuperClasses.empty()) { + OS << " switch (B) {\n"; + OS << " default: return false;\n"; + for (StringRef SC : SuperClasses) + OS << " case " << SC << ": return true;\n"; + OS << " }\n"; + } else { + // No case statement to emit + OS << " return false;\n"; + } + } + + // If there were case statements emitted into the string stream write the + // default. + if (EmittedSwitch) + OS << " }\n"; + else + OS << " return false;\n"; + + OS << "}\n\n"; +} + +/// emitMatchTokenString - Emit the function to match a token string to the +/// appropriate match class value. +static void emitMatchTokenString(CodeGenTarget &Target, + std::forward_list<ClassInfo> &Infos, + raw_ostream &OS) { + // Construct the match list. + std::vector<StringMatcher::StringPair> Matches; + for (const auto &CI : Infos) { + if (CI.Kind == ClassInfo::Token) + Matches.emplace_back(CI.ValueName, "return " + CI.Name + ";"); + } + + OS << "static MatchClassKind matchTokenString(StringRef Name) {\n"; + + StringMatcher("Name", Matches, OS).Emit(); + + OS << " return InvalidMatchClass;\n"; + OS << "}\n\n"; +} + +/// emitMatchRegisterName - Emit the function to match a string to the target +/// specific register enum. +static void emitMatchRegisterName(CodeGenTarget &Target, Record *AsmParser, + raw_ostream &OS) { + // Construct the match list. + std::vector<StringMatcher::StringPair> Matches; + const auto &Regs = Target.getRegBank().getRegisters(); + for (const CodeGenRegister &Reg : Regs) { + if (Reg.TheDef->getValueAsString("AsmName").empty()) + continue; + + Matches.emplace_back(std::string(Reg.TheDef->getValueAsString("AsmName")), + "return " + utostr(Reg.EnumValue) + ";"); + } + + OS << "static unsigned MatchRegisterName(StringRef Name) {\n"; + + bool IgnoreDuplicates = + AsmParser->getValueAsBit("AllowDuplicateRegisterNames"); + StringMatcher("Name", Matches, OS).Emit(0, IgnoreDuplicates); + + OS << " return 0;\n"; + OS << "}\n\n"; +} + +/// Emit the function to match a string to the target +/// specific register enum. +static void emitMatchRegisterAltName(CodeGenTarget &Target, Record *AsmParser, + raw_ostream &OS) { + // Construct the match list. + std::vector<StringMatcher::StringPair> Matches; + const auto &Regs = Target.getRegBank().getRegisters(); + for (const CodeGenRegister &Reg : Regs) { + + auto AltNames = Reg.TheDef->getValueAsListOfStrings("AltNames"); + + for (auto AltName : AltNames) { + AltName = StringRef(AltName).trim(); + + // don't handle empty alternative names + if (AltName.empty()) + continue; + + Matches.emplace_back(std::string(AltName), + "return " + utostr(Reg.EnumValue) + ";"); + } + } + + OS << "static unsigned MatchRegisterAltName(StringRef Name) {\n"; + + bool IgnoreDuplicates = + AsmParser->getValueAsBit("AllowDuplicateRegisterNames"); + StringMatcher("Name", Matches, OS).Emit(0, IgnoreDuplicates); + + OS << " return 0;\n"; + OS << "}\n\n"; +} + +/// emitOperandDiagnosticTypes - Emit the operand matching diagnostic types. +static void emitOperandDiagnosticTypes(AsmMatcherInfo &Info, raw_ostream &OS) { + // Get the set of diagnostic types from all of the operand classes. + std::set<StringRef> Types; + for (const auto &OpClassEntry : Info.AsmOperandClasses) { + if (!OpClassEntry.second->DiagnosticType.empty()) + Types.insert(OpClassEntry.second->DiagnosticType); + } + for (const auto &OpClassEntry : Info.RegisterClassClasses) { + if (!OpClassEntry.second->DiagnosticType.empty()) + Types.insert(OpClassEntry.second->DiagnosticType); + } + + if (Types.empty()) return; + + // Now emit the enum entries. + for (StringRef Type : Types) + OS << " Match_" << Type << ",\n"; + OS << " END_OPERAND_DIAGNOSTIC_TYPES\n"; +} + +/// emitGetSubtargetFeatureName - Emit the helper function to get the +/// user-level name for a subtarget feature. +static void emitGetSubtargetFeatureName(AsmMatcherInfo &Info, raw_ostream &OS) { + OS << "// User-level names for subtarget features that participate in\n" + << "// instruction matching.\n" + << "static const char *getSubtargetFeatureName(uint64_t Val) {\n"; + if (!Info.SubtargetFeatures.empty()) { + OS << " switch(Val) {\n"; + for (const auto &SF : Info.SubtargetFeatures) { + const SubtargetFeatureInfo &SFI = SF.second; + // FIXME: Totally just a placeholder name to get the algorithm working. + OS << " case " << SFI.getEnumBitName() << ": return \"" + << SFI.TheDef->getValueAsString("PredicateName") << "\";\n"; + } + OS << " default: return \"(unknown)\";\n"; + OS << " }\n"; + } else { + // Nothing to emit, so skip the switch + OS << " return \"(unknown)\";\n"; + } + OS << "}\n\n"; +} + +static std::string GetAliasRequiredFeatures(Record *R, + const AsmMatcherInfo &Info) { + std::vector<Record*> ReqFeatures = R->getValueAsListOfDefs("Predicates"); + std::string Result; + + if (ReqFeatures.empty()) + return Result; + + for (unsigned i = 0, e = ReqFeatures.size(); i != e; ++i) { + const SubtargetFeatureInfo *F = Info.getSubtargetFeature(ReqFeatures[i]); + + if (!F) + PrintFatalError(R->getLoc(), "Predicate '" + ReqFeatures[i]->getName() + + "' is not marked as an AssemblerPredicate!"); + + if (i) + Result += " && "; + + Result += "Features.test(" + F->getEnumBitName() + ')'; + } + + return Result; +} + +static void emitMnemonicAliasVariant(raw_ostream &OS,const AsmMatcherInfo &Info, + std::vector<Record*> &Aliases, + unsigned Indent = 0, + StringRef AsmParserVariantName = StringRef()){ + // Keep track of all the aliases from a mnemonic. Use an std::map so that the + // iteration order of the map is stable. + std::map<std::string, std::vector<Record*> > AliasesFromMnemonic; + + for (Record *R : Aliases) { + // FIXME: Allow AssemblerVariantName to be a comma separated list. + StringRef AsmVariantName = R->getValueAsString("AsmVariantName"); + if (AsmVariantName != AsmParserVariantName) + continue; + AliasesFromMnemonic[std::string(R->getValueAsString("FromMnemonic"))] + .push_back(R); + } + if (AliasesFromMnemonic.empty()) + return; + + // Process each alias a "from" mnemonic at a time, building the code executed + // by the string remapper. + std::vector<StringMatcher::StringPair> Cases; + for (const auto &AliasEntry : AliasesFromMnemonic) { + const std::vector<Record*> &ToVec = AliasEntry.second; + + // Loop through each alias and emit code that handles each case. If there + // are two instructions without predicates, emit an error. If there is one, + // emit it last. + std::string MatchCode; + int AliasWithNoPredicate = -1; + + for (unsigned i = 0, e = ToVec.size(); i != e; ++i) { + Record *R = ToVec[i]; + std::string FeatureMask = GetAliasRequiredFeatures(R, Info); + + // If this unconditionally matches, remember it for later and diagnose + // duplicates. + if (FeatureMask.empty()) { + if (AliasWithNoPredicate != -1) { + // We can't have two aliases from the same mnemonic with no predicate. + PrintError(ToVec[AliasWithNoPredicate]->getLoc(), + "two MnemonicAliases with the same 'from' mnemonic!"); + PrintFatalError(R->getLoc(), "this is the other MnemonicAlias."); + } + + AliasWithNoPredicate = i; + continue; + } + if (R->getValueAsString("ToMnemonic") == AliasEntry.first) + PrintFatalError(R->getLoc(), "MnemonicAlias to the same string"); + + if (!MatchCode.empty()) + MatchCode += "else "; + MatchCode += "if (" + FeatureMask + ")\n"; + MatchCode += " Mnemonic = \""; + MatchCode += R->getValueAsString("ToMnemonic"); + MatchCode += "\";\n"; + } + + if (AliasWithNoPredicate != -1) { + Record *R = ToVec[AliasWithNoPredicate]; + if (!MatchCode.empty()) + MatchCode += "else\n "; + MatchCode += "Mnemonic = \""; + MatchCode += R->getValueAsString("ToMnemonic"); + MatchCode += "\";\n"; + } + + MatchCode += "return;"; + + Cases.push_back(std::make_pair(AliasEntry.first, MatchCode)); + } + StringMatcher("Mnemonic", Cases, OS).Emit(Indent); +} + +/// emitMnemonicAliases - If the target has any MnemonicAlias<> definitions, +/// emit a function for them and return true, otherwise return false. +static bool emitMnemonicAliases(raw_ostream &OS, const AsmMatcherInfo &Info, + CodeGenTarget &Target) { + // Ignore aliases when match-prefix is set. + if (!MatchPrefix.empty()) + return false; + + std::vector<Record*> Aliases = + Info.getRecords().getAllDerivedDefinitions("MnemonicAlias"); + if (Aliases.empty()) return false; + + OS << "static void applyMnemonicAliases(StringRef &Mnemonic, " + "const FeatureBitset &Features, unsigned VariantID) {\n"; + OS << " switch (VariantID) {\n"; + unsigned VariantCount = Target.getAsmParserVariantCount(); + for (unsigned VC = 0; VC != VariantCount; ++VC) { + Record *AsmVariant = Target.getAsmParserVariant(VC); + int AsmParserVariantNo = AsmVariant->getValueAsInt("Variant"); + StringRef AsmParserVariantName = AsmVariant->getValueAsString("Name"); + OS << " case " << AsmParserVariantNo << ":\n"; + emitMnemonicAliasVariant(OS, Info, Aliases, /*Indent=*/2, + AsmParserVariantName); + OS << " break;\n"; + } + OS << " }\n"; + + // Emit aliases that apply to all variants. + emitMnemonicAliasVariant(OS, Info, Aliases); + + OS << "}\n\n"; + + return true; +} + +static void emitCustomOperandParsing(raw_ostream &OS, CodeGenTarget &Target, + const AsmMatcherInfo &Info, StringRef ClassName, + StringToOffsetTable &StringTable, + unsigned MaxMnemonicIndex, + unsigned MaxFeaturesIndex, + bool HasMnemonicFirst) { + unsigned MaxMask = 0; + for (const OperandMatchEntry &OMI : Info.OperandMatchInfo) { + MaxMask |= OMI.OperandMask; + } + + // Emit the static custom operand parsing table; + OS << "namespace {\n"; + OS << " struct OperandMatchEntry {\n"; + OS << " " << getMinimalTypeForRange(MaxMnemonicIndex) + << " Mnemonic;\n"; + OS << " " << getMinimalTypeForRange(MaxMask) + << " OperandMask;\n"; + OS << " " << getMinimalTypeForRange(std::distance( + Info.Classes.begin(), Info.Classes.end())) << " Class;\n"; + OS << " " << getMinimalTypeForRange(MaxFeaturesIndex) + << " RequiredFeaturesIdx;\n\n"; + OS << " StringRef getMnemonic() const {\n"; + OS << " return StringRef(MnemonicTable + Mnemonic + 1,\n"; + OS << " MnemonicTable[Mnemonic]);\n"; + OS << " }\n"; + OS << " };\n\n"; + + OS << " // Predicate for searching for an opcode.\n"; + OS << " struct LessOpcodeOperand {\n"; + OS << " bool operator()(const OperandMatchEntry &LHS, StringRef RHS) {\n"; + OS << " return LHS.getMnemonic() < RHS;\n"; + OS << " }\n"; + OS << " bool operator()(StringRef LHS, const OperandMatchEntry &RHS) {\n"; + OS << " return LHS < RHS.getMnemonic();\n"; + OS << " }\n"; + OS << " bool operator()(const OperandMatchEntry &LHS,"; + OS << " const OperandMatchEntry &RHS) {\n"; + OS << " return LHS.getMnemonic() < RHS.getMnemonic();\n"; + OS << " }\n"; + OS << " };\n"; + + OS << "} // end anonymous namespace\n\n"; + + OS << "static const OperandMatchEntry OperandMatchTable[" + << Info.OperandMatchInfo.size() << "] = {\n"; + + OS << " /* Operand List Mnemonic, Mask, Operand Class, Features */\n"; + for (const OperandMatchEntry &OMI : Info.OperandMatchInfo) { + const MatchableInfo &II = *OMI.MI; + + OS << " { "; + + // Store a pascal-style length byte in the mnemonic. + std::string LenMnemonic = char(II.Mnemonic.size()) + II.Mnemonic.str(); + OS << StringTable.GetOrAddStringOffset(LenMnemonic, false) + << " /* " << II.Mnemonic << " */, "; + + OS << OMI.OperandMask; + OS << " /* "; + bool printComma = false; + for (int i = 0, e = 31; i !=e; ++i) + if (OMI.OperandMask & (1 << i)) { + if (printComma) + OS << ", "; + OS << i; + printComma = true; + } + OS << " */, "; + + OS << OMI.CI->Name; + + // Write the required features mask. + OS << ", AMFBS"; + if (II.RequiredFeatures.empty()) + OS << "_None"; + else + for (unsigned i = 0, e = II.RequiredFeatures.size(); i != e; ++i) + OS << '_' << II.RequiredFeatures[i]->TheDef->getName(); + + OS << " },\n"; + } + OS << "};\n\n"; + + // Emit the operand class switch to call the correct custom parser for + // the found operand class. + OS << "OperandMatchResultTy " << Target.getName() << ClassName << "::\n" + << "tryCustomParseOperand(OperandVector" + << " &Operands,\n unsigned MCK) {\n\n" + << " switch(MCK) {\n"; + + for (const auto &CI : Info.Classes) { + if (CI.ParserMethod.empty()) + continue; + OS << " case " << CI.Name << ":\n" + << " return " << CI.ParserMethod << "(Operands);\n"; + } + + OS << " default:\n"; + OS << " return MatchOperand_NoMatch;\n"; + OS << " }\n"; + OS << " return MatchOperand_NoMatch;\n"; + OS << "}\n\n"; + + // Emit the static custom operand parser. This code is very similar with + // the other matcher. Also use MatchResultTy here just in case we go for + // a better error handling. + OS << "OperandMatchResultTy " << Target.getName() << ClassName << "::\n" + << "MatchOperandParserImpl(OperandVector" + << " &Operands,\n StringRef Mnemonic,\n" + << " bool ParseForAllFeatures) {\n"; + + // Emit code to get the available features. + OS << " // Get the current feature set.\n"; + OS << " const FeatureBitset &AvailableFeatures = getAvailableFeatures();\n\n"; + + OS << " // Get the next operand index.\n"; + OS << " unsigned NextOpNum = Operands.size()" + << (HasMnemonicFirst ? " - 1" : "") << ";\n"; + + // Emit code to search the table. + OS << " // Search the table.\n"; + if (HasMnemonicFirst) { + OS << " auto MnemonicRange =\n"; + OS << " std::equal_range(std::begin(OperandMatchTable), " + "std::end(OperandMatchTable),\n"; + OS << " Mnemonic, LessOpcodeOperand());\n\n"; + } else { + OS << " auto MnemonicRange = std::make_pair(std::begin(OperandMatchTable)," + " std::end(OperandMatchTable));\n"; + OS << " if (!Mnemonic.empty())\n"; + OS << " MnemonicRange =\n"; + OS << " std::equal_range(std::begin(OperandMatchTable), " + "std::end(OperandMatchTable),\n"; + OS << " Mnemonic, LessOpcodeOperand());\n\n"; + } + + OS << " if (MnemonicRange.first == MnemonicRange.second)\n"; + OS << " return MatchOperand_NoMatch;\n\n"; + + OS << " for (const OperandMatchEntry *it = MnemonicRange.first,\n" + << " *ie = MnemonicRange.second; it != ie; ++it) {\n"; + + OS << " // equal_range guarantees that instruction mnemonic matches.\n"; + OS << " assert(Mnemonic == it->getMnemonic());\n\n"; + + // Emit check that the required features are available. + OS << " // check if the available features match\n"; + OS << " const FeatureBitset &RequiredFeatures = " + "FeatureBitsets[it->RequiredFeaturesIdx];\n"; + OS << " if (!ParseForAllFeatures && (AvailableFeatures & " + "RequiredFeatures) != RequiredFeatures)\n"; + OS << " continue;\n\n"; + + // Emit check to ensure the operand number matches. + OS << " // check if the operand in question has a custom parser.\n"; + OS << " if (!(it->OperandMask & (1 << NextOpNum)))\n"; + OS << " continue;\n\n"; + + // Emit call to the custom parser method + OS << " // call custom parse method to handle the operand\n"; + OS << " OperandMatchResultTy Result = "; + OS << "tryCustomParseOperand(Operands, it->Class);\n"; + OS << " if (Result != MatchOperand_NoMatch)\n"; + OS << " return Result;\n"; + OS << " }\n\n"; + + OS << " // Okay, we had no match.\n"; + OS << " return MatchOperand_NoMatch;\n"; + OS << "}\n\n"; +} + +static void emitAsmTiedOperandConstraints(CodeGenTarget &Target, + AsmMatcherInfo &Info, + raw_ostream &OS) { + std::string AsmParserName = + std::string(Info.AsmParser->getValueAsString("AsmParserClassName")); + OS << "static bool "; + OS << "checkAsmTiedOperandConstraints(const " << Target.getName() + << AsmParserName << "&AsmParser,\n"; + OS << " unsigned Kind,\n"; + OS << " const OperandVector &Operands,\n"; + OS << " uint64_t &ErrorInfo) {\n"; + OS << " assert(Kind < CVT_NUM_SIGNATURES && \"Invalid signature!\");\n"; + OS << " const uint8_t *Converter = ConversionTable[Kind];\n"; + OS << " for (const uint8_t *p = Converter; *p; p+= 2) {\n"; + OS << " switch (*p) {\n"; + OS << " case CVT_Tied: {\n"; + OS << " unsigned OpIdx = *(p+1);\n"; + OS << " assert(OpIdx < (size_t)(std::end(TiedAsmOperandTable) -\n"; + OS << " std::begin(TiedAsmOperandTable)) &&\n"; + OS << " \"Tied operand not found\");\n"; + OS << " unsigned OpndNum1 = TiedAsmOperandTable[OpIdx][1];\n"; + OS << " unsigned OpndNum2 = TiedAsmOperandTable[OpIdx][2];\n"; + OS << " if (OpndNum1 != OpndNum2) {\n"; + OS << " auto &SrcOp1 = Operands[OpndNum1];\n"; + OS << " auto &SrcOp2 = Operands[OpndNum2];\n"; + OS << " if (SrcOp1->isReg() && SrcOp2->isReg()) {\n"; + OS << " if (!AsmParser.regsEqual(*SrcOp1, *SrcOp2)) {\n"; + OS << " ErrorInfo = OpndNum2;\n"; + OS << " return false;\n"; + OS << " }\n"; + OS << " }\n"; + OS << " }\n"; + OS << " break;\n"; + OS << " }\n"; + OS << " default:\n"; + OS << " break;\n"; + OS << " }\n"; + OS << " }\n"; + OS << " return true;\n"; + OS << "}\n\n"; +} + +static void emitMnemonicSpellChecker(raw_ostream &OS, CodeGenTarget &Target, + unsigned VariantCount) { + OS << "static std::string " << Target.getName() + << "MnemonicSpellCheck(StringRef S, const FeatureBitset &FBS," + << " unsigned VariantID) {\n"; + if (!VariantCount) + OS << " return \"\";"; + else { + OS << " const unsigned MaxEditDist = 2;\n"; + OS << " std::vector<StringRef> Candidates;\n"; + OS << " StringRef Prev = \"\";\n\n"; + + OS << " // Find the appropriate table for this asm variant.\n"; + OS << " const MatchEntry *Start, *End;\n"; + OS << " switch (VariantID) {\n"; + OS << " default: llvm_unreachable(\"invalid variant!\");\n"; + for (unsigned VC = 0; VC != VariantCount; ++VC) { + Record *AsmVariant = Target.getAsmParserVariant(VC); + int AsmVariantNo = AsmVariant->getValueAsInt("Variant"); + OS << " case " << AsmVariantNo << ": Start = std::begin(MatchTable" << VC + << "); End = std::end(MatchTable" << VC << "); break;\n"; + } + OS << " }\n\n"; + OS << " for (auto I = Start; I < End; I++) {\n"; + OS << " // Ignore unsupported instructions.\n"; + OS << " const FeatureBitset &RequiredFeatures = " + "FeatureBitsets[I->RequiredFeaturesIdx];\n"; + OS << " if ((FBS & RequiredFeatures) != RequiredFeatures)\n"; + OS << " continue;\n"; + OS << "\n"; + OS << " StringRef T = I->getMnemonic();\n"; + OS << " // Avoid recomputing the edit distance for the same string.\n"; + OS << " if (T.equals(Prev))\n"; + OS << " continue;\n"; + OS << "\n"; + OS << " Prev = T;\n"; + OS << " unsigned Dist = S.edit_distance(T, false, MaxEditDist);\n"; + OS << " if (Dist <= MaxEditDist)\n"; + OS << " Candidates.push_back(T);\n"; + OS << " }\n"; + OS << "\n"; + OS << " if (Candidates.empty())\n"; + OS << " return \"\";\n"; + OS << "\n"; + OS << " std::string Res = \", did you mean: \";\n"; + OS << " unsigned i = 0;\n"; + OS << " for( ; i < Candidates.size() - 1; i++)\n"; + OS << " Res += Candidates[i].str() + \", \";\n"; + OS << " return Res + Candidates[i].str() + \"?\";\n"; + } + OS << "}\n"; + OS << "\n"; +} + + +// Emit a function mapping match classes to strings, for debugging. +static void emitMatchClassKindNames(std::forward_list<ClassInfo> &Infos, + raw_ostream &OS) { + OS << "#ifndef NDEBUG\n"; + OS << "const char *getMatchClassName(MatchClassKind Kind) {\n"; + OS << " switch (Kind) {\n"; + + OS << " case InvalidMatchClass: return \"InvalidMatchClass\";\n"; + OS << " case OptionalMatchClass: return \"OptionalMatchClass\";\n"; + for (const auto &CI : Infos) { + OS << " case " << CI.Name << ": return \"" << CI.Name << "\";\n"; + } + OS << " case NumMatchClassKinds: return \"NumMatchClassKinds\";\n"; + + OS << " }\n"; + OS << " llvm_unreachable(\"unhandled MatchClassKind!\");\n"; + OS << "}\n\n"; + OS << "#endif // NDEBUG\n"; +} + +static std::string +getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset) { + std::string Name = "AMFBS"; + for (const auto &Feature : FeatureBitset) + Name += ("_" + Feature->getName()).str(); + return Name; +} + +void AsmMatcherEmitter::run(raw_ostream &OS) { + CodeGenTarget Target(Records); + Record *AsmParser = Target.getAsmParser(); + StringRef ClassName = AsmParser->getValueAsString("AsmParserClassName"); + + // Compute the information on the instructions to match. + AsmMatcherInfo Info(AsmParser, Target, Records); + Info.buildInfo(); + + // Sort the instruction table using the partial order on classes. We use + // stable_sort to ensure that ambiguous instructions are still + // deterministically ordered. + llvm::stable_sort( + Info.Matchables, + [](const std::unique_ptr<MatchableInfo> &a, + const std::unique_ptr<MatchableInfo> &b) { return *a < *b; }); + +#ifdef EXPENSIVE_CHECKS + // Verify that the table is sorted and operator < works transitively. + for (auto I = Info.Matchables.begin(), E = Info.Matchables.end(); I != E; + ++I) { + for (auto J = I; J != E; ++J) { + assert(!(**J < **I)); + } + } +#endif + + DEBUG_WITH_TYPE("instruction_info", { + for (const auto &MI : Info.Matchables) + MI->dump(); + }); + + // Check for ambiguous matchables. + DEBUG_WITH_TYPE("ambiguous_instrs", { + unsigned NumAmbiguous = 0; + for (auto I = Info.Matchables.begin(), E = Info.Matchables.end(); I != E; + ++I) { + for (auto J = std::next(I); J != E; ++J) { + const MatchableInfo &A = **I; + const MatchableInfo &B = **J; + + if (A.couldMatchAmbiguouslyWith(B)) { + errs() << "warning: ambiguous matchables:\n"; + A.dump(); + errs() << "\nis incomparable with:\n"; + B.dump(); + errs() << "\n\n"; + ++NumAmbiguous; + } + } + } + if (NumAmbiguous) + errs() << "warning: " << NumAmbiguous + << " ambiguous matchables!\n"; + }); + + // Compute the information on the custom operand parsing. + Info.buildOperandMatchInfo(); + + bool HasMnemonicFirst = AsmParser->getValueAsBit("HasMnemonicFirst"); + bool HasOptionalOperands = Info.hasOptionalOperands(); + bool ReportMultipleNearMisses = + AsmParser->getValueAsBit("ReportMultipleNearMisses"); + + // Write the output. + + // Information for the class declaration. + OS << "\n#ifdef GET_ASSEMBLER_HEADER\n"; + OS << "#undef GET_ASSEMBLER_HEADER\n"; + OS << " // This should be included into the middle of the declaration of\n"; + OS << " // your subclasses implementation of MCTargetAsmParser.\n"; + OS << " FeatureBitset ComputeAvailableFeatures(const FeatureBitset& FB) const;\n"; + if (HasOptionalOperands) { + OS << " void convertToMCInst(unsigned Kind, MCInst &Inst, " + << "unsigned Opcode,\n" + << " const OperandVector &Operands,\n" + << " const SmallBitVector &OptionalOperandsMask);\n"; + } else { + OS << " void convertToMCInst(unsigned Kind, MCInst &Inst, " + << "unsigned Opcode,\n" + << " const OperandVector &Operands);\n"; + } + OS << " void convertToMapAndConstraints(unsigned Kind,\n "; + OS << " const OperandVector &Operands) override;\n"; + OS << " unsigned MatchInstructionImpl(const OperandVector &Operands,\n" + << " MCInst &Inst,\n"; + if (ReportMultipleNearMisses) + OS << " SmallVectorImpl<NearMissInfo> *NearMisses,\n"; + else + OS << " uint64_t &ErrorInfo,\n" + << " FeatureBitset &MissingFeatures,\n"; + OS << " bool matchingInlineAsm,\n" + << " unsigned VariantID = 0);\n"; + if (!ReportMultipleNearMisses) + OS << " unsigned MatchInstructionImpl(const OperandVector &Operands,\n" + << " MCInst &Inst,\n" + << " uint64_t &ErrorInfo,\n" + << " bool matchingInlineAsm,\n" + << " unsigned VariantID = 0) {\n" + << " FeatureBitset MissingFeatures;\n" + << " return MatchInstructionImpl(Operands, Inst, ErrorInfo, MissingFeatures,\n" + << " matchingInlineAsm, VariantID);\n" + << " }\n\n"; + + + if (!Info.OperandMatchInfo.empty()) { + OS << " OperandMatchResultTy MatchOperandParserImpl(\n"; + OS << " OperandVector &Operands,\n"; + OS << " StringRef Mnemonic,\n"; + OS << " bool ParseForAllFeatures = false);\n"; + + OS << " OperandMatchResultTy tryCustomParseOperand(\n"; + OS << " OperandVector &Operands,\n"; + OS << " unsigned MCK);\n\n"; + } + + OS << "#endif // GET_ASSEMBLER_HEADER_INFO\n\n"; + + // Emit the operand match diagnostic enum names. + OS << "\n#ifdef GET_OPERAND_DIAGNOSTIC_TYPES\n"; + OS << "#undef GET_OPERAND_DIAGNOSTIC_TYPES\n\n"; + emitOperandDiagnosticTypes(Info, OS); + OS << "#endif // GET_OPERAND_DIAGNOSTIC_TYPES\n\n"; + + OS << "\n#ifdef GET_REGISTER_MATCHER\n"; + OS << "#undef GET_REGISTER_MATCHER\n\n"; + + // Emit the subtarget feature enumeration. + SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration( + Info.SubtargetFeatures, OS); + + // Emit the function to match a register name to number. + // This should be omitted for Mips target + if (AsmParser->getValueAsBit("ShouldEmitMatchRegisterName")) + emitMatchRegisterName(Target, AsmParser, OS); + + if (AsmParser->getValueAsBit("ShouldEmitMatchRegisterAltName")) + emitMatchRegisterAltName(Target, AsmParser, OS); + + OS << "#endif // GET_REGISTER_MATCHER\n\n"; + + OS << "\n#ifdef GET_SUBTARGET_FEATURE_NAME\n"; + OS << "#undef GET_SUBTARGET_FEATURE_NAME\n\n"; + + // Generate the helper function to get the names for subtarget features. + emitGetSubtargetFeatureName(Info, OS); + + OS << "#endif // GET_SUBTARGET_FEATURE_NAME\n\n"; + + OS << "\n#ifdef GET_MATCHER_IMPLEMENTATION\n"; + OS << "#undef GET_MATCHER_IMPLEMENTATION\n\n"; + + // Generate the function that remaps for mnemonic aliases. + bool HasMnemonicAliases = emitMnemonicAliases(OS, Info, Target); + + // Generate the convertToMCInst function to convert operands into an MCInst. + // Also, generate the convertToMapAndConstraints function for MS-style inline + // assembly. The latter doesn't actually generate a MCInst. + unsigned NumConverters = emitConvertFuncs(Target, ClassName, Info.Matchables, + HasMnemonicFirst, + HasOptionalOperands, OS); + + // Emit the enumeration for classes which participate in matching. + emitMatchClassEnumeration(Target, Info.Classes, OS); + + // Emit a function to get the user-visible string to describe an operand + // match failure in diagnostics. + emitOperandMatchErrorDiagStrings(Info, OS); + + // Emit a function to map register classes to operand match failure codes. + emitRegisterMatchErrorFunc(Info, OS); + + // Emit the routine to match token strings to their match class. + emitMatchTokenString(Target, Info.Classes, OS); + + // Emit the subclass predicate routine. + emitIsSubclass(Target, Info.Classes, OS); + + // Emit the routine to validate an operand against a match class. + emitValidateOperandClass(Info, OS); + + emitMatchClassKindNames(Info.Classes, OS); + + // Emit the available features compute function. + SubtargetFeatureInfo::emitComputeAssemblerAvailableFeatures( + Info.Target.getName(), ClassName, "ComputeAvailableFeatures", + Info.SubtargetFeatures, OS); + + if (!ReportMultipleNearMisses) + emitAsmTiedOperandConstraints(Target, Info, OS); + + StringToOffsetTable StringTable; + + size_t MaxNumOperands = 0; + unsigned MaxMnemonicIndex = 0; + bool HasDeprecation = false; + for (const auto &MI : Info.Matchables) { + MaxNumOperands = std::max(MaxNumOperands, MI->AsmOperands.size()); + HasDeprecation |= MI->HasDeprecation; + + // Store a pascal-style length byte in the mnemonic. + std::string LenMnemonic = char(MI->Mnemonic.size()) + MI->Mnemonic.str(); + MaxMnemonicIndex = std::max(MaxMnemonicIndex, + StringTable.GetOrAddStringOffset(LenMnemonic, false)); + } + + OS << "static const char *const MnemonicTable =\n"; + StringTable.EmitString(OS); + OS << ";\n\n"; + + std::vector<std::vector<Record *>> FeatureBitsets; + for (const auto &MI : Info.Matchables) { + if (MI->RequiredFeatures.empty()) + continue; + FeatureBitsets.emplace_back(); + for (unsigned I = 0, E = MI->RequiredFeatures.size(); I != E; ++I) + FeatureBitsets.back().push_back(MI->RequiredFeatures[I]->TheDef); + } + + llvm::sort(FeatureBitsets, [&](const std::vector<Record *> &A, + const std::vector<Record *> &B) { + if (A.size() < B.size()) + return true; + if (A.size() > B.size()) + return false; + for (auto Pair : zip(A, B)) { + if (std::get<0>(Pair)->getName() < std::get<1>(Pair)->getName()) + return true; + if (std::get<0>(Pair)->getName() > std::get<1>(Pair)->getName()) + return false; + } + return false; + }); + FeatureBitsets.erase( + std::unique(FeatureBitsets.begin(), FeatureBitsets.end()), + FeatureBitsets.end()); + OS << "// Feature bitsets.\n" + << "enum : " << getMinimalTypeForRange(FeatureBitsets.size()) << " {\n" + << " AMFBS_None,\n"; + for (const auto &FeatureBitset : FeatureBitsets) { + if (FeatureBitset.empty()) + continue; + OS << " " << getNameForFeatureBitset(FeatureBitset) << ",\n"; + } + OS << "};\n\n" + << "static constexpr FeatureBitset FeatureBitsets[] = {\n" + << " {}, // AMFBS_None\n"; + for (const auto &FeatureBitset : FeatureBitsets) { + if (FeatureBitset.empty()) + continue; + OS << " {"; + for (const auto &Feature : FeatureBitset) { + const auto &I = Info.SubtargetFeatures.find(Feature); + assert(I != Info.SubtargetFeatures.end() && "Didn't import predicate?"); + OS << I->second.getEnumBitName() << ", "; + } + OS << "},\n"; + } + OS << "};\n\n"; + + // Emit the static match table; unused classes get initialized to 0 which is + // guaranteed to be InvalidMatchClass. + // + // FIXME: We can reduce the size of this table very easily. First, we change + // it so that store the kinds in separate bit-fields for each index, which + // only needs to be the max width used for classes at that index (we also need + // to reject based on this during classification). If we then make sure to + // order the match kinds appropriately (putting mnemonics last), then we + // should only end up using a few bits for each class, especially the ones + // following the mnemonic. + OS << "namespace {\n"; + OS << " struct MatchEntry {\n"; + OS << " " << getMinimalTypeForRange(MaxMnemonicIndex) + << " Mnemonic;\n"; + OS << " uint16_t Opcode;\n"; + OS << " " << getMinimalTypeForRange(NumConverters) + << " ConvertFn;\n"; + OS << " " << getMinimalTypeForRange(FeatureBitsets.size()) + << " RequiredFeaturesIdx;\n"; + OS << " " << getMinimalTypeForRange( + std::distance(Info.Classes.begin(), Info.Classes.end())) + << " Classes[" << MaxNumOperands << "];\n"; + OS << " StringRef getMnemonic() const {\n"; + OS << " return StringRef(MnemonicTable + Mnemonic + 1,\n"; + OS << " MnemonicTable[Mnemonic]);\n"; + OS << " }\n"; + OS << " };\n\n"; + + OS << " // Predicate for searching for an opcode.\n"; + OS << " struct LessOpcode {\n"; + OS << " bool operator()(const MatchEntry &LHS, StringRef RHS) {\n"; + OS << " return LHS.getMnemonic() < RHS;\n"; + OS << " }\n"; + OS << " bool operator()(StringRef LHS, const MatchEntry &RHS) {\n"; + OS << " return LHS < RHS.getMnemonic();\n"; + OS << " }\n"; + OS << " bool operator()(const MatchEntry &LHS, const MatchEntry &RHS) {\n"; + OS << " return LHS.getMnemonic() < RHS.getMnemonic();\n"; + OS << " }\n"; + OS << " };\n"; + + OS << "} // end anonymous namespace\n\n"; + + unsigned VariantCount = Target.getAsmParserVariantCount(); + for (unsigned VC = 0; VC != VariantCount; ++VC) { + Record *AsmVariant = Target.getAsmParserVariant(VC); + int AsmVariantNo = AsmVariant->getValueAsInt("Variant"); + + OS << "static const MatchEntry MatchTable" << VC << "[] = {\n"; + + for (const auto &MI : Info.Matchables) { + if (MI->AsmVariantID != AsmVariantNo) + continue; + + // Store a pascal-style length byte in the mnemonic. + std::string LenMnemonic = char(MI->Mnemonic.size()) + MI->Mnemonic.str(); + OS << " { " << StringTable.GetOrAddStringOffset(LenMnemonic, false) + << " /* " << MI->Mnemonic << " */, " + << Target.getInstNamespace() << "::" + << MI->getResultInst()->TheDef->getName() << ", " + << MI->ConversionFnKind << ", "; + + // Write the required features mask. + OS << "AMFBS"; + if (MI->RequiredFeatures.empty()) + OS << "_None"; + else + for (unsigned i = 0, e = MI->RequiredFeatures.size(); i != e; ++i) + OS << '_' << MI->RequiredFeatures[i]->TheDef->getName(); + + OS << ", { "; + for (unsigned i = 0, e = MI->AsmOperands.size(); i != e; ++i) { + const MatchableInfo::AsmOperand &Op = MI->AsmOperands[i]; + + if (i) OS << ", "; + OS << Op.Class->Name; + } + OS << " }, },\n"; + } + + OS << "};\n\n"; + } + + OS << "#include \"llvm/Support/Debug.h\"\n"; + OS << "#include \"llvm/Support/Format.h\"\n\n"; + + // Finally, build the match function. + OS << "unsigned " << Target.getName() << ClassName << "::\n" + << "MatchInstructionImpl(const OperandVector &Operands,\n"; + OS << " MCInst &Inst,\n"; + if (ReportMultipleNearMisses) + OS << " SmallVectorImpl<NearMissInfo> *NearMisses,\n"; + else + OS << " uint64_t &ErrorInfo,\n" + << " FeatureBitset &MissingFeatures,\n"; + OS << " bool matchingInlineAsm, unsigned VariantID) {\n"; + + if (!ReportMultipleNearMisses) { + OS << " // Eliminate obvious mismatches.\n"; + OS << " if (Operands.size() > " + << (MaxNumOperands + HasMnemonicFirst) << ") {\n"; + OS << " ErrorInfo = " + << (MaxNumOperands + HasMnemonicFirst) << ";\n"; + OS << " return Match_InvalidOperand;\n"; + OS << " }\n\n"; + } + + // Emit code to get the available features. + OS << " // Get the current feature set.\n"; + OS << " const FeatureBitset &AvailableFeatures = getAvailableFeatures();\n\n"; + + OS << " // Get the instruction mnemonic, which is the first token.\n"; + if (HasMnemonicFirst) { + OS << " StringRef Mnemonic = ((" << Target.getName() + << "Operand&)*Operands[0]).getToken();\n\n"; + } else { + OS << " StringRef Mnemonic;\n"; + OS << " if (Operands[0]->isToken())\n"; + OS << " Mnemonic = ((" << Target.getName() + << "Operand&)*Operands[0]).getToken();\n\n"; + } + + if (HasMnemonicAliases) { + OS << " // Process all MnemonicAliases to remap the mnemonic.\n"; + OS << " applyMnemonicAliases(Mnemonic, AvailableFeatures, VariantID);\n\n"; + } + + // Emit code to compute the class list for this operand vector. + if (!ReportMultipleNearMisses) { + OS << " // Some state to try to produce better error messages.\n"; + OS << " bool HadMatchOtherThanFeatures = false;\n"; + OS << " bool HadMatchOtherThanPredicate = false;\n"; + OS << " unsigned RetCode = Match_InvalidOperand;\n"; + OS << " MissingFeatures.set();\n"; + OS << " // Set ErrorInfo to the operand that mismatches if it is\n"; + OS << " // wrong for all instances of the instruction.\n"; + OS << " ErrorInfo = ~0ULL;\n"; + } + + if (HasOptionalOperands) { + OS << " SmallBitVector OptionalOperandsMask(" << MaxNumOperands << ");\n"; + } + + // Emit code to search the table. + OS << " // Find the appropriate table for this asm variant.\n"; + OS << " const MatchEntry *Start, *End;\n"; + OS << " switch (VariantID) {\n"; + OS << " default: llvm_unreachable(\"invalid variant!\");\n"; + for (unsigned VC = 0; VC != VariantCount; ++VC) { + Record *AsmVariant = Target.getAsmParserVariant(VC); + int AsmVariantNo = AsmVariant->getValueAsInt("Variant"); + OS << " case " << AsmVariantNo << ": Start = std::begin(MatchTable" << VC + << "); End = std::end(MatchTable" << VC << "); break;\n"; + } + OS << " }\n"; + + OS << " // Search the table.\n"; + if (HasMnemonicFirst) { + OS << " auto MnemonicRange = " + "std::equal_range(Start, End, Mnemonic, LessOpcode());\n\n"; + } else { + OS << " auto MnemonicRange = std::make_pair(Start, End);\n"; + OS << " unsigned SIndex = Mnemonic.empty() ? 0 : 1;\n"; + OS << " if (!Mnemonic.empty())\n"; + OS << " MnemonicRange = " + "std::equal_range(Start, End, Mnemonic.lower(), LessOpcode());\n\n"; + } + + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"AsmMatcher: found \" <<\n" + << " std::distance(MnemonicRange.first, MnemonicRange.second) << \n" + << " \" encodings with mnemonic '\" << Mnemonic << \"'\\n\");\n\n"; + + OS << " // Return a more specific error code if no mnemonics match.\n"; + OS << " if (MnemonicRange.first == MnemonicRange.second)\n"; + OS << " return Match_MnemonicFail;\n\n"; + + OS << " for (const MatchEntry *it = MnemonicRange.first, " + << "*ie = MnemonicRange.second;\n"; + OS << " it != ie; ++it) {\n"; + OS << " const FeatureBitset &RequiredFeatures = " + "FeatureBitsets[it->RequiredFeaturesIdx];\n"; + OS << " bool HasRequiredFeatures =\n"; + OS << " (AvailableFeatures & RequiredFeatures) == RequiredFeatures;\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"Trying to match opcode \"\n"; + OS << " << MII.getName(it->Opcode) << \"\\n\");\n"; + + if (ReportMultipleNearMisses) { + OS << " // Some state to record ways in which this instruction did not match.\n"; + OS << " NearMissInfo OperandNearMiss = NearMissInfo::getSuccess();\n"; + OS << " NearMissInfo FeaturesNearMiss = NearMissInfo::getSuccess();\n"; + OS << " NearMissInfo EarlyPredicateNearMiss = NearMissInfo::getSuccess();\n"; + OS << " NearMissInfo LatePredicateNearMiss = NearMissInfo::getSuccess();\n"; + OS << " bool MultipleInvalidOperands = false;\n"; + } + + if (HasMnemonicFirst) { + OS << " // equal_range guarantees that instruction mnemonic matches.\n"; + OS << " assert(Mnemonic == it->getMnemonic());\n"; + } + + // Emit check that the subclasses match. + if (!ReportMultipleNearMisses) + OS << " bool OperandsValid = true;\n"; + if (HasOptionalOperands) { + OS << " OptionalOperandsMask.reset(0, " << MaxNumOperands << ");\n"; + } + OS << " for (unsigned FormalIdx = " << (HasMnemonicFirst ? "0" : "SIndex") + << ", ActualIdx = " << (HasMnemonicFirst ? "1" : "SIndex") + << "; FormalIdx != " << MaxNumOperands << "; ++FormalIdx) {\n"; + OS << " auto Formal = " + << "static_cast<MatchClassKind>(it->Classes[FormalIdx]);\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\",\n"; + OS << " dbgs() << \" Matching formal operand class \" << getMatchClassName(Formal)\n"; + OS << " << \" against actual operand at index \" << ActualIdx);\n"; + OS << " if (ActualIdx < Operands.size())\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \" (\";\n"; + OS << " Operands[ActualIdx]->print(dbgs()); dbgs() << \"): \");\n"; + OS << " else\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \": \");\n"; + OS << " if (ActualIdx >= Operands.size()) {\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"actual operand index out of range \");\n"; + if (ReportMultipleNearMisses) { + OS << " bool ThisOperandValid = (Formal == " <<"InvalidMatchClass) || " + "isSubclass(Formal, OptionalMatchClass);\n"; + OS << " if (!ThisOperandValid) {\n"; + OS << " if (!OperandNearMiss) {\n"; + OS << " // Record info about match failure for later use.\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"recording too-few-operands near miss\\n\");\n"; + OS << " OperandNearMiss =\n"; + OS << " NearMissInfo::getTooFewOperands(Formal, it->Opcode);\n"; + OS << " } else if (OperandNearMiss.getKind() != NearMissInfo::NearMissTooFewOperands) {\n"; + OS << " // If more than one operand is invalid, give up on this match entry.\n"; + OS << " DEBUG_WITH_TYPE(\n"; + OS << " \"asm-matcher\",\n"; + OS << " dbgs() << \"second invalid operand, giving up on this opcode\\n\");\n"; + OS << " MultipleInvalidOperands = true;\n"; + OS << " break;\n"; + OS << " }\n"; + OS << " } else {\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"but formal operand not required\\n\");\n"; + OS << " break;\n"; + OS << " }\n"; + OS << " continue;\n"; + } else { + OS << " OperandsValid = (Formal == InvalidMatchClass) || isSubclass(Formal, OptionalMatchClass);\n"; + OS << " if (!OperandsValid) ErrorInfo = ActualIdx;\n"; + if (HasOptionalOperands) { + OS << " OptionalOperandsMask.set(FormalIdx, " << MaxNumOperands + << ");\n"; + } + OS << " break;\n"; + } + OS << " }\n"; + OS << " MCParsedAsmOperand &Actual = *Operands[ActualIdx];\n"; + OS << " unsigned Diag = validateOperandClass(Actual, Formal);\n"; + OS << " if (Diag == Match_Success) {\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\",\n"; + OS << " dbgs() << \"match success using generic matcher\\n\");\n"; + OS << " ++ActualIdx;\n"; + OS << " continue;\n"; + OS << " }\n"; + OS << " // If the generic handler indicates an invalid operand\n"; + OS << " // failure, check for a special case.\n"; + OS << " if (Diag != Match_Success) {\n"; + OS << " unsigned TargetDiag = validateTargetOperandClass(Actual, Formal);\n"; + OS << " if (TargetDiag == Match_Success) {\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\",\n"; + OS << " dbgs() << \"match success using target matcher\\n\");\n"; + OS << " ++ActualIdx;\n"; + OS << " continue;\n"; + OS << " }\n"; + OS << " // If the target matcher returned a specific error code use\n"; + OS << " // that, else use the one from the generic matcher.\n"; + OS << " if (TargetDiag != Match_InvalidOperand && " + "HasRequiredFeatures)\n"; + OS << " Diag = TargetDiag;\n"; + OS << " }\n"; + OS << " // If current formal operand wasn't matched and it is optional\n" + << " // then try to match next formal operand\n"; + OS << " if (Diag == Match_InvalidOperand " + << "&& isSubclass(Formal, OptionalMatchClass)) {\n"; + if (HasOptionalOperands) { + OS << " OptionalOperandsMask.set(FormalIdx);\n"; + } + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"ignoring optional operand\\n\");\n"; + OS << " continue;\n"; + OS << " }\n"; + + if (ReportMultipleNearMisses) { + OS << " if (!OperandNearMiss) {\n"; + OS << " // If this is the first invalid operand we have seen, record some\n"; + OS << " // information about it.\n"; + OS << " DEBUG_WITH_TYPE(\n"; + OS << " \"asm-matcher\",\n"; + OS << " dbgs()\n"; + OS << " << \"operand match failed, recording near-miss with diag code \"\n"; + OS << " << Diag << \"\\n\");\n"; + OS << " OperandNearMiss =\n"; + OS << " NearMissInfo::getMissedOperand(Diag, Formal, it->Opcode, ActualIdx);\n"; + OS << " ++ActualIdx;\n"; + OS << " } else {\n"; + OS << " // If more than one operand is invalid, give up on this match entry.\n"; + OS << " DEBUG_WITH_TYPE(\n"; + OS << " \"asm-matcher\",\n"; + OS << " dbgs() << \"second operand mismatch, skipping this opcode\\n\");\n"; + OS << " MultipleInvalidOperands = true;\n"; + OS << " break;\n"; + OS << " }\n"; + OS << " }\n\n"; + } else { + OS << " // If this operand is broken for all of the instances of this\n"; + OS << " // mnemonic, keep track of it so we can report loc info.\n"; + OS << " // If we already had a match that only failed due to a\n"; + OS << " // target predicate, that diagnostic is preferred.\n"; + OS << " if (!HadMatchOtherThanPredicate &&\n"; + OS << " (it == MnemonicRange.first || ErrorInfo <= ActualIdx)) {\n"; + OS << " if (HasRequiredFeatures && (ErrorInfo != ActualIdx || Diag " + "!= Match_InvalidOperand))\n"; + OS << " RetCode = Diag;\n"; + OS << " ErrorInfo = ActualIdx;\n"; + OS << " }\n"; + OS << " // Otherwise, just reject this instance of the mnemonic.\n"; + OS << " OperandsValid = false;\n"; + OS << " break;\n"; + OS << " }\n\n"; + } + + if (ReportMultipleNearMisses) + OS << " if (MultipleInvalidOperands) {\n"; + else + OS << " if (!OperandsValid) {\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"Opcode result: multiple \"\n"; + OS << " \"operand mismatches, ignoring \"\n"; + OS << " \"this opcode\\n\");\n"; + OS << " continue;\n"; + OS << " }\n"; + + // Emit check that the required features are available. + OS << " if (!HasRequiredFeatures) {\n"; + if (!ReportMultipleNearMisses) + OS << " HadMatchOtherThanFeatures = true;\n"; + OS << " FeatureBitset NewMissingFeatures = RequiredFeatures & " + "~AvailableFeatures;\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"Missing target features:\";\n"; + OS << " for (unsigned I = 0, E = NewMissingFeatures.size(); I != E; ++I)\n"; + OS << " if (NewMissingFeatures[I])\n"; + OS << " dbgs() << ' ' << I;\n"; + OS << " dbgs() << \"\\n\");\n"; + if (ReportMultipleNearMisses) { + OS << " FeaturesNearMiss = NearMissInfo::getMissedFeature(NewMissingFeatures);\n"; + } else { + OS << " if (NewMissingFeatures.count() <=\n" + " MissingFeatures.count())\n"; + OS << " MissingFeatures = NewMissingFeatures;\n"; + OS << " continue;\n"; + } + OS << " }\n"; + OS << "\n"; + OS << " Inst.clear();\n\n"; + OS << " Inst.setOpcode(it->Opcode);\n"; + // Verify the instruction with the target-specific match predicate function. + OS << " // We have a potential match but have not rendered the operands.\n" + << " // Check the target predicate to handle any context sensitive\n" + " // constraints.\n" + << " // For example, Ties that are referenced multiple times must be\n" + " // checked here to ensure the input is the same for each match\n" + " // constraints. If we leave it any later the ties will have been\n" + " // canonicalized\n" + << " unsigned MatchResult;\n" + << " if ((MatchResult = checkEarlyTargetMatchPredicate(Inst, " + "Operands)) != Match_Success) {\n" + << " Inst.clear();\n"; + OS << " DEBUG_WITH_TYPE(\n"; + OS << " \"asm-matcher\",\n"; + OS << " dbgs() << \"Early target match predicate failed with diag code \"\n"; + OS << " << MatchResult << \"\\n\");\n"; + if (ReportMultipleNearMisses) { + OS << " EarlyPredicateNearMiss = NearMissInfo::getMissedPredicate(MatchResult);\n"; + } else { + OS << " RetCode = MatchResult;\n" + << " HadMatchOtherThanPredicate = true;\n" + << " continue;\n"; + } + OS << " }\n\n"; + + if (ReportMultipleNearMisses) { + OS << " // If we did not successfully match the operands, then we can't convert to\n"; + OS << " // an MCInst, so bail out on this instruction variant now.\n"; + OS << " if (OperandNearMiss) {\n"; + OS << " // If the operand mismatch was the only problem, reprrt it as a near-miss.\n"; + OS << " if (NearMisses && !FeaturesNearMiss && !EarlyPredicateNearMiss) {\n"; + OS << " DEBUG_WITH_TYPE(\n"; + OS << " \"asm-matcher\",\n"; + OS << " dbgs()\n"; + OS << " << \"Opcode result: one mismatched operand, adding near-miss\\n\");\n"; + OS << " NearMisses->push_back(OperandNearMiss);\n"; + OS << " } else {\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"Opcode result: multiple \"\n"; + OS << " \"types of mismatch, so not \"\n"; + OS << " \"reporting near-miss\\n\");\n"; + OS << " }\n"; + OS << " continue;\n"; + OS << " }\n\n"; + } + + OS << " if (matchingInlineAsm) {\n"; + OS << " convertToMapAndConstraints(it->ConvertFn, Operands);\n"; + if (!ReportMultipleNearMisses) { + OS << " if (!checkAsmTiedOperandConstraints(*this, it->ConvertFn, " + "Operands, ErrorInfo))\n"; + OS << " return Match_InvalidTiedOperand;\n"; + OS << "\n"; + } + OS << " return Match_Success;\n"; + OS << " }\n\n"; + OS << " // We have selected a definite instruction, convert the parsed\n" + << " // operands into the appropriate MCInst.\n"; + if (HasOptionalOperands) { + OS << " convertToMCInst(it->ConvertFn, Inst, it->Opcode, Operands,\n" + << " OptionalOperandsMask);\n"; + } else { + OS << " convertToMCInst(it->ConvertFn, Inst, it->Opcode, Operands);\n"; + } + OS << "\n"; + + // Verify the instruction with the target-specific match predicate function. + OS << " // We have a potential match. Check the target predicate to\n" + << " // handle any context sensitive constraints.\n" + << " if ((MatchResult = checkTargetMatchPredicate(Inst)) !=" + << " Match_Success) {\n" + << " DEBUG_WITH_TYPE(\"asm-matcher\",\n" + << " dbgs() << \"Target match predicate failed with diag code \"\n" + << " << MatchResult << \"\\n\");\n" + << " Inst.clear();\n"; + if (ReportMultipleNearMisses) { + OS << " LatePredicateNearMiss = NearMissInfo::getMissedPredicate(MatchResult);\n"; + } else { + OS << " RetCode = MatchResult;\n" + << " HadMatchOtherThanPredicate = true;\n" + << " continue;\n"; + } + OS << " }\n\n"; + + if (ReportMultipleNearMisses) { + OS << " int NumNearMisses = ((int)(bool)OperandNearMiss +\n"; + OS << " (int)(bool)FeaturesNearMiss +\n"; + OS << " (int)(bool)EarlyPredicateNearMiss +\n"; + OS << " (int)(bool)LatePredicateNearMiss);\n"; + OS << " if (NumNearMisses == 1) {\n"; + OS << " // We had exactly one type of near-miss, so add that to the list.\n"; + OS << " assert(!OperandNearMiss && \"OperandNearMiss was handled earlier\");\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"Opcode result: found one type of \"\n"; + OS << " \"mismatch, so reporting a \"\n"; + OS << " \"near-miss\\n\");\n"; + OS << " if (NearMisses && FeaturesNearMiss)\n"; + OS << " NearMisses->push_back(FeaturesNearMiss);\n"; + OS << " else if (NearMisses && EarlyPredicateNearMiss)\n"; + OS << " NearMisses->push_back(EarlyPredicateNearMiss);\n"; + OS << " else if (NearMisses && LatePredicateNearMiss)\n"; + OS << " NearMisses->push_back(LatePredicateNearMiss);\n"; + OS << "\n"; + OS << " continue;\n"; + OS << " } else if (NumNearMisses > 1) {\n"; + OS << " // This instruction missed in more than one way, so ignore it.\n"; + OS << " DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \"Opcode result: multiple \"\n"; + OS << " \"types of mismatch, so not \"\n"; + OS << " \"reporting near-miss\\n\");\n"; + OS << " continue;\n"; + OS << " }\n"; + } + + // Call the post-processing function, if used. + StringRef InsnCleanupFn = AsmParser->getValueAsString("AsmParserInstCleanup"); + if (!InsnCleanupFn.empty()) + OS << " " << InsnCleanupFn << "(Inst);\n"; + + if (HasDeprecation) { + OS << " std::string Info;\n"; + OS << " if (!getParser().getTargetParser().\n"; + OS << " getTargetOptions().MCNoDeprecatedWarn &&\n"; + OS << " MII.getDeprecatedInfo(Inst, getSTI(), Info)) {\n"; + OS << " SMLoc Loc = ((" << Target.getName() + << "Operand&)*Operands[0]).getStartLoc();\n"; + OS << " getParser().Warning(Loc, Info, None);\n"; + OS << " }\n"; + } + + if (!ReportMultipleNearMisses) { + OS << " if (!checkAsmTiedOperandConstraints(*this, it->ConvertFn, " + "Operands, ErrorInfo))\n"; + OS << " return Match_InvalidTiedOperand;\n"; + OS << "\n"; + } + + OS << " DEBUG_WITH_TYPE(\n"; + OS << " \"asm-matcher\",\n"; + OS << " dbgs() << \"Opcode result: complete match, selecting this opcode\\n\");\n"; + OS << " return Match_Success;\n"; + OS << " }\n\n"; + + if (ReportMultipleNearMisses) { + OS << " // No instruction variants matched exactly.\n"; + OS << " return Match_NearMisses;\n"; + } else { + OS << " // Okay, we had no match. Try to return a useful error code.\n"; + OS << " if (HadMatchOtherThanPredicate || !HadMatchOtherThanFeatures)\n"; + OS << " return RetCode;\n\n"; + OS << " ErrorInfo = 0;\n"; + OS << " return Match_MissingFeature;\n"; + } + OS << "}\n\n"; + + if (!Info.OperandMatchInfo.empty()) + emitCustomOperandParsing(OS, Target, Info, ClassName, StringTable, + MaxMnemonicIndex, FeatureBitsets.size(), + HasMnemonicFirst); + + OS << "#endif // GET_MATCHER_IMPLEMENTATION\n\n"; + + OS << "\n#ifdef GET_MNEMONIC_SPELL_CHECKER\n"; + OS << "#undef GET_MNEMONIC_SPELL_CHECKER\n\n"; + + emitMnemonicSpellChecker(OS, Target, VariantCount); + + OS << "#endif // GET_MNEMONIC_SPELL_CHECKER\n\n"; +} + +namespace llvm { + +void EmitAsmMatcher(RecordKeeper &RK, raw_ostream &OS) { + emitSourceFileHeader("Assembly Matcher Source Fragment", OS); + AsmMatcherEmitter(RK).run(OS); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp new file mode 100644 index 000000000000..d10ea71e97e3 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/AsmWriterEmitter.cpp @@ -0,0 +1,1277 @@ +//===- AsmWriterEmitter.cpp - Generate an assembly writer -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend emits an assembly printer for the current target. +// Note that this is currently fairly skeletal, but will grow over time. +// +//===----------------------------------------------------------------------===// + +#include "AsmWriterInst.h" +#include "CodeGenInstruction.h" +#include "CodeGenRegisters.h" +#include "CodeGenTarget.h" +#include "SequenceToOffsetTable.h" +#include "Types.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <deque> +#include <iterator> +#include <map> +#include <set> +#include <string> +#include <tuple> +#include <utility> +#include <vector> + +using namespace llvm; + +#define DEBUG_TYPE "asm-writer-emitter" + +namespace { + +class AsmWriterEmitter { + RecordKeeper &Records; + CodeGenTarget Target; + ArrayRef<const CodeGenInstruction *> NumberedInstructions; + std::vector<AsmWriterInst> Instructions; + +public: + AsmWriterEmitter(RecordKeeper &R); + + void run(raw_ostream &o); + +private: + void EmitPrintInstruction(raw_ostream &o); + void EmitGetRegisterName(raw_ostream &o); + void EmitPrintAliasInstruction(raw_ostream &O); + + void FindUniqueOperandCommands(std::vector<std::string> &UOC, + std::vector<std::vector<unsigned>> &InstIdxs, + std::vector<unsigned> &InstOpsUsed, + bool PassSubtarget) const; +}; + +} // end anonymous namespace + +static void PrintCases(std::vector<std::pair<std::string, + AsmWriterOperand>> &OpsToPrint, raw_ostream &O, + bool PassSubtarget) { + O << " case " << OpsToPrint.back().first << ":"; + AsmWriterOperand TheOp = OpsToPrint.back().second; + OpsToPrint.pop_back(); + + // Check to see if any other operands are identical in this list, and if so, + // emit a case label for them. + for (unsigned i = OpsToPrint.size(); i != 0; --i) + if (OpsToPrint[i-1].second == TheOp) { + O << "\n case " << OpsToPrint[i-1].first << ":"; + OpsToPrint.erase(OpsToPrint.begin()+i-1); + } + + // Finally, emit the code. + O << "\n " << TheOp.getCode(PassSubtarget); + O << "\n break;\n"; +} + +/// EmitInstructions - Emit the last instruction in the vector and any other +/// instructions that are suitably similar to it. +static void EmitInstructions(std::vector<AsmWriterInst> &Insts, + raw_ostream &O, bool PassSubtarget) { + AsmWriterInst FirstInst = Insts.back(); + Insts.pop_back(); + + std::vector<AsmWriterInst> SimilarInsts; + unsigned DifferingOperand = ~0; + for (unsigned i = Insts.size(); i != 0; --i) { + unsigned DiffOp = Insts[i-1].MatchesAllButOneOp(FirstInst); + if (DiffOp != ~1U) { + if (DifferingOperand == ~0U) // First match! + DifferingOperand = DiffOp; + + // If this differs in the same operand as the rest of the instructions in + // this class, move it to the SimilarInsts list. + if (DifferingOperand == DiffOp || DiffOp == ~0U) { + SimilarInsts.push_back(Insts[i-1]); + Insts.erase(Insts.begin()+i-1); + } + } + } + + O << " case " << FirstInst.CGI->Namespace << "::" + << FirstInst.CGI->TheDef->getName() << ":\n"; + for (const AsmWriterInst &AWI : SimilarInsts) + O << " case " << AWI.CGI->Namespace << "::" + << AWI.CGI->TheDef->getName() << ":\n"; + for (unsigned i = 0, e = FirstInst.Operands.size(); i != e; ++i) { + if (i != DifferingOperand) { + // If the operand is the same for all instructions, just print it. + O << " " << FirstInst.Operands[i].getCode(PassSubtarget); + } else { + // If this is the operand that varies between all of the instructions, + // emit a switch for just this operand now. + O << " switch (MI->getOpcode()) {\n"; + O << " default: llvm_unreachable(\"Unexpected opcode.\");\n"; + std::vector<std::pair<std::string, AsmWriterOperand>> OpsToPrint; + OpsToPrint.push_back(std::make_pair(FirstInst.CGI->Namespace.str() + "::" + + FirstInst.CGI->TheDef->getName().str(), + FirstInst.Operands[i])); + + for (const AsmWriterInst &AWI : SimilarInsts) { + OpsToPrint.push_back(std::make_pair(AWI.CGI->Namespace.str()+"::" + + AWI.CGI->TheDef->getName().str(), + AWI.Operands[i])); + } + std::reverse(OpsToPrint.begin(), OpsToPrint.end()); + while (!OpsToPrint.empty()) + PrintCases(OpsToPrint, O, PassSubtarget); + O << " }"; + } + O << "\n"; + } + O << " break;\n"; +} + +void AsmWriterEmitter:: +FindUniqueOperandCommands(std::vector<std::string> &UniqueOperandCommands, + std::vector<std::vector<unsigned>> &InstIdxs, + std::vector<unsigned> &InstOpsUsed, + bool PassSubtarget) const { + // This vector parallels UniqueOperandCommands, keeping track of which + // instructions each case are used for. It is a comma separated string of + // enums. + std::vector<std::string> InstrsForCase; + InstrsForCase.resize(UniqueOperandCommands.size()); + InstOpsUsed.assign(UniqueOperandCommands.size(), 0); + + for (size_t i = 0, e = Instructions.size(); i != e; ++i) { + const AsmWriterInst &Inst = Instructions[i]; + if (Inst.Operands.empty()) + continue; // Instruction already done. + + std::string Command = " "+Inst.Operands[0].getCode(PassSubtarget)+"\n"; + + // Check to see if we already have 'Command' in UniqueOperandCommands. + // If not, add it. + auto I = llvm::find(UniqueOperandCommands, Command); + if (I != UniqueOperandCommands.end()) { + size_t idx = I - UniqueOperandCommands.begin(); + InstrsForCase[idx] += ", "; + InstrsForCase[idx] += Inst.CGI->TheDef->getName(); + InstIdxs[idx].push_back(i); + } else { + UniqueOperandCommands.push_back(std::move(Command)); + InstrsForCase.push_back(std::string(Inst.CGI->TheDef->getName())); + InstIdxs.emplace_back(); + InstIdxs.back().push_back(i); + + // This command matches one operand so far. + InstOpsUsed.push_back(1); + } + } + + // For each entry of UniqueOperandCommands, there is a set of instructions + // that uses it. If the next command of all instructions in the set are + // identical, fold it into the command. + for (size_t CommandIdx = 0, e = UniqueOperandCommands.size(); + CommandIdx != e; ++CommandIdx) { + + const auto &Idxs = InstIdxs[CommandIdx]; + + for (unsigned Op = 1; ; ++Op) { + // Find the first instruction in the set. + const AsmWriterInst &FirstInst = Instructions[Idxs.front()]; + // If this instruction has no more operands, we isn't anything to merge + // into this command. + if (FirstInst.Operands.size() == Op) + break; + + // Otherwise, scan to see if all of the other instructions in this command + // set share the operand. + if (std::any_of(Idxs.begin()+1, Idxs.end(), + [&](unsigned Idx) { + const AsmWriterInst &OtherInst = Instructions[Idx]; + return OtherInst.Operands.size() == Op || + OtherInst.Operands[Op] != FirstInst.Operands[Op]; + })) + break; + + // Okay, everything in this command set has the same next operand. Add it + // to UniqueOperandCommands and remember that it was consumed. + std::string Command = " " + + FirstInst.Operands[Op].getCode(PassSubtarget) + "\n"; + + UniqueOperandCommands[CommandIdx] += Command; + InstOpsUsed[CommandIdx]++; + } + } + + // Prepend some of the instructions each case is used for onto the case val. + for (unsigned i = 0, e = InstrsForCase.size(); i != e; ++i) { + std::string Instrs = InstrsForCase[i]; + if (Instrs.size() > 70) { + Instrs.erase(Instrs.begin()+70, Instrs.end()); + Instrs += "..."; + } + + if (!Instrs.empty()) + UniqueOperandCommands[i] = " // " + Instrs + "\n" + + UniqueOperandCommands[i]; + } +} + +static void UnescapeString(std::string &Str) { + for (unsigned i = 0; i != Str.size(); ++i) { + if (Str[i] == '\\' && i != Str.size()-1) { + switch (Str[i+1]) { + default: continue; // Don't execute the code after the switch. + case 'a': Str[i] = '\a'; break; + case 'b': Str[i] = '\b'; break; + case 'e': Str[i] = 27; break; + case 'f': Str[i] = '\f'; break; + case 'n': Str[i] = '\n'; break; + case 'r': Str[i] = '\r'; break; + case 't': Str[i] = '\t'; break; + case 'v': Str[i] = '\v'; break; + case '"': Str[i] = '\"'; break; + case '\'': Str[i] = '\''; break; + case '\\': Str[i] = '\\'; break; + } + // Nuke the second character. + Str.erase(Str.begin()+i+1); + } + } +} + +/// UnescapeAliasString - Supports literal braces in InstAlias asm string which +/// are escaped with '\\' to avoid being interpreted as variants. Braces must +/// be unescaped before c++ code is generated as (e.g.): +/// +/// AsmString = "foo \{$\x01\}"; +/// +/// causes non-standard escape character warnings. +static void UnescapeAliasString(std::string &Str) { + for (unsigned i = 0; i != Str.size(); ++i) { + if (Str[i] == '\\' && i != Str.size()-1) { + switch (Str[i+1]) { + default: continue; // Don't execute the code after the switch. + case '{': Str[i] = '{'; break; + case '}': Str[i] = '}'; break; + } + // Nuke the second character. + Str.erase(Str.begin()+i+1); + } + } +} + +/// EmitPrintInstruction - Generate the code for the "printInstruction" method +/// implementation. Destroys all instances of AsmWriterInst information, by +/// clearing the Instructions vector. +void AsmWriterEmitter::EmitPrintInstruction(raw_ostream &O) { + Record *AsmWriter = Target.getAsmWriter(); + StringRef ClassName = AsmWriter->getValueAsString("AsmWriterClassName"); + bool PassSubtarget = AsmWriter->getValueAsInt("PassSubtarget"); + + O << "/// printInstruction - This method is automatically generated by " + "tablegen\n" + "/// from the instruction set description.\n" + "void " + << Target.getName() << ClassName + << "::printInstruction(const MCInst *MI, uint64_t Address, " + << (PassSubtarget ? "const MCSubtargetInfo &STI, " : "") + << "raw_ostream &O) {\n"; + + // Build an aggregate string, and build a table of offsets into it. + SequenceToOffsetTable<std::string> StringTable; + + /// OpcodeInfo - This encodes the index of the string to use for the first + /// chunk of the output as well as indices used for operand printing. + std::vector<uint64_t> OpcodeInfo(NumberedInstructions.size()); + const unsigned OpcodeInfoBits = 64; + + // Add all strings to the string table upfront so it can generate an optimized + // representation. + for (AsmWriterInst &AWI : Instructions) { + if (AWI.Operands[0].OperandType == + AsmWriterOperand::isLiteralTextOperand && + !AWI.Operands[0].Str.empty()) { + std::string Str = AWI.Operands[0].Str; + UnescapeString(Str); + StringTable.add(Str); + } + } + + StringTable.layout(); + + unsigned MaxStringIdx = 0; + for (AsmWriterInst &AWI : Instructions) { + unsigned Idx; + if (AWI.Operands[0].OperandType != AsmWriterOperand::isLiteralTextOperand || + AWI.Operands[0].Str.empty()) { + // Something handled by the asmwriter printer, but with no leading string. + Idx = StringTable.get(""); + } else { + std::string Str = AWI.Operands[0].Str; + UnescapeString(Str); + Idx = StringTable.get(Str); + MaxStringIdx = std::max(MaxStringIdx, Idx); + + // Nuke the string from the operand list. It is now handled! + AWI.Operands.erase(AWI.Operands.begin()); + } + + // Bias offset by one since we want 0 as a sentinel. + OpcodeInfo[AWI.CGIIndex] = Idx+1; + } + + // Figure out how many bits we used for the string index. + unsigned AsmStrBits = Log2_32_Ceil(MaxStringIdx+2); + + // To reduce code size, we compactify common instructions into a few bits + // in the opcode-indexed table. + unsigned BitsLeft = OpcodeInfoBits-AsmStrBits; + + std::vector<std::vector<std::string>> TableDrivenOperandPrinters; + + while (true) { + std::vector<std::string> UniqueOperandCommands; + std::vector<std::vector<unsigned>> InstIdxs; + std::vector<unsigned> NumInstOpsHandled; + FindUniqueOperandCommands(UniqueOperandCommands, InstIdxs, + NumInstOpsHandled, PassSubtarget); + + // If we ran out of operands to print, we're done. + if (UniqueOperandCommands.empty()) break; + + // Compute the number of bits we need to represent these cases, this is + // ceil(log2(numentries)). + unsigned NumBits = Log2_32_Ceil(UniqueOperandCommands.size()); + + // If we don't have enough bits for this operand, don't include it. + if (NumBits > BitsLeft) { + LLVM_DEBUG(errs() << "Not enough bits to densely encode " << NumBits + << " more bits\n"); + break; + } + + // Otherwise, we can include this in the initial lookup table. Add it in. + for (size_t i = 0, e = InstIdxs.size(); i != e; ++i) { + unsigned NumOps = NumInstOpsHandled[i]; + for (unsigned Idx : InstIdxs[i]) { + OpcodeInfo[Instructions[Idx].CGIIndex] |= + (uint64_t)i << (OpcodeInfoBits-BitsLeft); + // Remove the info about this operand from the instruction. + AsmWriterInst &Inst = Instructions[Idx]; + if (!Inst.Operands.empty()) { + assert(NumOps <= Inst.Operands.size() && + "Can't remove this many ops!"); + Inst.Operands.erase(Inst.Operands.begin(), + Inst.Operands.begin()+NumOps); + } + } + } + BitsLeft -= NumBits; + + // Remember the handlers for this set of operands. + TableDrivenOperandPrinters.push_back(std::move(UniqueOperandCommands)); + } + + // Emit the string table itself. + StringTable.emitStringLiteralDef(O, " static const char AsmStrs[]"); + + // Emit the lookup tables in pieces to minimize wasted bytes. + unsigned BytesNeeded = ((OpcodeInfoBits - BitsLeft) + 7) / 8; + unsigned Table = 0, Shift = 0; + SmallString<128> BitsString; + raw_svector_ostream BitsOS(BitsString); + // If the total bits is more than 32-bits we need to use a 64-bit type. + BitsOS << " uint" << ((BitsLeft < (OpcodeInfoBits - 32)) ? 64 : 32) + << "_t Bits = 0;\n"; + while (BytesNeeded != 0) { + // Figure out how big this table section needs to be, but no bigger than 4. + unsigned TableSize = std::min(1 << Log2_32(BytesNeeded), 4); + BytesNeeded -= TableSize; + TableSize *= 8; // Convert to bits; + uint64_t Mask = (1ULL << TableSize) - 1; + O << " static const uint" << TableSize << "_t OpInfo" << Table + << "[] = {\n"; + for (unsigned i = 0, e = NumberedInstructions.size(); i != e; ++i) { + O << " " << ((OpcodeInfo[i] >> Shift) & Mask) << "U,\t// " + << NumberedInstructions[i]->TheDef->getName() << "\n"; + } + O << " };\n\n"; + // Emit string to combine the individual table lookups. + BitsOS << " Bits |= "; + // If the total bits is more than 32-bits we need to use a 64-bit type. + if (BitsLeft < (OpcodeInfoBits - 32)) + BitsOS << "(uint64_t)"; + BitsOS << "OpInfo" << Table << "[MI->getOpcode()] << " << Shift << ";\n"; + // Prepare the shift for the next iteration and increment the table count. + Shift += TableSize; + ++Table; + } + + // Emit the initial tab character. + O << " O << \"\\t\";\n\n"; + + O << " // Emit the opcode for the instruction.\n"; + O << BitsString; + + // Emit the starting string. + O << " assert(Bits != 0 && \"Cannot print this instruction.\");\n" + << " O << AsmStrs+(Bits & " << (1 << AsmStrBits)-1 << ")-1;\n\n"; + + // Output the table driven operand information. + BitsLeft = OpcodeInfoBits-AsmStrBits; + for (unsigned i = 0, e = TableDrivenOperandPrinters.size(); i != e; ++i) { + std::vector<std::string> &Commands = TableDrivenOperandPrinters[i]; + + // Compute the number of bits we need to represent these cases, this is + // ceil(log2(numentries)). + unsigned NumBits = Log2_32_Ceil(Commands.size()); + assert(NumBits <= BitsLeft && "consistency error"); + + // Emit code to extract this field from Bits. + O << "\n // Fragment " << i << " encoded into " << NumBits + << " bits for " << Commands.size() << " unique commands.\n"; + + if (Commands.size() == 2) { + // Emit two possibilitys with if/else. + O << " if ((Bits >> " + << (OpcodeInfoBits-BitsLeft) << ") & " + << ((1 << NumBits)-1) << ") {\n" + << Commands[1] + << " } else {\n" + << Commands[0] + << " }\n\n"; + } else if (Commands.size() == 1) { + // Emit a single possibility. + O << Commands[0] << "\n\n"; + } else { + O << " switch ((Bits >> " + << (OpcodeInfoBits-BitsLeft) << ") & " + << ((1 << NumBits)-1) << ") {\n" + << " default: llvm_unreachable(\"Invalid command number.\");\n"; + + // Print out all the cases. + for (unsigned j = 0, e = Commands.size(); j != e; ++j) { + O << " case " << j << ":\n"; + O << Commands[j]; + O << " break;\n"; + } + O << " }\n\n"; + } + BitsLeft -= NumBits; + } + + // Okay, delete instructions with no operand info left. + auto I = llvm::remove_if(Instructions, + [](AsmWriterInst &Inst) { return Inst.Operands.empty(); }); + Instructions.erase(I, Instructions.end()); + + + // Because this is a vector, we want to emit from the end. Reverse all of the + // elements in the vector. + std::reverse(Instructions.begin(), Instructions.end()); + + + // Now that we've emitted all of the operand info that fit into 64 bits, emit + // information for those instructions that are left. This is a less dense + // encoding, but we expect the main 64-bit table to handle the majority of + // instructions. + if (!Instructions.empty()) { + // Find the opcode # of inline asm. + O << " switch (MI->getOpcode()) {\n"; + O << " default: llvm_unreachable(\"Unexpected opcode.\");\n"; + while (!Instructions.empty()) + EmitInstructions(Instructions, O, PassSubtarget); + + O << " }\n"; + } + + O << "}\n"; +} + +static void +emitRegisterNameString(raw_ostream &O, StringRef AltName, + const std::deque<CodeGenRegister> &Registers) { + SequenceToOffsetTable<std::string> StringTable; + SmallVector<std::string, 4> AsmNames(Registers.size()); + unsigned i = 0; + for (const auto &Reg : Registers) { + std::string &AsmName = AsmNames[i++]; + + // "NoRegAltName" is special. We don't need to do a lookup for that, + // as it's just a reference to the default register name. + if (AltName == "" || AltName == "NoRegAltName") { + AsmName = std::string(Reg.TheDef->getValueAsString("AsmName")); + if (AsmName.empty()) + AsmName = std::string(Reg.getName()); + } else { + // Make sure the register has an alternate name for this index. + std::vector<Record*> AltNameList = + Reg.TheDef->getValueAsListOfDefs("RegAltNameIndices"); + unsigned Idx = 0, e; + for (e = AltNameList.size(); + Idx < e && (AltNameList[Idx]->getName() != AltName); + ++Idx) + ; + // If the register has an alternate name for this index, use it. + // Otherwise, leave it empty as an error flag. + if (Idx < e) { + std::vector<StringRef> AltNames = + Reg.TheDef->getValueAsListOfStrings("AltNames"); + if (AltNames.size() <= Idx) + PrintFatalError(Reg.TheDef->getLoc(), + "Register definition missing alt name for '" + + AltName + "'."); + AsmName = std::string(AltNames[Idx]); + } + } + StringTable.add(AsmName); + } + + StringTable.layout(); + StringTable.emitStringLiteralDef(O, Twine(" static const char AsmStrs") + + AltName + "[]"); + + O << " static const " << getMinimalTypeForRange(StringTable.size() - 1, 32) + << " RegAsmOffset" << AltName << "[] = {"; + for (unsigned i = 0, e = Registers.size(); i != e; ++i) { + if ((i % 14) == 0) + O << "\n "; + O << StringTable.get(AsmNames[i]) << ", "; + } + O << "\n };\n" + << "\n"; +} + +void AsmWriterEmitter::EmitGetRegisterName(raw_ostream &O) { + Record *AsmWriter = Target.getAsmWriter(); + StringRef ClassName = AsmWriter->getValueAsString("AsmWriterClassName"); + const auto &Registers = Target.getRegBank().getRegisters(); + const std::vector<Record*> &AltNameIndices = Target.getRegAltNameIndices(); + bool hasAltNames = AltNameIndices.size() > 1; + StringRef Namespace = Registers.front().TheDef->getValueAsString("Namespace"); + + O << + "\n\n/// getRegisterName - This method is automatically generated by tblgen\n" + "/// from the register set description. This returns the assembler name\n" + "/// for the specified register.\n" + "const char *" << Target.getName() << ClassName << "::"; + if (hasAltNames) + O << "\ngetRegisterName(unsigned RegNo, unsigned AltIdx) {\n"; + else + O << "getRegisterName(unsigned RegNo) {\n"; + O << " assert(RegNo && RegNo < " << (Registers.size()+1) + << " && \"Invalid register number!\");\n" + << "\n"; + + if (hasAltNames) { + for (const Record *R : AltNameIndices) + emitRegisterNameString(O, R->getName(), Registers); + } else + emitRegisterNameString(O, "", Registers); + + if (hasAltNames) { + O << " switch(AltIdx) {\n" + << " default: llvm_unreachable(\"Invalid register alt name index!\");\n"; + for (const Record *R : AltNameIndices) { + StringRef AltName = R->getName(); + O << " case "; + if (!Namespace.empty()) + O << Namespace << "::"; + O << AltName << ":\n"; + if (R->isValueUnset("FallbackRegAltNameIndex")) + O << " assert(*(AsmStrs" << AltName << "+RegAsmOffset" << AltName + << "[RegNo-1]) &&\n" + << " \"Invalid alt name index for register!\");\n"; + else { + O << " if (!*(AsmStrs" << AltName << "+RegAsmOffset" << AltName + << "[RegNo-1]))\n" + << " return getRegisterName(RegNo, "; + if (!Namespace.empty()) + O << Namespace << "::"; + O << R->getValueAsDef("FallbackRegAltNameIndex")->getName() << ");\n"; + } + O << " return AsmStrs" << AltName << "+RegAsmOffset" << AltName + << "[RegNo-1];\n"; + } + O << " }\n"; + } else { + O << " assert (*(AsmStrs+RegAsmOffset[RegNo-1]) &&\n" + << " \"Invalid alt name index for register!\");\n" + << " return AsmStrs+RegAsmOffset[RegNo-1];\n"; + } + O << "}\n"; +} + +namespace { + +// IAPrinter - Holds information about an InstAlias. Two InstAliases match if +// they both have the same conditionals. In which case, we cannot print out the +// alias for that pattern. +class IAPrinter { + std::map<StringRef, std::pair<int, int>> OpMap; + + std::vector<std::string> Conds; + + std::string Result; + std::string AsmString; + + unsigned NumMIOps; + +public: + IAPrinter(std::string R, std::string AS, unsigned NumMIOps) + : Result(std::move(R)), AsmString(std::move(AS)), NumMIOps(NumMIOps) {} + + void addCond(std::string C) { Conds.push_back(std::move(C)); } + ArrayRef<std::string> getConds() const { return Conds; } + size_t getCondCount() const { return Conds.size(); } + + void addOperand(StringRef Op, int OpIdx, int PrintMethodIdx = -1) { + assert(OpIdx >= 0 && OpIdx < 0xFE && "Idx out of range"); + assert(PrintMethodIdx >= -1 && PrintMethodIdx < 0xFF && + "Idx out of range"); + OpMap[Op] = std::make_pair(OpIdx, PrintMethodIdx); + } + + unsigned getNumMIOps() { return NumMIOps; } + + StringRef getResult() { return Result; } + + bool isOpMapped(StringRef Op) { return OpMap.find(Op) != OpMap.end(); } + int getOpIndex(StringRef Op) { return OpMap[Op].first; } + std::pair<int, int> &getOpData(StringRef Op) { return OpMap[Op]; } + + std::pair<StringRef, StringRef::iterator> parseName(StringRef::iterator Start, + StringRef::iterator End) { + StringRef::iterator I = Start; + StringRef::iterator Next; + if (*I == '{') { + // ${some_name} + Start = ++I; + while (I != End && *I != '}') + ++I; + Next = I; + // eat the final '}' + if (Next != End) + ++Next; + } else { + // $name, just eat the usual suspects. + while (I != End && + ((*I >= 'a' && *I <= 'z') || (*I >= 'A' && *I <= 'Z') || + (*I >= '0' && *I <= '9') || *I == '_')) + ++I; + Next = I; + } + + return std::make_pair(StringRef(Start, I - Start), Next); + } + + std::string formatAliasString(uint32_t &UnescapedSize) { + // Directly mangle mapped operands into the string. Each operand is + // identified by a '$' sign followed by a byte identifying the number of the + // operand. We add one to the index to avoid zero bytes. + StringRef ASM(AsmString); + std::string OutString; + raw_string_ostream OS(OutString); + for (StringRef::iterator I = ASM.begin(), E = ASM.end(); I != E;) { + OS << *I; + ++UnescapedSize; + if (*I == '$') { + StringRef Name; + std::tie(Name, I) = parseName(++I, E); + assert(isOpMapped(Name) && "Unmapped operand!"); + + int OpIndex, PrintIndex; + std::tie(OpIndex, PrintIndex) = getOpData(Name); + if (PrintIndex == -1) { + // Can use the default printOperand route. + OS << format("\\x%02X", (unsigned char)OpIndex + 1); + ++UnescapedSize; + } else { + // 3 bytes if a PrintMethod is needed: 0xFF, the MCInst operand + // number, and which of our pre-detected Methods to call. + OS << format("\\xFF\\x%02X\\x%02X", OpIndex + 1, PrintIndex + 1); + UnescapedSize += 3; + } + } else { + ++I; + } + } + + OS.flush(); + return OutString; + } + + bool operator==(const IAPrinter &RHS) const { + if (NumMIOps != RHS.NumMIOps) + return false; + if (Conds.size() != RHS.Conds.size()) + return false; + + unsigned Idx = 0; + for (const auto &str : Conds) + if (str != RHS.Conds[Idx++]) + return false; + + return true; + } +}; + +} // end anonymous namespace + +static unsigned CountNumOperands(StringRef AsmString, unsigned Variant) { + return AsmString.count(' ') + AsmString.count('\t'); +} + +namespace { + +struct AliasPriorityComparator { + typedef std::pair<CodeGenInstAlias, int> ValueType; + bool operator()(const ValueType &LHS, const ValueType &RHS) const { + if (LHS.second == RHS.second) { + // We don't actually care about the order, but for consistency it + // shouldn't depend on pointer comparisons. + return LessRecordByID()(LHS.first.TheDef, RHS.first.TheDef); + } + + // Aliases with larger priorities should be considered first. + return LHS.second > RHS.second; + } +}; + +} // end anonymous namespace + +void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) { + Record *AsmWriter = Target.getAsmWriter(); + + O << "\n#ifdef PRINT_ALIAS_INSTR\n"; + O << "#undef PRINT_ALIAS_INSTR\n\n"; + + ////////////////////////////// + // Gather information about aliases we need to print + ////////////////////////////// + + // Emit the method that prints the alias instruction. + StringRef ClassName = AsmWriter->getValueAsString("AsmWriterClassName"); + unsigned Variant = AsmWriter->getValueAsInt("Variant"); + bool PassSubtarget = AsmWriter->getValueAsInt("PassSubtarget"); + + std::vector<Record*> AllInstAliases = + Records.getAllDerivedDefinitions("InstAlias"); + + // Create a map from the qualified name to a list of potential matches. + typedef std::set<std::pair<CodeGenInstAlias, int>, AliasPriorityComparator> + AliasWithPriority; + std::map<std::string, AliasWithPriority> AliasMap; + for (Record *R : AllInstAliases) { + int Priority = R->getValueAsInt("EmitPriority"); + if (Priority < 1) + continue; // Aliases with priority 0 are never emitted. + + const DagInit *DI = R->getValueAsDag("ResultInst"); + AliasMap[getQualifiedName(DI->getOperatorAsDef(R->getLoc()))].insert( + std::make_pair(CodeGenInstAlias(R, Target), Priority)); + } + + // A map of which conditions need to be met for each instruction operand + // before it can be matched to the mnemonic. + std::map<std::string, std::vector<IAPrinter>> IAPrinterMap; + + std::vector<std::pair<std::string, bool>> PrintMethods; + + // A list of MCOperandPredicates for all operands in use, and the reverse map + std::vector<const Record*> MCOpPredicates; + DenseMap<const Record*, unsigned> MCOpPredicateMap; + + for (auto &Aliases : AliasMap) { + // Collection of instruction alias rules. May contain ambiguous rules. + std::vector<IAPrinter> IAPs; + + for (auto &Alias : Aliases.second) { + const CodeGenInstAlias &CGA = Alias.first; + unsigned LastOpNo = CGA.ResultInstOperandIndex.size(); + std::string FlatInstAsmString = + CodeGenInstruction::FlattenAsmStringVariants(CGA.ResultInst->AsmString, + Variant); + unsigned NumResultOps = CountNumOperands(FlatInstAsmString, Variant); + + std::string FlatAliasAsmString = + CodeGenInstruction::FlattenAsmStringVariants(CGA.AsmString, Variant); + UnescapeAliasString(FlatAliasAsmString); + + // Don't emit the alias if it has more operands than what it's aliasing. + if (NumResultOps < CountNumOperands(FlatAliasAsmString, Variant)) + continue; + + StringRef Namespace = Target.getName(); + unsigned NumMIOps = 0; + for (auto &ResultInstOpnd : CGA.ResultInst->Operands) + NumMIOps += ResultInstOpnd.MINumOperands; + + IAPrinter IAP(CGA.Result->getAsString(), FlatAliasAsmString, NumMIOps); + + bool CantHandle = false; + + unsigned MIOpNum = 0; + for (unsigned i = 0, e = LastOpNo; i != e; ++i) { + // Skip over tied operands as they're not part of an alias declaration. + auto &Operands = CGA.ResultInst->Operands; + while (true) { + unsigned OpNum = Operands.getSubOperandNumber(MIOpNum).first; + if (Operands[OpNum].MINumOperands == 1 && + Operands[OpNum].getTiedRegister() != -1) { + // Tied operands of different RegisterClass should be explicit within + // an instruction's syntax and so cannot be skipped. + int TiedOpNum = Operands[OpNum].getTiedRegister(); + if (Operands[OpNum].Rec->getName() == + Operands[TiedOpNum].Rec->getName()) { + ++MIOpNum; + continue; + } + } + break; + } + + // Ignore unchecked result operands. + while (IAP.getCondCount() < MIOpNum) + IAP.addCond("AliasPatternCond::K_Ignore, 0"); + + const CodeGenInstAlias::ResultOperand &RO = CGA.ResultOperands[i]; + + switch (RO.Kind) { + case CodeGenInstAlias::ResultOperand::K_Record: { + const Record *Rec = RO.getRecord(); + StringRef ROName = RO.getName(); + int PrintMethodIdx = -1; + + // These two may have a PrintMethod, which we want to record (if it's + // the first time we've seen it) and provide an index for the aliasing + // code to use. + if (Rec->isSubClassOf("RegisterOperand") || + Rec->isSubClassOf("Operand")) { + StringRef PrintMethod = Rec->getValueAsString("PrintMethod"); + bool IsPCRel = + Rec->getValueAsString("OperandType") == "OPERAND_PCREL"; + if (PrintMethod != "" && PrintMethod != "printOperand") { + PrintMethodIdx = llvm::find_if(PrintMethods, + [&](auto &X) { + return X.first == PrintMethod; + }) - + PrintMethods.begin(); + if (static_cast<unsigned>(PrintMethodIdx) == PrintMethods.size()) + PrintMethods.emplace_back(std::string(PrintMethod), IsPCRel); + } + } + + if (Rec->isSubClassOf("RegisterOperand")) + Rec = Rec->getValueAsDef("RegClass"); + if (Rec->isSubClassOf("RegisterClass")) { + if (!IAP.isOpMapped(ROName)) { + IAP.addOperand(ROName, MIOpNum, PrintMethodIdx); + Record *R = CGA.ResultOperands[i].getRecord(); + if (R->isSubClassOf("RegisterOperand")) + R = R->getValueAsDef("RegClass"); + IAP.addCond(std::string( + formatv("AliasPatternCond::K_RegClass, {0}::{1}RegClassID", + Namespace, R->getName()))); + } else { + IAP.addCond(std::string(formatv( + "AliasPatternCond::K_TiedReg, {0}", IAP.getOpIndex(ROName)))); + } + } else { + // Assume all printable operands are desired for now. This can be + // overridden in the InstAlias instantiation if necessary. + IAP.addOperand(ROName, MIOpNum, PrintMethodIdx); + + // There might be an additional predicate on the MCOperand + unsigned Entry = MCOpPredicateMap[Rec]; + if (!Entry) { + if (!Rec->isValueUnset("MCOperandPredicate")) { + MCOpPredicates.push_back(Rec); + Entry = MCOpPredicates.size(); + MCOpPredicateMap[Rec] = Entry; + } else + break; // No conditions on this operand at all + } + IAP.addCond( + std::string(formatv("AliasPatternCond::K_Custom, {0}", Entry))); + } + break; + } + case CodeGenInstAlias::ResultOperand::K_Imm: { + // Just because the alias has an immediate result, doesn't mean the + // MCInst will. An MCExpr could be present, for example. + auto Imm = CGA.ResultOperands[i].getImm(); + int32_t Imm32 = int32_t(Imm); + if (Imm != Imm32) + PrintFatalError("Matching an alias with an immediate out of the " + "range of int32_t is not supported"); + IAP.addCond(std::string( + formatv("AliasPatternCond::K_Imm, uint32_t({0})", Imm32))); + break; + } + case CodeGenInstAlias::ResultOperand::K_Reg: + // If this is zero_reg, something's playing tricks we're not + // equipped to handle. + if (!CGA.ResultOperands[i].getRegister()) { + CantHandle = true; + break; + } + + StringRef Reg = CGA.ResultOperands[i].getRegister()->getName(); + IAP.addCond(std::string( + formatv("AliasPatternCond::K_Reg, {0}::{1}", Namespace, Reg))); + break; + } + + MIOpNum += RO.getMINumOperands(); + } + + if (CantHandle) continue; + + std::vector<Record *> ReqFeatures; + if (PassSubtarget) { + // We only consider ReqFeatures predicates if PassSubtarget + std::vector<Record *> RF = + CGA.TheDef->getValueAsListOfDefs("Predicates"); + copy_if(RF, std::back_inserter(ReqFeatures), [](Record *R) { + return R->getValueAsBit("AssemblerMatcherPredicate"); + }); + } + + for (auto I = ReqFeatures.cbegin(); I != ReqFeatures.cend(); I++) { + Record *R = *I; + const DagInit *D = R->getValueAsDag("AssemblerCondDag"); + std::string CombineType = D->getOperator()->getAsString(); + if (CombineType != "any_of" && CombineType != "all_of") + PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!"); + if (D->getNumArgs() == 0) + PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!"); + bool IsOr = CombineType == "any_of"; + + for (auto *Arg : D->getArgs()) { + bool IsNeg = false; + if (auto *NotArg = dyn_cast<DagInit>(Arg)) { + if (NotArg->getOperator()->getAsString() != "not" || + NotArg->getNumArgs() != 1) + PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!"); + Arg = NotArg->getArg(0); + IsNeg = true; + } + if (!isa<DefInit>(Arg) || + !cast<DefInit>(Arg)->getDef()->isSubClassOf("SubtargetFeature")) + PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!"); + + IAP.addCond(std::string(formatv( + "AliasPatternCond::K_{0}{1}Feature, {2}::{3}", IsOr ? "Or" : "", + IsNeg ? "Neg" : "", Namespace, Arg->getAsString()))); + } + // If an AssemblerPredicate with ors is used, note end of list should + // these be combined. + if (IsOr) + IAP.addCond("AliasPatternCond::K_EndOrFeatures, 0"); + } + + IAPrinterMap[Aliases.first].push_back(std::move(IAP)); + } + } + + ////////////////////////////// + // Write out the printAliasInstr function + ////////////////////////////// + + std::string Header; + raw_string_ostream HeaderO(Header); + + HeaderO << "bool " << Target.getName() << ClassName + << "::printAliasInstr(const MCInst" + << " *MI, uint64_t Address, " + << (PassSubtarget ? "const MCSubtargetInfo &STI, " : "") + << "raw_ostream &OS) {\n"; + + std::string PatternsForOpcode; + raw_string_ostream OpcodeO(PatternsForOpcode); + + unsigned PatternCount = 0; + std::string Patterns; + raw_string_ostream PatternO(Patterns); + + unsigned CondCount = 0; + std::string Conds; + raw_string_ostream CondO(Conds); + + // All flattened alias strings. + std::map<std::string, uint32_t> AsmStringOffsets; + std::vector<std::pair<uint32_t, std::string>> AsmStrings; + size_t AsmStringsSize = 0; + + // Iterate over the opcodes in enum order so they are sorted by opcode for + // binary search. + for (const CodeGenInstruction *Inst : NumberedInstructions) { + auto It = IAPrinterMap.find(getQualifiedName(Inst->TheDef)); + if (It == IAPrinterMap.end()) + continue; + std::vector<IAPrinter> &IAPs = It->second; + std::vector<IAPrinter*> UniqueIAPs; + + // Remove any ambiguous alias rules. + for (auto &LHS : IAPs) { + bool IsDup = false; + for (const auto &RHS : IAPs) { + if (&LHS != &RHS && LHS == RHS) { + IsDup = true; + break; + } + } + + if (!IsDup) + UniqueIAPs.push_back(&LHS); + } + + if (UniqueIAPs.empty()) continue; + + unsigned PatternStart = PatternCount; + + // Insert the pattern start and opcode in the pattern list for debugging. + PatternO << formatv(" // {0} - {1}\n", It->first, PatternStart); + + for (IAPrinter *IAP : UniqueIAPs) { + // Start each condition list with a comment of the resulting pattern that + // we're trying to match. + unsigned CondStart = CondCount; + CondO << formatv(" // {0} - {1}\n", IAP->getResult(), CondStart); + for (const auto &Cond : IAP->getConds()) + CondO << " {" << Cond << "},\n"; + CondCount += IAP->getCondCount(); + + // After operands have been examined, re-encode the alias string with + // escapes indicating how operands should be printed. + uint32_t UnescapedSize = 0; + std::string EncodedAsmString = IAP->formatAliasString(UnescapedSize); + auto Insertion = + AsmStringOffsets.insert({EncodedAsmString, AsmStringsSize}); + if (Insertion.second) { + // If the string is new, add it to the vector. + AsmStrings.push_back({AsmStringsSize, EncodedAsmString}); + AsmStringsSize += UnescapedSize + 1; + } + unsigned AsmStrOffset = Insertion.first->second; + + PatternO << formatv(" {{{0}, {1}, {2}, {3} },\n", AsmStrOffset, + CondStart, IAP->getNumMIOps(), IAP->getCondCount()); + ++PatternCount; + } + + OpcodeO << formatv(" {{{0}, {1}, {2} },\n", It->first, PatternStart, + PatternCount - PatternStart); + } + + if (OpcodeO.str().empty()) { + O << HeaderO.str(); + O << " return false;\n"; + O << "}\n\n"; + O << "#endif // PRINT_ALIAS_INSTR\n"; + return; + } + + // Forward declare the validation method if needed. + if (!MCOpPredicates.empty()) + O << "static bool " << Target.getName() << ClassName + << "ValidateMCOperand(const MCOperand &MCOp,\n" + << " const MCSubtargetInfo &STI,\n" + << " unsigned PredicateIndex);\n"; + + O << HeaderO.str(); + O.indent(2) << "static const PatternsForOpcode OpToPatterns[] = {\n"; + O << OpcodeO.str(); + O.indent(2) << "};\n\n"; + O.indent(2) << "static const AliasPattern Patterns[] = {\n"; + O << PatternO.str(); + O.indent(2) << "};\n\n"; + O.indent(2) << "static const AliasPatternCond Conds[] = {\n"; + O << CondO.str(); + O.indent(2) << "};\n\n"; + O.indent(2) << "static const char AsmStrings[] =\n"; + for (const auto &P : AsmStrings) { + O.indent(4) << "/* " << P.first << " */ \"" << P.second << "\\0\"\n"; + } + + O.indent(2) << ";\n\n"; + + // Assert that the opcode table is sorted. Use a static local constructor to + // ensure that the check only happens once on first run. + O << "#ifndef NDEBUG\n"; + O.indent(2) << "static struct SortCheck {\n"; + O.indent(2) << " SortCheck(ArrayRef<PatternsForOpcode> OpToPatterns) {\n"; + O.indent(2) << " assert(std::is_sorted(\n"; + O.indent(2) << " OpToPatterns.begin(), OpToPatterns.end(),\n"; + O.indent(2) << " [](const PatternsForOpcode &L, const " + "PatternsForOpcode &R) {\n"; + O.indent(2) << " return L.Opcode < R.Opcode;\n"; + O.indent(2) << " }) &&\n"; + O.indent(2) << " \"tablegen failed to sort opcode patterns\");\n"; + O.indent(2) << " }\n"; + O.indent(2) << "} sortCheckVar(OpToPatterns);\n"; + O << "#endif\n\n"; + + O.indent(2) << "AliasMatchingData M {\n"; + O.indent(2) << " makeArrayRef(OpToPatterns),\n"; + O.indent(2) << " makeArrayRef(Patterns),\n"; + O.indent(2) << " makeArrayRef(Conds),\n"; + O.indent(2) << " StringRef(AsmStrings, array_lengthof(AsmStrings)),\n"; + if (MCOpPredicates.empty()) + O.indent(2) << " nullptr,\n"; + else + O.indent(2) << " &" << Target.getName() << ClassName << "ValidateMCOperand,\n"; + O.indent(2) << "};\n"; + + O.indent(2) << "const char *AsmString = matchAliasPatterns(MI, " + << (PassSubtarget ? "&STI" : "nullptr") << ", M);\n"; + O.indent(2) << "if (!AsmString) return false;\n\n"; + + // Code that prints the alias, replacing the operands with the ones from the + // MCInst. + O << " unsigned I = 0;\n"; + O << " while (AsmString[I] != ' ' && AsmString[I] != '\\t' &&\n"; + O << " AsmString[I] != '$' && AsmString[I] != '\\0')\n"; + O << " ++I;\n"; + O << " OS << '\\t' << StringRef(AsmString, I);\n"; + + O << " if (AsmString[I] != '\\0') {\n"; + O << " if (AsmString[I] == ' ' || AsmString[I] == '\\t') {\n"; + O << " OS << '\\t';\n"; + O << " ++I;\n"; + O << " }\n"; + O << " do {\n"; + O << " if (AsmString[I] == '$') {\n"; + O << " ++I;\n"; + O << " if (AsmString[I] == (char)0xff) {\n"; + O << " ++I;\n"; + O << " int OpIdx = AsmString[I++] - 1;\n"; + O << " int PrintMethodIdx = AsmString[I++] - 1;\n"; + O << " printCustomAliasOperand(MI, Address, OpIdx, PrintMethodIdx, "; + O << (PassSubtarget ? "STI, " : ""); + O << "OS);\n"; + O << " } else\n"; + O << " printOperand(MI, unsigned(AsmString[I++]) - 1, "; + O << (PassSubtarget ? "STI, " : ""); + O << "OS);\n"; + O << " } else {\n"; + O << " OS << AsmString[I++];\n"; + O << " }\n"; + O << " } while (AsmString[I] != '\\0');\n"; + O << " }\n\n"; + + O << " return true;\n"; + O << "}\n\n"; + + ////////////////////////////// + // Write out the printCustomAliasOperand function + ////////////////////////////// + + O << "void " << Target.getName() << ClassName << "::" + << "printCustomAliasOperand(\n" + << " const MCInst *MI, uint64_t Address, unsigned OpIdx,\n" + << " unsigned PrintMethodIdx,\n" + << (PassSubtarget ? " const MCSubtargetInfo &STI,\n" : "") + << " raw_ostream &OS) {\n"; + if (PrintMethods.empty()) + O << " llvm_unreachable(\"Unknown PrintMethod kind\");\n"; + else { + O << " switch (PrintMethodIdx) {\n" + << " default:\n" + << " llvm_unreachable(\"Unknown PrintMethod kind\");\n" + << " break;\n"; + + for (unsigned i = 0; i < PrintMethods.size(); ++i) { + O << " case " << i << ":\n" + << " " << PrintMethods[i].first << "(MI, " + << (PrintMethods[i].second ? "Address, " : "") << "OpIdx, " + << (PassSubtarget ? "STI, " : "") << "OS);\n" + << " break;\n"; + } + O << " }\n"; + } + O << "}\n\n"; + + if (!MCOpPredicates.empty()) { + O << "static bool " << Target.getName() << ClassName + << "ValidateMCOperand(const MCOperand &MCOp,\n" + << " const MCSubtargetInfo &STI,\n" + << " unsigned PredicateIndex) {\n" + << " switch (PredicateIndex) {\n" + << " default:\n" + << " llvm_unreachable(\"Unknown MCOperandPredicate kind\");\n" + << " break;\n"; + + for (unsigned i = 0; i < MCOpPredicates.size(); ++i) { + Init *MCOpPred = MCOpPredicates[i]->getValueInit("MCOperandPredicate"); + if (CodeInit *SI = dyn_cast<CodeInit>(MCOpPred)) { + O << " case " << i + 1 << ": {\n" + << SI->getValue() << "\n" + << " }\n"; + } else + llvm_unreachable("Unexpected MCOperandPredicate field!"); + } + O << " }\n" + << "}\n\n"; + } + + O << "#endif // PRINT_ALIAS_INSTR\n"; +} + +AsmWriterEmitter::AsmWriterEmitter(RecordKeeper &R) : Records(R), Target(R) { + Record *AsmWriter = Target.getAsmWriter(); + unsigned Variant = AsmWriter->getValueAsInt("Variant"); + + // Get the instruction numbering. + NumberedInstructions = Target.getInstructionsByEnumValue(); + + for (unsigned i = 0, e = NumberedInstructions.size(); i != e; ++i) { + const CodeGenInstruction *I = NumberedInstructions[i]; + if (!I->AsmString.empty() && I->TheDef->getName() != "PHI") + Instructions.emplace_back(*I, i, Variant); + } +} + +void AsmWriterEmitter::run(raw_ostream &O) { + EmitPrintInstruction(O); + EmitGetRegisterName(O); + EmitPrintAliasInstruction(O); +} + +namespace llvm { + +void EmitAsmWriter(RecordKeeper &RK, raw_ostream &OS) { + emitSourceFileHeader("Assembly Writer Source Fragment", OS); + AsmWriterEmitter(RK).run(OS); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/AsmWriterInst.cpp b/contrib/llvm-project/llvm/utils/TableGen/AsmWriterInst.cpp new file mode 100644 index 000000000000..24d29ffc28e5 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/AsmWriterInst.cpp @@ -0,0 +1,211 @@ +//===- AsmWriterInst.h - Classes encapsulating a printable inst -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// These classes implement a parser for assembly strings. +// +//===----------------------------------------------------------------------===// + +#include "AsmWriterInst.h" +#include "CodeGenTarget.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" + +using namespace llvm; + +static bool isIdentChar(char C) { + return (C >= 'a' && C <= 'z') || + (C >= 'A' && C <= 'Z') || + (C >= '0' && C <= '9') || + C == '_'; +} + +std::string AsmWriterOperand::getCode(bool PassSubtarget) const { + if (OperandType == isLiteralTextOperand) { + if (Str.size() == 1) + return "O << '" + Str + "';"; + return "O << \"" + Str + "\";"; + } + + if (OperandType == isLiteralStatementOperand) + return Str; + + std::string Result = Str + "(MI"; + if (PCRel) + Result += ", Address"; + if (MIOpNo != ~0U) + Result += ", " + utostr(MIOpNo); + if (PassSubtarget) + Result += ", STI"; + Result += ", O"; + if (!MiModifier.empty()) + Result += ", \"" + MiModifier + '"'; + return Result + ");"; +} + +/// ParseAsmString - Parse the specified Instruction's AsmString into this +/// AsmWriterInst. +/// +AsmWriterInst::AsmWriterInst(const CodeGenInstruction &CGI, unsigned CGIIndex, + unsigned Variant) + : CGI(&CGI), CGIIndex(CGIIndex) { + + // NOTE: Any extensions to this code need to be mirrored in the + // AsmPrinter::printInlineAsm code that executes as compile time (assuming + // that inline asm strings should also get the new feature)! + std::string AsmString = CGI.FlattenAsmStringVariants(CGI.AsmString, Variant); + std::string::size_type LastEmitted = 0; + while (LastEmitted != AsmString.size()) { + std::string::size_type DollarPos = + AsmString.find_first_of("$\\", LastEmitted); + if (DollarPos == std::string::npos) DollarPos = AsmString.size(); + + // Emit a constant string fragment. + if (DollarPos != LastEmitted) { + for (; LastEmitted != DollarPos; ++LastEmitted) + switch (AsmString[LastEmitted]) { + case '\n': + AddLiteralString("\\n"); + break; + case '\t': + AddLiteralString("\\t"); + break; + case '"': + AddLiteralString("\\\""); + break; + case '\\': + AddLiteralString("\\\\"); + break; + default: + AddLiteralString(std::string(1, AsmString[LastEmitted])); + break; + } + } else if (AsmString[DollarPos] == '\\') { + if (DollarPos+1 != AsmString.size()) { + if (AsmString[DollarPos+1] == 'n') { + AddLiteralString("\\n"); + } else if (AsmString[DollarPos+1] == 't') { + AddLiteralString("\\t"); + } else if (std::string("${|}\\").find(AsmString[DollarPos+1]) + != std::string::npos) { + AddLiteralString(std::string(1, AsmString[DollarPos+1])); + } else { + PrintFatalError( + CGI.TheDef->getLoc(), + "Non-supported escaped character found in instruction '" + + CGI.TheDef->getName() + "'!"); + } + LastEmitted = DollarPos+2; + continue; + } + } else if (DollarPos+1 != AsmString.size() && + AsmString[DollarPos+1] == '$') { + AddLiteralString("$"); // "$$" -> $ + LastEmitted = DollarPos+2; + } else { + // Get the name of the variable. + std::string::size_type VarEnd = DollarPos+1; + + // handle ${foo}bar as $foo by detecting whether the character following + // the dollar sign is a curly brace. If so, advance VarEnd and DollarPos + // so the variable name does not contain the leading curly brace. + bool hasCurlyBraces = false; + if (VarEnd < AsmString.size() && '{' == AsmString[VarEnd]) { + hasCurlyBraces = true; + ++DollarPos; + ++VarEnd; + } + + while (VarEnd < AsmString.size() && isIdentChar(AsmString[VarEnd])) + ++VarEnd; + StringRef VarName(AsmString.data()+DollarPos+1, VarEnd-DollarPos-1); + + // Modifier - Support ${foo:modifier} syntax, where "modifier" is passed + // into printOperand. Also support ${:feature}, which is passed into + // PrintSpecial. + std::string Modifier; + + // In order to avoid starting the next string at the terminating curly + // brace, advance the end position past it if we found an opening curly + // brace. + if (hasCurlyBraces) { + if (VarEnd >= AsmString.size()) + PrintFatalError( + CGI.TheDef->getLoc(), + "Reached end of string before terminating curly brace in '" + + CGI.TheDef->getName() + "'"); + + // Look for a modifier string. + if (AsmString[VarEnd] == ':') { + ++VarEnd; + if (VarEnd >= AsmString.size()) + PrintFatalError( + CGI.TheDef->getLoc(), + "Reached end of string before terminating curly brace in '" + + CGI.TheDef->getName() + "'"); + + std::string::size_type ModifierStart = VarEnd; + while (VarEnd < AsmString.size() && isIdentChar(AsmString[VarEnd])) + ++VarEnd; + Modifier = std::string(AsmString.begin()+ModifierStart, + AsmString.begin()+VarEnd); + if (Modifier.empty()) + PrintFatalError(CGI.TheDef->getLoc(), + "Bad operand modifier name in '" + + CGI.TheDef->getName() + "'"); + } + + if (AsmString[VarEnd] != '}') + PrintFatalError( + CGI.TheDef->getLoc(), + "Variable name beginning with '{' did not end with '}' in '" + + CGI.TheDef->getName() + "'"); + ++VarEnd; + } + if (VarName.empty() && Modifier.empty()) + PrintFatalError(CGI.TheDef->getLoc(), + "Stray '$' in '" + CGI.TheDef->getName() + + "' asm string, maybe you want $$?"); + + if (VarName.empty()) { + // Just a modifier, pass this into PrintSpecial. + Operands.emplace_back("PrintSpecial", ~0U, Modifier); + } else { + // Otherwise, normal operand. + unsigned OpNo = CGI.Operands.getOperandNamed(VarName); + CGIOperandList::OperandInfo OpInfo = CGI.Operands[OpNo]; + + unsigned MIOp = OpInfo.MIOperandNo; + Operands.emplace_back(OpInfo.PrinterMethodName, MIOp, Modifier, + AsmWriterOperand::isMachineInstrOperand, + OpInfo.OperandType == "MCOI::OPERAND_PCREL"); + } + LastEmitted = VarEnd; + } + } + + Operands.emplace_back("return;", AsmWriterOperand::isLiteralStatementOperand); +} + +/// MatchesAllButOneOp - If this instruction is exactly identical to the +/// specified instruction except for one differing operand, return the differing +/// operand number. If more than one operand mismatches, return ~1, otherwise +/// if the instructions are identical return ~0. +unsigned AsmWriterInst::MatchesAllButOneOp(const AsmWriterInst &Other)const{ + if (Operands.size() != Other.Operands.size()) return ~1; + + unsigned MismatchOperand = ~0U; + for (unsigned i = 0, e = Operands.size(); i != e; ++i) { + if (Operands[i] != Other.Operands[i]) { + if (MismatchOperand != ~0U) // Already have one mismatch? + return ~1U; + MismatchOperand = i; + } + } + return MismatchOperand; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/AsmWriterInst.h b/contrib/llvm-project/llvm/utils/TableGen/AsmWriterInst.h new file mode 100644 index 000000000000..366c9eca664f --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/AsmWriterInst.h @@ -0,0 +1,107 @@ +//===- AsmWriterInst.h - Classes encapsulating a printable inst -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// These classes implement a parser for assembly strings. The parser splits +// the string into operands, which can be literal strings (the constant bits of +// the string), actual operands (i.e., operands from the MachineInstr), and +// dynamically-generated text, specified by raw C++ code. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_ASMWRITERINST_H +#define LLVM_UTILS_TABLEGEN_ASMWRITERINST_H + +#include <string> +#include <vector> + +namespace llvm { + class CodeGenInstruction; + class Record; + + struct AsmWriterOperand { + enum OpType { + // Output this text surrounded by quotes to the asm. + isLiteralTextOperand, + // This is the name of a routine to call to print the operand. + isMachineInstrOperand, + // Output this text verbatim to the asm writer. It is code that + // will output some text to the asm. + isLiteralStatementOperand + } OperandType; + + /// MiOpNo - For isMachineInstrOperand, this is the operand number of the + /// machine instruction. + unsigned MIOpNo = 0; + + /// Str - For isLiteralTextOperand, this IS the literal text. For + /// isMachineInstrOperand, this is the PrinterMethodName for the operand.. + /// For isLiteralStatementOperand, this is the code to insert verbatim + /// into the asm writer. + std::string Str; + + /// MiModifier - For isMachineInstrOperand, this is the modifier string for + /// an operand, specified with syntax like ${opname:modifier}. + std::string MiModifier; + + bool PCRel = false; + + // To make VS STL happy + AsmWriterOperand(OpType op = isLiteralTextOperand):OperandType(op) {} + + AsmWriterOperand(const std::string &LitStr, + OpType op = isLiteralTextOperand) + : OperandType(op), Str(LitStr) {} + + AsmWriterOperand(const std::string &Printer, unsigned _MIOpNo, + const std::string &Modifier, + OpType op = isMachineInstrOperand, bool PCRel = false) + : OperandType(op), MIOpNo(_MIOpNo), Str(Printer), MiModifier(Modifier), + PCRel(PCRel) {} + + bool operator!=(const AsmWriterOperand &Other) const { + if (OperandType != Other.OperandType || Str != Other.Str) return true; + if (OperandType == isMachineInstrOperand) + return MIOpNo != Other.MIOpNo || MiModifier != Other.MiModifier; + return false; + } + bool operator==(const AsmWriterOperand &Other) const { + return !operator!=(Other); + } + + /// getCode - Return the code that prints this operand. + std::string getCode(bool PassSubtarget) const; + }; + + class AsmWriterInst { + public: + std::vector<AsmWriterOperand> Operands; + const CodeGenInstruction *CGI; + unsigned CGIIndex; + + AsmWriterInst(const CodeGenInstruction &CGI, unsigned CGIIndex, + unsigned Variant); + + /// MatchesAllButOneOp - If this instruction is exactly identical to the + /// specified instruction except for one differing operand, return the + /// differing operand number. Otherwise return ~0. + unsigned MatchesAllButOneOp(const AsmWriterInst &Other) const; + + private: + void AddLiteralString(const std::string &Str) { + // If the last operand was already a literal text string, append this to + // it, otherwise add a new operand. + if (!Operands.empty() && + Operands.back().OperandType == AsmWriterOperand::isLiteralTextOperand) + Operands.back().Str.append(Str); + else + Operands.push_back(AsmWriterOperand(Str)); + } + }; +} + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/Attributes.cpp b/contrib/llvm-project/llvm/utils/TableGen/Attributes.cpp new file mode 100644 index 000000000000..f3f875e8ce0b --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/Attributes.cpp @@ -0,0 +1,110 @@ +//===- Attributes.cpp - Generate attributes -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/TableGen/Record.h" +#include <algorithm> +#include <string> +#include <vector> +using namespace llvm; + +#define DEBUG_TYPE "attr-enum" + +namespace { + +class Attributes { +public: + Attributes(RecordKeeper &R) : Records(R) {} + void emit(raw_ostream &OS); + +private: + void emitTargetIndependentNames(raw_ostream &OS); + void emitFnAttrCompatCheck(raw_ostream &OS, bool IsStringAttr); + + RecordKeeper &Records; +}; + +} // End anonymous namespace. + +void Attributes::emitTargetIndependentNames(raw_ostream &OS) { + OS << "#ifdef GET_ATTR_NAMES\n"; + OS << "#undef GET_ATTR_NAMES\n"; + + OS << "#ifndef ATTRIBUTE_ALL\n"; + OS << "#define ATTRIBUTE_ALL(FIRST, SECOND)\n"; + OS << "#endif\n\n"; + + auto Emit = [&](ArrayRef<StringRef> KindNames, StringRef MacroName) { + OS << "#ifndef " << MacroName << "\n"; + OS << "#define " << MacroName + << "(FIRST, SECOND) ATTRIBUTE_ALL(FIRST, SECOND)\n"; + OS << "#endif\n\n"; + for (StringRef KindName : KindNames) { + for (auto A : Records.getAllDerivedDefinitions(KindName)) { + OS << MacroName << "(" << A->getName() << "," + << A->getValueAsString("AttrString") << ")\n"; + } + } + OS << "#undef " << MacroName << "\n\n"; + }; + + // Emit attribute enums in the same order llvm::Attribute::operator< expects. + Emit({"EnumAttr", "TypeAttr", "IntAttr"}, "ATTRIBUTE_ENUM"); + Emit({"StrBoolAttr"}, "ATTRIBUTE_STRBOOL"); + + OS << "#undef ATTRIBUTE_ALL\n"; + OS << "#endif\n"; +} + +void Attributes::emitFnAttrCompatCheck(raw_ostream &OS, bool IsStringAttr) { + OS << "#ifdef GET_ATTR_COMPAT_FUNC\n"; + OS << "#undef GET_ATTR_COMPAT_FUNC\n"; + + OS << "static inline bool hasCompatibleFnAttrs(const Function &Caller,\n" + << " const Function &Callee) {\n"; + OS << " bool Ret = true;\n\n"; + + std::vector<Record *> CompatRules = + Records.getAllDerivedDefinitions("CompatRule"); + + for (auto *Rule : CompatRules) { + StringRef FuncName = Rule->getValueAsString("CompatFunc"); + OS << " Ret &= " << FuncName << "(Caller, Callee);\n"; + } + + OS << "\n"; + OS << " return Ret;\n"; + OS << "}\n\n"; + + std::vector<Record *> MergeRules = + Records.getAllDerivedDefinitions("MergeRule"); + OS << "static inline void mergeFnAttrs(Function &Caller,\n" + << " const Function &Callee) {\n"; + + for (auto *Rule : MergeRules) { + StringRef FuncName = Rule->getValueAsString("MergeFunc"); + OS << " " << FuncName << "(Caller, Callee);\n"; + } + + OS << "}\n\n"; + + OS << "#endif\n"; +} + +void Attributes::emit(raw_ostream &OS) { + emitTargetIndependentNames(OS); + emitFnAttrCompatCheck(OS, false); +} + +namespace llvm { + +void EmitAttributes(RecordKeeper &RK, raw_ostream &OS) { + Attributes(RK).emit(OS); +} + +} // End llvm namespace. diff --git a/contrib/llvm-project/llvm/utils/TableGen/CTagsEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/CTagsEmitter.cpp new file mode 100644 index 000000000000..ccb7f3300dde --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CTagsEmitter.cpp @@ -0,0 +1,86 @@ +//===- CTagsEmitter.cpp - Generate ctags-compatible index ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend emits an index of definitions in ctags(1) format. +// A helper script, utils/TableGen/tdtags, provides an easier-to-use +// interface; run 'tdtags -H' for documentation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include <algorithm> +#include <string> +#include <vector> +using namespace llvm; + +#define DEBUG_TYPE "ctags-emitter" + +namespace { + +class Tag { +private: + const std::string *Id; + SMLoc Loc; +public: + Tag(const std::string &Name, const SMLoc Location) + : Id(&Name), Loc(Location) {} + int operator<(const Tag &B) const { return *Id < *B.Id; } + void emit(raw_ostream &OS) const { + const MemoryBuffer *CurMB = + SrcMgr.getMemoryBuffer(SrcMgr.FindBufferContainingLoc(Loc)); + auto BufferName = CurMB->getBufferIdentifier(); + std::pair<unsigned, unsigned> LineAndColumn = SrcMgr.getLineAndColumn(Loc); + OS << *Id << "\t" << BufferName << "\t" << LineAndColumn.first << "\n"; + } +}; + +class CTagsEmitter { +private: + RecordKeeper &Records; +public: + CTagsEmitter(RecordKeeper &R) : Records(R) {} + + void run(raw_ostream &OS); + +private: + static SMLoc locate(const Record *R); +}; + +} // End anonymous namespace. + +SMLoc CTagsEmitter::locate(const Record *R) { + ArrayRef<SMLoc> Locs = R->getLoc(); + return !Locs.empty() ? Locs.front() : SMLoc(); +} + +void CTagsEmitter::run(raw_ostream &OS) { + const auto &Classes = Records.getClasses(); + const auto &Defs = Records.getDefs(); + std::vector<Tag> Tags; + // Collect tags. + Tags.reserve(Classes.size() + Defs.size()); + for (const auto &C : Classes) + Tags.push_back(Tag(C.first, locate(C.second.get()))); + for (const auto &D : Defs) + Tags.push_back(Tag(D.first, locate(D.second.get()))); + // Emit tags. + llvm::sort(Tags); + OS << "!_TAG_FILE_FORMAT\t1\t/original ctags format/\n"; + OS << "!_TAG_FILE_SORTED\t1\t/0=unsorted, 1=sorted, 2=foldcase/\n"; + for (const Tag &T : Tags) + T.emit(OS); +} + +namespace llvm { + +void EmitCTags(RecordKeeper &RK, raw_ostream &OS) { CTagsEmitter(RK).run(OS); } + +} // End llvm namespace. diff --git a/contrib/llvm-project/llvm/utils/TableGen/CallingConvEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/CallingConvEmitter.cpp new file mode 100644 index 000000000000..a4e993f80ec9 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CallingConvEmitter.cpp @@ -0,0 +1,300 @@ +//===- CallingConvEmitter.cpp - Generate calling conventions --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend is responsible for emitting descriptions of the calling +// conventions supported by this target. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cassert> +using namespace llvm; + +namespace { +class CallingConvEmitter { + RecordKeeper &Records; +public: + explicit CallingConvEmitter(RecordKeeper &R) : Records(R) {} + + void run(raw_ostream &o); + +private: + void EmitCallingConv(Record *CC, raw_ostream &O); + void EmitAction(Record *Action, unsigned Indent, raw_ostream &O); + unsigned Counter; +}; +} // End anonymous namespace + +void CallingConvEmitter::run(raw_ostream &O) { + std::vector<Record*> CCs = Records.getAllDerivedDefinitions("CallingConv"); + + // Emit prototypes for all of the non-custom CC's so that they can forward ref + // each other. + for (Record *CC : CCs) { + if (!CC->getValueAsBit("Custom")) { + unsigned Pad = CC->getName().size(); + if (CC->getValueAsBit("Entry")) { + O << "bool llvm::"; + Pad += 12; + } else { + O << "static bool "; + Pad += 13; + } + O << CC->getName() << "(unsigned ValNo, MVT ValVT,\n" + << std::string(Pad, ' ') << "MVT LocVT, CCValAssign::LocInfo LocInfo,\n" + << std::string(Pad, ' ') + << "ISD::ArgFlagsTy ArgFlags, CCState &State);\n"; + } + } + + // Emit each non-custom calling convention description in full. + for (Record *CC : CCs) { + if (!CC->getValueAsBit("Custom")) + EmitCallingConv(CC, O); + } +} + + +void CallingConvEmitter::EmitCallingConv(Record *CC, raw_ostream &O) { + ListInit *CCActions = CC->getValueAsListInit("Actions"); + Counter = 0; + + O << "\n\n"; + unsigned Pad = CC->getName().size(); + if (CC->getValueAsBit("Entry")) { + O << "bool llvm::"; + Pad += 12; + } else { + O << "static bool "; + Pad += 13; + } + O << CC->getName() << "(unsigned ValNo, MVT ValVT,\n" + << std::string(Pad, ' ') << "MVT LocVT, CCValAssign::LocInfo LocInfo,\n" + << std::string(Pad, ' ') << "ISD::ArgFlagsTy ArgFlags, CCState &State) {\n"; + // Emit all of the actions, in order. + for (unsigned i = 0, e = CCActions->size(); i != e; ++i) { + O << "\n"; + EmitAction(CCActions->getElementAsRecord(i), 2, O); + } + + O << "\n return true; // CC didn't match.\n"; + O << "}\n"; +} + +void CallingConvEmitter::EmitAction(Record *Action, + unsigned Indent, raw_ostream &O) { + std::string IndentStr = std::string(Indent, ' '); + + if (Action->isSubClassOf("CCPredicateAction")) { + O << IndentStr << "if ("; + + if (Action->isSubClassOf("CCIfType")) { + ListInit *VTs = Action->getValueAsListInit("VTs"); + for (unsigned i = 0, e = VTs->size(); i != e; ++i) { + Record *VT = VTs->getElementAsRecord(i); + if (i != 0) O << " ||\n " << IndentStr; + O << "LocVT == " << getEnumName(getValueType(VT)); + } + + } else if (Action->isSubClassOf("CCIf")) { + O << Action->getValueAsString("Predicate"); + } else { + errs() << *Action; + PrintFatalError(Action->getLoc(), "Unknown CCPredicateAction!"); + } + + O << ") {\n"; + EmitAction(Action->getValueAsDef("SubAction"), Indent+2, O); + O << IndentStr << "}\n"; + } else { + if (Action->isSubClassOf("CCDelegateTo")) { + Record *CC = Action->getValueAsDef("CC"); + O << IndentStr << "if (!" << CC->getName() + << "(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State))\n" + << IndentStr << " return false;\n"; + } else if (Action->isSubClassOf("CCAssignToReg")) { + ListInit *RegList = Action->getValueAsListInit("RegList"); + if (RegList->size() == 1) { + O << IndentStr << "if (unsigned Reg = State.AllocateReg("; + O << getQualifiedName(RegList->getElementAsRecord(0)) << ")) {\n"; + } else { + O << IndentStr << "static const MCPhysReg RegList" << ++Counter + << "[] = {\n"; + O << IndentStr << " "; + for (unsigned i = 0, e = RegList->size(); i != e; ++i) { + if (i != 0) O << ", "; + O << getQualifiedName(RegList->getElementAsRecord(i)); + } + O << "\n" << IndentStr << "};\n"; + O << IndentStr << "if (unsigned Reg = State.AllocateReg(RegList" + << Counter << ")) {\n"; + } + O << IndentStr << " State.addLoc(CCValAssign::getReg(ValNo, ValVT, " + << "Reg, LocVT, LocInfo));\n"; + O << IndentStr << " return false;\n"; + O << IndentStr << "}\n"; + } else if (Action->isSubClassOf("CCAssignToRegWithShadow")) { + ListInit *RegList = Action->getValueAsListInit("RegList"); + ListInit *ShadowRegList = Action->getValueAsListInit("ShadowRegList"); + if (!ShadowRegList->empty() && ShadowRegList->size() != RegList->size()) + PrintFatalError(Action->getLoc(), + "Invalid length of list of shadowed registers"); + + if (RegList->size() == 1) { + O << IndentStr << "if (unsigned Reg = State.AllocateReg("; + O << getQualifiedName(RegList->getElementAsRecord(0)); + O << ", " << getQualifiedName(ShadowRegList->getElementAsRecord(0)); + O << ")) {\n"; + } else { + unsigned RegListNumber = ++Counter; + unsigned ShadowRegListNumber = ++Counter; + + O << IndentStr << "static const MCPhysReg RegList" << RegListNumber + << "[] = {\n"; + O << IndentStr << " "; + for (unsigned i = 0, e = RegList->size(); i != e; ++i) { + if (i != 0) O << ", "; + O << getQualifiedName(RegList->getElementAsRecord(i)); + } + O << "\n" << IndentStr << "};\n"; + + O << IndentStr << "static const MCPhysReg RegList" + << ShadowRegListNumber << "[] = {\n"; + O << IndentStr << " "; + for (unsigned i = 0, e = ShadowRegList->size(); i != e; ++i) { + if (i != 0) O << ", "; + O << getQualifiedName(ShadowRegList->getElementAsRecord(i)); + } + O << "\n" << IndentStr << "};\n"; + + O << IndentStr << "if (unsigned Reg = State.AllocateReg(RegList" + << RegListNumber << ", " << "RegList" << ShadowRegListNumber + << ")) {\n"; + } + O << IndentStr << " State.addLoc(CCValAssign::getReg(ValNo, ValVT, " + << "Reg, LocVT, LocInfo));\n"; + O << IndentStr << " return false;\n"; + O << IndentStr << "}\n"; + } else if (Action->isSubClassOf("CCAssignToStack")) { + int Size = Action->getValueAsInt("Size"); + int Align = Action->getValueAsInt("Align"); + + O << IndentStr << "unsigned Offset" << ++Counter + << " = State.AllocateStack("; + if (Size) + O << Size << ", "; + else + O << "\n" << IndentStr + << " State.getMachineFunction().getDataLayout()." + "getTypeAllocSize(EVT(LocVT).getTypeForEVT(State.getContext()))," + " "; + if (Align) + O << "Align(" << Align << ")"; + else + O << "\n" + << IndentStr + << " State.getMachineFunction().getDataLayout()." + "getABITypeAlign(EVT(LocVT).getTypeForEVT(State.getContext()" + "))"; + O << ");\n" << IndentStr + << "State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset" + << Counter << ", LocVT, LocInfo));\n"; + O << IndentStr << "return false;\n"; + } else if (Action->isSubClassOf("CCAssignToStackWithShadow")) { + int Size = Action->getValueAsInt("Size"); + int Align = Action->getValueAsInt("Align"); + ListInit *ShadowRegList = Action->getValueAsListInit("ShadowRegList"); + + unsigned ShadowRegListNumber = ++Counter; + + O << IndentStr << "static const MCPhysReg ShadowRegList" + << ShadowRegListNumber << "[] = {\n"; + O << IndentStr << " "; + for (unsigned i = 0, e = ShadowRegList->size(); i != e; ++i) { + if (i != 0) O << ", "; + O << getQualifiedName(ShadowRegList->getElementAsRecord(i)); + } + O << "\n" << IndentStr << "};\n"; + + O << IndentStr << "unsigned Offset" << ++Counter + << " = State.AllocateStack(" << Size << ", Align(" << Align << "), " + << "ShadowRegList" << ShadowRegListNumber << ");\n"; + O << IndentStr << "State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset" + << Counter << ", LocVT, LocInfo));\n"; + O << IndentStr << "return false;\n"; + } else if (Action->isSubClassOf("CCPromoteToType")) { + Record *DestTy = Action->getValueAsDef("DestTy"); + MVT::SimpleValueType DestVT = getValueType(DestTy); + O << IndentStr << "LocVT = " << getEnumName(DestVT) <<";\n"; + if (MVT(DestVT).isFloatingPoint()) { + O << IndentStr << "LocInfo = CCValAssign::FPExt;\n"; + } else { + O << IndentStr << "if (ArgFlags.isSExt())\n" + << IndentStr << IndentStr << "LocInfo = CCValAssign::SExt;\n" + << IndentStr << "else if (ArgFlags.isZExt())\n" + << IndentStr << IndentStr << "LocInfo = CCValAssign::ZExt;\n" + << IndentStr << "else\n" + << IndentStr << IndentStr << "LocInfo = CCValAssign::AExt;\n"; + } + } else if (Action->isSubClassOf("CCPromoteToUpperBitsInType")) { + Record *DestTy = Action->getValueAsDef("DestTy"); + MVT::SimpleValueType DestVT = getValueType(DestTy); + O << IndentStr << "LocVT = " << getEnumName(DestVT) << ";\n"; + if (MVT(DestVT).isFloatingPoint()) { + PrintFatalError(Action->getLoc(), + "CCPromoteToUpperBitsInType does not handle floating " + "point"); + } else { + O << IndentStr << "if (ArgFlags.isSExt())\n" + << IndentStr << IndentStr << "LocInfo = CCValAssign::SExtUpper;\n" + << IndentStr << "else if (ArgFlags.isZExt())\n" + << IndentStr << IndentStr << "LocInfo = CCValAssign::ZExtUpper;\n" + << IndentStr << "else\n" + << IndentStr << IndentStr << "LocInfo = CCValAssign::AExtUpper;\n"; + } + } else if (Action->isSubClassOf("CCBitConvertToType")) { + Record *DestTy = Action->getValueAsDef("DestTy"); + O << IndentStr << "LocVT = " << getEnumName(getValueType(DestTy)) <<";\n"; + O << IndentStr << "LocInfo = CCValAssign::BCvt;\n"; + } else if (Action->isSubClassOf("CCTruncToType")) { + Record *DestTy = Action->getValueAsDef("DestTy"); + O << IndentStr << "LocVT = " << getEnumName(getValueType(DestTy)) <<";\n"; + O << IndentStr << "LocInfo = CCValAssign::Trunc;\n"; + } else if (Action->isSubClassOf("CCPassIndirect")) { + Record *DestTy = Action->getValueAsDef("DestTy"); + O << IndentStr << "LocVT = " << getEnumName(getValueType(DestTy)) <<";\n"; + O << IndentStr << "LocInfo = CCValAssign::Indirect;\n"; + } else if (Action->isSubClassOf("CCPassByVal")) { + int Size = Action->getValueAsInt("Size"); + int Align = Action->getValueAsInt("Align"); + O << IndentStr << "State.HandleByVal(ValNo, ValVT, LocVT, LocInfo, " + << Size << ", Align(" << Align << "), ArgFlags);\n"; + O << IndentStr << "return false;\n"; + } else if (Action->isSubClassOf("CCCustom")) { + O << IndentStr + << "if (" << Action->getValueAsString("FuncName") << "(ValNo, ValVT, " + << "LocVT, LocInfo, ArgFlags, State))\n"; + O << IndentStr << IndentStr << "return false;\n"; + } else { + errs() << *Action; + PrintFatalError(Action->getLoc(), "Unknown CCAction!"); + } + } +} + +namespace llvm { + +void EmitCallingConv(RecordKeeper &RK, raw_ostream &OS) { + emitSourceFileHeader("Calling Convention Implementation Fragment", OS); + CallingConvEmitter(RK).run(OS); +} + +} // End llvm namespace diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeEmitterGen.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeEmitterGen.cpp new file mode 100644 index 000000000000..6338d44fb2a7 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeEmitterGen.cpp @@ -0,0 +1,663 @@ +//===- CodeEmitterGen.cpp - Code Emitter Generator ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// CodeEmitterGen uses the descriptions of instructions and their fields to +// construct an automated code emitter: a function that, given a MachineInstr, +// returns the (currently, 32-bit unsigned) value of the instruction. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenInstruction.h" +#include "CodeGenTarget.h" +#include "SubtargetFeatureInfo.h" +#include "Types.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cassert> +#include <cstdint> +#include <map> +#include <set> +#include <string> +#include <utility> +#include <vector> + +using namespace llvm; + +namespace { + +class CodeEmitterGen { + RecordKeeper &Records; + +public: + CodeEmitterGen(RecordKeeper &R) : Records(R) {} + + void run(raw_ostream &o); + +private: + int getVariableBit(const std::string &VarName, BitsInit *BI, int bit); + std::string getInstructionCase(Record *R, CodeGenTarget &Target); + std::string getInstructionCaseForEncoding(Record *R, Record *EncodingDef, + CodeGenTarget &Target); + void AddCodeToMergeInOperand(Record *R, BitsInit *BI, + const std::string &VarName, + unsigned &NumberedOp, + std::set<unsigned> &NamedOpIndices, + std::string &Case, CodeGenTarget &Target); + + void emitInstructionBaseValues( + raw_ostream &o, ArrayRef<const CodeGenInstruction *> NumberedInstructions, + CodeGenTarget &Target, int HwMode = -1); + unsigned BitWidth; + bool UseAPInt; +}; + +// If the VarBitInit at position 'bit' matches the specified variable then +// return the variable bit position. Otherwise return -1. +int CodeEmitterGen::getVariableBit(const std::string &VarName, + BitsInit *BI, int bit) { + if (VarBitInit *VBI = dyn_cast<VarBitInit>(BI->getBit(bit))) { + if (VarInit *VI = dyn_cast<VarInit>(VBI->getBitVar())) + if (VI->getName() == VarName) + return VBI->getBitNum(); + } else if (VarInit *VI = dyn_cast<VarInit>(BI->getBit(bit))) { + if (VI->getName() == VarName) + return 0; + } + + return -1; +} + +void CodeEmitterGen:: +AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName, + unsigned &NumberedOp, + std::set<unsigned> &NamedOpIndices, + std::string &Case, CodeGenTarget &Target) { + CodeGenInstruction &CGI = Target.getInstruction(R); + + // Determine if VarName actually contributes to the Inst encoding. + int bit = BI->getNumBits()-1; + + // Scan for a bit that this contributed to. + for (; bit >= 0; ) { + if (getVariableBit(VarName, BI, bit) != -1) + break; + + --bit; + } + + // If we found no bits, ignore this value, otherwise emit the call to get the + // operand encoding. + if (bit < 0) return; + + // If the operand matches by name, reference according to that + // operand number. Non-matching operands are assumed to be in + // order. + unsigned OpIdx; + if (CGI.Operands.hasOperandNamed(VarName, OpIdx)) { + // Get the machine operand number for the indicated operand. + OpIdx = CGI.Operands[OpIdx].MIOperandNo; + assert(!CGI.Operands.isFlatOperandNotEmitted(OpIdx) && + "Explicitly used operand also marked as not emitted!"); + } else { + unsigned NumberOps = CGI.Operands.size(); + /// If this operand is not supposed to be emitted by the + /// generated emitter, skip it. + while (NumberedOp < NumberOps && + (CGI.Operands.isFlatOperandNotEmitted(NumberedOp) || + (!NamedOpIndices.empty() && NamedOpIndices.count( + CGI.Operands.getSubOperandNumber(NumberedOp).first)))) { + ++NumberedOp; + + if (NumberedOp >= CGI.Operands.back().MIOperandNo + + CGI.Operands.back().MINumOperands) { + errs() << "Too few operands in record " << R->getName() << + " (no match for variable " << VarName << "):\n"; + errs() << *R; + errs() << '\n'; + + return; + } + } + + OpIdx = NumberedOp++; + } + + std::pair<unsigned, unsigned> SO = CGI.Operands.getSubOperandNumber(OpIdx); + std::string &EncoderMethodName = CGI.Operands[SO.first].EncoderMethodName; + + if (UseAPInt) + Case += " op.clearAllBits();\n"; + + // If the source operand has a custom encoder, use it. This will + // get the encoding for all of the suboperands. + if (!EncoderMethodName.empty()) { + // A custom encoder has all of the information for the + // sub-operands, if there are more than one, so only + // query the encoder once per source operand. + if (SO.second == 0) { + Case += " // op: " + VarName + "\n"; + if (UseAPInt) { + Case += " " + EncoderMethodName + "(MI, " + utostr(OpIdx); + Case += ", op"; + } else { + Case += " op = " + EncoderMethodName + "(MI, " + utostr(OpIdx); + } + Case += ", Fixups, STI);\n"; + } + } else { + Case += " // op: " + VarName + "\n"; + if (UseAPInt) { + Case += " getMachineOpValue(MI, MI.getOperand(" + utostr(OpIdx) + ")"; + Case += ", op, Fixups, STI"; + } else { + Case += " op = getMachineOpValue(MI, MI.getOperand(" + utostr(OpIdx) + ")"; + Case += ", Fixups, STI"; + } + Case += ");\n"; + } + + // Precalculate the number of lits this variable contributes to in the + // operand. If there is a single lit (consecutive range of bits) we can use a + // destructive sequence on APInt that reduces memory allocations. + int numOperandLits = 0; + for (int tmpBit = bit; tmpBit >= 0;) { + int varBit = getVariableBit(VarName, BI, tmpBit); + + // If this bit isn't from a variable, skip it. + if (varBit == -1) { + --tmpBit; + continue; + } + + // Figure out the consecutive range of bits covered by this operand, in + // order to generate better encoding code. + int beginVarBit = varBit; + int N = 1; + for (--tmpBit; tmpBit >= 0;) { + varBit = getVariableBit(VarName, BI, tmpBit); + if (varBit == -1 || varBit != (beginVarBit - N)) + break; + ++N; + --tmpBit; + } + ++numOperandLits; + } + + for (; bit >= 0; ) { + int varBit = getVariableBit(VarName, BI, bit); + + // If this bit isn't from a variable, skip it. + if (varBit == -1) { + --bit; + continue; + } + + // Figure out the consecutive range of bits covered by this operand, in + // order to generate better encoding code. + int beginInstBit = bit; + int beginVarBit = varBit; + int N = 1; + for (--bit; bit >= 0;) { + varBit = getVariableBit(VarName, BI, bit); + if (varBit == -1 || varBit != (beginVarBit - N)) break; + ++N; + --bit; + } + + std::string maskStr; + int opShift; + + unsigned loBit = beginVarBit - N + 1; + unsigned hiBit = loBit + N; + unsigned loInstBit = beginInstBit - N + 1; + if (UseAPInt) { + std::string extractStr; + if (N >= 64) { + extractStr = "op.extractBits(" + itostr(hiBit - loBit) + ", " + + itostr(loBit) + ")"; + Case += " Value.insertBits(" + extractStr + ", " + + itostr(loInstBit) + ");\n"; + } else { + extractStr = "op.extractBitsAsZExtValue(" + itostr(hiBit - loBit) + + ", " + itostr(loBit) + ")"; + Case += " Value.insertBits(" + extractStr + ", " + + itostr(loInstBit) + ", " + itostr(hiBit - loBit) + ");\n"; + } + } else { + uint64_t opMask = ~(uint64_t)0 >> (64 - N); + opShift = beginVarBit - N + 1; + opMask <<= opShift; + maskStr = "UINT64_C(" + utostr(opMask) + ")"; + opShift = beginInstBit - beginVarBit; + + if (numOperandLits == 1) { + Case += " op &= " + maskStr + ";\n"; + if (opShift > 0) { + Case += " op <<= " + itostr(opShift) + ";\n"; + } else if (opShift < 0) { + Case += " op >>= " + itostr(-opShift) + ";\n"; + } + Case += " Value |= op;\n"; + } else { + if (opShift > 0) { + Case += " Value |= (op & " + maskStr + ") << " + + itostr(opShift) + ";\n"; + } else if (opShift < 0) { + Case += " Value |= (op & " + maskStr + ") >> " + + itostr(-opShift) + ";\n"; + } else { + Case += " Value |= (op & " + maskStr + ");\n"; + } + } + } + } +} + +std::string CodeEmitterGen::getInstructionCase(Record *R, + CodeGenTarget &Target) { + std::string Case; + if (const RecordVal *RV = R->getValue("EncodingInfos")) { + if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + const CodeGenHwModes &HWM = Target.getHwModes(); + EncodingInfoByHwMode EBM(DI->getDef(), HWM); + Case += " switch (HwMode) {\n"; + Case += " default: llvm_unreachable(\"Unhandled HwMode\");\n"; + for (auto &KV : EBM.Map) { + Case += " case " + itostr(KV.first) + ": {\n"; + Case += getInstructionCaseForEncoding(R, KV.second, Target); + Case += " break;\n"; + Case += " }\n"; + } + Case += " }\n"; + return Case; + } + } + return getInstructionCaseForEncoding(R, R, Target); +} + +std::string CodeEmitterGen::getInstructionCaseForEncoding(Record *R, Record *EncodingDef, + CodeGenTarget &Target) { + std::string Case; + BitsInit *BI = EncodingDef->getValueAsBitsInit("Inst"); + unsigned NumberedOp = 0; + std::set<unsigned> NamedOpIndices; + + // Collect the set of operand indices that might correspond to named + // operand, and skip these when assigning operands based on position. + if (Target.getInstructionSet()-> + getValueAsBit("noNamedPositionallyEncodedOperands")) { + CodeGenInstruction &CGI = Target.getInstruction(R); + for (const RecordVal &RV : R->getValues()) { + unsigned OpIdx; + if (!CGI.Operands.hasOperandNamed(RV.getName(), OpIdx)) + continue; + + NamedOpIndices.insert(OpIdx); + } + } + + // Loop over all of the fields in the instruction, determining which are the + // operands to the instruction. + for (const RecordVal &RV : EncodingDef->getValues()) { + // Ignore fixed fields in the record, we're looking for values like: + // bits<5> RST = { ?, ?, ?, ?, ? }; + if (RV.getPrefix() || RV.getValue()->isComplete()) + continue; + + AddCodeToMergeInOperand(R, BI, std::string(RV.getName()), NumberedOp, + NamedOpIndices, Case, Target); + } + + StringRef PostEmitter = R->getValueAsString("PostEncoderMethod"); + if (!PostEmitter.empty()) { + Case += " Value = "; + Case += PostEmitter; + Case += "(MI, Value"; + Case += ", STI"; + Case += ");\n"; + } + + return Case; +} + +static std::string +getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset) { + std::string Name = "CEFBS"; + for (const auto &Feature : FeatureBitset) + Name += ("_" + Feature->getName()).str(); + return Name; +} + +static void emitInstBits(raw_ostream &OS, const APInt &Bits) { + for (unsigned I = 0; I < Bits.getNumWords(); ++I) + OS << ((I > 0) ? ", " : "") << "UINT64_C(" << utostr(Bits.getRawData()[I]) + << ")"; +} + +void CodeEmitterGen::emitInstructionBaseValues( + raw_ostream &o, ArrayRef<const CodeGenInstruction *> NumberedInstructions, + CodeGenTarget &Target, int HwMode) { + const CodeGenHwModes &HWM = Target.getHwModes(); + if (HwMode == -1) + o << " static const uint64_t InstBits[] = {\n"; + else + o << " static const uint64_t InstBits_" << HWM.getMode(HwMode).Name + << "[] = {\n"; + + for (const CodeGenInstruction *CGI : NumberedInstructions) { + Record *R = CGI->TheDef; + + if (R->getValueAsString("Namespace") == "TargetOpcode" || + R->getValueAsBit("isPseudo")) { + o << " "; emitInstBits(o, APInt(BitWidth, 0)); o << ",\n"; + continue; + } + + Record *EncodingDef = R; + if (const RecordVal *RV = R->getValue("EncodingInfos")) { + if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + EncodingInfoByHwMode EBM(DI->getDef(), HWM); + if (EBM.hasMode(HwMode)) + EncodingDef = EBM.get(HwMode); + } + } + BitsInit *BI = EncodingDef->getValueAsBitsInit("Inst"); + + // Start by filling in fixed values. + APInt Value(BitWidth, 0); + for (unsigned i = 0, e = BI->getNumBits(); i != e; ++i) { + if (BitInit *B = dyn_cast<BitInit>(BI->getBit(e - i - 1))) + Value |= APInt(BitWidth, (uint64_t)B->getValue()) << (e - i - 1); + } + o << " "; + emitInstBits(o, Value); + o << "," << '\t' << "// " << R->getName() << "\n"; + } + o << " UINT64_C(0)\n };\n"; +} + +void CodeEmitterGen::run(raw_ostream &o) { + CodeGenTarget Target(Records); + std::vector<Record*> Insts = Records.getAllDerivedDefinitions("Instruction"); + + // For little-endian instruction bit encodings, reverse the bit order + Target.reverseBitsForLittleEndianEncoding(); + + ArrayRef<const CodeGenInstruction*> NumberedInstructions = + Target.getInstructionsByEnumValue(); + + const CodeGenHwModes &HWM = Target.getHwModes(); + // The set of HwModes used by instruction encodings. + std::set<unsigned> HwModes; + BitWidth = 0; + for (const CodeGenInstruction *CGI : NumberedInstructions) { + Record *R = CGI->TheDef; + if (R->getValueAsString("Namespace") == "TargetOpcode" || + R->getValueAsBit("isPseudo")) + continue; + + if (const RecordVal *RV = R->getValue("EncodingInfos")) { + if (DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + EncodingInfoByHwMode EBM(DI->getDef(), HWM); + for (auto &KV : EBM.Map) { + BitsInit *BI = KV.second->getValueAsBitsInit("Inst"); + BitWidth = std::max(BitWidth, BI->getNumBits()); + HwModes.insert(KV.first); + } + continue; + } + } + BitsInit *BI = R->getValueAsBitsInit("Inst"); + BitWidth = std::max(BitWidth, BI->getNumBits()); + } + UseAPInt = BitWidth > 64; + + // Emit function declaration + if (UseAPInt) { + o << "void " << Target.getName() + << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n" + << " SmallVectorImpl<MCFixup> &Fixups,\n" + << " APInt &Inst,\n" + << " APInt &Scratch,\n" + << " const MCSubtargetInfo &STI) const {\n"; + } else { + o << "uint64_t " << Target.getName(); + o << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n" + << " SmallVectorImpl<MCFixup> &Fixups,\n" + << " const MCSubtargetInfo &STI) const {\n"; + } + + // Emit instruction base values + if (HwModes.empty()) { + emitInstructionBaseValues(o, NumberedInstructions, Target, -1); + } else { + for (unsigned HwMode : HwModes) + emitInstructionBaseValues(o, NumberedInstructions, Target, (int)HwMode); + } + + if (!HwModes.empty()) { + o << " const uint64_t *InstBits;\n"; + o << " unsigned HwMode = STI.getHwMode();\n"; + o << " switch (HwMode) {\n"; + o << " default: llvm_unreachable(\"Unknown hardware mode!\"); break;\n"; + for (unsigned I : HwModes) { + o << " case " << I << ": InstBits = InstBits_" << HWM.getMode(I).Name + << "; break;\n"; + } + o << " };\n"; + } + + // Map to accumulate all the cases. + std::map<std::string, std::vector<std::string>> CaseMap; + + // Construct all cases statement for each opcode + for (std::vector<Record*>::iterator IC = Insts.begin(), EC = Insts.end(); + IC != EC; ++IC) { + Record *R = *IC; + if (R->getValueAsString("Namespace") == "TargetOpcode" || + R->getValueAsBit("isPseudo")) + continue; + std::string InstName = + (R->getValueAsString("Namespace") + "::" + R->getName()).str(); + std::string Case = getInstructionCase(R, Target); + + CaseMap[Case].push_back(std::move(InstName)); + } + + // Emit initial function code + if (UseAPInt) { + int NumWords = APInt::getNumWords(BitWidth); + int NumBytes = (BitWidth + 7) / 8; + o << " const unsigned opcode = MI.getOpcode();\n" + << " if (Inst.getBitWidth() != " << BitWidth << ")\n" + << " Inst = Inst.zext(" << BitWidth << ");\n" + << " if (Scratch.getBitWidth() != " << BitWidth << ")\n" + << " Scratch = Scratch.zext(" << BitWidth << ");\n" + << " LoadIntFromMemory(Inst, (uint8_t*)&InstBits[opcode * " << NumWords + << "], " << NumBytes << ");\n" + << " APInt &Value = Inst;\n" + << " APInt &op = Scratch;\n" + << " switch (opcode) {\n"; + } else { + o << " const unsigned opcode = MI.getOpcode();\n" + << " uint64_t Value = InstBits[opcode];\n" + << " uint64_t op = 0;\n" + << " (void)op; // suppress warning\n" + << " switch (opcode) {\n"; + } + + // Emit each case statement + std::map<std::string, std::vector<std::string>>::iterator IE, EE; + for (IE = CaseMap.begin(), EE = CaseMap.end(); IE != EE; ++IE) { + const std::string &Case = IE->first; + std::vector<std::string> &InstList = IE->second; + + for (int i = 0, N = InstList.size(); i < N; i++) { + if (i) o << "\n"; + o << " case " << InstList[i] << ":"; + } + o << " {\n"; + o << Case; + o << " break;\n" + << " }\n"; + } + + // Default case: unhandled opcode + o << " default:\n" + << " std::string msg;\n" + << " raw_string_ostream Msg(msg);\n" + << " Msg << \"Not supported instr: \" << MI;\n" + << " report_fatal_error(Msg.str());\n" + << " }\n"; + if (UseAPInt) + o << " Inst = Value;\n"; + else + o << " return Value;\n"; + o << "}\n\n"; + + const auto &All = SubtargetFeatureInfo::getAll(Records); + std::map<Record *, SubtargetFeatureInfo, LessRecordByID> SubtargetFeatures; + SubtargetFeatures.insert(All.begin(), All.end()); + + o << "#ifdef ENABLE_INSTR_PREDICATE_VERIFIER\n" + << "#undef ENABLE_INSTR_PREDICATE_VERIFIER\n" + << "#include <sstream>\n\n"; + + // Emit the subtarget feature enumeration. + SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(SubtargetFeatures, + o); + + // Emit the name table for error messages. + o << "#ifndef NDEBUG\n"; + SubtargetFeatureInfo::emitNameTable(SubtargetFeatures, o); + o << "#endif // NDEBUG\n"; + + // Emit the available features compute function. + SubtargetFeatureInfo::emitComputeAssemblerAvailableFeatures( + Target.getName(), "MCCodeEmitter", "computeAvailableFeatures", + SubtargetFeatures, o); + + std::vector<std::vector<Record *>> FeatureBitsets; + for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) { + FeatureBitsets.emplace_back(); + for (Record *Predicate : Inst->TheDef->getValueAsListOfDefs("Predicates")) { + const auto &I = SubtargetFeatures.find(Predicate); + if (I != SubtargetFeatures.end()) + FeatureBitsets.back().push_back(I->second.TheDef); + } + } + + llvm::sort(FeatureBitsets, [&](const std::vector<Record *> &A, + const std::vector<Record *> &B) { + if (A.size() < B.size()) + return true; + if (A.size() > B.size()) + return false; + for (auto Pair : zip(A, B)) { + if (std::get<0>(Pair)->getName() < std::get<1>(Pair)->getName()) + return true; + if (std::get<0>(Pair)->getName() > std::get<1>(Pair)->getName()) + return false; + } + return false; + }); + FeatureBitsets.erase( + std::unique(FeatureBitsets.begin(), FeatureBitsets.end()), + FeatureBitsets.end()); + o << "#ifndef NDEBUG\n" + << "// Feature bitsets.\n" + << "enum : " << getMinimalTypeForRange(FeatureBitsets.size()) << " {\n" + << " CEFBS_None,\n"; + for (const auto &FeatureBitset : FeatureBitsets) { + if (FeatureBitset.empty()) + continue; + o << " " << getNameForFeatureBitset(FeatureBitset) << ",\n"; + } + o << "};\n\n" + << "static constexpr FeatureBitset FeatureBitsets[] = {\n" + << " {}, // CEFBS_None\n"; + for (const auto &FeatureBitset : FeatureBitsets) { + if (FeatureBitset.empty()) + continue; + o << " {"; + for (const auto &Feature : FeatureBitset) { + const auto &I = SubtargetFeatures.find(Feature); + assert(I != SubtargetFeatures.end() && "Didn't import predicate?"); + o << I->second.getEnumBitName() << ", "; + } + o << "},\n"; + } + o << "};\n" + << "#endif // NDEBUG\n\n"; + + + // Emit the predicate verifier. + o << "void " << Target.getName() + << "MCCodeEmitter::verifyInstructionPredicates(\n" + << " const MCInst &Inst, const FeatureBitset &AvailableFeatures) const {\n" + << "#ifndef NDEBUG\n" + << " static " << getMinimalTypeForRange(FeatureBitsets.size()) + << " RequiredFeaturesRefs[] = {\n"; + unsigned InstIdx = 0; + for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) { + o << " CEFBS"; + unsigned NumPredicates = 0; + for (Record *Predicate : Inst->TheDef->getValueAsListOfDefs("Predicates")) { + const auto &I = SubtargetFeatures.find(Predicate); + if (I != SubtargetFeatures.end()) { + o << '_' << I->second.TheDef->getName(); + NumPredicates++; + } + } + if (!NumPredicates) + o << "_None"; + o << ", // " << Inst->TheDef->getName() << " = " << InstIdx << "\n"; + InstIdx++; + } + o << " };\n\n"; + o << " assert(Inst.getOpcode() < " << InstIdx << ");\n"; + o << " const FeatureBitset &RequiredFeatures = " + "FeatureBitsets[RequiredFeaturesRefs[Inst.getOpcode()]];\n"; + o << " FeatureBitset MissingFeatures =\n" + << " (AvailableFeatures & RequiredFeatures) ^\n" + << " RequiredFeatures;\n" + << " if (MissingFeatures.any()) {\n" + << " std::ostringstream Msg;\n" + << " Msg << \"Attempting to emit \" << " + "MCII.getName(Inst.getOpcode()).str()\n" + << " << \" instruction but the \";\n" + << " for (unsigned i = 0, e = MissingFeatures.size(); i != e; ++i)\n" + << " if (MissingFeatures.test(i))\n" + << " Msg << SubtargetFeatureNames[i] << \" \";\n" + << " Msg << \"predicate(s) are not met\";\n" + << " report_fatal_error(Msg.str());\n" + << " }\n" + << "#else\n" + << "// Silence unused variable warning on targets that don't use MCII for " + "other purposes (e.g. BPF).\n" + << "(void)MCII;\n" + << "#endif // NDEBUG\n"; + o << "}\n"; + o << "#endif\n"; +} + +} // end anonymous namespace + +namespace llvm { + +void EmitCodeEmitter(RecordKeeper &RK, raw_ostream &OS) { + emitSourceFileHeader("Machine Code Emitter", OS); + CodeEmitterGen(RK).run(OS); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.cpp new file mode 100644 index 000000000000..6fdc116721f3 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.cpp @@ -0,0 +1,4729 @@ +//===- CodeGenDAGPatterns.cpp - Read DAG patterns from .td file -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the CodeGenDAGPatterns class, which is used to read and +// represent the patterns present in a .td file for instructions. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenDAGPatterns.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/TypeSize.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include <algorithm> +#include <cstdio> +#include <iterator> +#include <set> +using namespace llvm; + +#define DEBUG_TYPE "dag-patterns" + +static inline bool isIntegerOrPtr(MVT VT) { + return VT.isInteger() || VT == MVT::iPTR; +} +static inline bool isFloatingPoint(MVT VT) { + return VT.isFloatingPoint(); +} +static inline bool isVector(MVT VT) { + return VT.isVector(); +} +static inline bool isScalar(MVT VT) { + return !VT.isVector(); +} + +template <typename Predicate> +static bool berase_if(MachineValueTypeSet &S, Predicate P) { + bool Erased = false; + // It is ok to iterate over MachineValueTypeSet and remove elements from it + // at the same time. + for (MVT T : S) { + if (!P(T)) + continue; + Erased = true; + S.erase(T); + } + return Erased; +} + +// --- TypeSetByHwMode + +// This is a parameterized type-set class. For each mode there is a list +// of types that are currently possible for a given tree node. Type +// inference will apply to each mode separately. + +TypeSetByHwMode::TypeSetByHwMode(ArrayRef<ValueTypeByHwMode> VTList) { + for (const ValueTypeByHwMode &VVT : VTList) { + insert(VVT); + AddrSpaces.push_back(VVT.PtrAddrSpace); + } +} + +bool TypeSetByHwMode::isValueTypeByHwMode(bool AllowEmpty) const { + for (const auto &I : *this) { + if (I.second.size() > 1) + return false; + if (!AllowEmpty && I.second.empty()) + return false; + } + return true; +} + +ValueTypeByHwMode TypeSetByHwMode::getValueTypeByHwMode() const { + assert(isValueTypeByHwMode(true) && + "The type set has multiple types for at least one HW mode"); + ValueTypeByHwMode VVT; + auto ASI = AddrSpaces.begin(); + + for (const auto &I : *this) { + MVT T = I.second.empty() ? MVT::Other : *I.second.begin(); + VVT.getOrCreateTypeForMode(I.first, T); + if (ASI != AddrSpaces.end()) + VVT.PtrAddrSpace = *ASI++; + } + return VVT; +} + +bool TypeSetByHwMode::isPossible() const { + for (const auto &I : *this) + if (!I.second.empty()) + return true; + return false; +} + +bool TypeSetByHwMode::insert(const ValueTypeByHwMode &VVT) { + bool Changed = false; + bool ContainsDefault = false; + MVT DT = MVT::Other; + + SmallDenseSet<unsigned, 4> Modes; + for (const auto &P : VVT) { + unsigned M = P.first; + Modes.insert(M); + // Make sure there exists a set for each specific mode from VVT. + Changed |= getOrCreate(M).insert(P.second).second; + // Cache VVT's default mode. + if (DefaultMode == M) { + ContainsDefault = true; + DT = P.second; + } + } + + // If VVT has a default mode, add the corresponding type to all + // modes in "this" that do not exist in VVT. + if (ContainsDefault) + for (auto &I : *this) + if (!Modes.count(I.first)) + Changed |= I.second.insert(DT).second; + + return Changed; +} + +// Constrain the type set to be the intersection with VTS. +bool TypeSetByHwMode::constrain(const TypeSetByHwMode &VTS) { + bool Changed = false; + if (hasDefault()) { + for (const auto &I : VTS) { + unsigned M = I.first; + if (M == DefaultMode || hasMode(M)) + continue; + Map.insert({M, Map.at(DefaultMode)}); + Changed = true; + } + } + + for (auto &I : *this) { + unsigned M = I.first; + SetType &S = I.second; + if (VTS.hasMode(M) || VTS.hasDefault()) { + Changed |= intersect(I.second, VTS.get(M)); + } else if (!S.empty()) { + S.clear(); + Changed = true; + } + } + return Changed; +} + +template <typename Predicate> +bool TypeSetByHwMode::constrain(Predicate P) { + bool Changed = false; + for (auto &I : *this) + Changed |= berase_if(I.second, [&P](MVT VT) { return !P(VT); }); + return Changed; +} + +template <typename Predicate> +bool TypeSetByHwMode::assign_if(const TypeSetByHwMode &VTS, Predicate P) { + assert(empty()); + for (const auto &I : VTS) { + SetType &S = getOrCreate(I.first); + for (auto J : I.second) + if (P(J)) + S.insert(J); + } + return !empty(); +} + +void TypeSetByHwMode::writeToStream(raw_ostream &OS) const { + SmallVector<unsigned, 4> Modes; + Modes.reserve(Map.size()); + + for (const auto &I : *this) + Modes.push_back(I.first); + if (Modes.empty()) { + OS << "{}"; + return; + } + array_pod_sort(Modes.begin(), Modes.end()); + + OS << '{'; + for (unsigned M : Modes) { + OS << ' ' << getModeName(M) << ':'; + writeToStream(get(M), OS); + } + OS << " }"; +} + +void TypeSetByHwMode::writeToStream(const SetType &S, raw_ostream &OS) { + SmallVector<MVT, 4> Types(S.begin(), S.end()); + array_pod_sort(Types.begin(), Types.end()); + + OS << '['; + for (unsigned i = 0, e = Types.size(); i != e; ++i) { + OS << ValueTypeByHwMode::getMVTName(Types[i]); + if (i != e-1) + OS << ' '; + } + OS << ']'; +} + +bool TypeSetByHwMode::operator==(const TypeSetByHwMode &VTS) const { + // The isSimple call is much quicker than hasDefault - check this first. + bool IsSimple = isSimple(); + bool VTSIsSimple = VTS.isSimple(); + if (IsSimple && VTSIsSimple) + return *begin() == *VTS.begin(); + + // Speedup: We have a default if the set is simple. + bool HaveDefault = IsSimple || hasDefault(); + bool VTSHaveDefault = VTSIsSimple || VTS.hasDefault(); + if (HaveDefault != VTSHaveDefault) + return false; + + SmallDenseSet<unsigned, 4> Modes; + for (auto &I : *this) + Modes.insert(I.first); + for (const auto &I : VTS) + Modes.insert(I.first); + + if (HaveDefault) { + // Both sets have default mode. + for (unsigned M : Modes) { + if (get(M) != VTS.get(M)) + return false; + } + } else { + // Neither set has default mode. + for (unsigned M : Modes) { + // If there is no default mode, an empty set is equivalent to not having + // the corresponding mode. + bool NoModeThis = !hasMode(M) || get(M).empty(); + bool NoModeVTS = !VTS.hasMode(M) || VTS.get(M).empty(); + if (NoModeThis != NoModeVTS) + return false; + if (!NoModeThis) + if (get(M) != VTS.get(M)) + return false; + } + } + + return true; +} + +namespace llvm { + raw_ostream &operator<<(raw_ostream &OS, const TypeSetByHwMode &T) { + T.writeToStream(OS); + return OS; + } +} + +LLVM_DUMP_METHOD +void TypeSetByHwMode::dump() const { + dbgs() << *this << '\n'; +} + +bool TypeSetByHwMode::intersect(SetType &Out, const SetType &In) { + bool OutP = Out.count(MVT::iPTR), InP = In.count(MVT::iPTR); + auto Int = [&In](MVT T) -> bool { return !In.count(T); }; + + if (OutP == InP) + return berase_if(Out, Int); + + // Compute the intersection of scalars separately to account for only + // one set containing iPTR. + // The intersection of iPTR with a set of integer scalar types that does not + // include iPTR will result in the most specific scalar type: + // - iPTR is more specific than any set with two elements or more + // - iPTR is less specific than any single integer scalar type. + // For example + // { iPTR } * { i32 } -> { i32 } + // { iPTR } * { i32 i64 } -> { iPTR } + // and + // { iPTR i32 } * { i32 } -> { i32 } + // { iPTR i32 } * { i32 i64 } -> { i32 i64 } + // { iPTR i32 } * { i32 i64 i128 } -> { iPTR i32 } + + // Compute the difference between the two sets in such a way that the + // iPTR is in the set that is being subtracted. This is to see if there + // are any extra scalars in the set without iPTR that are not in the + // set containing iPTR. Then the iPTR could be considered a "wildcard" + // matching these scalars. If there is only one such scalar, it would + // replace the iPTR, if there are more, the iPTR would be retained. + SetType Diff; + if (InP) { + Diff = Out; + berase_if(Diff, [&In](MVT T) { return In.count(T); }); + // Pre-remove these elements and rely only on InP/OutP to determine + // whether a change has been made. + berase_if(Out, [&Diff](MVT T) { return Diff.count(T); }); + } else { + Diff = In; + berase_if(Diff, [&Out](MVT T) { return Out.count(T); }); + Out.erase(MVT::iPTR); + } + + // The actual intersection. + bool Changed = berase_if(Out, Int); + unsigned NumD = Diff.size(); + if (NumD == 0) + return Changed; + + if (NumD == 1) { + Out.insert(*Diff.begin()); + // This is a change only if Out was the one with iPTR (which is now + // being replaced). + Changed |= OutP; + } else { + // Multiple elements from Out are now replaced with iPTR. + Out.insert(MVT::iPTR); + Changed |= !OutP; + } + return Changed; +} + +bool TypeSetByHwMode::validate() const { +#ifndef NDEBUG + if (empty()) + return true; + bool AllEmpty = true; + for (const auto &I : *this) + AllEmpty &= I.second.empty(); + return !AllEmpty; +#endif + return true; +} + +// --- TypeInfer + +bool TypeInfer::MergeInTypeInfo(TypeSetByHwMode &Out, + const TypeSetByHwMode &In) { + ValidateOnExit _1(Out, *this); + In.validate(); + if (In.empty() || Out == In || TP.hasError()) + return false; + if (Out.empty()) { + Out = In; + return true; + } + + bool Changed = Out.constrain(In); + if (Changed && Out.empty()) + TP.error("Type contradiction"); + + return Changed; +} + +bool TypeInfer::forceArbitrary(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out, *this); + if (TP.hasError()) + return false; + assert(!Out.empty() && "cannot pick from an empty set"); + + bool Changed = false; + for (auto &I : Out) { + TypeSetByHwMode::SetType &S = I.second; + if (S.size() <= 1) + continue; + MVT T = *S.begin(); // Pick the first element. + S.clear(); + S.insert(T); + Changed = true; + } + return Changed; +} + +bool TypeInfer::EnforceInteger(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out, *this); + if (TP.hasError()) + return false; + if (!Out.empty()) + return Out.constrain(isIntegerOrPtr); + + return Out.assign_if(getLegalTypes(), isIntegerOrPtr); +} + +bool TypeInfer::EnforceFloatingPoint(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out, *this); + if (TP.hasError()) + return false; + if (!Out.empty()) + return Out.constrain(isFloatingPoint); + + return Out.assign_if(getLegalTypes(), isFloatingPoint); +} + +bool TypeInfer::EnforceScalar(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out, *this); + if (TP.hasError()) + return false; + if (!Out.empty()) + return Out.constrain(isScalar); + + return Out.assign_if(getLegalTypes(), isScalar); +} + +bool TypeInfer::EnforceVector(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out, *this); + if (TP.hasError()) + return false; + if (!Out.empty()) + return Out.constrain(isVector); + + return Out.assign_if(getLegalTypes(), isVector); +} + +bool TypeInfer::EnforceAny(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out, *this); + if (TP.hasError() || !Out.empty()) + return false; + + Out = getLegalTypes(); + return true; +} + +template <typename Iter, typename Pred, typename Less> +static Iter min_if(Iter B, Iter E, Pred P, Less L) { + if (B == E) + return E; + Iter Min = E; + for (Iter I = B; I != E; ++I) { + if (!P(*I)) + continue; + if (Min == E || L(*I, *Min)) + Min = I; + } + return Min; +} + +template <typename Iter, typename Pred, typename Less> +static Iter max_if(Iter B, Iter E, Pred P, Less L) { + if (B == E) + return E; + Iter Max = E; + for (Iter I = B; I != E; ++I) { + if (!P(*I)) + continue; + if (Max == E || L(*Max, *I)) + Max = I; + } + return Max; +} + +/// Make sure that for each type in Small, there exists a larger type in Big. +bool TypeInfer::EnforceSmallerThan(TypeSetByHwMode &Small, + TypeSetByHwMode &Big) { + ValidateOnExit _1(Small, *this), _2(Big, *this); + if (TP.hasError()) + return false; + bool Changed = false; + + if (Small.empty()) + Changed |= EnforceAny(Small); + if (Big.empty()) + Changed |= EnforceAny(Big); + + assert(Small.hasDefault() && Big.hasDefault()); + + std::vector<unsigned> Modes = union_modes(Small, Big); + + // 1. Only allow integer or floating point types and make sure that + // both sides are both integer or both floating point. + // 2. Make sure that either both sides have vector types, or neither + // of them does. + for (unsigned M : Modes) { + TypeSetByHwMode::SetType &S = Small.get(M); + TypeSetByHwMode::SetType &B = Big.get(M); + + if (any_of(S, isIntegerOrPtr) && any_of(S, isIntegerOrPtr)) { + auto NotInt = [](MVT VT) { return !isIntegerOrPtr(VT); }; + Changed |= berase_if(S, NotInt); + Changed |= berase_if(B, NotInt); + } else if (any_of(S, isFloatingPoint) && any_of(B, isFloatingPoint)) { + auto NotFP = [](MVT VT) { return !isFloatingPoint(VT); }; + Changed |= berase_if(S, NotFP); + Changed |= berase_if(B, NotFP); + } else if (S.empty() || B.empty()) { + Changed = !S.empty() || !B.empty(); + S.clear(); + B.clear(); + } else { + TP.error("Incompatible types"); + return Changed; + } + + if (none_of(S, isVector) || none_of(B, isVector)) { + Changed |= berase_if(S, isVector); + Changed |= berase_if(B, isVector); + } + } + + auto LT = [](MVT A, MVT B) -> bool { + // Always treat non-scalable MVTs as smaller than scalable MVTs for the + // purposes of ordering. + auto ASize = std::make_tuple(A.isScalableVector(), A.getScalarSizeInBits(), + A.getSizeInBits()); + auto BSize = std::make_tuple(B.isScalableVector(), B.getScalarSizeInBits(), + B.getSizeInBits()); + return ASize < BSize; + }; + auto SameKindLE = [](MVT A, MVT B) -> bool { + // This function is used when removing elements: when a vector is compared + // to a non-vector or a scalable vector to any non-scalable MVT, it should + // return false (to avoid removal). + if (std::make_tuple(A.isVector(), A.isScalableVector()) != + std::make_tuple(B.isVector(), B.isScalableVector())) + return false; + + return std::make_tuple(A.getScalarSizeInBits(), A.getSizeInBits()) <= + std::make_tuple(B.getScalarSizeInBits(), B.getSizeInBits()); + }; + + for (unsigned M : Modes) { + TypeSetByHwMode::SetType &S = Small.get(M); + TypeSetByHwMode::SetType &B = Big.get(M); + // MinS = min scalar in Small, remove all scalars from Big that are + // smaller-or-equal than MinS. + auto MinS = min_if(S.begin(), S.end(), isScalar, LT); + if (MinS != S.end()) + Changed |= berase_if(B, std::bind(SameKindLE, + std::placeholders::_1, *MinS)); + + // MaxS = max scalar in Big, remove all scalars from Small that are + // larger than MaxS. + auto MaxS = max_if(B.begin(), B.end(), isScalar, LT); + if (MaxS != B.end()) + Changed |= berase_if(S, std::bind(SameKindLE, + *MaxS, std::placeholders::_1)); + + // MinV = min vector in Small, remove all vectors from Big that are + // smaller-or-equal than MinV. + auto MinV = min_if(S.begin(), S.end(), isVector, LT); + if (MinV != S.end()) + Changed |= berase_if(B, std::bind(SameKindLE, + std::placeholders::_1, *MinV)); + + // MaxV = max vector in Big, remove all vectors from Small that are + // larger than MaxV. + auto MaxV = max_if(B.begin(), B.end(), isVector, LT); + if (MaxV != B.end()) + Changed |= berase_if(S, std::bind(SameKindLE, + *MaxV, std::placeholders::_1)); + } + + return Changed; +} + +/// 1. Ensure that for each type T in Vec, T is a vector type, and that +/// for each type U in Elem, U is a scalar type. +/// 2. Ensure that for each (scalar) type U in Elem, there exists a (vector) +/// type T in Vec, such that U is the element type of T. +bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, + TypeSetByHwMode &Elem) { + ValidateOnExit _1(Vec, *this), _2(Elem, *this); + if (TP.hasError()) + return false; + bool Changed = false; + + if (Vec.empty()) + Changed |= EnforceVector(Vec); + if (Elem.empty()) + Changed |= EnforceScalar(Elem); + + for (unsigned M : union_modes(Vec, Elem)) { + TypeSetByHwMode::SetType &V = Vec.get(M); + TypeSetByHwMode::SetType &E = Elem.get(M); + + Changed |= berase_if(V, isScalar); // Scalar = !vector + Changed |= berase_if(E, isVector); // Vector = !scalar + assert(!V.empty() && !E.empty()); + + SmallSet<MVT,4> VT, ST; + // Collect element types from the "vector" set. + for (MVT T : V) + VT.insert(T.getVectorElementType()); + // Collect scalar types from the "element" set. + for (MVT T : E) + ST.insert(T); + + // Remove from V all (vector) types whose element type is not in S. + Changed |= berase_if(V, [&ST](MVT T) -> bool { + return !ST.count(T.getVectorElementType()); + }); + // Remove from E all (scalar) types, for which there is no corresponding + // type in V. + Changed |= berase_if(E, [&VT](MVT T) -> bool { return !VT.count(T); }); + } + + return Changed; +} + +bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, + const ValueTypeByHwMode &VVT) { + TypeSetByHwMode Tmp(VVT); + ValidateOnExit _1(Vec, *this), _2(Tmp, *this); + return EnforceVectorEltTypeIs(Vec, Tmp); +} + +/// Ensure that for each type T in Sub, T is a vector type, and there +/// exists a type U in Vec such that U is a vector type with the same +/// element type as T and at least as many elements as T. +bool TypeInfer::EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec, + TypeSetByHwMode &Sub) { + ValidateOnExit _1(Vec, *this), _2(Sub, *this); + if (TP.hasError()) + return false; + + /// Return true if B is a suB-vector of P, i.e. P is a suPer-vector of B. + auto IsSubVec = [](MVT B, MVT P) -> bool { + if (!B.isVector() || !P.isVector()) + return false; + // Logically a <4 x i32> is a valid subvector of <n x 4 x i32> + // but until there are obvious use-cases for this, keep the + // types separate. + if (B.isScalableVector() != P.isScalableVector()) + return false; + if (B.getVectorElementType() != P.getVectorElementType()) + return false; + return B.getVectorNumElements() < P.getVectorNumElements(); + }; + + /// Return true if S has no element (vector type) that T is a sub-vector of, + /// i.e. has the same element type as T and more elements. + auto NoSubV = [&IsSubVec](const TypeSetByHwMode::SetType &S, MVT T) -> bool { + for (auto I : S) + if (IsSubVec(T, I)) + return false; + return true; + }; + + /// Return true if S has no element (vector type) that T is a super-vector + /// of, i.e. has the same element type as T and fewer elements. + auto NoSupV = [&IsSubVec](const TypeSetByHwMode::SetType &S, MVT T) -> bool { + for (auto I : S) + if (IsSubVec(I, T)) + return false; + return true; + }; + + bool Changed = false; + + if (Vec.empty()) + Changed |= EnforceVector(Vec); + if (Sub.empty()) + Changed |= EnforceVector(Sub); + + for (unsigned M : union_modes(Vec, Sub)) { + TypeSetByHwMode::SetType &S = Sub.get(M); + TypeSetByHwMode::SetType &V = Vec.get(M); + + Changed |= berase_if(S, isScalar); + + // Erase all types from S that are not sub-vectors of a type in V. + Changed |= berase_if(S, std::bind(NoSubV, V, std::placeholders::_1)); + + // Erase all types from V that are not super-vectors of a type in S. + Changed |= berase_if(V, std::bind(NoSupV, S, std::placeholders::_1)); + } + + return Changed; +} + +/// 1. Ensure that V has a scalar type iff W has a scalar type. +/// 2. Ensure that for each vector type T in V, there exists a vector +/// type U in W, such that T and U have the same number of elements. +/// 3. Ensure that for each vector type U in W, there exists a vector +/// type T in V, such that T and U have the same number of elements +/// (reverse of 2). +bool TypeInfer::EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W) { + ValidateOnExit _1(V, *this), _2(W, *this); + if (TP.hasError()) + return false; + + bool Changed = false; + if (V.empty()) + Changed |= EnforceAny(V); + if (W.empty()) + Changed |= EnforceAny(W); + + // An actual vector type cannot have 0 elements, so we can treat scalars + // as zero-length vectors. This way both vectors and scalars can be + // processed identically. + auto NoLength = [](const SmallSet<unsigned,2> &Lengths, MVT T) -> bool { + return !Lengths.count(T.isVector() ? T.getVectorNumElements() : 0); + }; + + for (unsigned M : union_modes(V, W)) { + TypeSetByHwMode::SetType &VS = V.get(M); + TypeSetByHwMode::SetType &WS = W.get(M); + + SmallSet<unsigned,2> VN, WN; + for (MVT T : VS) + VN.insert(T.isVector() ? T.getVectorNumElements() : 0); + for (MVT T : WS) + WN.insert(T.isVector() ? T.getVectorNumElements() : 0); + + Changed |= berase_if(VS, std::bind(NoLength, WN, std::placeholders::_1)); + Changed |= berase_if(WS, std::bind(NoLength, VN, std::placeholders::_1)); + } + return Changed; +} + +/// 1. Ensure that for each type T in A, there exists a type U in B, +/// such that T and U have equal size in bits. +/// 2. Ensure that for each type U in B, there exists a type T in A +/// such that T and U have equal size in bits (reverse of 1). +bool TypeInfer::EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B) { + ValidateOnExit _1(A, *this), _2(B, *this); + if (TP.hasError()) + return false; + bool Changed = false; + if (A.empty()) + Changed |= EnforceAny(A); + if (B.empty()) + Changed |= EnforceAny(B); + + auto NoSize = [](const SmallSet<unsigned,2> &Sizes, MVT T) -> bool { + return !Sizes.count(T.getSizeInBits()); + }; + + for (unsigned M : union_modes(A, B)) { + TypeSetByHwMode::SetType &AS = A.get(M); + TypeSetByHwMode::SetType &BS = B.get(M); + SmallSet<unsigned,2> AN, BN; + + for (MVT T : AS) + AN.insert(T.getSizeInBits()); + for (MVT T : BS) + BN.insert(T.getSizeInBits()); + + Changed |= berase_if(AS, std::bind(NoSize, BN, std::placeholders::_1)); + Changed |= berase_if(BS, std::bind(NoSize, AN, std::placeholders::_1)); + } + + return Changed; +} + +void TypeInfer::expandOverloads(TypeSetByHwMode &VTS) { + ValidateOnExit _1(VTS, *this); + const TypeSetByHwMode &Legal = getLegalTypes(); + assert(Legal.isDefaultOnly() && "Default-mode only expected"); + const TypeSetByHwMode::SetType &LegalTypes = Legal.get(DefaultMode); + + for (auto &I : VTS) + expandOverloads(I.second, LegalTypes); +} + +void TypeInfer::expandOverloads(TypeSetByHwMode::SetType &Out, + const TypeSetByHwMode::SetType &Legal) { + std::set<MVT> Ovs; + for (MVT T : Out) { + if (!T.isOverloaded()) + continue; + + Ovs.insert(T); + // MachineValueTypeSet allows iteration and erasing. + Out.erase(T); + } + + for (MVT Ov : Ovs) { + switch (Ov.SimpleTy) { + case MVT::iPTRAny: + Out.insert(MVT::iPTR); + return; + case MVT::iAny: + for (MVT T : MVT::integer_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + for (MVT T : MVT::integer_fixedlen_vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + for (MVT T : MVT::integer_scalable_vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + return; + case MVT::fAny: + for (MVT T : MVT::fp_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + for (MVT T : MVT::fp_fixedlen_vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + for (MVT T : MVT::fp_scalable_vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + return; + case MVT::vAny: + for (MVT T : MVT::vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + return; + case MVT::Any: + for (MVT T : MVT::all_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + return; + default: + break; + } + } +} + +const TypeSetByHwMode &TypeInfer::getLegalTypes() { + if (!LegalTypesCached) { + TypeSetByHwMode::SetType &LegalTypes = LegalCache.getOrCreate(DefaultMode); + // Stuff all types from all modes into the default mode. + const TypeSetByHwMode <S = TP.getDAGPatterns().getLegalTypes(); + for (const auto &I : LTS) + LegalTypes.insert(I.second); + LegalTypesCached = true; + } + assert(LegalCache.isDefaultOnly() && "Default-mode only expected"); + return LegalCache; +} + +#ifndef NDEBUG +TypeInfer::ValidateOnExit::~ValidateOnExit() { + if (Infer.Validate && !VTS.validate()) { + dbgs() << "Type set is empty for each HW mode:\n" + "possible type contradiction in the pattern below " + "(use -print-records with llvm-tblgen to see all " + "expanded records).\n"; + Infer.TP.dump(); + llvm_unreachable(nullptr); + } +} +#endif + + +//===----------------------------------------------------------------------===// +// ScopedName Implementation +//===----------------------------------------------------------------------===// + +bool ScopedName::operator==(const ScopedName &o) const { + return Scope == o.Scope && Identifier == o.Identifier; +} + +bool ScopedName::operator!=(const ScopedName &o) const { + return !(*this == o); +} + + +//===----------------------------------------------------------------------===// +// TreePredicateFn Implementation +//===----------------------------------------------------------------------===// + +/// TreePredicateFn constructor. Here 'N' is a subclass of PatFrag. +TreePredicateFn::TreePredicateFn(TreePattern *N) : PatFragRec(N) { + assert( + (!hasPredCode() || !hasImmCode()) && + ".td file corrupt: can't have a node predicate *and* an imm predicate"); +} + +bool TreePredicateFn::hasPredCode() const { + return isLoad() || isStore() || isAtomic() || + !PatFragRec->getRecord()->getValueAsString("PredicateCode").empty(); +} + +std::string TreePredicateFn::getPredCode() const { + std::string Code = ""; + + if (!isLoad() && !isStore() && !isAtomic()) { + Record *MemoryVT = getMemoryVT(); + + if (MemoryVT) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "MemoryVT requires IsLoad or IsStore"); + } + + if (!isLoad() && !isStore()) { + if (isUnindexed()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsUnindexed requires IsLoad or IsStore"); + + Record *ScalarMemoryVT = getScalarMemoryVT(); + + if (ScalarMemoryVT) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "ScalarMemoryVT requires IsLoad or IsStore"); + } + + if (isLoad() + isStore() + isAtomic() > 1) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsLoad, IsStore, and IsAtomic are mutually exclusive"); + + if (isLoad()) { + if (!isUnindexed() && !isNonExtLoad() && !isAnyExtLoad() && + !isSignExtLoad() && !isZeroExtLoad() && getMemoryVT() == nullptr && + getScalarMemoryVT() == nullptr && getAddressSpaces() == nullptr && + getMinAlignment() < 1) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsLoad cannot be used by itself"); + } else { + if (isNonExtLoad()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsNonExtLoad requires IsLoad"); + if (isAnyExtLoad()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAnyExtLoad requires IsLoad"); + if (isSignExtLoad()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsSignExtLoad requires IsLoad"); + if (isZeroExtLoad()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsZeroExtLoad requires IsLoad"); + } + + if (isStore()) { + if (!isUnindexed() && !isTruncStore() && !isNonTruncStore() && + getMemoryVT() == nullptr && getScalarMemoryVT() == nullptr && + getAddressSpaces() == nullptr && getMinAlignment() < 1) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsStore cannot be used by itself"); + } else { + if (isNonTruncStore()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsNonTruncStore requires IsStore"); + if (isTruncStore()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsTruncStore requires IsStore"); + } + + if (isAtomic()) { + if (getMemoryVT() == nullptr && !isAtomicOrderingMonotonic() && + getAddressSpaces() == nullptr && + !isAtomicOrderingAcquire() && !isAtomicOrderingRelease() && + !isAtomicOrderingAcquireRelease() && + !isAtomicOrderingSequentiallyConsistent() && + !isAtomicOrderingAcquireOrStronger() && + !isAtomicOrderingReleaseOrStronger() && + !isAtomicOrderingWeakerThanAcquire() && + !isAtomicOrderingWeakerThanRelease()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAtomic cannot be used by itself"); + } else { + if (isAtomicOrderingMonotonic()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAtomicOrderingMonotonic requires IsAtomic"); + if (isAtomicOrderingAcquire()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAtomicOrderingAcquire requires IsAtomic"); + if (isAtomicOrderingRelease()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAtomicOrderingRelease requires IsAtomic"); + if (isAtomicOrderingAcquireRelease()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAtomicOrderingAcquireRelease requires IsAtomic"); + if (isAtomicOrderingSequentiallyConsistent()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAtomicOrderingSequentiallyConsistent requires IsAtomic"); + if (isAtomicOrderingAcquireOrStronger()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAtomicOrderingAcquireOrStronger requires IsAtomic"); + if (isAtomicOrderingReleaseOrStronger()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAtomicOrderingReleaseOrStronger requires IsAtomic"); + if (isAtomicOrderingWeakerThanAcquire()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAtomicOrderingWeakerThanAcquire requires IsAtomic"); + } + + if (isLoad() || isStore() || isAtomic()) { + if (ListInit *AddressSpaces = getAddressSpaces()) { + Code += "unsigned AddrSpace = cast<MemSDNode>(N)->getAddressSpace();\n" + " if ("; + + bool First = true; + for (Init *Val : AddressSpaces->getValues()) { + if (First) + First = false; + else + Code += " && "; + + IntInit *IntVal = dyn_cast<IntInit>(Val); + if (!IntVal) { + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "AddressSpaces element must be integer"); + } + + Code += "AddrSpace != " + utostr(IntVal->getValue()); + } + + Code += ")\nreturn false;\n"; + } + + int64_t MinAlign = getMinAlignment(); + if (MinAlign > 0) { + Code += "if (cast<MemSDNode>(N)->getAlign() < Align("; + Code += utostr(MinAlign); + Code += "))\nreturn false;\n"; + } + + Record *MemoryVT = getMemoryVT(); + + if (MemoryVT) + Code += ("if (cast<MemSDNode>(N)->getMemoryVT() != MVT::" + + MemoryVT->getName() + ") return false;\n") + .str(); + } + + if (isAtomic() && isAtomicOrderingMonotonic()) + Code += "if (cast<AtomicSDNode>(N)->getOrdering() != " + "AtomicOrdering::Monotonic) return false;\n"; + if (isAtomic() && isAtomicOrderingAcquire()) + Code += "if (cast<AtomicSDNode>(N)->getOrdering() != " + "AtomicOrdering::Acquire) return false;\n"; + if (isAtomic() && isAtomicOrderingRelease()) + Code += "if (cast<AtomicSDNode>(N)->getOrdering() != " + "AtomicOrdering::Release) return false;\n"; + if (isAtomic() && isAtomicOrderingAcquireRelease()) + Code += "if (cast<AtomicSDNode>(N)->getOrdering() != " + "AtomicOrdering::AcquireRelease) return false;\n"; + if (isAtomic() && isAtomicOrderingSequentiallyConsistent()) + Code += "if (cast<AtomicSDNode>(N)->getOrdering() != " + "AtomicOrdering::SequentiallyConsistent) return false;\n"; + + if (isAtomic() && isAtomicOrderingAcquireOrStronger()) + Code += "if (!isAcquireOrStronger(cast<AtomicSDNode>(N)->getOrdering())) " + "return false;\n"; + if (isAtomic() && isAtomicOrderingWeakerThanAcquire()) + Code += "if (isAcquireOrStronger(cast<AtomicSDNode>(N)->getOrdering())) " + "return false;\n"; + + if (isAtomic() && isAtomicOrderingReleaseOrStronger()) + Code += "if (!isReleaseOrStronger(cast<AtomicSDNode>(N)->getOrdering())) " + "return false;\n"; + if (isAtomic() && isAtomicOrderingWeakerThanRelease()) + Code += "if (isReleaseOrStronger(cast<AtomicSDNode>(N)->getOrdering())) " + "return false;\n"; + + if (isLoad() || isStore()) { + StringRef SDNodeName = isLoad() ? "LoadSDNode" : "StoreSDNode"; + + if (isUnindexed()) + Code += ("if (cast<" + SDNodeName + + ">(N)->getAddressingMode() != ISD::UNINDEXED) " + "return false;\n") + .str(); + + if (isLoad()) { + if ((isNonExtLoad() + isAnyExtLoad() + isSignExtLoad() + + isZeroExtLoad()) > 1) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsNonExtLoad, IsAnyExtLoad, IsSignExtLoad, and " + "IsZeroExtLoad are mutually exclusive"); + if (isNonExtLoad()) + Code += "if (cast<LoadSDNode>(N)->getExtensionType() != " + "ISD::NON_EXTLOAD) return false;\n"; + if (isAnyExtLoad()) + Code += "if (cast<LoadSDNode>(N)->getExtensionType() != ISD::EXTLOAD) " + "return false;\n"; + if (isSignExtLoad()) + Code += "if (cast<LoadSDNode>(N)->getExtensionType() != ISD::SEXTLOAD) " + "return false;\n"; + if (isZeroExtLoad()) + Code += "if (cast<LoadSDNode>(N)->getExtensionType() != ISD::ZEXTLOAD) " + "return false;\n"; + } else { + if ((isNonTruncStore() + isTruncStore()) > 1) + PrintFatalError( + getOrigPatFragRecord()->getRecord()->getLoc(), + "IsNonTruncStore, and IsTruncStore are mutually exclusive"); + if (isNonTruncStore()) + Code += + " if (cast<StoreSDNode>(N)->isTruncatingStore()) return false;\n"; + if (isTruncStore()) + Code += + " if (!cast<StoreSDNode>(N)->isTruncatingStore()) return false;\n"; + } + + Record *ScalarMemoryVT = getScalarMemoryVT(); + + if (ScalarMemoryVT) + Code += ("if (cast<" + SDNodeName + + ">(N)->getMemoryVT().getScalarType() != MVT::" + + ScalarMemoryVT->getName() + ") return false;\n") + .str(); + } + + std::string PredicateCode = + std::string(PatFragRec->getRecord()->getValueAsString("PredicateCode")); + + Code += PredicateCode; + + if (PredicateCode.empty() && !Code.empty()) + Code += "return true;\n"; + + return Code; +} + +bool TreePredicateFn::hasImmCode() const { + return !PatFragRec->getRecord()->getValueAsString("ImmediateCode").empty(); +} + +std::string TreePredicateFn::getImmCode() const { + return std::string( + PatFragRec->getRecord()->getValueAsString("ImmediateCode")); +} + +bool TreePredicateFn::immCodeUsesAPInt() const { + return getOrigPatFragRecord()->getRecord()->getValueAsBit("IsAPInt"); +} + +bool TreePredicateFn::immCodeUsesAPFloat() const { + bool Unset; + // The return value will be false when IsAPFloat is unset. + return getOrigPatFragRecord()->getRecord()->getValueAsBitOrUnset("IsAPFloat", + Unset); +} + +bool TreePredicateFn::isPredefinedPredicateEqualTo(StringRef Field, + bool Value) const { + bool Unset; + bool Result = + getOrigPatFragRecord()->getRecord()->getValueAsBitOrUnset(Field, Unset); + if (Unset) + return false; + return Result == Value; +} +bool TreePredicateFn::usesOperands() const { + return isPredefinedPredicateEqualTo("PredicateCodeUsesOperands", true); +} +bool TreePredicateFn::isLoad() const { + return isPredefinedPredicateEqualTo("IsLoad", true); +} +bool TreePredicateFn::isStore() const { + return isPredefinedPredicateEqualTo("IsStore", true); +} +bool TreePredicateFn::isAtomic() const { + return isPredefinedPredicateEqualTo("IsAtomic", true); +} +bool TreePredicateFn::isUnindexed() const { + return isPredefinedPredicateEqualTo("IsUnindexed", true); +} +bool TreePredicateFn::isNonExtLoad() const { + return isPredefinedPredicateEqualTo("IsNonExtLoad", true); +} +bool TreePredicateFn::isAnyExtLoad() const { + return isPredefinedPredicateEqualTo("IsAnyExtLoad", true); +} +bool TreePredicateFn::isSignExtLoad() const { + return isPredefinedPredicateEqualTo("IsSignExtLoad", true); +} +bool TreePredicateFn::isZeroExtLoad() const { + return isPredefinedPredicateEqualTo("IsZeroExtLoad", true); +} +bool TreePredicateFn::isNonTruncStore() const { + return isPredefinedPredicateEqualTo("IsTruncStore", false); +} +bool TreePredicateFn::isTruncStore() const { + return isPredefinedPredicateEqualTo("IsTruncStore", true); +} +bool TreePredicateFn::isAtomicOrderingMonotonic() const { + return isPredefinedPredicateEqualTo("IsAtomicOrderingMonotonic", true); +} +bool TreePredicateFn::isAtomicOrderingAcquire() const { + return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquire", true); +} +bool TreePredicateFn::isAtomicOrderingRelease() const { + return isPredefinedPredicateEqualTo("IsAtomicOrderingRelease", true); +} +bool TreePredicateFn::isAtomicOrderingAcquireRelease() const { + return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquireRelease", true); +} +bool TreePredicateFn::isAtomicOrderingSequentiallyConsistent() const { + return isPredefinedPredicateEqualTo("IsAtomicOrderingSequentiallyConsistent", + true); +} +bool TreePredicateFn::isAtomicOrderingAcquireOrStronger() const { + return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquireOrStronger", true); +} +bool TreePredicateFn::isAtomicOrderingWeakerThanAcquire() const { + return isPredefinedPredicateEqualTo("IsAtomicOrderingAcquireOrStronger", false); +} +bool TreePredicateFn::isAtomicOrderingReleaseOrStronger() const { + return isPredefinedPredicateEqualTo("IsAtomicOrderingReleaseOrStronger", true); +} +bool TreePredicateFn::isAtomicOrderingWeakerThanRelease() const { + return isPredefinedPredicateEqualTo("IsAtomicOrderingReleaseOrStronger", false); +} +Record *TreePredicateFn::getMemoryVT() const { + Record *R = getOrigPatFragRecord()->getRecord(); + if (R->isValueUnset("MemoryVT")) + return nullptr; + return R->getValueAsDef("MemoryVT"); +} + +ListInit *TreePredicateFn::getAddressSpaces() const { + Record *R = getOrigPatFragRecord()->getRecord(); + if (R->isValueUnset("AddressSpaces")) + return nullptr; + return R->getValueAsListInit("AddressSpaces"); +} + +int64_t TreePredicateFn::getMinAlignment() const { + Record *R = getOrigPatFragRecord()->getRecord(); + if (R->isValueUnset("MinAlignment")) + return 0; + return R->getValueAsInt("MinAlignment"); +} + +Record *TreePredicateFn::getScalarMemoryVT() const { + Record *R = getOrigPatFragRecord()->getRecord(); + if (R->isValueUnset("ScalarMemoryVT")) + return nullptr; + return R->getValueAsDef("ScalarMemoryVT"); +} +bool TreePredicateFn::hasGISelPredicateCode() const { + return !PatFragRec->getRecord() + ->getValueAsString("GISelPredicateCode") + .empty(); +} +std::string TreePredicateFn::getGISelPredicateCode() const { + return std::string( + PatFragRec->getRecord()->getValueAsString("GISelPredicateCode")); +} + +StringRef TreePredicateFn::getImmType() const { + if (immCodeUsesAPInt()) + return "const APInt &"; + if (immCodeUsesAPFloat()) + return "const APFloat &"; + return "int64_t"; +} + +StringRef TreePredicateFn::getImmTypeIdentifier() const { + if (immCodeUsesAPInt()) + return "APInt"; + else if (immCodeUsesAPFloat()) + return "APFloat"; + return "I64"; +} + +/// isAlwaysTrue - Return true if this is a noop predicate. +bool TreePredicateFn::isAlwaysTrue() const { + return !hasPredCode() && !hasImmCode(); +} + +/// Return the name to use in the generated code to reference this, this is +/// "Predicate_foo" if from a pattern fragment "foo". +std::string TreePredicateFn::getFnName() const { + return "Predicate_" + PatFragRec->getRecord()->getName().str(); +} + +/// getCodeToRunOnSDNode - Return the code for the function body that +/// evaluates this predicate. The argument is expected to be in "Node", +/// not N. This handles casting and conversion to a concrete node type as +/// appropriate. +std::string TreePredicateFn::getCodeToRunOnSDNode() const { + // Handle immediate predicates first. + std::string ImmCode = getImmCode(); + if (!ImmCode.empty()) { + if (isLoad()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsLoad cannot be used with ImmLeaf or its subclasses"); + if (isStore()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "IsStore cannot be used with ImmLeaf or its subclasses"); + if (isUnindexed()) + PrintFatalError( + getOrigPatFragRecord()->getRecord()->getLoc(), + "IsUnindexed cannot be used with ImmLeaf or its subclasses"); + if (isNonExtLoad()) + PrintFatalError( + getOrigPatFragRecord()->getRecord()->getLoc(), + "IsNonExtLoad cannot be used with ImmLeaf or its subclasses"); + if (isAnyExtLoad()) + PrintFatalError( + getOrigPatFragRecord()->getRecord()->getLoc(), + "IsAnyExtLoad cannot be used with ImmLeaf or its subclasses"); + if (isSignExtLoad()) + PrintFatalError( + getOrigPatFragRecord()->getRecord()->getLoc(), + "IsSignExtLoad cannot be used with ImmLeaf or its subclasses"); + if (isZeroExtLoad()) + PrintFatalError( + getOrigPatFragRecord()->getRecord()->getLoc(), + "IsZeroExtLoad cannot be used with ImmLeaf or its subclasses"); + if (isNonTruncStore()) + PrintFatalError( + getOrigPatFragRecord()->getRecord()->getLoc(), + "IsNonTruncStore cannot be used with ImmLeaf or its subclasses"); + if (isTruncStore()) + PrintFatalError( + getOrigPatFragRecord()->getRecord()->getLoc(), + "IsTruncStore cannot be used with ImmLeaf or its subclasses"); + if (getMemoryVT()) + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "MemoryVT cannot be used with ImmLeaf or its subclasses"); + if (getScalarMemoryVT()) + PrintFatalError( + getOrigPatFragRecord()->getRecord()->getLoc(), + "ScalarMemoryVT cannot be used with ImmLeaf or its subclasses"); + + std::string Result = (" " + getImmType() + " Imm = ").str(); + if (immCodeUsesAPFloat()) + Result += "cast<ConstantFPSDNode>(Node)->getValueAPF();\n"; + else if (immCodeUsesAPInt()) + Result += "cast<ConstantSDNode>(Node)->getAPIntValue();\n"; + else + Result += "cast<ConstantSDNode>(Node)->getSExtValue();\n"; + return Result + ImmCode; + } + + // Handle arbitrary node predicates. + assert(hasPredCode() && "Don't have any predicate code!"); + + // If this is using PatFrags, there are multiple trees to search. They should + // all have the same class. FIXME: Is there a way to find a common + // superclass? + StringRef ClassName; + for (const auto &Tree : PatFragRec->getTrees()) { + StringRef TreeClassName; + if (Tree->isLeaf()) + TreeClassName = "SDNode"; + else { + Record *Op = Tree->getOperator(); + const SDNodeInfo &Info = PatFragRec->getDAGPatterns().getSDNodeInfo(Op); + TreeClassName = Info.getSDClassName(); + } + + if (ClassName.empty()) + ClassName = TreeClassName; + else if (ClassName != TreeClassName) { + PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(), + "PatFrags trees do not have consistent class"); + } + } + + std::string Result; + if (ClassName == "SDNode") + Result = " SDNode *N = Node;\n"; + else + Result = " auto *N = cast<" + ClassName.str() + ">(Node);\n"; + + return (Twine(Result) + " (void)N;\n" + getPredCode()).str(); +} + +//===----------------------------------------------------------------------===// +// PatternToMatch implementation +// + +static bool isImmAllOnesAllZerosMatch(const TreePatternNode *P) { + if (!P->isLeaf()) + return false; + DefInit *DI = dyn_cast<DefInit>(P->getLeafValue()); + if (!DI) + return false; + + Record *R = DI->getDef(); + return R->getName() == "immAllOnesV" || R->getName() == "immAllZerosV"; +} + +/// getPatternSize - Return the 'size' of this pattern. We want to match large +/// patterns before small ones. This is used to determine the size of a +/// pattern. +static unsigned getPatternSize(const TreePatternNode *P, + const CodeGenDAGPatterns &CGP) { + unsigned Size = 3; // The node itself. + // If the root node is a ConstantSDNode, increases its size. + // e.g. (set R32:$dst, 0). + if (P->isLeaf() && isa<IntInit>(P->getLeafValue())) + Size += 2; + + if (const ComplexPattern *AM = P->getComplexPatternInfo(CGP)) { + Size += AM->getComplexity(); + // We don't want to count any children twice, so return early. + return Size; + } + + // If this node has some predicate function that must match, it adds to the + // complexity of this node. + if (!P->getPredicateCalls().empty()) + ++Size; + + // Count children in the count if they are also nodes. + for (unsigned i = 0, e = P->getNumChildren(); i != e; ++i) { + const TreePatternNode *Child = P->getChild(i); + if (!Child->isLeaf() && Child->getNumTypes()) { + const TypeSetByHwMode &T0 = Child->getExtType(0); + // At this point, all variable type sets should be simple, i.e. only + // have a default mode. + if (T0.getMachineValueType() != MVT::Other) { + Size += getPatternSize(Child, CGP); + continue; + } + } + if (Child->isLeaf()) { + if (isa<IntInit>(Child->getLeafValue())) + Size += 5; // Matches a ConstantSDNode (+3) and a specific value (+2). + else if (Child->getComplexPatternInfo(CGP)) + Size += getPatternSize(Child, CGP); + else if (isImmAllOnesAllZerosMatch(Child)) + Size += 4; // Matches a build_vector(+3) and a predicate (+1). + else if (!Child->getPredicateCalls().empty()) + ++Size; + } + } + + return Size; +} + +/// Compute the complexity metric for the input pattern. This roughly +/// corresponds to the number of nodes that are covered. +int PatternToMatch:: +getPatternComplexity(const CodeGenDAGPatterns &CGP) const { + return getPatternSize(getSrcPattern(), CGP) + getAddedComplexity(); +} + +/// getPredicateCheck - Return a single string containing all of this +/// pattern's predicates concatenated with "&&" operators. +/// +std::string PatternToMatch::getPredicateCheck() const { + SmallVector<const Predicate*,4> PredList; + for (const Predicate &P : Predicates) { + if (!P.getCondString().empty()) + PredList.push_back(&P); + } + llvm::sort(PredList, deref<std::less<>>()); + + std::string Check; + for (unsigned i = 0, e = PredList.size(); i != e; ++i) { + if (i != 0) + Check += " && "; + Check += '(' + PredList[i]->getCondString() + ')'; + } + return Check; +} + +//===----------------------------------------------------------------------===// +// SDTypeConstraint implementation +// + +SDTypeConstraint::SDTypeConstraint(Record *R, const CodeGenHwModes &CGH) { + OperandNo = R->getValueAsInt("OperandNum"); + + if (R->isSubClassOf("SDTCisVT")) { + ConstraintType = SDTCisVT; + VVT = getValueTypeByHwMode(R->getValueAsDef("VT"), CGH); + for (const auto &P : VVT) + if (P.second == MVT::isVoid) + PrintFatalError(R->getLoc(), "Cannot use 'Void' as type to SDTCisVT"); + } else if (R->isSubClassOf("SDTCisPtrTy")) { + ConstraintType = SDTCisPtrTy; + } else if (R->isSubClassOf("SDTCisInt")) { + ConstraintType = SDTCisInt; + } else if (R->isSubClassOf("SDTCisFP")) { + ConstraintType = SDTCisFP; + } else if (R->isSubClassOf("SDTCisVec")) { + ConstraintType = SDTCisVec; + } else if (R->isSubClassOf("SDTCisSameAs")) { + ConstraintType = SDTCisSameAs; + x.SDTCisSameAs_Info.OtherOperandNum = R->getValueAsInt("OtherOperandNum"); + } else if (R->isSubClassOf("SDTCisVTSmallerThanOp")) { + ConstraintType = SDTCisVTSmallerThanOp; + x.SDTCisVTSmallerThanOp_Info.OtherOperandNum = + R->getValueAsInt("OtherOperandNum"); + } else if (R->isSubClassOf("SDTCisOpSmallerThanOp")) { + ConstraintType = SDTCisOpSmallerThanOp; + x.SDTCisOpSmallerThanOp_Info.BigOperandNum = + R->getValueAsInt("BigOperandNum"); + } else if (R->isSubClassOf("SDTCisEltOfVec")) { + ConstraintType = SDTCisEltOfVec; + x.SDTCisEltOfVec_Info.OtherOperandNum = R->getValueAsInt("OtherOpNum"); + } else if (R->isSubClassOf("SDTCisSubVecOfVec")) { + ConstraintType = SDTCisSubVecOfVec; + x.SDTCisSubVecOfVec_Info.OtherOperandNum = + R->getValueAsInt("OtherOpNum"); + } else if (R->isSubClassOf("SDTCVecEltisVT")) { + ConstraintType = SDTCVecEltisVT; + VVT = getValueTypeByHwMode(R->getValueAsDef("VT"), CGH); + for (const auto &P : VVT) { + MVT T = P.second; + if (T.isVector()) + PrintFatalError(R->getLoc(), + "Cannot use vector type as SDTCVecEltisVT"); + if (!T.isInteger() && !T.isFloatingPoint()) + PrintFatalError(R->getLoc(), "Must use integer or floating point type " + "as SDTCVecEltisVT"); + } + } else if (R->isSubClassOf("SDTCisSameNumEltsAs")) { + ConstraintType = SDTCisSameNumEltsAs; + x.SDTCisSameNumEltsAs_Info.OtherOperandNum = + R->getValueAsInt("OtherOperandNum"); + } else if (R->isSubClassOf("SDTCisSameSizeAs")) { + ConstraintType = SDTCisSameSizeAs; + x.SDTCisSameSizeAs_Info.OtherOperandNum = + R->getValueAsInt("OtherOperandNum"); + } else { + PrintFatalError(R->getLoc(), + "Unrecognized SDTypeConstraint '" + R->getName() + "'!\n"); + } +} + +/// getOperandNum - Return the node corresponding to operand #OpNo in tree +/// N, and the result number in ResNo. +static TreePatternNode *getOperandNum(unsigned OpNo, TreePatternNode *N, + const SDNodeInfo &NodeInfo, + unsigned &ResNo) { + unsigned NumResults = NodeInfo.getNumResults(); + if (OpNo < NumResults) { + ResNo = OpNo; + return N; + } + + OpNo -= NumResults; + + if (OpNo >= N->getNumChildren()) { + std::string S; + raw_string_ostream OS(S); + OS << "Invalid operand number in type constraint " + << (OpNo+NumResults) << " "; + N->print(OS); + PrintFatalError(OS.str()); + } + + return N->getChild(OpNo); +} + +/// ApplyTypeConstraint - Given a node in a pattern, apply this type +/// constraint to the nodes operands. This returns true if it makes a +/// change, false otherwise. If a type contradiction is found, flag an error. +bool SDTypeConstraint::ApplyTypeConstraint(TreePatternNode *N, + const SDNodeInfo &NodeInfo, + TreePattern &TP) const { + if (TP.hasError()) + return false; + + unsigned ResNo = 0; // The result number being referenced. + TreePatternNode *NodeToApply = getOperandNum(OperandNo, N, NodeInfo, ResNo); + TypeInfer &TI = TP.getInfer(); + + switch (ConstraintType) { + case SDTCisVT: + // Operand must be a particular type. + return NodeToApply->UpdateNodeType(ResNo, VVT, TP); + case SDTCisPtrTy: + // Operand must be same as target pointer type. + return NodeToApply->UpdateNodeType(ResNo, MVT::iPTR, TP); + case SDTCisInt: + // Require it to be one of the legal integer VTs. + return TI.EnforceInteger(NodeToApply->getExtType(ResNo)); + case SDTCisFP: + // Require it to be one of the legal fp VTs. + return TI.EnforceFloatingPoint(NodeToApply->getExtType(ResNo)); + case SDTCisVec: + // Require it to be one of the legal vector VTs. + return TI.EnforceVector(NodeToApply->getExtType(ResNo)); + case SDTCisSameAs: { + unsigned OResNo = 0; + TreePatternNode *OtherNode = + getOperandNum(x.SDTCisSameAs_Info.OtherOperandNum, N, NodeInfo, OResNo); + return NodeToApply->UpdateNodeType(ResNo, OtherNode->getExtType(OResNo),TP)| + OtherNode->UpdateNodeType(OResNo,NodeToApply->getExtType(ResNo),TP); + } + case SDTCisVTSmallerThanOp: { + // The NodeToApply must be a leaf node that is a VT. OtherOperandNum must + // have an integer type that is smaller than the VT. + if (!NodeToApply->isLeaf() || + !isa<DefInit>(NodeToApply->getLeafValue()) || + !static_cast<DefInit*>(NodeToApply->getLeafValue())->getDef() + ->isSubClassOf("ValueType")) { + TP.error(N->getOperator()->getName() + " expects a VT operand!"); + return false; + } + DefInit *DI = static_cast<DefInit*>(NodeToApply->getLeafValue()); + const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); + auto VVT = getValueTypeByHwMode(DI->getDef(), T.getHwModes()); + TypeSetByHwMode TypeListTmp(VVT); + + unsigned OResNo = 0; + TreePatternNode *OtherNode = + getOperandNum(x.SDTCisVTSmallerThanOp_Info.OtherOperandNum, N, NodeInfo, + OResNo); + + return TI.EnforceSmallerThan(TypeListTmp, OtherNode->getExtType(OResNo)); + } + case SDTCisOpSmallerThanOp: { + unsigned BResNo = 0; + TreePatternNode *BigOperand = + getOperandNum(x.SDTCisOpSmallerThanOp_Info.BigOperandNum, N, NodeInfo, + BResNo); + return TI.EnforceSmallerThan(NodeToApply->getExtType(ResNo), + BigOperand->getExtType(BResNo)); + } + case SDTCisEltOfVec: { + unsigned VResNo = 0; + TreePatternNode *VecOperand = + getOperandNum(x.SDTCisEltOfVec_Info.OtherOperandNum, N, NodeInfo, + VResNo); + // Filter vector types out of VecOperand that don't have the right element + // type. + return TI.EnforceVectorEltTypeIs(VecOperand->getExtType(VResNo), + NodeToApply->getExtType(ResNo)); + } + case SDTCisSubVecOfVec: { + unsigned VResNo = 0; + TreePatternNode *BigVecOperand = + getOperandNum(x.SDTCisSubVecOfVec_Info.OtherOperandNum, N, NodeInfo, + VResNo); + + // Filter vector types out of BigVecOperand that don't have the + // right subvector type. + return TI.EnforceVectorSubVectorTypeIs(BigVecOperand->getExtType(VResNo), + NodeToApply->getExtType(ResNo)); + } + case SDTCVecEltisVT: { + return TI.EnforceVectorEltTypeIs(NodeToApply->getExtType(ResNo), VVT); + } + case SDTCisSameNumEltsAs: { + unsigned OResNo = 0; + TreePatternNode *OtherNode = + getOperandNum(x.SDTCisSameNumEltsAs_Info.OtherOperandNum, + N, NodeInfo, OResNo); + return TI.EnforceSameNumElts(OtherNode->getExtType(OResNo), + NodeToApply->getExtType(ResNo)); + } + case SDTCisSameSizeAs: { + unsigned OResNo = 0; + TreePatternNode *OtherNode = + getOperandNum(x.SDTCisSameSizeAs_Info.OtherOperandNum, + N, NodeInfo, OResNo); + return TI.EnforceSameSize(OtherNode->getExtType(OResNo), + NodeToApply->getExtType(ResNo)); + } + } + llvm_unreachable("Invalid ConstraintType!"); +} + +// Update the node type to match an instruction operand or result as specified +// in the ins or outs lists on the instruction definition. Return true if the +// type was actually changed. +bool TreePatternNode::UpdateNodeTypeFromInst(unsigned ResNo, + Record *Operand, + TreePattern &TP) { + // The 'unknown' operand indicates that types should be inferred from the + // context. + if (Operand->isSubClassOf("unknown_class")) + return false; + + // The Operand class specifies a type directly. + if (Operand->isSubClassOf("Operand")) { + Record *R = Operand->getValueAsDef("Type"); + const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); + return UpdateNodeType(ResNo, getValueTypeByHwMode(R, T.getHwModes()), TP); + } + + // PointerLikeRegClass has a type that is determined at runtime. + if (Operand->isSubClassOf("PointerLikeRegClass")) + return UpdateNodeType(ResNo, MVT::iPTR, TP); + + // Both RegisterClass and RegisterOperand operands derive their types from a + // register class def. + Record *RC = nullptr; + if (Operand->isSubClassOf("RegisterClass")) + RC = Operand; + else if (Operand->isSubClassOf("RegisterOperand")) + RC = Operand->getValueAsDef("RegClass"); + + assert(RC && "Unknown operand type"); + CodeGenTarget &Tgt = TP.getDAGPatterns().getTargetInfo(); + return UpdateNodeType(ResNo, Tgt.getRegisterClass(RC).getValueTypes(), TP); +} + +bool TreePatternNode::ContainsUnresolvedType(TreePattern &TP) const { + for (unsigned i = 0, e = Types.size(); i != e; ++i) + if (!TP.getInfer().isConcrete(Types[i], true)) + return true; + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) + if (getChild(i)->ContainsUnresolvedType(TP)) + return true; + return false; +} + +bool TreePatternNode::hasProperTypeByHwMode() const { + for (const TypeSetByHwMode &S : Types) + if (!S.isDefaultOnly()) + return true; + for (const TreePatternNodePtr &C : Children) + if (C->hasProperTypeByHwMode()) + return true; + return false; +} + +bool TreePatternNode::hasPossibleType() const { + for (const TypeSetByHwMode &S : Types) + if (!S.isPossible()) + return false; + for (const TreePatternNodePtr &C : Children) + if (!C->hasPossibleType()) + return false; + return true; +} + +bool TreePatternNode::setDefaultMode(unsigned Mode) { + for (TypeSetByHwMode &S : Types) { + S.makeSimple(Mode); + // Check if the selected mode had a type conflict. + if (S.get(DefaultMode).empty()) + return false; + } + for (const TreePatternNodePtr &C : Children) + if (!C->setDefaultMode(Mode)) + return false; + return true; +} + +//===----------------------------------------------------------------------===// +// SDNodeInfo implementation +// +SDNodeInfo::SDNodeInfo(Record *R, const CodeGenHwModes &CGH) : Def(R) { + EnumName = R->getValueAsString("Opcode"); + SDClassName = R->getValueAsString("SDClass"); + Record *TypeProfile = R->getValueAsDef("TypeProfile"); + NumResults = TypeProfile->getValueAsInt("NumResults"); + NumOperands = TypeProfile->getValueAsInt("NumOperands"); + + // Parse the properties. + Properties = parseSDPatternOperatorProperties(R); + + // Parse the type constraints. + std::vector<Record*> ConstraintList = + TypeProfile->getValueAsListOfDefs("Constraints"); + for (Record *R : ConstraintList) + TypeConstraints.emplace_back(R, CGH); +} + +/// getKnownType - If the type constraints on this node imply a fixed type +/// (e.g. all stores return void, etc), then return it as an +/// MVT::SimpleValueType. Otherwise, return EEVT::Other. +MVT::SimpleValueType SDNodeInfo::getKnownType(unsigned ResNo) const { + unsigned NumResults = getNumResults(); + assert(NumResults <= 1 && + "We only work with nodes with zero or one result so far!"); + assert(ResNo == 0 && "Only handles single result nodes so far"); + + for (const SDTypeConstraint &Constraint : TypeConstraints) { + // Make sure that this applies to the correct node result. + if (Constraint.OperandNo >= NumResults) // FIXME: need value # + continue; + + switch (Constraint.ConstraintType) { + default: break; + case SDTypeConstraint::SDTCisVT: + if (Constraint.VVT.isSimple()) + return Constraint.VVT.getSimple().SimpleTy; + break; + case SDTypeConstraint::SDTCisPtrTy: + return MVT::iPTR; + } + } + return MVT::Other; +} + +//===----------------------------------------------------------------------===// +// TreePatternNode implementation +// + +static unsigned GetNumNodeResults(Record *Operator, CodeGenDAGPatterns &CDP) { + if (Operator->getName() == "set" || + Operator->getName() == "implicit") + return 0; // All return nothing. + + if (Operator->isSubClassOf("Intrinsic")) + return CDP.getIntrinsic(Operator).IS.RetVTs.size(); + + if (Operator->isSubClassOf("SDNode")) + return CDP.getSDNodeInfo(Operator).getNumResults(); + + if (Operator->isSubClassOf("PatFrags")) { + // If we've already parsed this pattern fragment, get it. Otherwise, handle + // the forward reference case where one pattern fragment references another + // before it is processed. + if (TreePattern *PFRec = CDP.getPatternFragmentIfRead(Operator)) { + // The number of results of a fragment with alternative records is the + // maximum number of results across all alternatives. + unsigned NumResults = 0; + for (auto T : PFRec->getTrees()) + NumResults = std::max(NumResults, T->getNumTypes()); + return NumResults; + } + + ListInit *LI = Operator->getValueAsListInit("Fragments"); + assert(LI && "Invalid Fragment"); + unsigned NumResults = 0; + for (Init *I : LI->getValues()) { + Record *Op = nullptr; + if (DagInit *Dag = dyn_cast<DagInit>(I)) + if (DefInit *DI = dyn_cast<DefInit>(Dag->getOperator())) + Op = DI->getDef(); + assert(Op && "Invalid Fragment"); + NumResults = std::max(NumResults, GetNumNodeResults(Op, CDP)); + } + return NumResults; + } + + if (Operator->isSubClassOf("Instruction")) { + CodeGenInstruction &InstInfo = CDP.getTargetInfo().getInstruction(Operator); + + unsigned NumDefsToAdd = InstInfo.Operands.NumDefs; + + // Subtract any defaulted outputs. + for (unsigned i = 0; i != InstInfo.Operands.NumDefs; ++i) { + Record *OperandNode = InstInfo.Operands[i].Rec; + + if (OperandNode->isSubClassOf("OperandWithDefaultOps") && + !CDP.getDefaultOperand(OperandNode).DefaultOps.empty()) + --NumDefsToAdd; + } + + // Add on one implicit def if it has a resolvable type. + if (InstInfo.HasOneImplicitDefWithKnownVT(CDP.getTargetInfo()) !=MVT::Other) + ++NumDefsToAdd; + return NumDefsToAdd; + } + + if (Operator->isSubClassOf("SDNodeXForm")) + return 1; // FIXME: Generalize SDNodeXForm + + if (Operator->isSubClassOf("ValueType")) + return 1; // A type-cast of one result. + + if (Operator->isSubClassOf("ComplexPattern")) + return 1; + + errs() << *Operator; + PrintFatalError("Unhandled node in GetNumNodeResults"); +} + +void TreePatternNode::print(raw_ostream &OS) const { + if (isLeaf()) + OS << *getLeafValue(); + else + OS << '(' << getOperator()->getName(); + + for (unsigned i = 0, e = Types.size(); i != e; ++i) { + OS << ':'; + getExtType(i).writeToStream(OS); + } + + if (!isLeaf()) { + if (getNumChildren() != 0) { + OS << " "; + getChild(0)->print(OS); + for (unsigned i = 1, e = getNumChildren(); i != e; ++i) { + OS << ", "; + getChild(i)->print(OS); + } + } + OS << ")"; + } + + for (const TreePredicateCall &Pred : PredicateCalls) { + OS << "<<P:"; + if (Pred.Scope) + OS << Pred.Scope << ":"; + OS << Pred.Fn.getFnName() << ">>"; + } + if (TransformFn) + OS << "<<X:" << TransformFn->getName() << ">>"; + if (!getName().empty()) + OS << ":$" << getName(); + + for (const ScopedName &Name : NamesAsPredicateArg) + OS << ":$pred:" << Name.getScope() << ":" << Name.getIdentifier(); +} +void TreePatternNode::dump() const { + print(errs()); +} + +/// isIsomorphicTo - Return true if this node is recursively +/// isomorphic to the specified node. For this comparison, the node's +/// entire state is considered. The assigned name is ignored, since +/// nodes with differing names are considered isomorphic. However, if +/// the assigned name is present in the dependent variable set, then +/// the assigned name is considered significant and the node is +/// isomorphic if the names match. +bool TreePatternNode::isIsomorphicTo(const TreePatternNode *N, + const MultipleUseVarSet &DepVars) const { + if (N == this) return true; + if (N->isLeaf() != isLeaf() || getExtTypes() != N->getExtTypes() || + getPredicateCalls() != N->getPredicateCalls() || + getTransformFn() != N->getTransformFn()) + return false; + + if (isLeaf()) { + if (DefInit *DI = dyn_cast<DefInit>(getLeafValue())) { + if (DefInit *NDI = dyn_cast<DefInit>(N->getLeafValue())) { + return ((DI->getDef() == NDI->getDef()) + && (DepVars.find(getName()) == DepVars.end() + || getName() == N->getName())); + } + } + return getLeafValue() == N->getLeafValue(); + } + + if (N->getOperator() != getOperator() || + N->getNumChildren() != getNumChildren()) return false; + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) + if (!getChild(i)->isIsomorphicTo(N->getChild(i), DepVars)) + return false; + return true; +} + +/// clone - Make a copy of this tree and all of its children. +/// +TreePatternNodePtr TreePatternNode::clone() const { + TreePatternNodePtr New; + if (isLeaf()) { + New = std::make_shared<TreePatternNode>(getLeafValue(), getNumTypes()); + } else { + std::vector<TreePatternNodePtr> CChildren; + CChildren.reserve(Children.size()); + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) + CChildren.push_back(getChild(i)->clone()); + New = std::make_shared<TreePatternNode>(getOperator(), std::move(CChildren), + getNumTypes()); + } + New->setName(getName()); + New->setNamesAsPredicateArg(getNamesAsPredicateArg()); + New->Types = Types; + New->setPredicateCalls(getPredicateCalls()); + New->setTransformFn(getTransformFn()); + return New; +} + +/// RemoveAllTypes - Recursively strip all the types of this tree. +void TreePatternNode::RemoveAllTypes() { + // Reset to unknown type. + std::fill(Types.begin(), Types.end(), TypeSetByHwMode()); + if (isLeaf()) return; + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) + getChild(i)->RemoveAllTypes(); +} + + +/// SubstituteFormalArguments - Replace the formal arguments in this tree +/// with actual values specified by ArgMap. +void TreePatternNode::SubstituteFormalArguments( + std::map<std::string, TreePatternNodePtr> &ArgMap) { + if (isLeaf()) return; + + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) { + TreePatternNode *Child = getChild(i); + if (Child->isLeaf()) { + Init *Val = Child->getLeafValue(); + // Note that, when substituting into an output pattern, Val might be an + // UnsetInit. + if (isa<UnsetInit>(Val) || (isa<DefInit>(Val) && + cast<DefInit>(Val)->getDef()->getName() == "node")) { + // We found a use of a formal argument, replace it with its value. + TreePatternNodePtr NewChild = ArgMap[Child->getName()]; + assert(NewChild && "Couldn't find formal argument!"); + assert((Child->getPredicateCalls().empty() || + NewChild->getPredicateCalls() == Child->getPredicateCalls()) && + "Non-empty child predicate clobbered!"); + setChild(i, std::move(NewChild)); + } + } else { + getChild(i)->SubstituteFormalArguments(ArgMap); + } + } +} + + +/// InlinePatternFragments - If this pattern refers to any pattern +/// fragments, return the set of inlined versions (this can be more than +/// one if a PatFrags record has multiple alternatives). +void TreePatternNode::InlinePatternFragments( + TreePatternNodePtr T, TreePattern &TP, + std::vector<TreePatternNodePtr> &OutAlternatives) { + + if (TP.hasError()) + return; + + if (isLeaf()) { + OutAlternatives.push_back(T); // nothing to do. + return; + } + + Record *Op = getOperator(); + + if (!Op->isSubClassOf("PatFrags")) { + if (getNumChildren() == 0) { + OutAlternatives.push_back(T); + return; + } + + // Recursively inline children nodes. + std::vector<std::vector<TreePatternNodePtr> > ChildAlternatives; + ChildAlternatives.resize(getNumChildren()); + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) { + TreePatternNodePtr Child = getChildShared(i); + Child->InlinePatternFragments(Child, TP, ChildAlternatives[i]); + // If there are no alternatives for any child, there are no + // alternatives for this expression as whole. + if (ChildAlternatives[i].empty()) + return; + + for (auto NewChild : ChildAlternatives[i]) + assert((Child->getPredicateCalls().empty() || + NewChild->getPredicateCalls() == Child->getPredicateCalls()) && + "Non-empty child predicate clobbered!"); + } + + // The end result is an all-pairs construction of the resultant pattern. + std::vector<unsigned> Idxs; + Idxs.resize(ChildAlternatives.size()); + bool NotDone; + do { + // Create the variant and add it to the output list. + std::vector<TreePatternNodePtr> NewChildren; + for (unsigned i = 0, e = ChildAlternatives.size(); i != e; ++i) + NewChildren.push_back(ChildAlternatives[i][Idxs[i]]); + TreePatternNodePtr R = std::make_shared<TreePatternNode>( + getOperator(), std::move(NewChildren), getNumTypes()); + + // Copy over properties. + R->setName(getName()); + R->setNamesAsPredicateArg(getNamesAsPredicateArg()); + R->setPredicateCalls(getPredicateCalls()); + R->setTransformFn(getTransformFn()); + for (unsigned i = 0, e = getNumTypes(); i != e; ++i) + R->setType(i, getExtType(i)); + for (unsigned i = 0, e = getNumResults(); i != e; ++i) + R->setResultIndex(i, getResultIndex(i)); + + // Register alternative. + OutAlternatives.push_back(R); + + // Increment indices to the next permutation by incrementing the + // indices from last index backward, e.g., generate the sequence + // [0, 0], [0, 1], [1, 0], [1, 1]. + int IdxsIdx; + for (IdxsIdx = Idxs.size() - 1; IdxsIdx >= 0; --IdxsIdx) { + if (++Idxs[IdxsIdx] == ChildAlternatives[IdxsIdx].size()) + Idxs[IdxsIdx] = 0; + else + break; + } + NotDone = (IdxsIdx >= 0); + } while (NotDone); + + return; + } + + // Otherwise, we found a reference to a fragment. First, look up its + // TreePattern record. + TreePattern *Frag = TP.getDAGPatterns().getPatternFragment(Op); + + // Verify that we are passing the right number of operands. + if (Frag->getNumArgs() != Children.size()) { + TP.error("'" + Op->getName() + "' fragment requires " + + Twine(Frag->getNumArgs()) + " operands!"); + return; + } + + TreePredicateFn PredFn(Frag); + unsigned Scope = 0; + if (TreePredicateFn(Frag).usesOperands()) + Scope = TP.getDAGPatterns().allocateScope(); + + // Compute the map of formal to actual arguments. + std::map<std::string, TreePatternNodePtr> ArgMap; + for (unsigned i = 0, e = Frag->getNumArgs(); i != e; ++i) { + TreePatternNodePtr Child = getChildShared(i); + if (Scope != 0) { + Child = Child->clone(); + Child->addNameAsPredicateArg(ScopedName(Scope, Frag->getArgName(i))); + } + ArgMap[Frag->getArgName(i)] = Child; + } + + // Loop over all fragment alternatives. + for (auto Alternative : Frag->getTrees()) { + TreePatternNodePtr FragTree = Alternative->clone(); + + if (!PredFn.isAlwaysTrue()) + FragTree->addPredicateCall(PredFn, Scope); + + // Resolve formal arguments to their actual value. + if (Frag->getNumArgs()) + FragTree->SubstituteFormalArguments(ArgMap); + + // Transfer types. Note that the resolved alternative may have fewer + // (but not more) results than the PatFrags node. + FragTree->setName(getName()); + for (unsigned i = 0, e = FragTree->getNumTypes(); i != e; ++i) + FragTree->UpdateNodeType(i, getExtType(i), TP); + + // Transfer in the old predicates. + for (const TreePredicateCall &Pred : getPredicateCalls()) + FragTree->addPredicateCall(Pred); + + // The fragment we inlined could have recursive inlining that is needed. See + // if there are any pattern fragments in it and inline them as needed. + FragTree->InlinePatternFragments(FragTree, TP, OutAlternatives); + } +} + +/// getImplicitType - Check to see if the specified record has an implicit +/// type which should be applied to it. This will infer the type of register +/// references from the register file information, for example. +/// +/// When Unnamed is set, return the type of a DAG operand with no name, such as +/// the F8RC register class argument in: +/// +/// (COPY_TO_REGCLASS GPR:$src, F8RC) +/// +/// When Unnamed is false, return the type of a named DAG operand such as the +/// GPR:$src operand above. +/// +static TypeSetByHwMode getImplicitType(Record *R, unsigned ResNo, + bool NotRegisters, + bool Unnamed, + TreePattern &TP) { + CodeGenDAGPatterns &CDP = TP.getDAGPatterns(); + + // Check to see if this is a register operand. + if (R->isSubClassOf("RegisterOperand")) { + assert(ResNo == 0 && "Regoperand ref only has one result!"); + if (NotRegisters) + return TypeSetByHwMode(); // Unknown. + Record *RegClass = R->getValueAsDef("RegClass"); + const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); + return TypeSetByHwMode(T.getRegisterClass(RegClass).getValueTypes()); + } + + // Check to see if this is a register or a register class. + if (R->isSubClassOf("RegisterClass")) { + assert(ResNo == 0 && "Regclass ref only has one result!"); + // An unnamed register class represents itself as an i32 immediate, for + // example on a COPY_TO_REGCLASS instruction. + if (Unnamed) + return TypeSetByHwMode(MVT::i32); + + // In a named operand, the register class provides the possible set of + // types. + if (NotRegisters) + return TypeSetByHwMode(); // Unknown. + const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); + return TypeSetByHwMode(T.getRegisterClass(R).getValueTypes()); + } + + if (R->isSubClassOf("PatFrags")) { + assert(ResNo == 0 && "FIXME: PatFrag with multiple results?"); + // Pattern fragment types will be resolved when they are inlined. + return TypeSetByHwMode(); // Unknown. + } + + if (R->isSubClassOf("Register")) { + assert(ResNo == 0 && "Registers only produce one result!"); + if (NotRegisters) + return TypeSetByHwMode(); // Unknown. + const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); + return TypeSetByHwMode(T.getRegisterVTs(R)); + } + + if (R->isSubClassOf("SubRegIndex")) { + assert(ResNo == 0 && "SubRegisterIndices only produce one result!"); + return TypeSetByHwMode(MVT::i32); + } + + if (R->isSubClassOf("ValueType")) { + assert(ResNo == 0 && "This node only has one result!"); + // An unnamed VTSDNode represents itself as an MVT::Other immediate. + // + // (sext_inreg GPR:$src, i16) + // ~~~ + if (Unnamed) + return TypeSetByHwMode(MVT::Other); + // With a name, the ValueType simply provides the type of the named + // variable. + // + // (sext_inreg i32:$src, i16) + // ~~~~~~~~ + if (NotRegisters) + return TypeSetByHwMode(); // Unknown. + const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); + return TypeSetByHwMode(getValueTypeByHwMode(R, CGH)); + } + + if (R->isSubClassOf("CondCode")) { + assert(ResNo == 0 && "This node only has one result!"); + // Using a CondCodeSDNode. + return TypeSetByHwMode(MVT::Other); + } + + if (R->isSubClassOf("ComplexPattern")) { + assert(ResNo == 0 && "FIXME: ComplexPattern with multiple results?"); + if (NotRegisters) + return TypeSetByHwMode(); // Unknown. + return TypeSetByHwMode(CDP.getComplexPattern(R).getValueType()); + } + if (R->isSubClassOf("PointerLikeRegClass")) { + assert(ResNo == 0 && "Regclass can only have one result!"); + TypeSetByHwMode VTS(MVT::iPTR); + TP.getInfer().expandOverloads(VTS); + return VTS; + } + + if (R->getName() == "node" || R->getName() == "srcvalue" || + R->getName() == "zero_reg" || R->getName() == "immAllOnesV" || + R->getName() == "immAllZerosV" || R->getName() == "undef_tied_input") { + // Placeholder. + return TypeSetByHwMode(); // Unknown. + } + + if (R->isSubClassOf("Operand")) { + const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); + Record *T = R->getValueAsDef("Type"); + return TypeSetByHwMode(getValueTypeByHwMode(T, CGH)); + } + + TP.error("Unknown node flavor used in pattern: " + R->getName()); + return TypeSetByHwMode(MVT::Other); +} + + +/// getIntrinsicInfo - If this node corresponds to an intrinsic, return the +/// CodeGenIntrinsic information for it, otherwise return a null pointer. +const CodeGenIntrinsic *TreePatternNode:: +getIntrinsicInfo(const CodeGenDAGPatterns &CDP) const { + if (getOperator() != CDP.get_intrinsic_void_sdnode() && + getOperator() != CDP.get_intrinsic_w_chain_sdnode() && + getOperator() != CDP.get_intrinsic_wo_chain_sdnode()) + return nullptr; + + unsigned IID = cast<IntInit>(getChild(0)->getLeafValue())->getValue(); + return &CDP.getIntrinsicInfo(IID); +} + +/// getComplexPatternInfo - If this node corresponds to a ComplexPattern, +/// return the ComplexPattern information, otherwise return null. +const ComplexPattern * +TreePatternNode::getComplexPatternInfo(const CodeGenDAGPatterns &CGP) const { + Record *Rec; + if (isLeaf()) { + DefInit *DI = dyn_cast<DefInit>(getLeafValue()); + if (!DI) + return nullptr; + Rec = DI->getDef(); + } else + Rec = getOperator(); + + if (!Rec->isSubClassOf("ComplexPattern")) + return nullptr; + return &CGP.getComplexPattern(Rec); +} + +unsigned TreePatternNode::getNumMIResults(const CodeGenDAGPatterns &CGP) const { + // A ComplexPattern specifically declares how many results it fills in. + if (const ComplexPattern *CP = getComplexPatternInfo(CGP)) + return CP->getNumOperands(); + + // If MIOperandInfo is specified, that gives the count. + if (isLeaf()) { + DefInit *DI = dyn_cast<DefInit>(getLeafValue()); + if (DI && DI->getDef()->isSubClassOf("Operand")) { + DagInit *MIOps = DI->getDef()->getValueAsDag("MIOperandInfo"); + if (MIOps->getNumArgs()) + return MIOps->getNumArgs(); + } + } + + // Otherwise there is just one result. + return 1; +} + +/// NodeHasProperty - Return true if this node has the specified property. +bool TreePatternNode::NodeHasProperty(SDNP Property, + const CodeGenDAGPatterns &CGP) const { + if (isLeaf()) { + if (const ComplexPattern *CP = getComplexPatternInfo(CGP)) + return CP->hasProperty(Property); + + return false; + } + + if (Property != SDNPHasChain) { + // The chain proprety is already present on the different intrinsic node + // types (intrinsic_w_chain, intrinsic_void), and is not explicitly listed + // on the intrinsic. Anything else is specific to the individual intrinsic. + if (const CodeGenIntrinsic *Int = getIntrinsicInfo(CGP)) + return Int->hasProperty(Property); + } + + if (!Operator->isSubClassOf("SDPatternOperator")) + return false; + + return CGP.getSDNodeInfo(Operator).hasProperty(Property); +} + + + + +/// TreeHasProperty - Return true if any node in this tree has the specified +/// property. +bool TreePatternNode::TreeHasProperty(SDNP Property, + const CodeGenDAGPatterns &CGP) const { + if (NodeHasProperty(Property, CGP)) + return true; + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) + if (getChild(i)->TreeHasProperty(Property, CGP)) + return true; + return false; +} + +/// isCommutativeIntrinsic - Return true if the node corresponds to a +/// commutative intrinsic. +bool +TreePatternNode::isCommutativeIntrinsic(const CodeGenDAGPatterns &CDP) const { + if (const CodeGenIntrinsic *Int = getIntrinsicInfo(CDP)) + return Int->isCommutative; + return false; +} + +static bool isOperandClass(const TreePatternNode *N, StringRef Class) { + if (!N->isLeaf()) + return N->getOperator()->isSubClassOf(Class); + + DefInit *DI = dyn_cast<DefInit>(N->getLeafValue()); + if (DI && DI->getDef()->isSubClassOf(Class)) + return true; + + return false; +} + +static void emitTooManyOperandsError(TreePattern &TP, + StringRef InstName, + unsigned Expected, + unsigned Actual) { + TP.error("Instruction '" + InstName + "' was provided " + Twine(Actual) + + " operands but expected only " + Twine(Expected) + "!"); +} + +static void emitTooFewOperandsError(TreePattern &TP, + StringRef InstName, + unsigned Actual) { + TP.error("Instruction '" + InstName + + "' expects more than the provided " + Twine(Actual) + " operands!"); +} + +/// ApplyTypeConstraints - Apply all of the type constraints relevant to +/// this node and its children in the tree. This returns true if it makes a +/// change, false otherwise. If a type contradiction is found, flag an error. +bool TreePatternNode::ApplyTypeConstraints(TreePattern &TP, bool NotRegisters) { + if (TP.hasError()) + return false; + + CodeGenDAGPatterns &CDP = TP.getDAGPatterns(); + if (isLeaf()) { + if (DefInit *DI = dyn_cast<DefInit>(getLeafValue())) { + // If it's a regclass or something else known, include the type. + bool MadeChange = false; + for (unsigned i = 0, e = Types.size(); i != e; ++i) + MadeChange |= UpdateNodeType(i, getImplicitType(DI->getDef(), i, + NotRegisters, + !hasName(), TP), TP); + return MadeChange; + } + + if (IntInit *II = dyn_cast<IntInit>(getLeafValue())) { + assert(Types.size() == 1 && "Invalid IntInit"); + + // Int inits are always integers. :) + bool MadeChange = TP.getInfer().EnforceInteger(Types[0]); + + if (!TP.getInfer().isConcrete(Types[0], false)) + return MadeChange; + + ValueTypeByHwMode VVT = TP.getInfer().getConcrete(Types[0], false); + for (auto &P : VVT) { + MVT::SimpleValueType VT = P.second.SimpleTy; + if (VT == MVT::iPTR || VT == MVT::iPTRAny) + continue; + unsigned Size = MVT(VT).getSizeInBits(); + // Make sure that the value is representable for this type. + if (Size >= 32) + continue; + // Check that the value doesn't use more bits than we have. It must + // either be a sign- or zero-extended equivalent of the original. + int64_t SignBitAndAbove = II->getValue() >> (Size - 1); + if (SignBitAndAbove == -1 || SignBitAndAbove == 0 || + SignBitAndAbove == 1) + continue; + + TP.error("Integer value '" + Twine(II->getValue()) + + "' is out of range for type '" + getEnumName(VT) + "'!"); + break; + } + return MadeChange; + } + + return false; + } + + if (const CodeGenIntrinsic *Int = getIntrinsicInfo(CDP)) { + bool MadeChange = false; + + // Apply the result type to the node. + unsigned NumRetVTs = Int->IS.RetVTs.size(); + unsigned NumParamVTs = Int->IS.ParamVTs.size(); + + for (unsigned i = 0, e = NumRetVTs; i != e; ++i) + MadeChange |= UpdateNodeType(i, Int->IS.RetVTs[i], TP); + + if (getNumChildren() != NumParamVTs + 1) { + TP.error("Intrinsic '" + Int->Name + "' expects " + Twine(NumParamVTs) + + " operands, not " + Twine(getNumChildren() - 1) + " operands!"); + return false; + } + + // Apply type info to the intrinsic ID. + MadeChange |= getChild(0)->UpdateNodeType(0, MVT::iPTR, TP); + + for (unsigned i = 0, e = getNumChildren()-1; i != e; ++i) { + MadeChange |= getChild(i+1)->ApplyTypeConstraints(TP, NotRegisters); + + MVT::SimpleValueType OpVT = Int->IS.ParamVTs[i]; + assert(getChild(i+1)->getNumTypes() == 1 && "Unhandled case"); + MadeChange |= getChild(i+1)->UpdateNodeType(0, OpVT, TP); + } + return MadeChange; + } + + if (getOperator()->isSubClassOf("SDNode")) { + const SDNodeInfo &NI = CDP.getSDNodeInfo(getOperator()); + + // Check that the number of operands is sane. Negative operands -> varargs. + if (NI.getNumOperands() >= 0 && + getNumChildren() != (unsigned)NI.getNumOperands()) { + TP.error(getOperator()->getName() + " node requires exactly " + + Twine(NI.getNumOperands()) + " operands!"); + return false; + } + + bool MadeChange = false; + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) + MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); + MadeChange |= NI.ApplyTypeConstraints(this, TP); + return MadeChange; + } + + if (getOperator()->isSubClassOf("Instruction")) { + const DAGInstruction &Inst = CDP.getInstruction(getOperator()); + CodeGenInstruction &InstInfo = + CDP.getTargetInfo().getInstruction(getOperator()); + + bool MadeChange = false; + + // Apply the result types to the node, these come from the things in the + // (outs) list of the instruction. + unsigned NumResultsToAdd = std::min(InstInfo.Operands.NumDefs, + Inst.getNumResults()); + for (unsigned ResNo = 0; ResNo != NumResultsToAdd; ++ResNo) + MadeChange |= UpdateNodeTypeFromInst(ResNo, Inst.getResult(ResNo), TP); + + // If the instruction has implicit defs, we apply the first one as a result. + // FIXME: This sucks, it should apply all implicit defs. + if (!InstInfo.ImplicitDefs.empty()) { + unsigned ResNo = NumResultsToAdd; + + // FIXME: Generalize to multiple possible types and multiple possible + // ImplicitDefs. + MVT::SimpleValueType VT = + InstInfo.HasOneImplicitDefWithKnownVT(CDP.getTargetInfo()); + + if (VT != MVT::Other) + MadeChange |= UpdateNodeType(ResNo, VT, TP); + } + + // If this is an INSERT_SUBREG, constrain the source and destination VTs to + // be the same. + if (getOperator()->getName() == "INSERT_SUBREG") { + assert(getChild(0)->getNumTypes() == 1 && "FIXME: Unhandled"); + MadeChange |= UpdateNodeType(0, getChild(0)->getExtType(0), TP); + MadeChange |= getChild(0)->UpdateNodeType(0, getExtType(0), TP); + } else if (getOperator()->getName() == "REG_SEQUENCE") { + // We need to do extra, custom typechecking for REG_SEQUENCE since it is + // variadic. + + unsigned NChild = getNumChildren(); + if (NChild < 3) { + TP.error("REG_SEQUENCE requires at least 3 operands!"); + return false; + } + + if (NChild % 2 == 0) { + TP.error("REG_SEQUENCE requires an odd number of operands!"); + return false; + } + + if (!isOperandClass(getChild(0), "RegisterClass")) { + TP.error("REG_SEQUENCE requires a RegisterClass for first operand!"); + return false; + } + + for (unsigned I = 1; I < NChild; I += 2) { + TreePatternNode *SubIdxChild = getChild(I + 1); + if (!isOperandClass(SubIdxChild, "SubRegIndex")) { + TP.error("REG_SEQUENCE requires a SubRegIndex for operand " + + Twine(I + 1) + "!"); + return false; + } + } + } + + unsigned NumResults = Inst.getNumResults(); + unsigned NumFixedOperands = InstInfo.Operands.size(); + + // If one or more operands with a default value appear at the end of the + // formal operand list for an instruction, we allow them to be overridden + // by optional operands provided in the pattern. + // + // But if an operand B without a default appears at any point after an + // operand A with a default, then we don't allow A to be overridden, + // because there would be no way to specify whether the next operand in + // the pattern was intended to override A or skip it. + unsigned NonOverridableOperands = NumFixedOperands; + while (NonOverridableOperands > NumResults && + CDP.operandHasDefault(InstInfo.Operands[NonOverridableOperands-1].Rec)) + --NonOverridableOperands; + + unsigned ChildNo = 0; + assert(NumResults <= NumFixedOperands); + for (unsigned i = NumResults, e = NumFixedOperands; i != e; ++i) { + Record *OperandNode = InstInfo.Operands[i].Rec; + + // If the operand has a default value, do we use it? We must use the + // default if we've run out of children of the pattern DAG to consume, + // or if the operand is followed by a non-defaulted one. + if (CDP.operandHasDefault(OperandNode) && + (i < NonOverridableOperands || ChildNo >= getNumChildren())) + continue; + + // If we have run out of child nodes and there _isn't_ a default + // value we can use for the next operand, give an error. + if (ChildNo >= getNumChildren()) { + emitTooFewOperandsError(TP, getOperator()->getName(), getNumChildren()); + return false; + } + + TreePatternNode *Child = getChild(ChildNo++); + unsigned ChildResNo = 0; // Instructions always use res #0 of their op. + + // If the operand has sub-operands, they may be provided by distinct + // child patterns, so attempt to match each sub-operand separately. + if (OperandNode->isSubClassOf("Operand")) { + DagInit *MIOpInfo = OperandNode->getValueAsDag("MIOperandInfo"); + if (unsigned NumArgs = MIOpInfo->getNumArgs()) { + // But don't do that if the whole operand is being provided by + // a single ComplexPattern-related Operand. + + if (Child->getNumMIResults(CDP) < NumArgs) { + // Match first sub-operand against the child we already have. + Record *SubRec = cast<DefInit>(MIOpInfo->getArg(0))->getDef(); + MadeChange |= + Child->UpdateNodeTypeFromInst(ChildResNo, SubRec, TP); + + // And the remaining sub-operands against subsequent children. + for (unsigned Arg = 1; Arg < NumArgs; ++Arg) { + if (ChildNo >= getNumChildren()) { + emitTooFewOperandsError(TP, getOperator()->getName(), + getNumChildren()); + return false; + } + Child = getChild(ChildNo++); + + SubRec = cast<DefInit>(MIOpInfo->getArg(Arg))->getDef(); + MadeChange |= + Child->UpdateNodeTypeFromInst(ChildResNo, SubRec, TP); + } + continue; + } + } + } + + // If we didn't match by pieces above, attempt to match the whole + // operand now. + MadeChange |= Child->UpdateNodeTypeFromInst(ChildResNo, OperandNode, TP); + } + + if (!InstInfo.Operands.isVariadic && ChildNo != getNumChildren()) { + emitTooManyOperandsError(TP, getOperator()->getName(), + ChildNo, getNumChildren()); + return false; + } + + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) + MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); + return MadeChange; + } + + if (getOperator()->isSubClassOf("ComplexPattern")) { + bool MadeChange = false; + + for (unsigned i = 0; i < getNumChildren(); ++i) + MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); + + return MadeChange; + } + + assert(getOperator()->isSubClassOf("SDNodeXForm") && "Unknown node type!"); + + // Node transforms always take one operand. + if (getNumChildren() != 1) { + TP.error("Node transform '" + getOperator()->getName() + + "' requires one operand!"); + return false; + } + + bool MadeChange = getChild(0)->ApplyTypeConstraints(TP, NotRegisters); + return MadeChange; +} + +/// OnlyOnRHSOfCommutative - Return true if this value is only allowed on the +/// RHS of a commutative operation, not the on LHS. +static bool OnlyOnRHSOfCommutative(TreePatternNode *N) { + if (!N->isLeaf() && N->getOperator()->getName() == "imm") + return true; + if (N->isLeaf() && isa<IntInit>(N->getLeafValue())) + return true; + return false; +} + + +/// canPatternMatch - If it is impossible for this pattern to match on this +/// target, fill in Reason and return false. Otherwise, return true. This is +/// used as a sanity check for .td files (to prevent people from writing stuff +/// that can never possibly work), and to prevent the pattern permuter from +/// generating stuff that is useless. +bool TreePatternNode::canPatternMatch(std::string &Reason, + const CodeGenDAGPatterns &CDP) { + if (isLeaf()) return true; + + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) + if (!getChild(i)->canPatternMatch(Reason, CDP)) + return false; + + // If this is an intrinsic, handle cases that would make it not match. For + // example, if an operand is required to be an immediate. + if (getOperator()->isSubClassOf("Intrinsic")) { + // TODO: + return true; + } + + if (getOperator()->isSubClassOf("ComplexPattern")) + return true; + + // If this node is a commutative operator, check that the LHS isn't an + // immediate. + const SDNodeInfo &NodeInfo = CDP.getSDNodeInfo(getOperator()); + bool isCommIntrinsic = isCommutativeIntrinsic(CDP); + if (NodeInfo.hasProperty(SDNPCommutative) || isCommIntrinsic) { + // Scan all of the operands of the node and make sure that only the last one + // is a constant node, unless the RHS also is. + if (!OnlyOnRHSOfCommutative(getChild(getNumChildren()-1))) { + unsigned Skip = isCommIntrinsic ? 1 : 0; // First operand is intrinsic id. + for (unsigned i = Skip, e = getNumChildren()-1; i != e; ++i) + if (OnlyOnRHSOfCommutative(getChild(i))) { + Reason="Immediate value must be on the RHS of commutative operators!"; + return false; + } + } + } + + return true; +} + +//===----------------------------------------------------------------------===// +// TreePattern implementation +// + +TreePattern::TreePattern(Record *TheRec, ListInit *RawPat, bool isInput, + CodeGenDAGPatterns &cdp) : TheRecord(TheRec), CDP(cdp), + isInputPattern(isInput), HasError(false), + Infer(*this) { + for (Init *I : RawPat->getValues()) + Trees.push_back(ParseTreePattern(I, "")); +} + +TreePattern::TreePattern(Record *TheRec, DagInit *Pat, bool isInput, + CodeGenDAGPatterns &cdp) : TheRecord(TheRec), CDP(cdp), + isInputPattern(isInput), HasError(false), + Infer(*this) { + Trees.push_back(ParseTreePattern(Pat, "")); +} + +TreePattern::TreePattern(Record *TheRec, TreePatternNodePtr Pat, bool isInput, + CodeGenDAGPatterns &cdp) + : TheRecord(TheRec), CDP(cdp), isInputPattern(isInput), HasError(false), + Infer(*this) { + Trees.push_back(Pat); +} + +void TreePattern::error(const Twine &Msg) { + if (HasError) + return; + dump(); + PrintError(TheRecord->getLoc(), "In " + TheRecord->getName() + ": " + Msg); + HasError = true; +} + +void TreePattern::ComputeNamedNodes() { + for (TreePatternNodePtr &Tree : Trees) + ComputeNamedNodes(Tree.get()); +} + +void TreePattern::ComputeNamedNodes(TreePatternNode *N) { + if (!N->getName().empty()) + NamedNodes[N->getName()].push_back(N); + + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) + ComputeNamedNodes(N->getChild(i)); +} + +TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit, + StringRef OpName) { + if (DefInit *DI = dyn_cast<DefInit>(TheInit)) { + Record *R = DI->getDef(); + + // Direct reference to a leaf DagNode or PatFrag? Turn it into a + // TreePatternNode of its own. For example: + /// (foo GPR, imm) -> (foo GPR, (imm)) + if (R->isSubClassOf("SDNode") || R->isSubClassOf("PatFrags")) + return ParseTreePattern( + DagInit::get(DI, nullptr, + std::vector<std::pair<Init*, StringInit*> >()), + OpName); + + // Input argument? + TreePatternNodePtr Res = std::make_shared<TreePatternNode>(DI, 1); + if (R->getName() == "node" && !OpName.empty()) { + if (OpName.empty()) + error("'node' argument requires a name to match with operand list"); + Args.push_back(std::string(OpName)); + } + + Res->setName(OpName); + return Res; + } + + // ?:$name or just $name. + if (isa<UnsetInit>(TheInit)) { + if (OpName.empty()) + error("'?' argument requires a name to match with operand list"); + TreePatternNodePtr Res = std::make_shared<TreePatternNode>(TheInit, 1); + Args.push_back(std::string(OpName)); + Res->setName(OpName); + return Res; + } + + if (isa<IntInit>(TheInit) || isa<BitInit>(TheInit)) { + if (!OpName.empty()) + error("Constant int or bit argument should not have a name!"); + if (isa<BitInit>(TheInit)) + TheInit = TheInit->convertInitializerTo(IntRecTy::get()); + return std::make_shared<TreePatternNode>(TheInit, 1); + } + + if (BitsInit *BI = dyn_cast<BitsInit>(TheInit)) { + // Turn this into an IntInit. + Init *II = BI->convertInitializerTo(IntRecTy::get()); + if (!II || !isa<IntInit>(II)) + error("Bits value must be constants!"); + return ParseTreePattern(II, OpName); + } + + DagInit *Dag = dyn_cast<DagInit>(TheInit); + if (!Dag) { + TheInit->print(errs()); + error("Pattern has unexpected init kind!"); + } + DefInit *OpDef = dyn_cast<DefInit>(Dag->getOperator()); + if (!OpDef) error("Pattern has unexpected operator type!"); + Record *Operator = OpDef->getDef(); + + if (Operator->isSubClassOf("ValueType")) { + // If the operator is a ValueType, then this must be "type cast" of a leaf + // node. + if (Dag->getNumArgs() != 1) + error("Type cast only takes one operand!"); + + TreePatternNodePtr New = + ParseTreePattern(Dag->getArg(0), Dag->getArgNameStr(0)); + + // Apply the type cast. + assert(New->getNumTypes() == 1 && "FIXME: Unhandled"); + const CodeGenHwModes &CGH = getDAGPatterns().getTargetInfo().getHwModes(); + New->UpdateNodeType(0, getValueTypeByHwMode(Operator, CGH), *this); + + if (!OpName.empty()) + error("ValueType cast should not have a name!"); + return New; + } + + // Verify that this is something that makes sense for an operator. + if (!Operator->isSubClassOf("PatFrags") && + !Operator->isSubClassOf("SDNode") && + !Operator->isSubClassOf("Instruction") && + !Operator->isSubClassOf("SDNodeXForm") && + !Operator->isSubClassOf("Intrinsic") && + !Operator->isSubClassOf("ComplexPattern") && + Operator->getName() != "set" && + Operator->getName() != "implicit") + error("Unrecognized node '" + Operator->getName() + "'!"); + + // Check to see if this is something that is illegal in an input pattern. + if (isInputPattern) { + if (Operator->isSubClassOf("Instruction") || + Operator->isSubClassOf("SDNodeXForm")) + error("Cannot use '" + Operator->getName() + "' in an input pattern!"); + } else { + if (Operator->isSubClassOf("Intrinsic")) + error("Cannot use '" + Operator->getName() + "' in an output pattern!"); + + if (Operator->isSubClassOf("SDNode") && + Operator->getName() != "imm" && + Operator->getName() != "timm" && + Operator->getName() != "fpimm" && + Operator->getName() != "tglobaltlsaddr" && + Operator->getName() != "tconstpool" && + Operator->getName() != "tjumptable" && + Operator->getName() != "tframeindex" && + Operator->getName() != "texternalsym" && + Operator->getName() != "tblockaddress" && + Operator->getName() != "tglobaladdr" && + Operator->getName() != "bb" && + Operator->getName() != "vt" && + Operator->getName() != "mcsym") + error("Cannot use '" + Operator->getName() + "' in an output pattern!"); + } + + std::vector<TreePatternNodePtr> Children; + + // Parse all the operands. + for (unsigned i = 0, e = Dag->getNumArgs(); i != e; ++i) + Children.push_back(ParseTreePattern(Dag->getArg(i), Dag->getArgNameStr(i))); + + // Get the actual number of results before Operator is converted to an intrinsic + // node (which is hard-coded to have either zero or one result). + unsigned NumResults = GetNumNodeResults(Operator, CDP); + + // If the operator is an intrinsic, then this is just syntactic sugar for + // (intrinsic_* <number>, ..children..). Pick the right intrinsic node, and + // convert the intrinsic name to a number. + if (Operator->isSubClassOf("Intrinsic")) { + const CodeGenIntrinsic &Int = getDAGPatterns().getIntrinsic(Operator); + unsigned IID = getDAGPatterns().getIntrinsicID(Operator)+1; + + // If this intrinsic returns void, it must have side-effects and thus a + // chain. + if (Int.IS.RetVTs.empty()) + Operator = getDAGPatterns().get_intrinsic_void_sdnode(); + else if (Int.ModRef != CodeGenIntrinsic::NoMem || Int.hasSideEffects) + // Has side-effects, requires chain. + Operator = getDAGPatterns().get_intrinsic_w_chain_sdnode(); + else // Otherwise, no chain. + Operator = getDAGPatterns().get_intrinsic_wo_chain_sdnode(); + + Children.insert(Children.begin(), + std::make_shared<TreePatternNode>(IntInit::get(IID), 1)); + } + + if (Operator->isSubClassOf("ComplexPattern")) { + for (unsigned i = 0; i < Children.size(); ++i) { + TreePatternNodePtr Child = Children[i]; + + if (Child->getName().empty()) + error("All arguments to a ComplexPattern must be named"); + + // Check that the ComplexPattern uses are consistent: "(MY_PAT $a, $b)" + // and "(MY_PAT $b, $a)" should not be allowed in the same pattern; + // neither should "(MY_PAT_1 $a, $b)" and "(MY_PAT_2 $a, $b)". + auto OperandId = std::make_pair(Operator, i); + auto PrevOp = ComplexPatternOperands.find(Child->getName()); + if (PrevOp != ComplexPatternOperands.end()) { + if (PrevOp->getValue() != OperandId) + error("All ComplexPattern operands must appear consistently: " + "in the same order in just one ComplexPattern instance."); + } else + ComplexPatternOperands[Child->getName()] = OperandId; + } + } + + TreePatternNodePtr Result = + std::make_shared<TreePatternNode>(Operator, std::move(Children), + NumResults); + Result->setName(OpName); + + if (Dag->getName()) { + assert(Result->getName().empty()); + Result->setName(Dag->getNameStr()); + } + return Result; +} + +/// SimplifyTree - See if we can simplify this tree to eliminate something that +/// will never match in favor of something obvious that will. This is here +/// strictly as a convenience to target authors because it allows them to write +/// more type generic things and have useless type casts fold away. +/// +/// This returns true if any change is made. +static bool SimplifyTree(TreePatternNodePtr &N) { + if (N->isLeaf()) + return false; + + // If we have a bitconvert with a resolved type and if the source and + // destination types are the same, then the bitconvert is useless, remove it. + // + // We make an exception if the types are completely empty. This can come up + // when the pattern being simplified is in the Fragments list of a PatFrags, + // so that the operand is just an untyped "node". In that situation we leave + // bitconverts unsimplified, and simplify them later once the fragment is + // expanded into its true context. + if (N->getOperator()->getName() == "bitconvert" && + N->getExtType(0).isValueTypeByHwMode(false) && + !N->getExtType(0).empty() && + N->getExtType(0) == N->getChild(0)->getExtType(0) && + N->getName().empty()) { + N = N->getChildShared(0); + SimplifyTree(N); + return true; + } + + // Walk all children. + bool MadeChange = false; + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) { + TreePatternNodePtr Child = N->getChildShared(i); + MadeChange |= SimplifyTree(Child); + N->setChild(i, std::move(Child)); + } + return MadeChange; +} + + + +/// InferAllTypes - Infer/propagate as many types throughout the expression +/// patterns as possible. Return true if all types are inferred, false +/// otherwise. Flags an error if a type contradiction is found. +bool TreePattern:: +InferAllTypes(const StringMap<SmallVector<TreePatternNode*,1> > *InNamedTypes) { + if (NamedNodes.empty()) + ComputeNamedNodes(); + + bool MadeChange = true; + while (MadeChange) { + MadeChange = false; + for (TreePatternNodePtr &Tree : Trees) { + MadeChange |= Tree->ApplyTypeConstraints(*this, false); + MadeChange |= SimplifyTree(Tree); + } + + // If there are constraints on our named nodes, apply them. + for (auto &Entry : NamedNodes) { + SmallVectorImpl<TreePatternNode*> &Nodes = Entry.second; + + // If we have input named node types, propagate their types to the named + // values here. + if (InNamedTypes) { + if (!InNamedTypes->count(Entry.getKey())) { + error("Node '" + std::string(Entry.getKey()) + + "' in output pattern but not input pattern"); + return true; + } + + const SmallVectorImpl<TreePatternNode*> &InNodes = + InNamedTypes->find(Entry.getKey())->second; + + // The input types should be fully resolved by now. + for (TreePatternNode *Node : Nodes) { + // If this node is a register class, and it is the root of the pattern + // then we're mapping something onto an input register. We allow + // changing the type of the input register in this case. This allows + // us to match things like: + // def : Pat<(v1i64 (bitconvert(v2i32 DPR:$src))), (v1i64 DPR:$src)>; + if (Node == Trees[0].get() && Node->isLeaf()) { + DefInit *DI = dyn_cast<DefInit>(Node->getLeafValue()); + if (DI && (DI->getDef()->isSubClassOf("RegisterClass") || + DI->getDef()->isSubClassOf("RegisterOperand"))) + continue; + } + + assert(Node->getNumTypes() == 1 && + InNodes[0]->getNumTypes() == 1 && + "FIXME: cannot name multiple result nodes yet"); + MadeChange |= Node->UpdateNodeType(0, InNodes[0]->getExtType(0), + *this); + } + } + + // If there are multiple nodes with the same name, they must all have the + // same type. + if (Entry.second.size() > 1) { + for (unsigned i = 0, e = Nodes.size()-1; i != e; ++i) { + TreePatternNode *N1 = Nodes[i], *N2 = Nodes[i+1]; + assert(N1->getNumTypes() == 1 && N2->getNumTypes() == 1 && + "FIXME: cannot name multiple result nodes yet"); + + MadeChange |= N1->UpdateNodeType(0, N2->getExtType(0), *this); + MadeChange |= N2->UpdateNodeType(0, N1->getExtType(0), *this); + } + } + } + } + + bool HasUnresolvedTypes = false; + for (const TreePatternNodePtr &Tree : Trees) + HasUnresolvedTypes |= Tree->ContainsUnresolvedType(*this); + return !HasUnresolvedTypes; +} + +void TreePattern::print(raw_ostream &OS) const { + OS << getRecord()->getName(); + if (!Args.empty()) { + OS << "(" << Args[0]; + for (unsigned i = 1, e = Args.size(); i != e; ++i) + OS << ", " << Args[i]; + OS << ")"; + } + OS << ": "; + + if (Trees.size() > 1) + OS << "[\n"; + for (const TreePatternNodePtr &Tree : Trees) { + OS << "\t"; + Tree->print(OS); + OS << "\n"; + } + + if (Trees.size() > 1) + OS << "]\n"; +} + +void TreePattern::dump() const { print(errs()); } + +//===----------------------------------------------------------------------===// +// CodeGenDAGPatterns implementation +// + +CodeGenDAGPatterns::CodeGenDAGPatterns(RecordKeeper &R, + PatternRewriterFn PatternRewriter) + : Records(R), Target(R), LegalVTS(Target.getLegalValueTypes()), + PatternRewriter(PatternRewriter) { + + Intrinsics = CodeGenIntrinsicTable(Records); + ParseNodeInfo(); + ParseNodeTransforms(); + ParseComplexPatterns(); + ParsePatternFragments(); + ParseDefaultOperands(); + ParseInstructions(); + ParsePatternFragments(/*OutFrags*/true); + ParsePatterns(); + + // Break patterns with parameterized types into a series of patterns, + // where each one has a fixed type and is predicated on the conditions + // of the associated HW mode. + ExpandHwModeBasedTypes(); + + // Generate variants. For example, commutative patterns can match + // multiple ways. Add them to PatternsToMatch as well. + GenerateVariants(); + + // Infer instruction flags. For example, we can detect loads, + // stores, and side effects in many cases by examining an + // instruction's pattern. + InferInstructionFlags(); + + // Verify that instruction flags match the patterns. + VerifyInstructionFlags(); +} + +Record *CodeGenDAGPatterns::getSDNodeNamed(const std::string &Name) const { + Record *N = Records.getDef(Name); + if (!N || !N->isSubClassOf("SDNode")) + PrintFatalError("Error getting SDNode '" + Name + "'!"); + + return N; +} + +// Parse all of the SDNode definitions for the target, populating SDNodes. +void CodeGenDAGPatterns::ParseNodeInfo() { + std::vector<Record*> Nodes = Records.getAllDerivedDefinitions("SDNode"); + const CodeGenHwModes &CGH = getTargetInfo().getHwModes(); + + while (!Nodes.empty()) { + Record *R = Nodes.back(); + SDNodes.insert(std::make_pair(R, SDNodeInfo(R, CGH))); + Nodes.pop_back(); + } + + // Get the builtin intrinsic nodes. + intrinsic_void_sdnode = getSDNodeNamed("intrinsic_void"); + intrinsic_w_chain_sdnode = getSDNodeNamed("intrinsic_w_chain"); + intrinsic_wo_chain_sdnode = getSDNodeNamed("intrinsic_wo_chain"); +} + +/// ParseNodeTransforms - Parse all SDNodeXForm instances into the SDNodeXForms +/// map, and emit them to the file as functions. +void CodeGenDAGPatterns::ParseNodeTransforms() { + std::vector<Record*> Xforms = Records.getAllDerivedDefinitions("SDNodeXForm"); + while (!Xforms.empty()) { + Record *XFormNode = Xforms.back(); + Record *SDNode = XFormNode->getValueAsDef("Opcode"); + StringRef Code = XFormNode->getValueAsString("XFormFunction"); + SDNodeXForms.insert( + std::make_pair(XFormNode, NodeXForm(SDNode, std::string(Code)))); + + Xforms.pop_back(); + } +} + +void CodeGenDAGPatterns::ParseComplexPatterns() { + std::vector<Record*> AMs = Records.getAllDerivedDefinitions("ComplexPattern"); + while (!AMs.empty()) { + ComplexPatterns.insert(std::make_pair(AMs.back(), AMs.back())); + AMs.pop_back(); + } +} + + +/// ParsePatternFragments - Parse all of the PatFrag definitions in the .td +/// file, building up the PatternFragments map. After we've collected them all, +/// inline fragments together as necessary, so that there are no references left +/// inside a pattern fragment to a pattern fragment. +/// +void CodeGenDAGPatterns::ParsePatternFragments(bool OutFrags) { + std::vector<Record*> Fragments = Records.getAllDerivedDefinitions("PatFrags"); + + // First step, parse all of the fragments. + for (Record *Frag : Fragments) { + if (OutFrags != Frag->isSubClassOf("OutPatFrag")) + continue; + + ListInit *LI = Frag->getValueAsListInit("Fragments"); + TreePattern *P = + (PatternFragments[Frag] = std::make_unique<TreePattern>( + Frag, LI, !Frag->isSubClassOf("OutPatFrag"), + *this)).get(); + + // Validate the argument list, converting it to set, to discard duplicates. + std::vector<std::string> &Args = P->getArgList(); + // Copy the args so we can take StringRefs to them. + auto ArgsCopy = Args; + SmallDenseSet<StringRef, 4> OperandsSet; + OperandsSet.insert(ArgsCopy.begin(), ArgsCopy.end()); + + if (OperandsSet.count("")) + P->error("Cannot have unnamed 'node' values in pattern fragment!"); + + // Parse the operands list. + DagInit *OpsList = Frag->getValueAsDag("Operands"); + DefInit *OpsOp = dyn_cast<DefInit>(OpsList->getOperator()); + // Special cases: ops == outs == ins. Different names are used to + // improve readability. + if (!OpsOp || + (OpsOp->getDef()->getName() != "ops" && + OpsOp->getDef()->getName() != "outs" && + OpsOp->getDef()->getName() != "ins")) + P->error("Operands list should start with '(ops ... '!"); + + // Copy over the arguments. + Args.clear(); + for (unsigned j = 0, e = OpsList->getNumArgs(); j != e; ++j) { + if (!isa<DefInit>(OpsList->getArg(j)) || + cast<DefInit>(OpsList->getArg(j))->getDef()->getName() != "node") + P->error("Operands list should all be 'node' values."); + if (!OpsList->getArgName(j)) + P->error("Operands list should have names for each operand!"); + StringRef ArgNameStr = OpsList->getArgNameStr(j); + if (!OperandsSet.count(ArgNameStr)) + P->error("'" + ArgNameStr + + "' does not occur in pattern or was multiply specified!"); + OperandsSet.erase(ArgNameStr); + Args.push_back(std::string(ArgNameStr)); + } + + if (!OperandsSet.empty()) + P->error("Operands list does not contain an entry for operand '" + + *OperandsSet.begin() + "'!"); + + // If there is a node transformation corresponding to this, keep track of + // it. + Record *Transform = Frag->getValueAsDef("OperandTransform"); + if (!getSDNodeTransform(Transform).second.empty()) // not noop xform? + for (auto T : P->getTrees()) + T->setTransformFn(Transform); + } + + // Now that we've parsed all of the tree fragments, do a closure on them so + // that there are not references to PatFrags left inside of them. + for (Record *Frag : Fragments) { + if (OutFrags != Frag->isSubClassOf("OutPatFrag")) + continue; + + TreePattern &ThePat = *PatternFragments[Frag]; + ThePat.InlinePatternFragments(); + + // Infer as many types as possible. Don't worry about it if we don't infer + // all of them, some may depend on the inputs of the pattern. Also, don't + // validate type sets; validation may cause spurious failures e.g. if a + // fragment needs floating-point types but the current target does not have + // any (this is only an error if that fragment is ever used!). + { + TypeInfer::SuppressValidation SV(ThePat.getInfer()); + ThePat.InferAllTypes(); + ThePat.resetError(); + } + + // If debugging, print out the pattern fragment result. + LLVM_DEBUG(ThePat.dump()); + } +} + +void CodeGenDAGPatterns::ParseDefaultOperands() { + std::vector<Record*> DefaultOps; + DefaultOps = Records.getAllDerivedDefinitions("OperandWithDefaultOps"); + + // Find some SDNode. + assert(!SDNodes.empty() && "No SDNodes parsed?"); + Init *SomeSDNode = DefInit::get(SDNodes.begin()->first); + + for (unsigned i = 0, e = DefaultOps.size(); i != e; ++i) { + DagInit *DefaultInfo = DefaultOps[i]->getValueAsDag("DefaultOps"); + + // Clone the DefaultInfo dag node, changing the operator from 'ops' to + // SomeSDnode so that we can parse this. + std::vector<std::pair<Init*, StringInit*> > Ops; + for (unsigned op = 0, e = DefaultInfo->getNumArgs(); op != e; ++op) + Ops.push_back(std::make_pair(DefaultInfo->getArg(op), + DefaultInfo->getArgName(op))); + DagInit *DI = DagInit::get(SomeSDNode, nullptr, Ops); + + // Create a TreePattern to parse this. + TreePattern P(DefaultOps[i], DI, false, *this); + assert(P.getNumTrees() == 1 && "This ctor can only produce one tree!"); + + // Copy the operands over into a DAGDefaultOperand. + DAGDefaultOperand DefaultOpInfo; + + const TreePatternNodePtr &T = P.getTree(0); + for (unsigned op = 0, e = T->getNumChildren(); op != e; ++op) { + TreePatternNodePtr TPN = T->getChildShared(op); + while (TPN->ApplyTypeConstraints(P, false)) + /* Resolve all types */; + + if (TPN->ContainsUnresolvedType(P)) { + PrintFatalError("Value #" + Twine(i) + " of OperandWithDefaultOps '" + + DefaultOps[i]->getName() + + "' doesn't have a concrete type!"); + } + DefaultOpInfo.DefaultOps.push_back(std::move(TPN)); + } + + // Insert it into the DefaultOperands map so we can find it later. + DefaultOperands[DefaultOps[i]] = DefaultOpInfo; + } +} + +/// HandleUse - Given "Pat" a leaf in the pattern, check to see if it is an +/// instruction input. Return true if this is a real use. +static bool HandleUse(TreePattern &I, TreePatternNodePtr Pat, + std::map<std::string, TreePatternNodePtr> &InstInputs) { + // No name -> not interesting. + if (Pat->getName().empty()) { + if (Pat->isLeaf()) { + DefInit *DI = dyn_cast<DefInit>(Pat->getLeafValue()); + if (DI && (DI->getDef()->isSubClassOf("RegisterClass") || + DI->getDef()->isSubClassOf("RegisterOperand"))) + I.error("Input " + DI->getDef()->getName() + " must be named!"); + } + return false; + } + + Record *Rec; + if (Pat->isLeaf()) { + DefInit *DI = dyn_cast<DefInit>(Pat->getLeafValue()); + if (!DI) + I.error("Input $" + Pat->getName() + " must be an identifier!"); + Rec = DI->getDef(); + } else { + Rec = Pat->getOperator(); + } + + // SRCVALUE nodes are ignored. + if (Rec->getName() == "srcvalue") + return false; + + TreePatternNodePtr &Slot = InstInputs[Pat->getName()]; + if (!Slot) { + Slot = Pat; + return true; + } + Record *SlotRec; + if (Slot->isLeaf()) { + SlotRec = cast<DefInit>(Slot->getLeafValue())->getDef(); + } else { + assert(Slot->getNumChildren() == 0 && "can't be a use with children!"); + SlotRec = Slot->getOperator(); + } + + // Ensure that the inputs agree if we've already seen this input. + if (Rec != SlotRec) + I.error("All $" + Pat->getName() + " inputs must agree with each other"); + // Ensure that the types can agree as well. + Slot->UpdateNodeType(0, Pat->getExtType(0), I); + Pat->UpdateNodeType(0, Slot->getExtType(0), I); + if (Slot->getExtTypes() != Pat->getExtTypes()) + I.error("All $" + Pat->getName() + " inputs must agree with each other"); + return true; +} + +/// FindPatternInputsAndOutputs - Scan the specified TreePatternNode (which is +/// part of "I", the instruction), computing the set of inputs and outputs of +/// the pattern. Report errors if we see anything naughty. +void CodeGenDAGPatterns::FindPatternInputsAndOutputs( + TreePattern &I, TreePatternNodePtr Pat, + std::map<std::string, TreePatternNodePtr> &InstInputs, + MapVector<std::string, TreePatternNodePtr, std::map<std::string, unsigned>> + &InstResults, + std::vector<Record *> &InstImpResults) { + + // The instruction pattern still has unresolved fragments. For *named* + // nodes we must resolve those here. This may not result in multiple + // alternatives. + if (!Pat->getName().empty()) { + TreePattern SrcPattern(I.getRecord(), Pat, true, *this); + SrcPattern.InlinePatternFragments(); + SrcPattern.InferAllTypes(); + Pat = SrcPattern.getOnlyTree(); + } + + if (Pat->isLeaf()) { + bool isUse = HandleUse(I, Pat, InstInputs); + if (!isUse && Pat->getTransformFn()) + I.error("Cannot specify a transform function for a non-input value!"); + return; + } + + if (Pat->getOperator()->getName() == "implicit") { + for (unsigned i = 0, e = Pat->getNumChildren(); i != e; ++i) { + TreePatternNode *Dest = Pat->getChild(i); + if (!Dest->isLeaf()) + I.error("implicitly defined value should be a register!"); + + DefInit *Val = dyn_cast<DefInit>(Dest->getLeafValue()); + if (!Val || !Val->getDef()->isSubClassOf("Register")) + I.error("implicitly defined value should be a register!"); + InstImpResults.push_back(Val->getDef()); + } + return; + } + + if (Pat->getOperator()->getName() != "set") { + // If this is not a set, verify that the children nodes are not void typed, + // and recurse. + for (unsigned i = 0, e = Pat->getNumChildren(); i != e; ++i) { + if (Pat->getChild(i)->getNumTypes() == 0) + I.error("Cannot have void nodes inside of patterns!"); + FindPatternInputsAndOutputs(I, Pat->getChildShared(i), InstInputs, + InstResults, InstImpResults); + } + + // If this is a non-leaf node with no children, treat it basically as if + // it were a leaf. This handles nodes like (imm). + bool isUse = HandleUse(I, Pat, InstInputs); + + if (!isUse && Pat->getTransformFn()) + I.error("Cannot specify a transform function for a non-input value!"); + return; + } + + // Otherwise, this is a set, validate and collect instruction results. + if (Pat->getNumChildren() == 0) + I.error("set requires operands!"); + + if (Pat->getTransformFn()) + I.error("Cannot specify a transform function on a set node!"); + + // Check the set destinations. + unsigned NumDests = Pat->getNumChildren()-1; + for (unsigned i = 0; i != NumDests; ++i) { + TreePatternNodePtr Dest = Pat->getChildShared(i); + // For set destinations we also must resolve fragments here. + TreePattern DestPattern(I.getRecord(), Dest, false, *this); + DestPattern.InlinePatternFragments(); + DestPattern.InferAllTypes(); + Dest = DestPattern.getOnlyTree(); + + if (!Dest->isLeaf()) + I.error("set destination should be a register!"); + + DefInit *Val = dyn_cast<DefInit>(Dest->getLeafValue()); + if (!Val) { + I.error("set destination should be a register!"); + continue; + } + + if (Val->getDef()->isSubClassOf("RegisterClass") || + Val->getDef()->isSubClassOf("ValueType") || + Val->getDef()->isSubClassOf("RegisterOperand") || + Val->getDef()->isSubClassOf("PointerLikeRegClass")) { + if (Dest->getName().empty()) + I.error("set destination must have a name!"); + if (InstResults.count(Dest->getName())) + I.error("cannot set '" + Dest->getName() + "' multiple times"); + InstResults[Dest->getName()] = Dest; + } else if (Val->getDef()->isSubClassOf("Register")) { + InstImpResults.push_back(Val->getDef()); + } else { + I.error("set destination should be a register!"); + } + } + + // Verify and collect info from the computation. + FindPatternInputsAndOutputs(I, Pat->getChildShared(NumDests), InstInputs, + InstResults, InstImpResults); +} + +//===----------------------------------------------------------------------===// +// Instruction Analysis +//===----------------------------------------------------------------------===// + +class InstAnalyzer { + const CodeGenDAGPatterns &CDP; +public: + bool hasSideEffects; + bool mayStore; + bool mayLoad; + bool isBitcast; + bool isVariadic; + bool hasChain; + + InstAnalyzer(const CodeGenDAGPatterns &cdp) + : CDP(cdp), hasSideEffects(false), mayStore(false), mayLoad(false), + isBitcast(false), isVariadic(false), hasChain(false) {} + + void Analyze(const PatternToMatch &Pat) { + const TreePatternNode *N = Pat.getSrcPattern(); + AnalyzeNode(N); + // These properties are detected only on the root node. + isBitcast = IsNodeBitcast(N); + } + +private: + bool IsNodeBitcast(const TreePatternNode *N) const { + if (hasSideEffects || mayLoad || mayStore || isVariadic) + return false; + + if (N->isLeaf()) + return false; + if (N->getNumChildren() != 1 || !N->getChild(0)->isLeaf()) + return false; + + const SDNodeInfo &OpInfo = CDP.getSDNodeInfo(N->getOperator()); + if (OpInfo.getNumResults() != 1 || OpInfo.getNumOperands() != 1) + return false; + return OpInfo.getEnumName() == "ISD::BITCAST"; + } + +public: + void AnalyzeNode(const TreePatternNode *N) { + if (N->isLeaf()) { + if (DefInit *DI = dyn_cast<DefInit>(N->getLeafValue())) { + Record *LeafRec = DI->getDef(); + // Handle ComplexPattern leaves. + if (LeafRec->isSubClassOf("ComplexPattern")) { + const ComplexPattern &CP = CDP.getComplexPattern(LeafRec); + if (CP.hasProperty(SDNPMayStore)) mayStore = true; + if (CP.hasProperty(SDNPMayLoad)) mayLoad = true; + if (CP.hasProperty(SDNPSideEffect)) hasSideEffects = true; + } + } + return; + } + + // Analyze children. + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) + AnalyzeNode(N->getChild(i)); + + // Notice properties of the node. + if (N->NodeHasProperty(SDNPMayStore, CDP)) mayStore = true; + if (N->NodeHasProperty(SDNPMayLoad, CDP)) mayLoad = true; + if (N->NodeHasProperty(SDNPSideEffect, CDP)) hasSideEffects = true; + if (N->NodeHasProperty(SDNPVariadic, CDP)) isVariadic = true; + if (N->NodeHasProperty(SDNPHasChain, CDP)) hasChain = true; + + if (const CodeGenIntrinsic *IntInfo = N->getIntrinsicInfo(CDP)) { + // If this is an intrinsic, analyze it. + if (IntInfo->ModRef & CodeGenIntrinsic::MR_Ref) + mayLoad = true;// These may load memory. + + if (IntInfo->ModRef & CodeGenIntrinsic::MR_Mod) + mayStore = true;// Intrinsics that can write to memory are 'mayStore'. + + if (IntInfo->ModRef >= CodeGenIntrinsic::ReadWriteMem || + IntInfo->hasSideEffects) + // ReadWriteMem intrinsics can have other strange effects. + hasSideEffects = true; + } + } + +}; + +static bool InferFromPattern(CodeGenInstruction &InstInfo, + const InstAnalyzer &PatInfo, + Record *PatDef) { + bool Error = false; + + // Remember where InstInfo got its flags. + if (InstInfo.hasUndefFlags()) + InstInfo.InferredFrom = PatDef; + + // Check explicitly set flags for consistency. + if (InstInfo.hasSideEffects != PatInfo.hasSideEffects && + !InstInfo.hasSideEffects_Unset) { + // Allow explicitly setting hasSideEffects = 1 on instructions, even when + // the pattern has no side effects. That could be useful for div/rem + // instructions that may trap. + if (!InstInfo.hasSideEffects) { + Error = true; + PrintError(PatDef->getLoc(), "Pattern doesn't match hasSideEffects = " + + Twine(InstInfo.hasSideEffects)); + } + } + + if (InstInfo.mayStore != PatInfo.mayStore && !InstInfo.mayStore_Unset) { + Error = true; + PrintError(PatDef->getLoc(), "Pattern doesn't match mayStore = " + + Twine(InstInfo.mayStore)); + } + + if (InstInfo.mayLoad != PatInfo.mayLoad && !InstInfo.mayLoad_Unset) { + // Allow explicitly setting mayLoad = 1, even when the pattern has no loads. + // Some targets translate immediates to loads. + if (!InstInfo.mayLoad) { + Error = true; + PrintError(PatDef->getLoc(), "Pattern doesn't match mayLoad = " + + Twine(InstInfo.mayLoad)); + } + } + + // Transfer inferred flags. + InstInfo.hasSideEffects |= PatInfo.hasSideEffects; + InstInfo.mayStore |= PatInfo.mayStore; + InstInfo.mayLoad |= PatInfo.mayLoad; + + // These flags are silently added without any verification. + // FIXME: To match historical behavior of TableGen, for now add those flags + // only when we're inferring from the primary instruction pattern. + if (PatDef->isSubClassOf("Instruction")) { + InstInfo.isBitcast |= PatInfo.isBitcast; + InstInfo.hasChain |= PatInfo.hasChain; + InstInfo.hasChain_Inferred = true; + } + + // Don't infer isVariadic. This flag means something different on SDNodes and + // instructions. For example, a CALL SDNode is variadic because it has the + // call arguments as operands, but a CALL instruction is not variadic - it + // has argument registers as implicit, not explicit uses. + + return Error; +} + +/// hasNullFragReference - Return true if the DAG has any reference to the +/// null_frag operator. +static bool hasNullFragReference(DagInit *DI) { + DefInit *OpDef = dyn_cast<DefInit>(DI->getOperator()); + if (!OpDef) return false; + Record *Operator = OpDef->getDef(); + + // If this is the null fragment, return true. + if (Operator->getName() == "null_frag") return true; + // If any of the arguments reference the null fragment, return true. + for (unsigned i = 0, e = DI->getNumArgs(); i != e; ++i) { + DagInit *Arg = dyn_cast<DagInit>(DI->getArg(i)); + if (Arg && hasNullFragReference(Arg)) + return true; + } + + return false; +} + +/// hasNullFragReference - Return true if any DAG in the list references +/// the null_frag operator. +static bool hasNullFragReference(ListInit *LI) { + for (Init *I : LI->getValues()) { + DagInit *DI = dyn_cast<DagInit>(I); + assert(DI && "non-dag in an instruction Pattern list?!"); + if (hasNullFragReference(DI)) + return true; + } + return false; +} + +/// Get all the instructions in a tree. +static void +getInstructionsInTree(TreePatternNode *Tree, SmallVectorImpl<Record*> &Instrs) { + if (Tree->isLeaf()) + return; + if (Tree->getOperator()->isSubClassOf("Instruction")) + Instrs.push_back(Tree->getOperator()); + for (unsigned i = 0, e = Tree->getNumChildren(); i != e; ++i) + getInstructionsInTree(Tree->getChild(i), Instrs); +} + +/// Check the class of a pattern leaf node against the instruction operand it +/// represents. +static bool checkOperandClass(CGIOperandList::OperandInfo &OI, + Record *Leaf) { + if (OI.Rec == Leaf) + return true; + + // Allow direct value types to be used in instruction set patterns. + // The type will be checked later. + if (Leaf->isSubClassOf("ValueType")) + return true; + + // Patterns can also be ComplexPattern instances. + if (Leaf->isSubClassOf("ComplexPattern")) + return true; + + return false; +} + +void CodeGenDAGPatterns::parseInstructionPattern( + CodeGenInstruction &CGI, ListInit *Pat, DAGInstMap &DAGInsts) { + + assert(!DAGInsts.count(CGI.TheDef) && "Instruction already parsed!"); + + // Parse the instruction. + TreePattern I(CGI.TheDef, Pat, true, *this); + + // InstInputs - Keep track of all of the inputs of the instruction, along + // with the record they are declared as. + std::map<std::string, TreePatternNodePtr> InstInputs; + + // InstResults - Keep track of all the virtual registers that are 'set' + // in the instruction, including what reg class they are. + MapVector<std::string, TreePatternNodePtr, std::map<std::string, unsigned>> + InstResults; + + std::vector<Record*> InstImpResults; + + // Verify that the top-level forms in the instruction are of void type, and + // fill in the InstResults map. + SmallString<32> TypesString; + for (unsigned j = 0, e = I.getNumTrees(); j != e; ++j) { + TypesString.clear(); + TreePatternNodePtr Pat = I.getTree(j); + if (Pat->getNumTypes() != 0) { + raw_svector_ostream OS(TypesString); + for (unsigned k = 0, ke = Pat->getNumTypes(); k != ke; ++k) { + if (k > 0) + OS << ", "; + Pat->getExtType(k).writeToStream(OS); + } + I.error("Top-level forms in instruction pattern should have" + " void types, has types " + + OS.str()); + } + + // Find inputs and outputs, and verify the structure of the uses/defs. + FindPatternInputsAndOutputs(I, Pat, InstInputs, InstResults, + InstImpResults); + } + + // Now that we have inputs and outputs of the pattern, inspect the operands + // list for the instruction. This determines the order that operands are + // added to the machine instruction the node corresponds to. + unsigned NumResults = InstResults.size(); + + // Parse the operands list from the (ops) list, validating it. + assert(I.getArgList().empty() && "Args list should still be empty here!"); + + // Check that all of the results occur first in the list. + std::vector<Record*> Results; + std::vector<unsigned> ResultIndices; + SmallVector<TreePatternNodePtr, 2> ResNodes; + for (unsigned i = 0; i != NumResults; ++i) { + if (i == CGI.Operands.size()) { + const std::string &OpName = + std::find_if(InstResults.begin(), InstResults.end(), + [](const std::pair<std::string, TreePatternNodePtr> &P) { + return P.second; + }) + ->first; + + I.error("'" + OpName + "' set but does not appear in operand list!"); + } + + const std::string &OpName = CGI.Operands[i].Name; + + // Check that it exists in InstResults. + auto InstResultIter = InstResults.find(OpName); + if (InstResultIter == InstResults.end() || !InstResultIter->second) + I.error("Operand $" + OpName + " does not exist in operand list!"); + + TreePatternNodePtr RNode = InstResultIter->second; + Record *R = cast<DefInit>(RNode->getLeafValue())->getDef(); + ResNodes.push_back(std::move(RNode)); + if (!R) + I.error("Operand $" + OpName + " should be a set destination: all " + "outputs must occur before inputs in operand list!"); + + if (!checkOperandClass(CGI.Operands[i], R)) + I.error("Operand $" + OpName + " class mismatch!"); + + // Remember the return type. + Results.push_back(CGI.Operands[i].Rec); + + // Remember the result index. + ResultIndices.push_back(std::distance(InstResults.begin(), InstResultIter)); + + // Okay, this one checks out. + InstResultIter->second = nullptr; + } + + // Loop over the inputs next. + std::vector<TreePatternNodePtr> ResultNodeOperands; + std::vector<Record*> Operands; + for (unsigned i = NumResults, e = CGI.Operands.size(); i != e; ++i) { + CGIOperandList::OperandInfo &Op = CGI.Operands[i]; + const std::string &OpName = Op.Name; + if (OpName.empty()) + I.error("Operand #" + Twine(i) + " in operands list has no name!"); + + if (!InstInputs.count(OpName)) { + // If this is an operand with a DefaultOps set filled in, we can ignore + // this. When we codegen it, we will do so as always executed. + if (Op.Rec->isSubClassOf("OperandWithDefaultOps")) { + // Does it have a non-empty DefaultOps field? If so, ignore this + // operand. + if (!getDefaultOperand(Op.Rec).DefaultOps.empty()) + continue; + } + I.error("Operand $" + OpName + + " does not appear in the instruction pattern"); + } + TreePatternNodePtr InVal = InstInputs[OpName]; + InstInputs.erase(OpName); // It occurred, remove from map. + + if (InVal->isLeaf() && isa<DefInit>(InVal->getLeafValue())) { + Record *InRec = static_cast<DefInit*>(InVal->getLeafValue())->getDef(); + if (!checkOperandClass(Op, InRec)) + I.error("Operand $" + OpName + "'s register class disagrees" + " between the operand and pattern"); + } + Operands.push_back(Op.Rec); + + // Construct the result for the dest-pattern operand list. + TreePatternNodePtr OpNode = InVal->clone(); + + // No predicate is useful on the result. + OpNode->clearPredicateCalls(); + + // Promote the xform function to be an explicit node if set. + if (Record *Xform = OpNode->getTransformFn()) { + OpNode->setTransformFn(nullptr); + std::vector<TreePatternNodePtr> Children; + Children.push_back(OpNode); + OpNode = std::make_shared<TreePatternNode>(Xform, std::move(Children), + OpNode->getNumTypes()); + } + + ResultNodeOperands.push_back(std::move(OpNode)); + } + + if (!InstInputs.empty()) + I.error("Input operand $" + InstInputs.begin()->first + + " occurs in pattern but not in operands list!"); + + TreePatternNodePtr ResultPattern = std::make_shared<TreePatternNode>( + I.getRecord(), std::move(ResultNodeOperands), + GetNumNodeResults(I.getRecord(), *this)); + // Copy fully inferred output node types to instruction result pattern. + for (unsigned i = 0; i != NumResults; ++i) { + assert(ResNodes[i]->getNumTypes() == 1 && "FIXME: Unhandled"); + ResultPattern->setType(i, ResNodes[i]->getExtType(0)); + ResultPattern->setResultIndex(i, ResultIndices[i]); + } + + // FIXME: Assume only the first tree is the pattern. The others are clobber + // nodes. + TreePatternNodePtr Pattern = I.getTree(0); + TreePatternNodePtr SrcPattern; + if (Pattern->getOperator()->getName() == "set") { + SrcPattern = Pattern->getChild(Pattern->getNumChildren()-1)->clone(); + } else{ + // Not a set (store or something?) + SrcPattern = Pattern; + } + + // Create and insert the instruction. + // FIXME: InstImpResults should not be part of DAGInstruction. + Record *R = I.getRecord(); + DAGInsts.emplace(std::piecewise_construct, std::forward_as_tuple(R), + std::forward_as_tuple(Results, Operands, InstImpResults, + SrcPattern, ResultPattern)); + + LLVM_DEBUG(I.dump()); +} + +/// ParseInstructions - Parse all of the instructions, inlining and resolving +/// any fragments involved. This populates the Instructions list with fully +/// resolved instructions. +void CodeGenDAGPatterns::ParseInstructions() { + std::vector<Record*> Instrs = Records.getAllDerivedDefinitions("Instruction"); + + for (Record *Instr : Instrs) { + ListInit *LI = nullptr; + + if (isa<ListInit>(Instr->getValueInit("Pattern"))) + LI = Instr->getValueAsListInit("Pattern"); + + // If there is no pattern, only collect minimal information about the + // instruction for its operand list. We have to assume that there is one + // result, as we have no detailed info. A pattern which references the + // null_frag operator is as-if no pattern were specified. Normally this + // is from a multiclass expansion w/ a SDPatternOperator passed in as + // null_frag. + if (!LI || LI->empty() || hasNullFragReference(LI)) { + std::vector<Record*> Results; + std::vector<Record*> Operands; + + CodeGenInstruction &InstInfo = Target.getInstruction(Instr); + + if (InstInfo.Operands.size() != 0) { + for (unsigned j = 0, e = InstInfo.Operands.NumDefs; j < e; ++j) + Results.push_back(InstInfo.Operands[j].Rec); + + // The rest are inputs. + for (unsigned j = InstInfo.Operands.NumDefs, + e = InstInfo.Operands.size(); j < e; ++j) + Operands.push_back(InstInfo.Operands[j].Rec); + } + + // Create and insert the instruction. + std::vector<Record*> ImpResults; + Instructions.insert(std::make_pair(Instr, + DAGInstruction(Results, Operands, ImpResults))); + continue; // no pattern. + } + + CodeGenInstruction &CGI = Target.getInstruction(Instr); + parseInstructionPattern(CGI, LI, Instructions); + } + + // If we can, convert the instructions to be patterns that are matched! + for (auto &Entry : Instructions) { + Record *Instr = Entry.first; + DAGInstruction &TheInst = Entry.second; + TreePatternNodePtr SrcPattern = TheInst.getSrcPattern(); + TreePatternNodePtr ResultPattern = TheInst.getResultPattern(); + + if (SrcPattern && ResultPattern) { + TreePattern Pattern(Instr, SrcPattern, true, *this); + TreePattern Result(Instr, ResultPattern, false, *this); + ParseOnePattern(Instr, Pattern, Result, TheInst.getImpResults()); + } + } +} + +typedef std::pair<TreePatternNode *, unsigned> NameRecord; + +static void FindNames(TreePatternNode *P, + std::map<std::string, NameRecord> &Names, + TreePattern *PatternTop) { + if (!P->getName().empty()) { + NameRecord &Rec = Names[P->getName()]; + // If this is the first instance of the name, remember the node. + if (Rec.second++ == 0) + Rec.first = P; + else if (Rec.first->getExtTypes() != P->getExtTypes()) + PatternTop->error("repetition of value: $" + P->getName() + + " where different uses have different types!"); + } + + if (!P->isLeaf()) { + for (unsigned i = 0, e = P->getNumChildren(); i != e; ++i) + FindNames(P->getChild(i), Names, PatternTop); + } +} + +std::vector<Predicate> CodeGenDAGPatterns::makePredList(ListInit *L) { + std::vector<Predicate> Preds; + for (Init *I : L->getValues()) { + if (DefInit *Pred = dyn_cast<DefInit>(I)) + Preds.push_back(Pred->getDef()); + else + llvm_unreachable("Non-def on the list"); + } + + // Sort so that different orders get canonicalized to the same string. + llvm::sort(Preds); + return Preds; +} + +void CodeGenDAGPatterns::AddPatternToMatch(TreePattern *Pattern, + PatternToMatch &&PTM) { + // Do some sanity checking on the pattern we're about to match. + std::string Reason; + if (!PTM.getSrcPattern()->canPatternMatch(Reason, *this)) { + PrintWarning(Pattern->getRecord()->getLoc(), + Twine("Pattern can never match: ") + Reason); + return; + } + + // If the source pattern's root is a complex pattern, that complex pattern + // must specify the nodes it can potentially match. + if (const ComplexPattern *CP = + PTM.getSrcPattern()->getComplexPatternInfo(*this)) + if (CP->getRootNodes().empty()) + Pattern->error("ComplexPattern at root must specify list of opcodes it" + " could match"); + + + // Find all of the named values in the input and output, ensure they have the + // same type. + std::map<std::string, NameRecord> SrcNames, DstNames; + FindNames(PTM.getSrcPattern(), SrcNames, Pattern); + FindNames(PTM.getDstPattern(), DstNames, Pattern); + + // Scan all of the named values in the destination pattern, rejecting them if + // they don't exist in the input pattern. + for (const auto &Entry : DstNames) { + if (SrcNames[Entry.first].first == nullptr) + Pattern->error("Pattern has input without matching name in output: $" + + Entry.first); + } + + // Scan all of the named values in the source pattern, rejecting them if the + // name isn't used in the dest, and isn't used to tie two values together. + for (const auto &Entry : SrcNames) + if (DstNames[Entry.first].first == nullptr && + SrcNames[Entry.first].second == 1) + Pattern->error("Pattern has dead named input: $" + Entry.first); + + PatternsToMatch.push_back(PTM); +} + +void CodeGenDAGPatterns::InferInstructionFlags() { + ArrayRef<const CodeGenInstruction*> Instructions = + Target.getInstructionsByEnumValue(); + + unsigned Errors = 0; + + // Try to infer flags from all patterns in PatternToMatch. These include + // both the primary instruction patterns (which always come first) and + // patterns defined outside the instruction. + for (const PatternToMatch &PTM : ptms()) { + // We can only infer from single-instruction patterns, otherwise we won't + // know which instruction should get the flags. + SmallVector<Record*, 8> PatInstrs; + getInstructionsInTree(PTM.getDstPattern(), PatInstrs); + if (PatInstrs.size() != 1) + continue; + + // Get the single instruction. + CodeGenInstruction &InstInfo = Target.getInstruction(PatInstrs.front()); + + // Only infer properties from the first pattern. We'll verify the others. + if (InstInfo.InferredFrom) + continue; + + InstAnalyzer PatInfo(*this); + PatInfo.Analyze(PTM); + Errors += InferFromPattern(InstInfo, PatInfo, PTM.getSrcRecord()); + } + + if (Errors) + PrintFatalError("pattern conflicts"); + + // If requested by the target, guess any undefined properties. + if (Target.guessInstructionProperties()) { + for (unsigned i = 0, e = Instructions.size(); i != e; ++i) { + CodeGenInstruction *InstInfo = + const_cast<CodeGenInstruction *>(Instructions[i]); + if (InstInfo->InferredFrom) + continue; + // The mayLoad and mayStore flags default to false. + // Conservatively assume hasSideEffects if it wasn't explicit. + if (InstInfo->hasSideEffects_Unset) + InstInfo->hasSideEffects = true; + } + return; + } + + // Complain about any flags that are still undefined. + for (unsigned i = 0, e = Instructions.size(); i != e; ++i) { + CodeGenInstruction *InstInfo = + const_cast<CodeGenInstruction *>(Instructions[i]); + if (InstInfo->InferredFrom) + continue; + if (InstInfo->hasSideEffects_Unset) + PrintError(InstInfo->TheDef->getLoc(), + "Can't infer hasSideEffects from patterns"); + if (InstInfo->mayStore_Unset) + PrintError(InstInfo->TheDef->getLoc(), + "Can't infer mayStore from patterns"); + if (InstInfo->mayLoad_Unset) + PrintError(InstInfo->TheDef->getLoc(), + "Can't infer mayLoad from patterns"); + } +} + + +/// Verify instruction flags against pattern node properties. +void CodeGenDAGPatterns::VerifyInstructionFlags() { + unsigned Errors = 0; + for (ptm_iterator I = ptm_begin(), E = ptm_end(); I != E; ++I) { + const PatternToMatch &PTM = *I; + SmallVector<Record*, 8> Instrs; + getInstructionsInTree(PTM.getDstPattern(), Instrs); + if (Instrs.empty()) + continue; + + // Count the number of instructions with each flag set. + unsigned NumSideEffects = 0; + unsigned NumStores = 0; + unsigned NumLoads = 0; + for (const Record *Instr : Instrs) { + const CodeGenInstruction &InstInfo = Target.getInstruction(Instr); + NumSideEffects += InstInfo.hasSideEffects; + NumStores += InstInfo.mayStore; + NumLoads += InstInfo.mayLoad; + } + + // Analyze the source pattern. + InstAnalyzer PatInfo(*this); + PatInfo.Analyze(PTM); + + // Collect error messages. + SmallVector<std::string, 4> Msgs; + + // Check for missing flags in the output. + // Permit extra flags for now at least. + if (PatInfo.hasSideEffects && !NumSideEffects) + Msgs.push_back("pattern has side effects, but hasSideEffects isn't set"); + + // Don't verify store flags on instructions with side effects. At least for + // intrinsics, side effects implies mayStore. + if (!PatInfo.hasSideEffects && PatInfo.mayStore && !NumStores) + Msgs.push_back("pattern may store, but mayStore isn't set"); + + // Similarly, mayStore implies mayLoad on intrinsics. + if (!PatInfo.mayStore && PatInfo.mayLoad && !NumLoads) + Msgs.push_back("pattern may load, but mayLoad isn't set"); + + // Print error messages. + if (Msgs.empty()) + continue; + ++Errors; + + for (const std::string &Msg : Msgs) + PrintError(PTM.getSrcRecord()->getLoc(), Twine(Msg) + " on the " + + (Instrs.size() == 1 ? + "instruction" : "output instructions")); + // Provide the location of the relevant instruction definitions. + for (const Record *Instr : Instrs) { + if (Instr != PTM.getSrcRecord()) + PrintError(Instr->getLoc(), "defined here"); + const CodeGenInstruction &InstInfo = Target.getInstruction(Instr); + if (InstInfo.InferredFrom && + InstInfo.InferredFrom != InstInfo.TheDef && + InstInfo.InferredFrom != PTM.getSrcRecord()) + PrintError(InstInfo.InferredFrom->getLoc(), "inferred from pattern"); + } + } + if (Errors) + PrintFatalError("Errors in DAG patterns"); +} + +/// Given a pattern result with an unresolved type, see if we can find one +/// instruction with an unresolved result type. Force this result type to an +/// arbitrary element if it's possible types to converge results. +static bool ForceArbitraryInstResultType(TreePatternNode *N, TreePattern &TP) { + if (N->isLeaf()) + return false; + + // Analyze children. + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) + if (ForceArbitraryInstResultType(N->getChild(i), TP)) + return true; + + if (!N->getOperator()->isSubClassOf("Instruction")) + return false; + + // If this type is already concrete or completely unknown we can't do + // anything. + TypeInfer &TI = TP.getInfer(); + for (unsigned i = 0, e = N->getNumTypes(); i != e; ++i) { + if (N->getExtType(i).empty() || TI.isConcrete(N->getExtType(i), false)) + continue; + + // Otherwise, force its type to an arbitrary choice. + if (TI.forceArbitrary(N->getExtType(i))) + return true; + } + + return false; +} + +// Promote xform function to be an explicit node wherever set. +static TreePatternNodePtr PromoteXForms(TreePatternNodePtr N) { + if (Record *Xform = N->getTransformFn()) { + N->setTransformFn(nullptr); + std::vector<TreePatternNodePtr> Children; + Children.push_back(PromoteXForms(N)); + return std::make_shared<TreePatternNode>(Xform, std::move(Children), + N->getNumTypes()); + } + + if (!N->isLeaf()) + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) { + TreePatternNodePtr Child = N->getChildShared(i); + N->setChild(i, PromoteXForms(Child)); + } + return N; +} + +void CodeGenDAGPatterns::ParseOnePattern(Record *TheDef, + TreePattern &Pattern, TreePattern &Result, + const std::vector<Record *> &InstImpResults) { + + // Inline pattern fragments and expand multiple alternatives. + Pattern.InlinePatternFragments(); + Result.InlinePatternFragments(); + + if (Result.getNumTrees() != 1) + Result.error("Cannot use multi-alternative fragments in result pattern!"); + + // Infer types. + bool IterateInference; + bool InferredAllPatternTypes, InferredAllResultTypes; + do { + // Infer as many types as possible. If we cannot infer all of them, we + // can never do anything with this pattern: report it to the user. + InferredAllPatternTypes = + Pattern.InferAllTypes(&Pattern.getNamedNodesMap()); + + // Infer as many types as possible. If we cannot infer all of them, we + // can never do anything with this pattern: report it to the user. + InferredAllResultTypes = + Result.InferAllTypes(&Pattern.getNamedNodesMap()); + + IterateInference = false; + + // Apply the type of the result to the source pattern. This helps us + // resolve cases where the input type is known to be a pointer type (which + // is considered resolved), but the result knows it needs to be 32- or + // 64-bits. Infer the other way for good measure. + for (auto T : Pattern.getTrees()) + for (unsigned i = 0, e = std::min(Result.getOnlyTree()->getNumTypes(), + T->getNumTypes()); + i != e; ++i) { + IterateInference |= T->UpdateNodeType( + i, Result.getOnlyTree()->getExtType(i), Result); + IterateInference |= Result.getOnlyTree()->UpdateNodeType( + i, T->getExtType(i), Result); + } + + // If our iteration has converged and the input pattern's types are fully + // resolved but the result pattern is not fully resolved, we may have a + // situation where we have two instructions in the result pattern and + // the instructions require a common register class, but don't care about + // what actual MVT is used. This is actually a bug in our modelling: + // output patterns should have register classes, not MVTs. + // + // In any case, to handle this, we just go through and disambiguate some + // arbitrary types to the result pattern's nodes. + if (!IterateInference && InferredAllPatternTypes && + !InferredAllResultTypes) + IterateInference = + ForceArbitraryInstResultType(Result.getTree(0).get(), Result); + } while (IterateInference); + + // Verify that we inferred enough types that we can do something with the + // pattern and result. If these fire the user has to add type casts. + if (!InferredAllPatternTypes) + Pattern.error("Could not infer all types in pattern!"); + if (!InferredAllResultTypes) { + Pattern.dump(); + Result.error("Could not infer all types in pattern result!"); + } + + // Promote xform function to be an explicit node wherever set. + TreePatternNodePtr DstShared = PromoteXForms(Result.getOnlyTree()); + + TreePattern Temp(Result.getRecord(), DstShared, false, *this); + Temp.InferAllTypes(); + + ListInit *Preds = TheDef->getValueAsListInit("Predicates"); + int Complexity = TheDef->getValueAsInt("AddedComplexity"); + + if (PatternRewriter) + PatternRewriter(&Pattern); + + // A pattern may end up with an "impossible" type, i.e. a situation + // where all types have been eliminated for some node in this pattern. + // This could occur for intrinsics that only make sense for a specific + // value type, and use a specific register class. If, for some mode, + // that register class does not accept that type, the type inference + // will lead to a contradiction, which is not an error however, but + // a sign that this pattern will simply never match. + if (Temp.getOnlyTree()->hasPossibleType()) + for (auto T : Pattern.getTrees()) + if (T->hasPossibleType()) + AddPatternToMatch(&Pattern, + PatternToMatch(TheDef, makePredList(Preds), + T, Temp.getOnlyTree(), + InstImpResults, Complexity, + TheDef->getID())); +} + +void CodeGenDAGPatterns::ParsePatterns() { + std::vector<Record*> Patterns = Records.getAllDerivedDefinitions("Pattern"); + + for (Record *CurPattern : Patterns) { + DagInit *Tree = CurPattern->getValueAsDag("PatternToMatch"); + + // If the pattern references the null_frag, there's nothing to do. + if (hasNullFragReference(Tree)) + continue; + + TreePattern Pattern(CurPattern, Tree, true, *this); + + ListInit *LI = CurPattern->getValueAsListInit("ResultInstrs"); + if (LI->empty()) continue; // no pattern. + + // Parse the instruction. + TreePattern Result(CurPattern, LI, false, *this); + + if (Result.getNumTrees() != 1) + Result.error("Cannot handle instructions producing instructions " + "with temporaries yet!"); + + // Validate that the input pattern is correct. + std::map<std::string, TreePatternNodePtr> InstInputs; + MapVector<std::string, TreePatternNodePtr, std::map<std::string, unsigned>> + InstResults; + std::vector<Record*> InstImpResults; + for (unsigned j = 0, ee = Pattern.getNumTrees(); j != ee; ++j) + FindPatternInputsAndOutputs(Pattern, Pattern.getTree(j), InstInputs, + InstResults, InstImpResults); + + ParseOnePattern(CurPattern, Pattern, Result, InstImpResults); + } +} + +static void collectModes(std::set<unsigned> &Modes, const TreePatternNode *N) { + for (const TypeSetByHwMode &VTS : N->getExtTypes()) + for (const auto &I : VTS) + Modes.insert(I.first); + + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) + collectModes(Modes, N->getChild(i)); +} + +void CodeGenDAGPatterns::ExpandHwModeBasedTypes() { + const CodeGenHwModes &CGH = getTargetInfo().getHwModes(); + std::map<unsigned,std::vector<Predicate>> ModeChecks; + std::vector<PatternToMatch> Copy = PatternsToMatch; + PatternsToMatch.clear(); + + auto AppendPattern = [this, &ModeChecks](PatternToMatch &P, unsigned Mode) { + TreePatternNodePtr NewSrc = P.SrcPattern->clone(); + TreePatternNodePtr NewDst = P.DstPattern->clone(); + if (!NewSrc->setDefaultMode(Mode) || !NewDst->setDefaultMode(Mode)) { + return; + } + + std::vector<Predicate> Preds = P.Predicates; + const std::vector<Predicate> &MC = ModeChecks[Mode]; + Preds.insert(Preds.end(), MC.begin(), MC.end()); + PatternsToMatch.emplace_back(P.getSrcRecord(), Preds, std::move(NewSrc), + std::move(NewDst), P.getDstRegs(), + P.getAddedComplexity(), Record::getNewUID(), + Mode); + }; + + for (PatternToMatch &P : Copy) { + TreePatternNodePtr SrcP = nullptr, DstP = nullptr; + if (P.SrcPattern->hasProperTypeByHwMode()) + SrcP = P.SrcPattern; + if (P.DstPattern->hasProperTypeByHwMode()) + DstP = P.DstPattern; + if (!SrcP && !DstP) { + PatternsToMatch.push_back(P); + continue; + } + + std::set<unsigned> Modes; + if (SrcP) + collectModes(Modes, SrcP.get()); + if (DstP) + collectModes(Modes, DstP.get()); + + // The predicate for the default mode needs to be constructed for each + // pattern separately. + // Since not all modes must be present in each pattern, if a mode m is + // absent, then there is no point in constructing a check for m. If such + // a check was created, it would be equivalent to checking the default + // mode, except not all modes' predicates would be a part of the checking + // code. The subsequently generated check for the default mode would then + // have the exact same patterns, but a different predicate code. To avoid + // duplicated patterns with different predicate checks, construct the + // default check as a negation of all predicates that are actually present + // in the source/destination patterns. + std::vector<Predicate> DefaultPred; + + for (unsigned M : Modes) { + if (M == DefaultMode) + continue; + if (ModeChecks.find(M) != ModeChecks.end()) + continue; + + // Fill the map entry for this mode. + const HwMode &HM = CGH.getMode(M); + ModeChecks[M].emplace_back(Predicate(HM.Features, true)); + + // Add negations of the HM's predicates to the default predicate. + DefaultPred.emplace_back(Predicate(HM.Features, false)); + } + + for (unsigned M : Modes) { + if (M == DefaultMode) + continue; + AppendPattern(P, M); + } + + bool HasDefault = Modes.count(DefaultMode); + if (HasDefault) + AppendPattern(P, DefaultMode); + } +} + +/// Dependent variable map for CodeGenDAGPattern variant generation +typedef StringMap<int> DepVarMap; + +static void FindDepVarsOf(TreePatternNode *N, DepVarMap &DepMap) { + if (N->isLeaf()) { + if (N->hasName() && isa<DefInit>(N->getLeafValue())) + DepMap[N->getName()]++; + } else { + for (size_t i = 0, e = N->getNumChildren(); i != e; ++i) + FindDepVarsOf(N->getChild(i), DepMap); + } +} + +/// Find dependent variables within child patterns +static void FindDepVars(TreePatternNode *N, MultipleUseVarSet &DepVars) { + DepVarMap depcounts; + FindDepVarsOf(N, depcounts); + for (const auto &Pair : depcounts) { + if (Pair.getValue() > 1) + DepVars.insert(Pair.getKey()); + } +} + +#ifndef NDEBUG +/// Dump the dependent variable set: +static void DumpDepVars(MultipleUseVarSet &DepVars) { + if (DepVars.empty()) { + LLVM_DEBUG(errs() << "<empty set>"); + } else { + LLVM_DEBUG(errs() << "[ "); + for (const auto &DepVar : DepVars) { + LLVM_DEBUG(errs() << DepVar.getKey() << " "); + } + LLVM_DEBUG(errs() << "]"); + } +} +#endif + + +/// CombineChildVariants - Given a bunch of permutations of each child of the +/// 'operator' node, put them together in all possible ways. +static void CombineChildVariants( + TreePatternNodePtr Orig, + const std::vector<std::vector<TreePatternNodePtr>> &ChildVariants, + std::vector<TreePatternNodePtr> &OutVariants, CodeGenDAGPatterns &CDP, + const MultipleUseVarSet &DepVars) { + // Make sure that each operand has at least one variant to choose from. + for (const auto &Variants : ChildVariants) + if (Variants.empty()) + return; + + // The end result is an all-pairs construction of the resultant pattern. + std::vector<unsigned> Idxs; + Idxs.resize(ChildVariants.size()); + bool NotDone; + do { +#ifndef NDEBUG + LLVM_DEBUG(if (!Idxs.empty()) { + errs() << Orig->getOperator()->getName() << ": Idxs = [ "; + for (unsigned Idx : Idxs) { + errs() << Idx << " "; + } + errs() << "]\n"; + }); +#endif + // Create the variant and add it to the output list. + std::vector<TreePatternNodePtr> NewChildren; + for (unsigned i = 0, e = ChildVariants.size(); i != e; ++i) + NewChildren.push_back(ChildVariants[i][Idxs[i]]); + TreePatternNodePtr R = std::make_shared<TreePatternNode>( + Orig->getOperator(), std::move(NewChildren), Orig->getNumTypes()); + + // Copy over properties. + R->setName(Orig->getName()); + R->setNamesAsPredicateArg(Orig->getNamesAsPredicateArg()); + R->setPredicateCalls(Orig->getPredicateCalls()); + R->setTransformFn(Orig->getTransformFn()); + for (unsigned i = 0, e = Orig->getNumTypes(); i != e; ++i) + R->setType(i, Orig->getExtType(i)); + + // If this pattern cannot match, do not include it as a variant. + std::string ErrString; + // Scan to see if this pattern has already been emitted. We can get + // duplication due to things like commuting: + // (and GPRC:$a, GPRC:$b) -> (and GPRC:$b, GPRC:$a) + // which are the same pattern. Ignore the dups. + if (R->canPatternMatch(ErrString, CDP) && + none_of(OutVariants, [&](TreePatternNodePtr Variant) { + return R->isIsomorphicTo(Variant.get(), DepVars); + })) + OutVariants.push_back(R); + + // Increment indices to the next permutation by incrementing the + // indices from last index backward, e.g., generate the sequence + // [0, 0], [0, 1], [1, 0], [1, 1]. + int IdxsIdx; + for (IdxsIdx = Idxs.size() - 1; IdxsIdx >= 0; --IdxsIdx) { + if (++Idxs[IdxsIdx] == ChildVariants[IdxsIdx].size()) + Idxs[IdxsIdx] = 0; + else + break; + } + NotDone = (IdxsIdx >= 0); + } while (NotDone); +} + +/// CombineChildVariants - A helper function for binary operators. +/// +static void CombineChildVariants(TreePatternNodePtr Orig, + const std::vector<TreePatternNodePtr> &LHS, + const std::vector<TreePatternNodePtr> &RHS, + std::vector<TreePatternNodePtr> &OutVariants, + CodeGenDAGPatterns &CDP, + const MultipleUseVarSet &DepVars) { + std::vector<std::vector<TreePatternNodePtr>> ChildVariants; + ChildVariants.push_back(LHS); + ChildVariants.push_back(RHS); + CombineChildVariants(Orig, ChildVariants, OutVariants, CDP, DepVars); +} + +static void +GatherChildrenOfAssociativeOpcode(TreePatternNodePtr N, + std::vector<TreePatternNodePtr> &Children) { + assert(N->getNumChildren()==2 &&"Associative but doesn't have 2 children!"); + Record *Operator = N->getOperator(); + + // Only permit raw nodes. + if (!N->getName().empty() || !N->getPredicateCalls().empty() || + N->getTransformFn()) { + Children.push_back(N); + return; + } + + if (N->getChild(0)->isLeaf() || N->getChild(0)->getOperator() != Operator) + Children.push_back(N->getChildShared(0)); + else + GatherChildrenOfAssociativeOpcode(N->getChildShared(0), Children); + + if (N->getChild(1)->isLeaf() || N->getChild(1)->getOperator() != Operator) + Children.push_back(N->getChildShared(1)); + else + GatherChildrenOfAssociativeOpcode(N->getChildShared(1), Children); +} + +/// GenerateVariantsOf - Given a pattern N, generate all permutations we can of +/// the (potentially recursive) pattern by using algebraic laws. +/// +static void GenerateVariantsOf(TreePatternNodePtr N, + std::vector<TreePatternNodePtr> &OutVariants, + CodeGenDAGPatterns &CDP, + const MultipleUseVarSet &DepVars) { + // We cannot permute leaves or ComplexPattern uses. + if (N->isLeaf() || N->getOperator()->isSubClassOf("ComplexPattern")) { + OutVariants.push_back(N); + return; + } + + // Look up interesting info about the node. + const SDNodeInfo &NodeInfo = CDP.getSDNodeInfo(N->getOperator()); + + // If this node is associative, re-associate. + if (NodeInfo.hasProperty(SDNPAssociative)) { + // Re-associate by pulling together all of the linked operators + std::vector<TreePatternNodePtr> MaximalChildren; + GatherChildrenOfAssociativeOpcode(N, MaximalChildren); + + // Only handle child sizes of 3. Otherwise we'll end up trying too many + // permutations. + if (MaximalChildren.size() == 3) { + // Find the variants of all of our maximal children. + std::vector<TreePatternNodePtr> AVariants, BVariants, CVariants; + GenerateVariantsOf(MaximalChildren[0], AVariants, CDP, DepVars); + GenerateVariantsOf(MaximalChildren[1], BVariants, CDP, DepVars); + GenerateVariantsOf(MaximalChildren[2], CVariants, CDP, DepVars); + + // There are only two ways we can permute the tree: + // (A op B) op C and A op (B op C) + // Within these forms, we can also permute A/B/C. + + // Generate legal pair permutations of A/B/C. + std::vector<TreePatternNodePtr> ABVariants; + std::vector<TreePatternNodePtr> BAVariants; + std::vector<TreePatternNodePtr> ACVariants; + std::vector<TreePatternNodePtr> CAVariants; + std::vector<TreePatternNodePtr> BCVariants; + std::vector<TreePatternNodePtr> CBVariants; + CombineChildVariants(N, AVariants, BVariants, ABVariants, CDP, DepVars); + CombineChildVariants(N, BVariants, AVariants, BAVariants, CDP, DepVars); + CombineChildVariants(N, AVariants, CVariants, ACVariants, CDP, DepVars); + CombineChildVariants(N, CVariants, AVariants, CAVariants, CDP, DepVars); + CombineChildVariants(N, BVariants, CVariants, BCVariants, CDP, DepVars); + CombineChildVariants(N, CVariants, BVariants, CBVariants, CDP, DepVars); + + // Combine those into the result: (x op x) op x + CombineChildVariants(N, ABVariants, CVariants, OutVariants, CDP, DepVars); + CombineChildVariants(N, BAVariants, CVariants, OutVariants, CDP, DepVars); + CombineChildVariants(N, ACVariants, BVariants, OutVariants, CDP, DepVars); + CombineChildVariants(N, CAVariants, BVariants, OutVariants, CDP, DepVars); + CombineChildVariants(N, BCVariants, AVariants, OutVariants, CDP, DepVars); + CombineChildVariants(N, CBVariants, AVariants, OutVariants, CDP, DepVars); + + // Combine those into the result: x op (x op x) + CombineChildVariants(N, CVariants, ABVariants, OutVariants, CDP, DepVars); + CombineChildVariants(N, CVariants, BAVariants, OutVariants, CDP, DepVars); + CombineChildVariants(N, BVariants, ACVariants, OutVariants, CDP, DepVars); + CombineChildVariants(N, BVariants, CAVariants, OutVariants, CDP, DepVars); + CombineChildVariants(N, AVariants, BCVariants, OutVariants, CDP, DepVars); + CombineChildVariants(N, AVariants, CBVariants, OutVariants, CDP, DepVars); + return; + } + } + + // Compute permutations of all children. + std::vector<std::vector<TreePatternNodePtr>> ChildVariants; + ChildVariants.resize(N->getNumChildren()); + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) + GenerateVariantsOf(N->getChildShared(i), ChildVariants[i], CDP, DepVars); + + // Build all permutations based on how the children were formed. + CombineChildVariants(N, ChildVariants, OutVariants, CDP, DepVars); + + // If this node is commutative, consider the commuted order. + bool isCommIntrinsic = N->isCommutativeIntrinsic(CDP); + if (NodeInfo.hasProperty(SDNPCommutative) || isCommIntrinsic) { + assert((N->getNumChildren()>=2 || isCommIntrinsic) && + "Commutative but doesn't have 2 children!"); + // Don't count children which are actually register references. + unsigned NC = 0; + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) { + TreePatternNode *Child = N->getChild(i); + if (Child->isLeaf()) + if (DefInit *DI = dyn_cast<DefInit>(Child->getLeafValue())) { + Record *RR = DI->getDef(); + if (RR->isSubClassOf("Register")) + continue; + } + NC++; + } + // Consider the commuted order. + if (isCommIntrinsic) { + // Commutative intrinsic. First operand is the intrinsic id, 2nd and 3rd + // operands are the commutative operands, and there might be more operands + // after those. + assert(NC >= 3 && + "Commutative intrinsic should have at least 3 children!"); + std::vector<std::vector<TreePatternNodePtr>> Variants; + Variants.push_back(std::move(ChildVariants[0])); // Intrinsic id. + Variants.push_back(std::move(ChildVariants[2])); + Variants.push_back(std::move(ChildVariants[1])); + for (unsigned i = 3; i != NC; ++i) + Variants.push_back(std::move(ChildVariants[i])); + CombineChildVariants(N, Variants, OutVariants, CDP, DepVars); + } else if (NC == N->getNumChildren()) { + std::vector<std::vector<TreePatternNodePtr>> Variants; + Variants.push_back(std::move(ChildVariants[1])); + Variants.push_back(std::move(ChildVariants[0])); + for (unsigned i = 2; i != NC; ++i) + Variants.push_back(std::move(ChildVariants[i])); + CombineChildVariants(N, Variants, OutVariants, CDP, DepVars); + } + } +} + + +// GenerateVariants - Generate variants. For example, commutative patterns can +// match multiple ways. Add them to PatternsToMatch as well. +void CodeGenDAGPatterns::GenerateVariants() { + LLVM_DEBUG(errs() << "Generating instruction variants.\n"); + + // Loop over all of the patterns we've collected, checking to see if we can + // generate variants of the instruction, through the exploitation of + // identities. This permits the target to provide aggressive matching without + // the .td file having to contain tons of variants of instructions. + // + // Note that this loop adds new patterns to the PatternsToMatch list, but we + // intentionally do not reconsider these. Any variants of added patterns have + // already been added. + // + const unsigned NumOriginalPatterns = PatternsToMatch.size(); + BitVector MatchedPatterns(NumOriginalPatterns); + std::vector<BitVector> MatchedPredicates(NumOriginalPatterns, + BitVector(NumOriginalPatterns)); + + typedef std::pair<MultipleUseVarSet, std::vector<TreePatternNodePtr>> + DepsAndVariants; + std::map<unsigned, DepsAndVariants> PatternsWithVariants; + + // Collect patterns with more than one variant. + for (unsigned i = 0; i != NumOriginalPatterns; ++i) { + MultipleUseVarSet DepVars; + std::vector<TreePatternNodePtr> Variants; + FindDepVars(PatternsToMatch[i].getSrcPattern(), DepVars); + LLVM_DEBUG(errs() << "Dependent/multiply used variables: "); + LLVM_DEBUG(DumpDepVars(DepVars)); + LLVM_DEBUG(errs() << "\n"); + GenerateVariantsOf(PatternsToMatch[i].getSrcPatternShared(), Variants, + *this, DepVars); + + assert(!Variants.empty() && "Must create at least original variant!"); + if (Variants.size() == 1) // No additional variants for this pattern. + continue; + + LLVM_DEBUG(errs() << "FOUND VARIANTS OF: "; + PatternsToMatch[i].getSrcPattern()->dump(); errs() << "\n"); + + PatternsWithVariants[i] = std::make_pair(DepVars, Variants); + + // Cache matching predicates. + if (MatchedPatterns[i]) + continue; + + const std::vector<Predicate> &Predicates = + PatternsToMatch[i].getPredicates(); + + BitVector &Matches = MatchedPredicates[i]; + MatchedPatterns.set(i); + Matches.set(i); + + // Don't test patterns that have already been cached - it won't match. + for (unsigned p = 0; p != NumOriginalPatterns; ++p) + if (!MatchedPatterns[p]) + Matches[p] = (Predicates == PatternsToMatch[p].getPredicates()); + + // Copy this to all the matching patterns. + for (int p = Matches.find_first(); p != -1; p = Matches.find_next(p)) + if (p != (int)i) { + MatchedPatterns.set(p); + MatchedPredicates[p] = Matches; + } + } + + for (auto it : PatternsWithVariants) { + unsigned i = it.first; + const MultipleUseVarSet &DepVars = it.second.first; + const std::vector<TreePatternNodePtr> &Variants = it.second.second; + + for (unsigned v = 0, e = Variants.size(); v != e; ++v) { + TreePatternNodePtr Variant = Variants[v]; + BitVector &Matches = MatchedPredicates[i]; + + LLVM_DEBUG(errs() << " VAR#" << v << ": "; Variant->dump(); + errs() << "\n"); + + // Scan to see if an instruction or explicit pattern already matches this. + bool AlreadyExists = false; + for (unsigned p = 0, e = PatternsToMatch.size(); p != e; ++p) { + // Skip if the top level predicates do not match. + if (!Matches[p]) + continue; + // Check to see if this variant already exists. + if (Variant->isIsomorphicTo(PatternsToMatch[p].getSrcPattern(), + DepVars)) { + LLVM_DEBUG(errs() << " *** ALREADY EXISTS, ignoring variant.\n"); + AlreadyExists = true; + break; + } + } + // If we already have it, ignore the variant. + if (AlreadyExists) continue; + + // Otherwise, add it to the list of patterns we have. + PatternsToMatch.push_back(PatternToMatch( + PatternsToMatch[i].getSrcRecord(), PatternsToMatch[i].getPredicates(), + Variant, PatternsToMatch[i].getDstPatternShared(), + PatternsToMatch[i].getDstRegs(), + PatternsToMatch[i].getAddedComplexity(), Record::getNewUID())); + MatchedPredicates.push_back(Matches); + + // Add a new match the same as this pattern. + for (auto &P : MatchedPredicates) + P.push_back(P[i]); + } + + LLVM_DEBUG(errs() << "\n"); + } +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.h new file mode 100644 index 000000000000..a3b84d76fde9 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenDAGPatterns.h @@ -0,0 +1,1321 @@ +//===- CodeGenDAGPatterns.h - Read DAG patterns from .td file ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the CodeGenDAGPatterns class, which is used to read and +// represent the patterns present in a .td file for instructions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H +#define LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H + +#include "CodeGenHwModes.h" +#include "CodeGenIntrinsics.h" +#include "CodeGenTarget.h" +#include "SDNodeProperties.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" +#include <algorithm> +#include <array> +#include <functional> +#include <map> +#include <numeric> +#include <set> +#include <vector> + +namespace llvm { + +class Record; +class Init; +class ListInit; +class DagInit; +class SDNodeInfo; +class TreePattern; +class TreePatternNode; +class CodeGenDAGPatterns; +class ComplexPattern; + +/// Shared pointer for TreePatternNode. +using TreePatternNodePtr = std::shared_ptr<TreePatternNode>; + +/// This represents a set of MVTs. Since the underlying type for the MVT +/// is uint8_t, there are at most 256 values. To reduce the number of memory +/// allocations and deallocations, represent the set as a sequence of bits. +/// To reduce the allocations even further, make MachineValueTypeSet own +/// the storage and use std::array as the bit container. +struct MachineValueTypeSet { + static_assert(std::is_same<std::underlying_type<MVT::SimpleValueType>::type, + uint8_t>::value, + "Change uint8_t here to the SimpleValueType's type"); + static unsigned constexpr Capacity = std::numeric_limits<uint8_t>::max()+1; + using WordType = uint64_t; + static unsigned constexpr WordWidth = CHAR_BIT*sizeof(WordType); + static unsigned constexpr NumWords = Capacity/WordWidth; + static_assert(NumWords*WordWidth == Capacity, + "Capacity should be a multiple of WordWidth"); + + LLVM_ATTRIBUTE_ALWAYS_INLINE + MachineValueTypeSet() { + clear(); + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + unsigned size() const { + unsigned Count = 0; + for (WordType W : Words) + Count += countPopulation(W); + return Count; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE + void clear() { + std::memset(Words.data(), 0, NumWords*sizeof(WordType)); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool empty() const { + for (WordType W : Words) + if (W != 0) + return false; + return true; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE + unsigned count(MVT T) const { + return (Words[T.SimpleTy / WordWidth] >> (T.SimpleTy % WordWidth)) & 1; + } + std::pair<MachineValueTypeSet&,bool> insert(MVT T) { + bool V = count(T.SimpleTy); + Words[T.SimpleTy / WordWidth] |= WordType(1) << (T.SimpleTy % WordWidth); + return {*this, V}; + } + MachineValueTypeSet &insert(const MachineValueTypeSet &S) { + for (unsigned i = 0; i != NumWords; ++i) + Words[i] |= S.Words[i]; + return *this; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE + void erase(MVT T) { + Words[T.SimpleTy / WordWidth] &= ~(WordType(1) << (T.SimpleTy % WordWidth)); + } + + struct const_iterator { + // Some implementations of the C++ library require these traits to be + // defined. + using iterator_category = std::forward_iterator_tag; + using value_type = MVT; + using difference_type = ptrdiff_t; + using pointer = const MVT*; + using reference = const MVT&; + + LLVM_ATTRIBUTE_ALWAYS_INLINE + MVT operator*() const { + assert(Pos != Capacity); + return MVT::SimpleValueType(Pos); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE + const_iterator(const MachineValueTypeSet *S, bool End) : Set(S) { + Pos = End ? Capacity : find_from_pos(0); + } + LLVM_ATTRIBUTE_ALWAYS_INLINE + const_iterator &operator++() { + assert(Pos != Capacity); + Pos = find_from_pos(Pos+1); + return *this; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool operator==(const const_iterator &It) const { + return Set == It.Set && Pos == It.Pos; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool operator!=(const const_iterator &It) const { + return !operator==(It); + } + + private: + unsigned find_from_pos(unsigned P) const { + unsigned SkipWords = P / WordWidth; + unsigned SkipBits = P % WordWidth; + unsigned Count = SkipWords * WordWidth; + + // If P is in the middle of a word, process it manually here, because + // the trailing bits need to be masked off to use findFirstSet. + if (SkipBits != 0) { + WordType W = Set->Words[SkipWords]; + W &= maskLeadingOnes<WordType>(WordWidth-SkipBits); + if (W != 0) + return Count + findFirstSet(W); + Count += WordWidth; + SkipWords++; + } + + for (unsigned i = SkipWords; i != NumWords; ++i) { + WordType W = Set->Words[i]; + if (W != 0) + return Count + findFirstSet(W); + Count += WordWidth; + } + return Capacity; + } + + const MachineValueTypeSet *Set; + unsigned Pos; + }; + + LLVM_ATTRIBUTE_ALWAYS_INLINE + const_iterator begin() const { return const_iterator(this, false); } + LLVM_ATTRIBUTE_ALWAYS_INLINE + const_iterator end() const { return const_iterator(this, true); } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool operator==(const MachineValueTypeSet &S) const { + return Words == S.Words; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool operator!=(const MachineValueTypeSet &S) const { + return !operator==(S); + } + +private: + friend struct const_iterator; + std::array<WordType,NumWords> Words; +}; + +struct TypeSetByHwMode : public InfoByHwMode<MachineValueTypeSet> { + using SetType = MachineValueTypeSet; + std::vector<unsigned> AddrSpaces; + + TypeSetByHwMode() = default; + TypeSetByHwMode(const TypeSetByHwMode &VTS) = default; + TypeSetByHwMode &operator=(const TypeSetByHwMode &) = default; + TypeSetByHwMode(MVT::SimpleValueType VT) + : TypeSetByHwMode(ValueTypeByHwMode(VT)) {} + TypeSetByHwMode(ValueTypeByHwMode VT) + : TypeSetByHwMode(ArrayRef<ValueTypeByHwMode>(&VT, 1)) {} + TypeSetByHwMode(ArrayRef<ValueTypeByHwMode> VTList); + + SetType &getOrCreate(unsigned Mode) { + if (hasMode(Mode)) + return get(Mode); + return Map.insert({Mode,SetType()}).first->second; + } + + bool isValueTypeByHwMode(bool AllowEmpty) const; + ValueTypeByHwMode getValueTypeByHwMode() const; + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool isMachineValueType() const { + return isDefaultOnly() && Map.begin()->second.size() == 1; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + MVT getMachineValueType() const { + assert(isMachineValueType()); + return *Map.begin()->second.begin(); + } + + bool isPossible() const; + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool isDefaultOnly() const { + return Map.size() == 1 && Map.begin()->first == DefaultMode; + } + + bool isPointer() const { + return getValueTypeByHwMode().isPointer(); + } + + unsigned getPtrAddrSpace() const { + assert(isPointer()); + return getValueTypeByHwMode().PtrAddrSpace; + } + + bool insert(const ValueTypeByHwMode &VVT); + bool constrain(const TypeSetByHwMode &VTS); + template <typename Predicate> bool constrain(Predicate P); + template <typename Predicate> + bool assign_if(const TypeSetByHwMode &VTS, Predicate P); + + void writeToStream(raw_ostream &OS) const; + static void writeToStream(const SetType &S, raw_ostream &OS); + + bool operator==(const TypeSetByHwMode &VTS) const; + bool operator!=(const TypeSetByHwMode &VTS) const { return !(*this == VTS); } + + void dump() const; + bool validate() const; + +private: + unsigned PtrAddrSpace = std::numeric_limits<unsigned>::max(); + /// Intersect two sets. Return true if anything has changed. + bool intersect(SetType &Out, const SetType &In); +}; + +raw_ostream &operator<<(raw_ostream &OS, const TypeSetByHwMode &T); + +struct TypeInfer { + TypeInfer(TreePattern &T) : TP(T), ForceMode(0) {} + + bool isConcrete(const TypeSetByHwMode &VTS, bool AllowEmpty) const { + return VTS.isValueTypeByHwMode(AllowEmpty); + } + ValueTypeByHwMode getConcrete(const TypeSetByHwMode &VTS, + bool AllowEmpty) const { + assert(VTS.isValueTypeByHwMode(AllowEmpty)); + return VTS.getValueTypeByHwMode(); + } + + /// The protocol in the following functions (Merge*, force*, Enforce*, + /// expand*) is to return "true" if a change has been made, "false" + /// otherwise. + + bool MergeInTypeInfo(TypeSetByHwMode &Out, const TypeSetByHwMode &In); + bool MergeInTypeInfo(TypeSetByHwMode &Out, MVT::SimpleValueType InVT) { + return MergeInTypeInfo(Out, TypeSetByHwMode(InVT)); + } + bool MergeInTypeInfo(TypeSetByHwMode &Out, ValueTypeByHwMode InVT) { + return MergeInTypeInfo(Out, TypeSetByHwMode(InVT)); + } + + /// Reduce the set \p Out to have at most one element for each mode. + bool forceArbitrary(TypeSetByHwMode &Out); + + /// The following four functions ensure that upon return the set \p Out + /// will only contain types of the specified kind: integer, floating-point, + /// scalar, or vector. + /// If \p Out is empty, all legal types of the specified kind will be added + /// to it. Otherwise, all types that are not of the specified kind will be + /// removed from \p Out. + bool EnforceInteger(TypeSetByHwMode &Out); + bool EnforceFloatingPoint(TypeSetByHwMode &Out); + bool EnforceScalar(TypeSetByHwMode &Out); + bool EnforceVector(TypeSetByHwMode &Out); + + /// If \p Out is empty, fill it with all legal types. Otherwise, leave it + /// unchanged. + bool EnforceAny(TypeSetByHwMode &Out); + /// Make sure that for each type in \p Small, there exists a larger type + /// in \p Big. + bool EnforceSmallerThan(TypeSetByHwMode &Small, TypeSetByHwMode &Big); + /// 1. Ensure that for each type T in \p Vec, T is a vector type, and that + /// for each type U in \p Elem, U is a scalar type. + /// 2. Ensure that for each (scalar) type U in \p Elem, there exists a + /// (vector) type T in \p Vec, such that U is the element type of T. + bool EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, TypeSetByHwMode &Elem); + bool EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, + const ValueTypeByHwMode &VVT); + /// Ensure that for each type T in \p Sub, T is a vector type, and there + /// exists a type U in \p Vec such that U is a vector type with the same + /// element type as T and at least as many elements as T. + bool EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec, + TypeSetByHwMode &Sub); + /// 1. Ensure that \p V has a scalar type iff \p W has a scalar type. + /// 2. Ensure that for each vector type T in \p V, there exists a vector + /// type U in \p W, such that T and U have the same number of elements. + /// 3. Ensure that for each vector type U in \p W, there exists a vector + /// type T in \p V, such that T and U have the same number of elements + /// (reverse of 2). + bool EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W); + /// 1. Ensure that for each type T in \p A, there exists a type U in \p B, + /// such that T and U have equal size in bits. + /// 2. Ensure that for each type U in \p B, there exists a type T in \p A + /// such that T and U have equal size in bits (reverse of 1). + bool EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B); + + /// For each overloaded type (i.e. of form *Any), replace it with the + /// corresponding subset of legal, specific types. + void expandOverloads(TypeSetByHwMode &VTS); + void expandOverloads(TypeSetByHwMode::SetType &Out, + const TypeSetByHwMode::SetType &Legal); + + struct ValidateOnExit { + ValidateOnExit(TypeSetByHwMode &T, TypeInfer &TI) : Infer(TI), VTS(T) {} + #ifndef NDEBUG + ~ValidateOnExit(); + #else + ~ValidateOnExit() {} // Empty destructor with NDEBUG. + #endif + TypeInfer &Infer; + TypeSetByHwMode &VTS; + }; + + struct SuppressValidation { + SuppressValidation(TypeInfer &TI) : Infer(TI), SavedValidate(TI.Validate) { + Infer.Validate = false; + } + ~SuppressValidation() { + Infer.Validate = SavedValidate; + } + TypeInfer &Infer; + bool SavedValidate; + }; + + TreePattern &TP; + unsigned ForceMode; // Mode to use when set. + bool CodeGen = false; // Set during generation of matcher code. + bool Validate = true; // Indicate whether to validate types. + +private: + const TypeSetByHwMode &getLegalTypes(); + + /// Cached legal types (in default mode). + bool LegalTypesCached = false; + TypeSetByHwMode LegalCache; +}; + +/// Set type used to track multiply used variables in patterns +typedef StringSet<> MultipleUseVarSet; + +/// SDTypeConstraint - This is a discriminated union of constraints, +/// corresponding to the SDTypeConstraint tablegen class in Target.td. +struct SDTypeConstraint { + SDTypeConstraint(Record *R, const CodeGenHwModes &CGH); + + unsigned OperandNo; // The operand # this constraint applies to. + enum { + SDTCisVT, SDTCisPtrTy, SDTCisInt, SDTCisFP, SDTCisVec, SDTCisSameAs, + SDTCisVTSmallerThanOp, SDTCisOpSmallerThanOp, SDTCisEltOfVec, + SDTCisSubVecOfVec, SDTCVecEltisVT, SDTCisSameNumEltsAs, SDTCisSameSizeAs + } ConstraintType; + + union { // The discriminated union. + struct { + unsigned OtherOperandNum; + } SDTCisSameAs_Info; + struct { + unsigned OtherOperandNum; + } SDTCisVTSmallerThanOp_Info; + struct { + unsigned BigOperandNum; + } SDTCisOpSmallerThanOp_Info; + struct { + unsigned OtherOperandNum; + } SDTCisEltOfVec_Info; + struct { + unsigned OtherOperandNum; + } SDTCisSubVecOfVec_Info; + struct { + unsigned OtherOperandNum; + } SDTCisSameNumEltsAs_Info; + struct { + unsigned OtherOperandNum; + } SDTCisSameSizeAs_Info; + } x; + + // The VT for SDTCisVT and SDTCVecEltisVT. + // Must not be in the union because it has a non-trivial destructor. + ValueTypeByHwMode VVT; + + /// ApplyTypeConstraint - Given a node in a pattern, apply this type + /// constraint to the nodes operands. This returns true if it makes a + /// change, false otherwise. If a type contradiction is found, an error + /// is flagged. + bool ApplyTypeConstraint(TreePatternNode *N, const SDNodeInfo &NodeInfo, + TreePattern &TP) const; +}; + +/// ScopedName - A name of a node associated with a "scope" that indicates +/// the context (e.g. instance of Pattern or PatFrag) in which the name was +/// used. This enables substitution of pattern fragments while keeping track +/// of what name(s) were originally given to various nodes in the tree. +class ScopedName { + unsigned Scope; + std::string Identifier; +public: + ScopedName(unsigned Scope, StringRef Identifier) + : Scope(Scope), Identifier(std::string(Identifier)) { + assert(Scope != 0 && + "Scope == 0 is used to indicate predicates without arguments"); + } + + unsigned getScope() const { return Scope; } + const std::string &getIdentifier() const { return Identifier; } + + std::string getFullName() const; + + bool operator==(const ScopedName &o) const; + bool operator!=(const ScopedName &o) const; +}; + +/// SDNodeInfo - One of these records is created for each SDNode instance in +/// the target .td file. This represents the various dag nodes we will be +/// processing. +class SDNodeInfo { + Record *Def; + StringRef EnumName; + StringRef SDClassName; + unsigned Properties; + unsigned NumResults; + int NumOperands; + std::vector<SDTypeConstraint> TypeConstraints; +public: + // Parse the specified record. + SDNodeInfo(Record *R, const CodeGenHwModes &CGH); + + unsigned getNumResults() const { return NumResults; } + + /// getNumOperands - This is the number of operands required or -1 if + /// variadic. + int getNumOperands() const { return NumOperands; } + Record *getRecord() const { return Def; } + StringRef getEnumName() const { return EnumName; } + StringRef getSDClassName() const { return SDClassName; } + + const std::vector<SDTypeConstraint> &getTypeConstraints() const { + return TypeConstraints; + } + + /// getKnownType - If the type constraints on this node imply a fixed type + /// (e.g. all stores return void, etc), then return it as an + /// MVT::SimpleValueType. Otherwise, return MVT::Other. + MVT::SimpleValueType getKnownType(unsigned ResNo) const; + + /// hasProperty - Return true if this node has the specified property. + /// + bool hasProperty(enum SDNP Prop) const { return Properties & (1 << Prop); } + + /// ApplyTypeConstraints - Given a node in a pattern, apply the type + /// constraints for this node to the operands of the node. This returns + /// true if it makes a change, false otherwise. If a type contradiction is + /// found, an error is flagged. + bool ApplyTypeConstraints(TreePatternNode *N, TreePattern &TP) const; +}; + +/// TreePredicateFn - This is an abstraction that represents the predicates on +/// a PatFrag node. This is a simple one-word wrapper around a pointer to +/// provide nice accessors. +class TreePredicateFn { + /// PatFragRec - This is the TreePattern for the PatFrag that we + /// originally came from. + TreePattern *PatFragRec; +public: + /// TreePredicateFn constructor. Here 'N' is a subclass of PatFrag. + TreePredicateFn(TreePattern *N); + + + TreePattern *getOrigPatFragRecord() const { return PatFragRec; } + + /// isAlwaysTrue - Return true if this is a noop predicate. + bool isAlwaysTrue() const; + + bool isImmediatePattern() const { return hasImmCode(); } + + /// getImmediatePredicateCode - Return the code that evaluates this pattern if + /// this is an immediate predicate. It is an error to call this on a + /// non-immediate pattern. + std::string getImmediatePredicateCode() const { + std::string Result = getImmCode(); + assert(!Result.empty() && "Isn't an immediate pattern!"); + return Result; + } + + bool operator==(const TreePredicateFn &RHS) const { + return PatFragRec == RHS.PatFragRec; + } + + bool operator!=(const TreePredicateFn &RHS) const { return !(*this == RHS); } + + /// Return the name to use in the generated code to reference this, this is + /// "Predicate_foo" if from a pattern fragment "foo". + std::string getFnName() const; + + /// getCodeToRunOnSDNode - Return the code for the function body that + /// evaluates this predicate. The argument is expected to be in "Node", + /// not N. This handles casting and conversion to a concrete node type as + /// appropriate. + std::string getCodeToRunOnSDNode() const; + + /// Get the data type of the argument to getImmediatePredicateCode(). + StringRef getImmType() const; + + /// Get a string that describes the type returned by getImmType() but is + /// usable as part of an identifier. + StringRef getImmTypeIdentifier() const; + + // Predicate code uses the PatFrag's captured operands. + bool usesOperands() const; + + // Is the desired predefined predicate for a load? + bool isLoad() const; + // Is the desired predefined predicate for a store? + bool isStore() const; + // Is the desired predefined predicate for an atomic? + bool isAtomic() const; + + /// Is this predicate the predefined unindexed load predicate? + /// Is this predicate the predefined unindexed store predicate? + bool isUnindexed() const; + /// Is this predicate the predefined non-extending load predicate? + bool isNonExtLoad() const; + /// Is this predicate the predefined any-extend load predicate? + bool isAnyExtLoad() const; + /// Is this predicate the predefined sign-extend load predicate? + bool isSignExtLoad() const; + /// Is this predicate the predefined zero-extend load predicate? + bool isZeroExtLoad() const; + /// Is this predicate the predefined non-truncating store predicate? + bool isNonTruncStore() const; + /// Is this predicate the predefined truncating store predicate? + bool isTruncStore() const; + + /// Is this predicate the predefined monotonic atomic predicate? + bool isAtomicOrderingMonotonic() const; + /// Is this predicate the predefined acquire atomic predicate? + bool isAtomicOrderingAcquire() const; + /// Is this predicate the predefined release atomic predicate? + bool isAtomicOrderingRelease() const; + /// Is this predicate the predefined acquire-release atomic predicate? + bool isAtomicOrderingAcquireRelease() const; + /// Is this predicate the predefined sequentially consistent atomic predicate? + bool isAtomicOrderingSequentiallyConsistent() const; + + /// Is this predicate the predefined acquire-or-stronger atomic predicate? + bool isAtomicOrderingAcquireOrStronger() const; + /// Is this predicate the predefined weaker-than-acquire atomic predicate? + bool isAtomicOrderingWeakerThanAcquire() const; + + /// Is this predicate the predefined release-or-stronger atomic predicate? + bool isAtomicOrderingReleaseOrStronger() const; + /// Is this predicate the predefined weaker-than-release atomic predicate? + bool isAtomicOrderingWeakerThanRelease() const; + + /// If non-null, indicates that this predicate is a predefined memory VT + /// predicate for a load/store and returns the ValueType record for the memory VT. + Record *getMemoryVT() const; + /// If non-null, indicates that this predicate is a predefined memory VT + /// predicate (checking only the scalar type) for load/store and returns the + /// ValueType record for the memory VT. + Record *getScalarMemoryVT() const; + + ListInit *getAddressSpaces() const; + int64_t getMinAlignment() const; + + // If true, indicates that GlobalISel-based C++ code was supplied. + bool hasGISelPredicateCode() const; + std::string getGISelPredicateCode() const; + +private: + bool hasPredCode() const; + bool hasImmCode() const; + std::string getPredCode() const; + std::string getImmCode() const; + bool immCodeUsesAPInt() const; + bool immCodeUsesAPFloat() const; + + bool isPredefinedPredicateEqualTo(StringRef Field, bool Value) const; +}; + +struct TreePredicateCall { + TreePredicateFn Fn; + + // Scope -- unique identifier for retrieving named arguments. 0 is used when + // the predicate does not use named arguments. + unsigned Scope; + + TreePredicateCall(const TreePredicateFn &Fn, unsigned Scope) + : Fn(Fn), Scope(Scope) {} + + bool operator==(const TreePredicateCall &o) const { + return Fn == o.Fn && Scope == o.Scope; + } + bool operator!=(const TreePredicateCall &o) const { + return !(*this == o); + } +}; + +class TreePatternNode { + /// The type of each node result. Before and during type inference, each + /// result may be a set of possible types. After (successful) type inference, + /// each is a single concrete type. + std::vector<TypeSetByHwMode> Types; + + /// The index of each result in results of the pattern. + std::vector<unsigned> ResultPerm; + + /// Operator - The Record for the operator if this is an interior node (not + /// a leaf). + Record *Operator; + + /// Val - The init value (e.g. the "GPRC" record, or "7") for a leaf. + /// + Init *Val; + + /// Name - The name given to this node with the :$foo notation. + /// + std::string Name; + + std::vector<ScopedName> NamesAsPredicateArg; + + /// PredicateCalls - The predicate functions to execute on this node to check + /// for a match. If this list is empty, no predicate is involved. + std::vector<TreePredicateCall> PredicateCalls; + + /// TransformFn - The transformation function to execute on this node before + /// it can be substituted into the resulting instruction on a pattern match. + Record *TransformFn; + + std::vector<TreePatternNodePtr> Children; + +public: + TreePatternNode(Record *Op, std::vector<TreePatternNodePtr> Ch, + unsigned NumResults) + : Operator(Op), Val(nullptr), TransformFn(nullptr), + Children(std::move(Ch)) { + Types.resize(NumResults); + ResultPerm.resize(NumResults); + std::iota(ResultPerm.begin(), ResultPerm.end(), 0); + } + TreePatternNode(Init *val, unsigned NumResults) // leaf ctor + : Operator(nullptr), Val(val), TransformFn(nullptr) { + Types.resize(NumResults); + ResultPerm.resize(NumResults); + std::iota(ResultPerm.begin(), ResultPerm.end(), 0); + } + + bool hasName() const { return !Name.empty(); } + const std::string &getName() const { return Name; } + void setName(StringRef N) { Name.assign(N.begin(), N.end()); } + + const std::vector<ScopedName> &getNamesAsPredicateArg() const { + return NamesAsPredicateArg; + } + void setNamesAsPredicateArg(const std::vector<ScopedName>& Names) { + NamesAsPredicateArg = Names; + } + void addNameAsPredicateArg(const ScopedName &N) { + NamesAsPredicateArg.push_back(N); + } + + bool isLeaf() const { return Val != nullptr; } + + // Type accessors. + unsigned getNumTypes() const { return Types.size(); } + ValueTypeByHwMode getType(unsigned ResNo) const { + return Types[ResNo].getValueTypeByHwMode(); + } + const std::vector<TypeSetByHwMode> &getExtTypes() const { return Types; } + const TypeSetByHwMode &getExtType(unsigned ResNo) const { + return Types[ResNo]; + } + TypeSetByHwMode &getExtType(unsigned ResNo) { return Types[ResNo]; } + void setType(unsigned ResNo, const TypeSetByHwMode &T) { Types[ResNo] = T; } + MVT::SimpleValueType getSimpleType(unsigned ResNo) const { + return Types[ResNo].getMachineValueType().SimpleTy; + } + + bool hasConcreteType(unsigned ResNo) const { + return Types[ResNo].isValueTypeByHwMode(false); + } + bool isTypeCompletelyUnknown(unsigned ResNo, TreePattern &TP) const { + return Types[ResNo].empty(); + } + + unsigned getNumResults() const { return ResultPerm.size(); } + unsigned getResultIndex(unsigned ResNo) const { return ResultPerm[ResNo]; } + void setResultIndex(unsigned ResNo, unsigned RI) { ResultPerm[ResNo] = RI; } + + Init *getLeafValue() const { assert(isLeaf()); return Val; } + Record *getOperator() const { assert(!isLeaf()); return Operator; } + + unsigned getNumChildren() const { return Children.size(); } + TreePatternNode *getChild(unsigned N) const { return Children[N].get(); } + const TreePatternNodePtr &getChildShared(unsigned N) const { + return Children[N]; + } + void setChild(unsigned i, TreePatternNodePtr N) { Children[i] = N; } + + /// hasChild - Return true if N is any of our children. + bool hasChild(const TreePatternNode *N) const { + for (unsigned i = 0, e = Children.size(); i != e; ++i) + if (Children[i].get() == N) + return true; + return false; + } + + bool hasProperTypeByHwMode() const; + bool hasPossibleType() const; + bool setDefaultMode(unsigned Mode); + + bool hasAnyPredicate() const { return !PredicateCalls.empty(); } + + const std::vector<TreePredicateCall> &getPredicateCalls() const { + return PredicateCalls; + } + void clearPredicateCalls() { PredicateCalls.clear(); } + void setPredicateCalls(const std::vector<TreePredicateCall> &Calls) { + assert(PredicateCalls.empty() && "Overwriting non-empty predicate list!"); + PredicateCalls = Calls; + } + void addPredicateCall(const TreePredicateCall &Call) { + assert(!Call.Fn.isAlwaysTrue() && "Empty predicate string!"); + assert(!is_contained(PredicateCalls, Call) && "predicate applied recursively"); + PredicateCalls.push_back(Call); + } + void addPredicateCall(const TreePredicateFn &Fn, unsigned Scope) { + assert((Scope != 0) == Fn.usesOperands()); + addPredicateCall(TreePredicateCall(Fn, Scope)); + } + + Record *getTransformFn() const { return TransformFn; } + void setTransformFn(Record *Fn) { TransformFn = Fn; } + + /// getIntrinsicInfo - If this node corresponds to an intrinsic, return the + /// CodeGenIntrinsic information for it, otherwise return a null pointer. + const CodeGenIntrinsic *getIntrinsicInfo(const CodeGenDAGPatterns &CDP) const; + + /// getComplexPatternInfo - If this node corresponds to a ComplexPattern, + /// return the ComplexPattern information, otherwise return null. + const ComplexPattern * + getComplexPatternInfo(const CodeGenDAGPatterns &CGP) const; + + /// Returns the number of MachineInstr operands that would be produced by this + /// node if it mapped directly to an output Instruction's + /// operand. ComplexPattern specifies this explicitly; MIOperandInfo gives it + /// for Operands; otherwise 1. + unsigned getNumMIResults(const CodeGenDAGPatterns &CGP) const; + + /// NodeHasProperty - Return true if this node has the specified property. + bool NodeHasProperty(SDNP Property, const CodeGenDAGPatterns &CGP) const; + + /// TreeHasProperty - Return true if any node in this tree has the specified + /// property. + bool TreeHasProperty(SDNP Property, const CodeGenDAGPatterns &CGP) const; + + /// isCommutativeIntrinsic - Return true if the node is an intrinsic which is + /// marked isCommutative. + bool isCommutativeIntrinsic(const CodeGenDAGPatterns &CDP) const; + + void print(raw_ostream &OS) const; + void dump() const; + +public: // Higher level manipulation routines. + + /// clone - Return a new copy of this tree. + /// + TreePatternNodePtr clone() const; + + /// RemoveAllTypes - Recursively strip all the types of this tree. + void RemoveAllTypes(); + + /// isIsomorphicTo - Return true if this node is recursively isomorphic to + /// the specified node. For this comparison, all of the state of the node + /// is considered, except for the assigned name. Nodes with differing names + /// that are otherwise identical are considered isomorphic. + bool isIsomorphicTo(const TreePatternNode *N, + const MultipleUseVarSet &DepVars) const; + + /// SubstituteFormalArguments - Replace the formal arguments in this tree + /// with actual values specified by ArgMap. + void + SubstituteFormalArguments(std::map<std::string, TreePatternNodePtr> &ArgMap); + + /// InlinePatternFragments - If this pattern refers to any pattern + /// fragments, return the set of inlined versions (this can be more than + /// one if a PatFrags record has multiple alternatives). + void InlinePatternFragments(TreePatternNodePtr T, + TreePattern &TP, + std::vector<TreePatternNodePtr> &OutAlternatives); + + /// ApplyTypeConstraints - Apply all of the type constraints relevant to + /// this node and its children in the tree. This returns true if it makes a + /// change, false otherwise. If a type contradiction is found, flag an error. + bool ApplyTypeConstraints(TreePattern &TP, bool NotRegisters); + + /// UpdateNodeType - Set the node type of N to VT if VT contains + /// information. If N already contains a conflicting type, then flag an + /// error. This returns true if any information was updated. + /// + bool UpdateNodeType(unsigned ResNo, const TypeSetByHwMode &InTy, + TreePattern &TP); + bool UpdateNodeType(unsigned ResNo, MVT::SimpleValueType InTy, + TreePattern &TP); + bool UpdateNodeType(unsigned ResNo, ValueTypeByHwMode InTy, + TreePattern &TP); + + // Update node type with types inferred from an instruction operand or result + // def from the ins/outs lists. + // Return true if the type changed. + bool UpdateNodeTypeFromInst(unsigned ResNo, Record *Operand, TreePattern &TP); + + /// ContainsUnresolvedType - Return true if this tree contains any + /// unresolved types. + bool ContainsUnresolvedType(TreePattern &TP) const; + + /// canPatternMatch - If it is impossible for this pattern to match on this + /// target, fill in Reason and return false. Otherwise, return true. + bool canPatternMatch(std::string &Reason, const CodeGenDAGPatterns &CDP); +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const TreePatternNode &TPN) { + TPN.print(OS); + return OS; +} + + +/// TreePattern - Represent a pattern, used for instructions, pattern +/// fragments, etc. +/// +class TreePattern { + /// Trees - The list of pattern trees which corresponds to this pattern. + /// Note that PatFrag's only have a single tree. + /// + std::vector<TreePatternNodePtr> Trees; + + /// NamedNodes - This is all of the nodes that have names in the trees in this + /// pattern. + StringMap<SmallVector<TreePatternNode *, 1>> NamedNodes; + + /// TheRecord - The actual TableGen record corresponding to this pattern. + /// + Record *TheRecord; + + /// Args - This is a list of all of the arguments to this pattern (for + /// PatFrag patterns), which are the 'node' markers in this pattern. + std::vector<std::string> Args; + + /// CDP - the top-level object coordinating this madness. + /// + CodeGenDAGPatterns &CDP; + + /// isInputPattern - True if this is an input pattern, something to match. + /// False if this is an output pattern, something to emit. + bool isInputPattern; + + /// hasError - True if the currently processed nodes have unresolvable types + /// or other non-fatal errors + bool HasError; + + /// It's important that the usage of operands in ComplexPatterns is + /// consistent: each named operand can be defined by at most one + /// ComplexPattern. This records the ComplexPattern instance and the operand + /// number for each operand encountered in a ComplexPattern to aid in that + /// check. + StringMap<std::pair<Record *, unsigned>> ComplexPatternOperands; + + TypeInfer Infer; + +public: + + /// TreePattern constructor - Parse the specified DagInits into the + /// current record. + TreePattern(Record *TheRec, ListInit *RawPat, bool isInput, + CodeGenDAGPatterns &ise); + TreePattern(Record *TheRec, DagInit *Pat, bool isInput, + CodeGenDAGPatterns &ise); + TreePattern(Record *TheRec, TreePatternNodePtr Pat, bool isInput, + CodeGenDAGPatterns &ise); + + /// getTrees - Return the tree patterns which corresponds to this pattern. + /// + const std::vector<TreePatternNodePtr> &getTrees() const { return Trees; } + unsigned getNumTrees() const { return Trees.size(); } + const TreePatternNodePtr &getTree(unsigned i) const { return Trees[i]; } + void setTree(unsigned i, TreePatternNodePtr Tree) { Trees[i] = Tree; } + const TreePatternNodePtr &getOnlyTree() const { + assert(Trees.size() == 1 && "Doesn't have exactly one pattern!"); + return Trees[0]; + } + + const StringMap<SmallVector<TreePatternNode *, 1>> &getNamedNodesMap() { + if (NamedNodes.empty()) + ComputeNamedNodes(); + return NamedNodes; + } + + /// getRecord - Return the actual TableGen record corresponding to this + /// pattern. + /// + Record *getRecord() const { return TheRecord; } + + unsigned getNumArgs() const { return Args.size(); } + const std::string &getArgName(unsigned i) const { + assert(i < Args.size() && "Argument reference out of range!"); + return Args[i]; + } + std::vector<std::string> &getArgList() { return Args; } + + CodeGenDAGPatterns &getDAGPatterns() const { return CDP; } + + /// InlinePatternFragments - If this pattern refers to any pattern + /// fragments, inline them into place, giving us a pattern without any + /// PatFrags references. This may increase the number of trees in the + /// pattern if a PatFrags has multiple alternatives. + void InlinePatternFragments() { + std::vector<TreePatternNodePtr> Copy = Trees; + Trees.clear(); + for (unsigned i = 0, e = Copy.size(); i != e; ++i) + Copy[i]->InlinePatternFragments(Copy[i], *this, Trees); + } + + /// InferAllTypes - Infer/propagate as many types throughout the expression + /// patterns as possible. Return true if all types are inferred, false + /// otherwise. Bail out if a type contradiction is found. + bool InferAllTypes( + const StringMap<SmallVector<TreePatternNode *, 1>> *NamedTypes = nullptr); + + /// error - If this is the first error in the current resolution step, + /// print it and set the error flag. Otherwise, continue silently. + void error(const Twine &Msg); + bool hasError() const { + return HasError; + } + void resetError() { + HasError = false; + } + + TypeInfer &getInfer() { return Infer; } + + void print(raw_ostream &OS) const; + void dump() const; + +private: + TreePatternNodePtr ParseTreePattern(Init *DI, StringRef OpName); + void ComputeNamedNodes(); + void ComputeNamedNodes(TreePatternNode *N); +}; + + +inline bool TreePatternNode::UpdateNodeType(unsigned ResNo, + const TypeSetByHwMode &InTy, + TreePattern &TP) { + TypeSetByHwMode VTS(InTy); + TP.getInfer().expandOverloads(VTS); + return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS); +} + +inline bool TreePatternNode::UpdateNodeType(unsigned ResNo, + MVT::SimpleValueType InTy, + TreePattern &TP) { + TypeSetByHwMode VTS(InTy); + TP.getInfer().expandOverloads(VTS); + return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS); +} + +inline bool TreePatternNode::UpdateNodeType(unsigned ResNo, + ValueTypeByHwMode InTy, + TreePattern &TP) { + TypeSetByHwMode VTS(InTy); + TP.getInfer().expandOverloads(VTS); + return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS); +} + + +/// DAGDefaultOperand - One of these is created for each OperandWithDefaultOps +/// that has a set ExecuteAlways / DefaultOps field. +struct DAGDefaultOperand { + std::vector<TreePatternNodePtr> DefaultOps; +}; + +class DAGInstruction { + std::vector<Record*> Results; + std::vector<Record*> Operands; + std::vector<Record*> ImpResults; + TreePatternNodePtr SrcPattern; + TreePatternNodePtr ResultPattern; + +public: + DAGInstruction(const std::vector<Record*> &results, + const std::vector<Record*> &operands, + const std::vector<Record*> &impresults, + TreePatternNodePtr srcpattern = nullptr, + TreePatternNodePtr resultpattern = nullptr) + : Results(results), Operands(operands), ImpResults(impresults), + SrcPattern(srcpattern), ResultPattern(resultpattern) {} + + unsigned getNumResults() const { return Results.size(); } + unsigned getNumOperands() const { return Operands.size(); } + unsigned getNumImpResults() const { return ImpResults.size(); } + const std::vector<Record*>& getImpResults() const { return ImpResults; } + + Record *getResult(unsigned RN) const { + assert(RN < Results.size()); + return Results[RN]; + } + + Record *getOperand(unsigned ON) const { + assert(ON < Operands.size()); + return Operands[ON]; + } + + Record *getImpResult(unsigned RN) const { + assert(RN < ImpResults.size()); + return ImpResults[RN]; + } + + TreePatternNodePtr getSrcPattern() const { return SrcPattern; } + TreePatternNodePtr getResultPattern() const { return ResultPattern; } +}; + +/// This class represents a condition that has to be satisfied for a pattern +/// to be tried. It is a generalization of a class "Pattern" from Target.td: +/// in addition to the Target.td's predicates, this class can also represent +/// conditions associated with HW modes. Both types will eventually become +/// strings containing C++ code to be executed, the difference is in how +/// these strings are generated. +class Predicate { +public: + Predicate(Record *R, bool C = true) : Def(R), IfCond(C), IsHwMode(false) { + assert(R->isSubClassOf("Predicate") && + "Predicate objects should only be created for records derived" + "from Predicate class"); + } + Predicate(StringRef FS, bool C = true) : Def(nullptr), Features(FS.str()), + IfCond(C), IsHwMode(true) {} + + /// Return a string which contains the C++ condition code that will serve + /// as a predicate during instruction selection. + std::string getCondString() const { + // The string will excute in a subclass of SelectionDAGISel. + // Cast to std::string explicitly to avoid ambiguity with StringRef. + std::string C = IsHwMode + ? std::string("MF->getSubtarget().checkFeatures(\"" + + Features + "\")") + : std::string(Def->getValueAsString("CondString")); + if (C.empty()) + return ""; + return IfCond ? C : "!("+C+')'; + } + + bool operator==(const Predicate &P) const { + return IfCond == P.IfCond && IsHwMode == P.IsHwMode && Def == P.Def; + } + bool operator<(const Predicate &P) const { + if (IsHwMode != P.IsHwMode) + return IsHwMode < P.IsHwMode; + assert(!Def == !P.Def && "Inconsistency between Def and IsHwMode"); + if (IfCond != P.IfCond) + return IfCond < P.IfCond; + if (Def) + return LessRecord()(Def, P.Def); + return Features < P.Features; + } + Record *Def; ///< Predicate definition from .td file, null for + ///< HW modes. + std::string Features; ///< Feature string for HW mode. + bool IfCond; ///< The boolean value that the condition has to + ///< evaluate to for this predicate to be true. + bool IsHwMode; ///< Does this predicate correspond to a HW mode? +}; + +/// PatternToMatch - Used by CodeGenDAGPatterns to keep tab of patterns +/// processed to produce isel. +class PatternToMatch { +public: + PatternToMatch(Record *srcrecord, std::vector<Predicate> preds, + TreePatternNodePtr src, TreePatternNodePtr dst, + std::vector<Record *> dstregs, int complexity, + unsigned uid, unsigned setmode = 0) + : SrcRecord(srcrecord), SrcPattern(src), DstPattern(dst), + Predicates(std::move(preds)), Dstregs(std::move(dstregs)), + AddedComplexity(complexity), ID(uid), ForceMode(setmode) {} + + Record *SrcRecord; // Originating Record for the pattern. + TreePatternNodePtr SrcPattern; // Source pattern to match. + TreePatternNodePtr DstPattern; // Resulting pattern. + std::vector<Predicate> Predicates; // Top level predicate conditions + // to match. + std::vector<Record*> Dstregs; // Physical register defs being matched. + int AddedComplexity; // Add to matching pattern complexity. + unsigned ID; // Unique ID for the record. + unsigned ForceMode; // Force this mode in type inference when set. + + Record *getSrcRecord() const { return SrcRecord; } + TreePatternNode *getSrcPattern() const { return SrcPattern.get(); } + TreePatternNodePtr getSrcPatternShared() const { return SrcPattern; } + TreePatternNode *getDstPattern() const { return DstPattern.get(); } + TreePatternNodePtr getDstPatternShared() const { return DstPattern; } + const std::vector<Record*> &getDstRegs() const { return Dstregs; } + int getAddedComplexity() const { return AddedComplexity; } + const std::vector<Predicate> &getPredicates() const { return Predicates; } + + std::string getPredicateCheck() const; + + /// Compute the complexity metric for the input pattern. This roughly + /// corresponds to the number of nodes that are covered. + int getPatternComplexity(const CodeGenDAGPatterns &CGP) const; +}; + +class CodeGenDAGPatterns { + RecordKeeper &Records; + CodeGenTarget Target; + CodeGenIntrinsicTable Intrinsics; + + std::map<Record*, SDNodeInfo, LessRecordByID> SDNodes; + std::map<Record*, std::pair<Record*, std::string>, LessRecordByID> + SDNodeXForms; + std::map<Record*, ComplexPattern, LessRecordByID> ComplexPatterns; + std::map<Record *, std::unique_ptr<TreePattern>, LessRecordByID> + PatternFragments; + std::map<Record*, DAGDefaultOperand, LessRecordByID> DefaultOperands; + std::map<Record*, DAGInstruction, LessRecordByID> Instructions; + + // Specific SDNode definitions: + Record *intrinsic_void_sdnode; + Record *intrinsic_w_chain_sdnode, *intrinsic_wo_chain_sdnode; + + /// PatternsToMatch - All of the things we are matching on the DAG. The first + /// value is the pattern to match, the second pattern is the result to + /// emit. + std::vector<PatternToMatch> PatternsToMatch; + + TypeSetByHwMode LegalVTS; + + using PatternRewriterFn = std::function<void (TreePattern *)>; + PatternRewriterFn PatternRewriter; + + unsigned NumScopes = 0; + +public: + CodeGenDAGPatterns(RecordKeeper &R, + PatternRewriterFn PatternRewriter = nullptr); + + CodeGenTarget &getTargetInfo() { return Target; } + const CodeGenTarget &getTargetInfo() const { return Target; } + const TypeSetByHwMode &getLegalTypes() const { return LegalVTS; } + + Record *getSDNodeNamed(const std::string &Name) const; + + const SDNodeInfo &getSDNodeInfo(Record *R) const { + auto F = SDNodes.find(R); + assert(F != SDNodes.end() && "Unknown node!"); + return F->second; + } + + // Node transformation lookups. + typedef std::pair<Record*, std::string> NodeXForm; + const NodeXForm &getSDNodeTransform(Record *R) const { + auto F = SDNodeXForms.find(R); + assert(F != SDNodeXForms.end() && "Invalid transform!"); + return F->second; + } + + const ComplexPattern &getComplexPattern(Record *R) const { + auto F = ComplexPatterns.find(R); + assert(F != ComplexPatterns.end() && "Unknown addressing mode!"); + return F->second; + } + + const CodeGenIntrinsic &getIntrinsic(Record *R) const { + for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i) + if (Intrinsics[i].TheDef == R) return Intrinsics[i]; + llvm_unreachable("Unknown intrinsic!"); + } + + const CodeGenIntrinsic &getIntrinsicInfo(unsigned IID) const { + if (IID-1 < Intrinsics.size()) + return Intrinsics[IID-1]; + llvm_unreachable("Bad intrinsic ID!"); + } + + unsigned getIntrinsicID(Record *R) const { + for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i) + if (Intrinsics[i].TheDef == R) return i; + llvm_unreachable("Unknown intrinsic!"); + } + + const DAGDefaultOperand &getDefaultOperand(Record *R) const { + auto F = DefaultOperands.find(R); + assert(F != DefaultOperands.end() &&"Isn't an analyzed default operand!"); + return F->second; + } + + // Pattern Fragment information. + TreePattern *getPatternFragment(Record *R) const { + auto F = PatternFragments.find(R); + assert(F != PatternFragments.end() && "Invalid pattern fragment request!"); + return F->second.get(); + } + TreePattern *getPatternFragmentIfRead(Record *R) const { + auto F = PatternFragments.find(R); + if (F == PatternFragments.end()) + return nullptr; + return F->second.get(); + } + + typedef std::map<Record *, std::unique_ptr<TreePattern>, + LessRecordByID>::const_iterator pf_iterator; + pf_iterator pf_begin() const { return PatternFragments.begin(); } + pf_iterator pf_end() const { return PatternFragments.end(); } + iterator_range<pf_iterator> ptfs() const { return PatternFragments; } + + // Patterns to match information. + typedef std::vector<PatternToMatch>::const_iterator ptm_iterator; + ptm_iterator ptm_begin() const { return PatternsToMatch.begin(); } + ptm_iterator ptm_end() const { return PatternsToMatch.end(); } + iterator_range<ptm_iterator> ptms() const { return PatternsToMatch; } + + /// Parse the Pattern for an instruction, and insert the result in DAGInsts. + typedef std::map<Record*, DAGInstruction, LessRecordByID> DAGInstMap; + void parseInstructionPattern( + CodeGenInstruction &CGI, ListInit *Pattern, + DAGInstMap &DAGInsts); + + const DAGInstruction &getInstruction(Record *R) const { + auto F = Instructions.find(R); + assert(F != Instructions.end() && "Unknown instruction!"); + return F->second; + } + + Record *get_intrinsic_void_sdnode() const { + return intrinsic_void_sdnode; + } + Record *get_intrinsic_w_chain_sdnode() const { + return intrinsic_w_chain_sdnode; + } + Record *get_intrinsic_wo_chain_sdnode() const { + return intrinsic_wo_chain_sdnode; + } + + unsigned allocateScope() { return ++NumScopes; } + + bool operandHasDefault(Record *Op) const { + return Op->isSubClassOf("OperandWithDefaultOps") && + !getDefaultOperand(Op).DefaultOps.empty(); + } + +private: + void ParseNodeInfo(); + void ParseNodeTransforms(); + void ParseComplexPatterns(); + void ParsePatternFragments(bool OutFrags = false); + void ParseDefaultOperands(); + void ParseInstructions(); + void ParsePatterns(); + void ExpandHwModeBasedTypes(); + void InferInstructionFlags(); + void GenerateVariants(); + void VerifyInstructionFlags(); + + std::vector<Predicate> makePredList(ListInit *L); + + void ParseOnePattern(Record *TheDef, + TreePattern &Pattern, TreePattern &Result, + const std::vector<Record *> &InstImpResults); + void AddPatternToMatch(TreePattern *Pattern, PatternToMatch &&PTM); + void FindPatternInputsAndOutputs( + TreePattern &I, TreePatternNodePtr Pat, + std::map<std::string, TreePatternNodePtr> &InstInputs, + MapVector<std::string, TreePatternNodePtr, + std::map<std::string, unsigned>> &InstResults, + std::vector<Record *> &InstImpResults); +}; + + +inline bool SDNodeInfo::ApplyTypeConstraints(TreePatternNode *N, + TreePattern &TP) const { + bool MadeChange = false; + for (unsigned i = 0, e = TypeConstraints.size(); i != e; ++i) + MadeChange |= TypeConstraints[i].ApplyTypeConstraint(N, *this, TP); + return MadeChange; + } + +} // end namespace llvm + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenHwModes.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenHwModes.cpp new file mode 100644 index 000000000000..2fec46c44100 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenHwModes.cpp @@ -0,0 +1,113 @@ +//===--- CodeGenHwModes.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Classes to parse and store HW mode information for instruction selection +//===----------------------------------------------------------------------===// + +#include "CodeGenHwModes.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" + +using namespace llvm; + +StringRef CodeGenHwModes::DefaultModeName = "DefaultMode"; + +HwMode::HwMode(Record *R) { + Name = R->getName(); + Features = std::string(R->getValueAsString("Features")); +} + +LLVM_DUMP_METHOD +void HwMode::dump() const { + dbgs() << Name << ": " << Features << '\n'; +} + +HwModeSelect::HwModeSelect(Record *R, CodeGenHwModes &CGH) { + std::vector<Record*> Modes = R->getValueAsListOfDefs("Modes"); + std::vector<Record*> Objects = R->getValueAsListOfDefs("Objects"); + if (Modes.size() != Objects.size()) { + PrintError(R->getLoc(), "in record " + R->getName() + + " derived from HwModeSelect: the lists Modes and Objects should " + "have the same size"); + report_fatal_error("error in target description."); + } + for (unsigned i = 0, e = Modes.size(); i != e; ++i) { + unsigned ModeId = CGH.getHwModeId(Modes[i]->getName()); + Items.push_back(std::make_pair(ModeId, Objects[i])); + } +} + +LLVM_DUMP_METHOD +void HwModeSelect::dump() const { + dbgs() << '{'; + for (const PairType &P : Items) + dbgs() << " (" << P.first << ',' << P.second->getName() << ')'; + dbgs() << " }\n"; +} + +CodeGenHwModes::CodeGenHwModes(RecordKeeper &RK) : Records(RK) { + std::vector<Record*> MRs = Records.getAllDerivedDefinitions("HwMode"); + // The default mode needs a definition in the .td sources for TableGen + // to accept references to it. We need to ignore the definition here. + for (auto I = MRs.begin(), E = MRs.end(); I != E; ++I) { + if ((*I)->getName() != DefaultModeName) + continue; + MRs.erase(I); + break; + } + + for (Record *R : MRs) { + Modes.emplace_back(R); + unsigned NewId = Modes.size(); + ModeIds.insert(std::make_pair(Modes[NewId-1].Name, NewId)); + } + + std::vector<Record*> MSs = Records.getAllDerivedDefinitions("HwModeSelect"); + for (Record *R : MSs) { + auto P = ModeSelects.emplace(std::make_pair(R, HwModeSelect(R, *this))); + assert(P.second); + (void)P; + } +} + +unsigned CodeGenHwModes::getHwModeId(StringRef Name) const { + if (Name == DefaultModeName) + return DefaultMode; + auto F = ModeIds.find(Name); + assert(F != ModeIds.end() && "Unknown mode name"); + return F->second; +} + +const HwModeSelect &CodeGenHwModes::getHwModeSelect(Record *R) const { + auto F = ModeSelects.find(R); + assert(F != ModeSelects.end() && "Record is not a \"mode select\""); + return F->second; +} + +LLVM_DUMP_METHOD +void CodeGenHwModes::dump() const { + dbgs() << "Modes: {\n"; + for (const HwMode &M : Modes) { + dbgs() << " "; + M.dump(); + } + dbgs() << "}\n"; + + dbgs() << "ModeIds: {\n"; + for (const auto &P : ModeIds) + dbgs() << " " << P.first() << " -> " << P.second << '\n'; + dbgs() << "}\n"; + + dbgs() << "ModeSelects: {\n"; + for (const auto &P : ModeSelects) { + dbgs() << " " << P.first->getName() << " -> "; + P.second.dump(); + } + dbgs() << "}\n"; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenHwModes.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenHwModes.h new file mode 100644 index 000000000000..55507cbca37d --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenHwModes.h @@ -0,0 +1,64 @@ +//===--- CodeGenHwModes.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Classes to parse and store HW mode information for instruction selection. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEGENHWMODES_H +#define LLVM_UTILS_TABLEGEN_CODEGENHWMODES_H + +#include "llvm/ADT/StringMap.h" +#include <cassert> +#include <map> +#include <string> +#include <vector> + +// HwModeId -> list of predicates (definition) + +namespace llvm { + class Record; + class RecordKeeper; + + struct CodeGenHwModes; + + struct HwMode { + HwMode(Record *R); + StringRef Name; + std::string Features; + void dump() const; + }; + + struct HwModeSelect { + HwModeSelect(Record *R, CodeGenHwModes &CGH); + typedef std::pair<unsigned, Record*> PairType; + std::vector<PairType> Items; + void dump() const; + }; + + struct CodeGenHwModes { + enum : unsigned { DefaultMode = 0 }; + static StringRef DefaultModeName; + + CodeGenHwModes(RecordKeeper &R); + unsigned getHwModeId(StringRef Name) const; + const HwMode &getMode(unsigned Id) const { + assert(Id != 0 && "Mode id of 0 is reserved for the default mode"); + return Modes[Id-1]; + } + const HwModeSelect &getHwModeSelect(Record *R) const; + unsigned getNumModeIds() const { return Modes.size()+1; } + void dump() const; + + private: + RecordKeeper &Records; + StringMap<unsigned> ModeIds; // HwMode (string) -> HwModeId + std::vector<HwMode> Modes; + std::map<Record*,HwModeSelect> ModeSelects; + }; +} + +#endif // LLVM_UTILS_TABLEGEN_CODEGENHWMODES_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.cpp new file mode 100644 index 000000000000..1df5902b081e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.cpp @@ -0,0 +1,793 @@ +//===- CodeGenInstruction.cpp - CodeGen Instruction Class Wrapper ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the CodeGenInstruction class. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenInstruction.h" +#include "CodeGenTarget.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include <set> +using namespace llvm; + +//===----------------------------------------------------------------------===// +// CGIOperandList Implementation +//===----------------------------------------------------------------------===// + +CGIOperandList::CGIOperandList(Record *R) : TheDef(R) { + isPredicable = false; + hasOptionalDef = false; + isVariadic = false; + + DagInit *OutDI = R->getValueAsDag("OutOperandList"); + + if (DefInit *Init = dyn_cast<DefInit>(OutDI->getOperator())) { + if (Init->getDef()->getName() != "outs") + PrintFatalError(R->getLoc(), + R->getName() + + ": invalid def name for output list: use 'outs'"); + } else + PrintFatalError(R->getLoc(), + R->getName() + ": invalid output list: use 'outs'"); + + NumDefs = OutDI->getNumArgs(); + + DagInit *InDI = R->getValueAsDag("InOperandList"); + if (DefInit *Init = dyn_cast<DefInit>(InDI->getOperator())) { + if (Init->getDef()->getName() != "ins") + PrintFatalError(R->getLoc(), + R->getName() + + ": invalid def name for input list: use 'ins'"); + } else + PrintFatalError(R->getLoc(), + R->getName() + ": invalid input list: use 'ins'"); + + unsigned MIOperandNo = 0; + std::set<std::string> OperandNames; + unsigned e = InDI->getNumArgs() + OutDI->getNumArgs(); + OperandList.reserve(e); + bool VariadicOuts = false; + for (unsigned i = 0; i != e; ++i){ + Init *ArgInit; + StringRef ArgName; + if (i < NumDefs) { + ArgInit = OutDI->getArg(i); + ArgName = OutDI->getArgNameStr(i); + } else { + ArgInit = InDI->getArg(i-NumDefs); + ArgName = InDI->getArgNameStr(i-NumDefs); + } + + DefInit *Arg = dyn_cast<DefInit>(ArgInit); + if (!Arg) + PrintFatalError(R->getLoc(), "Illegal operand for the '" + R->getName() + + "' instruction!"); + + Record *Rec = Arg->getDef(); + std::string PrintMethod = "printOperand"; + std::string EncoderMethod; + std::string OperandType = "OPERAND_UNKNOWN"; + std::string OperandNamespace = "MCOI"; + unsigned NumOps = 1; + DagInit *MIOpInfo = nullptr; + if (Rec->isSubClassOf("RegisterOperand")) { + PrintMethod = std::string(Rec->getValueAsString("PrintMethod")); + OperandType = std::string(Rec->getValueAsString("OperandType")); + OperandNamespace = std::string(Rec->getValueAsString("OperandNamespace")); + EncoderMethod = std::string(Rec->getValueAsString("EncoderMethod")); + } else if (Rec->isSubClassOf("Operand")) { + PrintMethod = std::string(Rec->getValueAsString("PrintMethod")); + OperandType = std::string(Rec->getValueAsString("OperandType")); + OperandNamespace = std::string(Rec->getValueAsString("OperandNamespace")); + // If there is an explicit encoder method, use it. + EncoderMethod = std::string(Rec->getValueAsString("EncoderMethod")); + MIOpInfo = Rec->getValueAsDag("MIOperandInfo"); + + // Verify that MIOpInfo has an 'ops' root value. + if (!isa<DefInit>(MIOpInfo->getOperator()) || + cast<DefInit>(MIOpInfo->getOperator())->getDef()->getName() != "ops") + PrintFatalError(R->getLoc(), + "Bad value for MIOperandInfo in operand '" + + Rec->getName() + "'\n"); + + // If we have MIOpInfo, then we have #operands equal to number of entries + // in MIOperandInfo. + if (unsigned NumArgs = MIOpInfo->getNumArgs()) + NumOps = NumArgs; + + if (Rec->isSubClassOf("PredicateOp")) + isPredicable = true; + else if (Rec->isSubClassOf("OptionalDefOperand")) + hasOptionalDef = true; + } else if (Rec->getName() == "variable_ops") { + if (i < NumDefs) + VariadicOuts = true; + isVariadic = true; + continue; + } else if (Rec->isSubClassOf("RegisterClass")) { + OperandType = "OPERAND_REGISTER"; + } else if (!Rec->isSubClassOf("PointerLikeRegClass") && + !Rec->isSubClassOf("unknown_class")) + PrintFatalError(R->getLoc(), "Unknown operand class '" + Rec->getName() + + "' in '" + R->getName() + + "' instruction!"); + + // Check that the operand has a name and that it's unique. + if (ArgName.empty()) + PrintFatalError(R->getLoc(), "In instruction '" + R->getName() + + "', operand #" + Twine(i) + + " has no name!"); + if (!OperandNames.insert(std::string(ArgName)).second) + PrintFatalError(R->getLoc(), + "In instruction '" + R->getName() + "', operand #" + + Twine(i) + + " has the same name as a previous operand!"); + + OperandList.emplace_back( + Rec, std::string(ArgName), std::string(PrintMethod), + std::string(EncoderMethod), OperandNamespace + "::" + OperandType, + MIOperandNo, NumOps, MIOpInfo); + MIOperandNo += NumOps; + } + + if (VariadicOuts) + --NumDefs; + + // Make sure the constraints list for each operand is large enough to hold + // constraint info, even if none is present. + for (OperandInfo &OpInfo : OperandList) + OpInfo.Constraints.resize(OpInfo.MINumOperands); +} + + +/// getOperandNamed - Return the index of the operand with the specified +/// non-empty name. If the instruction does not have an operand with the +/// specified name, abort. +/// +unsigned CGIOperandList::getOperandNamed(StringRef Name) const { + unsigned OpIdx; + if (hasOperandNamed(Name, OpIdx)) + return OpIdx; + PrintFatalError(TheDef->getLoc(), "'" + TheDef->getName() + + "' does not have an operand named '$" + + Name + "'!"); +} + +/// hasOperandNamed - Query whether the instruction has an operand of the +/// given name. If so, return true and set OpIdx to the index of the +/// operand. Otherwise, return false. +bool CGIOperandList::hasOperandNamed(StringRef Name, unsigned &OpIdx) const { + assert(!Name.empty() && "Cannot search for operand with no name!"); + for (unsigned i = 0, e = OperandList.size(); i != e; ++i) + if (OperandList[i].Name == Name) { + OpIdx = i; + return true; + } + return false; +} + +std::pair<unsigned,unsigned> +CGIOperandList::ParseOperandName(const std::string &Op, bool AllowWholeOp) { + if (Op.empty() || Op[0] != '$') + PrintFatalError(TheDef->getLoc(), + TheDef->getName() + ": Illegal operand name: '" + Op + "'"); + + std::string OpName = Op.substr(1); + std::string SubOpName; + + // Check to see if this is $foo.bar. + std::string::size_type DotIdx = OpName.find_first_of('.'); + if (DotIdx != std::string::npos) { + SubOpName = OpName.substr(DotIdx+1); + if (SubOpName.empty()) + PrintFatalError(TheDef->getLoc(), + TheDef->getName() + + ": illegal empty suboperand name in '" + Op + "'"); + OpName = OpName.substr(0, DotIdx); + } + + unsigned OpIdx = getOperandNamed(OpName); + + if (SubOpName.empty()) { // If no suboperand name was specified: + // If one was needed, throw. + if (OperandList[OpIdx].MINumOperands > 1 && !AllowWholeOp && + SubOpName.empty()) + PrintFatalError(TheDef->getLoc(), + TheDef->getName() + + ": Illegal to refer to" + " whole operand part of complex operand '" + + Op + "'"); + + // Otherwise, return the operand. + return std::make_pair(OpIdx, 0U); + } + + // Find the suboperand number involved. + DagInit *MIOpInfo = OperandList[OpIdx].MIOperandInfo; + if (!MIOpInfo) + PrintFatalError(TheDef->getLoc(), TheDef->getName() + + ": unknown suboperand name in '" + + Op + "'"); + + // Find the operand with the right name. + for (unsigned i = 0, e = MIOpInfo->getNumArgs(); i != e; ++i) + if (MIOpInfo->getArgNameStr(i) == SubOpName) + return std::make_pair(OpIdx, i); + + // Otherwise, didn't find it! + PrintFatalError(TheDef->getLoc(), TheDef->getName() + + ": unknown suboperand name in '" + Op + + "'"); + return std::make_pair(0U, 0U); +} + +static void ParseConstraint(const std::string &CStr, CGIOperandList &Ops, + Record *Rec) { + // EARLY_CLOBBER: @early $reg + std::string::size_type wpos = CStr.find_first_of(" \t"); + std::string::size_type start = CStr.find_first_not_of(" \t"); + std::string Tok = CStr.substr(start, wpos - start); + if (Tok == "@earlyclobber") { + std::string Name = CStr.substr(wpos+1); + wpos = Name.find_first_not_of(" \t"); + if (wpos == std::string::npos) + PrintFatalError( + Rec->getLoc(), "Illegal format for @earlyclobber constraint in '" + + Rec->getName() + "': '" + CStr + "'"); + Name = Name.substr(wpos); + std::pair<unsigned,unsigned> Op = Ops.ParseOperandName(Name, false); + + // Build the string for the operand + if (!Ops[Op.first].Constraints[Op.second].isNone()) + PrintFatalError( + Rec->getLoc(), "Operand '" + Name + "' of '" + Rec->getName() + + "' cannot have multiple constraints!"); + Ops[Op.first].Constraints[Op.second] = + CGIOperandList::ConstraintInfo::getEarlyClobber(); + return; + } + + // Only other constraint is "TIED_TO" for now. + std::string::size_type pos = CStr.find_first_of('='); + if (pos == std::string::npos) + PrintFatalError( + Rec->getLoc(), "Unrecognized constraint '" + CStr + + "' in '" + Rec->getName() + "'"); + start = CStr.find_first_not_of(" \t"); + + // TIED_TO: $src1 = $dst + wpos = CStr.find_first_of(" \t", start); + if (wpos == std::string::npos || wpos > pos) + PrintFatalError( + Rec->getLoc(), "Illegal format for tied-to constraint in '" + + Rec->getName() + "': '" + CStr + "'"); + std::string LHSOpName = + std::string(StringRef(CStr).substr(start, wpos - start)); + std::pair<unsigned,unsigned> LHSOp = Ops.ParseOperandName(LHSOpName, false); + + wpos = CStr.find_first_not_of(" \t", pos + 1); + if (wpos == std::string::npos) + PrintFatalError( + Rec->getLoc(), "Illegal format for tied-to constraint: '" + CStr + "'"); + + std::string RHSOpName = std::string(StringRef(CStr).substr(wpos)); + std::pair<unsigned,unsigned> RHSOp = Ops.ParseOperandName(RHSOpName, false); + + // Sort the operands into order, which should put the output one + // first. But keep the original order, for use in diagnostics. + bool FirstIsDest = (LHSOp < RHSOp); + std::pair<unsigned,unsigned> DestOp = (FirstIsDest ? LHSOp : RHSOp); + StringRef DestOpName = (FirstIsDest ? LHSOpName : RHSOpName); + std::pair<unsigned,unsigned> SrcOp = (FirstIsDest ? RHSOp : LHSOp); + StringRef SrcOpName = (FirstIsDest ? RHSOpName : LHSOpName); + + // Ensure one operand is a def and the other is a use. + if (DestOp.first >= Ops.NumDefs) + PrintFatalError( + Rec->getLoc(), "Input operands '" + LHSOpName + "' and '" + RHSOpName + + "' of '" + Rec->getName() + "' cannot be tied!"); + if (SrcOp.first < Ops.NumDefs) + PrintFatalError( + Rec->getLoc(), "Output operands '" + LHSOpName + "' and '" + RHSOpName + + "' of '" + Rec->getName() + "' cannot be tied!"); + + // The constraint has to go on the operand with higher index, i.e. + // the source one. Check there isn't another constraint there + // already. + if (!Ops[SrcOp.first].Constraints[SrcOp.second].isNone()) + PrintFatalError( + Rec->getLoc(), "Operand '" + SrcOpName + "' of '" + Rec->getName() + + "' cannot have multiple constraints!"); + + unsigned DestFlatOpNo = Ops.getFlattenedOperandNumber(DestOp); + auto NewConstraint = CGIOperandList::ConstraintInfo::getTied(DestFlatOpNo); + + // Check that the earlier operand is not the target of another tie + // before making it the target of this one. + for (const CGIOperandList::OperandInfo &Op : Ops) { + for (unsigned i = 0; i < Op.MINumOperands; i++) + if (Op.Constraints[i] == NewConstraint) + PrintFatalError( + Rec->getLoc(), "Operand '" + DestOpName + "' of '" + Rec->getName() + + "' cannot have multiple operands tied to it!"); + } + + Ops[SrcOp.first].Constraints[SrcOp.second] = NewConstraint; +} + +static void ParseConstraints(const std::string &CStr, CGIOperandList &Ops, + Record *Rec) { + if (CStr.empty()) return; + + const std::string delims(","); + std::string::size_type bidx, eidx; + + bidx = CStr.find_first_not_of(delims); + while (bidx != std::string::npos) { + eidx = CStr.find_first_of(delims, bidx); + if (eidx == std::string::npos) + eidx = CStr.length(); + + ParseConstraint(CStr.substr(bidx, eidx - bidx), Ops, Rec); + bidx = CStr.find_first_not_of(delims, eidx); + } +} + +void CGIOperandList::ProcessDisableEncoding(std::string DisableEncoding) { + while (1) { + std::pair<StringRef, StringRef> P = getToken(DisableEncoding, " ,\t"); + std::string OpName = std::string(P.first); + DisableEncoding = std::string(P.second); + if (OpName.empty()) break; + + // Figure out which operand this is. + std::pair<unsigned,unsigned> Op = ParseOperandName(OpName, false); + + // Mark the operand as not-to-be encoded. + if (Op.second >= OperandList[Op.first].DoNotEncode.size()) + OperandList[Op.first].DoNotEncode.resize(Op.second+1); + OperandList[Op.first].DoNotEncode[Op.second] = true; + } + +} + +//===----------------------------------------------------------------------===// +// CodeGenInstruction Implementation +//===----------------------------------------------------------------------===// + +CodeGenInstruction::CodeGenInstruction(Record *R) + : TheDef(R), Operands(R), InferredFrom(nullptr) { + Namespace = R->getValueAsString("Namespace"); + AsmString = std::string(R->getValueAsString("AsmString")); + + isPreISelOpcode = R->getValueAsBit("isPreISelOpcode"); + isReturn = R->getValueAsBit("isReturn"); + isEHScopeReturn = R->getValueAsBit("isEHScopeReturn"); + isBranch = R->getValueAsBit("isBranch"); + isIndirectBranch = R->getValueAsBit("isIndirectBranch"); + isCompare = R->getValueAsBit("isCompare"); + isMoveImm = R->getValueAsBit("isMoveImm"); + isMoveReg = R->getValueAsBit("isMoveReg"); + isBitcast = R->getValueAsBit("isBitcast"); + isSelect = R->getValueAsBit("isSelect"); + isBarrier = R->getValueAsBit("isBarrier"); + isCall = R->getValueAsBit("isCall"); + isAdd = R->getValueAsBit("isAdd"); + isTrap = R->getValueAsBit("isTrap"); + canFoldAsLoad = R->getValueAsBit("canFoldAsLoad"); + isPredicable = !R->getValueAsBit("isUnpredicable") && ( + Operands.isPredicable || R->getValueAsBit("isPredicable")); + isConvertibleToThreeAddress = R->getValueAsBit("isConvertibleToThreeAddress"); + isCommutable = R->getValueAsBit("isCommutable"); + isTerminator = R->getValueAsBit("isTerminator"); + isReMaterializable = R->getValueAsBit("isReMaterializable"); + hasDelaySlot = R->getValueAsBit("hasDelaySlot"); + usesCustomInserter = R->getValueAsBit("usesCustomInserter"); + hasPostISelHook = R->getValueAsBit("hasPostISelHook"); + hasCtrlDep = R->getValueAsBit("hasCtrlDep"); + isNotDuplicable = R->getValueAsBit("isNotDuplicable"); + isRegSequence = R->getValueAsBit("isRegSequence"); + isExtractSubreg = R->getValueAsBit("isExtractSubreg"); + isInsertSubreg = R->getValueAsBit("isInsertSubreg"); + isConvergent = R->getValueAsBit("isConvergent"); + hasNoSchedulingInfo = R->getValueAsBit("hasNoSchedulingInfo"); + FastISelShouldIgnore = R->getValueAsBit("FastISelShouldIgnore"); + variadicOpsAreDefs = R->getValueAsBit("variadicOpsAreDefs"); + isAuthenticated = R->getValueAsBit("isAuthenticated"); + + bool Unset; + mayLoad = R->getValueAsBitOrUnset("mayLoad", Unset); + mayLoad_Unset = Unset; + mayStore = R->getValueAsBitOrUnset("mayStore", Unset); + mayStore_Unset = Unset; + mayRaiseFPException = R->getValueAsBit("mayRaiseFPException"); + hasSideEffects = R->getValueAsBitOrUnset("hasSideEffects", Unset); + hasSideEffects_Unset = Unset; + + isAsCheapAsAMove = R->getValueAsBit("isAsCheapAsAMove"); + hasExtraSrcRegAllocReq = R->getValueAsBit("hasExtraSrcRegAllocReq"); + hasExtraDefRegAllocReq = R->getValueAsBit("hasExtraDefRegAllocReq"); + isCodeGenOnly = R->getValueAsBit("isCodeGenOnly"); + isPseudo = R->getValueAsBit("isPseudo"); + ImplicitDefs = R->getValueAsListOfDefs("Defs"); + ImplicitUses = R->getValueAsListOfDefs("Uses"); + + // This flag is only inferred from the pattern. + hasChain = false; + hasChain_Inferred = false; + + // Parse Constraints. + ParseConstraints(std::string(R->getValueAsString("Constraints")), Operands, + R); + + // Parse the DisableEncoding field. + Operands.ProcessDisableEncoding( + std::string(R->getValueAsString("DisableEncoding"))); + + // First check for a ComplexDeprecationPredicate. + if (R->getValue("ComplexDeprecationPredicate")) { + HasComplexDeprecationPredicate = true; + DeprecatedReason = + std::string(R->getValueAsString("ComplexDeprecationPredicate")); + } else if (RecordVal *Dep = R->getValue("DeprecatedFeatureMask")) { + // Check if we have a Subtarget feature mask. + HasComplexDeprecationPredicate = false; + DeprecatedReason = Dep->getValue()->getAsString(); + } else { + // This instruction isn't deprecated. + HasComplexDeprecationPredicate = false; + DeprecatedReason = ""; + } +} + +/// HasOneImplicitDefWithKnownVT - If the instruction has at least one +/// implicit def and it has a known VT, return the VT, otherwise return +/// MVT::Other. +MVT::SimpleValueType CodeGenInstruction:: +HasOneImplicitDefWithKnownVT(const CodeGenTarget &TargetInfo) const { + if (ImplicitDefs.empty()) return MVT::Other; + + // Check to see if the first implicit def has a resolvable type. + Record *FirstImplicitDef = ImplicitDefs[0]; + assert(FirstImplicitDef->isSubClassOf("Register")); + const std::vector<ValueTypeByHwMode> &RegVTs = + TargetInfo.getRegisterVTs(FirstImplicitDef); + if (RegVTs.size() == 1 && RegVTs[0].isSimple()) + return RegVTs[0].getSimple().SimpleTy; + return MVT::Other; +} + + +/// FlattenAsmStringVariants - Flatten the specified AsmString to only +/// include text from the specified variant, returning the new string. +std::string CodeGenInstruction:: +FlattenAsmStringVariants(StringRef Cur, unsigned Variant) { + std::string Res = ""; + + for (;;) { + // Find the start of the next variant string. + size_t VariantsStart = 0; + for (size_t e = Cur.size(); VariantsStart != e; ++VariantsStart) + if (Cur[VariantsStart] == '{' && + (VariantsStart == 0 || (Cur[VariantsStart-1] != '$' && + Cur[VariantsStart-1] != '\\'))) + break; + + // Add the prefix to the result. + Res += Cur.slice(0, VariantsStart); + if (VariantsStart == Cur.size()) + break; + + ++VariantsStart; // Skip the '{'. + + // Scan to the end of the variants string. + size_t VariantsEnd = VariantsStart; + unsigned NestedBraces = 1; + for (size_t e = Cur.size(); VariantsEnd != e; ++VariantsEnd) { + if (Cur[VariantsEnd] == '}' && Cur[VariantsEnd-1] != '\\') { + if (--NestedBraces == 0) + break; + } else if (Cur[VariantsEnd] == '{') + ++NestedBraces; + } + + // Select the Nth variant (or empty). + StringRef Selection = Cur.slice(VariantsStart, VariantsEnd); + for (unsigned i = 0; i != Variant; ++i) + Selection = Selection.split('|').second; + Res += Selection.split('|').first; + + assert(VariantsEnd != Cur.size() && + "Unterminated variants in assembly string!"); + Cur = Cur.substr(VariantsEnd + 1); + } + + return Res; +} + +bool CodeGenInstruction::isOperandImpl(unsigned i, + StringRef PropertyName) const { + DagInit *ConstraintList = TheDef->getValueAsDag("InOperandList"); + if (!ConstraintList || i >= ConstraintList->getNumArgs()) + return false; + + DefInit *Constraint = dyn_cast<DefInit>(ConstraintList->getArg(i)); + if (!Constraint) + return false; + + return Constraint->getDef()->isSubClassOf("TypedOperand") && + Constraint->getDef()->getValueAsBit(PropertyName); +} + +//===----------------------------------------------------------------------===// +/// CodeGenInstAlias Implementation +//===----------------------------------------------------------------------===// + +/// tryAliasOpMatch - This is a helper function for the CodeGenInstAlias +/// constructor. It checks if an argument in an InstAlias pattern matches +/// the corresponding operand of the instruction. It returns true on a +/// successful match, with ResOp set to the result operand to be used. +bool CodeGenInstAlias::tryAliasOpMatch(DagInit *Result, unsigned AliasOpNo, + Record *InstOpRec, bool hasSubOps, + ArrayRef<SMLoc> Loc, CodeGenTarget &T, + ResultOperand &ResOp) { + Init *Arg = Result->getArg(AliasOpNo); + DefInit *ADI = dyn_cast<DefInit>(Arg); + Record *ResultRecord = ADI ? ADI->getDef() : nullptr; + + if (ADI && ADI->getDef() == InstOpRec) { + // If the operand is a record, it must have a name, and the record type + // must match up with the instruction's argument type. + if (!Result->getArgName(AliasOpNo)) + PrintFatalError(Loc, "result argument #" + Twine(AliasOpNo) + + " must have a name!"); + ResOp = ResultOperand(std::string(Result->getArgNameStr(AliasOpNo)), + ResultRecord); + return true; + } + + // For register operands, the source register class can be a subclass + // of the instruction register class, not just an exact match. + if (InstOpRec->isSubClassOf("RegisterOperand")) + InstOpRec = InstOpRec->getValueAsDef("RegClass"); + + if (ADI && ADI->getDef()->isSubClassOf("RegisterOperand")) + ADI = ADI->getDef()->getValueAsDef("RegClass")->getDefInit(); + + if (ADI && ADI->getDef()->isSubClassOf("RegisterClass")) { + if (!InstOpRec->isSubClassOf("RegisterClass")) + return false; + if (!T.getRegisterClass(InstOpRec) + .hasSubClass(&T.getRegisterClass(ADI->getDef()))) + return false; + ResOp = ResultOperand(std::string(Result->getArgNameStr(AliasOpNo)), + ResultRecord); + return true; + } + + // Handle explicit registers. + if (ADI && ADI->getDef()->isSubClassOf("Register")) { + if (InstOpRec->isSubClassOf("OptionalDefOperand")) { + DagInit *DI = InstOpRec->getValueAsDag("MIOperandInfo"); + // The operand info should only have a single (register) entry. We + // want the register class of it. + InstOpRec = cast<DefInit>(DI->getArg(0))->getDef(); + } + + if (!InstOpRec->isSubClassOf("RegisterClass")) + return false; + + if (!T.getRegisterClass(InstOpRec) + .contains(T.getRegBank().getReg(ADI->getDef()))) + PrintFatalError(Loc, "fixed register " + ADI->getDef()->getName() + + " is not a member of the " + InstOpRec->getName() + + " register class!"); + + if (Result->getArgName(AliasOpNo)) + PrintFatalError(Loc, "result fixed register argument must " + "not have a name!"); + + ResOp = ResultOperand(ResultRecord); + return true; + } + + // Handle "zero_reg" for optional def operands. + if (ADI && ADI->getDef()->getName() == "zero_reg") { + + // Check if this is an optional def. + // Tied operands where the source is a sub-operand of a complex operand + // need to represent both operands in the alias destination instruction. + // Allow zero_reg for the tied portion. This can and should go away once + // the MC representation of things doesn't use tied operands at all. + //if (!InstOpRec->isSubClassOf("OptionalDefOperand")) + // throw TGError(Loc, "reg0 used for result that is not an " + // "OptionalDefOperand!"); + + ResOp = ResultOperand(static_cast<Record*>(nullptr)); + return true; + } + + // Literal integers. + if (IntInit *II = dyn_cast<IntInit>(Arg)) { + if (hasSubOps || !InstOpRec->isSubClassOf("Operand")) + return false; + // Integer arguments can't have names. + if (Result->getArgName(AliasOpNo)) + PrintFatalError(Loc, "result argument #" + Twine(AliasOpNo) + + " must not have a name!"); + ResOp = ResultOperand(II->getValue()); + return true; + } + + // Bits<n> (also used for 0bxx literals) + if (BitsInit *BI = dyn_cast<BitsInit>(Arg)) { + if (hasSubOps || !InstOpRec->isSubClassOf("Operand")) + return false; + if (!BI->isComplete()) + return false; + // Convert the bits init to an integer and use that for the result. + IntInit *II = + dyn_cast_or_null<IntInit>(BI->convertInitializerTo(IntRecTy::get())); + if (!II) + return false; + ResOp = ResultOperand(II->getValue()); + return true; + } + + // If both are Operands with the same MVT, allow the conversion. It's + // up to the user to make sure the values are appropriate, just like + // for isel Pat's. + if (InstOpRec->isSubClassOf("Operand") && ADI && + ADI->getDef()->isSubClassOf("Operand")) { + // FIXME: What other attributes should we check here? Identical + // MIOperandInfo perhaps? + if (InstOpRec->getValueInit("Type") != ADI->getDef()->getValueInit("Type")) + return false; + ResOp = ResultOperand(std::string(Result->getArgNameStr(AliasOpNo)), + ADI->getDef()); + return true; + } + + return false; +} + +unsigned CodeGenInstAlias::ResultOperand::getMINumOperands() const { + if (!isRecord()) + return 1; + + Record *Rec = getRecord(); + if (!Rec->isSubClassOf("Operand")) + return 1; + + DagInit *MIOpInfo = Rec->getValueAsDag("MIOperandInfo"); + if (MIOpInfo->getNumArgs() == 0) { + // Unspecified, so it defaults to 1 + return 1; + } + + return MIOpInfo->getNumArgs(); +} + +CodeGenInstAlias::CodeGenInstAlias(Record *R, CodeGenTarget &T) + : TheDef(R) { + Result = R->getValueAsDag("ResultInst"); + AsmString = std::string(R->getValueAsString("AsmString")); + + // Verify that the root of the result is an instruction. + DefInit *DI = dyn_cast<DefInit>(Result->getOperator()); + if (!DI || !DI->getDef()->isSubClassOf("Instruction")) + PrintFatalError(R->getLoc(), + "result of inst alias should be an instruction"); + + ResultInst = &T.getInstruction(DI->getDef()); + + // NameClass - If argument names are repeated, we need to verify they have + // the same class. + StringMap<Record*> NameClass; + for (unsigned i = 0, e = Result->getNumArgs(); i != e; ++i) { + DefInit *ADI = dyn_cast<DefInit>(Result->getArg(i)); + if (!ADI || !Result->getArgName(i)) + continue; + // Verify we don't have something like: (someinst GR16:$foo, GR32:$foo) + // $foo can exist multiple times in the result list, but it must have the + // same type. + Record *&Entry = NameClass[Result->getArgNameStr(i)]; + if (Entry && Entry != ADI->getDef()) + PrintFatalError(R->getLoc(), "result value $" + Result->getArgNameStr(i) + + " is both " + Entry->getName() + " and " + + ADI->getDef()->getName() + "!"); + Entry = ADI->getDef(); + } + + // Decode and validate the arguments of the result. + unsigned AliasOpNo = 0; + for (unsigned i = 0, e = ResultInst->Operands.size(); i != e; ++i) { + + // Tied registers don't have an entry in the result dag unless they're part + // of a complex operand, in which case we include them anyways, as we + // don't have any other way to specify the whole operand. + if (ResultInst->Operands[i].MINumOperands == 1 && + ResultInst->Operands[i].getTiedRegister() != -1) { + // Tied operands of different RegisterClass should be explicit within an + // instruction's syntax and so cannot be skipped. + int TiedOpNum = ResultInst->Operands[i].getTiedRegister(); + if (ResultInst->Operands[i].Rec->getName() == + ResultInst->Operands[TiedOpNum].Rec->getName()) + continue; + } + + if (AliasOpNo >= Result->getNumArgs()) + PrintFatalError(R->getLoc(), "not enough arguments for instruction!"); + + Record *InstOpRec = ResultInst->Operands[i].Rec; + unsigned NumSubOps = ResultInst->Operands[i].MINumOperands; + ResultOperand ResOp(static_cast<int64_t>(0)); + if (tryAliasOpMatch(Result, AliasOpNo, InstOpRec, (NumSubOps > 1), + R->getLoc(), T, ResOp)) { + // If this is a simple operand, or a complex operand with a custom match + // class, then we can match is verbatim. + if (NumSubOps == 1 || + (InstOpRec->getValue("ParserMatchClass") && + InstOpRec->getValueAsDef("ParserMatchClass") + ->getValueAsString("Name") != "Imm")) { + ResultOperands.push_back(ResOp); + ResultInstOperandIndex.push_back(std::make_pair(i, -1)); + ++AliasOpNo; + + // Otherwise, we need to match each of the suboperands individually. + } else { + DagInit *MIOI = ResultInst->Operands[i].MIOperandInfo; + for (unsigned SubOp = 0; SubOp != NumSubOps; ++SubOp) { + Record *SubRec = cast<DefInit>(MIOI->getArg(SubOp))->getDef(); + + // Take care to instantiate each of the suboperands with the correct + // nomenclature: $foo.bar + ResultOperands.emplace_back( + Result->getArgName(AliasOpNo)->getAsUnquotedString() + "." + + MIOI->getArgName(SubOp)->getAsUnquotedString(), SubRec); + ResultInstOperandIndex.push_back(std::make_pair(i, SubOp)); + } + ++AliasOpNo; + } + continue; + } + + // If the argument did not match the instruction operand, and the operand + // is composed of multiple suboperands, try matching the suboperands. + if (NumSubOps > 1) { + DagInit *MIOI = ResultInst->Operands[i].MIOperandInfo; + for (unsigned SubOp = 0; SubOp != NumSubOps; ++SubOp) { + if (AliasOpNo >= Result->getNumArgs()) + PrintFatalError(R->getLoc(), "not enough arguments for instruction!"); + Record *SubRec = cast<DefInit>(MIOI->getArg(SubOp))->getDef(); + if (tryAliasOpMatch(Result, AliasOpNo, SubRec, false, + R->getLoc(), T, ResOp)) { + ResultOperands.push_back(ResOp); + ResultInstOperandIndex.push_back(std::make_pair(i, SubOp)); + ++AliasOpNo; + } else { + PrintFatalError(R->getLoc(), "result argument #" + Twine(AliasOpNo) + + " does not match instruction operand class " + + (SubOp == 0 ? InstOpRec->getName() :SubRec->getName())); + } + } + continue; + } + PrintFatalError(R->getLoc(), "result argument #" + Twine(AliasOpNo) + + " does not match instruction operand class " + + InstOpRec->getName()); + } + + if (AliasOpNo != Result->getNumArgs()) + PrintFatalError(R->getLoc(), "too many operands for instruction!"); +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.h new file mode 100644 index 000000000000..af851a11676b --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenInstruction.h @@ -0,0 +1,393 @@ +//===- CodeGenInstruction.h - Instruction Class Wrapper ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines a wrapper class for the 'Instruction' TableGen class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEGENINSTRUCTION_H +#define LLVM_UTILS_TABLEGEN_CODEGENINSTRUCTION_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MachineValueType.h" +#include "llvm/Support/SMLoc.h" +#include <cassert> +#include <string> +#include <utility> +#include <vector> + +namespace llvm { +template <typename T> class ArrayRef; + class Record; + class DagInit; + class CodeGenTarget; + + class CGIOperandList { + public: + class ConstraintInfo { + enum { None, EarlyClobber, Tied } Kind = None; + unsigned OtherTiedOperand = 0; + + public: + ConstraintInfo() = default; + + static ConstraintInfo getEarlyClobber() { + ConstraintInfo I; + I.Kind = EarlyClobber; + I.OtherTiedOperand = 0; + return I; + } + + static ConstraintInfo getTied(unsigned Op) { + ConstraintInfo I; + I.Kind = Tied; + I.OtherTiedOperand = Op; + return I; + } + + bool isNone() const { return Kind == None; } + bool isEarlyClobber() const { return Kind == EarlyClobber; } + bool isTied() const { return Kind == Tied; } + + unsigned getTiedOperand() const { + assert(isTied()); + return OtherTiedOperand; + } + + bool operator==(const ConstraintInfo &RHS) const { + if (Kind != RHS.Kind) + return false; + if (Kind == Tied && OtherTiedOperand != RHS.OtherTiedOperand) + return false; + return true; + } + bool operator!=(const ConstraintInfo &RHS) const { + return !(*this == RHS); + } + }; + + /// OperandInfo - The information we keep track of for each operand in the + /// operand list for a tablegen instruction. + struct OperandInfo { + /// Rec - The definition this operand is declared as. + /// + Record *Rec; + + /// Name - If this operand was assigned a symbolic name, this is it, + /// otherwise, it's empty. + std::string Name; + + /// PrinterMethodName - The method used to print operands of this type in + /// the asmprinter. + std::string PrinterMethodName; + + /// EncoderMethodName - The method used to get the machine operand value + /// for binary encoding. "getMachineOpValue" by default. + std::string EncoderMethodName; + + /// OperandType - A value from MCOI::OperandType representing the type of + /// the operand. + std::string OperandType; + + /// MIOperandNo - Currently (this is meant to be phased out), some logical + /// operands correspond to multiple MachineInstr operands. In the X86 + /// target for example, one address operand is represented as 4 + /// MachineOperands. Because of this, the operand number in the + /// OperandList may not match the MachineInstr operand num. Until it + /// does, this contains the MI operand index of this operand. + unsigned MIOperandNo; + unsigned MINumOperands; // The number of operands. + + /// DoNotEncode - Bools are set to true in this vector for each operand in + /// the DisableEncoding list. These should not be emitted by the code + /// emitter. + std::vector<bool> DoNotEncode; + + /// MIOperandInfo - Default MI operand type. Note an operand may be made + /// up of multiple MI operands. + DagInit *MIOperandInfo; + + /// Constraint info for this operand. This operand can have pieces, so we + /// track constraint info for each. + std::vector<ConstraintInfo> Constraints; + + OperandInfo(Record *R, const std::string &N, const std::string &PMN, + const std::string &EMN, const std::string &OT, unsigned MION, + unsigned MINO, DagInit *MIOI) + : Rec(R), Name(N), PrinterMethodName(PMN), EncoderMethodName(EMN), + OperandType(OT), MIOperandNo(MION), MINumOperands(MINO), + MIOperandInfo(MIOI) {} + + + /// getTiedOperand - If this operand is tied to another one, return the + /// other operand number. Otherwise, return -1. + int getTiedRegister() const { + for (unsigned j = 0, e = Constraints.size(); j != e; ++j) { + const CGIOperandList::ConstraintInfo &CI = Constraints[j]; + if (CI.isTied()) return CI.getTiedOperand(); + } + return -1; + } + }; + + CGIOperandList(Record *D); + + Record *TheDef; // The actual record containing this OperandList. + + /// NumDefs - Number of def operands declared, this is the number of + /// elements in the instruction's (outs) list. + /// + unsigned NumDefs; + + /// OperandList - The list of declared operands, along with their declared + /// type (which is a record). + std::vector<OperandInfo> OperandList; + + // Information gleaned from the operand list. + bool isPredicable; + bool hasOptionalDef; + bool isVariadic; + + // Provide transparent accessors to the operand list. + bool empty() const { return OperandList.empty(); } + unsigned size() const { return OperandList.size(); } + const OperandInfo &operator[](unsigned i) const { return OperandList[i]; } + OperandInfo &operator[](unsigned i) { return OperandList[i]; } + OperandInfo &back() { return OperandList.back(); } + const OperandInfo &back() const { return OperandList.back(); } + + typedef std::vector<OperandInfo>::iterator iterator; + typedef std::vector<OperandInfo>::const_iterator const_iterator; + iterator begin() { return OperandList.begin(); } + const_iterator begin() const { return OperandList.begin(); } + iterator end() { return OperandList.end(); } + const_iterator end() const { return OperandList.end(); } + + /// getOperandNamed - Return the index of the operand with the specified + /// non-empty name. If the instruction does not have an operand with the + /// specified name, abort. + unsigned getOperandNamed(StringRef Name) const; + + /// hasOperandNamed - Query whether the instruction has an operand of the + /// given name. If so, return true and set OpIdx to the index of the + /// operand. Otherwise, return false. + bool hasOperandNamed(StringRef Name, unsigned &OpIdx) const; + + /// ParseOperandName - Parse an operand name like "$foo" or "$foo.bar", + /// where $foo is a whole operand and $foo.bar refers to a suboperand. + /// This aborts if the name is invalid. If AllowWholeOp is true, references + /// to operands with suboperands are allowed, otherwise not. + std::pair<unsigned,unsigned> ParseOperandName(const std::string &Op, + bool AllowWholeOp = true); + + /// getFlattenedOperandNumber - Flatten a operand/suboperand pair into a + /// flat machineinstr operand #. + unsigned getFlattenedOperandNumber(std::pair<unsigned,unsigned> Op) const { + return OperandList[Op.first].MIOperandNo + Op.second; + } + + /// getSubOperandNumber - Unflatten a operand number into an + /// operand/suboperand pair. + std::pair<unsigned,unsigned> getSubOperandNumber(unsigned Op) const { + for (unsigned i = 0; ; ++i) { + assert(i < OperandList.size() && "Invalid flat operand #"); + if (OperandList[i].MIOperandNo+OperandList[i].MINumOperands > Op) + return std::make_pair(i, Op-OperandList[i].MIOperandNo); + } + } + + + /// isFlatOperandNotEmitted - Return true if the specified flat operand # + /// should not be emitted with the code emitter. + bool isFlatOperandNotEmitted(unsigned FlatOpNo) const { + std::pair<unsigned,unsigned> Op = getSubOperandNumber(FlatOpNo); + if (OperandList[Op.first].DoNotEncode.size() > Op.second) + return OperandList[Op.first].DoNotEncode[Op.second]; + return false; + } + + void ProcessDisableEncoding(std::string Value); + }; + + + class CodeGenInstruction { + public: + Record *TheDef; // The actual record defining this instruction. + StringRef Namespace; // The namespace the instruction is in. + + /// AsmString - The format string used to emit a .s file for the + /// instruction. + std::string AsmString; + + /// Operands - This is information about the (ins) and (outs) list specified + /// to the instruction. + CGIOperandList Operands; + + /// ImplicitDefs/ImplicitUses - These are lists of registers that are + /// implicitly defined and used by the instruction. + std::vector<Record*> ImplicitDefs, ImplicitUses; + + // Various boolean values we track for the instruction. + bool isPreISelOpcode : 1; + bool isReturn : 1; + bool isEHScopeReturn : 1; + bool isBranch : 1; + bool isIndirectBranch : 1; + bool isCompare : 1; + bool isMoveImm : 1; + bool isMoveReg : 1; + bool isBitcast : 1; + bool isSelect : 1; + bool isBarrier : 1; + bool isCall : 1; + bool isAdd : 1; + bool isTrap : 1; + bool canFoldAsLoad : 1; + bool mayLoad : 1; + bool mayLoad_Unset : 1; + bool mayStore : 1; + bool mayStore_Unset : 1; + bool mayRaiseFPException : 1; + bool isPredicable : 1; + bool isConvertibleToThreeAddress : 1; + bool isCommutable : 1; + bool isTerminator : 1; + bool isReMaterializable : 1; + bool hasDelaySlot : 1; + bool usesCustomInserter : 1; + bool hasPostISelHook : 1; + bool hasCtrlDep : 1; + bool isNotDuplicable : 1; + bool hasSideEffects : 1; + bool hasSideEffects_Unset : 1; + bool isAsCheapAsAMove : 1; + bool hasExtraSrcRegAllocReq : 1; + bool hasExtraDefRegAllocReq : 1; + bool isCodeGenOnly : 1; + bool isPseudo : 1; + bool isRegSequence : 1; + bool isExtractSubreg : 1; + bool isInsertSubreg : 1; + bool isConvergent : 1; + bool hasNoSchedulingInfo : 1; + bool FastISelShouldIgnore : 1; + bool hasChain : 1; + bool hasChain_Inferred : 1; + bool variadicOpsAreDefs : 1; + bool isAuthenticated : 1; + + std::string DeprecatedReason; + bool HasComplexDeprecationPredicate; + + /// Are there any undefined flags? + bool hasUndefFlags() const { + return mayLoad_Unset || mayStore_Unset || hasSideEffects_Unset; + } + + // The record used to infer instruction flags, or NULL if no flag values + // have been inferred. + Record *InferredFrom; + + CodeGenInstruction(Record *R); + + /// HasOneImplicitDefWithKnownVT - If the instruction has at least one + /// implicit def and it has a known VT, return the VT, otherwise return + /// MVT::Other. + MVT::SimpleValueType + HasOneImplicitDefWithKnownVT(const CodeGenTarget &TargetInfo) const; + + + /// FlattenAsmStringVariants - Flatten the specified AsmString to only + /// include text from the specified variant, returning the new string. + static std::string FlattenAsmStringVariants(StringRef AsmString, + unsigned Variant); + + // Is the specified operand in a generic instruction implicitly a pointer. + // This can be used on intructions that use typeN or ptypeN to identify + // operands that should be considered as pointers even though SelectionDAG + // didn't make a distinction between integer and pointers. + bool isOperandAPointer(unsigned i) const { + return isOperandImpl(i, "IsPointer"); + } + + /// Check if the operand is required to be an immediate. + bool isOperandImmArg(unsigned i) const { + return isOperandImpl(i, "IsImmediate"); + } + + private: + bool isOperandImpl(unsigned i, StringRef PropertyName) const; + }; + + + /// CodeGenInstAlias - This represents an InstAlias definition. + class CodeGenInstAlias { + public: + Record *TheDef; // The actual record defining this InstAlias. + + /// AsmString - The format string used to emit a .s file for the + /// instruction. + std::string AsmString; + + /// Result - The result instruction. + DagInit *Result; + + /// ResultInst - The instruction generated by the alias (decoded from + /// Result). + CodeGenInstruction *ResultInst; + + + struct ResultOperand { + private: + std::string Name; + Record *R = nullptr; + int64_t Imm = 0; + + public: + enum { + K_Record, + K_Imm, + K_Reg + } Kind; + + ResultOperand(std::string N, Record *r) + : Name(std::move(N)), R(r), Kind(K_Record) {} + ResultOperand(int64_t I) : Imm(I), Kind(K_Imm) {} + ResultOperand(Record *r) : R(r), Kind(K_Reg) {} + + bool isRecord() const { return Kind == K_Record; } + bool isImm() const { return Kind == K_Imm; } + bool isReg() const { return Kind == K_Reg; } + + StringRef getName() const { assert(isRecord()); return Name; } + Record *getRecord() const { assert(isRecord()); return R; } + int64_t getImm() const { assert(isImm()); return Imm; } + Record *getRegister() const { assert(isReg()); return R; } + + unsigned getMINumOperands() const; + }; + + /// ResultOperands - The decoded operands for the result instruction. + std::vector<ResultOperand> ResultOperands; + + /// ResultInstOperandIndex - For each operand, this vector holds a pair of + /// indices to identify the corresponding operand in the result + /// instruction. The first index specifies the operand and the second + /// index specifies the suboperand. If there are no suboperands or if all + /// of them are matched by the operand, the second value should be -1. + std::vector<std::pair<unsigned, int> > ResultInstOperandIndex; + + CodeGenInstAlias(Record *R, CodeGenTarget &T); + + bool tryAliasOpMatch(DagInit *Result, unsigned AliasOpNo, + Record *InstOpRec, bool hasSubOps, ArrayRef<SMLoc> Loc, + CodeGenTarget &T, ResultOperand &ResOp); + }; +} + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenIntrinsics.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenIntrinsics.h new file mode 100644 index 000000000000..af59c1f3d833 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenIntrinsics.h @@ -0,0 +1,214 @@ +//===- CodeGenIntrinsic.h - Intrinsic Class Wrapper ------------*- C++ -*--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines a wrapper class for the 'Intrinsic' TableGen class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEGENINTRINSICS_H +#define LLVM_UTILS_TABLEGEN_CODEGENINTRINSICS_H + +#include "SDNodeProperties.h" +#include "llvm/Support/MachineValueType.h" +#include <string> +#include <vector> + +namespace llvm { +class Record; +class RecordKeeper; +class CodeGenTarget; + +struct CodeGenIntrinsic { + Record *TheDef; // The actual record defining this intrinsic. + std::string Name; // The name of the LLVM function "llvm.bswap.i32" + std::string EnumName; // The name of the enum "bswap_i32" + std::string GCCBuiltinName; // Name of the corresponding GCC builtin, or "". + std::string MSBuiltinName; // Name of the corresponding MS builtin, or "". + std::string TargetPrefix; // Target prefix, e.g. "ppc" for t-s intrinsics. + + /// This structure holds the return values and parameter values of an + /// intrinsic. If the number of return values is > 1, then the intrinsic + /// implicitly returns a first-class aggregate. The numbering of the types + /// starts at 0 with the first return value and continues from there through + /// the parameter list. This is useful for "matching" types. + struct IntrinsicSignature { + /// The MVT::SimpleValueType for each return type. Note that this list is + /// only populated when in the context of a target .td file. When building + /// Intrinsics.td, this isn't available, because we don't know the target + /// pointer size. + std::vector<MVT::SimpleValueType> RetVTs; + + /// The records for each return type. + std::vector<Record *> RetTypeDefs; + + /// The MVT::SimpleValueType for each parameter type. Note that this list is + /// only populated when in the context of a target .td file. When building + /// Intrinsics.td, this isn't available, because we don't know the target + /// pointer size. + std::vector<MVT::SimpleValueType> ParamVTs; + + /// The records for each parameter type. + std::vector<Record *> ParamTypeDefs; + }; + + IntrinsicSignature IS; + + /// Bit flags describing the type (ref/mod) and location of memory + /// accesses that may be performed by the intrinsics. Analogous to + /// \c FunctionModRefBehaviour. + enum ModRefBits { + /// The intrinsic may access memory that is otherwise inaccessible via + /// LLVM IR. + MR_InaccessibleMem = 1, + + /// The intrinsic may access memory through pointer arguments. + /// LLVM IR. + MR_ArgMem = 2, + + /// The intrinsic may access memory anywhere, i.e. it is not restricted + /// to access through pointer arguments. + MR_Anywhere = 4 | MR_ArgMem | MR_InaccessibleMem, + + /// The intrinsic may read memory. + MR_Ref = 8, + + /// The intrinsic may write memory. + MR_Mod = 16, + + /// The intrinsic may both read and write memory. + MR_ModRef = MR_Ref | MR_Mod, + }; + + /// Memory mod/ref behavior of this intrinsic, corresponding to intrinsic + /// properties (IntrReadMem, IntrArgMemOnly, etc.). + enum ModRefBehavior { + NoMem = 0, + ReadArgMem = MR_Ref | MR_ArgMem, + ReadInaccessibleMem = MR_Ref | MR_InaccessibleMem, + ReadInaccessibleMemOrArgMem = MR_Ref | MR_ArgMem | MR_InaccessibleMem, + ReadMem = MR_Ref | MR_Anywhere, + WriteArgMem = MR_Mod | MR_ArgMem, + WriteInaccessibleMem = MR_Mod | MR_InaccessibleMem, + WriteInaccessibleMemOrArgMem = MR_Mod | MR_ArgMem | MR_InaccessibleMem, + WriteMem = MR_Mod | MR_Anywhere, + ReadWriteArgMem = MR_ModRef | MR_ArgMem, + ReadWriteInaccessibleMem = MR_ModRef | MR_InaccessibleMem, + ReadWriteInaccessibleMemOrArgMem = MR_ModRef | MR_ArgMem | + MR_InaccessibleMem, + ReadWriteMem = MR_ModRef | MR_Anywhere, + }; + ModRefBehavior ModRef; + + /// SDPatternOperator Properties applied to the intrinsic. + unsigned Properties; + + /// This is set to true if the intrinsic is overloaded by its argument + /// types. + bool isOverloaded; + + /// True if the intrinsic is commutative. + bool isCommutative; + + /// True if the intrinsic can throw. + bool canThrow; + + /// True if the intrinsic is marked as noduplicate. + bool isNoDuplicate; + + /// True if the intrinsic is no-return. + bool isNoReturn; + + /// True if the intrinsic is no-sync. + bool isNoSync; + + /// True if the intrinsic is no-free. + bool isNoFree; + + /// True if the intrinsic is will-return. + bool isWillReturn; + + /// True if the intrinsic is cold. + bool isCold; + + /// True if the intrinsic is marked as convergent. + bool isConvergent; + + /// True if the intrinsic has side effects that aren't captured by any + /// of the other flags. + bool hasSideEffects; + + // True if the intrinsic is marked as speculatable. + bool isSpeculatable; + + enum ArgAttrKind { + NoCapture, + NoAlias, + Returned, + ReadOnly, + WriteOnly, + ReadNone, + ImmArg, + Alignment + }; + + struct ArgAttribute { + unsigned Index; + ArgAttrKind Kind; + uint64_t Value; + + ArgAttribute(unsigned Idx, ArgAttrKind K, uint64_t V) + : Index(Idx), Kind(K), Value(V) {} + + bool operator<(const ArgAttribute &Other) const { + return std::tie(Index, Kind, Value) < + std::tie(Other.Index, Other.Kind, Other.Value); + } + }; + + std::vector<ArgAttribute> ArgumentAttributes; + + bool hasProperty(enum SDNP Prop) const { + return Properties & (1 << Prop); + } + + /// Returns true if the parameter at \p ParamIdx is a pointer type. Returns + /// false if the parameter is not a pointer, or \p ParamIdx is greater than + /// the size of \p IS.ParamVTs. + /// + /// Note that this requires that \p IS.ParamVTs is available. + bool isParamAPointer(unsigned ParamIdx) const; + + bool isParamImmArg(unsigned ParamIdx) const; + + CodeGenIntrinsic(Record *R); +}; + +class CodeGenIntrinsicTable { + std::vector<CodeGenIntrinsic> Intrinsics; + +public: + struct TargetSet { + std::string Name; + size_t Offset; + size_t Count; + }; + std::vector<TargetSet> Targets; + + explicit CodeGenIntrinsicTable(const RecordKeeper &RC); + CodeGenIntrinsicTable() = default; + + bool empty() const { return Intrinsics.empty(); } + size_t size() const { return Intrinsics.size(); } + CodeGenIntrinsic &operator[](size_t Pos) { return Intrinsics[Pos]; } + const CodeGenIntrinsic &operator[](size_t Pos) const { + return Intrinsics[Pos]; + } +}; +} + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenMapTable.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenMapTable.cpp new file mode 100644 index 000000000000..baca0768b26b --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenMapTable.cpp @@ -0,0 +1,612 @@ +//===- CodeGenMapTable.cpp - Instruction Mapping Table Generator ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// CodeGenMapTable provides functionality for the TabelGen to create +// relation mapping between instructions. Relation models are defined using +// InstrMapping as a base class. This file implements the functionality which +// parses these definitions and generates relation maps using the information +// specified there. These maps are emitted as tables in the XXXGenInstrInfo.inc +// file along with the functions to query them. +// +// A relationship model to relate non-predicate instructions with their +// predicated true/false forms can be defined as follows: +// +// def getPredOpcode : InstrMapping { +// let FilterClass = "PredRel"; +// let RowFields = ["BaseOpcode"]; +// let ColFields = ["PredSense"]; +// let KeyCol = ["none"]; +// let ValueCols = [["true"], ["false"]]; } +// +// CodeGenMapTable parses this map and generates a table in XXXGenInstrInfo.inc +// file that contains the instructions modeling this relationship. This table +// is defined in the function +// "int getPredOpcode(uint16_t Opcode, enum PredSense inPredSense)" +// that can be used to retrieve the predicated form of the instruction by +// passing its opcode value and the predicate sense (true/false) of the desired +// instruction as arguments. +// +// Short description of the algorithm: +// +// 1) Iterate through all the records that derive from "InstrMapping" class. +// 2) For each record, filter out instructions based on the FilterClass value. +// 3) Iterate through this set of instructions and insert them into +// RowInstrMap map based on their RowFields values. RowInstrMap is keyed by the +// vector of RowFields values and contains vectors of Records (instructions) as +// values. RowFields is a list of fields that are required to have the same +// values for all the instructions appearing in the same row of the relation +// table. All the instructions in a given row of the relation table have some +// sort of relationship with the key instruction defined by the corresponding +// relationship model. +// +// Ex: RowInstrMap(RowVal1, RowVal2, ...) -> [Instr1, Instr2, Instr3, ... ] +// Here Instr1, Instr2, Instr3 have same values (RowVal1, RowVal2) for +// RowFields. These groups of instructions are later matched against ValueCols +// to determine the column they belong to, if any. +// +// While building the RowInstrMap map, collect all the key instructions in +// KeyInstrVec. These are the instructions having the same values as KeyCol +// for all the fields listed in ColFields. +// +// For Example: +// +// Relate non-predicate instructions with their predicated true/false forms. +// +// def getPredOpcode : InstrMapping { +// let FilterClass = "PredRel"; +// let RowFields = ["BaseOpcode"]; +// let ColFields = ["PredSense"]; +// let KeyCol = ["none"]; +// let ValueCols = [["true"], ["false"]]; } +// +// Here, only instructions that have "none" as PredSense will be selected as key +// instructions. +// +// 4) For each key instruction, get the group of instructions that share the +// same key-value as the key instruction from RowInstrMap. Iterate over the list +// of columns in ValueCols (it is defined as a list<list<string> >. Therefore, +// it can specify multi-column relationships). For each column, find the +// instruction from the group that matches all the values for the column. +// Multiple matches are not allowed. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "llvm/Support/Format.h" +#include "llvm/TableGen/Error.h" +using namespace llvm; +typedef std::map<std::string, std::vector<Record*> > InstrRelMapTy; + +typedef std::map<std::vector<Init*>, std::vector<Record*> > RowInstrMapTy; + +namespace { + +//===----------------------------------------------------------------------===// +// This class is used to represent InstrMapping class defined in Target.td file. +class InstrMap { +private: + std::string Name; + std::string FilterClass; + ListInit *RowFields; + ListInit *ColFields; + ListInit *KeyCol; + std::vector<ListInit*> ValueCols; + +public: + InstrMap(Record* MapRec) { + Name = std::string(MapRec->getName()); + + // FilterClass - It's used to reduce the search space only to the + // instructions that define the kind of relationship modeled by + // this InstrMapping object/record. + const RecordVal *Filter = MapRec->getValue("FilterClass"); + FilterClass = Filter->getValue()->getAsUnquotedString(); + + // List of fields/attributes that need to be same across all the + // instructions in a row of the relation table. + RowFields = MapRec->getValueAsListInit("RowFields"); + + // List of fields/attributes that are constant across all the instruction + // in a column of the relation table. Ex: ColFields = 'predSense' + ColFields = MapRec->getValueAsListInit("ColFields"); + + // Values for the fields/attributes listed in 'ColFields'. + // Ex: KeyCol = 'noPred' -- key instruction is non-predicated + KeyCol = MapRec->getValueAsListInit("KeyCol"); + + // List of values for the fields/attributes listed in 'ColFields', one for + // each column in the relation table. + // + // Ex: ValueCols = [['true'],['false']] -- it results two columns in the + // table. First column requires all the instructions to have predSense + // set to 'true' and second column requires it to be 'false'. + ListInit *ColValList = MapRec->getValueAsListInit("ValueCols"); + + // Each instruction map must specify at least one column for it to be valid. + if (ColValList->empty()) + PrintFatalError(MapRec->getLoc(), "InstrMapping record `" + + MapRec->getName() + "' has empty " + "`ValueCols' field!"); + + for (Init *I : ColValList->getValues()) { + auto *ColI = cast<ListInit>(I); + + // Make sure that all the sub-lists in 'ValueCols' have same number of + // elements as the fields in 'ColFields'. + if (ColI->size() != ColFields->size()) + PrintFatalError(MapRec->getLoc(), "Record `" + MapRec->getName() + + "', field `ValueCols' entries don't match with " + + " the entries in 'ColFields'!"); + ValueCols.push_back(ColI); + } + } + + std::string getName() const { + return Name; + } + + std::string getFilterClass() { + return FilterClass; + } + + ListInit *getRowFields() const { + return RowFields; + } + + ListInit *getColFields() const { + return ColFields; + } + + ListInit *getKeyCol() const { + return KeyCol; + } + + const std::vector<ListInit*> &getValueCols() const { + return ValueCols; + } +}; +} // end anonymous namespace + + +//===----------------------------------------------------------------------===// +// class MapTableEmitter : It builds the instruction relation maps using +// the information provided in InstrMapping records. It outputs these +// relationship maps as tables into XXXGenInstrInfo.inc file along with the +// functions to query them. + +namespace { +class MapTableEmitter { +private: +// std::string TargetName; + const CodeGenTarget &Target; + // InstrMapDesc - InstrMapping record to be processed. + InstrMap InstrMapDesc; + + // InstrDefs - list of instructions filtered using FilterClass defined + // in InstrMapDesc. + std::vector<Record*> InstrDefs; + + // RowInstrMap - maps RowFields values to the instructions. It's keyed by the + // values of the row fields and contains vector of records as values. + RowInstrMapTy RowInstrMap; + + // KeyInstrVec - list of key instructions. + std::vector<Record*> KeyInstrVec; + DenseMap<Record*, std::vector<Record*> > MapTable; + +public: + MapTableEmitter(CodeGenTarget &Target, RecordKeeper &Records, Record *IMRec): + Target(Target), InstrMapDesc(IMRec) { + const std::string FilterClass = InstrMapDesc.getFilterClass(); + InstrDefs = Records.getAllDerivedDefinitions(FilterClass); + } + + void buildRowInstrMap(); + + // Returns true if an instruction is a key instruction, i.e., its ColFields + // have same values as KeyCol. + bool isKeyColInstr(Record* CurInstr); + + // Find column instruction corresponding to a key instruction based on the + // constraints for that column. + Record *getInstrForColumn(Record *KeyInstr, ListInit *CurValueCol); + + // Find column instructions for each key instruction based + // on ValueCols and store them into MapTable. + void buildMapTable(); + + void emitBinSearch(raw_ostream &OS, unsigned TableSize); + void emitTablesWithFunc(raw_ostream &OS); + unsigned emitBinSearchTable(raw_ostream &OS); + + // Lookup functions to query binary search tables. + void emitMapFuncBody(raw_ostream &OS, unsigned TableSize); + +}; +} // end anonymous namespace + + +//===----------------------------------------------------------------------===// +// Process all the instructions that model this relation (alreday present in +// InstrDefs) and insert them into RowInstrMap which is keyed by the values of +// the fields listed as RowFields. It stores vectors of records as values. +// All the related instructions have the same values for the RowFields thus are +// part of the same key-value pair. +//===----------------------------------------------------------------------===// + +void MapTableEmitter::buildRowInstrMap() { + for (Record *CurInstr : InstrDefs) { + std::vector<Init*> KeyValue; + ListInit *RowFields = InstrMapDesc.getRowFields(); + for (Init *RowField : RowFields->getValues()) { + RecordVal *RecVal = CurInstr->getValue(RowField); + if (RecVal == nullptr) + PrintFatalError(CurInstr->getLoc(), "No value " + + RowField->getAsString() + " found in \"" + + CurInstr->getName() + "\" instruction description."); + Init *CurInstrVal = RecVal->getValue(); + KeyValue.push_back(CurInstrVal); + } + + // Collect key instructions into KeyInstrVec. Later, these instructions are + // processed to assign column position to the instructions sharing + // their KeyValue in RowInstrMap. + if (isKeyColInstr(CurInstr)) + KeyInstrVec.push_back(CurInstr); + + RowInstrMap[KeyValue].push_back(CurInstr); + } +} + +//===----------------------------------------------------------------------===// +// Return true if an instruction is a KeyCol instruction. +//===----------------------------------------------------------------------===// + +bool MapTableEmitter::isKeyColInstr(Record* CurInstr) { + ListInit *ColFields = InstrMapDesc.getColFields(); + ListInit *KeyCol = InstrMapDesc.getKeyCol(); + + // Check if the instruction is a KeyCol instruction. + bool MatchFound = true; + for (unsigned j = 0, endCF = ColFields->size(); + (j < endCF) && MatchFound; j++) { + RecordVal *ColFieldName = CurInstr->getValue(ColFields->getElement(j)); + std::string CurInstrVal = ColFieldName->getValue()->getAsUnquotedString(); + std::string KeyColValue = KeyCol->getElement(j)->getAsUnquotedString(); + MatchFound = (CurInstrVal == KeyColValue); + } + return MatchFound; +} + +//===----------------------------------------------------------------------===// +// Build a map to link key instructions with the column instructions arranged +// according to their column positions. +//===----------------------------------------------------------------------===// + +void MapTableEmitter::buildMapTable() { + // Find column instructions for a given key based on the ColField + // constraints. + const std::vector<ListInit*> &ValueCols = InstrMapDesc.getValueCols(); + unsigned NumOfCols = ValueCols.size(); + for (Record *CurKeyInstr : KeyInstrVec) { + std::vector<Record*> ColInstrVec(NumOfCols); + + // Find the column instruction based on the constraints for the column. + for (unsigned ColIdx = 0; ColIdx < NumOfCols; ColIdx++) { + ListInit *CurValueCol = ValueCols[ColIdx]; + Record *ColInstr = getInstrForColumn(CurKeyInstr, CurValueCol); + ColInstrVec[ColIdx] = ColInstr; + } + MapTable[CurKeyInstr] = ColInstrVec; + } +} + +//===----------------------------------------------------------------------===// +// Find column instruction based on the constraints for that column. +//===----------------------------------------------------------------------===// + +Record *MapTableEmitter::getInstrForColumn(Record *KeyInstr, + ListInit *CurValueCol) { + ListInit *RowFields = InstrMapDesc.getRowFields(); + std::vector<Init*> KeyValue; + + // Construct KeyValue using KeyInstr's values for RowFields. + for (Init *RowField : RowFields->getValues()) { + Init *KeyInstrVal = KeyInstr->getValue(RowField)->getValue(); + KeyValue.push_back(KeyInstrVal); + } + + // Get all the instructions that share the same KeyValue as the KeyInstr + // in RowInstrMap. We search through these instructions to find a match + // for the current column, i.e., the instruction which has the same values + // as CurValueCol for all the fields in ColFields. + const std::vector<Record*> &RelatedInstrVec = RowInstrMap[KeyValue]; + + ListInit *ColFields = InstrMapDesc.getColFields(); + Record *MatchInstr = nullptr; + + for (unsigned i = 0, e = RelatedInstrVec.size(); i < e; i++) { + bool MatchFound = true; + Record *CurInstr = RelatedInstrVec[i]; + for (unsigned j = 0, endCF = ColFields->size(); + (j < endCF) && MatchFound; j++) { + Init *ColFieldJ = ColFields->getElement(j); + Init *CurInstrInit = CurInstr->getValue(ColFieldJ)->getValue(); + std::string CurInstrVal = CurInstrInit->getAsUnquotedString(); + Init *ColFieldJVallue = CurValueCol->getElement(j); + MatchFound = (CurInstrVal == ColFieldJVallue->getAsUnquotedString()); + } + + if (MatchFound) { + if (MatchInstr) { + // Already had a match + // Error if multiple matches are found for a column. + std::string KeyValueStr; + for (Init *Value : KeyValue) { + if (!KeyValueStr.empty()) + KeyValueStr += ", "; + KeyValueStr += Value->getAsString(); + } + + PrintFatalError("Multiple matches found for `" + KeyInstr->getName() + + "', for the relation `" + InstrMapDesc.getName() + "', row fields [" + + KeyValueStr + "], column `" + CurValueCol->getAsString() + "'"); + } + MatchInstr = CurInstr; + } + } + return MatchInstr; +} + +//===----------------------------------------------------------------------===// +// Emit one table per relation. Only instructions with a valid relation of a +// given type are included in the table sorted by their enum values (opcodes). +// Binary search is used for locating instructions in the table. +//===----------------------------------------------------------------------===// + +unsigned MapTableEmitter::emitBinSearchTable(raw_ostream &OS) { + + ArrayRef<const CodeGenInstruction*> NumberedInstructions = + Target.getInstructionsByEnumValue(); + StringRef Namespace = Target.getInstNamespace(); + const std::vector<ListInit*> &ValueCols = InstrMapDesc.getValueCols(); + unsigned NumCol = ValueCols.size(); + unsigned TotalNumInstr = NumberedInstructions.size(); + unsigned TableSize = 0; + + OS << "static const uint16_t "<<InstrMapDesc.getName(); + // Number of columns in the table are NumCol+1 because key instructions are + // emitted as first column. + OS << "Table[]["<< NumCol+1 << "] = {\n"; + for (unsigned i = 0; i < TotalNumInstr; i++) { + Record *CurInstr = NumberedInstructions[i]->TheDef; + std::vector<Record*> ColInstrs = MapTable[CurInstr]; + std::string OutStr(""); + unsigned RelExists = 0; + if (!ColInstrs.empty()) { + for (unsigned j = 0; j < NumCol; j++) { + if (ColInstrs[j] != nullptr) { + RelExists = 1; + OutStr += ", "; + OutStr += Namespace; + OutStr += "::"; + OutStr += ColInstrs[j]->getName(); + } else { OutStr += ", (uint16_t)-1U";} + } + + if (RelExists) { + OS << " { " << Namespace << "::" << CurInstr->getName(); + OS << OutStr <<" },\n"; + TableSize++; + } + } + } + if (!TableSize) { + OS << " { " << Namespace << "::" << "INSTRUCTION_LIST_END, "; + OS << Namespace << "::" << "INSTRUCTION_LIST_END }"; + } + OS << "}; // End of " << InstrMapDesc.getName() << "Table\n\n"; + return TableSize; +} + +//===----------------------------------------------------------------------===// +// Emit binary search algorithm as part of the functions used to query +// relation tables. +//===----------------------------------------------------------------------===// + +void MapTableEmitter::emitBinSearch(raw_ostream &OS, unsigned TableSize) { + OS << " unsigned mid;\n"; + OS << " unsigned start = 0;\n"; + OS << " unsigned end = " << TableSize << ";\n"; + OS << " while (start < end) {\n"; + OS << " mid = start + (end - start)/2;\n"; + OS << " if (Opcode == " << InstrMapDesc.getName() << "Table[mid][0]) {\n"; + OS << " break;\n"; + OS << " }\n"; + OS << " if (Opcode < " << InstrMapDesc.getName() << "Table[mid][0])\n"; + OS << " end = mid;\n"; + OS << " else\n"; + OS << " start = mid + 1;\n"; + OS << " }\n"; + OS << " if (start == end)\n"; + OS << " return -1; // Instruction doesn't exist in this table.\n\n"; +} + +//===----------------------------------------------------------------------===// +// Emit functions to query relation tables. +//===----------------------------------------------------------------------===// + +void MapTableEmitter::emitMapFuncBody(raw_ostream &OS, + unsigned TableSize) { + + ListInit *ColFields = InstrMapDesc.getColFields(); + const std::vector<ListInit*> &ValueCols = InstrMapDesc.getValueCols(); + + // Emit binary search algorithm to locate instructions in the + // relation table. If found, return opcode value from the appropriate column + // of the table. + emitBinSearch(OS, TableSize); + + if (ValueCols.size() > 1) { + for (unsigned i = 0, e = ValueCols.size(); i < e; i++) { + ListInit *ColumnI = ValueCols[i]; + for (unsigned j = 0, ColSize = ColumnI->size(); j < ColSize; ++j) { + std::string ColName = ColFields->getElement(j)->getAsUnquotedString(); + OS << " if (in" << ColName; + OS << " == "; + OS << ColName << "_" << ColumnI->getElement(j)->getAsUnquotedString(); + if (j < ColumnI->size() - 1) OS << " && "; + else OS << ")\n"; + } + OS << " return " << InstrMapDesc.getName(); + OS << "Table[mid]["<<i+1<<"];\n"; + } + OS << " return -1;"; + } + else + OS << " return " << InstrMapDesc.getName() << "Table[mid][1];\n"; + + OS <<"}\n\n"; +} + +//===----------------------------------------------------------------------===// +// Emit relation tables and the functions to query them. +//===----------------------------------------------------------------------===// + +void MapTableEmitter::emitTablesWithFunc(raw_ostream &OS) { + + // Emit function name and the input parameters : mostly opcode value of the + // current instruction. However, if a table has multiple columns (more than 2 + // since first column is used for the key instructions), then we also need + // to pass another input to indicate the column to be selected. + + ListInit *ColFields = InstrMapDesc.getColFields(); + const std::vector<ListInit*> &ValueCols = InstrMapDesc.getValueCols(); + OS << "// "<< InstrMapDesc.getName() << "\nLLVM_READONLY\n"; + OS << "int "<< InstrMapDesc.getName() << "(uint16_t Opcode"; + if (ValueCols.size() > 1) { + for (Init *CF : ColFields->getValues()) { + std::string ColName = CF->getAsUnquotedString(); + OS << ", enum " << ColName << " in" << ColName << ") {\n"; + } + } else { OS << ") {\n"; } + + // Emit map table. + unsigned TableSize = emitBinSearchTable(OS); + + // Emit rest of the function body. + emitMapFuncBody(OS, TableSize); +} + +//===----------------------------------------------------------------------===// +// Emit enums for the column fields across all the instruction maps. +//===----------------------------------------------------------------------===// + +static void emitEnums(raw_ostream &OS, RecordKeeper &Records) { + + std::vector<Record*> InstrMapVec; + InstrMapVec = Records.getAllDerivedDefinitions("InstrMapping"); + std::map<std::string, std::vector<Init*> > ColFieldValueMap; + + // Iterate over all InstrMapping records and create a map between column + // fields and their possible values across all records. + for (Record *CurMap : InstrMapVec) { + ListInit *ColFields; + ColFields = CurMap->getValueAsListInit("ColFields"); + ListInit *List = CurMap->getValueAsListInit("ValueCols"); + std::vector<ListInit*> ValueCols; + unsigned ListSize = List->size(); + + for (unsigned j = 0; j < ListSize; j++) { + auto *ListJ = cast<ListInit>(List->getElement(j)); + + if (ListJ->size() != ColFields->size()) + PrintFatalError("Record `" + CurMap->getName() + "', field " + "`ValueCols' entries don't match with the entries in 'ColFields' !"); + ValueCols.push_back(ListJ); + } + + for (unsigned j = 0, endCF = ColFields->size(); j < endCF; j++) { + for (unsigned k = 0; k < ListSize; k++){ + std::string ColName = ColFields->getElement(j)->getAsUnquotedString(); + ColFieldValueMap[ColName].push_back((ValueCols[k])->getElement(j)); + } + } + } + + for (auto &Entry : ColFieldValueMap) { + std::vector<Init*> FieldValues = Entry.second; + + // Delete duplicate entries from ColFieldValueMap + for (unsigned i = 0; i < FieldValues.size() - 1; i++) { + Init *CurVal = FieldValues[i]; + for (unsigned j = i+1; j < FieldValues.size(); j++) { + if (CurVal == FieldValues[j]) { + FieldValues.erase(FieldValues.begin()+j); + --j; + } + } + } + + // Emit enumerated values for the column fields. + OS << "enum " << Entry.first << " {\n"; + for (unsigned i = 0, endFV = FieldValues.size(); i < endFV; i++) { + OS << "\t" << Entry.first << "_" << FieldValues[i]->getAsUnquotedString(); + if (i != endFV - 1) + OS << ",\n"; + else + OS << "\n};\n\n"; + } + } +} + +namespace llvm { +//===----------------------------------------------------------------------===// +// Parse 'InstrMapping' records and use the information to form relationship +// between instructions. These relations are emitted as a tables along with the +// functions to query them. +//===----------------------------------------------------------------------===// +void EmitMapTable(RecordKeeper &Records, raw_ostream &OS) { + CodeGenTarget Target(Records); + StringRef NameSpace = Target.getInstNamespace(); + std::vector<Record*> InstrMapVec; + InstrMapVec = Records.getAllDerivedDefinitions("InstrMapping"); + + if (InstrMapVec.empty()) + return; + + OS << "#ifdef GET_INSTRMAP_INFO\n"; + OS << "#undef GET_INSTRMAP_INFO\n"; + OS << "namespace llvm {\n\n"; + OS << "namespace " << NameSpace << " {\n\n"; + + // Emit coulumn field names and their values as enums. + emitEnums(OS, Records); + + // Iterate over all instruction mapping records and construct relationship + // maps based on the information specified there. + // + for (Record *CurMap : InstrMapVec) { + MapTableEmitter IMap(Target, Records, CurMap); + + // Build RowInstrMap to group instructions based on their values for + // RowFields. In the process, also collect key instructions into + // KeyInstrVec. + IMap.buildRowInstrMap(); + + // Build MapTable to map key instructions with the corresponding column + // instructions. + IMap.buildMapTable(); + + // Emit map tables and the functions to query them. + IMap.emitTablesWithFunc(OS); + } + OS << "} // end namespace " << NameSpace << "\n"; + OS << "} // end namespace llvm\n"; + OS << "#endif // GET_INSTRMAP_INFO\n\n"; +} + +} // End llvm namespace diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.cpp new file mode 100644 index 000000000000..4584bc7cfae3 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.cpp @@ -0,0 +1,2449 @@ +//===- CodeGenRegisters.cpp - Register and RegisterClass Info -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines structures to encapsulate information gleaned from the +// target register and register class definitions. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenRegisters.h" +#include "CodeGenTarget.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/IntEqClasses.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <iterator> +#include <map> +#include <queue> +#include <set> +#include <string> +#include <tuple> +#include <utility> +#include <vector> + +using namespace llvm; + +#define DEBUG_TYPE "regalloc-emitter" + +//===----------------------------------------------------------------------===// +// CodeGenSubRegIndex +//===----------------------------------------------------------------------===// + +CodeGenSubRegIndex::CodeGenSubRegIndex(Record *R, unsigned Enum) + : TheDef(R), EnumValue(Enum), AllSuperRegsCovered(true), Artificial(true) { + Name = std::string(R->getName()); + if (R->getValue("Namespace")) + Namespace = std::string(R->getValueAsString("Namespace")); + Size = R->getValueAsInt("Size"); + Offset = R->getValueAsInt("Offset"); +} + +CodeGenSubRegIndex::CodeGenSubRegIndex(StringRef N, StringRef Nspace, + unsigned Enum) + : TheDef(nullptr), Name(std::string(N)), Namespace(std::string(Nspace)), + Size(-1), Offset(-1), EnumValue(Enum), AllSuperRegsCovered(true), + Artificial(true) {} + +std::string CodeGenSubRegIndex::getQualifiedName() const { + std::string N = getNamespace(); + if (!N.empty()) + N += "::"; + N += getName(); + return N; +} + +void CodeGenSubRegIndex::updateComponents(CodeGenRegBank &RegBank) { + if (!TheDef) + return; + + std::vector<Record*> Comps = TheDef->getValueAsListOfDefs("ComposedOf"); + if (!Comps.empty()) { + if (Comps.size() != 2) + PrintFatalError(TheDef->getLoc(), + "ComposedOf must have exactly two entries"); + CodeGenSubRegIndex *A = RegBank.getSubRegIdx(Comps[0]); + CodeGenSubRegIndex *B = RegBank.getSubRegIdx(Comps[1]); + CodeGenSubRegIndex *X = A->addComposite(B, this); + if (X) + PrintFatalError(TheDef->getLoc(), "Ambiguous ComposedOf entries"); + } + + std::vector<Record*> Parts = + TheDef->getValueAsListOfDefs("CoveringSubRegIndices"); + if (!Parts.empty()) { + if (Parts.size() < 2) + PrintFatalError(TheDef->getLoc(), + "CoveredBySubRegs must have two or more entries"); + SmallVector<CodeGenSubRegIndex*, 8> IdxParts; + for (Record *Part : Parts) + IdxParts.push_back(RegBank.getSubRegIdx(Part)); + setConcatenationOf(IdxParts); + } +} + +LaneBitmask CodeGenSubRegIndex::computeLaneMask() const { + // Already computed? + if (LaneMask.any()) + return LaneMask; + + // Recursion guard, shouldn't be required. + LaneMask = LaneBitmask::getAll(); + + // The lane mask is simply the union of all sub-indices. + LaneBitmask M; + for (const auto &C : Composed) + M |= C.second->computeLaneMask(); + assert(M.any() && "Missing lane mask, sub-register cycle?"); + LaneMask = M; + return LaneMask; +} + +void CodeGenSubRegIndex::setConcatenationOf( + ArrayRef<CodeGenSubRegIndex*> Parts) { + if (ConcatenationOf.empty()) + ConcatenationOf.assign(Parts.begin(), Parts.end()); + else + assert(std::equal(Parts.begin(), Parts.end(), + ConcatenationOf.begin()) && "parts consistent"); +} + +void CodeGenSubRegIndex::computeConcatTransitiveClosure() { + for (SmallVectorImpl<CodeGenSubRegIndex*>::iterator + I = ConcatenationOf.begin(); I != ConcatenationOf.end(); /*empty*/) { + CodeGenSubRegIndex *SubIdx = *I; + SubIdx->computeConcatTransitiveClosure(); +#ifndef NDEBUG + for (CodeGenSubRegIndex *SRI : SubIdx->ConcatenationOf) + assert(SRI->ConcatenationOf.empty() && "No transitive closure?"); +#endif + + if (SubIdx->ConcatenationOf.empty()) { + ++I; + } else { + I = ConcatenationOf.erase(I); + I = ConcatenationOf.insert(I, SubIdx->ConcatenationOf.begin(), + SubIdx->ConcatenationOf.end()); + I += SubIdx->ConcatenationOf.size(); + } + } +} + +//===----------------------------------------------------------------------===// +// CodeGenRegister +//===----------------------------------------------------------------------===// + +CodeGenRegister::CodeGenRegister(Record *R, unsigned Enum) + : TheDef(R), + EnumValue(Enum), + CostPerUse(R->getValueAsInt("CostPerUse")), + CoveredBySubRegs(R->getValueAsBit("CoveredBySubRegs")), + HasDisjunctSubRegs(false), + SubRegsComplete(false), + SuperRegsComplete(false), + TopoSig(~0u) { + Artificial = R->getValueAsBit("isArtificial"); +} + +void CodeGenRegister::buildObjectGraph(CodeGenRegBank &RegBank) { + std::vector<Record*> SRIs = TheDef->getValueAsListOfDefs("SubRegIndices"); + std::vector<Record*> SRs = TheDef->getValueAsListOfDefs("SubRegs"); + + if (SRIs.size() != SRs.size()) + PrintFatalError(TheDef->getLoc(), + "SubRegs and SubRegIndices must have the same size"); + + for (unsigned i = 0, e = SRIs.size(); i != e; ++i) { + ExplicitSubRegIndices.push_back(RegBank.getSubRegIdx(SRIs[i])); + ExplicitSubRegs.push_back(RegBank.getReg(SRs[i])); + } + + // Also compute leading super-registers. Each register has a list of + // covered-by-subregs super-registers where it appears as the first explicit + // sub-register. + // + // This is used by computeSecondarySubRegs() to find candidates. + if (CoveredBySubRegs && !ExplicitSubRegs.empty()) + ExplicitSubRegs.front()->LeadingSuperRegs.push_back(this); + + // Add ad hoc alias links. This is a symmetric relationship between two + // registers, so build a symmetric graph by adding links in both ends. + std::vector<Record*> Aliases = TheDef->getValueAsListOfDefs("Aliases"); + for (Record *Alias : Aliases) { + CodeGenRegister *Reg = RegBank.getReg(Alias); + ExplicitAliases.push_back(Reg); + Reg->ExplicitAliases.push_back(this); + } +} + +const StringRef CodeGenRegister::getName() const { + assert(TheDef && "no def"); + return TheDef->getName(); +} + +namespace { + +// Iterate over all register units in a set of registers. +class RegUnitIterator { + CodeGenRegister::Vec::const_iterator RegI, RegE; + CodeGenRegister::RegUnitList::iterator UnitI, UnitE; + +public: + RegUnitIterator(const CodeGenRegister::Vec &Regs): + RegI(Regs.begin()), RegE(Regs.end()) { + + if (RegI != RegE) { + UnitI = (*RegI)->getRegUnits().begin(); + UnitE = (*RegI)->getRegUnits().end(); + advance(); + } + } + + bool isValid() const { return UnitI != UnitE; } + + unsigned operator* () const { assert(isValid()); return *UnitI; } + + const CodeGenRegister *getReg() const { assert(isValid()); return *RegI; } + + /// Preincrement. Move to the next unit. + void operator++() { + assert(isValid() && "Cannot advance beyond the last operand"); + ++UnitI; + advance(); + } + +protected: + void advance() { + while (UnitI == UnitE) { + if (++RegI == RegE) + break; + UnitI = (*RegI)->getRegUnits().begin(); + UnitE = (*RegI)->getRegUnits().end(); + } + } +}; + +} // end anonymous namespace + +// Return true of this unit appears in RegUnits. +static bool hasRegUnit(CodeGenRegister::RegUnitList &RegUnits, unsigned Unit) { + return RegUnits.test(Unit); +} + +// Inherit register units from subregisters. +// Return true if the RegUnits changed. +bool CodeGenRegister::inheritRegUnits(CodeGenRegBank &RegBank) { + bool changed = false; + for (const auto &SubReg : SubRegs) { + CodeGenRegister *SR = SubReg.second; + // Merge the subregister's units into this register's RegUnits. + changed |= (RegUnits |= SR->RegUnits); + } + + return changed; +} + +const CodeGenRegister::SubRegMap & +CodeGenRegister::computeSubRegs(CodeGenRegBank &RegBank) { + // Only compute this map once. + if (SubRegsComplete) + return SubRegs; + SubRegsComplete = true; + + HasDisjunctSubRegs = ExplicitSubRegs.size() > 1; + + // First insert the explicit subregs and make sure they are fully indexed. + for (unsigned i = 0, e = ExplicitSubRegs.size(); i != e; ++i) { + CodeGenRegister *SR = ExplicitSubRegs[i]; + CodeGenSubRegIndex *Idx = ExplicitSubRegIndices[i]; + if (!SR->Artificial) + Idx->Artificial = false; + if (!SubRegs.insert(std::make_pair(Idx, SR)).second) + PrintFatalError(TheDef->getLoc(), "SubRegIndex " + Idx->getName() + + " appears twice in Register " + getName()); + // Map explicit sub-registers first, so the names take precedence. + // The inherited sub-registers are mapped below. + SubReg2Idx.insert(std::make_pair(SR, Idx)); + } + + // Keep track of inherited subregs and how they can be reached. + SmallPtrSet<CodeGenRegister*, 8> Orphans; + + // Clone inherited subregs and place duplicate entries in Orphans. + // Here the order is important - earlier subregs take precedence. + for (CodeGenRegister *ESR : ExplicitSubRegs) { + const SubRegMap &Map = ESR->computeSubRegs(RegBank); + HasDisjunctSubRegs |= ESR->HasDisjunctSubRegs; + + for (const auto &SR : Map) { + if (!SubRegs.insert(SR).second) + Orphans.insert(SR.second); + } + } + + // Expand any composed subreg indices. + // If dsub_2 has ComposedOf = [qsub_1, dsub_0], and this register has a + // qsub_1 subreg, add a dsub_2 subreg. Keep growing Indices and process + // expanded subreg indices recursively. + SmallVector<CodeGenSubRegIndex*, 8> Indices = ExplicitSubRegIndices; + for (unsigned i = 0; i != Indices.size(); ++i) { + CodeGenSubRegIndex *Idx = Indices[i]; + const CodeGenSubRegIndex::CompMap &Comps = Idx->getComposites(); + CodeGenRegister *SR = SubRegs[Idx]; + const SubRegMap &Map = SR->computeSubRegs(RegBank); + + // Look at the possible compositions of Idx. + // They may not all be supported by SR. + for (CodeGenSubRegIndex::CompMap::const_iterator I = Comps.begin(), + E = Comps.end(); I != E; ++I) { + SubRegMap::const_iterator SRI = Map.find(I->first); + if (SRI == Map.end()) + continue; // Idx + I->first doesn't exist in SR. + // Add I->second as a name for the subreg SRI->second, assuming it is + // orphaned, and the name isn't already used for something else. + if (SubRegs.count(I->second) || !Orphans.erase(SRI->second)) + continue; + // We found a new name for the orphaned sub-register. + SubRegs.insert(std::make_pair(I->second, SRI->second)); + Indices.push_back(I->second); + } + } + + // Now Orphans contains the inherited subregisters without a direct index. + // Create inferred indexes for all missing entries. + // Work backwards in the Indices vector in order to compose subregs bottom-up. + // Consider this subreg sequence: + // + // qsub_1 -> dsub_0 -> ssub_0 + // + // The qsub_1 -> dsub_0 composition becomes dsub_2, so the ssub_0 register + // can be reached in two different ways: + // + // qsub_1 -> ssub_0 + // dsub_2 -> ssub_0 + // + // We pick the latter composition because another register may have [dsub_0, + // dsub_1, dsub_2] subregs without necessarily having a qsub_1 subreg. The + // dsub_2 -> ssub_0 composition can be shared. + while (!Indices.empty() && !Orphans.empty()) { + CodeGenSubRegIndex *Idx = Indices.pop_back_val(); + CodeGenRegister *SR = SubRegs[Idx]; + const SubRegMap &Map = SR->computeSubRegs(RegBank); + for (const auto &SubReg : Map) + if (Orphans.erase(SubReg.second)) + SubRegs[RegBank.getCompositeSubRegIndex(Idx, SubReg.first)] = SubReg.second; + } + + // Compute the inverse SubReg -> Idx map. + for (const auto &SubReg : SubRegs) { + if (SubReg.second == this) { + ArrayRef<SMLoc> Loc; + if (TheDef) + Loc = TheDef->getLoc(); + PrintFatalError(Loc, "Register " + getName() + + " has itself as a sub-register"); + } + + // Compute AllSuperRegsCovered. + if (!CoveredBySubRegs) + SubReg.first->AllSuperRegsCovered = false; + + // Ensure that every sub-register has a unique name. + DenseMap<const CodeGenRegister*, CodeGenSubRegIndex*>::iterator Ins = + SubReg2Idx.insert(std::make_pair(SubReg.second, SubReg.first)).first; + if (Ins->second == SubReg.first) + continue; + // Trouble: Two different names for SubReg.second. + ArrayRef<SMLoc> Loc; + if (TheDef) + Loc = TheDef->getLoc(); + PrintFatalError(Loc, "Sub-register can't have two names: " + + SubReg.second->getName() + " available as " + + SubReg.first->getName() + " and " + Ins->second->getName()); + } + + // Derive possible names for sub-register concatenations from any explicit + // sub-registers. By doing this before computeSecondarySubRegs(), we ensure + // that getConcatSubRegIndex() won't invent any concatenated indices that the + // user already specified. + for (unsigned i = 0, e = ExplicitSubRegs.size(); i != e; ++i) { + CodeGenRegister *SR = ExplicitSubRegs[i]; + if (!SR->CoveredBySubRegs || SR->ExplicitSubRegs.size() <= 1 || + SR->Artificial) + continue; + + // SR is composed of multiple sub-regs. Find their names in this register. + SmallVector<CodeGenSubRegIndex*, 8> Parts; + for (unsigned j = 0, e = SR->ExplicitSubRegs.size(); j != e; ++j) { + CodeGenSubRegIndex &I = *SR->ExplicitSubRegIndices[j]; + if (!I.Artificial) + Parts.push_back(getSubRegIndex(SR->ExplicitSubRegs[j])); + } + + // Offer this as an existing spelling for the concatenation of Parts. + CodeGenSubRegIndex &Idx = *ExplicitSubRegIndices[i]; + Idx.setConcatenationOf(Parts); + } + + // Initialize RegUnitList. Because getSubRegs is called recursively, this + // processes the register hierarchy in postorder. + // + // Inherit all sub-register units. It is good enough to look at the explicit + // sub-registers, the other registers won't contribute any more units. + for (unsigned i = 0, e = ExplicitSubRegs.size(); i != e; ++i) { + CodeGenRegister *SR = ExplicitSubRegs[i]; + RegUnits |= SR->RegUnits; + } + + // Absent any ad hoc aliasing, we create one register unit per leaf register. + // These units correspond to the maximal cliques in the register overlap + // graph which is optimal. + // + // When there is ad hoc aliasing, we simply create one unit per edge in the + // undirected ad hoc aliasing graph. Technically, we could do better by + // identifying maximal cliques in the ad hoc graph, but cliques larger than 2 + // are extremely rare anyway (I've never seen one), so we don't bother with + // the added complexity. + for (unsigned i = 0, e = ExplicitAliases.size(); i != e; ++i) { + CodeGenRegister *AR = ExplicitAliases[i]; + // Only visit each edge once. + if (AR->SubRegsComplete) + continue; + // Create a RegUnit representing this alias edge, and add it to both + // registers. + unsigned Unit = RegBank.newRegUnit(this, AR); + RegUnits.set(Unit); + AR->RegUnits.set(Unit); + } + + // Finally, create units for leaf registers without ad hoc aliases. Note that + // a leaf register with ad hoc aliases doesn't get its own unit - it isn't + // necessary. This means the aliasing leaf registers can share a single unit. + if (RegUnits.empty()) + RegUnits.set(RegBank.newRegUnit(this)); + + // We have now computed the native register units. More may be adopted later + // for balancing purposes. + NativeRegUnits = RegUnits; + + return SubRegs; +} + +// In a register that is covered by its sub-registers, try to find redundant +// sub-registers. For example: +// +// QQ0 = {Q0, Q1} +// Q0 = {D0, D1} +// Q1 = {D2, D3} +// +// We can infer that D1_D2 is also a sub-register, even if it wasn't named in +// the register definition. +// +// The explicitly specified registers form a tree. This function discovers +// sub-register relationships that would force a DAG. +// +void CodeGenRegister::computeSecondarySubRegs(CodeGenRegBank &RegBank) { + SmallVector<SubRegMap::value_type, 8> NewSubRegs; + + std::queue<std::pair<CodeGenSubRegIndex*,CodeGenRegister*>> SubRegQueue; + for (std::pair<CodeGenSubRegIndex*,CodeGenRegister*> P : SubRegs) + SubRegQueue.push(P); + + // Look at the leading super-registers of each sub-register. Those are the + // candidates for new sub-registers, assuming they are fully contained in + // this register. + while (!SubRegQueue.empty()) { + CodeGenSubRegIndex *SubRegIdx; + const CodeGenRegister *SubReg; + std::tie(SubRegIdx, SubReg) = SubRegQueue.front(); + SubRegQueue.pop(); + + const CodeGenRegister::SuperRegList &Leads = SubReg->LeadingSuperRegs; + for (unsigned i = 0, e = Leads.size(); i != e; ++i) { + CodeGenRegister *Cand = const_cast<CodeGenRegister*>(Leads[i]); + // Already got this sub-register? + if (Cand == this || getSubRegIndex(Cand)) + continue; + // Check if each component of Cand is already a sub-register. + assert(!Cand->ExplicitSubRegs.empty() && + "Super-register has no sub-registers"); + if (Cand->ExplicitSubRegs.size() == 1) + continue; + SmallVector<CodeGenSubRegIndex*, 8> Parts; + // We know that the first component is (SubRegIdx,SubReg). However we + // may still need to split it into smaller subregister parts. + assert(Cand->ExplicitSubRegs[0] == SubReg && "LeadingSuperRegs correct"); + assert(getSubRegIndex(SubReg) == SubRegIdx && "LeadingSuperRegs correct"); + for (CodeGenRegister *SubReg : Cand->ExplicitSubRegs) { + if (CodeGenSubRegIndex *SubRegIdx = getSubRegIndex(SubReg)) { + if (SubRegIdx->ConcatenationOf.empty()) { + Parts.push_back(SubRegIdx); + } else + for (CodeGenSubRegIndex *SubIdx : SubRegIdx->ConcatenationOf) + Parts.push_back(SubIdx); + } else { + // Sub-register doesn't exist. + Parts.clear(); + break; + } + } + // There is nothing to do if some Cand sub-register is not part of this + // register. + if (Parts.empty()) + continue; + + // Each part of Cand is a sub-register of this. Make the full Cand also + // a sub-register with a concatenated sub-register index. + CodeGenSubRegIndex *Concat = RegBank.getConcatSubRegIndex(Parts); + std::pair<CodeGenSubRegIndex*,CodeGenRegister*> NewSubReg = + std::make_pair(Concat, Cand); + + if (!SubRegs.insert(NewSubReg).second) + continue; + + // We inserted a new subregister. + NewSubRegs.push_back(NewSubReg); + SubRegQueue.push(NewSubReg); + SubReg2Idx.insert(std::make_pair(Cand, Concat)); + } + } + + // Create sub-register index composition maps for the synthesized indices. + for (unsigned i = 0, e = NewSubRegs.size(); i != e; ++i) { + CodeGenSubRegIndex *NewIdx = NewSubRegs[i].first; + CodeGenRegister *NewSubReg = NewSubRegs[i].second; + for (SubRegMap::const_iterator SI = NewSubReg->SubRegs.begin(), + SE = NewSubReg->SubRegs.end(); SI != SE; ++SI) { + CodeGenSubRegIndex *SubIdx = getSubRegIndex(SI->second); + if (!SubIdx) + PrintFatalError(TheDef->getLoc(), "No SubRegIndex for " + + SI->second->getName() + " in " + getName()); + NewIdx->addComposite(SI->first, SubIdx); + } + } +} + +void CodeGenRegister::computeSuperRegs(CodeGenRegBank &RegBank) { + // Only visit each register once. + if (SuperRegsComplete) + return; + SuperRegsComplete = true; + + // Make sure all sub-registers have been visited first, so the super-reg + // lists will be topologically ordered. + for (SubRegMap::const_iterator I = SubRegs.begin(), E = SubRegs.end(); + I != E; ++I) + I->second->computeSuperRegs(RegBank); + + // Now add this as a super-register on all sub-registers. + // Also compute the TopoSigId in post-order. + TopoSigId Id; + for (SubRegMap::const_iterator I = SubRegs.begin(), E = SubRegs.end(); + I != E; ++I) { + // Topological signature computed from SubIdx, TopoId(SubReg). + // Loops and idempotent indices have TopoSig = ~0u. + Id.push_back(I->first->EnumValue); + Id.push_back(I->second->TopoSig); + + // Don't add duplicate entries. + if (!I->second->SuperRegs.empty() && I->second->SuperRegs.back() == this) + continue; + I->second->SuperRegs.push_back(this); + } + TopoSig = RegBank.getTopoSig(Id); +} + +void +CodeGenRegister::addSubRegsPreOrder(SetVector<const CodeGenRegister*> &OSet, + CodeGenRegBank &RegBank) const { + assert(SubRegsComplete && "Must precompute sub-registers"); + for (unsigned i = 0, e = ExplicitSubRegs.size(); i != e; ++i) { + CodeGenRegister *SR = ExplicitSubRegs[i]; + if (OSet.insert(SR)) + SR->addSubRegsPreOrder(OSet, RegBank); + } + // Add any secondary sub-registers that weren't part of the explicit tree. + for (SubRegMap::const_iterator I = SubRegs.begin(), E = SubRegs.end(); + I != E; ++I) + OSet.insert(I->second); +} + +// Get the sum of this register's unit weights. +unsigned CodeGenRegister::getWeight(const CodeGenRegBank &RegBank) const { + unsigned Weight = 0; + for (RegUnitList::iterator I = RegUnits.begin(), E = RegUnits.end(); + I != E; ++I) { + Weight += RegBank.getRegUnit(*I).Weight; + } + return Weight; +} + +//===----------------------------------------------------------------------===// +// RegisterTuples +//===----------------------------------------------------------------------===// + +// A RegisterTuples def is used to generate pseudo-registers from lists of +// sub-registers. We provide a SetTheory expander class that returns the new +// registers. +namespace { + +struct TupleExpander : SetTheory::Expander { + // Reference to SynthDefs in the containing CodeGenRegBank, to keep track of + // the synthesized definitions for their lifetime. + std::vector<std::unique_ptr<Record>> &SynthDefs; + + TupleExpander(std::vector<std::unique_ptr<Record>> &SynthDefs) + : SynthDefs(SynthDefs) {} + + void expand(SetTheory &ST, Record *Def, SetTheory::RecSet &Elts) override { + std::vector<Record*> Indices = Def->getValueAsListOfDefs("SubRegIndices"); + unsigned Dim = Indices.size(); + ListInit *SubRegs = Def->getValueAsListInit("SubRegs"); + if (Dim != SubRegs->size()) + PrintFatalError(Def->getLoc(), "SubRegIndices and SubRegs size mismatch"); + if (Dim < 2) + PrintFatalError(Def->getLoc(), + "Tuples must have at least 2 sub-registers"); + + // Evaluate the sub-register lists to be zipped. + unsigned Length = ~0u; + SmallVector<SetTheory::RecSet, 4> Lists(Dim); + for (unsigned i = 0; i != Dim; ++i) { + ST.evaluate(SubRegs->getElement(i), Lists[i], Def->getLoc()); + Length = std::min(Length, unsigned(Lists[i].size())); + } + + if (Length == 0) + return; + + // Precompute some types. + Record *RegisterCl = Def->getRecords().getClass("Register"); + RecTy *RegisterRecTy = RecordRecTy::get(RegisterCl); + std::vector<StringRef> RegNames = + Def->getValueAsListOfStrings("RegAsmNames"); + + // Zip them up. + for (unsigned n = 0; n != Length; ++n) { + std::string Name; + Record *Proto = Lists[0][n]; + std::vector<Init*> Tuple; + unsigned CostPerUse = 0; + for (unsigned i = 0; i != Dim; ++i) { + Record *Reg = Lists[i][n]; + if (i) Name += '_'; + Name += Reg->getName(); + Tuple.push_back(DefInit::get(Reg)); + CostPerUse = std::max(CostPerUse, + unsigned(Reg->getValueAsInt("CostPerUse"))); + } + + StringInit *AsmName = StringInit::get(""); + if (!RegNames.empty()) { + if (RegNames.size() <= n) + PrintFatalError(Def->getLoc(), + "Register tuple definition missing name for '" + + Name + "'."); + AsmName = StringInit::get(RegNames[n]); + } + + // Create a new Record representing the synthesized register. This record + // is only for consumption by CodeGenRegister, it is not added to the + // RecordKeeper. + SynthDefs.emplace_back( + std::make_unique<Record>(Name, Def->getLoc(), Def->getRecords())); + Record *NewReg = SynthDefs.back().get(); + Elts.insert(NewReg); + + // Copy Proto super-classes. + ArrayRef<std::pair<Record *, SMRange>> Supers = Proto->getSuperClasses(); + for (const auto &SuperPair : Supers) + NewReg->addSuperClass(SuperPair.first, SuperPair.second); + + // Copy Proto fields. + for (unsigned i = 0, e = Proto->getValues().size(); i != e; ++i) { + RecordVal RV = Proto->getValues()[i]; + + // Skip existing fields, like NAME. + if (NewReg->getValue(RV.getNameInit())) + continue; + + StringRef Field = RV.getName(); + + // Replace the sub-register list with Tuple. + if (Field == "SubRegs") + RV.setValue(ListInit::get(Tuple, RegisterRecTy)); + + if (Field == "AsmName") + RV.setValue(AsmName); + + // CostPerUse is aggregated from all Tuple members. + if (Field == "CostPerUse") + RV.setValue(IntInit::get(CostPerUse)); + + // Composite registers are always covered by sub-registers. + if (Field == "CoveredBySubRegs") + RV.setValue(BitInit::get(true)); + + // Copy fields from the RegisterTuples def. + if (Field == "SubRegIndices" || + Field == "CompositeIndices") { + NewReg->addValue(*Def->getValue(Field)); + continue; + } + + // Some fields get their default uninitialized value. + if (Field == "DwarfNumbers" || + Field == "DwarfAlias" || + Field == "Aliases") { + if (const RecordVal *DefRV = RegisterCl->getValue(Field)) + NewReg->addValue(*DefRV); + continue; + } + + // Everything else is copied from Proto. + NewReg->addValue(RV); + } + } + } +}; + +} // end anonymous namespace + +//===----------------------------------------------------------------------===// +// CodeGenRegisterClass +//===----------------------------------------------------------------------===// + +static void sortAndUniqueRegisters(CodeGenRegister::Vec &M) { + llvm::sort(M, deref<std::less<>>()); + M.erase(std::unique(M.begin(), M.end(), deref<std::equal_to<>>()), M.end()); +} + +CodeGenRegisterClass::CodeGenRegisterClass(CodeGenRegBank &RegBank, Record *R) + : TheDef(R), Name(std::string(R->getName())), + TopoSigs(RegBank.getNumTopoSigs()), EnumValue(-1) { + GeneratePressureSet = R->getValueAsBit("GeneratePressureSet"); + std::vector<Record*> TypeList = R->getValueAsListOfDefs("RegTypes"); + for (unsigned i = 0, e = TypeList.size(); i != e; ++i) { + Record *Type = TypeList[i]; + if (!Type->isSubClassOf("ValueType")) + PrintFatalError(R->getLoc(), + "RegTypes list member '" + Type->getName() + + "' does not derive from the ValueType class!"); + VTs.push_back(getValueTypeByHwMode(Type, RegBank.getHwModes())); + } + assert(!VTs.empty() && "RegisterClass must contain at least one ValueType!"); + + // Allocation order 0 is the full set. AltOrders provides others. + const SetTheory::RecVec *Elements = RegBank.getSets().expand(R); + ListInit *AltOrders = R->getValueAsListInit("AltOrders"); + Orders.resize(1 + AltOrders->size()); + + // Default allocation order always contains all registers. + Artificial = true; + for (unsigned i = 0, e = Elements->size(); i != e; ++i) { + Orders[0].push_back((*Elements)[i]); + const CodeGenRegister *Reg = RegBank.getReg((*Elements)[i]); + Members.push_back(Reg); + Artificial &= Reg->Artificial; + TopoSigs.set(Reg->getTopoSig()); + } + sortAndUniqueRegisters(Members); + + // Alternative allocation orders may be subsets. + SetTheory::RecSet Order; + for (unsigned i = 0, e = AltOrders->size(); i != e; ++i) { + RegBank.getSets().evaluate(AltOrders->getElement(i), Order, R->getLoc()); + Orders[1 + i].append(Order.begin(), Order.end()); + // Verify that all altorder members are regclass members. + while (!Order.empty()) { + CodeGenRegister *Reg = RegBank.getReg(Order.back()); + Order.pop_back(); + if (!contains(Reg)) + PrintFatalError(R->getLoc(), " AltOrder register " + Reg->getName() + + " is not a class member"); + } + } + + Namespace = R->getValueAsString("Namespace"); + + if (const RecordVal *RV = R->getValue("RegInfos")) + if (DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) + RSI = RegSizeInfoByHwMode(DI->getDef(), RegBank.getHwModes()); + unsigned Size = R->getValueAsInt("Size"); + assert((RSI.hasDefault() || Size != 0 || VTs[0].isSimple()) && + "Impossible to determine register size"); + if (!RSI.hasDefault()) { + RegSizeInfo RI; + RI.RegSize = RI.SpillSize = Size ? Size + : VTs[0].getSimple().getSizeInBits(); + RI.SpillAlignment = R->getValueAsInt("Alignment"); + RSI.Map.insert({DefaultMode, RI}); + } + + CopyCost = R->getValueAsInt("CopyCost"); + Allocatable = R->getValueAsBit("isAllocatable"); + AltOrderSelect = R->getValueAsString("AltOrderSelect"); + int AllocationPriority = R->getValueAsInt("AllocationPriority"); + if (AllocationPriority < 0 || AllocationPriority > 63) + PrintFatalError(R->getLoc(), "AllocationPriority out of range [0,63]"); + this->AllocationPriority = AllocationPriority; +} + +// Create an inferred register class that was missing from the .td files. +// Most properties will be inherited from the closest super-class after the +// class structure has been computed. +CodeGenRegisterClass::CodeGenRegisterClass(CodeGenRegBank &RegBank, + StringRef Name, Key Props) + : Members(*Props.Members), TheDef(nullptr), Name(std::string(Name)), + TopoSigs(RegBank.getNumTopoSigs()), EnumValue(-1), RSI(Props.RSI), + CopyCost(0), Allocatable(true), AllocationPriority(0) { + Artificial = true; + GeneratePressureSet = false; + for (const auto R : Members) { + TopoSigs.set(R->getTopoSig()); + Artificial &= R->Artificial; + } +} + +// Compute inherited propertied for a synthesized register class. +void CodeGenRegisterClass::inheritProperties(CodeGenRegBank &RegBank) { + assert(!getDef() && "Only synthesized classes can inherit properties"); + assert(!SuperClasses.empty() && "Synthesized class without super class"); + + // The last super-class is the smallest one. + CodeGenRegisterClass &Super = *SuperClasses.back(); + + // Most properties are copied directly. + // Exceptions are members, size, and alignment + Namespace = Super.Namespace; + VTs = Super.VTs; + CopyCost = Super.CopyCost; + Allocatable = Super.Allocatable; + AltOrderSelect = Super.AltOrderSelect; + AllocationPriority = Super.AllocationPriority; + GeneratePressureSet |= Super.GeneratePressureSet; + + // Copy all allocation orders, filter out foreign registers from the larger + // super-class. + Orders.resize(Super.Orders.size()); + for (unsigned i = 0, ie = Super.Orders.size(); i != ie; ++i) + for (unsigned j = 0, je = Super.Orders[i].size(); j != je; ++j) + if (contains(RegBank.getReg(Super.Orders[i][j]))) + Orders[i].push_back(Super.Orders[i][j]); +} + +bool CodeGenRegisterClass::contains(const CodeGenRegister *Reg) const { + return std::binary_search(Members.begin(), Members.end(), Reg, + deref<std::less<>>()); +} + +unsigned CodeGenRegisterClass::getWeight(const CodeGenRegBank& RegBank) const { + if (TheDef && !TheDef->isValueUnset("Weight")) + return TheDef->getValueAsInt("Weight"); + + if (Members.empty() || Artificial) + return 0; + + return (*Members.begin())->getWeight(RegBank); +} + +namespace llvm { + + raw_ostream &operator<<(raw_ostream &OS, const CodeGenRegisterClass::Key &K) { + OS << "{ " << K.RSI; + for (const auto R : *K.Members) + OS << ", " << R->getName(); + return OS << " }"; + } + +} // end namespace llvm + +// This is a simple lexicographical order that can be used to search for sets. +// It is not the same as the topological order provided by TopoOrderRC. +bool CodeGenRegisterClass::Key:: +operator<(const CodeGenRegisterClass::Key &B) const { + assert(Members && B.Members); + return std::tie(*Members, RSI) < std::tie(*B.Members, B.RSI); +} + +// Returns true if RC is a strict subclass. +// RC is a sub-class of this class if it is a valid replacement for any +// instruction operand where a register of this classis required. It must +// satisfy these conditions: +// +// 1. All RC registers are also in this. +// 2. The RC spill size must not be smaller than our spill size. +// 3. RC spill alignment must be compatible with ours. +// +static bool testSubClass(const CodeGenRegisterClass *A, + const CodeGenRegisterClass *B) { + return A->RSI.isSubClassOf(B->RSI) && + std::includes(A->getMembers().begin(), A->getMembers().end(), + B->getMembers().begin(), B->getMembers().end(), + deref<std::less<>>()); +} + +/// Sorting predicate for register classes. This provides a topological +/// ordering that arranges all register classes before their sub-classes. +/// +/// Register classes with the same registers, spill size, and alignment form a +/// clique. They will be ordered alphabetically. +/// +static bool TopoOrderRC(const CodeGenRegisterClass &PA, + const CodeGenRegisterClass &PB) { + auto *A = &PA; + auto *B = &PB; + if (A == B) + return false; + + if (A->RSI < B->RSI) + return true; + if (A->RSI != B->RSI) + return false; + + // Order by descending set size. Note that the classes' allocation order may + // not have been computed yet. The Members set is always vaild. + if (A->getMembers().size() > B->getMembers().size()) + return true; + if (A->getMembers().size() < B->getMembers().size()) + return false; + + // Finally order by name as a tie breaker. + return StringRef(A->getName()) < B->getName(); +} + +std::string CodeGenRegisterClass::getQualifiedName() const { + if (Namespace.empty()) + return getName(); + else + return (Namespace + "::" + getName()).str(); +} + +// Compute sub-classes of all register classes. +// Assume the classes are ordered topologically. +void CodeGenRegisterClass::computeSubClasses(CodeGenRegBank &RegBank) { + auto &RegClasses = RegBank.getRegClasses(); + + // Visit backwards so sub-classes are seen first. + for (auto I = RegClasses.rbegin(), E = RegClasses.rend(); I != E; ++I) { + CodeGenRegisterClass &RC = *I; + RC.SubClasses.resize(RegClasses.size()); + RC.SubClasses.set(RC.EnumValue); + if (RC.Artificial) + continue; + + // Normally, all subclasses have IDs >= rci, unless RC is part of a clique. + for (auto I2 = I.base(), E2 = RegClasses.end(); I2 != E2; ++I2) { + CodeGenRegisterClass &SubRC = *I2; + if (RC.SubClasses.test(SubRC.EnumValue)) + continue; + if (!testSubClass(&RC, &SubRC)) + continue; + // SubRC is a sub-class. Grap all its sub-classes so we won't have to + // check them again. + RC.SubClasses |= SubRC.SubClasses; + } + + // Sweep up missed clique members. They will be immediately preceding RC. + for (auto I2 = std::next(I); I2 != E && testSubClass(&RC, &*I2); ++I2) + RC.SubClasses.set(I2->EnumValue); + } + + // Compute the SuperClasses lists from the SubClasses vectors. + for (auto &RC : RegClasses) { + const BitVector &SC = RC.getSubClasses(); + auto I = RegClasses.begin(); + for (int s = 0, next_s = SC.find_first(); next_s != -1; + next_s = SC.find_next(s)) { + std::advance(I, next_s - s); + s = next_s; + if (&*I == &RC) + continue; + I->SuperClasses.push_back(&RC); + } + } + + // With the class hierarchy in place, let synthesized register classes inherit + // properties from their closest super-class. The iteration order here can + // propagate properties down multiple levels. + for (auto &RC : RegClasses) + if (!RC.getDef()) + RC.inheritProperties(RegBank); +} + +Optional<std::pair<CodeGenRegisterClass *, CodeGenRegisterClass *>> +CodeGenRegisterClass::getMatchingSubClassWithSubRegs( + CodeGenRegBank &RegBank, const CodeGenSubRegIndex *SubIdx) const { + auto SizeOrder = [this](const CodeGenRegisterClass *A, + const CodeGenRegisterClass *B) { + // If there are multiple, identical register classes, prefer the original + // register class. + if (A->getMembers().size() == B->getMembers().size()) + return A == this; + return A->getMembers().size() > B->getMembers().size(); + }; + + auto &RegClasses = RegBank.getRegClasses(); + + // Find all the subclasses of this one that fully support the sub-register + // index and order them by size. BiggestSuperRC should always be first. + CodeGenRegisterClass *BiggestSuperRegRC = getSubClassWithSubReg(SubIdx); + if (!BiggestSuperRegRC) + return None; + BitVector SuperRegRCsBV = BiggestSuperRegRC->getSubClasses(); + std::vector<CodeGenRegisterClass *> SuperRegRCs; + for (auto &RC : RegClasses) + if (SuperRegRCsBV[RC.EnumValue]) + SuperRegRCs.emplace_back(&RC); + llvm::stable_sort(SuperRegRCs, SizeOrder); + + assert(SuperRegRCs.front() == BiggestSuperRegRC && + "Biggest class wasn't first"); + + // Find all the subreg classes and order them by size too. + std::vector<std::pair<CodeGenRegisterClass *, BitVector>> SuperRegClasses; + for (auto &RC: RegClasses) { + BitVector SuperRegClassesBV(RegClasses.size()); + RC.getSuperRegClasses(SubIdx, SuperRegClassesBV); + if (SuperRegClassesBV.any()) + SuperRegClasses.push_back(std::make_pair(&RC, SuperRegClassesBV)); + } + llvm::sort(SuperRegClasses, + [&](const std::pair<CodeGenRegisterClass *, BitVector> &A, + const std::pair<CodeGenRegisterClass *, BitVector> &B) { + return SizeOrder(A.first, B.first); + }); + + // Find the biggest subclass and subreg class such that R:subidx is in the + // subreg class for all R in subclass. + // + // For example: + // All registers in X86's GR64 have a sub_32bit subregister but no class + // exists that contains all the 32-bit subregisters because GR64 contains RIP + // but GR32 does not contain EIP. Instead, we constrain SuperRegRC to + // GR32_with_sub_8bit (which is identical to GR32_with_sub_32bit) and then, + // having excluded RIP, we are able to find a SubRegRC (GR32). + CodeGenRegisterClass *ChosenSuperRegClass = nullptr; + CodeGenRegisterClass *SubRegRC = nullptr; + for (auto *SuperRegRC : SuperRegRCs) { + for (const auto &SuperRegClassPair : SuperRegClasses) { + const BitVector &SuperRegClassBV = SuperRegClassPair.second; + if (SuperRegClassBV[SuperRegRC->EnumValue]) { + SubRegRC = SuperRegClassPair.first; + ChosenSuperRegClass = SuperRegRC; + + // If SubRegRC is bigger than SuperRegRC then there are members of + // SubRegRC that don't have super registers via SubIdx. Keep looking to + // find a better fit and fall back on this one if there isn't one. + // + // This is intended to prevent X86 from making odd choices such as + // picking LOW32_ADDR_ACCESS_RBP instead of GR32 in the example above. + // LOW32_ADDR_ACCESS_RBP is a valid choice but contains registers that + // aren't subregisters of SuperRegRC whereas GR32 has a direct 1:1 + // mapping. + if (SuperRegRC->getMembers().size() >= SubRegRC->getMembers().size()) + return std::make_pair(ChosenSuperRegClass, SubRegRC); + } + } + + // If we found a fit but it wasn't quite ideal because SubRegRC had excess + // registers, then we're done. + if (ChosenSuperRegClass) + return std::make_pair(ChosenSuperRegClass, SubRegRC); + } + + return None; +} + +void CodeGenRegisterClass::getSuperRegClasses(const CodeGenSubRegIndex *SubIdx, + BitVector &Out) const { + auto FindI = SuperRegClasses.find(SubIdx); + if (FindI == SuperRegClasses.end()) + return; + for (CodeGenRegisterClass *RC : FindI->second) + Out.set(RC->EnumValue); +} + +// Populate a unique sorted list of units from a register set. +void CodeGenRegisterClass::buildRegUnitSet(const CodeGenRegBank &RegBank, + std::vector<unsigned> &RegUnits) const { + std::vector<unsigned> TmpUnits; + for (RegUnitIterator UnitI(Members); UnitI.isValid(); ++UnitI) { + const RegUnit &RU = RegBank.getRegUnit(*UnitI); + if (!RU.Artificial) + TmpUnits.push_back(*UnitI); + } + llvm::sort(TmpUnits); + std::unique_copy(TmpUnits.begin(), TmpUnits.end(), + std::back_inserter(RegUnits)); +} + +//===----------------------------------------------------------------------===// +// CodeGenRegBank +//===----------------------------------------------------------------------===// + +CodeGenRegBank::CodeGenRegBank(RecordKeeper &Records, + const CodeGenHwModes &Modes) : CGH(Modes) { + // Configure register Sets to understand register classes and tuples. + Sets.addFieldExpander("RegisterClass", "MemberList"); + Sets.addFieldExpander("CalleeSavedRegs", "SaveList"); + Sets.addExpander("RegisterTuples", + std::make_unique<TupleExpander>(SynthDefs)); + + // Read in the user-defined (named) sub-register indices. + // More indices will be synthesized later. + std::vector<Record*> SRIs = Records.getAllDerivedDefinitions("SubRegIndex"); + llvm::sort(SRIs, LessRecord()); + for (unsigned i = 0, e = SRIs.size(); i != e; ++i) + getSubRegIdx(SRIs[i]); + // Build composite maps from ComposedOf fields. + for (auto &Idx : SubRegIndices) + Idx.updateComponents(*this); + + // Read in the register definitions. + std::vector<Record*> Regs = Records.getAllDerivedDefinitions("Register"); + llvm::sort(Regs, LessRecordRegister()); + // Assign the enumeration values. + for (unsigned i = 0, e = Regs.size(); i != e; ++i) + getReg(Regs[i]); + + // Expand tuples and number the new registers. + std::vector<Record*> Tups = + Records.getAllDerivedDefinitions("RegisterTuples"); + + for (Record *R : Tups) { + std::vector<Record *> TupRegs = *Sets.expand(R); + llvm::sort(TupRegs, LessRecordRegister()); + for (Record *RC : TupRegs) + getReg(RC); + } + + // Now all the registers are known. Build the object graph of explicit + // register-register references. + for (auto &Reg : Registers) + Reg.buildObjectGraph(*this); + + // Compute register name map. + for (auto &Reg : Registers) + // FIXME: This could just be RegistersByName[name] = register, except that + // causes some failures in MIPS - perhaps they have duplicate register name + // entries? (or maybe there's a reason for it - I don't know much about this + // code, just drive-by refactoring) + RegistersByName.insert( + std::make_pair(Reg.TheDef->getValueAsString("AsmName"), &Reg)); + + // Precompute all sub-register maps. + // This will create Composite entries for all inferred sub-register indices. + for (auto &Reg : Registers) + Reg.computeSubRegs(*this); + + // Compute transitive closure of subregister index ConcatenationOf vectors + // and initialize ConcatIdx map. + for (CodeGenSubRegIndex &SRI : SubRegIndices) { + SRI.computeConcatTransitiveClosure(); + if (!SRI.ConcatenationOf.empty()) + ConcatIdx.insert(std::make_pair( + SmallVector<CodeGenSubRegIndex*,8>(SRI.ConcatenationOf.begin(), + SRI.ConcatenationOf.end()), &SRI)); + } + + // Infer even more sub-registers by combining leading super-registers. + for (auto &Reg : Registers) + if (Reg.CoveredBySubRegs) + Reg.computeSecondarySubRegs(*this); + + // After the sub-register graph is complete, compute the topologically + // ordered SuperRegs list. + for (auto &Reg : Registers) + Reg.computeSuperRegs(*this); + + // For each pair of Reg:SR, if both are non-artificial, mark the + // corresponding sub-register index as non-artificial. + for (auto &Reg : Registers) { + if (Reg.Artificial) + continue; + for (auto P : Reg.getSubRegs()) { + const CodeGenRegister *SR = P.second; + if (!SR->Artificial) + P.first->Artificial = false; + } + } + + // Native register units are associated with a leaf register. They've all been + // discovered now. + NumNativeRegUnits = RegUnits.size(); + + // Read in register class definitions. + std::vector<Record*> RCs = Records.getAllDerivedDefinitions("RegisterClass"); + if (RCs.empty()) + PrintFatalError("No 'RegisterClass' subclasses defined!"); + + // Allocate user-defined register classes. + for (auto *R : RCs) { + RegClasses.emplace_back(*this, R); + CodeGenRegisterClass &RC = RegClasses.back(); + if (!RC.Artificial) + addToMaps(&RC); + } + + // Infer missing classes to create a full algebra. + computeInferredRegisterClasses(); + + // Order register classes topologically and assign enum values. + RegClasses.sort(TopoOrderRC); + unsigned i = 0; + for (auto &RC : RegClasses) + RC.EnumValue = i++; + CodeGenRegisterClass::computeSubClasses(*this); +} + +// Create a synthetic CodeGenSubRegIndex without a corresponding Record. +CodeGenSubRegIndex* +CodeGenRegBank::createSubRegIndex(StringRef Name, StringRef Namespace) { + SubRegIndices.emplace_back(Name, Namespace, SubRegIndices.size() + 1); + return &SubRegIndices.back(); +} + +CodeGenSubRegIndex *CodeGenRegBank::getSubRegIdx(Record *Def) { + CodeGenSubRegIndex *&Idx = Def2SubRegIdx[Def]; + if (Idx) + return Idx; + SubRegIndices.emplace_back(Def, SubRegIndices.size() + 1); + Idx = &SubRegIndices.back(); + return Idx; +} + +const CodeGenSubRegIndex * +CodeGenRegBank::findSubRegIdx(const Record* Def) const { + auto I = Def2SubRegIdx.find(Def); + return (I == Def2SubRegIdx.end()) ? nullptr : I->second; +} + +CodeGenRegister *CodeGenRegBank::getReg(Record *Def) { + CodeGenRegister *&Reg = Def2Reg[Def]; + if (Reg) + return Reg; + Registers.emplace_back(Def, Registers.size() + 1); + Reg = &Registers.back(); + return Reg; +} + +void CodeGenRegBank::addToMaps(CodeGenRegisterClass *RC) { + if (Record *Def = RC->getDef()) + Def2RC.insert(std::make_pair(Def, RC)); + + // Duplicate classes are rejected by insert(). + // That's OK, we only care about the properties handled by CGRC::Key. + CodeGenRegisterClass::Key K(*RC); + Key2RC.insert(std::make_pair(K, RC)); +} + +// Create a synthetic sub-class if it is missing. +CodeGenRegisterClass* +CodeGenRegBank::getOrCreateSubClass(const CodeGenRegisterClass *RC, + const CodeGenRegister::Vec *Members, + StringRef Name) { + // Synthetic sub-class has the same size and alignment as RC. + CodeGenRegisterClass::Key K(Members, RC->RSI); + RCKeyMap::const_iterator FoundI = Key2RC.find(K); + if (FoundI != Key2RC.end()) + return FoundI->second; + + // Sub-class doesn't exist, create a new one. + RegClasses.emplace_back(*this, Name, K); + addToMaps(&RegClasses.back()); + return &RegClasses.back(); +} + +CodeGenRegisterClass *CodeGenRegBank::getRegClass(const Record *Def) const { + if (CodeGenRegisterClass *RC = Def2RC.lookup(Def)) + return RC; + + PrintFatalError(Def->getLoc(), "Not a known RegisterClass!"); +} + +CodeGenSubRegIndex* +CodeGenRegBank::getCompositeSubRegIndex(CodeGenSubRegIndex *A, + CodeGenSubRegIndex *B) { + // Look for an existing entry. + CodeGenSubRegIndex *Comp = A->compose(B); + if (Comp) + return Comp; + + // None exists, synthesize one. + std::string Name = A->getName() + "_then_" + B->getName(); + Comp = createSubRegIndex(Name, A->getNamespace()); + A->addComposite(B, Comp); + return Comp; +} + +CodeGenSubRegIndex *CodeGenRegBank:: +getConcatSubRegIndex(const SmallVector<CodeGenSubRegIndex *, 8> &Parts) { + assert(Parts.size() > 1 && "Need two parts to concatenate"); +#ifndef NDEBUG + for (CodeGenSubRegIndex *Idx : Parts) { + assert(Idx->ConcatenationOf.empty() && "No transitive closure?"); + } +#endif + + // Look for an existing entry. + CodeGenSubRegIndex *&Idx = ConcatIdx[Parts]; + if (Idx) + return Idx; + + // None exists, synthesize one. + std::string Name = Parts.front()->getName(); + // Determine whether all parts are contiguous. + bool isContinuous = true; + unsigned Size = Parts.front()->Size; + unsigned LastOffset = Parts.front()->Offset; + unsigned LastSize = Parts.front()->Size; + for (unsigned i = 1, e = Parts.size(); i != e; ++i) { + Name += '_'; + Name += Parts[i]->getName(); + Size += Parts[i]->Size; + if (Parts[i]->Offset != (LastOffset + LastSize)) + isContinuous = false; + LastOffset = Parts[i]->Offset; + LastSize = Parts[i]->Size; + } + Idx = createSubRegIndex(Name, Parts.front()->getNamespace()); + Idx->Size = Size; + Idx->Offset = isContinuous ? Parts.front()->Offset : -1; + Idx->ConcatenationOf.assign(Parts.begin(), Parts.end()); + return Idx; +} + +void CodeGenRegBank::computeComposites() { + using RegMap = std::map<const CodeGenRegister*, const CodeGenRegister*>; + + // Subreg -> { Reg->Reg }, where the right-hand side is the mapping from + // register to (sub)register associated with the action of the left-hand + // side subregister. + std::map<const CodeGenSubRegIndex*, RegMap> SubRegAction; + for (const CodeGenRegister &R : Registers) { + const CodeGenRegister::SubRegMap &SM = R.getSubRegs(); + for (std::pair<const CodeGenSubRegIndex*, const CodeGenRegister*> P : SM) + SubRegAction[P.first].insert({&R, P.second}); + } + + // Calculate the composition of two subregisters as compositions of their + // associated actions. + auto compose = [&SubRegAction] (const CodeGenSubRegIndex *Sub1, + const CodeGenSubRegIndex *Sub2) { + RegMap C; + const RegMap &Img1 = SubRegAction.at(Sub1); + const RegMap &Img2 = SubRegAction.at(Sub2); + for (std::pair<const CodeGenRegister*, const CodeGenRegister*> P : Img1) { + auto F = Img2.find(P.second); + if (F != Img2.end()) + C.insert({P.first, F->second}); + } + return C; + }; + + // Check if the two maps agree on the intersection of their domains. + auto agree = [] (const RegMap &Map1, const RegMap &Map2) { + // Technically speaking, an empty map agrees with any other map, but + // this could flag false positives. We're interested in non-vacuous + // agreements. + if (Map1.empty() || Map2.empty()) + return false; + for (std::pair<const CodeGenRegister*, const CodeGenRegister*> P : Map1) { + auto F = Map2.find(P.first); + if (F == Map2.end() || P.second != F->second) + return false; + } + return true; + }; + + using CompositePair = std::pair<const CodeGenSubRegIndex*, + const CodeGenSubRegIndex*>; + SmallSet<CompositePair,4> UserDefined; + for (const CodeGenSubRegIndex &Idx : SubRegIndices) + for (auto P : Idx.getComposites()) + UserDefined.insert(std::make_pair(&Idx, P.first)); + + // Keep track of TopoSigs visited. We only need to visit each TopoSig once, + // and many registers will share TopoSigs on regular architectures. + BitVector TopoSigs(getNumTopoSigs()); + + for (const auto &Reg1 : Registers) { + // Skip identical subreg structures already processed. + if (TopoSigs.test(Reg1.getTopoSig())) + continue; + TopoSigs.set(Reg1.getTopoSig()); + + const CodeGenRegister::SubRegMap &SRM1 = Reg1.getSubRegs(); + for (CodeGenRegister::SubRegMap::const_iterator i1 = SRM1.begin(), + e1 = SRM1.end(); i1 != e1; ++i1) { + CodeGenSubRegIndex *Idx1 = i1->first; + CodeGenRegister *Reg2 = i1->second; + // Ignore identity compositions. + if (&Reg1 == Reg2) + continue; + const CodeGenRegister::SubRegMap &SRM2 = Reg2->getSubRegs(); + // Try composing Idx1 with another SubRegIndex. + for (CodeGenRegister::SubRegMap::const_iterator i2 = SRM2.begin(), + e2 = SRM2.end(); i2 != e2; ++i2) { + CodeGenSubRegIndex *Idx2 = i2->first; + CodeGenRegister *Reg3 = i2->second; + // Ignore identity compositions. + if (Reg2 == Reg3) + continue; + // OK Reg1:IdxPair == Reg3. Find the index with Reg:Idx == Reg3. + CodeGenSubRegIndex *Idx3 = Reg1.getSubRegIndex(Reg3); + assert(Idx3 && "Sub-register doesn't have an index"); + + // Conflicting composition? Emit a warning but allow it. + if (CodeGenSubRegIndex *Prev = Idx1->addComposite(Idx2, Idx3)) { + // If the composition was not user-defined, always emit a warning. + if (!UserDefined.count({Idx1, Idx2}) || + agree(compose(Idx1, Idx2), SubRegAction.at(Idx3))) + PrintWarning(Twine("SubRegIndex ") + Idx1->getQualifiedName() + + " and " + Idx2->getQualifiedName() + + " compose ambiguously as " + Prev->getQualifiedName() + + " or " + Idx3->getQualifiedName()); + } + } + } + } +} + +// Compute lane masks. This is similar to register units, but at the +// sub-register index level. Each bit in the lane mask is like a register unit +// class, and two lane masks will have a bit in common if two sub-register +// indices overlap in some register. +// +// Conservatively share a lane mask bit if two sub-register indices overlap in +// some registers, but not in others. That shouldn't happen a lot. +void CodeGenRegBank::computeSubRegLaneMasks() { + // First assign individual bits to all the leaf indices. + unsigned Bit = 0; + // Determine mask of lanes that cover their registers. + CoveringLanes = LaneBitmask::getAll(); + for (auto &Idx : SubRegIndices) { + if (Idx.getComposites().empty()) { + if (Bit > LaneBitmask::BitWidth) { + PrintFatalError( + Twine("Ran out of lanemask bits to represent subregister ") + + Idx.getName()); + } + Idx.LaneMask = LaneBitmask::getLane(Bit); + ++Bit; + } else { + Idx.LaneMask = LaneBitmask::getNone(); + } + } + + // Compute transformation sequences for composeSubRegIndexLaneMask. The idea + // here is that for each possible target subregister we look at the leafs + // in the subregister graph that compose for this target and create + // transformation sequences for the lanemasks. Each step in the sequence + // consists of a bitmask and a bitrotate operation. As the rotation amounts + // are usually the same for many subregisters we can easily combine the steps + // by combining the masks. + for (const auto &Idx : SubRegIndices) { + const auto &Composites = Idx.getComposites(); + auto &LaneTransforms = Idx.CompositionLaneMaskTransform; + + if (Composites.empty()) { + // Moving from a class with no subregisters we just had a single lane: + // The subregister must be a leaf subregister and only occupies 1 bit. + // Move the bit from the class without subregisters into that position. + unsigned DstBit = Idx.LaneMask.getHighestLane(); + assert(Idx.LaneMask == LaneBitmask::getLane(DstBit) && + "Must be a leaf subregister"); + MaskRolPair MaskRol = { LaneBitmask::getLane(0), (uint8_t)DstBit }; + LaneTransforms.push_back(MaskRol); + } else { + // Go through all leaf subregisters and find the ones that compose with + // Idx. These make out all possible valid bits in the lane mask we want to + // transform. Looking only at the leafs ensure that only a single bit in + // the mask is set. + unsigned NextBit = 0; + for (auto &Idx2 : SubRegIndices) { + // Skip non-leaf subregisters. + if (!Idx2.getComposites().empty()) + continue; + // Replicate the behaviour from the lane mask generation loop above. + unsigned SrcBit = NextBit; + LaneBitmask SrcMask = LaneBitmask::getLane(SrcBit); + if (NextBit < LaneBitmask::BitWidth-1) + ++NextBit; + assert(Idx2.LaneMask == SrcMask); + + // Get the composed subregister if there is any. + auto C = Composites.find(&Idx2); + if (C == Composites.end()) + continue; + const CodeGenSubRegIndex *Composite = C->second; + // The Composed subreg should be a leaf subreg too + assert(Composite->getComposites().empty()); + + // Create Mask+Rotate operation and merge with existing ops if possible. + unsigned DstBit = Composite->LaneMask.getHighestLane(); + int Shift = DstBit - SrcBit; + uint8_t RotateLeft = Shift >= 0 ? (uint8_t)Shift + : LaneBitmask::BitWidth + Shift; + for (auto &I : LaneTransforms) { + if (I.RotateLeft == RotateLeft) { + I.Mask |= SrcMask; + SrcMask = LaneBitmask::getNone(); + } + } + if (SrcMask.any()) { + MaskRolPair MaskRol = { SrcMask, RotateLeft }; + LaneTransforms.push_back(MaskRol); + } + } + } + + // Optimize if the transformation consists of one step only: Set mask to + // 0xffffffff (including some irrelevant invalid bits) so that it should + // merge with more entries later while compressing the table. + if (LaneTransforms.size() == 1) + LaneTransforms[0].Mask = LaneBitmask::getAll(); + + // Further compression optimization: For invalid compositions resulting + // in a sequence with 0 entries we can just pick any other. Choose + // Mask 0xffffffff with Rotation 0. + if (LaneTransforms.size() == 0) { + MaskRolPair P = { LaneBitmask::getAll(), 0 }; + LaneTransforms.push_back(P); + } + } + + // FIXME: What if ad-hoc aliasing introduces overlaps that aren't represented + // by the sub-register graph? This doesn't occur in any known targets. + + // Inherit lanes from composites. + for (const auto &Idx : SubRegIndices) { + LaneBitmask Mask = Idx.computeLaneMask(); + // If some super-registers without CoveredBySubRegs use this index, we can + // no longer assume that the lanes are covering their registers. + if (!Idx.AllSuperRegsCovered) + CoveringLanes &= ~Mask; + } + + // Compute lane mask combinations for register classes. + for (auto &RegClass : RegClasses) { + LaneBitmask LaneMask; + for (const auto &SubRegIndex : SubRegIndices) { + if (RegClass.getSubClassWithSubReg(&SubRegIndex) == nullptr) + continue; + LaneMask |= SubRegIndex.LaneMask; + } + + // For classes without any subregisters set LaneMask to 1 instead of 0. + // This makes it easier for client code to handle classes uniformly. + if (LaneMask.none()) + LaneMask = LaneBitmask::getLane(0); + + RegClass.LaneMask = LaneMask; + } +} + +namespace { + +// UberRegSet is a helper class for computeRegUnitWeights. Each UberRegSet is +// the transitive closure of the union of overlapping register +// classes. Together, the UberRegSets form a partition of the registers. If we +// consider overlapping register classes to be connected, then each UberRegSet +// is a set of connected components. +// +// An UberRegSet will likely be a horizontal slice of register names of +// the same width. Nontrivial subregisters should then be in a separate +// UberRegSet. But this property isn't required for valid computation of +// register unit weights. +// +// A Weight field caches the max per-register unit weight in each UberRegSet. +// +// A set of SingularDeterminants flags single units of some register in this set +// for which the unit weight equals the set weight. These units should not have +// their weight increased. +struct UberRegSet { + CodeGenRegister::Vec Regs; + unsigned Weight = 0; + CodeGenRegister::RegUnitList SingularDeterminants; + + UberRegSet() = default; +}; + +} // end anonymous namespace + +// Partition registers into UberRegSets, where each set is the transitive +// closure of the union of overlapping register classes. +// +// UberRegSets[0] is a special non-allocatable set. +static void computeUberSets(std::vector<UberRegSet> &UberSets, + std::vector<UberRegSet*> &RegSets, + CodeGenRegBank &RegBank) { + const auto &Registers = RegBank.getRegisters(); + + // The Register EnumValue is one greater than its index into Registers. + assert(Registers.size() == Registers.back().EnumValue && + "register enum value mismatch"); + + // For simplicitly make the SetID the same as EnumValue. + IntEqClasses UberSetIDs(Registers.size()+1); + std::set<unsigned> AllocatableRegs; + for (auto &RegClass : RegBank.getRegClasses()) { + if (!RegClass.Allocatable) + continue; + + const CodeGenRegister::Vec &Regs = RegClass.getMembers(); + if (Regs.empty()) + continue; + + unsigned USetID = UberSetIDs.findLeader((*Regs.begin())->EnumValue); + assert(USetID && "register number 0 is invalid"); + + AllocatableRegs.insert((*Regs.begin())->EnumValue); + for (auto I = std::next(Regs.begin()), E = Regs.end(); I != E; ++I) { + AllocatableRegs.insert((*I)->EnumValue); + UberSetIDs.join(USetID, (*I)->EnumValue); + } + } + // Combine non-allocatable regs. + for (const auto &Reg : Registers) { + unsigned RegNum = Reg.EnumValue; + if (AllocatableRegs.count(RegNum)) + continue; + + UberSetIDs.join(0, RegNum); + } + UberSetIDs.compress(); + + // Make the first UberSet a special unallocatable set. + unsigned ZeroID = UberSetIDs[0]; + + // Insert Registers into the UberSets formed by union-find. + // Do not resize after this. + UberSets.resize(UberSetIDs.getNumClasses()); + unsigned i = 0; + for (const CodeGenRegister &Reg : Registers) { + unsigned USetID = UberSetIDs[Reg.EnumValue]; + if (!USetID) + USetID = ZeroID; + else if (USetID == ZeroID) + USetID = 0; + + UberRegSet *USet = &UberSets[USetID]; + USet->Regs.push_back(&Reg); + sortAndUniqueRegisters(USet->Regs); + RegSets[i++] = USet; + } +} + +// Recompute each UberSet weight after changing unit weights. +static void computeUberWeights(std::vector<UberRegSet> &UberSets, + CodeGenRegBank &RegBank) { + // Skip the first unallocatable set. + for (std::vector<UberRegSet>::iterator I = std::next(UberSets.begin()), + E = UberSets.end(); I != E; ++I) { + + // Initialize all unit weights in this set, and remember the max units/reg. + const CodeGenRegister *Reg = nullptr; + unsigned MaxWeight = 0, Weight = 0; + for (RegUnitIterator UnitI(I->Regs); UnitI.isValid(); ++UnitI) { + if (Reg != UnitI.getReg()) { + if (Weight > MaxWeight) + MaxWeight = Weight; + Reg = UnitI.getReg(); + Weight = 0; + } + if (!RegBank.getRegUnit(*UnitI).Artificial) { + unsigned UWeight = RegBank.getRegUnit(*UnitI).Weight; + if (!UWeight) { + UWeight = 1; + RegBank.increaseRegUnitWeight(*UnitI, UWeight); + } + Weight += UWeight; + } + } + if (Weight > MaxWeight) + MaxWeight = Weight; + if (I->Weight != MaxWeight) { + LLVM_DEBUG(dbgs() << "UberSet " << I - UberSets.begin() << " Weight " + << MaxWeight; + for (auto &Unit + : I->Regs) dbgs() + << " " << Unit->getName(); + dbgs() << "\n"); + // Update the set weight. + I->Weight = MaxWeight; + } + + // Find singular determinants. + for (const auto R : I->Regs) { + if (R->getRegUnits().count() == 1 && R->getWeight(RegBank) == I->Weight) { + I->SingularDeterminants |= R->getRegUnits(); + } + } + } +} + +// normalizeWeight is a computeRegUnitWeights helper that adjusts the weight of +// a register and its subregisters so that they have the same weight as their +// UberSet. Self-recursion processes the subregister tree in postorder so +// subregisters are normalized first. +// +// Side effects: +// - creates new adopted register units +// - causes superregisters to inherit adopted units +// - increases the weight of "singular" units +// - induces recomputation of UberWeights. +static bool normalizeWeight(CodeGenRegister *Reg, + std::vector<UberRegSet> &UberSets, + std::vector<UberRegSet*> &RegSets, + BitVector &NormalRegs, + CodeGenRegister::RegUnitList &NormalUnits, + CodeGenRegBank &RegBank) { + NormalRegs.resize(std::max(Reg->EnumValue + 1, NormalRegs.size())); + if (NormalRegs.test(Reg->EnumValue)) + return false; + NormalRegs.set(Reg->EnumValue); + + bool Changed = false; + const CodeGenRegister::SubRegMap &SRM = Reg->getSubRegs(); + for (CodeGenRegister::SubRegMap::const_iterator SRI = SRM.begin(), + SRE = SRM.end(); SRI != SRE; ++SRI) { + if (SRI->second == Reg) + continue; // self-cycles happen + + Changed |= normalizeWeight(SRI->second, UberSets, RegSets, + NormalRegs, NormalUnits, RegBank); + } + // Postorder register normalization. + + // Inherit register units newly adopted by subregisters. + if (Reg->inheritRegUnits(RegBank)) + computeUberWeights(UberSets, RegBank); + + // Check if this register is too skinny for its UberRegSet. + UberRegSet *UberSet = RegSets[RegBank.getRegIndex(Reg)]; + + unsigned RegWeight = Reg->getWeight(RegBank); + if (UberSet->Weight > RegWeight) { + // A register unit's weight can be adjusted only if it is the singular unit + // for this register, has not been used to normalize a subregister's set, + // and has not already been used to singularly determine this UberRegSet. + unsigned AdjustUnit = *Reg->getRegUnits().begin(); + if (Reg->getRegUnits().count() != 1 + || hasRegUnit(NormalUnits, AdjustUnit) + || hasRegUnit(UberSet->SingularDeterminants, AdjustUnit)) { + // We don't have an adjustable unit, so adopt a new one. + AdjustUnit = RegBank.newRegUnit(UberSet->Weight - RegWeight); + Reg->adoptRegUnit(AdjustUnit); + // Adopting a unit does not immediately require recomputing set weights. + } + else { + // Adjust the existing single unit. + if (!RegBank.getRegUnit(AdjustUnit).Artificial) + RegBank.increaseRegUnitWeight(AdjustUnit, UberSet->Weight - RegWeight); + // The unit may be shared among sets and registers within this set. + computeUberWeights(UberSets, RegBank); + } + Changed = true; + } + + // Mark these units normalized so superregisters can't change their weights. + NormalUnits |= Reg->getRegUnits(); + + return Changed; +} + +// Compute a weight for each register unit created during getSubRegs. +// +// The goal is that two registers in the same class will have the same weight, +// where each register's weight is defined as sum of its units' weights. +void CodeGenRegBank::computeRegUnitWeights() { + std::vector<UberRegSet> UberSets; + std::vector<UberRegSet*> RegSets(Registers.size()); + computeUberSets(UberSets, RegSets, *this); + // UberSets and RegSets are now immutable. + + computeUberWeights(UberSets, *this); + + // Iterate over each Register, normalizing the unit weights until reaching + // a fix point. + unsigned NumIters = 0; + for (bool Changed = true; Changed; ++NumIters) { + assert(NumIters <= NumNativeRegUnits && "Runaway register unit weights"); + Changed = false; + for (auto &Reg : Registers) { + CodeGenRegister::RegUnitList NormalUnits; + BitVector NormalRegs; + Changed |= normalizeWeight(&Reg, UberSets, RegSets, NormalRegs, + NormalUnits, *this); + } + } +} + +// Find a set in UniqueSets with the same elements as Set. +// Return an iterator into UniqueSets. +static std::vector<RegUnitSet>::const_iterator +findRegUnitSet(const std::vector<RegUnitSet> &UniqueSets, + const RegUnitSet &Set) { + std::vector<RegUnitSet>::const_iterator + I = UniqueSets.begin(), E = UniqueSets.end(); + for(;I != E; ++I) { + if (I->Units == Set.Units) + break; + } + return I; +} + +// Return true if the RUSubSet is a subset of RUSuperSet. +static bool isRegUnitSubSet(const std::vector<unsigned> &RUSubSet, + const std::vector<unsigned> &RUSuperSet) { + return std::includes(RUSuperSet.begin(), RUSuperSet.end(), + RUSubSet.begin(), RUSubSet.end()); +} + +/// Iteratively prune unit sets. Prune subsets that are close to the superset, +/// but with one or two registers removed. We occasionally have registers like +/// APSR and PC thrown in with the general registers. We also see many +/// special-purpose register subsets, such as tail-call and Thumb +/// encodings. Generating all possible overlapping sets is combinatorial and +/// overkill for modeling pressure. Ideally we could fix this statically in +/// tablegen by (1) having the target define register classes that only include +/// the allocatable registers and marking other classes as non-allocatable and +/// (2) having a way to mark special purpose classes as "don't-care" classes for +/// the purpose of pressure. However, we make an attempt to handle targets that +/// are not nicely defined by merging nearly identical register unit sets +/// statically. This generates smaller tables. Then, dynamically, we adjust the +/// set limit by filtering the reserved registers. +/// +/// Merge sets only if the units have the same weight. For example, on ARM, +/// Q-tuples with ssub index 0 include all S regs but also include D16+. We +/// should not expand the S set to include D regs. +void CodeGenRegBank::pruneUnitSets() { + assert(RegClassUnitSets.empty() && "this invalidates RegClassUnitSets"); + + // Form an equivalence class of UnitSets with no significant difference. + std::vector<unsigned> SuperSetIDs; + for (unsigned SubIdx = 0, EndIdx = RegUnitSets.size(); + SubIdx != EndIdx; ++SubIdx) { + const RegUnitSet &SubSet = RegUnitSets[SubIdx]; + unsigned SuperIdx = 0; + for (; SuperIdx != EndIdx; ++SuperIdx) { + if (SuperIdx == SubIdx) + continue; + + unsigned UnitWeight = RegUnits[SubSet.Units[0]].Weight; + const RegUnitSet &SuperSet = RegUnitSets[SuperIdx]; + if (isRegUnitSubSet(SubSet.Units, SuperSet.Units) + && (SubSet.Units.size() + 3 > SuperSet.Units.size()) + && UnitWeight == RegUnits[SuperSet.Units[0]].Weight + && UnitWeight == RegUnits[SuperSet.Units.back()].Weight) { + LLVM_DEBUG(dbgs() << "UnitSet " << SubIdx << " subsumed by " << SuperIdx + << "\n"); + // We can pick any of the set names for the merged set. Go for the + // shortest one to avoid picking the name of one of the classes that are + // artificially created by tablegen. So "FPR128_lo" instead of + // "QQQQ_with_qsub3_in_FPR128_lo". + if (RegUnitSets[SubIdx].Name.size() < RegUnitSets[SuperIdx].Name.size()) + RegUnitSets[SuperIdx].Name = RegUnitSets[SubIdx].Name; + break; + } + } + if (SuperIdx == EndIdx) + SuperSetIDs.push_back(SubIdx); + } + // Populate PrunedUnitSets with each equivalence class's superset. + std::vector<RegUnitSet> PrunedUnitSets(SuperSetIDs.size()); + for (unsigned i = 0, e = SuperSetIDs.size(); i != e; ++i) { + unsigned SuperIdx = SuperSetIDs[i]; + PrunedUnitSets[i].Name = RegUnitSets[SuperIdx].Name; + PrunedUnitSets[i].Units.swap(RegUnitSets[SuperIdx].Units); + } + RegUnitSets.swap(PrunedUnitSets); +} + +// Create a RegUnitSet for each RegClass that contains all units in the class +// including adopted units that are necessary to model register pressure. Then +// iteratively compute RegUnitSets such that the union of any two overlapping +// RegUnitSets is repreresented. +// +// RegisterInfoEmitter will map each RegClass to its RegUnitClass and any +// RegUnitSet that is a superset of that RegUnitClass. +void CodeGenRegBank::computeRegUnitSets() { + assert(RegUnitSets.empty() && "dirty RegUnitSets"); + + // Compute a unique RegUnitSet for each RegClass. + auto &RegClasses = getRegClasses(); + for (auto &RC : RegClasses) { + if (!RC.Allocatable || RC.Artificial || !RC.GeneratePressureSet) + continue; + + // Speculatively grow the RegUnitSets to hold the new set. + RegUnitSets.resize(RegUnitSets.size() + 1); + RegUnitSets.back().Name = RC.getName(); + + // Compute a sorted list of units in this class. + RC.buildRegUnitSet(*this, RegUnitSets.back().Units); + + // Find an existing RegUnitSet. + std::vector<RegUnitSet>::const_iterator SetI = + findRegUnitSet(RegUnitSets, RegUnitSets.back()); + if (SetI != std::prev(RegUnitSets.end())) + RegUnitSets.pop_back(); + } + + LLVM_DEBUG(dbgs() << "\nBefore pruning:\n"; for (unsigned USIdx = 0, + USEnd = RegUnitSets.size(); + USIdx < USEnd; ++USIdx) { + dbgs() << "UnitSet " << USIdx << " " << RegUnitSets[USIdx].Name << ":"; + for (auto &U : RegUnitSets[USIdx].Units) + printRegUnitName(U); + dbgs() << "\n"; + }); + + // Iteratively prune unit sets. + pruneUnitSets(); + + LLVM_DEBUG(dbgs() << "\nBefore union:\n"; for (unsigned USIdx = 0, + USEnd = RegUnitSets.size(); + USIdx < USEnd; ++USIdx) { + dbgs() << "UnitSet " << USIdx << " " << RegUnitSets[USIdx].Name << ":"; + for (auto &U : RegUnitSets[USIdx].Units) + printRegUnitName(U); + dbgs() << "\n"; + } dbgs() << "\nUnion sets:\n"); + + // Iterate over all unit sets, including new ones added by this loop. + unsigned NumRegUnitSubSets = RegUnitSets.size(); + for (unsigned Idx = 0, EndIdx = RegUnitSets.size(); Idx != EndIdx; ++Idx) { + // In theory, this is combinatorial. In practice, it needs to be bounded + // by a small number of sets for regpressure to be efficient. + // If the assert is hit, we need to implement pruning. + assert(Idx < (2*NumRegUnitSubSets) && "runaway unit set inference"); + + // Compare new sets with all original classes. + for (unsigned SearchIdx = (Idx >= NumRegUnitSubSets) ? 0 : Idx+1; + SearchIdx != EndIdx; ++SearchIdx) { + std::set<unsigned> Intersection; + std::set_intersection(RegUnitSets[Idx].Units.begin(), + RegUnitSets[Idx].Units.end(), + RegUnitSets[SearchIdx].Units.begin(), + RegUnitSets[SearchIdx].Units.end(), + std::inserter(Intersection, Intersection.begin())); + if (Intersection.empty()) + continue; + + // Speculatively grow the RegUnitSets to hold the new set. + RegUnitSets.resize(RegUnitSets.size() + 1); + RegUnitSets.back().Name = + RegUnitSets[Idx].Name + "_with_" + RegUnitSets[SearchIdx].Name; + + std::set_union(RegUnitSets[Idx].Units.begin(), + RegUnitSets[Idx].Units.end(), + RegUnitSets[SearchIdx].Units.begin(), + RegUnitSets[SearchIdx].Units.end(), + std::inserter(RegUnitSets.back().Units, + RegUnitSets.back().Units.begin())); + + // Find an existing RegUnitSet, or add the union to the unique sets. + std::vector<RegUnitSet>::const_iterator SetI = + findRegUnitSet(RegUnitSets, RegUnitSets.back()); + if (SetI != std::prev(RegUnitSets.end())) + RegUnitSets.pop_back(); + else { + LLVM_DEBUG(dbgs() << "UnitSet " << RegUnitSets.size() - 1 << " " + << RegUnitSets.back().Name << ":"; + for (auto &U + : RegUnitSets.back().Units) printRegUnitName(U); + dbgs() << "\n";); + } + } + } + + // Iteratively prune unit sets after inferring supersets. + pruneUnitSets(); + + LLVM_DEBUG( + dbgs() << "\n"; for (unsigned USIdx = 0, USEnd = RegUnitSets.size(); + USIdx < USEnd; ++USIdx) { + dbgs() << "UnitSet " << USIdx << " " << RegUnitSets[USIdx].Name << ":"; + for (auto &U : RegUnitSets[USIdx].Units) + printRegUnitName(U); + dbgs() << "\n"; + }); + + // For each register class, list the UnitSets that are supersets. + RegClassUnitSets.resize(RegClasses.size()); + int RCIdx = -1; + for (auto &RC : RegClasses) { + ++RCIdx; + if (!RC.Allocatable) + continue; + + // Recompute the sorted list of units in this class. + std::vector<unsigned> RCRegUnits; + RC.buildRegUnitSet(*this, RCRegUnits); + + // Don't increase pressure for unallocatable regclasses. + if (RCRegUnits.empty()) + continue; + + LLVM_DEBUG(dbgs() << "RC " << RC.getName() << " Units: \n"; + for (auto U + : RCRegUnits) printRegUnitName(U); + dbgs() << "\n UnitSetIDs:"); + + // Find all supersets. + for (unsigned USIdx = 0, USEnd = RegUnitSets.size(); + USIdx != USEnd; ++USIdx) { + if (isRegUnitSubSet(RCRegUnits, RegUnitSets[USIdx].Units)) { + LLVM_DEBUG(dbgs() << " " << USIdx); + RegClassUnitSets[RCIdx].push_back(USIdx); + } + } + LLVM_DEBUG(dbgs() << "\n"); + assert(!RegClassUnitSets[RCIdx].empty() && "missing unit set for regclass"); + } + + // For each register unit, ensure that we have the list of UnitSets that + // contain the unit. Normally, this matches an existing list of UnitSets for a + // register class. If not, we create a new entry in RegClassUnitSets as a + // "fake" register class. + for (unsigned UnitIdx = 0, UnitEnd = NumNativeRegUnits; + UnitIdx < UnitEnd; ++UnitIdx) { + std::vector<unsigned> RUSets; + for (unsigned i = 0, e = RegUnitSets.size(); i != e; ++i) { + RegUnitSet &RUSet = RegUnitSets[i]; + if (!is_contained(RUSet.Units, UnitIdx)) + continue; + RUSets.push_back(i); + } + unsigned RCUnitSetsIdx = 0; + for (unsigned e = RegClassUnitSets.size(); + RCUnitSetsIdx != e; ++RCUnitSetsIdx) { + if (RegClassUnitSets[RCUnitSetsIdx] == RUSets) { + break; + } + } + RegUnits[UnitIdx].RegClassUnitSetsIdx = RCUnitSetsIdx; + if (RCUnitSetsIdx == RegClassUnitSets.size()) { + // Create a new list of UnitSets as a "fake" register class. + RegClassUnitSets.resize(RCUnitSetsIdx + 1); + RegClassUnitSets[RCUnitSetsIdx].swap(RUSets); + } + } +} + +void CodeGenRegBank::computeRegUnitLaneMasks() { + for (auto &Register : Registers) { + // Create an initial lane mask for all register units. + const auto &RegUnits = Register.getRegUnits(); + CodeGenRegister::RegUnitLaneMaskList + RegUnitLaneMasks(RegUnits.count(), LaneBitmask::getNone()); + // Iterate through SubRegisters. + typedef CodeGenRegister::SubRegMap SubRegMap; + const SubRegMap &SubRegs = Register.getSubRegs(); + for (SubRegMap::const_iterator S = SubRegs.begin(), + SE = SubRegs.end(); S != SE; ++S) { + CodeGenRegister *SubReg = S->second; + // Ignore non-leaf subregisters, their lane masks are fully covered by + // the leaf subregisters anyway. + if (!SubReg->getSubRegs().empty()) + continue; + CodeGenSubRegIndex *SubRegIndex = S->first; + const CodeGenRegister *SubRegister = S->second; + LaneBitmask LaneMask = SubRegIndex->LaneMask; + // Distribute LaneMask to Register Units touched. + for (unsigned SUI : SubRegister->getRegUnits()) { + bool Found = false; + unsigned u = 0; + for (unsigned RU : RegUnits) { + if (SUI == RU) { + RegUnitLaneMasks[u] |= LaneMask; + assert(!Found); + Found = true; + } + ++u; + } + (void)Found; + assert(Found); + } + } + Register.setRegUnitLaneMasks(RegUnitLaneMasks); + } +} + +void CodeGenRegBank::computeDerivedInfo() { + computeComposites(); + computeSubRegLaneMasks(); + + // Compute a weight for each register unit created during getSubRegs. + // This may create adopted register units (with unit # >= NumNativeRegUnits). + computeRegUnitWeights(); + + // Compute a unique set of RegUnitSets. One for each RegClass and inferred + // supersets for the union of overlapping sets. + computeRegUnitSets(); + + computeRegUnitLaneMasks(); + + // Compute register class HasDisjunctSubRegs/CoveredBySubRegs flag. + for (CodeGenRegisterClass &RC : RegClasses) { + RC.HasDisjunctSubRegs = false; + RC.CoveredBySubRegs = true; + for (const CodeGenRegister *Reg : RC.getMembers()) { + RC.HasDisjunctSubRegs |= Reg->HasDisjunctSubRegs; + RC.CoveredBySubRegs &= Reg->CoveredBySubRegs; + } + } + + // Get the weight of each set. + for (unsigned Idx = 0, EndIdx = RegUnitSets.size(); Idx != EndIdx; ++Idx) + RegUnitSets[Idx].Weight = getRegUnitSetWeight(RegUnitSets[Idx].Units); + + // Find the order of each set. + RegUnitSetOrder.reserve(RegUnitSets.size()); + for (unsigned Idx = 0, EndIdx = RegUnitSets.size(); Idx != EndIdx; ++Idx) + RegUnitSetOrder.push_back(Idx); + + llvm::stable_sort(RegUnitSetOrder, [this](unsigned ID1, unsigned ID2) { + return getRegPressureSet(ID1).Units.size() < + getRegPressureSet(ID2).Units.size(); + }); + for (unsigned Idx = 0, EndIdx = RegUnitSets.size(); Idx != EndIdx; ++Idx) { + RegUnitSets[RegUnitSetOrder[Idx]].Order = Idx; + } +} + +// +// Synthesize missing register class intersections. +// +// Make sure that sub-classes of RC exists such that getCommonSubClass(RC, X) +// returns a maximal register class for all X. +// +void CodeGenRegBank::inferCommonSubClass(CodeGenRegisterClass *RC) { + assert(!RegClasses.empty()); + // Stash the iterator to the last element so that this loop doesn't visit + // elements added by the getOrCreateSubClass call within it. + for (auto I = RegClasses.begin(), E = std::prev(RegClasses.end()); + I != std::next(E); ++I) { + CodeGenRegisterClass *RC1 = RC; + CodeGenRegisterClass *RC2 = &*I; + if (RC1 == RC2) + continue; + + // Compute the set intersection of RC1 and RC2. + const CodeGenRegister::Vec &Memb1 = RC1->getMembers(); + const CodeGenRegister::Vec &Memb2 = RC2->getMembers(); + CodeGenRegister::Vec Intersection; + std::set_intersection(Memb1.begin(), Memb1.end(), Memb2.begin(), + Memb2.end(), + std::inserter(Intersection, Intersection.begin()), + deref<std::less<>>()); + + // Skip disjoint class pairs. + if (Intersection.empty()) + continue; + + // If RC1 and RC2 have different spill sizes or alignments, use the + // stricter one for sub-classing. If they are equal, prefer RC1. + if (RC2->RSI.hasStricterSpillThan(RC1->RSI)) + std::swap(RC1, RC2); + + getOrCreateSubClass(RC1, &Intersection, + RC1->getName() + "_and_" + RC2->getName()); + } +} + +// +// Synthesize missing sub-classes for getSubClassWithSubReg(). +// +// Make sure that the set of registers in RC with a given SubIdx sub-register +// form a register class. Update RC->SubClassWithSubReg. +// +void CodeGenRegBank::inferSubClassWithSubReg(CodeGenRegisterClass *RC) { + // Map SubRegIndex to set of registers in RC supporting that SubRegIndex. + typedef std::map<const CodeGenSubRegIndex *, CodeGenRegister::Vec, + deref<std::less<>>> + SubReg2SetMap; + + // Compute the set of registers supporting each SubRegIndex. + SubReg2SetMap SRSets; + for (const auto R : RC->getMembers()) { + if (R->Artificial) + continue; + const CodeGenRegister::SubRegMap &SRM = R->getSubRegs(); + for (CodeGenRegister::SubRegMap::const_iterator I = SRM.begin(), + E = SRM.end(); I != E; ++I) { + if (!I->first->Artificial) + SRSets[I->first].push_back(R); + } + } + + for (auto I : SRSets) + sortAndUniqueRegisters(I.second); + + // Find matching classes for all SRSets entries. Iterate in SubRegIndex + // numerical order to visit synthetic indices last. + for (const auto &SubIdx : SubRegIndices) { + if (SubIdx.Artificial) + continue; + SubReg2SetMap::const_iterator I = SRSets.find(&SubIdx); + // Unsupported SubRegIndex. Skip it. + if (I == SRSets.end()) + continue; + // In most cases, all RC registers support the SubRegIndex. + if (I->second.size() == RC->getMembers().size()) { + RC->setSubClassWithSubReg(&SubIdx, RC); + continue; + } + // This is a real subset. See if we have a matching class. + CodeGenRegisterClass *SubRC = + getOrCreateSubClass(RC, &I->second, + RC->getName() + "_with_" + I->first->getName()); + RC->setSubClassWithSubReg(&SubIdx, SubRC); + } +} + +// +// Synthesize missing sub-classes of RC for getMatchingSuperRegClass(). +// +// Create sub-classes of RC such that getMatchingSuperRegClass(RC, SubIdx, X) +// has a maximal result for any SubIdx and any X >= FirstSubRegRC. +// + +void CodeGenRegBank::inferMatchingSuperRegClass(CodeGenRegisterClass *RC, + std::list<CodeGenRegisterClass>::iterator FirstSubRegRC) { + SmallVector<std::pair<const CodeGenRegister*, + const CodeGenRegister*>, 16> SSPairs; + BitVector TopoSigs(getNumTopoSigs()); + + // Iterate in SubRegIndex numerical order to visit synthetic indices last. + for (auto &SubIdx : SubRegIndices) { + // Skip indexes that aren't fully supported by RC's registers. This was + // computed by inferSubClassWithSubReg() above which should have been + // called first. + if (RC->getSubClassWithSubReg(&SubIdx) != RC) + continue; + + // Build list of (Super, Sub) pairs for this SubIdx. + SSPairs.clear(); + TopoSigs.reset(); + for (const auto Super : RC->getMembers()) { + const CodeGenRegister *Sub = Super->getSubRegs().find(&SubIdx)->second; + assert(Sub && "Missing sub-register"); + SSPairs.push_back(std::make_pair(Super, Sub)); + TopoSigs.set(Sub->getTopoSig()); + } + + // Iterate over sub-register class candidates. Ignore classes created by + // this loop. They will never be useful. + // Store an iterator to the last element (not end) so that this loop doesn't + // visit newly inserted elements. + assert(!RegClasses.empty()); + for (auto I = FirstSubRegRC, E = std::prev(RegClasses.end()); + I != std::next(E); ++I) { + CodeGenRegisterClass &SubRC = *I; + if (SubRC.Artificial) + continue; + // Topological shortcut: SubRC members have the wrong shape. + if (!TopoSigs.anyCommon(SubRC.getTopoSigs())) + continue; + // Compute the subset of RC that maps into SubRC. + CodeGenRegister::Vec SubSetVec; + for (unsigned i = 0, e = SSPairs.size(); i != e; ++i) + if (SubRC.contains(SSPairs[i].second)) + SubSetVec.push_back(SSPairs[i].first); + + if (SubSetVec.empty()) + continue; + + // RC injects completely into SubRC. + sortAndUniqueRegisters(SubSetVec); + if (SubSetVec.size() == SSPairs.size()) { + SubRC.addSuperRegClass(&SubIdx, RC); + continue; + } + + // Only a subset of RC maps into SubRC. Make sure it is represented by a + // class. + getOrCreateSubClass(RC, &SubSetVec, RC->getName() + "_with_" + + SubIdx.getName() + "_in_" + + SubRC.getName()); + } + } +} + +// +// Infer missing register classes. +// +void CodeGenRegBank::computeInferredRegisterClasses() { + assert(!RegClasses.empty()); + // When this function is called, the register classes have not been sorted + // and assigned EnumValues yet. That means getSubClasses(), + // getSuperClasses(), and hasSubClass() functions are defunct. + + // Use one-before-the-end so it doesn't move forward when new elements are + // added. + auto FirstNewRC = std::prev(RegClasses.end()); + + // Visit all register classes, including the ones being added by the loop. + // Watch out for iterator invalidation here. + for (auto I = RegClasses.begin(), E = RegClasses.end(); I != E; ++I) { + CodeGenRegisterClass *RC = &*I; + if (RC->Artificial) + continue; + + // Synthesize answers for getSubClassWithSubReg(). + inferSubClassWithSubReg(RC); + + // Synthesize answers for getCommonSubClass(). + inferCommonSubClass(RC); + + // Synthesize answers for getMatchingSuperRegClass(). + inferMatchingSuperRegClass(RC); + + // New register classes are created while this loop is running, and we need + // to visit all of them. I particular, inferMatchingSuperRegClass needs + // to match old super-register classes with sub-register classes created + // after inferMatchingSuperRegClass was called. At this point, + // inferMatchingSuperRegClass has checked SuperRC = [0..rci] with SubRC = + // [0..FirstNewRC). We need to cover SubRC = [FirstNewRC..rci]. + if (I == FirstNewRC) { + auto NextNewRC = std::prev(RegClasses.end()); + for (auto I2 = RegClasses.begin(), E2 = std::next(FirstNewRC); I2 != E2; + ++I2) + inferMatchingSuperRegClass(&*I2, E2); + FirstNewRC = NextNewRC; + } + } +} + +/// getRegisterClassForRegister - Find the register class that contains the +/// specified physical register. If the register is not in a register class, +/// return null. If the register is in multiple classes, and the classes have a +/// superset-subset relationship and the same set of types, return the +/// superclass. Otherwise return null. +const CodeGenRegisterClass* +CodeGenRegBank::getRegClassForRegister(Record *R) { + const CodeGenRegister *Reg = getReg(R); + const CodeGenRegisterClass *FoundRC = nullptr; + for (const auto &RC : getRegClasses()) { + if (!RC.contains(Reg)) + continue; + + // If this is the first class that contains the register, + // make a note of it and go on to the next class. + if (!FoundRC) { + FoundRC = &RC; + continue; + } + + // If a register's classes have different types, return null. + if (RC.getValueTypes() != FoundRC->getValueTypes()) + return nullptr; + + // Check to see if the previously found class that contains + // the register is a subclass of the current class. If so, + // prefer the superclass. + if (RC.hasSubClass(FoundRC)) { + FoundRC = &RC; + continue; + } + + // Check to see if the previously found class that contains + // the register is a superclass of the current class. If so, + // prefer the superclass. + if (FoundRC->hasSubClass(&RC)) + continue; + + // Multiple classes, and neither is a superclass of the other. + // Return null. + return nullptr; + } + return FoundRC; +} + +const CodeGenRegisterClass * +CodeGenRegBank::getMinimalPhysRegClass(Record *RegRecord, + ValueTypeByHwMode *VT) { + const CodeGenRegister *Reg = getReg(RegRecord); + const CodeGenRegisterClass *BestRC = nullptr; + for (const auto &RC : getRegClasses()) { + if ((!VT || RC.hasType(*VT)) && + RC.contains(Reg) && (!BestRC || BestRC->hasSubClass(&RC))) + BestRC = &RC; + } + + assert(BestRC && "Couldn't find the register class"); + return BestRC; +} + +BitVector CodeGenRegBank::computeCoveredRegisters(ArrayRef<Record*> Regs) { + SetVector<const CodeGenRegister*> Set; + + // First add Regs with all sub-registers. + for (unsigned i = 0, e = Regs.size(); i != e; ++i) { + CodeGenRegister *Reg = getReg(Regs[i]); + if (Set.insert(Reg)) + // Reg is new, add all sub-registers. + // The pre-ordering is not important here. + Reg->addSubRegsPreOrder(Set, *this); + } + + // Second, find all super-registers that are completely covered by the set. + for (unsigned i = 0; i != Set.size(); ++i) { + const CodeGenRegister::SuperRegList &SR = Set[i]->getSuperRegs(); + for (unsigned j = 0, e = SR.size(); j != e; ++j) { + const CodeGenRegister *Super = SR[j]; + if (!Super->CoveredBySubRegs || Set.count(Super)) + continue; + // This new super-register is covered by its sub-registers. + bool AllSubsInSet = true; + const CodeGenRegister::SubRegMap &SRM = Super->getSubRegs(); + for (CodeGenRegister::SubRegMap::const_iterator I = SRM.begin(), + E = SRM.end(); I != E; ++I) + if (!Set.count(I->second)) { + AllSubsInSet = false; + break; + } + // All sub-registers in Set, add Super as well. + // We will visit Super later to recheck its super-registers. + if (AllSubsInSet) + Set.insert(Super); + } + } + + // Convert to BitVector. + BitVector BV(Registers.size() + 1); + for (unsigned i = 0, e = Set.size(); i != e; ++i) + BV.set(Set[i]->EnumValue); + return BV; +} + +void CodeGenRegBank::printRegUnitName(unsigned Unit) const { + if (Unit < NumNativeRegUnits) + dbgs() << ' ' << RegUnits[Unit].Roots[0]->getName(); + else + dbgs() << " #" << Unit; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.h new file mode 100644 index 000000000000..2b200adef312 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenRegisters.h @@ -0,0 +1,807 @@ +//===- CodeGenRegisters.h - Register and RegisterClass Info -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines structures to encapsulate information gleaned from the +// target register and register class definitions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEGENREGISTERS_H +#define LLVM_UTILS_TABLEGEN_CODEGENREGISTERS_H + +#include "InfoByHwMode.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/SparseBitVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/MC/LaneBitmask.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MachineValueType.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/SetTheory.h" +#include <cassert> +#include <cstdint> +#include <deque> +#include <list> +#include <map> +#include <string> +#include <utility> +#include <vector> + +namespace llvm { + + class CodeGenRegBank; + template <typename T, typename Vector, typename Set> class SetVector; + + /// Used to encode a step in a register lane mask transformation. + /// Mask the bits specified in Mask, then rotate them Rol bits to the left + /// assuming a wraparound at 32bits. + struct MaskRolPair { + LaneBitmask Mask; + uint8_t RotateLeft; + + bool operator==(const MaskRolPair Other) const { + return Mask == Other.Mask && RotateLeft == Other.RotateLeft; + } + bool operator!=(const MaskRolPair Other) const { + return Mask != Other.Mask || RotateLeft != Other.RotateLeft; + } + }; + + /// CodeGenSubRegIndex - Represents a sub-register index. + class CodeGenSubRegIndex { + Record *const TheDef; + std::string Name; + std::string Namespace; + + public: + uint16_t Size; + uint16_t Offset; + const unsigned EnumValue; + mutable LaneBitmask LaneMask; + mutable SmallVector<MaskRolPair,1> CompositionLaneMaskTransform; + + /// A list of subregister indexes concatenated resulting in this + /// subregister index. This is the reverse of CodeGenRegBank::ConcatIdx. + SmallVector<CodeGenSubRegIndex*,4> ConcatenationOf; + + // Are all super-registers containing this SubRegIndex covered by their + // sub-registers? + bool AllSuperRegsCovered; + // A subregister index is "artificial" if every subregister obtained + // from applying this index is artificial. Artificial subregister + // indexes are not used to create new register classes. + bool Artificial; + + CodeGenSubRegIndex(Record *R, unsigned Enum); + CodeGenSubRegIndex(StringRef N, StringRef Nspace, unsigned Enum); + CodeGenSubRegIndex(CodeGenSubRegIndex&) = delete; + + const std::string &getName() const { return Name; } + const std::string &getNamespace() const { return Namespace; } + std::string getQualifiedName() const; + + // Map of composite subreg indices. + typedef std::map<CodeGenSubRegIndex *, CodeGenSubRegIndex *, + deref<std::less<>>> + CompMap; + + // Returns the subreg index that results from composing this with Idx. + // Returns NULL if this and Idx don't compose. + CodeGenSubRegIndex *compose(CodeGenSubRegIndex *Idx) const { + CompMap::const_iterator I = Composed.find(Idx); + return I == Composed.end() ? nullptr : I->second; + } + + // Add a composite subreg index: this+A = B. + // Return a conflicting composite, or NULL + CodeGenSubRegIndex *addComposite(CodeGenSubRegIndex *A, + CodeGenSubRegIndex *B) { + assert(A && B); + std::pair<CompMap::iterator, bool> Ins = + Composed.insert(std::make_pair(A, B)); + // Synthetic subreg indices that aren't contiguous (for instance ARM + // register tuples) don't have a bit range, so it's OK to let + // B->Offset == -1. For the other cases, accumulate the offset and set + // the size here. Only do so if there is no offset yet though. + if ((Offset != (uint16_t)-1 && A->Offset != (uint16_t)-1) && + (B->Offset == (uint16_t)-1)) { + B->Offset = Offset + A->Offset; + B->Size = A->Size; + } + return (Ins.second || Ins.first->second == B) ? nullptr + : Ins.first->second; + } + + // Update the composite maps of components specified in 'ComposedOf'. + void updateComponents(CodeGenRegBank&); + + // Return the map of composites. + const CompMap &getComposites() const { return Composed; } + + // Compute LaneMask from Composed. Return LaneMask. + LaneBitmask computeLaneMask() const; + + void setConcatenationOf(ArrayRef<CodeGenSubRegIndex*> Parts); + + /// Replaces subregister indexes in the `ConcatenationOf` list with + /// list of subregisters they are composed of (if any). Do this recursively. + void computeConcatTransitiveClosure(); + + bool operator<(const CodeGenSubRegIndex &RHS) const { + return this->EnumValue < RHS.EnumValue; + } + + private: + CompMap Composed; + }; + + /// CodeGenRegister - Represents a register definition. + struct CodeGenRegister { + Record *TheDef; + unsigned EnumValue; + unsigned CostPerUse; + bool CoveredBySubRegs; + bool HasDisjunctSubRegs; + bool Artificial; + + // Map SubRegIndex -> Register. + typedef std::map<CodeGenSubRegIndex *, CodeGenRegister *, + deref<std::less<>>> + SubRegMap; + + CodeGenRegister(Record *R, unsigned Enum); + + const StringRef getName() const; + + // Extract more information from TheDef. This is used to build an object + // graph after all CodeGenRegister objects have been created. + void buildObjectGraph(CodeGenRegBank&); + + // Lazily compute a map of all sub-registers. + // This includes unique entries for all sub-sub-registers. + const SubRegMap &computeSubRegs(CodeGenRegBank&); + + // Compute extra sub-registers by combining the existing sub-registers. + void computeSecondarySubRegs(CodeGenRegBank&); + + // Add this as a super-register to all sub-registers after the sub-register + // graph has been built. + void computeSuperRegs(CodeGenRegBank&); + + const SubRegMap &getSubRegs() const { + assert(SubRegsComplete && "Must precompute sub-registers"); + return SubRegs; + } + + // Add sub-registers to OSet following a pre-order defined by the .td file. + void addSubRegsPreOrder(SetVector<const CodeGenRegister*> &OSet, + CodeGenRegBank&) const; + + // Return the sub-register index naming Reg as a sub-register of this + // register. Returns NULL if Reg is not a sub-register. + CodeGenSubRegIndex *getSubRegIndex(const CodeGenRegister *Reg) const { + return SubReg2Idx.lookup(Reg); + } + + typedef std::vector<const CodeGenRegister*> SuperRegList; + + // Get the list of super-registers in topological order, small to large. + // This is valid after computeSubRegs visits all registers during RegBank + // construction. + const SuperRegList &getSuperRegs() const { + assert(SubRegsComplete && "Must precompute sub-registers"); + return SuperRegs; + } + + // Get the list of ad hoc aliases. The graph is symmetric, so the list + // contains all registers in 'Aliases', and all registers that mention this + // register in 'Aliases'. + ArrayRef<CodeGenRegister*> getExplicitAliases() const { + return ExplicitAliases; + } + + // Get the topological signature of this register. This is a small integer + // less than RegBank.getNumTopoSigs(). Registers with the same TopoSig have + // identical sub-register structure. That is, they support the same set of + // sub-register indices mapping to the same kind of sub-registers + // (TopoSig-wise). + unsigned getTopoSig() const { + assert(SuperRegsComplete && "TopoSigs haven't been computed yet."); + return TopoSig; + } + + // List of register units in ascending order. + typedef SparseBitVector<> RegUnitList; + typedef SmallVector<LaneBitmask, 16> RegUnitLaneMaskList; + + // How many entries in RegUnitList are native? + RegUnitList NativeRegUnits; + + // Get the list of register units. + // This is only valid after computeSubRegs() completes. + const RegUnitList &getRegUnits() const { return RegUnits; } + + ArrayRef<LaneBitmask> getRegUnitLaneMasks() const { + return makeArrayRef(RegUnitLaneMasks).slice(0, NativeRegUnits.count()); + } + + // Get the native register units. This is a prefix of getRegUnits(). + RegUnitList getNativeRegUnits() const { + return NativeRegUnits; + } + + void setRegUnitLaneMasks(const RegUnitLaneMaskList &LaneMasks) { + RegUnitLaneMasks = LaneMasks; + } + + // Inherit register units from subregisters. + // Return true if the RegUnits changed. + bool inheritRegUnits(CodeGenRegBank &RegBank); + + // Adopt a register unit for pressure tracking. + // A unit is adopted iff its unit number is >= NativeRegUnits.count(). + void adoptRegUnit(unsigned RUID) { RegUnits.set(RUID); } + + // Get the sum of this register's register unit weights. + unsigned getWeight(const CodeGenRegBank &RegBank) const; + + // Canonically ordered set. + typedef std::vector<const CodeGenRegister*> Vec; + + private: + bool SubRegsComplete; + bool SuperRegsComplete; + unsigned TopoSig; + + // The sub-registers explicit in the .td file form a tree. + SmallVector<CodeGenSubRegIndex*, 8> ExplicitSubRegIndices; + SmallVector<CodeGenRegister*, 8> ExplicitSubRegs; + + // Explicit ad hoc aliases, symmetrized to form an undirected graph. + SmallVector<CodeGenRegister*, 8> ExplicitAliases; + + // Super-registers where this is the first explicit sub-register. + SuperRegList LeadingSuperRegs; + + SubRegMap SubRegs; + SuperRegList SuperRegs; + DenseMap<const CodeGenRegister*, CodeGenSubRegIndex*> SubReg2Idx; + RegUnitList RegUnits; + RegUnitLaneMaskList RegUnitLaneMasks; + }; + + inline bool operator<(const CodeGenRegister &A, const CodeGenRegister &B) { + return A.EnumValue < B.EnumValue; + } + + inline bool operator==(const CodeGenRegister &A, const CodeGenRegister &B) { + return A.EnumValue == B.EnumValue; + } + + class CodeGenRegisterClass { + CodeGenRegister::Vec Members; + // Allocation orders. Order[0] always contains all registers in Members. + std::vector<SmallVector<Record*, 16>> Orders; + // Bit mask of sub-classes including this, indexed by their EnumValue. + BitVector SubClasses; + // List of super-classes, topologocally ordered to have the larger classes + // first. This is the same as sorting by EnumValue. + SmallVector<CodeGenRegisterClass*, 4> SuperClasses; + Record *TheDef; + std::string Name; + + // For a synthesized class, inherit missing properties from the nearest + // super-class. + void inheritProperties(CodeGenRegBank&); + + // Map SubRegIndex -> sub-class. This is the largest sub-class where all + // registers have a SubRegIndex sub-register. + DenseMap<const CodeGenSubRegIndex *, CodeGenRegisterClass *> + SubClassWithSubReg; + + // Map SubRegIndex -> set of super-reg classes. This is all register + // classes SuperRC such that: + // + // R:SubRegIndex in this RC for all R in SuperRC. + // + DenseMap<const CodeGenSubRegIndex *, SmallPtrSet<CodeGenRegisterClass *, 8>> + SuperRegClasses; + + // Bit vector of TopoSigs for the registers in this class. This will be + // very sparse on regular architectures. + BitVector TopoSigs; + + public: + unsigned EnumValue; + StringRef Namespace; + SmallVector<ValueTypeByHwMode, 4> VTs; + RegSizeInfoByHwMode RSI; + int CopyCost; + bool Allocatable; + StringRef AltOrderSelect; + uint8_t AllocationPriority; + /// Contains the combination of the lane masks of all subregisters. + LaneBitmask LaneMask; + /// True if there are at least 2 subregisters which do not interfere. + bool HasDisjunctSubRegs; + bool CoveredBySubRegs; + /// A register class is artificial if all its members are artificial. + bool Artificial; + /// Generate register pressure set for this register class and any class + /// synthesized from it. + bool GeneratePressureSet; + + // Return the Record that defined this class, or NULL if the class was + // created by TableGen. + Record *getDef() const { return TheDef; } + + const std::string &getName() const { return Name; } + std::string getQualifiedName() const; + ArrayRef<ValueTypeByHwMode> getValueTypes() const { return VTs; } + unsigned getNumValueTypes() const { return VTs.size(); } + + bool hasType(const ValueTypeByHwMode &VT) const { + return std::find(VTs.begin(), VTs.end(), VT) != VTs.end(); + } + + const ValueTypeByHwMode &getValueTypeNum(unsigned VTNum) const { + if (VTNum < VTs.size()) + return VTs[VTNum]; + llvm_unreachable("VTNum greater than number of ValueTypes in RegClass!"); + } + + // Return true if this this class contains the register. + bool contains(const CodeGenRegister*) const; + + // Returns true if RC is a subclass. + // RC is a sub-class of this class if it is a valid replacement for any + // instruction operand where a register of this classis required. It must + // satisfy these conditions: + // + // 1. All RC registers are also in this. + // 2. The RC spill size must not be smaller than our spill size. + // 3. RC spill alignment must be compatible with ours. + // + bool hasSubClass(const CodeGenRegisterClass *RC) const { + return SubClasses.test(RC->EnumValue); + } + + // getSubClassWithSubReg - Returns the largest sub-class where all + // registers have a SubIdx sub-register. + CodeGenRegisterClass * + getSubClassWithSubReg(const CodeGenSubRegIndex *SubIdx) const { + return SubClassWithSubReg.lookup(SubIdx); + } + + /// Find largest subclass where all registers have SubIdx subregisters in + /// SubRegClass and the largest subregister class that contains those + /// subregisters without (as far as possible) also containing additional registers. + /// + /// This can be used to find a suitable pair of classes for subregister copies. + /// \return std::pair<SubClass, SubRegClass> where SubClass is a SubClass is + /// a class where every register has SubIdx and SubRegClass is a class where + /// every register is covered by the SubIdx subregister of SubClass. + Optional<std::pair<CodeGenRegisterClass *, CodeGenRegisterClass *>> + getMatchingSubClassWithSubRegs(CodeGenRegBank &RegBank, + const CodeGenSubRegIndex *SubIdx) const; + + void setSubClassWithSubReg(const CodeGenSubRegIndex *SubIdx, + CodeGenRegisterClass *SubRC) { + SubClassWithSubReg[SubIdx] = SubRC; + } + + // getSuperRegClasses - Returns a bit vector of all register classes + // containing only SubIdx super-registers of this class. + void getSuperRegClasses(const CodeGenSubRegIndex *SubIdx, + BitVector &Out) const; + + // addSuperRegClass - Add a class containing only SubIdx super-registers. + void addSuperRegClass(CodeGenSubRegIndex *SubIdx, + CodeGenRegisterClass *SuperRC) { + SuperRegClasses[SubIdx].insert(SuperRC); + } + + // getSubClasses - Returns a constant BitVector of subclasses indexed by + // EnumValue. + // The SubClasses vector includes an entry for this class. + const BitVector &getSubClasses() const { return SubClasses; } + + // getSuperClasses - Returns a list of super classes ordered by EnumValue. + // The array does not include an entry for this class. + ArrayRef<CodeGenRegisterClass*> getSuperClasses() const { + return SuperClasses; + } + + // Returns an ordered list of class members. + // The order of registers is the same as in the .td file. + // No = 0 is the default allocation order, No = 1 is the first alternative. + ArrayRef<Record*> getOrder(unsigned No = 0) const { + return Orders[No]; + } + + // Return the total number of allocation orders available. + unsigned getNumOrders() const { return Orders.size(); } + + // Get the set of registers. This set contains the same registers as + // getOrder(0). + const CodeGenRegister::Vec &getMembers() const { return Members; } + + // Get a bit vector of TopoSigs present in this register class. + const BitVector &getTopoSigs() const { return TopoSigs; } + + // Get a weight of this register class. + unsigned getWeight(const CodeGenRegBank&) const; + + // Populate a unique sorted list of units from a register set. + void buildRegUnitSet(const CodeGenRegBank &RegBank, + std::vector<unsigned> &RegUnits) const; + + CodeGenRegisterClass(CodeGenRegBank&, Record *R); + CodeGenRegisterClass(CodeGenRegisterClass&) = delete; + + // A key representing the parts of a register class used for forming + // sub-classes. Note the ordering provided by this key is not the same as + // the topological order used for the EnumValues. + struct Key { + const CodeGenRegister::Vec *Members; + RegSizeInfoByHwMode RSI; + + Key(const CodeGenRegister::Vec *M, const RegSizeInfoByHwMode &I) + : Members(M), RSI(I) {} + + Key(const CodeGenRegisterClass &RC) + : Members(&RC.getMembers()), RSI(RC.RSI) {} + + // Lexicographical order of (Members, RegSizeInfoByHwMode). + bool operator<(const Key&) const; + }; + + // Create a non-user defined register class. + CodeGenRegisterClass(CodeGenRegBank&, StringRef Name, Key Props); + + // Called by CodeGenRegBank::CodeGenRegBank(). + static void computeSubClasses(CodeGenRegBank&); + }; + + // Register units are used to model interference and register pressure. + // Every register is assigned one or more register units such that two + // registers overlap if and only if they have a register unit in common. + // + // Normally, one register unit is created per leaf register. Non-leaf + // registers inherit the units of their sub-registers. + struct RegUnit { + // Weight assigned to this RegUnit for estimating register pressure. + // This is useful when equalizing weights in register classes with mixed + // register topologies. + unsigned Weight; + + // Each native RegUnit corresponds to one or two root registers. The full + // set of registers containing this unit can be computed as the union of + // these two registers and their super-registers. + const CodeGenRegister *Roots[2]; + + // Index into RegClassUnitSets where we can find the list of UnitSets that + // contain this unit. + unsigned RegClassUnitSetsIdx; + // A register unit is artificial if at least one of its roots is + // artificial. + bool Artificial; + + RegUnit() : Weight(0), RegClassUnitSetsIdx(0), Artificial(false) { + Roots[0] = Roots[1] = nullptr; + } + + ArrayRef<const CodeGenRegister*> getRoots() const { + assert(!(Roots[1] && !Roots[0]) && "Invalid roots array"); + return makeArrayRef(Roots, !!Roots[0] + !!Roots[1]); + } + }; + + // Each RegUnitSet is a sorted vector with a name. + struct RegUnitSet { + typedef std::vector<unsigned>::const_iterator iterator; + + std::string Name; + std::vector<unsigned> Units; + unsigned Weight = 0; // Cache the sum of all unit weights. + unsigned Order = 0; // Cache the sort key. + + RegUnitSet() = default; + }; + + // Base vector for identifying TopoSigs. The contents uniquely identify a + // TopoSig, only computeSuperRegs needs to know how. + typedef SmallVector<unsigned, 16> TopoSigId; + + // CodeGenRegBank - Represent a target's registers and the relations between + // them. + class CodeGenRegBank { + SetTheory Sets; + + const CodeGenHwModes &CGH; + + std::deque<CodeGenSubRegIndex> SubRegIndices; + DenseMap<Record*, CodeGenSubRegIndex*> Def2SubRegIdx; + + CodeGenSubRegIndex *createSubRegIndex(StringRef Name, StringRef NameSpace); + + typedef std::map<SmallVector<CodeGenSubRegIndex*, 8>, + CodeGenSubRegIndex*> ConcatIdxMap; + ConcatIdxMap ConcatIdx; + + // Registers. + std::deque<CodeGenRegister> Registers; + StringMap<CodeGenRegister*> RegistersByName; + DenseMap<Record*, CodeGenRegister*> Def2Reg; + unsigned NumNativeRegUnits; + + std::map<TopoSigId, unsigned> TopoSigs; + + // Includes native (0..NumNativeRegUnits-1) and adopted register units. + SmallVector<RegUnit, 8> RegUnits; + + // Register classes. + std::list<CodeGenRegisterClass> RegClasses; + DenseMap<Record*, CodeGenRegisterClass*> Def2RC; + typedef std::map<CodeGenRegisterClass::Key, CodeGenRegisterClass*> RCKeyMap; + RCKeyMap Key2RC; + + // Remember each unique set of register units. Initially, this contains a + // unique set for each register class. Simliar sets are coalesced with + // pruneUnitSets and new supersets are inferred during computeRegUnitSets. + std::vector<RegUnitSet> RegUnitSets; + + // Map RegisterClass index to the index of the RegUnitSet that contains the + // class's units and any inferred RegUnit supersets. + // + // NOTE: This could grow beyond the number of register classes when we map + // register units to lists of unit sets. If the list of unit sets does not + // already exist for a register class, we create a new entry in this vector. + std::vector<std::vector<unsigned>> RegClassUnitSets; + + // Give each register unit set an order based on sorting criteria. + std::vector<unsigned> RegUnitSetOrder; + + // Keep track of synthesized definitions generated in TupleExpander. + std::vector<std::unique_ptr<Record>> SynthDefs; + + // Add RC to *2RC maps. + void addToMaps(CodeGenRegisterClass*); + + // Create a synthetic sub-class if it is missing. + CodeGenRegisterClass *getOrCreateSubClass(const CodeGenRegisterClass *RC, + const CodeGenRegister::Vec *Membs, + StringRef Name); + + // Infer missing register classes. + void computeInferredRegisterClasses(); + void inferCommonSubClass(CodeGenRegisterClass *RC); + void inferSubClassWithSubReg(CodeGenRegisterClass *RC); + + void inferMatchingSuperRegClass(CodeGenRegisterClass *RC) { + inferMatchingSuperRegClass(RC, RegClasses.begin()); + } + + void inferMatchingSuperRegClass( + CodeGenRegisterClass *RC, + std::list<CodeGenRegisterClass>::iterator FirstSubRegRC); + + // Iteratively prune unit sets. + void pruneUnitSets(); + + // Compute a weight for each register unit created during getSubRegs. + void computeRegUnitWeights(); + + // Create a RegUnitSet for each RegClass and infer superclasses. + void computeRegUnitSets(); + + // Populate the Composite map from sub-register relationships. + void computeComposites(); + + // Compute a lane mask for each sub-register index. + void computeSubRegLaneMasks(); + + /// Computes a lane mask for each register unit enumerated by a physical + /// register. + void computeRegUnitLaneMasks(); + + public: + CodeGenRegBank(RecordKeeper&, const CodeGenHwModes&); + CodeGenRegBank(CodeGenRegBank&) = delete; + + SetTheory &getSets() { return Sets; } + + const CodeGenHwModes &getHwModes() const { return CGH; } + + // Sub-register indices. The first NumNamedIndices are defined by the user + // in the .td files. The rest are synthesized such that all sub-registers + // have a unique name. + const std::deque<CodeGenSubRegIndex> &getSubRegIndices() const { + return SubRegIndices; + } + + // Find a SubRegIndex from its Record def or add to the list if it does + // not exist there yet. + CodeGenSubRegIndex *getSubRegIdx(Record*); + + // Find a SubRegIndex from its Record def. + const CodeGenSubRegIndex *findSubRegIdx(const Record* Def) const; + + // Find or create a sub-register index representing the A+B composition. + CodeGenSubRegIndex *getCompositeSubRegIndex(CodeGenSubRegIndex *A, + CodeGenSubRegIndex *B); + + // Find or create a sub-register index representing the concatenation of + // non-overlapping sibling indices. + CodeGenSubRegIndex * + getConcatSubRegIndex(const SmallVector<CodeGenSubRegIndex *, 8>&); + + const std::deque<CodeGenRegister> &getRegisters() const { + return Registers; + } + + const StringMap<CodeGenRegister *> &getRegistersByName() const { + return RegistersByName; + } + + // Find a register from its Record def. + CodeGenRegister *getReg(Record*); + + // Get a Register's index into the Registers array. + unsigned getRegIndex(const CodeGenRegister *Reg) const { + return Reg->EnumValue - 1; + } + + // Return the number of allocated TopoSigs. The first TopoSig representing + // leaf registers is allocated number 0. + unsigned getNumTopoSigs() const { + return TopoSigs.size(); + } + + // Find or create a TopoSig for the given TopoSigId. + // This function is only for use by CodeGenRegister::computeSuperRegs(). + // Others should simply use Reg->getTopoSig(). + unsigned getTopoSig(const TopoSigId &Id) { + return TopoSigs.insert(std::make_pair(Id, TopoSigs.size())).first->second; + } + + // Create a native register unit that is associated with one or two root + // registers. + unsigned newRegUnit(CodeGenRegister *R0, CodeGenRegister *R1 = nullptr) { + RegUnits.resize(RegUnits.size() + 1); + RegUnit &RU = RegUnits.back(); + RU.Roots[0] = R0; + RU.Roots[1] = R1; + RU.Artificial = R0->Artificial; + if (R1) + RU.Artificial |= R1->Artificial; + return RegUnits.size() - 1; + } + + // Create a new non-native register unit that can be adopted by a register + // to increase its pressure. Note that NumNativeRegUnits is not increased. + unsigned newRegUnit(unsigned Weight) { + RegUnits.resize(RegUnits.size() + 1); + RegUnits.back().Weight = Weight; + return RegUnits.size() - 1; + } + + // Native units are the singular unit of a leaf register. Register aliasing + // is completely characterized by native units. Adopted units exist to give + // register additional weight but don't affect aliasing. + bool isNativeUnit(unsigned RUID) const { + return RUID < NumNativeRegUnits; + } + + unsigned getNumNativeRegUnits() const { + return NumNativeRegUnits; + } + + RegUnit &getRegUnit(unsigned RUID) { return RegUnits[RUID]; } + const RegUnit &getRegUnit(unsigned RUID) const { return RegUnits[RUID]; } + + std::list<CodeGenRegisterClass> &getRegClasses() { return RegClasses; } + + const std::list<CodeGenRegisterClass> &getRegClasses() const { + return RegClasses; + } + + // Find a register class from its def. + CodeGenRegisterClass *getRegClass(const Record *) const; + + /// getRegisterClassForRegister - Find the register class that contains the + /// specified physical register. If the register is not in a register + /// class, return null. If the register is in multiple classes, and the + /// classes have a superset-subset relationship and the same set of types, + /// return the superclass. Otherwise return null. + const CodeGenRegisterClass* getRegClassForRegister(Record *R); + + // Analog of TargetRegisterInfo::getMinimalPhysRegClass. Unlike + // getRegClassForRegister, this tries to find the smallest class containing + // the physical register. If \p VT is specified, it will only find classes + // with a matching type + const CodeGenRegisterClass * + getMinimalPhysRegClass(Record *RegRecord, ValueTypeByHwMode *VT = nullptr); + + // Get the sum of unit weights. + unsigned getRegUnitSetWeight(const std::vector<unsigned> &Units) const { + unsigned Weight = 0; + for (std::vector<unsigned>::const_iterator + I = Units.begin(), E = Units.end(); I != E; ++I) + Weight += getRegUnit(*I).Weight; + return Weight; + } + + unsigned getRegSetIDAt(unsigned Order) const { + return RegUnitSetOrder[Order]; + } + + const RegUnitSet &getRegSetAt(unsigned Order) const { + return RegUnitSets[RegUnitSetOrder[Order]]; + } + + // Increase a RegUnitWeight. + void increaseRegUnitWeight(unsigned RUID, unsigned Inc) { + getRegUnit(RUID).Weight += Inc; + } + + // Get the number of register pressure dimensions. + unsigned getNumRegPressureSets() const { return RegUnitSets.size(); } + + // Get a set of register unit IDs for a given dimension of pressure. + const RegUnitSet &getRegPressureSet(unsigned Idx) const { + return RegUnitSets[Idx]; + } + + // The number of pressure set lists may be larget than the number of + // register classes if some register units appeared in a list of sets that + // did not correspond to an existing register class. + unsigned getNumRegClassPressureSetLists() const { + return RegClassUnitSets.size(); + } + + // Get a list of pressure set IDs for a register class. Liveness of a + // register in this class impacts each pressure set in this list by the + // weight of the register. An exact solution requires all registers in a + // class to have the same class, but it is not strictly guaranteed. + ArrayRef<unsigned> getRCPressureSetIDs(unsigned RCIdx) const { + return RegClassUnitSets[RCIdx]; + } + + // Computed derived records such as missing sub-register indices. + void computeDerivedInfo(); + + // Compute the set of registers completely covered by the registers in Regs. + // The returned BitVector will have a bit set for each register in Regs, + // all sub-registers, and all super-registers that are covered by the + // registers in Regs. + // + // This is used to compute the mask of call-preserved registers from a list + // of callee-saves. + BitVector computeCoveredRegisters(ArrayRef<Record*> Regs); + + // Bit mask of lanes that cover their registers. A sub-register index whose + // LaneMask is contained in CoveringLanes will be completely covered by + // another sub-register with the same or larger lane mask. + LaneBitmask CoveringLanes; + + // Helper function for printing debug information. Handles artificial + // (non-native) reg units. + void printRegUnitName(unsigned Unit) const; + }; + +} // end namespace llvm + +#endif // LLVM_UTILS_TABLEGEN_CODEGENREGISTERS_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenSchedule.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenSchedule.cpp new file mode 100644 index 000000000000..67583c736cd2 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenSchedule.cpp @@ -0,0 +1,2232 @@ +//===- CodeGenSchedule.cpp - Scheduling MachineModels ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines structures to encapsulate the machine model as described in +// the target description. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenSchedule.h" +#include "CodeGenInstruction.h" +#include "CodeGenTarget.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include <algorithm> +#include <iterator> +#include <utility> + +using namespace llvm; + +#define DEBUG_TYPE "subtarget-emitter" + +#ifndef NDEBUG +static void dumpIdxVec(ArrayRef<unsigned> V) { + for (unsigned Idx : V) + dbgs() << Idx << ", "; +} +#endif + +namespace { + +// (instrs a, b, ...) Evaluate and union all arguments. Identical to AddOp. +struct InstrsOp : public SetTheory::Operator { + void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts, + ArrayRef<SMLoc> Loc) override { + ST.evaluate(Expr->arg_begin(), Expr->arg_end(), Elts, Loc); + } +}; + +// (instregex "OpcPat",...) Find all instructions matching an opcode pattern. +struct InstRegexOp : public SetTheory::Operator { + const CodeGenTarget &Target; + InstRegexOp(const CodeGenTarget &t): Target(t) {} + + /// Remove any text inside of parentheses from S. + static std::string removeParens(llvm::StringRef S) { + std::string Result; + unsigned Paren = 0; + // NB: We don't care about escaped parens here. + for (char C : S) { + switch (C) { + case '(': + ++Paren; + break; + case ')': + --Paren; + break; + default: + if (Paren == 0) + Result += C; + } + } + return Result; + } + + void apply(SetTheory &ST, DagInit *Expr, SetTheory::RecSet &Elts, + ArrayRef<SMLoc> Loc) override { + ArrayRef<const CodeGenInstruction *> Instructions = + Target.getInstructionsByEnumValue(); + + unsigned NumGeneric = Target.getNumFixedInstructions(); + unsigned NumPseudos = Target.getNumPseudoInstructions(); + auto Generics = Instructions.slice(0, NumGeneric); + auto Pseudos = Instructions.slice(NumGeneric, NumPseudos); + auto NonPseudos = Instructions.slice(NumGeneric + NumPseudos); + + for (Init *Arg : make_range(Expr->arg_begin(), Expr->arg_end())) { + StringInit *SI = dyn_cast<StringInit>(Arg); + if (!SI) + PrintFatalError(Loc, "instregex requires pattern string: " + + Expr->getAsString()); + StringRef Original = SI->getValue(); + + // Extract a prefix that we can binary search on. + static const char RegexMetachars[] = "()^$|*+?.[]\\{}"; + auto FirstMeta = Original.find_first_of(RegexMetachars); + + // Look for top-level | or ?. We cannot optimize them to binary search. + if (removeParens(Original).find_first_of("|?") != std::string::npos) + FirstMeta = 0; + + Optional<Regex> Regexpr = None; + StringRef Prefix = Original.substr(0, FirstMeta); + StringRef PatStr = Original.substr(FirstMeta); + if (!PatStr.empty()) { + // For the rest use a python-style prefix match. + std::string pat = std::string(PatStr); + if (pat[0] != '^') { + pat.insert(0, "^("); + pat.insert(pat.end(), ')'); + } + Regexpr = Regex(pat); + } + + int NumMatches = 0; + + // The generic opcodes are unsorted, handle them manually. + for (auto *Inst : Generics) { + StringRef InstName = Inst->TheDef->getName(); + if (InstName.startswith(Prefix) && + (!Regexpr || Regexpr->match(InstName.substr(Prefix.size())))) { + Elts.insert(Inst->TheDef); + NumMatches++; + } + } + + // Target instructions are split into two ranges: pseudo instructions + // first, than non-pseudos. Each range is in lexicographical order + // sorted by name. Find the sub-ranges that start with our prefix. + struct Comp { + bool operator()(const CodeGenInstruction *LHS, StringRef RHS) { + return LHS->TheDef->getName() < RHS; + } + bool operator()(StringRef LHS, const CodeGenInstruction *RHS) { + return LHS < RHS->TheDef->getName() && + !RHS->TheDef->getName().startswith(LHS); + } + }; + auto Range1 = + std::equal_range(Pseudos.begin(), Pseudos.end(), Prefix, Comp()); + auto Range2 = std::equal_range(NonPseudos.begin(), NonPseudos.end(), + Prefix, Comp()); + + // For these ranges we know that instruction names start with the prefix. + // Check if there's a regex that needs to be checked. + const auto HandleNonGeneric = [&](const CodeGenInstruction *Inst) { + StringRef InstName = Inst->TheDef->getName(); + if (!Regexpr || Regexpr->match(InstName.substr(Prefix.size()))) { + Elts.insert(Inst->TheDef); + NumMatches++; + } + }; + std::for_each(Range1.first, Range1.second, HandleNonGeneric); + std::for_each(Range2.first, Range2.second, HandleNonGeneric); + + if (0 == NumMatches) + PrintFatalError(Loc, "instregex has no matches: " + Original); + } + } +}; + +} // end anonymous namespace + +/// CodeGenModels ctor interprets machine model records and populates maps. +CodeGenSchedModels::CodeGenSchedModels(RecordKeeper &RK, + const CodeGenTarget &TGT): + Records(RK), Target(TGT) { + + Sets.addFieldExpander("InstRW", "Instrs"); + + // Allow Set evaluation to recognize the dags used in InstRW records: + // (instrs Op1, Op1...) + Sets.addOperator("instrs", std::make_unique<InstrsOp>()); + Sets.addOperator("instregex", std::make_unique<InstRegexOp>(Target)); + + // Instantiate a CodeGenProcModel for each SchedMachineModel with the values + // that are explicitly referenced in tablegen records. Resources associated + // with each processor will be derived later. Populate ProcModelMap with the + // CodeGenProcModel instances. + collectProcModels(); + + // Instantiate a CodeGenSchedRW for each SchedReadWrite record explicitly + // defined, and populate SchedReads and SchedWrites vectors. Implicit + // SchedReadWrites that represent sequences derived from expanded variant will + // be inferred later. + collectSchedRW(); + + // Instantiate a CodeGenSchedClass for each unique SchedRW signature directly + // required by an instruction definition, and populate SchedClassIdxMap. Set + // NumItineraryClasses to the number of explicit itinerary classes referenced + // by instructions. Set NumInstrSchedClasses to the number of itinerary + // classes plus any classes implied by instructions that derive from class + // Sched and provide SchedRW list. This does not infer any new classes from + // SchedVariant. + collectSchedClasses(); + + // Find instruction itineraries for each processor. Sort and populate + // CodeGenProcModel::ItinDefList. (Cycle-to-cycle itineraries). This requires + // all itinerary classes to be discovered. + collectProcItins(); + + // Find ItinRW records for each processor and itinerary class. + // (For per-operand resources mapped to itinerary classes). + collectProcItinRW(); + + // Find UnsupportedFeatures records for each processor. + // (For per-operand resources mapped to itinerary classes). + collectProcUnsupportedFeatures(); + + // Infer new SchedClasses from SchedVariant. + inferSchedClasses(); + + // Populate each CodeGenProcModel's WriteResDefs, ReadAdvanceDefs, and + // ProcResourceDefs. + LLVM_DEBUG( + dbgs() << "\n+++ RESOURCE DEFINITIONS (collectProcResources) +++\n"); + collectProcResources(); + + // Collect optional processor description. + collectOptionalProcessorInfo(); + + // Check MCInstPredicate definitions. + checkMCInstPredicates(); + + // Check STIPredicate definitions. + checkSTIPredicates(); + + // Find STIPredicate definitions for each processor model, and construct + // STIPredicateFunction objects. + collectSTIPredicates(); + + checkCompleteness(); +} + +void CodeGenSchedModels::checkSTIPredicates() const { + DenseMap<StringRef, const Record *> Declarations; + + // There cannot be multiple declarations with the same name. + const RecVec Decls = Records.getAllDerivedDefinitions("STIPredicateDecl"); + for (const Record *R : Decls) { + StringRef Name = R->getValueAsString("Name"); + const auto It = Declarations.find(Name); + if (It == Declarations.end()) { + Declarations[Name] = R; + continue; + } + + PrintError(R->getLoc(), "STIPredicate " + Name + " multiply declared."); + PrintNote(It->second->getLoc(), "Previous declaration was here."); + PrintFatalError(R->getLoc(), "Invalid STIPredicateDecl found."); + } + + // Disallow InstructionEquivalenceClasses with an empty instruction list. + const RecVec Defs = + Records.getAllDerivedDefinitions("InstructionEquivalenceClass"); + for (const Record *R : Defs) { + RecVec Opcodes = R->getValueAsListOfDefs("Opcodes"); + if (Opcodes.empty()) { + PrintFatalError(R->getLoc(), "Invalid InstructionEquivalenceClass " + "defined with an empty opcode list."); + } + } +} + +// Used by function `processSTIPredicate` to construct a mask of machine +// instruction operands. +static APInt constructOperandMask(ArrayRef<int64_t> Indices) { + APInt OperandMask; + if (Indices.empty()) + return OperandMask; + + int64_t MaxIndex = *std::max_element(Indices.begin(), Indices.end()); + assert(MaxIndex >= 0 && "Invalid negative indices in input!"); + OperandMask = OperandMask.zext(MaxIndex + 1); + for (const int64_t Index : Indices) { + assert(Index >= 0 && "Invalid negative indices!"); + OperandMask.setBit(Index); + } + + return OperandMask; +} + +static void +processSTIPredicate(STIPredicateFunction &Fn, + const DenseMap<Record *, unsigned> &ProcModelMap) { + DenseMap<const Record *, unsigned> Opcode2Index; + using OpcodeMapPair = std::pair<const Record *, OpcodeInfo>; + std::vector<OpcodeMapPair> OpcodeMappings; + std::vector<std::pair<APInt, APInt>> OpcodeMasks; + + DenseMap<const Record *, unsigned> Predicate2Index; + unsigned NumUniquePredicates = 0; + + // Number unique predicates and opcodes used by InstructionEquivalenceClass + // definitions. Each unique opcode will be associated with an OpcodeInfo + // object. + for (const Record *Def : Fn.getDefinitions()) { + RecVec Classes = Def->getValueAsListOfDefs("Classes"); + for (const Record *EC : Classes) { + const Record *Pred = EC->getValueAsDef("Predicate"); + if (Predicate2Index.find(Pred) == Predicate2Index.end()) + Predicate2Index[Pred] = NumUniquePredicates++; + + RecVec Opcodes = EC->getValueAsListOfDefs("Opcodes"); + for (const Record *Opcode : Opcodes) { + if (Opcode2Index.find(Opcode) == Opcode2Index.end()) { + Opcode2Index[Opcode] = OpcodeMappings.size(); + OpcodeMappings.emplace_back(Opcode, OpcodeInfo()); + } + } + } + } + + // Initialize vector `OpcodeMasks` with default values. We want to keep track + // of which processors "use" which opcodes. We also want to be able to + // identify predicates that are used by different processors for a same + // opcode. + // This information is used later on by this algorithm to sort OpcodeMapping + // elements based on their processor and predicate sets. + OpcodeMasks.resize(OpcodeMappings.size()); + APInt DefaultProcMask(ProcModelMap.size(), 0); + APInt DefaultPredMask(NumUniquePredicates, 0); + for (std::pair<APInt, APInt> &MaskPair : OpcodeMasks) + MaskPair = std::make_pair(DefaultProcMask, DefaultPredMask); + + // Construct a OpcodeInfo object for every unique opcode declared by an + // InstructionEquivalenceClass definition. + for (const Record *Def : Fn.getDefinitions()) { + RecVec Classes = Def->getValueAsListOfDefs("Classes"); + const Record *SchedModel = Def->getValueAsDef("SchedModel"); + unsigned ProcIndex = ProcModelMap.find(SchedModel)->second; + APInt ProcMask(ProcModelMap.size(), 0); + ProcMask.setBit(ProcIndex); + + for (const Record *EC : Classes) { + RecVec Opcodes = EC->getValueAsListOfDefs("Opcodes"); + + std::vector<int64_t> OpIndices = + EC->getValueAsListOfInts("OperandIndices"); + APInt OperandMask = constructOperandMask(OpIndices); + + const Record *Pred = EC->getValueAsDef("Predicate"); + APInt PredMask(NumUniquePredicates, 0); + PredMask.setBit(Predicate2Index[Pred]); + + for (const Record *Opcode : Opcodes) { + unsigned OpcodeIdx = Opcode2Index[Opcode]; + if (OpcodeMasks[OpcodeIdx].first[ProcIndex]) { + std::string Message = + "Opcode " + Opcode->getName().str() + + " used by multiple InstructionEquivalenceClass definitions."; + PrintFatalError(EC->getLoc(), Message); + } + OpcodeMasks[OpcodeIdx].first |= ProcMask; + OpcodeMasks[OpcodeIdx].second |= PredMask; + OpcodeInfo &OI = OpcodeMappings[OpcodeIdx].second; + + OI.addPredicateForProcModel(ProcMask, OperandMask, Pred); + } + } + } + + // Sort OpcodeMappings elements based on their CPU and predicate masks. + // As a last resort, order elements by opcode identifier. + llvm::sort(OpcodeMappings, + [&](const OpcodeMapPair &Lhs, const OpcodeMapPair &Rhs) { + unsigned LhsIdx = Opcode2Index[Lhs.first]; + unsigned RhsIdx = Opcode2Index[Rhs.first]; + const std::pair<APInt, APInt> &LhsMasks = OpcodeMasks[LhsIdx]; + const std::pair<APInt, APInt> &RhsMasks = OpcodeMasks[RhsIdx]; + + auto LessThan = [](const APInt &Lhs, const APInt &Rhs) { + unsigned LhsCountPopulation = Lhs.countPopulation(); + unsigned RhsCountPopulation = Rhs.countPopulation(); + return ((LhsCountPopulation < RhsCountPopulation) || + ((LhsCountPopulation == RhsCountPopulation) && + (Lhs.countLeadingZeros() > Rhs.countLeadingZeros()))); + }; + + if (LhsMasks.first != RhsMasks.first) + return LessThan(LhsMasks.first, RhsMasks.first); + + if (LhsMasks.second != RhsMasks.second) + return LessThan(LhsMasks.second, RhsMasks.second); + + return LhsIdx < RhsIdx; + }); + + // Now construct opcode groups. Groups are used by the SubtargetEmitter when + // expanding the body of a STIPredicate function. In particular, each opcode + // group is expanded into a sequence of labels in a switch statement. + // It identifies opcodes for which different processors define same predicates + // and same opcode masks. + for (OpcodeMapPair &Info : OpcodeMappings) + Fn.addOpcode(Info.first, std::move(Info.second)); +} + +void CodeGenSchedModels::collectSTIPredicates() { + // Map STIPredicateDecl records to elements of vector + // CodeGenSchedModels::STIPredicates. + DenseMap<const Record *, unsigned> Decl2Index; + + RecVec RV = Records.getAllDerivedDefinitions("STIPredicate"); + for (const Record *R : RV) { + const Record *Decl = R->getValueAsDef("Declaration"); + + const auto It = Decl2Index.find(Decl); + if (It == Decl2Index.end()) { + Decl2Index[Decl] = STIPredicates.size(); + STIPredicateFunction Predicate(Decl); + Predicate.addDefinition(R); + STIPredicates.emplace_back(std::move(Predicate)); + continue; + } + + STIPredicateFunction &PreviousDef = STIPredicates[It->second]; + PreviousDef.addDefinition(R); + } + + for (STIPredicateFunction &Fn : STIPredicates) + processSTIPredicate(Fn, ProcModelMap); +} + +void OpcodeInfo::addPredicateForProcModel(const llvm::APInt &CpuMask, + const llvm::APInt &OperandMask, + const Record *Predicate) { + auto It = llvm::find_if( + Predicates, [&OperandMask, &Predicate](const PredicateInfo &P) { + return P.Predicate == Predicate && P.OperandMask == OperandMask; + }); + if (It == Predicates.end()) { + Predicates.emplace_back(CpuMask, OperandMask, Predicate); + return; + } + It->ProcModelMask |= CpuMask; +} + +void CodeGenSchedModels::checkMCInstPredicates() const { + RecVec MCPredicates = Records.getAllDerivedDefinitions("TIIPredicate"); + if (MCPredicates.empty()) + return; + + // A target cannot have multiple TIIPredicate definitions with a same name. + llvm::StringMap<const Record *> TIIPredicates(MCPredicates.size()); + for (const Record *TIIPred : MCPredicates) { + StringRef Name = TIIPred->getValueAsString("FunctionName"); + StringMap<const Record *>::const_iterator It = TIIPredicates.find(Name); + if (It == TIIPredicates.end()) { + TIIPredicates[Name] = TIIPred; + continue; + } + + PrintError(TIIPred->getLoc(), + "TIIPredicate " + Name + " is multiply defined."); + PrintNote(It->second->getLoc(), + " Previous definition of " + Name + " was here."); + PrintFatalError(TIIPred->getLoc(), + "Found conflicting definitions of TIIPredicate."); + } +} + +void CodeGenSchedModels::collectRetireControlUnits() { + RecVec Units = Records.getAllDerivedDefinitions("RetireControlUnit"); + + for (Record *RCU : Units) { + CodeGenProcModel &PM = getProcModel(RCU->getValueAsDef("SchedModel")); + if (PM.RetireControlUnit) { + PrintError(RCU->getLoc(), + "Expected a single RetireControlUnit definition"); + PrintNote(PM.RetireControlUnit->getLoc(), + "Previous definition of RetireControlUnit was here"); + } + PM.RetireControlUnit = RCU; + } +} + +void CodeGenSchedModels::collectLoadStoreQueueInfo() { + RecVec Queues = Records.getAllDerivedDefinitions("MemoryQueue"); + + for (Record *Queue : Queues) { + CodeGenProcModel &PM = getProcModel(Queue->getValueAsDef("SchedModel")); + if (Queue->isSubClassOf("LoadQueue")) { + if (PM.LoadQueue) { + PrintError(Queue->getLoc(), + "Expected a single LoadQueue definition"); + PrintNote(PM.LoadQueue->getLoc(), + "Previous definition of LoadQueue was here"); + } + + PM.LoadQueue = Queue; + } + + if (Queue->isSubClassOf("StoreQueue")) { + if (PM.StoreQueue) { + PrintError(Queue->getLoc(), + "Expected a single StoreQueue definition"); + PrintNote(PM.LoadQueue->getLoc(), + "Previous definition of StoreQueue was here"); + } + + PM.StoreQueue = Queue; + } + } +} + +/// Collect optional processor information. +void CodeGenSchedModels::collectOptionalProcessorInfo() { + // Find register file definitions for each processor. + collectRegisterFiles(); + + // Collect processor RetireControlUnit descriptors if available. + collectRetireControlUnits(); + + // Collect information about load/store queues. + collectLoadStoreQueueInfo(); + + checkCompleteness(); +} + +/// Gather all processor models. +void CodeGenSchedModels::collectProcModels() { + RecVec ProcRecords = Records.getAllDerivedDefinitions("Processor"); + llvm::sort(ProcRecords, LessRecordFieldName()); + + // Reserve space because we can. Reallocation would be ok. + ProcModels.reserve(ProcRecords.size()+1); + + // Use idx=0 for NoModel/NoItineraries. + Record *NoModelDef = Records.getDef("NoSchedModel"); + Record *NoItinsDef = Records.getDef("NoItineraries"); + ProcModels.emplace_back(0, "NoSchedModel", NoModelDef, NoItinsDef); + ProcModelMap[NoModelDef] = 0; + + // For each processor, find a unique machine model. + LLVM_DEBUG(dbgs() << "+++ PROCESSOR MODELs (addProcModel) +++\n"); + for (Record *ProcRecord : ProcRecords) + addProcModel(ProcRecord); +} + +/// Get a unique processor model based on the defined MachineModel and +/// ProcessorItineraries. +void CodeGenSchedModels::addProcModel(Record *ProcDef) { + Record *ModelKey = getModelOrItinDef(ProcDef); + if (!ProcModelMap.insert(std::make_pair(ModelKey, ProcModels.size())).second) + return; + + std::string Name = std::string(ModelKey->getName()); + if (ModelKey->isSubClassOf("SchedMachineModel")) { + Record *ItinsDef = ModelKey->getValueAsDef("Itineraries"); + ProcModels.emplace_back(ProcModels.size(), Name, ModelKey, ItinsDef); + } + else { + // An itinerary is defined without a machine model. Infer a new model. + if (!ModelKey->getValueAsListOfDefs("IID").empty()) + Name = Name + "Model"; + ProcModels.emplace_back(ProcModels.size(), Name, + ProcDef->getValueAsDef("SchedModel"), ModelKey); + } + LLVM_DEBUG(ProcModels.back().dump()); +} + +// Recursively find all reachable SchedReadWrite records. +static void scanSchedRW(Record *RWDef, RecVec &RWDefs, + SmallPtrSet<Record*, 16> &RWSet) { + if (!RWSet.insert(RWDef).second) + return; + RWDefs.push_back(RWDef); + // Reads don't currently have sequence records, but it can be added later. + if (RWDef->isSubClassOf("WriteSequence")) { + RecVec Seq = RWDef->getValueAsListOfDefs("Writes"); + for (Record *WSRec : Seq) + scanSchedRW(WSRec, RWDefs, RWSet); + } + else if (RWDef->isSubClassOf("SchedVariant")) { + // Visit each variant (guarded by a different predicate). + RecVec Vars = RWDef->getValueAsListOfDefs("Variants"); + for (Record *Variant : Vars) { + // Visit each RW in the sequence selected by the current variant. + RecVec Selected = Variant->getValueAsListOfDefs("Selected"); + for (Record *SelDef : Selected) + scanSchedRW(SelDef, RWDefs, RWSet); + } + } +} + +// Collect and sort all SchedReadWrites reachable via tablegen records. +// More may be inferred later when inferring new SchedClasses from variants. +void CodeGenSchedModels::collectSchedRW() { + // Reserve idx=0 for invalid writes/reads. + SchedWrites.resize(1); + SchedReads.resize(1); + + SmallPtrSet<Record*, 16> RWSet; + + // Find all SchedReadWrites referenced by instruction defs. + RecVec SWDefs, SRDefs; + for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) { + Record *SchedDef = Inst->TheDef; + if (SchedDef->isValueUnset("SchedRW")) + continue; + RecVec RWs = SchedDef->getValueAsListOfDefs("SchedRW"); + for (Record *RW : RWs) { + if (RW->isSubClassOf("SchedWrite")) + scanSchedRW(RW, SWDefs, RWSet); + else { + assert(RW->isSubClassOf("SchedRead") && "Unknown SchedReadWrite"); + scanSchedRW(RW, SRDefs, RWSet); + } + } + } + // Find all ReadWrites referenced by InstRW. + RecVec InstRWDefs = Records.getAllDerivedDefinitions("InstRW"); + for (Record *InstRWDef : InstRWDefs) { + // For all OperandReadWrites. + RecVec RWDefs = InstRWDef->getValueAsListOfDefs("OperandReadWrites"); + for (Record *RWDef : RWDefs) { + if (RWDef->isSubClassOf("SchedWrite")) + scanSchedRW(RWDef, SWDefs, RWSet); + else { + assert(RWDef->isSubClassOf("SchedRead") && "Unknown SchedReadWrite"); + scanSchedRW(RWDef, SRDefs, RWSet); + } + } + } + // Find all ReadWrites referenced by ItinRW. + RecVec ItinRWDefs = Records.getAllDerivedDefinitions("ItinRW"); + for (Record *ItinRWDef : ItinRWDefs) { + // For all OperandReadWrites. + RecVec RWDefs = ItinRWDef->getValueAsListOfDefs("OperandReadWrites"); + for (Record *RWDef : RWDefs) { + if (RWDef->isSubClassOf("SchedWrite")) + scanSchedRW(RWDef, SWDefs, RWSet); + else { + assert(RWDef->isSubClassOf("SchedRead") && "Unknown SchedReadWrite"); + scanSchedRW(RWDef, SRDefs, RWSet); + } + } + } + // Find all ReadWrites referenced by SchedAlias. AliasDefs needs to be sorted + // for the loop below that initializes Alias vectors. + RecVec AliasDefs = Records.getAllDerivedDefinitions("SchedAlias"); + llvm::sort(AliasDefs, LessRecord()); + for (Record *ADef : AliasDefs) { + Record *MatchDef = ADef->getValueAsDef("MatchRW"); + Record *AliasDef = ADef->getValueAsDef("AliasRW"); + if (MatchDef->isSubClassOf("SchedWrite")) { + if (!AliasDef->isSubClassOf("SchedWrite")) + PrintFatalError(ADef->getLoc(), "SchedWrite Alias must be SchedWrite"); + scanSchedRW(AliasDef, SWDefs, RWSet); + } + else { + assert(MatchDef->isSubClassOf("SchedRead") && "Unknown SchedReadWrite"); + if (!AliasDef->isSubClassOf("SchedRead")) + PrintFatalError(ADef->getLoc(), "SchedRead Alias must be SchedRead"); + scanSchedRW(AliasDef, SRDefs, RWSet); + } + } + // Sort and add the SchedReadWrites directly referenced by instructions or + // itinerary resources. Index reads and writes in separate domains. + llvm::sort(SWDefs, LessRecord()); + for (Record *SWDef : SWDefs) { + assert(!getSchedRWIdx(SWDef, /*IsRead=*/false) && "duplicate SchedWrite"); + SchedWrites.emplace_back(SchedWrites.size(), SWDef); + } + llvm::sort(SRDefs, LessRecord()); + for (Record *SRDef : SRDefs) { + assert(!getSchedRWIdx(SRDef, /*IsRead-*/true) && "duplicate SchedWrite"); + SchedReads.emplace_back(SchedReads.size(), SRDef); + } + // Initialize WriteSequence vectors. + for (CodeGenSchedRW &CGRW : SchedWrites) { + if (!CGRW.IsSequence) + continue; + findRWs(CGRW.TheDef->getValueAsListOfDefs("Writes"), CGRW.Sequence, + /*IsRead=*/false); + } + // Initialize Aliases vectors. + for (Record *ADef : AliasDefs) { + Record *AliasDef = ADef->getValueAsDef("AliasRW"); + getSchedRW(AliasDef).IsAlias = true; + Record *MatchDef = ADef->getValueAsDef("MatchRW"); + CodeGenSchedRW &RW = getSchedRW(MatchDef); + if (RW.IsAlias) + PrintFatalError(ADef->getLoc(), "Cannot Alias an Alias"); + RW.Aliases.push_back(ADef); + } + LLVM_DEBUG( + dbgs() << "\n+++ SCHED READS and WRITES (collectSchedRW) +++\n"; + for (unsigned WIdx = 0, WEnd = SchedWrites.size(); WIdx != WEnd; ++WIdx) { + dbgs() << WIdx << ": "; + SchedWrites[WIdx].dump(); + dbgs() << '\n'; + } for (unsigned RIdx = 0, REnd = SchedReads.size(); RIdx != REnd; + ++RIdx) { + dbgs() << RIdx << ": "; + SchedReads[RIdx].dump(); + dbgs() << '\n'; + } RecVec RWDefs = Records.getAllDerivedDefinitions("SchedReadWrite"); + for (Record *RWDef + : RWDefs) { + if (!getSchedRWIdx(RWDef, RWDef->isSubClassOf("SchedRead"))) { + StringRef Name = RWDef->getName(); + if (Name != "NoWrite" && Name != "ReadDefault") + dbgs() << "Unused SchedReadWrite " << Name << '\n'; + } + }); +} + +/// Compute a SchedWrite name from a sequence of writes. +std::string CodeGenSchedModels::genRWName(ArrayRef<unsigned> Seq, bool IsRead) { + std::string Name("("); + for (auto I = Seq.begin(), E = Seq.end(); I != E; ++I) { + if (I != Seq.begin()) + Name += '_'; + Name += getSchedRW(*I, IsRead).Name; + } + Name += ')'; + return Name; +} + +unsigned CodeGenSchedModels::getSchedRWIdx(const Record *Def, + bool IsRead) const { + const std::vector<CodeGenSchedRW> &RWVec = IsRead ? SchedReads : SchedWrites; + const auto I = find_if( + RWVec, [Def](const CodeGenSchedRW &RW) { return RW.TheDef == Def; }); + return I == RWVec.end() ? 0 : std::distance(RWVec.begin(), I); +} + +bool CodeGenSchedModels::hasReadOfWrite(Record *WriteDef) const { + for (const CodeGenSchedRW &Read : SchedReads) { + Record *ReadDef = Read.TheDef; + if (!ReadDef || !ReadDef->isSubClassOf("ProcReadAdvance")) + continue; + + RecVec ValidWrites = ReadDef->getValueAsListOfDefs("ValidWrites"); + if (is_contained(ValidWrites, WriteDef)) { + return true; + } + } + return false; +} + +static void splitSchedReadWrites(const RecVec &RWDefs, + RecVec &WriteDefs, RecVec &ReadDefs) { + for (Record *RWDef : RWDefs) { + if (RWDef->isSubClassOf("SchedWrite")) + WriteDefs.push_back(RWDef); + else { + assert(RWDef->isSubClassOf("SchedRead") && "unknown SchedReadWrite"); + ReadDefs.push_back(RWDef); + } + } +} + +// Split the SchedReadWrites defs and call findRWs for each list. +void CodeGenSchedModels::findRWs(const RecVec &RWDefs, + IdxVec &Writes, IdxVec &Reads) const { + RecVec WriteDefs; + RecVec ReadDefs; + splitSchedReadWrites(RWDefs, WriteDefs, ReadDefs); + findRWs(WriteDefs, Writes, false); + findRWs(ReadDefs, Reads, true); +} + +// Call getSchedRWIdx for all elements in a sequence of SchedRW defs. +void CodeGenSchedModels::findRWs(const RecVec &RWDefs, IdxVec &RWs, + bool IsRead) const { + for (Record *RWDef : RWDefs) { + unsigned Idx = getSchedRWIdx(RWDef, IsRead); + assert(Idx && "failed to collect SchedReadWrite"); + RWs.push_back(Idx); + } +} + +void CodeGenSchedModels::expandRWSequence(unsigned RWIdx, IdxVec &RWSeq, + bool IsRead) const { + const CodeGenSchedRW &SchedRW = getSchedRW(RWIdx, IsRead); + if (!SchedRW.IsSequence) { + RWSeq.push_back(RWIdx); + return; + } + int Repeat = + SchedRW.TheDef ? SchedRW.TheDef->getValueAsInt("Repeat") : 1; + for (int i = 0; i < Repeat; ++i) { + for (unsigned I : SchedRW.Sequence) { + expandRWSequence(I, RWSeq, IsRead); + } + } +} + +// Expand a SchedWrite as a sequence following any aliases that coincide with +// the given processor model. +void CodeGenSchedModels::expandRWSeqForProc( + unsigned RWIdx, IdxVec &RWSeq, bool IsRead, + const CodeGenProcModel &ProcModel) const { + + const CodeGenSchedRW &SchedWrite = getSchedRW(RWIdx, IsRead); + Record *AliasDef = nullptr; + for (const Record *Rec : SchedWrite.Aliases) { + const CodeGenSchedRW &AliasRW = getSchedRW(Rec->getValueAsDef("AliasRW")); + if (Rec->getValueInit("SchedModel")->isComplete()) { + Record *ModelDef = Rec->getValueAsDef("SchedModel"); + if (&getProcModel(ModelDef) != &ProcModel) + continue; + } + if (AliasDef) + PrintFatalError(AliasRW.TheDef->getLoc(), "Multiple aliases " + "defined for processor " + ProcModel.ModelName + + " Ensure only one SchedAlias exists per RW."); + AliasDef = AliasRW.TheDef; + } + if (AliasDef) { + expandRWSeqForProc(getSchedRWIdx(AliasDef, IsRead), + RWSeq, IsRead,ProcModel); + return; + } + if (!SchedWrite.IsSequence) { + RWSeq.push_back(RWIdx); + return; + } + int Repeat = + SchedWrite.TheDef ? SchedWrite.TheDef->getValueAsInt("Repeat") : 1; + for (int I = 0, E = Repeat; I < E; ++I) { + for (unsigned Idx : SchedWrite.Sequence) { + expandRWSeqForProc(Idx, RWSeq, IsRead, ProcModel); + } + } +} + +// Find the existing SchedWrite that models this sequence of writes. +unsigned CodeGenSchedModels::findRWForSequence(ArrayRef<unsigned> Seq, + bool IsRead) { + std::vector<CodeGenSchedRW> &RWVec = IsRead ? SchedReads : SchedWrites; + + auto I = find_if(RWVec, [Seq](CodeGenSchedRW &RW) { + return makeArrayRef(RW.Sequence) == Seq; + }); + // Index zero reserved for invalid RW. + return I == RWVec.end() ? 0 : std::distance(RWVec.begin(), I); +} + +/// Add this ReadWrite if it doesn't already exist. +unsigned CodeGenSchedModels::findOrInsertRW(ArrayRef<unsigned> Seq, + bool IsRead) { + assert(!Seq.empty() && "cannot insert empty sequence"); + if (Seq.size() == 1) + return Seq.back(); + + unsigned Idx = findRWForSequence(Seq, IsRead); + if (Idx) + return Idx; + + std::vector<CodeGenSchedRW> &RWVec = IsRead ? SchedReads : SchedWrites; + unsigned RWIdx = RWVec.size(); + CodeGenSchedRW SchedRW(RWIdx, IsRead, Seq, genRWName(Seq, IsRead)); + RWVec.push_back(SchedRW); + return RWIdx; +} + +/// Visit all the instruction definitions for this target to gather and +/// enumerate the itinerary classes. These are the explicitly specified +/// SchedClasses. More SchedClasses may be inferred. +void CodeGenSchedModels::collectSchedClasses() { + + // NoItinerary is always the first class at Idx=0 + assert(SchedClasses.empty() && "Expected empty sched class"); + SchedClasses.emplace_back(0, "NoInstrModel", + Records.getDef("NoItinerary")); + SchedClasses.back().ProcIndices.push_back(0); + + // Create a SchedClass for each unique combination of itinerary class and + // SchedRW list. + for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) { + Record *ItinDef = Inst->TheDef->getValueAsDef("Itinerary"); + IdxVec Writes, Reads; + if (!Inst->TheDef->isValueUnset("SchedRW")) + findRWs(Inst->TheDef->getValueAsListOfDefs("SchedRW"), Writes, Reads); + + // ProcIdx == 0 indicates the class applies to all processors. + unsigned SCIdx = addSchedClass(ItinDef, Writes, Reads, /*ProcIndices*/{0}); + InstrClassMap[Inst->TheDef] = SCIdx; + } + // Create classes for InstRW defs. + RecVec InstRWDefs = Records.getAllDerivedDefinitions("InstRW"); + llvm::sort(InstRWDefs, LessRecord()); + LLVM_DEBUG(dbgs() << "\n+++ SCHED CLASSES (createInstRWClass) +++\n"); + for (Record *RWDef : InstRWDefs) + createInstRWClass(RWDef); + + NumInstrSchedClasses = SchedClasses.size(); + + bool EnableDump = false; + LLVM_DEBUG(EnableDump = true); + if (!EnableDump) + return; + + LLVM_DEBUG( + dbgs() + << "\n+++ ITINERARIES and/or MACHINE MODELS (collectSchedClasses) +++\n"); + for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) { + StringRef InstName = Inst->TheDef->getName(); + unsigned SCIdx = getSchedClassIdx(*Inst); + if (!SCIdx) { + LLVM_DEBUG({ + if (!Inst->hasNoSchedulingInfo) + dbgs() << "No machine model for " << Inst->TheDef->getName() << '\n'; + }); + continue; + } + CodeGenSchedClass &SC = getSchedClass(SCIdx); + if (SC.ProcIndices[0] != 0) + PrintFatalError(Inst->TheDef->getLoc(), "Instruction's sched class " + "must not be subtarget specific."); + + IdxVec ProcIndices; + if (SC.ItinClassDef->getName() != "NoItinerary") { + ProcIndices.push_back(0); + dbgs() << "Itinerary for " << InstName << ": " + << SC.ItinClassDef->getName() << '\n'; + } + if (!SC.Writes.empty()) { + ProcIndices.push_back(0); + LLVM_DEBUG({ + dbgs() << "SchedRW machine model for " << InstName; + for (IdxIter WI = SC.Writes.begin(), WE = SC.Writes.end(); WI != WE; + ++WI) + dbgs() << " " << SchedWrites[*WI].Name; + for (IdxIter RI = SC.Reads.begin(), RE = SC.Reads.end(); RI != RE; ++RI) + dbgs() << " " << SchedReads[*RI].Name; + dbgs() << '\n'; + }); + } + const RecVec &RWDefs = SchedClasses[SCIdx].InstRWs; + for (Record *RWDef : RWDefs) { + const CodeGenProcModel &ProcModel = + getProcModel(RWDef->getValueAsDef("SchedModel")); + ProcIndices.push_back(ProcModel.Index); + LLVM_DEBUG(dbgs() << "InstRW on " << ProcModel.ModelName << " for " + << InstName); + IdxVec Writes; + IdxVec Reads; + findRWs(RWDef->getValueAsListOfDefs("OperandReadWrites"), + Writes, Reads); + LLVM_DEBUG({ + for (unsigned WIdx : Writes) + dbgs() << " " << SchedWrites[WIdx].Name; + for (unsigned RIdx : Reads) + dbgs() << " " << SchedReads[RIdx].Name; + dbgs() << '\n'; + }); + } + // If ProcIndices contains zero, the class applies to all processors. + LLVM_DEBUG({ + if (!std::count(ProcIndices.begin(), ProcIndices.end(), 0)) { + for (const CodeGenProcModel &PM : ProcModels) { + if (!std::count(ProcIndices.begin(), ProcIndices.end(), PM.Index)) + dbgs() << "No machine model for " << Inst->TheDef->getName() + << " on processor " << PM.ModelName << '\n'; + } + } + }); + } +} + +// Get the SchedClass index for an instruction. +unsigned +CodeGenSchedModels::getSchedClassIdx(const CodeGenInstruction &Inst) const { + return InstrClassMap.lookup(Inst.TheDef); +} + +std::string +CodeGenSchedModels::createSchedClassName(Record *ItinClassDef, + ArrayRef<unsigned> OperWrites, + ArrayRef<unsigned> OperReads) { + + std::string Name; + if (ItinClassDef && ItinClassDef->getName() != "NoItinerary") + Name = std::string(ItinClassDef->getName()); + for (unsigned Idx : OperWrites) { + if (!Name.empty()) + Name += '_'; + Name += SchedWrites[Idx].Name; + } + for (unsigned Idx : OperReads) { + Name += '_'; + Name += SchedReads[Idx].Name; + } + return Name; +} + +std::string CodeGenSchedModels::createSchedClassName(const RecVec &InstDefs) { + + std::string Name; + for (RecIter I = InstDefs.begin(), E = InstDefs.end(); I != E; ++I) { + if (I != InstDefs.begin()) + Name += '_'; + Name += (*I)->getName(); + } + return Name; +} + +/// Add an inferred sched class from an itinerary class and per-operand list of +/// SchedWrites and SchedReads. ProcIndices contains the set of IDs of +/// processors that may utilize this class. +unsigned CodeGenSchedModels::addSchedClass(Record *ItinClassDef, + ArrayRef<unsigned> OperWrites, + ArrayRef<unsigned> OperReads, + ArrayRef<unsigned> ProcIndices) { + assert(!ProcIndices.empty() && "expect at least one ProcIdx"); + + auto IsKeyEqual = [=](const CodeGenSchedClass &SC) { + return SC.isKeyEqual(ItinClassDef, OperWrites, OperReads); + }; + + auto I = find_if(make_range(schedClassBegin(), schedClassEnd()), IsKeyEqual); + unsigned Idx = I == schedClassEnd() ? 0 : std::distance(schedClassBegin(), I); + if (Idx || SchedClasses[0].isKeyEqual(ItinClassDef, OperWrites, OperReads)) { + IdxVec PI; + std::set_union(SchedClasses[Idx].ProcIndices.begin(), + SchedClasses[Idx].ProcIndices.end(), + ProcIndices.begin(), ProcIndices.end(), + std::back_inserter(PI)); + SchedClasses[Idx].ProcIndices = std::move(PI); + return Idx; + } + Idx = SchedClasses.size(); + SchedClasses.emplace_back(Idx, + createSchedClassName(ItinClassDef, OperWrites, + OperReads), + ItinClassDef); + CodeGenSchedClass &SC = SchedClasses.back(); + SC.Writes = OperWrites; + SC.Reads = OperReads; + SC.ProcIndices = ProcIndices; + + return Idx; +} + +// Create classes for each set of opcodes that are in the same InstReadWrite +// definition across all processors. +void CodeGenSchedModels::createInstRWClass(Record *InstRWDef) { + // ClassInstrs will hold an entry for each subset of Instrs in InstRWDef that + // intersects with an existing class via a previous InstRWDef. Instrs that do + // not intersect with an existing class refer back to their former class as + // determined from ItinDef or SchedRW. + SmallMapVector<unsigned, SmallVector<Record *, 8>, 4> ClassInstrs; + // Sort Instrs into sets. + const RecVec *InstDefs = Sets.expand(InstRWDef); + if (InstDefs->empty()) + PrintFatalError(InstRWDef->getLoc(), "No matching instruction opcodes"); + + for (Record *InstDef : *InstDefs) { + InstClassMapTy::const_iterator Pos = InstrClassMap.find(InstDef); + if (Pos == InstrClassMap.end()) + PrintFatalError(InstDef->getLoc(), "No sched class for instruction."); + unsigned SCIdx = Pos->second; + ClassInstrs[SCIdx].push_back(InstDef); + } + // For each set of Instrs, create a new class if necessary, and map or remap + // the Instrs to it. + for (auto &Entry : ClassInstrs) { + unsigned OldSCIdx = Entry.first; + ArrayRef<Record*> InstDefs = Entry.second; + // If the all instrs in the current class are accounted for, then leave + // them mapped to their old class. + if (OldSCIdx) { + const RecVec &RWDefs = SchedClasses[OldSCIdx].InstRWs; + if (!RWDefs.empty()) { + const RecVec *OrigInstDefs = Sets.expand(RWDefs[0]); + unsigned OrigNumInstrs = + count_if(*OrigInstDefs, [&](Record *OIDef) { + return InstrClassMap[OIDef] == OldSCIdx; + }); + if (OrigNumInstrs == InstDefs.size()) { + assert(SchedClasses[OldSCIdx].ProcIndices[0] == 0 && + "expected a generic SchedClass"); + Record *RWModelDef = InstRWDef->getValueAsDef("SchedModel"); + // Make sure we didn't already have a InstRW containing this + // instruction on this model. + for (Record *RWD : RWDefs) { + if (RWD->getValueAsDef("SchedModel") == RWModelDef && + RWModelDef->getValueAsBit("FullInstRWOverlapCheck")) { + assert(!InstDefs.empty()); // Checked at function start. + PrintFatalError + (InstRWDef->getLoc(), + "Overlapping InstRW definition for \"" + + InstDefs.front()->getName() + + "\" also matches previous \"" + + RWD->getValue("Instrs")->getValue()->getAsString() + + "\"."); + } + } + LLVM_DEBUG(dbgs() << "InstRW: Reuse SC " << OldSCIdx << ":" + << SchedClasses[OldSCIdx].Name << " on " + << RWModelDef->getName() << "\n"); + SchedClasses[OldSCIdx].InstRWs.push_back(InstRWDef); + continue; + } + } + } + unsigned SCIdx = SchedClasses.size(); + SchedClasses.emplace_back(SCIdx, createSchedClassName(InstDefs), nullptr); + CodeGenSchedClass &SC = SchedClasses.back(); + LLVM_DEBUG(dbgs() << "InstRW: New SC " << SCIdx << ":" << SC.Name << " on " + << InstRWDef->getValueAsDef("SchedModel")->getName() + << "\n"); + + // Preserve ItinDef and Writes/Reads for processors without an InstRW entry. + SC.ItinClassDef = SchedClasses[OldSCIdx].ItinClassDef; + SC.Writes = SchedClasses[OldSCIdx].Writes; + SC.Reads = SchedClasses[OldSCIdx].Reads; + SC.ProcIndices.push_back(0); + // If we had an old class, copy it's InstRWs to this new class. + if (OldSCIdx) { + Record *RWModelDef = InstRWDef->getValueAsDef("SchedModel"); + for (Record *OldRWDef : SchedClasses[OldSCIdx].InstRWs) { + if (OldRWDef->getValueAsDef("SchedModel") == RWModelDef) { + assert(!InstDefs.empty()); // Checked at function start. + PrintFatalError + (InstRWDef->getLoc(), + "Overlapping InstRW definition for \"" + + InstDefs.front()->getName() + + "\" also matches previous \"" + + OldRWDef->getValue("Instrs")->getValue()->getAsString() + + "\"."); + } + assert(OldRWDef != InstRWDef && + "SchedClass has duplicate InstRW def"); + SC.InstRWs.push_back(OldRWDef); + } + } + // Map each Instr to this new class. + for (Record *InstDef : InstDefs) + InstrClassMap[InstDef] = SCIdx; + SC.InstRWs.push_back(InstRWDef); + } +} + +// True if collectProcItins found anything. +bool CodeGenSchedModels::hasItineraries() const { + for (const CodeGenProcModel &PM : make_range(procModelBegin(),procModelEnd())) + if (PM.hasItineraries()) + return true; + return false; +} + +// Gather the processor itineraries. +void CodeGenSchedModels::collectProcItins() { + LLVM_DEBUG(dbgs() << "\n+++ PROBLEM ITINERARIES (collectProcItins) +++\n"); + for (CodeGenProcModel &ProcModel : ProcModels) { + if (!ProcModel.hasItineraries()) + continue; + + RecVec ItinRecords = ProcModel.ItinsDef->getValueAsListOfDefs("IID"); + assert(!ItinRecords.empty() && "ProcModel.hasItineraries is incorrect"); + + // Populate ItinDefList with Itinerary records. + ProcModel.ItinDefList.resize(NumInstrSchedClasses); + + // Insert each itinerary data record in the correct position within + // the processor model's ItinDefList. + for (Record *ItinData : ItinRecords) { + const Record *ItinDef = ItinData->getValueAsDef("TheClass"); + bool FoundClass = false; + + for (const CodeGenSchedClass &SC : + make_range(schedClassBegin(), schedClassEnd())) { + // Multiple SchedClasses may share an itinerary. Update all of them. + if (SC.ItinClassDef == ItinDef) { + ProcModel.ItinDefList[SC.Index] = ItinData; + FoundClass = true; + } + } + if (!FoundClass) { + LLVM_DEBUG(dbgs() << ProcModel.ItinsDef->getName() + << " missing class for itinerary " + << ItinDef->getName() << '\n'); + } + } + // Check for missing itinerary entries. + assert(!ProcModel.ItinDefList[0] && "NoItinerary class can't have rec"); + LLVM_DEBUG( + for (unsigned i = 1, N = ProcModel.ItinDefList.size(); i < N; ++i) { + if (!ProcModel.ItinDefList[i]) + dbgs() << ProcModel.ItinsDef->getName() + << " missing itinerary for class " << SchedClasses[i].Name + << '\n'; + }); + } +} + +// Gather the read/write types for each itinerary class. +void CodeGenSchedModels::collectProcItinRW() { + RecVec ItinRWDefs = Records.getAllDerivedDefinitions("ItinRW"); + llvm::sort(ItinRWDefs, LessRecord()); + for (Record *RWDef : ItinRWDefs) { + if (!RWDef->getValueInit("SchedModel")->isComplete()) + PrintFatalError(RWDef->getLoc(), "SchedModel is undefined"); + Record *ModelDef = RWDef->getValueAsDef("SchedModel"); + ProcModelMapTy::const_iterator I = ProcModelMap.find(ModelDef); + if (I == ProcModelMap.end()) { + PrintFatalError(RWDef->getLoc(), "Undefined SchedMachineModel " + + ModelDef->getName()); + } + ProcModels[I->second].ItinRWDefs.push_back(RWDef); + } +} + +// Gather the unsupported features for processor models. +void CodeGenSchedModels::collectProcUnsupportedFeatures() { + for (CodeGenProcModel &ProcModel : ProcModels) { + for (Record *Pred : ProcModel.ModelDef->getValueAsListOfDefs("UnsupportedFeatures")) { + ProcModel.UnsupportedFeaturesDefs.push_back(Pred); + } + } +} + +/// Infer new classes from existing classes. In the process, this may create new +/// SchedWrites from sequences of existing SchedWrites. +void CodeGenSchedModels::inferSchedClasses() { + LLVM_DEBUG( + dbgs() << "\n+++ INFERRING SCHED CLASSES (inferSchedClasses) +++\n"); + LLVM_DEBUG(dbgs() << NumInstrSchedClasses << " instr sched classes.\n"); + + // Visit all existing classes and newly created classes. + for (unsigned Idx = 0; Idx != SchedClasses.size(); ++Idx) { + assert(SchedClasses[Idx].Index == Idx && "bad SCIdx"); + + if (SchedClasses[Idx].ItinClassDef) + inferFromItinClass(SchedClasses[Idx].ItinClassDef, Idx); + if (!SchedClasses[Idx].InstRWs.empty()) + inferFromInstRWs(Idx); + if (!SchedClasses[Idx].Writes.empty()) { + inferFromRW(SchedClasses[Idx].Writes, SchedClasses[Idx].Reads, + Idx, SchedClasses[Idx].ProcIndices); + } + assert(SchedClasses.size() < (NumInstrSchedClasses*6) && + "too many SchedVariants"); + } +} + +/// Infer classes from per-processor itinerary resources. +void CodeGenSchedModels::inferFromItinClass(Record *ItinClassDef, + unsigned FromClassIdx) { + for (unsigned PIdx = 0, PEnd = ProcModels.size(); PIdx != PEnd; ++PIdx) { + const CodeGenProcModel &PM = ProcModels[PIdx]; + // For all ItinRW entries. + bool HasMatch = false; + for (const Record *Rec : PM.ItinRWDefs) { + RecVec Matched = Rec->getValueAsListOfDefs("MatchedItinClasses"); + if (!std::count(Matched.begin(), Matched.end(), ItinClassDef)) + continue; + if (HasMatch) + PrintFatalError(Rec->getLoc(), "Duplicate itinerary class " + + ItinClassDef->getName() + + " in ItinResources for " + PM.ModelName); + HasMatch = true; + IdxVec Writes, Reads; + findRWs(Rec->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads); + inferFromRW(Writes, Reads, FromClassIdx, PIdx); + } + } +} + +/// Infer classes from per-processor InstReadWrite definitions. +void CodeGenSchedModels::inferFromInstRWs(unsigned SCIdx) { + for (unsigned I = 0, E = SchedClasses[SCIdx].InstRWs.size(); I != E; ++I) { + assert(SchedClasses[SCIdx].InstRWs.size() == E && "InstrRWs was mutated!"); + Record *Rec = SchedClasses[SCIdx].InstRWs[I]; + const RecVec *InstDefs = Sets.expand(Rec); + RecIter II = InstDefs->begin(), IE = InstDefs->end(); + for (; II != IE; ++II) { + if (InstrClassMap[*II] == SCIdx) + break; + } + // If this class no longer has any instructions mapped to it, it has become + // irrelevant. + if (II == IE) + continue; + IdxVec Writes, Reads; + findRWs(Rec->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads); + unsigned PIdx = getProcModel(Rec->getValueAsDef("SchedModel")).Index; + inferFromRW(Writes, Reads, SCIdx, PIdx); // May mutate SchedClasses. + } +} + +namespace { + +// Helper for substituteVariantOperand. +struct TransVariant { + Record *VarOrSeqDef; // Variant or sequence. + unsigned RWIdx; // Index of this variant or sequence's matched type. + unsigned ProcIdx; // Processor model index or zero for any. + unsigned TransVecIdx; // Index into PredTransitions::TransVec. + + TransVariant(Record *def, unsigned rwi, unsigned pi, unsigned ti): + VarOrSeqDef(def), RWIdx(rwi), ProcIdx(pi), TransVecIdx(ti) {} +}; + +// Associate a predicate with the SchedReadWrite that it guards. +// RWIdx is the index of the read/write variant. +struct PredCheck { + bool IsRead; + unsigned RWIdx; + Record *Predicate; + + PredCheck(bool r, unsigned w, Record *p): IsRead(r), RWIdx(w), Predicate(p) {} +}; + +// A Predicate transition is a list of RW sequences guarded by a PredTerm. +struct PredTransition { + // A predicate term is a conjunction of PredChecks. + SmallVector<PredCheck, 4> PredTerm; + SmallVector<SmallVector<unsigned,4>, 16> WriteSequences; + SmallVector<SmallVector<unsigned,4>, 16> ReadSequences; + SmallVector<unsigned, 4> ProcIndices; +}; + +// Encapsulate a set of partially constructed transitions. +// The results are built by repeated calls to substituteVariants. +class PredTransitions { + CodeGenSchedModels &SchedModels; + +public: + std::vector<PredTransition> TransVec; + + PredTransitions(CodeGenSchedModels &sm): SchedModels(sm) {} + + void substituteVariantOperand(const SmallVectorImpl<unsigned> &RWSeq, + bool IsRead, unsigned StartIdx); + + void substituteVariants(const PredTransition &Trans); + +#ifndef NDEBUG + void dump() const; +#endif + +private: + bool mutuallyExclusive(Record *PredDef, ArrayRef<PredCheck> Term); + void getIntersectingVariants( + const CodeGenSchedRW &SchedRW, unsigned TransIdx, + std::vector<TransVariant> &IntersectingVariants); + void pushVariant(const TransVariant &VInfo, bool IsRead); +}; + +} // end anonymous namespace + +// Return true if this predicate is mutually exclusive with a PredTerm. This +// degenerates into checking if the predicate is mutually exclusive with any +// predicate in the Term's conjunction. +// +// All predicates associated with a given SchedRW are considered mutually +// exclusive. This should work even if the conditions expressed by the +// predicates are not exclusive because the predicates for a given SchedWrite +// are always checked in the order they are defined in the .td file. Later +// conditions implicitly negate any prior condition. +bool PredTransitions::mutuallyExclusive(Record *PredDef, + ArrayRef<PredCheck> Term) { + for (const PredCheck &PC: Term) { + if (PC.Predicate == PredDef) + return false; + + const CodeGenSchedRW &SchedRW = SchedModels.getSchedRW(PC.RWIdx, PC.IsRead); + assert(SchedRW.HasVariants && "PredCheck must refer to a SchedVariant"); + RecVec Variants = SchedRW.TheDef->getValueAsListOfDefs("Variants"); + if (any_of(Variants, [PredDef](const Record *R) { + return R->getValueAsDef("Predicate") == PredDef; + })) + return true; + } + return false; +} + +static bool hasAliasedVariants(const CodeGenSchedRW &RW, + CodeGenSchedModels &SchedModels) { + if (RW.HasVariants) + return true; + + for (Record *Alias : RW.Aliases) { + const CodeGenSchedRW &AliasRW = + SchedModels.getSchedRW(Alias->getValueAsDef("AliasRW")); + if (AliasRW.HasVariants) + return true; + if (AliasRW.IsSequence) { + IdxVec ExpandedRWs; + SchedModels.expandRWSequence(AliasRW.Index, ExpandedRWs, AliasRW.IsRead); + for (unsigned SI : ExpandedRWs) { + if (hasAliasedVariants(SchedModels.getSchedRW(SI, AliasRW.IsRead), + SchedModels)) + return true; + } + } + } + return false; +} + +static bool hasVariant(ArrayRef<PredTransition> Transitions, + CodeGenSchedModels &SchedModels) { + for (const PredTransition &PTI : Transitions) { + for (const SmallVectorImpl<unsigned> &WSI : PTI.WriteSequences) + for (unsigned WI : WSI) + if (hasAliasedVariants(SchedModels.getSchedWrite(WI), SchedModels)) + return true; + + for (const SmallVectorImpl<unsigned> &RSI : PTI.ReadSequences) + for (unsigned RI : RSI) + if (hasAliasedVariants(SchedModels.getSchedRead(RI), SchedModels)) + return true; + } + return false; +} + +// Populate IntersectingVariants with any variants or aliased sequences of the +// given SchedRW whose processor indices and predicates are not mutually +// exclusive with the given transition. +void PredTransitions::getIntersectingVariants( + const CodeGenSchedRW &SchedRW, unsigned TransIdx, + std::vector<TransVariant> &IntersectingVariants) { + + bool GenericRW = false; + + std::vector<TransVariant> Variants; + if (SchedRW.HasVariants) { + unsigned VarProcIdx = 0; + if (SchedRW.TheDef->getValueInit("SchedModel")->isComplete()) { + Record *ModelDef = SchedRW.TheDef->getValueAsDef("SchedModel"); + VarProcIdx = SchedModels.getProcModel(ModelDef).Index; + } + // Push each variant. Assign TransVecIdx later. + const RecVec VarDefs = SchedRW.TheDef->getValueAsListOfDefs("Variants"); + for (Record *VarDef : VarDefs) + Variants.emplace_back(VarDef, SchedRW.Index, VarProcIdx, 0); + if (VarProcIdx == 0) + GenericRW = true; + } + for (RecIter AI = SchedRW.Aliases.begin(), AE = SchedRW.Aliases.end(); + AI != AE; ++AI) { + // If either the SchedAlias itself or the SchedReadWrite that it aliases + // to is defined within a processor model, constrain all variants to + // that processor. + unsigned AliasProcIdx = 0; + if ((*AI)->getValueInit("SchedModel")->isComplete()) { + Record *ModelDef = (*AI)->getValueAsDef("SchedModel"); + AliasProcIdx = SchedModels.getProcModel(ModelDef).Index; + } + const CodeGenSchedRW &AliasRW = + SchedModels.getSchedRW((*AI)->getValueAsDef("AliasRW")); + + if (AliasRW.HasVariants) { + const RecVec VarDefs = AliasRW.TheDef->getValueAsListOfDefs("Variants"); + for (Record *VD : VarDefs) + Variants.emplace_back(VD, AliasRW.Index, AliasProcIdx, 0); + } + if (AliasRW.IsSequence) + Variants.emplace_back(AliasRW.TheDef, SchedRW.Index, AliasProcIdx, 0); + if (AliasProcIdx == 0) + GenericRW = true; + } + for (TransVariant &Variant : Variants) { + // Don't expand variants if the processor models don't intersect. + // A zero processor index means any processor. + SmallVectorImpl<unsigned> &ProcIndices = TransVec[TransIdx].ProcIndices; + if (ProcIndices[0] && Variant.ProcIdx) { + unsigned Cnt = std::count(ProcIndices.begin(), ProcIndices.end(), + Variant.ProcIdx); + if (!Cnt) + continue; + if (Cnt > 1) { + const CodeGenProcModel &PM = + *(SchedModels.procModelBegin() + Variant.ProcIdx); + PrintFatalError(Variant.VarOrSeqDef->getLoc(), + "Multiple variants defined for processor " + + PM.ModelName + + " Ensure only one SchedAlias exists per RW."); + } + } + if (Variant.VarOrSeqDef->isSubClassOf("SchedVar")) { + Record *PredDef = Variant.VarOrSeqDef->getValueAsDef("Predicate"); + if (mutuallyExclusive(PredDef, TransVec[TransIdx].PredTerm)) + continue; + } + if (IntersectingVariants.empty()) { + // The first variant builds on the existing transition. + Variant.TransVecIdx = TransIdx; + IntersectingVariants.push_back(Variant); + } + else { + // Push another copy of the current transition for more variants. + Variant.TransVecIdx = TransVec.size(); + IntersectingVariants.push_back(Variant); + TransVec.push_back(TransVec[TransIdx]); + } + } + if (GenericRW && IntersectingVariants.empty()) { + PrintFatalError(SchedRW.TheDef->getLoc(), "No variant of this type has " + "a matching predicate on any processor"); + } +} + +// Push the Reads/Writes selected by this variant onto the PredTransition +// specified by VInfo. +void PredTransitions:: +pushVariant(const TransVariant &VInfo, bool IsRead) { + PredTransition &Trans = TransVec[VInfo.TransVecIdx]; + + // If this operand transition is reached through a processor-specific alias, + // then the whole transition is specific to this processor. + if (VInfo.ProcIdx != 0) + Trans.ProcIndices.assign(1, VInfo.ProcIdx); + + IdxVec SelectedRWs; + if (VInfo.VarOrSeqDef->isSubClassOf("SchedVar")) { + Record *PredDef = VInfo.VarOrSeqDef->getValueAsDef("Predicate"); + Trans.PredTerm.emplace_back(IsRead, VInfo.RWIdx,PredDef); + RecVec SelectedDefs = VInfo.VarOrSeqDef->getValueAsListOfDefs("Selected"); + SchedModels.findRWs(SelectedDefs, SelectedRWs, IsRead); + } + else { + assert(VInfo.VarOrSeqDef->isSubClassOf("WriteSequence") && + "variant must be a SchedVariant or aliased WriteSequence"); + SelectedRWs.push_back(SchedModels.getSchedRWIdx(VInfo.VarOrSeqDef, IsRead)); + } + + const CodeGenSchedRW &SchedRW = SchedModels.getSchedRW(VInfo.RWIdx, IsRead); + + SmallVectorImpl<SmallVector<unsigned,4>> &RWSequences = IsRead + ? Trans.ReadSequences : Trans.WriteSequences; + if (SchedRW.IsVariadic) { + unsigned OperIdx = RWSequences.size()-1; + // Make N-1 copies of this transition's last sequence. + RWSequences.insert(RWSequences.end(), SelectedRWs.size() - 1, + RWSequences[OperIdx]); + // Push each of the N elements of the SelectedRWs onto a copy of the last + // sequence (split the current operand into N operands). + // Note that write sequences should be expanded within this loop--the entire + // sequence belongs to a single operand. + for (IdxIter RWI = SelectedRWs.begin(), RWE = SelectedRWs.end(); + RWI != RWE; ++RWI, ++OperIdx) { + IdxVec ExpandedRWs; + if (IsRead) + ExpandedRWs.push_back(*RWI); + else + SchedModels.expandRWSequence(*RWI, ExpandedRWs, IsRead); + RWSequences[OperIdx].insert(RWSequences[OperIdx].end(), + ExpandedRWs.begin(), ExpandedRWs.end()); + } + assert(OperIdx == RWSequences.size() && "missed a sequence"); + } + else { + // Push this transition's expanded sequence onto this transition's last + // sequence (add to the current operand's sequence). + SmallVectorImpl<unsigned> &Seq = RWSequences.back(); + IdxVec ExpandedRWs; + for (IdxIter RWI = SelectedRWs.begin(), RWE = SelectedRWs.end(); + RWI != RWE; ++RWI) { + if (IsRead) + ExpandedRWs.push_back(*RWI); + else + SchedModels.expandRWSequence(*RWI, ExpandedRWs, IsRead); + } + Seq.insert(Seq.end(), ExpandedRWs.begin(), ExpandedRWs.end()); + } +} + +// RWSeq is a sequence of all Reads or all Writes for the next read or write +// operand. StartIdx is an index into TransVec where partial results +// starts. RWSeq must be applied to all transitions between StartIdx and the end +// of TransVec. +void PredTransitions::substituteVariantOperand( + const SmallVectorImpl<unsigned> &RWSeq, bool IsRead, unsigned StartIdx) { + + // Visit each original RW within the current sequence. + for (SmallVectorImpl<unsigned>::const_iterator + RWI = RWSeq.begin(), RWE = RWSeq.end(); RWI != RWE; ++RWI) { + const CodeGenSchedRW &SchedRW = SchedModels.getSchedRW(*RWI, IsRead); + // Push this RW on all partial PredTransitions or distribute variants. + // New PredTransitions may be pushed within this loop which should not be + // revisited (TransEnd must be loop invariant). + for (unsigned TransIdx = StartIdx, TransEnd = TransVec.size(); + TransIdx != TransEnd; ++TransIdx) { + // In the common case, push RW onto the current operand's sequence. + if (!hasAliasedVariants(SchedRW, SchedModels)) { + if (IsRead) + TransVec[TransIdx].ReadSequences.back().push_back(*RWI); + else + TransVec[TransIdx].WriteSequences.back().push_back(*RWI); + continue; + } + // Distribute this partial PredTransition across intersecting variants. + // This will push a copies of TransVec[TransIdx] on the back of TransVec. + std::vector<TransVariant> IntersectingVariants; + getIntersectingVariants(SchedRW, TransIdx, IntersectingVariants); + // Now expand each variant on top of its copy of the transition. + for (std::vector<TransVariant>::const_iterator + IVI = IntersectingVariants.begin(), + IVE = IntersectingVariants.end(); + IVI != IVE; ++IVI) { + pushVariant(*IVI, IsRead); + } + } + } +} + +// For each variant of a Read/Write in Trans, substitute the sequence of +// Read/Writes guarded by the variant. This is exponential in the number of +// variant Read/Writes, but in practice detection of mutually exclusive +// predicates should result in linear growth in the total number variants. +// +// This is one step in a breadth-first search of nested variants. +void PredTransitions::substituteVariants(const PredTransition &Trans) { + // Build up a set of partial results starting at the back of + // PredTransitions. Remember the first new transition. + unsigned StartIdx = TransVec.size(); + TransVec.emplace_back(); + TransVec.back().PredTerm = Trans.PredTerm; + TransVec.back().ProcIndices = Trans.ProcIndices; + + // Visit each original write sequence. + for (SmallVectorImpl<SmallVector<unsigned,4>>::const_iterator + WSI = Trans.WriteSequences.begin(), WSE = Trans.WriteSequences.end(); + WSI != WSE; ++WSI) { + // Push a new (empty) write sequence onto all partial Transitions. + for (std::vector<PredTransition>::iterator I = + TransVec.begin() + StartIdx, E = TransVec.end(); I != E; ++I) { + I->WriteSequences.emplace_back(); + } + substituteVariantOperand(*WSI, /*IsRead=*/false, StartIdx); + } + // Visit each original read sequence. + for (SmallVectorImpl<SmallVector<unsigned,4>>::const_iterator + RSI = Trans.ReadSequences.begin(), RSE = Trans.ReadSequences.end(); + RSI != RSE; ++RSI) { + // Push a new (empty) read sequence onto all partial Transitions. + for (std::vector<PredTransition>::iterator I = + TransVec.begin() + StartIdx, E = TransVec.end(); I != E; ++I) { + I->ReadSequences.emplace_back(); + } + substituteVariantOperand(*RSI, /*IsRead=*/true, StartIdx); + } +} + +// Create a new SchedClass for each variant found by inferFromRW. Pass +static void inferFromTransitions(ArrayRef<PredTransition> LastTransitions, + unsigned FromClassIdx, + CodeGenSchedModels &SchedModels) { + // For each PredTransition, create a new CodeGenSchedTransition, which usually + // requires creating a new SchedClass. + for (ArrayRef<PredTransition>::iterator + I = LastTransitions.begin(), E = LastTransitions.end(); I != E; ++I) { + IdxVec OperWritesVariant; + transform(I->WriteSequences, std::back_inserter(OperWritesVariant), + [&SchedModels](ArrayRef<unsigned> WS) { + return SchedModels.findOrInsertRW(WS, /*IsRead=*/false); + }); + IdxVec OperReadsVariant; + transform(I->ReadSequences, std::back_inserter(OperReadsVariant), + [&SchedModels](ArrayRef<unsigned> RS) { + return SchedModels.findOrInsertRW(RS, /*IsRead=*/true); + }); + CodeGenSchedTransition SCTrans; + SCTrans.ToClassIdx = + SchedModels.addSchedClass(/*ItinClassDef=*/nullptr, OperWritesVariant, + OperReadsVariant, I->ProcIndices); + SCTrans.ProcIndices.assign(I->ProcIndices.begin(), I->ProcIndices.end()); + // The final PredTerm is unique set of predicates guarding the transition. + RecVec Preds; + transform(I->PredTerm, std::back_inserter(Preds), + [](const PredCheck &P) { + return P.Predicate; + }); + Preds.erase(std::unique(Preds.begin(), Preds.end()), Preds.end()); + SCTrans.PredTerm = std::move(Preds); + SchedModels.getSchedClass(FromClassIdx) + .Transitions.push_back(std::move(SCTrans)); + } +} + +// Create new SchedClasses for the given ReadWrite list. If any of the +// ReadWrites refers to a SchedVariant, create a new SchedClass for each variant +// of the ReadWrite list, following Aliases if necessary. +void CodeGenSchedModels::inferFromRW(ArrayRef<unsigned> OperWrites, + ArrayRef<unsigned> OperReads, + unsigned FromClassIdx, + ArrayRef<unsigned> ProcIndices) { + LLVM_DEBUG(dbgs() << "INFER RW proc("; dumpIdxVec(ProcIndices); + dbgs() << ") "); + + // Create a seed transition with an empty PredTerm and the expanded sequences + // of SchedWrites for the current SchedClass. + std::vector<PredTransition> LastTransitions; + LastTransitions.emplace_back(); + LastTransitions.back().ProcIndices.append(ProcIndices.begin(), + ProcIndices.end()); + + for (unsigned WriteIdx : OperWrites) { + IdxVec WriteSeq; + expandRWSequence(WriteIdx, WriteSeq, /*IsRead=*/false); + LastTransitions[0].WriteSequences.emplace_back(); + SmallVectorImpl<unsigned> &Seq = LastTransitions[0].WriteSequences.back(); + Seq.append(WriteSeq.begin(), WriteSeq.end()); + LLVM_DEBUG(dbgs() << "("; dumpIdxVec(Seq); dbgs() << ") "); + } + LLVM_DEBUG(dbgs() << " Reads: "); + for (unsigned ReadIdx : OperReads) { + IdxVec ReadSeq; + expandRWSequence(ReadIdx, ReadSeq, /*IsRead=*/true); + LastTransitions[0].ReadSequences.emplace_back(); + SmallVectorImpl<unsigned> &Seq = LastTransitions[0].ReadSequences.back(); + Seq.append(ReadSeq.begin(), ReadSeq.end()); + LLVM_DEBUG(dbgs() << "("; dumpIdxVec(Seq); dbgs() << ") "); + } + LLVM_DEBUG(dbgs() << '\n'); + + // Collect all PredTransitions for individual operands. + // Iterate until no variant writes remain. + while (hasVariant(LastTransitions, *this)) { + PredTransitions Transitions(*this); + for (const PredTransition &Trans : LastTransitions) + Transitions.substituteVariants(Trans); + LLVM_DEBUG(Transitions.dump()); + LastTransitions.swap(Transitions.TransVec); + } + // If the first transition has no variants, nothing to do. + if (LastTransitions[0].PredTerm.empty()) + return; + + // WARNING: We are about to mutate the SchedClasses vector. Do not refer to + // OperWrites, OperReads, or ProcIndices after calling inferFromTransitions. + inferFromTransitions(LastTransitions, FromClassIdx, *this); +} + +// Check if any processor resource group contains all resource records in +// SubUnits. +bool CodeGenSchedModels::hasSuperGroup(RecVec &SubUnits, CodeGenProcModel &PM) { + for (unsigned i = 0, e = PM.ProcResourceDefs.size(); i < e; ++i) { + if (!PM.ProcResourceDefs[i]->isSubClassOf("ProcResGroup")) + continue; + RecVec SuperUnits = + PM.ProcResourceDefs[i]->getValueAsListOfDefs("Resources"); + RecIter RI = SubUnits.begin(), RE = SubUnits.end(); + for ( ; RI != RE; ++RI) { + if (!is_contained(SuperUnits, *RI)) { + break; + } + } + if (RI == RE) + return true; + } + return false; +} + +// Verify that overlapping groups have a common supergroup. +void CodeGenSchedModels::verifyProcResourceGroups(CodeGenProcModel &PM) { + for (unsigned i = 0, e = PM.ProcResourceDefs.size(); i < e; ++i) { + if (!PM.ProcResourceDefs[i]->isSubClassOf("ProcResGroup")) + continue; + RecVec CheckUnits = + PM.ProcResourceDefs[i]->getValueAsListOfDefs("Resources"); + for (unsigned j = i+1; j < e; ++j) { + if (!PM.ProcResourceDefs[j]->isSubClassOf("ProcResGroup")) + continue; + RecVec OtherUnits = + PM.ProcResourceDefs[j]->getValueAsListOfDefs("Resources"); + if (std::find_first_of(CheckUnits.begin(), CheckUnits.end(), + OtherUnits.begin(), OtherUnits.end()) + != CheckUnits.end()) { + // CheckUnits and OtherUnits overlap + OtherUnits.insert(OtherUnits.end(), CheckUnits.begin(), + CheckUnits.end()); + if (!hasSuperGroup(OtherUnits, PM)) { + PrintFatalError((PM.ProcResourceDefs[i])->getLoc(), + "proc resource group overlaps with " + + PM.ProcResourceDefs[j]->getName() + + " but no supergroup contains both."); + } + } + } + } +} + +// Collect all the RegisterFile definitions available in this target. +void CodeGenSchedModels::collectRegisterFiles() { + RecVec RegisterFileDefs = Records.getAllDerivedDefinitions("RegisterFile"); + + // RegisterFiles is the vector of CodeGenRegisterFile. + for (Record *RF : RegisterFileDefs) { + // For each register file definition, construct a CodeGenRegisterFile object + // and add it to the appropriate scheduling model. + CodeGenProcModel &PM = getProcModel(RF->getValueAsDef("SchedModel")); + PM.RegisterFiles.emplace_back(CodeGenRegisterFile(RF->getName(),RF)); + CodeGenRegisterFile &CGRF = PM.RegisterFiles.back(); + CGRF.MaxMovesEliminatedPerCycle = + RF->getValueAsInt("MaxMovesEliminatedPerCycle"); + CGRF.AllowZeroMoveEliminationOnly = + RF->getValueAsBit("AllowZeroMoveEliminationOnly"); + + // Now set the number of physical registers as well as the cost of registers + // in each register class. + CGRF.NumPhysRegs = RF->getValueAsInt("NumPhysRegs"); + if (!CGRF.NumPhysRegs) { + PrintFatalError(RF->getLoc(), + "Invalid RegisterFile with zero physical registers"); + } + + RecVec RegisterClasses = RF->getValueAsListOfDefs("RegClasses"); + std::vector<int64_t> RegisterCosts = RF->getValueAsListOfInts("RegCosts"); + ListInit *MoveElimInfo = RF->getValueAsListInit("AllowMoveElimination"); + for (unsigned I = 0, E = RegisterClasses.size(); I < E; ++I) { + int Cost = RegisterCosts.size() > I ? RegisterCosts[I] : 1; + + bool AllowMoveElim = false; + if (MoveElimInfo->size() > I) { + BitInit *Val = cast<BitInit>(MoveElimInfo->getElement(I)); + AllowMoveElim = Val->getValue(); + } + + CGRF.Costs.emplace_back(RegisterClasses[I], Cost, AllowMoveElim); + } + } +} + +// Collect and sort WriteRes, ReadAdvance, and ProcResources. +void CodeGenSchedModels::collectProcResources() { + ProcResourceDefs = Records.getAllDerivedDefinitions("ProcResourceUnits"); + ProcResGroups = Records.getAllDerivedDefinitions("ProcResGroup"); + + // Add any subtarget-specific SchedReadWrites that are directly associated + // with processor resources. Refer to the parent SchedClass's ProcIndices to + // determine which processors they apply to. + for (const CodeGenSchedClass &SC : + make_range(schedClassBegin(), schedClassEnd())) { + if (SC.ItinClassDef) { + collectItinProcResources(SC.ItinClassDef); + continue; + } + + // This class may have a default ReadWrite list which can be overriden by + // InstRW definitions. + for (Record *RW : SC.InstRWs) { + Record *RWModelDef = RW->getValueAsDef("SchedModel"); + unsigned PIdx = getProcModel(RWModelDef).Index; + IdxVec Writes, Reads; + findRWs(RW->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads); + collectRWResources(Writes, Reads, PIdx); + } + + collectRWResources(SC.Writes, SC.Reads, SC.ProcIndices); + } + // Add resources separately defined by each subtarget. + RecVec WRDefs = Records.getAllDerivedDefinitions("WriteRes"); + for (Record *WR : WRDefs) { + Record *ModelDef = WR->getValueAsDef("SchedModel"); + addWriteRes(WR, getProcModel(ModelDef).Index); + } + RecVec SWRDefs = Records.getAllDerivedDefinitions("SchedWriteRes"); + for (Record *SWR : SWRDefs) { + Record *ModelDef = SWR->getValueAsDef("SchedModel"); + addWriteRes(SWR, getProcModel(ModelDef).Index); + } + RecVec RADefs = Records.getAllDerivedDefinitions("ReadAdvance"); + for (Record *RA : RADefs) { + Record *ModelDef = RA->getValueAsDef("SchedModel"); + addReadAdvance(RA, getProcModel(ModelDef).Index); + } + RecVec SRADefs = Records.getAllDerivedDefinitions("SchedReadAdvance"); + for (Record *SRA : SRADefs) { + if (SRA->getValueInit("SchedModel")->isComplete()) { + Record *ModelDef = SRA->getValueAsDef("SchedModel"); + addReadAdvance(SRA, getProcModel(ModelDef).Index); + } + } + // Add ProcResGroups that are defined within this processor model, which may + // not be directly referenced but may directly specify a buffer size. + RecVec ProcResGroups = Records.getAllDerivedDefinitions("ProcResGroup"); + for (Record *PRG : ProcResGroups) { + if (!PRG->getValueInit("SchedModel")->isComplete()) + continue; + CodeGenProcModel &PM = getProcModel(PRG->getValueAsDef("SchedModel")); + if (!is_contained(PM.ProcResourceDefs, PRG)) + PM.ProcResourceDefs.push_back(PRG); + } + // Add ProcResourceUnits unconditionally. + for (Record *PRU : Records.getAllDerivedDefinitions("ProcResourceUnits")) { + if (!PRU->getValueInit("SchedModel")->isComplete()) + continue; + CodeGenProcModel &PM = getProcModel(PRU->getValueAsDef("SchedModel")); + if (!is_contained(PM.ProcResourceDefs, PRU)) + PM.ProcResourceDefs.push_back(PRU); + } + // Finalize each ProcModel by sorting the record arrays. + for (CodeGenProcModel &PM : ProcModels) { + llvm::sort(PM.WriteResDefs, LessRecord()); + llvm::sort(PM.ReadAdvanceDefs, LessRecord()); + llvm::sort(PM.ProcResourceDefs, LessRecord()); + LLVM_DEBUG( + PM.dump(); + dbgs() << "WriteResDefs: "; for (RecIter RI = PM.WriteResDefs.begin(), + RE = PM.WriteResDefs.end(); + RI != RE; ++RI) { + if ((*RI)->isSubClassOf("WriteRes")) + dbgs() << (*RI)->getValueAsDef("WriteType")->getName() << " "; + else + dbgs() << (*RI)->getName() << " "; + } dbgs() << "\nReadAdvanceDefs: "; + for (RecIter RI = PM.ReadAdvanceDefs.begin(), + RE = PM.ReadAdvanceDefs.end(); + RI != RE; ++RI) { + if ((*RI)->isSubClassOf("ReadAdvance")) + dbgs() << (*RI)->getValueAsDef("ReadType")->getName() << " "; + else + dbgs() << (*RI)->getName() << " "; + } dbgs() + << "\nProcResourceDefs: "; + for (RecIter RI = PM.ProcResourceDefs.begin(), + RE = PM.ProcResourceDefs.end(); + RI != RE; ++RI) { dbgs() << (*RI)->getName() << " "; } dbgs() + << '\n'); + verifyProcResourceGroups(PM); + } + + ProcResourceDefs.clear(); + ProcResGroups.clear(); +} + +void CodeGenSchedModels::checkCompleteness() { + bool Complete = true; + bool HadCompleteModel = false; + for (const CodeGenProcModel &ProcModel : procModels()) { + const bool HasItineraries = ProcModel.hasItineraries(); + if (!ProcModel.ModelDef->getValueAsBit("CompleteModel")) + continue; + for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) { + if (Inst->hasNoSchedulingInfo) + continue; + if (ProcModel.isUnsupported(*Inst)) + continue; + unsigned SCIdx = getSchedClassIdx(*Inst); + if (!SCIdx) { + if (Inst->TheDef->isValueUnset("SchedRW") && !HadCompleteModel) { + PrintError(Inst->TheDef->getLoc(), + "No schedule information for instruction '" + + Inst->TheDef->getName() + "' in SchedMachineModel '" + + ProcModel.ModelDef->getName() + "'"); + Complete = false; + } + continue; + } + + const CodeGenSchedClass &SC = getSchedClass(SCIdx); + if (!SC.Writes.empty()) + continue; + if (HasItineraries && SC.ItinClassDef != nullptr && + SC.ItinClassDef->getName() != "NoItinerary") + continue; + + const RecVec &InstRWs = SC.InstRWs; + auto I = find_if(InstRWs, [&ProcModel](const Record *R) { + return R->getValueAsDef("SchedModel") == ProcModel.ModelDef; + }); + if (I == InstRWs.end()) { + PrintError(Inst->TheDef->getLoc(), "'" + ProcModel.ModelName + + "' lacks information for '" + + Inst->TheDef->getName() + "'"); + Complete = false; + } + } + HadCompleteModel = true; + } + if (!Complete) { + errs() << "\n\nIncomplete schedule models found.\n" + << "- Consider setting 'CompleteModel = 0' while developing new models.\n" + << "- Pseudo instructions can be marked with 'hasNoSchedulingInfo = 1'.\n" + << "- Instructions should usually have Sched<[...]> as a superclass, " + "you may temporarily use an empty list.\n" + << "- Instructions related to unsupported features can be excluded with " + "list<Predicate> UnsupportedFeatures = [HasA,..,HasY]; in the " + "processor model.\n\n"; + PrintFatalError("Incomplete schedule model"); + } +} + +// Collect itinerary class resources for each processor. +void CodeGenSchedModels::collectItinProcResources(Record *ItinClassDef) { + for (unsigned PIdx = 0, PEnd = ProcModels.size(); PIdx != PEnd; ++PIdx) { + const CodeGenProcModel &PM = ProcModels[PIdx]; + // For all ItinRW entries. + bool HasMatch = false; + for (RecIter II = PM.ItinRWDefs.begin(), IE = PM.ItinRWDefs.end(); + II != IE; ++II) { + RecVec Matched = (*II)->getValueAsListOfDefs("MatchedItinClasses"); + if (!std::count(Matched.begin(), Matched.end(), ItinClassDef)) + continue; + if (HasMatch) + PrintFatalError((*II)->getLoc(), "Duplicate itinerary class " + + ItinClassDef->getName() + + " in ItinResources for " + PM.ModelName); + HasMatch = true; + IdxVec Writes, Reads; + findRWs((*II)->getValueAsListOfDefs("OperandReadWrites"), Writes, Reads); + collectRWResources(Writes, Reads, PIdx); + } + } +} + +void CodeGenSchedModels::collectRWResources(unsigned RWIdx, bool IsRead, + ArrayRef<unsigned> ProcIndices) { + const CodeGenSchedRW &SchedRW = getSchedRW(RWIdx, IsRead); + if (SchedRW.TheDef) { + if (!IsRead && SchedRW.TheDef->isSubClassOf("SchedWriteRes")) { + for (unsigned Idx : ProcIndices) + addWriteRes(SchedRW.TheDef, Idx); + } + else if (IsRead && SchedRW.TheDef->isSubClassOf("SchedReadAdvance")) { + for (unsigned Idx : ProcIndices) + addReadAdvance(SchedRW.TheDef, Idx); + } + } + for (RecIter AI = SchedRW.Aliases.begin(), AE = SchedRW.Aliases.end(); + AI != AE; ++AI) { + IdxVec AliasProcIndices; + if ((*AI)->getValueInit("SchedModel")->isComplete()) { + AliasProcIndices.push_back( + getProcModel((*AI)->getValueAsDef("SchedModel")).Index); + } + else + AliasProcIndices = ProcIndices; + const CodeGenSchedRW &AliasRW = getSchedRW((*AI)->getValueAsDef("AliasRW")); + assert(AliasRW.IsRead == IsRead && "cannot alias reads to writes"); + + IdxVec ExpandedRWs; + expandRWSequence(AliasRW.Index, ExpandedRWs, IsRead); + for (IdxIter SI = ExpandedRWs.begin(), SE = ExpandedRWs.end(); + SI != SE; ++SI) { + collectRWResources(*SI, IsRead, AliasProcIndices); + } + } +} + +// Collect resources for a set of read/write types and processor indices. +void CodeGenSchedModels::collectRWResources(ArrayRef<unsigned> Writes, + ArrayRef<unsigned> Reads, + ArrayRef<unsigned> ProcIndices) { + for (unsigned Idx : Writes) + collectRWResources(Idx, /*IsRead=*/false, ProcIndices); + + for (unsigned Idx : Reads) + collectRWResources(Idx, /*IsRead=*/true, ProcIndices); +} + +// Find the processor's resource units for this kind of resource. +Record *CodeGenSchedModels::findProcResUnits(Record *ProcResKind, + const CodeGenProcModel &PM, + ArrayRef<SMLoc> Loc) const { + if (ProcResKind->isSubClassOf("ProcResourceUnits")) + return ProcResKind; + + Record *ProcUnitDef = nullptr; + assert(!ProcResourceDefs.empty()); + assert(!ProcResGroups.empty()); + + for (Record *ProcResDef : ProcResourceDefs) { + if (ProcResDef->getValueAsDef("Kind") == ProcResKind + && ProcResDef->getValueAsDef("SchedModel") == PM.ModelDef) { + if (ProcUnitDef) { + PrintFatalError(Loc, + "Multiple ProcessorResourceUnits associated with " + + ProcResKind->getName()); + } + ProcUnitDef = ProcResDef; + } + } + for (Record *ProcResGroup : ProcResGroups) { + if (ProcResGroup == ProcResKind + && ProcResGroup->getValueAsDef("SchedModel") == PM.ModelDef) { + if (ProcUnitDef) { + PrintFatalError(Loc, + "Multiple ProcessorResourceUnits associated with " + + ProcResKind->getName()); + } + ProcUnitDef = ProcResGroup; + } + } + if (!ProcUnitDef) { + PrintFatalError(Loc, + "No ProcessorResources associated with " + + ProcResKind->getName()); + } + return ProcUnitDef; +} + +// Iteratively add a resource and its super resources. +void CodeGenSchedModels::addProcResource(Record *ProcResKind, + CodeGenProcModel &PM, + ArrayRef<SMLoc> Loc) { + while (true) { + Record *ProcResUnits = findProcResUnits(ProcResKind, PM, Loc); + + // See if this ProcResource is already associated with this processor. + if (is_contained(PM.ProcResourceDefs, ProcResUnits)) + return; + + PM.ProcResourceDefs.push_back(ProcResUnits); + if (ProcResUnits->isSubClassOf("ProcResGroup")) + return; + + if (!ProcResUnits->getValueInit("Super")->isComplete()) + return; + + ProcResKind = ProcResUnits->getValueAsDef("Super"); + } +} + +// Add resources for a SchedWrite to this processor if they don't exist. +void CodeGenSchedModels::addWriteRes(Record *ProcWriteResDef, unsigned PIdx) { + assert(PIdx && "don't add resources to an invalid Processor model"); + + RecVec &WRDefs = ProcModels[PIdx].WriteResDefs; + if (is_contained(WRDefs, ProcWriteResDef)) + return; + WRDefs.push_back(ProcWriteResDef); + + // Visit ProcResourceKinds referenced by the newly discovered WriteRes. + RecVec ProcResDefs = ProcWriteResDef->getValueAsListOfDefs("ProcResources"); + for (RecIter WritePRI = ProcResDefs.begin(), WritePRE = ProcResDefs.end(); + WritePRI != WritePRE; ++WritePRI) { + addProcResource(*WritePRI, ProcModels[PIdx], ProcWriteResDef->getLoc()); + } +} + +// Add resources for a ReadAdvance to this processor if they don't exist. +void CodeGenSchedModels::addReadAdvance(Record *ProcReadAdvanceDef, + unsigned PIdx) { + RecVec &RADefs = ProcModels[PIdx].ReadAdvanceDefs; + if (is_contained(RADefs, ProcReadAdvanceDef)) + return; + RADefs.push_back(ProcReadAdvanceDef); +} + +unsigned CodeGenProcModel::getProcResourceIdx(Record *PRDef) const { + RecIter PRPos = find(ProcResourceDefs, PRDef); + if (PRPos == ProcResourceDefs.end()) + PrintFatalError(PRDef->getLoc(), "ProcResource def is not included in " + "the ProcResources list for " + ModelName); + // Idx=0 is reserved for invalid. + return 1 + (PRPos - ProcResourceDefs.begin()); +} + +bool CodeGenProcModel::isUnsupported(const CodeGenInstruction &Inst) const { + for (const Record *TheDef : UnsupportedFeaturesDefs) { + for (const Record *PredDef : Inst.TheDef->getValueAsListOfDefs("Predicates")) { + if (TheDef->getName() == PredDef->getName()) + return true; + } + } + return false; +} + +#ifndef NDEBUG +void CodeGenProcModel::dump() const { + dbgs() << Index << ": " << ModelName << " " + << (ModelDef ? ModelDef->getName() : "inferred") << " " + << (ItinsDef ? ItinsDef->getName() : "no itinerary") << '\n'; +} + +void CodeGenSchedRW::dump() const { + dbgs() << Name << (IsVariadic ? " (V) " : " "); + if (IsSequence) { + dbgs() << "("; + dumpIdxVec(Sequence); + dbgs() << ")"; + } +} + +void CodeGenSchedClass::dump(const CodeGenSchedModels* SchedModels) const { + dbgs() << "SCHEDCLASS " << Index << ":" << Name << '\n' + << " Writes: "; + for (unsigned i = 0, N = Writes.size(); i < N; ++i) { + SchedModels->getSchedWrite(Writes[i]).dump(); + if (i < N-1) { + dbgs() << '\n'; + dbgs().indent(10); + } + } + dbgs() << "\n Reads: "; + for (unsigned i = 0, N = Reads.size(); i < N; ++i) { + SchedModels->getSchedRead(Reads[i]).dump(); + if (i < N-1) { + dbgs() << '\n'; + dbgs().indent(10); + } + } + dbgs() << "\n ProcIdx: "; dumpIdxVec(ProcIndices); dbgs() << '\n'; + if (!Transitions.empty()) { + dbgs() << "\n Transitions for Proc "; + for (const CodeGenSchedTransition &Transition : Transitions) { + dumpIdxVec(Transition.ProcIndices); + } + } +} + +void PredTransitions::dump() const { + dbgs() << "Expanded Variants:\n"; + for (std::vector<PredTransition>::const_iterator + TI = TransVec.begin(), TE = TransVec.end(); TI != TE; ++TI) { + dbgs() << "{"; + for (SmallVectorImpl<PredCheck>::const_iterator + PCI = TI->PredTerm.begin(), PCE = TI->PredTerm.end(); + PCI != PCE; ++PCI) { + if (PCI != TI->PredTerm.begin()) + dbgs() << ", "; + dbgs() << SchedModels.getSchedRW(PCI->RWIdx, PCI->IsRead).Name + << ":" << PCI->Predicate->getName(); + } + dbgs() << "},\n => {"; + for (SmallVectorImpl<SmallVector<unsigned,4>>::const_iterator + WSI = TI->WriteSequences.begin(), WSE = TI->WriteSequences.end(); + WSI != WSE; ++WSI) { + dbgs() << "("; + for (SmallVectorImpl<unsigned>::const_iterator + WI = WSI->begin(), WE = WSI->end(); WI != WE; ++WI) { + if (WI != WSI->begin()) + dbgs() << ", "; + dbgs() << SchedModels.getSchedWrite(*WI).Name; + } + dbgs() << "),"; + } + dbgs() << "}\n"; + } +} +#endif // NDEBUG diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenSchedule.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenSchedule.h new file mode 100644 index 000000000000..c487d142d46c --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenSchedule.h @@ -0,0 +1,646 @@ +//===- CodeGenSchedule.h - Scheduling Machine Models ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines structures to encapsulate the machine model as described in +// the target description. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEGENSCHEDULE_H +#define LLVM_UTILS_TABLEGEN_CODEGENSCHEDULE_H + +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/SetTheory.h" + +namespace llvm { + +class CodeGenTarget; +class CodeGenSchedModels; +class CodeGenInstruction; +class CodeGenRegisterClass; + +using RecVec = std::vector<Record*>; +using RecIter = std::vector<Record*>::const_iterator; + +using IdxVec = std::vector<unsigned>; +using IdxIter = std::vector<unsigned>::const_iterator; + +/// We have two kinds of SchedReadWrites. Explicitly defined and inferred +/// sequences. TheDef is nonnull for explicit SchedWrites, but Sequence may or +/// may not be empty. TheDef is null for inferred sequences, and Sequence must +/// be nonempty. +/// +/// IsVariadic controls whether the variants are expanded into multiple operands +/// or a sequence of writes on one operand. +struct CodeGenSchedRW { + unsigned Index; + std::string Name; + Record *TheDef; + bool IsRead; + bool IsAlias; + bool HasVariants; + bool IsVariadic; + bool IsSequence; + IdxVec Sequence; + RecVec Aliases; + + CodeGenSchedRW() + : Index(0), TheDef(nullptr), IsRead(false), IsAlias(false), + HasVariants(false), IsVariadic(false), IsSequence(false) {} + CodeGenSchedRW(unsigned Idx, Record *Def) + : Index(Idx), TheDef(Def), IsAlias(false), IsVariadic(false) { + Name = std::string(Def->getName()); + IsRead = Def->isSubClassOf("SchedRead"); + HasVariants = Def->isSubClassOf("SchedVariant"); + if (HasVariants) + IsVariadic = Def->getValueAsBit("Variadic"); + + // Read records don't currently have sequences, but it can be easily + // added. Note that implicit Reads (from ReadVariant) may have a Sequence + // (but no record). + IsSequence = Def->isSubClassOf("WriteSequence"); + } + + CodeGenSchedRW(unsigned Idx, bool Read, ArrayRef<unsigned> Seq, + const std::string &Name) + : Index(Idx), Name(Name), TheDef(nullptr), IsRead(Read), IsAlias(false), + HasVariants(false), IsVariadic(false), IsSequence(true), Sequence(Seq) { + assert(Sequence.size() > 1 && "implied sequence needs >1 RWs"); + } + + bool isValid() const { + assert((!HasVariants || TheDef) && "Variant write needs record def"); + assert((!IsVariadic || HasVariants) && "Variadic write needs variants"); + assert((!IsSequence || !HasVariants) && "Sequence can't have variant"); + assert((!IsSequence || !Sequence.empty()) && "Sequence should be nonempty"); + assert((!IsAlias || Aliases.empty()) && "Alias cannot have aliases"); + return TheDef || !Sequence.empty(); + } + +#ifndef NDEBUG + void dump() const; +#endif +}; + +/// Represent a transition between SchedClasses induced by SchedVariant. +struct CodeGenSchedTransition { + unsigned ToClassIdx; + IdxVec ProcIndices; + RecVec PredTerm; +}; + +/// Scheduling class. +/// +/// Each instruction description will be mapped to a scheduling class. There are +/// four types of classes: +/// +/// 1) An explicitly defined itinerary class with ItinClassDef set. +/// Writes and ReadDefs are empty. ProcIndices contains 0 for any processor. +/// +/// 2) An implied class with a list of SchedWrites and SchedReads that are +/// defined in an instruction definition and which are common across all +/// subtargets. ProcIndices contains 0 for any processor. +/// +/// 3) An implied class with a list of InstRW records that map instructions to +/// SchedWrites and SchedReads per-processor. InstrClassMap should map the same +/// instructions to this class. ProcIndices contains all the processors that +/// provided InstrRW records for this class. ItinClassDef or Writes/Reads may +/// still be defined for processors with no InstRW entry. +/// +/// 4) An inferred class represents a variant of another class that may be +/// resolved at runtime. ProcIndices contains the set of processors that may +/// require the class. ProcIndices are propagated through SchedClasses as +/// variants are expanded. Multiple SchedClasses may be inferred from an +/// itinerary class. Each inherits the processor index from the ItinRW record +/// that mapped the itinerary class to the variant Writes or Reads. +struct CodeGenSchedClass { + unsigned Index; + std::string Name; + Record *ItinClassDef; + + IdxVec Writes; + IdxVec Reads; + // Sorted list of ProcIdx, where ProcIdx==0 implies any processor. + IdxVec ProcIndices; + + std::vector<CodeGenSchedTransition> Transitions; + + // InstRW records associated with this class. These records may refer to an + // Instruction no longer mapped to this class by InstrClassMap. These + // Instructions should be ignored by this class because they have been split + // off to join another inferred class. + RecVec InstRWs; + + CodeGenSchedClass(unsigned Index, std::string Name, Record *ItinClassDef) + : Index(Index), Name(std::move(Name)), ItinClassDef(ItinClassDef) {} + + bool isKeyEqual(Record *IC, ArrayRef<unsigned> W, + ArrayRef<unsigned> R) const { + return ItinClassDef == IC && makeArrayRef(Writes) == W && + makeArrayRef(Reads) == R; + } + + // Is this class generated from a variants if existing classes? Instructions + // are never mapped directly to inferred scheduling classes. + bool isInferred() const { return !ItinClassDef; } + +#ifndef NDEBUG + void dump(const CodeGenSchedModels *SchedModels) const; +#endif +}; + +/// Represent the cost of allocating a register of register class RCDef. +/// +/// The cost of allocating a register is equivalent to the number of physical +/// registers used by the register renamer. Register costs are defined at +/// register class granularity. +struct CodeGenRegisterCost { + Record *RCDef; + unsigned Cost; + bool AllowMoveElimination; + CodeGenRegisterCost(Record *RC, unsigned RegisterCost, bool AllowMoveElim = false) + : RCDef(RC), Cost(RegisterCost), AllowMoveElimination(AllowMoveElim) {} + CodeGenRegisterCost(const CodeGenRegisterCost &) = default; + CodeGenRegisterCost &operator=(const CodeGenRegisterCost &) = delete; +}; + +/// A processor register file. +/// +/// This class describes a processor register file. Register file information is +/// currently consumed by external tools like llvm-mca to predict dispatch +/// stalls due to register pressure. +struct CodeGenRegisterFile { + std::string Name; + Record *RegisterFileDef; + unsigned MaxMovesEliminatedPerCycle; + bool AllowZeroMoveEliminationOnly; + + unsigned NumPhysRegs; + std::vector<CodeGenRegisterCost> Costs; + + CodeGenRegisterFile(StringRef name, Record *def, unsigned MaxMoveElimPerCy = 0, + bool AllowZeroMoveElimOnly = false) + : Name(name), RegisterFileDef(def), + MaxMovesEliminatedPerCycle(MaxMoveElimPerCy), + AllowZeroMoveEliminationOnly(AllowZeroMoveElimOnly), + NumPhysRegs(0) {} + + bool hasDefaultCosts() const { return Costs.empty(); } +}; + +// Processor model. +// +// ModelName is a unique name used to name an instantiation of MCSchedModel. +// +// ModelDef is NULL for inferred Models. This happens when a processor defines +// an itinerary but no machine model. If the processor defines neither a machine +// model nor itinerary, then ModelDef remains pointing to NoModel. NoModel has +// the special "NoModel" field set to true. +// +// ItinsDef always points to a valid record definition, but may point to the +// default NoItineraries. NoItineraries has an empty list of InstrItinData +// records. +// +// ItinDefList orders this processor's InstrItinData records by SchedClass idx. +struct CodeGenProcModel { + unsigned Index; + std::string ModelName; + Record *ModelDef; + Record *ItinsDef; + + // Derived members... + + // Array of InstrItinData records indexed by a CodeGenSchedClass index. + // This list is empty if the Processor has no value for Itineraries. + // Initialized by collectProcItins(). + RecVec ItinDefList; + + // Map itinerary classes to per-operand resources. + // This list is empty if no ItinRW refers to this Processor. + RecVec ItinRWDefs; + + // List of unsupported feature. + // This list is empty if the Processor has no UnsupportedFeatures. + RecVec UnsupportedFeaturesDefs; + + // All read/write resources associated with this processor. + RecVec WriteResDefs; + RecVec ReadAdvanceDefs; + + // Per-operand machine model resources associated with this processor. + RecVec ProcResourceDefs; + + // List of Register Files. + std::vector<CodeGenRegisterFile> RegisterFiles; + + // Optional Retire Control Unit definition. + Record *RetireControlUnit; + + // Load/Store queue descriptors. + Record *LoadQueue; + Record *StoreQueue; + + CodeGenProcModel(unsigned Idx, std::string Name, Record *MDef, + Record *IDef) : + Index(Idx), ModelName(std::move(Name)), ModelDef(MDef), ItinsDef(IDef), + RetireControlUnit(nullptr), LoadQueue(nullptr), StoreQueue(nullptr) {} + + bool hasItineraries() const { + return !ItinsDef->getValueAsListOfDefs("IID").empty(); + } + + bool hasInstrSchedModel() const { + return !WriteResDefs.empty() || !ItinRWDefs.empty(); + } + + bool hasExtraProcessorInfo() const { + return RetireControlUnit || LoadQueue || StoreQueue || + !RegisterFiles.empty(); + } + + unsigned getProcResourceIdx(Record *PRDef) const; + + bool isUnsupported(const CodeGenInstruction &Inst) const; + +#ifndef NDEBUG + void dump() const; +#endif +}; + +/// Used to correlate instructions to MCInstPredicates specified by +/// InstructionEquivalentClass tablegen definitions. +/// +/// Example: a XOR of a register with self, is a known zero-idiom for most +/// X86 processors. +/// +/// Each processor can use a (potentially different) InstructionEquivalenceClass +/// definition to classify zero-idioms. That means, XORrr is likely to appear +/// in more than one equivalence class (where each class definition is +/// contributed by a different processor). +/// +/// There is no guarantee that the same MCInstPredicate will be used to describe +/// equivalence classes that identify XORrr as a zero-idiom. +/// +/// To be more specific, the requirements for being a zero-idiom XORrr may be +/// different for different processors. +/// +/// Class PredicateInfo identifies a subset of processors that specify the same +/// requirements (i.e. same MCInstPredicate and OperandMask) for an instruction +/// opcode. +/// +/// Back to the example. Field `ProcModelMask` will have one bit set for every +/// processor model that sees XORrr as a zero-idiom, and that specifies the same +/// set of constraints. +/// +/// By construction, there can be multiple instances of PredicateInfo associated +/// with a same instruction opcode. For example, different processors may define +/// different constraints on the same opcode. +/// +/// Field OperandMask can be used as an extra constraint. +/// It may be used to describe conditions that appy only to a subset of the +/// operands of a machine instruction, and the operands subset may not be the +/// same for all processor models. +struct PredicateInfo { + llvm::APInt ProcModelMask; // A set of processor model indices. + llvm::APInt OperandMask; // An operand mask. + const Record *Predicate; // MCInstrPredicate definition. + PredicateInfo(llvm::APInt CpuMask, llvm::APInt Operands, const Record *Pred) + : ProcModelMask(CpuMask), OperandMask(Operands), Predicate(Pred) {} + + bool operator==(const PredicateInfo &Other) const { + return ProcModelMask == Other.ProcModelMask && + OperandMask == Other.OperandMask && Predicate == Other.Predicate; + } +}; + +/// A collection of PredicateInfo objects. +/// +/// There is at least one OpcodeInfo object for every opcode specified by a +/// TIPredicate definition. +class OpcodeInfo { + std::vector<PredicateInfo> Predicates; + + OpcodeInfo(const OpcodeInfo &Other) = delete; + OpcodeInfo &operator=(const OpcodeInfo &Other) = delete; + +public: + OpcodeInfo() = default; + OpcodeInfo &operator=(OpcodeInfo &&Other) = default; + OpcodeInfo(OpcodeInfo &&Other) = default; + + ArrayRef<PredicateInfo> getPredicates() const { return Predicates; } + + void addPredicateForProcModel(const llvm::APInt &CpuMask, + const llvm::APInt &OperandMask, + const Record *Predicate); +}; + +/// Used to group together tablegen instruction definitions that are subject +/// to a same set of constraints (identified by an instance of OpcodeInfo). +class OpcodeGroup { + OpcodeInfo Info; + std::vector<const Record *> Opcodes; + + OpcodeGroup(const OpcodeGroup &Other) = delete; + OpcodeGroup &operator=(const OpcodeGroup &Other) = delete; + +public: + OpcodeGroup(OpcodeInfo &&OpInfo) : Info(std::move(OpInfo)) {} + OpcodeGroup(OpcodeGroup &&Other) = default; + + void addOpcode(const Record *Opcode) { + assert(std::find(Opcodes.begin(), Opcodes.end(), Opcode) == Opcodes.end() && + "Opcode already in set!"); + Opcodes.push_back(Opcode); + } + + ArrayRef<const Record *> getOpcodes() const { return Opcodes; } + const OpcodeInfo &getOpcodeInfo() const { return Info; } +}; + +/// An STIPredicateFunction descriptor used by tablegen backends to +/// auto-generate the body of a predicate function as a member of tablegen'd +/// class XXXGenSubtargetInfo. +class STIPredicateFunction { + const Record *FunctionDeclaration; + + std::vector<const Record *> Definitions; + std::vector<OpcodeGroup> Groups; + + STIPredicateFunction(const STIPredicateFunction &Other) = delete; + STIPredicateFunction &operator=(const STIPredicateFunction &Other) = delete; + +public: + STIPredicateFunction(const Record *Rec) : FunctionDeclaration(Rec) {} + STIPredicateFunction(STIPredicateFunction &&Other) = default; + + bool isCompatibleWith(const STIPredicateFunction &Other) const { + return FunctionDeclaration == Other.FunctionDeclaration; + } + + void addDefinition(const Record *Def) { Definitions.push_back(Def); } + void addOpcode(const Record *OpcodeRec, OpcodeInfo &&Info) { + if (Groups.empty() || + Groups.back().getOpcodeInfo().getPredicates() != Info.getPredicates()) + Groups.emplace_back(std::move(Info)); + Groups.back().addOpcode(OpcodeRec); + } + + StringRef getName() const { + return FunctionDeclaration->getValueAsString("Name"); + } + const Record *getDefaultReturnPredicate() const { + return FunctionDeclaration->getValueAsDef("DefaultReturnValue"); + } + + const Record *getDeclaration() const { return FunctionDeclaration; } + ArrayRef<const Record *> getDefinitions() const { return Definitions; } + ArrayRef<OpcodeGroup> getGroups() const { return Groups; } +}; + +/// Top level container for machine model data. +class CodeGenSchedModels { + RecordKeeper &Records; + const CodeGenTarget &Target; + + // Map dag expressions to Instruction lists. + SetTheory Sets; + + // List of unique processor models. + std::vector<CodeGenProcModel> ProcModels; + + // Map Processor's MachineModel or ProcItin to a CodeGenProcModel index. + using ProcModelMapTy = DenseMap<Record*, unsigned>; + ProcModelMapTy ProcModelMap; + + // Per-operand SchedReadWrite types. + std::vector<CodeGenSchedRW> SchedWrites; + std::vector<CodeGenSchedRW> SchedReads; + + // List of unique SchedClasses. + std::vector<CodeGenSchedClass> SchedClasses; + + // Any inferred SchedClass has an index greater than NumInstrSchedClassses. + unsigned NumInstrSchedClasses; + + RecVec ProcResourceDefs; + RecVec ProcResGroups; + + // Map each instruction to its unique SchedClass index considering the + // combination of it's itinerary class, SchedRW list, and InstRW records. + using InstClassMapTy = DenseMap<Record*, unsigned>; + InstClassMapTy InstrClassMap; + + std::vector<STIPredicateFunction> STIPredicates; + +public: + CodeGenSchedModels(RecordKeeper& RK, const CodeGenTarget &TGT); + + // iterator access to the scheduling classes. + using class_iterator = std::vector<CodeGenSchedClass>::iterator; + using const_class_iterator = std::vector<CodeGenSchedClass>::const_iterator; + class_iterator classes_begin() { return SchedClasses.begin(); } + const_class_iterator classes_begin() const { return SchedClasses.begin(); } + class_iterator classes_end() { return SchedClasses.end(); } + const_class_iterator classes_end() const { return SchedClasses.end(); } + iterator_range<class_iterator> classes() { + return make_range(classes_begin(), classes_end()); + } + iterator_range<const_class_iterator> classes() const { + return make_range(classes_begin(), classes_end()); + } + iterator_range<class_iterator> explicit_classes() { + return make_range(classes_begin(), classes_begin() + NumInstrSchedClasses); + } + iterator_range<const_class_iterator> explicit_classes() const { + return make_range(classes_begin(), classes_begin() + NumInstrSchedClasses); + } + + Record *getModelOrItinDef(Record *ProcDef) const { + Record *ModelDef = ProcDef->getValueAsDef("SchedModel"); + Record *ItinsDef = ProcDef->getValueAsDef("ProcItin"); + if (!ItinsDef->getValueAsListOfDefs("IID").empty()) { + assert(ModelDef->getValueAsBit("NoModel") + && "Itineraries must be defined within SchedMachineModel"); + return ItinsDef; + } + return ModelDef; + } + + const CodeGenProcModel &getModelForProc(Record *ProcDef) const { + Record *ModelDef = getModelOrItinDef(ProcDef); + ProcModelMapTy::const_iterator I = ProcModelMap.find(ModelDef); + assert(I != ProcModelMap.end() && "missing machine model"); + return ProcModels[I->second]; + } + + CodeGenProcModel &getProcModel(Record *ModelDef) { + ProcModelMapTy::const_iterator I = ProcModelMap.find(ModelDef); + assert(I != ProcModelMap.end() && "missing machine model"); + return ProcModels[I->second]; + } + const CodeGenProcModel &getProcModel(Record *ModelDef) const { + return const_cast<CodeGenSchedModels*>(this)->getProcModel(ModelDef); + } + + // Iterate over the unique processor models. + using ProcIter = std::vector<CodeGenProcModel>::const_iterator; + ProcIter procModelBegin() const { return ProcModels.begin(); } + ProcIter procModelEnd() const { return ProcModels.end(); } + ArrayRef<CodeGenProcModel> procModels() const { return ProcModels; } + + // Return true if any processors have itineraries. + bool hasItineraries() const; + + // Get a SchedWrite from its index. + const CodeGenSchedRW &getSchedWrite(unsigned Idx) const { + assert(Idx < SchedWrites.size() && "bad SchedWrite index"); + assert(SchedWrites[Idx].isValid() && "invalid SchedWrite"); + return SchedWrites[Idx]; + } + // Get a SchedWrite from its index. + const CodeGenSchedRW &getSchedRead(unsigned Idx) const { + assert(Idx < SchedReads.size() && "bad SchedRead index"); + assert(SchedReads[Idx].isValid() && "invalid SchedRead"); + return SchedReads[Idx]; + } + + const CodeGenSchedRW &getSchedRW(unsigned Idx, bool IsRead) const { + return IsRead ? getSchedRead(Idx) : getSchedWrite(Idx); + } + CodeGenSchedRW &getSchedRW(Record *Def) { + bool IsRead = Def->isSubClassOf("SchedRead"); + unsigned Idx = getSchedRWIdx(Def, IsRead); + return const_cast<CodeGenSchedRW&>( + IsRead ? getSchedRead(Idx) : getSchedWrite(Idx)); + } + const CodeGenSchedRW &getSchedRW(Record *Def) const { + return const_cast<CodeGenSchedModels&>(*this).getSchedRW(Def); + } + + unsigned getSchedRWIdx(const Record *Def, bool IsRead) const; + + // Return true if the given write record is referenced by a ReadAdvance. + bool hasReadOfWrite(Record *WriteDef) const; + + // Get a SchedClass from its index. + CodeGenSchedClass &getSchedClass(unsigned Idx) { + assert(Idx < SchedClasses.size() && "bad SchedClass index"); + return SchedClasses[Idx]; + } + const CodeGenSchedClass &getSchedClass(unsigned Idx) const { + assert(Idx < SchedClasses.size() && "bad SchedClass index"); + return SchedClasses[Idx]; + } + + // Get the SchedClass index for an instruction. Instructions with no + // itinerary, no SchedReadWrites, and no InstrReadWrites references return 0 + // for NoItinerary. + unsigned getSchedClassIdx(const CodeGenInstruction &Inst) const; + + using SchedClassIter = std::vector<CodeGenSchedClass>::const_iterator; + SchedClassIter schedClassBegin() const { return SchedClasses.begin(); } + SchedClassIter schedClassEnd() const { return SchedClasses.end(); } + ArrayRef<CodeGenSchedClass> schedClasses() const { return SchedClasses; } + + unsigned numInstrSchedClasses() const { return NumInstrSchedClasses; } + + void findRWs(const RecVec &RWDefs, IdxVec &Writes, IdxVec &Reads) const; + void findRWs(const RecVec &RWDefs, IdxVec &RWs, bool IsRead) const; + void expandRWSequence(unsigned RWIdx, IdxVec &RWSeq, bool IsRead) const; + void expandRWSeqForProc(unsigned RWIdx, IdxVec &RWSeq, bool IsRead, + const CodeGenProcModel &ProcModel) const; + + unsigned addSchedClass(Record *ItinDef, ArrayRef<unsigned> OperWrites, + ArrayRef<unsigned> OperReads, + ArrayRef<unsigned> ProcIndices); + + unsigned findOrInsertRW(ArrayRef<unsigned> Seq, bool IsRead); + + Record *findProcResUnits(Record *ProcResKind, const CodeGenProcModel &PM, + ArrayRef<SMLoc> Loc) const; + + ArrayRef<STIPredicateFunction> getSTIPredicates() const { + return STIPredicates; + } +private: + void collectProcModels(); + + // Initialize a new processor model if it is unique. + void addProcModel(Record *ProcDef); + + void collectSchedRW(); + + std::string genRWName(ArrayRef<unsigned> Seq, bool IsRead); + unsigned findRWForSequence(ArrayRef<unsigned> Seq, bool IsRead); + + void collectSchedClasses(); + + void collectRetireControlUnits(); + + void collectRegisterFiles(); + + void collectOptionalProcessorInfo(); + + std::string createSchedClassName(Record *ItinClassDef, + ArrayRef<unsigned> OperWrites, + ArrayRef<unsigned> OperReads); + std::string createSchedClassName(const RecVec &InstDefs); + void createInstRWClass(Record *InstRWDef); + + void collectProcItins(); + + void collectProcItinRW(); + + void collectProcUnsupportedFeatures(); + + void inferSchedClasses(); + + void checkMCInstPredicates() const; + + void checkSTIPredicates() const; + + void collectSTIPredicates(); + + void collectLoadStoreQueueInfo(); + + void checkCompleteness(); + + void inferFromRW(ArrayRef<unsigned> OperWrites, ArrayRef<unsigned> OperReads, + unsigned FromClassIdx, ArrayRef<unsigned> ProcIndices); + void inferFromItinClass(Record *ItinClassDef, unsigned FromClassIdx); + void inferFromInstRWs(unsigned SCIdx); + + bool hasSuperGroup(RecVec &SubUnits, CodeGenProcModel &PM); + void verifyProcResourceGroups(CodeGenProcModel &PM); + + void collectProcResources(); + + void collectItinProcResources(Record *ItinClassDef); + + void collectRWResources(unsigned RWIdx, bool IsRead, + ArrayRef<unsigned> ProcIndices); + + void collectRWResources(ArrayRef<unsigned> Writes, ArrayRef<unsigned> Reads, + ArrayRef<unsigned> ProcIndices); + + void addProcResource(Record *ProcResourceKind, CodeGenProcModel &PM, + ArrayRef<SMLoc> Loc); + + void addWriteRes(Record *ProcWriteResDef, unsigned PIdx); + + void addReadAdvance(Record *ProcReadAdvanceDef, unsigned PIdx); +}; + +} // namespace llvm + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.cpp b/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.cpp new file mode 100644 index 000000000000..891a08ea590e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.cpp @@ -0,0 +1,859 @@ +//===- CodeGenTarget.cpp - CodeGen Target Class Wrapper -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This class wraps target description classes used by the various code +// generation TableGen backends. This makes it easier to access the data and +// provides a single place that needs to check it for validity. All of these +// classes abort on error conditions. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "CodeGenDAGPatterns.h" +#include "CodeGenIntrinsics.h" +#include "CodeGenSchedule.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Timer.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <algorithm> +using namespace llvm; + +cl::OptionCategory AsmParserCat("Options for -gen-asm-parser"); +cl::OptionCategory AsmWriterCat("Options for -gen-asm-writer"); + +static cl::opt<unsigned> + AsmParserNum("asmparsernum", cl::init(0), + cl::desc("Make -gen-asm-parser emit assembly parser #N"), + cl::cat(AsmParserCat)); + +static cl::opt<unsigned> + AsmWriterNum("asmwriternum", cl::init(0), + cl::desc("Make -gen-asm-writer emit assembly writer #N"), + cl::cat(AsmWriterCat)); + +/// getValueType - Return the MVT::SimpleValueType that the specified TableGen +/// record corresponds to. +MVT::SimpleValueType llvm::getValueType(Record *Rec) { + return (MVT::SimpleValueType)Rec->getValueAsInt("Value"); +} + +StringRef llvm::getName(MVT::SimpleValueType T) { + switch (T) { + case MVT::Other: return "UNKNOWN"; + case MVT::iPTR: return "TLI.getPointerTy()"; + case MVT::iPTRAny: return "TLI.getPointerTy()"; + default: return getEnumName(T); + } +} + +StringRef llvm::getEnumName(MVT::SimpleValueType T) { + switch (T) { + case MVT::Other: return "MVT::Other"; + case MVT::i1: return "MVT::i1"; + case MVT::i8: return "MVT::i8"; + case MVT::i16: return "MVT::i16"; + case MVT::i32: return "MVT::i32"; + case MVT::i64: return "MVT::i64"; + case MVT::i128: return "MVT::i128"; + case MVT::Any: return "MVT::Any"; + case MVT::iAny: return "MVT::iAny"; + case MVT::fAny: return "MVT::fAny"; + case MVT::vAny: return "MVT::vAny"; + case MVT::f16: return "MVT::f16"; + case MVT::bf16: return "MVT::bf16"; + case MVT::f32: return "MVT::f32"; + case MVT::f64: return "MVT::f64"; + case MVT::f80: return "MVT::f80"; + case MVT::f128: return "MVT::f128"; + case MVT::ppcf128: return "MVT::ppcf128"; + case MVT::x86mmx: return "MVT::x86mmx"; + case MVT::Glue: return "MVT::Glue"; + case MVT::isVoid: return "MVT::isVoid"; + case MVT::v1i1: return "MVT::v1i1"; + case MVT::v2i1: return "MVT::v2i1"; + case MVT::v4i1: return "MVT::v4i1"; + case MVT::v8i1: return "MVT::v8i1"; + case MVT::v16i1: return "MVT::v16i1"; + case MVT::v32i1: return "MVT::v32i1"; + case MVT::v64i1: return "MVT::v64i1"; + case MVT::v128i1: return "MVT::v128i1"; + case MVT::v512i1: return "MVT::v512i1"; + case MVT::v1024i1: return "MVT::v1024i1"; + case MVT::v1i8: return "MVT::v1i8"; + case MVT::v2i8: return "MVT::v2i8"; + case MVT::v4i8: return "MVT::v4i8"; + case MVT::v8i8: return "MVT::v8i8"; + case MVT::v16i8: return "MVT::v16i8"; + case MVT::v32i8: return "MVT::v32i8"; + case MVT::v64i8: return "MVT::v64i8"; + case MVT::v128i8: return "MVT::v128i8"; + case MVT::v256i8: return "MVT::v256i8"; + case MVT::v1i16: return "MVT::v1i16"; + case MVT::v2i16: return "MVT::v2i16"; + case MVT::v3i16: return "MVT::v3i16"; + case MVT::v4i16: return "MVT::v4i16"; + case MVT::v8i16: return "MVT::v8i16"; + case MVT::v16i16: return "MVT::v16i16"; + case MVT::v32i16: return "MVT::v32i16"; + case MVT::v64i16: return "MVT::v64i16"; + case MVT::v128i16: return "MVT::v128i16"; + case MVT::v1i32: return "MVT::v1i32"; + case MVT::v2i32: return "MVT::v2i32"; + case MVT::v3i32: return "MVT::v3i32"; + case MVT::v4i32: return "MVT::v4i32"; + case MVT::v5i32: return "MVT::v5i32"; + case MVT::v8i32: return "MVT::v8i32"; + case MVT::v16i32: return "MVT::v16i32"; + case MVT::v32i32: return "MVT::v32i32"; + case MVT::v64i32: return "MVT::v64i32"; + case MVT::v128i32: return "MVT::v128i32"; + case MVT::v256i32: return "MVT::v256i32"; + case MVT::v512i32: return "MVT::v512i32"; + case MVT::v1024i32: return "MVT::v1024i32"; + case MVT::v2048i32: return "MVT::v2048i32"; + case MVT::v1i64: return "MVT::v1i64"; + case MVT::v2i64: return "MVT::v2i64"; + case MVT::v4i64: return "MVT::v4i64"; + case MVT::v8i64: return "MVT::v8i64"; + case MVT::v16i64: return "MVT::v16i64"; + case MVT::v32i64: return "MVT::v32i64"; + case MVT::v1i128: return "MVT::v1i128"; + case MVT::v2f16: return "MVT::v2f16"; + case MVT::v3f16: return "MVT::v3f16"; + case MVT::v4f16: return "MVT::v4f16"; + case MVT::v8f16: return "MVT::v8f16"; + case MVT::v16f16: return "MVT::v16f16"; + case MVT::v32f16: return "MVT::v32f16"; + case MVT::v64f16: return "MVT::v64f16"; + case MVT::v128f16: return "MVT::v128f16"; + case MVT::v2bf16: return "MVT::v2bf16"; + case MVT::v3bf16: return "MVT::v3bf16"; + case MVT::v4bf16: return "MVT::v4bf16"; + case MVT::v8bf16: return "MVT::v8bf16"; + case MVT::v16bf16: return "MVT::v16bf16"; + case MVT::v32bf16: return "MVT::v32bf16"; + case MVT::v64bf16: return "MVT::v64bf16"; + case MVT::v128bf16: return "MVT::v128bf16"; + case MVT::v1f32: return "MVT::v1f32"; + case MVT::v2f32: return "MVT::v2f32"; + case MVT::v3f32: return "MVT::v3f32"; + case MVT::v4f32: return "MVT::v4f32"; + case MVT::v5f32: return "MVT::v5f32"; + case MVT::v8f32: return "MVT::v8f32"; + case MVT::v16f32: return "MVT::v16f32"; + case MVT::v32f32: return "MVT::v32f32"; + case MVT::v64f32: return "MVT::v64f32"; + case MVT::v128f32: return "MVT::v128f32"; + case MVT::v256f32: return "MVT::v256f32"; + case MVT::v512f32: return "MVT::v512f32"; + case MVT::v1024f32: return "MVT::v1024f32"; + case MVT::v2048f32: return "MVT::v2048f32"; + case MVT::v1f64: return "MVT::v1f64"; + case MVT::v2f64: return "MVT::v2f64"; + case MVT::v4f64: return "MVT::v4f64"; + case MVT::v8f64: return "MVT::v8f64"; + case MVT::v16f64: return "MVT::v16f64"; + case MVT::v32f64: return "MVT::v32f64"; + case MVT::nxv1i1: return "MVT::nxv1i1"; + case MVT::nxv2i1: return "MVT::nxv2i1"; + case MVT::nxv4i1: return "MVT::nxv4i1"; + case MVT::nxv8i1: return "MVT::nxv8i1"; + case MVT::nxv16i1: return "MVT::nxv16i1"; + case MVT::nxv32i1: return "MVT::nxv32i1"; + case MVT::nxv64i1: return "MVT::nxv64i1"; + case MVT::nxv1i8: return "MVT::nxv1i8"; + case MVT::nxv2i8: return "MVT::nxv2i8"; + case MVT::nxv4i8: return "MVT::nxv4i8"; + case MVT::nxv8i8: return "MVT::nxv8i8"; + case MVT::nxv16i8: return "MVT::nxv16i8"; + case MVT::nxv32i8: return "MVT::nxv32i8"; + case MVT::nxv64i8: return "MVT::nxv64i8"; + case MVT::nxv1i16: return "MVT::nxv1i16"; + case MVT::nxv2i16: return "MVT::nxv2i16"; + case MVT::nxv4i16: return "MVT::nxv4i16"; + case MVT::nxv8i16: return "MVT::nxv8i16"; + case MVT::nxv16i16: return "MVT::nxv16i16"; + case MVT::nxv32i16: return "MVT::nxv32i16"; + case MVT::nxv1i32: return "MVT::nxv1i32"; + case MVT::nxv2i32: return "MVT::nxv2i32"; + case MVT::nxv4i32: return "MVT::nxv4i32"; + case MVT::nxv8i32: return "MVT::nxv8i32"; + case MVT::nxv16i32: return "MVT::nxv16i32"; + case MVT::nxv32i32: return "MVT::nxv32i32"; + case MVT::nxv1i64: return "MVT::nxv1i64"; + case MVT::nxv2i64: return "MVT::nxv2i64"; + case MVT::nxv4i64: return "MVT::nxv4i64"; + case MVT::nxv8i64: return "MVT::nxv8i64"; + case MVT::nxv16i64: return "MVT::nxv16i64"; + case MVT::nxv32i64: return "MVT::nxv32i64"; + case MVT::nxv1f16: return "MVT::nxv1f16"; + case MVT::nxv2f16: return "MVT::nxv2f16"; + case MVT::nxv4f16: return "MVT::nxv4f16"; + case MVT::nxv8f16: return "MVT::nxv8f16"; + case MVT::nxv16f16: return "MVT::nxv16f16"; + case MVT::nxv32f16: return "MVT::nxv32f16"; + case MVT::nxv2bf16: return "MVT::nxv2bf16"; + case MVT::nxv4bf16: return "MVT::nxv4bf16"; + case MVT::nxv8bf16: return "MVT::nxv8bf16"; + case MVT::nxv1f32: return "MVT::nxv1f32"; + case MVT::nxv2f32: return "MVT::nxv2f32"; + case MVT::nxv4f32: return "MVT::nxv4f32"; + case MVT::nxv8f32: return "MVT::nxv8f32"; + case MVT::nxv16f32: return "MVT::nxv16f32"; + case MVT::nxv1f64: return "MVT::nxv1f64"; + case MVT::nxv2f64: return "MVT::nxv2f64"; + case MVT::nxv4f64: return "MVT::nxv4f64"; + case MVT::nxv8f64: return "MVT::nxv8f64"; + case MVT::token: return "MVT::token"; + case MVT::Metadata: return "MVT::Metadata"; + case MVT::iPTR: return "MVT::iPTR"; + case MVT::iPTRAny: return "MVT::iPTRAny"; + case MVT::Untyped: return "MVT::Untyped"; + case MVT::exnref: return "MVT::exnref"; + default: llvm_unreachable("ILLEGAL VALUE TYPE!"); + } +} + +/// getQualifiedName - Return the name of the specified record, with a +/// namespace qualifier if the record contains one. +/// +std::string llvm::getQualifiedName(const Record *R) { + std::string Namespace; + if (R->getValue("Namespace")) + Namespace = std::string(R->getValueAsString("Namespace")); + if (Namespace.empty()) + return std::string(R->getName()); + return Namespace + "::" + R->getName().str(); +} + + +/// getTarget - Return the current instance of the Target class. +/// +CodeGenTarget::CodeGenTarget(RecordKeeper &records) + : Records(records), CGH(records) { + std::vector<Record*> Targets = Records.getAllDerivedDefinitions("Target"); + if (Targets.size() == 0) + PrintFatalError("ERROR: No 'Target' subclasses defined!"); + if (Targets.size() != 1) + PrintFatalError("ERROR: Multiple subclasses of Target defined!"); + TargetRec = Targets[0]; +} + +CodeGenTarget::~CodeGenTarget() { +} + +const StringRef CodeGenTarget::getName() const { + return TargetRec->getName(); +} + +StringRef CodeGenTarget::getInstNamespace() const { + for (const CodeGenInstruction *Inst : getInstructionsByEnumValue()) { + // Make sure not to pick up "TargetOpcode" by accidentally getting + // the namespace off the PHI instruction or something. + if (Inst->Namespace != "TargetOpcode") + return Inst->Namespace; + } + + return ""; +} + +Record *CodeGenTarget::getInstructionSet() const { + return TargetRec->getValueAsDef("InstructionSet"); +} + +bool CodeGenTarget::getAllowRegisterRenaming() const { + return TargetRec->getValueAsInt("AllowRegisterRenaming"); +} + +/// getAsmParser - Return the AssemblyParser definition for this target. +/// +Record *CodeGenTarget::getAsmParser() const { + std::vector<Record*> LI = TargetRec->getValueAsListOfDefs("AssemblyParsers"); + if (AsmParserNum >= LI.size()) + PrintFatalError("Target does not have an AsmParser #" + + Twine(AsmParserNum) + "!"); + return LI[AsmParserNum]; +} + +/// getAsmParserVariant - Return the AssemblyParserVariant definition for +/// this target. +/// +Record *CodeGenTarget::getAsmParserVariant(unsigned i) const { + std::vector<Record*> LI = + TargetRec->getValueAsListOfDefs("AssemblyParserVariants"); + if (i >= LI.size()) + PrintFatalError("Target does not have an AsmParserVariant #" + Twine(i) + + "!"); + return LI[i]; +} + +/// getAsmParserVariantCount - Return the AssemblyParserVariant definition +/// available for this target. +/// +unsigned CodeGenTarget::getAsmParserVariantCount() const { + std::vector<Record*> LI = + TargetRec->getValueAsListOfDefs("AssemblyParserVariants"); + return LI.size(); +} + +/// getAsmWriter - Return the AssemblyWriter definition for this target. +/// +Record *CodeGenTarget::getAsmWriter() const { + std::vector<Record*> LI = TargetRec->getValueAsListOfDefs("AssemblyWriters"); + if (AsmWriterNum >= LI.size()) + PrintFatalError("Target does not have an AsmWriter #" + + Twine(AsmWriterNum) + "!"); + return LI[AsmWriterNum]; +} + +CodeGenRegBank &CodeGenTarget::getRegBank() const { + if (!RegBank) + RegBank = std::make_unique<CodeGenRegBank>(Records, getHwModes()); + return *RegBank; +} + +Optional<CodeGenRegisterClass *> +CodeGenTarget::getSuperRegForSubReg(const ValueTypeByHwMode &ValueTy, + CodeGenRegBank &RegBank, + const CodeGenSubRegIndex *SubIdx) const { + std::vector<CodeGenRegisterClass *> Candidates; + auto &RegClasses = RegBank.getRegClasses(); + + // Try to find a register class which supports ValueTy, and also contains + // SubIdx. + for (CodeGenRegisterClass &RC : RegClasses) { + // Is there a subclass of this class which contains this subregister index? + CodeGenRegisterClass *SubClassWithSubReg = RC.getSubClassWithSubReg(SubIdx); + if (!SubClassWithSubReg) + continue; + + // We have a class. Check if it supports this value type. + if (llvm::none_of(SubClassWithSubReg->VTs, + [&ValueTy](const ValueTypeByHwMode &ClassVT) { + return ClassVT == ValueTy; + })) + continue; + + // We have a register class which supports both the value type and + // subregister index. Remember it. + Candidates.push_back(SubClassWithSubReg); + } + + // If we didn't find anything, we're done. + if (Candidates.empty()) + return None; + + // Find and return the largest of our candidate classes. + llvm::stable_sort(Candidates, [&](const CodeGenRegisterClass *A, + const CodeGenRegisterClass *B) { + if (A->getMembers().size() > B->getMembers().size()) + return true; + + if (A->getMembers().size() < B->getMembers().size()) + return false; + + // Order by name as a tie-breaker. + return StringRef(A->getName()) < B->getName(); + }); + + return Candidates[0]; +} + +void CodeGenTarget::ReadRegAltNameIndices() const { + RegAltNameIndices = Records.getAllDerivedDefinitions("RegAltNameIndex"); + llvm::sort(RegAltNameIndices, LessRecord()); +} + +/// getRegisterByName - If there is a register with the specific AsmName, +/// return it. +const CodeGenRegister *CodeGenTarget::getRegisterByName(StringRef Name) const { + const StringMap<CodeGenRegister*> &Regs = getRegBank().getRegistersByName(); + StringMap<CodeGenRegister*>::const_iterator I = Regs.find(Name); + if (I == Regs.end()) + return nullptr; + return I->second; +} + +std::vector<ValueTypeByHwMode> CodeGenTarget::getRegisterVTs(Record *R) + const { + const CodeGenRegister *Reg = getRegBank().getReg(R); + std::vector<ValueTypeByHwMode> Result; + for (const auto &RC : getRegBank().getRegClasses()) { + if (RC.contains(Reg)) { + ArrayRef<ValueTypeByHwMode> InVTs = RC.getValueTypes(); + Result.insert(Result.end(), InVTs.begin(), InVTs.end()); + } + } + + // Remove duplicates. + llvm::sort(Result); + Result.erase(std::unique(Result.begin(), Result.end()), Result.end()); + return Result; +} + + +void CodeGenTarget::ReadLegalValueTypes() const { + for (const auto &RC : getRegBank().getRegClasses()) + LegalValueTypes.insert(LegalValueTypes.end(), RC.VTs.begin(), RC.VTs.end()); + + // Remove duplicates. + llvm::sort(LegalValueTypes); + LegalValueTypes.erase(std::unique(LegalValueTypes.begin(), + LegalValueTypes.end()), + LegalValueTypes.end()); +} + +CodeGenSchedModels &CodeGenTarget::getSchedModels() const { + if (!SchedModels) + SchedModels = std::make_unique<CodeGenSchedModels>(Records, *this); + return *SchedModels; +} + +void CodeGenTarget::ReadInstructions() const { + NamedRegionTimer T("Read Instructions", "Time spent reading instructions", + "CodeGenTarget", "CodeGenTarget", TimeRegions); + std::vector<Record*> Insts = Records.getAllDerivedDefinitions("Instruction"); + if (Insts.size() <= 2) + PrintFatalError("No 'Instruction' subclasses defined!"); + + // Parse the instructions defined in the .td file. + for (unsigned i = 0, e = Insts.size(); i != e; ++i) + Instructions[Insts[i]] = std::make_unique<CodeGenInstruction>(Insts[i]); +} + +static const CodeGenInstruction * +GetInstByName(const char *Name, + const DenseMap<const Record*, + std::unique_ptr<CodeGenInstruction>> &Insts, + RecordKeeper &Records) { + const Record *Rec = Records.getDef(Name); + + const auto I = Insts.find(Rec); + if (!Rec || I == Insts.end()) + PrintFatalError(Twine("Could not find '") + Name + "' instruction!"); + return I->second.get(); +} + +static const char *const FixedInstrs[] = { +#define HANDLE_TARGET_OPCODE(OPC) #OPC, +#include "llvm/Support/TargetOpcodes.def" + nullptr}; + +unsigned CodeGenTarget::getNumFixedInstructions() { + return array_lengthof(FixedInstrs) - 1; +} + +/// Return all of the instructions defined by the target, ordered by +/// their enum value. +void CodeGenTarget::ComputeInstrsByEnum() const { + const auto &Insts = getInstructions(); + for (const char *const *p = FixedInstrs; *p; ++p) { + const CodeGenInstruction *Instr = GetInstByName(*p, Insts, Records); + assert(Instr && "Missing target independent instruction"); + assert(Instr->Namespace == "TargetOpcode" && "Bad namespace"); + InstrsByEnum.push_back(Instr); + } + unsigned EndOfPredefines = InstrsByEnum.size(); + assert(EndOfPredefines == getNumFixedInstructions() && + "Missing generic opcode"); + + for (const auto &I : Insts) { + const CodeGenInstruction *CGI = I.second.get(); + if (CGI->Namespace != "TargetOpcode") { + InstrsByEnum.push_back(CGI); + if (CGI->TheDef->getValueAsBit("isPseudo")) + ++NumPseudoInstructions; + } + } + + assert(InstrsByEnum.size() == Insts.size() && "Missing predefined instr"); + + // All of the instructions are now in random order based on the map iteration. + llvm::sort( + InstrsByEnum.begin() + EndOfPredefines, InstrsByEnum.end(), + [](const CodeGenInstruction *Rec1, const CodeGenInstruction *Rec2) { + const auto &D1 = *Rec1->TheDef; + const auto &D2 = *Rec2->TheDef; + return std::make_tuple(!D1.getValueAsBit("isPseudo"), D1.getName()) < + std::make_tuple(!D2.getValueAsBit("isPseudo"), D2.getName()); + }); +} + + +/// isLittleEndianEncoding - Return whether this target encodes its instruction +/// in little-endian format, i.e. bits laid out in the order [0..n] +/// +bool CodeGenTarget::isLittleEndianEncoding() const { + return getInstructionSet()->getValueAsBit("isLittleEndianEncoding"); +} + +/// reverseBitsForLittleEndianEncoding - For little-endian instruction bit +/// encodings, reverse the bit order of all instructions. +void CodeGenTarget::reverseBitsForLittleEndianEncoding() { + if (!isLittleEndianEncoding()) + return; + + std::vector<Record *> Insts = + Records.getAllDerivedDefinitions("InstructionEncoding"); + for (Record *R : Insts) { + if (R->getValueAsString("Namespace") == "TargetOpcode" || + R->getValueAsBit("isPseudo")) + continue; + + BitsInit *BI = R->getValueAsBitsInit("Inst"); + + unsigned numBits = BI->getNumBits(); + + SmallVector<Init *, 16> NewBits(numBits); + + for (unsigned bit = 0, end = numBits / 2; bit != end; ++bit) { + unsigned bitSwapIdx = numBits - bit - 1; + Init *OrigBit = BI->getBit(bit); + Init *BitSwap = BI->getBit(bitSwapIdx); + NewBits[bit] = BitSwap; + NewBits[bitSwapIdx] = OrigBit; + } + if (numBits % 2) { + unsigned middle = (numBits + 1) / 2; + NewBits[middle] = BI->getBit(middle); + } + + BitsInit *NewBI = BitsInit::get(NewBits); + + // Update the bits in reversed order so that emitInstrOpBits will get the + // correct endianness. + R->getValue("Inst")->setValue(NewBI); + } +} + +/// guessInstructionProperties - Return true if it's OK to guess instruction +/// properties instead of raising an error. +/// +/// This is configurable as a temporary migration aid. It will eventually be +/// permanently false. +bool CodeGenTarget::guessInstructionProperties() const { + return getInstructionSet()->getValueAsBit("guessInstructionProperties"); +} + +//===----------------------------------------------------------------------===// +// ComplexPattern implementation +// +ComplexPattern::ComplexPattern(Record *R) { + Ty = ::getValueType(R->getValueAsDef("Ty")); + NumOperands = R->getValueAsInt("NumOperands"); + SelectFunc = std::string(R->getValueAsString("SelectFunc")); + RootNodes = R->getValueAsListOfDefs("RootNodes"); + + // FIXME: This is a hack to statically increase the priority of patterns which + // maps a sub-dag to a complex pattern. e.g. favors LEA over ADD. To get best + // possible pattern match we'll need to dynamically calculate the complexity + // of all patterns a dag can potentially map to. + int64_t RawComplexity = R->getValueAsInt("Complexity"); + if (RawComplexity == -1) + Complexity = NumOperands * 3; + else + Complexity = RawComplexity; + + // FIXME: Why is this different from parseSDPatternOperatorProperties? + // Parse the properties. + Properties = 0; + std::vector<Record*> PropList = R->getValueAsListOfDefs("Properties"); + for (unsigned i = 0, e = PropList.size(); i != e; ++i) + if (PropList[i]->getName() == "SDNPHasChain") { + Properties |= 1 << SDNPHasChain; + } else if (PropList[i]->getName() == "SDNPOptInGlue") { + Properties |= 1 << SDNPOptInGlue; + } else if (PropList[i]->getName() == "SDNPMayStore") { + Properties |= 1 << SDNPMayStore; + } else if (PropList[i]->getName() == "SDNPMayLoad") { + Properties |= 1 << SDNPMayLoad; + } else if (PropList[i]->getName() == "SDNPSideEffect") { + Properties |= 1 << SDNPSideEffect; + } else if (PropList[i]->getName() == "SDNPMemOperand") { + Properties |= 1 << SDNPMemOperand; + } else if (PropList[i]->getName() == "SDNPVariadic") { + Properties |= 1 << SDNPVariadic; + } else if (PropList[i]->getName() == "SDNPWantRoot") { + Properties |= 1 << SDNPWantRoot; + } else if (PropList[i]->getName() == "SDNPWantParent") { + Properties |= 1 << SDNPWantParent; + } else { + PrintFatalError(R->getLoc(), "Unsupported SD Node property '" + + PropList[i]->getName() + + "' on ComplexPattern '" + R->getName() + + "'!"); + } +} + +//===----------------------------------------------------------------------===// +// CodeGenIntrinsic Implementation +//===----------------------------------------------------------------------===// + +CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) { + std::vector<Record*> Defs = RC.getAllDerivedDefinitions("Intrinsic"); + + Intrinsics.reserve(Defs.size()); + + for (unsigned I = 0, e = Defs.size(); I != e; ++I) + Intrinsics.push_back(CodeGenIntrinsic(Defs[I])); + + llvm::sort(Intrinsics, + [](const CodeGenIntrinsic &LHS, const CodeGenIntrinsic &RHS) { + return std::tie(LHS.TargetPrefix, LHS.Name) < + std::tie(RHS.TargetPrefix, RHS.Name); + }); + Targets.push_back({"", 0, 0}); + for (size_t I = 0, E = Intrinsics.size(); I < E; ++I) + if (Intrinsics[I].TargetPrefix != Targets.back().Name) { + Targets.back().Count = I - Targets.back().Offset; + Targets.push_back({Intrinsics[I].TargetPrefix, I, 0}); + } + Targets.back().Count = Intrinsics.size() - Targets.back().Offset; +} + +CodeGenIntrinsic::CodeGenIntrinsic(Record *R) { + TheDef = R; + std::string DefName = std::string(R->getName()); + ArrayRef<SMLoc> DefLoc = R->getLoc(); + ModRef = ReadWriteMem; + Properties = 0; + isOverloaded = false; + isCommutative = false; + canThrow = false; + isNoReturn = false; + isNoSync = false; + isNoFree = false; + isWillReturn = false; + isCold = false; + isNoDuplicate = false; + isConvergent = false; + isSpeculatable = false; + hasSideEffects = false; + + if (DefName.size() <= 4 || + std::string(DefName.begin(), DefName.begin() + 4) != "int_") + PrintFatalError(DefLoc, + "Intrinsic '" + DefName + "' does not start with 'int_'!"); + + EnumName = std::string(DefName.begin()+4, DefName.end()); + + if (R->getValue("GCCBuiltinName")) // Ignore a missing GCCBuiltinName field. + GCCBuiltinName = std::string(R->getValueAsString("GCCBuiltinName")); + if (R->getValue("MSBuiltinName")) // Ignore a missing MSBuiltinName field. + MSBuiltinName = std::string(R->getValueAsString("MSBuiltinName")); + + TargetPrefix = std::string(R->getValueAsString("TargetPrefix")); + Name = std::string(R->getValueAsString("LLVMName")); + + if (Name == "") { + // If an explicit name isn't specified, derive one from the DefName. + Name = "llvm."; + + for (unsigned i = 0, e = EnumName.size(); i != e; ++i) + Name += (EnumName[i] == '_') ? '.' : EnumName[i]; + } else { + // Verify it starts with "llvm.". + if (Name.size() <= 5 || + std::string(Name.begin(), Name.begin() + 5) != "llvm.") + PrintFatalError(DefLoc, "Intrinsic '" + DefName + + "'s name does not start with 'llvm.'!"); + } + + // If TargetPrefix is specified, make sure that Name starts with + // "llvm.<targetprefix>.". + if (!TargetPrefix.empty()) { + if (Name.size() < 6+TargetPrefix.size() || + std::string(Name.begin() + 5, Name.begin() + 6 + TargetPrefix.size()) + != (TargetPrefix + ".")) + PrintFatalError(DefLoc, "Intrinsic '" + DefName + + "' does not start with 'llvm." + + TargetPrefix + ".'!"); + } + + ListInit *RetTypes = R->getValueAsListInit("RetTypes"); + ListInit *ParamTypes = R->getValueAsListInit("ParamTypes"); + + // First collate a list of overloaded types. + std::vector<MVT::SimpleValueType> OverloadedVTs; + for (ListInit *TypeList : {RetTypes, ParamTypes}) { + for (unsigned i = 0, e = TypeList->size(); i != e; ++i) { + Record *TyEl = TypeList->getElementAsRecord(i); + assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!"); + + if (TyEl->isSubClassOf("LLVMMatchType")) + continue; + + MVT::SimpleValueType VT = getValueType(TyEl->getValueAsDef("VT")); + if (MVT(VT).isOverloaded()) { + OverloadedVTs.push_back(VT); + isOverloaded = true; + } + } + } + + // Parse the list of return types. + ListInit *TypeList = RetTypes; + for (unsigned i = 0, e = TypeList->size(); i != e; ++i) { + Record *TyEl = TypeList->getElementAsRecord(i); + assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!"); + MVT::SimpleValueType VT; + if (TyEl->isSubClassOf("LLVMMatchType")) { + unsigned MatchTy = TyEl->getValueAsInt("Number"); + assert(MatchTy < OverloadedVTs.size() && + "Invalid matching number!"); + VT = OverloadedVTs[MatchTy]; + // It only makes sense to use the extended and truncated vector element + // variants with iAny types; otherwise, if the intrinsic is not + // overloaded, all the types can be specified directly. + assert(((!TyEl->isSubClassOf("LLVMExtendedType") && + !TyEl->isSubClassOf("LLVMTruncatedType")) || + VT == MVT::iAny || VT == MVT::vAny) && + "Expected iAny or vAny type"); + } else { + VT = getValueType(TyEl->getValueAsDef("VT")); + } + + // Reject invalid types. + if (VT == MVT::isVoid) + PrintFatalError(DefLoc, "Intrinsic '" + DefName + + " has void in result type list!"); + + IS.RetVTs.push_back(VT); + IS.RetTypeDefs.push_back(TyEl); + } + + // Parse the list of parameter types. + TypeList = ParamTypes; + for (unsigned i = 0, e = TypeList->size(); i != e; ++i) { + Record *TyEl = TypeList->getElementAsRecord(i); + assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!"); + MVT::SimpleValueType VT; + if (TyEl->isSubClassOf("LLVMMatchType")) { + unsigned MatchTy = TyEl->getValueAsInt("Number"); + if (MatchTy >= OverloadedVTs.size()) { + PrintError(R->getLoc(), + "Parameter #" + Twine(i) + " has out of bounds matching " + "number " + Twine(MatchTy)); + PrintFatalError(DefLoc, + Twine("ParamTypes is ") + TypeList->getAsString()); + } + VT = OverloadedVTs[MatchTy]; + // It only makes sense to use the extended and truncated vector element + // variants with iAny types; otherwise, if the intrinsic is not + // overloaded, all the types can be specified directly. + assert(((!TyEl->isSubClassOf("LLVMExtendedType") && + !TyEl->isSubClassOf("LLVMTruncatedType")) || + VT == MVT::iAny || VT == MVT::vAny) && + "Expected iAny or vAny type"); + } else + VT = getValueType(TyEl->getValueAsDef("VT")); + + // Reject invalid types. + if (VT == MVT::isVoid && i != e-1 /*void at end means varargs*/) + PrintFatalError(DefLoc, "Intrinsic '" + DefName + + " has void in result type list!"); + + IS.ParamVTs.push_back(VT); + IS.ParamTypeDefs.push_back(TyEl); + } + + // Parse the intrinsic properties. + ListInit *PropList = R->getValueAsListInit("IntrProperties"); + for (unsigned i = 0, e = PropList->size(); i != e; ++i) { + Record *Property = PropList->getElementAsRecord(i); + assert(Property->isSubClassOf("IntrinsicProperty") && + "Expected a property!"); + + if (Property->getName() == "IntrNoMem") + ModRef = NoMem; + else if (Property->getName() == "IntrReadMem") + ModRef = ModRefBehavior(ModRef & ~MR_Mod); + else if (Property->getName() == "IntrWriteMem") + ModRef = ModRefBehavior(ModRef & ~MR_Ref); + else if (Property->getName() == "IntrArgMemOnly") + ModRef = ModRefBehavior((ModRef & ~MR_Anywhere) | MR_ArgMem); + else if (Property->getName() == "IntrInaccessibleMemOnly") + ModRef = ModRefBehavior((ModRef & ~MR_Anywhere) | MR_InaccessibleMem); + else if (Property->getName() == "IntrInaccessibleMemOrArgMemOnly") + ModRef = ModRefBehavior((ModRef & ~MR_Anywhere) | MR_ArgMem | + MR_InaccessibleMem); + else if (Property->getName() == "Commutative") + isCommutative = true; + else if (Property->getName() == "Throws") + canThrow = true; + else if (Property->getName() == "IntrNoDuplicate") + isNoDuplicate = true; + else if (Property->getName() == "IntrConvergent") + isConvergent = true; + else if (Property->getName() == "IntrNoReturn") + isNoReturn = true; + else if (Property->getName() == "IntrNoSync") + isNoSync = true; + else if (Property->getName() == "IntrNoFree") + isNoFree = true; + else if (Property->getName() == "IntrWillReturn") + isWillReturn = true; + else if (Property->getName() == "IntrCold") + isCold = true; + else if (Property->getName() == "IntrSpeculatable") + isSpeculatable = true; + else if (Property->getName() == "IntrHasSideEffects") + hasSideEffects = true; + else if (Property->isSubClassOf("NoCapture")) { + unsigned ArgNo = Property->getValueAsInt("ArgNo"); + ArgumentAttributes.emplace_back(ArgNo, NoCapture, 0); + } else if (Property->isSubClassOf("NoAlias")) { + unsigned ArgNo = Property->getValueAsInt("ArgNo"); + ArgumentAttributes.emplace_back(ArgNo, NoAlias, 0); + } else if (Property->isSubClassOf("Returned")) { + unsigned ArgNo = Property->getValueAsInt("ArgNo"); + ArgumentAttributes.emplace_back(ArgNo, Returned, 0); + } else if (Property->isSubClassOf("ReadOnly")) { + unsigned ArgNo = Property->getValueAsInt("ArgNo"); + ArgumentAttributes.emplace_back(ArgNo, ReadOnly, 0); + } else if (Property->isSubClassOf("WriteOnly")) { + unsigned ArgNo = Property->getValueAsInt("ArgNo"); + ArgumentAttributes.emplace_back(ArgNo, WriteOnly, 0); + } else if (Property->isSubClassOf("ReadNone")) { + unsigned ArgNo = Property->getValueAsInt("ArgNo"); + ArgumentAttributes.emplace_back(ArgNo, ReadNone, 0); + } else if (Property->isSubClassOf("ImmArg")) { + unsigned ArgNo = Property->getValueAsInt("ArgNo"); + ArgumentAttributes.emplace_back(ArgNo, ImmArg, 0); + } else if (Property->isSubClassOf("Align")) { + unsigned ArgNo = Property->getValueAsInt("ArgNo"); + uint64_t Align = Property->getValueAsInt("Align"); + ArgumentAttributes.emplace_back(ArgNo, Alignment, Align); + } else + llvm_unreachable("Unknown property!"); + } + + // Also record the SDPatternOperator Properties. + Properties = parseSDPatternOperatorProperties(R); + + // Sort the argument attributes for later benefit. + llvm::sort(ArgumentAttributes); +} + +bool CodeGenIntrinsic::isParamAPointer(unsigned ParamIdx) const { + if (ParamIdx >= IS.ParamVTs.size()) + return false; + MVT ParamType = MVT(IS.ParamVTs[ParamIdx]); + return ParamType == MVT::iPTR || ParamType == MVT::iPTRAny; +} + +bool CodeGenIntrinsic::isParamImmArg(unsigned ParamIdx) const { + // Convert argument index to attribute index starting from `FirstArgIndex`. + ArgAttribute Val{ParamIdx + 1, ImmArg, 0}; + return std::binary_search(ArgumentAttributes.begin(), + ArgumentAttributes.end(), Val); +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.h b/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.h new file mode 100644 index 000000000000..6c89f34c50ec --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/CodeGenTarget.h @@ -0,0 +1,221 @@ +//===- CodeGenTarget.h - Target Class Wrapper -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines wrappers for the Target class and related global +// functionality. This makes it easier to access the data and provides a single +// place that needs to check it for validity. All of these classes abort +// on error conditions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEGENTARGET_H +#define LLVM_UTILS_TABLEGEN_CODEGENTARGET_H + +#include "CodeGenHwModes.h" +#include "CodeGenInstruction.h" +#include "CodeGenRegisters.h" +#include "InfoByHwMode.h" +#include "SDNodeProperties.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Record.h" +#include <algorithm> + +namespace llvm { + +struct CodeGenRegister; +class CodeGenSchedModels; +class CodeGenTarget; + +/// getValueType - Return the MVT::SimpleValueType that the specified TableGen +/// record corresponds to. +MVT::SimpleValueType getValueType(Record *Rec); + +StringRef getName(MVT::SimpleValueType T); +StringRef getEnumName(MVT::SimpleValueType T); + +/// getQualifiedName - Return the name of the specified record, with a +/// namespace qualifier if the record contains one. +std::string getQualifiedName(const Record *R); + +/// CodeGenTarget - This class corresponds to the Target class in the .td files. +/// +class CodeGenTarget { + RecordKeeper &Records; + Record *TargetRec; + + mutable DenseMap<const Record*, + std::unique_ptr<CodeGenInstruction>> Instructions; + mutable std::unique_ptr<CodeGenRegBank> RegBank; + mutable std::vector<Record*> RegAltNameIndices; + mutable SmallVector<ValueTypeByHwMode, 8> LegalValueTypes; + CodeGenHwModes CGH; + void ReadRegAltNameIndices() const; + void ReadInstructions() const; + void ReadLegalValueTypes() const; + + mutable std::unique_ptr<CodeGenSchedModels> SchedModels; + + mutable std::vector<const CodeGenInstruction*> InstrsByEnum; + mutable unsigned NumPseudoInstructions = 0; +public: + CodeGenTarget(RecordKeeper &Records); + ~CodeGenTarget(); + + Record *getTargetRecord() const { return TargetRec; } + const StringRef getName() const; + + /// getInstNamespace - Return the target-specific instruction namespace. + /// + StringRef getInstNamespace() const; + + /// getInstructionSet - Return the InstructionSet object. + /// + Record *getInstructionSet() const; + + /// getAllowRegisterRenaming - Return the AllowRegisterRenaming flag value for + /// this target. + /// + bool getAllowRegisterRenaming() const; + + /// getAsmParser - Return the AssemblyParser definition for this target. + /// + Record *getAsmParser() const; + + /// getAsmParserVariant - Return the AssemblyParserVariant definition for + /// this target. + /// + Record *getAsmParserVariant(unsigned i) const; + + /// getAsmParserVariantCount - Return the AssemblyParserVariant definition + /// available for this target. + /// + unsigned getAsmParserVariantCount() const; + + /// getAsmWriter - Return the AssemblyWriter definition for this target. + /// + Record *getAsmWriter() const; + + /// getRegBank - Return the register bank description. + CodeGenRegBank &getRegBank() const; + + /// Return the largest register class on \p RegBank which supports \p Ty and + /// covers \p SubIdx if it exists. + Optional<CodeGenRegisterClass *> + getSuperRegForSubReg(const ValueTypeByHwMode &Ty, CodeGenRegBank &RegBank, + const CodeGenSubRegIndex *SubIdx) const; + + /// getRegisterByName - If there is a register with the specific AsmName, + /// return it. + const CodeGenRegister *getRegisterByName(StringRef Name) const; + + const std::vector<Record*> &getRegAltNameIndices() const { + if (RegAltNameIndices.empty()) ReadRegAltNameIndices(); + return RegAltNameIndices; + } + + const CodeGenRegisterClass &getRegisterClass(Record *R) const { + return *getRegBank().getRegClass(R); + } + + /// getRegisterVTs - Find the union of all possible SimpleValueTypes for the + /// specified physical register. + std::vector<ValueTypeByHwMode> getRegisterVTs(Record *R) const; + + ArrayRef<ValueTypeByHwMode> getLegalValueTypes() const { + if (LegalValueTypes.empty()) + ReadLegalValueTypes(); + return LegalValueTypes; + } + + CodeGenSchedModels &getSchedModels() const; + + const CodeGenHwModes &getHwModes() const { return CGH; } + +private: + DenseMap<const Record*, std::unique_ptr<CodeGenInstruction>> & + getInstructions() const { + if (Instructions.empty()) ReadInstructions(); + return Instructions; + } +public: + + CodeGenInstruction &getInstruction(const Record *InstRec) const { + if (Instructions.empty()) ReadInstructions(); + auto I = Instructions.find(InstRec); + assert(I != Instructions.end() && "Not an instruction"); + return *I->second; + } + + /// Returns the number of predefined instructions. + static unsigned getNumFixedInstructions(); + + /// Returns the number of pseudo instructions. + unsigned getNumPseudoInstructions() const { + if (InstrsByEnum.empty()) + ComputeInstrsByEnum(); + return NumPseudoInstructions; + } + + /// Return all of the instructions defined by the target, ordered by their + /// enum value. + /// The following order of instructions is also guaranteed: + /// - fixed / generic instructions as declared in TargetOpcodes.def, in order; + /// - pseudo instructions in lexicographical order sorted by name; + /// - other instructions in lexicographical order sorted by name. + ArrayRef<const CodeGenInstruction *> getInstructionsByEnumValue() const { + if (InstrsByEnum.empty()) + ComputeInstrsByEnum(); + return InstrsByEnum; + } + + typedef ArrayRef<const CodeGenInstruction *>::const_iterator inst_iterator; + inst_iterator inst_begin() const{return getInstructionsByEnumValue().begin();} + inst_iterator inst_end() const { return getInstructionsByEnumValue().end(); } + + + /// isLittleEndianEncoding - are instruction bit patterns defined as [0..n]? + /// + bool isLittleEndianEncoding() const; + + /// reverseBitsForLittleEndianEncoding - For little-endian instruction bit + /// encodings, reverse the bit order of all instructions. + void reverseBitsForLittleEndianEncoding(); + + /// guessInstructionProperties - should we just guess unset instruction + /// properties? + bool guessInstructionProperties() const; + +private: + void ComputeInstrsByEnum() const; +}; + +/// ComplexPattern - ComplexPattern info, corresponding to the ComplexPattern +/// tablegen class in TargetSelectionDAG.td +class ComplexPattern { + MVT::SimpleValueType Ty; + unsigned NumOperands; + std::string SelectFunc; + std::vector<Record*> RootNodes; + unsigned Properties; // Node properties + unsigned Complexity; +public: + ComplexPattern(Record *R); + + MVT::SimpleValueType getValueType() const { return Ty; } + unsigned getNumOperands() const { return NumOperands; } + const std::string &getSelectFunc() const { return SelectFunc; } + const std::vector<Record*> &getRootNodes() const { + return RootNodes; + } + bool hasProperty(enum SDNP Prop) const { return Properties & (1 << Prop); } + unsigned getComplexity() const { return Complexity; } +}; + +} // End llvm namespace + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DAGISelEmitter.cpp new file mode 100644 index 000000000000..d8e78ce55c7b --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelEmitter.cpp @@ -0,0 +1,189 @@ +//===- DAGISelEmitter.cpp - Generate an instruction selector --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend emits a DAG instruction selector. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenDAGPatterns.h" +#include "DAGISelMatcher.h" +#include "llvm/Support/Debug.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +using namespace llvm; + +#define DEBUG_TYPE "dag-isel-emitter" + +namespace { +/// DAGISelEmitter - The top-level class which coordinates construction +/// and emission of the instruction selector. +class DAGISelEmitter { + CodeGenDAGPatterns CGP; +public: + explicit DAGISelEmitter(RecordKeeper &R) : CGP(R) {} + void run(raw_ostream &OS); +}; +} // End anonymous namespace + +//===----------------------------------------------------------------------===// +// DAGISelEmitter Helper methods +// + +/// getResultPatternCost - Compute the number of instructions for this pattern. +/// This is a temporary hack. We should really include the instruction +/// latencies in this calculation. +static unsigned getResultPatternCost(TreePatternNode *P, + CodeGenDAGPatterns &CGP) { + if (P->isLeaf()) return 0; + + unsigned Cost = 0; + Record *Op = P->getOperator(); + if (Op->isSubClassOf("Instruction")) { + Cost++; + CodeGenInstruction &II = CGP.getTargetInfo().getInstruction(Op); + if (II.usesCustomInserter) + Cost += 10; + } + for (unsigned i = 0, e = P->getNumChildren(); i != e; ++i) + Cost += getResultPatternCost(P->getChild(i), CGP); + return Cost; +} + +/// getResultPatternCodeSize - Compute the code size of instructions for this +/// pattern. +static unsigned getResultPatternSize(TreePatternNode *P, + CodeGenDAGPatterns &CGP) { + if (P->isLeaf()) return 0; + + unsigned Cost = 0; + Record *Op = P->getOperator(); + if (Op->isSubClassOf("Instruction")) { + Cost += Op->getValueAsInt("CodeSize"); + } + for (unsigned i = 0, e = P->getNumChildren(); i != e; ++i) + Cost += getResultPatternSize(P->getChild(i), CGP); + return Cost; +} + +namespace { +// PatternSortingPredicate - return true if we prefer to match LHS before RHS. +// In particular, we want to match maximal patterns first and lowest cost within +// a particular complexity first. +struct PatternSortingPredicate { + PatternSortingPredicate(CodeGenDAGPatterns &cgp) : CGP(cgp) {} + CodeGenDAGPatterns &CGP; + + bool operator()(const PatternToMatch *LHS, const PatternToMatch *RHS) { + const TreePatternNode *LT = LHS->getSrcPattern(); + const TreePatternNode *RT = RHS->getSrcPattern(); + + MVT LHSVT = LT->getNumTypes() != 0 ? LT->getSimpleType(0) : MVT::Other; + MVT RHSVT = RT->getNumTypes() != 0 ? RT->getSimpleType(0) : MVT::Other; + if (LHSVT.isVector() != RHSVT.isVector()) + return RHSVT.isVector(); + + if (LHSVT.isFloatingPoint() != RHSVT.isFloatingPoint()) + return RHSVT.isFloatingPoint(); + + // Otherwise, if the patterns might both match, sort based on complexity, + // which means that we prefer to match patterns that cover more nodes in the + // input over nodes that cover fewer. + int LHSSize = LHS->getPatternComplexity(CGP); + int RHSSize = RHS->getPatternComplexity(CGP); + if (LHSSize > RHSSize) return true; // LHS -> bigger -> less cost + if (LHSSize < RHSSize) return false; + + // If the patterns have equal complexity, compare generated instruction cost + unsigned LHSCost = getResultPatternCost(LHS->getDstPattern(), CGP); + unsigned RHSCost = getResultPatternCost(RHS->getDstPattern(), CGP); + if (LHSCost < RHSCost) return true; + if (LHSCost > RHSCost) return false; + + unsigned LHSPatSize = getResultPatternSize(LHS->getDstPattern(), CGP); + unsigned RHSPatSize = getResultPatternSize(RHS->getDstPattern(), CGP); + if (LHSPatSize < RHSPatSize) return true; + if (LHSPatSize > RHSPatSize) return false; + + // Sort based on the UID of the pattern, to reflect source order. + // Note that this is not guaranteed to be unique, since a single source + // pattern may have been resolved into multiple match patterns due to + // alternative fragments. To ensure deterministic output, always use + // std::stable_sort with this predicate. + return LHS->ID < RHS->ID; + } +}; +} // End anonymous namespace + + +void DAGISelEmitter::run(raw_ostream &OS) { + emitSourceFileHeader("DAG Instruction Selector for the " + + CGP.getTargetInfo().getName().str() + " target", OS); + + OS << "// *** NOTE: This file is #included into the middle of the target\n" + << "// *** instruction selector class. These functions are really " + << "methods.\n\n"; + + OS << "// If GET_DAGISEL_DECL is #defined with any value, only function\n" + "// declarations will be included when this file is included.\n" + "// If GET_DAGISEL_BODY is #defined, its value should be the name of\n" + "// the instruction selector class. Function bodies will be emitted\n" + "// and each function's name will be qualified with the name of the\n" + "// class.\n" + "//\n" + "// When neither of the GET_DAGISEL* macros is defined, the functions\n" + "// are emitted inline.\n\n"; + + LLVM_DEBUG(errs() << "\n\nALL PATTERNS TO MATCH:\n\n"; + for (CodeGenDAGPatterns::ptm_iterator I = CGP.ptm_begin(), + E = CGP.ptm_end(); + I != E; ++I) { + errs() << "PATTERN: "; + I->getSrcPattern()->dump(); + errs() << "\nRESULT: "; + I->getDstPattern()->dump(); + errs() << "\n"; + }); + + // Add all the patterns to a temporary list so we can sort them. + std::vector<const PatternToMatch*> Patterns; + for (CodeGenDAGPatterns::ptm_iterator I = CGP.ptm_begin(), E = CGP.ptm_end(); + I != E; ++I) + Patterns.push_back(&*I); + + // We want to process the matches in order of minimal cost. Sort the patterns + // so the least cost one is at the start. + std::stable_sort(Patterns.begin(), Patterns.end(), + PatternSortingPredicate(CGP)); + + + // Convert each variant of each pattern into a Matcher. + std::vector<Matcher*> PatternMatchers; + for (unsigned i = 0, e = Patterns.size(); i != e; ++i) { + for (unsigned Variant = 0; ; ++Variant) { + if (Matcher *M = ConvertPatternToMatcher(*Patterns[i], Variant, CGP)) + PatternMatchers.push_back(M); + else + break; + } + } + + std::unique_ptr<Matcher> TheMatcher = + std::make_unique<ScopeMatcher>(PatternMatchers); + + OptimizeMatcher(TheMatcher, CGP); + //Matcher->dump(); + EmitMatcherTable(TheMatcher.get(), CGP, OS); +} + +namespace llvm { + +void EmitDAGISel(RecordKeeper &RK, raw_ostream &OS) { + DAGISelEmitter(RK).run(OS); +} + +} // End llvm namespace diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcher.cpp b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcher.cpp new file mode 100644 index 000000000000..bebd205ad58f --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcher.cpp @@ -0,0 +1,423 @@ +//===- DAGISelMatcher.cpp - Representation of DAG pattern matcher ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DAGISelMatcher.h" +#include "CodeGenDAGPatterns.h" +#include "CodeGenTarget.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Record.h" +using namespace llvm; + +void Matcher::anchor() { } + +void Matcher::dump() const { + print(errs(), 0); +} + +void Matcher::print(raw_ostream &OS, unsigned indent) const { + printImpl(OS, indent); + if (Next) + return Next->print(OS, indent); +} + +void Matcher::printOne(raw_ostream &OS) const { + printImpl(OS, 0); +} + +/// unlinkNode - Unlink the specified node from this chain. If Other == this, +/// we unlink the next pointer and return it. Otherwise we unlink Other from +/// the list and return this. +Matcher *Matcher::unlinkNode(Matcher *Other) { + if (this == Other) + return takeNext(); + + // Scan until we find the predecessor of Other. + Matcher *Cur = this; + for (; Cur && Cur->getNext() != Other; Cur = Cur->getNext()) + /*empty*/; + + if (!Cur) return nullptr; + Cur->takeNext(); + Cur->setNext(Other->takeNext()); + return this; +} + +/// canMoveBefore - Return true if this matcher is the same as Other, or if +/// we can move this matcher past all of the nodes in-between Other and this +/// node. Other must be equal to or before this. +bool Matcher::canMoveBefore(const Matcher *Other) const { + for (;; Other = Other->getNext()) { + assert(Other && "Other didn't come before 'this'?"); + if (this == Other) return true; + + // We have to be able to move this node across the Other node. + if (!canMoveBeforeNode(Other)) + return false; + } +} + +/// canMoveBeforeNode - Return true if it is safe to move the current matcher +/// across the specified one. +bool Matcher::canMoveBeforeNode(const Matcher *Other) const { + // We can move simple predicates before record nodes. + if (isSimplePredicateNode()) + return Other->isSimplePredicateOrRecordNode(); + + // We can move record nodes across simple predicates. + if (isSimplePredicateOrRecordNode()) + return isSimplePredicateNode(); + + // We can't move record nodes across each other etc. + return false; +} + + +ScopeMatcher::~ScopeMatcher() { + for (Matcher *C : Children) + delete C; +} + +SwitchOpcodeMatcher::~SwitchOpcodeMatcher() { + for (auto &C : Cases) + delete C.second; +} + +SwitchTypeMatcher::~SwitchTypeMatcher() { + for (auto &C : Cases) + delete C.second; +} + +CheckPredicateMatcher::CheckPredicateMatcher( + const TreePredicateFn &pred, const SmallVectorImpl<unsigned> &Ops) + : Matcher(CheckPredicate), Pred(pred.getOrigPatFragRecord()), + Operands(Ops.begin(), Ops.end()) {} + +TreePredicateFn CheckPredicateMatcher::getPredicate() const { + return TreePredicateFn(Pred); +} + +unsigned CheckPredicateMatcher::getNumOperands() const { + return Operands.size(); +} + +unsigned CheckPredicateMatcher::getOperandNo(unsigned i) const { + assert(i < Operands.size()); + return Operands[i]; +} + + +// printImpl methods. + +void ScopeMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "Scope\n"; + for (const Matcher *C : Children) { + if (!C) + OS.indent(indent+1) << "NULL POINTER\n"; + else + C->print(OS, indent+2); + } +} + +void RecordMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "Record\n"; +} + +void RecordChildMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "RecordChild: " << ChildNo << '\n'; +} + +void RecordMemRefMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "RecordMemRef\n"; +} + +void CaptureGlueInputMatcher::printImpl(raw_ostream &OS, unsigned indent) const{ + OS.indent(indent) << "CaptureGlueInput\n"; +} + +void MoveChildMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "MoveChild " << ChildNo << '\n'; +} + +void MoveParentMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "MoveParent\n"; +} + +void CheckSameMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckSame " << MatchNumber << '\n'; +} + +void CheckChildSameMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckChild" << ChildNo << "Same\n"; +} + +void CheckPatternPredicateMatcher:: +printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckPatternPredicate " << Predicate << '\n'; +} + +void CheckPredicateMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckPredicate " << getPredicate().getFnName() << '\n'; +} + +void CheckOpcodeMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckOpcode " << Opcode.getEnumName() << '\n'; +} + +void SwitchOpcodeMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "SwitchOpcode: {\n"; + for (const auto &C : Cases) { + OS.indent(indent) << "case " << C.first->getEnumName() << ":\n"; + C.second->print(OS, indent+2); + } + OS.indent(indent) << "}\n"; +} + + +void CheckTypeMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckType " << getEnumName(Type) << ", ResNo=" + << ResNo << '\n'; +} + +void SwitchTypeMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "SwitchType: {\n"; + for (const auto &C : Cases) { + OS.indent(indent) << "case " << getEnumName(C.first) << ":\n"; + C.second->print(OS, indent+2); + } + OS.indent(indent) << "}\n"; +} + +void CheckChildTypeMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckChildType " << ChildNo << " " + << getEnumName(Type) << '\n'; +} + + +void CheckIntegerMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckInteger " << Value << '\n'; +} + +void CheckChildIntegerMatcher::printImpl(raw_ostream &OS, + unsigned indent) const { + OS.indent(indent) << "CheckChildInteger " << ChildNo << " " << Value << '\n'; +} + +void CheckCondCodeMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckCondCode ISD::" << CondCodeName << '\n'; +} + +void CheckChild2CondCodeMatcher::printImpl(raw_ostream &OS, + unsigned indent) const { + OS.indent(indent) << "CheckChild2CondCode ISD::" << CondCodeName << '\n'; +} + +void CheckValueTypeMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckValueType MVT::" << TypeName << '\n'; +} + +void CheckComplexPatMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckComplexPat " << Pattern.getSelectFunc() << '\n'; +} + +void CheckAndImmMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckAndImm " << Value << '\n'; +} + +void CheckOrImmMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CheckOrImm " << Value << '\n'; +} + +void CheckFoldableChainNodeMatcher::printImpl(raw_ostream &OS, + unsigned indent) const { + OS.indent(indent) << "CheckFoldableChainNode\n"; +} + +void CheckImmAllOnesVMatcher::printImpl(raw_ostream &OS, + unsigned indent) const { + OS.indent(indent) << "CheckAllOnesV\n"; +} + +void CheckImmAllZerosVMatcher::printImpl(raw_ostream &OS, + unsigned indent) const { + OS.indent(indent) << "CheckAllZerosV\n"; +} + +void EmitIntegerMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "EmitInteger " << Val << " VT=" << getEnumName(VT) + << '\n'; +} + +void EmitStringIntegerMatcher:: +printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "EmitStringInteger " << Val << " VT=" << getEnumName(VT) + << '\n'; +} + +void EmitRegisterMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "EmitRegister "; + if (Reg) + OS << Reg->getName(); + else + OS << "zero_reg"; + OS << " VT=" << getEnumName(VT) << '\n'; +} + +void EmitConvertToTargetMatcher:: +printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "EmitConvertToTarget " << Slot << '\n'; +} + +void EmitMergeInputChainsMatcher:: +printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "EmitMergeInputChains <todo: args>\n"; +} + +void EmitCopyToRegMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "EmitCopyToReg <todo: args>\n"; +} + +void EmitNodeXFormMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "EmitNodeXForm " << NodeXForm->getName() + << " Slot=" << Slot << '\n'; +} + + +void EmitNodeMatcherCommon::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent); + OS << (isa<MorphNodeToMatcher>(this) ? "MorphNodeTo: " : "EmitNode: ") + << OpcodeName << ": <todo flags> "; + + for (unsigned i = 0, e = VTs.size(); i != e; ++i) + OS << ' ' << getEnumName(VTs[i]); + OS << '('; + for (unsigned i = 0, e = Operands.size(); i != e; ++i) + OS << Operands[i] << ' '; + OS << ")\n"; +} + +void CompleteMatchMatcher::printImpl(raw_ostream &OS, unsigned indent) const { + OS.indent(indent) << "CompleteMatch <todo args>\n"; + OS.indent(indent) << "Src = " << *Pattern.getSrcPattern() << "\n"; + OS.indent(indent) << "Dst = " << *Pattern.getDstPattern() << "\n"; +} + +bool CheckOpcodeMatcher::isEqualImpl(const Matcher *M) const { + // Note: pointer equality isn't enough here, we have to check the enum names + // to ensure that the nodes are for the same opcode. + return cast<CheckOpcodeMatcher>(M)->Opcode.getEnumName() == + Opcode.getEnumName(); +} + +bool EmitNodeMatcherCommon::isEqualImpl(const Matcher *m) const { + const EmitNodeMatcherCommon *M = cast<EmitNodeMatcherCommon>(m); + return M->OpcodeName == OpcodeName && M->VTs == VTs && + M->Operands == Operands && M->HasChain == HasChain && + M->HasInGlue == HasInGlue && M->HasOutGlue == HasOutGlue && + M->HasMemRefs == HasMemRefs && + M->NumFixedArityOperands == NumFixedArityOperands; +} + +void EmitNodeMatcher::anchor() { } + +void MorphNodeToMatcher::anchor() { } + +// isContradictoryImpl Implementations. + +static bool TypesAreContradictory(MVT::SimpleValueType T1, + MVT::SimpleValueType T2) { + // If the two types are the same, then they are the same, so they don't + // contradict. + if (T1 == T2) return false; + + // If either type is about iPtr, then they don't conflict unless the other + // one is not a scalar integer type. + if (T1 == MVT::iPTR) + return !MVT(T2).isInteger() || MVT(T2).isVector(); + + if (T2 == MVT::iPTR) + return !MVT(T1).isInteger() || MVT(T1).isVector(); + + // Otherwise, they are two different non-iPTR types, they conflict. + return true; +} + +bool CheckOpcodeMatcher::isContradictoryImpl(const Matcher *M) const { + if (const CheckOpcodeMatcher *COM = dyn_cast<CheckOpcodeMatcher>(M)) { + // One node can't have two different opcodes! + // Note: pointer equality isn't enough here, we have to check the enum names + // to ensure that the nodes are for the same opcode. + return COM->getOpcode().getEnumName() != getOpcode().getEnumName(); + } + + // If the node has a known type, and if the type we're checking for is + // different, then we know they contradict. For example, a check for + // ISD::STORE will never be true at the same time a check for Type i32 is. + if (const CheckTypeMatcher *CT = dyn_cast<CheckTypeMatcher>(M)) { + // If checking for a result the opcode doesn't have, it can't match. + if (CT->getResNo() >= getOpcode().getNumResults()) + return true; + + MVT::SimpleValueType NodeType = getOpcode().getKnownType(CT->getResNo()); + if (NodeType != MVT::Other) + return TypesAreContradictory(NodeType, CT->getType()); + } + + return false; +} + +bool CheckTypeMatcher::isContradictoryImpl(const Matcher *M) const { + if (const CheckTypeMatcher *CT = dyn_cast<CheckTypeMatcher>(M)) + return TypesAreContradictory(getType(), CT->getType()); + return false; +} + +bool CheckChildTypeMatcher::isContradictoryImpl(const Matcher *M) const { + if (const CheckChildTypeMatcher *CC = dyn_cast<CheckChildTypeMatcher>(M)) { + // If the two checks are about different nodes, we don't know if they + // conflict! + if (CC->getChildNo() != getChildNo()) + return false; + + return TypesAreContradictory(getType(), CC->getType()); + } + return false; +} + +bool CheckIntegerMatcher::isContradictoryImpl(const Matcher *M) const { + if (const CheckIntegerMatcher *CIM = dyn_cast<CheckIntegerMatcher>(M)) + return CIM->getValue() != getValue(); + return false; +} + +bool CheckChildIntegerMatcher::isContradictoryImpl(const Matcher *M) const { + if (const CheckChildIntegerMatcher *CCIM = dyn_cast<CheckChildIntegerMatcher>(M)) { + // If the two checks are about different nodes, we don't know if they + // conflict! + if (CCIM->getChildNo() != getChildNo()) + return false; + + return CCIM->getValue() != getValue(); + } + return false; +} + +bool CheckValueTypeMatcher::isContradictoryImpl(const Matcher *M) const { + if (const CheckValueTypeMatcher *CVT = dyn_cast<CheckValueTypeMatcher>(M)) + return CVT->getTypeName() != getTypeName(); + return false; +} + +bool CheckImmAllOnesVMatcher::isContradictoryImpl(const Matcher *M) const { + // AllZeros is contradictory. + return isa<CheckImmAllZerosVMatcher>(M); +} + +bool CheckImmAllZerosVMatcher::isContradictoryImpl(const Matcher *M) const { + // AllOnes is contradictory. + return isa<CheckImmAllOnesVMatcher>(M); +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcher.h b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcher.h new file mode 100644 index 000000000000..223513fc8d38 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcher.h @@ -0,0 +1,1117 @@ +//===- DAGISelMatcher.h - Representation of DAG pattern matcher -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_DAGISELMATCHER_H +#define LLVM_UTILS_TABLEGEN_DAGISELMATCHER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/MachineValueType.h" + +namespace llvm { + struct CodeGenRegister; + class CodeGenDAGPatterns; + class Matcher; + class PatternToMatch; + class raw_ostream; + class ComplexPattern; + class Record; + class SDNodeInfo; + class TreePredicateFn; + class TreePattern; + +Matcher *ConvertPatternToMatcher(const PatternToMatch &Pattern,unsigned Variant, + const CodeGenDAGPatterns &CGP); +void OptimizeMatcher(std::unique_ptr<Matcher> &Matcher, + const CodeGenDAGPatterns &CGP); +void EmitMatcherTable(const Matcher *Matcher, const CodeGenDAGPatterns &CGP, + raw_ostream &OS); + + +/// Matcher - Base class for all the DAG ISel Matcher representation +/// nodes. +class Matcher { + // The next matcher node that is executed after this one. Null if this is the + // last stage of a match. + std::unique_ptr<Matcher> Next; + virtual void anchor(); +public: + enum KindTy { + // Matcher state manipulation. + Scope, // Push a checking scope. + RecordNode, // Record the current node. + RecordChild, // Record a child of the current node. + RecordMemRef, // Record the memref in the current node. + CaptureGlueInput, // If the current node has an input glue, save it. + MoveChild, // Move current node to specified child. + MoveParent, // Move current node to parent. + + // Predicate checking. + CheckSame, // Fail if not same as prev match. + CheckChildSame, // Fail if child not same as prev match. + CheckPatternPredicate, + CheckPredicate, // Fail if node predicate fails. + CheckOpcode, // Fail if not opcode. + SwitchOpcode, // Dispatch based on opcode. + CheckType, // Fail if not correct type. + SwitchType, // Dispatch based on type. + CheckChildType, // Fail if child has wrong type. + CheckInteger, // Fail if wrong val. + CheckChildInteger, // Fail if child is wrong val. + CheckCondCode, // Fail if not condcode. + CheckChild2CondCode, // Fail if child is wrong condcode. + CheckValueType, + CheckComplexPat, + CheckAndImm, + CheckOrImm, + CheckImmAllOnesV, + CheckImmAllZerosV, + CheckFoldableChainNode, + + // Node creation/emisssion. + EmitInteger, // Create a TargetConstant + EmitStringInteger, // Create a TargetConstant from a string. + EmitRegister, // Create a register. + EmitConvertToTarget, // Convert a imm/fpimm to target imm/fpimm + EmitMergeInputChains, // Merge together a chains for an input. + EmitCopyToReg, // Emit a copytoreg into a physreg. + EmitNode, // Create a DAG node + EmitNodeXForm, // Run a SDNodeXForm + CompleteMatch, // Finish a match and update the results. + MorphNodeTo // Build a node, finish a match and update results. + }; + const KindTy Kind; + +protected: + Matcher(KindTy K) : Kind(K) {} +public: + virtual ~Matcher() {} + + KindTy getKind() const { return Kind; } + + Matcher *getNext() { return Next.get(); } + const Matcher *getNext() const { return Next.get(); } + void setNext(Matcher *C) { Next.reset(C); } + Matcher *takeNext() { return Next.release(); } + + std::unique_ptr<Matcher> &getNextPtr() { return Next; } + + bool isEqual(const Matcher *M) const { + if (getKind() != M->getKind()) return false; + return isEqualImpl(M); + } + + /// isSimplePredicateNode - Return true if this is a simple predicate that + /// operates on the node or its children without potential side effects or a + /// change of the current node. + bool isSimplePredicateNode() const { + switch (getKind()) { + default: return false; + case CheckSame: + case CheckChildSame: + case CheckPatternPredicate: + case CheckPredicate: + case CheckOpcode: + case CheckType: + case CheckChildType: + case CheckInteger: + case CheckChildInteger: + case CheckCondCode: + case CheckChild2CondCode: + case CheckValueType: + case CheckAndImm: + case CheckOrImm: + case CheckImmAllOnesV: + case CheckImmAllZerosV: + case CheckFoldableChainNode: + return true; + } + } + + /// isSimplePredicateOrRecordNode - Return true if this is a record node or + /// a simple predicate. + bool isSimplePredicateOrRecordNode() const { + return isSimplePredicateNode() || + getKind() == RecordNode || getKind() == RecordChild; + } + + /// unlinkNode - Unlink the specified node from this chain. If Other == this, + /// we unlink the next pointer and return it. Otherwise we unlink Other from + /// the list and return this. + Matcher *unlinkNode(Matcher *Other); + + /// canMoveBefore - Return true if this matcher is the same as Other, or if + /// we can move this matcher past all of the nodes in-between Other and this + /// node. Other must be equal to or before this. + bool canMoveBefore(const Matcher *Other) const; + + /// canMoveBeforeNode - Return true if it is safe to move the current matcher + /// across the specified one. + bool canMoveBeforeNode(const Matcher *Other) const; + + /// isContradictory - Return true of these two matchers could never match on + /// the same node. + bool isContradictory(const Matcher *Other) const { + // Since this predicate is reflexive, we canonicalize the ordering so that + // we always match a node against nodes with kinds that are greater or equal + // to them. For example, we'll pass in a CheckType node as an argument to + // the CheckOpcode method, not the other way around. + if (getKind() < Other->getKind()) + return isContradictoryImpl(Other); + return Other->isContradictoryImpl(this); + } + + void print(raw_ostream &OS, unsigned indent = 0) const; + void printOne(raw_ostream &OS) const; + void dump() const; +protected: + virtual void printImpl(raw_ostream &OS, unsigned indent) const = 0; + virtual bool isEqualImpl(const Matcher *M) const = 0; + virtual bool isContradictoryImpl(const Matcher *M) const { return false; } +}; + +/// ScopeMatcher - This attempts to match each of its children to find the first +/// one that successfully matches. If one child fails, it tries the next child. +/// If none of the children match then this check fails. It never has a 'next'. +class ScopeMatcher : public Matcher { + SmallVector<Matcher*, 4> Children; +public: + ScopeMatcher(ArrayRef<Matcher *> children) + : Matcher(Scope), Children(children.begin(), children.end()) { + } + ~ScopeMatcher() override; + + unsigned getNumChildren() const { return Children.size(); } + + Matcher *getChild(unsigned i) { return Children[i]; } + const Matcher *getChild(unsigned i) const { return Children[i]; } + + void resetChild(unsigned i, Matcher *N) { + delete Children[i]; + Children[i] = N; + } + + Matcher *takeChild(unsigned i) { + Matcher *Res = Children[i]; + Children[i] = nullptr; + return Res; + } + + void setNumChildren(unsigned NC) { + if (NC < Children.size()) { + // delete any children we're about to lose pointers to. + for (unsigned i = NC, e = Children.size(); i != e; ++i) + delete Children[i]; + } + Children.resize(NC); + } + + static bool classof(const Matcher *N) { + return N->getKind() == Scope; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { return false; } +}; + +/// RecordMatcher - Save the current node in the operand list. +class RecordMatcher : public Matcher { + /// WhatFor - This is a string indicating why we're recording this. This + /// should only be used for comment generation not anything semantic. + std::string WhatFor; + + /// ResultNo - The slot number in the RecordedNodes vector that this will be, + /// just printed as a comment. + unsigned ResultNo; +public: + RecordMatcher(const std::string &whatfor, unsigned resultNo) + : Matcher(RecordNode), WhatFor(whatfor), ResultNo(resultNo) {} + + const std::string &getWhatFor() const { return WhatFor; } + unsigned getResultNo() const { return ResultNo; } + + static bool classof(const Matcher *N) { + return N->getKind() == RecordNode; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { return true; } +}; + +/// RecordChildMatcher - Save a numbered child of the current node, or fail +/// the match if it doesn't exist. This is logically equivalent to: +/// MoveChild N + RecordNode + MoveParent. +class RecordChildMatcher : public Matcher { + unsigned ChildNo; + + /// WhatFor - This is a string indicating why we're recording this. This + /// should only be used for comment generation not anything semantic. + std::string WhatFor; + + /// ResultNo - The slot number in the RecordedNodes vector that this will be, + /// just printed as a comment. + unsigned ResultNo; +public: + RecordChildMatcher(unsigned childno, const std::string &whatfor, + unsigned resultNo) + : Matcher(RecordChild), ChildNo(childno), WhatFor(whatfor), + ResultNo(resultNo) {} + + unsigned getChildNo() const { return ChildNo; } + const std::string &getWhatFor() const { return WhatFor; } + unsigned getResultNo() const { return ResultNo; } + + static bool classof(const Matcher *N) { + return N->getKind() == RecordChild; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<RecordChildMatcher>(M)->getChildNo() == getChildNo(); + } +}; + +/// RecordMemRefMatcher - Save the current node's memref. +class RecordMemRefMatcher : public Matcher { +public: + RecordMemRefMatcher() : Matcher(RecordMemRef) {} + + static bool classof(const Matcher *N) { + return N->getKind() == RecordMemRef; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { return true; } +}; + + +/// CaptureGlueInputMatcher - If the current record has a glue input, record +/// it so that it is used as an input to the generated code. +class CaptureGlueInputMatcher : public Matcher { +public: + CaptureGlueInputMatcher() : Matcher(CaptureGlueInput) {} + + static bool classof(const Matcher *N) { + return N->getKind() == CaptureGlueInput; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { return true; } +}; + +/// MoveChildMatcher - This tells the interpreter to move into the +/// specified child node. +class MoveChildMatcher : public Matcher { + unsigned ChildNo; +public: + MoveChildMatcher(unsigned childNo) : Matcher(MoveChild), ChildNo(childNo) {} + + unsigned getChildNo() const { return ChildNo; } + + static bool classof(const Matcher *N) { + return N->getKind() == MoveChild; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<MoveChildMatcher>(M)->getChildNo() == getChildNo(); + } +}; + +/// MoveParentMatcher - This tells the interpreter to move to the parent +/// of the current node. +class MoveParentMatcher : public Matcher { +public: + MoveParentMatcher() : Matcher(MoveParent) {} + + static bool classof(const Matcher *N) { + return N->getKind() == MoveParent; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { return true; } +}; + +/// CheckSameMatcher - This checks to see if this node is exactly the same +/// node as the specified match that was recorded with 'Record'. This is used +/// when patterns have the same name in them, like '(mul GPR:$in, GPR:$in)'. +class CheckSameMatcher : public Matcher { + unsigned MatchNumber; +public: + CheckSameMatcher(unsigned matchnumber) + : Matcher(CheckSame), MatchNumber(matchnumber) {} + + unsigned getMatchNumber() const { return MatchNumber; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckSame; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckSameMatcher>(M)->getMatchNumber() == getMatchNumber(); + } +}; + +/// CheckChildSameMatcher - This checks to see if child node is exactly the same +/// node as the specified match that was recorded with 'Record'. This is used +/// when patterns have the same name in them, like '(mul GPR:$in, GPR:$in)'. +class CheckChildSameMatcher : public Matcher { + unsigned ChildNo; + unsigned MatchNumber; +public: + CheckChildSameMatcher(unsigned childno, unsigned matchnumber) + : Matcher(CheckChildSame), ChildNo(childno), MatchNumber(matchnumber) {} + + unsigned getChildNo() const { return ChildNo; } + unsigned getMatchNumber() const { return MatchNumber; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckChildSame; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckChildSameMatcher>(M)->ChildNo == ChildNo && + cast<CheckChildSameMatcher>(M)->MatchNumber == MatchNumber; + } +}; + +/// CheckPatternPredicateMatcher - This checks the target-specific predicate +/// to see if the entire pattern is capable of matching. This predicate does +/// not take a node as input. This is used for subtarget feature checks etc. +class CheckPatternPredicateMatcher : public Matcher { + std::string Predicate; +public: + CheckPatternPredicateMatcher(StringRef predicate) + : Matcher(CheckPatternPredicate), Predicate(predicate) {} + + StringRef getPredicate() const { return Predicate; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckPatternPredicate; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckPatternPredicateMatcher>(M)->getPredicate() == Predicate; + } +}; + +/// CheckPredicateMatcher - This checks the target-specific predicate to +/// see if the node is acceptable. +class CheckPredicateMatcher : public Matcher { + TreePattern *Pred; + const SmallVector<unsigned, 4> Operands; +public: + CheckPredicateMatcher(const TreePredicateFn &pred, + const SmallVectorImpl<unsigned> &Operands); + + TreePredicateFn getPredicate() const; + unsigned getNumOperands() const; + unsigned getOperandNo(unsigned i) const; + + static bool classof(const Matcher *N) { + return N->getKind() == CheckPredicate; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckPredicateMatcher>(M)->Pred == Pred; + } +}; + + +/// CheckOpcodeMatcher - This checks to see if the current node has the +/// specified opcode, if not it fails to match. +class CheckOpcodeMatcher : public Matcher { + const SDNodeInfo &Opcode; +public: + CheckOpcodeMatcher(const SDNodeInfo &opcode) + : Matcher(CheckOpcode), Opcode(opcode) {} + + const SDNodeInfo &getOpcode() const { return Opcode; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckOpcode; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override; + bool isContradictoryImpl(const Matcher *M) const override; +}; + +/// SwitchOpcodeMatcher - Switch based on the current node's opcode, dispatching +/// to one matcher per opcode. If the opcode doesn't match any of the cases, +/// then the match fails. This is semantically equivalent to a Scope node where +/// every child does a CheckOpcode, but is much faster. +class SwitchOpcodeMatcher : public Matcher { + SmallVector<std::pair<const SDNodeInfo*, Matcher*>, 8> Cases; +public: + SwitchOpcodeMatcher(ArrayRef<std::pair<const SDNodeInfo*, Matcher*> > cases) + : Matcher(SwitchOpcode), Cases(cases.begin(), cases.end()) {} + ~SwitchOpcodeMatcher() override; + + static bool classof(const Matcher *N) { + return N->getKind() == SwitchOpcode; + } + + unsigned getNumCases() const { return Cases.size(); } + + const SDNodeInfo &getCaseOpcode(unsigned i) const { return *Cases[i].first; } + Matcher *getCaseMatcher(unsigned i) { return Cases[i].second; } + const Matcher *getCaseMatcher(unsigned i) const { return Cases[i].second; } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { return false; } +}; + +/// CheckTypeMatcher - This checks to see if the current node has the +/// specified type at the specified result, if not it fails to match. +class CheckTypeMatcher : public Matcher { + MVT::SimpleValueType Type; + unsigned ResNo; +public: + CheckTypeMatcher(MVT::SimpleValueType type, unsigned resno) + : Matcher(CheckType), Type(type), ResNo(resno) {} + + MVT::SimpleValueType getType() const { return Type; } + unsigned getResNo() const { return ResNo; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckType; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckTypeMatcher>(M)->Type == Type; + } + bool isContradictoryImpl(const Matcher *M) const override; +}; + +/// SwitchTypeMatcher - Switch based on the current node's type, dispatching +/// to one matcher per case. If the type doesn't match any of the cases, +/// then the match fails. This is semantically equivalent to a Scope node where +/// every child does a CheckType, but is much faster. +class SwitchTypeMatcher : public Matcher { + SmallVector<std::pair<MVT::SimpleValueType, Matcher*>, 8> Cases; +public: + SwitchTypeMatcher(ArrayRef<std::pair<MVT::SimpleValueType, Matcher*> > cases) + : Matcher(SwitchType), Cases(cases.begin(), cases.end()) {} + ~SwitchTypeMatcher() override; + + static bool classof(const Matcher *N) { + return N->getKind() == SwitchType; + } + + unsigned getNumCases() const { return Cases.size(); } + + MVT::SimpleValueType getCaseType(unsigned i) const { return Cases[i].first; } + Matcher *getCaseMatcher(unsigned i) { return Cases[i].second; } + const Matcher *getCaseMatcher(unsigned i) const { return Cases[i].second; } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { return false; } +}; + + +/// CheckChildTypeMatcher - This checks to see if a child node has the +/// specified type, if not it fails to match. +class CheckChildTypeMatcher : public Matcher { + unsigned ChildNo; + MVT::SimpleValueType Type; +public: + CheckChildTypeMatcher(unsigned childno, MVT::SimpleValueType type) + : Matcher(CheckChildType), ChildNo(childno), Type(type) {} + + unsigned getChildNo() const { return ChildNo; } + MVT::SimpleValueType getType() const { return Type; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckChildType; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckChildTypeMatcher>(M)->ChildNo == ChildNo && + cast<CheckChildTypeMatcher>(M)->Type == Type; + } + bool isContradictoryImpl(const Matcher *M) const override; +}; + + +/// CheckIntegerMatcher - This checks to see if the current node is a +/// ConstantSDNode with the specified integer value, if not it fails to match. +class CheckIntegerMatcher : public Matcher { + int64_t Value; +public: + CheckIntegerMatcher(int64_t value) + : Matcher(CheckInteger), Value(value) {} + + int64_t getValue() const { return Value; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckInteger; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckIntegerMatcher>(M)->Value == Value; + } + bool isContradictoryImpl(const Matcher *M) const override; +}; + +/// CheckChildIntegerMatcher - This checks to see if the child node is a +/// ConstantSDNode with a specified integer value, if not it fails to match. +class CheckChildIntegerMatcher : public Matcher { + unsigned ChildNo; + int64_t Value; +public: + CheckChildIntegerMatcher(unsigned childno, int64_t value) + : Matcher(CheckChildInteger), ChildNo(childno), Value(value) {} + + unsigned getChildNo() const { return ChildNo; } + int64_t getValue() const { return Value; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckChildInteger; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckChildIntegerMatcher>(M)->ChildNo == ChildNo && + cast<CheckChildIntegerMatcher>(M)->Value == Value; + } + bool isContradictoryImpl(const Matcher *M) const override; +}; + +/// CheckCondCodeMatcher - This checks to see if the current node is a +/// CondCodeSDNode with the specified condition, if not it fails to match. +class CheckCondCodeMatcher : public Matcher { + StringRef CondCodeName; +public: + CheckCondCodeMatcher(StringRef condcodename) + : Matcher(CheckCondCode), CondCodeName(condcodename) {} + + StringRef getCondCodeName() const { return CondCodeName; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckCondCode; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckCondCodeMatcher>(M)->CondCodeName == CondCodeName; + } +}; + +/// CheckChild2CondCodeMatcher - This checks to see if child 2 node is a +/// CondCodeSDNode with the specified condition, if not it fails to match. +class CheckChild2CondCodeMatcher : public Matcher { + StringRef CondCodeName; +public: + CheckChild2CondCodeMatcher(StringRef condcodename) + : Matcher(CheckChild2CondCode), CondCodeName(condcodename) {} + + StringRef getCondCodeName() const { return CondCodeName; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckChild2CondCode; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckChild2CondCodeMatcher>(M)->CondCodeName == CondCodeName; + } +}; + +/// CheckValueTypeMatcher - This checks to see if the current node is a +/// VTSDNode with the specified type, if not it fails to match. +class CheckValueTypeMatcher : public Matcher { + StringRef TypeName; +public: + CheckValueTypeMatcher(StringRef type_name) + : Matcher(CheckValueType), TypeName(type_name) {} + + StringRef getTypeName() const { return TypeName; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckValueType; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckValueTypeMatcher>(M)->TypeName == TypeName; + } + bool isContradictoryImpl(const Matcher *M) const override; +}; + + + +/// CheckComplexPatMatcher - This node runs the specified ComplexPattern on +/// the current node. +class CheckComplexPatMatcher : public Matcher { + const ComplexPattern &Pattern; + + /// MatchNumber - This is the recorded nodes slot that contains the node we + /// want to match against. + unsigned MatchNumber; + + /// Name - The name of the node we're matching, for comment emission. + std::string Name; + + /// FirstResult - This is the first slot in the RecordedNodes list that the + /// result of the match populates. + unsigned FirstResult; +public: + CheckComplexPatMatcher(const ComplexPattern &pattern, unsigned matchnumber, + const std::string &name, unsigned firstresult) + : Matcher(CheckComplexPat), Pattern(pattern), MatchNumber(matchnumber), + Name(name), FirstResult(firstresult) {} + + const ComplexPattern &getPattern() const { return Pattern; } + unsigned getMatchNumber() const { return MatchNumber; } + + const std::string getName() const { return Name; } + unsigned getFirstResult() const { return FirstResult; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckComplexPat; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return &cast<CheckComplexPatMatcher>(M)->Pattern == &Pattern && + cast<CheckComplexPatMatcher>(M)->MatchNumber == MatchNumber; + } +}; + +/// CheckAndImmMatcher - This checks to see if the current node is an 'and' +/// with something equivalent to the specified immediate. +class CheckAndImmMatcher : public Matcher { + int64_t Value; +public: + CheckAndImmMatcher(int64_t value) + : Matcher(CheckAndImm), Value(value) {} + + int64_t getValue() const { return Value; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckAndImm; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckAndImmMatcher>(M)->Value == Value; + } +}; + +/// CheckOrImmMatcher - This checks to see if the current node is an 'and' +/// with something equivalent to the specified immediate. +class CheckOrImmMatcher : public Matcher { + int64_t Value; +public: + CheckOrImmMatcher(int64_t value) + : Matcher(CheckOrImm), Value(value) {} + + int64_t getValue() const { return Value; } + + static bool classof(const Matcher *N) { + return N->getKind() == CheckOrImm; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CheckOrImmMatcher>(M)->Value == Value; + } +}; + +/// CheckImmAllOnesVMatcher - This check if the current node is an build vector +/// of all ones. +class CheckImmAllOnesVMatcher : public Matcher { +public: + CheckImmAllOnesVMatcher() : Matcher(CheckImmAllOnesV) {} + + static bool classof(const Matcher *N) { + return N->getKind() == CheckImmAllOnesV; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { return true; } + bool isContradictoryImpl(const Matcher *M) const override; +}; + +/// CheckImmAllZerosVMatcher - This check if the current node is an build vector +/// of all zeros. +class CheckImmAllZerosVMatcher : public Matcher { +public: + CheckImmAllZerosVMatcher() : Matcher(CheckImmAllZerosV) {} + + static bool classof(const Matcher *N) { + return N->getKind() == CheckImmAllZerosV; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { return true; } + bool isContradictoryImpl(const Matcher *M) const override; +}; + +/// CheckFoldableChainNodeMatcher - This checks to see if the current node +/// (which defines a chain operand) is safe to fold into a larger pattern. +class CheckFoldableChainNodeMatcher : public Matcher { +public: + CheckFoldableChainNodeMatcher() + : Matcher(CheckFoldableChainNode) {} + + static bool classof(const Matcher *N) { + return N->getKind() == CheckFoldableChainNode; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { return true; } +}; + +/// EmitIntegerMatcher - This creates a new TargetConstant. +class EmitIntegerMatcher : public Matcher { + int64_t Val; + MVT::SimpleValueType VT; +public: + EmitIntegerMatcher(int64_t val, MVT::SimpleValueType vt) + : Matcher(EmitInteger), Val(val), VT(vt) {} + + int64_t getValue() const { return Val; } + MVT::SimpleValueType getVT() const { return VT; } + + static bool classof(const Matcher *N) { + return N->getKind() == EmitInteger; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<EmitIntegerMatcher>(M)->Val == Val && + cast<EmitIntegerMatcher>(M)->VT == VT; + } +}; + +/// EmitStringIntegerMatcher - A target constant whose value is represented +/// by a string. +class EmitStringIntegerMatcher : public Matcher { + std::string Val; + MVT::SimpleValueType VT; +public: + EmitStringIntegerMatcher(const std::string &val, MVT::SimpleValueType vt) + : Matcher(EmitStringInteger), Val(val), VT(vt) {} + + const std::string &getValue() const { return Val; } + MVT::SimpleValueType getVT() const { return VT; } + + static bool classof(const Matcher *N) { + return N->getKind() == EmitStringInteger; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<EmitStringIntegerMatcher>(M)->Val == Val && + cast<EmitStringIntegerMatcher>(M)->VT == VT; + } +}; + +/// EmitRegisterMatcher - This creates a new TargetConstant. +class EmitRegisterMatcher : public Matcher { + /// Reg - The def for the register that we're emitting. If this is null, then + /// this is a reference to zero_reg. + const CodeGenRegister *Reg; + MVT::SimpleValueType VT; +public: + EmitRegisterMatcher(const CodeGenRegister *reg, MVT::SimpleValueType vt) + : Matcher(EmitRegister), Reg(reg), VT(vt) {} + + const CodeGenRegister *getReg() const { return Reg; } + MVT::SimpleValueType getVT() const { return VT; } + + static bool classof(const Matcher *N) { + return N->getKind() == EmitRegister; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<EmitRegisterMatcher>(M)->Reg == Reg && + cast<EmitRegisterMatcher>(M)->VT == VT; + } +}; + +/// EmitConvertToTargetMatcher - Emit an operation that reads a specified +/// recorded node and converts it from being a ISD::Constant to +/// ISD::TargetConstant, likewise for ConstantFP. +class EmitConvertToTargetMatcher : public Matcher { + unsigned Slot; +public: + EmitConvertToTargetMatcher(unsigned slot) + : Matcher(EmitConvertToTarget), Slot(slot) {} + + unsigned getSlot() const { return Slot; } + + static bool classof(const Matcher *N) { + return N->getKind() == EmitConvertToTarget; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<EmitConvertToTargetMatcher>(M)->Slot == Slot; + } +}; + +/// EmitMergeInputChainsMatcher - Emit a node that merges a list of input +/// chains together with a token factor. The list of nodes are the nodes in the +/// matched pattern that have chain input/outputs. This node adds all input +/// chains of these nodes if they are not themselves a node in the pattern. +class EmitMergeInputChainsMatcher : public Matcher { + SmallVector<unsigned, 3> ChainNodes; +public: + EmitMergeInputChainsMatcher(ArrayRef<unsigned> nodes) + : Matcher(EmitMergeInputChains), ChainNodes(nodes.begin(), nodes.end()) {} + + unsigned getNumNodes() const { return ChainNodes.size(); } + + unsigned getNode(unsigned i) const { + assert(i < ChainNodes.size()); + return ChainNodes[i]; + } + + static bool classof(const Matcher *N) { + return N->getKind() == EmitMergeInputChains; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<EmitMergeInputChainsMatcher>(M)->ChainNodes == ChainNodes; + } +}; + +/// EmitCopyToRegMatcher - Emit a CopyToReg node from a value to a physreg, +/// pushing the chain and glue results. +/// +class EmitCopyToRegMatcher : public Matcher { + unsigned SrcSlot; // Value to copy into the physreg. + const CodeGenRegister *DestPhysReg; + +public: + EmitCopyToRegMatcher(unsigned srcSlot, + const CodeGenRegister *destPhysReg) + : Matcher(EmitCopyToReg), SrcSlot(srcSlot), DestPhysReg(destPhysReg) {} + + unsigned getSrcSlot() const { return SrcSlot; } + const CodeGenRegister *getDestPhysReg() const { return DestPhysReg; } + + static bool classof(const Matcher *N) { + return N->getKind() == EmitCopyToReg; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<EmitCopyToRegMatcher>(M)->SrcSlot == SrcSlot && + cast<EmitCopyToRegMatcher>(M)->DestPhysReg == DestPhysReg; + } +}; + + + +/// EmitNodeXFormMatcher - Emit an operation that runs an SDNodeXForm on a +/// recorded node and records the result. +class EmitNodeXFormMatcher : public Matcher { + unsigned Slot; + Record *NodeXForm; +public: + EmitNodeXFormMatcher(unsigned slot, Record *nodeXForm) + : Matcher(EmitNodeXForm), Slot(slot), NodeXForm(nodeXForm) {} + + unsigned getSlot() const { return Slot; } + Record *getNodeXForm() const { return NodeXForm; } + + static bool classof(const Matcher *N) { + return N->getKind() == EmitNodeXForm; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<EmitNodeXFormMatcher>(M)->Slot == Slot && + cast<EmitNodeXFormMatcher>(M)->NodeXForm == NodeXForm; + } +}; + +/// EmitNodeMatcherCommon - Common class shared between EmitNode and +/// MorphNodeTo. +class EmitNodeMatcherCommon : public Matcher { + std::string OpcodeName; + const SmallVector<MVT::SimpleValueType, 3> VTs; + const SmallVector<unsigned, 6> Operands; + bool HasChain, HasInGlue, HasOutGlue, HasMemRefs; + + /// NumFixedArityOperands - If this is a fixed arity node, this is set to -1. + /// If this is a varidic node, this is set to the number of fixed arity + /// operands in the root of the pattern. The rest are appended to this node. + int NumFixedArityOperands; +public: + EmitNodeMatcherCommon(const std::string &opcodeName, + ArrayRef<MVT::SimpleValueType> vts, + ArrayRef<unsigned> operands, + bool hasChain, bool hasInGlue, bool hasOutGlue, + bool hasmemrefs, + int numfixedarityoperands, bool isMorphNodeTo) + : Matcher(isMorphNodeTo ? MorphNodeTo : EmitNode), OpcodeName(opcodeName), + VTs(vts.begin(), vts.end()), Operands(operands.begin(), operands.end()), + HasChain(hasChain), HasInGlue(hasInGlue), HasOutGlue(hasOutGlue), + HasMemRefs(hasmemrefs), NumFixedArityOperands(numfixedarityoperands) {} + + const std::string &getOpcodeName() const { return OpcodeName; } + + unsigned getNumVTs() const { return VTs.size(); } + MVT::SimpleValueType getVT(unsigned i) const { + assert(i < VTs.size()); + return VTs[i]; + } + + unsigned getNumOperands() const { return Operands.size(); } + unsigned getOperand(unsigned i) const { + assert(i < Operands.size()); + return Operands[i]; + } + + const SmallVectorImpl<MVT::SimpleValueType> &getVTList() const { return VTs; } + const SmallVectorImpl<unsigned> &getOperandList() const { return Operands; } + + + bool hasChain() const { return HasChain; } + bool hasInFlag() const { return HasInGlue; } + bool hasOutFlag() const { return HasOutGlue; } + bool hasMemRefs() const { return HasMemRefs; } + int getNumFixedArityOperands() const { return NumFixedArityOperands; } + + static bool classof(const Matcher *N) { + return N->getKind() == EmitNode || N->getKind() == MorphNodeTo; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override; +}; + +/// EmitNodeMatcher - This signals a successful match and generates a node. +class EmitNodeMatcher : public EmitNodeMatcherCommon { + void anchor() override; + unsigned FirstResultSlot; +public: + EmitNodeMatcher(const std::string &opcodeName, + ArrayRef<MVT::SimpleValueType> vts, + ArrayRef<unsigned> operands, + bool hasChain, bool hasInFlag, bool hasOutFlag, + bool hasmemrefs, + int numfixedarityoperands, unsigned firstresultslot) + : EmitNodeMatcherCommon(opcodeName, vts, operands, hasChain, + hasInFlag, hasOutFlag, hasmemrefs, + numfixedarityoperands, false), + FirstResultSlot(firstresultslot) {} + + unsigned getFirstResultSlot() const { return FirstResultSlot; } + + static bool classof(const Matcher *N) { + return N->getKind() == EmitNode; + } + +}; + +class MorphNodeToMatcher : public EmitNodeMatcherCommon { + void anchor() override; + const PatternToMatch &Pattern; +public: + MorphNodeToMatcher(const std::string &opcodeName, + ArrayRef<MVT::SimpleValueType> vts, + ArrayRef<unsigned> operands, + bool hasChain, bool hasInFlag, bool hasOutFlag, + bool hasmemrefs, + int numfixedarityoperands, const PatternToMatch &pattern) + : EmitNodeMatcherCommon(opcodeName, vts, operands, hasChain, + hasInFlag, hasOutFlag, hasmemrefs, + numfixedarityoperands, true), + Pattern(pattern) { + } + + const PatternToMatch &getPattern() const { return Pattern; } + + static bool classof(const Matcher *N) { + return N->getKind() == MorphNodeTo; + } +}; + +/// CompleteMatchMatcher - Complete a match by replacing the results of the +/// pattern with the newly generated nodes. This also prints a comment +/// indicating the source and dest patterns. +class CompleteMatchMatcher : public Matcher { + SmallVector<unsigned, 2> Results; + const PatternToMatch &Pattern; +public: + CompleteMatchMatcher(ArrayRef<unsigned> results, + const PatternToMatch &pattern) + : Matcher(CompleteMatch), Results(results.begin(), results.end()), + Pattern(pattern) {} + + unsigned getNumResults() const { return Results.size(); } + unsigned getResult(unsigned R) const { return Results[R]; } + const PatternToMatch &getPattern() const { return Pattern; } + + static bool classof(const Matcher *N) { + return N->getKind() == CompleteMatch; + } + +private: + void printImpl(raw_ostream &OS, unsigned indent) const override; + bool isEqualImpl(const Matcher *M) const override { + return cast<CompleteMatchMatcher>(M)->Results == Results && + &cast<CompleteMatchMatcher>(M)->Pattern == &Pattern; + } +}; + +} // end namespace llvm + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp new file mode 100644 index 000000000000..d9ec14aab8a8 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -0,0 +1,1134 @@ +//===- DAGISelMatcherEmitter.cpp - Matcher Emitter ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains code to generate C++ code for a matcher. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenDAGPatterns.h" +#include "DAGISelMatcher.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/TinyPtrVector.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +using namespace llvm; + +enum { + IndexWidth = 6, + FullIndexWidth = IndexWidth + 4, + HistOpcWidth = 40, +}; + +cl::OptionCategory DAGISelCat("Options for -gen-dag-isel"); + +// To reduce generated source code size. +static cl::opt<bool> OmitComments("omit-comments", + cl::desc("Do not generate comments"), + cl::init(false), cl::cat(DAGISelCat)); + +static cl::opt<bool> InstrumentCoverage( + "instrument-coverage", + cl::desc("Generates tables to help identify patterns matched"), + cl::init(false), cl::cat(DAGISelCat)); + +namespace { +class MatcherTableEmitter { + const CodeGenDAGPatterns &CGP; + + DenseMap<TreePattern *, unsigned> NodePredicateMap; + std::vector<TreePredicateFn> NodePredicates; + std::vector<TreePredicateFn> NodePredicatesWithOperands; + + // We de-duplicate the predicates by code string, and use this map to track + // all the patterns with "identical" predicates. + StringMap<TinyPtrVector<TreePattern *>> NodePredicatesByCodeToRun; + + StringMap<unsigned> PatternPredicateMap; + std::vector<std::string> PatternPredicates; + + DenseMap<const ComplexPattern*, unsigned> ComplexPatternMap; + std::vector<const ComplexPattern*> ComplexPatterns; + + + DenseMap<Record*, unsigned> NodeXFormMap; + std::vector<Record*> NodeXForms; + + std::vector<std::string> VecIncludeStrings; + MapVector<std::string, unsigned, StringMap<unsigned> > VecPatterns; + + unsigned getPatternIdxFromTable(std::string &&P, std::string &&include_loc) { + const auto It = VecPatterns.find(P); + if (It == VecPatterns.end()) { + VecPatterns.insert(make_pair(std::move(P), VecPatterns.size())); + VecIncludeStrings.push_back(std::move(include_loc)); + return VecIncludeStrings.size() - 1; + } + return It->second; + } + +public: + MatcherTableEmitter(const CodeGenDAGPatterns &cgp) + : CGP(cgp) {} + + unsigned EmitMatcherList(const Matcher *N, unsigned Indent, + unsigned StartIdx, raw_ostream &OS); + + void EmitPredicateFunctions(raw_ostream &OS); + + void EmitHistogram(const Matcher *N, raw_ostream &OS); + + void EmitPatternMatchTable(raw_ostream &OS); + +private: + void EmitNodePredicatesFunction(const std::vector<TreePredicateFn> &Preds, + StringRef Decl, raw_ostream &OS); + + unsigned EmitMatcher(const Matcher *N, unsigned Indent, unsigned CurrentIdx, + raw_ostream &OS); + + unsigned getNodePredicate(TreePredicateFn Pred) { + TreePattern *TP = Pred.getOrigPatFragRecord(); + unsigned &Entry = NodePredicateMap[TP]; + if (Entry == 0) { + TinyPtrVector<TreePattern *> &SameCodePreds = + NodePredicatesByCodeToRun[Pred.getCodeToRunOnSDNode()]; + if (SameCodePreds.empty()) { + // We've never seen a predicate with the same code: allocate an entry. + if (Pred.usesOperands()) { + NodePredicatesWithOperands.push_back(Pred); + Entry = NodePredicatesWithOperands.size(); + } else { + NodePredicates.push_back(Pred); + Entry = NodePredicates.size(); + } + } else { + // We did see an identical predicate: re-use it. + Entry = NodePredicateMap[SameCodePreds.front()]; + assert(Entry != 0); + assert(TreePredicateFn(SameCodePreds.front()).usesOperands() == + Pred.usesOperands() && + "PatFrags with some code must have same usesOperands setting"); + } + // In both cases, we've never seen this particular predicate before, so + // mark it in the list of predicates sharing the same code. + SameCodePreds.push_back(TP); + } + return Entry-1; + } + + unsigned getPatternPredicate(StringRef PredName) { + unsigned &Entry = PatternPredicateMap[PredName]; + if (Entry == 0) { + PatternPredicates.push_back(PredName.str()); + Entry = PatternPredicates.size(); + } + return Entry-1; + } + unsigned getComplexPat(const ComplexPattern &P) { + unsigned &Entry = ComplexPatternMap[&P]; + if (Entry == 0) { + ComplexPatterns.push_back(&P); + Entry = ComplexPatterns.size(); + } + return Entry-1; + } + + unsigned getNodeXFormID(Record *Rec) { + unsigned &Entry = NodeXFormMap[Rec]; + if (Entry == 0) { + NodeXForms.push_back(Rec); + Entry = NodeXForms.size(); + } + return Entry-1; + } + +}; +} // end anonymous namespace. + +static std::string GetPatFromTreePatternNode(const TreePatternNode *N) { + std::string str; + raw_string_ostream Stream(str); + Stream << *N; + Stream.str(); + return str; +} + +static unsigned GetVBRSize(unsigned Val) { + if (Val <= 127) return 1; + + unsigned NumBytes = 0; + while (Val >= 128) { + Val >>= 7; + ++NumBytes; + } + return NumBytes+1; +} + +/// EmitVBRValue - Emit the specified value as a VBR, returning the number of +/// bytes emitted. +static uint64_t EmitVBRValue(uint64_t Val, raw_ostream &OS) { + if (Val <= 127) { + OS << Val << ", "; + return 1; + } + + uint64_t InVal = Val; + unsigned NumBytes = 0; + while (Val >= 128) { + OS << (Val&127) << "|128,"; + Val >>= 7; + ++NumBytes; + } + OS << Val; + if (!OmitComments) + OS << "/*" << InVal << "*/"; + OS << ", "; + return NumBytes+1; +} + +// This is expensive and slow. +static std::string getIncludePath(const Record *R) { + std::string str; + raw_string_ostream Stream(str); + auto Locs = R->getLoc(); + SMLoc L; + if (Locs.size() > 1) { + // Get where the pattern prototype was instantiated + L = Locs[1]; + } else if (Locs.size() == 1) { + L = Locs[0]; + } + unsigned CurBuf = SrcMgr.FindBufferContainingLoc(L); + assert(CurBuf && "Invalid or unspecified location!"); + + Stream << SrcMgr.getBufferInfo(CurBuf).Buffer->getBufferIdentifier() << ":" + << SrcMgr.FindLineNumber(L, CurBuf); + Stream.str(); + return str; +} + +static void BeginEmitFunction(raw_ostream &OS, StringRef RetType, + StringRef Decl, bool AddOverride) { + OS << "#ifdef GET_DAGISEL_DECL\n"; + OS << RetType << ' ' << Decl; + if (AddOverride) + OS << " override"; + OS << ";\n" + "#endif\n" + "#if defined(GET_DAGISEL_BODY) || DAGISEL_INLINE\n"; + OS << RetType << " DAGISEL_CLASS_COLONCOLON " << Decl << "\n"; + if (AddOverride) { + OS << "#if DAGISEL_INLINE\n" + " override\n" + "#endif\n"; + } +} + +static void EndEmitFunction(raw_ostream &OS) { + OS << "#endif // GET_DAGISEL_BODY\n\n"; +} + +void MatcherTableEmitter::EmitPatternMatchTable(raw_ostream &OS) { + + assert(isUInt<16>(VecPatterns.size()) && + "Using only 16 bits to encode offset into Pattern Table"); + assert(VecPatterns.size() == VecIncludeStrings.size() && + "The sizes of Pattern and include vectors should be the same"); + + BeginEmitFunction(OS, "StringRef", "getPatternForIndex(unsigned Index)", + true/*AddOverride*/); + OS << "{\n"; + OS << "static const char * PATTERN_MATCH_TABLE[] = {\n"; + + for (const auto &It : VecPatterns) { + OS << "\"" << It.first << "\",\n"; + } + + OS << "\n};"; + OS << "\nreturn StringRef(PATTERN_MATCH_TABLE[Index]);"; + OS << "\n}\n"; + EndEmitFunction(OS); + + BeginEmitFunction(OS, "StringRef", "getIncludePathForIndex(unsigned Index)", + true/*AddOverride*/); + OS << "{\n"; + OS << "static const char * INCLUDE_PATH_TABLE[] = {\n"; + + for (const auto &It : VecIncludeStrings) { + OS << "\"" << It << "\",\n"; + } + + OS << "\n};"; + OS << "\nreturn StringRef(INCLUDE_PATH_TABLE[Index]);"; + OS << "\n}\n"; + EndEmitFunction(OS); +} + +/// EmitMatcher - Emit bytes for the specified matcher and return +/// the number of bytes emitted. +unsigned MatcherTableEmitter:: +EmitMatcher(const Matcher *N, unsigned Indent, unsigned CurrentIdx, + raw_ostream &OS) { + OS.indent(Indent); + + switch (N->getKind()) { + case Matcher::Scope: { + const ScopeMatcher *SM = cast<ScopeMatcher>(N); + assert(SM->getNext() == nullptr && "Shouldn't have next after scope"); + + unsigned StartIdx = CurrentIdx; + + // Emit all of the children. + SmallString<128> TmpBuf; + for (unsigned i = 0, e = SM->getNumChildren(); i != e; ++i) { + if (i == 0) { + OS << "OPC_Scope, "; + ++CurrentIdx; + } else { + if (!OmitComments) { + OS << "/*" << format_decimal(CurrentIdx, IndexWidth) << "*/"; + OS.indent(Indent) << "/*Scope*/ "; + } else + OS.indent(Indent); + } + + // We need to encode the child and the offset of the failure code before + // emitting either of them. Handle this by buffering the output into a + // string while we get the size. Unfortunately, the offset of the + // children depends on the VBR size of the child, so for large children we + // have to iterate a bit. + unsigned ChildSize = 0; + unsigned VBRSize = 0; + do { + VBRSize = GetVBRSize(ChildSize); + + TmpBuf.clear(); + raw_svector_ostream OS(TmpBuf); + ChildSize = EmitMatcherList(SM->getChild(i), Indent+1, + CurrentIdx+VBRSize, OS); + } while (GetVBRSize(ChildSize) != VBRSize); + + assert(ChildSize != 0 && "Should not have a zero-sized child!"); + + CurrentIdx += EmitVBRValue(ChildSize, OS); + if (!OmitComments) { + OS << "/*->" << CurrentIdx+ChildSize << "*/"; + + if (i == 0) + OS << " // " << SM->getNumChildren() << " children in Scope"; + } + + OS << '\n' << TmpBuf; + CurrentIdx += ChildSize; + } + + // Emit a zero as a sentinel indicating end of 'Scope'. + if (!OmitComments) + OS << "/*" << format_decimal(CurrentIdx, IndexWidth) << "*/"; + OS.indent(Indent) << "0, "; + if (!OmitComments) + OS << "/*End of Scope*/"; + OS << '\n'; + return CurrentIdx - StartIdx + 1; + } + + case Matcher::RecordNode: + OS << "OPC_RecordNode,"; + if (!OmitComments) + OS << " // #" + << cast<RecordMatcher>(N)->getResultNo() << " = " + << cast<RecordMatcher>(N)->getWhatFor(); + OS << '\n'; + return 1; + + case Matcher::RecordChild: + OS << "OPC_RecordChild" << cast<RecordChildMatcher>(N)->getChildNo() + << ','; + if (!OmitComments) + OS << " // #" + << cast<RecordChildMatcher>(N)->getResultNo() << " = " + << cast<RecordChildMatcher>(N)->getWhatFor(); + OS << '\n'; + return 1; + + case Matcher::RecordMemRef: + OS << "OPC_RecordMemRef,\n"; + return 1; + + case Matcher::CaptureGlueInput: + OS << "OPC_CaptureGlueInput,\n"; + return 1; + + case Matcher::MoveChild: { + const auto *MCM = cast<MoveChildMatcher>(N); + + OS << "OPC_MoveChild"; + // Handle the specialized forms. + if (MCM->getChildNo() >= 8) + OS << ", "; + OS << MCM->getChildNo() << ",\n"; + return (MCM->getChildNo() >= 8) ? 2 : 1; + } + + case Matcher::MoveParent: + OS << "OPC_MoveParent,\n"; + return 1; + + case Matcher::CheckSame: + OS << "OPC_CheckSame, " + << cast<CheckSameMatcher>(N)->getMatchNumber() << ",\n"; + return 2; + + case Matcher::CheckChildSame: + OS << "OPC_CheckChild" + << cast<CheckChildSameMatcher>(N)->getChildNo() << "Same, " + << cast<CheckChildSameMatcher>(N)->getMatchNumber() << ",\n"; + return 2; + + case Matcher::CheckPatternPredicate: { + StringRef Pred =cast<CheckPatternPredicateMatcher>(N)->getPredicate(); + OS << "OPC_CheckPatternPredicate, " << getPatternPredicate(Pred) << ','; + if (!OmitComments) + OS << " // " << Pred; + OS << '\n'; + return 2; + } + case Matcher::CheckPredicate: { + TreePredicateFn Pred = cast<CheckPredicateMatcher>(N)->getPredicate(); + unsigned OperandBytes = 0; + + if (Pred.usesOperands()) { + unsigned NumOps = cast<CheckPredicateMatcher>(N)->getNumOperands(); + OS << "OPC_CheckPredicateWithOperands, " << NumOps << "/*#Ops*/, "; + for (unsigned i = 0; i < NumOps; ++i) + OS << cast<CheckPredicateMatcher>(N)->getOperandNo(i) << ", "; + OperandBytes = 1 + NumOps; + } else { + OS << "OPC_CheckPredicate, "; + } + + OS << getNodePredicate(Pred) << ','; + if (!OmitComments) + OS << " // " << Pred.getFnName(); + OS << '\n'; + return 2 + OperandBytes; + } + + case Matcher::CheckOpcode: + OS << "OPC_CheckOpcode, TARGET_VAL(" + << cast<CheckOpcodeMatcher>(N)->getOpcode().getEnumName() << "),\n"; + return 3; + + case Matcher::SwitchOpcode: + case Matcher::SwitchType: { + unsigned StartIdx = CurrentIdx; + + unsigned NumCases; + if (const SwitchOpcodeMatcher *SOM = dyn_cast<SwitchOpcodeMatcher>(N)) { + OS << "OPC_SwitchOpcode "; + NumCases = SOM->getNumCases(); + } else { + OS << "OPC_SwitchType "; + NumCases = cast<SwitchTypeMatcher>(N)->getNumCases(); + } + + if (!OmitComments) + OS << "/*" << NumCases << " cases */"; + OS << ", "; + ++CurrentIdx; + + // For each case we emit the size, then the opcode, then the matcher. + SmallString<128> TmpBuf; + for (unsigned i = 0, e = NumCases; i != e; ++i) { + const Matcher *Child; + unsigned IdxSize; + if (const SwitchOpcodeMatcher *SOM = dyn_cast<SwitchOpcodeMatcher>(N)) { + Child = SOM->getCaseMatcher(i); + IdxSize = 2; // size of opcode in table is 2 bytes. + } else { + Child = cast<SwitchTypeMatcher>(N)->getCaseMatcher(i); + IdxSize = 1; // size of type in table is 1 byte. + } + + // We need to encode the opcode and the offset of the case code before + // emitting the case code. Handle this by buffering the output into a + // string while we get the size. Unfortunately, the offset of the + // children depends on the VBR size of the child, so for large children we + // have to iterate a bit. + unsigned ChildSize = 0; + unsigned VBRSize = 0; + do { + VBRSize = GetVBRSize(ChildSize); + + TmpBuf.clear(); + raw_svector_ostream OS(TmpBuf); + ChildSize = EmitMatcherList(Child, Indent+1, CurrentIdx+VBRSize+IdxSize, + OS); + } while (GetVBRSize(ChildSize) != VBRSize); + + assert(ChildSize != 0 && "Should not have a zero-sized child!"); + + if (i != 0) { + if (!OmitComments) + OS << "/*" << format_decimal(CurrentIdx, IndexWidth) << "*/"; + OS.indent(Indent); + if (!OmitComments) + OS << (isa<SwitchOpcodeMatcher>(N) ? + "/*SwitchOpcode*/ " : "/*SwitchType*/ "); + } + + // Emit the VBR. + CurrentIdx += EmitVBRValue(ChildSize, OS); + + if (const SwitchOpcodeMatcher *SOM = dyn_cast<SwitchOpcodeMatcher>(N)) + OS << "TARGET_VAL(" << SOM->getCaseOpcode(i).getEnumName() << "),"; + else + OS << getEnumName(cast<SwitchTypeMatcher>(N)->getCaseType(i)) << ','; + + CurrentIdx += IdxSize; + + if (!OmitComments) + OS << "// ->" << CurrentIdx+ChildSize; + OS << '\n'; + OS << TmpBuf; + CurrentIdx += ChildSize; + } + + // Emit the final zero to terminate the switch. + if (!OmitComments) + OS << "/*" << format_decimal(CurrentIdx, IndexWidth) << "*/"; + OS.indent(Indent) << "0,"; + if (!OmitComments) + OS << (isa<SwitchOpcodeMatcher>(N) ? + " // EndSwitchOpcode" : " // EndSwitchType"); + + OS << '\n'; + ++CurrentIdx; + return CurrentIdx-StartIdx; + } + + case Matcher::CheckType: + if (cast<CheckTypeMatcher>(N)->getResNo() == 0) { + OS << "OPC_CheckType, " + << getEnumName(cast<CheckTypeMatcher>(N)->getType()) << ",\n"; + return 2; + } + OS << "OPC_CheckTypeRes, " << cast<CheckTypeMatcher>(N)->getResNo() + << ", " << getEnumName(cast<CheckTypeMatcher>(N)->getType()) << ",\n"; + return 3; + + case Matcher::CheckChildType: + OS << "OPC_CheckChild" + << cast<CheckChildTypeMatcher>(N)->getChildNo() << "Type, " + << getEnumName(cast<CheckChildTypeMatcher>(N)->getType()) << ",\n"; + return 2; + + case Matcher::CheckInteger: { + OS << "OPC_CheckInteger, "; + unsigned Bytes=1+EmitVBRValue(cast<CheckIntegerMatcher>(N)->getValue(), OS); + OS << '\n'; + return Bytes; + } + case Matcher::CheckChildInteger: { + OS << "OPC_CheckChild" << cast<CheckChildIntegerMatcher>(N)->getChildNo() + << "Integer, "; + unsigned Bytes=1+EmitVBRValue(cast<CheckChildIntegerMatcher>(N)->getValue(), + OS); + OS << '\n'; + return Bytes; + } + case Matcher::CheckCondCode: + OS << "OPC_CheckCondCode, ISD::" + << cast<CheckCondCodeMatcher>(N)->getCondCodeName() << ",\n"; + return 2; + + case Matcher::CheckChild2CondCode: + OS << "OPC_CheckChild2CondCode, ISD::" + << cast<CheckChild2CondCodeMatcher>(N)->getCondCodeName() << ",\n"; + return 2; + + case Matcher::CheckValueType: + OS << "OPC_CheckValueType, MVT::" + << cast<CheckValueTypeMatcher>(N)->getTypeName() << ",\n"; + return 2; + + case Matcher::CheckComplexPat: { + const CheckComplexPatMatcher *CCPM = cast<CheckComplexPatMatcher>(N); + const ComplexPattern &Pattern = CCPM->getPattern(); + OS << "OPC_CheckComplexPat, /*CP*/" << getComplexPat(Pattern) << ", /*#*/" + << CCPM->getMatchNumber() << ','; + + if (!OmitComments) { + OS << " // " << Pattern.getSelectFunc(); + OS << ":$" << CCPM->getName(); + for (unsigned i = 0, e = Pattern.getNumOperands(); i != e; ++i) + OS << " #" << CCPM->getFirstResult()+i; + + if (Pattern.hasProperty(SDNPHasChain)) + OS << " + chain result"; + } + OS << '\n'; + return 3; + } + + case Matcher::CheckAndImm: { + OS << "OPC_CheckAndImm, "; + unsigned Bytes=1+EmitVBRValue(cast<CheckAndImmMatcher>(N)->getValue(), OS); + OS << '\n'; + return Bytes; + } + + case Matcher::CheckOrImm: { + OS << "OPC_CheckOrImm, "; + unsigned Bytes = 1+EmitVBRValue(cast<CheckOrImmMatcher>(N)->getValue(), OS); + OS << '\n'; + return Bytes; + } + + case Matcher::CheckFoldableChainNode: + OS << "OPC_CheckFoldableChainNode,\n"; + return 1; + + case Matcher::CheckImmAllOnesV: + OS << "OPC_CheckImmAllOnesV,\n"; + return 1; + + case Matcher::CheckImmAllZerosV: + OS << "OPC_CheckImmAllZerosV,\n"; + return 1; + + case Matcher::EmitInteger: { + int64_t Val = cast<EmitIntegerMatcher>(N)->getValue(); + OS << "OPC_EmitInteger, " + << getEnumName(cast<EmitIntegerMatcher>(N)->getVT()) << ", "; + unsigned Bytes = 2+EmitVBRValue(Val, OS); + OS << '\n'; + return Bytes; + } + case Matcher::EmitStringInteger: { + const std::string &Val = cast<EmitStringIntegerMatcher>(N)->getValue(); + // These should always fit into 7 bits. + OS << "OPC_EmitInteger, " + << getEnumName(cast<EmitStringIntegerMatcher>(N)->getVT()) << ", " + << Val << ",\n"; + return 3; + } + + case Matcher::EmitRegister: { + const EmitRegisterMatcher *Matcher = cast<EmitRegisterMatcher>(N); + const CodeGenRegister *Reg = Matcher->getReg(); + // If the enum value of the register is larger than one byte can handle, + // use EmitRegister2. + if (Reg && Reg->EnumValue > 255) { + OS << "OPC_EmitRegister2, " << getEnumName(Matcher->getVT()) << ", "; + OS << "TARGET_VAL(" << getQualifiedName(Reg->TheDef) << "),\n"; + return 4; + } else { + OS << "OPC_EmitRegister, " << getEnumName(Matcher->getVT()) << ", "; + if (Reg) { + OS << getQualifiedName(Reg->TheDef) << ",\n"; + } else { + OS << "0 "; + if (!OmitComments) + OS << "/*zero_reg*/"; + OS << ",\n"; + } + return 3; + } + } + + case Matcher::EmitConvertToTarget: + OS << "OPC_EmitConvertToTarget, " + << cast<EmitConvertToTargetMatcher>(N)->getSlot() << ",\n"; + return 2; + + case Matcher::EmitMergeInputChains: { + const EmitMergeInputChainsMatcher *MN = + cast<EmitMergeInputChainsMatcher>(N); + + // Handle the specialized forms OPC_EmitMergeInputChains1_0, 1_1, and 1_2. + if (MN->getNumNodes() == 1 && MN->getNode(0) < 3) { + OS << "OPC_EmitMergeInputChains1_" << MN->getNode(0) << ",\n"; + return 1; + } + + OS << "OPC_EmitMergeInputChains, " << MN->getNumNodes() << ", "; + for (unsigned i = 0, e = MN->getNumNodes(); i != e; ++i) + OS << MN->getNode(i) << ", "; + OS << '\n'; + return 2+MN->getNumNodes(); + } + case Matcher::EmitCopyToReg: { + const auto *C2RMatcher = cast<EmitCopyToRegMatcher>(N); + int Bytes = 3; + const CodeGenRegister *Reg = C2RMatcher->getDestPhysReg(); + if (Reg->EnumValue > 255) { + assert(isUInt<16>(Reg->EnumValue) && "not handled"); + OS << "OPC_EmitCopyToReg2, " << C2RMatcher->getSrcSlot() << ", " + << "TARGET_VAL(" << getQualifiedName(Reg->TheDef) << "),\n"; + ++Bytes; + } else { + OS << "OPC_EmitCopyToReg, " << C2RMatcher->getSrcSlot() << ", " + << getQualifiedName(Reg->TheDef) << ",\n"; + } + + return Bytes; + } + case Matcher::EmitNodeXForm: { + const EmitNodeXFormMatcher *XF = cast<EmitNodeXFormMatcher>(N); + OS << "OPC_EmitNodeXForm, " << getNodeXFormID(XF->getNodeXForm()) << ", " + << XF->getSlot() << ','; + if (!OmitComments) + OS << " // "<<XF->getNodeXForm()->getName(); + OS <<'\n'; + return 3; + } + + case Matcher::EmitNode: + case Matcher::MorphNodeTo: { + auto NumCoveredBytes = 0; + if (InstrumentCoverage) { + if (const MorphNodeToMatcher *SNT = dyn_cast<MorphNodeToMatcher>(N)) { + NumCoveredBytes = 3; + OS << "OPC_Coverage, "; + std::string src = + GetPatFromTreePatternNode(SNT->getPattern().getSrcPattern()); + std::string dst = + GetPatFromTreePatternNode(SNT->getPattern().getDstPattern()); + Record *PatRecord = SNT->getPattern().getSrcRecord(); + std::string include_src = getIncludePath(PatRecord); + unsigned Offset = + getPatternIdxFromTable(src + " -> " + dst, std::move(include_src)); + OS << "TARGET_VAL(" << Offset << "),\n"; + OS.indent(FullIndexWidth + Indent); + } + } + const EmitNodeMatcherCommon *EN = cast<EmitNodeMatcherCommon>(N); + OS << (isa<EmitNodeMatcher>(EN) ? "OPC_EmitNode" : "OPC_MorphNodeTo"); + bool CompressVTs = EN->getNumVTs() < 3; + if (CompressVTs) + OS << EN->getNumVTs(); + + OS << ", TARGET_VAL(" << EN->getOpcodeName() << "), 0"; + + if (EN->hasChain()) OS << "|OPFL_Chain"; + if (EN->hasInFlag()) OS << "|OPFL_GlueInput"; + if (EN->hasOutFlag()) OS << "|OPFL_GlueOutput"; + if (EN->hasMemRefs()) OS << "|OPFL_MemRefs"; + if (EN->getNumFixedArityOperands() != -1) + OS << "|OPFL_Variadic" << EN->getNumFixedArityOperands(); + OS << ",\n"; + + OS.indent(FullIndexWidth + Indent+4); + if (!CompressVTs) { + OS << EN->getNumVTs(); + if (!OmitComments) + OS << "/*#VTs*/"; + OS << ", "; + } + for (unsigned i = 0, e = EN->getNumVTs(); i != e; ++i) + OS << getEnumName(EN->getVT(i)) << ", "; + + OS << EN->getNumOperands(); + if (!OmitComments) + OS << "/*#Ops*/"; + OS << ", "; + unsigned NumOperandBytes = 0; + for (unsigned i = 0, e = EN->getNumOperands(); i != e; ++i) + NumOperandBytes += EmitVBRValue(EN->getOperand(i), OS); + + if (!OmitComments) { + // Print the result #'s for EmitNode. + if (const EmitNodeMatcher *E = dyn_cast<EmitNodeMatcher>(EN)) { + if (unsigned NumResults = EN->getNumVTs()) { + OS << " // Results ="; + unsigned First = E->getFirstResultSlot(); + for (unsigned i = 0; i != NumResults; ++i) + OS << " #" << First+i; + } + } + OS << '\n'; + + if (const MorphNodeToMatcher *SNT = dyn_cast<MorphNodeToMatcher>(N)) { + OS.indent(FullIndexWidth + Indent) << "// Src: " + << *SNT->getPattern().getSrcPattern() << " - Complexity = " + << SNT->getPattern().getPatternComplexity(CGP) << '\n'; + OS.indent(FullIndexWidth + Indent) << "// Dst: " + << *SNT->getPattern().getDstPattern() << '\n'; + } + } else + OS << '\n'; + + return 5 + !CompressVTs + EN->getNumVTs() + NumOperandBytes + + NumCoveredBytes; + } + case Matcher::CompleteMatch: { + const CompleteMatchMatcher *CM = cast<CompleteMatchMatcher>(N); + auto NumCoveredBytes = 0; + if (InstrumentCoverage) { + NumCoveredBytes = 3; + OS << "OPC_Coverage, "; + std::string src = + GetPatFromTreePatternNode(CM->getPattern().getSrcPattern()); + std::string dst = + GetPatFromTreePatternNode(CM->getPattern().getDstPattern()); + Record *PatRecord = CM->getPattern().getSrcRecord(); + std::string include_src = getIncludePath(PatRecord); + unsigned Offset = + getPatternIdxFromTable(src + " -> " + dst, std::move(include_src)); + OS << "TARGET_VAL(" << Offset << "),\n"; + OS.indent(FullIndexWidth + Indent); + } + OS << "OPC_CompleteMatch, " << CM->getNumResults() << ", "; + unsigned NumResultBytes = 0; + for (unsigned i = 0, e = CM->getNumResults(); i != e; ++i) + NumResultBytes += EmitVBRValue(CM->getResult(i), OS); + OS << '\n'; + if (!OmitComments) { + OS.indent(FullIndexWidth + Indent) << " // Src: " + << *CM->getPattern().getSrcPattern() << " - Complexity = " + << CM->getPattern().getPatternComplexity(CGP) << '\n'; + OS.indent(FullIndexWidth + Indent) << " // Dst: " + << *CM->getPattern().getDstPattern(); + } + OS << '\n'; + return 2 + NumResultBytes + NumCoveredBytes; + } + } + llvm_unreachable("Unreachable"); +} + +/// EmitMatcherList - Emit the bytes for the specified matcher subtree. +unsigned MatcherTableEmitter:: +EmitMatcherList(const Matcher *N, unsigned Indent, unsigned CurrentIdx, + raw_ostream &OS) { + unsigned Size = 0; + while (N) { + if (!OmitComments) + OS << "/*" << format_decimal(CurrentIdx, IndexWidth) << "*/"; + unsigned MatcherSize = EmitMatcher(N, Indent, CurrentIdx, OS); + Size += MatcherSize; + CurrentIdx += MatcherSize; + + // If there are other nodes in this list, iterate to them, otherwise we're + // done. + N = N->getNext(); + } + return Size; +} + +void MatcherTableEmitter::EmitNodePredicatesFunction( + const std::vector<TreePredicateFn> &Preds, StringRef Decl, + raw_ostream &OS) { + if (Preds.empty()) + return; + + BeginEmitFunction(OS, "bool", Decl, true/*AddOverride*/); + OS << "{\n"; + OS << " switch (PredNo) {\n"; + OS << " default: llvm_unreachable(\"Invalid predicate in table?\");\n"; + for (unsigned i = 0, e = Preds.size(); i != e; ++i) { + // Emit the predicate code corresponding to this pattern. + TreePredicateFn PredFn = Preds[i]; + + assert(!PredFn.isAlwaysTrue() && "No code in this predicate"); + OS << " case " << i << ": { \n"; + for (auto *SimilarPred : + NodePredicatesByCodeToRun[PredFn.getCodeToRunOnSDNode()]) + OS << " // " << TreePredicateFn(SimilarPred).getFnName() <<'\n'; + + OS << PredFn.getCodeToRunOnSDNode() << "\n }\n"; + } + OS << " }\n"; + OS << "}\n"; + EndEmitFunction(OS); +} + +void MatcherTableEmitter::EmitPredicateFunctions(raw_ostream &OS) { + // Emit pattern predicates. + if (!PatternPredicates.empty()) { + BeginEmitFunction(OS, "bool", + "CheckPatternPredicate(unsigned PredNo) const", true/*AddOverride*/); + OS << "{\n"; + OS << " switch (PredNo) {\n"; + OS << " default: llvm_unreachable(\"Invalid predicate in table?\");\n"; + for (unsigned i = 0, e = PatternPredicates.size(); i != e; ++i) + OS << " case " << i << ": return " << PatternPredicates[i] << ";\n"; + OS << " }\n"; + OS << "}\n"; + EndEmitFunction(OS); + } + + // Emit Node predicates. + EmitNodePredicatesFunction( + NodePredicates, "CheckNodePredicate(SDNode *Node, unsigned PredNo) const", + OS); + EmitNodePredicatesFunction( + NodePredicatesWithOperands, + "CheckNodePredicateWithOperands(SDNode *Node, unsigned PredNo, " + "const SmallVectorImpl<SDValue> &Operands) const", + OS); + + // Emit CompletePattern matchers. + // FIXME: This should be const. + if (!ComplexPatterns.empty()) { + BeginEmitFunction(OS, "bool", + "CheckComplexPattern(SDNode *Root, SDNode *Parent,\n" + " SDValue N, unsigned PatternNo,\n" + " SmallVectorImpl<std::pair<SDValue, SDNode*>> &Result)", + true/*AddOverride*/); + OS << "{\n"; + OS << " unsigned NextRes = Result.size();\n"; + OS << " switch (PatternNo) {\n"; + OS << " default: llvm_unreachable(\"Invalid pattern # in table?\");\n"; + for (unsigned i = 0, e = ComplexPatterns.size(); i != e; ++i) { + const ComplexPattern &P = *ComplexPatterns[i]; + unsigned NumOps = P.getNumOperands(); + + if (P.hasProperty(SDNPHasChain)) + ++NumOps; // Get the chained node too. + + OS << " case " << i << ":\n"; + if (InstrumentCoverage) + OS << " {\n"; + OS << " Result.resize(NextRes+" << NumOps << ");\n"; + if (InstrumentCoverage) + OS << " bool Succeeded = " << P.getSelectFunc(); + else + OS << " return " << P.getSelectFunc(); + + OS << "("; + // If the complex pattern wants the root of the match, pass it in as the + // first argument. + if (P.hasProperty(SDNPWantRoot)) + OS << "Root, "; + + // If the complex pattern wants the parent of the operand being matched, + // pass it in as the next argument. + if (P.hasProperty(SDNPWantParent)) + OS << "Parent, "; + + OS << "N"; + for (unsigned i = 0; i != NumOps; ++i) + OS << ", Result[NextRes+" << i << "].first"; + OS << ");\n"; + if (InstrumentCoverage) { + OS << " if (Succeeded)\n"; + OS << " dbgs() << \"\\nCOMPLEX_PATTERN: " << P.getSelectFunc() + << "\\n\" ;\n"; + OS << " return Succeeded;\n"; + OS << " }\n"; + } + } + OS << " }\n"; + OS << "}\n"; + EndEmitFunction(OS); + } + + + // Emit SDNodeXForm handlers. + // FIXME: This should be const. + if (!NodeXForms.empty()) { + BeginEmitFunction(OS, "SDValue", + "RunSDNodeXForm(SDValue V, unsigned XFormNo)", true/*AddOverride*/); + OS << "{\n"; + OS << " switch (XFormNo) {\n"; + OS << " default: llvm_unreachable(\"Invalid xform # in table?\");\n"; + + // FIXME: The node xform could take SDValue's instead of SDNode*'s. + for (unsigned i = 0, e = NodeXForms.size(); i != e; ++i) { + const CodeGenDAGPatterns::NodeXForm &Entry = + CGP.getSDNodeTransform(NodeXForms[i]); + + Record *SDNode = Entry.first; + const std::string &Code = Entry.second; + + OS << " case " << i << ": { "; + if (!OmitComments) + OS << "// " << NodeXForms[i]->getName(); + OS << '\n'; + + std::string ClassName = + std::string(CGP.getSDNodeInfo(SDNode).getSDClassName()); + if (ClassName == "SDNode") + OS << " SDNode *N = V.getNode();\n"; + else + OS << " " << ClassName << " *N = cast<" << ClassName + << ">(V.getNode());\n"; + OS << Code << "\n }\n"; + } + OS << " }\n"; + OS << "}\n"; + EndEmitFunction(OS); + } +} + +static void BuildHistogram(const Matcher *M, std::vector<unsigned> &OpcodeFreq){ + for (; M != nullptr; M = M->getNext()) { + // Count this node. + if (unsigned(M->getKind()) >= OpcodeFreq.size()) + OpcodeFreq.resize(M->getKind()+1); + OpcodeFreq[M->getKind()]++; + + // Handle recursive nodes. + if (const ScopeMatcher *SM = dyn_cast<ScopeMatcher>(M)) { + for (unsigned i = 0, e = SM->getNumChildren(); i != e; ++i) + BuildHistogram(SM->getChild(i), OpcodeFreq); + } else if (const SwitchOpcodeMatcher *SOM = + dyn_cast<SwitchOpcodeMatcher>(M)) { + for (unsigned i = 0, e = SOM->getNumCases(); i != e; ++i) + BuildHistogram(SOM->getCaseMatcher(i), OpcodeFreq); + } else if (const SwitchTypeMatcher *STM = dyn_cast<SwitchTypeMatcher>(M)) { + for (unsigned i = 0, e = STM->getNumCases(); i != e; ++i) + BuildHistogram(STM->getCaseMatcher(i), OpcodeFreq); + } + } +} + +static StringRef getOpcodeString(Matcher::KindTy Kind) { + switch (Kind) { + case Matcher::Scope: return "OPC_Scope"; break; + case Matcher::RecordNode: return "OPC_RecordNode"; break; + case Matcher::RecordChild: return "OPC_RecordChild"; break; + case Matcher::RecordMemRef: return "OPC_RecordMemRef"; break; + case Matcher::CaptureGlueInput: return "OPC_CaptureGlueInput"; break; + case Matcher::MoveChild: return "OPC_MoveChild"; break; + case Matcher::MoveParent: return "OPC_MoveParent"; break; + case Matcher::CheckSame: return "OPC_CheckSame"; break; + case Matcher::CheckChildSame: return "OPC_CheckChildSame"; break; + case Matcher::CheckPatternPredicate: + return "OPC_CheckPatternPredicate"; break; + case Matcher::CheckPredicate: return "OPC_CheckPredicate"; break; + case Matcher::CheckOpcode: return "OPC_CheckOpcode"; break; + case Matcher::SwitchOpcode: return "OPC_SwitchOpcode"; break; + case Matcher::CheckType: return "OPC_CheckType"; break; + case Matcher::SwitchType: return "OPC_SwitchType"; break; + case Matcher::CheckChildType: return "OPC_CheckChildType"; break; + case Matcher::CheckInteger: return "OPC_CheckInteger"; break; + case Matcher::CheckChildInteger: return "OPC_CheckChildInteger"; break; + case Matcher::CheckCondCode: return "OPC_CheckCondCode"; break; + case Matcher::CheckChild2CondCode: return "OPC_CheckChild2CondCode"; break; + case Matcher::CheckValueType: return "OPC_CheckValueType"; break; + case Matcher::CheckComplexPat: return "OPC_CheckComplexPat"; break; + case Matcher::CheckAndImm: return "OPC_CheckAndImm"; break; + case Matcher::CheckOrImm: return "OPC_CheckOrImm"; break; + case Matcher::CheckFoldableChainNode: + return "OPC_CheckFoldableChainNode"; break; + case Matcher::CheckImmAllOnesV: return "OPC_CheckImmAllOnesV"; break; + case Matcher::CheckImmAllZerosV: return "OPC_CheckImmAllZerosV"; break; + case Matcher::EmitInteger: return "OPC_EmitInteger"; break; + case Matcher::EmitStringInteger: return "OPC_EmitStringInteger"; break; + case Matcher::EmitRegister: return "OPC_EmitRegister"; break; + case Matcher::EmitConvertToTarget: return "OPC_EmitConvertToTarget"; break; + case Matcher::EmitMergeInputChains: return "OPC_EmitMergeInputChains"; break; + case Matcher::EmitCopyToReg: return "OPC_EmitCopyToReg"; break; + case Matcher::EmitNode: return "OPC_EmitNode"; break; + case Matcher::MorphNodeTo: return "OPC_MorphNodeTo"; break; + case Matcher::EmitNodeXForm: return "OPC_EmitNodeXForm"; break; + case Matcher::CompleteMatch: return "OPC_CompleteMatch"; break; + } + + llvm_unreachable("Unhandled opcode?"); +} + +void MatcherTableEmitter::EmitHistogram(const Matcher *M, + raw_ostream &OS) { + if (OmitComments) + return; + + std::vector<unsigned> OpcodeFreq; + BuildHistogram(M, OpcodeFreq); + + OS << " // Opcode Histogram:\n"; + for (unsigned i = 0, e = OpcodeFreq.size(); i != e; ++i) { + OS << " // #" + << left_justify(getOpcodeString((Matcher::KindTy)i), HistOpcWidth) + << " = " << OpcodeFreq[i] << '\n'; + } + OS << '\n'; +} + + +void llvm::EmitMatcherTable(const Matcher *TheMatcher, + const CodeGenDAGPatterns &CGP, + raw_ostream &OS) { + OS << "#if defined(GET_DAGISEL_DECL) && defined(GET_DAGISEL_BODY)\n"; + OS << "#error GET_DAGISEL_DECL and GET_DAGISEL_BODY cannot be both defined, "; + OS << "undef both for inline definitions\n"; + OS << "#endif\n\n"; + + // Emit a check for omitted class name. + OS << "#ifdef GET_DAGISEL_BODY\n"; + OS << "#define LOCAL_DAGISEL_STRINGIZE(X) LOCAL_DAGISEL_STRINGIZE_(X)\n"; + OS << "#define LOCAL_DAGISEL_STRINGIZE_(X) #X\n"; + OS << "static_assert(sizeof(LOCAL_DAGISEL_STRINGIZE(GET_DAGISEL_BODY)) > 1," + "\n"; + OS << " \"GET_DAGISEL_BODY is empty: it should be defined with the class " + "name\");\n"; + OS << "#undef LOCAL_DAGISEL_STRINGIZE_\n"; + OS << "#undef LOCAL_DAGISEL_STRINGIZE\n"; + OS << "#endif\n\n"; + + OS << "#if !defined(GET_DAGISEL_DECL) && !defined(GET_DAGISEL_BODY)\n"; + OS << "#define DAGISEL_INLINE 1\n"; + OS << "#else\n"; + OS << "#define DAGISEL_INLINE 0\n"; + OS << "#endif\n\n"; + + OS << "#if !DAGISEL_INLINE\n"; + OS << "#define DAGISEL_CLASS_COLONCOLON GET_DAGISEL_BODY ::\n"; + OS << "#else\n"; + OS << "#define DAGISEL_CLASS_COLONCOLON\n"; + OS << "#endif\n\n"; + + BeginEmitFunction(OS, "void", "SelectCode(SDNode *N)", false/*AddOverride*/); + MatcherTableEmitter MatcherEmitter(CGP); + + OS << "{\n"; + OS << " // Some target values are emitted as 2 bytes, TARGET_VAL handles\n"; + OS << " // this.\n"; + OS << " #define TARGET_VAL(X) X & 255, unsigned(X) >> 8\n"; + OS << " static const unsigned char MatcherTable[] = {\n"; + unsigned TotalSize = MatcherEmitter.EmitMatcherList(TheMatcher, 1, 0, OS); + OS << " 0\n }; // Total Array size is " << (TotalSize+1) << " bytes\n\n"; + + MatcherEmitter.EmitHistogram(TheMatcher, OS); + + OS << " #undef TARGET_VAL\n"; + OS << " SelectCodeCommon(N, MatcherTable,sizeof(MatcherTable));\n"; + OS << "}\n"; + EndEmitFunction(OS); + + // Next up, emit the function for node and pattern predicates: + MatcherEmitter.EmitPredicateFunctions(OS); + + if (InstrumentCoverage) + MatcherEmitter.EmitPatternMatchTable(OS); + + // Clean up the preprocessor macros. + OS << "\n"; + OS << "#ifdef DAGISEL_INLINE\n"; + OS << "#undef DAGISEL_INLINE\n"; + OS << "#endif\n"; + OS << "#ifdef DAGISEL_CLASS_COLONCOLON\n"; + OS << "#undef DAGISEL_CLASS_COLONCOLON\n"; + OS << "#endif\n"; + OS << "#ifdef GET_DAGISEL_DECL\n"; + OS << "#undef GET_DAGISEL_DECL\n"; + OS << "#endif\n"; + OS << "#ifdef GET_DAGISEL_BODY\n"; + OS << "#undef GET_DAGISEL_BODY\n"; + OS << "#endif\n"; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp new file mode 100644 index 000000000000..123ea3374c74 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherGen.cpp @@ -0,0 +1,1106 @@ +//===- DAGISelMatcherGen.cpp - Matcher generator --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DAGISelMatcher.h" +#include "CodeGenDAGPatterns.h" +#include "CodeGenRegisters.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include <utility> +using namespace llvm; + + +/// getRegisterValueType - Look up and return the ValueType of the specified +/// register. If the register is a member of multiple register classes which +/// have different associated types, return MVT::Other. +static MVT::SimpleValueType getRegisterValueType(Record *R, + const CodeGenTarget &T) { + bool FoundRC = false; + MVT::SimpleValueType VT = MVT::Other; + const CodeGenRegister *Reg = T.getRegBank().getReg(R); + + for (const auto &RC : T.getRegBank().getRegClasses()) { + if (!RC.contains(Reg)) + continue; + + if (!FoundRC) { + FoundRC = true; + const ValueTypeByHwMode &VVT = RC.getValueTypeNum(0); + if (VVT.isSimple()) + VT = VVT.getSimple().SimpleTy; + continue; + } + +#ifndef NDEBUG + // If this occurs in multiple register classes, they all have to agree. + const ValueTypeByHwMode &T = RC.getValueTypeNum(0); + assert((!T.isSimple() || T.getSimple().SimpleTy == VT) && + "ValueType mismatch between register classes for this register"); +#endif + } + return VT; +} + + +namespace { + class MatcherGen { + const PatternToMatch &Pattern; + const CodeGenDAGPatterns &CGP; + + /// PatWithNoTypes - This is a clone of Pattern.getSrcPattern() that starts + /// out with all of the types removed. This allows us to insert type checks + /// as we scan the tree. + TreePatternNodePtr PatWithNoTypes; + + /// VariableMap - A map from variable names ('$dst') to the recorded operand + /// number that they were captured as. These are biased by 1 to make + /// insertion easier. + StringMap<unsigned> VariableMap; + + /// This maintains the recorded operand number that OPC_CheckComplexPattern + /// drops each sub-operand into. We don't want to insert these into + /// VariableMap because that leads to identity checking if they are + /// encountered multiple times. Biased by 1 like VariableMap for + /// consistency. + StringMap<unsigned> NamedComplexPatternOperands; + + /// NextRecordedOperandNo - As we emit opcodes to record matched values in + /// the RecordedNodes array, this keeps track of which slot will be next to + /// record into. + unsigned NextRecordedOperandNo; + + /// MatchedChainNodes - This maintains the position in the recorded nodes + /// array of all of the recorded input nodes that have chains. + SmallVector<unsigned, 2> MatchedChainNodes; + + /// MatchedComplexPatterns - This maintains a list of all of the + /// ComplexPatterns that we need to check. The second element of each pair + /// is the recorded operand number of the input node. + SmallVector<std::pair<const TreePatternNode*, + unsigned>, 2> MatchedComplexPatterns; + + /// PhysRegInputs - List list has an entry for each explicitly specified + /// physreg input to the pattern. The first elt is the Register node, the + /// second is the recorded slot number the input pattern match saved it in. + SmallVector<std::pair<Record*, unsigned>, 2> PhysRegInputs; + + /// Matcher - This is the top level of the generated matcher, the result. + Matcher *TheMatcher; + + /// CurPredicate - As we emit matcher nodes, this points to the latest check + /// which should have future checks stuck into its Next position. + Matcher *CurPredicate; + public: + MatcherGen(const PatternToMatch &pattern, const CodeGenDAGPatterns &cgp); + + bool EmitMatcherCode(unsigned Variant); + void EmitResultCode(); + + Matcher *GetMatcher() const { return TheMatcher; } + private: + void AddMatcher(Matcher *NewNode); + void InferPossibleTypes(unsigned ForceMode); + + // Matcher Generation. + void EmitMatchCode(const TreePatternNode *N, TreePatternNode *NodeNoTypes, + unsigned ForceMode); + void EmitLeafMatchCode(const TreePatternNode *N); + void EmitOperatorMatchCode(const TreePatternNode *N, + TreePatternNode *NodeNoTypes, + unsigned ForceMode); + + /// If this is the first time a node with unique identifier Name has been + /// seen, record it. Otherwise, emit a check to make sure this is the same + /// node. Returns true if this is the first encounter. + bool recordUniqueNode(ArrayRef<std::string> Names); + + // Result Code Generation. + unsigned getNamedArgumentSlot(StringRef Name) { + unsigned VarMapEntry = VariableMap[Name]; + assert(VarMapEntry != 0 && + "Variable referenced but not defined and not caught earlier!"); + return VarMapEntry-1; + } + + void EmitResultOperand(const TreePatternNode *N, + SmallVectorImpl<unsigned> &ResultOps); + void EmitResultOfNamedOperand(const TreePatternNode *N, + SmallVectorImpl<unsigned> &ResultOps); + void EmitResultLeafAsOperand(const TreePatternNode *N, + SmallVectorImpl<unsigned> &ResultOps); + void EmitResultInstructionAsOperand(const TreePatternNode *N, + SmallVectorImpl<unsigned> &ResultOps); + void EmitResultSDNodeXFormAsOperand(const TreePatternNode *N, + SmallVectorImpl<unsigned> &ResultOps); + }; + +} // end anonymous namespace + +MatcherGen::MatcherGen(const PatternToMatch &pattern, + const CodeGenDAGPatterns &cgp) +: Pattern(pattern), CGP(cgp), NextRecordedOperandNo(0), + TheMatcher(nullptr), CurPredicate(nullptr) { + // We need to produce the matcher tree for the patterns source pattern. To do + // this we need to match the structure as well as the types. To do the type + // matching, we want to figure out the fewest number of type checks we need to + // emit. For example, if there is only one integer type supported by a + // target, there should be no type comparisons at all for integer patterns! + // + // To figure out the fewest number of type checks needed, clone the pattern, + // remove the types, then perform type inference on the pattern as a whole. + // If there are unresolved types, emit an explicit check for those types, + // apply the type to the tree, then rerun type inference. Iterate until all + // types are resolved. + // + PatWithNoTypes = Pattern.getSrcPattern()->clone(); + PatWithNoTypes->RemoveAllTypes(); + + // If there are types that are manifestly known, infer them. + InferPossibleTypes(Pattern.ForceMode); +} + +/// InferPossibleTypes - As we emit the pattern, we end up generating type +/// checks and applying them to the 'PatWithNoTypes' tree. As we do this, we +/// want to propagate implied types as far throughout the tree as possible so +/// that we avoid doing redundant type checks. This does the type propagation. +void MatcherGen::InferPossibleTypes(unsigned ForceMode) { + // TP - Get *SOME* tree pattern, we don't care which. It is only used for + // diagnostics, which we know are impossible at this point. + TreePattern &TP = *CGP.pf_begin()->second; + TP.getInfer().CodeGen = true; + TP.getInfer().ForceMode = ForceMode; + + bool MadeChange = true; + while (MadeChange) + MadeChange = PatWithNoTypes->ApplyTypeConstraints(TP, + true/*Ignore reg constraints*/); +} + + +/// AddMatcher - Add a matcher node to the current graph we're building. +void MatcherGen::AddMatcher(Matcher *NewNode) { + if (CurPredicate) + CurPredicate->setNext(NewNode); + else + TheMatcher = NewNode; + CurPredicate = NewNode; +} + + +//===----------------------------------------------------------------------===// +// Pattern Match Generation +//===----------------------------------------------------------------------===// + +/// EmitLeafMatchCode - Generate matching code for leaf nodes. +void MatcherGen::EmitLeafMatchCode(const TreePatternNode *N) { + assert(N->isLeaf() && "Not a leaf?"); + + // Direct match against an integer constant. + if (IntInit *II = dyn_cast<IntInit>(N->getLeafValue())) { + // If this is the root of the dag we're matching, we emit a redundant opcode + // check to ensure that this gets folded into the normal top-level + // OpcodeSwitch. + if (N == Pattern.getSrcPattern()) { + const SDNodeInfo &NI = CGP.getSDNodeInfo(CGP.getSDNodeNamed("imm")); + AddMatcher(new CheckOpcodeMatcher(NI)); + } + + return AddMatcher(new CheckIntegerMatcher(II->getValue())); + } + + // An UnsetInit represents a named node without any constraints. + if (isa<UnsetInit>(N->getLeafValue())) { + assert(N->hasName() && "Unnamed ? leaf"); + return; + } + + DefInit *DI = dyn_cast<DefInit>(N->getLeafValue()); + if (!DI) { + errs() << "Unknown leaf kind: " << *N << "\n"; + abort(); + } + + Record *LeafRec = DI->getDef(); + + // A ValueType leaf node can represent a register when named, or itself when + // unnamed. + if (LeafRec->isSubClassOf("ValueType")) { + // A named ValueType leaf always matches: (add i32:$a, i32:$b). + if (N->hasName()) + return; + // An unnamed ValueType as in (sext_inreg GPR:$foo, i8). + return AddMatcher(new CheckValueTypeMatcher(LeafRec->getName())); + } + + if (// Handle register references. Nothing to do here, they always match. + LeafRec->isSubClassOf("RegisterClass") || + LeafRec->isSubClassOf("RegisterOperand") || + LeafRec->isSubClassOf("PointerLikeRegClass") || + LeafRec->isSubClassOf("SubRegIndex") || + // Place holder for SRCVALUE nodes. Nothing to do here. + LeafRec->getName() == "srcvalue") + return; + + // If we have a physreg reference like (mul gpr:$src, EAX) then we need to + // record the register + if (LeafRec->isSubClassOf("Register")) { + AddMatcher(new RecordMatcher("physreg input "+LeafRec->getName().str(), + NextRecordedOperandNo)); + PhysRegInputs.push_back(std::make_pair(LeafRec, NextRecordedOperandNo++)); + return; + } + + if (LeafRec->isSubClassOf("CondCode")) + return AddMatcher(new CheckCondCodeMatcher(LeafRec->getName())); + + if (LeafRec->isSubClassOf("ComplexPattern")) { + // We can't model ComplexPattern uses that don't have their name taken yet. + // The OPC_CheckComplexPattern operation implicitly records the results. + if (N->getName().empty()) { + std::string S; + raw_string_ostream OS(S); + OS << "We expect complex pattern uses to have names: " << *N; + PrintFatalError(OS.str()); + } + + // Remember this ComplexPattern so that we can emit it after all the other + // structural matches are done. + unsigned InputOperand = VariableMap[N->getName()] - 1; + MatchedComplexPatterns.push_back(std::make_pair(N, InputOperand)); + return; + } + + if (LeafRec->getName() == "immAllOnesV") { + // If this is the root of the dag we're matching, we emit a redundant opcode + // check to ensure that this gets folded into the normal top-level + // OpcodeSwitch. + if (N == Pattern.getSrcPattern()) { + const SDNodeInfo &NI = CGP.getSDNodeInfo(CGP.getSDNodeNamed("build_vector")); + AddMatcher(new CheckOpcodeMatcher(NI)); + } + return AddMatcher(new CheckImmAllOnesVMatcher()); + } + if (LeafRec->getName() == "immAllZerosV") { + // If this is the root of the dag we're matching, we emit a redundant opcode + // check to ensure that this gets folded into the normal top-level + // OpcodeSwitch. + if (N == Pattern.getSrcPattern()) { + const SDNodeInfo &NI = CGP.getSDNodeInfo(CGP.getSDNodeNamed("build_vector")); + AddMatcher(new CheckOpcodeMatcher(NI)); + } + return AddMatcher(new CheckImmAllZerosVMatcher()); + } + + errs() << "Unknown leaf kind: " << *N << "\n"; + abort(); +} + +void MatcherGen::EmitOperatorMatchCode(const TreePatternNode *N, + TreePatternNode *NodeNoTypes, + unsigned ForceMode) { + assert(!N->isLeaf() && "Not an operator?"); + + if (N->getOperator()->isSubClassOf("ComplexPattern")) { + // The "name" of a non-leaf complex pattern (MY_PAT $op1, $op2) is + // "MY_PAT:op1:op2". We should already have validated that the uses are + // consistent. + std::string PatternName = std::string(N->getOperator()->getName()); + for (unsigned i = 0; i < N->getNumChildren(); ++i) { + PatternName += ":"; + PatternName += N->getChild(i)->getName(); + } + + if (recordUniqueNode(PatternName)) { + auto NodeAndOpNum = std::make_pair(N, NextRecordedOperandNo - 1); + MatchedComplexPatterns.push_back(NodeAndOpNum); + } + + return; + } + + const SDNodeInfo &CInfo = CGP.getSDNodeInfo(N->getOperator()); + + // If this is an 'and R, 1234' where the operation is AND/OR and the RHS is + // a constant without a predicate fn that has more than one bit set, handle + // this as a special case. This is usually for targets that have special + // handling of certain large constants (e.g. alpha with it's 8/16/32-bit + // handling stuff). Using these instructions is often far more efficient + // than materializing the constant. Unfortunately, both the instcombiner + // and the dag combiner can often infer that bits are dead, and thus drop + // them from the mask in the dag. For example, it might turn 'AND X, 255' + // into 'AND X, 254' if it knows the low bit is set. Emit code that checks + // to handle this. + if ((N->getOperator()->getName() == "and" || + N->getOperator()->getName() == "or") && + N->getChild(1)->isLeaf() && N->getChild(1)->getPredicateCalls().empty() && + N->getPredicateCalls().empty()) { + if (IntInit *II = dyn_cast<IntInit>(N->getChild(1)->getLeafValue())) { + if (!isPowerOf2_32(II->getValue())) { // Don't bother with single bits. + // If this is at the root of the pattern, we emit a redundant + // CheckOpcode so that the following checks get factored properly under + // a single opcode check. + if (N == Pattern.getSrcPattern()) + AddMatcher(new CheckOpcodeMatcher(CInfo)); + + // Emit the CheckAndImm/CheckOrImm node. + if (N->getOperator()->getName() == "and") + AddMatcher(new CheckAndImmMatcher(II->getValue())); + else + AddMatcher(new CheckOrImmMatcher(II->getValue())); + + // Match the LHS of the AND as appropriate. + AddMatcher(new MoveChildMatcher(0)); + EmitMatchCode(N->getChild(0), NodeNoTypes->getChild(0), ForceMode); + AddMatcher(new MoveParentMatcher()); + return; + } + } + } + + // Check that the current opcode lines up. + AddMatcher(new CheckOpcodeMatcher(CInfo)); + + // If this node has memory references (i.e. is a load or store), tell the + // interpreter to capture them in the memref array. + if (N->NodeHasProperty(SDNPMemOperand, CGP)) + AddMatcher(new RecordMemRefMatcher()); + + // If this node has a chain, then the chain is operand #0 is the SDNode, and + // the child numbers of the node are all offset by one. + unsigned OpNo = 0; + if (N->NodeHasProperty(SDNPHasChain, CGP)) { + // Record the node and remember it in our chained nodes list. + AddMatcher(new RecordMatcher("'" + N->getOperator()->getName().str() + + "' chained node", + NextRecordedOperandNo)); + // Remember all of the input chains our pattern will match. + MatchedChainNodes.push_back(NextRecordedOperandNo++); + + // Don't look at the input chain when matching the tree pattern to the + // SDNode. + OpNo = 1; + + // If this node is not the root and the subtree underneath it produces a + // chain, then the result of matching the node is also produce a chain. + // Beyond that, this means that we're also folding (at least) the root node + // into the node that produce the chain (for example, matching + // "(add reg, (load ptr))" as a add_with_memory on X86). This is + // problematic, if the 'reg' node also uses the load (say, its chain). + // Graphically: + // + // [LD] + // ^ ^ + // | \ DAG's like cheese. + // / | + // / [YY] + // | ^ + // [XX]--/ + // + // It would be invalid to fold XX and LD. In this case, folding the two + // nodes together would induce a cycle in the DAG, making it a 'cyclic DAG' + // To prevent this, we emit a dynamic check for legality before allowing + // this to be folded. + // + const TreePatternNode *Root = Pattern.getSrcPattern(); + if (N != Root) { // Not the root of the pattern. + // If there is a node between the root and this node, then we definitely + // need to emit the check. + bool NeedCheck = !Root->hasChild(N); + + // If it *is* an immediate child of the root, we can still need a check if + // the root SDNode has multiple inputs. For us, this means that it is an + // intrinsic, has multiple operands, or has other inputs like chain or + // glue). + if (!NeedCheck) { + const SDNodeInfo &PInfo = CGP.getSDNodeInfo(Root->getOperator()); + NeedCheck = + Root->getOperator() == CGP.get_intrinsic_void_sdnode() || + Root->getOperator() == CGP.get_intrinsic_w_chain_sdnode() || + Root->getOperator() == CGP.get_intrinsic_wo_chain_sdnode() || + PInfo.getNumOperands() > 1 || + PInfo.hasProperty(SDNPHasChain) || + PInfo.hasProperty(SDNPInGlue) || + PInfo.hasProperty(SDNPOptInGlue); + } + + if (NeedCheck) + AddMatcher(new CheckFoldableChainNodeMatcher()); + } + } + + // If this node has an output glue and isn't the root, remember it. + if (N->NodeHasProperty(SDNPOutGlue, CGP) && + N != Pattern.getSrcPattern()) { + // TODO: This redundantly records nodes with both glues and chains. + + // Record the node and remember it in our chained nodes list. + AddMatcher(new RecordMatcher("'" + N->getOperator()->getName().str() + + "' glue output node", + NextRecordedOperandNo)); + } + + // If this node is known to have an input glue or if it *might* have an input + // glue, capture it as the glue input of the pattern. + if (N->NodeHasProperty(SDNPOptInGlue, CGP) || + N->NodeHasProperty(SDNPInGlue, CGP)) + AddMatcher(new CaptureGlueInputMatcher()); + + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i, ++OpNo) { + // Get the code suitable for matching this child. Move to the child, check + // it then move back to the parent. + AddMatcher(new MoveChildMatcher(OpNo)); + EmitMatchCode(N->getChild(i), NodeNoTypes->getChild(i), ForceMode); + AddMatcher(new MoveParentMatcher()); + } +} + +bool MatcherGen::recordUniqueNode(ArrayRef<std::string> Names) { + unsigned Entry = 0; + for (const std::string &Name : Names) { + unsigned &VarMapEntry = VariableMap[Name]; + if (!Entry) + Entry = VarMapEntry; + assert(Entry == VarMapEntry); + } + + bool NewRecord = false; + if (Entry == 0) { + // If it is a named node, we must emit a 'Record' opcode. + std::string WhatFor; + for (const std::string &Name : Names) { + if (!WhatFor.empty()) + WhatFor += ','; + WhatFor += "$" + Name; + } + AddMatcher(new RecordMatcher(WhatFor, NextRecordedOperandNo)); + Entry = ++NextRecordedOperandNo; + NewRecord = true; + } else { + // If we get here, this is a second reference to a specific name. Since + // we already have checked that the first reference is valid, we don't + // have to recursively match it, just check that it's the same as the + // previously named thing. + AddMatcher(new CheckSameMatcher(Entry-1)); + } + + for (const std::string &Name : Names) + VariableMap[Name] = Entry; + + return NewRecord; +} + +void MatcherGen::EmitMatchCode(const TreePatternNode *N, + TreePatternNode *NodeNoTypes, + unsigned ForceMode) { + // If N and NodeNoTypes don't agree on a type, then this is a case where we + // need to do a type check. Emit the check, apply the type to NodeNoTypes and + // reinfer any correlated types. + SmallVector<unsigned, 2> ResultsToTypeCheck; + + for (unsigned i = 0, e = NodeNoTypes->getNumTypes(); i != e; ++i) { + if (NodeNoTypes->getExtType(i) == N->getExtType(i)) continue; + NodeNoTypes->setType(i, N->getExtType(i)); + InferPossibleTypes(ForceMode); + ResultsToTypeCheck.push_back(i); + } + + // If this node has a name associated with it, capture it in VariableMap. If + // we already saw this in the pattern, emit code to verify dagness. + SmallVector<std::string, 4> Names; + if (!N->getName().empty()) + Names.push_back(N->getName()); + + for (const ScopedName &Name : N->getNamesAsPredicateArg()) { + Names.push_back(("pred:" + Twine(Name.getScope()) + ":" + Name.getIdentifier()).str()); + } + + if (!Names.empty()) { + if (!recordUniqueNode(Names)) + return; + } + + if (N->isLeaf()) + EmitLeafMatchCode(N); + else + EmitOperatorMatchCode(N, NodeNoTypes, ForceMode); + + // If there are node predicates for this node, generate their checks. + for (unsigned i = 0, e = N->getPredicateCalls().size(); i != e; ++i) { + const TreePredicateCall &Pred = N->getPredicateCalls()[i]; + SmallVector<unsigned, 4> Operands; + if (Pred.Fn.usesOperands()) { + TreePattern *TP = Pred.Fn.getOrigPatFragRecord(); + for (unsigned i = 0; i < TP->getNumArgs(); ++i) { + std::string Name = + ("pred:" + Twine(Pred.Scope) + ":" + TP->getArgName(i)).str(); + Operands.push_back(getNamedArgumentSlot(Name)); + } + } + AddMatcher(new CheckPredicateMatcher(Pred.Fn, Operands)); + } + + for (unsigned i = 0, e = ResultsToTypeCheck.size(); i != e; ++i) + AddMatcher(new CheckTypeMatcher(N->getSimpleType(ResultsToTypeCheck[i]), + ResultsToTypeCheck[i])); +} + +/// EmitMatcherCode - Generate the code that matches the predicate of this +/// pattern for the specified Variant. If the variant is invalid this returns +/// true and does not generate code, if it is valid, it returns false. +bool MatcherGen::EmitMatcherCode(unsigned Variant) { + // If the root of the pattern is a ComplexPattern and if it is specified to + // match some number of root opcodes, these are considered to be our variants. + // Depending on which variant we're generating code for, emit the root opcode + // check. + if (const ComplexPattern *CP = + Pattern.getSrcPattern()->getComplexPatternInfo(CGP)) { + const std::vector<Record*> &OpNodes = CP->getRootNodes(); + assert(!OpNodes.empty() &&"Complex Pattern must specify what it can match"); + if (Variant >= OpNodes.size()) return true; + + AddMatcher(new CheckOpcodeMatcher(CGP.getSDNodeInfo(OpNodes[Variant]))); + } else { + if (Variant != 0) return true; + } + + // Emit the matcher for the pattern structure and types. + EmitMatchCode(Pattern.getSrcPattern(), PatWithNoTypes.get(), + Pattern.ForceMode); + + // If the pattern has a predicate on it (e.g. only enabled when a subtarget + // feature is around, do the check). + if (!Pattern.getPredicateCheck().empty()) + AddMatcher(new CheckPatternPredicateMatcher(Pattern.getPredicateCheck())); + + // Now that we've completed the structural type match, emit any ComplexPattern + // checks (e.g. addrmode matches). We emit this after the structural match + // because they are generally more expensive to evaluate and more difficult to + // factor. + for (unsigned i = 0, e = MatchedComplexPatterns.size(); i != e; ++i) { + auto N = MatchedComplexPatterns[i].first; + + // Remember where the results of this match get stuck. + if (N->isLeaf()) { + NamedComplexPatternOperands[N->getName()] = NextRecordedOperandNo + 1; + } else { + unsigned CurOp = NextRecordedOperandNo; + for (unsigned i = 0; i < N->getNumChildren(); ++i) { + NamedComplexPatternOperands[N->getChild(i)->getName()] = CurOp + 1; + CurOp += N->getChild(i)->getNumMIResults(CGP); + } + } + + // Get the slot we recorded the value in from the name on the node. + unsigned RecNodeEntry = MatchedComplexPatterns[i].second; + + const ComplexPattern &CP = *N->getComplexPatternInfo(CGP); + + // Emit a CheckComplexPat operation, which does the match (aborting if it + // fails) and pushes the matched operands onto the recorded nodes list. + AddMatcher(new CheckComplexPatMatcher(CP, RecNodeEntry, + N->getName(), NextRecordedOperandNo)); + + // Record the right number of operands. + NextRecordedOperandNo += CP.getNumOperands(); + if (CP.hasProperty(SDNPHasChain)) { + // If the complex pattern has a chain, then we need to keep track of the + // fact that we just recorded a chain input. The chain input will be + // matched as the last operand of the predicate if it was successful. + ++NextRecordedOperandNo; // Chained node operand. + + // It is the last operand recorded. + assert(NextRecordedOperandNo > 1 && + "Should have recorded input/result chains at least!"); + MatchedChainNodes.push_back(NextRecordedOperandNo-1); + } + + // TODO: Complex patterns can't have output glues, if they did, we'd want + // to record them. + } + + return false; +} + + +//===----------------------------------------------------------------------===// +// Node Result Generation +//===----------------------------------------------------------------------===// + +void MatcherGen::EmitResultOfNamedOperand(const TreePatternNode *N, + SmallVectorImpl<unsigned> &ResultOps){ + assert(!N->getName().empty() && "Operand not named!"); + + if (unsigned SlotNo = NamedComplexPatternOperands[N->getName()]) { + // Complex operands have already been completely selected, just find the + // right slot ant add the arguments directly. + for (unsigned i = 0; i < N->getNumMIResults(CGP); ++i) + ResultOps.push_back(SlotNo - 1 + i); + + return; + } + + unsigned SlotNo = getNamedArgumentSlot(N->getName()); + + // If this is an 'imm' or 'fpimm' node, make sure to convert it to the target + // version of the immediate so that it doesn't get selected due to some other + // node use. + if (!N->isLeaf()) { + StringRef OperatorName = N->getOperator()->getName(); + if (OperatorName == "imm" || OperatorName == "fpimm") { + AddMatcher(new EmitConvertToTargetMatcher(SlotNo)); + ResultOps.push_back(NextRecordedOperandNo++); + return; + } + } + + for (unsigned i = 0; i < N->getNumMIResults(CGP); ++i) + ResultOps.push_back(SlotNo + i); +} + +void MatcherGen::EmitResultLeafAsOperand(const TreePatternNode *N, + SmallVectorImpl<unsigned> &ResultOps) { + assert(N->isLeaf() && "Must be a leaf"); + + if (IntInit *II = dyn_cast<IntInit>(N->getLeafValue())) { + AddMatcher(new EmitIntegerMatcher(II->getValue(), N->getSimpleType(0))); + ResultOps.push_back(NextRecordedOperandNo++); + return; + } + + // If this is an explicit register reference, handle it. + if (DefInit *DI = dyn_cast<DefInit>(N->getLeafValue())) { + Record *Def = DI->getDef(); + if (Def->isSubClassOf("Register")) { + const CodeGenRegister *Reg = + CGP.getTargetInfo().getRegBank().getReg(Def); + AddMatcher(new EmitRegisterMatcher(Reg, N->getSimpleType(0))); + ResultOps.push_back(NextRecordedOperandNo++); + return; + } + + if (Def->getName() == "zero_reg") { + AddMatcher(new EmitRegisterMatcher(nullptr, N->getSimpleType(0))); + ResultOps.push_back(NextRecordedOperandNo++); + return; + } + + if (Def->getName() == "undef_tied_input") { + std::array<MVT::SimpleValueType, 1> ResultVTs = {{ N->getSimpleType(0) }}; + std::array<unsigned, 0> InstOps; + auto IDOperandNo = NextRecordedOperandNo++; + AddMatcher(new EmitNodeMatcher("TargetOpcode::IMPLICIT_DEF", + ResultVTs, InstOps, false, false, false, + false, -1, IDOperandNo)); + ResultOps.push_back(IDOperandNo); + return; + } + + // Handle a reference to a register class. This is used + // in COPY_TO_SUBREG instructions. + if (Def->isSubClassOf("RegisterOperand")) + Def = Def->getValueAsDef("RegClass"); + if (Def->isSubClassOf("RegisterClass")) { + // If the register class has an enum integer value greater than 127, the + // encoding overflows the limit of 7 bits, which precludes the use of + // StringIntegerMatcher. In this case, fallback to using IntegerMatcher. + const CodeGenRegisterClass &RC = + CGP.getTargetInfo().getRegisterClass(Def); + if (RC.EnumValue <= 127) { + std::string Value = getQualifiedName(Def) + "RegClassID"; + AddMatcher(new EmitStringIntegerMatcher(Value, MVT::i32)); + ResultOps.push_back(NextRecordedOperandNo++); + } else { + AddMatcher(new EmitIntegerMatcher(RC.EnumValue, MVT::i32)); + ResultOps.push_back(NextRecordedOperandNo++); + } + return; + } + + // Handle a subregister index. This is used for INSERT_SUBREG etc. + if (Def->isSubClassOf("SubRegIndex")) { + const CodeGenRegBank &RB = CGP.getTargetInfo().getRegBank(); + // If we have more than 127 subreg indices the encoding can overflow + // 7 bit and we cannot use StringInteger. + if (RB.getSubRegIndices().size() > 127) { + const CodeGenSubRegIndex *I = RB.findSubRegIdx(Def); + assert(I && "Cannot find subreg index by name!"); + if (I->EnumValue > 127) { + AddMatcher(new EmitIntegerMatcher(I->EnumValue, MVT::i32)); + ResultOps.push_back(NextRecordedOperandNo++); + return; + } + } + std::string Value = getQualifiedName(Def); + AddMatcher(new EmitStringIntegerMatcher(Value, MVT::i32)); + ResultOps.push_back(NextRecordedOperandNo++); + return; + } + } + + errs() << "unhandled leaf node: \n"; + N->dump(); +} + +static bool +mayInstNodeLoadOrStore(const TreePatternNode *N, + const CodeGenDAGPatterns &CGP) { + Record *Op = N->getOperator(); + const CodeGenTarget &CGT = CGP.getTargetInfo(); + CodeGenInstruction &II = CGT.getInstruction(Op); + return II.mayLoad || II.mayStore; +} + +static unsigned +numNodesThatMayLoadOrStore(const TreePatternNode *N, + const CodeGenDAGPatterns &CGP) { + if (N->isLeaf()) + return 0; + + Record *OpRec = N->getOperator(); + if (!OpRec->isSubClassOf("Instruction")) + return 0; + + unsigned Count = 0; + if (mayInstNodeLoadOrStore(N, CGP)) + ++Count; + + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) + Count += numNodesThatMayLoadOrStore(N->getChild(i), CGP); + + return Count; +} + +void MatcherGen:: +EmitResultInstructionAsOperand(const TreePatternNode *N, + SmallVectorImpl<unsigned> &OutputOps) { + Record *Op = N->getOperator(); + const CodeGenTarget &CGT = CGP.getTargetInfo(); + CodeGenInstruction &II = CGT.getInstruction(Op); + const DAGInstruction &Inst = CGP.getInstruction(Op); + + bool isRoot = N == Pattern.getDstPattern(); + + // TreeHasOutGlue - True if this tree has glue. + bool TreeHasInGlue = false, TreeHasOutGlue = false; + if (isRoot) { + const TreePatternNode *SrcPat = Pattern.getSrcPattern(); + TreeHasInGlue = SrcPat->TreeHasProperty(SDNPOptInGlue, CGP) || + SrcPat->TreeHasProperty(SDNPInGlue, CGP); + + // FIXME2: this is checking the entire pattern, not just the node in + // question, doing this just for the root seems like a total hack. + TreeHasOutGlue = SrcPat->TreeHasProperty(SDNPOutGlue, CGP); + } + + // NumResults - This is the number of results produced by the instruction in + // the "outs" list. + unsigned NumResults = Inst.getNumResults(); + + // Number of operands we know the output instruction must have. If it is + // variadic, we could have more operands. + unsigned NumFixedOperands = II.Operands.size(); + + SmallVector<unsigned, 8> InstOps; + + // Loop over all of the fixed operands of the instruction pattern, emitting + // code to fill them all in. The node 'N' usually has number children equal to + // the number of input operands of the instruction. However, in cases where + // there are predicate operands for an instruction, we need to fill in the + // 'execute always' values. Match up the node operands to the instruction + // operands to do this. + unsigned ChildNo = 0; + + // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the + // number of operands at the end of the list which have default values. + // Those can come from the pattern if it provides enough arguments, or be + // filled in with the default if the pattern hasn't provided them. But any + // operand with a default value _before_ the last mandatory one will be + // filled in with their defaults unconditionally. + unsigned NonOverridableOperands = NumFixedOperands; + while (NonOverridableOperands > NumResults && + CGP.operandHasDefault(II.Operands[NonOverridableOperands-1].Rec)) + --NonOverridableOperands; + + for (unsigned InstOpNo = NumResults, e = NumFixedOperands; + InstOpNo != e; ++InstOpNo) { + // Determine what to emit for this operand. + Record *OperandNode = II.Operands[InstOpNo].Rec; + if (CGP.operandHasDefault(OperandNode) && + (InstOpNo < NonOverridableOperands || ChildNo >= N->getNumChildren())) { + // This is a predicate or optional def operand which the pattern has not + // overridden, or which we aren't letting it override; emit the 'default + // ops' operands. + const DAGDefaultOperand &DefaultOp + = CGP.getDefaultOperand(OperandNode); + for (unsigned i = 0, e = DefaultOp.DefaultOps.size(); i != e; ++i) + EmitResultOperand(DefaultOp.DefaultOps[i].get(), InstOps); + continue; + } + + // Otherwise this is a normal operand or a predicate operand without + // 'execute always'; emit it. + + // For operands with multiple sub-operands we may need to emit + // multiple child patterns to cover them all. However, ComplexPattern + // children may themselves emit multiple MI operands. + unsigned NumSubOps = 1; + if (OperandNode->isSubClassOf("Operand")) { + DagInit *MIOpInfo = OperandNode->getValueAsDag("MIOperandInfo"); + if (unsigned NumArgs = MIOpInfo->getNumArgs()) + NumSubOps = NumArgs; + } + + unsigned FinalNumOps = InstOps.size() + NumSubOps; + while (InstOps.size() < FinalNumOps) { + const TreePatternNode *Child = N->getChild(ChildNo); + unsigned BeforeAddingNumOps = InstOps.size(); + EmitResultOperand(Child, InstOps); + assert(InstOps.size() > BeforeAddingNumOps && "Didn't add any operands"); + + // If the operand is an instruction and it produced multiple results, just + // take the first one. + if (!Child->isLeaf() && Child->getOperator()->isSubClassOf("Instruction")) + InstOps.resize(BeforeAddingNumOps+1); + + ++ChildNo; + } + } + + // If this is a variadic output instruction (i.e. REG_SEQUENCE), we can't + // expand suboperands, use default operands, or other features determined from + // the CodeGenInstruction after the fixed operands, which were handled + // above. Emit the remaining instructions implicitly added by the use for + // variable_ops. + if (II.Operands.isVariadic) { + for (unsigned I = ChildNo, E = N->getNumChildren(); I < E; ++I) + EmitResultOperand(N->getChild(I), InstOps); + } + + // If this node has input glue or explicitly specified input physregs, we + // need to add chained and glued copyfromreg nodes and materialize the glue + // input. + if (isRoot && !PhysRegInputs.empty()) { + // Emit all of the CopyToReg nodes for the input physical registers. These + // occur in patterns like (mul:i8 AL:i8, GR8:i8:$src). + for (unsigned i = 0, e = PhysRegInputs.size(); i != e; ++i) { + const CodeGenRegister *Reg = + CGP.getTargetInfo().getRegBank().getReg(PhysRegInputs[i].first); + AddMatcher(new EmitCopyToRegMatcher(PhysRegInputs[i].second, + Reg)); + } + + // Even if the node has no other glue inputs, the resultant node must be + // glued to the CopyFromReg nodes we just generated. + TreeHasInGlue = true; + } + + // Result order: node results, chain, glue + + // Determine the result types. + SmallVector<MVT::SimpleValueType, 4> ResultVTs; + for (unsigned i = 0, e = N->getNumTypes(); i != e; ++i) + ResultVTs.push_back(N->getSimpleType(i)); + + // If this is the root instruction of a pattern that has physical registers in + // its result pattern, add output VTs for them. For example, X86 has: + // (set AL, (mul ...)) + // This also handles implicit results like: + // (implicit EFLAGS) + if (isRoot && !Pattern.getDstRegs().empty()) { + // If the root came from an implicit def in the instruction handling stuff, + // don't re-add it. + Record *HandledReg = nullptr; + if (II.HasOneImplicitDefWithKnownVT(CGT) != MVT::Other) + HandledReg = II.ImplicitDefs[0]; + + for (Record *Reg : Pattern.getDstRegs()) { + if (!Reg->isSubClassOf("Register") || Reg == HandledReg) continue; + ResultVTs.push_back(getRegisterValueType(Reg, CGT)); + } + } + + // If this is the root of the pattern and the pattern we're matching includes + // a node that is variadic, mark the generated node as variadic so that it + // gets the excess operands from the input DAG. + int NumFixedArityOperands = -1; + if (isRoot && + Pattern.getSrcPattern()->NodeHasProperty(SDNPVariadic, CGP)) + NumFixedArityOperands = Pattern.getSrcPattern()->getNumChildren(); + + // If this is the root node and multiple matched nodes in the input pattern + // have MemRefs in them, have the interpreter collect them and plop them onto + // this node. If there is just one node with MemRefs, leave them on that node + // even if it is not the root. + // + // FIXME3: This is actively incorrect for result patterns with multiple + // memory-referencing instructions. + bool PatternHasMemOperands = + Pattern.getSrcPattern()->TreeHasProperty(SDNPMemOperand, CGP); + + bool NodeHasMemRefs = false; + if (PatternHasMemOperands) { + unsigned NumNodesThatLoadOrStore = + numNodesThatMayLoadOrStore(Pattern.getDstPattern(), CGP); + bool NodeIsUniqueLoadOrStore = mayInstNodeLoadOrStore(N, CGP) && + NumNodesThatLoadOrStore == 1; + NodeHasMemRefs = + NodeIsUniqueLoadOrStore || (isRoot && (mayInstNodeLoadOrStore(N, CGP) || + NumNodesThatLoadOrStore != 1)); + } + + // Determine whether we need to attach a chain to this node. + bool NodeHasChain = false; + if (Pattern.getSrcPattern()->TreeHasProperty(SDNPHasChain, CGP)) { + // For some instructions, we were able to infer from the pattern whether + // they should have a chain. Otherwise, attach the chain to the root. + // + // FIXME2: This is extremely dubious for several reasons, not the least of + // which it gives special status to instructions with patterns that Pat<> + // nodes can't duplicate. + if (II.hasChain_Inferred) + NodeHasChain = II.hasChain; + else + NodeHasChain = isRoot; + // Instructions which load and store from memory should have a chain, + // regardless of whether they happen to have a pattern saying so. + if (II.hasCtrlDep || II.mayLoad || II.mayStore || II.canFoldAsLoad || + II.hasSideEffects) + NodeHasChain = true; + } + + assert((!ResultVTs.empty() || TreeHasOutGlue || NodeHasChain) && + "Node has no result"); + + AddMatcher(new EmitNodeMatcher(II.Namespace.str()+"::"+II.TheDef->getName().str(), + ResultVTs, InstOps, + NodeHasChain, TreeHasInGlue, TreeHasOutGlue, + NodeHasMemRefs, NumFixedArityOperands, + NextRecordedOperandNo)); + + // The non-chain and non-glue results of the newly emitted node get recorded. + for (unsigned i = 0, e = ResultVTs.size(); i != e; ++i) { + if (ResultVTs[i] == MVT::Other || ResultVTs[i] == MVT::Glue) break; + OutputOps.push_back(NextRecordedOperandNo++); + } +} + +void MatcherGen:: +EmitResultSDNodeXFormAsOperand(const TreePatternNode *N, + SmallVectorImpl<unsigned> &ResultOps) { + assert(N->getOperator()->isSubClassOf("SDNodeXForm") && "Not SDNodeXForm?"); + + // Emit the operand. + SmallVector<unsigned, 8> InputOps; + + // FIXME2: Could easily generalize this to support multiple inputs and outputs + // to the SDNodeXForm. For now we just support one input and one output like + // the old instruction selector. + assert(N->getNumChildren() == 1); + EmitResultOperand(N->getChild(0), InputOps); + + // The input currently must have produced exactly one result. + assert(InputOps.size() == 1 && "Unexpected input to SDNodeXForm"); + + AddMatcher(new EmitNodeXFormMatcher(InputOps[0], N->getOperator())); + ResultOps.push_back(NextRecordedOperandNo++); +} + +void MatcherGen::EmitResultOperand(const TreePatternNode *N, + SmallVectorImpl<unsigned> &ResultOps) { + // This is something selected from the pattern we matched. + if (!N->getName().empty()) + return EmitResultOfNamedOperand(N, ResultOps); + + if (N->isLeaf()) + return EmitResultLeafAsOperand(N, ResultOps); + + Record *OpRec = N->getOperator(); + if (OpRec->isSubClassOf("Instruction")) + return EmitResultInstructionAsOperand(N, ResultOps); + if (OpRec->isSubClassOf("SDNodeXForm")) + return EmitResultSDNodeXFormAsOperand(N, ResultOps); + errs() << "Unknown result node to emit code for: " << *N << '\n'; + PrintFatalError("Unknown node in result pattern!"); +} + +void MatcherGen::EmitResultCode() { + // Patterns that match nodes with (potentially multiple) chain inputs have to + // merge them together into a token factor. This informs the generated code + // what all the chained nodes are. + if (!MatchedChainNodes.empty()) + AddMatcher(new EmitMergeInputChainsMatcher(MatchedChainNodes)); + + // Codegen the root of the result pattern, capturing the resulting values. + SmallVector<unsigned, 8> Ops; + EmitResultOperand(Pattern.getDstPattern(), Ops); + + // At this point, we have however many values the result pattern produces. + // However, the input pattern might not need all of these. If there are + // excess values at the end (such as implicit defs of condition codes etc) + // just lop them off. This doesn't need to worry about glue or chains, just + // explicit results. + // + unsigned NumSrcResults = Pattern.getSrcPattern()->getNumTypes(); + + // If the pattern also has (implicit) results, count them as well. + if (!Pattern.getDstRegs().empty()) { + // If the root came from an implicit def in the instruction handling stuff, + // don't re-add it. + Record *HandledReg = nullptr; + const TreePatternNode *DstPat = Pattern.getDstPattern(); + if (!DstPat->isLeaf() &&DstPat->getOperator()->isSubClassOf("Instruction")){ + const CodeGenTarget &CGT = CGP.getTargetInfo(); + CodeGenInstruction &II = CGT.getInstruction(DstPat->getOperator()); + + if (II.HasOneImplicitDefWithKnownVT(CGT) != MVT::Other) + HandledReg = II.ImplicitDefs[0]; + } + + for (Record *Reg : Pattern.getDstRegs()) { + if (!Reg->isSubClassOf("Register") || Reg == HandledReg) continue; + ++NumSrcResults; + } + } + + SmallVector<unsigned, 8> Results(Ops); + + // Apply result permutation. + for (unsigned ResNo = 0; ResNo < Pattern.getDstPattern()->getNumResults(); + ++ResNo) { + Results[ResNo] = Ops[Pattern.getDstPattern()->getResultIndex(ResNo)]; + } + + Results.resize(NumSrcResults); + AddMatcher(new CompleteMatchMatcher(Results, Pattern)); +} + + +/// ConvertPatternToMatcher - Create the matcher for the specified pattern with +/// the specified variant. If the variant number is invalid, this returns null. +Matcher *llvm::ConvertPatternToMatcher(const PatternToMatch &Pattern, + unsigned Variant, + const CodeGenDAGPatterns &CGP) { + MatcherGen Gen(Pattern, CGP); + + // Generate the code for the matcher. + if (Gen.EmitMatcherCode(Variant)) + return nullptr; + + // FIXME2: Kill extra MoveParent commands at the end of the matcher sequence. + // FIXME2: Split result code out to another table, and make the matcher end + // with an "Emit <index>" command. This allows result generation stuff to be + // shared and factored? + + // If the match succeeds, then we generate Pattern. + Gen.EmitResultCode(); + + // Unconditional match. + return Gen.GetMatcher(); +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherOpt.cpp new file mode 100644 index 000000000000..6746fdd676a7 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DAGISelMatcherOpt.cpp @@ -0,0 +1,471 @@ +//===- DAGISelMatcherOpt.cpp - Optimize a DAG Matcher ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the DAG Matcher optimizer. +// +//===----------------------------------------------------------------------===// + +#include "DAGISelMatcher.h" +#include "CodeGenDAGPatterns.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +using namespace llvm; + +#define DEBUG_TYPE "isel-opt" + +/// ContractNodes - Turn multiple matcher node patterns like 'MoveChild+Record' +/// into single compound nodes like RecordChild. +static void ContractNodes(std::unique_ptr<Matcher> &MatcherPtr, + const CodeGenDAGPatterns &CGP) { + // If we reached the end of the chain, we're done. + Matcher *N = MatcherPtr.get(); + if (!N) return; + + // If we have a scope node, walk down all of the children. + if (ScopeMatcher *Scope = dyn_cast<ScopeMatcher>(N)) { + for (unsigned i = 0, e = Scope->getNumChildren(); i != e; ++i) { + std::unique_ptr<Matcher> Child(Scope->takeChild(i)); + ContractNodes(Child, CGP); + Scope->resetChild(i, Child.release()); + } + return; + } + + // If we found a movechild node with a node that comes in a 'foochild' form, + // transform it. + if (MoveChildMatcher *MC = dyn_cast<MoveChildMatcher>(N)) { + Matcher *New = nullptr; + if (RecordMatcher *RM = dyn_cast<RecordMatcher>(MC->getNext())) + if (MC->getChildNo() < 8) // Only have RecordChild0...7 + New = new RecordChildMatcher(MC->getChildNo(), RM->getWhatFor(), + RM->getResultNo()); + + if (CheckTypeMatcher *CT = dyn_cast<CheckTypeMatcher>(MC->getNext())) + if (MC->getChildNo() < 8 && // Only have CheckChildType0...7 + CT->getResNo() == 0) // CheckChildType checks res #0 + New = new CheckChildTypeMatcher(MC->getChildNo(), CT->getType()); + + if (CheckSameMatcher *CS = dyn_cast<CheckSameMatcher>(MC->getNext())) + if (MC->getChildNo() < 4) // Only have CheckChildSame0...3 + New = new CheckChildSameMatcher(MC->getChildNo(), CS->getMatchNumber()); + + if (CheckIntegerMatcher *CI = dyn_cast<CheckIntegerMatcher>(MC->getNext())) + if (MC->getChildNo() < 5) // Only have CheckChildInteger0...4 + New = new CheckChildIntegerMatcher(MC->getChildNo(), CI->getValue()); + + if (auto *CCC = dyn_cast<CheckCondCodeMatcher>(MC->getNext())) + if (MC->getChildNo() == 2) // Only have CheckChild2CondCode + New = new CheckChild2CondCodeMatcher(CCC->getCondCodeName()); + + if (New) { + // Insert the new node. + New->setNext(MatcherPtr.release()); + MatcherPtr.reset(New); + // Remove the old one. + MC->setNext(MC->getNext()->takeNext()); + return ContractNodes(MatcherPtr, CGP); + } + } + + // Zap movechild -> moveparent. + if (MoveChildMatcher *MC = dyn_cast<MoveChildMatcher>(N)) + if (MoveParentMatcher *MP = + dyn_cast<MoveParentMatcher>(MC->getNext())) { + MatcherPtr.reset(MP->takeNext()); + return ContractNodes(MatcherPtr, CGP); + } + + // Turn EmitNode->CompleteMatch into MorphNodeTo if we can. + if (EmitNodeMatcher *EN = dyn_cast<EmitNodeMatcher>(N)) + if (CompleteMatchMatcher *CM = + dyn_cast<CompleteMatchMatcher>(EN->getNext())) { + // We can only use MorphNodeTo if the result values match up. + unsigned RootResultFirst = EN->getFirstResultSlot(); + bool ResultsMatch = true; + for (unsigned i = 0, e = CM->getNumResults(); i != e; ++i) + if (CM->getResult(i) != RootResultFirst+i) + ResultsMatch = false; + + // If the selected node defines a subset of the glue/chain results, we + // can't use MorphNodeTo. For example, we can't use MorphNodeTo if the + // matched pattern has a chain but the root node doesn't. + const PatternToMatch &Pattern = CM->getPattern(); + + if (!EN->hasChain() && + Pattern.getSrcPattern()->NodeHasProperty(SDNPHasChain, CGP)) + ResultsMatch = false; + + // If the matched node has glue and the output root doesn't, we can't + // use MorphNodeTo. + // + // NOTE: Strictly speaking, we don't have to check for glue here + // because the code in the pattern generator doesn't handle it right. We + // do it anyway for thoroughness. + if (!EN->hasOutFlag() && + Pattern.getSrcPattern()->NodeHasProperty(SDNPOutGlue, CGP)) + ResultsMatch = false; + + + // If the root result node defines more results than the source root node + // *and* has a chain or glue input, then we can't match it because it + // would end up replacing the extra result with the chain/glue. +#if 0 + if ((EN->hasGlue() || EN->hasChain()) && + EN->getNumNonChainGlueVTs() > ... need to get no results reliably ...) + ResultMatch = false; +#endif + + if (ResultsMatch) { + const SmallVectorImpl<MVT::SimpleValueType> &VTs = EN->getVTList(); + const SmallVectorImpl<unsigned> &Operands = EN->getOperandList(); + MatcherPtr.reset(new MorphNodeToMatcher(EN->getOpcodeName(), + VTs, Operands, + EN->hasChain(), EN->hasInFlag(), + EN->hasOutFlag(), + EN->hasMemRefs(), + EN->getNumFixedArityOperands(), + Pattern)); + return; + } + + // FIXME2: Kill off all the SelectionDAG::SelectNodeTo and getMachineNode + // variants. + } + + ContractNodes(N->getNextPtr(), CGP); + + + // If we have a CheckType/CheckChildType/Record node followed by a + // CheckOpcode, invert the two nodes. We prefer to do structural checks + // before type checks, as this opens opportunities for factoring on targets + // like X86 where many operations are valid on multiple types. + if ((isa<CheckTypeMatcher>(N) || isa<CheckChildTypeMatcher>(N) || + isa<RecordMatcher>(N)) && + isa<CheckOpcodeMatcher>(N->getNext())) { + // Unlink the two nodes from the list. + Matcher *CheckType = MatcherPtr.release(); + Matcher *CheckOpcode = CheckType->takeNext(); + Matcher *Tail = CheckOpcode->takeNext(); + + // Relink them. + MatcherPtr.reset(CheckOpcode); + CheckOpcode->setNext(CheckType); + CheckType->setNext(Tail); + return ContractNodes(MatcherPtr, CGP); + } +} + +/// FindNodeWithKind - Scan a series of matchers looking for a matcher with a +/// specified kind. Return null if we didn't find one otherwise return the +/// matcher. +static Matcher *FindNodeWithKind(Matcher *M, Matcher::KindTy Kind) { + for (; M; M = M->getNext()) + if (M->getKind() == Kind) + return M; + return nullptr; +} + + +/// FactorNodes - Turn matches like this: +/// Scope +/// OPC_CheckType i32 +/// ABC +/// OPC_CheckType i32 +/// XYZ +/// into: +/// OPC_CheckType i32 +/// Scope +/// ABC +/// XYZ +/// +static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) { + // Look for a push node. Iterates instead of recurses to reduce stack usage. + ScopeMatcher *Scope = nullptr; + std::unique_ptr<Matcher> *RebindableMatcherPtr = &InputMatcherPtr; + while (!Scope) { + // If we reached the end of the chain, we're done. + Matcher *N = RebindableMatcherPtr->get(); + if (!N) return; + + // If this is not a push node, just scan for one. + Scope = dyn_cast<ScopeMatcher>(N); + if (!Scope) + RebindableMatcherPtr = &(N->getNextPtr()); + } + std::unique_ptr<Matcher> &MatcherPtr = *RebindableMatcherPtr; + + // Okay, pull together the children of the scope node into a vector so we can + // inspect it more easily. + SmallVector<Matcher*, 32> OptionsToMatch; + + for (unsigned i = 0, e = Scope->getNumChildren(); i != e; ++i) { + // Factor the subexpression. + std::unique_ptr<Matcher> Child(Scope->takeChild(i)); + FactorNodes(Child); + + if (Child) { + // If the child is a ScopeMatcher we can just merge its contents. + if (auto *SM = dyn_cast<ScopeMatcher>(Child.get())) { + for (unsigned j = 0, e = SM->getNumChildren(); j != e; ++j) + OptionsToMatch.push_back(SM->takeChild(j)); + } else { + OptionsToMatch.push_back(Child.release()); + } + } + } + + SmallVector<Matcher*, 32> NewOptionsToMatch; + + // Loop over options to match, merging neighboring patterns with identical + // starting nodes into a shared matcher. + for (unsigned OptionIdx = 0, e = OptionsToMatch.size(); OptionIdx != e;) { + // Find the set of matchers that start with this node. + Matcher *Optn = OptionsToMatch[OptionIdx++]; + + if (OptionIdx == e) { + NewOptionsToMatch.push_back(Optn); + continue; + } + + // See if the next option starts with the same matcher. If the two + // neighbors *do* start with the same matcher, we can factor the matcher out + // of at least these two patterns. See what the maximal set we can merge + // together is. + SmallVector<Matcher*, 8> EqualMatchers; + EqualMatchers.push_back(Optn); + + // Factor all of the known-equal matchers after this one into the same + // group. + while (OptionIdx != e && OptionsToMatch[OptionIdx]->isEqual(Optn)) + EqualMatchers.push_back(OptionsToMatch[OptionIdx++]); + + // If we found a non-equal matcher, see if it is contradictory with the + // current node. If so, we know that the ordering relation between the + // current sets of nodes and this node don't matter. Look past it to see if + // we can merge anything else into this matching group. + unsigned Scan = OptionIdx; + while (1) { + // If we ran out of stuff to scan, we're done. + if (Scan == e) break; + + Matcher *ScanMatcher = OptionsToMatch[Scan]; + + // If we found an entry that matches out matcher, merge it into the set to + // handle. + if (Optn->isEqual(ScanMatcher)) { + // If is equal after all, add the option to EqualMatchers and remove it + // from OptionsToMatch. + EqualMatchers.push_back(ScanMatcher); + OptionsToMatch.erase(OptionsToMatch.begin()+Scan); + --e; + continue; + } + + // If the option we're checking for contradicts the start of the list, + // skip over it. + if (Optn->isContradictory(ScanMatcher)) { + ++Scan; + continue; + } + + // If we're scanning for a simple node, see if it occurs later in the + // sequence. If so, and if we can move it up, it might be contradictory + // or the same as what we're looking for. If so, reorder it. + if (Optn->isSimplePredicateOrRecordNode()) { + Matcher *M2 = FindNodeWithKind(ScanMatcher, Optn->getKind()); + if (M2 && M2 != ScanMatcher && + M2->canMoveBefore(ScanMatcher) && + (M2->isEqual(Optn) || M2->isContradictory(Optn))) { + Matcher *MatcherWithoutM2 = ScanMatcher->unlinkNode(M2); + M2->setNext(MatcherWithoutM2); + OptionsToMatch[Scan] = M2; + continue; + } + } + + // Otherwise, we don't know how to handle this entry, we have to bail. + break; + } + + if (Scan != e && + // Don't print it's obvious nothing extra could be merged anyway. + Scan+1 != e) { + LLVM_DEBUG(errs() << "Couldn't merge this:\n"; Optn->print(errs(), 4); + errs() << "into this:\n"; + OptionsToMatch[Scan]->print(errs(), 4); + if (Scan + 1 != e) OptionsToMatch[Scan + 1]->printOne(errs()); + if (Scan + 2 < e) OptionsToMatch[Scan + 2]->printOne(errs()); + errs() << "\n"); + } + + // If we only found one option starting with this matcher, no factoring is + // possible. + if (EqualMatchers.size() == 1) { + NewOptionsToMatch.push_back(EqualMatchers[0]); + continue; + } + + // Factor these checks by pulling the first node off each entry and + // discarding it. Take the first one off the first entry to reuse. + Matcher *Shared = Optn; + Optn = Optn->takeNext(); + EqualMatchers[0] = Optn; + + // Remove and delete the first node from the other matchers we're factoring. + for (unsigned i = 1, e = EqualMatchers.size(); i != e; ++i) { + Matcher *Tmp = EqualMatchers[i]->takeNext(); + delete EqualMatchers[i]; + EqualMatchers[i] = Tmp; + } + + Shared->setNext(new ScopeMatcher(EqualMatchers)); + + // Recursively factor the newly created node. + FactorNodes(Shared->getNextPtr()); + + NewOptionsToMatch.push_back(Shared); + } + + // If we're down to a single pattern to match, then we don't need this scope + // anymore. + if (NewOptionsToMatch.size() == 1) { + MatcherPtr.reset(NewOptionsToMatch[0]); + return; + } + + if (NewOptionsToMatch.empty()) { + MatcherPtr.reset(); + return; + } + + // If our factoring failed (didn't achieve anything) see if we can simplify in + // other ways. + + // Check to see if all of the leading entries are now opcode checks. If so, + // we can convert this Scope to be a OpcodeSwitch instead. + bool AllOpcodeChecks = true, AllTypeChecks = true; + for (unsigned i = 0, e = NewOptionsToMatch.size(); i != e; ++i) { + // Check to see if this breaks a series of CheckOpcodeMatchers. + if (AllOpcodeChecks && + !isa<CheckOpcodeMatcher>(NewOptionsToMatch[i])) { +#if 0 + if (i > 3) { + errs() << "FAILING OPC #" << i << "\n"; + NewOptionsToMatch[i]->dump(); + } +#endif + AllOpcodeChecks = false; + } + + // Check to see if this breaks a series of CheckTypeMatcher's. + if (AllTypeChecks) { + CheckTypeMatcher *CTM = + cast_or_null<CheckTypeMatcher>(FindNodeWithKind(NewOptionsToMatch[i], + Matcher::CheckType)); + if (!CTM || + // iPTR checks could alias any other case without us knowing, don't + // bother with them. + CTM->getType() == MVT::iPTR || + // SwitchType only works for result #0. + CTM->getResNo() != 0 || + // If the CheckType isn't at the start of the list, see if we can move + // it there. + !CTM->canMoveBefore(NewOptionsToMatch[i])) { +#if 0 + if (i > 3 && AllTypeChecks) { + errs() << "FAILING TYPE #" << i << "\n"; + NewOptionsToMatch[i]->dump(); + } +#endif + AllTypeChecks = false; + } + } + } + + // If all the options are CheckOpcode's, we can form the SwitchOpcode, woot. + if (AllOpcodeChecks) { + StringSet<> Opcodes; + SmallVector<std::pair<const SDNodeInfo*, Matcher*>, 8> Cases; + for (unsigned i = 0, e = NewOptionsToMatch.size(); i != e; ++i) { + CheckOpcodeMatcher *COM = cast<CheckOpcodeMatcher>(NewOptionsToMatch[i]); + assert(Opcodes.insert(COM->getOpcode().getEnumName()).second && + "Duplicate opcodes not factored?"); + Cases.push_back(std::make_pair(&COM->getOpcode(), COM->takeNext())); + delete COM; + } + + MatcherPtr.reset(new SwitchOpcodeMatcher(Cases)); + return; + } + + // If all the options are CheckType's, we can form the SwitchType, woot. + if (AllTypeChecks) { + DenseMap<unsigned, unsigned> TypeEntry; + SmallVector<std::pair<MVT::SimpleValueType, Matcher*>, 8> Cases; + for (unsigned i = 0, e = NewOptionsToMatch.size(); i != e; ++i) { + Matcher* M = FindNodeWithKind(NewOptionsToMatch[i], Matcher::CheckType); + assert(M && isa<CheckTypeMatcher>(M) && "Unknown Matcher type"); + + auto *CTM = cast<CheckTypeMatcher>(M); + Matcher *MatcherWithoutCTM = NewOptionsToMatch[i]->unlinkNode(CTM); + MVT::SimpleValueType CTMTy = CTM->getType(); + delete CTM; + + unsigned &Entry = TypeEntry[CTMTy]; + if (Entry != 0) { + // If we have unfactored duplicate types, then we should factor them. + Matcher *PrevMatcher = Cases[Entry-1].second; + if (ScopeMatcher *SM = dyn_cast<ScopeMatcher>(PrevMatcher)) { + SM->setNumChildren(SM->getNumChildren()+1); + SM->resetChild(SM->getNumChildren()-1, MatcherWithoutCTM); + continue; + } + + Matcher *Entries[2] = { PrevMatcher, MatcherWithoutCTM }; + Cases[Entry-1].second = new ScopeMatcher(Entries); + continue; + } + + Entry = Cases.size()+1; + Cases.push_back(std::make_pair(CTMTy, MatcherWithoutCTM)); + } + + // Make sure we recursively factor any scopes we may have created. + for (auto &M : Cases) { + if (ScopeMatcher *SM = dyn_cast<ScopeMatcher>(M.second)) { + std::unique_ptr<Matcher> Scope(SM); + FactorNodes(Scope); + M.second = Scope.release(); + assert(M.second && "null matcher"); + } + } + + if (Cases.size() != 1) { + MatcherPtr.reset(new SwitchTypeMatcher(Cases)); + } else { + // If we factored and ended up with one case, create it now. + MatcherPtr.reset(new CheckTypeMatcher(Cases[0].first, 0)); + MatcherPtr->setNext(Cases[0].second); + } + return; + } + + + // Reassemble the Scope node with the adjusted children. + Scope->setNumChildren(NewOptionsToMatch.size()); + for (unsigned i = 0, e = NewOptionsToMatch.size(); i != e; ++i) + Scope->resetChild(i, NewOptionsToMatch[i]); +} + +void +llvm::OptimizeMatcher(std::unique_ptr<Matcher> &MatcherPtr, + const CodeGenDAGPatterns &CGP) { + ContractNodes(MatcherPtr, CGP); + FactorNodes(MatcherPtr); +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.cpp new file mode 100644 index 000000000000..7391f6845a4b --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.cpp @@ -0,0 +1,397 @@ +//===- DFAEmitter.cpp - Finite state automaton emitter --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This class can produce a generic deterministic finite state automaton (DFA), +// given a set of possible states and transitions. +// +// The input transitions can be nondeterministic - this class will produce the +// deterministic equivalent state machine. +// +// The generated code can run the DFA and produce an accepted / not accepted +// state and also produce, given a sequence of transitions that results in an +// accepted state, the sequence of intermediate states. This is useful if the +// initial automaton was nondeterministic - it allows mapping back from the DFA +// to the NFA. +// +//===----------------------------------------------------------------------===// +#define DEBUG_TYPE "dfa-emitter" + +#include "DFAEmitter.h" +#include "CodeGenTarget.h" +#include "SequenceToOffsetTable.h" +#include "TableGenBackends.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/UniqueVector.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cassert> +#include <cstdint> +#include <map> +#include <set> +#include <string> +#include <vector> + +using namespace llvm; + +//===----------------------------------------------------------------------===// +// DfaEmitter implementation. This is independent of the GenAutomaton backend. +//===----------------------------------------------------------------------===// + +void DfaEmitter::addTransition(state_type From, state_type To, action_type A) { + Actions.insert(A); + NfaStates.insert(From); + NfaStates.insert(To); + NfaTransitions[{From, A}].push_back(To); + ++NumNfaTransitions; +} + +void DfaEmitter::visitDfaState(const DfaState &DS) { + // For every possible action... + auto FromId = DfaStates.idFor(DS); + for (action_type A : Actions) { + DfaState NewStates; + DfaTransitionInfo TI; + // For every represented state, word pair in the original NFA... + for (state_type FromState : DS) { + // If this action is possible from this state add the transitioned-to + // states to NewStates. + auto I = NfaTransitions.find({FromState, A}); + if (I == NfaTransitions.end()) + continue; + for (state_type &ToState : I->second) { + NewStates.push_back(ToState); + TI.emplace_back(FromState, ToState); + } + } + if (NewStates.empty()) + continue; + // Sort and unique. + sort(NewStates); + NewStates.erase(std::unique(NewStates.begin(), NewStates.end()), + NewStates.end()); + sort(TI); + TI.erase(std::unique(TI.begin(), TI.end()), TI.end()); + unsigned ToId = DfaStates.insert(NewStates); + DfaTransitions.emplace(std::make_pair(FromId, A), std::make_pair(ToId, TI)); + } +} + +void DfaEmitter::constructDfa() { + DfaState Initial(1, /*NFA initial state=*/0); + DfaStates.insert(Initial); + + // Note that UniqueVector starts indices at 1, not zero. + unsigned DfaStateId = 1; + while (DfaStateId <= DfaStates.size()) { + DfaState S = DfaStates[DfaStateId]; + visitDfaState(S); + DfaStateId++; + } +} + +void DfaEmitter::emit(StringRef Name, raw_ostream &OS) { + constructDfa(); + + OS << "// Input NFA has " << NfaStates.size() << " states with " + << NumNfaTransitions << " transitions.\n"; + OS << "// Generated DFA has " << DfaStates.size() << " states with " + << DfaTransitions.size() << " transitions.\n\n"; + + // Implementation note: We don't bake a simple std::pair<> here as it requires + // significantly more effort to parse. A simple test with a large array of + // struct-pairs (N=100000) took clang-10 6s to parse. The same array of + // std::pair<uint64_t, uint64_t> took 242s. Instead we allow the user to + // define the pair type. + // + // FIXME: It may make sense to emit these as ULEB sequences instead of + // pairs of uint64_t. + OS << "// A zero-terminated sequence of NFA state transitions. Every DFA\n"; + OS << "// transition implies a set of NFA transitions. These are referred\n"; + OS << "// to by index in " << Name << "Transitions[].\n"; + + SequenceToOffsetTable<DfaTransitionInfo> Table; + std::map<DfaTransitionInfo, unsigned> EmittedIndices; + for (auto &T : DfaTransitions) + Table.add(T.second.second); + Table.layout(); + OS << "const std::array<NfaStatePair, " << Table.size() << "> " << Name + << "TransitionInfo = {{\n"; + Table.emit( + OS, + [](raw_ostream &OS, std::pair<uint64_t, uint64_t> P) { + OS << "{" << P.first << ", " << P.second << "}"; + }, + "{0ULL, 0ULL}"); + + OS << "}};\n\n"; + + OS << "// A transition in the generated " << Name << " DFA.\n"; + OS << "struct " << Name << "Transition {\n"; + OS << " unsigned FromDfaState; // The transitioned-from DFA state.\n"; + OS << " "; + printActionType(OS); + OS << " Action; // The input symbol that causes this transition.\n"; + OS << " unsigned ToDfaState; // The transitioned-to DFA state.\n"; + OS << " unsigned InfoIdx; // Start index into " << Name + << "TransitionInfo.\n"; + OS << "};\n\n"; + + OS << "// A table of DFA transitions, ordered by {FromDfaState, Action}.\n"; + OS << "// The initial state is 1, not zero.\n"; + OS << "const std::array<" << Name << "Transition, " + << DfaTransitions.size() << "> " << Name << "Transitions = {{\n"; + for (auto &KV : DfaTransitions) { + dfa_state_type From = KV.first.first; + dfa_state_type To = KV.second.first; + action_type A = KV.first.second; + unsigned InfoIdx = Table.get(KV.second.second); + OS << " {" << From << ", "; + printActionValue(A, OS); + OS << ", " << To << ", " << InfoIdx << "},\n"; + } + OS << "\n}};\n\n"; +} + +void DfaEmitter::printActionType(raw_ostream &OS) { OS << "uint64_t"; } + +void DfaEmitter::printActionValue(action_type A, raw_ostream &OS) { OS << A; } + +//===----------------------------------------------------------------------===// +// AutomatonEmitter implementation +//===----------------------------------------------------------------------===// + +namespace { +// FIXME: This entire discriminated union could be removed with c++17: +// using Action = std::variant<Record *, unsigned, std::string>; +struct Action { + Record *R = nullptr; + unsigned I = 0; + std::string S = nullptr; + + Action() = default; + Action(Record *R, unsigned I, std::string S) : R(R), I(I), S(S) {} + + void print(raw_ostream &OS) const { + if (R) + OS << R->getName(); + else if (!S.empty()) + OS << '"' << S << '"'; + else + OS << I; + } + bool operator<(const Action &Other) const { + return std::make_tuple(R, I, S) < + std::make_tuple(Other.R, Other.I, Other.S); + } +}; + +using ActionTuple = std::vector<Action>; +class Automaton; + +class Transition { + uint64_t NewState; + // The tuple of actions that causes this transition. + ActionTuple Actions; + // The types of the actions; this is the same across all transitions. + SmallVector<std::string, 4> Types; + +public: + Transition(Record *R, Automaton *Parent); + const ActionTuple &getActions() { return Actions; } + SmallVector<std::string, 4> getTypes() { return Types; } + + bool canTransitionFrom(uint64_t State); + uint64_t transitionFrom(uint64_t State); +}; + +class Automaton { + RecordKeeper &Records; + Record *R; + std::vector<Transition> Transitions; + /// All possible action tuples, uniqued. + UniqueVector<ActionTuple> Actions; + /// The fields within each Transition object to find the action symbols. + std::vector<StringRef> ActionSymbolFields; + +public: + Automaton(RecordKeeper &Records, Record *R); + void emit(raw_ostream &OS); + + ArrayRef<StringRef> getActionSymbolFields() { return ActionSymbolFields; } + /// If the type of action A has been overridden (there exists a field + /// "TypeOf_A") return that, otherwise return the empty string. + StringRef getActionSymbolType(StringRef A); +}; + +class AutomatonEmitter { + RecordKeeper &Records; + +public: + AutomatonEmitter(RecordKeeper &R) : Records(R) {} + void run(raw_ostream &OS); +}; + +/// A DfaEmitter implementation that can print our variant action type. +class CustomDfaEmitter : public DfaEmitter { + const UniqueVector<ActionTuple> &Actions; + std::string TypeName; + +public: + CustomDfaEmitter(const UniqueVector<ActionTuple> &Actions, StringRef TypeName) + : Actions(Actions), TypeName(TypeName) {} + + void printActionType(raw_ostream &OS) override; + void printActionValue(action_type A, raw_ostream &OS) override; +}; +} // namespace + +void AutomatonEmitter::run(raw_ostream &OS) { + for (Record *R : Records.getAllDerivedDefinitions("GenericAutomaton")) { + Automaton A(Records, R); + OS << "#ifdef GET_" << R->getName() << "_DECL\n"; + A.emit(OS); + OS << "#endif // GET_" << R->getName() << "_DECL\n"; + } +} + +Automaton::Automaton(RecordKeeper &Records, Record *R) + : Records(Records), R(R) { + LLVM_DEBUG(dbgs() << "Emitting automaton for " << R->getName() << "\n"); + ActionSymbolFields = R->getValueAsListOfStrings("SymbolFields"); +} + +void Automaton::emit(raw_ostream &OS) { + StringRef TransitionClass = R->getValueAsString("TransitionClass"); + for (Record *T : Records.getAllDerivedDefinitions(TransitionClass)) { + assert(T->isSubClassOf("Transition")); + Transitions.emplace_back(T, this); + Actions.insert(Transitions.back().getActions()); + } + + LLVM_DEBUG(dbgs() << " Action alphabet cardinality: " << Actions.size() + << "\n"); + LLVM_DEBUG(dbgs() << " Each state has " << Transitions.size() + << " potential transitions.\n"); + + StringRef Name = R->getName(); + + CustomDfaEmitter Emitter(Actions, std::string(Name) + "Action"); + // Starting from the initial state, build up a list of possible states and + // transitions. + std::deque<uint64_t> Worklist(1, 0); + std::set<uint64_t> SeenStates; + unsigned NumTransitions = 0; + SeenStates.insert(Worklist.front()); + while (!Worklist.empty()) { + uint64_t State = Worklist.front(); + Worklist.pop_front(); + for (Transition &T : Transitions) { + if (!T.canTransitionFrom(State)) + continue; + uint64_t NewState = T.transitionFrom(State); + if (SeenStates.emplace(NewState).second) + Worklist.emplace_back(NewState); + ++NumTransitions; + Emitter.addTransition(State, NewState, Actions.idFor(T.getActions())); + } + } + LLVM_DEBUG(dbgs() << " NFA automaton has " << SeenStates.size() + << " states with " << NumTransitions << " transitions.\n"); + + const auto &ActionTypes = Transitions.back().getTypes(); + OS << "// The type of an action in the " << Name << " automaton.\n"; + if (ActionTypes.size() == 1) { + OS << "using " << Name << "Action = " << ActionTypes[0] << ";\n"; + } else { + OS << "using " << Name << "Action = std::tuple<" << join(ActionTypes, ", ") + << ">;\n"; + } + OS << "\n"; + + Emitter.emit(Name, OS); +} + +StringRef Automaton::getActionSymbolType(StringRef A) { + Twine Ty = "TypeOf_" + A; + if (!R->getValue(Ty.str())) + return ""; + return R->getValueAsString(Ty.str()); +} + +Transition::Transition(Record *R, Automaton *Parent) { + BitsInit *NewStateInit = R->getValueAsBitsInit("NewState"); + NewState = 0; + assert(NewStateInit->getNumBits() <= sizeof(uint64_t) * 8 && + "State cannot be represented in 64 bits!"); + for (unsigned I = 0; I < NewStateInit->getNumBits(); ++I) { + if (auto *Bit = dyn_cast<BitInit>(NewStateInit->getBit(I))) { + if (Bit->getValue()) + NewState |= 1ULL << I; + } + } + + for (StringRef A : Parent->getActionSymbolFields()) { + RecordVal *SymbolV = R->getValue(A); + if (auto *Ty = dyn_cast<RecordRecTy>(SymbolV->getType())) { + Actions.emplace_back(R->getValueAsDef(A), 0, ""); + Types.emplace_back(Ty->getAsString()); + } else if (isa<IntRecTy>(SymbolV->getType())) { + Actions.emplace_back(nullptr, R->getValueAsInt(A), ""); + Types.emplace_back("unsigned"); + } else if (isa<StringRecTy>(SymbolV->getType()) || + isa<CodeRecTy>(SymbolV->getType())) { + Actions.emplace_back(nullptr, 0, std::string(R->getValueAsString(A))); + Types.emplace_back("std::string"); + } else { + report_fatal_error("Unhandled symbol type!"); + } + + StringRef TypeOverride = Parent->getActionSymbolType(A); + if (!TypeOverride.empty()) + Types.back() = std::string(TypeOverride); + } +} + +bool Transition::canTransitionFrom(uint64_t State) { + if ((State & NewState) == 0) + // The bits we want to set are not set; + return true; + return false; +} + +uint64_t Transition::transitionFrom(uint64_t State) { + return State | NewState; +} + +void CustomDfaEmitter::printActionType(raw_ostream &OS) { OS << TypeName; } + +void CustomDfaEmitter::printActionValue(action_type A, raw_ostream &OS) { + const ActionTuple &AT = Actions[A]; + if (AT.size() > 1) + OS << "std::make_tuple("; + bool First = true; + for (const auto &SingleAction : AT) { + if (!First) + OS << ", "; + First = false; + SingleAction.print(OS); + } + if (AT.size() > 1) + OS << ")"; +} + +namespace llvm { + +void EmitAutomata(RecordKeeper &RK, raw_ostream &OS) { + AutomatonEmitter(RK).run(OS); +} + +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.h b/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.h new file mode 100644 index 000000000000..44e5d97d544f --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DFAEmitter.h @@ -0,0 +1,107 @@ +//===--------------------- DfaEmitter.h -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Defines a generic automaton builder. This takes a set of transitions and +// states that represent a nondeterministic finite state automaton (NFA) and +// emits a determinized DFA in a form that include/llvm/Support/Automaton.h can +// drive. +// +// See file llvm/TableGen/Automaton.td for the TableGen API definition. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_DFAEMITTER_H +#define LLVM_UTILS_TABLEGEN_DFAEMITTER_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/UniqueVector.h" +#include <map> +#include <set> + +namespace llvm { + +class raw_ostream; +class StringRef; + +/// Construct a deterministic finite state automaton from possible +/// nondeterministic state and transition data. +/// +/// The state type is a 64-bit unsigned integer. The generated automaton is +/// invariant to the sparsity of the state representation - its size is only +/// a function of the cardinality of the set of states. +/// +/// The inputs to this emitter are considered to define a nondeterministic +/// finite state automaton (NFA). This is then converted to a DFA during +/// emission. The emitted tables can be used to by +/// include/llvm/Support/Automaton.h. +class DfaEmitter { +public: + // The type of an NFA state. The initial state is always zero. + using state_type = uint64_t; + // The type of an action. + using action_type = uint64_t; + + DfaEmitter() = default; + virtual ~DfaEmitter() = default; + + void addTransition(state_type From, state_type To, action_type A); + void emit(StringRef Name, raw_ostream &OS); + +protected: + /// Emit the C++ type of an action to OS. + virtual void printActionType(raw_ostream &OS); + /// Emit the C++ value of an action A to OS. + virtual void printActionValue(action_type A, raw_ostream &OS); + +private: + /// The state type of deterministic states. These are only used internally to + /// this class. This is an ID into the DfaStates UniqueVector. + using dfa_state_type = unsigned; + + /// The actual representation of a DFA state, which is a union of one or more + /// NFA states. + using DfaState = SmallVector<state_type, 4>; + + /// A DFA transition consists of a set of NFA states transitioning to a + /// new set of NFA states. The DfaTransitionInfo tracks, for every + /// transitioned-from NFA state, a set of valid transitioned-to states. + /// + /// Emission of this transition relation allows algorithmic determination of + /// the possible candidate NFA paths taken under a given input sequence to + /// reach a given DFA state. + using DfaTransitionInfo = SmallVector<std::pair<state_type, state_type>, 4>; + + /// The set of all possible actions. + std::set<action_type> Actions; + + /// The set of nondeterministic transitions. A state-action pair can + /// transition to multiple target states. + std::map<std::pair<state_type, action_type>, std::vector<state_type>> + NfaTransitions; + std::set<state_type> NfaStates; + unsigned NumNfaTransitions = 0; + + /// The set of deterministic states. DfaStates.getId(DfaState) returns an ID, + /// which is dfa_state_type. Note that because UniqueVector reserves state + /// zero, the initial DFA state is always 1. + UniqueVector<DfaState> DfaStates; + /// The set of deterministic transitions. A state-action pair has only a + /// single target state. + std::map<std::pair<dfa_state_type, action_type>, + std::pair<dfa_state_type, DfaTransitionInfo>> + DfaTransitions; + + /// Visit all NFA states and construct the DFA. + void constructDfa(); + /// Visit a single DFA state and construct all possible transitions to new DFA + /// states. + void visitDfaState(const DfaState &DS); +}; + +} // namespace llvm + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/DFAPacketizerEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DFAPacketizerEmitter.cpp new file mode 100644 index 000000000000..bc4a084b3224 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DFAPacketizerEmitter.cpp @@ -0,0 +1,364 @@ +//===- DFAPacketizerEmitter.cpp - Packetization DFA for a VLIW machine ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This class parses the Schedule.td file and produces an API that can be used +// to reason about whether an instruction can be added to a packet on a VLIW +// architecture. The class internally generates a deterministic finite +// automaton (DFA) that models all possible mappings of machine instructions +// to functional units as instructions are added to a packet. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "dfa-emitter" + +#include "CodeGenSchedule.h" +#include "CodeGenTarget.h" +#include "DFAEmitter.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cassert> +#include <cstdint> +#include <map> +#include <set> +#include <string> +#include <unordered_map> +#include <vector> + +using namespace llvm; + +// We use a uint64_t to represent a resource bitmask. +#define DFA_MAX_RESOURCES 64 + +namespace { +using ResourceVector = SmallVector<uint64_t, 4>; + +struct ScheduleClass { + /// The parent itinerary index (processor model ID). + unsigned ItineraryID; + + /// Index within this itinerary of the schedule class. + unsigned Idx; + + /// The index within the uniqued set of required resources of Resources. + unsigned ResourcesIdx; + + /// Conjunctive list of resource requirements: + /// {a|b, b|c} => (a OR b) AND (b or c). + /// Resources are unique across all itineraries. + ResourceVector Resources; +}; + +// Generates and prints out the DFA for resource tracking. +class DFAPacketizerEmitter { +private: + std::string TargetName; + RecordKeeper &Records; + + UniqueVector<ResourceVector> UniqueResources; + std::vector<ScheduleClass> ScheduleClasses; + std::map<std::string, uint64_t> FUNameToBitsMap; + std::map<unsigned, uint64_t> ComboBitToBitsMap; + +public: + DFAPacketizerEmitter(RecordKeeper &R); + + // Construct a map of function unit names to bits. + int collectAllFuncUnits( + ArrayRef<const CodeGenProcModel *> ProcModels); + + // Construct a map from a combo function unit bit to the bits of all included + // functional units. + int collectAllComboFuncs(ArrayRef<Record *> ComboFuncList); + + ResourceVector getResourcesForItinerary(Record *Itinerary); + void createScheduleClasses(unsigned ItineraryIdx, const RecVec &Itineraries); + + // Emit code for a subset of itineraries. + void emitForItineraries(raw_ostream &OS, + std::vector<const CodeGenProcModel *> &ProcItinList, + std::string DFAName); + + void run(raw_ostream &OS); +}; +} // end anonymous namespace + +DFAPacketizerEmitter::DFAPacketizerEmitter(RecordKeeper &R) + : TargetName(std::string(CodeGenTarget(R).getName())), Records(R) {} + +int DFAPacketizerEmitter::collectAllFuncUnits( + ArrayRef<const CodeGenProcModel *> ProcModels) { + LLVM_DEBUG(dbgs() << "-------------------------------------------------------" + "----------------------\n"); + LLVM_DEBUG(dbgs() << "collectAllFuncUnits"); + LLVM_DEBUG(dbgs() << " (" << ProcModels.size() << " itineraries)\n"); + + std::set<Record *> ProcItinList; + for (const CodeGenProcModel *Model : ProcModels) + ProcItinList.insert(Model->ItinsDef); + + int totalFUs = 0; + // Parse functional units for all the itineraries. + for (Record *Proc : ProcItinList) { + std::vector<Record *> FUs = Proc->getValueAsListOfDefs("FU"); + + LLVM_DEBUG(dbgs() << " FU:" + << " (" << FUs.size() << " FUs) " << Proc->getName()); + + // Convert macros to bits for each stage. + unsigned numFUs = FUs.size(); + for (unsigned j = 0; j < numFUs; ++j) { + assert((j < DFA_MAX_RESOURCES) && + "Exceeded maximum number of representable resources"); + uint64_t FuncResources = 1ULL << j; + FUNameToBitsMap[std::string(FUs[j]->getName())] = FuncResources; + LLVM_DEBUG(dbgs() << " " << FUs[j]->getName() << ":0x" + << Twine::utohexstr(FuncResources)); + } + totalFUs += numFUs; + LLVM_DEBUG(dbgs() << "\n"); + } + return totalFUs; +} + +int DFAPacketizerEmitter::collectAllComboFuncs(ArrayRef<Record *> ComboFuncList) { + LLVM_DEBUG(dbgs() << "-------------------------------------------------------" + "----------------------\n"); + LLVM_DEBUG(dbgs() << "collectAllComboFuncs"); + LLVM_DEBUG(dbgs() << " (" << ComboFuncList.size() << " sets)\n"); + + int numCombos = 0; + for (unsigned i = 0, N = ComboFuncList.size(); i < N; ++i) { + Record *Func = ComboFuncList[i]; + std::vector<Record *> FUs = Func->getValueAsListOfDefs("CFD"); + + LLVM_DEBUG(dbgs() << " CFD:" << i << " (" << FUs.size() << " combo FUs) " + << Func->getName() << "\n"); + + // Convert macros to bits for each stage. + for (unsigned j = 0, N = FUs.size(); j < N; ++j) { + assert((j < DFA_MAX_RESOURCES) && + "Exceeded maximum number of DFA resources"); + Record *FuncData = FUs[j]; + Record *ComboFunc = FuncData->getValueAsDef("TheComboFunc"); + const std::vector<Record *> &FuncList = + FuncData->getValueAsListOfDefs("FuncList"); + const std::string &ComboFuncName = std::string(ComboFunc->getName()); + uint64_t ComboBit = FUNameToBitsMap[ComboFuncName]; + uint64_t ComboResources = ComboBit; + LLVM_DEBUG(dbgs() << " combo: " << ComboFuncName << ":0x" + << Twine::utohexstr(ComboResources) << "\n"); + for (unsigned k = 0, M = FuncList.size(); k < M; ++k) { + std::string FuncName = std::string(FuncList[k]->getName()); + uint64_t FuncResources = FUNameToBitsMap[FuncName]; + LLVM_DEBUG(dbgs() << " " << FuncName << ":0x" + << Twine::utohexstr(FuncResources) << "\n"); + ComboResources |= FuncResources; + } + ComboBitToBitsMap[ComboBit] = ComboResources; + numCombos++; + LLVM_DEBUG(dbgs() << " => combo bits: " << ComboFuncName << ":0x" + << Twine::utohexstr(ComboBit) << " = 0x" + << Twine::utohexstr(ComboResources) << "\n"); + } + } + return numCombos; +} + +ResourceVector +DFAPacketizerEmitter::getResourcesForItinerary(Record *Itinerary) { + ResourceVector Resources; + assert(Itinerary); + for (Record *StageDef : Itinerary->getValueAsListOfDefs("Stages")) { + uint64_t StageResources = 0; + for (Record *Unit : StageDef->getValueAsListOfDefs("Units")) { + StageResources |= FUNameToBitsMap[std::string(Unit->getName())]; + } + if (StageResources != 0) + Resources.push_back(StageResources); + } + return Resources; +} + +void DFAPacketizerEmitter::createScheduleClasses(unsigned ItineraryIdx, + const RecVec &Itineraries) { + unsigned Idx = 0; + for (Record *Itinerary : Itineraries) { + if (!Itinerary) { + ScheduleClasses.push_back({ItineraryIdx, Idx++, 0, ResourceVector{}}); + continue; + } + ResourceVector Resources = getResourcesForItinerary(Itinerary); + ScheduleClasses.push_back( + {ItineraryIdx, Idx++, UniqueResources.insert(Resources), Resources}); + } +} + +// +// Run the worklist algorithm to generate the DFA. +// +void DFAPacketizerEmitter::run(raw_ostream &OS) { + OS << "\n" + << "#include \"llvm/CodeGen/DFAPacketizer.h\"\n"; + OS << "namespace llvm {\n"; + + CodeGenTarget CGT(Records); + CodeGenSchedModels CGS(Records, CGT); + + std::unordered_map<std::string, std::vector<const CodeGenProcModel *>> + ItinsByNamespace; + for (const CodeGenProcModel &ProcModel : CGS.procModels()) { + if (ProcModel.hasItineraries()) { + auto NS = ProcModel.ItinsDef->getValueAsString("PacketizerNamespace"); + ItinsByNamespace[std::string(NS)].push_back(&ProcModel); + } + } + + for (auto &KV : ItinsByNamespace) + emitForItineraries(OS, KV.second, KV.first); + OS << "} // end namespace llvm\n"; +} + +void DFAPacketizerEmitter::emitForItineraries( + raw_ostream &OS, std::vector<const CodeGenProcModel *> &ProcModels, + std::string DFAName) { + OS << "} // end namespace llvm\n\n"; + OS << "namespace {\n"; + collectAllFuncUnits(ProcModels); + collectAllComboFuncs(Records.getAllDerivedDefinitions("ComboFuncUnits")); + + // Collect the itineraries. + DenseMap<const CodeGenProcModel *, unsigned> ProcModelStartIdx; + for (const CodeGenProcModel *Model : ProcModels) { + assert(Model->hasItineraries()); + ProcModelStartIdx[Model] = ScheduleClasses.size(); + createScheduleClasses(Model->Index, Model->ItinDefList); + } + + // Output the mapping from ScheduleClass to ResourcesIdx. + unsigned Idx = 0; + OS << "constexpr unsigned " << TargetName << DFAName + << "ResourceIndices[] = {"; + for (const ScheduleClass &SC : ScheduleClasses) { + if (Idx++ % 32 == 0) + OS << "\n "; + OS << SC.ResourcesIdx << ", "; + } + OS << "\n};\n\n"; + + // And the mapping from Itinerary index into the previous table. + OS << "constexpr unsigned " << TargetName << DFAName + << "ProcResourceIndexStart[] = {\n"; + OS << " 0, // NoSchedModel\n"; + for (const CodeGenProcModel *Model : ProcModels) { + OS << " " << ProcModelStartIdx[Model] << ", // " << Model->ModelName + << "\n"; + } + OS << ScheduleClasses.size() << "\n};\n\n"; + + // The type of a state in the nondeterministic automaton we're defining. + using NfaStateTy = uint64_t; + + // Given a resource state, return all resource states by applying + // InsnClass. + auto applyInsnClass = [&](const ResourceVector &InsnClass, + NfaStateTy State) -> std::deque<NfaStateTy> { + std::deque<NfaStateTy> V(1, State); + // Apply every stage in the class individually. + for (NfaStateTy Stage : InsnClass) { + // Apply this stage to every existing member of V in turn. + size_t Sz = V.size(); + for (unsigned I = 0; I < Sz; ++I) { + NfaStateTy S = V.front(); + V.pop_front(); + + // For this stage, state combination, try all possible resources. + for (unsigned J = 0; J < DFA_MAX_RESOURCES; ++J) { + NfaStateTy ResourceMask = 1ULL << J; + if ((ResourceMask & Stage) == 0) + // This resource isn't required by this stage. + continue; + NfaStateTy Combo = ComboBitToBitsMap[ResourceMask]; + if (Combo && ((~S & Combo) != Combo)) + // This combo units bits are not available. + continue; + NfaStateTy ResultingResourceState = S | ResourceMask | Combo; + if (ResultingResourceState == S) + continue; + V.push_back(ResultingResourceState); + } + } + } + return V; + }; + + // Given a resource state, return a quick (conservative) guess as to whether + // InsnClass can be applied. This is a filter for the more heavyweight + // applyInsnClass. + auto canApplyInsnClass = [](const ResourceVector &InsnClass, + NfaStateTy State) -> bool { + for (NfaStateTy Resources : InsnClass) { + if ((State | Resources) == State) + return false; + } + return true; + }; + + DfaEmitter Emitter; + std::deque<NfaStateTy> Worklist(1, 0); + std::set<NfaStateTy> SeenStates; + SeenStates.insert(Worklist.front()); + while (!Worklist.empty()) { + NfaStateTy State = Worklist.front(); + Worklist.pop_front(); + for (const ResourceVector &Resources : UniqueResources) { + if (!canApplyInsnClass(Resources, State)) + continue; + unsigned ResourcesID = UniqueResources.idFor(Resources); + for (uint64_t NewState : applyInsnClass(Resources, State)) { + if (SeenStates.emplace(NewState).second) + Worklist.emplace_back(NewState); + Emitter.addTransition(State, NewState, ResourcesID); + } + } + } + + std::string TargetAndDFAName = TargetName + DFAName; + Emitter.emit(TargetAndDFAName, OS); + OS << "} // end anonymous namespace\n\n"; + + std::string SubTargetClassName = TargetName + "GenSubtargetInfo"; + OS << "namespace llvm {\n"; + OS << "DFAPacketizer *" << SubTargetClassName << "::" + << "create" << DFAName + << "DFAPacketizer(const InstrItineraryData *IID) const {\n" + << " static Automaton<uint64_t> A(ArrayRef<" << TargetAndDFAName + << "Transition>(" << TargetAndDFAName << "Transitions), " + << TargetAndDFAName << "TransitionInfo);\n" + << " unsigned ProcResIdxStart = " << TargetAndDFAName + << "ProcResourceIndexStart[IID->SchedModel.ProcID];\n" + << " unsigned ProcResIdxNum = " << TargetAndDFAName + << "ProcResourceIndexStart[IID->SchedModel.ProcID + 1] - " + "ProcResIdxStart;\n" + << " return new DFAPacketizer(IID, A, {&" << TargetAndDFAName + << "ResourceIndices[ProcResIdxStart], ProcResIdxNum});\n" + << "\n}\n\n"; +} + +namespace llvm { + +void EmitDFAPacketizer(RecordKeeper &RK, raw_ostream &OS) { + emitSourceFileHeader("Target DFA Packetizer Tables", OS); + DFAPacketizerEmitter(RK).run(OS); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/DirectiveEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DirectiveEmitter.cpp new file mode 100644 index 000000000000..2061ff1fdd1a --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DirectiveEmitter.cpp @@ -0,0 +1,524 @@ +//===- DirectiveEmitter.cpp - Directive Language Emitter ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// DirectiveEmitter uses the descriptions of directives and clauses to construct +// common code declarations to be used in Frontends. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" + +using namespace llvm; + +namespace { +// Simple RAII helper for defining ifdef-undef-endif scopes. +class IfDefScope { +public: + IfDefScope(StringRef Name, raw_ostream &OS) : Name(Name), OS(OS) { + OS << "#ifdef " << Name << "\n" + << "#undef " << Name << "\n"; + } + + ~IfDefScope() { OS << "\n#endif // " << Name << "\n\n"; } + +private: + StringRef Name; + raw_ostream &OS; +}; +} // end anonymous namespace + +namespace llvm { + +// Get Directive or Clause name formatted by replacing whitespaces with +// underscores. +std::string getFormattedName(StringRef Name) { + std::string N = Name.str(); + std::replace(N.begin(), N.end(), ' ', '_'); + return N; +} + +// Generate enum class +void GenerateEnumClass(const std::vector<Record *> &Records, raw_ostream &OS, + StringRef Enum, StringRef Prefix, StringRef CppNamespace, + bool MakeEnumAvailableInNamespace) { + OS << "\n"; + OS << "enum class " << Enum << " {\n"; + for (const auto &R : Records) { + const auto Name = R->getValueAsString("name"); + OS << " " << Prefix << getFormattedName(Name) << ",\n"; + } + OS << "};\n"; + OS << "\n"; + OS << "static constexpr std::size_t " << Enum + << "_enumSize = " << Records.size() << ";\n"; + + // Make the enum values available in the defined namespace. This allows us to + // write something like Enum_X if we have a `using namespace <CppNamespace>`. + // At the same time we do not loose the strong type guarantees of the enum + // class, that is we cannot pass an unsigned as Directive without an explicit + // cast. + if (MakeEnumAvailableInNamespace) { + OS << "\n"; + for (const auto &R : Records) { + const auto FormattedName = getFormattedName(R->getValueAsString("name")); + OS << "constexpr auto " << Prefix << FormattedName << " = " + << "llvm::" << CppNamespace << "::" << Enum << "::" << Prefix + << FormattedName << ";\n"; + } + } +} + +// Generate the declaration section for the enumeration in the directive +// language +void EmitDirectivesDecl(RecordKeeper &Records, raw_ostream &OS) { + + const auto &DirectiveLanguages = + Records.getAllDerivedDefinitions("DirectiveLanguage"); + + if (DirectiveLanguages.size() != 1) { + PrintError("A single definition of DirectiveLanguage is needed."); + return; + } + + const auto &DirectiveLanguage = DirectiveLanguages[0]; + StringRef LanguageName = DirectiveLanguage->getValueAsString("name"); + StringRef DirectivePrefix = + DirectiveLanguage->getValueAsString("directivePrefix"); + StringRef ClausePrefix = DirectiveLanguage->getValueAsString("clausePrefix"); + StringRef CppNamespace = DirectiveLanguage->getValueAsString("cppNamespace"); + bool MakeEnumAvailableInNamespace = + DirectiveLanguage->getValueAsBit("makeEnumAvailableInNamespace"); + bool EnableBitmaskEnumInNamespace = + DirectiveLanguage->getValueAsBit("enableBitmaskEnumInNamespace"); + + OS << "#ifndef LLVM_" << LanguageName << "_INC\n"; + OS << "#define LLVM_" << LanguageName << "_INC\n"; + + if (EnableBitmaskEnumInNamespace) + OS << "\n#include \"llvm/ADT/BitmaskEnum.h\"\n"; + + OS << "\n"; + OS << "namespace llvm {\n"; + OS << "class StringRef;\n"; + + // Open namespaces defined in the directive language + llvm::SmallVector<StringRef, 2> Namespaces; + llvm::SplitString(CppNamespace, Namespaces, "::"); + for (auto Ns : Namespaces) + OS << "namespace " << Ns << " {\n"; + + if (EnableBitmaskEnumInNamespace) + OS << "\nLLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();\n"; + + // Emit Directive enumeration + const auto &Directives = Records.getAllDerivedDefinitions("Directive"); + GenerateEnumClass(Directives, OS, "Directive", DirectivePrefix, CppNamespace, + MakeEnumAvailableInNamespace); + + // Emit Clause enumeration + const auto &Clauses = Records.getAllDerivedDefinitions("Clause"); + GenerateEnumClass(Clauses, OS, "Clause", ClausePrefix, CppNamespace, + MakeEnumAvailableInNamespace); + + // Generic function signatures + OS << "\n"; + OS << "// Enumeration helper functions\n"; + OS << "Directive get" << LanguageName + << "DirectiveKind(llvm::StringRef Str);\n"; + OS << "\n"; + OS << "llvm::StringRef get" << LanguageName + << "DirectiveName(Directive D);\n"; + OS << "\n"; + OS << "Clause get" << LanguageName << "ClauseKind(llvm::StringRef Str);\n"; + OS << "\n"; + OS << "llvm::StringRef get" << LanguageName << "ClauseName(Clause C);\n"; + OS << "\n"; + OS << "/// Return true if \\p C is a valid clause for \\p D in version \\p " + << "Version.\n"; + OS << "bool isAllowedClauseForDirective(Directive D, " + << "Clause C, unsigned Version);\n"; + OS << "\n"; + + // Closing namespaces + for (auto Ns : llvm::reverse(Namespaces)) + OS << "} // namespace " << Ns << "\n"; + + OS << "} // namespace llvm\n"; + + OS << "#endif // LLVM_" << LanguageName << "_INC\n"; +} + +// Generate function implementation for get<Enum>Name(StringRef Str) +void GenerateGetName(const std::vector<Record *> &Records, raw_ostream &OS, + StringRef Enum, StringRef Prefix, StringRef LanguageName, + StringRef Namespace) { + OS << "\n"; + OS << "llvm::StringRef llvm::" << Namespace << "::get" << LanguageName << Enum + << "Name(" << Enum << " Kind) {\n"; + OS << " switch (Kind) {\n"; + for (const auto &R : Records) { + const auto Name = R->getValueAsString("name"); + const auto AlternativeName = R->getValueAsString("alternativeName"); + OS << " case " << Prefix << getFormattedName(Name) << ":\n"; + OS << " return \""; + if (AlternativeName.empty()) + OS << Name; + else + OS << AlternativeName; + OS << "\";\n"; + } + OS << " }\n"; // switch + OS << " llvm_unreachable(\"Invalid " << LanguageName << " " << Enum + << " kind\");\n"; + OS << "}\n"; +} + +// Generate function implementation for get<Enum>Kind(StringRef Str) +void GenerateGetKind(const std::vector<Record *> &Records, raw_ostream &OS, + StringRef Enum, StringRef Prefix, StringRef LanguageName, + StringRef Namespace, bool ImplicitAsUnknown) { + + auto DefaultIt = std::find_if(Records.begin(), Records.end(), [](Record *R) { + return R->getValueAsBit("isDefault") == true; + }); + + if (DefaultIt == Records.end()) { + PrintError("A least one " + Enum + " must be defined as default."); + return; + } + + const auto FormattedDefaultName = + getFormattedName((*DefaultIt)->getValueAsString("name")); + + OS << "\n"; + OS << Enum << " llvm::" << Namespace << "::get" << LanguageName << Enum + << "Kind(llvm::StringRef Str) {\n"; + OS << " return llvm::StringSwitch<" << Enum << ">(Str)\n"; + + for (const auto &R : Records) { + const auto Name = R->getValueAsString("name"); + if (ImplicitAsUnknown && R->getValueAsBit("isImplicit")) { + OS << " .Case(\"" << Name << "\"," << Prefix << FormattedDefaultName + << ")\n"; + } else { + OS << " .Case(\"" << Name << "\"," << Prefix << getFormattedName(Name) + << ")\n"; + } + } + OS << " .Default(" << Prefix << FormattedDefaultName << ");\n"; + OS << "}\n"; +} + +void GenerateCaseForVersionedClauses(const std::vector<Record *> &Clauses, + raw_ostream &OS, StringRef DirectiveName, + StringRef DirectivePrefix, + StringRef ClausePrefix, + llvm::StringSet<> &Cases) { + for (const auto &C : Clauses) { + const auto MinVersion = C->getValueAsInt("minVersion"); + const auto MaxVersion = C->getValueAsInt("maxVersion"); + const auto SpecificClause = C->getValueAsDef("clause"); + const auto ClauseName = + getFormattedName(SpecificClause->getValueAsString("name")); + + if (Cases.find(ClauseName) == Cases.end()) { + Cases.insert(ClauseName); + OS << " case " << ClausePrefix << ClauseName << ":\n"; + OS << " return " << MinVersion << " <= Version && " << MaxVersion + << " >= Version;\n"; + } + } +} + +// Generate the isAllowedClauseForDirective function implementation. +void GenerateIsAllowedClause(const std::vector<Record *> &Directives, + raw_ostream &OS, StringRef LanguageName, + StringRef DirectivePrefix, StringRef ClausePrefix, + StringRef CppNamespace) { + OS << "\n"; + OS << "bool llvm::" << CppNamespace << "::isAllowedClauseForDirective(" + << "Directive D, Clause C, unsigned Version) {\n"; + OS << " assert(unsigned(D) <= llvm::" << CppNamespace + << "::Directive_enumSize);\n"; + OS << " assert(unsigned(C) <= llvm::" << CppNamespace + << "::Clause_enumSize);\n"; + + OS << " switch (D) {\n"; + + for (const auto &D : Directives) { + + const auto DirectiveName = D->getValueAsString("name"); + const auto &AllowedClauses = D->getValueAsListOfDefs("allowedClauses"); + const auto &AllowedOnceClauses = + D->getValueAsListOfDefs("allowedOnceClauses"); + const auto &AllowedExclusiveClauses = + D->getValueAsListOfDefs("allowedExclusiveClauses"); + const auto &RequiredClauses = D->getValueAsListOfDefs("requiredClauses"); + + OS << " case " << DirectivePrefix << getFormattedName(DirectiveName) + << ":\n"; + if (AllowedClauses.size() == 0 && AllowedOnceClauses.size() == 0 && + AllowedExclusiveClauses.size() == 0 && RequiredClauses.size() == 0) { + OS << " return false;\n"; + } else { + OS << " switch (C) {\n"; + + llvm::StringSet<> Cases; + + GenerateCaseForVersionedClauses(AllowedClauses, OS, DirectiveName, + DirectivePrefix, ClausePrefix, Cases); + + GenerateCaseForVersionedClauses(AllowedOnceClauses, OS, DirectiveName, + DirectivePrefix, ClausePrefix, Cases); + + GenerateCaseForVersionedClauses(AllowedExclusiveClauses, OS, + DirectiveName, DirectivePrefix, + ClausePrefix, Cases); + + GenerateCaseForVersionedClauses(RequiredClauses, OS, DirectiveName, + DirectivePrefix, ClausePrefix, Cases); + + OS << " default:\n"; + OS << " return false;\n"; + OS << " }\n"; // End of clauses switch + } + OS << " break;\n"; + } + + OS << " }\n"; // End of directives switch + OS << " llvm_unreachable(\"Invalid " << LanguageName + << " Directive kind\");\n"; + OS << "}\n"; // End of function isAllowedClauseForDirective +} + +// Generate a simple enum set with the give clauses. +void GenerateClauseSet(const std::vector<Record *> &Clauses, raw_ostream &OS, + StringRef ClauseEnumSetClass, StringRef ClauseSetPrefix, + StringRef DirectiveName, StringRef DirectivePrefix, + StringRef ClausePrefix, StringRef CppNamespace) { + + OS << "\n"; + OS << " static " << ClauseEnumSetClass << " " << ClauseSetPrefix + << DirectivePrefix << getFormattedName(DirectiveName) << " {\n"; + + for (const auto &C : Clauses) { + const auto SpecificClause = C->getValueAsDef("clause"); + const auto ClauseName = SpecificClause->getValueAsString("name"); + OS << " llvm::" << CppNamespace << "::Clause::" << ClausePrefix + << getFormattedName(ClauseName) << ",\n"; + } + OS << " };\n"; +} + +// Generate an enum set for the 4 kinds of clauses linked to a directive. +void GenerateDirectiveClauseSets(const std::vector<Record *> &Directives, + raw_ostream &OS, StringRef LanguageName, + StringRef ClauseEnumSetClass, + StringRef DirectivePrefix, + StringRef ClausePrefix, + StringRef CppNamespace) { + + IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_SETS", OS); + + OS << "\n"; + OS << "namespace llvm {\n"; + + // Open namespaces defined in the directive language. + llvm::SmallVector<StringRef, 2> Namespaces; + llvm::SplitString(CppNamespace, Namespaces, "::"); + for (auto Ns : Namespaces) + OS << "namespace " << Ns << " {\n"; + + for (const auto &D : Directives) { + const auto DirectiveName = D->getValueAsString("name"); + + const auto &AllowedClauses = D->getValueAsListOfDefs("allowedClauses"); + const auto &AllowedOnceClauses = + D->getValueAsListOfDefs("allowedOnceClauses"); + const auto &AllowedExclusiveClauses = + D->getValueAsListOfDefs("allowedExclusiveClauses"); + const auto &RequiredClauses = D->getValueAsListOfDefs("requiredClauses"); + + OS << "\n"; + OS << " // Sets for " << DirectiveName << "\n"; + + GenerateClauseSet(AllowedClauses, OS, ClauseEnumSetClass, "allowedClauses_", + DirectiveName, DirectivePrefix, ClausePrefix, + CppNamespace); + GenerateClauseSet(AllowedOnceClauses, OS, ClauseEnumSetClass, + "allowedOnceClauses_", DirectiveName, DirectivePrefix, + ClausePrefix, CppNamespace); + GenerateClauseSet(AllowedExclusiveClauses, OS, ClauseEnumSetClass, + "allowedExclusiveClauses_", DirectiveName, + DirectivePrefix, ClausePrefix, CppNamespace); + GenerateClauseSet(RequiredClauses, OS, ClauseEnumSetClass, + "requiredClauses_", DirectiveName, DirectivePrefix, + ClausePrefix, CppNamespace); + } + + // Closing namespaces + for (auto Ns : llvm::reverse(Namespaces)) + OS << "} // namespace " << Ns << "\n"; + + OS << "} // namespace llvm\n"; +} + +// Generate a map of directive (key) with DirectiveClauses struct as values. +// The struct holds the 4 sets of enumeration for the 4 kinds of clauses +// allowances (allowed, allowed once, allowed exclusive and required). +void GenerateDirectiveClauseMap(const std::vector<Record *> &Directives, + raw_ostream &OS, StringRef LanguageName, + StringRef ClauseEnumSetClass, + StringRef DirectivePrefix, + StringRef ClausePrefix, + StringRef CppNamespace) { + + IfDefScope Scope("GEN_FLANG_DIRECTIVE_CLAUSE_MAP", OS); + + OS << "\n"; + OS << "struct " << LanguageName << "DirectiveClauses {\n"; + OS << " const " << ClauseEnumSetClass << " allowed;\n"; + OS << " const " << ClauseEnumSetClass << " allowedOnce;\n"; + OS << " const " << ClauseEnumSetClass << " allowedExclusive;\n"; + OS << " const " << ClauseEnumSetClass << " requiredOneOf;\n"; + OS << "};\n"; + + OS << "\n"; + + OS << "std::unordered_map<llvm::" << CppNamespace << "::Directive, " + << LanguageName << "DirectiveClauses>\n"; + OS << " directiveClausesTable = {\n"; + + for (const auto &D : Directives) { + const auto FormattedDirectiveName = + getFormattedName(D->getValueAsString("name")); + OS << " {llvm::" << CppNamespace << "::Directive::" << DirectivePrefix + << FormattedDirectiveName << ",\n"; + OS << " {\n"; + OS << " llvm::" << CppNamespace << "::allowedClauses_" + << DirectivePrefix << FormattedDirectiveName << ",\n"; + OS << " llvm::" << CppNamespace << "::allowedOnceClauses_" + << DirectivePrefix << FormattedDirectiveName << ",\n"; + OS << " llvm::" << CppNamespace << "::allowedExclusiveClauses_" + << DirectivePrefix << FormattedDirectiveName << ",\n"; + OS << " llvm::" << CppNamespace << "::requiredClauses_" + << DirectivePrefix << FormattedDirectiveName << ",\n"; + OS << " }\n"; + OS << " },\n"; + } + + OS << "};\n"; +} + +// Generate the implemenation section for the enumeration in the directive +// language +void EmitDirectivesFlangImpl(const std::vector<Record *> &Directives, + raw_ostream &OS, StringRef LanguageName, + StringRef ClauseEnumSetClass, + StringRef DirectivePrefix, StringRef ClausePrefix, + StringRef CppNamespace) { + + GenerateDirectiveClauseSets(Directives, OS, LanguageName, ClauseEnumSetClass, + DirectivePrefix, ClausePrefix, CppNamespace); + + GenerateDirectiveClauseMap(Directives, OS, LanguageName, ClauseEnumSetClass, + DirectivePrefix, ClausePrefix, CppNamespace); +} + +// Generate the implemenation section for the enumeration in the directive +// language. +void EmitDirectivesGen(RecordKeeper &Records, raw_ostream &OS) { + + const auto &DirectiveLanguages = + Records.getAllDerivedDefinitions("DirectiveLanguage"); + + if (DirectiveLanguages.size() != 1) { + PrintError("A single definition of DirectiveLanguage is needed."); + return; + } + + const auto &DirectiveLanguage = DirectiveLanguages[0]; + StringRef DirectivePrefix = + DirectiveLanguage->getValueAsString("directivePrefix"); + StringRef LanguageName = DirectiveLanguage->getValueAsString("name"); + StringRef ClausePrefix = DirectiveLanguage->getValueAsString("clausePrefix"); + StringRef CppNamespace = DirectiveLanguage->getValueAsString("cppNamespace"); + StringRef ClauseEnumSetClass = + DirectiveLanguage->getValueAsString("clauseEnumSetClass"); + + const auto &Directives = Records.getAllDerivedDefinitions("Directive"); + + EmitDirectivesFlangImpl(Directives, OS, LanguageName, ClauseEnumSetClass, + DirectivePrefix, ClausePrefix, CppNamespace); +} + +// Generate the implemenation for the enumeration in the directive +// language. This code can be included in library. +void EmitDirectivesImpl(RecordKeeper &Records, raw_ostream &OS) { + + const auto &DirectiveLanguages = + Records.getAllDerivedDefinitions("DirectiveLanguage"); + + if (DirectiveLanguages.size() != 1) { + PrintError("A single definition of DirectiveLanguage is needed."); + return; + } + + const auto &DirectiveLanguage = DirectiveLanguages[0]; + StringRef DirectivePrefix = + DirectiveLanguage->getValueAsString("directivePrefix"); + StringRef LanguageName = DirectiveLanguage->getValueAsString("name"); + StringRef ClausePrefix = DirectiveLanguage->getValueAsString("clausePrefix"); + StringRef CppNamespace = DirectiveLanguage->getValueAsString("cppNamespace"); + const auto &Directives = Records.getAllDerivedDefinitions("Directive"); + const auto &Clauses = Records.getAllDerivedDefinitions("Clause"); + + StringRef IncludeHeader = + DirectiveLanguage->getValueAsString("includeHeader"); + + if (!IncludeHeader.empty()) + OS << "#include \"" << IncludeHeader << "\"\n\n"; + + OS << "#include \"llvm/ADT/StringRef.h\"\n"; + OS << "#include \"llvm/ADT/StringSwitch.h\"\n"; + OS << "#include \"llvm/Support/ErrorHandling.h\"\n"; + OS << "\n"; + OS << "using namespace llvm;\n"; + llvm::SmallVector<StringRef, 2> Namespaces; + llvm::SplitString(CppNamespace, Namespaces, "::"); + for (auto Ns : Namespaces) + OS << "using namespace " << Ns << ";\n"; + + // getDirectiveKind(StringRef Str) + GenerateGetKind(Directives, OS, "Directive", DirectivePrefix, LanguageName, + CppNamespace, /*ImplicitAsUnknown=*/false); + + // getDirectiveName(Directive Kind) + GenerateGetName(Directives, OS, "Directive", DirectivePrefix, LanguageName, + CppNamespace); + + // getClauseKind(StringRef Str) + GenerateGetKind(Clauses, OS, "Clause", ClausePrefix, LanguageName, + CppNamespace, /*ImplicitAsUnknown=*/true); + + // getClauseName(Clause Kind) + GenerateGetName(Clauses, OS, "Clause", ClausePrefix, LanguageName, + CppNamespace); + + // isAllowedClauseForDirective(Directive D, Clause C, unsigned Version) + GenerateIsAllowedClause(Directives, OS, LanguageName, DirectivePrefix, + ClausePrefix, CppNamespace); +} + +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/DisassemblerEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/DisassemblerEmitter.cpp new file mode 100644 index 000000000000..7c3f53b31bf4 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/DisassemblerEmitter.cpp @@ -0,0 +1,156 @@ +//===- DisassemblerEmitter.cpp - Generate a disassembler ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "WebAssemblyDisassemblerEmitter.h" +#include "X86DisassemblerTables.h" +#include "X86RecognizableInstr.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" + +using namespace llvm; +using namespace llvm::X86Disassembler; + +/// DisassemblerEmitter - Contains disassembler table emitters for various +/// architectures. + +/// X86 Disassembler Emitter +/// +/// *** IF YOU'RE HERE TO RESOLVE A "Primary decode conflict", LOOK DOWN NEAR +/// THE END OF THIS COMMENT! +/// +/// The X86 disassembler emitter is part of the X86 Disassembler, which is +/// documented in lib/Target/X86/X86Disassembler.h. +/// +/// The emitter produces the tables that the disassembler uses to translate +/// instructions. The emitter generates the following tables: +/// +/// - One table (CONTEXTS_SYM) that contains a mapping of attribute masks to +/// instruction contexts. Although for each attribute there are cases where +/// that attribute determines decoding, in the majority of cases decoding is +/// the same whether or not an attribute is present. For example, a 64-bit +/// instruction with an OPSIZE prefix and an XS prefix decodes the same way in +/// all cases as a 64-bit instruction with only OPSIZE set. (The XS prefix +/// may have effects on its execution, but does not change the instruction +/// returned.) This allows considerable space savings in other tables. +/// - Six tables (ONEBYTE_SYM, TWOBYTE_SYM, THREEBYTE38_SYM, THREEBYTE3A_SYM, +/// THREEBYTEA6_SYM, and THREEBYTEA7_SYM contain the hierarchy that the +/// decoder traverses while decoding an instruction. At the lowest level of +/// this hierarchy are instruction UIDs, 16-bit integers that can be used to +/// uniquely identify the instruction and correspond exactly to its position +/// in the list of CodeGenInstructions for the target. +/// - One table (INSTRUCTIONS_SYM) contains information about the operands of +/// each instruction and how to decode them. +/// +/// During table generation, there may be conflicts between instructions that +/// occupy the same space in the decode tables. These conflicts are resolved as +/// follows in setTableFields() (X86DisassemblerTables.cpp) +/// +/// - If the current context is the native context for one of the instructions +/// (that is, the attributes specified for it in the LLVM tables specify +/// precisely the current context), then it has priority. +/// - If the current context isn't native for either of the instructions, then +/// the higher-priority context wins (that is, the one that is more specific). +/// That hierarchy is determined by outranks() (X86DisassemblerTables.cpp) +/// - If the current context is native for both instructions, then the table +/// emitter reports a conflict and dies. +/// +/// *** RESOLUTION FOR "Primary decode conflict"S +/// +/// If two instructions collide, typically the solution is (in order of +/// likelihood): +/// +/// (1) to filter out one of the instructions by editing filter() +/// (X86RecognizableInstr.cpp). This is the most common resolution, but +/// check the Intel manuals first to make sure that (2) and (3) are not the +/// problem. +/// (2) to fix the tables (X86.td and its subsidiaries) so the opcodes are +/// accurate. Sometimes they are not. +/// (3) to fix the tables to reflect the actual context (for example, required +/// prefixes), and possibly to add a new context by editing +/// include/llvm/Support/X86DisassemblerDecoderCommon.h. This is unlikely +/// to be the cause. +/// +/// DisassemblerEmitter.cpp contains the implementation for the emitter, +/// which simply pulls out instructions from the CodeGenTarget and pushes them +/// into X86DisassemblerTables. +/// X86DisassemblerTables.h contains the interface for the instruction tables, +/// which manage and emit the structures discussed above. +/// X86DisassemblerTables.cpp contains the implementation for the instruction +/// tables. +/// X86ModRMFilters.h contains filters that can be used to determine which +/// ModR/M values are valid for a particular instruction. These are used to +/// populate ModRMDecisions. +/// X86RecognizableInstr.h contains the interface for a single instruction, +/// which knows how to translate itself from a CodeGenInstruction and provide +/// the information necessary for integration into the tables. +/// X86RecognizableInstr.cpp contains the implementation for a single +/// instruction. + +namespace llvm { + +extern void EmitFixedLenDecoder(RecordKeeper &RK, raw_ostream &OS, + const std::string &PredicateNamespace, + const std::string &GPrefix, + const std::string &GPostfix, + const std::string &ROK, + const std::string &RFail, const std::string &L); + +void EmitDisassembler(RecordKeeper &Records, raw_ostream &OS) { + CodeGenTarget Target(Records); + emitSourceFileHeader(" * " + Target.getName().str() + " Disassembler", OS); + + // X86 uses a custom disassembler. + if (Target.getName() == "X86") { + DisassemblerTables Tables; + + ArrayRef<const CodeGenInstruction*> numberedInstructions = + Target.getInstructionsByEnumValue(); + + for (unsigned i = 0, e = numberedInstructions.size(); i != e; ++i) + RecognizableInstr::processInstr(Tables, *numberedInstructions[i], i); + + if (Tables.hasConflicts()) { + PrintError(Target.getTargetRecord()->getLoc(), "Primary decode conflict"); + return; + } + + Tables.emit(OS); + return; + } + + // WebAssembly has variable length opcodes, so can't use EmitFixedLenDecoder + // below (which depends on a Size table-gen Record), and also uses a custom + // disassembler. + if (Target.getName() == "WebAssembly") { + emitWebAssemblyDisassemblerTables(OS, Target.getInstructionsByEnumValue()); + return; + } + + // ARM and Thumb have a CHECK() macro to deal with DecodeStatuses. + if (Target.getName() == "ARM" || Target.getName() == "Thumb" || + Target.getName() == "AArch64" || Target.getName() == "ARM64") { + std::string PredicateNamespace = std::string(Target.getName()); + if (PredicateNamespace == "Thumb") + PredicateNamespace = "ARM"; + + EmitFixedLenDecoder(Records, OS, PredicateNamespace, + "if (!Check(S, ", "))", + "S", "MCDisassembler::Fail", + " MCDisassembler::DecodeStatus S = " + "MCDisassembler::Success;\n(void)S;"); + return; + } + + EmitFixedLenDecoder(Records, OS, std::string(Target.getName()), "if (", + " == MCDisassembler::Fail)", "MCDisassembler::Success", + "MCDisassembler::Fail", ""); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/ExegesisEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/ExegesisEmitter.cpp new file mode 100644 index 000000000000..8f784e4a4121 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/ExegesisEmitter.cpp @@ -0,0 +1,215 @@ +//===- ExegesisEmitter.cpp - Generate exegesis target data ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend emits llvm-exegesis information. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <map> +#include <string> +#include <vector> + +using namespace llvm; + +#define DEBUG_TYPE "exegesis-emitter" + +namespace { + +class ExegesisEmitter { +public: + ExegesisEmitter(RecordKeeper &RK); + + void run(raw_ostream &OS) const; + +private: + unsigned getPfmCounterId(llvm::StringRef Name) const { + const auto It = PfmCounterNameTable.find(Name); + if (It == PfmCounterNameTable.end()) + PrintFatalError("no pfm counter id for " + Name); + return It->second; + } + + // Collects all the ProcPfmCounters definitions available in this target. + void emitPfmCounters(raw_ostream &OS) const; + + void emitPfmCountersInfo(const Record &Def, + unsigned &IssueCountersTableOffset, + raw_ostream &OS) const; + + void emitPfmCountersLookupTable(raw_ostream &OS) const; + + RecordKeeper &Records; + std::string Target; + + // Table of counter name -> counter index. + const std::map<llvm::StringRef, unsigned> PfmCounterNameTable; +}; + +static std::map<llvm::StringRef, unsigned> +collectPfmCounters(const RecordKeeper &Records) { + std::map<llvm::StringRef, unsigned> PfmCounterNameTable; + const auto AddPfmCounterName = [&PfmCounterNameTable]( + const Record *PfmCounterDef) { + const llvm::StringRef Counter = PfmCounterDef->getValueAsString("Counter"); + if (!Counter.empty()) + PfmCounterNameTable.emplace(Counter, 0); + }; + for (Record *Def : Records.getAllDerivedDefinitions("ProcPfmCounters")) { + // Check that ResourceNames are unique. + llvm::SmallSet<llvm::StringRef, 16> Seen; + for (const Record *IssueCounter : + Def->getValueAsListOfDefs("IssueCounters")) { + const llvm::StringRef ResourceName = + IssueCounter->getValueAsString("ResourceName"); + if (ResourceName.empty()) + PrintFatalError(IssueCounter->getLoc(), "invalid empty ResourceName"); + if (!Seen.insert(ResourceName).second) + PrintFatalError(IssueCounter->getLoc(), + "duplicate ResourceName " + ResourceName); + AddPfmCounterName(IssueCounter); + } + AddPfmCounterName(Def->getValueAsDef("CycleCounter")); + AddPfmCounterName(Def->getValueAsDef("UopsCounter")); + } + unsigned Index = 0; + for (auto &NameAndIndex : PfmCounterNameTable) + NameAndIndex.second = Index++; + return PfmCounterNameTable; +} + +ExegesisEmitter::ExegesisEmitter(RecordKeeper &RK) + : Records(RK), PfmCounterNameTable(collectPfmCounters(RK)) { + std::vector<Record *> Targets = Records.getAllDerivedDefinitions("Target"); + if (Targets.size() == 0) + PrintFatalError("ERROR: No 'Target' subclasses defined!"); + if (Targets.size() != 1) + PrintFatalError("ERROR: Multiple subclasses of Target defined!"); + Target = std::string(Targets[0]->getName()); +} + +void ExegesisEmitter::emitPfmCountersInfo(const Record &Def, + unsigned &IssueCountersTableOffset, + raw_ostream &OS) const { + const auto CycleCounter = + Def.getValueAsDef("CycleCounter")->getValueAsString("Counter"); + const auto UopsCounter = + Def.getValueAsDef("UopsCounter")->getValueAsString("Counter"); + const size_t NumIssueCounters = + Def.getValueAsListOfDefs("IssueCounters").size(); + + OS << "\nstatic const PfmCountersInfo " << Target << Def.getName() + << " = {\n"; + + // Cycle Counter. + if (CycleCounter.empty()) + OS << " nullptr, // No cycle counter.\n"; + else + OS << " " << Target << "PfmCounterNames[" << getPfmCounterId(CycleCounter) + << "], // Cycle counter\n"; + + // Uops Counter. + if (UopsCounter.empty()) + OS << " nullptr, // No uops counter.\n"; + else + OS << " " << Target << "PfmCounterNames[" << getPfmCounterId(UopsCounter) + << "], // Uops counter\n"; + + // Issue Counters + if (NumIssueCounters == 0) + OS << " nullptr, // No issue counters.\n 0\n"; + else + OS << " " << Target << "PfmIssueCounters + " << IssueCountersTableOffset + << ", " << NumIssueCounters << " // Issue counters.\n"; + + OS << "};\n"; + IssueCountersTableOffset += NumIssueCounters; +} + +void ExegesisEmitter::emitPfmCounters(raw_ostream &OS) const { + // Emit the counter name table. + OS << "\nstatic const char* " << Target << "PfmCounterNames[] = {\n"; + for (const auto &NameAndIndex : PfmCounterNameTable) + OS << " \"" << NameAndIndex.first << "\", // " << NameAndIndex.second + << "\n"; + OS << "};\n\n"; + + // Emit the IssueCounters table. + const auto PfmCounterDefs = + Records.getAllDerivedDefinitions("ProcPfmCounters"); + // Only emit if non-empty. + const bool HasAtLeastOnePfmIssueCounter = + llvm::any_of(PfmCounterDefs, [](const Record *Def) { + return !Def->getValueAsListOfDefs("IssueCounters").empty(); + }); + if (HasAtLeastOnePfmIssueCounter) { + OS << "static const PfmCountersInfo::IssueCounter " << Target + << "PfmIssueCounters[] = {\n"; + for (const Record *Def : PfmCounterDefs) { + for (const Record *ICDef : Def->getValueAsListOfDefs("IssueCounters")) + OS << " { " << Target << "PfmCounterNames[" + << getPfmCounterId(ICDef->getValueAsString("Counter")) << "], \"" + << ICDef->getValueAsString("ResourceName") << "\"},\n"; + } + OS << "};\n"; + } + + // Now generate the PfmCountersInfo. + unsigned IssueCountersTableOffset = 0; + for (const Record *Def : PfmCounterDefs) + emitPfmCountersInfo(*Def, IssueCountersTableOffset, OS); + + OS << "\n"; +} // namespace + +void ExegesisEmitter::emitPfmCountersLookupTable(raw_ostream &OS) const { + std::vector<Record *> Bindings = + Records.getAllDerivedDefinitions("PfmCountersBinding"); + assert(!Bindings.empty() && "there must be at least one binding"); + llvm::sort(Bindings, [](const Record *L, const Record *R) { + return L->getValueAsString("CpuName") < R->getValueAsString("CpuName"); + }); + + OS << "// Sorted (by CpuName) array of pfm counters.\n" + << "static const CpuAndPfmCounters " << Target << "CpuPfmCounters[] = {\n"; + for (Record *Binding : Bindings) { + // Emit as { "cpu", procinit }, + OS << " { \"" // + << Binding->getValueAsString("CpuName") << "\"," // + << " &" << Target << Binding->getValueAsDef("Counters")->getName() // + << " },\n"; + } + OS << "};\n\n"; +} + +void ExegesisEmitter::run(raw_ostream &OS) const { + emitSourceFileHeader("Exegesis Tables", OS); + emitPfmCounters(OS); + emitPfmCountersLookupTable(OS); +} + +} // end anonymous namespace + +namespace llvm { + +void EmitExegesis(RecordKeeper &RK, raw_ostream &OS) { + ExegesisEmitter(RK).run(OS); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/FastISelEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/FastISelEmitter.cpp new file mode 100644 index 000000000000..0729ab70d696 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/FastISelEmitter.cpp @@ -0,0 +1,893 @@ +///===- FastISelEmitter.cpp - Generate an instruction selector -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend emits code for use by the "fast" instruction +// selection algorithm. See the comments at the top of +// lib/CodeGen/SelectionDAG/FastISel.cpp for background. +// +// This file scans through the target's tablegen instruction-info files +// and extracts instructions with obvious-looking patterns, and it emits +// code to look up these instructions by type and operator. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenDAGPatterns.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <utility> +using namespace llvm; + + +/// InstructionMemo - This class holds additional information about an +/// instruction needed to emit code for it. +/// +namespace { +struct InstructionMemo { + std::string Name; + const CodeGenRegisterClass *RC; + std::string SubRegNo; + std::vector<std::string> PhysRegs; + std::string PredicateCheck; + + InstructionMemo(StringRef Name, const CodeGenRegisterClass *RC, + std::string SubRegNo, std::vector<std::string> PhysRegs, + std::string PredicateCheck) + : Name(Name), RC(RC), SubRegNo(std::move(SubRegNo)), + PhysRegs(std::move(PhysRegs)), + PredicateCheck(std::move(PredicateCheck)) {} + + // Make sure we do not copy InstructionMemo. + InstructionMemo(const InstructionMemo &Other) = delete; + InstructionMemo(InstructionMemo &&Other) = default; +}; +} // End anonymous namespace + +/// ImmPredicateSet - This uniques predicates (represented as a string) and +/// gives them unique (small) integer ID's that start at 0. +namespace { +class ImmPredicateSet { + DenseMap<TreePattern *, unsigned> ImmIDs; + std::vector<TreePredicateFn> PredsByName; +public: + + unsigned getIDFor(TreePredicateFn Pred) { + unsigned &Entry = ImmIDs[Pred.getOrigPatFragRecord()]; + if (Entry == 0) { + PredsByName.push_back(Pred); + Entry = PredsByName.size(); + } + return Entry-1; + } + + const TreePredicateFn &getPredicate(unsigned i) { + assert(i < PredsByName.size()); + return PredsByName[i]; + } + + typedef std::vector<TreePredicateFn>::const_iterator iterator; + iterator begin() const { return PredsByName.begin(); } + iterator end() const { return PredsByName.end(); } + +}; +} // End anonymous namespace + +/// OperandsSignature - This class holds a description of a list of operand +/// types. It has utility methods for emitting text based on the operands. +/// +namespace { +struct OperandsSignature { + class OpKind { + enum { OK_Reg, OK_FP, OK_Imm, OK_Invalid = -1 }; + char Repr; + public: + + OpKind() : Repr(OK_Invalid) {} + + bool operator<(OpKind RHS) const { return Repr < RHS.Repr; } + bool operator==(OpKind RHS) const { return Repr == RHS.Repr; } + + static OpKind getReg() { OpKind K; K.Repr = OK_Reg; return K; } + static OpKind getFP() { OpKind K; K.Repr = OK_FP; return K; } + static OpKind getImm(unsigned V) { + assert((unsigned)OK_Imm+V < 128 && + "Too many integer predicates for the 'Repr' char"); + OpKind K; K.Repr = OK_Imm+V; return K; + } + + bool isReg() const { return Repr == OK_Reg; } + bool isFP() const { return Repr == OK_FP; } + bool isImm() const { return Repr >= OK_Imm; } + + unsigned getImmCode() const { assert(isImm()); return Repr-OK_Imm; } + + void printManglingSuffix(raw_ostream &OS, ImmPredicateSet &ImmPredicates, + bool StripImmCodes) const { + if (isReg()) + OS << 'r'; + else if (isFP()) + OS << 'f'; + else { + OS << 'i'; + if (!StripImmCodes) + if (unsigned Code = getImmCode()) + OS << "_" << ImmPredicates.getPredicate(Code-1).getFnName(); + } + } + }; + + + SmallVector<OpKind, 3> Operands; + + bool operator<(const OperandsSignature &O) const { + return Operands < O.Operands; + } + bool operator==(const OperandsSignature &O) const { + return Operands == O.Operands; + } + + bool empty() const { return Operands.empty(); } + + bool hasAnyImmediateCodes() const { + for (unsigned i = 0, e = Operands.size(); i != e; ++i) + if (Operands[i].isImm() && Operands[i].getImmCode() != 0) + return true; + return false; + } + + /// getWithoutImmCodes - Return a copy of this with any immediate codes forced + /// to zero. + OperandsSignature getWithoutImmCodes() const { + OperandsSignature Result; + for (unsigned i = 0, e = Operands.size(); i != e; ++i) + if (!Operands[i].isImm()) + Result.Operands.push_back(Operands[i]); + else + Result.Operands.push_back(OpKind::getImm(0)); + return Result; + } + + void emitImmediatePredicate(raw_ostream &OS, ImmPredicateSet &ImmPredicates) { + bool EmittedAnything = false; + for (unsigned i = 0, e = Operands.size(); i != e; ++i) { + if (!Operands[i].isImm()) continue; + + unsigned Code = Operands[i].getImmCode(); + if (Code == 0) continue; + + if (EmittedAnything) + OS << " &&\n "; + + TreePredicateFn PredFn = ImmPredicates.getPredicate(Code-1); + + // Emit the type check. + TreePattern *TP = PredFn.getOrigPatFragRecord(); + ValueTypeByHwMode VVT = TP->getTree(0)->getType(0); + assert(VVT.isSimple() && + "Cannot use variable value types with fast isel"); + OS << "VT == " << getEnumName(VVT.getSimple().SimpleTy) << " && "; + + OS << PredFn.getFnName() << "(imm" << i <<')'; + EmittedAnything = true; + } + } + + /// initialize - Examine the given pattern and initialize the contents + /// of the Operands array accordingly. Return true if all the operands + /// are supported, false otherwise. + /// + bool initialize(TreePatternNode *InstPatNode, const CodeGenTarget &Target, + MVT::SimpleValueType VT, + ImmPredicateSet &ImmediatePredicates, + const CodeGenRegisterClass *OrigDstRC) { + if (InstPatNode->isLeaf()) + return false; + + if (InstPatNode->getOperator()->getName() == "imm") { + Operands.push_back(OpKind::getImm(0)); + return true; + } + + if (InstPatNode->getOperator()->getName() == "fpimm") { + Operands.push_back(OpKind::getFP()); + return true; + } + + const CodeGenRegisterClass *DstRC = nullptr; + + for (unsigned i = 0, e = InstPatNode->getNumChildren(); i != e; ++i) { + TreePatternNode *Op = InstPatNode->getChild(i); + + // Handle imm operands specially. + if (!Op->isLeaf() && Op->getOperator()->getName() == "imm") { + unsigned PredNo = 0; + if (!Op->getPredicateCalls().empty()) { + TreePredicateFn PredFn = Op->getPredicateCalls()[0].Fn; + // If there is more than one predicate weighing in on this operand + // then we don't handle it. This doesn't typically happen for + // immediates anyway. + if (Op->getPredicateCalls().size() > 1 || + !PredFn.isImmediatePattern() || PredFn.usesOperands()) + return false; + // Ignore any instruction with 'FastIselShouldIgnore', these are + // not needed and just bloat the fast instruction selector. For + // example, X86 doesn't need to generate code to match ADD16ri8 since + // ADD16ri will do just fine. + Record *Rec = PredFn.getOrigPatFragRecord()->getRecord(); + if (Rec->getValueAsBit("FastIselShouldIgnore")) + return false; + + PredNo = ImmediatePredicates.getIDFor(PredFn)+1; + } + + Operands.push_back(OpKind::getImm(PredNo)); + continue; + } + + + // For now, filter out any operand with a predicate. + // For now, filter out any operand with multiple values. + if (!Op->getPredicateCalls().empty() || Op->getNumTypes() != 1) + return false; + + if (!Op->isLeaf()) { + if (Op->getOperator()->getName() == "fpimm") { + Operands.push_back(OpKind::getFP()); + continue; + } + // For now, ignore other non-leaf nodes. + return false; + } + + assert(Op->hasConcreteType(0) && "Type infererence not done?"); + + // For now, all the operands must have the same type (if they aren't + // immediates). Note that this causes us to reject variable sized shifts + // on X86. + if (Op->getSimpleType(0) != VT) + return false; + + DefInit *OpDI = dyn_cast<DefInit>(Op->getLeafValue()); + if (!OpDI) + return false; + Record *OpLeafRec = OpDI->getDef(); + + // For now, the only other thing we accept is register operands. + const CodeGenRegisterClass *RC = nullptr; + if (OpLeafRec->isSubClassOf("RegisterOperand")) + OpLeafRec = OpLeafRec->getValueAsDef("RegClass"); + if (OpLeafRec->isSubClassOf("RegisterClass")) + RC = &Target.getRegisterClass(OpLeafRec); + else if (OpLeafRec->isSubClassOf("Register")) + RC = Target.getRegBank().getRegClassForRegister(OpLeafRec); + else if (OpLeafRec->isSubClassOf("ValueType")) { + RC = OrigDstRC; + } else + return false; + + // For now, this needs to be a register class of some sort. + if (!RC) + return false; + + // For now, all the operands must have the same register class or be + // a strict subclass of the destination. + if (DstRC) { + if (DstRC != RC && !DstRC->hasSubClass(RC)) + return false; + } else + DstRC = RC; + Operands.push_back(OpKind::getReg()); + } + return true; + } + + void PrintParameters(raw_ostream &OS) const { + for (unsigned i = 0, e = Operands.size(); i != e; ++i) { + if (Operands[i].isReg()) { + OS << "unsigned Op" << i << ", bool Op" << i << "IsKill"; + } else if (Operands[i].isImm()) { + OS << "uint64_t imm" << i; + } else if (Operands[i].isFP()) { + OS << "const ConstantFP *f" << i; + } else { + llvm_unreachable("Unknown operand kind!"); + } + if (i + 1 != e) + OS << ", "; + } + } + + void PrintArguments(raw_ostream &OS, + const std::vector<std::string> &PR) const { + assert(PR.size() == Operands.size()); + bool PrintedArg = false; + for (unsigned i = 0, e = Operands.size(); i != e; ++i) { + if (PR[i] != "") + // Implicit physical register operand. + continue; + + if (PrintedArg) + OS << ", "; + if (Operands[i].isReg()) { + OS << "Op" << i << ", Op" << i << "IsKill"; + PrintedArg = true; + } else if (Operands[i].isImm()) { + OS << "imm" << i; + PrintedArg = true; + } else if (Operands[i].isFP()) { + OS << "f" << i; + PrintedArg = true; + } else { + llvm_unreachable("Unknown operand kind!"); + } + } + } + + void PrintArguments(raw_ostream &OS) const { + for (unsigned i = 0, e = Operands.size(); i != e; ++i) { + if (Operands[i].isReg()) { + OS << "Op" << i << ", Op" << i << "IsKill"; + } else if (Operands[i].isImm()) { + OS << "imm" << i; + } else if (Operands[i].isFP()) { + OS << "f" << i; + } else { + llvm_unreachable("Unknown operand kind!"); + } + if (i + 1 != e) + OS << ", "; + } + } + + + void PrintManglingSuffix(raw_ostream &OS, const std::vector<std::string> &PR, + ImmPredicateSet &ImmPredicates, + bool StripImmCodes = false) const { + for (unsigned i = 0, e = Operands.size(); i != e; ++i) { + if (PR[i] != "") + // Implicit physical register operand. e.g. Instruction::Mul expect to + // select to a binary op. On x86, mul may take a single operand with + // the other operand being implicit. We must emit something that looks + // like a binary instruction except for the very inner fastEmitInst_* + // call. + continue; + Operands[i].printManglingSuffix(OS, ImmPredicates, StripImmCodes); + } + } + + void PrintManglingSuffix(raw_ostream &OS, ImmPredicateSet &ImmPredicates, + bool StripImmCodes = false) const { + for (unsigned i = 0, e = Operands.size(); i != e; ++i) + Operands[i].printManglingSuffix(OS, ImmPredicates, StripImmCodes); + } +}; +} // End anonymous namespace + +namespace { +class FastISelMap { + // A multimap is needed instead of a "plain" map because the key is + // the instruction's complexity (an int) and they are not unique. + typedef std::multimap<int, InstructionMemo> PredMap; + typedef std::map<MVT::SimpleValueType, PredMap> RetPredMap; + typedef std::map<MVT::SimpleValueType, RetPredMap> TypeRetPredMap; + typedef std::map<std::string, TypeRetPredMap> OpcodeTypeRetPredMap; + typedef std::map<OperandsSignature, OpcodeTypeRetPredMap> + OperandsOpcodeTypeRetPredMap; + + OperandsOpcodeTypeRetPredMap SimplePatterns; + + // This is used to check that there are no duplicate predicates + typedef std::multimap<std::string, bool> PredCheckMap; + typedef std::map<MVT::SimpleValueType, PredCheckMap> RetPredCheckMap; + typedef std::map<MVT::SimpleValueType, RetPredCheckMap> TypeRetPredCheckMap; + typedef std::map<std::string, TypeRetPredCheckMap> OpcodeTypeRetPredCheckMap; + typedef std::map<OperandsSignature, OpcodeTypeRetPredCheckMap> + OperandsOpcodeTypeRetPredCheckMap; + + OperandsOpcodeTypeRetPredCheckMap SimplePatternsCheck; + + std::map<OperandsSignature, std::vector<OperandsSignature> > + SignaturesWithConstantForms; + + StringRef InstNS; + ImmPredicateSet ImmediatePredicates; +public: + explicit FastISelMap(StringRef InstNS); + + void collectPatterns(CodeGenDAGPatterns &CGP); + void printImmediatePredicates(raw_ostream &OS); + void printFunctionDefinitions(raw_ostream &OS); +private: + void emitInstructionCode(raw_ostream &OS, + const OperandsSignature &Operands, + const PredMap &PM, + const std::string &RetVTName); +}; +} // End anonymous namespace + +static std::string getOpcodeName(Record *Op, CodeGenDAGPatterns &CGP) { + return std::string(CGP.getSDNodeInfo(Op).getEnumName()); +} + +static std::string getLegalCName(std::string OpName) { + std::string::size_type pos = OpName.find("::"); + if (pos != std::string::npos) + OpName.replace(pos, 2, "_"); + return OpName; +} + +FastISelMap::FastISelMap(StringRef instns) : InstNS(instns) {} + +static std::string PhyRegForNode(TreePatternNode *Op, + const CodeGenTarget &Target) { + std::string PhysReg; + + if (!Op->isLeaf()) + return PhysReg; + + Record *OpLeafRec = cast<DefInit>(Op->getLeafValue())->getDef(); + if (!OpLeafRec->isSubClassOf("Register")) + return PhysReg; + + PhysReg += cast<StringInit>(OpLeafRec->getValue("Namespace")->getValue()) + ->getValue(); + PhysReg += "::"; + PhysReg += Target.getRegBank().getReg(OpLeafRec)->getName(); + return PhysReg; +} + +void FastISelMap::collectPatterns(CodeGenDAGPatterns &CGP) { + const CodeGenTarget &Target = CGP.getTargetInfo(); + + // Scan through all the patterns and record the simple ones. + for (CodeGenDAGPatterns::ptm_iterator I = CGP.ptm_begin(), + E = CGP.ptm_end(); I != E; ++I) { + const PatternToMatch &Pattern = *I; + + // For now, just look at Instructions, so that we don't have to worry + // about emitting multiple instructions for a pattern. + TreePatternNode *Dst = Pattern.getDstPattern(); + if (Dst->isLeaf()) continue; + Record *Op = Dst->getOperator(); + if (!Op->isSubClassOf("Instruction")) + continue; + CodeGenInstruction &II = CGP.getTargetInfo().getInstruction(Op); + if (II.Operands.empty()) + continue; + + // Allow instructions to be marked as unavailable for FastISel for + // certain cases, i.e. an ISA has two 'and' instruction which differ + // by what registers they can use but are otherwise identical for + // codegen purposes. + if (II.FastISelShouldIgnore) + continue; + + // For now, ignore multi-instruction patterns. + bool MultiInsts = false; + for (unsigned i = 0, e = Dst->getNumChildren(); i != e; ++i) { + TreePatternNode *ChildOp = Dst->getChild(i); + if (ChildOp->isLeaf()) + continue; + if (ChildOp->getOperator()->isSubClassOf("Instruction")) { + MultiInsts = true; + break; + } + } + if (MultiInsts) + continue; + + // For now, ignore instructions where the first operand is not an + // output register. + const CodeGenRegisterClass *DstRC = nullptr; + std::string SubRegNo; + if (Op->getName() != "EXTRACT_SUBREG") { + Record *Op0Rec = II.Operands[0].Rec; + if (Op0Rec->isSubClassOf("RegisterOperand")) + Op0Rec = Op0Rec->getValueAsDef("RegClass"); + if (!Op0Rec->isSubClassOf("RegisterClass")) + continue; + DstRC = &Target.getRegisterClass(Op0Rec); + if (!DstRC) + continue; + } else { + // If this isn't a leaf, then continue since the register classes are + // a bit too complicated for now. + if (!Dst->getChild(1)->isLeaf()) continue; + + DefInit *SR = dyn_cast<DefInit>(Dst->getChild(1)->getLeafValue()); + if (SR) + SubRegNo = getQualifiedName(SR->getDef()); + else + SubRegNo = Dst->getChild(1)->getLeafValue()->getAsString(); + } + + // Inspect the pattern. + TreePatternNode *InstPatNode = Pattern.getSrcPattern(); + if (!InstPatNode) continue; + if (InstPatNode->isLeaf()) continue; + + // Ignore multiple result nodes for now. + if (InstPatNode->getNumTypes() > 1) continue; + + Record *InstPatOp = InstPatNode->getOperator(); + std::string OpcodeName = getOpcodeName(InstPatOp, CGP); + MVT::SimpleValueType RetVT = MVT::isVoid; + if (InstPatNode->getNumTypes()) RetVT = InstPatNode->getSimpleType(0); + MVT::SimpleValueType VT = RetVT; + if (InstPatNode->getNumChildren()) { + assert(InstPatNode->getChild(0)->getNumTypes() == 1); + VT = InstPatNode->getChild(0)->getSimpleType(0); + } + + // For now, filter out any instructions with predicates. + if (!InstPatNode->getPredicateCalls().empty()) + continue; + + // Check all the operands. + OperandsSignature Operands; + if (!Operands.initialize(InstPatNode, Target, VT, ImmediatePredicates, + DstRC)) + continue; + + std::vector<std::string> PhysRegInputs; + if (InstPatNode->getOperator()->getName() == "imm" || + InstPatNode->getOperator()->getName() == "fpimm") + PhysRegInputs.push_back(""); + else { + // Compute the PhysRegs used by the given pattern, and check that + // the mapping from the src to dst patterns is simple. + bool FoundNonSimplePattern = false; + unsigned DstIndex = 0; + for (unsigned i = 0, e = InstPatNode->getNumChildren(); i != e; ++i) { + std::string PhysReg = PhyRegForNode(InstPatNode->getChild(i), Target); + if (PhysReg.empty()) { + if (DstIndex >= Dst->getNumChildren() || + Dst->getChild(DstIndex)->getName() != + InstPatNode->getChild(i)->getName()) { + FoundNonSimplePattern = true; + break; + } + ++DstIndex; + } + + PhysRegInputs.push_back(PhysReg); + } + + if (Op->getName() != "EXTRACT_SUBREG" && DstIndex < Dst->getNumChildren()) + FoundNonSimplePattern = true; + + if (FoundNonSimplePattern) + continue; + } + + // Check if the operands match one of the patterns handled by FastISel. + std::string ManglingSuffix; + raw_string_ostream SuffixOS(ManglingSuffix); + Operands.PrintManglingSuffix(SuffixOS, ImmediatePredicates, true); + SuffixOS.flush(); + if (!StringSwitch<bool>(ManglingSuffix) + .Cases("", "r", "rr", "ri", "i", "f", true) + .Default(false)) + continue; + + // Get the predicate that guards this pattern. + std::string PredicateCheck = Pattern.getPredicateCheck(); + + // Ok, we found a pattern that we can handle. Remember it. + InstructionMemo Memo( + Pattern.getDstPattern()->getOperator()->getName(), + DstRC, + SubRegNo, + PhysRegInputs, + PredicateCheck + ); + + int complexity = Pattern.getPatternComplexity(CGP); + + if (SimplePatternsCheck[Operands][OpcodeName][VT] + [RetVT].count(PredicateCheck)) { + PrintFatalError(Pattern.getSrcRecord()->getLoc(), + "Duplicate predicate in FastISel table!"); + } + SimplePatternsCheck[Operands][OpcodeName][VT][RetVT].insert( + std::make_pair(PredicateCheck, true)); + + // Note: Instructions with the same complexity will appear in the order + // that they are encountered. + SimplePatterns[Operands][OpcodeName][VT][RetVT].emplace(complexity, + std::move(Memo)); + + // If any of the operands were immediates with predicates on them, strip + // them down to a signature that doesn't have predicates so that we can + // associate them with the stripped predicate version. + if (Operands.hasAnyImmediateCodes()) { + SignaturesWithConstantForms[Operands.getWithoutImmCodes()] + .push_back(Operands); + } + } +} + +void FastISelMap::printImmediatePredicates(raw_ostream &OS) { + if (ImmediatePredicates.begin() == ImmediatePredicates.end()) + return; + + OS << "\n// FastEmit Immediate Predicate functions.\n"; + for (ImmPredicateSet::iterator I = ImmediatePredicates.begin(), + E = ImmediatePredicates.end(); I != E; ++I) { + OS << "static bool " << I->getFnName() << "(int64_t Imm) {\n"; + OS << I->getImmediatePredicateCode() << "\n}\n"; + } + + OS << "\n\n"; +} + +void FastISelMap::emitInstructionCode(raw_ostream &OS, + const OperandsSignature &Operands, + const PredMap &PM, + const std::string &RetVTName) { + // Emit code for each possible instruction. There may be + // multiple if there are subtarget concerns. A reverse iterator + // is used to produce the ones with highest complexity first. + + bool OneHadNoPredicate = false; + for (PredMap::const_reverse_iterator PI = PM.rbegin(), PE = PM.rend(); + PI != PE; ++PI) { + const InstructionMemo &Memo = PI->second; + std::string PredicateCheck = Memo.PredicateCheck; + + if (PredicateCheck.empty()) { + assert(!OneHadNoPredicate && + "Multiple instructions match and more than one had " + "no predicate!"); + OneHadNoPredicate = true; + } else { + if (OneHadNoPredicate) { + PrintFatalError("Multiple instructions match and one with no " + "predicate came before one with a predicate! " + "name:" + Memo.Name + " predicate: " + PredicateCheck); + } + OS << " if (" + PredicateCheck + ") {\n"; + OS << " "; + } + + for (unsigned i = 0; i < Memo.PhysRegs.size(); ++i) { + if (Memo.PhysRegs[i] != "") + OS << " BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, " + << "TII.get(TargetOpcode::COPY), " << Memo.PhysRegs[i] + << ").addReg(Op" << i << ");\n"; + } + + OS << " return fastEmitInst_"; + if (Memo.SubRegNo.empty()) { + Operands.PrintManglingSuffix(OS, Memo.PhysRegs, ImmediatePredicates, + true); + OS << "(" << InstNS << "::" << Memo.Name << ", "; + OS << "&" << InstNS << "::" << Memo.RC->getName() << "RegClass"; + if (!Operands.empty()) + OS << ", "; + Operands.PrintArguments(OS, Memo.PhysRegs); + OS << ");\n"; + } else { + OS << "extractsubreg(" << RetVTName + << ", Op0, Op0IsKill, " << Memo.SubRegNo << ");\n"; + } + + if (!PredicateCheck.empty()) { + OS << " }\n"; + } + } + // Return 0 if all of the possibilities had predicates but none + // were satisfied. + if (!OneHadNoPredicate) + OS << " return 0;\n"; + OS << "}\n"; + OS << "\n"; +} + + +void FastISelMap::printFunctionDefinitions(raw_ostream &OS) { + // Now emit code for all the patterns that we collected. + for (OperandsOpcodeTypeRetPredMap::const_iterator OI = SimplePatterns.begin(), + OE = SimplePatterns.end(); OI != OE; ++OI) { + const OperandsSignature &Operands = OI->first; + const OpcodeTypeRetPredMap &OTM = OI->second; + + for (OpcodeTypeRetPredMap::const_iterator I = OTM.begin(), E = OTM.end(); + I != E; ++I) { + const std::string &Opcode = I->first; + const TypeRetPredMap &TM = I->second; + + OS << "// FastEmit functions for " << Opcode << ".\n"; + OS << "\n"; + + // Emit one function for each opcode,type pair. + for (TypeRetPredMap::const_iterator TI = TM.begin(), TE = TM.end(); + TI != TE; ++TI) { + MVT::SimpleValueType VT = TI->first; + const RetPredMap &RM = TI->second; + if (RM.size() != 1) { + for (RetPredMap::const_iterator RI = RM.begin(), RE = RM.end(); + RI != RE; ++RI) { + MVT::SimpleValueType RetVT = RI->first; + const PredMap &PM = RI->second; + + OS << "unsigned fastEmit_" << getLegalCName(Opcode) << "_" + << getLegalCName(std::string(getName(VT))) << "_" + << getLegalCName(std::string(getName(RetVT))) << "_"; + Operands.PrintManglingSuffix(OS, ImmediatePredicates); + OS << "("; + Operands.PrintParameters(OS); + OS << ") {\n"; + + emitInstructionCode(OS, Operands, PM, std::string(getName(RetVT))); + } + + // Emit one function for the type that demultiplexes on return type. + OS << "unsigned fastEmit_" << getLegalCName(Opcode) << "_" + << getLegalCName(std::string(getName(VT))) << "_"; + Operands.PrintManglingSuffix(OS, ImmediatePredicates); + OS << "(MVT RetVT"; + if (!Operands.empty()) + OS << ", "; + Operands.PrintParameters(OS); + OS << ") {\nswitch (RetVT.SimpleTy) {\n"; + for (RetPredMap::const_iterator RI = RM.begin(), RE = RM.end(); + RI != RE; ++RI) { + MVT::SimpleValueType RetVT = RI->first; + OS << " case " << getName(RetVT) << ": return fastEmit_" + << getLegalCName(Opcode) << "_" + << getLegalCName(std::string(getName(VT))) << "_" + << getLegalCName(std::string(getName(RetVT))) << "_"; + Operands.PrintManglingSuffix(OS, ImmediatePredicates); + OS << "("; + Operands.PrintArguments(OS); + OS << ");\n"; + } + OS << " default: return 0;\n}\n}\n\n"; + + } else { + // Non-variadic return type. + OS << "unsigned fastEmit_" << getLegalCName(Opcode) << "_" + << getLegalCName(std::string(getName(VT))) << "_"; + Operands.PrintManglingSuffix(OS, ImmediatePredicates); + OS << "(MVT RetVT"; + if (!Operands.empty()) + OS << ", "; + Operands.PrintParameters(OS); + OS << ") {\n"; + + OS << " if (RetVT.SimpleTy != " << getName(RM.begin()->first) + << ")\n return 0;\n"; + + const PredMap &PM = RM.begin()->second; + + emitInstructionCode(OS, Operands, PM, "RetVT"); + } + } + + // Emit one function for the opcode that demultiplexes based on the type. + OS << "unsigned fastEmit_" + << getLegalCName(Opcode) << "_"; + Operands.PrintManglingSuffix(OS, ImmediatePredicates); + OS << "(MVT VT, MVT RetVT"; + if (!Operands.empty()) + OS << ", "; + Operands.PrintParameters(OS); + OS << ") {\n"; + OS << " switch (VT.SimpleTy) {\n"; + for (TypeRetPredMap::const_iterator TI = TM.begin(), TE = TM.end(); + TI != TE; ++TI) { + MVT::SimpleValueType VT = TI->first; + std::string TypeName = std::string(getName(VT)); + OS << " case " << TypeName << ": return fastEmit_" + << getLegalCName(Opcode) << "_" << getLegalCName(TypeName) << "_"; + Operands.PrintManglingSuffix(OS, ImmediatePredicates); + OS << "(RetVT"; + if (!Operands.empty()) + OS << ", "; + Operands.PrintArguments(OS); + OS << ");\n"; + } + OS << " default: return 0;\n"; + OS << " }\n"; + OS << "}\n"; + OS << "\n"; + } + + OS << "// Top-level FastEmit function.\n"; + OS << "\n"; + + // Emit one function for the operand signature that demultiplexes based + // on opcode and type. + OS << "unsigned fastEmit_"; + Operands.PrintManglingSuffix(OS, ImmediatePredicates); + OS << "(MVT VT, MVT RetVT, unsigned Opcode"; + if (!Operands.empty()) + OS << ", "; + Operands.PrintParameters(OS); + OS << ") "; + if (!Operands.hasAnyImmediateCodes()) + OS << "override "; + OS << "{\n"; + + // If there are any forms of this signature available that operate on + // constrained forms of the immediate (e.g., 32-bit sext immediate in a + // 64-bit operand), check them first. + + std::map<OperandsSignature, std::vector<OperandsSignature> >::iterator MI + = SignaturesWithConstantForms.find(Operands); + if (MI != SignaturesWithConstantForms.end()) { + // Unique any duplicates out of the list. + llvm::sort(MI->second); + MI->second.erase(std::unique(MI->second.begin(), MI->second.end()), + MI->second.end()); + + // Check each in order it was seen. It would be nice to have a good + // relative ordering between them, but we're not going for optimality + // here. + for (unsigned i = 0, e = MI->second.size(); i != e; ++i) { + OS << " if ("; + MI->second[i].emitImmediatePredicate(OS, ImmediatePredicates); + OS << ")\n if (unsigned Reg = fastEmit_"; + MI->second[i].PrintManglingSuffix(OS, ImmediatePredicates); + OS << "(VT, RetVT, Opcode"; + if (!MI->second[i].empty()) + OS << ", "; + MI->second[i].PrintArguments(OS); + OS << "))\n return Reg;\n\n"; + } + + // Done with this, remove it. + SignaturesWithConstantForms.erase(MI); + } + + OS << " switch (Opcode) {\n"; + for (OpcodeTypeRetPredMap::const_iterator I = OTM.begin(), E = OTM.end(); + I != E; ++I) { + const std::string &Opcode = I->first; + + OS << " case " << Opcode << ": return fastEmit_" + << getLegalCName(Opcode) << "_"; + Operands.PrintManglingSuffix(OS, ImmediatePredicates); + OS << "(VT, RetVT"; + if (!Operands.empty()) + OS << ", "; + Operands.PrintArguments(OS); + OS << ");\n"; + } + OS << " default: return 0;\n"; + OS << " }\n"; + OS << "}\n"; + OS << "\n"; + } + + // TODO: SignaturesWithConstantForms should be empty here. +} + +namespace llvm { + +void EmitFastISel(RecordKeeper &RK, raw_ostream &OS) { + CodeGenDAGPatterns CGP(RK); + const CodeGenTarget &Target = CGP.getTargetInfo(); + emitSourceFileHeader("\"Fast\" Instruction Selector for the " + + Target.getName().str() + " target", OS); + + // Determine the target's namespace name. + StringRef InstNS = Target.getInstNamespace(); + assert(!InstNS.empty() && "Can't determine target-specific namespace!"); + + FastISelMap F(InstNS); + F.collectPatterns(CGP); + F.printImmediatePredicates(OS); + F.printFunctionDefinitions(OS); +} + +} // End llvm namespace diff --git a/contrib/llvm-project/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp new file mode 100644 index 000000000000..88d210f7fd39 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp @@ -0,0 +1,2549 @@ +//===------------ FixedLenDecoderEmitter.cpp - Decoder Generator ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// It contains the tablegen backend that emits the decoder functions for +// targets with fixed length instruction set. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenInstruction.h" +#include "CodeGenTarget.h" +#include "InfoByHwMode.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/CachedHashString.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/MC/MCFixedLenDisassembler.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/LEB128.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> +#include <vector> + +using namespace llvm; + +#define DEBUG_TYPE "decoder-emitter" + +namespace { + +STATISTIC(NumEncodings, "Number of encodings considered"); +STATISTIC(NumEncodingsLackingDisasm, "Number of encodings without disassembler info"); +STATISTIC(NumInstructions, "Number of instructions considered"); +STATISTIC(NumEncodingsSupported, "Number of encodings supported"); +STATISTIC(NumEncodingsOmitted, "Number of encodings omitted"); + +struct EncodingField { + unsigned Base, Width, Offset; + EncodingField(unsigned B, unsigned W, unsigned O) + : Base(B), Width(W), Offset(O) { } +}; + +struct OperandInfo { + std::vector<EncodingField> Fields; + std::string Decoder; + bool HasCompleteDecoder; + uint64_t InitValue; + + OperandInfo(std::string D, bool HCD) + : Decoder(std::move(D)), HasCompleteDecoder(HCD), InitValue(0) {} + + void addField(unsigned Base, unsigned Width, unsigned Offset) { + Fields.push_back(EncodingField(Base, Width, Offset)); + } + + unsigned numFields() const { return Fields.size(); } + + typedef std::vector<EncodingField>::const_iterator const_iterator; + + const_iterator begin() const { return Fields.begin(); } + const_iterator end() const { return Fields.end(); } +}; + +typedef std::vector<uint8_t> DecoderTable; +typedef uint32_t DecoderFixup; +typedef std::vector<DecoderFixup> FixupList; +typedef std::vector<FixupList> FixupScopeList; +typedef SmallSetVector<CachedHashString, 16> PredicateSet; +typedef SmallSetVector<CachedHashString, 16> DecoderSet; +struct DecoderTableInfo { + DecoderTable Table; + FixupScopeList FixupStack; + PredicateSet Predicates; + DecoderSet Decoders; +}; + +struct EncodingAndInst { + const Record *EncodingDef; + const CodeGenInstruction *Inst; + StringRef HwModeName; + + EncodingAndInst(const Record *EncodingDef, const CodeGenInstruction *Inst, + StringRef HwModeName = "") + : EncodingDef(EncodingDef), Inst(Inst), HwModeName(HwModeName) {} +}; + +struct EncodingIDAndOpcode { + unsigned EncodingID; + unsigned Opcode; + + EncodingIDAndOpcode() : EncodingID(0), Opcode(0) {} + EncodingIDAndOpcode(unsigned EncodingID, unsigned Opcode) + : EncodingID(EncodingID), Opcode(Opcode) {} +}; + +raw_ostream &operator<<(raw_ostream &OS, const EncodingAndInst &Value) { + if (Value.EncodingDef != Value.Inst->TheDef) + OS << Value.EncodingDef->getName() << ":"; + OS << Value.Inst->TheDef->getName(); + return OS; +} + +class FixedLenDecoderEmitter { + RecordKeeper &RK; + std::vector<EncodingAndInst> NumberedEncodings; + +public: + // Defaults preserved here for documentation, even though they aren't + // strictly necessary given the way that this is currently being called. + FixedLenDecoderEmitter(RecordKeeper &R, std::string PredicateNamespace, + std::string GPrefix = "if (", + std::string GPostfix = " == MCDisassembler::Fail)", + std::string ROK = "MCDisassembler::Success", + std::string RFail = "MCDisassembler::Fail", + std::string L = "") + : RK(R), Target(R), PredicateNamespace(std::move(PredicateNamespace)), + GuardPrefix(std::move(GPrefix)), GuardPostfix(std::move(GPostfix)), + ReturnOK(std::move(ROK)), ReturnFail(std::move(RFail)), + Locals(std::move(L)) {} + + // Emit the decoder state machine table. + void emitTable(formatted_raw_ostream &o, DecoderTable &Table, + unsigned Indentation, unsigned BitWidth, + StringRef Namespace) const; + void emitPredicateFunction(formatted_raw_ostream &OS, + PredicateSet &Predicates, + unsigned Indentation) const; + void emitDecoderFunction(formatted_raw_ostream &OS, + DecoderSet &Decoders, + unsigned Indentation) const; + + // run - Output the code emitter + void run(raw_ostream &o); + +private: + CodeGenTarget Target; + +public: + std::string PredicateNamespace; + std::string GuardPrefix, GuardPostfix; + std::string ReturnOK, ReturnFail; + std::string Locals; +}; + +} // end anonymous namespace + +// The set (BIT_TRUE, BIT_FALSE, BIT_UNSET) represents a ternary logic system +// for a bit value. +// +// BIT_UNFILTERED is used as the init value for a filter position. It is used +// only for filter processings. +typedef enum { + BIT_TRUE, // '1' + BIT_FALSE, // '0' + BIT_UNSET, // '?' + BIT_UNFILTERED // unfiltered +} bit_value_t; + +static bool ValueSet(bit_value_t V) { + return (V == BIT_TRUE || V == BIT_FALSE); +} + +static bool ValueNotSet(bit_value_t V) { + return (V == BIT_UNSET); +} + +static int Value(bit_value_t V) { + return ValueNotSet(V) ? -1 : (V == BIT_FALSE ? 0 : 1); +} + +static bit_value_t bitFromBits(const BitsInit &bits, unsigned index) { + if (BitInit *bit = dyn_cast<BitInit>(bits.getBit(index))) + return bit->getValue() ? BIT_TRUE : BIT_FALSE; + + // The bit is uninitialized. + return BIT_UNSET; +} + +// Prints the bit value for each position. +static void dumpBits(raw_ostream &o, const BitsInit &bits) { + for (unsigned index = bits.getNumBits(); index > 0; --index) { + switch (bitFromBits(bits, index - 1)) { + case BIT_TRUE: + o << "1"; + break; + case BIT_FALSE: + o << "0"; + break; + case BIT_UNSET: + o << "_"; + break; + default: + llvm_unreachable("unexpected return value from bitFromBits"); + } + } +} + +static BitsInit &getBitsField(const Record &def, StringRef str) { + BitsInit *bits = def.getValueAsBitsInit(str); + return *bits; +} + +// Representation of the instruction to work on. +typedef std::vector<bit_value_t> insn_t; + +namespace { + +class FilterChooser; + +/// Filter - Filter works with FilterChooser to produce the decoding tree for +/// the ISA. +/// +/// It is useful to think of a Filter as governing the switch stmts of the +/// decoding tree in a certain level. Each case stmt delegates to an inferior +/// FilterChooser to decide what further decoding logic to employ, or in another +/// words, what other remaining bits to look at. The FilterChooser eventually +/// chooses a best Filter to do its job. +/// +/// This recursive scheme ends when the number of Opcodes assigned to the +/// FilterChooser becomes 1 or if there is a conflict. A conflict happens when +/// the Filter/FilterChooser combo does not know how to distinguish among the +/// Opcodes assigned. +/// +/// An example of a conflict is +/// +/// Conflict: +/// 111101000.00........00010000.... +/// 111101000.00........0001........ +/// 1111010...00........0001........ +/// 1111010...00.................... +/// 1111010......................... +/// 1111............................ +/// ................................ +/// VST4q8a 111101000_00________00010000____ +/// VST4q8b 111101000_00________00010000____ +/// +/// The Debug output shows the path that the decoding tree follows to reach the +/// the conclusion that there is a conflict. VST4q8a is a vst4 to double-spaced +/// even registers, while VST4q8b is a vst4 to double-spaced odd registers. +/// +/// The encoding info in the .td files does not specify this meta information, +/// which could have been used by the decoder to resolve the conflict. The +/// decoder could try to decode the even/odd register numbering and assign to +/// VST4q8a or VST4q8b, but for the time being, the decoder chooses the "a" +/// version and return the Opcode since the two have the same Asm format string. +class Filter { +protected: + const FilterChooser *Owner;// points to the FilterChooser who owns this filter + unsigned StartBit; // the starting bit position + unsigned NumBits; // number of bits to filter + bool Mixed; // a mixed region contains both set and unset bits + + // Map of well-known segment value to the set of uid's with that value. + std::map<uint64_t, std::vector<EncodingIDAndOpcode>> + FilteredInstructions; + + // Set of uid's with non-constant segment values. + std::vector<EncodingIDAndOpcode> VariableInstructions; + + // Map of well-known segment value to its delegate. + std::map<unsigned, std::unique_ptr<const FilterChooser>> FilterChooserMap; + + // Number of instructions which fall under FilteredInstructions category. + unsigned NumFiltered; + + // Keeps track of the last opcode in the filtered bucket. + EncodingIDAndOpcode LastOpcFiltered; + +public: + Filter(Filter &&f); + Filter(FilterChooser &owner, unsigned startBit, unsigned numBits, bool mixed); + + ~Filter() = default; + + unsigned getNumFiltered() const { return NumFiltered; } + + EncodingIDAndOpcode getSingletonOpc() const { + assert(NumFiltered == 1); + return LastOpcFiltered; + } + + // Return the filter chooser for the group of instructions without constant + // segment values. + const FilterChooser &getVariableFC() const { + assert(NumFiltered == 1); + assert(FilterChooserMap.size() == 1); + return *(FilterChooserMap.find((unsigned)-1)->second); + } + + // Divides the decoding task into sub tasks and delegates them to the + // inferior FilterChooser's. + // + // A special case arises when there's only one entry in the filtered + // instructions. In order to unambiguously decode the singleton, we need to + // match the remaining undecoded encoding bits against the singleton. + void recurse(); + + // Emit table entries to decode instructions given a segment or segments of + // bits. + void emitTableEntry(DecoderTableInfo &TableInfo) const; + + // Returns the number of fanout produced by the filter. More fanout implies + // the filter distinguishes more categories of instructions. + unsigned usefulness() const; +}; // end class Filter + +} // end anonymous namespace + +// These are states of our finite state machines used in FilterChooser's +// filterProcessor() which produces the filter candidates to use. +typedef enum { + ATTR_NONE, + ATTR_FILTERED, + ATTR_ALL_SET, + ATTR_ALL_UNSET, + ATTR_MIXED +} bitAttr_t; + +/// FilterChooser - FilterChooser chooses the best filter among a set of Filters +/// in order to perform the decoding of instructions at the current level. +/// +/// Decoding proceeds from the top down. Based on the well-known encoding bits +/// of instructions available, FilterChooser builds up the possible Filters that +/// can further the task of decoding by distinguishing among the remaining +/// candidate instructions. +/// +/// Once a filter has been chosen, it is called upon to divide the decoding task +/// into sub-tasks and delegates them to its inferior FilterChoosers for further +/// processings. +/// +/// It is useful to think of a Filter as governing the switch stmts of the +/// decoding tree. And each case is delegated to an inferior FilterChooser to +/// decide what further remaining bits to look at. +namespace { + +class FilterChooser { +protected: + friend class Filter; + + // Vector of codegen instructions to choose our filter. + ArrayRef<EncodingAndInst> AllInstructions; + + // Vector of uid's for this filter chooser to work on. + // The first member of the pair is the opcode id being decoded, the second is + // the opcode id that should be emitted. + const std::vector<EncodingIDAndOpcode> &Opcodes; + + // Lookup table for the operand decoding of instructions. + const std::map<unsigned, std::vector<OperandInfo>> &Operands; + + // Vector of candidate filters. + std::vector<Filter> Filters; + + // Array of bit values passed down from our parent. + // Set to all BIT_UNFILTERED's for Parent == NULL. + std::vector<bit_value_t> FilterBitValues; + + // Links to the FilterChooser above us in the decoding tree. + const FilterChooser *Parent; + + // Index of the best filter from Filters. + int BestIndex; + + // Width of instructions + unsigned BitWidth; + + // Parent emitter + const FixedLenDecoderEmitter *Emitter; + +public: + FilterChooser(ArrayRef<EncodingAndInst> Insts, + const std::vector<EncodingIDAndOpcode> &IDs, + const std::map<unsigned, std::vector<OperandInfo>> &Ops, + unsigned BW, const FixedLenDecoderEmitter *E) + : AllInstructions(Insts), Opcodes(IDs), Operands(Ops), + FilterBitValues(BW, BIT_UNFILTERED), Parent(nullptr), BestIndex(-1), + BitWidth(BW), Emitter(E) { + doFilter(); + } + + FilterChooser(ArrayRef<EncodingAndInst> Insts, + const std::vector<EncodingIDAndOpcode> &IDs, + const std::map<unsigned, std::vector<OperandInfo>> &Ops, + const std::vector<bit_value_t> &ParentFilterBitValues, + const FilterChooser &parent) + : AllInstructions(Insts), Opcodes(IDs), Operands(Ops), + FilterBitValues(ParentFilterBitValues), Parent(&parent), BestIndex(-1), + BitWidth(parent.BitWidth), Emitter(parent.Emitter) { + doFilter(); + } + + FilterChooser(const FilterChooser &) = delete; + void operator=(const FilterChooser &) = delete; + + unsigned getBitWidth() const { return BitWidth; } + +protected: + // Populates the insn given the uid. + void insnWithID(insn_t &Insn, unsigned Opcode) const { + BitsInit &Bits = getBitsField(*AllInstructions[Opcode].EncodingDef, "Inst"); + + // We may have a SoftFail bitmask, which specifies a mask where an encoding + // may differ from the value in "Inst" and yet still be valid, but the + // disassembler should return SoftFail instead of Success. + // + // This is used for marking UNPREDICTABLE instructions in the ARM world. + BitsInit *SFBits = + AllInstructions[Opcode].EncodingDef->getValueAsBitsInit("SoftFail"); + + for (unsigned i = 0; i < BitWidth; ++i) { + if (SFBits && bitFromBits(*SFBits, i) == BIT_TRUE) + Insn.push_back(BIT_UNSET); + else + Insn.push_back(bitFromBits(Bits, i)); + } + } + + // Emit the name of the encoding/instruction pair. + void emitNameWithID(raw_ostream &OS, unsigned Opcode) const { + const Record *EncodingDef = AllInstructions[Opcode].EncodingDef; + const Record *InstDef = AllInstructions[Opcode].Inst->TheDef; + if (EncodingDef != InstDef) + OS << EncodingDef->getName() << ":"; + OS << InstDef->getName(); + } + + // Populates the field of the insn given the start position and the number of + // consecutive bits to scan for. + // + // Returns false if there exists any uninitialized bit value in the range. + // Returns true, otherwise. + bool fieldFromInsn(uint64_t &Field, insn_t &Insn, unsigned StartBit, + unsigned NumBits) const; + + /// dumpFilterArray - dumpFilterArray prints out debugging info for the given + /// filter array as a series of chars. + void dumpFilterArray(raw_ostream &o, + const std::vector<bit_value_t> & filter) const; + + /// dumpStack - dumpStack traverses the filter chooser chain and calls + /// dumpFilterArray on each filter chooser up to the top level one. + void dumpStack(raw_ostream &o, const char *prefix) const; + + Filter &bestFilter() { + assert(BestIndex != -1 && "BestIndex not set"); + return Filters[BestIndex]; + } + + bool PositionFiltered(unsigned i) const { + return ValueSet(FilterBitValues[i]); + } + + // Calculates the island(s) needed to decode the instruction. + // This returns a lit of undecoded bits of an instructions, for example, + // Inst{20} = 1 && Inst{3-0} == 0b1111 represents two islands of yet-to-be + // decoded bits in order to verify that the instruction matches the Opcode. + unsigned getIslands(std::vector<unsigned> &StartBits, + std::vector<unsigned> &EndBits, + std::vector<uint64_t> &FieldVals, + const insn_t &Insn) const; + + // Emits code to check the Predicates member of an instruction are true. + // Returns true if predicate matches were emitted, false otherwise. + bool emitPredicateMatch(raw_ostream &o, unsigned &Indentation, + unsigned Opc) const; + + bool doesOpcodeNeedPredicate(unsigned Opc) const; + unsigned getPredicateIndex(DecoderTableInfo &TableInfo, StringRef P) const; + void emitPredicateTableEntry(DecoderTableInfo &TableInfo, + unsigned Opc) const; + + void emitSoftFailTableEntry(DecoderTableInfo &TableInfo, + unsigned Opc) const; + + // Emits table entries to decode the singleton. + void emitSingletonTableEntry(DecoderTableInfo &TableInfo, + EncodingIDAndOpcode Opc) const; + + // Emits code to decode the singleton, and then to decode the rest. + void emitSingletonTableEntry(DecoderTableInfo &TableInfo, + const Filter &Best) const; + + void emitBinaryParser(raw_ostream &o, unsigned &Indentation, + const OperandInfo &OpInfo, + bool &OpHasCompleteDecoder) const; + + void emitDecoder(raw_ostream &OS, unsigned Indentation, unsigned Opc, + bool &HasCompleteDecoder) const; + unsigned getDecoderIndex(DecoderSet &Decoders, unsigned Opc, + bool &HasCompleteDecoder) const; + + // Assign a single filter and run with it. + void runSingleFilter(unsigned startBit, unsigned numBit, bool mixed); + + // reportRegion is a helper function for filterProcessor to mark a region as + // eligible for use as a filter region. + void reportRegion(bitAttr_t RA, unsigned StartBit, unsigned BitIndex, + bool AllowMixed); + + // FilterProcessor scans the well-known encoding bits of the instructions and + // builds up a list of candidate filters. It chooses the best filter and + // recursively descends down the decoding tree. + bool filterProcessor(bool AllowMixed, bool Greedy = true); + + // Decides on the best configuration of filter(s) to use in order to decode + // the instructions. A conflict of instructions may occur, in which case we + // dump the conflict set to the standard error. + void doFilter(); + +public: + // emitTableEntries - Emit state machine entries to decode our share of + // instructions. + void emitTableEntries(DecoderTableInfo &TableInfo) const; +}; + +} // end anonymous namespace + +/////////////////////////// +// // +// Filter Implementation // +// // +/////////////////////////// + +Filter::Filter(Filter &&f) + : Owner(f.Owner), StartBit(f.StartBit), NumBits(f.NumBits), Mixed(f.Mixed), + FilteredInstructions(std::move(f.FilteredInstructions)), + VariableInstructions(std::move(f.VariableInstructions)), + FilterChooserMap(std::move(f.FilterChooserMap)), NumFiltered(f.NumFiltered), + LastOpcFiltered(f.LastOpcFiltered) { +} + +Filter::Filter(FilterChooser &owner, unsigned startBit, unsigned numBits, + bool mixed) + : Owner(&owner), StartBit(startBit), NumBits(numBits), Mixed(mixed) { + assert(StartBit + NumBits - 1 < Owner->BitWidth); + + NumFiltered = 0; + LastOpcFiltered = {0, 0}; + + for (unsigned i = 0, e = Owner->Opcodes.size(); i != e; ++i) { + insn_t Insn; + + // Populates the insn given the uid. + Owner->insnWithID(Insn, Owner->Opcodes[i].EncodingID); + + uint64_t Field; + // Scans the segment for possibly well-specified encoding bits. + bool ok = Owner->fieldFromInsn(Field, Insn, StartBit, NumBits); + + if (ok) { + // The encoding bits are well-known. Lets add the uid of the + // instruction into the bucket keyed off the constant field value. + LastOpcFiltered = Owner->Opcodes[i]; + FilteredInstructions[Field].push_back(LastOpcFiltered); + ++NumFiltered; + } else { + // Some of the encoding bit(s) are unspecified. This contributes to + // one additional member of "Variable" instructions. + VariableInstructions.push_back(Owner->Opcodes[i]); + } + } + + assert((FilteredInstructions.size() + VariableInstructions.size() > 0) + && "Filter returns no instruction categories"); +} + +// Divides the decoding task into sub tasks and delegates them to the +// inferior FilterChooser's. +// +// A special case arises when there's only one entry in the filtered +// instructions. In order to unambiguously decode the singleton, we need to +// match the remaining undecoded encoding bits against the singleton. +void Filter::recurse() { + // Starts by inheriting our parent filter chooser's filter bit values. + std::vector<bit_value_t> BitValueArray(Owner->FilterBitValues); + + if (!VariableInstructions.empty()) { + // Conservatively marks each segment position as BIT_UNSET. + for (unsigned bitIndex = 0; bitIndex < NumBits; ++bitIndex) + BitValueArray[StartBit + bitIndex] = BIT_UNSET; + + // Delegates to an inferior filter chooser for further processing on this + // group of instructions whose segment values are variable. + FilterChooserMap.insert( + std::make_pair(-1U, std::make_unique<FilterChooser>( + Owner->AllInstructions, VariableInstructions, + Owner->Operands, BitValueArray, *Owner))); + } + + // No need to recurse for a singleton filtered instruction. + // See also Filter::emit*(). + if (getNumFiltered() == 1) { + assert(FilterChooserMap.size() == 1); + return; + } + + // Otherwise, create sub choosers. + for (const auto &Inst : FilteredInstructions) { + + // Marks all the segment positions with either BIT_TRUE or BIT_FALSE. + for (unsigned bitIndex = 0; bitIndex < NumBits; ++bitIndex) { + if (Inst.first & (1ULL << bitIndex)) + BitValueArray[StartBit + bitIndex] = BIT_TRUE; + else + BitValueArray[StartBit + bitIndex] = BIT_FALSE; + } + + // Delegates to an inferior filter chooser for further processing on this + // category of instructions. + FilterChooserMap.insert(std::make_pair( + Inst.first, std::make_unique<FilterChooser>( + Owner->AllInstructions, Inst.second, + Owner->Operands, BitValueArray, *Owner))); + } +} + +static void resolveTableFixups(DecoderTable &Table, const FixupList &Fixups, + uint32_t DestIdx) { + // Any NumToSkip fixups in the current scope can resolve to the + // current location. + for (FixupList::const_reverse_iterator I = Fixups.rbegin(), + E = Fixups.rend(); + I != E; ++I) { + // Calculate the distance from the byte following the fixup entry byte + // to the destination. The Target is calculated from after the 16-bit + // NumToSkip entry itself, so subtract two from the displacement here + // to account for that. + uint32_t FixupIdx = *I; + uint32_t Delta = DestIdx - FixupIdx - 3; + // Our NumToSkip entries are 24-bits. Make sure our table isn't too + // big. + assert(Delta < (1u << 24)); + Table[FixupIdx] = (uint8_t)Delta; + Table[FixupIdx + 1] = (uint8_t)(Delta >> 8); + Table[FixupIdx + 2] = (uint8_t)(Delta >> 16); + } +} + +// Emit table entries to decode instructions given a segment or segments +// of bits. +void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const { + TableInfo.Table.push_back(MCD::OPC_ExtractField); + TableInfo.Table.push_back(StartBit); + TableInfo.Table.push_back(NumBits); + + // A new filter entry begins a new scope for fixup resolution. + TableInfo.FixupStack.emplace_back(); + + DecoderTable &Table = TableInfo.Table; + + size_t PrevFilter = 0; + bool HasFallthrough = false; + for (auto &Filter : FilterChooserMap) { + // Field value -1 implies a non-empty set of variable instructions. + // See also recurse(). + if (Filter.first == (unsigned)-1) { + HasFallthrough = true; + + // Each scope should always have at least one filter value to check + // for. + assert(PrevFilter != 0 && "empty filter set!"); + FixupList &CurScope = TableInfo.FixupStack.back(); + // Resolve any NumToSkip fixups in the current scope. + resolveTableFixups(Table, CurScope, Table.size()); + CurScope.clear(); + PrevFilter = 0; // Don't re-process the filter's fallthrough. + } else { + Table.push_back(MCD::OPC_FilterValue); + // Encode and emit the value to filter against. + uint8_t Buffer[16]; + unsigned Len = encodeULEB128(Filter.first, Buffer); + Table.insert(Table.end(), Buffer, Buffer + Len); + // Reserve space for the NumToSkip entry. We'll backpatch the value + // later. + PrevFilter = Table.size(); + Table.push_back(0); + Table.push_back(0); + Table.push_back(0); + } + + // We arrive at a category of instructions with the same segment value. + // Now delegate to the sub filter chooser for further decodings. + // The case may fallthrough, which happens if the remaining well-known + // encoding bits do not match exactly. + Filter.second->emitTableEntries(TableInfo); + + // Now that we've emitted the body of the handler, update the NumToSkip + // of the filter itself to be able to skip forward when false. Subtract + // two as to account for the width of the NumToSkip field itself. + if (PrevFilter) { + uint32_t NumToSkip = Table.size() - PrevFilter - 3; + assert(NumToSkip < (1u << 24) && "disassembler decoding table too large!"); + Table[PrevFilter] = (uint8_t)NumToSkip; + Table[PrevFilter + 1] = (uint8_t)(NumToSkip >> 8); + Table[PrevFilter + 2] = (uint8_t)(NumToSkip >> 16); + } + } + + // Any remaining unresolved fixups bubble up to the parent fixup scope. + assert(TableInfo.FixupStack.size() > 1 && "fixup stack underflow!"); + FixupScopeList::iterator Source = TableInfo.FixupStack.end() - 1; + FixupScopeList::iterator Dest = Source - 1; + Dest->insert(Dest->end(), Source->begin(), Source->end()); + TableInfo.FixupStack.pop_back(); + + // If there is no fallthrough, then the final filter should get fixed + // up according to the enclosing scope rather than the current position. + if (!HasFallthrough) + TableInfo.FixupStack.back().push_back(PrevFilter); +} + +// Returns the number of fanout produced by the filter. More fanout implies +// the filter distinguishes more categories of instructions. +unsigned Filter::usefulness() const { + if (!VariableInstructions.empty()) + return FilteredInstructions.size(); + else + return FilteredInstructions.size() + 1; +} + +////////////////////////////////// +// // +// Filterchooser Implementation // +// // +////////////////////////////////// + +// Emit the decoder state machine table. +void FixedLenDecoderEmitter::emitTable(formatted_raw_ostream &OS, + DecoderTable &Table, + unsigned Indentation, + unsigned BitWidth, + StringRef Namespace) const { + OS.indent(Indentation) << "static const uint8_t DecoderTable" << Namespace + << BitWidth << "[] = {\n"; + + Indentation += 2; + + // FIXME: We may be able to use the NumToSkip values to recover + // appropriate indentation levels. + DecoderTable::const_iterator I = Table.begin(); + DecoderTable::const_iterator E = Table.end(); + while (I != E) { + assert (I < E && "incomplete decode table entry!"); + + uint64_t Pos = I - Table.begin(); + OS << "/* " << Pos << " */"; + OS.PadToColumn(12); + + switch (*I) { + default: + PrintFatalError("invalid decode table opcode"); + case MCD::OPC_ExtractField: { + ++I; + unsigned Start = *I++; + unsigned Len = *I++; + OS.indent(Indentation) << "MCD::OPC_ExtractField, " << Start << ", " + << Len << ", // Inst{"; + if (Len > 1) + OS << (Start + Len - 1) << "-"; + OS << Start << "} ...\n"; + break; + } + case MCD::OPC_FilterValue: { + ++I; + OS.indent(Indentation) << "MCD::OPC_FilterValue, "; + // The filter value is ULEB128 encoded. + while (*I >= 128) + OS << (unsigned)*I++ << ", "; + OS << (unsigned)*I++ << ", "; + + // 24-bit numtoskip value. + uint8_t Byte = *I++; + uint32_t NumToSkip = Byte; + OS << (unsigned)Byte << ", "; + Byte = *I++; + OS << (unsigned)Byte << ", "; + NumToSkip |= Byte << 8; + Byte = *I++; + OS << utostr(Byte) << ", "; + NumToSkip |= Byte << 16; + OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n"; + break; + } + case MCD::OPC_CheckField: { + ++I; + unsigned Start = *I++; + unsigned Len = *I++; + OS.indent(Indentation) << "MCD::OPC_CheckField, " << Start << ", " + << Len << ", ";// << Val << ", " << NumToSkip << ",\n"; + // ULEB128 encoded field value. + for (; *I >= 128; ++I) + OS << (unsigned)*I << ", "; + OS << (unsigned)*I++ << ", "; + // 24-bit numtoskip value. + uint8_t Byte = *I++; + uint32_t NumToSkip = Byte; + OS << (unsigned)Byte << ", "; + Byte = *I++; + OS << (unsigned)Byte << ", "; + NumToSkip |= Byte << 8; + Byte = *I++; + OS << utostr(Byte) << ", "; + NumToSkip |= Byte << 16; + OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n"; + break; + } + case MCD::OPC_CheckPredicate: { + ++I; + OS.indent(Indentation) << "MCD::OPC_CheckPredicate, "; + for (; *I >= 128; ++I) + OS << (unsigned)*I << ", "; + OS << (unsigned)*I++ << ", "; + + // 24-bit numtoskip value. + uint8_t Byte = *I++; + uint32_t NumToSkip = Byte; + OS << (unsigned)Byte << ", "; + Byte = *I++; + OS << (unsigned)Byte << ", "; + NumToSkip |= Byte << 8; + Byte = *I++; + OS << utostr(Byte) << ", "; + NumToSkip |= Byte << 16; + OS << "// Skip to: " << ((I - Table.begin()) + NumToSkip) << "\n"; + break; + } + case MCD::OPC_Decode: + case MCD::OPC_TryDecode: { + bool IsTry = *I == MCD::OPC_TryDecode; + ++I; + // Extract the ULEB128 encoded Opcode to a buffer. + uint8_t Buffer[16], *p = Buffer; + while ((*p++ = *I++) >= 128) + assert((p - Buffer) <= (ptrdiff_t)sizeof(Buffer) + && "ULEB128 value too large!"); + // Decode the Opcode value. + unsigned Opc = decodeULEB128(Buffer); + OS.indent(Indentation) << "MCD::OPC_" << (IsTry ? "Try" : "") + << "Decode, "; + for (p = Buffer; *p >= 128; ++p) + OS << (unsigned)*p << ", "; + OS << (unsigned)*p << ", "; + + // Decoder index. + for (; *I >= 128; ++I) + OS << (unsigned)*I << ", "; + OS << (unsigned)*I++ << ", "; + + if (!IsTry) { + OS << "// Opcode: " << NumberedEncodings[Opc] << "\n"; + break; + } + + // Fallthrough for OPC_TryDecode. + + // 24-bit numtoskip value. + uint8_t Byte = *I++; + uint32_t NumToSkip = Byte; + OS << (unsigned)Byte << ", "; + Byte = *I++; + OS << (unsigned)Byte << ", "; + NumToSkip |= Byte << 8; + Byte = *I++; + OS << utostr(Byte) << ", "; + NumToSkip |= Byte << 16; + + OS << "// Opcode: " << NumberedEncodings[Opc] + << ", skip to: " << ((I - Table.begin()) + NumToSkip) << "\n"; + break; + } + case MCD::OPC_SoftFail: { + ++I; + OS.indent(Indentation) << "MCD::OPC_SoftFail"; + // Positive mask + uint64_t Value = 0; + unsigned Shift = 0; + do { + OS << ", " << (unsigned)*I; + Value += (*I & 0x7f) << Shift; + Shift += 7; + } while (*I++ >= 128); + if (Value > 127) { + OS << " /* 0x"; + OS.write_hex(Value); + OS << " */"; + } + // Negative mask + Value = 0; + Shift = 0; + do { + OS << ", " << (unsigned)*I; + Value += (*I & 0x7f) << Shift; + Shift += 7; + } while (*I++ >= 128); + if (Value > 127) { + OS << " /* 0x"; + OS.write_hex(Value); + OS << " */"; + } + OS << ",\n"; + break; + } + case MCD::OPC_Fail: { + ++I; + OS.indent(Indentation) << "MCD::OPC_Fail,\n"; + break; + } + } + } + OS.indent(Indentation) << "0\n"; + + Indentation -= 2; + + OS.indent(Indentation) << "};\n\n"; +} + +void FixedLenDecoderEmitter:: +emitPredicateFunction(formatted_raw_ostream &OS, PredicateSet &Predicates, + unsigned Indentation) const { + // The predicate function is just a big switch statement based on the + // input predicate index. + OS.indent(Indentation) << "static bool checkDecoderPredicate(unsigned Idx, " + << "const FeatureBitset& Bits) {\n"; + Indentation += 2; + if (!Predicates.empty()) { + OS.indent(Indentation) << "switch (Idx) {\n"; + OS.indent(Indentation) << "default: llvm_unreachable(\"Invalid index!\");\n"; + unsigned Index = 0; + for (const auto &Predicate : Predicates) { + OS.indent(Indentation) << "case " << Index++ << ":\n"; + OS.indent(Indentation+2) << "return (" << Predicate << ");\n"; + } + OS.indent(Indentation) << "}\n"; + } else { + // No case statement to emit + OS.indent(Indentation) << "llvm_unreachable(\"Invalid index!\");\n"; + } + Indentation -= 2; + OS.indent(Indentation) << "}\n\n"; +} + +void FixedLenDecoderEmitter:: +emitDecoderFunction(formatted_raw_ostream &OS, DecoderSet &Decoders, + unsigned Indentation) const { + // The decoder function is just a big switch statement based on the + // input decoder index. + OS.indent(Indentation) << "template<typename InsnType>\n"; + OS.indent(Indentation) << "static DecodeStatus decodeToMCInst(DecodeStatus S," + << " unsigned Idx, InsnType insn, MCInst &MI,\n"; + OS.indent(Indentation) << " uint64_t " + << "Address, const void *Decoder, bool &DecodeComplete) {\n"; + Indentation += 2; + OS.indent(Indentation) << "DecodeComplete = true;\n"; + OS.indent(Indentation) << "InsnType tmp;\n"; + OS.indent(Indentation) << "switch (Idx) {\n"; + OS.indent(Indentation) << "default: llvm_unreachable(\"Invalid index!\");\n"; + unsigned Index = 0; + for (const auto &Decoder : Decoders) { + OS.indent(Indentation) << "case " << Index++ << ":\n"; + OS << Decoder; + OS.indent(Indentation+2) << "return S;\n"; + } + OS.indent(Indentation) << "}\n"; + Indentation -= 2; + OS.indent(Indentation) << "}\n\n"; +} + +// Populates the field of the insn given the start position and the number of +// consecutive bits to scan for. +// +// Returns false if and on the first uninitialized bit value encountered. +// Returns true, otherwise. +bool FilterChooser::fieldFromInsn(uint64_t &Field, insn_t &Insn, + unsigned StartBit, unsigned NumBits) const { + Field = 0; + + for (unsigned i = 0; i < NumBits; ++i) { + if (Insn[StartBit + i] == BIT_UNSET) + return false; + + if (Insn[StartBit + i] == BIT_TRUE) + Field = Field | (1ULL << i); + } + + return true; +} + +/// dumpFilterArray - dumpFilterArray prints out debugging info for the given +/// filter array as a series of chars. +void FilterChooser::dumpFilterArray(raw_ostream &o, + const std::vector<bit_value_t> &filter) const { + for (unsigned bitIndex = BitWidth; bitIndex > 0; bitIndex--) { + switch (filter[bitIndex - 1]) { + case BIT_UNFILTERED: + o << "."; + break; + case BIT_UNSET: + o << "_"; + break; + case BIT_TRUE: + o << "1"; + break; + case BIT_FALSE: + o << "0"; + break; + } + } +} + +/// dumpStack - dumpStack traverses the filter chooser chain and calls +/// dumpFilterArray on each filter chooser up to the top level one. +void FilterChooser::dumpStack(raw_ostream &o, const char *prefix) const { + const FilterChooser *current = this; + + while (current) { + o << prefix; + dumpFilterArray(o, current->FilterBitValues); + o << '\n'; + current = current->Parent; + } +} + +// Calculates the island(s) needed to decode the instruction. +// This returns a list of undecoded bits of an instructions, for example, +// Inst{20} = 1 && Inst{3-0} == 0b1111 represents two islands of yet-to-be +// decoded bits in order to verify that the instruction matches the Opcode. +unsigned FilterChooser::getIslands(std::vector<unsigned> &StartBits, + std::vector<unsigned> &EndBits, + std::vector<uint64_t> &FieldVals, + const insn_t &Insn) const { + unsigned Num, BitNo; + Num = BitNo = 0; + + uint64_t FieldVal = 0; + + // 0: Init + // 1: Water (the bit value does not affect decoding) + // 2: Island (well-known bit value needed for decoding) + int State = 0; + + for (unsigned i = 0; i < BitWidth; ++i) { + int64_t Val = Value(Insn[i]); + bool Filtered = PositionFiltered(i); + switch (State) { + default: llvm_unreachable("Unreachable code!"); + case 0: + case 1: + if (Filtered || Val == -1) + State = 1; // Still in Water + else { + State = 2; // Into the Island + BitNo = 0; + StartBits.push_back(i); + FieldVal = Val; + } + break; + case 2: + if (Filtered || Val == -1) { + State = 1; // Into the Water + EndBits.push_back(i - 1); + FieldVals.push_back(FieldVal); + ++Num; + } else { + State = 2; // Still in Island + ++BitNo; + FieldVal = FieldVal | Val << BitNo; + } + break; + } + } + // If we are still in Island after the loop, do some housekeeping. + if (State == 2) { + EndBits.push_back(BitWidth - 1); + FieldVals.push_back(FieldVal); + ++Num; + } + + assert(StartBits.size() == Num && EndBits.size() == Num && + FieldVals.size() == Num); + return Num; +} + +void FilterChooser::emitBinaryParser(raw_ostream &o, unsigned &Indentation, + const OperandInfo &OpInfo, + bool &OpHasCompleteDecoder) const { + const std::string &Decoder = OpInfo.Decoder; + + if (OpInfo.numFields() != 1 || OpInfo.InitValue != 0) { + o.indent(Indentation) << "tmp = 0x"; + o.write_hex(OpInfo.InitValue); + o << ";\n"; + } + + for (const EncodingField &EF : OpInfo) { + o.indent(Indentation) << "tmp "; + if (OpInfo.numFields() != 1 || OpInfo.InitValue != 0) o << '|'; + o << "= fieldFromInstruction" + << "(insn, " << EF.Base << ", " << EF.Width << ')'; + if (OpInfo.numFields() != 1 || EF.Offset != 0) + o << " << " << EF.Offset; + o << ";\n"; + } + + if (Decoder != "") { + OpHasCompleteDecoder = OpInfo.HasCompleteDecoder; + o.indent(Indentation) << Emitter->GuardPrefix << Decoder + << "(MI, tmp, Address, Decoder)" + << Emitter->GuardPostfix + << " { " << (OpHasCompleteDecoder ? "" : "DecodeComplete = false; ") + << "return MCDisassembler::Fail; }\n"; + } else { + OpHasCompleteDecoder = true; + o.indent(Indentation) << "MI.addOperand(MCOperand::createImm(tmp));\n"; + } +} + +void FilterChooser::emitDecoder(raw_ostream &OS, unsigned Indentation, + unsigned Opc, bool &HasCompleteDecoder) const { + HasCompleteDecoder = true; + + for (const auto &Op : Operands.find(Opc)->second) { + // If a custom instruction decoder was specified, use that. + if (Op.numFields() == 0 && !Op.Decoder.empty()) { + HasCompleteDecoder = Op.HasCompleteDecoder; + OS.indent(Indentation) << Emitter->GuardPrefix << Op.Decoder + << "(MI, insn, Address, Decoder)" + << Emitter->GuardPostfix + << " { " << (HasCompleteDecoder ? "" : "DecodeComplete = false; ") + << "return MCDisassembler::Fail; }\n"; + break; + } + + bool OpHasCompleteDecoder; + emitBinaryParser(OS, Indentation, Op, OpHasCompleteDecoder); + if (!OpHasCompleteDecoder) + HasCompleteDecoder = false; + } +} + +unsigned FilterChooser::getDecoderIndex(DecoderSet &Decoders, + unsigned Opc, + bool &HasCompleteDecoder) const { + // Build up the predicate string. + SmallString<256> Decoder; + // FIXME: emitDecoder() function can take a buffer directly rather than + // a stream. + raw_svector_ostream S(Decoder); + unsigned I = 4; + emitDecoder(S, I, Opc, HasCompleteDecoder); + + // Using the full decoder string as the key value here is a bit + // heavyweight, but is effective. If the string comparisons become a + // performance concern, we can implement a mangling of the predicate + // data easily enough with a map back to the actual string. That's + // overkill for now, though. + + // Make sure the predicate is in the table. + Decoders.insert(CachedHashString(Decoder)); + // Now figure out the index for when we write out the table. + DecoderSet::const_iterator P = find(Decoders, Decoder.str()); + return (unsigned)(P - Decoders.begin()); +} + +bool FilterChooser::emitPredicateMatch(raw_ostream &o, unsigned &Indentation, + unsigned Opc) const { + ListInit *Predicates = + AllInstructions[Opc].EncodingDef->getValueAsListInit("Predicates"); + bool IsFirstEmission = true; + for (unsigned i = 0; i < Predicates->size(); ++i) { + Record *Pred = Predicates->getElementAsRecord(i); + if (!Pred->getValue("AssemblerMatcherPredicate")) + continue; + + if (!dyn_cast<DagInit>(Pred->getValue("AssemblerCondDag")->getValue())) + continue; + + const DagInit *D = Pred->getValueAsDag("AssemblerCondDag"); + std::string CombineType = D->getOperator()->getAsString(); + if (CombineType != "any_of" && CombineType != "all_of") + PrintFatalError(Pred->getLoc(), "Invalid AssemblerCondDag!"); + if (D->getNumArgs() == 0) + PrintFatalError(Pred->getLoc(), "Invalid AssemblerCondDag!"); + bool IsOr = CombineType == "any_of"; + + if (!IsFirstEmission) + o << " && "; + + if (IsOr) + o << "("; + + bool First = true; + for (auto *Arg : D->getArgs()) { + if (!First) { + if (IsOr) + o << " || "; + else + o << " && "; + } + if (auto *NotArg = dyn_cast<DagInit>(Arg)) { + if (NotArg->getOperator()->getAsString() != "not" || + NotArg->getNumArgs() != 1) + PrintFatalError(Pred->getLoc(), "Invalid AssemblerCondDag!"); + Arg = NotArg->getArg(0); + o << "!"; + } + if (!isa<DefInit>(Arg) || + !cast<DefInit>(Arg)->getDef()->isSubClassOf("SubtargetFeature")) + PrintFatalError(Pred->getLoc(), "Invalid AssemblerCondDag!"); + o << "Bits[" << Emitter->PredicateNamespace << "::" << Arg->getAsString() + << "]"; + + First = false; + } + + if (IsOr) + o << ")"; + + IsFirstEmission = false; + } + return !Predicates->empty(); +} + +bool FilterChooser::doesOpcodeNeedPredicate(unsigned Opc) const { + ListInit *Predicates = + AllInstructions[Opc].EncodingDef->getValueAsListInit("Predicates"); + for (unsigned i = 0; i < Predicates->size(); ++i) { + Record *Pred = Predicates->getElementAsRecord(i); + if (!Pred->getValue("AssemblerMatcherPredicate")) + continue; + + if (dyn_cast<DagInit>(Pred->getValue("AssemblerCondDag")->getValue())) + return true; + } + return false; +} + +unsigned FilterChooser::getPredicateIndex(DecoderTableInfo &TableInfo, + StringRef Predicate) const { + // Using the full predicate string as the key value here is a bit + // heavyweight, but is effective. If the string comparisons become a + // performance concern, we can implement a mangling of the predicate + // data easily enough with a map back to the actual string. That's + // overkill for now, though. + + // Make sure the predicate is in the table. + TableInfo.Predicates.insert(CachedHashString(Predicate)); + // Now figure out the index for when we write out the table. + PredicateSet::const_iterator P = find(TableInfo.Predicates, Predicate); + return (unsigned)(P - TableInfo.Predicates.begin()); +} + +void FilterChooser::emitPredicateTableEntry(DecoderTableInfo &TableInfo, + unsigned Opc) const { + if (!doesOpcodeNeedPredicate(Opc)) + return; + + // Build up the predicate string. + SmallString<256> Predicate; + // FIXME: emitPredicateMatch() functions can take a buffer directly rather + // than a stream. + raw_svector_ostream PS(Predicate); + unsigned I = 0; + emitPredicateMatch(PS, I, Opc); + + // Figure out the index into the predicate table for the predicate just + // computed. + unsigned PIdx = getPredicateIndex(TableInfo, PS.str()); + SmallString<16> PBytes; + raw_svector_ostream S(PBytes); + encodeULEB128(PIdx, S); + + TableInfo.Table.push_back(MCD::OPC_CheckPredicate); + // Predicate index + for (unsigned i = 0, e = PBytes.size(); i != e; ++i) + TableInfo.Table.push_back(PBytes[i]); + // Push location for NumToSkip backpatching. + TableInfo.FixupStack.back().push_back(TableInfo.Table.size()); + TableInfo.Table.push_back(0); + TableInfo.Table.push_back(0); + TableInfo.Table.push_back(0); +} + +void FilterChooser::emitSoftFailTableEntry(DecoderTableInfo &TableInfo, + unsigned Opc) const { + BitsInit *SFBits = + AllInstructions[Opc].EncodingDef->getValueAsBitsInit("SoftFail"); + if (!SFBits) return; + BitsInit *InstBits = + AllInstructions[Opc].EncodingDef->getValueAsBitsInit("Inst"); + + APInt PositiveMask(BitWidth, 0ULL); + APInt NegativeMask(BitWidth, 0ULL); + for (unsigned i = 0; i < BitWidth; ++i) { + bit_value_t B = bitFromBits(*SFBits, i); + bit_value_t IB = bitFromBits(*InstBits, i); + + if (B != BIT_TRUE) continue; + + switch (IB) { + case BIT_FALSE: + // The bit is meant to be false, so emit a check to see if it is true. + PositiveMask.setBit(i); + break; + case BIT_TRUE: + // The bit is meant to be true, so emit a check to see if it is false. + NegativeMask.setBit(i); + break; + default: + // The bit is not set; this must be an error! + errs() << "SoftFail Conflict: bit SoftFail{" << i << "} in " + << AllInstructions[Opc] << " is set but Inst{" << i + << "} is unset!\n" + << " - You can only mark a bit as SoftFail if it is fully defined" + << " (1/0 - not '?') in Inst\n"; + return; + } + } + + bool NeedPositiveMask = PositiveMask.getBoolValue(); + bool NeedNegativeMask = NegativeMask.getBoolValue(); + + if (!NeedPositiveMask && !NeedNegativeMask) + return; + + TableInfo.Table.push_back(MCD::OPC_SoftFail); + + SmallString<16> MaskBytes; + raw_svector_ostream S(MaskBytes); + if (NeedPositiveMask) { + encodeULEB128(PositiveMask.getZExtValue(), S); + for (unsigned i = 0, e = MaskBytes.size(); i != e; ++i) + TableInfo.Table.push_back(MaskBytes[i]); + } else + TableInfo.Table.push_back(0); + if (NeedNegativeMask) { + MaskBytes.clear(); + encodeULEB128(NegativeMask.getZExtValue(), S); + for (unsigned i = 0, e = MaskBytes.size(); i != e; ++i) + TableInfo.Table.push_back(MaskBytes[i]); + } else + TableInfo.Table.push_back(0); +} + +// Emits table entries to decode the singleton. +void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo, + EncodingIDAndOpcode Opc) const { + std::vector<unsigned> StartBits; + std::vector<unsigned> EndBits; + std::vector<uint64_t> FieldVals; + insn_t Insn; + insnWithID(Insn, Opc.EncodingID); + + // Look for islands of undecoded bits of the singleton. + getIslands(StartBits, EndBits, FieldVals, Insn); + + unsigned Size = StartBits.size(); + + // Emit the predicate table entry if one is needed. + emitPredicateTableEntry(TableInfo, Opc.EncodingID); + + // Check any additional encoding fields needed. + for (unsigned I = Size; I != 0; --I) { + unsigned NumBits = EndBits[I-1] - StartBits[I-1] + 1; + TableInfo.Table.push_back(MCD::OPC_CheckField); + TableInfo.Table.push_back(StartBits[I-1]); + TableInfo.Table.push_back(NumBits); + uint8_t Buffer[16], *p; + encodeULEB128(FieldVals[I-1], Buffer); + for (p = Buffer; *p >= 128 ; ++p) + TableInfo.Table.push_back(*p); + TableInfo.Table.push_back(*p); + // Push location for NumToSkip backpatching. + TableInfo.FixupStack.back().push_back(TableInfo.Table.size()); + // The fixup is always 24-bits, so go ahead and allocate the space + // in the table so all our relative position calculations work OK even + // before we fully resolve the real value here. + TableInfo.Table.push_back(0); + TableInfo.Table.push_back(0); + TableInfo.Table.push_back(0); + } + + // Check for soft failure of the match. + emitSoftFailTableEntry(TableInfo, Opc.EncodingID); + + bool HasCompleteDecoder; + unsigned DIdx = + getDecoderIndex(TableInfo.Decoders, Opc.EncodingID, HasCompleteDecoder); + + // Produce OPC_Decode or OPC_TryDecode opcode based on the information + // whether the instruction decoder is complete or not. If it is complete + // then it handles all possible values of remaining variable/unfiltered bits + // and for any value can determine if the bitpattern is a valid instruction + // or not. This means OPC_Decode will be the final step in the decoding + // process. If it is not complete, then the Fail return code from the + // decoder method indicates that additional processing should be done to see + // if there is any other instruction that also matches the bitpattern and + // can decode it. + TableInfo.Table.push_back(HasCompleteDecoder ? MCD::OPC_Decode : + MCD::OPC_TryDecode); + NumEncodingsSupported++; + uint8_t Buffer[16], *p; + encodeULEB128(Opc.Opcode, Buffer); + for (p = Buffer; *p >= 128 ; ++p) + TableInfo.Table.push_back(*p); + TableInfo.Table.push_back(*p); + + SmallString<16> Bytes; + raw_svector_ostream S(Bytes); + encodeULEB128(DIdx, S); + + // Decoder index + for (unsigned i = 0, e = Bytes.size(); i != e; ++i) + TableInfo.Table.push_back(Bytes[i]); + + if (!HasCompleteDecoder) { + // Push location for NumToSkip backpatching. + TableInfo.FixupStack.back().push_back(TableInfo.Table.size()); + // Allocate the space for the fixup. + TableInfo.Table.push_back(0); + TableInfo.Table.push_back(0); + TableInfo.Table.push_back(0); + } +} + +// Emits table entries to decode the singleton, and then to decode the rest. +void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo, + const Filter &Best) const { + EncodingIDAndOpcode Opc = Best.getSingletonOpc(); + + // complex singletons need predicate checks from the first singleton + // to refer forward to the variable filterchooser that follows. + TableInfo.FixupStack.emplace_back(); + + emitSingletonTableEntry(TableInfo, Opc); + + resolveTableFixups(TableInfo.Table, TableInfo.FixupStack.back(), + TableInfo.Table.size()); + TableInfo.FixupStack.pop_back(); + + Best.getVariableFC().emitTableEntries(TableInfo); +} + +// Assign a single filter and run with it. Top level API client can initialize +// with a single filter to start the filtering process. +void FilterChooser::runSingleFilter(unsigned startBit, unsigned numBit, + bool mixed) { + Filters.clear(); + Filters.emplace_back(*this, startBit, numBit, true); + BestIndex = 0; // Sole Filter instance to choose from. + bestFilter().recurse(); +} + +// reportRegion is a helper function for filterProcessor to mark a region as +// eligible for use as a filter region. +void FilterChooser::reportRegion(bitAttr_t RA, unsigned StartBit, + unsigned BitIndex, bool AllowMixed) { + if (RA == ATTR_MIXED && AllowMixed) + Filters.emplace_back(*this, StartBit, BitIndex - StartBit, true); + else if (RA == ATTR_ALL_SET && !AllowMixed) + Filters.emplace_back(*this, StartBit, BitIndex - StartBit, false); +} + +// FilterProcessor scans the well-known encoding bits of the instructions and +// builds up a list of candidate filters. It chooses the best filter and +// recursively descends down the decoding tree. +bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) { + Filters.clear(); + BestIndex = -1; + unsigned numInstructions = Opcodes.size(); + + assert(numInstructions && "Filter created with no instructions"); + + // No further filtering is necessary. + if (numInstructions == 1) + return true; + + // Heuristics. See also doFilter()'s "Heuristics" comment when num of + // instructions is 3. + if (AllowMixed && !Greedy) { + assert(numInstructions == 3); + + for (unsigned i = 0; i < Opcodes.size(); ++i) { + std::vector<unsigned> StartBits; + std::vector<unsigned> EndBits; + std::vector<uint64_t> FieldVals; + insn_t Insn; + + insnWithID(Insn, Opcodes[i].EncodingID); + + // Look for islands of undecoded bits of any instruction. + if (getIslands(StartBits, EndBits, FieldVals, Insn) > 0) { + // Found an instruction with island(s). Now just assign a filter. + runSingleFilter(StartBits[0], EndBits[0] - StartBits[0] + 1, true); + return true; + } + } + } + + unsigned BitIndex; + + // We maintain BIT_WIDTH copies of the bitAttrs automaton. + // The automaton consumes the corresponding bit from each + // instruction. + // + // Input symbols: 0, 1, and _ (unset). + // States: NONE, FILTERED, ALL_SET, ALL_UNSET, and MIXED. + // Initial state: NONE. + // + // (NONE) ------- [01] -> (ALL_SET) + // (NONE) ------- _ ----> (ALL_UNSET) + // (ALL_SET) ---- [01] -> (ALL_SET) + // (ALL_SET) ---- _ ----> (MIXED) + // (ALL_UNSET) -- [01] -> (MIXED) + // (ALL_UNSET) -- _ ----> (ALL_UNSET) + // (MIXED) ------ . ----> (MIXED) + // (FILTERED)---- . ----> (FILTERED) + + std::vector<bitAttr_t> bitAttrs; + + // FILTERED bit positions provide no entropy and are not worthy of pursuing. + // Filter::recurse() set either BIT_TRUE or BIT_FALSE for each position. + for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex) + if (FilterBitValues[BitIndex] == BIT_TRUE || + FilterBitValues[BitIndex] == BIT_FALSE) + bitAttrs.push_back(ATTR_FILTERED); + else + bitAttrs.push_back(ATTR_NONE); + + for (unsigned InsnIndex = 0; InsnIndex < numInstructions; ++InsnIndex) { + insn_t insn; + + insnWithID(insn, Opcodes[InsnIndex].EncodingID); + + for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex) { + switch (bitAttrs[BitIndex]) { + case ATTR_NONE: + if (insn[BitIndex] == BIT_UNSET) + bitAttrs[BitIndex] = ATTR_ALL_UNSET; + else + bitAttrs[BitIndex] = ATTR_ALL_SET; + break; + case ATTR_ALL_SET: + if (insn[BitIndex] == BIT_UNSET) + bitAttrs[BitIndex] = ATTR_MIXED; + break; + case ATTR_ALL_UNSET: + if (insn[BitIndex] != BIT_UNSET) + bitAttrs[BitIndex] = ATTR_MIXED; + break; + case ATTR_MIXED: + case ATTR_FILTERED: + break; + } + } + } + + // The regionAttr automaton consumes the bitAttrs automatons' state, + // lowest-to-highest. + // + // Input symbols: F(iltered), (all_)S(et), (all_)U(nset), M(ixed) + // States: NONE, ALL_SET, MIXED + // Initial state: NONE + // + // (NONE) ----- F --> (NONE) + // (NONE) ----- S --> (ALL_SET) ; and set region start + // (NONE) ----- U --> (NONE) + // (NONE) ----- M --> (MIXED) ; and set region start + // (ALL_SET) -- F --> (NONE) ; and report an ALL_SET region + // (ALL_SET) -- S --> (ALL_SET) + // (ALL_SET) -- U --> (NONE) ; and report an ALL_SET region + // (ALL_SET) -- M --> (MIXED) ; and report an ALL_SET region + // (MIXED) ---- F --> (NONE) ; and report a MIXED region + // (MIXED) ---- S --> (ALL_SET) ; and report a MIXED region + // (MIXED) ---- U --> (NONE) ; and report a MIXED region + // (MIXED) ---- M --> (MIXED) + + bitAttr_t RA = ATTR_NONE; + unsigned StartBit = 0; + + for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex) { + bitAttr_t bitAttr = bitAttrs[BitIndex]; + + assert(bitAttr != ATTR_NONE && "Bit without attributes"); + + switch (RA) { + case ATTR_NONE: + switch (bitAttr) { + case ATTR_FILTERED: + break; + case ATTR_ALL_SET: + StartBit = BitIndex; + RA = ATTR_ALL_SET; + break; + case ATTR_ALL_UNSET: + break; + case ATTR_MIXED: + StartBit = BitIndex; + RA = ATTR_MIXED; + break; + default: + llvm_unreachable("Unexpected bitAttr!"); + } + break; + case ATTR_ALL_SET: + switch (bitAttr) { + case ATTR_FILTERED: + reportRegion(RA, StartBit, BitIndex, AllowMixed); + RA = ATTR_NONE; + break; + case ATTR_ALL_SET: + break; + case ATTR_ALL_UNSET: + reportRegion(RA, StartBit, BitIndex, AllowMixed); + RA = ATTR_NONE; + break; + case ATTR_MIXED: + reportRegion(RA, StartBit, BitIndex, AllowMixed); + StartBit = BitIndex; + RA = ATTR_MIXED; + break; + default: + llvm_unreachable("Unexpected bitAttr!"); + } + break; + case ATTR_MIXED: + switch (bitAttr) { + case ATTR_FILTERED: + reportRegion(RA, StartBit, BitIndex, AllowMixed); + StartBit = BitIndex; + RA = ATTR_NONE; + break; + case ATTR_ALL_SET: + reportRegion(RA, StartBit, BitIndex, AllowMixed); + StartBit = BitIndex; + RA = ATTR_ALL_SET; + break; + case ATTR_ALL_UNSET: + reportRegion(RA, StartBit, BitIndex, AllowMixed); + RA = ATTR_NONE; + break; + case ATTR_MIXED: + break; + default: + llvm_unreachable("Unexpected bitAttr!"); + } + break; + case ATTR_ALL_UNSET: + llvm_unreachable("regionAttr state machine has no ATTR_UNSET state"); + case ATTR_FILTERED: + llvm_unreachable("regionAttr state machine has no ATTR_FILTERED state"); + } + } + + // At the end, if we're still in ALL_SET or MIXED states, report a region + switch (RA) { + case ATTR_NONE: + break; + case ATTR_FILTERED: + break; + case ATTR_ALL_SET: + reportRegion(RA, StartBit, BitIndex, AllowMixed); + break; + case ATTR_ALL_UNSET: + break; + case ATTR_MIXED: + reportRegion(RA, StartBit, BitIndex, AllowMixed); + break; + } + + // We have finished with the filter processings. Now it's time to choose + // the best performing filter. + BestIndex = 0; + bool AllUseless = true; + unsigned BestScore = 0; + + for (unsigned i = 0, e = Filters.size(); i != e; ++i) { + unsigned Usefulness = Filters[i].usefulness(); + + if (Usefulness) + AllUseless = false; + + if (Usefulness > BestScore) { + BestIndex = i; + BestScore = Usefulness; + } + } + + if (!AllUseless) + bestFilter().recurse(); + + return !AllUseless; +} // end of FilterChooser::filterProcessor(bool) + +// Decides on the best configuration of filter(s) to use in order to decode +// the instructions. A conflict of instructions may occur, in which case we +// dump the conflict set to the standard error. +void FilterChooser::doFilter() { + unsigned Num = Opcodes.size(); + assert(Num && "FilterChooser created with no instructions"); + + // Try regions of consecutive known bit values first. + if (filterProcessor(false)) + return; + + // Then regions of mixed bits (both known and unitialized bit values allowed). + if (filterProcessor(true)) + return; + + // Heuristics to cope with conflict set {t2CMPrs, t2SUBSrr, t2SUBSrs} where + // no single instruction for the maximum ATTR_MIXED region Inst{14-4} has a + // well-known encoding pattern. In such case, we backtrack and scan for the + // the very first consecutive ATTR_ALL_SET region and assign a filter to it. + if (Num == 3 && filterProcessor(true, false)) + return; + + // If we come to here, the instruction decoding has failed. + // Set the BestIndex to -1 to indicate so. + BestIndex = -1; +} + +// emitTableEntries - Emit state machine entries to decode our share of +// instructions. +void FilterChooser::emitTableEntries(DecoderTableInfo &TableInfo) const { + if (Opcodes.size() == 1) { + // There is only one instruction in the set, which is great! + // Call emitSingletonDecoder() to see whether there are any remaining + // encodings bits. + emitSingletonTableEntry(TableInfo, Opcodes[0]); + return; + } + + // Choose the best filter to do the decodings! + if (BestIndex != -1) { + const Filter &Best = Filters[BestIndex]; + if (Best.getNumFiltered() == 1) + emitSingletonTableEntry(TableInfo, Best); + else + Best.emitTableEntry(TableInfo); + return; + } + + // We don't know how to decode these instructions! Dump the + // conflict set and bail. + + // Print out useful conflict information for postmortem analysis. + errs() << "Decoding Conflict:\n"; + + dumpStack(errs(), "\t\t"); + + for (unsigned i = 0; i < Opcodes.size(); ++i) { + errs() << '\t'; + emitNameWithID(errs(), Opcodes[i].EncodingID); + errs() << " "; + dumpBits( + errs(), + getBitsField(*AllInstructions[Opcodes[i].EncodingID].EncodingDef, "Inst")); + errs() << '\n'; + } +} + +static std::string findOperandDecoderMethod(TypedInit *TI) { + std::string Decoder; + + Record *Record = cast<DefInit>(TI)->getDef(); + + RecordVal *DecoderString = Record->getValue("DecoderMethod"); + StringInit *String = DecoderString ? + dyn_cast<StringInit>(DecoderString->getValue()) : nullptr; + if (String) { + Decoder = std::string(String->getValue()); + if (!Decoder.empty()) + return Decoder; + } + + if (Record->isSubClassOf("RegisterOperand")) + Record = Record->getValueAsDef("RegClass"); + + if (Record->isSubClassOf("RegisterClass")) { + Decoder = "Decode" + Record->getName().str() + "RegisterClass"; + } else if (Record->isSubClassOf("PointerLikeRegClass")) { + Decoder = "DecodePointerLikeRegClass" + + utostr(Record->getValueAsInt("RegClassKind")); + } + + return Decoder; +} + +static bool +populateInstruction(CodeGenTarget &Target, const Record &EncodingDef, + const CodeGenInstruction &CGI, unsigned Opc, + std::map<unsigned, std::vector<OperandInfo>> &Operands) { + const Record &Def = *CGI.TheDef; + // If all the bit positions are not specified; do not decode this instruction. + // We are bound to fail! For proper disassembly, the well-known encoding bits + // of the instruction must be fully specified. + + BitsInit &Bits = getBitsField(EncodingDef, "Inst"); + if (Bits.allInComplete()) return false; + + std::vector<OperandInfo> InsnOperands; + + // If the instruction has specified a custom decoding hook, use that instead + // of trying to auto-generate the decoder. + StringRef InstDecoder = EncodingDef.getValueAsString("DecoderMethod"); + if (InstDecoder != "") { + bool HasCompleteInstDecoder = EncodingDef.getValueAsBit("hasCompleteDecoder"); + InsnOperands.push_back( + OperandInfo(std::string(InstDecoder), HasCompleteInstDecoder)); + Operands[Opc] = InsnOperands; + return true; + } + + // Generate a description of the operand of the instruction that we know + // how to decode automatically. + // FIXME: We'll need to have a way to manually override this as needed. + + // Gather the outputs/inputs of the instruction, so we can find their + // positions in the encoding. This assumes for now that they appear in the + // MCInst in the order that they're listed. + std::vector<std::pair<Init*, StringRef>> InOutOperands; + DagInit *Out = Def.getValueAsDag("OutOperandList"); + DagInit *In = Def.getValueAsDag("InOperandList"); + for (unsigned i = 0; i < Out->getNumArgs(); ++i) + InOutOperands.push_back(std::make_pair(Out->getArg(i), + Out->getArgNameStr(i))); + for (unsigned i = 0; i < In->getNumArgs(); ++i) + InOutOperands.push_back(std::make_pair(In->getArg(i), + In->getArgNameStr(i))); + + // Search for tied operands, so that we can correctly instantiate + // operands that are not explicitly represented in the encoding. + std::map<std::string, std::string> TiedNames; + for (unsigned i = 0; i < CGI.Operands.size(); ++i) { + int tiedTo = CGI.Operands[i].getTiedRegister(); + if (tiedTo != -1) { + std::pair<unsigned, unsigned> SO = + CGI.Operands.getSubOperandNumber(tiedTo); + TiedNames[std::string(InOutOperands[i].second)] = + std::string(InOutOperands[SO.first].second); + TiedNames[std::string(InOutOperands[SO.first].second)] = + std::string(InOutOperands[i].second); + } + } + + std::map<std::string, std::vector<OperandInfo>> NumberedInsnOperands; + std::set<std::string> NumberedInsnOperandsNoTie; + if (Target.getInstructionSet()-> + getValueAsBit("decodePositionallyEncodedOperands")) { + const std::vector<RecordVal> &Vals = Def.getValues(); + unsigned NumberedOp = 0; + + std::set<unsigned> NamedOpIndices; + if (Target.getInstructionSet()-> + getValueAsBit("noNamedPositionallyEncodedOperands")) + // Collect the set of operand indices that might correspond to named + // operand, and skip these when assigning operands based on position. + for (unsigned i = 0, e = Vals.size(); i != e; ++i) { + unsigned OpIdx; + if (!CGI.Operands.hasOperandNamed(Vals[i].getName(), OpIdx)) + continue; + + NamedOpIndices.insert(OpIdx); + } + + for (unsigned i = 0, e = Vals.size(); i != e; ++i) { + // Ignore fixed fields in the record, we're looking for values like: + // bits<5> RST = { ?, ?, ?, ?, ? }; + if (Vals[i].getPrefix() || Vals[i].getValue()->isComplete()) + continue; + + // Determine if Vals[i] actually contributes to the Inst encoding. + unsigned bi = 0; + for (; bi < Bits.getNumBits(); ++bi) { + VarInit *Var = nullptr; + VarBitInit *BI = dyn_cast<VarBitInit>(Bits.getBit(bi)); + if (BI) + Var = dyn_cast<VarInit>(BI->getBitVar()); + else + Var = dyn_cast<VarInit>(Bits.getBit(bi)); + + if (Var && Var->getName() == Vals[i].getName()) + break; + } + + if (bi == Bits.getNumBits()) + continue; + + // Skip variables that correspond to explicitly-named operands. + unsigned OpIdx; + if (CGI.Operands.hasOperandNamed(Vals[i].getName(), OpIdx)) + continue; + + // Get the bit range for this operand: + unsigned bitStart = bi++, bitWidth = 1; + for (; bi < Bits.getNumBits(); ++bi) { + VarInit *Var = nullptr; + VarBitInit *BI = dyn_cast<VarBitInit>(Bits.getBit(bi)); + if (BI) + Var = dyn_cast<VarInit>(BI->getBitVar()); + else + Var = dyn_cast<VarInit>(Bits.getBit(bi)); + + if (!Var) + break; + + if (Var->getName() != Vals[i].getName()) + break; + + ++bitWidth; + } + + unsigned NumberOps = CGI.Operands.size(); + while (NumberedOp < NumberOps && + (CGI.Operands.isFlatOperandNotEmitted(NumberedOp) || + (!NamedOpIndices.empty() && NamedOpIndices.count( + CGI.Operands.getSubOperandNumber(NumberedOp).first)))) + ++NumberedOp; + + OpIdx = NumberedOp++; + + // OpIdx now holds the ordered operand number of Vals[i]. + std::pair<unsigned, unsigned> SO = + CGI.Operands.getSubOperandNumber(OpIdx); + const std::string &Name = CGI.Operands[SO.first].Name; + + LLVM_DEBUG(dbgs() << "Numbered operand mapping for " << Def.getName() + << ": " << Name << "(" << SO.first << ", " << SO.second + << ") => " << Vals[i].getName() << "\n"); + + std::string Decoder; + Record *TypeRecord = CGI.Operands[SO.first].Rec; + + RecordVal *DecoderString = TypeRecord->getValue("DecoderMethod"); + StringInit *String = DecoderString ? + dyn_cast<StringInit>(DecoderString->getValue()) : nullptr; + if (String && String->getValue() != "") + Decoder = std::string(String->getValue()); + + if (Decoder == "" && + CGI.Operands[SO.first].MIOperandInfo && + CGI.Operands[SO.first].MIOperandInfo->getNumArgs()) { + Init *Arg = CGI.Operands[SO.first].MIOperandInfo-> + getArg(SO.second); + if (DefInit *DI = cast<DefInit>(Arg)) + TypeRecord = DI->getDef(); + } + + bool isReg = false; + if (TypeRecord->isSubClassOf("RegisterOperand")) + TypeRecord = TypeRecord->getValueAsDef("RegClass"); + if (TypeRecord->isSubClassOf("RegisterClass")) { + Decoder = "Decode" + TypeRecord->getName().str() + "RegisterClass"; + isReg = true; + } else if (TypeRecord->isSubClassOf("PointerLikeRegClass")) { + Decoder = "DecodePointerLikeRegClass" + + utostr(TypeRecord->getValueAsInt("RegClassKind")); + isReg = true; + } + + DecoderString = TypeRecord->getValue("DecoderMethod"); + String = DecoderString ? + dyn_cast<StringInit>(DecoderString->getValue()) : nullptr; + if (!isReg && String && String->getValue() != "") + Decoder = std::string(String->getValue()); + + RecordVal *HasCompleteDecoderVal = + TypeRecord->getValue("hasCompleteDecoder"); + BitInit *HasCompleteDecoderBit = HasCompleteDecoderVal ? + dyn_cast<BitInit>(HasCompleteDecoderVal->getValue()) : nullptr; + bool HasCompleteDecoder = HasCompleteDecoderBit ? + HasCompleteDecoderBit->getValue() : true; + + OperandInfo OpInfo(Decoder, HasCompleteDecoder); + OpInfo.addField(bitStart, bitWidth, 0); + + NumberedInsnOperands[Name].push_back(OpInfo); + + // FIXME: For complex operands with custom decoders we can't handle tied + // sub-operands automatically. Skip those here and assume that this is + // fixed up elsewhere. + if (CGI.Operands[SO.first].MIOperandInfo && + CGI.Operands[SO.first].MIOperandInfo->getNumArgs() > 1 && + String && String->getValue() != "") + NumberedInsnOperandsNoTie.insert(Name); + } + } + + // For each operand, see if we can figure out where it is encoded. + for (const auto &Op : InOutOperands) { + if (!NumberedInsnOperands[std::string(Op.second)].empty()) { + InsnOperands.insert(InsnOperands.end(), + NumberedInsnOperands[std::string(Op.second)].begin(), + NumberedInsnOperands[std::string(Op.second)].end()); + continue; + } + if (!NumberedInsnOperands[TiedNames[std::string(Op.second)]].empty()) { + if (!NumberedInsnOperandsNoTie.count(TiedNames[std::string(Op.second)])) { + // Figure out to which (sub)operand we're tied. + unsigned i = + CGI.Operands.getOperandNamed(TiedNames[std::string(Op.second)]); + int tiedTo = CGI.Operands[i].getTiedRegister(); + if (tiedTo == -1) { + i = CGI.Operands.getOperandNamed(Op.second); + tiedTo = CGI.Operands[i].getTiedRegister(); + } + + if (tiedTo != -1) { + std::pair<unsigned, unsigned> SO = + CGI.Operands.getSubOperandNumber(tiedTo); + + InsnOperands.push_back( + NumberedInsnOperands[TiedNames[std::string(Op.second)]] + [SO.second]); + } + } + continue; + } + + TypedInit *TI = cast<TypedInit>(Op.first); + + // At this point, we can locate the decoder field, but we need to know how + // to interpret it. As a first step, require the target to provide + // callbacks for decoding register classes. + std::string Decoder = findOperandDecoderMethod(TI); + Record *TypeRecord = cast<DefInit>(TI)->getDef(); + + RecordVal *HasCompleteDecoderVal = + TypeRecord->getValue("hasCompleteDecoder"); + BitInit *HasCompleteDecoderBit = HasCompleteDecoderVal ? + dyn_cast<BitInit>(HasCompleteDecoderVal->getValue()) : nullptr; + bool HasCompleteDecoder = HasCompleteDecoderBit ? + HasCompleteDecoderBit->getValue() : true; + + OperandInfo OpInfo(Decoder, HasCompleteDecoder); + + // Some bits of the operand may be required to be 1 depending on the + // instruction's encoding. Collect those bits. + if (const RecordVal *EncodedValue = EncodingDef.getValue(Op.second)) + if (const BitsInit *OpBits = dyn_cast<BitsInit>(EncodedValue->getValue())) + for (unsigned I = 0; I < OpBits->getNumBits(); ++I) + if (const BitInit *OpBit = dyn_cast<BitInit>(OpBits->getBit(I))) + if (OpBit->getValue()) + OpInfo.InitValue |= 1ULL << I; + + unsigned Base = ~0U; + unsigned Width = 0; + unsigned Offset = 0; + + for (unsigned bi = 0; bi < Bits.getNumBits(); ++bi) { + VarInit *Var = nullptr; + VarBitInit *BI = dyn_cast<VarBitInit>(Bits.getBit(bi)); + if (BI) + Var = dyn_cast<VarInit>(BI->getBitVar()); + else + Var = dyn_cast<VarInit>(Bits.getBit(bi)); + + if (!Var) { + if (Base != ~0U) { + OpInfo.addField(Base, Width, Offset); + Base = ~0U; + Width = 0; + Offset = 0; + } + continue; + } + + if (Var->getName() != Op.second && + Var->getName() != TiedNames[std::string(Op.second)]) { + if (Base != ~0U) { + OpInfo.addField(Base, Width, Offset); + Base = ~0U; + Width = 0; + Offset = 0; + } + continue; + } + + if (Base == ~0U) { + Base = bi; + Width = 1; + Offset = BI ? BI->getBitNum() : 0; + } else if (BI && BI->getBitNum() != Offset + Width) { + OpInfo.addField(Base, Width, Offset); + Base = bi; + Width = 1; + Offset = BI->getBitNum(); + } else { + ++Width; + } + } + + if (Base != ~0U) + OpInfo.addField(Base, Width, Offset); + + if (OpInfo.numFields() > 0) + InsnOperands.push_back(OpInfo); + } + + Operands[Opc] = InsnOperands; + +#if 0 + LLVM_DEBUG({ + // Dumps the instruction encoding bits. + dumpBits(errs(), Bits); + + errs() << '\n'; + + // Dumps the list of operand info. + for (unsigned i = 0, e = CGI.Operands.size(); i != e; ++i) { + const CGIOperandList::OperandInfo &Info = CGI.Operands[i]; + const std::string &OperandName = Info.Name; + const Record &OperandDef = *Info.Rec; + + errs() << "\t" << OperandName << " (" << OperandDef.getName() << ")\n"; + } + }); +#endif + + return true; +} + +// emitFieldFromInstruction - Emit the templated helper function +// fieldFromInstruction(). +// On Windows we make sure that this function is not inlined when +// using the VS compiler. It has a bug which causes the function +// to be optimized out in some circustances. See llvm.org/pr38292 +static void emitFieldFromInstruction(formatted_raw_ostream &OS) { + OS << "// Helper functions for extracting fields from encoded instructions.\n" + << "// InsnType must either be integral or an APInt-like object that " + "must:\n" + << "// * Have a static const max_size_in_bits equal to the number of bits " + "in the\n" + << "// encoding.\n" + << "// * be default-constructible and copy-constructible\n" + << "// * be constructible from a uint64_t\n" + << "// * be constructible from an APInt (this can be private)\n" + << "// * Support getBitsSet(loBit, hiBit)\n" + << "// * be convertible to uint64_t\n" + << "// * Support the ~, &, ==, !=, and |= operators with other objects of " + "the same type\n" + << "// * Support shift (<<, >>) with signed and unsigned integers on the " + "RHS\n" + << "// * Support put (<<) to raw_ostream&\n" + << "template<typename InsnType>\n" + << "#if defined(_MSC_VER) && !defined(__clang__)\n" + << "__declspec(noinline)\n" + << "#endif\n" + << "static InsnType fieldFromInstruction(InsnType insn, unsigned " + "startBit,\n" + << " unsigned numBits, " + "std::true_type) {\n" + << " assert(startBit + numBits <= 64 && \"Cannot support >64-bit " + "extractions!\");\n" + << " assert(startBit + numBits <= (sizeof(InsnType) * 8) &&\n" + << " \"Instruction field out of bounds!\");\n" + << " InsnType fieldMask;\n" + << " if (numBits == sizeof(InsnType) * 8)\n" + << " fieldMask = (InsnType)(-1LL);\n" + << " else\n" + << " fieldMask = (((InsnType)1 << numBits) - 1) << startBit;\n" + << " return (insn & fieldMask) >> startBit;\n" + << "}\n" + << "\n" + << "template<typename InsnType>\n" + << "static InsnType fieldFromInstruction(InsnType insn, unsigned " + "startBit,\n" + << " unsigned numBits, " + "std::false_type) {\n" + << " assert(startBit + numBits <= InsnType::max_size_in_bits && " + "\"Instruction field out of bounds!\");\n" + << " InsnType fieldMask = InsnType::getBitsSet(0, numBits);\n" + << " return (insn >> startBit) & fieldMask;\n" + << "}\n" + << "\n" + << "template<typename InsnType>\n" + << "static InsnType fieldFromInstruction(InsnType insn, unsigned " + "startBit,\n" + << " unsigned numBits) {\n" + << " return fieldFromInstruction(insn, startBit, numBits, " + "std::is_integral<InsnType>());\n" + << "}\n\n"; +} + +// emitDecodeInstruction - Emit the templated helper function +// decodeInstruction(). +static void emitDecodeInstruction(formatted_raw_ostream &OS) { + OS << "template<typename InsnType>\n" + << "static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], " + "MCInst &MI,\n" + << " InsnType insn, uint64_t " + "Address,\n" + << " const void *DisAsm,\n" + << " const MCSubtargetInfo &STI) {\n" + << " const FeatureBitset& Bits = STI.getFeatureBits();\n" + << "\n" + << " const uint8_t *Ptr = DecodeTable;\n" + << " InsnType CurFieldValue = 0;\n" + << " DecodeStatus S = MCDisassembler::Success;\n" + << " while (true) {\n" + << " ptrdiff_t Loc = Ptr - DecodeTable;\n" + << " switch (*Ptr) {\n" + << " default:\n" + << " errs() << Loc << \": Unexpected decode table opcode!\\n\";\n" + << " return MCDisassembler::Fail;\n" + << " case MCD::OPC_ExtractField: {\n" + << " unsigned Start = *++Ptr;\n" + << " unsigned Len = *++Ptr;\n" + << " ++Ptr;\n" + << " CurFieldValue = fieldFromInstruction(insn, Start, Len);\n" + << " LLVM_DEBUG(dbgs() << Loc << \": OPC_ExtractField(\" << Start << " + "\", \"\n" + << " << Len << \"): \" << CurFieldValue << \"\\n\");\n" + << " break;\n" + << " }\n" + << " case MCD::OPC_FilterValue: {\n" + << " // Decode the field value.\n" + << " unsigned Len;\n" + << " InsnType Val = decodeULEB128(++Ptr, &Len);\n" + << " Ptr += Len;\n" + << " // NumToSkip is a plain 24-bit integer.\n" + << " unsigned NumToSkip = *Ptr++;\n" + << " NumToSkip |= (*Ptr++) << 8;\n" + << " NumToSkip |= (*Ptr++) << 16;\n" + << "\n" + << " // Perform the filter operation.\n" + << " if (Val != CurFieldValue)\n" + << " Ptr += NumToSkip;\n" + << " LLVM_DEBUG(dbgs() << Loc << \": OPC_FilterValue(\" << Val << " + "\", \" << NumToSkip\n" + << " << \"): \" << ((Val != CurFieldValue) ? \"FAIL:\" " + ": \"PASS:\")\n" + << " << \" continuing at \" << (Ptr - DecodeTable) << " + "\"\\n\");\n" + << "\n" + << " break;\n" + << " }\n" + << " case MCD::OPC_CheckField: {\n" + << " unsigned Start = *++Ptr;\n" + << " unsigned Len = *++Ptr;\n" + << " InsnType FieldValue = fieldFromInstruction(insn, Start, Len);\n" + << " // Decode the field value.\n" + << " InsnType ExpectedValue = decodeULEB128(++Ptr, &Len);\n" + << " Ptr += Len;\n" + << " // NumToSkip is a plain 24-bit integer.\n" + << " unsigned NumToSkip = *Ptr++;\n" + << " NumToSkip |= (*Ptr++) << 8;\n" + << " NumToSkip |= (*Ptr++) << 16;\n" + << "\n" + << " // If the actual and expected values don't match, skip.\n" + << " if (ExpectedValue != FieldValue)\n" + << " Ptr += NumToSkip;\n" + << " LLVM_DEBUG(dbgs() << Loc << \": OPC_CheckField(\" << Start << " + "\", \"\n" + << " << Len << \", \" << ExpectedValue << \", \" << " + "NumToSkip\n" + << " << \"): FieldValue = \" << FieldValue << \", " + "ExpectedValue = \"\n" + << " << ExpectedValue << \": \"\n" + << " << ((ExpectedValue == FieldValue) ? \"PASS\\n\" : " + "\"FAIL\\n\"));\n" + << " break;\n" + << " }\n" + << " case MCD::OPC_CheckPredicate: {\n" + << " unsigned Len;\n" + << " // Decode the Predicate Index value.\n" + << " unsigned PIdx = decodeULEB128(++Ptr, &Len);\n" + << " Ptr += Len;\n" + << " // NumToSkip is a plain 24-bit integer.\n" + << " unsigned NumToSkip = *Ptr++;\n" + << " NumToSkip |= (*Ptr++) << 8;\n" + << " NumToSkip |= (*Ptr++) << 16;\n" + << " // Check the predicate.\n" + << " bool Pred;\n" + << " if (!(Pred = checkDecoderPredicate(PIdx, Bits)))\n" + << " Ptr += NumToSkip;\n" + << " (void)Pred;\n" + << " LLVM_DEBUG(dbgs() << Loc << \": OPC_CheckPredicate(\" << PIdx " + "<< \"): \"\n" + << " << (Pred ? \"PASS\\n\" : \"FAIL\\n\"));\n" + << "\n" + << " break;\n" + << " }\n" + << " case MCD::OPC_Decode: {\n" + << " unsigned Len;\n" + << " // Decode the Opcode value.\n" + << " unsigned Opc = decodeULEB128(++Ptr, &Len);\n" + << " Ptr += Len;\n" + << " unsigned DecodeIdx = decodeULEB128(Ptr, &Len);\n" + << " Ptr += Len;\n" + << "\n" + << " MI.clear();\n" + << " MI.setOpcode(Opc);\n" + << " bool DecodeComplete;\n" + << " S = decodeToMCInst(S, DecodeIdx, insn, MI, Address, DisAsm, " + "DecodeComplete);\n" + << " assert(DecodeComplete);\n" + << "\n" + << " LLVM_DEBUG(dbgs() << Loc << \": OPC_Decode: opcode \" << Opc\n" + << " << \", using decoder \" << DecodeIdx << \": \"\n" + << " << (S != MCDisassembler::Fail ? \"PASS\" : " + "\"FAIL\") << \"\\n\");\n" + << " return S;\n" + << " }\n" + << " case MCD::OPC_TryDecode: {\n" + << " unsigned Len;\n" + << " // Decode the Opcode value.\n" + << " unsigned Opc = decodeULEB128(++Ptr, &Len);\n" + << " Ptr += Len;\n" + << " unsigned DecodeIdx = decodeULEB128(Ptr, &Len);\n" + << " Ptr += Len;\n" + << " // NumToSkip is a plain 24-bit integer.\n" + << " unsigned NumToSkip = *Ptr++;\n" + << " NumToSkip |= (*Ptr++) << 8;\n" + << " NumToSkip |= (*Ptr++) << 16;\n" + << "\n" + << " // Perform the decode operation.\n" + << " MCInst TmpMI;\n" + << " TmpMI.setOpcode(Opc);\n" + << " bool DecodeComplete;\n" + << " S = decodeToMCInst(S, DecodeIdx, insn, TmpMI, Address, DisAsm, " + "DecodeComplete);\n" + << " LLVM_DEBUG(dbgs() << Loc << \": OPC_TryDecode: opcode \" << " + "Opc\n" + << " << \", using decoder \" << DecodeIdx << \": \");\n" + << "\n" + << " if (DecodeComplete) {\n" + << " // Decoding complete.\n" + << " LLVM_DEBUG(dbgs() << (S != MCDisassembler::Fail ? \"PASS\" : " + "\"FAIL\") << \"\\n\");\n" + << " MI = TmpMI;\n" + << " return S;\n" + << " } else {\n" + << " assert(S == MCDisassembler::Fail);\n" + << " // If the decoding was incomplete, skip.\n" + << " Ptr += NumToSkip;\n" + << " LLVM_DEBUG(dbgs() << \"FAIL: continuing at \" << (Ptr - " + "DecodeTable) << \"\\n\");\n" + << " // Reset decode status. This also drops a SoftFail status " + "that could be\n" + << " // set before the decode attempt.\n" + << " S = MCDisassembler::Success;\n" + << " }\n" + << " break;\n" + << " }\n" + << " case MCD::OPC_SoftFail: {\n" + << " // Decode the mask values.\n" + << " unsigned Len;\n" + << " InsnType PositiveMask = decodeULEB128(++Ptr, &Len);\n" + << " Ptr += Len;\n" + << " InsnType NegativeMask = decodeULEB128(Ptr, &Len);\n" + << " Ptr += Len;\n" + << " bool Fail = (insn & PositiveMask) || (~insn & NegativeMask);\n" + << " if (Fail)\n" + << " S = MCDisassembler::SoftFail;\n" + << " LLVM_DEBUG(dbgs() << Loc << \": OPC_SoftFail: \" << (Fail ? " + "\"FAIL\\n\":\"PASS\\n\"));\n" + << " break;\n" + << " }\n" + << " case MCD::OPC_Fail: {\n" + << " LLVM_DEBUG(dbgs() << Loc << \": OPC_Fail\\n\");\n" + << " return MCDisassembler::Fail;\n" + << " }\n" + << " }\n" + << " }\n" + << " llvm_unreachable(\"bogosity detected in disassembler state " + "machine!\");\n" + << "}\n\n"; +} + +// Emits disassembler code for instruction decoding. +void FixedLenDecoderEmitter::run(raw_ostream &o) { + formatted_raw_ostream OS(o); + OS << "#include \"llvm/MC/MCInst.h\"\n"; + OS << "#include \"llvm/Support/Debug.h\"\n"; + OS << "#include \"llvm/Support/DataTypes.h\"\n"; + OS << "#include \"llvm/Support/LEB128.h\"\n"; + OS << "#include \"llvm/Support/raw_ostream.h\"\n"; + OS << "#include <assert.h>\n"; + OS << '\n'; + OS << "namespace llvm {\n\n"; + + emitFieldFromInstruction(OS); + + Target.reverseBitsForLittleEndianEncoding(); + + // Parameterize the decoders based on namespace and instruction width. + std::set<StringRef> HwModeNames; + const auto &NumberedInstructions = Target.getInstructionsByEnumValue(); + NumberedEncodings.reserve(NumberedInstructions.size()); + DenseMap<Record *, unsigned> IndexOfInstruction; + // First, collect all HwModes referenced by the target. + for (const auto &NumberedInstruction : NumberedInstructions) { + IndexOfInstruction[NumberedInstruction->TheDef] = NumberedEncodings.size(); + + if (const RecordVal *RV = + NumberedInstruction->TheDef->getValue("EncodingInfos")) { + if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + const CodeGenHwModes &HWM = Target.getHwModes(); + EncodingInfoByHwMode EBM(DI->getDef(), HWM); + for (auto &KV : EBM.Map) + HwModeNames.insert(HWM.getMode(KV.first).Name); + } + } + } + + // If HwModeNames is empty, add the empty string so we always have one HwMode. + if (HwModeNames.empty()) + HwModeNames.insert(""); + + for (const auto &NumberedInstruction : NumberedInstructions) { + IndexOfInstruction[NumberedInstruction->TheDef] = NumberedEncodings.size(); + + if (const RecordVal *RV = + NumberedInstruction->TheDef->getValue("EncodingInfos")) { + if (DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) { + const CodeGenHwModes &HWM = Target.getHwModes(); + EncodingInfoByHwMode EBM(DI->getDef(), HWM); + for (auto &KV : EBM.Map) { + NumberedEncodings.emplace_back(KV.second, NumberedInstruction, + HWM.getMode(KV.first).Name); + HwModeNames.insert(HWM.getMode(KV.first).Name); + } + continue; + } + } + // This instruction is encoded the same on all HwModes. Emit it for all + // HwModes. + for (StringRef HwModeName : HwModeNames) + NumberedEncodings.emplace_back(NumberedInstruction->TheDef, + NumberedInstruction, HwModeName); + } + for (const auto &NumberedAlias : RK.getAllDerivedDefinitions("AdditionalEncoding")) + NumberedEncodings.emplace_back( + NumberedAlias, + &Target.getInstruction(NumberedAlias->getValueAsDef("AliasOf"))); + + std::map<std::pair<std::string, unsigned>, std::vector<EncodingIDAndOpcode>> + OpcMap; + std::map<unsigned, std::vector<OperandInfo>> Operands; + + for (unsigned i = 0; i < NumberedEncodings.size(); ++i) { + const Record *EncodingDef = NumberedEncodings[i].EncodingDef; + const CodeGenInstruction *Inst = NumberedEncodings[i].Inst; + const Record *Def = Inst->TheDef; + unsigned Size = EncodingDef->getValueAsInt("Size"); + if (Def->getValueAsString("Namespace") == "TargetOpcode" || + Def->getValueAsBit("isPseudo") || + Def->getValueAsBit("isAsmParserOnly") || + Def->getValueAsBit("isCodeGenOnly")) { + NumEncodingsLackingDisasm++; + continue; + } + + if (i < NumberedInstructions.size()) + NumInstructions++; + NumEncodings++; + + if (!Size) + continue; + + if (populateInstruction(Target, *EncodingDef, *Inst, i, Operands)) { + std::string DecoderNamespace = + std::string(EncodingDef->getValueAsString("DecoderNamespace")); + if (!NumberedEncodings[i].HwModeName.empty()) + DecoderNamespace += + std::string("_") + NumberedEncodings[i].HwModeName.str(); + OpcMap[std::make_pair(DecoderNamespace, Size)].emplace_back( + i, IndexOfInstruction.find(Def)->second); + } else { + NumEncodingsOmitted++; + } + } + + DecoderTableInfo TableInfo; + for (const auto &Opc : OpcMap) { + // Emit the decoder for this namespace+width combination. + ArrayRef<EncodingAndInst> NumberedEncodingsRef( + NumberedEncodings.data(), NumberedEncodings.size()); + FilterChooser FC(NumberedEncodingsRef, Opc.second, Operands, + 8 * Opc.first.second, this); + + // The decode table is cleared for each top level decoder function. The + // predicates and decoders themselves, however, are shared across all + // decoders to give more opportunities for uniqueing. + TableInfo.Table.clear(); + TableInfo.FixupStack.clear(); + TableInfo.Table.reserve(16384); + TableInfo.FixupStack.emplace_back(); + FC.emitTableEntries(TableInfo); + // Any NumToSkip fixups in the top level scope can resolve to the + // OPC_Fail at the end of the table. + assert(TableInfo.FixupStack.size() == 1 && "fixup stack phasing error!"); + // Resolve any NumToSkip fixups in the current scope. + resolveTableFixups(TableInfo.Table, TableInfo.FixupStack.back(), + TableInfo.Table.size()); + TableInfo.FixupStack.clear(); + + TableInfo.Table.push_back(MCD::OPC_Fail); + + // Print the table to the output stream. + emitTable(OS, TableInfo.Table, 0, FC.getBitWidth(), Opc.first.first); + OS.flush(); + } + + // Emit the predicate function. + emitPredicateFunction(OS, TableInfo.Predicates, 0); + + // Emit the decoder function. + emitDecoderFunction(OS, TableInfo.Decoders, 0); + + // Emit the main entry point for the decoder, decodeInstruction(). + emitDecodeInstruction(OS); + + OS << "\n} // end namespace llvm\n"; +} + +namespace llvm { + +void EmitFixedLenDecoder(RecordKeeper &RK, raw_ostream &OS, + const std::string &PredicateNamespace, + const std::string &GPrefix, + const std::string &GPostfix, const std::string &ROK, + const std::string &RFail, const std::string &L) { + FixedLenDecoderEmitter(RK, PredicateNamespace, GPrefix, GPostfix, + ROK, RFail, L).run(OS); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/GICombinerEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/GICombinerEmitter.cpp new file mode 100644 index 000000000000..e2a670070ae7 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GICombinerEmitter.cpp @@ -0,0 +1,1065 @@ +//===- GlobalCombinerEmitter.cpp - Generate a combiner --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Generate a combiner implementation for GlobalISel from a declarative +/// syntax +/// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/Timer.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/StringMatcher.h" +#include "llvm/TableGen/TableGenBackend.h" +#include "CodeGenTarget.h" +#include "GlobalISel/CodeExpander.h" +#include "GlobalISel/CodeExpansions.h" +#include "GlobalISel/GIMatchDag.h" +#include "GlobalISel/GIMatchTree.h" +#include <cstdint> + +using namespace llvm; + +#define DEBUG_TYPE "gicombiner-emitter" + +// FIXME: Use ALWAYS_ENABLED_STATISTIC once it's available. +unsigned NumPatternTotal = 0; +STATISTIC(NumPatternTotalStatistic, "Total number of patterns"); + +cl::OptionCategory + GICombinerEmitterCat("Options for -gen-global-isel-combiner"); +static cl::list<std::string> + SelectedCombiners("combiners", cl::desc("Emit the specified combiners"), + cl::cat(GICombinerEmitterCat), cl::CommaSeparated); +static cl::opt<bool> ShowExpansions( + "gicombiner-show-expansions", + cl::desc("Use C++ comments to indicate occurence of code expansion"), + cl::cat(GICombinerEmitterCat)); +static cl::opt<bool> StopAfterParse( + "gicombiner-stop-after-parse", + cl::desc("Stop processing after parsing rules and dump state"), + cl::cat(GICombinerEmitterCat)); +static cl::opt<bool> StopAfterBuild( + "gicombiner-stop-after-build", + cl::desc("Stop processing after building the match tree"), + cl::cat(GICombinerEmitterCat)); + +namespace { +typedef uint64_t RuleID; + +// We're going to be referencing the same small strings quite a lot for operand +// names and the like. Make their lifetime management simple with a global +// string table. +StringSet<> StrTab; + +StringRef insertStrTab(StringRef S) { + if (S.empty()) + return S; + return StrTab.insert(S).first->first(); +} + +class format_partition_name { + const GIMatchTree &Tree; + unsigned Idx; + +public: + format_partition_name(const GIMatchTree &Tree, unsigned Idx) + : Tree(Tree), Idx(Idx) {} + void print(raw_ostream &OS) const { + Tree.getPartitioner()->emitPartitionName(OS, Idx); + } +}; +raw_ostream &operator<<(raw_ostream &OS, const format_partition_name &Fmt) { + Fmt.print(OS); + return OS; +} + +/// Declares data that is passed from the match stage to the apply stage. +class MatchDataInfo { + /// The symbol used in the tablegen patterns + StringRef PatternSymbol; + /// The data type for the variable + StringRef Type; + /// The name of the variable as declared in the generated matcher. + std::string VariableName; + +public: + MatchDataInfo(StringRef PatternSymbol, StringRef Type, StringRef VariableName) + : PatternSymbol(PatternSymbol), Type(Type), VariableName(VariableName) {} + + StringRef getPatternSymbol() const { return PatternSymbol; }; + StringRef getType() const { return Type; }; + StringRef getVariableName() const { return VariableName; }; +}; + +class RootInfo { + StringRef PatternSymbol; + +public: + RootInfo(StringRef PatternSymbol) : PatternSymbol(PatternSymbol) {} + + StringRef getPatternSymbol() const { return PatternSymbol; } +}; + +class CombineRule { +public: + + using const_matchdata_iterator = std::vector<MatchDataInfo>::const_iterator; + + struct VarInfo { + const GIMatchDagInstr *N; + const GIMatchDagOperand *Op; + const DagInit *Matcher; + + public: + VarInfo(const GIMatchDagInstr *N, const GIMatchDagOperand *Op, + const DagInit *Matcher) + : N(N), Op(Op), Matcher(Matcher) {} + }; + +protected: + /// A unique ID for this rule + /// ID's are used for debugging and run-time disabling of rules among other + /// things. + RuleID ID; + + /// A unique ID that can be used for anonymous objects belonging to this rule. + /// Used to create unique names in makeNameForAnon*() without making tests + /// overly fragile. + unsigned UID = 0; + + /// The record defining this rule. + const Record &TheDef; + + /// The roots of a match. These are the leaves of the DAG that are closest to + /// the end of the function. I.e. the nodes that are encountered without + /// following any edges of the DAG described by the pattern as we work our way + /// from the bottom of the function to the top. + std::vector<RootInfo> Roots; + + GIMatchDag MatchDag; + + /// A block of arbitrary C++ to finish testing the match. + /// FIXME: This is a temporary measure until we have actual pattern matching + const CodeInit *MatchingFixupCode = nullptr; + + /// The MatchData defined by the match stage and required by the apply stage. + /// This allows the plumbing of arbitrary data from C++ predicates between the + /// stages. + /// + /// For example, suppose you have: + /// %A = <some-constant-expr> + /// %0 = G_ADD %1, %A + /// you could define a GIMatchPredicate that walks %A, constant folds as much + /// as possible and returns an APInt containing the discovered constant. You + /// could then declare: + /// def apint : GIDefMatchData<"APInt">; + /// add it to the rule with: + /// (defs root:$root, apint:$constant) + /// evaluate it in the pattern with a C++ function that takes a + /// MachineOperand& and an APInt& with: + /// (match [{MIR %root = G_ADD %0, %A }], + /// (constantfold operand:$A, apint:$constant)) + /// and finally use it in the apply stage with: + /// (apply (create_operand + /// [{ MachineOperand::CreateImm(${constant}.getZExtValue()); + /// ]}, apint:$constant), + /// [{MIR %root = FOO %0, %constant }]) + std::vector<MatchDataInfo> MatchDataDecls; + + void declareMatchData(StringRef PatternSymbol, StringRef Type, + StringRef VarName); + + bool parseInstructionMatcher(const CodeGenTarget &Target, StringInit *ArgName, + const Init &Arg, + StringMap<std::vector<VarInfo>> &NamedEdgeDefs, + StringMap<std::vector<VarInfo>> &NamedEdgeUses); + bool parseWipMatchOpcodeMatcher(const CodeGenTarget &Target, + StringInit *ArgName, const Init &Arg); + +public: + CombineRule(const CodeGenTarget &Target, GIMatchDagContext &Ctx, RuleID ID, + const Record &R) + : ID(ID), TheDef(R), MatchDag(Ctx) {} + CombineRule(const CombineRule &) = delete; + + bool parseDefs(); + bool parseMatcher(const CodeGenTarget &Target); + + RuleID getID() const { return ID; } + unsigned allocUID() { return UID++; } + StringRef getName() const { return TheDef.getName(); } + const Record &getDef() const { return TheDef; } + const CodeInit *getMatchingFixupCode() const { return MatchingFixupCode; } + size_t getNumRoots() const { return Roots.size(); } + + GIMatchDag &getMatchDag() { return MatchDag; } + const GIMatchDag &getMatchDag() const { return MatchDag; } + + using const_root_iterator = std::vector<RootInfo>::const_iterator; + const_root_iterator roots_begin() const { return Roots.begin(); } + const_root_iterator roots_end() const { return Roots.end(); } + iterator_range<const_root_iterator> roots() const { + return llvm::make_range(Roots.begin(), Roots.end()); + } + + iterator_range<const_matchdata_iterator> matchdata_decls() const { + return make_range(MatchDataDecls.begin(), MatchDataDecls.end()); + } + + /// Export expansions for this rule + void declareExpansions(CodeExpansions &Expansions) const { + for (const auto &I : matchdata_decls()) + Expansions.declare(I.getPatternSymbol(), I.getVariableName()); + } + + /// The matcher will begin from the roots and will perform the match by + /// traversing the edges to cover the whole DAG. This function reverses DAG + /// edges such that everything is reachable from a root. This is part of the + /// preparation work for flattening the DAG into a tree. + void reorientToRoots() { + SmallSet<const GIMatchDagInstr *, 5> Roots; + SmallSet<const GIMatchDagInstr *, 5> Visited; + SmallSet<GIMatchDagEdge *, 20> EdgesRemaining; + + for (auto &I : MatchDag.roots()) { + Roots.insert(I); + Visited.insert(I); + } + for (auto &I : MatchDag.edges()) + EdgesRemaining.insert(I); + + bool Progressed = false; + SmallSet<GIMatchDagEdge *, 20> EdgesToRemove; + while (!EdgesRemaining.empty()) { + for (auto EI = EdgesRemaining.begin(), EE = EdgesRemaining.end(); + EI != EE; ++EI) { + if (Visited.count((*EI)->getFromMI())) { + if (Roots.count((*EI)->getToMI())) + PrintError(TheDef.getLoc(), "One or more roots are unnecessary"); + Visited.insert((*EI)->getToMI()); + EdgesToRemove.insert(*EI); + Progressed = true; + } + } + for (GIMatchDagEdge *ToRemove : EdgesToRemove) + EdgesRemaining.erase(ToRemove); + EdgesToRemove.clear(); + + for (auto EI = EdgesRemaining.begin(), EE = EdgesRemaining.end(); + EI != EE; ++EI) { + if (Visited.count((*EI)->getToMI())) { + (*EI)->reverse(); + Visited.insert((*EI)->getToMI()); + EdgesToRemove.insert(*EI); + Progressed = true; + } + for (GIMatchDagEdge *ToRemove : EdgesToRemove) + EdgesRemaining.erase(ToRemove); + EdgesToRemove.clear(); + } + + if (!Progressed) { + LLVM_DEBUG(dbgs() << "No progress\n"); + return; + } + Progressed = false; + } + } +}; + +/// A convenience function to check that an Init refers to a specific def. This +/// is primarily useful for testing for defs and similar in DagInit's since +/// DagInit's support any type inside them. +static bool isSpecificDef(const Init &N, StringRef Def) { + if (const DefInit *OpI = dyn_cast<DefInit>(&N)) + if (OpI->getDef()->getName() == Def) + return true; + return false; +} + +/// A convenience function to check that an Init refers to a def that is a +/// subclass of the given class and coerce it to a def if it is. This is +/// primarily useful for testing for subclasses of GIMatchKind and similar in +/// DagInit's since DagInit's support any type inside them. +static Record *getDefOfSubClass(const Init &N, StringRef Cls) { + if (const DefInit *OpI = dyn_cast<DefInit>(&N)) + if (OpI->getDef()->isSubClassOf(Cls)) + return OpI->getDef(); + return nullptr; +} + +/// A convenience function to check that an Init refers to a dag whose operator +/// is a specific def and coerce it to a dag if it is. This is primarily useful +/// for testing for subclasses of GIMatchKind and similar in DagInit's since +/// DagInit's support any type inside them. +static const DagInit *getDagWithSpecificOperator(const Init &N, + StringRef Name) { + if (const DagInit *I = dyn_cast<DagInit>(&N)) + if (I->getNumArgs() > 0) + if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator())) + if (OpI->getDef()->getName() == Name) + return I; + return nullptr; +} + +/// A convenience function to check that an Init refers to a dag whose operator +/// is a def that is a subclass of the given class and coerce it to a dag if it +/// is. This is primarily useful for testing for subclasses of GIMatchKind and +/// similar in DagInit's since DagInit's support any type inside them. +static const DagInit *getDagWithOperatorOfSubClass(const Init &N, + StringRef Cls) { + if (const DagInit *I = dyn_cast<DagInit>(&N)) + if (I->getNumArgs() > 0) + if (const DefInit *OpI = dyn_cast<DefInit>(I->getOperator())) + if (OpI->getDef()->isSubClassOf(Cls)) + return I; + return nullptr; +} + +StringRef makeNameForAnonInstr(CombineRule &Rule) { + return insertStrTab(to_string( + format("__anon%" PRIu64 "_%u", Rule.getID(), Rule.allocUID()))); +} + +StringRef makeDebugName(CombineRule &Rule, StringRef Name) { + return insertStrTab(Name.empty() ? makeNameForAnonInstr(Rule) : StringRef(Name)); +} + +StringRef makeNameForAnonPredicate(CombineRule &Rule) { + return insertStrTab(to_string( + format("__anonpred%" PRIu64 "_%u", Rule.getID(), Rule.allocUID()))); +} + +void CombineRule::declareMatchData(StringRef PatternSymbol, StringRef Type, + StringRef VarName) { + MatchDataDecls.emplace_back(PatternSymbol, Type, VarName); +} + +bool CombineRule::parseDefs() { + NamedRegionTimer T("parseDefs", "Time spent parsing the defs", "Rule Parsing", + "Time spent on rule parsing", TimeRegions); + DagInit *Defs = TheDef.getValueAsDag("Defs"); + + if (Defs->getOperatorAsDef(TheDef.getLoc())->getName() != "defs") { + PrintError(TheDef.getLoc(), "Expected defs operator"); + return false; + } + + for (unsigned I = 0, E = Defs->getNumArgs(); I < E; ++I) { + // Roots should be collected into Roots + if (isSpecificDef(*Defs->getArg(I), "root")) { + Roots.emplace_back(Defs->getArgNameStr(I)); + continue; + } + + // Subclasses of GIDefMatchData should declare that this rule needs to pass + // data from the match stage to the apply stage, and ensure that the + // generated matcher has a suitable variable for it to do so. + if (Record *MatchDataRec = + getDefOfSubClass(*Defs->getArg(I), "GIDefMatchData")) { + declareMatchData(Defs->getArgNameStr(I), + MatchDataRec->getValueAsString("Type"), + llvm::to_string(llvm::format("MatchData%" PRIu64, ID))); + continue; + } + + // Otherwise emit an appropriate error message. + if (getDefOfSubClass(*Defs->getArg(I), "GIDefKind")) + PrintError(TheDef.getLoc(), + "This GIDefKind not implemented in tablegen"); + else if (getDefOfSubClass(*Defs->getArg(I), "GIDefKindWithArgs")) + PrintError(TheDef.getLoc(), + "This GIDefKindWithArgs not implemented in tablegen"); + else + PrintError(TheDef.getLoc(), + "Expected a subclass of GIDefKind or a sub-dag whose " + "operator is of type GIDefKindWithArgs"); + return false; + } + + if (Roots.empty()) { + PrintError(TheDef.getLoc(), "Combine rules must have at least one root"); + return false; + } + return true; +} + +// Parse an (Instruction $a:Arg1, $b:Arg2, ...) matcher. Edges are formed +// between matching operand names between different matchers. +bool CombineRule::parseInstructionMatcher( + const CodeGenTarget &Target, StringInit *ArgName, const Init &Arg, + StringMap<std::vector<VarInfo>> &NamedEdgeDefs, + StringMap<std::vector<VarInfo>> &NamedEdgeUses) { + if (const DagInit *Matcher = + getDagWithOperatorOfSubClass(Arg, "Instruction")) { + auto &Instr = + Target.getInstruction(Matcher->getOperatorAsDef(TheDef.getLoc())); + + StringRef Name = ArgName ? ArgName->getValue() : ""; + + GIMatchDagInstr *N = + MatchDag.addInstrNode(makeDebugName(*this, Name), insertStrTab(Name), + MatchDag.getContext().makeOperandList(Instr)); + + N->setOpcodeAnnotation(&Instr); + const auto &P = MatchDag.addPredicateNode<GIMatchDagOpcodePredicate>( + makeNameForAnonPredicate(*this), Instr); + MatchDag.addPredicateDependency(N, nullptr, P, &P->getOperandInfo()["mi"]); + unsigned OpIdx = 0; + for (const auto &NameInit : Matcher->getArgNames()) { + StringRef Name = insertStrTab(NameInit->getAsUnquotedString()); + if (Name.empty()) + continue; + N->assignNameToOperand(OpIdx, Name); + + // Record the endpoints of any named edges. We'll add the cartesian + // product of edges later. + const auto &InstrOperand = N->getOperandInfo()[OpIdx]; + if (InstrOperand.isDef()) { + NamedEdgeDefs.try_emplace(Name); + NamedEdgeDefs[Name].emplace_back(N, &InstrOperand, Matcher); + } else { + NamedEdgeUses.try_emplace(Name); + NamedEdgeUses[Name].emplace_back(N, &InstrOperand, Matcher); + } + + if (InstrOperand.isDef()) { + if (find_if(Roots, [&](const RootInfo &X) { + return X.getPatternSymbol() == Name; + }) != Roots.end()) { + N->setMatchRoot(); + } + } + + OpIdx++; + } + + return true; + } + return false; +} + +// Parse the wip_match_opcode placeholder that's temporarily present in lieu of +// implementing macros or choices between two matchers. +bool CombineRule::parseWipMatchOpcodeMatcher(const CodeGenTarget &Target, + StringInit *ArgName, + const Init &Arg) { + if (const DagInit *Matcher = + getDagWithSpecificOperator(Arg, "wip_match_opcode")) { + StringRef Name = ArgName ? ArgName->getValue() : ""; + + GIMatchDagInstr *N = + MatchDag.addInstrNode(makeDebugName(*this, Name), insertStrTab(Name), + MatchDag.getContext().makeEmptyOperandList()); + + if (find_if(Roots, [&](const RootInfo &X) { + return ArgName && X.getPatternSymbol() == ArgName->getValue(); + }) != Roots.end()) { + N->setMatchRoot(); + } + + const auto &P = MatchDag.addPredicateNode<GIMatchDagOneOfOpcodesPredicate>( + makeNameForAnonPredicate(*this)); + MatchDag.addPredicateDependency(N, nullptr, P, &P->getOperandInfo()["mi"]); + // Each argument is an opcode that will pass this predicate. Add them all to + // the predicate implementation + for (const auto &Arg : Matcher->getArgs()) { + Record *OpcodeDef = getDefOfSubClass(*Arg, "Instruction"); + if (OpcodeDef) { + P->addOpcode(&Target.getInstruction(OpcodeDef)); + continue; + } + PrintError(TheDef.getLoc(), + "Arguments to wip_match_opcode must be instructions"); + return false; + } + return true; + } + return false; +} +bool CombineRule::parseMatcher(const CodeGenTarget &Target) { + NamedRegionTimer T("parseMatcher", "Time spent parsing the matcher", + "Rule Parsing", "Time spent on rule parsing", TimeRegions); + StringMap<std::vector<VarInfo>> NamedEdgeDefs; + StringMap<std::vector<VarInfo>> NamedEdgeUses; + DagInit *Matchers = TheDef.getValueAsDag("Match"); + + if (Matchers->getOperatorAsDef(TheDef.getLoc())->getName() != "match") { + PrintError(TheDef.getLoc(), "Expected match operator"); + return false; + } + + if (Matchers->getNumArgs() == 0) { + PrintError(TheDef.getLoc(), "Matcher is empty"); + return false; + } + + // The match section consists of a list of matchers and predicates. Parse each + // one and add the equivalent GIMatchDag nodes, predicates, and edges. + for (unsigned I = 0; I < Matchers->getNumArgs(); ++I) { + if (parseInstructionMatcher(Target, Matchers->getArgName(I), + *Matchers->getArg(I), NamedEdgeDefs, + NamedEdgeUses)) + continue; + + if (parseWipMatchOpcodeMatcher(Target, Matchers->getArgName(I), + *Matchers->getArg(I))) + continue; + + + // Parse arbitrary C++ code we have in lieu of supporting MIR matching + if (const CodeInit *CodeI = dyn_cast<CodeInit>(Matchers->getArg(I))) { + assert(!MatchingFixupCode && + "Only one block of arbitrary code is currently permitted"); + MatchingFixupCode = CodeI; + MatchDag.setHasPostMatchPredicate(true); + continue; + } + + PrintError(TheDef.getLoc(), + "Expected a subclass of GIMatchKind or a sub-dag whose " + "operator is either of a GIMatchKindWithArgs or Instruction"); + PrintNote("Pattern was `" + Matchers->getArg(I)->getAsString() + "'"); + return false; + } + + // Add the cartesian product of use -> def edges. + bool FailedToAddEdges = false; + for (const auto &NameAndDefs : NamedEdgeDefs) { + if (NameAndDefs.getValue().size() > 1) { + PrintError(TheDef.getLoc(), + "Two different MachineInstrs cannot def the same vreg"); + for (const auto &NameAndDefOp : NameAndDefs.getValue()) + PrintNote("in " + to_string(*NameAndDefOp.N) + " created from " + + to_string(*NameAndDefOp.Matcher) + ""); + FailedToAddEdges = true; + } + const auto &Uses = NamedEdgeUses[NameAndDefs.getKey()]; + for (const VarInfo &DefVar : NameAndDefs.getValue()) { + for (const VarInfo &UseVar : Uses) { + MatchDag.addEdge(insertStrTab(NameAndDefs.getKey()), UseVar.N, UseVar.Op, + DefVar.N, DefVar.Op); + } + } + } + if (FailedToAddEdges) + return false; + + // If a variable is referenced in multiple use contexts then we need a + // predicate to confirm they are the same operand. We can elide this if it's + // also referenced in a def context and we're traversing the def-use chain + // from the def to the uses but we can't know which direction we're going + // until after reorientToRoots(). + for (const auto &NameAndUses : NamedEdgeUses) { + const auto &Uses = NameAndUses.getValue(); + if (Uses.size() > 1) { + const auto &LeadingVar = Uses.front(); + for (const auto &Var : ArrayRef<VarInfo>(Uses).drop_front()) { + // Add a predicate for each pair until we've covered the whole + // equivalence set. We could test the whole set in a single predicate + // but that means we can't test any equivalence until all the MO's are + // available which can lead to wasted work matching the DAG when this + // predicate can already be seen to have failed. + // + // We have a similar problem due to the need to wait for a particular MO + // before being able to test any of them. However, that is mitigated by + // the order in which we build the DAG. We build from the roots outwards + // so by using the first recorded use in all the predicates, we are + // making the dependency on one of the earliest visited references in + // the DAG. It's not guaranteed once the generated matcher is optimized + // (because the factoring the common portions of rules might change the + // visit order) but this should mean that these predicates depend on the + // first MO to become available. + const auto &P = MatchDag.addPredicateNode<GIMatchDagSameMOPredicate>( + makeNameForAnonPredicate(*this)); + MatchDag.addPredicateDependency(LeadingVar.N, LeadingVar.Op, P, + &P->getOperandInfo()["mi0"]); + MatchDag.addPredicateDependency(Var.N, Var.Op, P, + &P->getOperandInfo()["mi1"]); + } + } + } + return true; +} + +class GICombinerEmitter { + StringRef Name; + const CodeGenTarget &Target; + Record *Combiner; + std::vector<std::unique_ptr<CombineRule>> Rules; + GIMatchDagContext MatchDagCtx; + + std::unique_ptr<CombineRule> makeCombineRule(const Record &R); + + void gatherRules(std::vector<std::unique_ptr<CombineRule>> &ActiveRules, + const std::vector<Record *> &&RulesAndGroups); + +public: + explicit GICombinerEmitter(RecordKeeper &RK, const CodeGenTarget &Target, + StringRef Name, Record *Combiner); + ~GICombinerEmitter() {} + + StringRef getClassName() const { + return Combiner->getValueAsString("Classname"); + } + void run(raw_ostream &OS); + + /// Emit the name matcher (guarded by #ifndef NDEBUG) used to disable rules in + /// response to the generated cl::opt. + void emitNameMatcher(raw_ostream &OS) const; + + void generateDeclarationsCodeForTree(raw_ostream &OS, const GIMatchTree &Tree) const; + void generateCodeForTree(raw_ostream &OS, const GIMatchTree &Tree, + StringRef Indent) const; +}; + +GICombinerEmitter::GICombinerEmitter(RecordKeeper &RK, + const CodeGenTarget &Target, + StringRef Name, Record *Combiner) + : Name(Name), Target(Target), Combiner(Combiner) {} + +void GICombinerEmitter::emitNameMatcher(raw_ostream &OS) const { + std::vector<std::pair<std::string, std::string>> Cases; + Cases.reserve(Rules.size()); + + for (const CombineRule &EnumeratedRule : make_pointee_range(Rules)) { + std::string Code; + raw_string_ostream SS(Code); + SS << "return " << EnumeratedRule.getID() << ";\n"; + Cases.push_back( + std::make_pair(std::string(EnumeratedRule.getName()), SS.str())); + } + + OS << "static Optional<uint64_t> getRuleIdxForIdentifier(StringRef " + "RuleIdentifier) {\n" + << " uint64_t I;\n" + << " // getAtInteger(...) returns false on success\n" + << " bool Parsed = !RuleIdentifier.getAsInteger(0, I);\n" + << " if (Parsed)\n" + << " return I;\n\n" + << "#ifndef NDEBUG\n"; + StringMatcher Matcher("RuleIdentifier", Cases, OS); + Matcher.Emit(); + OS << "#endif // ifndef NDEBUG\n\n" + << " return None;\n" + << "}\n"; +} + +std::unique_ptr<CombineRule> +GICombinerEmitter::makeCombineRule(const Record &TheDef) { + std::unique_ptr<CombineRule> Rule = + std::make_unique<CombineRule>(Target, MatchDagCtx, NumPatternTotal, TheDef); + + if (!Rule->parseDefs()) + return nullptr; + if (!Rule->parseMatcher(Target)) + return nullptr; + + Rule->reorientToRoots(); + + LLVM_DEBUG({ + dbgs() << "Parsed rule defs/match for '" << Rule->getName() << "'\n"; + Rule->getMatchDag().dump(); + Rule->getMatchDag().writeDOTGraph(dbgs(), Rule->getName()); + }); + if (StopAfterParse) + return Rule; + + // For now, don't support traversing from def to use. We'll come back to + // this later once we have the algorithm changes to support it. + bool EmittedDefToUseError = false; + for (const auto &E : Rule->getMatchDag().edges()) { + if (E->isDefToUse()) { + if (!EmittedDefToUseError) { + PrintError( + TheDef.getLoc(), + "Generated state machine cannot lookup uses from a def (yet)"); + EmittedDefToUseError = true; + } + PrintNote("Node " + to_string(*E->getFromMI())); + PrintNote("Node " + to_string(*E->getToMI())); + PrintNote("Edge " + to_string(*E)); + } + } + if (EmittedDefToUseError) + return nullptr; + + // For now, don't support multi-root rules. We'll come back to this later + // once we have the algorithm changes to support it. + if (Rule->getNumRoots() > 1) { + PrintError(TheDef.getLoc(), "Multi-root matches are not supported (yet)"); + return nullptr; + } + return Rule; +} + +/// Recurse into GICombineGroup's and flatten the ruleset into a simple list. +void GICombinerEmitter::gatherRules( + std::vector<std::unique_ptr<CombineRule>> &ActiveRules, + const std::vector<Record *> &&RulesAndGroups) { + for (Record *R : RulesAndGroups) { + if (R->isValueUnset("Rules")) { + std::unique_ptr<CombineRule> Rule = makeCombineRule(*R); + if (Rule == nullptr) { + PrintError(R->getLoc(), "Failed to parse rule"); + continue; + } + ActiveRules.emplace_back(std::move(Rule)); + ++NumPatternTotal; + } else + gatherRules(ActiveRules, R->getValueAsListOfDefs("Rules")); + } +} + +void GICombinerEmitter::generateCodeForTree(raw_ostream &OS, + const GIMatchTree &Tree, + StringRef Indent) const { + if (Tree.getPartitioner() != nullptr) { + Tree.getPartitioner()->generatePartitionSelectorCode(OS, Indent); + for (const auto &EnumChildren : enumerate(Tree.children())) { + OS << Indent << "if (Partition == " << EnumChildren.index() << " /* " + << format_partition_name(Tree, EnumChildren.index()) << " */) {\n"; + generateCodeForTree(OS, EnumChildren.value(), (Indent + " ").str()); + OS << Indent << "}\n"; + } + return; + } + + bool AnyFullyTested = false; + for (const auto &Leaf : Tree.possible_leaves()) { + OS << Indent << "// Leaf name: " << Leaf.getName() << "\n"; + + const CombineRule *Rule = Leaf.getTargetData<CombineRule>(); + const Record &RuleDef = Rule->getDef(); + + OS << Indent << "// Rule: " << RuleDef.getName() << "\n" + << Indent << "if (!RuleConfig->isRuleDisabled(" << Rule->getID() + << ")) {\n"; + + CodeExpansions Expansions; + for (const auto &VarBinding : Leaf.var_bindings()) { + if (VarBinding.isInstr()) + Expansions.declare(VarBinding.getName(), + "MIs[" + to_string(VarBinding.getInstrID()) + "]"); + else + Expansions.declare(VarBinding.getName(), + "MIs[" + to_string(VarBinding.getInstrID()) + + "]->getOperand(" + + to_string(VarBinding.getOpIdx()) + ")"); + } + Rule->declareExpansions(Expansions); + + DagInit *Applyer = RuleDef.getValueAsDag("Apply"); + if (Applyer->getOperatorAsDef(RuleDef.getLoc())->getName() != + "apply") { + PrintError(RuleDef.getLoc(), "Expected apply operator"); + return; + } + + OS << Indent << " if (1\n"; + + // Attempt to emit code for any untested predicates left over. Note that + // isFullyTested() will remain false even if we succeed here and therefore + // combine rule elision will not be performed. This is because we do not + // know if there's any connection between the predicates for each leaf and + // therefore can't tell if one makes another unreachable. Ideally, the + // partitioner(s) would be sufficiently complete to prevent us from having + // untested predicates left over. + for (const GIMatchDagPredicate *Predicate : Leaf.untested_predicates()) { + if (Predicate->generateCheckCode(OS, (Indent + " ").str(), + Expansions)) + continue; + PrintError(RuleDef.getLoc(), + "Unable to test predicate used in rule"); + PrintNote(SMLoc(), + "This indicates an incomplete implementation in tablegen"); + Predicate->print(errs()); + errs() << "\n"; + OS << Indent + << "llvm_unreachable(\"TableGen did not emit complete code for this " + "path\");\n"; + break; + } + + if (Rule->getMatchingFixupCode() && + !Rule->getMatchingFixupCode()->getValue().empty()) { + // FIXME: Single-use lambda's like this are a serious compile-time + // performance and memory issue. It's convenient for this early stage to + // defer some work to successive patches but we need to eliminate this + // before the ruleset grows to small-moderate size. Last time, it became + // a big problem for low-mem systems around the 500 rule mark but by the + // time we grow that large we should have merged the ISel match table + // mechanism with the Combiner. + OS << Indent << " && [&]() {\n" + << Indent << " " + << CodeExpander(Rule->getMatchingFixupCode()->getValue(), Expansions, + Rule->getMatchingFixupCode()->getLoc(), ShowExpansions) + << "\n" + << Indent << " return true;\n" + << Indent << " }()"; + } + OS << ") {\n" << Indent << " "; + + if (const CodeInit *Code = dyn_cast<CodeInit>(Applyer->getArg(0))) { + OS << CodeExpander(Code->getAsUnquotedString(), Expansions, + Code->getLoc(), ShowExpansions) + << "\n" + << Indent << " return true;\n" + << Indent << " }\n"; + } else { + PrintError(RuleDef.getLoc(), "Expected apply code block"); + return; + } + + OS << Indent << "}\n"; + + assert(Leaf.isFullyTraversed()); + + // If we didn't have any predicates left over and we're not using the + // trap-door we have to support arbitrary C++ code while we're migrating to + // the declarative style then we know that subsequent leaves are + // unreachable. + if (Leaf.isFullyTested() && + (!Rule->getMatchingFixupCode() || + Rule->getMatchingFixupCode()->getValue().empty())) { + AnyFullyTested = true; + OS << Indent + << "llvm_unreachable(\"Combine rule elision was incorrect\");\n" + << Indent << "return false;\n"; + } + } + if (!AnyFullyTested) + OS << Indent << "return false;\n"; +} + +static void emitAdditionalHelperMethodArguments(raw_ostream &OS, + Record *Combiner) { + for (Record *Arg : Combiner->getValueAsListOfDefs("AdditionalArguments")) + OS << ",\n " << Arg->getValueAsString("Type") + << Arg->getValueAsString("Name"); +} + +void GICombinerEmitter::run(raw_ostream &OS) { + gatherRules(Rules, Combiner->getValueAsListOfDefs("Rules")); + if (StopAfterParse) { + MatchDagCtx.print(errs()); + PrintNote(Combiner->getLoc(), + "Terminating due to -gicombiner-stop-after-parse"); + return; + } + if (ErrorsPrinted) + PrintFatalError(Combiner->getLoc(), "Failed to parse one or more rules"); + LLVM_DEBUG(dbgs() << "Optimizing tree for " << Rules.size() << " rules\n"); + std::unique_ptr<GIMatchTree> Tree; + { + NamedRegionTimer T("Optimize", "Time spent optimizing the combiner", + "Code Generation", "Time spent generating code", + TimeRegions); + + GIMatchTreeBuilder TreeBuilder(0); + for (const auto &Rule : Rules) { + bool HadARoot = false; + for (const auto &Root : enumerate(Rule->getMatchDag().roots())) { + TreeBuilder.addLeaf(Rule->getName(), Root.index(), Rule->getMatchDag(), + Rule.get()); + HadARoot = true; + } + if (!HadARoot) + PrintFatalError(Rule->getDef().getLoc(), "All rules must have a root"); + } + + Tree = TreeBuilder.run(); + } + if (StopAfterBuild) { + Tree->writeDOTGraph(outs()); + PrintNote(Combiner->getLoc(), + "Terminating due to -gicombiner-stop-after-build"); + return; + } + + NamedRegionTimer T("Emit", "Time spent emitting the combiner", + "Code Generation", "Time spent generating code", + TimeRegions); + OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_DEPS\n" + << "#include \"llvm/ADT/SparseBitVector.h\"\n" + << "namespace llvm {\n" + << "extern cl::OptionCategory GICombinerOptionCategory;\n" + << "} // end namespace llvm\n" + << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_DEPS\n\n"; + + OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_H\n" + << "class " << getClassName() << "RuleConfig {\n" + << " SparseBitVector<> DisabledRules;\n" + << "\n" + << "public:\n" + << " bool parseCommandLineOption();\n" + << " bool isRuleDisabled(unsigned ID) const;\n" + << " bool setRuleEnabled(StringRef RuleIdentifier);\n" + << " bool setRuleDisabled(StringRef RuleIdentifier);\n" + << "\n" + << "};\n" + << "\n" + << "class " << getClassName(); + StringRef StateClass = Combiner->getValueAsString("StateClass"); + if (!StateClass.empty()) + OS << " : public " << StateClass; + OS << " {\n" + << " const " << getClassName() << "RuleConfig *RuleConfig;\n" + << "\n" + << "public:\n" + << " template<typename ... Args>" << getClassName() << "(const " + << getClassName() << "RuleConfig &RuleConfig, Args &&... args) : "; + if (!StateClass.empty()) + OS << StateClass << "(std::forward<Args>(args)...), "; + OS << "RuleConfig(&RuleConfig) {}\n" + << "\n" + << " bool tryCombineAll(\n" + << " GISelChangeObserver &Observer,\n" + << " MachineInstr &MI,\n" + << " MachineIRBuilder &B"; + emitAdditionalHelperMethodArguments(OS, Combiner); + OS << ") const;\n"; + OS << "};\n\n"; + + emitNameMatcher(OS); + + OS << "static Optional<std::pair<uint64_t, uint64_t>> " + "getRuleRangeForIdentifier(StringRef RuleIdentifier) {\n" + << " std::pair<StringRef, StringRef> RangePair = " + "RuleIdentifier.split('-');\n" + << " if (!RangePair.second.empty()) {\n" + << " const auto First = " + "getRuleIdxForIdentifier(RangePair.first);\n" + << " const auto Last = " + "getRuleIdxForIdentifier(RangePair.second);\n" + << " if (!First.hasValue() || !Last.hasValue())\n" + << " return None;\n" + << " if (First >= Last)\n" + << " report_fatal_error(\"Beginning of range should be before " + "end of range\");\n" + << " return {{ *First, *Last + 1 }};\n" + << " } else if (RangePair.first == \"*\") {\n" + << " return {{ 0, " << Rules.size() << " }};\n" + << " } else {\n" + << " const auto I = getRuleIdxForIdentifier(RangePair.first);\n" + << " if (!I.hasValue())\n" + << " return None;\n" + << " return {{*I, *I + 1}};\n" + << " }\n" + << " return None;\n" + << "}\n\n"; + + for (bool Enabled : {true, false}) { + OS << "bool " << getClassName() << "RuleConfig::setRule" + << (Enabled ? "Enabled" : "Disabled") << "(StringRef RuleIdentifier) {\n" + << " auto MaybeRange = getRuleRangeForIdentifier(RuleIdentifier);\n" + << " if(!MaybeRange.hasValue())\n" + << " return false;\n" + << " for (auto I = MaybeRange->first; I < MaybeRange->second; ++I)\n" + << " DisabledRules." << (Enabled ? "reset" : "set") << "(I);\n" + << " return true;\n" + << "}\n\n"; + } + + OS << "bool " << getClassName() + << "RuleConfig::isRuleDisabled(unsigned RuleID) const {\n" + << " return DisabledRules.test(RuleID);\n" + << "}\n"; + OS << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_H\n\n"; + + OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_CPP\n" + << "\n" + << "std::vector<std::string> " << Name << "Option;\n" + << "cl::list<std::string> " << Name << "DisableOption(\n" + << " \"" << Name.lower() << "-disable-rule\",\n" + << " cl::desc(\"Disable one or more combiner rules temporarily in " + << "the " << Name << " pass\"),\n" + << " cl::CommaSeparated,\n" + << " cl::Hidden,\n" + << " cl::cat(GICombinerOptionCategory),\n" + << " cl::callback([](const std::string &Str) {\n" + << " " << Name << "Option.push_back(Str);\n" + << " }));\n" + << "cl::list<std::string> " << Name << "OnlyEnableOption(\n" + << " \"" << Name.lower() << "-only-enable-rule\",\n" + << " cl::desc(\"Disable all rules in the " << Name + << " pass then re-enable the specified ones\"),\n" + << " cl::Hidden,\n" + << " cl::cat(GICombinerOptionCategory),\n" + << " cl::callback([](const std::string &CommaSeparatedArg) {\n" + << " StringRef Str = CommaSeparatedArg;\n" + << " " << Name << "Option.push_back(\"*\");\n" + << " do {\n" + << " auto X = Str.split(\",\");\n" + << " " << Name << "Option.push_back((\"!\" + X.first).str());\n" + << " Str = X.second;\n" + << " } while (!Str.empty());\n" + << " }));\n" + << "\n" + << "bool " << getClassName() << "RuleConfig::parseCommandLineOption() {\n" + << " for (StringRef Identifier : " << Name << "Option) {\n" + << " bool Enabled = Identifier.consume_front(\"!\");\n" + << " if (Enabled && !setRuleEnabled(Identifier))\n" + << " return false;\n" + << " if (!Enabled && !setRuleDisabled(Identifier))\n" + << " return false;\n" + << " }\n" + << " return true;\n" + << "}\n\n"; + + OS << "bool " << getClassName() << "::tryCombineAll(\n" + << " GISelChangeObserver &Observer,\n" + << " MachineInstr &MI,\n" + << " MachineIRBuilder &B"; + emitAdditionalHelperMethodArguments(OS, Combiner); + OS << ") const {\n" + << " MachineBasicBlock *MBB = MI.getParent();\n" + << " MachineFunction *MF = MBB->getParent();\n" + << " MachineRegisterInfo &MRI = MF->getRegInfo();\n" + << " SmallVector<MachineInstr *, 8> MIs = { &MI };\n\n" + << " (void)MBB; (void)MF; (void)MRI; (void)RuleConfig;\n\n"; + + OS << " // Match data\n"; + for (const auto &Rule : Rules) + for (const auto &I : Rule->matchdata_decls()) + OS << " " << I.getType() << " " << I.getVariableName() << ";\n"; + OS << "\n"; + + OS << " int Partition = -1;\n"; + generateCodeForTree(OS, *Tree, " "); + OS << "\n return false;\n" + << "}\n" + << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_CPP\n"; +} + +} // end anonymous namespace + +//===----------------------------------------------------------------------===// + +namespace llvm { +void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS) { + CodeGenTarget Target(RK); + emitSourceFileHeader("Global Combiner", OS); + + if (SelectedCombiners.empty()) + PrintFatalError("No combiners selected with -combiners"); + for (const auto &Combiner : SelectedCombiners) { + Record *CombinerDef = RK.getDef(Combiner); + if (!CombinerDef) + PrintFatalError("Could not find " + Combiner); + GICombinerEmitter(RK, Target, Combiner, CombinerDef).run(OS); + } + NumPatternTotalStatistic = NumPatternTotal; +} + +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp new file mode 100644 index 000000000000..d59a9b8e3b65 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp @@ -0,0 +1,93 @@ +//===- CodeExpander.cpp - Expand variables in a string --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Expand the variables in a string. +// +//===----------------------------------------------------------------------===// + +#include "CodeExpander.h" +#include "CodeExpansions.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" + +using namespace llvm; + +void CodeExpander::emit(raw_ostream &OS) const { + StringRef Current = Code; + + while (!Current.empty()) { + size_t Pos = Current.find_first_of("$\n\\"); + if (Pos == StringRef::npos) { + OS << Current; + Current = ""; + continue; + } + + OS << Current.substr(0, Pos); + Current = Current.substr(Pos); + + if (Current.startswith("\n")) { + OS << "\n" << Indent; + Current = Current.drop_front(1); + continue; + } + + if (Current.startswith("\\$") || Current.startswith("\\\\")) { + OS << Current[1]; + Current = Current.drop_front(2); + continue; + } + + if (Current.startswith("\\")) { + Current = Current.drop_front(1); + continue; + } + + if (Current.startswith("${")) { + StringRef StartVar = Current; + Current = Current.drop_front(2); + StringRef Var; + std::tie(Var, Current) = Current.split("}"); + + // Warn if we split because no terminator was found. + StringRef EndVar = StartVar.drop_front(2 /* ${ */ + Var.size()); + if (EndVar.empty()) { + size_t LocOffset = StartVar.data() - Code.data(); + PrintWarning( + Loc.size() > 0 && Loc[0].isValid() + ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset) + : SMLoc(), + "Unterminated expansion"); + } + + auto ValueI = Expansions.find(Var); + if (ValueI == Expansions.end()) { + size_t LocOffset = StartVar.data() - Code.data(); + PrintError(Loc.size() > 0 && Loc[0].isValid() + ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset) + : SMLoc(), + "Attempting to expand an undeclared variable " + Var); + } + if (ShowExpansions) + OS << "/*$" << Var << "{*/"; + OS << Expansions.lookup(Var); + if (ShowExpansions) + OS << "/*}*/"; + continue; + } + + size_t LocOffset = Current.data() - Code.data(); + PrintWarning(Loc.size() > 0 && Loc[0].isValid() + ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset) + : SMLoc(), + "Assuming missing escape character"); + OS << "$"; + Current = Current.drop_front(1); + } +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.h new file mode 100644 index 000000000000..bd6946de5925 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpander.h @@ -0,0 +1,55 @@ +//===- CodeExpander.h - Expand variables in a string ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Expand the variables in a string. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H +#define LLVM_UTILS_TABLEGEN_CODEEXPANDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/SMLoc.h" + +namespace llvm { +class CodeExpansions; +class raw_ostream; + +/// Emit the given code with all '${foo}' placeholders expanded to their +/// replacements. +/// +/// It's an error to use an undefined expansion and expansion-like output that +/// needs to be emitted verbatim can be escaped as '\${foo}' +/// +/// The emitted code can be given a custom indent to enable both indentation by +/// an arbitrary amount of whitespace and emission of the code as a comment. +class CodeExpander { + StringRef Code; + const CodeExpansions &Expansions; + const ArrayRef<SMLoc> &Loc; + bool ShowExpansions; + StringRef Indent; + +public: + CodeExpander(StringRef Code, const CodeExpansions &Expansions, + const ArrayRef<SMLoc> &Loc, bool ShowExpansions, + StringRef Indent = " ") + : Code(Code), Expansions(Expansions), Loc(Loc), + ShowExpansions(ShowExpansions), Indent(Indent) {} + + void emit(raw_ostream &OS) const; +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const CodeExpander &Expander) { + Expander.emit(OS); + return OS; +} +} // end namespace llvm + +#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpansions.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpansions.h new file mode 100644 index 000000000000..bb890ec8f57e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/CodeExpansions.h @@ -0,0 +1,43 @@ +//===- CodeExpansions.h - Record expansions for CodeExpander --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Record the expansions to use in a CodeExpander. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringMap.h" + +#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H +#define LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H +namespace llvm { +class CodeExpansions { +public: + using const_iterator = StringMap<std::string>::const_iterator; + +protected: + StringMap<std::string> Expansions; + +public: + void declare(StringRef Name, StringRef Expansion) { + bool Inserted = Expansions.try_emplace(Name, Expansion).second; + assert(Inserted && "Declared variable twice"); + (void)Inserted; + } + + std::string lookup(StringRef Variable) const { + return Expansions.lookup(Variable); + } + + const_iterator begin() const { return Expansions.begin(); } + const_iterator end() const { return Expansions.end(); } + const_iterator find(StringRef Variable) const { + return Expansions.find(Variable); + } +}; +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.cpp new file mode 100644 index 000000000000..a3a9b7d8b037 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.cpp @@ -0,0 +1,138 @@ +//===- GIMatchDag.cpp - A DAG representation of a pattern to be matched ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDag.h" + +#include "llvm/Support/Format.h" +#include "llvm/TableGen/Record.h" +#include "../CodeGenInstruction.h" + +using namespace llvm; + +void GIMatchDag::writeDOTGraph(raw_ostream &OS, StringRef ID) const { + const auto writePorts = [&](StringRef Prefix, + const GIMatchDagOperandList &Operands) { + StringRef Separator = ""; + OS << "{"; + for (const auto &Op : enumerate(Operands)) { + OS << Separator << "<" << Prefix << format("%d", Op.index()) << ">" + << "#" << Op.index() << " $" << Op.value().getName(); + Separator = "|"; + } + OS << "}"; + }; + + OS << "digraph \"" << ID << "\" {\n" + << " rankdir=\"BT\"\n"; + for (const auto &N : InstrNodes) { + OS << " " << format("Node%p", &*N) << " [shape=record,label=\"{"; + writePorts("s", N->getOperandInfo()); + OS << "|" << N->getName(); + if (N->getOpcodeAnnotation()) + OS << "|" << N->getOpcodeAnnotation()->TheDef->getName(); + if (N->isMatchRoot()) + OS << "|Match starts here"; + OS << "|"; + SmallVector<std::pair<unsigned, StringRef>, 8> ToPrint; + for (const auto &Assignment : N->user_assigned_operand_names()) + ToPrint.emplace_back(Assignment.first, Assignment.second); + llvm::sort(ToPrint.begin(), ToPrint.end()); + StringRef Separator = ""; + for (const auto &Assignment : ToPrint) { + OS << Separator << "$" << Assignment.second << "=getOperand(" + << Assignment.first << ")"; + Separator = ", "; + } + OS << format("|%p|", &N); + writePorts("d", N->getOperandInfo()); + OS << "}\""; + if (N->isMatchRoot()) + OS << ",color=red"; + OS << "]\n"; + } + + for (const auto &E : Edges) { + const char *FromFmt = "Node%p:s%d:n"; + const char *ToFmt = "Node%p:d%d:s"; + if (E->getFromMO()->isDef() && !E->getToMO()->isDef()) + std::swap(FromFmt, ToFmt); + auto From = format(FromFmt, E->getFromMI(), E->getFromMO()->getIdx()); + auto To = format(ToFmt, E->getToMI(), E->getToMO()->getIdx()); + if (E->getFromMO()->isDef() && !E->getToMO()->isDef()) + std::swap(From, To); + + OS << " " << From << " -> " << To << " [label=\"$" << E->getName(); + if (E->getFromMO()->isDef() == E->getToMO()->isDef()) + OS << " INVALID EDGE!"; + OS << "\""; + if (E->getFromMO()->isDef() == E->getToMO()->isDef()) + OS << ",color=red"; + else if (E->getFromMO()->isDef() && !E->getToMO()->isDef()) + OS << ",dir=back,arrowtail=crow"; + OS << "]\n"; + } + + for (const auto &N : PredicateNodes) { + OS << " " << format("Pred%p", &*N) << " [shape=record,label=\"{"; + writePorts("s", N->getOperandInfo()); + OS << "|" << N->getName() << "|"; + N->printDescription(OS); + OS << format("|%p|", &N); + writePorts("d", N->getOperandInfo()); + OS << "}\",style=dotted]\n"; + } + + for (const auto &E : PredicateDependencies) { + const char *FromMIFmt = "Node%p:e"; + const char *FromMOFmt = "Node%p:s%d:n"; + const char *ToFmt = "Pred%p:d%d:s"; + auto To = format(ToFmt, E->getPredicate(), E->getPredicateOp()->getIdx()); + auto Style = "[style=dotted]"; + if (E->getRequiredMO()) { + auto From = + format(FromMOFmt, E->getRequiredMI(), E->getRequiredMO()->getIdx()); + OS << " " << From << " -> " << To << " " << Style << "\n"; + continue; + } + auto From = format(FromMIFmt, E->getRequiredMI()); + OS << " " << From << " -> " << To << " " << Style << "\n"; + } + + OS << "}\n"; +} + +LLVM_DUMP_METHOD void GIMatchDag::print(raw_ostream &OS) const { + OS << "matchdag {\n"; + for (const auto &N : InstrNodes) { + OS << " "; + N->print(OS); + OS << "\n"; + } + for (const auto &E : Edges) { + OS << " "; + E->print(OS); + OS << "\n"; + } + + for (const auto &P : PredicateNodes) { + OS << " "; + P->print(OS); + OS << "\n"; + } + for (const auto &D : PredicateDependencies) { + OS << " "; + D->print(OS); + OS << "\n"; + } + OS << "}\n"; +} + +raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDag &G) { + G.print(OS); + return OS; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.h new file mode 100644 index 000000000000..567580540877 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDag.h @@ -0,0 +1,243 @@ +//===- GIMatchDag.h - Represent a DAG to be matched -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAG_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAG_H + +#include "GIMatchDagEdge.h" +#include "GIMatchDagInstr.h" +#include "GIMatchDagOperands.h" +#include "GIMatchDagPredicate.h" +#include "GIMatchDagPredicateDependencyEdge.h" + +namespace llvm { +class GIMatchDag; + +/// This class manages lifetimes for data associated with the GIMatchDag object. +class GIMatchDagContext { + GIMatchDagOperandListContext OperandListCtx; + +public: + const GIMatchDagOperandList &makeEmptyOperandList() { + return OperandListCtx.makeEmptyOperandList(); + } + + const GIMatchDagOperandList &makeOperandList(const CodeGenInstruction &I) { + return OperandListCtx.makeOperandList(I); + } + + const GIMatchDagOperandList &makeMIPredicateOperandList() { + return OperandListCtx.makeMIPredicateOperandList(); + } + + + const GIMatchDagOperandList &makeTwoMOPredicateOperandList() { + return OperandListCtx.makeTwoMOPredicateOperandList(); + } + + void print(raw_ostream &OS) const { + OperandListCtx.print(OS); + } + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +class GIMatchDag { +public: + using InstrNodesVec = std::vector<std::unique_ptr<GIMatchDagInstr>>; + using instr_node_iterator = raw_pointer_iterator<InstrNodesVec::iterator>; + using const_instr_node_iterator = + raw_pointer_iterator<InstrNodesVec::const_iterator>; + + using EdgesVec = std::vector<std::unique_ptr<GIMatchDagEdge>>; + using edge_iterator = raw_pointer_iterator<EdgesVec::iterator>; + using const_edge_iterator = raw_pointer_iterator<EdgesVec::const_iterator>; + + using PredicateNodesVec = std::vector<std::unique_ptr<GIMatchDagPredicate>>; + using predicate_iterator = raw_pointer_iterator<PredicateNodesVec::iterator>; + using const_predicate_iterator = + raw_pointer_iterator<PredicateNodesVec::const_iterator>; + + using PredicateDependencyEdgesVec = + std::vector<std::unique_ptr<GIMatchDagPredicateDependencyEdge>>; + using predicate_edge_iterator = + raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>; + using const_predicate_edge_iterator = + raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>; + +protected: + GIMatchDagContext &Ctx; + InstrNodesVec InstrNodes; + PredicateNodesVec PredicateNodes; + EdgesVec Edges; + PredicateDependencyEdgesVec PredicateDependencies; + std::vector<GIMatchDagInstr *> MatchRoots; + // FIXME: This is a temporary measure while we still accept arbitrary code + // blocks to fix up the matcher while it's being developed. + bool HasPostMatchPredicate = false; + +public: + GIMatchDag(GIMatchDagContext &Ctx) + : Ctx(Ctx), InstrNodes(), PredicateNodes(), Edges(), + PredicateDependencies() {} + GIMatchDag(const GIMatchDag &) = delete; + + GIMatchDagContext &getContext() const { return Ctx; } + edge_iterator edges_begin() { + return raw_pointer_iterator<EdgesVec::iterator>(Edges.begin()); + } + edge_iterator edges_end() { + return raw_pointer_iterator<EdgesVec::iterator>(Edges.end()); + } + const_edge_iterator edges_begin() const { + return raw_pointer_iterator<EdgesVec::const_iterator>(Edges.begin()); + } + const_edge_iterator edges_end() const { + return raw_pointer_iterator<EdgesVec::const_iterator>(Edges.end()); + } + iterator_range<edge_iterator> edges() { + return make_range(edges_begin(), edges_end()); + } + iterator_range<const_edge_iterator> edges() const { + return make_range(edges_begin(), edges_end()); + } + iterator_range<std::vector<GIMatchDagInstr *>::iterator> roots() { + return make_range(MatchRoots.begin(), MatchRoots.end()); + } + iterator_range<std::vector<GIMatchDagInstr *>::const_iterator> roots() const { + return make_range(MatchRoots.begin(), MatchRoots.end()); + } + + instr_node_iterator instr_nodes_begin() { + return raw_pointer_iterator<InstrNodesVec::iterator>(InstrNodes.begin()); + } + instr_node_iterator instr_nodes_end() { + return raw_pointer_iterator<InstrNodesVec::iterator>(InstrNodes.end()); + } + const_instr_node_iterator instr_nodes_begin() const { + return raw_pointer_iterator<InstrNodesVec::const_iterator>( + InstrNodes.begin()); + } + const_instr_node_iterator instr_nodes_end() const { + return raw_pointer_iterator<InstrNodesVec::const_iterator>( + InstrNodes.end()); + } + iterator_range<instr_node_iterator> instr_nodes() { + return make_range(instr_nodes_begin(), instr_nodes_end()); + } + iterator_range<const_instr_node_iterator> instr_nodes() const { + return make_range(instr_nodes_begin(), instr_nodes_end()); + } + predicate_edge_iterator predicate_edges_begin() { + return raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>( + PredicateDependencies.begin()); + } + predicate_edge_iterator predicate_edges_end() { + return raw_pointer_iterator<PredicateDependencyEdgesVec::iterator>( + PredicateDependencies.end()); + } + const_predicate_edge_iterator predicate_edges_begin() const { + return raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>( + PredicateDependencies.begin()); + } + const_predicate_edge_iterator predicate_edges_end() const { + return raw_pointer_iterator<PredicateDependencyEdgesVec::const_iterator>( + PredicateDependencies.end()); + } + iterator_range<predicate_edge_iterator> predicate_edges() { + return make_range(predicate_edges_begin(), predicate_edges_end()); + } + iterator_range<const_predicate_edge_iterator> predicate_edges() const { + return make_range(predicate_edges_begin(), predicate_edges_end()); + } + predicate_iterator predicates_begin() { + return raw_pointer_iterator<PredicateNodesVec::iterator>( + PredicateNodes.begin()); + } + predicate_iterator predicates_end() { + return raw_pointer_iterator<PredicateNodesVec::iterator>( + PredicateNodes.end()); + } + const_predicate_iterator predicates_begin() const { + return raw_pointer_iterator<PredicateNodesVec::const_iterator>( + PredicateNodes.begin()); + } + const_predicate_iterator predicates_end() const { + return raw_pointer_iterator<PredicateNodesVec::const_iterator>( + PredicateNodes.end()); + } + iterator_range<predicate_iterator> predicates() { + return make_range(predicates_begin(), predicates_end()); + } + iterator_range<const_predicate_iterator> predicates() const { + return make_range(predicates_begin(), predicates_end()); + } + + template <class... Args> GIMatchDagInstr *addInstrNode(Args &&... args) { + auto Obj = + std::make_unique<GIMatchDagInstr>(*this, std::forward<Args>(args)...); + auto ObjRaw = Obj.get(); + InstrNodes.push_back(std::move(Obj)); + return ObjRaw; + } + + template <class T, class... Args> + T *addPredicateNode(Args &&... args) { + auto Obj = std::make_unique<T>(getContext(), std::forward<Args>(args)...); + auto ObjRaw = Obj.get(); + PredicateNodes.push_back(std::move(Obj)); + return ObjRaw; + } + + template <class... Args> GIMatchDagEdge *addEdge(Args &&... args) { + auto Obj = std::make_unique<GIMatchDagEdge>(std::forward<Args>(args)...); + auto ObjRaw = Obj.get(); + Edges.push_back(std::move(Obj)); + return ObjRaw; + } + + template <class... Args> + GIMatchDagPredicateDependencyEdge *addPredicateDependency(Args &&... args) { + auto Obj = std::make_unique<GIMatchDagPredicateDependencyEdge>( + std::forward<Args>(args)...); + auto ObjRaw = Obj.get(); + PredicateDependencies.push_back(std::move(Obj)); + return ObjRaw; + } + + size_t getInstrNodeIdx(instr_node_iterator I) { + return std::distance(instr_nodes_begin(), I); + } + size_t getInstrNodeIdx(const_instr_node_iterator I) const { + return std::distance(instr_nodes_begin(), I); + } + size_t getNumInstrNodes() const { return InstrNodes.size(); } + size_t getNumEdges() const { return Edges.size(); } + size_t getNumPredicates() const { return PredicateNodes.size(); } + + void setHasPostMatchPredicate(bool V) { HasPostMatchPredicate = V; } + bool hasPostMatchPredicate() const { return HasPostMatchPredicate; } + + void addMatchRoot(GIMatchDagInstr *N) { MatchRoots.push_back(N); } + + LLVM_DUMP_METHOD void print(raw_ostream &OS) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + + void writeDOTGraph(raw_ostream &OS, StringRef ID) const; +}; + +raw_ostream &operator<<(raw_ostream &OS, const GIMatchDag &G); + +} // end namespace llvm + +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAG_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp new file mode 100644 index 000000000000..e59cb3aae49a --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.cpp @@ -0,0 +1,38 @@ +//===- GIMatchDagEdge.cpp - An edge describing a def/use lookup -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDagEdge.h" +#include "GIMatchDagInstr.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +LLVM_DUMP_METHOD void GIMatchDagEdge::print(raw_ostream &OS) const { + OS << getFromMI()->getName() << "[" << getFromMO()->getName() << "] --[" + << Name << "]--> " << getToMI()->getName() << "[" << getToMO()->getName() + << "]"; +} + +bool GIMatchDagEdge::isDefToUse() const { + // Def -> Def is invalid so we only need to check FromMO. + return FromMO->isDef(); +} + +void GIMatchDagEdge::reverse() { + std::swap(FromMI, ToMI); + std::swap(FromMO, ToMO); +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +LLVM_DUMP_METHOD void GIMatchDagEdge::dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + +raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDagEdge &E) { + E.print(OS); + return OS; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h new file mode 100644 index 000000000000..8e845ff0a51e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagEdge.h @@ -0,0 +1,70 @@ +//===- GIMatchDagEdge.h - Represent a shared operand list for nodes -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGEDGE_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAGEDGE_H + +#include "llvm/ADT/StringRef.h" + +namespace llvm { +class raw_ostream; +class GIMatchDagInstr; +class GIMatchDagOperand; + +/// Represents an edge that connects two instructions together via a pair of +/// operands. For example: +/// %a = FOO ... +/// %0 = BAR %a +/// %1 = BAZ %a +/// would have two edges for %a like so: +/// BAR:Op#1 --[a]----> Op#0:FOO +/// ^ +/// BAZ:Op#1 --[a]------/ +/// Ideally, all edges in the DAG are from a use to a def as this is a many +/// to one edge but edges from defs to uses are supported too. +class GIMatchDagEdge { + /// The name of the edge. For example, + /// (FOO $a, $b, $c) + /// (BAR $d, $e, $a) + /// will create an edge named 'a' to connect FOO to BAR. Although the name + /// refers to the edge, the canonical value of 'a' is the operand that defines + /// it. + StringRef Name; + const GIMatchDagInstr *FromMI; + const GIMatchDagOperand *FromMO; + const GIMatchDagInstr *ToMI; + const GIMatchDagOperand *ToMO; + +public: + GIMatchDagEdge(StringRef Name, const GIMatchDagInstr *FromMI, const GIMatchDagOperand *FromMO, + const GIMatchDagInstr *ToMI, const GIMatchDagOperand *ToMO) + : Name(Name), FromMI(FromMI), FromMO(FromMO), ToMI(ToMI), ToMO(ToMO) {} + + StringRef getName() const { return Name; } + const GIMatchDagInstr *getFromMI() const { return FromMI; } + const GIMatchDagOperand *getFromMO() const { return FromMO; } + const GIMatchDagInstr *getToMI() const { return ToMI; } + const GIMatchDagOperand *getToMO() const { return ToMO; } + + /// Flip the direction of the edge. + void reverse(); + + /// Does this edge run from a def to (one of many) uses? + bool isDefToUse() const; + + LLVM_DUMP_METHOD void print(raw_ostream &OS) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const; +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagEdge &E); + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGEDGE_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.cpp new file mode 100644 index 000000000000..218b741be20c --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.cpp @@ -0,0 +1,48 @@ +//===- GIMatchDagInstr.cpp - A shared operand list for nodes --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDagInstr.h" +#include "../CodeGenInstruction.h" +#include "GIMatchDag.h" +#include "llvm/TableGen/Record.h" + +using namespace llvm; + +void GIMatchDagInstr::print(raw_ostream &OS) const { + OS << "("; + if (const auto *Annotation = getOpcodeAnnotation()) + OS << Annotation->TheDef->getName(); + else + OS << "<unknown>"; + OS << " "; + OperandInfo.print(OS); + OS << "):$" << Name; + if (!UserAssignedNamesForOperands.empty()) { + OS << " // "; + SmallVector<std::pair<unsigned, StringRef>, 8> ToPrint; + for (const auto &Assignment : UserAssignedNamesForOperands) + ToPrint.emplace_back(Assignment.first, Assignment.second); + llvm::sort(ToPrint.begin(), ToPrint.end()); + StringRef Separator = ""; + for (const auto &Assignment : ToPrint) { + OS << Separator << "$" << Assignment.second << "=getOperand(" + << Assignment.first << ")"; + Separator = ", "; + } + } +} + +void GIMatchDagInstr::setMatchRoot() { + IsMatchRoot = true; + Dag.addMatchRoot(this); +} + +raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDagInstr &N) { + N.print(OS); + return OS; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h new file mode 100644 index 000000000000..4a07767a2e19 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagInstr.h @@ -0,0 +1,115 @@ +//===- GIMatchDagInstr.h - Represent a instruction to be matched ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGINSTR_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAGINSTR_H + +#include "GIMatchDagOperands.h" +#include "llvm/ADT/DenseMap.h" + +namespace llvm { +class GIMatchDag; + +/// Represents an instruction in the match DAG. This object knows very little +/// about the actual instruction to be matched as the bulk of that is in +/// predicates that are associated with the match DAG. It merely knows the names +/// and indices of any operands that need to be matched in order to allow edges +/// to link to them. +/// +/// Instances of this class objects are owned by the GIMatchDag and are not +/// shareable between instances of GIMatchDag. This is because the Name, +/// IsMatchRoot, and OpcodeAnnotation are likely to differ between GIMatchDag +/// instances. +class GIMatchDagInstr { +public: + using const_user_assigned_operand_names_iterator = + DenseMap<unsigned, StringRef>::const_iterator; + +protected: + /// The match DAG this instruction belongs to. + GIMatchDag &Dag; + + /// The name of the instruction in the pattern. For example: + /// (FOO $a, $b, $c):$name + /// will cause name to be assigned to this member. Anonymous instructions will + /// have a name assigned for debugging purposes. + StringRef Name; + + /// The name of the instruction in the pattern as assigned by the user. For + /// example: + /// (FOO $a, $b, $c):$name + /// will cause name to be assigned to this member. If a name is not provided, + /// this will be empty. This name is used to bind variables from rules to the + /// matched instruction. + StringRef UserAssignedName; + + /// The name of each operand (if any) that was assigned by the user. For + /// example: + /// (FOO $a, $b, $c):$name + /// will cause {0, "a"}, {1, "b"}, {2, "c} to be inserted into this map. + DenseMap<unsigned, StringRef> UserAssignedNamesForOperands; + + /// The operand list for this instruction. This object may be shared with + /// other instructions of a similar 'shape'. + const GIMatchDagOperandList &OperandInfo; + + /// For debugging purposes, it's helpful to have access to a description of + /// the Opcode. However, this object shouldn't use it for more than debugging + /// output since predicates are expected to be handled outside the DAG. + CodeGenInstruction *OpcodeAnnotation = 0; + + /// When true, this instruction will be a starting point for a match attempt. + bool IsMatchRoot = false; + +public: + GIMatchDagInstr(GIMatchDag &Dag, StringRef Name, StringRef UserAssignedName, + const GIMatchDagOperandList &OperandInfo) + : Dag(Dag), Name(Name), UserAssignedName(UserAssignedName), + OperandInfo(OperandInfo) {} + + const GIMatchDagOperandList &getOperandInfo() const { return OperandInfo; } + StringRef getName() const { return Name; } + StringRef getUserAssignedName() const { return UserAssignedName; } + void assignNameToOperand(unsigned Idx, StringRef Name) { + assert(UserAssignedNamesForOperands[Idx].empty() && "Cannot assign twice"); + UserAssignedNamesForOperands[Idx] = Name; + } + + const_user_assigned_operand_names_iterator + user_assigned_operand_names_begin() const { + return UserAssignedNamesForOperands.begin(); + } + const_user_assigned_operand_names_iterator + user_assigned_operand_names_end() const { + return UserAssignedNamesForOperands.end(); + } + iterator_range<const_user_assigned_operand_names_iterator> + user_assigned_operand_names() const { + return make_range(user_assigned_operand_names_begin(), + user_assigned_operand_names_end()); + } + + /// Mark this instruction as being a root of the match. This means that the + /// matcher will start from this node when attempting to match MIR. + void setMatchRoot(); + bool isMatchRoot() const { return IsMatchRoot; } + + void setOpcodeAnnotation(CodeGenInstruction *I) { OpcodeAnnotation = I; } + CodeGenInstruction *getOpcodeAnnotation() const { return OpcodeAnnotation; } + + void print(raw_ostream &OS) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagInstr &N); + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGINSTR_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp new file mode 100644 index 000000000000..e79e4686b91e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.cpp @@ -0,0 +1,153 @@ +//===- GIMatchDagOperands.cpp - A shared operand list for nodes -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDagOperands.h" + +#include "../CodeGenInstruction.h" + +using namespace llvm; + +void GIMatchDagOperand::Profile(FoldingSetNodeID &ID) const { + Profile(ID, Idx, Name, IsDef); +} + +void GIMatchDagOperand::Profile(FoldingSetNodeID &ID, size_t Idx, + StringRef Name, bool IsDef) { + ID.AddInteger(Idx); + ID.AddString(Name); + ID.AddBoolean(IsDef); +} + +void GIMatchDagOperandList::add(StringRef Name, unsigned Idx, bool IsDef) { + assert(Idx == Operands.size() && "Operands added in wrong order"); + Operands.emplace_back(Operands.size(), Name, IsDef); + OperandsByName.try_emplace(Operands.back().getName(), Operands.size() - 1); +} + +void GIMatchDagOperandList::Profile(FoldingSetNodeID &ID) const { + for (const auto &I : enumerate(Operands)) + GIMatchDagOperand::Profile(ID, I.index(), I.value().getName(), + I.value().isDef()); +} + +void GIMatchDagOperandList::print(raw_ostream &OS) const { + if (Operands.empty()) { + OS << "<empty>"; + return; + } + StringRef Separator = ""; + for (const auto &I : Operands) { + OS << Separator << I.getIdx() << ":" << I.getName(); + if (I.isDef()) + OS << "<def>"; + Separator = ", "; + } +} + +const GIMatchDagOperandList::value_type &GIMatchDagOperandList:: +operator[](StringRef K) const { + const auto &I = OperandsByName.find(K); + assert(I != OperandsByName.end() && "Operand not found by name"); + return Operands[I->second]; +} + +const GIMatchDagOperandList & +GIMatchDagOperandListContext::makeEmptyOperandList() { + FoldingSetNodeID ID; + + void *InsertPoint; + GIMatchDagOperandList *Value = + OperandLists.FindNodeOrInsertPos(ID, InsertPoint); + if (Value) + return *Value; + + std::unique_ptr<GIMatchDagOperandList> NewValue = + std::make_unique<GIMatchDagOperandList>(); + OperandLists.InsertNode(NewValue.get(), InsertPoint); + OperandListsOwner.push_back(std::move(NewValue)); + return *OperandListsOwner.back().get(); +} + +const GIMatchDagOperandList & +GIMatchDagOperandListContext::makeOperandList(const CodeGenInstruction &I) { + FoldingSetNodeID ID; + for (unsigned i = 0; i < I.Operands.size(); ++i) + GIMatchDagOperand::Profile(ID, i, I.Operands[i].Name, + i < I.Operands.NumDefs); + + void *InsertPoint; + GIMatchDagOperandList *Value = + OperandLists.FindNodeOrInsertPos(ID, InsertPoint); + if (Value) + return *Value; + + std::unique_ptr<GIMatchDagOperandList> NewValue = + std::make_unique<GIMatchDagOperandList>(); + for (unsigned i = 0; i < I.Operands.size(); ++i) + NewValue->add(I.Operands[i].Name, i, i < I.Operands.NumDefs); + OperandLists.InsertNode(NewValue.get(), InsertPoint); + OperandListsOwner.push_back(std::move(NewValue)); + return *OperandListsOwner.back().get(); +} + +const GIMatchDagOperandList & +GIMatchDagOperandListContext::makeMIPredicateOperandList() { + FoldingSetNodeID ID; + GIMatchDagOperand::Profile(ID, 0, "$", true); + GIMatchDagOperand::Profile(ID, 1, "mi", false); + + void *InsertPoint; + GIMatchDagOperandList *Value = + OperandLists.FindNodeOrInsertPos(ID, InsertPoint); + if (Value) + return *Value; + + std::unique_ptr<GIMatchDagOperandList> NewValue = + std::make_unique<GIMatchDagOperandList>(); + NewValue->add("$", 0, true); + NewValue->add("mi", 1, false); + OperandLists.InsertNode(NewValue.get(), InsertPoint); + OperandListsOwner.push_back(std::move(NewValue)); + return *OperandListsOwner.back().get(); +} + + +const GIMatchDagOperandList & +GIMatchDagOperandListContext::makeTwoMOPredicateOperandList() { + FoldingSetNodeID ID; + GIMatchDagOperand::Profile(ID, 0, "$", true); + GIMatchDagOperand::Profile(ID, 1, "mi0", false); + GIMatchDagOperand::Profile(ID, 2, "mi1", false); + + void *InsertPoint; + GIMatchDagOperandList *Value = + OperandLists.FindNodeOrInsertPos(ID, InsertPoint); + if (Value) + return *Value; + + std::unique_ptr<GIMatchDagOperandList> NewValue = + std::make_unique<GIMatchDagOperandList>(); + NewValue->add("$", 0, true); + NewValue->add("mi0", 1, false); + NewValue->add("mi1", 2, false); + OperandLists.InsertNode(NewValue.get(), InsertPoint); + OperandListsOwner.push_back(std::move(NewValue)); + return *OperandListsOwner.back().get(); +} + +void GIMatchDagOperandListContext::print(raw_ostream &OS) const { + OS << "GIMatchDagOperandListContext {\n" + << " OperandLists {\n"; + for (const auto &I : OperandListsOwner) { + OS << " "; + I->print(OS); + OS << "\n"; + } + OS << " }\n" + << "}\n"; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h new file mode 100644 index 000000000000..c2d30574231d --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagOperands.h @@ -0,0 +1,133 @@ +//===- GIMatchDagOperands.h - Represent a shared operand list for nodes ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGOPERANDS_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAGOPERANDS_H + +#include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" + +#include <vector> + +namespace llvm { +class CodeGenInstruction; +/// Describes an operand of a MachineInstr w.r.t the DAG Matching. This +/// information is derived from CodeGenInstruction::Operands but is more +/// readily available for context-less access as we don't need to know which +/// instruction it's used with or know how many defs that instruction had. +/// +/// There may be multiple GIMatchDagOperand's with the same contents. However, +/// they are uniqued within the set of instructions that have the same overall +/// operand list. For example, given: +/// Inst1 operands ($dst:<def>, $src1, $src2) +/// Inst2 operands ($dst:<def>, $src1, $src2) +/// Inst3 operands ($dst:<def>, $src) +/// $src1 will have a single instance of GIMatchDagOperand shared by Inst1 and +/// Inst2, as will $src2. $dst however, will have two instances one shared +/// between Inst1 and Inst2 and one unique to Inst3. We could potentially +/// fully de-dupe the GIMatchDagOperand instances but the saving is not expected +/// to be worth the overhead. +/// +/// The result of this is that the address of the object can be relied upon to +/// trivially identify commonality between two instructions which will be useful +/// when generating the matcher. When the pointers differ, the contents can be +/// inspected instead. +class GIMatchDagOperand { + unsigned Idx; + StringRef Name; + bool IsDef; + +public: + GIMatchDagOperand(unsigned Idx, StringRef Name, bool IsDef) + : Idx(Idx), Name(Name), IsDef(IsDef) {} + + unsigned getIdx() const { return Idx; } + StringRef getName() const { return Name; } + bool isDef() const { return IsDef; } + + /// This object isn't a FoldingSetNode but it's part of one. See FoldingSet + /// for details on the Profile function. + void Profile(FoldingSetNodeID &ID) const; + + /// A helper that behaves like Profile() but is also usable without the object. + /// We use size_t here to match enumerate<...>::index(). If we don't match + /// that the hashes won't be equal. + static void Profile(FoldingSetNodeID &ID, size_t Idx, StringRef Name, + bool IsDef); +}; + +/// A list of GIMatchDagOperands for an instruction without any association with +/// a particular instruction. +/// +/// An important detail to be aware of with this class is that they are shared +/// with other instructions of a similar 'shape'. For example, all the binary +/// instructions are likely to share a single GIMatchDagOperandList. This is +/// primarily a memory optimization as it's fairly common to have a large number +/// of instructions but only a few 'shapes'. +/// +/// See GIMatchDagOperandList::Profile() for the details on how they are folded. +class GIMatchDagOperandList : public FoldingSetNode { +public: + using value_type = GIMatchDagOperand; + +protected: + using vector_type = SmallVector<GIMatchDagOperand, 3>; + +public: + using iterator = vector_type::iterator; + using const_iterator = vector_type::const_iterator; + +protected: + vector_type Operands; + StringMap<unsigned> OperandsByName; + +public: + void add(StringRef Name, unsigned Idx, bool IsDef); + + /// See FoldingSet for details. + void Profile(FoldingSetNodeID &ID) const; + + iterator begin() { return Operands.begin(); } + const_iterator begin() const { return Operands.begin(); } + iterator end() { return Operands.end(); } + const_iterator end() const { return Operands.end(); } + + const value_type &operator[](unsigned I) const { return Operands[I]; } + const value_type &operator[](StringRef K) const; + + void print(raw_ostream &OS) const; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +/// This is the portion of GIMatchDagContext that directly relates to +/// GIMatchDagOperandList and GIMatchDagOperandList. +class GIMatchDagOperandListContext { + FoldingSet<GIMatchDagOperandList> OperandLists; + std::vector<std::unique_ptr<GIMatchDagOperandList>> OperandListsOwner; + +public: + const GIMatchDagOperandList &makeEmptyOperandList(); + const GIMatchDagOperandList &makeOperandList(const CodeGenInstruction &I); + const GIMatchDagOperandList &makeMIPredicateOperandList(); + const GIMatchDagOperandList &makeTwoMOPredicateOperandList(); + + void print(raw_ostream &OS) const; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGOPERANDS_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp new file mode 100644 index 000000000000..1aca2f9dc135 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.cpp @@ -0,0 +1,69 @@ +//===- GIMatchDagPredicate.cpp - Represent a predicate to check -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDagPredicate.h" + +#include "llvm/TableGen/Record.h" + +#include "GIMatchDagOperands.h" +#include "../CodeGenInstruction.h" + +using namespace llvm; + +void GIMatchDagPredicate::print(raw_ostream &OS) const { + OS << "<<"; + printDescription(OS); + OS << ">>:$" << Name; +} + +void GIMatchDagPredicate::printDescription(raw_ostream &OS) const { OS << ""; } + +GIMatchDagOpcodePredicate::GIMatchDagOpcodePredicate( + GIMatchDagContext &Ctx, StringRef Name, const CodeGenInstruction &Instr) + : GIMatchDagPredicate(GIMatchDagPredicateKind_Opcode, Name, + Ctx.makeMIPredicateOperandList()), + Instr(Instr) {} + +void GIMatchDagOpcodePredicate::printDescription(raw_ostream &OS) const { + OS << "$mi.getOpcode() == " << Instr.TheDef->getName(); +} + +GIMatchDagOneOfOpcodesPredicate::GIMatchDagOneOfOpcodesPredicate( + GIMatchDagContext &Ctx, StringRef Name) + : GIMatchDagPredicate(GIMatchDagPredicateKind_OneOfOpcodes, Name, + Ctx.makeMIPredicateOperandList()) {} + +void GIMatchDagOneOfOpcodesPredicate::printDescription(raw_ostream &OS) const { + OS << "$mi.getOpcode() == oneof("; + StringRef Separator = ""; + for (const CodeGenInstruction *Instr : Instrs) { + OS << Separator << Instr->TheDef->getName(); + Separator = ","; + } + OS << ")"; +} + +GIMatchDagSameMOPredicate::GIMatchDagSameMOPredicate(GIMatchDagContext &Ctx, + StringRef Name) + : GIMatchDagPredicate(GIMatchDagPredicateKind_SameMO, Name, + Ctx.makeTwoMOPredicateOperandList()) {} + +void GIMatchDagSameMOPredicate::printDescription(raw_ostream &OS) const { + OS << "$mi0 == $mi1"; +} + +raw_ostream &llvm::operator<<(raw_ostream &OS, const GIMatchDagPredicate &N) { + N.print(OS); + return OS; +} + +raw_ostream &llvm::operator<<(raw_ostream &OS, + const GIMatchDagOpcodePredicate &N) { + N.print(OS); + return OS; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h new file mode 100644 index 000000000000..9b030d6edb13 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicate.h @@ -0,0 +1,141 @@ +//===- GIMatchDagPredicate - Represent a predicate to check ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATE_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATE_H + +#include "llvm/ADT/StringRef.h" +#include "GIMatchDag.h" + +namespace llvm { +class CodeExpansions; +class CodeGenInstruction; +class GIMatchDagOperandList; +class GIMatchDagContext; +class raw_ostream; + +/// Represents a predicate on the match DAG. This records the details of the +/// predicate. The dependencies are stored in the GIMatchDag as edges. +/// +/// Instances of this class objects are owned by the GIMatchDag and are not +/// shareable between instances of GIMatchDag. +class GIMatchDagPredicate { +public: + enum GIMatchDagPredicateKind { + GIMatchDagPredicateKind_Opcode, + GIMatchDagPredicateKind_OneOfOpcodes, + GIMatchDagPredicateKind_SameMO, + }; + +protected: + const GIMatchDagPredicateKind Kind; + + /// The name of the predicate. For example: + /// (FOO $a:s32, $b, $c) + /// will cause 's32' to be assigned to this member for the $a predicate. + /// Similarly, the opcode predicate will cause 'FOO' to be assigned to this + /// member. Anonymous instructions will have a name assigned for debugging + /// purposes. + StringRef Name; + + /// The operand list for this predicate. This object may be shared with + /// other predicates of a similar 'shape'. + const GIMatchDagOperandList &OperandInfo; + +public: + GIMatchDagPredicate(GIMatchDagPredicateKind Kind, StringRef Name, + const GIMatchDagOperandList &OperandInfo) + : Kind(Kind), Name(Name), OperandInfo(OperandInfo) {} + virtual ~GIMatchDagPredicate() {} + + GIMatchDagPredicateKind getKind() const { return Kind; } + + StringRef getName() const { return Name; } + const GIMatchDagOperandList &getOperandInfo() const { return OperandInfo; } + + // Generate C++ code to check this predicate. If a partitioner has already + // tested this predicate then this function won't be called. If this function + // is called, it must emit code and return true to indicate that it did so. If + // it ever returns false, then the caller will abort due to an untested + // predicate. + virtual bool generateCheckCode(raw_ostream &OS, StringRef Indent, + const CodeExpansions &Expansions) const { + return false; + } + + virtual void print(raw_ostream &OS) const; + virtual void printDescription(raw_ostream &OS) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + virtual LLVM_DUMP_METHOD void dump() const { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +class GIMatchDagOpcodePredicate : public GIMatchDagPredicate { + const CodeGenInstruction &Instr; + +public: + GIMatchDagOpcodePredicate(GIMatchDagContext &Ctx, StringRef Name, + const CodeGenInstruction &Instr); + + static bool classof(const GIMatchDagPredicate *P) { + return P->getKind() == GIMatchDagPredicateKind_Opcode; + } + + const CodeGenInstruction *getInstr() const { return &Instr; } + + void printDescription(raw_ostream &OS) const override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + virtual LLVM_DUMP_METHOD void dump() const override { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +class GIMatchDagOneOfOpcodesPredicate : public GIMatchDagPredicate { + SmallVector<const CodeGenInstruction *, 4> Instrs; + +public: + GIMatchDagOneOfOpcodesPredicate(GIMatchDagContext &Ctx, StringRef Name); + + void addOpcode(const CodeGenInstruction *Instr) { Instrs.push_back(Instr); } + + static bool classof(const GIMatchDagPredicate *P) { + return P->getKind() == GIMatchDagPredicateKind_OneOfOpcodes; + } + + const SmallVectorImpl<const CodeGenInstruction *> &getInstrs() const { + return Instrs; + } + + void printDescription(raw_ostream &OS) const override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + virtual LLVM_DUMP_METHOD void dump() const override { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +class GIMatchDagSameMOPredicate : public GIMatchDagPredicate { +public: + GIMatchDagSameMOPredicate(GIMatchDagContext &Ctx, StringRef Name); + + static bool classof(const GIMatchDagPredicate *P) { + return P->getKind() == GIMatchDagPredicateKind_SameMO; + } + + void printDescription(raw_ostream &OS) const override; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + virtual LLVM_DUMP_METHOD void dump() const override { print(errs()); } +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagPredicate &N); +raw_ostream &operator<<(raw_ostream &OS, const GIMatchDagOpcodePredicate &N); + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATE_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp new file mode 100644 index 000000000000..2e804de1cd4e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.cpp @@ -0,0 +1,37 @@ +//===- GIMatchDagPredicateDependencyEdge.cpp - Have inputs before check ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchDagPredicateDependencyEdge.h" + +#include "GIMatchDagInstr.h" +#include "GIMatchDagPredicate.h" + +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +LLVM_DUMP_METHOD void +GIMatchDagPredicateDependencyEdge::print(raw_ostream &OS) const { + OS << getRequiredMI()->getName(); + if (getRequiredMO()) + OS << "[" << getRequiredMO()->getName() << "]"; + OS << " ==> " << getPredicate()->getName() << "[" + << getPredicateOp()->getName() << "]"; +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +LLVM_DUMP_METHOD void GIMatchDagPredicateDependencyEdge::dump() const { + print(errs()); +} +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + +raw_ostream &llvm::operator<<(raw_ostream &OS, + const GIMatchDagPredicateDependencyEdge &E) { + E.print(OS); + return OS; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.h new file mode 100644 index 000000000000..865455fe4e4d --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchDagPredicateDependencyEdge.h @@ -0,0 +1,60 @@ +//===- GIMatchDagPredicateDependencyEdge - Ensure predicates have inputs --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATEEDGE_H +#define LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATEEDGE_H + +#include "GIMatchDagOperands.h" + +namespace llvm { +class GIMatchDag; +class GIMatchDagInstr; +class GIMatchDagEdge; +class GIMatchDagPredicate; + +/// Represents a dependency that must be met to evaluate a predicate. +/// +/// Instances of this class objects are owned by the GIMatchDag and are not +/// shareable between instances of GIMatchDag. +class GIMatchDagPredicateDependencyEdge { + /// The MI that must be available in order to test the predicate. + const GIMatchDagInstr *RequiredMI; + /// The MO that must be available in order to test the predicate. May be + /// nullptr when only the MI is required. + const GIMatchDagOperand *RequiredMO; + /// The Predicate that requires information from RequiredMI/RequiredMO. + const GIMatchDagPredicate *Predicate; + /// The Predicate operand that requires information from + /// RequiredMI/RequiredMO. + const GIMatchDagOperand *PredicateOp; + +public: + GIMatchDagPredicateDependencyEdge(const GIMatchDagInstr *RequiredMI, + const GIMatchDagOperand *RequiredMO, + const GIMatchDagPredicate *Predicate, + const GIMatchDagOperand *PredicateOp) + : RequiredMI(RequiredMI), RequiredMO(RequiredMO), Predicate(Predicate), + PredicateOp(PredicateOp) {} + + const GIMatchDagInstr *getRequiredMI() const { return RequiredMI; } + const GIMatchDagOperand *getRequiredMO() const { return RequiredMO; } + const GIMatchDagPredicate *getPredicate() const { return Predicate; } + const GIMatchDagOperand *getPredicateOp() const { return PredicateOp; } + + void print(raw_ostream &OS) const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const; +#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +}; + +raw_ostream &operator<<(raw_ostream &OS, + const GIMatchDagPredicateDependencyEdge &N); + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHDAGPREDICATEEDGE_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp new file mode 100644 index 000000000000..96dc4fc94893 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.cpp @@ -0,0 +1,782 @@ +//===- GIMatchTree.cpp - A decision tree to match GIMatchDag's ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GIMatchTree.h" + +#include "../CodeGenInstruction.h" + +#include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" + +#define DEBUG_TYPE "gimatchtree" + +using namespace llvm; + +void GIMatchTree::writeDOTGraph(raw_ostream &OS) const { + OS << "digraph \"matchtree\" {\n"; + writeDOTGraphNode(OS); + OS << "}\n"; +} + +void GIMatchTree::writeDOTGraphNode(raw_ostream &OS) const { + OS << format(" Node%p", this) << " [shape=record,label=\"{"; + if (Partitioner) { + Partitioner->emitDescription(OS); + OS << "|" << Partitioner->getNumPartitions() << " partitions|"; + } else + OS << "No partitioner|"; + bool IsFullyTraversed = true; + bool IsFullyTested = true; + StringRef Separator = ""; + for (const auto &Leaf : PossibleLeaves) { + OS << Separator << Leaf.getName(); + Separator = ","; + if (!Leaf.isFullyTraversed()) + IsFullyTraversed = false; + if (!Leaf.isFullyTested()) + IsFullyTested = false; + } + if (!Partitioner && !IsFullyTraversed) + OS << "|Not fully traversed"; + if (!Partitioner && !IsFullyTested) { + OS << "|Not fully tested"; + if (IsFullyTraversed) { + for (const GIMatchTreeLeafInfo &Leaf : PossibleLeaves) { + if (Leaf.isFullyTested()) + continue; + OS << "\\n" << Leaf.getName() << ": " << &Leaf; + for (const GIMatchDagPredicate *P : Leaf.untested_predicates()) + OS << *P; + } + } + } + OS << "}\""; + if (!Partitioner && + (!IsFullyTraversed || !IsFullyTested || PossibleLeaves.size() > 1)) + OS << ",color=red"; + OS << "]\n"; + for (const auto &C : Children) + C.writeDOTGraphNode(OS); + writeDOTGraphEdges(OS); +} + +void GIMatchTree::writeDOTGraphEdges(raw_ostream &OS) const { + for (const auto &Child : enumerate(Children)) { + OS << format(" Node%p", this) << " -> " << format("Node%p", &Child.value()) + << " [label=\"#" << Child.index() << " "; + Partitioner->emitPartitionName(OS, Child.index()); + OS << "\"]\n"; + } +} + +GIMatchTreeBuilderLeafInfo::GIMatchTreeBuilderLeafInfo( + GIMatchTreeBuilder &Builder, StringRef Name, unsigned RootIdx, + const GIMatchDag &MatchDag, void *Data) + : Builder(Builder), Info(Name, RootIdx, Data), MatchDag(MatchDag), + InstrNodeToInfo(), + RemainingInstrNodes(BitVector(MatchDag.getNumInstrNodes(), true)), + RemainingEdges(BitVector(MatchDag.getNumEdges(), true)), + RemainingPredicates(BitVector(MatchDag.getNumPredicates(), true)), + TraversableEdges(MatchDag.getNumEdges()), + TestablePredicates(MatchDag.getNumPredicates()) { + // Number all the predicates in this DAG + for (auto &P : enumerate(MatchDag.predicates())) { + PredicateIDs.insert(std::make_pair(P.value(), P.index())); + } + + // Number all the predicate dependencies in this DAG and set up a bitvector + // for each predicate indicating the unsatisfied dependencies. + for (auto &Dep : enumerate(MatchDag.predicate_edges())) { + PredicateDepIDs.insert(std::make_pair(Dep.value(), Dep.index())); + } + UnsatisfiedPredDepsForPred.resize(MatchDag.getNumPredicates(), + BitVector(PredicateDepIDs.size())); + for (auto &Dep : enumerate(MatchDag.predicate_edges())) { + unsigned ID = PredicateIDs.lookup(Dep.value()->getPredicate()); + UnsatisfiedPredDepsForPred[ID].set(Dep.index()); + } +} + +void GIMatchTreeBuilderLeafInfo::declareInstr(const GIMatchDagInstr *Instr, unsigned ID) { + // Record the assignment of this instr to the given ID. + auto InfoI = InstrNodeToInfo.insert(std::make_pair( + Instr, GIMatchTreeInstrInfo(ID, Instr))); + InstrIDToInfo.insert(std::make_pair(ID, &InfoI.first->second)); + + if (Instr == nullptr) + return; + + if (!Instr->getUserAssignedName().empty()) + Info.bindInstrVariable(Instr->getUserAssignedName(), ID); + for (const auto &VarBinding : Instr->user_assigned_operand_names()) + Info.bindOperandVariable(VarBinding.second, ID, VarBinding.first); + + // Clear the bit indicating we haven't visited this instr. + const auto &NodeI = std::find(MatchDag.instr_nodes_begin(), + MatchDag.instr_nodes_end(), Instr); + assert(NodeI != MatchDag.instr_nodes_end() && "Instr isn't in this DAG"); + unsigned InstrIdx = MatchDag.getInstrNodeIdx(NodeI); + RemainingInstrNodes.reset(InstrIdx); + + // When we declare an instruction, we don't expose any traversable edges just + // yet. A partitioner has to check they exist and are registers before they + // are traversable. + + // When we declare an instruction, we potentially activate some predicates. + // Mark the dependencies that are now satisfied as a result of this + // instruction and mark any predicates whose dependencies are fully + // satisfied. + for (auto &Dep : enumerate(MatchDag.predicate_edges())) { + if (Dep.value()->getRequiredMI() == Instr && + Dep.value()->getRequiredMO() == nullptr) { + for (auto &DepsFor : enumerate(UnsatisfiedPredDepsForPred)) { + DepsFor.value().reset(Dep.index()); + if (DepsFor.value().none()) + TestablePredicates.set(DepsFor.index()); + } + } + } +} + +void GIMatchTreeBuilderLeafInfo::declareOperand(unsigned InstrID, + unsigned OpIdx) { + const GIMatchDagInstr *Instr = InstrIDToInfo.lookup(InstrID)->getInstrNode(); + + OperandIDToInfo.insert(std::make_pair( + std::make_pair(InstrID, OpIdx), + GIMatchTreeOperandInfo(Instr, OpIdx))); + + // When an operand becomes reachable, we potentially activate some traversals. + // Record the edges that can now be followed as a result of this + // instruction. + for (auto &E : enumerate(MatchDag.edges())) { + if (E.value()->getFromMI() == Instr && + E.value()->getFromMO()->getIdx() == OpIdx) { + TraversableEdges.set(E.index()); + } + } + + // When an operand becomes reachable, we potentially activate some predicates. + // Clear the dependencies that are now satisfied as a result of this + // operand and activate any predicates whose dependencies are fully + // satisfied. + for (auto &Dep : enumerate(MatchDag.predicate_edges())) { + if (Dep.value()->getRequiredMI() == Instr && Dep.value()->getRequiredMO() && + Dep.value()->getRequiredMO()->getIdx() == OpIdx) { + for (auto &DepsFor : enumerate(UnsatisfiedPredDepsForPred)) { + DepsFor.value().reset(Dep.index()); + if (DepsFor.value().none()) + TestablePredicates.set(DepsFor.index()); + } + } + } +} + +void GIMatchTreeBuilder::addPartitionersForInstr(unsigned InstrIdx) { + // Find the partitioners that can be used now that this node is + // uncovered. Our choices are: + // - Test the opcode + addPartitioner(std::make_unique<GIMatchTreeOpcodePartitioner>(InstrIdx)); +} + +void GIMatchTreeBuilder::addPartitionersForOperand(unsigned InstrID, + unsigned OpIdx) { + LLVM_DEBUG(dbgs() << "Add partitioners for Instrs[" << InstrID + << "].getOperand(" << OpIdx << ")\n"); + addPartitioner( + std::make_unique<GIMatchTreeVRegDefPartitioner>(InstrID, OpIdx)); +} + +void GIMatchTreeBuilder::filterRedundantPartitioners() { + // TODO: Filter partitioners for facts that are already known + // - If we know the opcode, we can elide the num operand check so long as + // the instruction has a fixed number of operands. + // - If we know an exact number of operands then we can elide further number + // of operand checks. + // - If the current min number of operands exceeds the one we want to check + // then we can elide it. +} + +void GIMatchTreeBuilder::evaluatePartitioners() { + // Determine the partitioning the partitioner would produce + for (auto &Partitioner : Partitioners) { + LLVM_DEBUG(dbgs() << " Weighing up "; + Partitioner->emitDescription(dbgs()); dbgs() << "\n"); + Partitioner->repartition(Leaves); + LLVM_DEBUG(Partitioner->emitPartitionResults(dbgs())); + } +} + +void GIMatchTreeBuilder::runStep() { + LLVM_DEBUG(dbgs() << "Building match tree node for " << TreeNode << "\n"); + LLVM_DEBUG(dbgs() << " Rules reachable at this node:\n"); + for (const auto &Leaf : Leaves) { + LLVM_DEBUG(dbgs() << " " << Leaf.getName() << " (" << &Leaf.getInfo() << "\n"); + TreeNode->addPossibleLeaf(Leaf.getInfo(), Leaf.isFullyTraversed(), + Leaf.isFullyTested()); + } + + LLVM_DEBUG(dbgs() << " Partitioners available at this node:\n"); +#ifndef NDEBUG + for (const auto &Partitioner : Partitioners) + LLVM_DEBUG(dbgs() << " "; Partitioner->emitDescription(dbgs()); + dbgs() << "\n"); +#endif // ifndef NDEBUG + + // Check for unreachable rules. Rules are unreachable if they are preceeded by + // a fully tested rule. + // Note: This is only true for the current algorithm, if we allow the + // algorithm to compare equally valid rules then they will become + // reachable. + { + auto FullyTestedLeafI = Leaves.end(); + for (auto LeafI = Leaves.begin(), LeafE = Leaves.end(); + LeafI != LeafE; ++LeafI) { + if (LeafI->isFullyTraversed() && LeafI->isFullyTested()) + FullyTestedLeafI = LeafI; + else if (FullyTestedLeafI != Leaves.end()) { + PrintError("Leaf " + LeafI->getName() + " is unreachable"); + PrintNote("Leaf " + FullyTestedLeafI->getName() + + " will have already matched"); + } + } + } + + LLVM_DEBUG(dbgs() << " Eliminating redundant partitioners:\n"); + filterRedundantPartitioners(); + LLVM_DEBUG(dbgs() << " Partitioners remaining:\n"); +#ifndef NDEBUG + for (const auto &Partitioner : Partitioners) + LLVM_DEBUG(dbgs() << " "; Partitioner->emitDescription(dbgs()); + dbgs() << "\n"); +#endif // ifndef NDEBUG + + if (Partitioners.empty()) { + // Nothing left to do but check we really did identify a single rule. + if (Leaves.size() > 1) { + LLVM_DEBUG(dbgs() << "Leaf contains multiple rules, drop after the first " + "fully tested rule\n"); + auto FirstFullyTested = + std::find_if(Leaves.begin(), Leaves.end(), + [](const GIMatchTreeBuilderLeafInfo &X) { + return X.isFullyTraversed() && X.isFullyTested() && + !X.getMatchDag().hasPostMatchPredicate(); + }); + if (FirstFullyTested != Leaves.end()) + FirstFullyTested++; + +#ifndef NDEBUG + for (auto &Leaf : make_range(Leaves.begin(), FirstFullyTested)) + LLVM_DEBUG(dbgs() << " Kept " << Leaf.getName() << "\n"); + for (const auto &Leaf : make_range(FirstFullyTested, Leaves.end())) + LLVM_DEBUG(dbgs() << " Dropped " << Leaf.getName() << "\n"); +#endif // ifndef NDEBUG + TreeNode->dropLeavesAfter( + std::distance(Leaves.begin(), FirstFullyTested)); + } + for (const auto &Leaf : Leaves) { + if (!Leaf.isFullyTraversed()) { + PrintError("Leaf " + Leaf.getName() + " is not fully traversed"); + PrintNote("This indicates a missing partitioner within tblgen"); + Leaf.dump(errs()); + for (unsigned InstrIdx : Leaf.untested_instrs()) + PrintNote("Instr " + llvm::to_string(*Leaf.getInstr(InstrIdx))); + for (unsigned EdgeIdx : Leaf.untested_edges()) + PrintNote("Edge " + llvm::to_string(*Leaf.getEdge(EdgeIdx))); + } + } + + // Copy out information about untested predicates so the user of the tree + // can deal with them. + for (auto LeafPair : zip(Leaves, TreeNode->possible_leaves())) { + const GIMatchTreeBuilderLeafInfo &BuilderLeaf = std::get<0>(LeafPair); + GIMatchTreeLeafInfo &TreeLeaf = std::get<1>(LeafPair); + if (!BuilderLeaf.isFullyTested()) + for (unsigned PredicateIdx : BuilderLeaf.untested_predicates()) + TreeLeaf.addUntestedPredicate(BuilderLeaf.getPredicate(PredicateIdx)); + } + return; + } + + LLVM_DEBUG(dbgs() << " Weighing up partitioners:\n"); + evaluatePartitioners(); + + // Select the best partitioner by its ability to partition + // - Prefer partitioners that don't distinguish between partitions. This + // is to fail early on decisions that must go a single way. + auto PartitionerI = std::max_element( + Partitioners.begin(), Partitioners.end(), + [](const std::unique_ptr<GIMatchTreePartitioner> &A, + const std::unique_ptr<GIMatchTreePartitioner> &B) { + // We generally want partitioners that subdivide the + // ruleset as much as possible since these take fewer + // checks to converge on a particular rule. However, + // it's important to note that one leaf can end up in + // multiple partitions if the check isn't mutually + // exclusive (e.g. getVRegDef() vs isReg()). + // We therefore minimize average leaves per partition. + return (double)A->getNumLeavesWithDupes() / A->getNumPartitions() > + (double)B->getNumLeavesWithDupes() / B->getNumPartitions(); + }); + + // Select a partitioner and partition the ruleset + // Note that it's possible for a single rule to end up in multiple + // partitions. For example, an opcode test on a rule without an opcode + // predicate will result in it being passed to all partitions. + std::unique_ptr<GIMatchTreePartitioner> Partitioner = std::move(*PartitionerI); + Partitioners.erase(PartitionerI); + LLVM_DEBUG(dbgs() << " Selected partitioner: "; + Partitioner->emitDescription(dbgs()); dbgs() << "\n"); + + assert(Partitioner->getNumPartitions() > 0 && + "Must always partition into at least one partition"); + + TreeNode->setNumChildren(Partitioner->getNumPartitions()); + for (auto &C : enumerate(TreeNode->children())) { + SubtreeBuilders.emplace_back(&C.value(), NextInstrID); + Partitioner->applyForPartition(C.index(), *this, SubtreeBuilders.back()); + } + + TreeNode->setPartitioner(std::move(Partitioner)); + + // Recurse into the subtree builders. Each one must get a copy of the + // remaining partitioners as each path has to check everything. + for (auto &SubtreeBuilder : SubtreeBuilders) { + for (const auto &Partitioner : Partitioners) + SubtreeBuilder.addPartitioner(Partitioner->clone()); + SubtreeBuilder.runStep(); + } +} + +std::unique_ptr<GIMatchTree> GIMatchTreeBuilder::run() { + unsigned NewInstrID = allocInstrID(); + // Start by recording the root instruction as instr #0 and set up the initial + // partitioners. + for (auto &Leaf : Leaves) { + LLVM_DEBUG(Leaf.getMatchDag().writeDOTGraph(dbgs(), Leaf.getName())); + GIMatchDagInstr *Root = + *(Leaf.getMatchDag().roots().begin() + Leaf.getRootIdx()); + Leaf.declareInstr(Root, NewInstrID); + } + + addPartitionersForInstr(NewInstrID); + + std::unique_ptr<GIMatchTree> TreeRoot = std::make_unique<GIMatchTree>(); + TreeNode = TreeRoot.get(); + runStep(); + + return TreeRoot; +} + +void GIMatchTreeOpcodePartitioner::emitPartitionName(raw_ostream &OS, unsigned Idx) const { + if (PartitionToInstr[Idx] == nullptr) { + OS << "* or nullptr"; + return; + } + OS << PartitionToInstr[Idx]->Namespace + << "::" << PartitionToInstr[Idx]->TheDef->getName(); +} + +void GIMatchTreeOpcodePartitioner::repartition( + GIMatchTreeBuilder::LeafVec &Leaves) { + Partitions.clear(); + InstrToPartition.clear(); + PartitionToInstr.clear(); + TestedPredicates.clear(); + + for (const auto &Leaf : enumerate(Leaves)) { + bool AllOpcodes = true; + GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); + BitVector TestedPredicatesForLeaf( + Leaf.value().getMatchDag().getNumPredicates()); + + // If the instruction isn't declared then we don't care about it. Ignore + // it for now and add it to all partitions later once we know what + // partitions we have. + if (!InstrInfo) { + LLVM_DEBUG(dbgs() << " " << Leaf.value().getName() + << " doesn't care about Instr[" << InstrID << "]\n"); + assert(TestedPredicatesForLeaf.size() == Leaf.value().getMatchDag().getNumPredicates()); + TestedPredicates.push_back(TestedPredicatesForLeaf); + continue; + } + + // If the opcode is available to test then any opcode predicates will have + // been enabled too. + for (unsigned PIdx : Leaf.value().TestablePredicates.set_bits()) { + const auto &P = Leaf.value().getPredicate(PIdx); + SmallVector<const CodeGenInstruction *, 1> OpcodesForThisPredicate; + if (const auto *OpcodeP = dyn_cast<const GIMatchDagOpcodePredicate>(P)) { + // We've found _an_ opcode predicate, but we don't know if it's + // checking this instruction yet. + bool IsThisPredicate = false; + for (const auto &PDep : Leaf.value().getMatchDag().predicate_edges()) { + if (PDep->getRequiredMI() == InstrInfo->getInstrNode() && + PDep->getRequiredMO() == nullptr && PDep->getPredicate() == P) { + IsThisPredicate = true; + break; + } + } + if (!IsThisPredicate) + continue; + + // If we get here twice then we've somehow ended up with two opcode + // predicates for one instruction in the same DAG. That should be + // impossible. + assert(AllOpcodes && "Conflicting opcode predicates"); + const CodeGenInstruction *Expected = OpcodeP->getInstr(); + OpcodesForThisPredicate.push_back(Expected); + } + + if (const auto *OpcodeP = + dyn_cast<const GIMatchDagOneOfOpcodesPredicate>(P)) { + // We've found _an_ oneof(opcodes) predicate, but we don't know if it's + // checking this instruction yet. + bool IsThisPredicate = false; + for (const auto &PDep : Leaf.value().getMatchDag().predicate_edges()) { + if (PDep->getRequiredMI() == InstrInfo->getInstrNode() && + PDep->getRequiredMO() == nullptr && PDep->getPredicate() == P) { + IsThisPredicate = true; + break; + } + } + if (!IsThisPredicate) + continue; + + // If we get here twice then we've somehow ended up with two opcode + // predicates for one instruction in the same DAG. That should be + // impossible. + assert(AllOpcodes && "Conflicting opcode predicates"); + for (const CodeGenInstruction *Expected : OpcodeP->getInstrs()) + OpcodesForThisPredicate.push_back(Expected); + } + + for (const CodeGenInstruction *Expected : OpcodesForThisPredicate) { + // Mark this predicate as one we're testing. + TestedPredicatesForLeaf.set(PIdx); + + // Partitions must be numbered 0, 1, .., N but instructions don't meet + // that requirement. Assign a partition number to each opcode if we + // lack one ... + auto Partition = InstrToPartition.find(Expected); + if (Partition == InstrToPartition.end()) { + BitVector Contents(Leaves.size()); + Partition = InstrToPartition + .insert(std::make_pair(Expected, Partitions.size())) + .first; + PartitionToInstr.push_back(Expected); + Partitions.insert(std::make_pair(Partitions.size(), Contents)); + } + // ... and mark this leaf as being in that partition. + Partitions.find(Partition->second)->second.set(Leaf.index()); + AllOpcodes = false; + LLVM_DEBUG(dbgs() << " " << Leaf.value().getName() + << " is in partition " << Partition->second << "\n"); + } + + // TODO: This is where we would handle multiple choices of opcode + // the end result will be that this leaf ends up in multiple + // partitions similarly to AllOpcodes. + } + + // If we never check the opcode, add it to every partition. + if (AllOpcodes) { + // Add a partition for the default case if we don't already have one. + if (InstrToPartition.insert(std::make_pair(nullptr, 0)).second) { + PartitionToInstr.push_back(nullptr); + BitVector Contents(Leaves.size()); + Partitions.insert(std::make_pair(Partitions.size(), Contents)); + } + LLVM_DEBUG(dbgs() << " " << Leaf.value().getName() + << " is in all partitions (opcode not checked)\n"); + for (auto &Partition : Partitions) + Partition.second.set(Leaf.index()); + } + + assert(TestedPredicatesForLeaf.size() == Leaf.value().getMatchDag().getNumPredicates()); + TestedPredicates.push_back(TestedPredicatesForLeaf); + } + + if (Partitions.size() == 0) { + // Add a partition for the default case if we don't already have one. + if (InstrToPartition.insert(std::make_pair(nullptr, 0)).second) { + PartitionToInstr.push_back(nullptr); + BitVector Contents(Leaves.size()); + Partitions.insert(std::make_pair(Partitions.size(), Contents)); + } + } + + // Add any leaves that don't care about this instruction to all partitions. + for (const auto &Leaf : enumerate(Leaves)) { + GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); + if (!InstrInfo) { + // Add a partition for the default case if we don't already have one. + if (InstrToPartition.insert(std::make_pair(nullptr, 0)).second) { + PartitionToInstr.push_back(nullptr); + BitVector Contents(Leaves.size()); + Partitions.insert(std::make_pair(Partitions.size(), Contents)); + } + for (auto &Partition : Partitions) + Partition.second.set(Leaf.index()); + } + } + +} + +void GIMatchTreeOpcodePartitioner::applyForPartition( + unsigned PartitionIdx, GIMatchTreeBuilder &Builder, GIMatchTreeBuilder &SubBuilder) { + LLVM_DEBUG(dbgs() << " Making partition " << PartitionIdx << "\n"); + const CodeGenInstruction *CGI = PartitionToInstr[PartitionIdx]; + + BitVector PossibleLeaves = getPossibleLeavesForPartition(PartitionIdx); + // Consume any predicates we handled. + for (auto &EnumeratedLeaf : enumerate(Builder.getPossibleLeaves())) { + if (!PossibleLeaves[EnumeratedLeaf.index()]) + continue; + + auto &Leaf = EnumeratedLeaf.value(); + const auto &TestedPredicatesForLeaf = + TestedPredicates[EnumeratedLeaf.index()]; + + for (unsigned PredIdx : TestedPredicatesForLeaf.set_bits()) { + LLVM_DEBUG(dbgs() << " " << Leaf.getName() << " tested predicate #" + << PredIdx << " of " << TestedPredicatesForLeaf.size() + << " " << *Leaf.getPredicate(PredIdx) << "\n"); + Leaf.RemainingPredicates.reset(PredIdx); + Leaf.TestablePredicates.reset(PredIdx); + } + SubBuilder.addLeaf(Leaf); + } + + // Nothing to do, we don't know anything about this instruction as a result + // of this partitioner. + if (CGI == nullptr) + return; + + GIMatchTreeBuilder::LeafVec &NewLeaves = SubBuilder.getPossibleLeaves(); + // Find all the operands we know to exist and are referenced. This will + // usually be all the referenced operands but there are some cases where + // instructions are variadic. Such operands must be handled by partitioners + // that check the number of operands. + BitVector ReferencedOperands(1); + for (auto &Leaf : NewLeaves) { + GIMatchTreeInstrInfo *InstrInfo = Leaf.getInstrInfo(InstrID); + // Skip any leaves that don't care about this instruction. + if (!InstrInfo) + continue; + const GIMatchDagInstr *Instr = InstrInfo->getInstrNode(); + for (auto &E : enumerate(Leaf.getMatchDag().edges())) { + if (E.value()->getFromMI() == Instr && + E.value()->getFromMO()->getIdx() < CGI->Operands.size()) { + ReferencedOperands.resize(E.value()->getFromMO()->getIdx() + 1); + ReferencedOperands.set(E.value()->getFromMO()->getIdx()); + } + } + } + for (auto &Leaf : NewLeaves) { + for (unsigned OpIdx : ReferencedOperands.set_bits()) { + Leaf.declareOperand(InstrID, OpIdx); + } + } + for (unsigned OpIdx : ReferencedOperands.set_bits()) { + SubBuilder.addPartitionersForOperand(InstrID, OpIdx); + } +} + +void GIMatchTreeOpcodePartitioner::emitPartitionResults( + raw_ostream &OS) const { + OS << "Partitioning by opcode would produce " << Partitions.size() + << " partitions\n"; + for (const auto &Partition : InstrToPartition) { + if (Partition.first == nullptr) + OS << "Default: "; + else + OS << Partition.first->TheDef->getName() << ": "; + StringRef Separator = ""; + for (unsigned I : Partitions.find(Partition.second)->second.set_bits()) { + OS << Separator << I; + Separator = ", "; + } + OS << "\n"; + } +} + +void GIMatchTreeOpcodePartitioner::generatePartitionSelectorCode( + raw_ostream &OS, StringRef Indent) const { + // Make sure not to emit empty switch or switch with just default + if (PartitionToInstr.size() == 1 && PartitionToInstr[0] == nullptr) { + OS << Indent << "Partition = 0;\n"; + } else if (PartitionToInstr.size()) { + OS << Indent << "Partition = -1;\n" + << Indent << "switch (MIs[" << InstrID << "]->getOpcode()) {\n"; + for (const auto &EnumInstr : enumerate(PartitionToInstr)) { + if (EnumInstr.value() == nullptr) + OS << Indent << "default:"; + else + OS << Indent << "case " << EnumInstr.value()->Namespace + << "::" << EnumInstr.value()->TheDef->getName() << ":"; + OS << " Partition = " << EnumInstr.index() << "; break;\n"; + } + OS << Indent << "}\n"; + } + OS << Indent + << "// Default case but without conflicting with potential default case " + "in selection.\n" + << Indent << "if (Partition == -1) return false;\n"; +} + +void GIMatchTreeVRegDefPartitioner::addToPartition(bool Result, + unsigned LeafIdx) { + auto I = ResultToPartition.find(Result); + if (I == ResultToPartition.end()) { + ResultToPartition.insert(std::make_pair(Result, PartitionToResult.size())); + PartitionToResult.push_back(Result); + } + I = ResultToPartition.find(Result); + auto P = Partitions.find(I->second); + if (P == Partitions.end()) + P = Partitions.insert(std::make_pair(I->second, BitVector())).first; + P->second.resize(LeafIdx + 1); + P->second.set(LeafIdx); +} + +void GIMatchTreeVRegDefPartitioner::repartition( + GIMatchTreeBuilder::LeafVec &Leaves) { + Partitions.clear(); + + for (const auto &Leaf : enumerate(Leaves)) { + GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); + BitVector TraversedEdgesForLeaf(Leaf.value().getMatchDag().getNumEdges()); + + // If the instruction isn't declared then we don't care about it. Ignore + // it for now and add it to all partitions later once we know what + // partitions we have. + if (!InstrInfo) { + TraversedEdges.push_back(TraversedEdgesForLeaf); + continue; + } + + // If this node has an use -> def edge from this operand then this + // instruction must be in partition 1 (isVRegDef()). + bool WantsEdge = false; + for (unsigned EIdx : Leaf.value().TraversableEdges.set_bits()) { + const auto &E = Leaf.value().getEdge(EIdx); + if (E->getFromMI() != InstrInfo->getInstrNode() || + E->getFromMO()->getIdx() != OpIdx || E->isDefToUse()) + continue; + + // We're looking at the right edge. This leaf wants a vreg def so we'll + // put it in partition 1. + addToPartition(true, Leaf.index()); + TraversedEdgesForLeaf.set(EIdx); + WantsEdge = true; + } + + bool isNotReg = false; + if (!WantsEdge && isNotReg) { + // If this leaf doesn't have an edge and we _don't_ want a register, + // then add it to partition 0. + addToPartition(false, Leaf.index()); + } else if (!WantsEdge) { + // If this leaf doesn't have an edge and we don't know what we want, + // then add it to partition 0 and 1. + addToPartition(false, Leaf.index()); + addToPartition(true, Leaf.index()); + } + + TraversedEdges.push_back(TraversedEdgesForLeaf); + } + + // Add any leaves that don't care about this instruction to all partitions. + for (const auto &Leaf : enumerate(Leaves)) { + GIMatchTreeInstrInfo *InstrInfo = Leaf.value().getInstrInfo(InstrID); + if (!InstrInfo) + for (auto &Partition : Partitions) + Partition.second.set(Leaf.index()); + } +} + +void GIMatchTreeVRegDefPartitioner::applyForPartition( + unsigned PartitionIdx, GIMatchTreeBuilder &Builder, + GIMatchTreeBuilder &SubBuilder) { + BitVector PossibleLeaves = getPossibleLeavesForPartition(PartitionIdx); + + std::vector<BitVector> TraversedEdgesByNewLeaves; + // Consume any edges we handled. + for (auto &EnumeratedLeaf : enumerate(Builder.getPossibleLeaves())) { + if (!PossibleLeaves[EnumeratedLeaf.index()]) + continue; + + auto &Leaf = EnumeratedLeaf.value(); + const auto &TraversedEdgesForLeaf = TraversedEdges[EnumeratedLeaf.index()]; + TraversedEdgesByNewLeaves.push_back(TraversedEdgesForLeaf); + Leaf.RemainingEdges.reset(TraversedEdgesForLeaf); + Leaf.TraversableEdges.reset(TraversedEdgesForLeaf); + SubBuilder.addLeaf(Leaf); + } + + // Nothing to do. The only thing we know is that it isn't a vreg-def. + if (PartitionToResult[PartitionIdx] == false) + return; + + NewInstrID = SubBuilder.allocInstrID(); + + GIMatchTreeBuilder::LeafVec &NewLeaves = SubBuilder.getPossibleLeaves(); + for (const auto I : zip(NewLeaves, TraversedEdgesByNewLeaves)) { + auto &Leaf = std::get<0>(I); + auto &TraversedEdgesForLeaf = std::get<1>(I); + GIMatchTreeInstrInfo *InstrInfo = Leaf.getInstrInfo(InstrID); + // Skip any leaves that don't care about this instruction. + if (!InstrInfo) + continue; + for (unsigned EIdx : TraversedEdgesForLeaf.set_bits()) { + const GIMatchDagEdge *E = Leaf.getEdge(EIdx); + Leaf.declareInstr(E->getToMI(), NewInstrID); + } + } + SubBuilder.addPartitionersForInstr(NewInstrID); +} + +void GIMatchTreeVRegDefPartitioner::emitPartitionResults( + raw_ostream &OS) const { + OS << "Partitioning by vreg-def would produce " << Partitions.size() + << " partitions\n"; + for (const auto &Partition : Partitions) { + OS << Partition.first << " ("; + emitPartitionName(OS, Partition.first); + OS << "): "; + StringRef Separator = ""; + for (unsigned I : Partition.second.set_bits()) { + OS << Separator << I; + Separator = ", "; + } + OS << "\n"; + } +} + +void GIMatchTreeVRegDefPartitioner::generatePartitionSelectorCode( + raw_ostream &OS, StringRef Indent) const { + OS << Indent << "Partition = -1\n" + << Indent << "if (MIs.size() <= NewInstrID) MIs.resize(NewInstrID + 1);\n" + << Indent << "MIs[" << NewInstrID << "] = nullptr;\n" + << Indent << "if (MIs[" << InstrID << "].getOperand(" << OpIdx + << ").isReg()))\n" + << Indent << " MIs[" << NewInstrID << "] = MRI.getVRegDef(MIs[" << InstrID + << "].getOperand(" << OpIdx << ").getReg()));\n"; + + for (const auto &Pair : ResultToPartition) + OS << Indent << "if (MIs[" << NewInstrID << "] " + << (Pair.first ? "==" : "!=") + << " nullptr) Partition = " << Pair.second << ";\n"; + + OS << Indent << "if (Partition == -1) return false;\n"; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.h b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.h new file mode 100644 index 000000000000..b86f6454589c --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISel/GIMatchTree.h @@ -0,0 +1,629 @@ +//===- GIMatchTree.h - A decision tree to match GIMatchDag's --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_GIMATCHTREE_H +#define LLVM_UTILS_TABLEGEN_GIMATCHTREE_H + +#include "GIMatchDag.h" +#include "llvm/ADT/BitVector.h" + +namespace llvm { +class raw_ostream; + +class GIMatchTreeBuilder; +class GIMatchTreePartitioner; + +/// Describes the binding of a variable to the matched MIR +class GIMatchTreeVariableBinding { + /// The name of the variable described by this binding. + StringRef Name; + // The matched instruction it is bound to. + unsigned InstrID; + // The matched operand (if appropriate) it is bound to. + Optional<unsigned> OpIdx; + +public: + GIMatchTreeVariableBinding(StringRef Name, unsigned InstrID, + Optional<unsigned> OpIdx = None) + : Name(Name), InstrID(InstrID), OpIdx(OpIdx) {} + + bool isInstr() const { return !OpIdx.hasValue(); } + StringRef getName() const { return Name; } + unsigned getInstrID() const { return InstrID; } + unsigned getOpIdx() const { + assert(OpIdx.hasValue() && "Is not an operand binding"); + return *OpIdx; + } +}; + +/// Associates a matchable with a leaf of the decision tree. +class GIMatchTreeLeafInfo { +public: + using const_var_binding_iterator = + std::vector<GIMatchTreeVariableBinding>::const_iterator; + using UntestedPredicatesTy = SmallVector<const GIMatchDagPredicate *, 1>; + using const_untested_predicates_iterator = UntestedPredicatesTy::const_iterator; + +protected: + /// A name for the matchable. This is primarily for debugging. + StringRef Name; + /// Where rules have multiple roots, this is which root we're starting from. + unsigned RootIdx; + /// Opaque data the caller of the tree building code understands. + void *Data; + /// Has the decision tree covered every edge traversal? If it hasn't then this + /// is an unrecoverable error indicating there's something wrong with the + /// partitioners. + bool IsFullyTraversed; + /// Has the decision tree covered every predicate test? If it has, then + /// subsequent matchables on the same leaf are unreachable. If it hasn't, the + /// code that requested the GIMatchTree is responsible for finishing off any + /// remaining predicates. + bool IsFullyTested; + /// The variable bindings associated with this leaf so far. + std::vector<GIMatchTreeVariableBinding> VarBindings; + /// Any predicates left untested by the time we reach this leaf. + UntestedPredicatesTy UntestedPredicates; + +public: + GIMatchTreeLeafInfo() { llvm_unreachable("Cannot default-construct"); } + GIMatchTreeLeafInfo(StringRef Name, unsigned RootIdx, void *Data) + : Name(Name), RootIdx(RootIdx), Data(Data), IsFullyTraversed(false), + IsFullyTested(false) {} + + StringRef getName() const { return Name; } + unsigned getRootIdx() const { return RootIdx; } + template <class Ty> Ty *getTargetData() const { + return static_cast<Ty *>(Data); + } + bool isFullyTraversed() const { return IsFullyTraversed; } + void setIsFullyTraversed(bool V) { IsFullyTraversed = V; } + bool isFullyTested() const { return IsFullyTested; } + void setIsFullyTested(bool V) { IsFullyTested = V; } + + void bindInstrVariable(StringRef Name, unsigned InstrID) { + VarBindings.emplace_back(Name, InstrID); + } + void bindOperandVariable(StringRef Name, unsigned InstrID, unsigned OpIdx) { + VarBindings.emplace_back(Name, InstrID, OpIdx); + } + + const_var_binding_iterator var_bindings_begin() const { + return VarBindings.begin(); + } + const_var_binding_iterator var_bindings_end() const { + return VarBindings.end(); + } + iterator_range<const_var_binding_iterator> var_bindings() const { + return make_range(VarBindings.begin(), VarBindings.end()); + } + iterator_range<const_untested_predicates_iterator> untested_predicates() const { + return make_range(UntestedPredicates.begin(), UntestedPredicates.end()); + } + void addUntestedPredicate(const GIMatchDagPredicate *P) { + UntestedPredicates.push_back(P); + } +}; + +/// The nodes of a decision tree used to perform the match. +/// This will be used to generate the C++ code or state machine equivalent. +/// +/// It should be noted that some nodes of this tree (most notably nodes handling +/// def -> use edges) will need to iterate over several possible matches. As +/// such, code generated from this will sometimes need to support backtracking. +class GIMatchTree { + using LeafVector = std::vector<GIMatchTreeLeafInfo>; + + /// The partitioner that has been chosen for this node. This may be nullptr if + /// a partitioner hasn't been chosen yet or if the node is a leaf. + std::unique_ptr<GIMatchTreePartitioner> Partitioner; + /// All the leaves that are possible for this node of the tree. + /// Note: This should be emptied after the tree is built when there are + /// children but this currently isn't done to aid debuggability of the DOT + /// graph for the decision tree. + LeafVector PossibleLeaves; + /// The children of this node. The index into this array must match the index + /// chosen by the partitioner. + std::vector<GIMatchTree> Children; + + void writeDOTGraphNode(raw_ostream &OS) const; + void writeDOTGraphEdges(raw_ostream &OS) const; + +public: + void writeDOTGraph(raw_ostream &OS) const; + + void setNumChildren(unsigned Num) { Children.resize(Num); } + void addPossibleLeaf(const GIMatchTreeLeafInfo &V, bool IsFullyTraversed, + bool IsFullyTested) { + PossibleLeaves.push_back(V); + PossibleLeaves.back().setIsFullyTraversed(IsFullyTraversed); + PossibleLeaves.back().setIsFullyTested(IsFullyTested); + } + void dropLeavesAfter(size_t Length) { + if (PossibleLeaves.size() > Length) + PossibleLeaves.resize(Length); + } + void setPartitioner(std::unique_ptr<GIMatchTreePartitioner> &&V) { + Partitioner = std::move(V); + } + GIMatchTreePartitioner *getPartitioner() const { return Partitioner.get(); } + + std::vector<GIMatchTree>::iterator children_begin() { + return Children.begin(); + } + std::vector<GIMatchTree>::iterator children_end() { return Children.end(); } + iterator_range<std::vector<GIMatchTree>::iterator> children() { + return make_range(children_begin(), children_end()); + } + std::vector<GIMatchTree>::const_iterator children_begin() const { + return Children.begin(); + } + std::vector<GIMatchTree>::const_iterator children_end() const { + return Children.end(); + } + iterator_range<std::vector<GIMatchTree>::const_iterator> children() const { + return make_range(children_begin(), children_end()); + } + + LeafVector::const_iterator possible_leaves_begin() const { + return PossibleLeaves.begin(); + } + LeafVector::const_iterator possible_leaves_end() const { + return PossibleLeaves.end(); + } + iterator_range<LeafVector::const_iterator> + possible_leaves() const { + return make_range(possible_leaves_begin(), possible_leaves_end()); + } + LeafVector::iterator possible_leaves_begin() { + return PossibleLeaves.begin(); + } + LeafVector::iterator possible_leaves_end() { + return PossibleLeaves.end(); + } + iterator_range<LeafVector::iterator> possible_leaves() { + return make_range(possible_leaves_begin(), possible_leaves_end()); + } +}; + +/// Record information that is known about the instruction bound to this ID and +/// GIMatchDagInstrNode. Every rule gets its own set of +/// GIMatchTreeInstrInfo to bind the shared IDs to an instr node in its +/// DAG. +/// +/// For example, if we know that there are 3 operands. We can record it here to +/// elide duplicate checks. +class GIMatchTreeInstrInfo { + /// The instruction ID for the matched instruction. + unsigned ID; + /// The corresponding instruction node in the MatchDAG. + const GIMatchDagInstr *InstrNode; + +public: + GIMatchTreeInstrInfo(unsigned ID, const GIMatchDagInstr *InstrNode) + : ID(ID), InstrNode(InstrNode) {} + + unsigned getID() const { return ID; } + const GIMatchDagInstr *getInstrNode() const { return InstrNode; } +}; + +/// Record information that is known about the operand bound to this ID, OpIdx, +/// and GIMatchDagInstrNode. Every rule gets its own set of +/// GIMatchTreeOperandInfo to bind the shared IDs to an operand of an +/// instr node from its DAG. +/// +/// For example, if we know that there the operand is a register. We can record +/// it here to elide duplicate checks. +class GIMatchTreeOperandInfo { + /// The corresponding instruction node in the MatchDAG that the operand + /// belongs to. + const GIMatchDagInstr *InstrNode; + unsigned OpIdx; + +public: + GIMatchTreeOperandInfo(const GIMatchDagInstr *InstrNode, unsigned OpIdx) + : InstrNode(InstrNode), OpIdx(OpIdx) {} + + const GIMatchDagInstr *getInstrNode() const { return InstrNode; } + unsigned getOpIdx() const { return OpIdx; } +}; + +/// Represent a leaf of the match tree and any working data we need to build the +/// tree. +/// +/// It's important to note that each rule can have multiple +/// GIMatchTreeBuilderLeafInfo's since the partitioners do not always partition +/// into mutually-exclusive partitions. For example: +/// R1: (FOO ..., ...) +/// R2: (oneof(FOO, BAR) ..., ...) +/// will partition by opcode into two partitions FOO=>[R1, R2], and BAR=>[R2] +/// +/// As an optimization, all instructions, edges, and predicates in the DAGs are +/// numbered and tracked in BitVectors. As such, the GIMatchDAG must not be +/// modified once construction of the tree has begun. +class GIMatchTreeBuilderLeafInfo { +protected: + GIMatchTreeBuilder &Builder; + GIMatchTreeLeafInfo Info; + const GIMatchDag &MatchDag; + /// The association between GIMatchDagInstr* and GIMatchTreeInstrInfo. + /// The primary reason for this members existence is to allow the use of + /// InstrIDToInfo.lookup() since that requires that the value is + /// default-constructible. + DenseMap<const GIMatchDagInstr *, GIMatchTreeInstrInfo> InstrNodeToInfo; + /// The instruction information for a given ID in the context of this + /// particular leaf. + DenseMap<unsigned, GIMatchTreeInstrInfo *> InstrIDToInfo; + /// The operand information for a given ID and OpIdx in the context of this + /// particular leaf. + DenseMap<std::pair<unsigned, unsigned>, GIMatchTreeOperandInfo> + OperandIDToInfo; + +public: + /// The remaining instrs/edges/predicates to visit + BitVector RemainingInstrNodes; + BitVector RemainingEdges; + BitVector RemainingPredicates; + + // The remaining predicate dependencies for each predicate + std::vector<BitVector> UnsatisfiedPredDepsForPred; + + /// The edges/predicates we can visit as a result of the declare*() calls we + /// have already made. We don't need an instrs version since edges imply the + /// instr. + BitVector TraversableEdges; + BitVector TestablePredicates; + + /// Map predicates from the DAG to their position in the DAG predicate + /// iterators. + DenseMap<GIMatchDagPredicate *, unsigned> PredicateIDs; + /// Map predicate dependency edges from the DAG to their position in the DAG + /// predicate dependency iterators. + DenseMap<GIMatchDagPredicateDependencyEdge *, unsigned> PredicateDepIDs; + +public: + GIMatchTreeBuilderLeafInfo(GIMatchTreeBuilder &Builder, StringRef Name, + unsigned RootIdx, const GIMatchDag &MatchDag, + void *Data); + + StringRef getName() const { return Info.getName(); } + GIMatchTreeLeafInfo &getInfo() { return Info; } + const GIMatchTreeLeafInfo &getInfo() const { return Info; } + const GIMatchDag &getMatchDag() const { return MatchDag; } + unsigned getRootIdx() const { return Info.getRootIdx(); } + + /// Has this DAG been fully traversed. This must be true by the time the tree + /// builder finishes. + bool isFullyTraversed() const { + // We don't need UnsatisfiedPredDepsForPred because RemainingPredicates + // can't be all-zero without satisfying all the dependencies. The same is + // almost true for Edges and Instrs but it's possible to have Instrs without + // Edges. + return RemainingInstrNodes.none() && RemainingEdges.none(); + } + + /// Has this DAG been fully tested. This hould be true by the time the tree + /// builder finishes but clients can finish any untested predicates left over + /// if it's not true. + bool isFullyTested() const { + // We don't need UnsatisfiedPredDepsForPred because RemainingPredicates + // can't be all-zero without satisfying all the dependencies. The same is + // almost true for Edges and Instrs but it's possible to have Instrs without + // Edges. + return RemainingInstrNodes.none() && RemainingEdges.none() && + RemainingPredicates.none(); + } + + const GIMatchDagInstr *getInstr(unsigned Idx) const { + return *(MatchDag.instr_nodes_begin() + Idx); + } + const GIMatchDagEdge *getEdge(unsigned Idx) const { + return *(MatchDag.edges_begin() + Idx); + } + GIMatchDagEdge *getEdge(unsigned Idx) { + return *(MatchDag.edges_begin() + Idx); + } + const GIMatchDagPredicate *getPredicate(unsigned Idx) const { + return *(MatchDag.predicates_begin() + Idx); + } + iterator_range<llvm::BitVector::const_set_bits_iterator> + untested_instrs() const { + return RemainingInstrNodes.set_bits(); + } + iterator_range<llvm::BitVector::const_set_bits_iterator> + untested_edges() const { + return RemainingEdges.set_bits(); + } + iterator_range<llvm::BitVector::const_set_bits_iterator> + untested_predicates() const { + return RemainingPredicates.set_bits(); + } + + /// Bind an instr node to the given ID and clear any blocking dependencies + /// that were waiting for it. + void declareInstr(const GIMatchDagInstr *Instr, unsigned ID); + + /// Bind an operand to the given ID and OpIdx and clear any blocking + /// dependencies that were waiting for it. + void declareOperand(unsigned InstrID, unsigned OpIdx); + + GIMatchTreeInstrInfo *getInstrInfo(unsigned ID) const { + auto I = InstrIDToInfo.find(ID); + if (I != InstrIDToInfo.end()) + return I->second; + return nullptr; + } + + void dump(raw_ostream &OS) const { + OS << "Leaf " << getName() << " for root #" << getRootIdx() << "\n"; + MatchDag.print(OS); + for (const auto &I : InstrIDToInfo) + OS << "Declared Instr #" << I.first << "\n"; + for (const auto &I : OperandIDToInfo) + OS << "Declared Instr #" << I.first.first << ", Op #" << I.first.second + << "\n"; + OS << RemainingInstrNodes.count() << " untested instrs of " + << RemainingInstrNodes.size() << "\n"; + OS << RemainingEdges.count() << " untested edges of " + << RemainingEdges.size() << "\n"; + OS << RemainingPredicates.count() << " untested predicates of " + << RemainingPredicates.size() << "\n"; + + OS << TraversableEdges.count() << " edges could be traversed\n"; + OS << TestablePredicates.count() << " predicates could be tested\n"; + } +}; + +/// The tree builder has a fairly tough job. It's purpose is to merge all the +/// DAGs from the ruleset into a decision tree that walks all of them +/// simultaneously and identifies the rule that was matched. In addition to +/// that, it also needs to find the most efficient order to make decisions +/// without violating any dependencies and ensure that every DAG covers every +/// instr/edge/predicate. +class GIMatchTreeBuilder { +public: + using LeafVec = std::vector<GIMatchTreeBuilderLeafInfo>; + +protected: + /// The leaves that the resulting decision tree will distinguish. + LeafVec Leaves; + /// The tree node being constructed. + GIMatchTree *TreeNode; + /// The builders for each subtree resulting from the current decision. + std::vector<GIMatchTreeBuilder> SubtreeBuilders; + /// The possible partitioners we could apply right now. + std::vector<std::unique_ptr<GIMatchTreePartitioner>> Partitioners; + /// The next instruction ID to allocate when requested by the chosen + /// Partitioner. + unsigned NextInstrID; + + /// Use any context we have stored to cull partitioners that only test things + /// we already know. At the time of writing, there's no need to do anything + /// here but it will become important once, for example, there is a + /// num-operands and an opcode partitioner. This is because applying an opcode + /// partitioner (usually) makes the number of operands known which makes + /// additional checking pointless. + void filterRedundantPartitioners(); + + /// Evaluate the available partioners and select the best one at the moment. + void evaluatePartitioners(); + + /// Construct the current tree node. + void runStep(); + +public: + GIMatchTreeBuilder(unsigned NextInstrID) : NextInstrID(NextInstrID) {} + GIMatchTreeBuilder(GIMatchTree *TreeNode, unsigned NextInstrID) + : TreeNode(TreeNode), NextInstrID(NextInstrID) {} + + void addLeaf(StringRef Name, unsigned RootIdx, const GIMatchDag &MatchDag, + void *Data) { + Leaves.emplace_back(*this, Name, RootIdx, MatchDag, Data); + } + void addLeaf(const GIMatchTreeBuilderLeafInfo &L) { Leaves.push_back(L); } + void addPartitioner(std::unique_ptr<GIMatchTreePartitioner> P) { + Partitioners.push_back(std::move(P)); + } + void addPartitionersForInstr(unsigned InstrIdx); + void addPartitionersForOperand(unsigned InstrID, unsigned OpIdx); + + LeafVec &getPossibleLeaves() { return Leaves; } + + unsigned allocInstrID() { return NextInstrID++; } + + /// Construct the decision tree. + std::unique_ptr<GIMatchTree> run(); +}; + +/// Partitioners are the core of the tree builder and are unfortunately rather +/// tricky to write. +class GIMatchTreePartitioner { +protected: + /// The partitions resulting from applying the partitioner to the possible + /// leaves. The keys must be consecutive integers starting from 0. This can + /// lead to some unfortunate situations where partitioners test a predicate + /// and use 0 for success and 1 for failure if the ruleset encounters a + /// success case first but is necessary to assign the partition to one of the + /// tree nodes children. As a result, you usually need some kind of + /// indirection to map the natural keys (e.g. ptrs/bools) to this linear + /// sequence. The values are a bitvector indicating which leaves belong to + /// this partition. + DenseMap<unsigned, BitVector> Partitions; + +public: + virtual ~GIMatchTreePartitioner() {} + virtual std::unique_ptr<GIMatchTreePartitioner> clone() const = 0; + + /// Determines which partitions the given leaves belong to. A leaf may belong + /// to multiple partitions in which case it will be duplicated during + /// applyForPartition(). + /// + /// This function can be rather complicated. A few particular things to be + /// aware of include: + /// * One leaf can be assigned to multiple partitions when there's some + /// ambiguity. + /// * Not all DAG's for the leaves may be able to perform the test. For + /// example, the opcode partitiioner must account for one DAG being a + /// superset of another such as [(ADD ..., ..., ...)], and [(MUL t, ..., + /// ...), (ADD ..., t, ...)] + /// * Attaching meaning to a particular partition index will generally not + /// work due to the '0, 1, ..., n' requirement. You might encounter cases + /// where only partition 1 is seen, leaving a missing 0. + /// * Finding a specific predicate such as the opcode predicate for a specific + /// instruction is non-trivial. It's often O(NumPredicates), leading to + /// O(NumPredicates*NumRules) when applied to the whole ruleset. The good + /// news there is that n is typically small thanks to predicate dependencies + /// limiting how many are testable at once. Also, with opcode and type + /// predicates being so frequent the value of m drops very fast too. It + /// wouldn't be terribly surprising to see a 10k ruleset drop down to an + /// average of 100 leaves per partition after a single opcode partitioner. + /// * The same goes for finding specific edges. The need to traverse them in + /// dependency order dramatically limits the search space at any given + /// moment. + /// * If you need to add a leaf to all partitions, make sure you don't forget + /// them when adding partitions later. + virtual void repartition(GIMatchTreeBuilder::LeafVec &Leaves) = 0; + + /// Delegate the leaves for a given partition to the corresponding subbuilder, + /// update any recorded context for this partition (e.g. allocate instr id's + /// for instrs recorder by the current node), and clear any blocking + /// dependencies this partitioner resolved. + virtual void applyForPartition(unsigned PartitionIdx, + GIMatchTreeBuilder &Builder, + GIMatchTreeBuilder &SubBuilder) = 0; + + /// Return a BitVector indicating which leaves should be transferred to the + /// specified partition. Note that the same leaf can be indicated for multiple + /// partitions. + BitVector getPossibleLeavesForPartition(unsigned Idx) { + const auto &I = Partitions.find(Idx); + assert(I != Partitions.end() && "Requested non-existant partition"); + return I->second; + } + + size_t getNumPartitions() const { return Partitions.size(); } + size_t getNumLeavesWithDupes() const { + size_t S = 0; + for (const auto &P : Partitions) + S += P.second.size(); + return S; + } + + /// Emit a brief description of the partitioner suitable for debug printing or + /// use in a DOT graph. + virtual void emitDescription(raw_ostream &OS) const = 0; + /// Emit a label for the given partition suitable for debug printing or use in + /// a DOT graph. + virtual void emitPartitionName(raw_ostream &OS, unsigned Idx) const = 0; + + /// Emit a long description of how the partitioner partitions the leaves. + virtual void emitPartitionResults(raw_ostream &OS) const = 0; + + /// Generate code to select between partitions based on the MIR being matched. + /// This is typically a switch statement that picks a partition index. + virtual void generatePartitionSelectorCode(raw_ostream &OS, + StringRef Indent) const = 0; +}; + +/// Partition according to the opcode of the instruction. +/// +/// Numbers CodeGenInstr ptrs for use as partition ID's. One special partition, +/// nullptr, represents the case where the instruction isn't known. +/// +/// * If the opcode can be tested and is a single opcode, create the partition +/// for that opcode and assign the leaf to it. This partition no longer needs +/// to test the opcode, and many details about the instruction will usually +/// become known (e.g. number of operands for non-variadic instrs) via the +/// CodeGenInstr ptr. +/// * (not implemented yet) If the opcode can be tested and is a choice of +/// opcodes, then the leaf can be treated like the single-opcode case but must +/// be added to all relevant partitions and not quite as much becomes known as +/// a result. That said, multiple-choice opcodes are likely similar enough +/// (because if they aren't then handling them together makes little sense) +/// that plenty still becomes known. The main implementation issue with this +/// is having a description to represent the commonality between instructions. +/// * If the opcode is not tested, the leaf must be added to all partitions +/// including the wildcard nullptr partition. What becomes known as a result +/// varies between partitions. +/// * If the instruction to be tested is not declared then add the leaf to all +/// partitions. This occurs when we encounter one rule that is a superset of +/// the other and we are still matching the remainder of the superset. The +/// result is that the cases that don't match the superset will match the +/// subset rule, while the ones that do match the superset will match either +/// (which one is algorithm dependent but will usually be the superset). +class GIMatchTreeOpcodePartitioner : public GIMatchTreePartitioner { + unsigned InstrID; + DenseMap<const CodeGenInstruction *, unsigned> InstrToPartition; + std::vector<const CodeGenInstruction *> PartitionToInstr; + std::vector<BitVector> TestedPredicates; + +public: + GIMatchTreeOpcodePartitioner(unsigned InstrID) : InstrID(InstrID) {} + + std::unique_ptr<GIMatchTreePartitioner> clone() const override { + return std::make_unique<GIMatchTreeOpcodePartitioner>(*this); + } + + void emitDescription(raw_ostream &OS) const override { + OS << "MI[" << InstrID << "].getOpcode()"; + } + + void emitPartitionName(raw_ostream &OS, unsigned Idx) const override; + + void repartition(GIMatchTreeBuilder::LeafVec &Leaves) override; + void applyForPartition(unsigned Idx, GIMatchTreeBuilder &SubBuilder, + GIMatchTreeBuilder &Builder) override; + + void emitPartitionResults(raw_ostream &OS) const override; + + void generatePartitionSelectorCode(raw_ostream &OS, + StringRef Indent) const override; +}; + +class GIMatchTreeVRegDefPartitioner : public GIMatchTreePartitioner { + unsigned NewInstrID = -1; + unsigned InstrID; + unsigned OpIdx; + std::vector<BitVector> TraversedEdges; + DenseMap<unsigned, unsigned> ResultToPartition; + std::vector<bool> PartitionToResult; + + void addToPartition(bool Result, unsigned LeafIdx); + +public: + GIMatchTreeVRegDefPartitioner(unsigned InstrID, unsigned OpIdx) + : InstrID(InstrID), OpIdx(OpIdx) {} + + std::unique_ptr<GIMatchTreePartitioner> clone() const override { + return std::make_unique<GIMatchTreeVRegDefPartitioner>(*this); + } + + void emitDescription(raw_ostream &OS) const override { + OS << "MI[" << NewInstrID << "] = getVRegDef(MI[" << InstrID + << "].getOperand(" << OpIdx << "))"; + } + + void emitPartitionName(raw_ostream &OS, unsigned Idx) const override { + bool Result = PartitionToResult[Idx]; + if (Result) + OS << "true"; + else + OS << "false"; + } + + void repartition(GIMatchTreeBuilder::LeafVec &Leaves) override; + void applyForPartition(unsigned PartitionIdx, GIMatchTreeBuilder &Builder, + GIMatchTreeBuilder &SubBuilder) override; + void emitPartitionResults(raw_ostream &OS) const override; + + void generatePartitionSelectorCode(raw_ostream &OS, + StringRef Indent) const override; +}; + +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_GIMATCHTREE_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp new file mode 100644 index 000000000000..4e8dcc52fc20 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -0,0 +1,5858 @@ +//===- GlobalISelEmitter.cpp - Generate an instruction selector -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// This tablegen backend emits code for use by the GlobalISel instruction +/// selector. See include/llvm/CodeGen/TargetGlobalISel.td. +/// +/// This file analyzes the patterns recognized by the SelectionDAGISel tablegen +/// backend, filters out the ones that are unsupported, maps +/// SelectionDAG-specific constructs to their GlobalISel counterpart +/// (when applicable: MVT to LLT; SDNode to generic Instruction). +/// +/// Not all patterns are supported: pass the tablegen invocation +/// "-warn-on-skipped-patterns" to emit a warning when a pattern is skipped, +/// as well as why. +/// +/// The generated file defines a single method: +/// bool <Target>InstructionSelector::selectImpl(MachineInstr &I) const; +/// intended to be used in InstructionSelector::select as the first-step +/// selector for the patterns that don't require complex C++. +/// +/// FIXME: We'll probably want to eventually define a base +/// "TargetGenInstructionSelector" class. +/// +//===----------------------------------------------------------------------===// + +#include "CodeGenDAGPatterns.h" +#include "SubtargetFeatureInfo.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Support/CodeGenCoverage.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/LowLevelTypeImpl.h" +#include "llvm/Support/MachineValueType.h" +#include "llvm/Support/ScopedPrinter.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <numeric> +#include <string> +using namespace llvm; + +#define DEBUG_TYPE "gisel-emitter" + +STATISTIC(NumPatternTotal, "Total number of patterns"); +STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG"); +STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped"); +STATISTIC(NumPatternsTested, "Number of patterns executed according to coverage information"); +STATISTIC(NumPatternEmitted, "Number of patterns emitted"); + +cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel"); + +static cl::opt<bool> WarnOnSkippedPatterns( + "warn-on-skipped-patterns", + cl::desc("Explain why a pattern was skipped for inclusion " + "in the GlobalISel selector"), + cl::init(false), cl::cat(GlobalISelEmitterCat)); + +static cl::opt<bool> GenerateCoverage( + "instrument-gisel-coverage", + cl::desc("Generate coverage instrumentation for GlobalISel"), + cl::init(false), cl::cat(GlobalISelEmitterCat)); + +static cl::opt<std::string> UseCoverageFile( + "gisel-coverage-file", cl::init(""), + cl::desc("Specify file to retrieve coverage information from"), + cl::cat(GlobalISelEmitterCat)); + +static cl::opt<bool> OptimizeMatchTable( + "optimize-match-table", + cl::desc("Generate an optimized version of the match table"), + cl::init(true), cl::cat(GlobalISelEmitterCat)); + +namespace { +//===- Helper functions ---------------------------------------------------===// + +/// Get the name of the enum value used to number the predicate function. +std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) { + if (Predicate.hasGISelPredicateCode()) + return "GIPFP_MI_" + Predicate.getFnName(); + return "GIPFP_" + Predicate.getImmTypeIdentifier().str() + "_" + + Predicate.getFnName(); +} + +/// Get the opcode used to check this predicate. +std::string getMatchOpcodeForPredicate(const TreePredicateFn &Predicate) { + return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate"; +} + +/// This class stands in for LLT wherever we want to tablegen-erate an +/// equivalent at compiler run-time. +class LLTCodeGen { +private: + LLT Ty; + +public: + LLTCodeGen() = default; + LLTCodeGen(const LLT &Ty) : Ty(Ty) {} + + std::string getCxxEnumValue() const { + std::string Str; + raw_string_ostream OS(Str); + + emitCxxEnumValue(OS); + return OS.str(); + } + + void emitCxxEnumValue(raw_ostream &OS) const { + if (Ty.isScalar()) { + OS << "GILLT_s" << Ty.getSizeInBits(); + return; + } + if (Ty.isVector()) { + OS << "GILLT_v" << Ty.getNumElements() << "s" << Ty.getScalarSizeInBits(); + return; + } + if (Ty.isPointer()) { + OS << "GILLT_p" << Ty.getAddressSpace(); + if (Ty.getSizeInBits() > 0) + OS << "s" << Ty.getSizeInBits(); + return; + } + llvm_unreachable("Unhandled LLT"); + } + + void emitCxxConstructorCall(raw_ostream &OS) const { + if (Ty.isScalar()) { + OS << "LLT::scalar(" << Ty.getSizeInBits() << ")"; + return; + } + if (Ty.isVector()) { + OS << "LLT::vector(" << Ty.getNumElements() << ", " + << Ty.getScalarSizeInBits() << ")"; + return; + } + if (Ty.isPointer() && Ty.getSizeInBits() > 0) { + OS << "LLT::pointer(" << Ty.getAddressSpace() << ", " + << Ty.getSizeInBits() << ")"; + return; + } + llvm_unreachable("Unhandled LLT"); + } + + const LLT &get() const { return Ty; } + + /// This ordering is used for std::unique() and llvm::sort(). There's no + /// particular logic behind the order but either A < B or B < A must be + /// true if A != B. + bool operator<(const LLTCodeGen &Other) const { + if (Ty.isValid() != Other.Ty.isValid()) + return Ty.isValid() < Other.Ty.isValid(); + if (!Ty.isValid()) + return false; + + if (Ty.isVector() != Other.Ty.isVector()) + return Ty.isVector() < Other.Ty.isVector(); + if (Ty.isScalar() != Other.Ty.isScalar()) + return Ty.isScalar() < Other.Ty.isScalar(); + if (Ty.isPointer() != Other.Ty.isPointer()) + return Ty.isPointer() < Other.Ty.isPointer(); + + if (Ty.isPointer() && Ty.getAddressSpace() != Other.Ty.getAddressSpace()) + return Ty.getAddressSpace() < Other.Ty.getAddressSpace(); + + if (Ty.isVector() && Ty.getNumElements() != Other.Ty.getNumElements()) + return Ty.getNumElements() < Other.Ty.getNumElements(); + + return Ty.getSizeInBits() < Other.Ty.getSizeInBits(); + } + + bool operator==(const LLTCodeGen &B) const { return Ty == B.Ty; } +}; + +// Track all types that are used so we can emit the corresponding enum. +std::set<LLTCodeGen> KnownTypes; + +class InstructionMatcher; +/// Convert an MVT to an equivalent LLT if possible, or the invalid LLT() for +/// MVTs that don't map cleanly to an LLT (e.g., iPTR, *any, ...). +static Optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT) { + MVT VT(SVT); + + if (VT.isVector() && VT.getVectorNumElements() != 1) + return LLTCodeGen( + LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits())); + + if (VT.isInteger() || VT.isFloatingPoint()) + return LLTCodeGen(LLT::scalar(VT.getSizeInBits())); + return None; +} + +static std::string explainPredicates(const TreePatternNode *N) { + std::string Explanation = ""; + StringRef Separator = ""; + for (const TreePredicateCall &Call : N->getPredicateCalls()) { + const TreePredicateFn &P = Call.Fn; + Explanation += + (Separator + P.getOrigPatFragRecord()->getRecord()->getName()).str(); + Separator = ", "; + + if (P.isAlwaysTrue()) + Explanation += " always-true"; + if (P.isImmediatePattern()) + Explanation += " immediate"; + + if (P.isUnindexed()) + Explanation += " unindexed"; + + if (P.isNonExtLoad()) + Explanation += " non-extload"; + if (P.isAnyExtLoad()) + Explanation += " extload"; + if (P.isSignExtLoad()) + Explanation += " sextload"; + if (P.isZeroExtLoad()) + Explanation += " zextload"; + + if (P.isNonTruncStore()) + Explanation += " non-truncstore"; + if (P.isTruncStore()) + Explanation += " truncstore"; + + if (Record *VT = P.getMemoryVT()) + Explanation += (" MemVT=" + VT->getName()).str(); + if (Record *VT = P.getScalarMemoryVT()) + Explanation += (" ScalarVT(MemVT)=" + VT->getName()).str(); + + if (ListInit *AddrSpaces = P.getAddressSpaces()) { + raw_string_ostream OS(Explanation); + OS << " AddressSpaces=["; + + StringRef AddrSpaceSeparator; + for (Init *Val : AddrSpaces->getValues()) { + IntInit *IntVal = dyn_cast<IntInit>(Val); + if (!IntVal) + continue; + + OS << AddrSpaceSeparator << IntVal->getValue(); + AddrSpaceSeparator = ", "; + } + + OS << ']'; + } + + int64_t MinAlign = P.getMinAlignment(); + if (MinAlign > 0) + Explanation += " MinAlign=" + utostr(MinAlign); + + if (P.isAtomicOrderingMonotonic()) + Explanation += " monotonic"; + if (P.isAtomicOrderingAcquire()) + Explanation += " acquire"; + if (P.isAtomicOrderingRelease()) + Explanation += " release"; + if (P.isAtomicOrderingAcquireRelease()) + Explanation += " acq_rel"; + if (P.isAtomicOrderingSequentiallyConsistent()) + Explanation += " seq_cst"; + if (P.isAtomicOrderingAcquireOrStronger()) + Explanation += " >=acquire"; + if (P.isAtomicOrderingWeakerThanAcquire()) + Explanation += " <acquire"; + if (P.isAtomicOrderingReleaseOrStronger()) + Explanation += " >=release"; + if (P.isAtomicOrderingWeakerThanRelease()) + Explanation += " <release"; + } + return Explanation; +} + +std::string explainOperator(Record *Operator) { + if (Operator->isSubClassOf("SDNode")) + return (" (" + Operator->getValueAsString("Opcode") + ")").str(); + + if (Operator->isSubClassOf("Intrinsic")) + return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str(); + + if (Operator->isSubClassOf("ComplexPattern")) + return (" (Operator is an unmapped ComplexPattern, " + Operator->getName() + + ")") + .str(); + + if (Operator->isSubClassOf("SDNodeXForm")) + return (" (Operator is an unmapped SDNodeXForm, " + Operator->getName() + + ")") + .str(); + + return (" (Operator " + Operator->getName() + " not understood)").str(); +} + +/// Helper function to let the emitter report skip reason error messages. +static Error failedImport(const Twine &Reason) { + return make_error<StringError>(Reason, inconvertibleErrorCode()); +} + +static Error isTrivialOperatorNode(const TreePatternNode *N) { + std::string Explanation = ""; + std::string Separator = ""; + + bool HasUnsupportedPredicate = false; + for (const TreePredicateCall &Call : N->getPredicateCalls()) { + const TreePredicateFn &Predicate = Call.Fn; + + if (Predicate.isAlwaysTrue()) + continue; + + if (Predicate.isImmediatePattern()) + continue; + + if (Predicate.isNonExtLoad() || Predicate.isAnyExtLoad() || + Predicate.isSignExtLoad() || Predicate.isZeroExtLoad()) + continue; + + if (Predicate.isNonTruncStore() || Predicate.isTruncStore()) + continue; + + if (Predicate.isLoad() && Predicate.getMemoryVT()) + continue; + + if (Predicate.isLoad() || Predicate.isStore()) { + if (Predicate.isUnindexed()) + continue; + } + + if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) { + const ListInit *AddrSpaces = Predicate.getAddressSpaces(); + if (AddrSpaces && !AddrSpaces->empty()) + continue; + + if (Predicate.getMinAlignment() > 0) + continue; + } + + if (Predicate.isAtomic() && Predicate.getMemoryVT()) + continue; + + if (Predicate.isAtomic() && + (Predicate.isAtomicOrderingMonotonic() || + Predicate.isAtomicOrderingAcquire() || + Predicate.isAtomicOrderingRelease() || + Predicate.isAtomicOrderingAcquireRelease() || + Predicate.isAtomicOrderingSequentiallyConsistent() || + Predicate.isAtomicOrderingAcquireOrStronger() || + Predicate.isAtomicOrderingWeakerThanAcquire() || + Predicate.isAtomicOrderingReleaseOrStronger() || + Predicate.isAtomicOrderingWeakerThanRelease())) + continue; + + if (Predicate.hasGISelPredicateCode()) + continue; + + HasUnsupportedPredicate = true; + Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")"; + Separator = ", "; + Explanation += (Separator + "first-failing:" + + Predicate.getOrigPatFragRecord()->getRecord()->getName()) + .str(); + break; + } + + if (!HasUnsupportedPredicate) + return Error::success(); + + return failedImport(Explanation); +} + +static Record *getInitValueAsRegClass(Init *V) { + if (DefInit *VDefInit = dyn_cast<DefInit>(V)) { + if (VDefInit->getDef()->isSubClassOf("RegisterOperand")) + return VDefInit->getDef()->getValueAsDef("RegClass"); + if (VDefInit->getDef()->isSubClassOf("RegisterClass")) + return VDefInit->getDef(); + } + return nullptr; +} + +std::string +getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset) { + std::string Name = "GIFBS"; + for (const auto &Feature : FeatureBitset) + Name += ("_" + Feature->getName()).str(); + return Name; +} + +//===- MatchTable Helpers -------------------------------------------------===// + +class MatchTable; + +/// A record to be stored in a MatchTable. +/// +/// This class represents any and all output that may be required to emit the +/// MatchTable. Instances are most often configured to represent an opcode or +/// value that will be emitted to the table with some formatting but it can also +/// represent commas, comments, and other formatting instructions. +struct MatchTableRecord { + enum RecordFlagsBits { + MTRF_None = 0x0, + /// Causes EmitStr to be formatted as comment when emitted. + MTRF_Comment = 0x1, + /// Causes the record value to be followed by a comma when emitted. + MTRF_CommaFollows = 0x2, + /// Causes the record value to be followed by a line break when emitted. + MTRF_LineBreakFollows = 0x4, + /// Indicates that the record defines a label and causes an additional + /// comment to be emitted containing the index of the label. + MTRF_Label = 0x8, + /// Causes the record to be emitted as the index of the label specified by + /// LabelID along with a comment indicating where that label is. + MTRF_JumpTarget = 0x10, + /// Causes the formatter to add a level of indentation before emitting the + /// record. + MTRF_Indent = 0x20, + /// Causes the formatter to remove a level of indentation after emitting the + /// record. + MTRF_Outdent = 0x40, + }; + + /// When MTRF_Label or MTRF_JumpTarget is used, indicates a label id to + /// reference or define. + unsigned LabelID; + /// The string to emit. Depending on the MTRF_* flags it may be a comment, a + /// value, a label name. + std::string EmitStr; + +private: + /// The number of MatchTable elements described by this record. Comments are 0 + /// while values are typically 1. Values >1 may occur when we need to emit + /// values that exceed the size of a MatchTable element. + unsigned NumElements; + +public: + /// A bitfield of RecordFlagsBits flags. + unsigned Flags; + + /// The actual run-time value, if known + int64_t RawValue; + + MatchTableRecord(Optional<unsigned> LabelID_, StringRef EmitStr, + unsigned NumElements, unsigned Flags, + int64_t RawValue = std::numeric_limits<int64_t>::min()) + : LabelID(LabelID_.hasValue() ? LabelID_.getValue() : ~0u), + EmitStr(EmitStr), NumElements(NumElements), Flags(Flags), + RawValue(RawValue) { + assert((!LabelID_.hasValue() || LabelID != ~0u) && + "This value is reserved for non-labels"); + } + MatchTableRecord(const MatchTableRecord &Other) = default; + MatchTableRecord(MatchTableRecord &&Other) = default; + + /// Useful if a Match Table Record gets optimized out + void turnIntoComment() { + Flags |= MTRF_Comment; + Flags &= ~MTRF_CommaFollows; + NumElements = 0; + } + + /// For Jump Table generation purposes + bool operator<(const MatchTableRecord &Other) const { + return RawValue < Other.RawValue; + } + int64_t getRawValue() const { return RawValue; } + + void emit(raw_ostream &OS, bool LineBreakNextAfterThis, + const MatchTable &Table) const; + unsigned size() const { return NumElements; } +}; + +class Matcher; + +/// Holds the contents of a generated MatchTable to enable formatting and the +/// necessary index tracking needed to support GIM_Try. +class MatchTable { + /// An unique identifier for the table. The generated table will be named + /// MatchTable${ID}. + unsigned ID; + /// The records that make up the table. Also includes comments describing the + /// values being emitted and line breaks to format it. + std::vector<MatchTableRecord> Contents; + /// The currently defined labels. + DenseMap<unsigned, unsigned> LabelMap; + /// Tracks the sum of MatchTableRecord::NumElements as the table is built. + unsigned CurrentSize = 0; + /// A unique identifier for a MatchTable label. + unsigned CurrentLabelID = 0; + /// Determines if the table should be instrumented for rule coverage tracking. + bool IsWithCoverage; + +public: + static MatchTableRecord LineBreak; + static MatchTableRecord Comment(StringRef Comment) { + return MatchTableRecord(None, Comment, 0, MatchTableRecord::MTRF_Comment); + } + static MatchTableRecord Opcode(StringRef Opcode, int IndentAdjust = 0) { + unsigned ExtraFlags = 0; + if (IndentAdjust > 0) + ExtraFlags |= MatchTableRecord::MTRF_Indent; + if (IndentAdjust < 0) + ExtraFlags |= MatchTableRecord::MTRF_Outdent; + + return MatchTableRecord(None, Opcode, 1, + MatchTableRecord::MTRF_CommaFollows | ExtraFlags); + } + static MatchTableRecord NamedValue(StringRef NamedValue) { + return MatchTableRecord(None, NamedValue, 1, + MatchTableRecord::MTRF_CommaFollows); + } + static MatchTableRecord NamedValue(StringRef NamedValue, int64_t RawValue) { + return MatchTableRecord(None, NamedValue, 1, + MatchTableRecord::MTRF_CommaFollows, RawValue); + } + static MatchTableRecord NamedValue(StringRef Namespace, + StringRef NamedValue) { + return MatchTableRecord(None, (Namespace + "::" + NamedValue).str(), 1, + MatchTableRecord::MTRF_CommaFollows); + } + static MatchTableRecord NamedValue(StringRef Namespace, StringRef NamedValue, + int64_t RawValue) { + return MatchTableRecord(None, (Namespace + "::" + NamedValue).str(), 1, + MatchTableRecord::MTRF_CommaFollows, RawValue); + } + static MatchTableRecord IntValue(int64_t IntValue) { + return MatchTableRecord(None, llvm::to_string(IntValue), 1, + MatchTableRecord::MTRF_CommaFollows); + } + static MatchTableRecord Label(unsigned LabelID) { + return MatchTableRecord(LabelID, "Label " + llvm::to_string(LabelID), 0, + MatchTableRecord::MTRF_Label | + MatchTableRecord::MTRF_Comment | + MatchTableRecord::MTRF_LineBreakFollows); + } + static MatchTableRecord JumpTarget(unsigned LabelID) { + return MatchTableRecord(LabelID, "Label " + llvm::to_string(LabelID), 1, + MatchTableRecord::MTRF_JumpTarget | + MatchTableRecord::MTRF_Comment | + MatchTableRecord::MTRF_CommaFollows); + } + + static MatchTable buildTable(ArrayRef<Matcher *> Rules, bool WithCoverage); + + MatchTable(bool WithCoverage, unsigned ID = 0) + : ID(ID), IsWithCoverage(WithCoverage) {} + + bool isWithCoverage() const { return IsWithCoverage; } + + void push_back(const MatchTableRecord &Value) { + if (Value.Flags & MatchTableRecord::MTRF_Label) + defineLabel(Value.LabelID); + Contents.push_back(Value); + CurrentSize += Value.size(); + } + + unsigned allocateLabelID() { return CurrentLabelID++; } + + void defineLabel(unsigned LabelID) { + LabelMap.insert(std::make_pair(LabelID, CurrentSize)); + } + + unsigned getLabelIndex(unsigned LabelID) const { + const auto I = LabelMap.find(LabelID); + assert(I != LabelMap.end() && "Use of undeclared label"); + return I->second; + } + + void emitUse(raw_ostream &OS) const { OS << "MatchTable" << ID; } + + void emitDeclaration(raw_ostream &OS) const { + unsigned Indentation = 4; + OS << " constexpr static int64_t MatchTable" << ID << "[] = {"; + LineBreak.emit(OS, true, *this); + OS << std::string(Indentation, ' '); + + for (auto I = Contents.begin(), E = Contents.end(); I != E; + ++I) { + bool LineBreakIsNext = false; + const auto &NextI = std::next(I); + + if (NextI != E) { + if (NextI->EmitStr == "" && + NextI->Flags == MatchTableRecord::MTRF_LineBreakFollows) + LineBreakIsNext = true; + } + + if (I->Flags & MatchTableRecord::MTRF_Indent) + Indentation += 2; + + I->emit(OS, LineBreakIsNext, *this); + if (I->Flags & MatchTableRecord::MTRF_LineBreakFollows) + OS << std::string(Indentation, ' '); + + if (I->Flags & MatchTableRecord::MTRF_Outdent) + Indentation -= 2; + } + OS << "};\n"; + } +}; + +MatchTableRecord MatchTable::LineBreak = { + None, "" /* Emit String */, 0 /* Elements */, + MatchTableRecord::MTRF_LineBreakFollows}; + +void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis, + const MatchTable &Table) const { + bool UseLineComment = + LineBreakIsNextAfterThis || (Flags & MTRF_LineBreakFollows); + if (Flags & (MTRF_JumpTarget | MTRF_CommaFollows)) + UseLineComment = false; + + if (Flags & MTRF_Comment) + OS << (UseLineComment ? "// " : "/*"); + + OS << EmitStr; + if (Flags & MTRF_Label) + OS << ": @" << Table.getLabelIndex(LabelID); + + if ((Flags & MTRF_Comment) && !UseLineComment) + OS << "*/"; + + if (Flags & MTRF_JumpTarget) { + if (Flags & MTRF_Comment) + OS << " "; + OS << Table.getLabelIndex(LabelID); + } + + if (Flags & MTRF_CommaFollows) { + OS << ","; + if (!LineBreakIsNextAfterThis && !(Flags & MTRF_LineBreakFollows)) + OS << " "; + } + + if (Flags & MTRF_LineBreakFollows) + OS << "\n"; +} + +MatchTable &operator<<(MatchTable &Table, const MatchTableRecord &Value) { + Table.push_back(Value); + return Table; +} + +//===- Matchers -----------------------------------------------------------===// + +class OperandMatcher; +class MatchAction; +class PredicateMatcher; +class RuleMatcher; + +class Matcher { +public: + virtual ~Matcher() = default; + virtual void optimize() {} + virtual void emit(MatchTable &Table) = 0; + + virtual bool hasFirstCondition() const = 0; + virtual const PredicateMatcher &getFirstCondition() const = 0; + virtual std::unique_ptr<PredicateMatcher> popFirstCondition() = 0; +}; + +MatchTable MatchTable::buildTable(ArrayRef<Matcher *> Rules, + bool WithCoverage) { + MatchTable Table(WithCoverage); + for (Matcher *Rule : Rules) + Rule->emit(Table); + + return Table << MatchTable::Opcode("GIM_Reject") << MatchTable::LineBreak; +} + +class GroupMatcher final : public Matcher { + /// Conditions that form a common prefix of all the matchers contained. + SmallVector<std::unique_ptr<PredicateMatcher>, 1> Conditions; + + /// All the nested matchers, sharing a common prefix. + std::vector<Matcher *> Matchers; + + /// An owning collection for any auxiliary matchers created while optimizing + /// nested matchers contained. + std::vector<std::unique_ptr<Matcher>> MatcherStorage; + +public: + /// Add a matcher to the collection of nested matchers if it meets the + /// requirements, and return true. If it doesn't, do nothing and return false. + /// + /// Expected to preserve its argument, so it could be moved out later on. + bool addMatcher(Matcher &Candidate); + + /// Mark the matcher as fully-built and ensure any invariants expected by both + /// optimize() and emit(...) methods. Generally, both sequences of calls + /// are expected to lead to a sensible result: + /// + /// addMatcher(...)*; finalize(); optimize(); emit(...); and + /// addMatcher(...)*; finalize(); emit(...); + /// + /// or generally + /// + /// addMatcher(...)*; finalize(); { optimize()*; emit(...); }* + /// + /// Multiple calls to optimize() are expected to be handled gracefully, though + /// optimize() is not expected to be idempotent. Multiple calls to finalize() + /// aren't generally supported. emit(...) is expected to be non-mutating and + /// producing the exact same results upon repeated calls. + /// + /// addMatcher() calls after the finalize() call are not supported. + /// + /// finalize() and optimize() are both allowed to mutate the contained + /// matchers, so moving them out after finalize() is not supported. + void finalize(); + void optimize() override; + void emit(MatchTable &Table) override; + + /// Could be used to move out the matchers added previously, unless finalize() + /// has been already called. If any of the matchers are moved out, the group + /// becomes safe to destroy, but not safe to re-use for anything else. + iterator_range<std::vector<Matcher *>::iterator> matchers() { + return make_range(Matchers.begin(), Matchers.end()); + } + size_t size() const { return Matchers.size(); } + bool empty() const { return Matchers.empty(); } + + std::unique_ptr<PredicateMatcher> popFirstCondition() override { + assert(!Conditions.empty() && + "Trying to pop a condition from a condition-less group"); + std::unique_ptr<PredicateMatcher> P = std::move(Conditions.front()); + Conditions.erase(Conditions.begin()); + return P; + } + const PredicateMatcher &getFirstCondition() const override { + assert(!Conditions.empty() && + "Trying to get a condition from a condition-less group"); + return *Conditions.front(); + } + bool hasFirstCondition() const override { return !Conditions.empty(); } + +private: + /// See if a candidate matcher could be added to this group solely by + /// analyzing its first condition. + bool candidateConditionMatches(const PredicateMatcher &Predicate) const; +}; + +class SwitchMatcher : public Matcher { + /// All the nested matchers, representing distinct switch-cases. The first + /// conditions (as Matcher::getFirstCondition() reports) of all the nested + /// matchers must share the same type and path to a value they check, in other + /// words, be isIdenticalDownToValue, but have different values they check + /// against. + std::vector<Matcher *> Matchers; + + /// The representative condition, with a type and a path (InsnVarID and OpIdx + /// in most cases) shared by all the matchers contained. + std::unique_ptr<PredicateMatcher> Condition = nullptr; + + /// Temporary set used to check that the case values don't repeat within the + /// same switch. + std::set<MatchTableRecord> Values; + + /// An owning collection for any auxiliary matchers created while optimizing + /// nested matchers contained. + std::vector<std::unique_ptr<Matcher>> MatcherStorage; + +public: + bool addMatcher(Matcher &Candidate); + + void finalize(); + void emit(MatchTable &Table) override; + + iterator_range<std::vector<Matcher *>::iterator> matchers() { + return make_range(Matchers.begin(), Matchers.end()); + } + size_t size() const { return Matchers.size(); } + bool empty() const { return Matchers.empty(); } + + std::unique_ptr<PredicateMatcher> popFirstCondition() override { + // SwitchMatcher doesn't have a common first condition for its cases, as all + // the cases only share a kind of a value (a type and a path to it) they + // match, but deliberately differ in the actual value they match. + llvm_unreachable("Trying to pop a condition from a condition-less group"); + } + const PredicateMatcher &getFirstCondition() const override { + llvm_unreachable("Trying to pop a condition from a condition-less group"); + } + bool hasFirstCondition() const override { return false; } + +private: + /// See if the predicate type has a Switch-implementation for it. + static bool isSupportedPredicateType(const PredicateMatcher &Predicate); + + bool candidateConditionMatches(const PredicateMatcher &Predicate) const; + + /// emit()-helper + static void emitPredicateSpecificOpcodes(const PredicateMatcher &P, + MatchTable &Table); +}; + +/// Generates code to check that a match rule matches. +class RuleMatcher : public Matcher { +public: + using ActionList = std::list<std::unique_ptr<MatchAction>>; + using action_iterator = ActionList::iterator; + +protected: + /// A list of matchers that all need to succeed for the current rule to match. + /// FIXME: This currently supports a single match position but could be + /// extended to support multiple positions to support div/rem fusion or + /// load-multiple instructions. + using MatchersTy = std::vector<std::unique_ptr<InstructionMatcher>> ; + MatchersTy Matchers; + + /// A list of actions that need to be taken when all predicates in this rule + /// have succeeded. + ActionList Actions; + + using DefinedInsnVariablesMap = std::map<InstructionMatcher *, unsigned>; + + /// A map of instruction matchers to the local variables + DefinedInsnVariablesMap InsnVariableIDs; + + using MutatableInsnSet = SmallPtrSet<InstructionMatcher *, 4>; + + // The set of instruction matchers that have not yet been claimed for mutation + // by a BuildMI. + MutatableInsnSet MutatableInsns; + + /// A map of named operands defined by the matchers that may be referenced by + /// the renderers. + StringMap<OperandMatcher *> DefinedOperands; + + /// A map of anonymous physical register operands defined by the matchers that + /// may be referenced by the renderers. + DenseMap<Record *, OperandMatcher *> PhysRegOperands; + + /// ID for the next instruction variable defined with implicitlyDefineInsnVar() + unsigned NextInsnVarID; + + /// ID for the next output instruction allocated with allocateOutputInsnID() + unsigned NextOutputInsnID; + + /// ID for the next temporary register ID allocated with allocateTempRegID() + unsigned NextTempRegID; + + std::vector<Record *> RequiredFeatures; + std::vector<std::unique_ptr<PredicateMatcher>> EpilogueMatchers; + + ArrayRef<SMLoc> SrcLoc; + + typedef std::tuple<Record *, unsigned, unsigned> + DefinedComplexPatternSubOperand; + typedef StringMap<DefinedComplexPatternSubOperand> + DefinedComplexPatternSubOperandMap; + /// A map of Symbolic Names to ComplexPattern sub-operands. + DefinedComplexPatternSubOperandMap ComplexSubOperands; + + uint64_t RuleID; + static uint64_t NextRuleID; + +public: + RuleMatcher(ArrayRef<SMLoc> SrcLoc) + : Matchers(), Actions(), InsnVariableIDs(), MutatableInsns(), + DefinedOperands(), NextInsnVarID(0), NextOutputInsnID(0), + NextTempRegID(0), SrcLoc(SrcLoc), ComplexSubOperands(), + RuleID(NextRuleID++) {} + RuleMatcher(RuleMatcher &&Other) = default; + RuleMatcher &operator=(RuleMatcher &&Other) = default; + + uint64_t getRuleID() const { return RuleID; } + + InstructionMatcher &addInstructionMatcher(StringRef SymbolicName); + void addRequiredFeature(Record *Feature); + const std::vector<Record *> &getRequiredFeatures() const; + + template <class Kind, class... Args> Kind &addAction(Args &&... args); + template <class Kind, class... Args> + action_iterator insertAction(action_iterator InsertPt, Args &&... args); + + /// Define an instruction without emitting any code to do so. + unsigned implicitlyDefineInsnVar(InstructionMatcher &Matcher); + + unsigned getInsnVarID(InstructionMatcher &InsnMatcher) const; + DefinedInsnVariablesMap::const_iterator defined_insn_vars_begin() const { + return InsnVariableIDs.begin(); + } + DefinedInsnVariablesMap::const_iterator defined_insn_vars_end() const { + return InsnVariableIDs.end(); + } + iterator_range<typename DefinedInsnVariablesMap::const_iterator> + defined_insn_vars() const { + return make_range(defined_insn_vars_begin(), defined_insn_vars_end()); + } + + MutatableInsnSet::const_iterator mutatable_insns_begin() const { + return MutatableInsns.begin(); + } + MutatableInsnSet::const_iterator mutatable_insns_end() const { + return MutatableInsns.end(); + } + iterator_range<typename MutatableInsnSet::const_iterator> + mutatable_insns() const { + return make_range(mutatable_insns_begin(), mutatable_insns_end()); + } + void reserveInsnMatcherForMutation(InstructionMatcher *InsnMatcher) { + bool R = MutatableInsns.erase(InsnMatcher); + assert(R && "Reserving a mutatable insn that isn't available"); + (void)R; + } + + action_iterator actions_begin() { return Actions.begin(); } + action_iterator actions_end() { return Actions.end(); } + iterator_range<action_iterator> actions() { + return make_range(actions_begin(), actions_end()); + } + + void defineOperand(StringRef SymbolicName, OperandMatcher &OM); + + void definePhysRegOperand(Record *Reg, OperandMatcher &OM); + + Error defineComplexSubOperand(StringRef SymbolicName, Record *ComplexPattern, + unsigned RendererID, unsigned SubOperandID) { + if (ComplexSubOperands.count(SymbolicName)) + return failedImport( + "Complex suboperand referenced more than once (Operand: " + + SymbolicName + ")"); + + ComplexSubOperands[SymbolicName] = + std::make_tuple(ComplexPattern, RendererID, SubOperandID); + + return Error::success(); + } + + Optional<DefinedComplexPatternSubOperand> + getComplexSubOperand(StringRef SymbolicName) const { + const auto &I = ComplexSubOperands.find(SymbolicName); + if (I == ComplexSubOperands.end()) + return None; + return I->second; + } + + InstructionMatcher &getInstructionMatcher(StringRef SymbolicName) const; + const OperandMatcher &getOperandMatcher(StringRef Name) const; + const OperandMatcher &getPhysRegOperandMatcher(Record *) const; + + void optimize() override; + void emit(MatchTable &Table) override; + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + bool isHigherPriorityThan(const RuleMatcher &B) const; + + /// Report the maximum number of temporary operands needed by the rule + /// matcher. + unsigned countRendererFns() const; + + std::unique_ptr<PredicateMatcher> popFirstCondition() override; + const PredicateMatcher &getFirstCondition() const override; + LLTCodeGen getFirstConditionAsRootType(); + bool hasFirstCondition() const override; + unsigned getNumOperands() const; + StringRef getOpcode() const; + + // FIXME: Remove this as soon as possible + InstructionMatcher &insnmatchers_front() const { return *Matchers.front(); } + + unsigned allocateOutputInsnID() { return NextOutputInsnID++; } + unsigned allocateTempRegID() { return NextTempRegID++; } + + iterator_range<MatchersTy::iterator> insnmatchers() { + return make_range(Matchers.begin(), Matchers.end()); + } + bool insnmatchers_empty() const { return Matchers.empty(); } + void insnmatchers_pop_front() { Matchers.erase(Matchers.begin()); } +}; + +uint64_t RuleMatcher::NextRuleID = 0; + +using action_iterator = RuleMatcher::action_iterator; + +template <class PredicateTy> class PredicateListMatcher { +private: + /// Template instantiations should specialize this to return a string to use + /// for the comment emitted when there are no predicates. + std::string getNoPredicateComment() const; + +protected: + using PredicatesTy = std::deque<std::unique_ptr<PredicateTy>>; + PredicatesTy Predicates; + + /// Track if the list of predicates was manipulated by one of the optimization + /// methods. + bool Optimized = false; + +public: + /// Construct a new predicate and add it to the matcher. + template <class Kind, class... Args> + Optional<Kind *> addPredicate(Args &&... args); + + typename PredicatesTy::iterator predicates_begin() { + return Predicates.begin(); + } + typename PredicatesTy::iterator predicates_end() { + return Predicates.end(); + } + iterator_range<typename PredicatesTy::iterator> predicates() { + return make_range(predicates_begin(), predicates_end()); + } + typename PredicatesTy::size_type predicates_size() const { + return Predicates.size(); + } + bool predicates_empty() const { return Predicates.empty(); } + + std::unique_ptr<PredicateTy> predicates_pop_front() { + std::unique_ptr<PredicateTy> Front = std::move(Predicates.front()); + Predicates.pop_front(); + Optimized = true; + return Front; + } + + void prependPredicate(std::unique_ptr<PredicateTy> &&Predicate) { + Predicates.push_front(std::move(Predicate)); + } + + void eraseNullPredicates() { + const auto NewEnd = + std::stable_partition(Predicates.begin(), Predicates.end(), + std::logical_not<std::unique_ptr<PredicateTy>>()); + if (NewEnd != Predicates.begin()) { + Predicates.erase(Predicates.begin(), NewEnd); + Optimized = true; + } + } + + /// Emit MatchTable opcodes that tests whether all the predicates are met. + template <class... Args> + void emitPredicateListOpcodes(MatchTable &Table, Args &&... args) { + if (Predicates.empty() && !Optimized) { + Table << MatchTable::Comment(getNoPredicateComment()) + << MatchTable::LineBreak; + return; + } + + for (const auto &Predicate : predicates()) + Predicate->emitPredicateOpcodes(Table, std::forward<Args>(args)...); + } + + /// Provide a function to avoid emitting certain predicates. This is used to + /// defer some predicate checks until after others + using PredicateFilterFunc = std::function<bool(const PredicateTy&)>; + + /// Emit MatchTable opcodes for predicates which satisfy \p + /// ShouldEmitPredicate. This should be called multiple times to ensure all + /// predicates are eventually added to the match table. + template <class... Args> + void emitFilteredPredicateListOpcodes(PredicateFilterFunc ShouldEmitPredicate, + MatchTable &Table, Args &&... args) { + if (Predicates.empty() && !Optimized) { + Table << MatchTable::Comment(getNoPredicateComment()) + << MatchTable::LineBreak; + return; + } + + for (const auto &Predicate : predicates()) { + if (ShouldEmitPredicate(*Predicate)) + Predicate->emitPredicateOpcodes(Table, std::forward<Args>(args)...); + } + } +}; + +class PredicateMatcher { +public: + /// This enum is used for RTTI and also defines the priority that is given to + /// the predicate when generating the matcher code. Kinds with higher priority + /// must be tested first. + /// + /// The relative priority of OPM_LLT, OPM_RegBank, and OPM_MBB do not matter + /// but OPM_Int must have priority over OPM_RegBank since constant integers + /// are represented by a virtual register defined by a G_CONSTANT instruction. + /// + /// Note: The relative priority between IPM_ and OPM_ does not matter, they + /// are currently not compared between each other. + enum PredicateKind { + IPM_Opcode, + IPM_NumOperands, + IPM_ImmPredicate, + IPM_Imm, + IPM_AtomicOrderingMMO, + IPM_MemoryLLTSize, + IPM_MemoryVsLLTSize, + IPM_MemoryAddressSpace, + IPM_MemoryAlignment, + IPM_GenericPredicate, + OPM_SameOperand, + OPM_ComplexPattern, + OPM_IntrinsicID, + OPM_CmpPredicate, + OPM_Instruction, + OPM_Int, + OPM_LiteralInt, + OPM_LLT, + OPM_PointerToAny, + OPM_RegBank, + OPM_MBB, + }; + +protected: + PredicateKind Kind; + unsigned InsnVarID; + unsigned OpIdx; + +public: + PredicateMatcher(PredicateKind Kind, unsigned InsnVarID, unsigned OpIdx = ~0) + : Kind(Kind), InsnVarID(InsnVarID), OpIdx(OpIdx) {} + + unsigned getInsnVarID() const { return InsnVarID; } + unsigned getOpIdx() const { return OpIdx; } + + virtual ~PredicateMatcher() = default; + /// Emit MatchTable opcodes that check the predicate for the given operand. + virtual void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const = 0; + + PredicateKind getKind() const { return Kind; } + + bool dependsOnOperands() const { + // Custom predicates really depend on the context pattern of the + // instruction, not just the individual instruction. This therefore + // implicitly depends on all other pattern constraints. + return Kind == IPM_GenericPredicate; + } + + virtual bool isIdentical(const PredicateMatcher &B) const { + return B.getKind() == getKind() && InsnVarID == B.InsnVarID && + OpIdx == B.OpIdx; + } + + virtual bool isIdenticalDownToValue(const PredicateMatcher &B) const { + return hasValue() && PredicateMatcher::isIdentical(B); + } + + virtual MatchTableRecord getValue() const { + assert(hasValue() && "Can not get a value of a value-less predicate!"); + llvm_unreachable("Not implemented yet"); + } + virtual bool hasValue() const { return false; } + + /// Report the maximum number of temporary operands needed by the predicate + /// matcher. + virtual unsigned countRendererFns() const { return 0; } +}; + +/// Generates code to check a predicate of an operand. +/// +/// Typical predicates include: +/// * Operand is a particular register. +/// * Operand is assigned a particular register bank. +/// * Operand is an MBB. +class OperandPredicateMatcher : public PredicateMatcher { +public: + OperandPredicateMatcher(PredicateKind Kind, unsigned InsnVarID, + unsigned OpIdx) + : PredicateMatcher(Kind, InsnVarID, OpIdx) {} + virtual ~OperandPredicateMatcher() {} + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + virtual bool isHigherPriorityThan(const OperandPredicateMatcher &B) const; +}; + +template <> +std::string +PredicateListMatcher<OperandPredicateMatcher>::getNoPredicateComment() const { + return "No operand predicates"; +} + +/// Generates code to check that a register operand is defined by the same exact +/// one as another. +class SameOperandMatcher : public OperandPredicateMatcher { + std::string MatchingName; + +public: + SameOperandMatcher(unsigned InsnVarID, unsigned OpIdx, StringRef MatchingName) + : OperandPredicateMatcher(OPM_SameOperand, InsnVarID, OpIdx), + MatchingName(MatchingName) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_SameOperand; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override; + + bool isIdentical(const PredicateMatcher &B) const override { + return OperandPredicateMatcher::isIdentical(B) && + MatchingName == cast<SameOperandMatcher>(&B)->MatchingName; + } +}; + +/// Generates code to check that an operand is a particular LLT. +class LLTOperandMatcher : public OperandPredicateMatcher { +protected: + LLTCodeGen Ty; + +public: + static std::map<LLTCodeGen, unsigned> TypeIDValues; + + static void initTypeIDValuesMap() { + TypeIDValues.clear(); + + unsigned ID = 0; + for (const LLTCodeGen &LLTy : KnownTypes) + TypeIDValues[LLTy] = ID++; + } + + LLTOperandMatcher(unsigned InsnVarID, unsigned OpIdx, const LLTCodeGen &Ty) + : OperandPredicateMatcher(OPM_LLT, InsnVarID, OpIdx), Ty(Ty) { + KnownTypes.insert(Ty); + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_LLT; + } + bool isIdentical(const PredicateMatcher &B) const override { + return OperandPredicateMatcher::isIdentical(B) && + Ty == cast<LLTOperandMatcher>(&B)->Ty; + } + MatchTableRecord getValue() const override { + const auto VI = TypeIDValues.find(Ty); + if (VI == TypeIDValues.end()) + return MatchTable::NamedValue(getTy().getCxxEnumValue()); + return MatchTable::NamedValue(getTy().getCxxEnumValue(), VI->second); + } + bool hasValue() const override { + if (TypeIDValues.size() != KnownTypes.size()) + initTypeIDValuesMap(); + return TypeIDValues.count(Ty); + } + + LLTCodeGen getTy() const { return Ty; } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckType") << MatchTable::Comment("MI") + << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op") + << MatchTable::IntValue(OpIdx) << MatchTable::Comment("Type") + << getValue() << MatchTable::LineBreak; + } +}; + +std::map<LLTCodeGen, unsigned> LLTOperandMatcher::TypeIDValues; + +/// Generates code to check that an operand is a pointer to any address space. +/// +/// In SelectionDAG, the types did not describe pointers or address spaces. As a +/// result, iN is used to describe a pointer of N bits to any address space and +/// PatFrag predicates are typically used to constrain the address space. There's +/// no reliable means to derive the missing type information from the pattern so +/// imported rules must test the components of a pointer separately. +/// +/// If SizeInBits is zero, then the pointer size will be obtained from the +/// subtarget. +class PointerToAnyOperandMatcher : public OperandPredicateMatcher { +protected: + unsigned SizeInBits; + +public: + PointerToAnyOperandMatcher(unsigned InsnVarID, unsigned OpIdx, + unsigned SizeInBits) + : OperandPredicateMatcher(OPM_PointerToAny, InsnVarID, OpIdx), + SizeInBits(SizeInBits) {} + + static bool classof(const OperandPredicateMatcher *P) { + return P->getKind() == OPM_PointerToAny; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckPointerToAny") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) + << MatchTable::Comment("SizeInBits") + << MatchTable::IntValue(SizeInBits) << MatchTable::LineBreak; + } +}; + +/// Generates code to check that an operand is a particular target constant. +class ComplexPatternOperandMatcher : public OperandPredicateMatcher { +protected: + const OperandMatcher &Operand; + const Record &TheDef; + + unsigned getAllocatedTemporariesBaseID() const; + +public: + bool isIdentical(const PredicateMatcher &B) const override { return false; } + + ComplexPatternOperandMatcher(unsigned InsnVarID, unsigned OpIdx, + const OperandMatcher &Operand, + const Record &TheDef) + : OperandPredicateMatcher(OPM_ComplexPattern, InsnVarID, OpIdx), + Operand(Operand), TheDef(TheDef) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_ComplexPattern; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + unsigned ID = getAllocatedTemporariesBaseID(); + Table << MatchTable::Opcode("GIM_CheckComplexPattern") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) + << MatchTable::Comment("Renderer") << MatchTable::IntValue(ID) + << MatchTable::NamedValue(("GICP_" + TheDef.getName()).str()) + << MatchTable::LineBreak; + } + + unsigned countRendererFns() const override { + return 1; + } +}; + +/// Generates code to check that an operand is in a particular register bank. +class RegisterBankOperandMatcher : public OperandPredicateMatcher { +protected: + const CodeGenRegisterClass &RC; + +public: + RegisterBankOperandMatcher(unsigned InsnVarID, unsigned OpIdx, + const CodeGenRegisterClass &RC) + : OperandPredicateMatcher(OPM_RegBank, InsnVarID, OpIdx), RC(RC) {} + + bool isIdentical(const PredicateMatcher &B) const override { + return OperandPredicateMatcher::isIdentical(B) && + RC.getDef() == cast<RegisterBankOperandMatcher>(&B)->RC.getDef(); + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_RegBank; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckRegBankForClass") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) + << MatchTable::Comment("RC") + << MatchTable::NamedValue(RC.getQualifiedName() + "RegClassID") + << MatchTable::LineBreak; + } +}; + +/// Generates code to check that an operand is a basic block. +class MBBOperandMatcher : public OperandPredicateMatcher { +public: + MBBOperandMatcher(unsigned InsnVarID, unsigned OpIdx) + : OperandPredicateMatcher(OPM_MBB, InsnVarID, OpIdx) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_MBB; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckIsMBB") << MatchTable::Comment("MI") + << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op") + << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak; + } +}; + +class ImmOperandMatcher : public OperandPredicateMatcher { +public: + ImmOperandMatcher(unsigned InsnVarID, unsigned OpIdx) + : OperandPredicateMatcher(IPM_Imm, InsnVarID, OpIdx) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_Imm; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckIsImm") << MatchTable::Comment("MI") + << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op") + << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak; + } +}; + +/// Generates code to check that an operand is a G_CONSTANT with a particular +/// int. +class ConstantIntOperandMatcher : public OperandPredicateMatcher { +protected: + int64_t Value; + +public: + ConstantIntOperandMatcher(unsigned InsnVarID, unsigned OpIdx, int64_t Value) + : OperandPredicateMatcher(OPM_Int, InsnVarID, OpIdx), Value(Value) {} + + bool isIdentical(const PredicateMatcher &B) const override { + return OperandPredicateMatcher::isIdentical(B) && + Value == cast<ConstantIntOperandMatcher>(&B)->Value; + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_Int; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckConstantInt") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) + << MatchTable::IntValue(Value) << MatchTable::LineBreak; + } +}; + +/// Generates code to check that an operand is a raw int (where MO.isImm() or +/// MO.isCImm() is true). +class LiteralIntOperandMatcher : public OperandPredicateMatcher { +protected: + int64_t Value; + +public: + LiteralIntOperandMatcher(unsigned InsnVarID, unsigned OpIdx, int64_t Value) + : OperandPredicateMatcher(OPM_LiteralInt, InsnVarID, OpIdx), + Value(Value) {} + + bool isIdentical(const PredicateMatcher &B) const override { + return OperandPredicateMatcher::isIdentical(B) && + Value == cast<LiteralIntOperandMatcher>(&B)->Value; + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_LiteralInt; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckLiteralInt") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) + << MatchTable::IntValue(Value) << MatchTable::LineBreak; + } +}; + +/// Generates code to check that an operand is an CmpInst predicate +class CmpPredicateOperandMatcher : public OperandPredicateMatcher { +protected: + std::string PredName; + +public: + CmpPredicateOperandMatcher(unsigned InsnVarID, unsigned OpIdx, + std::string P) + : OperandPredicateMatcher(OPM_CmpPredicate, InsnVarID, OpIdx), PredName(P) {} + + bool isIdentical(const PredicateMatcher &B) const override { + return OperandPredicateMatcher::isIdentical(B) && + PredName == cast<CmpPredicateOperandMatcher>(&B)->PredName; + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_CmpPredicate; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckCmpPredicate") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) + << MatchTable::Comment("Predicate") + << MatchTable::NamedValue("CmpInst", PredName) + << MatchTable::LineBreak; + } +}; + +/// Generates code to check that an operand is an intrinsic ID. +class IntrinsicIDOperandMatcher : public OperandPredicateMatcher { +protected: + const CodeGenIntrinsic *II; + +public: + IntrinsicIDOperandMatcher(unsigned InsnVarID, unsigned OpIdx, + const CodeGenIntrinsic *II) + : OperandPredicateMatcher(OPM_IntrinsicID, InsnVarID, OpIdx), II(II) {} + + bool isIdentical(const PredicateMatcher &B) const override { + return OperandPredicateMatcher::isIdentical(B) && + II == cast<IntrinsicIDOperandMatcher>(&B)->II; + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_IntrinsicID; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckIntrinsicID") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) + << MatchTable::NamedValue("Intrinsic::" + II->EnumName) + << MatchTable::LineBreak; + } +}; + +/// Generates code to check that a set of predicates match for a particular +/// operand. +class OperandMatcher : public PredicateListMatcher<OperandPredicateMatcher> { +protected: + InstructionMatcher &Insn; + unsigned OpIdx; + std::string SymbolicName; + + /// The index of the first temporary variable allocated to this operand. The + /// number of allocated temporaries can be found with + /// countRendererFns(). + unsigned AllocatedTemporariesBaseID; + +public: + OperandMatcher(InstructionMatcher &Insn, unsigned OpIdx, + const std::string &SymbolicName, + unsigned AllocatedTemporariesBaseID) + : Insn(Insn), OpIdx(OpIdx), SymbolicName(SymbolicName), + AllocatedTemporariesBaseID(AllocatedTemporariesBaseID) {} + + bool hasSymbolicName() const { return !SymbolicName.empty(); } + const StringRef getSymbolicName() const { return SymbolicName; } + void setSymbolicName(StringRef Name) { + assert(SymbolicName.empty() && "Operand already has a symbolic name"); + SymbolicName = std::string(Name); + } + + /// Construct a new operand predicate and add it to the matcher. + template <class Kind, class... Args> + Optional<Kind *> addPredicate(Args &&... args) { + if (isSameAsAnotherOperand()) + return None; + Predicates.emplace_back(std::make_unique<Kind>( + getInsnVarID(), getOpIdx(), std::forward<Args>(args)...)); + return static_cast<Kind *>(Predicates.back().get()); + } + + unsigned getOpIdx() const { return OpIdx; } + unsigned getInsnVarID() const; + + std::string getOperandExpr(unsigned InsnVarID) const { + return "State.MIs[" + llvm::to_string(InsnVarID) + "]->getOperand(" + + llvm::to_string(OpIdx) + ")"; + } + + InstructionMatcher &getInstructionMatcher() const { return Insn; } + + Error addTypeCheckPredicate(const TypeSetByHwMode &VTy, + bool OperandIsAPointer); + + /// Emit MatchTable opcodes that test whether the instruction named in + /// InsnVarID matches all the predicates and all the operands. + void emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule) { + if (!Optimized) { + std::string Comment; + raw_string_ostream CommentOS(Comment); + CommentOS << "MIs[" << getInsnVarID() << "] "; + if (SymbolicName.empty()) + CommentOS << "Operand " << OpIdx; + else + CommentOS << SymbolicName; + Table << MatchTable::Comment(CommentOS.str()) << MatchTable::LineBreak; + } + + emitPredicateListOpcodes(Table, Rule); + } + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + bool isHigherPriorityThan(OperandMatcher &B) { + // Operand matchers involving more predicates have higher priority. + if (predicates_size() > B.predicates_size()) + return true; + if (predicates_size() < B.predicates_size()) + return false; + + // This assumes that predicates are added in a consistent order. + for (auto &&Predicate : zip(predicates(), B.predicates())) { + if (std::get<0>(Predicate)->isHigherPriorityThan(*std::get<1>(Predicate))) + return true; + if (std::get<1>(Predicate)->isHigherPriorityThan(*std::get<0>(Predicate))) + return false; + } + + return false; + }; + + /// Report the maximum number of temporary operands needed by the operand + /// matcher. + unsigned countRendererFns() { + return std::accumulate( + predicates().begin(), predicates().end(), 0, + [](unsigned A, + const std::unique_ptr<OperandPredicateMatcher> &Predicate) { + return A + Predicate->countRendererFns(); + }); + } + + unsigned getAllocatedTemporariesBaseID() const { + return AllocatedTemporariesBaseID; + } + + bool isSameAsAnotherOperand() { + for (const auto &Predicate : predicates()) + if (isa<SameOperandMatcher>(Predicate)) + return true; + return false; + } +}; + +Error OperandMatcher::addTypeCheckPredicate(const TypeSetByHwMode &VTy, + bool OperandIsAPointer) { + if (!VTy.isMachineValueType()) + return failedImport("unsupported typeset"); + + if (VTy.getMachineValueType() == MVT::iPTR && OperandIsAPointer) { + addPredicate<PointerToAnyOperandMatcher>(0); + return Error::success(); + } + + auto OpTyOrNone = MVTToLLT(VTy.getMachineValueType().SimpleTy); + if (!OpTyOrNone) + return failedImport("unsupported type"); + + if (OperandIsAPointer) + addPredicate<PointerToAnyOperandMatcher>(OpTyOrNone->get().getSizeInBits()); + else if (VTy.isPointer()) + addPredicate<LLTOperandMatcher>(LLT::pointer(VTy.getPtrAddrSpace(), + OpTyOrNone->get().getSizeInBits())); + else + addPredicate<LLTOperandMatcher>(*OpTyOrNone); + return Error::success(); +} + +unsigned ComplexPatternOperandMatcher::getAllocatedTemporariesBaseID() const { + return Operand.getAllocatedTemporariesBaseID(); +} + +/// Generates code to check a predicate on an instruction. +/// +/// Typical predicates include: +/// * The opcode of the instruction is a particular value. +/// * The nsw/nuw flag is/isn't set. +class InstructionPredicateMatcher : public PredicateMatcher { +public: + InstructionPredicateMatcher(PredicateKind Kind, unsigned InsnVarID) + : PredicateMatcher(Kind, InsnVarID) {} + virtual ~InstructionPredicateMatcher() {} + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + virtual bool + isHigherPriorityThan(const InstructionPredicateMatcher &B) const { + return Kind < B.Kind; + }; +}; + +template <> +std::string +PredicateListMatcher<PredicateMatcher>::getNoPredicateComment() const { + return "No instruction predicates"; +} + +/// Generates code to check the opcode of an instruction. +class InstructionOpcodeMatcher : public InstructionPredicateMatcher { +protected: + const CodeGenInstruction *I; + + static DenseMap<const CodeGenInstruction *, unsigned> OpcodeValues; + +public: + static void initOpcodeValuesMap(const CodeGenTarget &Target) { + OpcodeValues.clear(); + + unsigned OpcodeValue = 0; + for (const CodeGenInstruction *I : Target.getInstructionsByEnumValue()) + OpcodeValues[I] = OpcodeValue++; + } + + InstructionOpcodeMatcher(unsigned InsnVarID, const CodeGenInstruction *I) + : InstructionPredicateMatcher(IPM_Opcode, InsnVarID), I(I) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_Opcode; + } + + bool isIdentical(const PredicateMatcher &B) const override { + return InstructionPredicateMatcher::isIdentical(B) && + I == cast<InstructionOpcodeMatcher>(&B)->I; + } + MatchTableRecord getValue() const override { + const auto VI = OpcodeValues.find(I); + if (VI != OpcodeValues.end()) + return MatchTable::NamedValue(I->Namespace, I->TheDef->getName(), + VI->second); + return MatchTable::NamedValue(I->Namespace, I->TheDef->getName()); + } + bool hasValue() const override { return OpcodeValues.count(I); } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckOpcode") << MatchTable::Comment("MI") + << MatchTable::IntValue(InsnVarID) << getValue() + << MatchTable::LineBreak; + } + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + bool + isHigherPriorityThan(const InstructionPredicateMatcher &B) const override { + if (InstructionPredicateMatcher::isHigherPriorityThan(B)) + return true; + if (B.InstructionPredicateMatcher::isHigherPriorityThan(*this)) + return false; + + // Prioritize opcodes for cosmetic reasons in the generated source. Although + // this is cosmetic at the moment, we may want to drive a similar ordering + // using instruction frequency information to improve compile time. + if (const InstructionOpcodeMatcher *BO = + dyn_cast<InstructionOpcodeMatcher>(&B)) + return I->TheDef->getName() < BO->I->TheDef->getName(); + + return false; + }; + + bool isConstantInstruction() const { + return I->TheDef->getName() == "G_CONSTANT"; + } + + StringRef getOpcode() const { return I->TheDef->getName(); } + bool isVariadicNumOperands() const { return I->Operands.isVariadic; } + + StringRef getOperandType(unsigned OpIdx) const { + return I->Operands[OpIdx].OperandType; + } +}; + +DenseMap<const CodeGenInstruction *, unsigned> + InstructionOpcodeMatcher::OpcodeValues; + +class InstructionNumOperandsMatcher final : public InstructionPredicateMatcher { + unsigned NumOperands = 0; + +public: + InstructionNumOperandsMatcher(unsigned InsnVarID, unsigned NumOperands) + : InstructionPredicateMatcher(IPM_NumOperands, InsnVarID), + NumOperands(NumOperands) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_NumOperands; + } + + bool isIdentical(const PredicateMatcher &B) const override { + return InstructionPredicateMatcher::isIdentical(B) && + NumOperands == cast<InstructionNumOperandsMatcher>(&B)->NumOperands; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckNumOperands") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Expected") + << MatchTable::IntValue(NumOperands) << MatchTable::LineBreak; + } +}; + +/// Generates code to check that this instruction is a constant whose value +/// meets an immediate predicate. +/// +/// Immediates are slightly odd since they are typically used like an operand +/// but are represented as an operator internally. We typically write simm8:$src +/// in a tablegen pattern, but this is just syntactic sugar for +/// (imm:i32)<<P:Predicate_simm8>>:$imm which more directly describes the nodes +/// that will be matched and the predicate (which is attached to the imm +/// operator) that will be tested. In SelectionDAG this describes a +/// ConstantSDNode whose internal value will be tested using the simm8 predicate. +/// +/// The corresponding GlobalISel representation is %1 = G_CONSTANT iN Value. In +/// this representation, the immediate could be tested with an +/// InstructionMatcher, InstructionOpcodeMatcher, OperandMatcher, and a +/// OperandPredicateMatcher-subclass to check the Value meets the predicate but +/// there are two implementation issues with producing that matcher +/// configuration from the SelectionDAG pattern: +/// * ImmLeaf is a PatFrag whose root is an InstructionMatcher. This means that +/// were we to sink the immediate predicate to the operand we would have to +/// have two partial implementations of PatFrag support, one for immediates +/// and one for non-immediates. +/// * At the point we handle the predicate, the OperandMatcher hasn't been +/// created yet. If we were to sink the predicate to the OperandMatcher we +/// would also have to complicate (or duplicate) the code that descends and +/// creates matchers for the subtree. +/// Overall, it's simpler to handle it in the place it was found. +class InstructionImmPredicateMatcher : public InstructionPredicateMatcher { +protected: + TreePredicateFn Predicate; + +public: + InstructionImmPredicateMatcher(unsigned InsnVarID, + const TreePredicateFn &Predicate) + : InstructionPredicateMatcher(IPM_ImmPredicate, InsnVarID), + Predicate(Predicate) {} + + bool isIdentical(const PredicateMatcher &B) const override { + return InstructionPredicateMatcher::isIdentical(B) && + Predicate.getOrigPatFragRecord() == + cast<InstructionImmPredicateMatcher>(&B) + ->Predicate.getOrigPatFragRecord(); + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_ImmPredicate; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode(getMatchOpcodeForPredicate(Predicate)) + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("Predicate") + << MatchTable::NamedValue(getEnumNameForPredicate(Predicate)) + << MatchTable::LineBreak; + } +}; + +/// Generates code to check that a memory instruction has a atomic ordering +/// MachineMemoryOperand. +class AtomicOrderingMMOPredicateMatcher : public InstructionPredicateMatcher { +public: + enum AOComparator { + AO_Exactly, + AO_OrStronger, + AO_WeakerThan, + }; + +protected: + StringRef Order; + AOComparator Comparator; + +public: + AtomicOrderingMMOPredicateMatcher(unsigned InsnVarID, StringRef Order, + AOComparator Comparator = AO_Exactly) + : InstructionPredicateMatcher(IPM_AtomicOrderingMMO, InsnVarID), + Order(Order), Comparator(Comparator) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_AtomicOrderingMMO; + } + + bool isIdentical(const PredicateMatcher &B) const override { + if (!InstructionPredicateMatcher::isIdentical(B)) + return false; + const auto &R = *cast<AtomicOrderingMMOPredicateMatcher>(&B); + return Order == R.Order && Comparator == R.Comparator; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + StringRef Opcode = "GIM_CheckAtomicOrdering"; + + if (Comparator == AO_OrStronger) + Opcode = "GIM_CheckAtomicOrderingOrStrongerThan"; + if (Comparator == AO_WeakerThan) + Opcode = "GIM_CheckAtomicOrderingWeakerThan"; + + Table << MatchTable::Opcode(Opcode) << MatchTable::Comment("MI") + << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Order") + << MatchTable::NamedValue(("(int64_t)AtomicOrdering::" + Order).str()) + << MatchTable::LineBreak; + } +}; + +/// Generates code to check that the size of an MMO is exactly N bytes. +class MemorySizePredicateMatcher : public InstructionPredicateMatcher { +protected: + unsigned MMOIdx; + uint64_t Size; + +public: + MemorySizePredicateMatcher(unsigned InsnVarID, unsigned MMOIdx, unsigned Size) + : InstructionPredicateMatcher(IPM_MemoryLLTSize, InsnVarID), + MMOIdx(MMOIdx), Size(Size) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_MemoryLLTSize; + } + bool isIdentical(const PredicateMatcher &B) const override { + return InstructionPredicateMatcher::isIdentical(B) && + MMOIdx == cast<MemorySizePredicateMatcher>(&B)->MMOIdx && + Size == cast<MemorySizePredicateMatcher>(&B)->Size; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckMemorySizeEqualTo") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx) + << MatchTable::Comment("Size") << MatchTable::IntValue(Size) + << MatchTable::LineBreak; + } +}; + +class MemoryAddressSpacePredicateMatcher : public InstructionPredicateMatcher { +protected: + unsigned MMOIdx; + SmallVector<unsigned, 4> AddrSpaces; + +public: + MemoryAddressSpacePredicateMatcher(unsigned InsnVarID, unsigned MMOIdx, + ArrayRef<unsigned> AddrSpaces) + : InstructionPredicateMatcher(IPM_MemoryAddressSpace, InsnVarID), + MMOIdx(MMOIdx), AddrSpaces(AddrSpaces.begin(), AddrSpaces.end()) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_MemoryAddressSpace; + } + bool isIdentical(const PredicateMatcher &B) const override { + if (!InstructionPredicateMatcher::isIdentical(B)) + return false; + auto *Other = cast<MemoryAddressSpacePredicateMatcher>(&B); + return MMOIdx == Other->MMOIdx && AddrSpaces == Other->AddrSpaces; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckMemoryAddressSpace") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx) + // Encode number of address spaces to expect. + << MatchTable::Comment("NumAddrSpace") + << MatchTable::IntValue(AddrSpaces.size()); + for (unsigned AS : AddrSpaces) + Table << MatchTable::Comment("AddrSpace") << MatchTable::IntValue(AS); + + Table << MatchTable::LineBreak; + } +}; + +class MemoryAlignmentPredicateMatcher : public InstructionPredicateMatcher { +protected: + unsigned MMOIdx; + int MinAlign; + +public: + MemoryAlignmentPredicateMatcher(unsigned InsnVarID, unsigned MMOIdx, + int MinAlign) + : InstructionPredicateMatcher(IPM_MemoryAlignment, InsnVarID), + MMOIdx(MMOIdx), MinAlign(MinAlign) { + assert(MinAlign > 0); + } + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_MemoryAlignment; + } + + bool isIdentical(const PredicateMatcher &B) const override { + if (!InstructionPredicateMatcher::isIdentical(B)) + return false; + auto *Other = cast<MemoryAlignmentPredicateMatcher>(&B); + return MMOIdx == Other->MMOIdx && MinAlign == Other->MinAlign; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckMemoryAlignment") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx) + << MatchTable::Comment("MinAlign") << MatchTable::IntValue(MinAlign) + << MatchTable::LineBreak; + } +}; + +/// Generates code to check that the size of an MMO is less-than, equal-to, or +/// greater than a given LLT. +class MemoryVsLLTSizePredicateMatcher : public InstructionPredicateMatcher { +public: + enum RelationKind { + GreaterThan, + EqualTo, + LessThan, + }; + +protected: + unsigned MMOIdx; + RelationKind Relation; + unsigned OpIdx; + +public: + MemoryVsLLTSizePredicateMatcher(unsigned InsnVarID, unsigned MMOIdx, + enum RelationKind Relation, + unsigned OpIdx) + : InstructionPredicateMatcher(IPM_MemoryVsLLTSize, InsnVarID), + MMOIdx(MMOIdx), Relation(Relation), OpIdx(OpIdx) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == IPM_MemoryVsLLTSize; + } + bool isIdentical(const PredicateMatcher &B) const override { + return InstructionPredicateMatcher::isIdentical(B) && + MMOIdx == cast<MemoryVsLLTSizePredicateMatcher>(&B)->MMOIdx && + Relation == cast<MemoryVsLLTSizePredicateMatcher>(&B)->Relation && + OpIdx == cast<MemoryVsLLTSizePredicateMatcher>(&B)->OpIdx; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode(Relation == EqualTo + ? "GIM_CheckMemorySizeEqualToLLT" + : Relation == GreaterThan + ? "GIM_CheckMemorySizeGreaterThanLLT" + : "GIM_CheckMemorySizeLessThanLLT") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx) + << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx) + << MatchTable::LineBreak; + } +}; + +/// Generates code to check an arbitrary C++ instruction predicate. +class GenericInstructionPredicateMatcher : public InstructionPredicateMatcher { +protected: + TreePredicateFn Predicate; + +public: + GenericInstructionPredicateMatcher(unsigned InsnVarID, + TreePredicateFn Predicate) + : InstructionPredicateMatcher(IPM_GenericPredicate, InsnVarID), + Predicate(Predicate) {} + + static bool classof(const InstructionPredicateMatcher *P) { + return P->getKind() == IPM_GenericPredicate; + } + bool isIdentical(const PredicateMatcher &B) const override { + return InstructionPredicateMatcher::isIdentical(B) && + Predicate == + static_cast<const GenericInstructionPredicateMatcher &>(B) + .Predicate; + } + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIM_CheckCxxInsnPredicate") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("FnId") + << MatchTable::NamedValue(getEnumNameForPredicate(Predicate)) + << MatchTable::LineBreak; + } +}; + +/// Generates code to check that a set of predicates and operands match for a +/// particular instruction. +/// +/// Typical predicates include: +/// * Has a specific opcode. +/// * Has an nsw/nuw flag or doesn't. +class InstructionMatcher final : public PredicateListMatcher<PredicateMatcher> { +protected: + typedef std::vector<std::unique_ptr<OperandMatcher>> OperandVec; + + RuleMatcher &Rule; + + /// The operands to match. All rendered operands must be present even if the + /// condition is always true. + OperandVec Operands; + bool NumOperandsCheck = true; + + std::string SymbolicName; + unsigned InsnVarID; + + /// PhysRegInputs - List list has an entry for each explicitly specified + /// physreg input to the pattern. The first elt is the Register node, the + /// second is the recorded slot number the input pattern match saved it in. + SmallVector<std::pair<Record *, unsigned>, 2> PhysRegInputs; + +public: + InstructionMatcher(RuleMatcher &Rule, StringRef SymbolicName) + : Rule(Rule), SymbolicName(SymbolicName) { + // We create a new instruction matcher. + // Get a new ID for that instruction. + InsnVarID = Rule.implicitlyDefineInsnVar(*this); + } + + /// Construct a new instruction predicate and add it to the matcher. + template <class Kind, class... Args> + Optional<Kind *> addPredicate(Args &&... args) { + Predicates.emplace_back( + std::make_unique<Kind>(getInsnVarID(), std::forward<Args>(args)...)); + return static_cast<Kind *>(Predicates.back().get()); + } + + RuleMatcher &getRuleMatcher() const { return Rule; } + + unsigned getInsnVarID() const { return InsnVarID; } + + /// Add an operand to the matcher. + OperandMatcher &addOperand(unsigned OpIdx, const std::string &SymbolicName, + unsigned AllocatedTemporariesBaseID) { + Operands.emplace_back(new OperandMatcher(*this, OpIdx, SymbolicName, + AllocatedTemporariesBaseID)); + if (!SymbolicName.empty()) + Rule.defineOperand(SymbolicName, *Operands.back()); + + return *Operands.back(); + } + + OperandMatcher &getOperand(unsigned OpIdx) { + auto I = std::find_if(Operands.begin(), Operands.end(), + [&OpIdx](const std::unique_ptr<OperandMatcher> &X) { + return X->getOpIdx() == OpIdx; + }); + if (I != Operands.end()) + return **I; + llvm_unreachable("Failed to lookup operand"); + } + + OperandMatcher &addPhysRegInput(Record *Reg, unsigned OpIdx, + unsigned TempOpIdx) { + assert(SymbolicName.empty()); + OperandMatcher *OM = new OperandMatcher(*this, OpIdx, "", TempOpIdx); + Operands.emplace_back(OM); + Rule.definePhysRegOperand(Reg, *OM); + PhysRegInputs.emplace_back(Reg, OpIdx); + return *OM; + } + + ArrayRef<std::pair<Record *, unsigned>> getPhysRegInputs() const { + return PhysRegInputs; + } + + StringRef getSymbolicName() const { return SymbolicName; } + unsigned getNumOperands() const { return Operands.size(); } + OperandVec::iterator operands_begin() { return Operands.begin(); } + OperandVec::iterator operands_end() { return Operands.end(); } + iterator_range<OperandVec::iterator> operands() { + return make_range(operands_begin(), operands_end()); + } + OperandVec::const_iterator operands_begin() const { return Operands.begin(); } + OperandVec::const_iterator operands_end() const { return Operands.end(); } + iterator_range<OperandVec::const_iterator> operands() const { + return make_range(operands_begin(), operands_end()); + } + bool operands_empty() const { return Operands.empty(); } + + void pop_front() { Operands.erase(Operands.begin()); } + + void optimize(); + + /// Emit MatchTable opcodes that test whether the instruction named in + /// InsnVarName matches all the predicates and all the operands. + void emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule) { + if (NumOperandsCheck) + InstructionNumOperandsMatcher(InsnVarID, getNumOperands()) + .emitPredicateOpcodes(Table, Rule); + + // First emit all instruction level predicates need to be verified before we + // can verify operands. + emitFilteredPredicateListOpcodes( + [](const PredicateMatcher &P) { + return !P.dependsOnOperands(); + }, Table, Rule); + + // Emit all operand constraints. + for (const auto &Operand : Operands) + Operand->emitPredicateOpcodes(Table, Rule); + + // All of the tablegen defined predicates should now be matched. Now emit + // any custom predicates that rely on all generated checks. + emitFilteredPredicateListOpcodes( + [](const PredicateMatcher &P) { + return P.dependsOnOperands(); + }, Table, Rule); + } + + /// Compare the priority of this object and B. + /// + /// Returns true if this object is more important than B. + bool isHigherPriorityThan(InstructionMatcher &B) { + // Instruction matchers involving more operands have higher priority. + if (Operands.size() > B.Operands.size()) + return true; + if (Operands.size() < B.Operands.size()) + return false; + + for (auto &&P : zip(predicates(), B.predicates())) { + auto L = static_cast<InstructionPredicateMatcher *>(std::get<0>(P).get()); + auto R = static_cast<InstructionPredicateMatcher *>(std::get<1>(P).get()); + if (L->isHigherPriorityThan(*R)) + return true; + if (R->isHigherPriorityThan(*L)) + return false; + } + + for (auto Operand : zip(Operands, B.Operands)) { + if (std::get<0>(Operand)->isHigherPriorityThan(*std::get<1>(Operand))) + return true; + if (std::get<1>(Operand)->isHigherPriorityThan(*std::get<0>(Operand))) + return false; + } + + return false; + }; + + /// Report the maximum number of temporary operands needed by the instruction + /// matcher. + unsigned countRendererFns() { + return std::accumulate( + predicates().begin(), predicates().end(), 0, + [](unsigned A, + const std::unique_ptr<PredicateMatcher> &Predicate) { + return A + Predicate->countRendererFns(); + }) + + std::accumulate( + Operands.begin(), Operands.end(), 0, + [](unsigned A, const std::unique_ptr<OperandMatcher> &Operand) { + return A + Operand->countRendererFns(); + }); + } + + InstructionOpcodeMatcher &getOpcodeMatcher() { + for (auto &P : predicates()) + if (auto *OpMatcher = dyn_cast<InstructionOpcodeMatcher>(P.get())) + return *OpMatcher; + llvm_unreachable("Didn't find an opcode matcher"); + } + + bool isConstantInstruction() { + return getOpcodeMatcher().isConstantInstruction(); + } + + StringRef getOpcode() { return getOpcodeMatcher().getOpcode(); } +}; + +StringRef RuleMatcher::getOpcode() const { + return Matchers.front()->getOpcode(); +} + +unsigned RuleMatcher::getNumOperands() const { + return Matchers.front()->getNumOperands(); +} + +LLTCodeGen RuleMatcher::getFirstConditionAsRootType() { + InstructionMatcher &InsnMatcher = *Matchers.front(); + if (!InsnMatcher.predicates_empty()) + if (const auto *TM = + dyn_cast<LLTOperandMatcher>(&**InsnMatcher.predicates_begin())) + if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0) + return TM->getTy(); + return {}; +} + +/// Generates code to check that the operand is a register defined by an +/// instruction that matches the given instruction matcher. +/// +/// For example, the pattern: +/// (set $dst, (G_MUL (G_ADD $src1, $src2), $src3)) +/// would use an InstructionOperandMatcher for operand 1 of the G_MUL to match +/// the: +/// (G_ADD $src1, $src2) +/// subpattern. +class InstructionOperandMatcher : public OperandPredicateMatcher { +protected: + std::unique_ptr<InstructionMatcher> InsnMatcher; + +public: + InstructionOperandMatcher(unsigned InsnVarID, unsigned OpIdx, + RuleMatcher &Rule, StringRef SymbolicName) + : OperandPredicateMatcher(OPM_Instruction, InsnVarID, OpIdx), + InsnMatcher(new InstructionMatcher(Rule, SymbolicName)) {} + + static bool classof(const PredicateMatcher *P) { + return P->getKind() == OPM_Instruction; + } + + InstructionMatcher &getInsnMatcher() const { return *InsnMatcher; } + + void emitCaptureOpcodes(MatchTable &Table, RuleMatcher &Rule) const { + const unsigned NewInsnVarID = InsnMatcher->getInsnVarID(); + Table << MatchTable::Opcode("GIM_RecordInsn") + << MatchTable::Comment("DefineMI") + << MatchTable::IntValue(NewInsnVarID) << MatchTable::Comment("MI") + << MatchTable::IntValue(getInsnVarID()) + << MatchTable::Comment("OpIdx") << MatchTable::IntValue(getOpIdx()) + << MatchTable::Comment("MIs[" + llvm::to_string(NewInsnVarID) + "]") + << MatchTable::LineBreak; + } + + void emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const override { + emitCaptureOpcodes(Table, Rule); + InsnMatcher->emitPredicateOpcodes(Table, Rule); + } + + bool isHigherPriorityThan(const OperandPredicateMatcher &B) const override { + if (OperandPredicateMatcher::isHigherPriorityThan(B)) + return true; + if (B.OperandPredicateMatcher::isHigherPriorityThan(*this)) + return false; + + if (const InstructionOperandMatcher *BP = + dyn_cast<InstructionOperandMatcher>(&B)) + if (InsnMatcher->isHigherPriorityThan(*BP->InsnMatcher)) + return true; + return false; + } +}; + +void InstructionMatcher::optimize() { + SmallVector<std::unique_ptr<PredicateMatcher>, 8> Stash; + const auto &OpcMatcher = getOpcodeMatcher(); + + Stash.push_back(predicates_pop_front()); + if (Stash.back().get() == &OpcMatcher) { + if (NumOperandsCheck && OpcMatcher.isVariadicNumOperands()) + Stash.emplace_back( + new InstructionNumOperandsMatcher(InsnVarID, getNumOperands())); + NumOperandsCheck = false; + + for (auto &OM : Operands) + for (auto &OP : OM->predicates()) + if (isa<IntrinsicIDOperandMatcher>(OP)) { + Stash.push_back(std::move(OP)); + OM->eraseNullPredicates(); + break; + } + } + + if (InsnVarID > 0) { + assert(!Operands.empty() && "Nested instruction is expected to def a vreg"); + for (auto &OP : Operands[0]->predicates()) + OP.reset(); + Operands[0]->eraseNullPredicates(); + } + for (auto &OM : Operands) { + for (auto &OP : OM->predicates()) + if (isa<LLTOperandMatcher>(OP)) + Stash.push_back(std::move(OP)); + OM->eraseNullPredicates(); + } + while (!Stash.empty()) + prependPredicate(Stash.pop_back_val()); +} + +//===- Actions ------------------------------------------------------------===// +class OperandRenderer { +public: + enum RendererKind { + OR_Copy, + OR_CopyOrAddZeroReg, + OR_CopySubReg, + OR_CopyPhysReg, + OR_CopyConstantAsImm, + OR_CopyFConstantAsFPImm, + OR_Imm, + OR_SubRegIndex, + OR_Register, + OR_TempRegister, + OR_ComplexPattern, + OR_Custom, + OR_CustomOperand + }; + +protected: + RendererKind Kind; + +public: + OperandRenderer(RendererKind Kind) : Kind(Kind) {} + virtual ~OperandRenderer() {} + + RendererKind getKind() const { return Kind; } + + virtual void emitRenderOpcodes(MatchTable &Table, + RuleMatcher &Rule) const = 0; +}; + +/// A CopyRenderer emits code to copy a single operand from an existing +/// instruction to the one being built. +class CopyRenderer : public OperandRenderer { +protected: + unsigned NewInsnID; + /// The name of the operand. + const StringRef SymbolicName; + +public: + CopyRenderer(unsigned NewInsnID, StringRef SymbolicName) + : OperandRenderer(OR_Copy), NewInsnID(NewInsnID), + SymbolicName(SymbolicName) { + assert(!SymbolicName.empty() && "Cannot copy from an unspecified source"); + } + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_Copy; + } + + const StringRef getSymbolicName() const { return SymbolicName; } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName); + unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher()); + Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID") + << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx") + << MatchTable::IntValue(Operand.getOpIdx()) + << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; + } +}; + +/// A CopyRenderer emits code to copy a virtual register to a specific physical +/// register. +class CopyPhysRegRenderer : public OperandRenderer { +protected: + unsigned NewInsnID; + Record *PhysReg; + +public: + CopyPhysRegRenderer(unsigned NewInsnID, Record *Reg) + : OperandRenderer(OR_CopyPhysReg), NewInsnID(NewInsnID), + PhysReg(Reg) { + assert(PhysReg); + } + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_CopyPhysReg; + } + + Record *getPhysReg() const { return PhysReg; } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + const OperandMatcher &Operand = Rule.getPhysRegOperandMatcher(PhysReg); + unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher()); + Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID") + << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx") + << MatchTable::IntValue(Operand.getOpIdx()) + << MatchTable::Comment(PhysReg->getName()) + << MatchTable::LineBreak; + } +}; + +/// A CopyOrAddZeroRegRenderer emits code to copy a single operand from an +/// existing instruction to the one being built. If the operand turns out to be +/// a 'G_CONSTANT 0' then it replaces the operand with a zero register. +class CopyOrAddZeroRegRenderer : public OperandRenderer { +protected: + unsigned NewInsnID; + /// The name of the operand. + const StringRef SymbolicName; + const Record *ZeroRegisterDef; + +public: + CopyOrAddZeroRegRenderer(unsigned NewInsnID, + StringRef SymbolicName, Record *ZeroRegisterDef) + : OperandRenderer(OR_CopyOrAddZeroReg), NewInsnID(NewInsnID), + SymbolicName(SymbolicName), ZeroRegisterDef(ZeroRegisterDef) { + assert(!SymbolicName.empty() && "Cannot copy from an unspecified source"); + } + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_CopyOrAddZeroReg; + } + + const StringRef getSymbolicName() const { return SymbolicName; } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName); + unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher()); + Table << MatchTable::Opcode("GIR_CopyOrAddZeroReg") + << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID) + << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx") + << MatchTable::IntValue(Operand.getOpIdx()) + << MatchTable::NamedValue( + (ZeroRegisterDef->getValue("Namespace") + ? ZeroRegisterDef->getValueAsString("Namespace") + : ""), + ZeroRegisterDef->getName()) + << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; + } +}; + +/// A CopyConstantAsImmRenderer emits code to render a G_CONSTANT instruction to +/// an extended immediate operand. +class CopyConstantAsImmRenderer : public OperandRenderer { +protected: + unsigned NewInsnID; + /// The name of the operand. + const std::string SymbolicName; + bool Signed; + +public: + CopyConstantAsImmRenderer(unsigned NewInsnID, StringRef SymbolicName) + : OperandRenderer(OR_CopyConstantAsImm), NewInsnID(NewInsnID), + SymbolicName(SymbolicName), Signed(true) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_CopyConstantAsImm; + } + + const StringRef getSymbolicName() const { return SymbolicName; } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName); + unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher); + Table << MatchTable::Opcode(Signed ? "GIR_CopyConstantAsSImm" + : "GIR_CopyConstantAsUImm") + << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID) + << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OldInsnVarID) + << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; + } +}; + +/// A CopyFConstantAsFPImmRenderer emits code to render a G_FCONSTANT +/// instruction to an extended immediate operand. +class CopyFConstantAsFPImmRenderer : public OperandRenderer { +protected: + unsigned NewInsnID; + /// The name of the operand. + const std::string SymbolicName; + +public: + CopyFConstantAsFPImmRenderer(unsigned NewInsnID, StringRef SymbolicName) + : OperandRenderer(OR_CopyFConstantAsFPImm), NewInsnID(NewInsnID), + SymbolicName(SymbolicName) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_CopyFConstantAsFPImm; + } + + const StringRef getSymbolicName() const { return SymbolicName; } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName); + unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher); + Table << MatchTable::Opcode("GIR_CopyFConstantAsFPImm") + << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID) + << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OldInsnVarID) + << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; + } +}; + +/// A CopySubRegRenderer emits code to copy a single register operand from an +/// existing instruction to the one being built and indicate that only a +/// subregister should be copied. +class CopySubRegRenderer : public OperandRenderer { +protected: + unsigned NewInsnID; + /// The name of the operand. + const StringRef SymbolicName; + /// The subregister to extract. + const CodeGenSubRegIndex *SubReg; + +public: + CopySubRegRenderer(unsigned NewInsnID, StringRef SymbolicName, + const CodeGenSubRegIndex *SubReg) + : OperandRenderer(OR_CopySubReg), NewInsnID(NewInsnID), + SymbolicName(SymbolicName), SubReg(SubReg) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_CopySubReg; + } + + const StringRef getSymbolicName() const { return SymbolicName; } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName); + unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher()); + Table << MatchTable::Opcode("GIR_CopySubReg") + << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID) + << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx") + << MatchTable::IntValue(Operand.getOpIdx()) + << MatchTable::Comment("SubRegIdx") + << MatchTable::IntValue(SubReg->EnumValue) + << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; + } +}; + +/// Adds a specific physical register to the instruction being built. +/// This is typically useful for WZR/XZR on AArch64. +class AddRegisterRenderer : public OperandRenderer { +protected: + unsigned InsnID; + const Record *RegisterDef; + bool IsDef; + +public: + AddRegisterRenderer(unsigned InsnID, const Record *RegisterDef, + bool IsDef = false) + : OperandRenderer(OR_Register), InsnID(InsnID), RegisterDef(RegisterDef), + IsDef(IsDef) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_Register; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIR_AddRegister") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::NamedValue( + (RegisterDef->getValue("Namespace") + ? RegisterDef->getValueAsString("Namespace") + : ""), + RegisterDef->getName()) + << MatchTable::Comment("AddRegisterRegFlags"); + + // TODO: This is encoded as a 64-bit element, but only 16 or 32-bits are + // really needed for a physical register reference. We can pack the + // register and flags in a single field. + if (IsDef) + Table << MatchTable::NamedValue("RegState::Define"); + else + Table << MatchTable::IntValue(0); + Table << MatchTable::LineBreak; + } +}; + +/// Adds a specific temporary virtual register to the instruction being built. +/// This is used to chain instructions together when emitting multiple +/// instructions. +class TempRegRenderer : public OperandRenderer { +protected: + unsigned InsnID; + unsigned TempRegID; + const CodeGenSubRegIndex *SubRegIdx; + bool IsDef; + +public: + TempRegRenderer(unsigned InsnID, unsigned TempRegID, bool IsDef = false, + const CodeGenSubRegIndex *SubReg = nullptr) + : OperandRenderer(OR_Register), InsnID(InsnID), TempRegID(TempRegID), + SubRegIdx(SubReg), IsDef(IsDef) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_TempRegister; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + if (SubRegIdx) { + assert(!IsDef); + Table << MatchTable::Opcode("GIR_AddTempSubRegister"); + } else + Table << MatchTable::Opcode("GIR_AddTempRegister"); + + Table << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID) + << MatchTable::Comment("TempRegFlags"); + + if (IsDef) + Table << MatchTable::NamedValue("RegState::Define"); + else + Table << MatchTable::IntValue(0); + + if (SubRegIdx) + Table << MatchTable::NamedValue(SubRegIdx->getQualifiedName()); + Table << MatchTable::LineBreak; + } +}; + +/// Adds a specific immediate to the instruction being built. +class ImmRenderer : public OperandRenderer { +protected: + unsigned InsnID; + int64_t Imm; + +public: + ImmRenderer(unsigned InsnID, int64_t Imm) + : OperandRenderer(OR_Imm), InsnID(InsnID), Imm(Imm) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_Imm; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID") + << MatchTable::IntValue(InsnID) << MatchTable::Comment("Imm") + << MatchTable::IntValue(Imm) << MatchTable::LineBreak; + } +}; + +/// Adds an enum value for a subreg index to the instruction being built. +class SubRegIndexRenderer : public OperandRenderer { +protected: + unsigned InsnID; + const CodeGenSubRegIndex *SubRegIdx; + +public: + SubRegIndexRenderer(unsigned InsnID, const CodeGenSubRegIndex *SRI) + : OperandRenderer(OR_SubRegIndex), InsnID(InsnID), SubRegIdx(SRI) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_SubRegIndex; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID") + << MatchTable::IntValue(InsnID) << MatchTable::Comment("SubRegIndex") + << MatchTable::IntValue(SubRegIdx->EnumValue) + << MatchTable::LineBreak; + } +}; + +/// Adds operands by calling a renderer function supplied by the ComplexPattern +/// matcher function. +class RenderComplexPatternOperand : public OperandRenderer { +private: + unsigned InsnID; + const Record &TheDef; + /// The name of the operand. + const StringRef SymbolicName; + /// The renderer number. This must be unique within a rule since it's used to + /// identify a temporary variable to hold the renderer function. + unsigned RendererID; + /// When provided, this is the suboperand of the ComplexPattern operand to + /// render. Otherwise all the suboperands will be rendered. + Optional<unsigned> SubOperand; + + unsigned getNumOperands() const { + return TheDef.getValueAsDag("Operands")->getNumArgs(); + } + +public: + RenderComplexPatternOperand(unsigned InsnID, const Record &TheDef, + StringRef SymbolicName, unsigned RendererID, + Optional<unsigned> SubOperand = None) + : OperandRenderer(OR_ComplexPattern), InsnID(InsnID), TheDef(TheDef), + SymbolicName(SymbolicName), RendererID(RendererID), + SubOperand(SubOperand) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_ComplexPattern; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + Table << MatchTable::Opcode(SubOperand.hasValue() ? "GIR_ComplexSubOperandRenderer" + : "GIR_ComplexRenderer") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("RendererID") + << MatchTable::IntValue(RendererID); + if (SubOperand.hasValue()) + Table << MatchTable::Comment("SubOperand") + << MatchTable::IntValue(SubOperand.getValue()); + Table << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; + } +}; + +class CustomRenderer : public OperandRenderer { +protected: + unsigned InsnID; + const Record &Renderer; + /// The name of the operand. + const std::string SymbolicName; + +public: + CustomRenderer(unsigned InsnID, const Record &Renderer, + StringRef SymbolicName) + : OperandRenderer(OR_Custom), InsnID(InsnID), Renderer(Renderer), + SymbolicName(SymbolicName) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_Custom; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName); + unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher); + Table << MatchTable::Opcode("GIR_CustomRenderer") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OldInsnVarID) + << MatchTable::Comment("Renderer") + << MatchTable::NamedValue( + "GICR_" + Renderer.getValueAsString("RendererFn").str()) + << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; + } +}; + +class CustomOperandRenderer : public OperandRenderer { +protected: + unsigned InsnID; + const Record &Renderer; + /// The name of the operand. + const std::string SymbolicName; + +public: + CustomOperandRenderer(unsigned InsnID, const Record &Renderer, + StringRef SymbolicName) + : OperandRenderer(OR_CustomOperand), InsnID(InsnID), Renderer(Renderer), + SymbolicName(SymbolicName) {} + + static bool classof(const OperandRenderer *R) { + return R->getKind() == OR_CustomOperand; + } + + void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + const OperandMatcher &OpdMatcher = Rule.getOperandMatcher(SymbolicName); + Table << MatchTable::Opcode("GIR_CustomOperandRenderer") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("OldInsnID") + << MatchTable::IntValue(OpdMatcher.getInsnVarID()) + << MatchTable::Comment("OpIdx") + << MatchTable::IntValue(OpdMatcher.getOpIdx()) + << MatchTable::Comment("OperandRenderer") + << MatchTable::NamedValue( + "GICR_" + Renderer.getValueAsString("RendererFn").str()) + << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak; + } +}; + +/// An action taken when all Matcher predicates succeeded for a parent rule. +/// +/// Typical actions include: +/// * Changing the opcode of an instruction. +/// * Adding an operand to an instruction. +class MatchAction { +public: + virtual ~MatchAction() {} + + /// Emit the MatchTable opcodes to implement the action. + virtual void emitActionOpcodes(MatchTable &Table, + RuleMatcher &Rule) const = 0; +}; + +/// Generates a comment describing the matched rule being acted upon. +class DebugCommentAction : public MatchAction { +private: + std::string S; + +public: + DebugCommentAction(StringRef S) : S(std::string(S)) {} + + void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + Table << MatchTable::Comment(S) << MatchTable::LineBreak; + } +}; + +/// Generates code to build an instruction or mutate an existing instruction +/// into the desired instruction when this is possible. +class BuildMIAction : public MatchAction { +private: + unsigned InsnID; + const CodeGenInstruction *I; + InstructionMatcher *Matched; + std::vector<std::unique_ptr<OperandRenderer>> OperandRenderers; + + /// True if the instruction can be built solely by mutating the opcode. + bool canMutate(RuleMatcher &Rule, const InstructionMatcher *Insn) const { + if (!Insn) + return false; + + if (OperandRenderers.size() != Insn->getNumOperands()) + return false; + + for (const auto &Renderer : enumerate(OperandRenderers)) { + if (const auto *Copy = dyn_cast<CopyRenderer>(&*Renderer.value())) { + const OperandMatcher &OM = Rule.getOperandMatcher(Copy->getSymbolicName()); + if (Insn != &OM.getInstructionMatcher() || + OM.getOpIdx() != Renderer.index()) + return false; + } else + return false; + } + + return true; + } + +public: + BuildMIAction(unsigned InsnID, const CodeGenInstruction *I) + : InsnID(InsnID), I(I), Matched(nullptr) {} + + unsigned getInsnID() const { return InsnID; } + const CodeGenInstruction *getCGI() const { return I; } + + void chooseInsnToMutate(RuleMatcher &Rule) { + for (auto *MutateCandidate : Rule.mutatable_insns()) { + if (canMutate(Rule, MutateCandidate)) { + // Take the first one we're offered that we're able to mutate. + Rule.reserveInsnMatcherForMutation(MutateCandidate); + Matched = MutateCandidate; + return; + } + } + } + + template <class Kind, class... Args> + Kind &addRenderer(Args&&... args) { + OperandRenderers.emplace_back( + std::make_unique<Kind>(InsnID, std::forward<Args>(args)...)); + return *static_cast<Kind *>(OperandRenderers.back().get()); + } + + void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + if (Matched) { + assert(canMutate(Rule, Matched) && + "Arranged to mutate an insn that isn't mutatable"); + + unsigned RecycleInsnID = Rule.getInsnVarID(*Matched); + Table << MatchTable::Opcode("GIR_MutateOpcode") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("RecycleInsnID") + << MatchTable::IntValue(RecycleInsnID) + << MatchTable::Comment("Opcode") + << MatchTable::NamedValue(I->Namespace, I->TheDef->getName()) + << MatchTable::LineBreak; + + if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) { + for (auto Def : I->ImplicitDefs) { + auto Namespace = Def->getValue("Namespace") + ? Def->getValueAsString("Namespace") + : ""; + Table << MatchTable::Opcode("GIR_AddImplicitDef") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::NamedValue(Namespace, Def->getName()) + << MatchTable::LineBreak; + } + for (auto Use : I->ImplicitUses) { + auto Namespace = Use->getValue("Namespace") + ? Use->getValueAsString("Namespace") + : ""; + Table << MatchTable::Opcode("GIR_AddImplicitUse") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::NamedValue(Namespace, Use->getName()) + << MatchTable::LineBreak; + } + } + return; + } + + // TODO: Simple permutation looks like it could be almost as common as + // mutation due to commutative operations. + + Table << MatchTable::Opcode("GIR_BuildMI") << MatchTable::Comment("InsnID") + << MatchTable::IntValue(InsnID) << MatchTable::Comment("Opcode") + << MatchTable::NamedValue(I->Namespace, I->TheDef->getName()) + << MatchTable::LineBreak; + for (const auto &Renderer : OperandRenderers) + Renderer->emitRenderOpcodes(Table, Rule); + + if (I->mayLoad || I->mayStore) { + Table << MatchTable::Opcode("GIR_MergeMemOperands") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("MergeInsnID's"); + // Emit the ID's for all the instructions that are matched by this rule. + // TODO: Limit this to matched instructions that mayLoad/mayStore or have + // some other means of having a memoperand. Also limit this to + // emitted instructions that expect to have a memoperand too. For + // example, (G_SEXT (G_LOAD x)) that results in separate load and + // sign-extend instructions shouldn't put the memoperand on the + // sign-extend since it has no effect there. + std::vector<unsigned> MergeInsnIDs; + for (const auto &IDMatcherPair : Rule.defined_insn_vars()) + MergeInsnIDs.push_back(IDMatcherPair.second); + llvm::sort(MergeInsnIDs); + for (const auto &MergeInsnID : MergeInsnIDs) + Table << MatchTable::IntValue(MergeInsnID); + Table << MatchTable::NamedValue("GIU_MergeMemOperands_EndOfList") + << MatchTable::LineBreak; + } + + // FIXME: This is a hack but it's sufficient for ISel. We'll need to do + // better for combines. Particularly when there are multiple match + // roots. + if (InsnID == 0) + Table << MatchTable::Opcode("GIR_EraseFromParent") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::LineBreak; + } +}; + +/// Generates code to constrain the operands of an output instruction to the +/// register classes specified by the definition of that instruction. +class ConstrainOperandsToDefinitionAction : public MatchAction { + unsigned InsnID; + +public: + ConstrainOperandsToDefinitionAction(unsigned InsnID) : InsnID(InsnID) {} + + void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIR_ConstrainSelectedInstOperands") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::LineBreak; + } +}; + +/// Generates code to constrain the specified operand of an output instruction +/// to the specified register class. +class ConstrainOperandToRegClassAction : public MatchAction { + unsigned InsnID; + unsigned OpIdx; + const CodeGenRegisterClass &RC; + +public: + ConstrainOperandToRegClassAction(unsigned InsnID, unsigned OpIdx, + const CodeGenRegisterClass &RC) + : InsnID(InsnID), OpIdx(OpIdx), RC(RC) {} + + void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIR_ConstrainOperandRC") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx) + << MatchTable::NamedValue(RC.getQualifiedName() + "RegClassID") + << MatchTable::LineBreak; + } +}; + +/// Generates code to create a temporary register which can be used to chain +/// instructions together. +class MakeTempRegisterAction : public MatchAction { +private: + LLTCodeGen Ty; + unsigned TempRegID; + +public: + MakeTempRegisterAction(const LLTCodeGen &Ty, unsigned TempRegID) + : Ty(Ty), TempRegID(TempRegID) { + KnownTypes.insert(Ty); + } + + void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override { + Table << MatchTable::Opcode("GIR_MakeTempReg") + << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID) + << MatchTable::Comment("TypeID") + << MatchTable::NamedValue(Ty.getCxxEnumValue()) + << MatchTable::LineBreak; + } +}; + +InstructionMatcher &RuleMatcher::addInstructionMatcher(StringRef SymbolicName) { + Matchers.emplace_back(new InstructionMatcher(*this, SymbolicName)); + MutatableInsns.insert(Matchers.back().get()); + return *Matchers.back(); +} + +void RuleMatcher::addRequiredFeature(Record *Feature) { + RequiredFeatures.push_back(Feature); +} + +const std::vector<Record *> &RuleMatcher::getRequiredFeatures() const { + return RequiredFeatures; +} + +// Emplaces an action of the specified Kind at the end of the action list. +// +// Returns a reference to the newly created action. +// +// Like std::vector::emplace_back(), may invalidate all iterators if the new +// size exceeds the capacity. Otherwise, only invalidates the past-the-end +// iterator. +template <class Kind, class... Args> +Kind &RuleMatcher::addAction(Args &&... args) { + Actions.emplace_back(std::make_unique<Kind>(std::forward<Args>(args)...)); + return *static_cast<Kind *>(Actions.back().get()); +} + +// Emplaces an action of the specified Kind before the given insertion point. +// +// Returns an iterator pointing at the newly created instruction. +// +// Like std::vector::insert(), may invalidate all iterators if the new size +// exceeds the capacity. Otherwise, only invalidates the iterators from the +// insertion point onwards. +template <class Kind, class... Args> +action_iterator RuleMatcher::insertAction(action_iterator InsertPt, + Args &&... args) { + return Actions.emplace(InsertPt, + std::make_unique<Kind>(std::forward<Args>(args)...)); +} + +unsigned RuleMatcher::implicitlyDefineInsnVar(InstructionMatcher &Matcher) { + unsigned NewInsnVarID = NextInsnVarID++; + InsnVariableIDs[&Matcher] = NewInsnVarID; + return NewInsnVarID; +} + +unsigned RuleMatcher::getInsnVarID(InstructionMatcher &InsnMatcher) const { + const auto &I = InsnVariableIDs.find(&InsnMatcher); + if (I != InsnVariableIDs.end()) + return I->second; + llvm_unreachable("Matched Insn was not captured in a local variable"); +} + +void RuleMatcher::defineOperand(StringRef SymbolicName, OperandMatcher &OM) { + if (DefinedOperands.find(SymbolicName) == DefinedOperands.end()) { + DefinedOperands[SymbolicName] = &OM; + return; + } + + // If the operand is already defined, then we must ensure both references in + // the matcher have the exact same node. + OM.addPredicate<SameOperandMatcher>(OM.getSymbolicName()); +} + +void RuleMatcher::definePhysRegOperand(Record *Reg, OperandMatcher &OM) { + if (PhysRegOperands.find(Reg) == PhysRegOperands.end()) { + PhysRegOperands[Reg] = &OM; + return; + } +} + +InstructionMatcher & +RuleMatcher::getInstructionMatcher(StringRef SymbolicName) const { + for (const auto &I : InsnVariableIDs) + if (I.first->getSymbolicName() == SymbolicName) + return *I.first; + llvm_unreachable( + ("Failed to lookup instruction " + SymbolicName).str().c_str()); +} + +const OperandMatcher & +RuleMatcher::getPhysRegOperandMatcher(Record *Reg) const { + const auto &I = PhysRegOperands.find(Reg); + + if (I == PhysRegOperands.end()) { + PrintFatalError(SrcLoc, "Register " + Reg->getName() + + " was not declared in matcher"); + } + + return *I->second; +} + +const OperandMatcher & +RuleMatcher::getOperandMatcher(StringRef Name) const { + const auto &I = DefinedOperands.find(Name); + + if (I == DefinedOperands.end()) + PrintFatalError(SrcLoc, "Operand " + Name + " was not declared in matcher"); + + return *I->second; +} + +void RuleMatcher::emit(MatchTable &Table) { + if (Matchers.empty()) + llvm_unreachable("Unexpected empty matcher!"); + + // The representation supports rules that require multiple roots such as: + // %ptr(p0) = ... + // %elt0(s32) = G_LOAD %ptr + // %1(p0) = G_ADD %ptr, 4 + // %elt1(s32) = G_LOAD p0 %1 + // which could be usefully folded into: + // %ptr(p0) = ... + // %elt0(s32), %elt1(s32) = TGT_LOAD_PAIR %ptr + // on some targets but we don't need to make use of that yet. + assert(Matchers.size() == 1 && "Cannot handle multi-root matchers yet"); + + unsigned LabelID = Table.allocateLabelID(); + Table << MatchTable::Opcode("GIM_Try", +1) + << MatchTable::Comment("On fail goto") + << MatchTable::JumpTarget(LabelID) + << MatchTable::Comment(("Rule ID " + Twine(RuleID) + " //").str()) + << MatchTable::LineBreak; + + if (!RequiredFeatures.empty()) { + Table << MatchTable::Opcode("GIM_CheckFeatures") + << MatchTable::NamedValue(getNameForFeatureBitset(RequiredFeatures)) + << MatchTable::LineBreak; + } + + Matchers.front()->emitPredicateOpcodes(Table, *this); + + // We must also check if it's safe to fold the matched instructions. + if (InsnVariableIDs.size() >= 2) { + // Invert the map to create stable ordering (by var names) + SmallVector<unsigned, 2> InsnIDs; + for (const auto &Pair : InsnVariableIDs) { + // Skip the root node since it isn't moving anywhere. Everything else is + // sinking to meet it. + if (Pair.first == Matchers.front().get()) + continue; + + InsnIDs.push_back(Pair.second); + } + llvm::sort(InsnIDs); + + for (const auto &InsnID : InsnIDs) { + // Reject the difficult cases until we have a more accurate check. + Table << MatchTable::Opcode("GIM_CheckIsSafeToFold") + << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID) + << MatchTable::LineBreak; + + // FIXME: Emit checks to determine it's _actually_ safe to fold and/or + // account for unsafe cases. + // + // Example: + // MI1--> %0 = ... + // %1 = ... %0 + // MI0--> %2 = ... %0 + // It's not safe to erase MI1. We currently handle this by not + // erasing %0 (even when it's dead). + // + // Example: + // MI1--> %0 = load volatile @a + // %1 = load volatile @a + // MI0--> %2 = ... %0 + // It's not safe to sink %0's def past %1. We currently handle + // this by rejecting all loads. + // + // Example: + // MI1--> %0 = load @a + // %1 = store @a + // MI0--> %2 = ... %0 + // It's not safe to sink %0's def past %1. We currently handle + // this by rejecting all loads. + // + // Example: + // G_CONDBR %cond, @BB1 + // BB0: + // MI1--> %0 = load @a + // G_BR @BB1 + // BB1: + // MI0--> %2 = ... %0 + // It's not always safe to sink %0 across control flow. In this + // case it may introduce a memory fault. We currentl handle this + // by rejecting all loads. + } + } + + for (const auto &PM : EpilogueMatchers) + PM->emitPredicateOpcodes(Table, *this); + + for (const auto &MA : Actions) + MA->emitActionOpcodes(Table, *this); + + if (Table.isWithCoverage()) + Table << MatchTable::Opcode("GIR_Coverage") << MatchTable::IntValue(RuleID) + << MatchTable::LineBreak; + else + Table << MatchTable::Comment(("GIR_Coverage, " + Twine(RuleID) + ",").str()) + << MatchTable::LineBreak; + + Table << MatchTable::Opcode("GIR_Done", -1) << MatchTable::LineBreak + << MatchTable::Label(LabelID); + ++NumPatternEmitted; +} + +bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const { + // Rules involving more match roots have higher priority. + if (Matchers.size() > B.Matchers.size()) + return true; + if (Matchers.size() < B.Matchers.size()) + return false; + + for (auto Matcher : zip(Matchers, B.Matchers)) { + if (std::get<0>(Matcher)->isHigherPriorityThan(*std::get<1>(Matcher))) + return true; + if (std::get<1>(Matcher)->isHigherPriorityThan(*std::get<0>(Matcher))) + return false; + } + + return false; +} + +unsigned RuleMatcher::countRendererFns() const { + return std::accumulate( + Matchers.begin(), Matchers.end(), 0, + [](unsigned A, const std::unique_ptr<InstructionMatcher> &Matcher) { + return A + Matcher->countRendererFns(); + }); +} + +bool OperandPredicateMatcher::isHigherPriorityThan( + const OperandPredicateMatcher &B) const { + // Generally speaking, an instruction is more important than an Int or a + // LiteralInt because it can cover more nodes but theres an exception to + // this. G_CONSTANT's are less important than either of those two because they + // are more permissive. + + const InstructionOperandMatcher *AOM = + dyn_cast<InstructionOperandMatcher>(this); + const InstructionOperandMatcher *BOM = + dyn_cast<InstructionOperandMatcher>(&B); + bool AIsConstantInsn = AOM && AOM->getInsnMatcher().isConstantInstruction(); + bool BIsConstantInsn = BOM && BOM->getInsnMatcher().isConstantInstruction(); + + if (AOM && BOM) { + // The relative priorities between a G_CONSTANT and any other instruction + // don't actually matter but this code is needed to ensure a strict weak + // ordering. This is particularly important on Windows where the rules will + // be incorrectly sorted without it. + if (AIsConstantInsn != BIsConstantInsn) + return AIsConstantInsn < BIsConstantInsn; + return false; + } + + if (AOM && AIsConstantInsn && (B.Kind == OPM_Int || B.Kind == OPM_LiteralInt)) + return false; + if (BOM && BIsConstantInsn && (Kind == OPM_Int || Kind == OPM_LiteralInt)) + return true; + + return Kind < B.Kind; +} + +void SameOperandMatcher::emitPredicateOpcodes(MatchTable &Table, + RuleMatcher &Rule) const { + const OperandMatcher &OtherOM = Rule.getOperandMatcher(MatchingName); + unsigned OtherInsnVarID = Rule.getInsnVarID(OtherOM.getInstructionMatcher()); + assert(OtherInsnVarID == OtherOM.getInstructionMatcher().getInsnVarID()); + + Table << MatchTable::Opcode("GIM_CheckIsSameOperand") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx) + << MatchTable::Comment("OtherMI") + << MatchTable::IntValue(OtherInsnVarID) + << MatchTable::Comment("OtherOpIdx") + << MatchTable::IntValue(OtherOM.getOpIdx()) + << MatchTable::LineBreak; +} + +//===- GlobalISelEmitter class --------------------------------------------===// + +static Expected<LLTCodeGen> getInstResultType(const TreePatternNode *Dst) { + ArrayRef<TypeSetByHwMode> ChildTypes = Dst->getExtTypes(); + if (ChildTypes.size() != 1) + return failedImport("Dst pattern child has multiple results"); + + Optional<LLTCodeGen> MaybeOpTy; + if (ChildTypes.front().isMachineValueType()) { + MaybeOpTy = + MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy); + } + + if (!MaybeOpTy) + return failedImport("Dst operand has an unsupported type"); + return *MaybeOpTy; +} + +class GlobalISelEmitter { +public: + explicit GlobalISelEmitter(RecordKeeper &RK); + void run(raw_ostream &OS); + +private: + const RecordKeeper &RK; + const CodeGenDAGPatterns CGP; + const CodeGenTarget &Target; + CodeGenRegBank &CGRegs; + + /// Keep track of the equivalence between SDNodes and Instruction by mapping + /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to + /// check for attributes on the relation such as CheckMMOIsNonAtomic. + /// This is defined using 'GINodeEquiv' in the target description. + DenseMap<Record *, Record *> NodeEquivs; + + /// Keep track of the equivalence between ComplexPattern's and + /// GIComplexOperandMatcher. Map entries are specified by subclassing + /// GIComplexPatternEquiv. + DenseMap<const Record *, const Record *> ComplexPatternEquivs; + + /// Keep track of the equivalence between SDNodeXForm's and + /// GICustomOperandRenderer. Map entries are specified by subclassing + /// GISDNodeXFormEquiv. + DenseMap<const Record *, const Record *> SDNodeXFormEquivs; + + /// Keep track of Scores of PatternsToMatch similar to how the DAG does. + /// This adds compatibility for RuleMatchers to use this for ordering rules. + DenseMap<uint64_t, int> RuleMatcherScores; + + // Map of predicates to their subtarget features. + SubtargetFeatureInfoMap SubtargetFeatures; + + // Rule coverage information. + Optional<CodeGenCoverage> RuleCoverage; + + void gatherOpcodeValues(); + void gatherTypeIDValues(); + void gatherNodeEquivs(); + + Record *findNodeEquiv(Record *N) const; + const CodeGenInstruction *getEquivNode(Record &Equiv, + const TreePatternNode *N) const; + + Error importRulePredicates(RuleMatcher &M, ArrayRef<Predicate> Predicates); + Expected<InstructionMatcher &> + createAndImportSelDAGMatcher(RuleMatcher &Rule, + InstructionMatcher &InsnMatcher, + const TreePatternNode *Src, unsigned &TempOpIdx); + Error importComplexPatternOperandMatcher(OperandMatcher &OM, Record *R, + unsigned &TempOpIdx) const; + Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher, + const TreePatternNode *SrcChild, + bool OperandIsAPointer, bool OperandIsImmArg, + unsigned OpIdx, unsigned &TempOpIdx); + + Expected<BuildMIAction &> createAndImportInstructionRenderer( + RuleMatcher &M, InstructionMatcher &InsnMatcher, + const TreePatternNode *Src, const TreePatternNode *Dst); + Expected<action_iterator> createAndImportSubInstructionRenderer( + action_iterator InsertPt, RuleMatcher &M, const TreePatternNode *Dst, + unsigned TempReg); + Expected<action_iterator> + createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M, + const TreePatternNode *Dst); + void importExplicitDefRenderers(BuildMIAction &DstMIBuilder); + + Expected<action_iterator> + importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M, + BuildMIAction &DstMIBuilder, + const llvm::TreePatternNode *Dst); + Expected<action_iterator> + importExplicitUseRenderer(action_iterator InsertPt, RuleMatcher &Rule, + BuildMIAction &DstMIBuilder, + TreePatternNode *DstChild); + Error importDefaultOperandRenderers(action_iterator InsertPt, RuleMatcher &M, + BuildMIAction &DstMIBuilder, + DagInit *DefaultOps) const; + Error + importImplicitDefRenderers(BuildMIAction &DstMIBuilder, + const std::vector<Record *> &ImplicitDefs) const; + + void emitCxxPredicateFns(raw_ostream &OS, StringRef CodeFieldName, + StringRef TypeIdentifier, StringRef ArgType, + StringRef ArgName, StringRef AdditionalDeclarations, + std::function<bool(const Record *R)> Filter); + void emitImmPredicateFns(raw_ostream &OS, StringRef TypeIdentifier, + StringRef ArgType, + std::function<bool(const Record *R)> Filter); + void emitMIPredicateFns(raw_ostream &OS); + + /// Analyze pattern \p P, returning a matcher for it if possible. + /// Otherwise, return an Error explaining why we don't support it. + Expected<RuleMatcher> runOnPattern(const PatternToMatch &P); + + void declareSubtargetFeature(Record *Predicate); + + MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize, + bool WithCoverage); + + /// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned + /// CodeGenRegisterClass will support the CodeGenRegisterClass of + /// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode. + /// If no register class is found, return None. + Optional<const CodeGenRegisterClass *> + inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty, + TreePatternNode *SuperRegNode, + TreePatternNode *SubRegIdxNode); + Optional<CodeGenSubRegIndex *> + inferSubRegIndexForNode(TreePatternNode *SubRegIdxNode); + + /// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode. + /// Return None if no such class exists. + Optional<const CodeGenRegisterClass *> + inferSuperRegisterClass(const TypeSetByHwMode &Ty, + TreePatternNode *SubRegIdxNode); + + /// Return the CodeGenRegisterClass associated with \p Leaf if it has one. + Optional<const CodeGenRegisterClass *> + getRegClassFromLeaf(TreePatternNode *Leaf); + + /// Return a CodeGenRegisterClass for \p N if one can be found. Return None + /// otherwise. + Optional<const CodeGenRegisterClass *> + inferRegClassFromPattern(TreePatternNode *N); + +public: + /// Takes a sequence of \p Rules and group them based on the predicates + /// they share. \p MatcherStorage is used as a memory container + /// for the group that are created as part of this process. + /// + /// What this optimization does looks like if GroupT = GroupMatcher: + /// Output without optimization: + /// \verbatim + /// # R1 + /// # predicate A + /// # predicate B + /// ... + /// # R2 + /// # predicate A // <-- effectively this is going to be checked twice. + /// // Once in R1 and once in R2. + /// # predicate C + /// \endverbatim + /// Output with optimization: + /// \verbatim + /// # Group1_2 + /// # predicate A // <-- Check is now shared. + /// # R1 + /// # predicate B + /// # R2 + /// # predicate C + /// \endverbatim + template <class GroupT> + static std::vector<Matcher *> optimizeRules( + ArrayRef<Matcher *> Rules, + std::vector<std::unique_ptr<Matcher>> &MatcherStorage); +}; + +void GlobalISelEmitter::gatherOpcodeValues() { + InstructionOpcodeMatcher::initOpcodeValuesMap(Target); +} + +void GlobalISelEmitter::gatherTypeIDValues() { + LLTOperandMatcher::initTypeIDValuesMap(); +} + +void GlobalISelEmitter::gatherNodeEquivs() { + assert(NodeEquivs.empty()); + for (Record *Equiv : RK.getAllDerivedDefinitions("GINodeEquiv")) + NodeEquivs[Equiv->getValueAsDef("Node")] = Equiv; + + assert(ComplexPatternEquivs.empty()); + for (Record *Equiv : RK.getAllDerivedDefinitions("GIComplexPatternEquiv")) { + Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent"); + if (!SelDAGEquiv) + continue; + ComplexPatternEquivs[SelDAGEquiv] = Equiv; + } + + assert(SDNodeXFormEquivs.empty()); + for (Record *Equiv : RK.getAllDerivedDefinitions("GISDNodeXFormEquiv")) { + Record *SelDAGEquiv = Equiv->getValueAsDef("SelDAGEquivalent"); + if (!SelDAGEquiv) + continue; + SDNodeXFormEquivs[SelDAGEquiv] = Equiv; + } +} + +Record *GlobalISelEmitter::findNodeEquiv(Record *N) const { + return NodeEquivs.lookup(N); +} + +const CodeGenInstruction * +GlobalISelEmitter::getEquivNode(Record &Equiv, const TreePatternNode *N) const { + if (N->getNumChildren() >= 1) { + // setcc operation maps to two different G_* instructions based on the type. + if (!Equiv.isValueUnset("IfFloatingPoint") && + MVT(N->getChild(0)->getSimpleType(0)).isFloatingPoint()) + return &Target.getInstruction(Equiv.getValueAsDef("IfFloatingPoint")); + } + + for (const TreePredicateCall &Call : N->getPredicateCalls()) { + const TreePredicateFn &Predicate = Call.Fn; + if (!Equiv.isValueUnset("IfSignExtend") && Predicate.isLoad() && + Predicate.isSignExtLoad()) + return &Target.getInstruction(Equiv.getValueAsDef("IfSignExtend")); + if (!Equiv.isValueUnset("IfZeroExtend") && Predicate.isLoad() && + Predicate.isZeroExtLoad()) + return &Target.getInstruction(Equiv.getValueAsDef("IfZeroExtend")); + } + + return &Target.getInstruction(Equiv.getValueAsDef("I")); +} + +GlobalISelEmitter::GlobalISelEmitter(RecordKeeper &RK) + : RK(RK), CGP(RK), Target(CGP.getTargetInfo()), + CGRegs(Target.getRegBank()) {} + +//===- Emitter ------------------------------------------------------------===// + +Error +GlobalISelEmitter::importRulePredicates(RuleMatcher &M, + ArrayRef<Predicate> Predicates) { + for (const Predicate &P : Predicates) { + if (!P.Def || P.getCondString().empty()) + continue; + declareSubtargetFeature(P.Def); + M.addRequiredFeature(P.Def); + } + + return Error::success(); +} + +Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher( + RuleMatcher &Rule, InstructionMatcher &InsnMatcher, + const TreePatternNode *Src, unsigned &TempOpIdx) { + Record *SrcGIEquivOrNull = nullptr; + const CodeGenInstruction *SrcGIOrNull = nullptr; + + // Start with the defined operands (i.e., the results of the root operator). + if (Src->getExtTypes().size() > 1) + return failedImport("Src pattern has multiple results"); + + if (Src->isLeaf()) { + Init *SrcInit = Src->getLeafValue(); + if (isa<IntInit>(SrcInit)) { + InsnMatcher.addPredicate<InstructionOpcodeMatcher>( + &Target.getInstruction(RK.getDef("G_CONSTANT"))); + } else + return failedImport( + "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); + } else { + SrcGIEquivOrNull = findNodeEquiv(Src->getOperator()); + if (!SrcGIEquivOrNull) + return failedImport("Pattern operator lacks an equivalent Instruction" + + explainOperator(Src->getOperator())); + SrcGIOrNull = getEquivNode(*SrcGIEquivOrNull, Src); + + // The operators look good: match the opcode + InsnMatcher.addPredicate<InstructionOpcodeMatcher>(SrcGIOrNull); + } + + unsigned OpIdx = 0; + for (const TypeSetByHwMode &VTy : Src->getExtTypes()) { + // Results don't have a name unless they are the root node. The caller will + // set the name if appropriate. + OperandMatcher &OM = InsnMatcher.addOperand(OpIdx++, "", TempOpIdx); + if (auto Error = OM.addTypeCheckPredicate(VTy, false /* OperandIsAPointer */)) + return failedImport(toString(std::move(Error)) + + " for result of Src pattern operator"); + } + + for (const TreePredicateCall &Call : Src->getPredicateCalls()) { + const TreePredicateFn &Predicate = Call.Fn; + if (Predicate.isAlwaysTrue()) + continue; + + if (Predicate.isImmediatePattern()) { + InsnMatcher.addPredicate<InstructionImmPredicateMatcher>(Predicate); + continue; + } + + // An address space check is needed in all contexts if there is one. + if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) { + if (const ListInit *AddrSpaces = Predicate.getAddressSpaces()) { + SmallVector<unsigned, 4> ParsedAddrSpaces; + + for (Init *Val : AddrSpaces->getValues()) { + IntInit *IntVal = dyn_cast<IntInit>(Val); + if (!IntVal) + return failedImport("Address space is not an integer"); + ParsedAddrSpaces.push_back(IntVal->getValue()); + } + + if (!ParsedAddrSpaces.empty()) { + InsnMatcher.addPredicate<MemoryAddressSpacePredicateMatcher>( + 0, ParsedAddrSpaces); + } + } + + int64_t MinAlign = Predicate.getMinAlignment(); + if (MinAlign > 0) + InsnMatcher.addPredicate<MemoryAlignmentPredicateMatcher>(0, MinAlign); + } + + // G_LOAD is used for both non-extending and any-extending loads. + if (Predicate.isLoad() && Predicate.isNonExtLoad()) { + InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( + 0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0); + continue; + } + if (Predicate.isLoad() && Predicate.isAnyExtLoad()) { + InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( + 0, MemoryVsLLTSizePredicateMatcher::LessThan, 0); + continue; + } + + if (Predicate.isStore()) { + if (Predicate.isTruncStore()) { + // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size. + InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( + 0, MemoryVsLLTSizePredicateMatcher::LessThan, 0); + continue; + } + if (Predicate.isNonTruncStore()) { + // We need to check the sizes match here otherwise we could incorrectly + // match truncating stores with non-truncating ones. + InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>( + 0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0); + } + } + + // No check required. We already did it by swapping the opcode. + if (!SrcGIEquivOrNull->isValueUnset("IfSignExtend") && + Predicate.isSignExtLoad()) + continue; + + // No check required. We already did it by swapping the opcode. + if (!SrcGIEquivOrNull->isValueUnset("IfZeroExtend") && + Predicate.isZeroExtLoad()) + continue; + + // No check required. G_STORE by itself is a non-extending store. + if (Predicate.isNonTruncStore()) + continue; + + if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) { + if (Predicate.getMemoryVT() != nullptr) { + Optional<LLTCodeGen> MemTyOrNone = + MVTToLLT(getValueType(Predicate.getMemoryVT())); + + if (!MemTyOrNone) + return failedImport("MemVT could not be converted to LLT"); + + // MMO's work in bytes so we must take care of unusual types like i1 + // don't round down. + unsigned MemSizeInBits = + llvm::alignTo(MemTyOrNone->get().getSizeInBits(), 8); + + InsnMatcher.addPredicate<MemorySizePredicateMatcher>( + 0, MemSizeInBits / 8); + continue; + } + } + + if (Predicate.isLoad() || Predicate.isStore()) { + // No check required. A G_LOAD/G_STORE is an unindexed load. + if (Predicate.isUnindexed()) + continue; + } + + if (Predicate.isAtomic()) { + if (Predicate.isAtomicOrderingMonotonic()) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( + "Monotonic"); + continue; + } + if (Predicate.isAtomicOrderingAcquire()) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Acquire"); + continue; + } + if (Predicate.isAtomicOrderingRelease()) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("Release"); + continue; + } + if (Predicate.isAtomicOrderingAcquireRelease()) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( + "AcquireRelease"); + continue; + } + if (Predicate.isAtomicOrderingSequentiallyConsistent()) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( + "SequentiallyConsistent"); + continue; + } + + if (Predicate.isAtomicOrderingAcquireOrStronger()) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( + "Acquire", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); + continue; + } + if (Predicate.isAtomicOrderingWeakerThanAcquire()) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( + "Acquire", AtomicOrderingMMOPredicateMatcher::AO_WeakerThan); + continue; + } + + if (Predicate.isAtomicOrderingReleaseOrStronger()) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( + "Release", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); + continue; + } + if (Predicate.isAtomicOrderingWeakerThanRelease()) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( + "Release", AtomicOrderingMMOPredicateMatcher::AO_WeakerThan); + continue; + } + } + + if (Predicate.hasGISelPredicateCode()) { + InsnMatcher.addPredicate<GenericInstructionPredicateMatcher>(Predicate); + continue; + } + + return failedImport("Src pattern child has predicate (" + + explainPredicates(Src) + ")"); + } + if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>("NotAtomic"); + else if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) { + InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>( + "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); + } + + if (Src->isLeaf()) { + Init *SrcInit = Src->getLeafValue(); + if (IntInit *SrcIntInit = dyn_cast<IntInit>(SrcInit)) { + OperandMatcher &OM = + InsnMatcher.addOperand(OpIdx++, Src->getName(), TempOpIdx); + OM.addPredicate<LiteralIntOperandMatcher>(SrcIntInit->getValue()); + } else + return failedImport( + "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); + } else { + assert(SrcGIOrNull && + "Expected to have already found an equivalent Instruction"); + if (SrcGIOrNull->TheDef->getName() == "G_CONSTANT" || + SrcGIOrNull->TheDef->getName() == "G_FCONSTANT") { + // imm/fpimm still have operands but we don't need to do anything with it + // here since we don't support ImmLeaf predicates yet. However, we still + // need to note the hidden operand to get GIM_CheckNumOperands correct. + InsnMatcher.addOperand(OpIdx++, "", TempOpIdx); + return InsnMatcher; + } + + // Special case because the operand order is changed from setcc. The + // predicate operand needs to be swapped from the last operand to the first + // source. + + unsigned NumChildren = Src->getNumChildren(); + bool IsFCmp = SrcGIOrNull->TheDef->getName() == "G_FCMP"; + + if (IsFCmp || SrcGIOrNull->TheDef->getName() == "G_ICMP") { + TreePatternNode *SrcChild = Src->getChild(NumChildren - 1); + if (SrcChild->isLeaf()) { + DefInit *DI = dyn_cast<DefInit>(SrcChild->getLeafValue()); + Record *CCDef = DI ? DI->getDef() : nullptr; + if (!CCDef || !CCDef->isSubClassOf("CondCode")) + return failedImport("Unable to handle CondCode"); + + OperandMatcher &OM = + InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx); + StringRef PredType = IsFCmp ? CCDef->getValueAsString("FCmpPredicate") : + CCDef->getValueAsString("ICmpPredicate"); + + if (!PredType.empty()) { + OM.addPredicate<CmpPredicateOperandMatcher>(std::string(PredType)); + // Process the other 2 operands normally. + --NumChildren; + } + } + } + + // Match the used operands (i.e. the children of the operator). + bool IsIntrinsic = + SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" || + SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_W_SIDE_EFFECTS"; + const CodeGenIntrinsic *II = Src->getIntrinsicInfo(CGP); + if (IsIntrinsic && !II) + return failedImport("Expected IntInit containing intrinsic ID)"); + + for (unsigned i = 0; i != NumChildren; ++i) { + TreePatternNode *SrcChild = Src->getChild(i); + + // We need to determine the meaning of a literal integer based on the + // context. If this is a field required to be an immediate (such as an + // immarg intrinsic argument), the required predicates are different than + // a constant which may be materialized in a register. If we have an + // argument that is required to be an immediate, we should not emit an LLT + // type check, and should not be looking for a G_CONSTANT defined + // register. + bool OperandIsImmArg = SrcGIOrNull->isOperandImmArg(i); + + // SelectionDAG allows pointers to be represented with iN since it doesn't + // distinguish between pointers and integers but they are different types in GlobalISel. + // Coerce integers to pointers to address space 0 if the context indicates a pointer. + // + bool OperandIsAPointer = SrcGIOrNull->isOperandAPointer(i); + + if (IsIntrinsic) { + // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately + // following the defs is an intrinsic ID. + if (i == 0) { + OperandMatcher &OM = + InsnMatcher.addOperand(OpIdx++, SrcChild->getName(), TempOpIdx); + OM.addPredicate<IntrinsicIDOperandMatcher>(II); + continue; + } + + // We have to check intrinsics for llvm_anyptr_ty and immarg parameters. + // + // Note that we have to look at the i-1th parameter, because we don't + // have the intrinsic ID in the intrinsic's parameter list. + OperandIsAPointer |= II->isParamAPointer(i - 1); + OperandIsImmArg |= II->isParamImmArg(i - 1); + } + + if (auto Error = + importChildMatcher(Rule, InsnMatcher, SrcChild, OperandIsAPointer, + OperandIsImmArg, OpIdx++, TempOpIdx)) + return std::move(Error); + } + } + + return InsnMatcher; +} + +Error GlobalISelEmitter::importComplexPatternOperandMatcher( + OperandMatcher &OM, Record *R, unsigned &TempOpIdx) const { + const auto &ComplexPattern = ComplexPatternEquivs.find(R); + if (ComplexPattern == ComplexPatternEquivs.end()) + return failedImport("SelectionDAG ComplexPattern (" + R->getName() + + ") not mapped to GlobalISel"); + + OM.addPredicate<ComplexPatternOperandMatcher>(OM, *ComplexPattern->second); + TempOpIdx++; + return Error::success(); +} + +// Get the name to use for a pattern operand. For an anonymous physical register +// input, this should use the register name. +static StringRef getSrcChildName(const TreePatternNode *SrcChild, + Record *&PhysReg) { + StringRef SrcChildName = SrcChild->getName(); + if (SrcChildName.empty() && SrcChild->isLeaf()) { + if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild->getLeafValue())) { + auto *ChildRec = ChildDefInit->getDef(); + if (ChildRec->isSubClassOf("Register")) { + SrcChildName = ChildRec->getName(); + PhysReg = ChildRec; + } + } + } + + return SrcChildName; +} + +Error GlobalISelEmitter::importChildMatcher( + RuleMatcher &Rule, InstructionMatcher &InsnMatcher, + const TreePatternNode *SrcChild, bool OperandIsAPointer, + bool OperandIsImmArg, unsigned OpIdx, unsigned &TempOpIdx) { + + Record *PhysReg = nullptr; + StringRef SrcChildName = getSrcChildName(SrcChild, PhysReg); + + OperandMatcher &OM = + PhysReg + ? InsnMatcher.addPhysRegInput(PhysReg, OpIdx, TempOpIdx) + : InsnMatcher.addOperand(OpIdx, std::string(SrcChildName), TempOpIdx); + if (OM.isSameAsAnotherOperand()) + return Error::success(); + + ArrayRef<TypeSetByHwMode> ChildTypes = SrcChild->getExtTypes(); + if (ChildTypes.size() != 1) + return failedImport("Src pattern child has multiple results"); + + // Check MBB's before the type check since they are not a known type. + if (!SrcChild->isLeaf()) { + if (SrcChild->getOperator()->isSubClassOf("SDNode")) { + auto &ChildSDNI = CGP.getSDNodeInfo(SrcChild->getOperator()); + if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") { + OM.addPredicate<MBBOperandMatcher>(); + return Error::success(); + } + if (SrcChild->getOperator()->getName() == "timm") { + OM.addPredicate<ImmOperandMatcher>(); + return Error::success(); + } + } + } + + // Immediate arguments have no meaningful type to check as they don't have + // registers. + if (!OperandIsImmArg) { + if (auto Error = + OM.addTypeCheckPredicate(ChildTypes.front(), OperandIsAPointer)) + return failedImport(toString(std::move(Error)) + " for Src operand (" + + to_string(*SrcChild) + ")"); + } + + // Check for nested instructions. + if (!SrcChild->isLeaf()) { + if (SrcChild->getOperator()->isSubClassOf("ComplexPattern")) { + // When a ComplexPattern is used as an operator, it should do the same + // thing as when used as a leaf. However, the children of the operator + // name the sub-operands that make up the complex operand and we must + // prepare to reference them in the renderer too. + unsigned RendererID = TempOpIdx; + if (auto Error = importComplexPatternOperandMatcher( + OM, SrcChild->getOperator(), TempOpIdx)) + return Error; + + for (unsigned i = 0, e = SrcChild->getNumChildren(); i != e; ++i) { + auto *SubOperand = SrcChild->getChild(i); + if (!SubOperand->getName().empty()) { + if (auto Error = Rule.defineComplexSubOperand(SubOperand->getName(), + SrcChild->getOperator(), + RendererID, i)) + return Error; + } + } + + return Error::success(); + } + + auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>( + InsnMatcher.getRuleMatcher(), SrcChild->getName()); + if (!MaybeInsnOperand.hasValue()) { + // This isn't strictly true. If the user were to provide exactly the same + // matchers as the original operand then we could allow it. However, it's + // simpler to not permit the redundant specification. + return failedImport("Nested instruction cannot be the same as another operand"); + } + + // Map the node to a gMIR instruction. + InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand; + auto InsnMatcherOrError = createAndImportSelDAGMatcher( + Rule, InsnOperand.getInsnMatcher(), SrcChild, TempOpIdx); + if (auto Error = InsnMatcherOrError.takeError()) + return Error; + + return Error::success(); + } + + if (SrcChild->hasAnyPredicate()) + return failedImport("Src pattern child has unsupported predicate"); + + // Check for constant immediates. + if (auto *ChildInt = dyn_cast<IntInit>(SrcChild->getLeafValue())) { + if (OperandIsImmArg) { + // Checks for argument directly in operand list + OM.addPredicate<LiteralIntOperandMatcher>(ChildInt->getValue()); + } else { + // Checks for materialized constant + OM.addPredicate<ConstantIntOperandMatcher>(ChildInt->getValue()); + } + return Error::success(); + } + + // Check for def's like register classes or ComplexPattern's. + if (auto *ChildDefInit = dyn_cast<DefInit>(SrcChild->getLeafValue())) { + auto *ChildRec = ChildDefInit->getDef(); + + // Check for register classes. + if (ChildRec->isSubClassOf("RegisterClass") || + ChildRec->isSubClassOf("RegisterOperand")) { + OM.addPredicate<RegisterBankOperandMatcher>( + Target.getRegisterClass(getInitValueAsRegClass(ChildDefInit))); + return Error::success(); + } + + if (ChildRec->isSubClassOf("Register")) { + // This just be emitted as a copy to the specific register. + ValueTypeByHwMode VT = ChildTypes.front().getValueTypeByHwMode(); + const CodeGenRegisterClass *RC + = CGRegs.getMinimalPhysRegClass(ChildRec, &VT); + if (!RC) { + return failedImport( + "Could not determine physical register class of pattern source"); + } + + OM.addPredicate<RegisterBankOperandMatcher>(*RC); + return Error::success(); + } + + // Check for ValueType. + if (ChildRec->isSubClassOf("ValueType")) { + // We already added a type check as standard practice so this doesn't need + // to do anything. + return Error::success(); + } + + // Check for ComplexPattern's. + if (ChildRec->isSubClassOf("ComplexPattern")) + return importComplexPatternOperandMatcher(OM, ChildRec, TempOpIdx); + + if (ChildRec->isSubClassOf("ImmLeaf")) { + return failedImport( + "Src pattern child def is an unsupported tablegen class (ImmLeaf)"); + } + + // Place holder for SRCVALUE nodes. Nothing to do here. + if (ChildRec->getName() == "srcvalue") + return Error::success(); + + return failedImport( + "Src pattern child def is an unsupported tablegen class"); + } + + return failedImport("Src pattern child is an unsupported kind"); +} + +Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer( + action_iterator InsertPt, RuleMatcher &Rule, BuildMIAction &DstMIBuilder, + TreePatternNode *DstChild) { + + const auto &SubOperand = Rule.getComplexSubOperand(DstChild->getName()); + if (SubOperand.hasValue()) { + DstMIBuilder.addRenderer<RenderComplexPatternOperand>( + *std::get<0>(*SubOperand), DstChild->getName(), + std::get<1>(*SubOperand), std::get<2>(*SubOperand)); + return InsertPt; + } + + if (!DstChild->isLeaf()) { + if (DstChild->getOperator()->isSubClassOf("SDNodeXForm")) { + auto Child = DstChild->getChild(0); + auto I = SDNodeXFormEquivs.find(DstChild->getOperator()); + if (I != SDNodeXFormEquivs.end()) { + Record *XFormOpc = DstChild->getOperator()->getValueAsDef("Opcode"); + if (XFormOpc->getName() == "timm") { + // If this is a TargetConstant, there won't be a corresponding + // instruction to transform. Instead, this will refer directly to an + // operand in an instruction's operand list. + DstMIBuilder.addRenderer<CustomOperandRenderer>(*I->second, + Child->getName()); + } else { + DstMIBuilder.addRenderer<CustomRenderer>(*I->second, + Child->getName()); + } + + return InsertPt; + } + return failedImport("SDNodeXForm " + Child->getName() + + " has no custom renderer"); + } + + // We accept 'bb' here. It's an operator because BasicBlockSDNode isn't + // inline, but in MI it's just another operand. + if (DstChild->getOperator()->isSubClassOf("SDNode")) { + auto &ChildSDNI = CGP.getSDNodeInfo(DstChild->getOperator()); + if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") { + DstMIBuilder.addRenderer<CopyRenderer>(DstChild->getName()); + return InsertPt; + } + } + + // Similarly, imm is an operator in TreePatternNode's view but must be + // rendered as operands. + // FIXME: The target should be able to choose sign-extended when appropriate + // (e.g. on Mips). + if (DstChild->getOperator()->getName() == "timm") { + DstMIBuilder.addRenderer<CopyRenderer>(DstChild->getName()); + return InsertPt; + } else if (DstChild->getOperator()->getName() == "imm") { + DstMIBuilder.addRenderer<CopyConstantAsImmRenderer>(DstChild->getName()); + return InsertPt; + } else if (DstChild->getOperator()->getName() == "fpimm") { + DstMIBuilder.addRenderer<CopyFConstantAsFPImmRenderer>( + DstChild->getName()); + return InsertPt; + } + + if (DstChild->getOperator()->isSubClassOf("Instruction")) { + auto OpTy = getInstResultType(DstChild); + if (!OpTy) + return OpTy.takeError(); + + unsigned TempRegID = Rule.allocateTempRegID(); + InsertPt = Rule.insertAction<MakeTempRegisterAction>( + InsertPt, *OpTy, TempRegID); + DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID); + + auto InsertPtOrError = createAndImportSubInstructionRenderer( + ++InsertPt, Rule, DstChild, TempRegID); + if (auto Error = InsertPtOrError.takeError()) + return std::move(Error); + return InsertPtOrError.get(); + } + + return failedImport("Dst pattern child isn't a leaf node or an MBB" + llvm::to_string(*DstChild)); + } + + // It could be a specific immediate in which case we should just check for + // that immediate. + if (const IntInit *ChildIntInit = + dyn_cast<IntInit>(DstChild->getLeafValue())) { + DstMIBuilder.addRenderer<ImmRenderer>(ChildIntInit->getValue()); + return InsertPt; + } + + // Otherwise, we're looking for a bog-standard RegisterClass operand. + if (auto *ChildDefInit = dyn_cast<DefInit>(DstChild->getLeafValue())) { + auto *ChildRec = ChildDefInit->getDef(); + + ArrayRef<TypeSetByHwMode> ChildTypes = DstChild->getExtTypes(); + if (ChildTypes.size() != 1) + return failedImport("Dst pattern child has multiple results"); + + Optional<LLTCodeGen> OpTyOrNone = None; + if (ChildTypes.front().isMachineValueType()) + OpTyOrNone = MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy); + if (!OpTyOrNone) + return failedImport("Dst operand has an unsupported type"); + + if (ChildRec->isSubClassOf("Register")) { + DstMIBuilder.addRenderer<AddRegisterRenderer>(ChildRec); + return InsertPt; + } + + if (ChildRec->isSubClassOf("RegisterClass") || + ChildRec->isSubClassOf("RegisterOperand") || + ChildRec->isSubClassOf("ValueType")) { + if (ChildRec->isSubClassOf("RegisterOperand") && + !ChildRec->isValueUnset("GIZeroRegister")) { + DstMIBuilder.addRenderer<CopyOrAddZeroRegRenderer>( + DstChild->getName(), ChildRec->getValueAsDef("GIZeroRegister")); + return InsertPt; + } + + DstMIBuilder.addRenderer<CopyRenderer>(DstChild->getName()); + return InsertPt; + } + + if (ChildRec->isSubClassOf("SubRegIndex")) { + CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(ChildRec); + DstMIBuilder.addRenderer<ImmRenderer>(SubIdx->EnumValue); + return InsertPt; + } + + if (ChildRec->isSubClassOf("ComplexPattern")) { + const auto &ComplexPattern = ComplexPatternEquivs.find(ChildRec); + if (ComplexPattern == ComplexPatternEquivs.end()) + return failedImport( + "SelectionDAG ComplexPattern not mapped to GlobalISel"); + + const OperandMatcher &OM = Rule.getOperandMatcher(DstChild->getName()); + DstMIBuilder.addRenderer<RenderComplexPatternOperand>( + *ComplexPattern->second, DstChild->getName(), + OM.getAllocatedTemporariesBaseID()); + return InsertPt; + } + + return failedImport( + "Dst pattern child def is an unsupported tablegen class"); + } + + return failedImport("Dst pattern child is an unsupported kind"); +} + +Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer( + RuleMatcher &M, InstructionMatcher &InsnMatcher, const TreePatternNode *Src, + const TreePatternNode *Dst) { + auto InsertPtOrError = createInstructionRenderer(M.actions_end(), M, Dst); + if (auto Error = InsertPtOrError.takeError()) + return std::move(Error); + + action_iterator InsertPt = InsertPtOrError.get(); + BuildMIAction &DstMIBuilder = *static_cast<BuildMIAction *>(InsertPt->get()); + + for (auto PhysInput : InsnMatcher.getPhysRegInputs()) { + InsertPt = M.insertAction<BuildMIAction>( + InsertPt, M.allocateOutputInsnID(), + &Target.getInstruction(RK.getDef("COPY"))); + BuildMIAction &CopyToPhysRegMIBuilder = + *static_cast<BuildMIAction *>(InsertPt->get()); + CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(PhysInput.first, + true); + CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(PhysInput.first); + } + + importExplicitDefRenderers(DstMIBuilder); + + if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst) + .takeError()) + return std::move(Error); + + return DstMIBuilder; +} + +Expected<action_iterator> +GlobalISelEmitter::createAndImportSubInstructionRenderer( + const action_iterator InsertPt, RuleMatcher &M, const TreePatternNode *Dst, + unsigned TempRegID) { + auto InsertPtOrError = createInstructionRenderer(InsertPt, M, Dst); + + // TODO: Assert there's exactly one result. + + if (auto Error = InsertPtOrError.takeError()) + return std::move(Error); + + BuildMIAction &DstMIBuilder = + *static_cast<BuildMIAction *>(InsertPtOrError.get()->get()); + + // Assign the result to TempReg. + DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, true); + + InsertPtOrError = + importExplicitUseRenderers(InsertPtOrError.get(), M, DstMIBuilder, Dst); + if (auto Error = InsertPtOrError.takeError()) + return std::move(Error); + + // We need to make sure that when we import an INSERT_SUBREG as a + // subinstruction that it ends up being constrained to the correct super + // register and subregister classes. + auto OpName = Target.getInstruction(Dst->getOperator()).TheDef->getName(); + if (OpName == "INSERT_SUBREG") { + auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); + if (!SubClass) + return failedImport( + "Cannot infer register class from INSERT_SUBREG operand #1"); + Optional<const CodeGenRegisterClass *> SuperClass = + inferSuperRegisterClassForNode(Dst->getExtType(0), Dst->getChild(0), + Dst->getChild(2)); + if (!SuperClass) + return failedImport( + "Cannot infer register class for INSERT_SUBREG operand #0"); + // The destination and the super register source of an INSERT_SUBREG must + // be the same register class. + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 1, **SuperClass); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 2, **SubClass); + return InsertPtOrError.get(); + } + + if (OpName == "EXTRACT_SUBREG") { + // EXTRACT_SUBREG selects into a subregister COPY but unlike most + // instructions, the result register class is controlled by the + // subregisters of the operand. As a result, we must constrain the result + // class rather than check that it's already the right one. + auto SuperClass = inferRegClassFromPattern(Dst->getChild(0)); + if (!SuperClass) + return failedImport( + "Cannot infer register class from EXTRACT_SUBREG operand #0"); + + auto SubIdx = inferSubRegIndexForNode(Dst->getChild(1)); + if (!SubIdx) + return failedImport("EXTRACT_SUBREG child #1 is not a subreg index"); + + const auto SrcRCDstRCPair = + (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx); + assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 0, *SrcRCDstRCPair->second); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 1, *SrcRCDstRCPair->first); + + // We're done with this pattern! It's eligible for GISel emission; return + // it. + return InsertPtOrError.get(); + } + + // Similar to INSERT_SUBREG, we also have to handle SUBREG_TO_REG as a + // subinstruction. + if (OpName == "SUBREG_TO_REG") { + auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); + if (!SubClass) + return failedImport( + "Cannot infer register class from SUBREG_TO_REG child #1"); + auto SuperClass = inferSuperRegisterClass(Dst->getExtType(0), + Dst->getChild(2)); + if (!SuperClass) + return failedImport( + "Cannot infer register class for SUBREG_TO_REG operand #0"); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 2, **SubClass); + return InsertPtOrError.get(); + } + + if (OpName == "REG_SEQUENCE") { + auto SuperClass = inferRegClassFromPattern(Dst->getChild(0)); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), 0, **SuperClass); + + unsigned Num = Dst->getNumChildren(); + for (unsigned I = 1; I != Num; I += 2) { + TreePatternNode *SubRegChild = Dst->getChild(I + 1); + + auto SubIdx = inferSubRegIndexForNode(SubRegChild); + if (!SubIdx) + return failedImport("REG_SEQUENCE child is not a subreg index"); + + const auto SrcRCDstRCPair = + (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx); + assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); + M.insertAction<ConstrainOperandToRegClassAction>( + InsertPt, DstMIBuilder.getInsnID(), I, *SrcRCDstRCPair->second); + } + + return InsertPtOrError.get(); + } + + M.insertAction<ConstrainOperandsToDefinitionAction>(InsertPt, + DstMIBuilder.getInsnID()); + return InsertPtOrError.get(); +} + +Expected<action_iterator> GlobalISelEmitter::createInstructionRenderer( + action_iterator InsertPt, RuleMatcher &M, const TreePatternNode *Dst) { + Record *DstOp = Dst->getOperator(); + if (!DstOp->isSubClassOf("Instruction")) { + if (DstOp->isSubClassOf("ValueType")) + return failedImport( + "Pattern operator isn't an instruction (it's a ValueType)"); + return failedImport("Pattern operator isn't an instruction"); + } + CodeGenInstruction *DstI = &Target.getInstruction(DstOp); + + // COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction + // attached. Similarly for EXTRACT_SUBREG except that's a subregister copy. + StringRef Name = DstI->TheDef->getName(); + if (Name == "COPY_TO_REGCLASS" || Name == "EXTRACT_SUBREG") + DstI = &Target.getInstruction(RK.getDef("COPY")); + + return M.insertAction<BuildMIAction>(InsertPt, M.allocateOutputInsnID(), + DstI); +} + +void GlobalISelEmitter::importExplicitDefRenderers( + BuildMIAction &DstMIBuilder) { + const CodeGenInstruction *DstI = DstMIBuilder.getCGI(); + for (unsigned I = 0; I < DstI->Operands.NumDefs; ++I) { + const CGIOperandList::OperandInfo &DstIOperand = DstI->Operands[I]; + DstMIBuilder.addRenderer<CopyRenderer>(DstIOperand.Name); + } +} + +Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers( + action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, + const llvm::TreePatternNode *Dst) { + const CodeGenInstruction *DstI = DstMIBuilder.getCGI(); + CodeGenInstruction *OrigDstI = &Target.getInstruction(Dst->getOperator()); + + StringRef Name = OrigDstI->TheDef->getName(); + unsigned ExpectedDstINumUses = Dst->getNumChildren(); + + // EXTRACT_SUBREG needs to use a subregister COPY. + if (Name == "EXTRACT_SUBREG") { + DefInit *SubRegInit = dyn_cast<DefInit>(Dst->getChild(1)->getLeafValue()); + if (!SubRegInit) + return failedImport("EXTRACT_SUBREG child #1 is not a subreg index"); + + CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef()); + TreePatternNode *ValChild = Dst->getChild(0); + if (!ValChild->isLeaf()) { + // We really have to handle the source instruction, and then insert a + // copy from the subregister. + auto ExtractSrcTy = getInstResultType(ValChild); + if (!ExtractSrcTy) + return ExtractSrcTy.takeError(); + + unsigned TempRegID = M.allocateTempRegID(); + InsertPt = M.insertAction<MakeTempRegisterAction>( + InsertPt, *ExtractSrcTy, TempRegID); + + auto InsertPtOrError = createAndImportSubInstructionRenderer( + ++InsertPt, M, ValChild, TempRegID); + if (auto Error = InsertPtOrError.takeError()) + return std::move(Error); + + DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID, false, SubIdx); + return InsertPt; + } + + // If this is a source operand, this is just a subregister copy. + Record *RCDef = getInitValueAsRegClass(ValChild->getLeafValue()); + if (!RCDef) + return failedImport("EXTRACT_SUBREG child #0 could not " + "be coerced to a register class"); + + CodeGenRegisterClass *RC = CGRegs.getRegClass(RCDef); + + const auto SrcRCDstRCPair = + RC->getMatchingSubClassWithSubRegs(CGRegs, SubIdx); + if (SrcRCDstRCPair.hasValue()) { + assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); + if (SrcRCDstRCPair->first != RC) + return failedImport("EXTRACT_SUBREG requires an additional COPY"); + } + + DstMIBuilder.addRenderer<CopySubRegRenderer>(Dst->getChild(0)->getName(), + SubIdx); + return InsertPt; + } + + if (Name == "REG_SEQUENCE") { + if (!Dst->getChild(0)->isLeaf()) + return failedImport("REG_SEQUENCE child #0 is not a leaf"); + + Record *RCDef = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); + if (!RCDef) + return failedImport("REG_SEQUENCE child #0 could not " + "be coerced to a register class"); + + if ((ExpectedDstINumUses - 1) % 2 != 0) + return failedImport("Malformed REG_SEQUENCE"); + + for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) { + TreePatternNode *ValChild = Dst->getChild(I); + TreePatternNode *SubRegChild = Dst->getChild(I + 1); + + if (DefInit *SubRegInit = + dyn_cast<DefInit>(SubRegChild->getLeafValue())) { + CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef()); + + auto InsertPtOrError = + importExplicitUseRenderer(InsertPt, M, DstMIBuilder, ValChild); + if (auto Error = InsertPtOrError.takeError()) + return std::move(Error); + InsertPt = InsertPtOrError.get(); + DstMIBuilder.addRenderer<SubRegIndexRenderer>(SubIdx); + } + } + + return InsertPt; + } + + // Render the explicit uses. + unsigned DstINumUses = OrigDstI->Operands.size() - OrigDstI->Operands.NumDefs; + if (Name == "COPY_TO_REGCLASS") { + DstINumUses--; // Ignore the class constraint. + ExpectedDstINumUses--; + } + + // NumResults - This is the number of results produced by the instruction in + // the "outs" list. + unsigned NumResults = OrigDstI->Operands.NumDefs; + + // Number of operands we know the output instruction must have. If it is + // variadic, we could have more operands. + unsigned NumFixedOperands = DstI->Operands.size(); + + // Loop over all of the fixed operands of the instruction pattern, emitting + // code to fill them all in. The node 'N' usually has number children equal to + // the number of input operands of the instruction. However, in cases where + // there are predicate operands for an instruction, we need to fill in the + // 'execute always' values. Match up the node operands to the instruction + // operands to do this. + unsigned Child = 0; + + // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the + // number of operands at the end of the list which have default values. + // Those can come from the pattern if it provides enough arguments, or be + // filled in with the default if the pattern hasn't provided them. But any + // operand with a default value _before_ the last mandatory one will be + // filled in with their defaults unconditionally. + unsigned NonOverridableOperands = NumFixedOperands; + while (NonOverridableOperands > NumResults && + CGP.operandHasDefault(DstI->Operands[NonOverridableOperands - 1].Rec)) + --NonOverridableOperands; + + unsigned NumDefaultOps = 0; + for (unsigned I = 0; I != DstINumUses; ++I) { + unsigned InstOpNo = DstI->Operands.NumDefs + I; + + // Determine what to emit for this operand. + Record *OperandNode = DstI->Operands[InstOpNo].Rec; + + // If the operand has default values, introduce them now. + if (CGP.operandHasDefault(OperandNode) && + (InstOpNo < NonOverridableOperands || Child >= Dst->getNumChildren())) { + // This is a predicate or optional def operand which the pattern has not + // overridden, or which we aren't letting it override; emit the 'default + // ops' operands. + + const CGIOperandList::OperandInfo &DstIOperand = DstI->Operands[InstOpNo]; + DagInit *DefaultOps = DstIOperand.Rec->getValueAsDag("DefaultOps"); + if (auto Error = importDefaultOperandRenderers( + InsertPt, M, DstMIBuilder, DefaultOps)) + return std::move(Error); + ++NumDefaultOps; + continue; + } + + auto InsertPtOrError = importExplicitUseRenderer(InsertPt, M, DstMIBuilder, + Dst->getChild(Child)); + if (auto Error = InsertPtOrError.takeError()) + return std::move(Error); + InsertPt = InsertPtOrError.get(); + ++Child; + } + + if (NumDefaultOps + ExpectedDstINumUses != DstINumUses) + return failedImport("Expected " + llvm::to_string(DstINumUses) + + " used operands but found " + + llvm::to_string(ExpectedDstINumUses) + + " explicit ones and " + llvm::to_string(NumDefaultOps) + + " default ones"); + + return InsertPt; +} + +Error GlobalISelEmitter::importDefaultOperandRenderers( + action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder, + DagInit *DefaultOps) const { + for (const auto *DefaultOp : DefaultOps->getArgs()) { + Optional<LLTCodeGen> OpTyOrNone = None; + + // Look through ValueType operators. + if (const DagInit *DefaultDagOp = dyn_cast<DagInit>(DefaultOp)) { + if (const DefInit *DefaultDagOperator = + dyn_cast<DefInit>(DefaultDagOp->getOperator())) { + if (DefaultDagOperator->getDef()->isSubClassOf("ValueType")) { + OpTyOrNone = MVTToLLT(getValueType( + DefaultDagOperator->getDef())); + DefaultOp = DefaultDagOp->getArg(0); + } + } + } + + if (const DefInit *DefaultDefOp = dyn_cast<DefInit>(DefaultOp)) { + auto Def = DefaultDefOp->getDef(); + if (Def->getName() == "undef_tied_input") { + unsigned TempRegID = M.allocateTempRegID(); + M.insertAction<MakeTempRegisterAction>( + InsertPt, OpTyOrNone.getValue(), TempRegID); + InsertPt = M.insertAction<BuildMIAction>( + InsertPt, M.allocateOutputInsnID(), + &Target.getInstruction(RK.getDef("IMPLICIT_DEF"))); + BuildMIAction &IDMIBuilder = *static_cast<BuildMIAction *>( + InsertPt->get()); + IDMIBuilder.addRenderer<TempRegRenderer>(TempRegID); + DstMIBuilder.addRenderer<TempRegRenderer>(TempRegID); + } else { + DstMIBuilder.addRenderer<AddRegisterRenderer>(Def); + } + continue; + } + + if (const IntInit *DefaultIntOp = dyn_cast<IntInit>(DefaultOp)) { + DstMIBuilder.addRenderer<ImmRenderer>(DefaultIntOp->getValue()); + continue; + } + + return failedImport("Could not add default op"); + } + + return Error::success(); +} + +Error GlobalISelEmitter::importImplicitDefRenderers( + BuildMIAction &DstMIBuilder, + const std::vector<Record *> &ImplicitDefs) const { + if (!ImplicitDefs.empty()) + return failedImport("Pattern defines a physical register"); + return Error::success(); +} + +Optional<const CodeGenRegisterClass *> +GlobalISelEmitter::getRegClassFromLeaf(TreePatternNode *Leaf) { + assert(Leaf && "Expected node?"); + assert(Leaf->isLeaf() && "Expected leaf?"); + Record *RCRec = getInitValueAsRegClass(Leaf->getLeafValue()); + if (!RCRec) + return None; + CodeGenRegisterClass *RC = CGRegs.getRegClass(RCRec); + if (!RC) + return None; + return RC; +} + +Optional<const CodeGenRegisterClass *> +GlobalISelEmitter::inferRegClassFromPattern(TreePatternNode *N) { + if (!N) + return None; + + if (N->isLeaf()) + return getRegClassFromLeaf(N); + + // We don't have a leaf node, so we have to try and infer something. Check + // that we have an instruction that we an infer something from. + + // Only handle things that produce a single type. + if (N->getNumTypes() != 1) + return None; + Record *OpRec = N->getOperator(); + + // We only want instructions. + if (!OpRec->isSubClassOf("Instruction")) + return None; + + // Don't want to try and infer things when there could potentially be more + // than one candidate register class. + auto &Inst = Target.getInstruction(OpRec); + if (Inst.Operands.NumDefs > 1) + return None; + + // Handle any special-case instructions which we can safely infer register + // classes from. + StringRef InstName = Inst.TheDef->getName(); + bool IsRegSequence = InstName == "REG_SEQUENCE"; + if (IsRegSequence || InstName == "COPY_TO_REGCLASS") { + // If we have a COPY_TO_REGCLASS, then we need to handle it specially. It + // has the desired register class as the first child. + TreePatternNode *RCChild = N->getChild(IsRegSequence ? 0 : 1); + if (!RCChild->isLeaf()) + return None; + return getRegClassFromLeaf(RCChild); + } + + // Handle destination record types that we can safely infer a register class + // from. + const auto &DstIOperand = Inst.Operands[0]; + Record *DstIOpRec = DstIOperand.Rec; + if (DstIOpRec->isSubClassOf("RegisterOperand")) { + DstIOpRec = DstIOpRec->getValueAsDef("RegClass"); + const CodeGenRegisterClass &RC = Target.getRegisterClass(DstIOpRec); + return &RC; + } + + if (DstIOpRec->isSubClassOf("RegisterClass")) { + const CodeGenRegisterClass &RC = Target.getRegisterClass(DstIOpRec); + return &RC; + } + + return None; +} + +Optional<const CodeGenRegisterClass *> +GlobalISelEmitter::inferSuperRegisterClass(const TypeSetByHwMode &Ty, + TreePatternNode *SubRegIdxNode) { + assert(SubRegIdxNode && "Expected subregister index node!"); + // We need a ValueTypeByHwMode for getSuperRegForSubReg. + if (!Ty.isValueTypeByHwMode(false)) + return None; + if (!SubRegIdxNode->isLeaf()) + return None; + DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode->getLeafValue()); + if (!SubRegInit) + return None; + CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef()); + + // Use the information we found above to find a minimal register class which + // supports the subregister and type we want. + auto RC = + Target.getSuperRegForSubReg(Ty.getValueTypeByHwMode(), CGRegs, SubIdx); + if (!RC) + return None; + return *RC; +} + +Optional<const CodeGenRegisterClass *> +GlobalISelEmitter::inferSuperRegisterClassForNode( + const TypeSetByHwMode &Ty, TreePatternNode *SuperRegNode, + TreePatternNode *SubRegIdxNode) { + assert(SuperRegNode && "Expected super register node!"); + // Check if we already have a defined register class for the super register + // node. If we do, then we should preserve that rather than inferring anything + // from the subregister index node. We can assume that whoever wrote the + // pattern in the first place made sure that the super register and + // subregister are compatible. + if (Optional<const CodeGenRegisterClass *> SuperRegisterClass = + inferRegClassFromPattern(SuperRegNode)) + return *SuperRegisterClass; + return inferSuperRegisterClass(Ty, SubRegIdxNode); +} + +Optional<CodeGenSubRegIndex *> +GlobalISelEmitter::inferSubRegIndexForNode(TreePatternNode *SubRegIdxNode) { + if (!SubRegIdxNode->isLeaf()) + return None; + + DefInit *SubRegInit = dyn_cast<DefInit>(SubRegIdxNode->getLeafValue()); + if (!SubRegInit) + return None; + return CGRegs.getSubRegIdx(SubRegInit->getDef()); +} + +Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { + // Keep track of the matchers and actions to emit. + int Score = P.getPatternComplexity(CGP); + RuleMatcher M(P.getSrcRecord()->getLoc()); + RuleMatcherScores[M.getRuleID()] = Score; + M.addAction<DebugCommentAction>(llvm::to_string(*P.getSrcPattern()) + + " => " + + llvm::to_string(*P.getDstPattern())); + + if (auto Error = importRulePredicates(M, P.getPredicates())) + return std::move(Error); + + // Next, analyze the pattern operators. + TreePatternNode *Src = P.getSrcPattern(); + TreePatternNode *Dst = P.getDstPattern(); + + // If the root of either pattern isn't a simple operator, ignore it. + if (auto Err = isTrivialOperatorNode(Dst)) + return failedImport("Dst pattern root isn't a trivial operator (" + + toString(std::move(Err)) + ")"); + if (auto Err = isTrivialOperatorNode(Src)) + return failedImport("Src pattern root isn't a trivial operator (" + + toString(std::move(Err)) + ")"); + + // The different predicates and matchers created during + // addInstructionMatcher use the RuleMatcher M to set up their + // instruction ID (InsnVarID) that are going to be used when + // M is going to be emitted. + // However, the code doing the emission still relies on the IDs + // returned during that process by the RuleMatcher when issuing + // the recordInsn opcodes. + // Because of that: + // 1. The order in which we created the predicates + // and such must be the same as the order in which we emit them, + // and + // 2. We need to reset the generation of the IDs in M somewhere between + // addInstructionMatcher and emit + // + // FIXME: Long term, we don't want to have to rely on this implicit + // naming being the same. One possible solution would be to have + // explicit operator for operation capture and reference those. + // The plus side is that it would expose opportunities to share + // the capture accross rules. The downside is that it would + // introduce a dependency between predicates (captures must happen + // before their first use.) + InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(Src->getName()); + unsigned TempOpIdx = 0; + auto InsnMatcherOrError = + createAndImportSelDAGMatcher(M, InsnMatcherTemp, Src, TempOpIdx); + if (auto Error = InsnMatcherOrError.takeError()) + return std::move(Error); + InstructionMatcher &InsnMatcher = InsnMatcherOrError.get(); + + if (Dst->isLeaf()) { + Record *RCDef = getInitValueAsRegClass(Dst->getLeafValue()); + + const CodeGenRegisterClass &RC = Target.getRegisterClass(RCDef); + if (RCDef) { + // We need to replace the def and all its uses with the specified + // operand. However, we must also insert COPY's wherever needed. + // For now, emit a copy and let the register allocator clean up. + auto &DstI = Target.getInstruction(RK.getDef("COPY")); + const auto &DstIOperand = DstI.Operands[0]; + + OperandMatcher &OM0 = InsnMatcher.getOperand(0); + OM0.setSymbolicName(DstIOperand.Name); + M.defineOperand(OM0.getSymbolicName(), OM0); + OM0.addPredicate<RegisterBankOperandMatcher>(RC); + + auto &DstMIBuilder = + M.addAction<BuildMIAction>(M.allocateOutputInsnID(), &DstI); + DstMIBuilder.addRenderer<CopyRenderer>(DstIOperand.Name); + DstMIBuilder.addRenderer<CopyRenderer>(Dst->getName()); + M.addAction<ConstrainOperandToRegClassAction>(0, 0, RC); + + // We're done with this pattern! It's eligible for GISel emission; return + // it. + ++NumPatternImported; + return std::move(M); + } + + return failedImport("Dst pattern root isn't a known leaf"); + } + + // Start with the defined operands (i.e., the results of the root operator). + Record *DstOp = Dst->getOperator(); + if (!DstOp->isSubClassOf("Instruction")) + return failedImport("Pattern operator isn't an instruction"); + + auto &DstI = Target.getInstruction(DstOp); + StringRef DstIName = DstI.TheDef->getName(); + + if (DstI.Operands.NumDefs != Src->getExtTypes().size()) + return failedImport("Src pattern results and dst MI defs are different (" + + to_string(Src->getExtTypes().size()) + " def(s) vs " + + to_string(DstI.Operands.NumDefs) + " def(s))"); + + // The root of the match also has constraints on the register bank so that it + // matches the result instruction. + unsigned OpIdx = 0; + for (const TypeSetByHwMode &VTy : Src->getExtTypes()) { + (void)VTy; + + const auto &DstIOperand = DstI.Operands[OpIdx]; + Record *DstIOpRec = DstIOperand.Rec; + if (DstIName == "COPY_TO_REGCLASS") { + DstIOpRec = getInitValueAsRegClass(Dst->getChild(1)->getLeafValue()); + + if (DstIOpRec == nullptr) + return failedImport( + "COPY_TO_REGCLASS operand #1 isn't a register class"); + } else if (DstIName == "REG_SEQUENCE") { + DstIOpRec = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); + if (DstIOpRec == nullptr) + return failedImport("REG_SEQUENCE operand #0 isn't a register class"); + } else if (DstIName == "EXTRACT_SUBREG") { + auto InferredClass = inferRegClassFromPattern(Dst->getChild(0)); + if (!InferredClass) + return failedImport("Could not infer class for EXTRACT_SUBREG operand #0"); + + // We can assume that a subregister is in the same bank as it's super + // register. + DstIOpRec = (*InferredClass)->getDef(); + } else if (DstIName == "INSERT_SUBREG") { + auto MaybeSuperClass = inferSuperRegisterClassForNode( + VTy, Dst->getChild(0), Dst->getChild(2)); + if (!MaybeSuperClass) + return failedImport( + "Cannot infer register class for INSERT_SUBREG operand #0"); + // Move to the next pattern here, because the register class we found + // doesn't necessarily have a record associated with it. So, we can't + // set DstIOpRec using this. + OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); + OM.setSymbolicName(DstIOperand.Name); + M.defineOperand(OM.getSymbolicName(), OM); + OM.addPredicate<RegisterBankOperandMatcher>(**MaybeSuperClass); + ++OpIdx; + continue; + } else if (DstIName == "SUBREG_TO_REG") { + auto MaybeRegClass = inferSuperRegisterClass(VTy, Dst->getChild(2)); + if (!MaybeRegClass) + return failedImport( + "Cannot infer register class for SUBREG_TO_REG operand #0"); + OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); + OM.setSymbolicName(DstIOperand.Name); + M.defineOperand(OM.getSymbolicName(), OM); + OM.addPredicate<RegisterBankOperandMatcher>(**MaybeRegClass); + ++OpIdx; + continue; + } else if (DstIOpRec->isSubClassOf("RegisterOperand")) + DstIOpRec = DstIOpRec->getValueAsDef("RegClass"); + else if (!DstIOpRec->isSubClassOf("RegisterClass")) + return failedImport("Dst MI def isn't a register class" + + to_string(*Dst)); + + OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); + OM.setSymbolicName(DstIOperand.Name); + M.defineOperand(OM.getSymbolicName(), OM); + OM.addPredicate<RegisterBankOperandMatcher>( + Target.getRegisterClass(DstIOpRec)); + ++OpIdx; + } + + auto DstMIBuilderOrError = + createAndImportInstructionRenderer(M, InsnMatcher, Src, Dst); + if (auto Error = DstMIBuilderOrError.takeError()) + return std::move(Error); + BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get(); + + // Render the implicit defs. + // These are only added to the root of the result. + if (auto Error = importImplicitDefRenderers(DstMIBuilder, P.getDstRegs())) + return std::move(Error); + + DstMIBuilder.chooseInsnToMutate(M); + + // Constrain the registers to classes. This is normally derived from the + // emitted instruction but a few instructions require special handling. + if (DstIName == "COPY_TO_REGCLASS") { + // COPY_TO_REGCLASS does not provide operand constraints itself but the + // result is constrained to the class given by the second child. + Record *DstIOpRec = + getInitValueAsRegClass(Dst->getChild(1)->getLeafValue()); + + if (DstIOpRec == nullptr) + return failedImport("COPY_TO_REGCLASS operand #1 isn't a register class"); + + M.addAction<ConstrainOperandToRegClassAction>( + 0, 0, Target.getRegisterClass(DstIOpRec)); + + // We're done with this pattern! It's eligible for GISel emission; return + // it. + ++NumPatternImported; + return std::move(M); + } + + if (DstIName == "EXTRACT_SUBREG") { + auto SuperClass = inferRegClassFromPattern(Dst->getChild(0)); + if (!SuperClass) + return failedImport( + "Cannot infer register class from EXTRACT_SUBREG operand #0"); + + auto SubIdx = inferSubRegIndexForNode(Dst->getChild(1)); + if (!SubIdx) + return failedImport("EXTRACT_SUBREG child #1 is not a subreg index"); + + // It would be nice to leave this constraint implicit but we're required + // to pick a register class so constrain the result to a register class + // that can hold the correct MVT. + // + // FIXME: This may introduce an extra copy if the chosen class doesn't + // actually contain the subregisters. + assert(Src->getExtTypes().size() == 1 && + "Expected Src of EXTRACT_SUBREG to have one result type"); + + const auto SrcRCDstRCPair = + (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx); + if (!SrcRCDstRCPair) { + return failedImport("subreg index is incompatible " + "with inferred reg class"); + } + + assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass"); + M.addAction<ConstrainOperandToRegClassAction>(0, 0, *SrcRCDstRCPair->second); + M.addAction<ConstrainOperandToRegClassAction>(0, 1, *SrcRCDstRCPair->first); + + // We're done with this pattern! It's eligible for GISel emission; return + // it. + ++NumPatternImported; + return std::move(M); + } + + if (DstIName == "INSERT_SUBREG") { + assert(Src->getExtTypes().size() == 1 && + "Expected Src of INSERT_SUBREG to have one result type"); + // We need to constrain the destination, a super regsister source, and a + // subregister source. + auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); + if (!SubClass) + return failedImport( + "Cannot infer register class from INSERT_SUBREG operand #1"); + auto SuperClass = inferSuperRegisterClassForNode( + Src->getExtType(0), Dst->getChild(0), Dst->getChild(2)); + if (!SuperClass) + return failedImport( + "Cannot infer register class for INSERT_SUBREG operand #0"); + M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass); + M.addAction<ConstrainOperandToRegClassAction>(0, 1, **SuperClass); + M.addAction<ConstrainOperandToRegClassAction>(0, 2, **SubClass); + ++NumPatternImported; + return std::move(M); + } + + if (DstIName == "SUBREG_TO_REG") { + // We need to constrain the destination and subregister source. + assert(Src->getExtTypes().size() == 1 && + "Expected Src of SUBREG_TO_REG to have one result type"); + + // Attempt to infer the subregister source from the first child. If it has + // an explicitly given register class, we'll use that. Otherwise, we will + // fail. + auto SubClass = inferRegClassFromPattern(Dst->getChild(1)); + if (!SubClass) + return failedImport( + "Cannot infer register class from SUBREG_TO_REG child #1"); + // We don't have a child to look at that might have a super register node. + auto SuperClass = + inferSuperRegisterClass(Src->getExtType(0), Dst->getChild(2)); + if (!SuperClass) + return failedImport( + "Cannot infer register class for SUBREG_TO_REG operand #0"); + M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass); + M.addAction<ConstrainOperandToRegClassAction>(0, 2, **SubClass); + ++NumPatternImported; + return std::move(M); + } + + if (DstIName == "REG_SEQUENCE") { + auto SuperClass = inferRegClassFromPattern(Dst->getChild(0)); + + M.addAction<ConstrainOperandToRegClassAction>(0, 0, **SuperClass); + + unsigned Num = Dst->getNumChildren(); + for (unsigned I = 1; I != Num; I += 2) { + TreePatternNode *SubRegChild = Dst->getChild(I + 1); + + auto SubIdx = inferSubRegIndexForNode(SubRegChild); + if (!SubIdx) + return failedImport("REG_SEQUENCE child is not a subreg index"); + + const auto SrcRCDstRCPair = + (*SuperClass)->getMatchingSubClassWithSubRegs(CGRegs, *SubIdx); + + M.addAction<ConstrainOperandToRegClassAction>(0, I, + *SrcRCDstRCPair->second); + } + + ++NumPatternImported; + return std::move(M); + } + + M.addAction<ConstrainOperandsToDefinitionAction>(0); + + // We're done with this pattern! It's eligible for GISel emission; return it. + ++NumPatternImported; + return std::move(M); +} + +// Emit imm predicate table and an enum to reference them with. +// The 'Predicate_' part of the name is redundant but eliminating it is more +// trouble than it's worth. +void GlobalISelEmitter::emitCxxPredicateFns( + raw_ostream &OS, StringRef CodeFieldName, StringRef TypeIdentifier, + StringRef ArgType, StringRef ArgName, StringRef AdditionalDeclarations, + std::function<bool(const Record *R)> Filter) { + std::vector<const Record *> MatchedRecords; + const auto &Defs = RK.getAllDerivedDefinitions("PatFrag"); + std::copy_if(Defs.begin(), Defs.end(), std::back_inserter(MatchedRecords), + [&](Record *Record) { + return !Record->getValueAsString(CodeFieldName).empty() && + Filter(Record); + }); + + if (!MatchedRecords.empty()) { + OS << "// PatFrag predicates.\n" + << "enum {\n"; + std::string EnumeratorSeparator = + (" = GIPFP_" + TypeIdentifier + "_Invalid + 1,\n").str(); + for (const auto *Record : MatchedRecords) { + OS << " GIPFP_" << TypeIdentifier << "_Predicate_" << Record->getName() + << EnumeratorSeparator; + EnumeratorSeparator = ",\n"; + } + OS << "};\n"; + } + + OS << "bool " << Target.getName() << "InstructionSelector::test" << ArgName + << "Predicate_" << TypeIdentifier << "(unsigned PredicateID, " << ArgType << " " + << ArgName << ") const {\n" + << AdditionalDeclarations; + if (!AdditionalDeclarations.empty()) + OS << "\n"; + if (!MatchedRecords.empty()) + OS << " switch (PredicateID) {\n"; + for (const auto *Record : MatchedRecords) { + OS << " case GIPFP_" << TypeIdentifier << "_Predicate_" + << Record->getName() << ": {\n" + << " " << Record->getValueAsString(CodeFieldName) << "\n" + << " llvm_unreachable(\"" << CodeFieldName + << " should have returned\");\n" + << " return false;\n" + << " }\n"; + } + if (!MatchedRecords.empty()) + OS << " }\n"; + OS << " llvm_unreachable(\"Unknown predicate\");\n" + << " return false;\n" + << "}\n"; +} + +void GlobalISelEmitter::emitImmPredicateFns( + raw_ostream &OS, StringRef TypeIdentifier, StringRef ArgType, + std::function<bool(const Record *R)> Filter) { + return emitCxxPredicateFns(OS, "ImmediateCode", TypeIdentifier, ArgType, + "Imm", "", Filter); +} + +void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) { + return emitCxxPredicateFns( + OS, "GISelPredicateCode", "MI", "const MachineInstr &", "MI", + " const MachineFunction &MF = *MI.getParent()->getParent();\n" + " const MachineRegisterInfo &MRI = MF.getRegInfo();\n" + " (void)MRI;", + [](const Record *R) { return true; }); +} + +template <class GroupT> +std::vector<Matcher *> GlobalISelEmitter::optimizeRules( + ArrayRef<Matcher *> Rules, + std::vector<std::unique_ptr<Matcher>> &MatcherStorage) { + + std::vector<Matcher *> OptRules; + std::unique_ptr<GroupT> CurrentGroup = std::make_unique<GroupT>(); + assert(CurrentGroup->empty() && "Newly created group isn't empty!"); + unsigned NumGroups = 0; + + auto ProcessCurrentGroup = [&]() { + if (CurrentGroup->empty()) + // An empty group is good to be reused: + return; + + // If the group isn't large enough to provide any benefit, move all the + // added rules out of it and make sure to re-create the group to properly + // re-initialize it: + if (CurrentGroup->size() < 2) + for (Matcher *M : CurrentGroup->matchers()) + OptRules.push_back(M); + else { + CurrentGroup->finalize(); + OptRules.push_back(CurrentGroup.get()); + MatcherStorage.emplace_back(std::move(CurrentGroup)); + ++NumGroups; + } + CurrentGroup = std::make_unique<GroupT>(); + }; + for (Matcher *Rule : Rules) { + // Greedily add as many matchers as possible to the current group: + if (CurrentGroup->addMatcher(*Rule)) + continue; + + ProcessCurrentGroup(); + assert(CurrentGroup->empty() && "A group wasn't properly re-initialized"); + + // Try to add the pending matcher to a newly created empty group: + if (!CurrentGroup->addMatcher(*Rule)) + // If we couldn't add the matcher to an empty group, that group type + // doesn't support that kind of matchers at all, so just skip it: + OptRules.push_back(Rule); + } + ProcessCurrentGroup(); + + LLVM_DEBUG(dbgs() << "NumGroups: " << NumGroups << "\n"); + assert(CurrentGroup->empty() && "The last group wasn't properly processed"); + return OptRules; +} + +MatchTable +GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules, + bool Optimize, bool WithCoverage) { + std::vector<Matcher *> InputRules; + for (Matcher &Rule : Rules) + InputRules.push_back(&Rule); + + if (!Optimize) + return MatchTable::buildTable(InputRules, WithCoverage); + + unsigned CurrentOrdering = 0; + StringMap<unsigned> OpcodeOrder; + for (RuleMatcher &Rule : Rules) { + const StringRef Opcode = Rule.getOpcode(); + assert(!Opcode.empty() && "Didn't expect an undefined opcode"); + if (OpcodeOrder.count(Opcode) == 0) + OpcodeOrder[Opcode] = CurrentOrdering++; + } + + std::stable_sort(InputRules.begin(), InputRules.end(), + [&OpcodeOrder](const Matcher *A, const Matcher *B) { + auto *L = static_cast<const RuleMatcher *>(A); + auto *R = static_cast<const RuleMatcher *>(B); + return std::make_tuple(OpcodeOrder[L->getOpcode()], + L->getNumOperands()) < + std::make_tuple(OpcodeOrder[R->getOpcode()], + R->getNumOperands()); + }); + + for (Matcher *Rule : InputRules) + Rule->optimize(); + + std::vector<std::unique_ptr<Matcher>> MatcherStorage; + std::vector<Matcher *> OptRules = + optimizeRules<GroupMatcher>(InputRules, MatcherStorage); + + for (Matcher *Rule : OptRules) + Rule->optimize(); + + OptRules = optimizeRules<SwitchMatcher>(OptRules, MatcherStorage); + + return MatchTable::buildTable(OptRules, WithCoverage); +} + +void GroupMatcher::optimize() { + // Make sure we only sort by a specific predicate within a range of rules that + // all have that predicate checked against a specific value (not a wildcard): + auto F = Matchers.begin(); + auto T = F; + auto E = Matchers.end(); + while (T != E) { + while (T != E) { + auto *R = static_cast<RuleMatcher *>(*T); + if (!R->getFirstConditionAsRootType().get().isValid()) + break; + ++T; + } + std::stable_sort(F, T, [](Matcher *A, Matcher *B) { + auto *L = static_cast<RuleMatcher *>(A); + auto *R = static_cast<RuleMatcher *>(B); + return L->getFirstConditionAsRootType() < + R->getFirstConditionAsRootType(); + }); + if (T != E) + F = ++T; + } + GlobalISelEmitter::optimizeRules<GroupMatcher>(Matchers, MatcherStorage) + .swap(Matchers); + GlobalISelEmitter::optimizeRules<SwitchMatcher>(Matchers, MatcherStorage) + .swap(Matchers); +} + +void GlobalISelEmitter::run(raw_ostream &OS) { + if (!UseCoverageFile.empty()) { + RuleCoverage = CodeGenCoverage(); + auto RuleCoverageBufOrErr = MemoryBuffer::getFile(UseCoverageFile); + if (!RuleCoverageBufOrErr) { + PrintWarning(SMLoc(), "Missing rule coverage data"); + RuleCoverage = None; + } else { + if (!RuleCoverage->parse(*RuleCoverageBufOrErr.get(), Target.getName())) { + PrintWarning(SMLoc(), "Ignoring invalid or missing rule coverage data"); + RuleCoverage = None; + } + } + } + + // Track the run-time opcode values + gatherOpcodeValues(); + // Track the run-time LLT ID values + gatherTypeIDValues(); + + // Track the GINodeEquiv definitions. + gatherNodeEquivs(); + + emitSourceFileHeader(("Global Instruction Selector for the " + + Target.getName() + " target").str(), OS); + std::vector<RuleMatcher> Rules; + // Look through the SelectionDAG patterns we found, possibly emitting some. + for (const PatternToMatch &Pat : CGP.ptms()) { + ++NumPatternTotal; + + auto MatcherOrErr = runOnPattern(Pat); + + // The pattern analysis can fail, indicating an unsupported pattern. + // Report that if we've been asked to do so. + if (auto Err = MatcherOrErr.takeError()) { + if (WarnOnSkippedPatterns) { + PrintWarning(Pat.getSrcRecord()->getLoc(), + "Skipped pattern: " + toString(std::move(Err))); + } else { + consumeError(std::move(Err)); + } + ++NumPatternImportsSkipped; + continue; + } + + if (RuleCoverage) { + if (RuleCoverage->isCovered(MatcherOrErr->getRuleID())) + ++NumPatternsTested; + else + PrintWarning(Pat.getSrcRecord()->getLoc(), + "Pattern is not covered by a test"); + } + Rules.push_back(std::move(MatcherOrErr.get())); + } + + // Comparison function to order records by name. + auto orderByName = [](const Record *A, const Record *B) { + return A->getName() < B->getName(); + }; + + std::vector<Record *> ComplexPredicates = + RK.getAllDerivedDefinitions("GIComplexOperandMatcher"); + llvm::sort(ComplexPredicates, orderByName); + + std::vector<Record *> CustomRendererFns = + RK.getAllDerivedDefinitions("GICustomOperandRenderer"); + llvm::sort(CustomRendererFns, orderByName); + + unsigned MaxTemporaries = 0; + for (const auto &Rule : Rules) + MaxTemporaries = std::max(MaxTemporaries, Rule.countRendererFns()); + + OS << "#ifdef GET_GLOBALISEL_PREDICATE_BITSET\n" + << "const unsigned MAX_SUBTARGET_PREDICATES = " << SubtargetFeatures.size() + << ";\n" + << "using PredicateBitset = " + "llvm::PredicateBitsetImpl<MAX_SUBTARGET_PREDICATES>;\n" + << "#endif // ifdef GET_GLOBALISEL_PREDICATE_BITSET\n\n"; + + OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n" + << " mutable MatcherState State;\n" + << " typedef " + "ComplexRendererFns(" + << Target.getName() + << "InstructionSelector::*ComplexMatcherMemFn)(MachineOperand &) const;\n" + + << " typedef void(" << Target.getName() + << "InstructionSelector::*CustomRendererFn)(MachineInstrBuilder &, const " + "MachineInstr&, int) " + "const;\n" + << " const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, " + "CustomRendererFn> " + "ISelInfo;\n"; + OS << " static " << Target.getName() + << "InstructionSelector::ComplexMatcherMemFn ComplexPredicateFns[];\n" + << " static " << Target.getName() + << "InstructionSelector::CustomRendererFn CustomRenderers[];\n" + << " bool testImmPredicate_I64(unsigned PredicateID, int64_t Imm) const " + "override;\n" + << " bool testImmPredicate_APInt(unsigned PredicateID, const APInt &Imm) " + "const override;\n" + << " bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat " + "&Imm) const override;\n" + << " const int64_t *getMatchTable() const override;\n" + << " bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI) " + "const override;\n" + << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_DECL\n\n"; + + OS << "#ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n" + << ", State(" << MaxTemporaries << "),\n" + << "ISelInfo(TypeObjects, NumTypeObjects, FeatureBitsets" + << ", ComplexPredicateFns, CustomRenderers)\n" + << "#endif // ifdef GET_GLOBALISEL_TEMPORARIES_INIT\n\n"; + + OS << "#ifdef GET_GLOBALISEL_IMPL\n"; + SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration(SubtargetFeatures, + OS); + + // Separate subtarget features by how often they must be recomputed. + SubtargetFeatureInfoMap ModuleFeatures; + std::copy_if(SubtargetFeatures.begin(), SubtargetFeatures.end(), + std::inserter(ModuleFeatures, ModuleFeatures.end()), + [](const SubtargetFeatureInfoMap::value_type &X) { + return !X.second.mustRecomputePerFunction(); + }); + SubtargetFeatureInfoMap FunctionFeatures; + std::copy_if(SubtargetFeatures.begin(), SubtargetFeatures.end(), + std::inserter(FunctionFeatures, FunctionFeatures.end()), + [](const SubtargetFeatureInfoMap::value_type &X) { + return X.second.mustRecomputePerFunction(); + }); + + SubtargetFeatureInfo::emitComputeAvailableFeatures( + Target.getName(), "InstructionSelector", "computeAvailableModuleFeatures", + ModuleFeatures, OS); + + + OS << "void " << Target.getName() << "InstructionSelector" + "::setupGeneratedPerFunctionState(MachineFunction &MF) {\n" + " AvailableFunctionFeatures = computeAvailableFunctionFeatures(" + "(const " << Target.getName() << "Subtarget*)&MF.getSubtarget(), &MF);\n" + "}\n"; + + if (Target.getName() == "X86" || Target.getName() == "AArch64") { + // TODO: Implement PGSO. + OS << "static bool shouldOptForSize(const MachineFunction *MF) {\n"; + OS << " return MF->getFunction().hasOptSize();\n"; + OS << "}\n\n"; + } + + SubtargetFeatureInfo::emitComputeAvailableFeatures( + Target.getName(), "InstructionSelector", + "computeAvailableFunctionFeatures", FunctionFeatures, OS, + "const MachineFunction *MF"); + + // Emit a table containing the LLT objects needed by the matcher and an enum + // for the matcher to reference them with. + std::vector<LLTCodeGen> TypeObjects; + for (const auto &Ty : KnownTypes) + TypeObjects.push_back(Ty); + llvm::sort(TypeObjects); + OS << "// LLT Objects.\n" + << "enum {\n"; + for (const auto &TypeObject : TypeObjects) { + OS << " "; + TypeObject.emitCxxEnumValue(OS); + OS << ",\n"; + } + OS << "};\n"; + OS << "const static size_t NumTypeObjects = " << TypeObjects.size() << ";\n" + << "const static LLT TypeObjects[] = {\n"; + for (const auto &TypeObject : TypeObjects) { + OS << " "; + TypeObject.emitCxxConstructorCall(OS); + OS << ",\n"; + } + OS << "};\n\n"; + + // Emit a table containing the PredicateBitsets objects needed by the matcher + // and an enum for the matcher to reference them with. + std::vector<std::vector<Record *>> FeatureBitsets; + for (auto &Rule : Rules) + FeatureBitsets.push_back(Rule.getRequiredFeatures()); + llvm::sort(FeatureBitsets, [&](const std::vector<Record *> &A, + const std::vector<Record *> &B) { + if (A.size() < B.size()) + return true; + if (A.size() > B.size()) + return false; + for (auto Pair : zip(A, B)) { + if (std::get<0>(Pair)->getName() < std::get<1>(Pair)->getName()) + return true; + if (std::get<0>(Pair)->getName() > std::get<1>(Pair)->getName()) + return false; + } + return false; + }); + FeatureBitsets.erase( + std::unique(FeatureBitsets.begin(), FeatureBitsets.end()), + FeatureBitsets.end()); + OS << "// Feature bitsets.\n" + << "enum {\n" + << " GIFBS_Invalid,\n"; + for (const auto &FeatureBitset : FeatureBitsets) { + if (FeatureBitset.empty()) + continue; + OS << " " << getNameForFeatureBitset(FeatureBitset) << ",\n"; + } + OS << "};\n" + << "const static PredicateBitset FeatureBitsets[] {\n" + << " {}, // GIFBS_Invalid\n"; + for (const auto &FeatureBitset : FeatureBitsets) { + if (FeatureBitset.empty()) + continue; + OS << " {"; + for (const auto &Feature : FeatureBitset) { + const auto &I = SubtargetFeatures.find(Feature); + assert(I != SubtargetFeatures.end() && "Didn't import predicate?"); + OS << I->second.getEnumBitName() << ", "; + } + OS << "},\n"; + } + OS << "};\n\n"; + + // Emit complex predicate table and an enum to reference them with. + OS << "// ComplexPattern predicates.\n" + << "enum {\n" + << " GICP_Invalid,\n"; + for (const auto &Record : ComplexPredicates) + OS << " GICP_" << Record->getName() << ",\n"; + OS << "};\n" + << "// See constructor for table contents\n\n"; + + emitImmPredicateFns(OS, "I64", "int64_t", [](const Record *R) { + bool Unset; + return !R->getValueAsBitOrUnset("IsAPFloat", Unset) && + !R->getValueAsBit("IsAPInt"); + }); + emitImmPredicateFns(OS, "APFloat", "const APFloat &", [](const Record *R) { + bool Unset; + return R->getValueAsBitOrUnset("IsAPFloat", Unset); + }); + emitImmPredicateFns(OS, "APInt", "const APInt &", [](const Record *R) { + return R->getValueAsBit("IsAPInt"); + }); + emitMIPredicateFns(OS); + OS << "\n"; + + OS << Target.getName() << "InstructionSelector::ComplexMatcherMemFn\n" + << Target.getName() << "InstructionSelector::ComplexPredicateFns[] = {\n" + << " nullptr, // GICP_Invalid\n"; + for (const auto &Record : ComplexPredicates) + OS << " &" << Target.getName() + << "InstructionSelector::" << Record->getValueAsString("MatcherFn") + << ", // " << Record->getName() << "\n"; + OS << "};\n\n"; + + OS << "// Custom renderers.\n" + << "enum {\n" + << " GICR_Invalid,\n"; + for (const auto &Record : CustomRendererFns) + OS << " GICR_" << Record->getValueAsString("RendererFn") << ", \n"; + OS << "};\n"; + + OS << Target.getName() << "InstructionSelector::CustomRendererFn\n" + << Target.getName() << "InstructionSelector::CustomRenderers[] = {\n" + << " nullptr, // GICR_Invalid\n"; + for (const auto &Record : CustomRendererFns) + OS << " &" << Target.getName() + << "InstructionSelector::" << Record->getValueAsString("RendererFn") + << ", // " << Record->getName() << "\n"; + OS << "};\n\n"; + + llvm::stable_sort(Rules, [&](const RuleMatcher &A, const RuleMatcher &B) { + int ScoreA = RuleMatcherScores[A.getRuleID()]; + int ScoreB = RuleMatcherScores[B.getRuleID()]; + if (ScoreA > ScoreB) + return true; + if (ScoreB > ScoreA) + return false; + if (A.isHigherPriorityThan(B)) { + assert(!B.isHigherPriorityThan(A) && "Cannot be more important " + "and less important at " + "the same time"); + return true; + } + return false; + }); + + OS << "bool " << Target.getName() + << "InstructionSelector::selectImpl(MachineInstr &I, CodeGenCoverage " + "&CoverageInfo) const {\n" + << " MachineFunction &MF = *I.getParent()->getParent();\n" + << " MachineRegisterInfo &MRI = MF.getRegInfo();\n" + << " const PredicateBitset AvailableFeatures = getAvailableFeatures();\n" + << " NewMIVector OutMIs;\n" + << " State.MIs.clear();\n" + << " State.MIs.push_back(&I);\n\n" + << " if (executeMatchTable(*this, OutMIs, State, ISelInfo" + << ", getMatchTable(), TII, MRI, TRI, RBI, AvailableFeatures" + << ", CoverageInfo)) {\n" + << " return true;\n" + << " }\n\n" + << " return false;\n" + << "}\n\n"; + + const MatchTable Table = + buildMatchTable(Rules, OptimizeMatchTable, GenerateCoverage); + OS << "const int64_t *" << Target.getName() + << "InstructionSelector::getMatchTable() const {\n"; + Table.emitDeclaration(OS); + OS << " return "; + Table.emitUse(OS); + OS << ";\n}\n"; + OS << "#endif // ifdef GET_GLOBALISEL_IMPL\n"; + + OS << "#ifdef GET_GLOBALISEL_PREDICATES_DECL\n" + << "PredicateBitset AvailableModuleFeatures;\n" + << "mutable PredicateBitset AvailableFunctionFeatures;\n" + << "PredicateBitset getAvailableFeatures() const {\n" + << " return AvailableModuleFeatures | AvailableFunctionFeatures;\n" + << "}\n" + << "PredicateBitset\n" + << "computeAvailableModuleFeatures(const " << Target.getName() + << "Subtarget *Subtarget) const;\n" + << "PredicateBitset\n" + << "computeAvailableFunctionFeatures(const " << Target.getName() + << "Subtarget *Subtarget,\n" + << " const MachineFunction *MF) const;\n" + << "void setupGeneratedPerFunctionState(MachineFunction &MF) override;\n" + << "#endif // ifdef GET_GLOBALISEL_PREDICATES_DECL\n"; + + OS << "#ifdef GET_GLOBALISEL_PREDICATES_INIT\n" + << "AvailableModuleFeatures(computeAvailableModuleFeatures(&STI)),\n" + << "AvailableFunctionFeatures()\n" + << "#endif // ifdef GET_GLOBALISEL_PREDICATES_INIT\n"; +} + +void GlobalISelEmitter::declareSubtargetFeature(Record *Predicate) { + if (SubtargetFeatures.count(Predicate) == 0) + SubtargetFeatures.emplace( + Predicate, SubtargetFeatureInfo(Predicate, SubtargetFeatures.size())); +} + +void RuleMatcher::optimize() { + for (auto &Item : InsnVariableIDs) { + InstructionMatcher &InsnMatcher = *Item.first; + for (auto &OM : InsnMatcher.operands()) { + // Complex Patterns are usually expensive and they relatively rarely fail + // on their own: more often we end up throwing away all the work done by a + // matching part of a complex pattern because some other part of the + // enclosing pattern didn't match. All of this makes it beneficial to + // delay complex patterns until the very end of the rule matching, + // especially for targets having lots of complex patterns. + for (auto &OP : OM->predicates()) + if (isa<ComplexPatternOperandMatcher>(OP)) + EpilogueMatchers.emplace_back(std::move(OP)); + OM->eraseNullPredicates(); + } + InsnMatcher.optimize(); + } + llvm::sort(EpilogueMatchers, [](const std::unique_ptr<PredicateMatcher> &L, + const std::unique_ptr<PredicateMatcher> &R) { + return std::make_tuple(L->getKind(), L->getInsnVarID(), L->getOpIdx()) < + std::make_tuple(R->getKind(), R->getInsnVarID(), R->getOpIdx()); + }); +} + +bool RuleMatcher::hasFirstCondition() const { + if (insnmatchers_empty()) + return false; + InstructionMatcher &Matcher = insnmatchers_front(); + if (!Matcher.predicates_empty()) + return true; + for (auto &OM : Matcher.operands()) + for (auto &OP : OM->predicates()) + if (!isa<InstructionOperandMatcher>(OP)) + return true; + return false; +} + +const PredicateMatcher &RuleMatcher::getFirstCondition() const { + assert(!insnmatchers_empty() && + "Trying to get a condition from an empty RuleMatcher"); + + InstructionMatcher &Matcher = insnmatchers_front(); + if (!Matcher.predicates_empty()) + return **Matcher.predicates_begin(); + // If there is no more predicate on the instruction itself, look at its + // operands. + for (auto &OM : Matcher.operands()) + for (auto &OP : OM->predicates()) + if (!isa<InstructionOperandMatcher>(OP)) + return *OP; + + llvm_unreachable("Trying to get a condition from an InstructionMatcher with " + "no conditions"); +} + +std::unique_ptr<PredicateMatcher> RuleMatcher::popFirstCondition() { + assert(!insnmatchers_empty() && + "Trying to pop a condition from an empty RuleMatcher"); + + InstructionMatcher &Matcher = insnmatchers_front(); + if (!Matcher.predicates_empty()) + return Matcher.predicates_pop_front(); + // If there is no more predicate on the instruction itself, look at its + // operands. + for (auto &OM : Matcher.operands()) + for (auto &OP : OM->predicates()) + if (!isa<InstructionOperandMatcher>(OP)) { + std::unique_ptr<PredicateMatcher> Result = std::move(OP); + OM->eraseNullPredicates(); + return Result; + } + + llvm_unreachable("Trying to pop a condition from an InstructionMatcher with " + "no conditions"); +} + +bool GroupMatcher::candidateConditionMatches( + const PredicateMatcher &Predicate) const { + + if (empty()) { + // Sharing predicates for nested instructions is not supported yet as we + // currently don't hoist the GIM_RecordInsn's properly, therefore we can + // only work on the original root instruction (InsnVarID == 0): + if (Predicate.getInsnVarID() != 0) + return false; + // ... otherwise an empty group can handle any predicate with no specific + // requirements: + return true; + } + + const Matcher &Representative = **Matchers.begin(); + const auto &RepresentativeCondition = Representative.getFirstCondition(); + // ... if not empty, the group can only accomodate matchers with the exact + // same first condition: + return Predicate.isIdentical(RepresentativeCondition); +} + +bool GroupMatcher::addMatcher(Matcher &Candidate) { + if (!Candidate.hasFirstCondition()) + return false; + + const PredicateMatcher &Predicate = Candidate.getFirstCondition(); + if (!candidateConditionMatches(Predicate)) + return false; + + Matchers.push_back(&Candidate); + return true; +} + +void GroupMatcher::finalize() { + assert(Conditions.empty() && "Already finalized?"); + if (empty()) + return; + + Matcher &FirstRule = **Matchers.begin(); + for (;;) { + // All the checks are expected to succeed during the first iteration: + for (const auto &Rule : Matchers) + if (!Rule->hasFirstCondition()) + return; + const auto &FirstCondition = FirstRule.getFirstCondition(); + for (unsigned I = 1, E = Matchers.size(); I < E; ++I) + if (!Matchers[I]->getFirstCondition().isIdentical(FirstCondition)) + return; + + Conditions.push_back(FirstRule.popFirstCondition()); + for (unsigned I = 1, E = Matchers.size(); I < E; ++I) + Matchers[I]->popFirstCondition(); + } +} + +void GroupMatcher::emit(MatchTable &Table) { + unsigned LabelID = ~0U; + if (!Conditions.empty()) { + LabelID = Table.allocateLabelID(); + Table << MatchTable::Opcode("GIM_Try", +1) + << MatchTable::Comment("On fail goto") + << MatchTable::JumpTarget(LabelID) << MatchTable::LineBreak; + } + for (auto &Condition : Conditions) + Condition->emitPredicateOpcodes( + Table, *static_cast<RuleMatcher *>(*Matchers.begin())); + + for (const auto &M : Matchers) + M->emit(Table); + + // Exit the group + if (!Conditions.empty()) + Table << MatchTable::Opcode("GIM_Reject", -1) << MatchTable::LineBreak + << MatchTable::Label(LabelID); +} + +bool SwitchMatcher::isSupportedPredicateType(const PredicateMatcher &P) { + return isa<InstructionOpcodeMatcher>(P) || isa<LLTOperandMatcher>(P); +} + +bool SwitchMatcher::candidateConditionMatches( + const PredicateMatcher &Predicate) const { + + if (empty()) { + // Sharing predicates for nested instructions is not supported yet as we + // currently don't hoist the GIM_RecordInsn's properly, therefore we can + // only work on the original root instruction (InsnVarID == 0): + if (Predicate.getInsnVarID() != 0) + return false; + // ... while an attempt to add even a root matcher to an empty SwitchMatcher + // could fail as not all the types of conditions are supported: + if (!isSupportedPredicateType(Predicate)) + return false; + // ... or the condition might not have a proper implementation of + // getValue() / isIdenticalDownToValue() yet: + if (!Predicate.hasValue()) + return false; + // ... otherwise an empty Switch can accomodate the condition with no + // further requirements: + return true; + } + + const Matcher &CaseRepresentative = **Matchers.begin(); + const auto &RepresentativeCondition = CaseRepresentative.getFirstCondition(); + // Switch-cases must share the same kind of condition and path to the value it + // checks: + if (!Predicate.isIdenticalDownToValue(RepresentativeCondition)) + return false; + + const auto Value = Predicate.getValue(); + // ... but be unique with respect to the actual value they check: + return Values.count(Value) == 0; +} + +bool SwitchMatcher::addMatcher(Matcher &Candidate) { + if (!Candidate.hasFirstCondition()) + return false; + + const PredicateMatcher &Predicate = Candidate.getFirstCondition(); + if (!candidateConditionMatches(Predicate)) + return false; + const auto Value = Predicate.getValue(); + Values.insert(Value); + + Matchers.push_back(&Candidate); + return true; +} + +void SwitchMatcher::finalize() { + assert(Condition == nullptr && "Already finalized"); + assert(Values.size() == Matchers.size() && "Broken SwitchMatcher"); + if (empty()) + return; + + std::stable_sort(Matchers.begin(), Matchers.end(), + [](const Matcher *L, const Matcher *R) { + return L->getFirstCondition().getValue() < + R->getFirstCondition().getValue(); + }); + Condition = Matchers[0]->popFirstCondition(); + for (unsigned I = 1, E = Values.size(); I < E; ++I) + Matchers[I]->popFirstCondition(); +} + +void SwitchMatcher::emitPredicateSpecificOpcodes(const PredicateMatcher &P, + MatchTable &Table) { + assert(isSupportedPredicateType(P) && "Predicate type is not supported"); + + if (const auto *Condition = dyn_cast<InstructionOpcodeMatcher>(&P)) { + Table << MatchTable::Opcode("GIM_SwitchOpcode") << MatchTable::Comment("MI") + << MatchTable::IntValue(Condition->getInsnVarID()); + return; + } + if (const auto *Condition = dyn_cast<LLTOperandMatcher>(&P)) { + Table << MatchTable::Opcode("GIM_SwitchType") << MatchTable::Comment("MI") + << MatchTable::IntValue(Condition->getInsnVarID()) + << MatchTable::Comment("Op") + << MatchTable::IntValue(Condition->getOpIdx()); + return; + } + + llvm_unreachable("emitPredicateSpecificOpcodes is broken: can not handle a " + "predicate type that is claimed to be supported"); +} + +void SwitchMatcher::emit(MatchTable &Table) { + assert(Values.size() == Matchers.size() && "Broken SwitchMatcher"); + if (empty()) + return; + assert(Condition != nullptr && + "Broken SwitchMatcher, hasn't been finalized?"); + + std::vector<unsigned> LabelIDs(Values.size()); + std::generate(LabelIDs.begin(), LabelIDs.end(), + [&Table]() { return Table.allocateLabelID(); }); + const unsigned Default = Table.allocateLabelID(); + + const int64_t LowerBound = Values.begin()->getRawValue(); + const int64_t UpperBound = Values.rbegin()->getRawValue() + 1; + + emitPredicateSpecificOpcodes(*Condition, Table); + + Table << MatchTable::Comment("[") << MatchTable::IntValue(LowerBound) + << MatchTable::IntValue(UpperBound) << MatchTable::Comment(")") + << MatchTable::Comment("default:") << MatchTable::JumpTarget(Default); + + int64_t J = LowerBound; + auto VI = Values.begin(); + for (unsigned I = 0, E = Values.size(); I < E; ++I) { + auto V = *VI++; + while (J++ < V.getRawValue()) + Table << MatchTable::IntValue(0); + V.turnIntoComment(); + Table << MatchTable::LineBreak << V << MatchTable::JumpTarget(LabelIDs[I]); + } + Table << MatchTable::LineBreak; + + for (unsigned I = 0, E = Values.size(); I < E; ++I) { + Table << MatchTable::Label(LabelIDs[I]); + Matchers[I]->emit(Table); + Table << MatchTable::Opcode("GIM_Reject") << MatchTable::LineBreak; + } + Table << MatchTable::Label(Default); +} + +unsigned OperandMatcher::getInsnVarID() const { return Insn.getInsnVarID(); } + +} // end anonymous namespace + +//===----------------------------------------------------------------------===// + +namespace llvm { +void EmitGlobalISel(RecordKeeper &RK, raw_ostream &OS) { + GlobalISelEmitter(RK).run(OS); +} +} // End llvm namespace diff --git a/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.cpp b/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.cpp new file mode 100644 index 000000000000..7cd1b0f08132 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.cpp @@ -0,0 +1,221 @@ +//===--- InfoByHwMode.cpp -------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Classes that implement data parameterized by HW modes for instruction +// selection. Currently it is ValueTypeByHwMode (parameterized ValueType), +// and RegSizeInfoByHwMode (parameterized register/spill size and alignment +// data). +//===----------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "InfoByHwMode.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +#include <set> +#include <string> + +using namespace llvm; + +std::string llvm::getModeName(unsigned Mode) { + if (Mode == DefaultMode) + return "*"; + return (Twine('m') + Twine(Mode)).str(); +} + +ValueTypeByHwMode::ValueTypeByHwMode(Record *R, const CodeGenHwModes &CGH) { + const HwModeSelect &MS = CGH.getHwModeSelect(R); + for (const HwModeSelect::PairType &P : MS.Items) { + auto I = Map.insert({P.first, MVT(llvm::getValueType(P.second))}); + assert(I.second && "Duplicate entry?"); + (void)I; + } +} + +ValueTypeByHwMode::ValueTypeByHwMode(Record *R, MVT T) : ValueTypeByHwMode(T) { + if (R->isSubClassOf("PtrValueType")) + PtrAddrSpace = R->getValueAsInt("AddrSpace"); +} + +bool ValueTypeByHwMode::operator== (const ValueTypeByHwMode &T) const { + assert(isValid() && T.isValid() && "Invalid type in assignment"); + bool Simple = isSimple(); + if (Simple != T.isSimple()) + return false; + if (Simple) + return getSimple() == T.getSimple(); + + return Map == T.Map; +} + +bool ValueTypeByHwMode::operator< (const ValueTypeByHwMode &T) const { + assert(isValid() && T.isValid() && "Invalid type in comparison"); + // Default order for maps. + return Map < T.Map; +} + +MVT &ValueTypeByHwMode::getOrCreateTypeForMode(unsigned Mode, MVT Type) { + auto F = Map.find(Mode); + if (F != Map.end()) + return F->second; + // If Mode is not in the map, look up the default mode. If it exists, + // make a copy of it for Mode and return it. + auto D = Map.find(DefaultMode); + if (D != Map.end()) + return Map.insert(std::make_pair(Mode, D->second)).first->second; + // If default mode is not present either, use provided Type. + return Map.insert(std::make_pair(Mode, Type)).first->second; +} + +StringRef ValueTypeByHwMode::getMVTName(MVT T) { + StringRef N = llvm::getEnumName(T.SimpleTy); + N.consume_front("MVT::"); + return N; +} + +void ValueTypeByHwMode::writeToStream(raw_ostream &OS) const { + if (isSimple()) { + OS << getMVTName(getSimple()); + return; + } + + std::vector<const PairType*> Pairs; + for (const auto &P : Map) + Pairs.push_back(&P); + llvm::sort(Pairs, deref<std::less<PairType>>()); + + OS << '{'; + for (unsigned i = 0, e = Pairs.size(); i != e; ++i) { + const PairType *P = Pairs[i]; + OS << '(' << getModeName(P->first) + << ':' << getMVTName(P->second).str() << ')'; + if (i != e-1) + OS << ','; + } + OS << '}'; +} + +LLVM_DUMP_METHOD +void ValueTypeByHwMode::dump() const { + dbgs() << *this << '\n'; +} + +ValueTypeByHwMode llvm::getValueTypeByHwMode(Record *Rec, + const CodeGenHwModes &CGH) { +#ifndef NDEBUG + if (!Rec->isSubClassOf("ValueType")) + Rec->dump(); +#endif + assert(Rec->isSubClassOf("ValueType") && + "Record must be derived from ValueType"); + if (Rec->isSubClassOf("HwModeSelect")) + return ValueTypeByHwMode(Rec, CGH); + return ValueTypeByHwMode(Rec, llvm::getValueType(Rec)); +} + +RegSizeInfo::RegSizeInfo(Record *R, const CodeGenHwModes &CGH) { + RegSize = R->getValueAsInt("RegSize"); + SpillSize = R->getValueAsInt("SpillSize"); + SpillAlignment = R->getValueAsInt("SpillAlignment"); +} + +bool RegSizeInfo::operator< (const RegSizeInfo &I) const { + return std::tie(RegSize, SpillSize, SpillAlignment) < + std::tie(I.RegSize, I.SpillSize, I.SpillAlignment); +} + +bool RegSizeInfo::isSubClassOf(const RegSizeInfo &I) const { + return RegSize <= I.RegSize && + SpillAlignment && I.SpillAlignment % SpillAlignment == 0 && + SpillSize <= I.SpillSize; +} + +void RegSizeInfo::writeToStream(raw_ostream &OS) const { + OS << "[R=" << RegSize << ",S=" << SpillSize + << ",A=" << SpillAlignment << ']'; +} + +RegSizeInfoByHwMode::RegSizeInfoByHwMode(Record *R, + const CodeGenHwModes &CGH) { + const HwModeSelect &MS = CGH.getHwModeSelect(R); + for (const HwModeSelect::PairType &P : MS.Items) { + auto I = Map.insert({P.first, RegSizeInfo(P.second, CGH)}); + assert(I.second && "Duplicate entry?"); + (void)I; + } +} + +bool RegSizeInfoByHwMode::operator< (const RegSizeInfoByHwMode &I) const { + unsigned M0 = Map.begin()->first; + return get(M0) < I.get(M0); +} + +bool RegSizeInfoByHwMode::operator== (const RegSizeInfoByHwMode &I) const { + unsigned M0 = Map.begin()->first; + return get(M0) == I.get(M0); +} + +bool RegSizeInfoByHwMode::isSubClassOf(const RegSizeInfoByHwMode &I) const { + unsigned M0 = Map.begin()->first; + return get(M0).isSubClassOf(I.get(M0)); +} + +bool RegSizeInfoByHwMode::hasStricterSpillThan(const RegSizeInfoByHwMode &I) + const { + unsigned M0 = Map.begin()->first; + const RegSizeInfo &A0 = get(M0); + const RegSizeInfo &B0 = I.get(M0); + return std::tie(A0.SpillSize, A0.SpillAlignment) > + std::tie(B0.SpillSize, B0.SpillAlignment); +} + +void RegSizeInfoByHwMode::writeToStream(raw_ostream &OS) const { + typedef typename decltype(Map)::value_type PairType; + std::vector<const PairType*> Pairs; + for (const auto &P : Map) + Pairs.push_back(&P); + llvm::sort(Pairs, deref<std::less<PairType>>()); + + OS << '{'; + for (unsigned i = 0, e = Pairs.size(); i != e; ++i) { + const PairType *P = Pairs[i]; + OS << '(' << getModeName(P->first) << ':' << P->second << ')'; + if (i != e-1) + OS << ','; + } + OS << '}'; +} + +EncodingInfoByHwMode::EncodingInfoByHwMode(Record *R, const CodeGenHwModes &CGH) { + const HwModeSelect &MS = CGH.getHwModeSelect(R); + for (const HwModeSelect::PairType &P : MS.Items) { + assert(P.second && P.second->isSubClassOf("InstructionEncoding") && + "Encoding must subclass InstructionEncoding"); + auto I = Map.insert({P.first, P.second}); + assert(I.second && "Duplicate entry?"); + (void)I; + } +} + +namespace llvm { + raw_ostream &operator<<(raw_ostream &OS, const ValueTypeByHwMode &T) { + T.writeToStream(OS); + return OS; + } + + raw_ostream &operator<<(raw_ostream &OS, const RegSizeInfo &T) { + T.writeToStream(OS); + return OS; + } + + raw_ostream &operator<<(raw_ostream &OS, const RegSizeInfoByHwMode &T) { + T.writeToStream(OS); + return OS; + } +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.h b/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.h new file mode 100644 index 000000000000..d92e5901a7f3 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/InfoByHwMode.h @@ -0,0 +1,194 @@ +//===--- InfoByHwMode.h -----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// Classes that implement data parameterized by HW modes for instruction +// selection. Currently it is ValueTypeByHwMode (parameterized ValueType), +// and RegSizeInfoByHwMode (parameterized register/spill size and alignment +// data). +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_INFOBYHWMODE_H +#define LLVM_UTILS_TABLEGEN_INFOBYHWMODE_H + +#include "CodeGenHwModes.h" +#include "llvm/Support/MachineValueType.h" + +#include <map> +#include <set> +#include <string> +#include <vector> + +namespace llvm { + +struct CodeGenHwModes; +class Record; +class raw_ostream; + +template <typename InfoT> struct InfoByHwMode; + +std::string getModeName(unsigned Mode); + +enum : unsigned { + DefaultMode = CodeGenHwModes::DefaultMode, +}; + +template <typename InfoT> +std::vector<unsigned> union_modes(const InfoByHwMode<InfoT> &A, + const InfoByHwMode<InfoT> &B) { + std::vector<unsigned> V; + std::set<unsigned> U; + for (const auto &P : A) + U.insert(P.first); + for (const auto &P : B) + U.insert(P.first); + // Make sure that the default mode is last on the list. + bool HasDefault = false; + for (unsigned M : U) + if (M != DefaultMode) + V.push_back(M); + else + HasDefault = true; + if (HasDefault) + V.push_back(DefaultMode); + return V; +} + +template <typename InfoT> +struct InfoByHwMode { + typedef std::map<unsigned,InfoT> MapType; + typedef typename MapType::value_type PairType; + typedef typename MapType::iterator iterator; + typedef typename MapType::const_iterator const_iterator; + + InfoByHwMode() = default; + InfoByHwMode(const MapType &M) : Map(M) {} + + LLVM_ATTRIBUTE_ALWAYS_INLINE + iterator begin() { return Map.begin(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE + iterator end() { return Map.end(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE + const_iterator begin() const { return Map.begin(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE + const_iterator end() const { return Map.end(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool empty() const { return Map.empty(); } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool hasMode(unsigned M) const { return Map.find(M) != Map.end(); } + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool hasDefault() const { return hasMode(DefaultMode); } + + InfoT &get(unsigned Mode) { + if (!hasMode(Mode)) { + assert(hasMode(DefaultMode)); + Map.insert({Mode, Map.at(DefaultMode)}); + } + return Map.at(Mode); + } + const InfoT &get(unsigned Mode) const { + auto F = Map.find(Mode); + if (Mode != DefaultMode && F == Map.end()) + F = Map.find(DefaultMode); + assert(F != Map.end()); + return F->second; + } + + LLVM_ATTRIBUTE_ALWAYS_INLINE + bool isSimple() const { + return Map.size() == 1 && Map.begin()->first == DefaultMode; + } + LLVM_ATTRIBUTE_ALWAYS_INLINE + InfoT getSimple() const { + assert(isSimple()); + return Map.begin()->second; + } + void makeSimple(unsigned Mode) { + assert(hasMode(Mode) || hasDefault()); + InfoT I = get(Mode); + Map.clear(); + Map.insert(std::make_pair(DefaultMode, I)); + } + + MapType Map; +}; + +struct ValueTypeByHwMode : public InfoByHwMode<MVT> { + ValueTypeByHwMode(Record *R, const CodeGenHwModes &CGH); + ValueTypeByHwMode(Record *R, MVT T); + ValueTypeByHwMode(MVT T) { Map.insert({DefaultMode,T}); } + ValueTypeByHwMode() = default; + + bool operator== (const ValueTypeByHwMode &T) const; + bool operator< (const ValueTypeByHwMode &T) const; + + bool isValid() const { + return !Map.empty(); + } + MVT getType(unsigned Mode) const { return get(Mode); } + MVT &getOrCreateTypeForMode(unsigned Mode, MVT Type); + + static StringRef getMVTName(MVT T); + void writeToStream(raw_ostream &OS) const; + void dump() const; + + unsigned PtrAddrSpace = std::numeric_limits<unsigned>::max(); + bool isPointer() const { + return PtrAddrSpace != std::numeric_limits<unsigned>::max(); + } +}; + +ValueTypeByHwMode getValueTypeByHwMode(Record *Rec, + const CodeGenHwModes &CGH); + +struct RegSizeInfo { + unsigned RegSize; + unsigned SpillSize; + unsigned SpillAlignment; + + RegSizeInfo(Record *R, const CodeGenHwModes &CGH); + RegSizeInfo() = default; + bool operator< (const RegSizeInfo &I) const; + bool operator== (const RegSizeInfo &I) const { + return std::tie(RegSize, SpillSize, SpillAlignment) == + std::tie(I.RegSize, I.SpillSize, I.SpillAlignment); + } + bool operator!= (const RegSizeInfo &I) const { + return !(*this == I); + } + + bool isSubClassOf(const RegSizeInfo &I) const; + void writeToStream(raw_ostream &OS) const; +}; + +struct RegSizeInfoByHwMode : public InfoByHwMode<RegSizeInfo> { + RegSizeInfoByHwMode(Record *R, const CodeGenHwModes &CGH); + RegSizeInfoByHwMode() = default; + bool operator< (const RegSizeInfoByHwMode &VI) const; + bool operator== (const RegSizeInfoByHwMode &VI) const; + bool operator!= (const RegSizeInfoByHwMode &VI) const { + return !(*this == VI); + } + + bool isSubClassOf(const RegSizeInfoByHwMode &I) const; + bool hasStricterSpillThan(const RegSizeInfoByHwMode &I) const; + + void writeToStream(raw_ostream &OS) const; +}; + +raw_ostream &operator<<(raw_ostream &OS, const ValueTypeByHwMode &T); +raw_ostream &operator<<(raw_ostream &OS, const RegSizeInfo &T); +raw_ostream &operator<<(raw_ostream &OS, const RegSizeInfoByHwMode &T); + +struct EncodingInfoByHwMode : public InfoByHwMode<Record*> { + EncodingInfoByHwMode(Record *R, const CodeGenHwModes &CGH); + EncodingInfoByHwMode() = default; +}; + +} // namespace llvm + +#endif // LLVM_UTILS_TABLEGEN_INFOBYHWMODE_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/InstrDocsEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/InstrDocsEmitter.cpp new file mode 100644 index 000000000000..66744bf9ecef --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/InstrDocsEmitter.cpp @@ -0,0 +1,235 @@ +//===- InstrDocsEmitter.cpp - Opcode Documentation Generator --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// InstrDocsEmitter generates restructured text documentation for the opcodes +// that can be used by MachineInstr. For each opcode, the documentation lists: +// * Opcode name +// * Assembly string +// * Flags (e.g. mayLoad, isBranch, ...) +// * Operands, including type and name +// * Operand constraints +// * Implicit register uses & defs +// * Predicates +// +//===----------------------------------------------------------------------===// + +#include "CodeGenDAGPatterns.h" +#include "CodeGenInstruction.h" +#include "CodeGenTarget.h" +#include "TableGenBackends.h" +#include "llvm/TableGen/Record.h" +#include <string> +#include <vector> + +using namespace llvm; + +namespace llvm { + +void writeTitle(StringRef Str, raw_ostream &OS, char Kind = '-') { + OS << std::string(Str.size(), Kind) << "\n" << Str << "\n" + << std::string(Str.size(), Kind) << "\n"; +} + +void writeHeader(StringRef Str, raw_ostream &OS, char Kind = '-') { + OS << Str << "\n" << std::string(Str.size(), Kind) << "\n"; +} + +std::string escapeForRST(StringRef Str) { + std::string Result; + Result.reserve(Str.size() + 4); + for (char C : Str) { + switch (C) { + // We want special characters to be shown as their C escape codes. + case '\n': Result += "\\n"; break; + case '\t': Result += "\\t"; break; + // Underscore at the end of a line has a special meaning in rst. + case '_': Result += "\\_"; break; + default: Result += C; + } + } + return Result; +} + +void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS) { + CodeGenDAGPatterns CDP(RK); + CodeGenTarget &Target = CDP.getTargetInfo(); + unsigned VariantCount = Target.getAsmParserVariantCount(); + + // Page title. + std::string Title = std::string(Target.getName()); + Title += " Instructions"; + writeTitle(Title, OS); + OS << "\n"; + + for (const CodeGenInstruction *II : Target.getInstructionsByEnumValue()) { + Record *Inst = II->TheDef; + + // Don't print the target-independent instructions. + if (II->Namespace == "TargetOpcode") + continue; + + // Heading (instruction name). + writeHeader(escapeForRST(Inst->getName()), OS, '='); + OS << "\n"; + + // Assembly string(s). + if (!II->AsmString.empty()) { + for (unsigned VarNum = 0; VarNum < VariantCount; ++VarNum) { + Record *AsmVariant = Target.getAsmParserVariant(VarNum); + OS << "Assembly string"; + if (VariantCount != 1) + OS << " (" << AsmVariant->getValueAsString("Name") << ")"; + std::string AsmString = + CodeGenInstruction::FlattenAsmStringVariants(II->AsmString, VarNum); + // We trim spaces at each end of the asm string because rst needs the + // formatting backticks to be next to a non-whitespace character. + OS << ": ``" << escapeForRST(StringRef(AsmString).trim(" ")) + << "``\n\n"; + } + } + + // Boolean flags. + std::vector<const char *> FlagStrings; +#define xstr(s) str(s) +#define str(s) #s +#define FLAG(f) if (II->f) { FlagStrings.push_back(str(f)); } + FLAG(isReturn) + FLAG(isEHScopeReturn) + FLAG(isBranch) + FLAG(isIndirectBranch) + FLAG(isCompare) + FLAG(isMoveImm) + FLAG(isBitcast) + FLAG(isSelect) + FLAG(isBarrier) + FLAG(isCall) + FLAG(isAdd) + FLAG(isTrap) + FLAG(canFoldAsLoad) + FLAG(mayLoad) + //FLAG(mayLoad_Unset) // Deliberately omitted. + FLAG(mayStore) + //FLAG(mayStore_Unset) // Deliberately omitted. + FLAG(isPredicable) + FLAG(isConvertibleToThreeAddress) + FLAG(isCommutable) + FLAG(isTerminator) + FLAG(isReMaterializable) + FLAG(hasDelaySlot) + FLAG(usesCustomInserter) + FLAG(hasPostISelHook) + FLAG(hasCtrlDep) + FLAG(isNotDuplicable) + FLAG(hasSideEffects) + //FLAG(hasSideEffects_Unset) // Deliberately omitted. + FLAG(isAsCheapAsAMove) + FLAG(hasExtraSrcRegAllocReq) + FLAG(hasExtraDefRegAllocReq) + FLAG(isCodeGenOnly) + FLAG(isPseudo) + FLAG(isRegSequence) + FLAG(isExtractSubreg) + FLAG(isInsertSubreg) + FLAG(isConvergent) + FLAG(hasNoSchedulingInfo) + FLAG(variadicOpsAreDefs) + FLAG(isAuthenticated) + if (!FlagStrings.empty()) { + OS << "Flags: "; + bool IsFirst = true; + for (auto FlagString : FlagStrings) { + if (!IsFirst) + OS << ", "; + OS << "``" << FlagString << "``"; + IsFirst = false; + } + OS << "\n\n"; + } + + // Operands. + for (unsigned i = 0; i < II->Operands.size(); ++i) { + bool IsDef = i < II->Operands.NumDefs; + auto Op = II->Operands[i]; + + if (Op.MINumOperands > 1) { + // This operand corresponds to multiple operands on the + // MachineInstruction, so print all of them, showing the types and + // names of both the compound operand and the basic operands it + // contains. + for (unsigned SubOpIdx = 0; SubOpIdx < Op.MINumOperands; ++SubOpIdx) { + Record *SubRec = + cast<DefInit>(Op.MIOperandInfo->getArg(SubOpIdx))->getDef(); + StringRef SubOpName = Op.MIOperandInfo->getArgNameStr(SubOpIdx); + StringRef SubOpTypeName = SubRec->getName(); + + OS << "* " << (IsDef ? "DEF" : "USE") << " ``" << Op.Rec->getName() + << "/" << SubOpTypeName << ":$" << Op.Name << "."; + // Not all sub-operands are named, make up a name for these. + if (SubOpName.empty()) + OS << "anon" << SubOpIdx; + else + OS << SubOpName; + OS << "``\n\n"; + } + } else { + // The operand corresponds to only one MachineInstruction operand. + OS << "* " << (IsDef ? "DEF" : "USE") << " ``" << Op.Rec->getName() + << ":$" << Op.Name << "``\n\n"; + } + } + + // Constraints. + StringRef Constraints = Inst->getValueAsString("Constraints"); + if (!Constraints.empty()) { + OS << "Constraints: ``" << Constraints << "``\n\n"; + } + + // Implicit definitions. + if (!II->ImplicitDefs.empty()) { + OS << "Implicit defs: "; + bool IsFirst = true; + for (Record *Def : II->ImplicitDefs) { + if (!IsFirst) + OS << ", "; + OS << "``" << Def->getName() << "``"; + IsFirst = false; + } + OS << "\n\n"; + } + + // Implicit uses. + if (!II->ImplicitUses.empty()) { + OS << "Implicit uses: "; + bool IsFirst = true; + for (Record *Use : II->ImplicitUses) { + if (!IsFirst) + OS << ", "; + OS << "``" << Use->getName() << "``"; + IsFirst = false; + } + OS << "\n\n"; + } + + // Predicates. + std::vector<Record *> Predicates = + II->TheDef->getValueAsListOfDefs("Predicates"); + if (!Predicates.empty()) { + OS << "Predicates: "; + bool IsFirst = true; + for (Record *P : Predicates) { + if (!IsFirst) + OS << ", "; + OS << "``" << P->getName() << "``"; + IsFirst = false; + } + OS << "\n\n"; + } + } +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp new file mode 100644 index 000000000000..f3141735a995 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/InstrInfoEmitter.cpp @@ -0,0 +1,869 @@ +//===- InstrInfoEmitter.cpp - Generate a Instruction Set Desc. --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend is responsible for emitting a description of the target +// instruction set for the code generator. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenDAGPatterns.h" +#include "CodeGenInstruction.h" +#include "CodeGenSchedule.h" +#include "CodeGenTarget.h" +#include "PredicateExpander.h" +#include "SequenceToOffsetTable.h" +#include "TableGenBackends.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cassert> +#include <cstdint> +#include <map> +#include <string> +#include <utility> +#include <vector> + +using namespace llvm; + +namespace { + +class InstrInfoEmitter { + RecordKeeper &Records; + CodeGenDAGPatterns CDP; + const CodeGenSchedModels &SchedModels; + +public: + InstrInfoEmitter(RecordKeeper &R): + Records(R), CDP(R), SchedModels(CDP.getTargetInfo().getSchedModels()) {} + + // run - Output the instruction set description. + void run(raw_ostream &OS); + +private: + void emitEnums(raw_ostream &OS); + + typedef std::map<std::vector<std::string>, unsigned> OperandInfoMapTy; + + /// The keys of this map are maps which have OpName enum values as their keys + /// and instruction operand indices as their values. The values of this map + /// are lists of instruction names. + typedef std::map<std::map<unsigned, unsigned>, + std::vector<std::string>> OpNameMapTy; + typedef std::map<std::string, unsigned>::iterator StrUintMapIter; + + /// Generate member functions in the target-specific GenInstrInfo class. + /// + /// This method is used to custom expand TIIPredicate definitions. + /// See file llvm/Target/TargetInstPredicates.td for a description of what is + /// a TIIPredicate and how to use it. + void emitTIIHelperMethods(raw_ostream &OS, StringRef TargetName, + bool ExpandDefinition = true); + + /// Expand TIIPredicate definitions to functions that accept a const MCInst + /// reference. + void emitMCIIHelperMethods(raw_ostream &OS, StringRef TargetName); + void emitRecord(const CodeGenInstruction &Inst, unsigned Num, + Record *InstrInfo, + std::map<std::vector<Record*>, unsigned> &EL, + const OperandInfoMapTy &OpInfo, + raw_ostream &OS); + void emitOperandTypeMappings( + raw_ostream &OS, const CodeGenTarget &Target, + ArrayRef<const CodeGenInstruction *> NumberedInstructions); + void initOperandMapData( + ArrayRef<const CodeGenInstruction *> NumberedInstructions, + StringRef Namespace, + std::map<std::string, unsigned> &Operands, + OpNameMapTy &OperandMap); + void emitOperandNameMappings(raw_ostream &OS, const CodeGenTarget &Target, + ArrayRef<const CodeGenInstruction*> NumberedInstructions); + + // Operand information. + void EmitOperandInfo(raw_ostream &OS, OperandInfoMapTy &OperandInfoIDs); + std::vector<std::string> GetOperandInfo(const CodeGenInstruction &Inst); +}; + +} // end anonymous namespace + +static void PrintDefList(const std::vector<Record*> &Uses, + unsigned Num, raw_ostream &OS) { + OS << "static const MCPhysReg ImplicitList" << Num << "[] = { "; + for (Record *U : Uses) + OS << getQualifiedName(U) << ", "; + OS << "0 };\n"; +} + +//===----------------------------------------------------------------------===// +// Operand Info Emission. +//===----------------------------------------------------------------------===// + +std::vector<std::string> +InstrInfoEmitter::GetOperandInfo(const CodeGenInstruction &Inst) { + std::vector<std::string> Result; + + for (auto &Op : Inst.Operands) { + // Handle aggregate operands and normal operands the same way by expanding + // either case into a list of operands for this op. + std::vector<CGIOperandList::OperandInfo> OperandList; + + // This might be a multiple operand thing. Targets like X86 have + // registers in their multi-operand operands. It may also be an anonymous + // operand, which has a single operand, but no declared class for the + // operand. + DagInit *MIOI = Op.MIOperandInfo; + + if (!MIOI || MIOI->getNumArgs() == 0) { + // Single, anonymous, operand. + OperandList.push_back(Op); + } else { + for (unsigned j = 0, e = Op.MINumOperands; j != e; ++j) { + OperandList.push_back(Op); + + auto *OpR = cast<DefInit>(MIOI->getArg(j))->getDef(); + OperandList.back().Rec = OpR; + } + } + + for (unsigned j = 0, e = OperandList.size(); j != e; ++j) { + Record *OpR = OperandList[j].Rec; + std::string Res; + + if (OpR->isSubClassOf("RegisterOperand")) + OpR = OpR->getValueAsDef("RegClass"); + if (OpR->isSubClassOf("RegisterClass")) + Res += getQualifiedName(OpR) + "RegClassID, "; + else if (OpR->isSubClassOf("PointerLikeRegClass")) + Res += utostr(OpR->getValueAsInt("RegClassKind")) + ", "; + else + // -1 means the operand does not have a fixed register class. + Res += "-1, "; + + // Fill in applicable flags. + Res += "0"; + + // Ptr value whose register class is resolved via callback. + if (OpR->isSubClassOf("PointerLikeRegClass")) + Res += "|(1<<MCOI::LookupPtrRegClass)"; + + // Predicate operands. Check to see if the original unexpanded operand + // was of type PredicateOp. + if (Op.Rec->isSubClassOf("PredicateOp")) + Res += "|(1<<MCOI::Predicate)"; + + // Optional def operands. Check to see if the original unexpanded operand + // was of type OptionalDefOperand. + if (Op.Rec->isSubClassOf("OptionalDefOperand")) + Res += "|(1<<MCOI::OptionalDef)"; + + // Branch target operands. Check to see if the original unexpanded + // operand was of type BranchTargetOperand. + if (Op.Rec->isSubClassOf("BranchTargetOperand")) + Res += "|(1<<MCOI::BranchTarget)"; + + // Fill in operand type. + Res += ", "; + assert(!Op.OperandType.empty() && "Invalid operand type."); + Res += Op.OperandType; + + // Fill in constraint info. + Res += ", "; + + const CGIOperandList::ConstraintInfo &Constraint = + Op.Constraints[j]; + if (Constraint.isNone()) + Res += "0"; + else if (Constraint.isEarlyClobber()) + Res += "(1 << MCOI::EARLY_CLOBBER)"; + else { + assert(Constraint.isTied()); + Res += "((" + utostr(Constraint.getTiedOperand()) + + " << 16) | (1 << MCOI::TIED_TO))"; + } + + Result.push_back(Res); + } + } + + return Result; +} + +void InstrInfoEmitter::EmitOperandInfo(raw_ostream &OS, + OperandInfoMapTy &OperandInfoIDs) { + // ID #0 is for no operand info. + unsigned OperandListNum = 0; + OperandInfoIDs[std::vector<std::string>()] = ++OperandListNum; + + OS << "\n"; + const CodeGenTarget &Target = CDP.getTargetInfo(); + for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) { + std::vector<std::string> OperandInfo = GetOperandInfo(*Inst); + unsigned &N = OperandInfoIDs[OperandInfo]; + if (N != 0) continue; + + N = ++OperandListNum; + OS << "static const MCOperandInfo OperandInfo" << N << "[] = { "; + for (const std::string &Info : OperandInfo) + OS << "{ " << Info << " }, "; + OS << "};\n"; + } +} + +/// Initialize data structures for generating operand name mappings. +/// +/// \param Operands [out] A map used to generate the OpName enum with operand +/// names as its keys and operand enum values as its values. +/// \param OperandMap [out] A map for representing the operand name mappings for +/// each instructions. This is used to generate the OperandMap table as +/// well as the getNamedOperandIdx() function. +void InstrInfoEmitter::initOperandMapData( + ArrayRef<const CodeGenInstruction *> NumberedInstructions, + StringRef Namespace, + std::map<std::string, unsigned> &Operands, + OpNameMapTy &OperandMap) { + unsigned NumOperands = 0; + for (const CodeGenInstruction *Inst : NumberedInstructions) { + if (!Inst->TheDef->getValueAsBit("UseNamedOperandTable")) + continue; + std::map<unsigned, unsigned> OpList; + for (const auto &Info : Inst->Operands) { + StrUintMapIter I = Operands.find(Info.Name); + + if (I == Operands.end()) { + I = Operands.insert(Operands.begin(), + std::pair<std::string, unsigned>(Info.Name, NumOperands++)); + } + OpList[I->second] = Info.MIOperandNo; + } + OperandMap[OpList].push_back(Namespace.str() + "::" + + Inst->TheDef->getName().str()); + } +} + +/// Generate a table and function for looking up the indices of operands by +/// name. +/// +/// This code generates: +/// - An enum in the llvm::TargetNamespace::OpName namespace, with one entry +/// for each operand name. +/// - A 2-dimensional table called OperandMap for mapping OpName enum values to +/// operand indices. +/// - A function called getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx) +/// for looking up the operand index for an instruction, given a value from +/// OpName enum +void InstrInfoEmitter::emitOperandNameMappings(raw_ostream &OS, + const CodeGenTarget &Target, + ArrayRef<const CodeGenInstruction*> NumberedInstructions) { + StringRef Namespace = Target.getInstNamespace(); + std::string OpNameNS = "OpName"; + // Map of operand names to their enumeration value. This will be used to + // generate the OpName enum. + std::map<std::string, unsigned> Operands; + OpNameMapTy OperandMap; + + initOperandMapData(NumberedInstructions, Namespace, Operands, OperandMap); + + OS << "#ifdef GET_INSTRINFO_OPERAND_ENUM\n"; + OS << "#undef GET_INSTRINFO_OPERAND_ENUM\n"; + OS << "namespace llvm {\n"; + OS << "namespace " << Namespace << " {\n"; + OS << "namespace " << OpNameNS << " {\n"; + OS << "enum {\n"; + for (const auto &Op : Operands) + OS << " " << Op.first << " = " << Op.second << ",\n"; + + OS << "OPERAND_LAST"; + OS << "\n};\n"; + OS << "} // end namespace OpName\n"; + OS << "} // end namespace " << Namespace << "\n"; + OS << "} // end namespace llvm\n"; + OS << "#endif //GET_INSTRINFO_OPERAND_ENUM\n\n"; + + OS << "#ifdef GET_INSTRINFO_NAMED_OPS\n"; + OS << "#undef GET_INSTRINFO_NAMED_OPS\n"; + OS << "namespace llvm {\n"; + OS << "namespace " << Namespace << " {\n"; + OS << "LLVM_READONLY\n"; + OS << "int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx) {\n"; + if (!Operands.empty()) { + OS << " static const int16_t OperandMap [][" << Operands.size() + << "] = {\n"; + for (const auto &Entry : OperandMap) { + const std::map<unsigned, unsigned> &OpList = Entry.first; + OS << "{"; + + // Emit a row of the OperandMap table + for (unsigned i = 0, e = Operands.size(); i != e; ++i) + OS << (OpList.count(i) == 0 ? -1 : (int)OpList.find(i)->second) << ", "; + + OS << "},\n"; + } + OS << "};\n"; + + OS << " switch(Opcode) {\n"; + unsigned TableIndex = 0; + for (const auto &Entry : OperandMap) { + for (const std::string &Name : Entry.second) + OS << " case " << Name << ":\n"; + + OS << " return OperandMap[" << TableIndex++ << "][NamedIdx];\n"; + } + OS << " default: return -1;\n"; + OS << " }\n"; + } else { + // There are no operands, so no need to emit anything + OS << " return -1;\n"; + } + OS << "}\n"; + OS << "} // end namespace " << Namespace << "\n"; + OS << "} // end namespace llvm\n"; + OS << "#endif //GET_INSTRINFO_NAMED_OPS\n\n"; +} + +/// Generate an enum for all the operand types for this target, under the +/// llvm::TargetNamespace::OpTypes namespace. +/// Operand types are all definitions derived of the Operand Target.td class. +void InstrInfoEmitter::emitOperandTypeMappings( + raw_ostream &OS, const CodeGenTarget &Target, + ArrayRef<const CodeGenInstruction *> NumberedInstructions) { + + StringRef Namespace = Target.getInstNamespace(); + std::vector<Record *> Operands = Records.getAllDerivedDefinitions("Operand"); + std::vector<Record *> RegisterOperands = + Records.getAllDerivedDefinitions("RegisterOperand"); + std::vector<Record *> RegisterClasses = + Records.getAllDerivedDefinitions("RegisterClass"); + + OS << "#ifdef GET_INSTRINFO_OPERAND_TYPES_ENUM\n"; + OS << "#undef GET_INSTRINFO_OPERAND_TYPES_ENUM\n"; + OS << "namespace llvm {\n"; + OS << "namespace " << Namespace << " {\n"; + OS << "namespace OpTypes {\n"; + OS << "enum OperandType {\n"; + + unsigned EnumVal = 0; + for (const std::vector<Record *> *RecordsToAdd : + {&Operands, &RegisterOperands, &RegisterClasses}) { + for (const Record *Op : *RecordsToAdd) { + if (!Op->isAnonymous()) + OS << " " << Op->getName() << " = " << EnumVal << ",\n"; + ++EnumVal; + } + } + + OS << " OPERAND_TYPE_LIST_END" << "\n};\n"; + OS << "} // end namespace OpTypes\n"; + OS << "} // end namespace " << Namespace << "\n"; + OS << "} // end namespace llvm\n"; + OS << "#endif // GET_INSTRINFO_OPERAND_TYPES_ENUM\n\n"; + + OS << "#ifdef GET_INSTRINFO_OPERAND_TYPE\n"; + OS << "#undef GET_INSTRINFO_OPERAND_TYPE\n"; + OS << "namespace llvm {\n"; + OS << "namespace " << Namespace << " {\n"; + OS << "LLVM_READONLY\n"; + OS << "static int getOperandType(uint16_t Opcode, uint16_t OpIdx) {\n"; + // TODO: Factor out instructions with same operands to compress the tables. + if (!NumberedInstructions.empty()) { + std::vector<int> OperandOffsets; + std::vector<Record *> OperandRecords; + int CurrentOffset = 0; + for (const CodeGenInstruction *Inst : NumberedInstructions) { + OperandOffsets.push_back(CurrentOffset); + for (const auto &Op : Inst->Operands) { + const DagInit *MIOI = Op.MIOperandInfo; + if (!MIOI || MIOI->getNumArgs() == 0) { + // Single, anonymous, operand. + OperandRecords.push_back(Op.Rec); + ++CurrentOffset; + } else { + for (Init *Arg : make_range(MIOI->arg_begin(), MIOI->arg_end())) { + OperandRecords.push_back(cast<DefInit>(Arg)->getDef()); + ++CurrentOffset; + } + } + } + } + + // Emit the table of offsets for the opcode lookup. + OS << " const int Offsets[] = {\n"; + for (int I = 0, E = OperandOffsets.size(); I != E; ++I) + OS << " " << OperandOffsets[I] << ",\n"; + OS << " };\n"; + + // Add an entry for the end so that we don't need to special case it below. + OperandOffsets.push_back(OperandRecords.size()); + // Emit the actual operand types in a flat table. + OS << " const int OpcodeOperandTypes[] = {\n "; + for (int I = 0, E = OperandRecords.size(), CurOffset = 1; I != E; ++I) { + // We print each Opcode's operands in its own row. + if (I == OperandOffsets[CurOffset]) { + OS << "\n "; + // If there are empty rows, mark them with an empty comment. + while (OperandOffsets[++CurOffset] == I) + OS << "/**/\n "; + } + Record *OpR = OperandRecords[I]; + if ((OpR->isSubClassOf("Operand") || + OpR->isSubClassOf("RegisterOperand") || + OpR->isSubClassOf("RegisterClass")) && + !OpR->isAnonymous()) + OS << "OpTypes::" << OpR->getName(); + else + OS << -1; + OS << ", "; + } + OS << "\n };\n"; + + OS << " return OpcodeOperandTypes[Offsets[Opcode] + OpIdx];\n"; + } else { + OS << " llvm_unreachable(\"No instructions defined\");\n"; + } + OS << "}\n"; + OS << "} // end namespace " << Namespace << "\n"; + OS << "} // end namespace llvm\n"; + OS << "#endif // GET_INSTRINFO_OPERAND_TYPE\n\n"; +} + +void InstrInfoEmitter::emitMCIIHelperMethods(raw_ostream &OS, + StringRef TargetName) { + RecVec TIIPredicates = Records.getAllDerivedDefinitions("TIIPredicate"); + if (TIIPredicates.empty()) + return; + + OS << "#ifdef GET_INSTRINFO_MC_HELPER_DECLS\n"; + OS << "#undef GET_INSTRINFO_MC_HELPER_DECLS\n\n"; + + OS << "namespace llvm {\n"; + OS << "class MCInst;\n\n"; + + OS << "namespace " << TargetName << "_MC {\n\n"; + + for (const Record *Rec : TIIPredicates) { + OS << "bool " << Rec->getValueAsString("FunctionName") + << "(const MCInst &MI);\n"; + } + + OS << "\n} // end namespace " << TargetName << "_MC\n"; + OS << "} // end namespace llvm\n\n"; + + OS << "#endif // GET_INSTRINFO_MC_HELPER_DECLS\n\n"; + + OS << "#ifdef GET_INSTRINFO_MC_HELPERS\n"; + OS << "#undef GET_INSTRINFO_MC_HELPERS\n\n"; + + OS << "namespace llvm {\n"; + OS << "namespace " << TargetName << "_MC {\n\n"; + + PredicateExpander PE(TargetName); + PE.setExpandForMC(true); + + for (const Record *Rec : TIIPredicates) { + OS << "bool " << Rec->getValueAsString("FunctionName"); + OS << "(const MCInst &MI) {\n"; + + OS.indent(PE.getIndentLevel() * 2); + PE.expandStatement(OS, Rec->getValueAsDef("Body")); + OS << "\n}\n\n"; + } + + OS << "} // end namespace " << TargetName << "_MC\n"; + OS << "} // end namespace llvm\n\n"; + + OS << "#endif // GET_GENISTRINFO_MC_HELPERS\n"; +} + +void InstrInfoEmitter::emitTIIHelperMethods(raw_ostream &OS, + StringRef TargetName, + bool ExpandDefinition) { + RecVec TIIPredicates = Records.getAllDerivedDefinitions("TIIPredicate"); + if (TIIPredicates.empty()) + return; + + PredicateExpander PE(TargetName); + PE.setExpandForMC(false); + + for (const Record *Rec : TIIPredicates) { + OS << (ExpandDefinition ? "" : "static ") << "bool "; + if (ExpandDefinition) + OS << TargetName << "InstrInfo::"; + OS << Rec->getValueAsString("FunctionName"); + OS << "(const MachineInstr &MI)"; + if (!ExpandDefinition) { + OS << ";\n"; + continue; + } + + OS << " {\n"; + OS.indent(PE.getIndentLevel() * 2); + PE.expandStatement(OS, Rec->getValueAsDef("Body")); + OS << "\n}\n\n"; + } +} + +//===----------------------------------------------------------------------===// +// Main Output. +//===----------------------------------------------------------------------===// + +// run - Emit the main instruction description records for the target... +void InstrInfoEmitter::run(raw_ostream &OS) { + emitSourceFileHeader("Target Instruction Enum Values and Descriptors", OS); + emitEnums(OS); + + OS << "#ifdef GET_INSTRINFO_MC_DESC\n"; + OS << "#undef GET_INSTRINFO_MC_DESC\n"; + + OS << "namespace llvm {\n\n"; + + CodeGenTarget &Target = CDP.getTargetInfo(); + const std::string &TargetName = std::string(Target.getName()); + Record *InstrInfo = Target.getInstructionSet(); + + // Keep track of all of the def lists we have emitted already. + std::map<std::vector<Record*>, unsigned> EmittedLists; + unsigned ListNumber = 0; + + // Emit all of the instruction's implicit uses and defs. + for (const CodeGenInstruction *II : Target.getInstructionsByEnumValue()) { + Record *Inst = II->TheDef; + std::vector<Record*> Uses = Inst->getValueAsListOfDefs("Uses"); + if (!Uses.empty()) { + unsigned &IL = EmittedLists[Uses]; + if (!IL) PrintDefList(Uses, IL = ++ListNumber, OS); + } + std::vector<Record*> Defs = Inst->getValueAsListOfDefs("Defs"); + if (!Defs.empty()) { + unsigned &IL = EmittedLists[Defs]; + if (!IL) PrintDefList(Defs, IL = ++ListNumber, OS); + } + } + + OperandInfoMapTy OperandInfoIDs; + + // Emit all of the operand info records. + EmitOperandInfo(OS, OperandInfoIDs); + + // Emit all of the MCInstrDesc records in their ENUM ordering. + // + OS << "\nextern const MCInstrDesc " << TargetName << "Insts[] = {\n"; + ArrayRef<const CodeGenInstruction*> NumberedInstructions = + Target.getInstructionsByEnumValue(); + + SequenceToOffsetTable<std::string> InstrNames; + unsigned Num = 0; + for (const CodeGenInstruction *Inst : NumberedInstructions) { + // Keep a list of the instruction names. + InstrNames.add(std::string(Inst->TheDef->getName())); + // Emit the record into the table. + emitRecord(*Inst, Num++, InstrInfo, EmittedLists, OperandInfoIDs, OS); + } + OS << "};\n\n"; + + // Emit the array of instruction names. + InstrNames.layout(); + InstrNames.emitStringLiteralDef(OS, Twine("extern const char ") + TargetName + + "InstrNameData[]"); + + OS << "extern const unsigned " << TargetName <<"InstrNameIndices[] = {"; + Num = 0; + for (const CodeGenInstruction *Inst : NumberedInstructions) { + // Newline every eight entries. + if (Num % 8 == 0) + OS << "\n "; + OS << InstrNames.get(std::string(Inst->TheDef->getName())) << "U, "; + ++Num; + } + OS << "\n};\n\n"; + + bool HasDeprecationFeatures = + llvm::any_of(NumberedInstructions, [](const CodeGenInstruction *Inst) { + return !Inst->HasComplexDeprecationPredicate && + !Inst->DeprecatedReason.empty(); + }); + if (HasDeprecationFeatures) { + OS << "extern const uint8_t " << TargetName + << "InstrDeprecationFeatures[] = {"; + Num = 0; + for (const CodeGenInstruction *Inst : NumberedInstructions) { + if (Num % 8 == 0) + OS << "\n "; + if (!Inst->HasComplexDeprecationPredicate && + !Inst->DeprecatedReason.empty()) + OS << Target.getInstNamespace() << "::" << Inst->DeprecatedReason + << ", "; + else + OS << "uint8_t(-1), "; + ++Num; + } + OS << "\n};\n\n"; + } + + bool HasComplexDeprecationInfos = + llvm::any_of(NumberedInstructions, [](const CodeGenInstruction *Inst) { + return Inst->HasComplexDeprecationPredicate; + }); + if (HasComplexDeprecationInfos) { + OS << "extern const MCInstrInfo::ComplexDeprecationPredicate " << TargetName + << "InstrComplexDeprecationInfos[] = {"; + Num = 0; + for (const CodeGenInstruction *Inst : NumberedInstructions) { + if (Num % 8 == 0) + OS << "\n "; + if (Inst->HasComplexDeprecationPredicate) + // Emit a function pointer to the complex predicate method. + OS << "&get" << Inst->DeprecatedReason << "DeprecationInfo, "; + else + OS << "nullptr, "; + ++Num; + } + OS << "\n};\n\n"; + } + + // MCInstrInfo initialization routine. + OS << "static inline void Init" << TargetName + << "MCInstrInfo(MCInstrInfo *II) {\n"; + OS << " II->InitMCInstrInfo(" << TargetName << "Insts, " << TargetName + << "InstrNameIndices, " << TargetName << "InstrNameData, "; + if (HasDeprecationFeatures) + OS << TargetName << "InstrDeprecationFeatures, "; + else + OS << "nullptr, "; + if (HasComplexDeprecationInfos) + OS << TargetName << "InstrComplexDeprecationInfos, "; + else + OS << "nullptr, "; + OS << NumberedInstructions.size() << ");\n}\n\n"; + + OS << "} // end namespace llvm\n"; + + OS << "#endif // GET_INSTRINFO_MC_DESC\n\n"; + + // Create a TargetInstrInfo subclass to hide the MC layer initialization. + OS << "#ifdef GET_INSTRINFO_HEADER\n"; + OS << "#undef GET_INSTRINFO_HEADER\n"; + + std::string ClassName = TargetName + "GenInstrInfo"; + OS << "namespace llvm {\n"; + OS << "struct " << ClassName << " : public TargetInstrInfo {\n" + << " explicit " << ClassName + << "(int CFSetupOpcode = -1, int CFDestroyOpcode = -1, int CatchRetOpcode = -1, int ReturnOpcode = -1);\n" + << " ~" << ClassName << "() override = default;\n"; + + + OS << "\n};\n} // end namespace llvm\n"; + + OS << "#endif // GET_INSTRINFO_HEADER\n\n"; + + OS << "#ifdef GET_INSTRINFO_HELPER_DECLS\n"; + OS << "#undef GET_INSTRINFO_HELPER_DECLS\n\n"; + emitTIIHelperMethods(OS, TargetName, /* ExpandDefintion = */false); + OS << "\n"; + OS << "#endif // GET_INSTRINFO_HELPER_DECLS\n\n"; + + OS << "#ifdef GET_INSTRINFO_HELPERS\n"; + OS << "#undef GET_INSTRINFO_HELPERS\n\n"; + emitTIIHelperMethods(OS, TargetName, /* ExpandDefintion = */true); + OS << "#endif // GET_INSTRINFO_HELPERS\n\n"; + + OS << "#ifdef GET_INSTRINFO_CTOR_DTOR\n"; + OS << "#undef GET_INSTRINFO_CTOR_DTOR\n"; + + OS << "namespace llvm {\n"; + OS << "extern const MCInstrDesc " << TargetName << "Insts[];\n"; + OS << "extern const unsigned " << TargetName << "InstrNameIndices[];\n"; + OS << "extern const char " << TargetName << "InstrNameData[];\n"; + if (HasDeprecationFeatures) + OS << "extern const uint8_t " << TargetName + << "InstrDeprecationFeatures[];\n"; + if (HasComplexDeprecationInfos) + OS << "extern const MCInstrInfo::ComplexDeprecationPredicate " << TargetName + << "InstrComplexDeprecationInfos[];\n"; + OS << ClassName << "::" << ClassName + << "(int CFSetupOpcode, int CFDestroyOpcode, int CatchRetOpcode, int " + "ReturnOpcode)\n" + << " : TargetInstrInfo(CFSetupOpcode, CFDestroyOpcode, CatchRetOpcode, " + "ReturnOpcode) {\n" + << " InitMCInstrInfo(" << TargetName << "Insts, " << TargetName + << "InstrNameIndices, " << TargetName << "InstrNameData, "; + if (HasDeprecationFeatures) + OS << TargetName << "InstrDeprecationFeatures, "; + else + OS << "nullptr, "; + if (HasComplexDeprecationInfos) + OS << TargetName << "InstrComplexDeprecationInfos, "; + else + OS << "nullptr, "; + OS << NumberedInstructions.size() << ");\n}\n"; + OS << "} // end namespace llvm\n"; + + OS << "#endif // GET_INSTRINFO_CTOR_DTOR\n\n"; + + emitOperandNameMappings(OS, Target, NumberedInstructions); + + emitOperandTypeMappings(OS, Target, NumberedInstructions); + + emitMCIIHelperMethods(OS, TargetName); +} + +void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num, + Record *InstrInfo, + std::map<std::vector<Record*>, unsigned> &EmittedLists, + const OperandInfoMapTy &OpInfo, + raw_ostream &OS) { + int MinOperands = 0; + if (!Inst.Operands.empty()) + // Each logical operand can be multiple MI operands. + MinOperands = Inst.Operands.back().MIOperandNo + + Inst.Operands.back().MINumOperands; + + OS << " { "; + OS << Num << ",\t" << MinOperands << ",\t" + << Inst.Operands.NumDefs << ",\t" + << Inst.TheDef->getValueAsInt("Size") << ",\t" + << SchedModels.getSchedClassIdx(Inst) << ",\t0"; + + CodeGenTarget &Target = CDP.getTargetInfo(); + + // Emit all of the target independent flags... + if (Inst.isPreISelOpcode) OS << "|(1ULL<<MCID::PreISelOpcode)"; + if (Inst.isPseudo) OS << "|(1ULL<<MCID::Pseudo)"; + if (Inst.isReturn) OS << "|(1ULL<<MCID::Return)"; + if (Inst.isEHScopeReturn) OS << "|(1ULL<<MCID::EHScopeReturn)"; + if (Inst.isBranch) OS << "|(1ULL<<MCID::Branch)"; + if (Inst.isIndirectBranch) OS << "|(1ULL<<MCID::IndirectBranch)"; + if (Inst.isCompare) OS << "|(1ULL<<MCID::Compare)"; + if (Inst.isMoveImm) OS << "|(1ULL<<MCID::MoveImm)"; + if (Inst.isMoveReg) OS << "|(1ULL<<MCID::MoveReg)"; + if (Inst.isBitcast) OS << "|(1ULL<<MCID::Bitcast)"; + if (Inst.isAdd) OS << "|(1ULL<<MCID::Add)"; + if (Inst.isTrap) OS << "|(1ULL<<MCID::Trap)"; + if (Inst.isSelect) OS << "|(1ULL<<MCID::Select)"; + if (Inst.isBarrier) OS << "|(1ULL<<MCID::Barrier)"; + if (Inst.hasDelaySlot) OS << "|(1ULL<<MCID::DelaySlot)"; + if (Inst.isCall) OS << "|(1ULL<<MCID::Call)"; + if (Inst.canFoldAsLoad) OS << "|(1ULL<<MCID::FoldableAsLoad)"; + if (Inst.mayLoad) OS << "|(1ULL<<MCID::MayLoad)"; + if (Inst.mayStore) OS << "|(1ULL<<MCID::MayStore)"; + if (Inst.mayRaiseFPException) OS << "|(1ULL<<MCID::MayRaiseFPException)"; + if (Inst.isPredicable) OS << "|(1ULL<<MCID::Predicable)"; + if (Inst.isConvertibleToThreeAddress) OS << "|(1ULL<<MCID::ConvertibleTo3Addr)"; + if (Inst.isCommutable) OS << "|(1ULL<<MCID::Commutable)"; + if (Inst.isTerminator) OS << "|(1ULL<<MCID::Terminator)"; + if (Inst.isReMaterializable) OS << "|(1ULL<<MCID::Rematerializable)"; + if (Inst.isNotDuplicable) OS << "|(1ULL<<MCID::NotDuplicable)"; + if (Inst.Operands.hasOptionalDef) OS << "|(1ULL<<MCID::HasOptionalDef)"; + if (Inst.usesCustomInserter) OS << "|(1ULL<<MCID::UsesCustomInserter)"; + if (Inst.hasPostISelHook) OS << "|(1ULL<<MCID::HasPostISelHook)"; + if (Inst.Operands.isVariadic)OS << "|(1ULL<<MCID::Variadic)"; + if (Inst.hasSideEffects) OS << "|(1ULL<<MCID::UnmodeledSideEffects)"; + if (Inst.isAsCheapAsAMove) OS << "|(1ULL<<MCID::CheapAsAMove)"; + if (!Target.getAllowRegisterRenaming() || Inst.hasExtraSrcRegAllocReq) + OS << "|(1ULL<<MCID::ExtraSrcRegAllocReq)"; + if (!Target.getAllowRegisterRenaming() || Inst.hasExtraDefRegAllocReq) + OS << "|(1ULL<<MCID::ExtraDefRegAllocReq)"; + if (Inst.isRegSequence) OS << "|(1ULL<<MCID::RegSequence)"; + if (Inst.isExtractSubreg) OS << "|(1ULL<<MCID::ExtractSubreg)"; + if (Inst.isInsertSubreg) OS << "|(1ULL<<MCID::InsertSubreg)"; + if (Inst.isConvergent) OS << "|(1ULL<<MCID::Convergent)"; + if (Inst.variadicOpsAreDefs) OS << "|(1ULL<<MCID::VariadicOpsAreDefs)"; + if (Inst.isAuthenticated) OS << "|(1ULL<<MCID::Authenticated)"; + + // Emit all of the target-specific flags... + BitsInit *TSF = Inst.TheDef->getValueAsBitsInit("TSFlags"); + if (!TSF) + PrintFatalError(Inst.TheDef->getLoc(), "no TSFlags?"); + uint64_t Value = 0; + for (unsigned i = 0, e = TSF->getNumBits(); i != e; ++i) { + if (const auto *Bit = dyn_cast<BitInit>(TSF->getBit(i))) + Value |= uint64_t(Bit->getValue()) << i; + else + PrintFatalError(Inst.TheDef->getLoc(), + "Invalid TSFlags bit in " + Inst.TheDef->getName()); + } + OS << ", 0x"; + OS.write_hex(Value); + OS << "ULL, "; + + // Emit the implicit uses and defs lists... + std::vector<Record*> UseList = Inst.TheDef->getValueAsListOfDefs("Uses"); + if (UseList.empty()) + OS << "nullptr, "; + else + OS << "ImplicitList" << EmittedLists[UseList] << ", "; + + std::vector<Record*> DefList = Inst.TheDef->getValueAsListOfDefs("Defs"); + if (DefList.empty()) + OS << "nullptr, "; + else + OS << "ImplicitList" << EmittedLists[DefList] << ", "; + + // Emit the operand info. + std::vector<std::string> OperandInfo = GetOperandInfo(Inst); + if (OperandInfo.empty()) + OS << "nullptr"; + else + OS << "OperandInfo" << OpInfo.find(OperandInfo)->second; + + OS << " }, // Inst #" << Num << " = " << Inst.TheDef->getName() << "\n"; +} + +// emitEnums - Print out enum values for all of the instructions. +void InstrInfoEmitter::emitEnums(raw_ostream &OS) { + OS << "#ifdef GET_INSTRINFO_ENUM\n"; + OS << "#undef GET_INSTRINFO_ENUM\n"; + + OS << "namespace llvm {\n\n"; + + const CodeGenTarget &Target = CDP.getTargetInfo(); + + // We must emit the PHI opcode first... + StringRef Namespace = Target.getInstNamespace(); + + if (Namespace.empty()) + PrintFatalError("No instructions defined!"); + + OS << "namespace " << Namespace << " {\n"; + OS << " enum {\n"; + unsigned Num = 0; + for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) + OS << " " << Inst->TheDef->getName() << "\t= " << Num++ << ",\n"; + OS << " INSTRUCTION_LIST_END = " << Num << "\n"; + OS << " };\n\n"; + OS << "} // end namespace " << Namespace << "\n"; + OS << "} // end namespace llvm\n"; + OS << "#endif // GET_INSTRINFO_ENUM\n\n"; + + OS << "#ifdef GET_INSTRINFO_SCHED_ENUM\n"; + OS << "#undef GET_INSTRINFO_SCHED_ENUM\n"; + OS << "namespace llvm {\n\n"; + OS << "namespace " << Namespace << " {\n"; + OS << "namespace Sched {\n"; + OS << " enum {\n"; + Num = 0; + for (const auto &Class : SchedModels.explicit_classes()) + OS << " " << Class.Name << "\t= " << Num++ << ",\n"; + OS << " SCHED_LIST_END = " << Num << "\n"; + OS << " };\n"; + OS << "} // end namespace Sched\n"; + OS << "} // end namespace " << Namespace << "\n"; + OS << "} // end namespace llvm\n"; + + OS << "#endif // GET_INSTRINFO_SCHED_ENUM\n\n"; +} + +namespace llvm { + +void EmitInstrInfo(RecordKeeper &RK, raw_ostream &OS) { + InstrInfoEmitter(RK).run(OS); + EmitMapTable(RK, OS); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp new file mode 100644 index 000000000000..7e4191494149 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -0,0 +1,1003 @@ +//===- IntrinsicEmitter.cpp - Generate intrinsic information --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend emits information about intrinsic functions. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenIntrinsics.h" +#include "CodeGenTarget.h" +#include "SequenceToOffsetTable.h" +#include "TableGenBackends.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/StringMatcher.h" +#include "llvm/TableGen/StringToOffsetTable.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <algorithm> +using namespace llvm; + +cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums"); +cl::opt<std::string> + IntrinsicPrefix("intrinsic-prefix", + cl::desc("Generate intrinsics with this target prefix"), + cl::value_desc("target prefix"), cl::cat(GenIntrinsicCat)); + +namespace { +class IntrinsicEmitter { + RecordKeeper &Records; + +public: + IntrinsicEmitter(RecordKeeper &R) : Records(R) {} + + void run(raw_ostream &OS, bool Enums); + + void EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); + void EmitTargetInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); + void EmitIntrinsicToNameTable(const CodeGenIntrinsicTable &Ints, + raw_ostream &OS); + void EmitIntrinsicToOverloadTable(const CodeGenIntrinsicTable &Ints, + raw_ostream &OS); + void EmitGenerator(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); + void EmitAttributes(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); + void EmitIntrinsicToBuiltinMap(const CodeGenIntrinsicTable &Ints, bool IsGCC, + raw_ostream &OS); +}; +} // End anonymous namespace + +//===----------------------------------------------------------------------===// +// IntrinsicEmitter Implementation +//===----------------------------------------------------------------------===// + +void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) { + emitSourceFileHeader("Intrinsic Function Source Fragment", OS); + + CodeGenIntrinsicTable Ints(Records); + + if (Enums) { + // Emit the enum information. + EmitEnumInfo(Ints, OS); + } else { + // Emit the target metadata. + EmitTargetInfo(Ints, OS); + + // Emit the intrinsic ID -> name table. + EmitIntrinsicToNameTable(Ints, OS); + + // Emit the intrinsic ID -> overload table. + EmitIntrinsicToOverloadTable(Ints, OS); + + // Emit the intrinsic declaration generator. + EmitGenerator(Ints, OS); + + // Emit the intrinsic parameter attributes. + EmitAttributes(Ints, OS); + + // Emit code to translate GCC builtins into LLVM intrinsics. + EmitIntrinsicToBuiltinMap(Ints, true, OS); + + // Emit code to translate MS builtins into LLVM intrinsics. + EmitIntrinsicToBuiltinMap(Ints, false, OS); + } +} + +void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints, + raw_ostream &OS) { + // Find the TargetSet for which to generate enums. There will be an initial + // set with an empty target prefix which will include target independent + // intrinsics like dbg.value. + const CodeGenIntrinsicTable::TargetSet *Set = nullptr; + for (const auto &Target : Ints.Targets) { + if (Target.Name == IntrinsicPrefix) { + Set = &Target; + break; + } + } + if (!Set) { + std::vector<std::string> KnownTargets; + for (const auto &Target : Ints.Targets) + if (!Target.Name.empty()) + KnownTargets.push_back(Target.Name); + PrintFatalError("tried to generate intrinsics for unknown target " + + IntrinsicPrefix + + "\nKnown targets are: " + join(KnownTargets, ", ") + "\n"); + } + + // Generate a complete header for target specific intrinsics. + if (!IntrinsicPrefix.empty()) { + std::string UpperPrefix = StringRef(IntrinsicPrefix).upper(); + OS << "#ifndef LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n"; + OS << "#define LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n\n"; + OS << "namespace llvm {\n"; + OS << "namespace Intrinsic {\n"; + OS << "enum " << UpperPrefix << "Intrinsics : unsigned {\n"; + } + + OS << "// Enum values for intrinsics\n"; + for (unsigned i = Set->Offset, e = Set->Offset + Set->Count; i != e; ++i) { + OS << " " << Ints[i].EnumName; + + // Assign a value to the first intrinsic in this target set so that all + // intrinsic ids are distinct. + if (i == Set->Offset) + OS << " = " << (Set->Offset + 1); + + OS << ", "; + if (Ints[i].EnumName.size() < 40) + OS.indent(40 - Ints[i].EnumName.size()); + OS << " // " << Ints[i].Name << "\n"; + } + + // Emit num_intrinsics into the target neutral enum. + if (IntrinsicPrefix.empty()) { + OS << " num_intrinsics = " << (Ints.size() + 1) << "\n"; + } else { + OS << "}; // enum\n"; + OS << "} // namespace Intrinsic\n"; + OS << "} // namespace llvm\n\n"; + OS << "#endif\n"; + } +} + +void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints, + raw_ostream &OS) { + OS << "// Target mapping\n"; + OS << "#ifdef GET_INTRINSIC_TARGET_DATA\n"; + OS << "struct IntrinsicTargetInfo {\n" + << " llvm::StringLiteral Name;\n" + << " size_t Offset;\n" + << " size_t Count;\n" + << "};\n"; + OS << "static constexpr IntrinsicTargetInfo TargetInfos[] = {\n"; + for (auto Target : Ints.Targets) + OS << " {llvm::StringLiteral(\"" << Target.Name << "\"), " << Target.Offset + << ", " << Target.Count << "},\n"; + OS << "};\n"; + OS << "#endif\n\n"; +} + +void IntrinsicEmitter::EmitIntrinsicToNameTable( + const CodeGenIntrinsicTable &Ints, raw_ostream &OS) { + OS << "// Intrinsic ID to name table\n"; + OS << "#ifdef GET_INTRINSIC_NAME_TABLE\n"; + OS << " // Note that entry #0 is the invalid intrinsic!\n"; + for (unsigned i = 0, e = Ints.size(); i != e; ++i) + OS << " \"" << Ints[i].Name << "\",\n"; + OS << "#endif\n\n"; +} + +void IntrinsicEmitter::EmitIntrinsicToOverloadTable( + const CodeGenIntrinsicTable &Ints, raw_ostream &OS) { + OS << "// Intrinsic ID to overload bitset\n"; + OS << "#ifdef GET_INTRINSIC_OVERLOAD_TABLE\n"; + OS << "static const uint8_t OTable[] = {\n"; + OS << " 0"; + for (unsigned i = 0, e = Ints.size(); i != e; ++i) { + // Add one to the index so we emit a null bit for the invalid #0 intrinsic. + if ((i+1)%8 == 0) + OS << ",\n 0"; + if (Ints[i].isOverloaded) + OS << " | (1<<" << (i+1)%8 << ')'; + } + OS << "\n};\n\n"; + // OTable contains a true bit at the position if the intrinsic is overloaded. + OS << "return (OTable[id/8] & (1 << (id%8))) != 0;\n"; + OS << "#endif\n\n"; +} + + +// NOTE: This must be kept in synch with the copy in lib/IR/Function.cpp! +enum IIT_Info { + // Common values should be encoded with 0-15. + IIT_Done = 0, + IIT_I1 = 1, + IIT_I8 = 2, + IIT_I16 = 3, + IIT_I32 = 4, + IIT_I64 = 5, + IIT_F16 = 6, + IIT_F32 = 7, + IIT_F64 = 8, + IIT_V2 = 9, + IIT_V4 = 10, + IIT_V8 = 11, + IIT_V16 = 12, + IIT_V32 = 13, + IIT_PTR = 14, + IIT_ARG = 15, + + // Values from 16+ are only encodable with the inefficient encoding. + IIT_V64 = 16, + IIT_MMX = 17, + IIT_TOKEN = 18, + IIT_METADATA = 19, + IIT_EMPTYSTRUCT = 20, + IIT_STRUCT2 = 21, + IIT_STRUCT3 = 22, + IIT_STRUCT4 = 23, + IIT_STRUCT5 = 24, + IIT_EXTEND_ARG = 25, + IIT_TRUNC_ARG = 26, + IIT_ANYPTR = 27, + IIT_V1 = 28, + IIT_VARARG = 29, + IIT_HALF_VEC_ARG = 30, + IIT_SAME_VEC_WIDTH_ARG = 31, + IIT_PTR_TO_ARG = 32, + IIT_PTR_TO_ELT = 33, + IIT_VEC_OF_ANYPTRS_TO_ELT = 34, + IIT_I128 = 35, + IIT_V512 = 36, + IIT_V1024 = 37, + IIT_STRUCT6 = 38, + IIT_STRUCT7 = 39, + IIT_STRUCT8 = 40, + IIT_F128 = 41, + IIT_VEC_ELEMENT = 42, + IIT_SCALABLE_VEC = 43, + IIT_SUBDIVIDE2_ARG = 44, + IIT_SUBDIVIDE4_ARG = 45, + IIT_VEC_OF_BITCASTS_TO_INT = 46, + IIT_V128 = 47, + IIT_BF16 = 48 +}; + +static void EncodeFixedValueType(MVT::SimpleValueType VT, + std::vector<unsigned char> &Sig) { + if (MVT(VT).isInteger()) { + unsigned BitWidth = MVT(VT).getSizeInBits(); + switch (BitWidth) { + default: PrintFatalError("unhandled integer type width in intrinsic!"); + case 1: return Sig.push_back(IIT_I1); + case 8: return Sig.push_back(IIT_I8); + case 16: return Sig.push_back(IIT_I16); + case 32: return Sig.push_back(IIT_I32); + case 64: return Sig.push_back(IIT_I64); + case 128: return Sig.push_back(IIT_I128); + } + } + + switch (VT) { + default: PrintFatalError("unhandled MVT in intrinsic!"); + case MVT::f16: return Sig.push_back(IIT_F16); + case MVT::bf16: return Sig.push_back(IIT_BF16); + case MVT::f32: return Sig.push_back(IIT_F32); + case MVT::f64: return Sig.push_back(IIT_F64); + case MVT::f128: return Sig.push_back(IIT_F128); + case MVT::token: return Sig.push_back(IIT_TOKEN); + case MVT::Metadata: return Sig.push_back(IIT_METADATA); + case MVT::x86mmx: return Sig.push_back(IIT_MMX); + // MVT::OtherVT is used to mean the empty struct type here. + case MVT::Other: return Sig.push_back(IIT_EMPTYSTRUCT); + // MVT::isVoid is used to represent varargs here. + case MVT::isVoid: return Sig.push_back(IIT_VARARG); + } +} + +#if defined(_MSC_VER) && !defined(__clang__) +#pragma optimize("",off) // MSVC 2015 optimizer can't deal with this function. +#endif + +static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes, + unsigned &NextArgCode, + std::vector<unsigned char> &Sig, + ArrayRef<unsigned char> Mapping) { + + if (R->isSubClassOf("LLVMMatchType")) { + unsigned Number = Mapping[R->getValueAsInt("Number")]; + assert(Number < ArgCodes.size() && "Invalid matching number!"); + if (R->isSubClassOf("LLVMExtendedType")) + Sig.push_back(IIT_EXTEND_ARG); + else if (R->isSubClassOf("LLVMTruncatedType")) + Sig.push_back(IIT_TRUNC_ARG); + else if (R->isSubClassOf("LLVMHalfElementsVectorType")) + Sig.push_back(IIT_HALF_VEC_ARG); + else if (R->isSubClassOf("LLVMScalarOrSameVectorWidth")) { + Sig.push_back(IIT_SAME_VEC_WIDTH_ARG); + Sig.push_back((Number << 3) | ArgCodes[Number]); + MVT::SimpleValueType VT = getValueType(R->getValueAsDef("ElTy")); + EncodeFixedValueType(VT, Sig); + return; + } + else if (R->isSubClassOf("LLVMPointerTo")) + Sig.push_back(IIT_PTR_TO_ARG); + else if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) { + Sig.push_back(IIT_VEC_OF_ANYPTRS_TO_ELT); + // Encode overloaded ArgNo + Sig.push_back(NextArgCode++); + // Encode LLVMMatchType<Number> ArgNo + Sig.push_back(Number); + return; + } else if (R->isSubClassOf("LLVMPointerToElt")) + Sig.push_back(IIT_PTR_TO_ELT); + else if (R->isSubClassOf("LLVMVectorElementType")) + Sig.push_back(IIT_VEC_ELEMENT); + else if (R->isSubClassOf("LLVMSubdivide2VectorType")) + Sig.push_back(IIT_SUBDIVIDE2_ARG); + else if (R->isSubClassOf("LLVMSubdivide4VectorType")) + Sig.push_back(IIT_SUBDIVIDE4_ARG); + else if (R->isSubClassOf("LLVMVectorOfBitcastsToInt")) + Sig.push_back(IIT_VEC_OF_BITCASTS_TO_INT); + else + Sig.push_back(IIT_ARG); + return Sig.push_back((Number << 3) | 7 /*IITDescriptor::AK_MatchType*/); + } + + MVT::SimpleValueType VT = getValueType(R->getValueAsDef("VT")); + + unsigned Tmp = 0; + switch (VT) { + default: break; + case MVT::iPTRAny: ++Tmp; LLVM_FALLTHROUGH; + case MVT::vAny: ++Tmp; LLVM_FALLTHROUGH; + case MVT::fAny: ++Tmp; LLVM_FALLTHROUGH; + case MVT::iAny: ++Tmp; LLVM_FALLTHROUGH; + case MVT::Any: { + // If this is an "any" valuetype, then the type is the type of the next + // type in the list specified to getIntrinsic(). + Sig.push_back(IIT_ARG); + + // Figure out what arg # this is consuming, and remember what kind it was. + assert(NextArgCode < ArgCodes.size() && ArgCodes[NextArgCode] == Tmp && + "Invalid or no ArgCode associated with overloaded VT!"); + unsigned ArgNo = NextArgCode++; + + // Encode what sort of argument it must be in the low 3 bits of the ArgNo. + return Sig.push_back((ArgNo << 3) | Tmp); + } + + case MVT::iPTR: { + unsigned AddrSpace = 0; + if (R->isSubClassOf("LLVMQualPointerType")) { + AddrSpace = R->getValueAsInt("AddrSpace"); + assert(AddrSpace < 256 && "Address space exceeds 255"); + } + if (AddrSpace) { + Sig.push_back(IIT_ANYPTR); + Sig.push_back(AddrSpace); + } else { + Sig.push_back(IIT_PTR); + } + return EncodeFixedType(R->getValueAsDef("ElTy"), ArgCodes, NextArgCode, Sig, + Mapping); + } + } + + if (MVT(VT).isVector()) { + MVT VVT = VT; + if (VVT.isScalableVector()) + Sig.push_back(IIT_SCALABLE_VEC); + switch (VVT.getVectorNumElements()) { + default: PrintFatalError("unhandled vector type width in intrinsic!"); + case 1: Sig.push_back(IIT_V1); break; + case 2: Sig.push_back(IIT_V2); break; + case 4: Sig.push_back(IIT_V4); break; + case 8: Sig.push_back(IIT_V8); break; + case 16: Sig.push_back(IIT_V16); break; + case 32: Sig.push_back(IIT_V32); break; + case 64: Sig.push_back(IIT_V64); break; + case 128: Sig.push_back(IIT_V128); break; + case 512: Sig.push_back(IIT_V512); break; + case 1024: Sig.push_back(IIT_V1024); break; + } + + return EncodeFixedValueType(VVT.getVectorElementType().SimpleTy, Sig); + } + + EncodeFixedValueType(VT, Sig); +} + +static void UpdateArgCodes(Record *R, std::vector<unsigned char> &ArgCodes, + unsigned int &NumInserted, + SmallVectorImpl<unsigned char> &Mapping) { + if (R->isSubClassOf("LLVMMatchType")) { + if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) { + ArgCodes.push_back(3 /*vAny*/); + ++NumInserted; + } + return; + } + + unsigned Tmp = 0; + switch (getValueType(R->getValueAsDef("VT"))) { + default: break; + case MVT::iPTR: + UpdateArgCodes(R->getValueAsDef("ElTy"), ArgCodes, NumInserted, Mapping); + break; + case MVT::iPTRAny: + ++Tmp; + LLVM_FALLTHROUGH; + case MVT::vAny: + ++Tmp; + LLVM_FALLTHROUGH; + case MVT::fAny: + ++Tmp; + LLVM_FALLTHROUGH; + case MVT::iAny: + ++Tmp; + LLVM_FALLTHROUGH; + case MVT::Any: + unsigned OriginalIdx = ArgCodes.size() - NumInserted; + assert(OriginalIdx >= Mapping.size()); + Mapping.resize(OriginalIdx+1); + Mapping[OriginalIdx] = ArgCodes.size(); + ArgCodes.push_back(Tmp); + break; + } +} + +#if defined(_MSC_VER) && !defined(__clang__) +#pragma optimize("",on) +#endif + +/// ComputeFixedEncoding - If we can encode the type signature for this +/// intrinsic into 32 bits, return it. If not, return ~0U. +static void ComputeFixedEncoding(const CodeGenIntrinsic &Int, + std::vector<unsigned char> &TypeSig) { + std::vector<unsigned char> ArgCodes; + + // Add codes for any overloaded result VTs. + unsigned int NumInserted = 0; + SmallVector<unsigned char, 8> ArgMapping; + for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i) + UpdateArgCodes(Int.IS.RetTypeDefs[i], ArgCodes, NumInserted, ArgMapping); + + // Add codes for any overloaded operand VTs. + for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i) + UpdateArgCodes(Int.IS.ParamTypeDefs[i], ArgCodes, NumInserted, ArgMapping); + + unsigned NextArgCode = 0; + if (Int.IS.RetVTs.empty()) + TypeSig.push_back(IIT_Done); + else if (Int.IS.RetVTs.size() == 1 && + Int.IS.RetVTs[0] == MVT::isVoid) + TypeSig.push_back(IIT_Done); + else { + switch (Int.IS.RetVTs.size()) { + case 1: break; + case 2: TypeSig.push_back(IIT_STRUCT2); break; + case 3: TypeSig.push_back(IIT_STRUCT3); break; + case 4: TypeSig.push_back(IIT_STRUCT4); break; + case 5: TypeSig.push_back(IIT_STRUCT5); break; + case 6: TypeSig.push_back(IIT_STRUCT6); break; + case 7: TypeSig.push_back(IIT_STRUCT7); break; + case 8: TypeSig.push_back(IIT_STRUCT8); break; + default: llvm_unreachable("Unhandled case in struct"); + } + + for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i) + EncodeFixedType(Int.IS.RetTypeDefs[i], ArgCodes, NextArgCode, TypeSig, + ArgMapping); + } + + for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i) + EncodeFixedType(Int.IS.ParamTypeDefs[i], ArgCodes, NextArgCode, TypeSig, + ArgMapping); +} + +static void printIITEntry(raw_ostream &OS, unsigned char X) { + OS << (unsigned)X; +} + +void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints, + raw_ostream &OS) { + // If we can compute a 32-bit fixed encoding for this intrinsic, do so and + // capture it in this vector, otherwise store a ~0U. + std::vector<unsigned> FixedEncodings; + + SequenceToOffsetTable<std::vector<unsigned char> > LongEncodingTable; + + std::vector<unsigned char> TypeSig; + + // Compute the unique argument type info. + for (unsigned i = 0, e = Ints.size(); i != e; ++i) { + // Get the signature for the intrinsic. + TypeSig.clear(); + ComputeFixedEncoding(Ints[i], TypeSig); + + // Check to see if we can encode it into a 32-bit word. We can only encode + // 8 nibbles into a 32-bit word. + if (TypeSig.size() <= 8) { + bool Failed = false; + unsigned Result = 0; + for (unsigned i = 0, e = TypeSig.size(); i != e; ++i) { + // If we had an unencodable argument, bail out. + if (TypeSig[i] > 15) { + Failed = true; + break; + } + Result = (Result << 4) | TypeSig[e-i-1]; + } + + // If this could be encoded into a 31-bit word, return it. + if (!Failed && (Result >> 31) == 0) { + FixedEncodings.push_back(Result); + continue; + } + } + + // Otherwise, we're going to unique the sequence into the + // LongEncodingTable, and use its offset in the 32-bit table instead. + LongEncodingTable.add(TypeSig); + + // This is a placehold that we'll replace after the table is laid out. + FixedEncodings.push_back(~0U); + } + + LongEncodingTable.layout(); + + OS << "// Global intrinsic function declaration type table.\n"; + OS << "#ifdef GET_INTRINSIC_GENERATOR_GLOBAL\n"; + + OS << "static const unsigned IIT_Table[] = {\n "; + + for (unsigned i = 0, e = FixedEncodings.size(); i != e; ++i) { + if ((i & 7) == 7) + OS << "\n "; + + // If the entry fit in the table, just emit it. + if (FixedEncodings[i] != ~0U) { + OS << "0x" << Twine::utohexstr(FixedEncodings[i]) << ", "; + continue; + } + + TypeSig.clear(); + ComputeFixedEncoding(Ints[i], TypeSig); + + + // Otherwise, emit the offset into the long encoding table. We emit it this + // way so that it is easier to read the offset in the .def file. + OS << "(1U<<31) | " << LongEncodingTable.get(TypeSig) << ", "; + } + + OS << "0\n};\n\n"; + + // Emit the shared table of register lists. + OS << "static const unsigned char IIT_LongEncodingTable[] = {\n"; + if (!LongEncodingTable.empty()) + LongEncodingTable.emit(OS, printIITEntry); + OS << " 255\n};\n\n"; + + OS << "#endif\n\n"; // End of GET_INTRINSIC_GENERATOR_GLOBAL +} + +namespace { +struct AttributeComparator { + bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const { + // Sort throwing intrinsics after non-throwing intrinsics. + if (L->canThrow != R->canThrow) + return R->canThrow; + + if (L->isNoDuplicate != R->isNoDuplicate) + return R->isNoDuplicate; + + if (L->isNoReturn != R->isNoReturn) + return R->isNoReturn; + + if (L->isNoSync != R->isNoSync) + return R->isNoSync; + + if (L->isNoFree != R->isNoFree) + return R->isNoFree; + + if (L->isWillReturn != R->isWillReturn) + return R->isWillReturn; + + if (L->isCold != R->isCold) + return R->isCold; + + if (L->isConvergent != R->isConvergent) + return R->isConvergent; + + if (L->isSpeculatable != R->isSpeculatable) + return R->isSpeculatable; + + if (L->hasSideEffects != R->hasSideEffects) + return R->hasSideEffects; + + // Try to order by readonly/readnone attribute. + CodeGenIntrinsic::ModRefBehavior LK = L->ModRef; + CodeGenIntrinsic::ModRefBehavior RK = R->ModRef; + if (LK != RK) return (LK > RK); + // Order by argument attributes. + // This is reliable because each side is already sorted internally. + return (L->ArgumentAttributes < R->ArgumentAttributes); + } +}; +} // End anonymous namespace + +/// EmitAttributes - This emits the Intrinsic::getAttributes method. +void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, + raw_ostream &OS) { + OS << "// Add parameter attributes that are not common to all intrinsics.\n"; + OS << "#ifdef GET_INTRINSIC_ATTRIBUTES\n"; + OS << "AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {\n"; + + // Compute the maximum number of attribute arguments and the map + typedef std::map<const CodeGenIntrinsic*, unsigned, + AttributeComparator> UniqAttrMapTy; + UniqAttrMapTy UniqAttributes; + unsigned maxArgAttrs = 0; + unsigned AttrNum = 0; + for (unsigned i = 0, e = Ints.size(); i != e; ++i) { + const CodeGenIntrinsic &intrinsic = Ints[i]; + maxArgAttrs = + std::max(maxArgAttrs, unsigned(intrinsic.ArgumentAttributes.size())); + unsigned &N = UniqAttributes[&intrinsic]; + if (N) continue; + assert(AttrNum < 256 && "Too many unique attributes for table!"); + N = ++AttrNum; + } + + // Emit an array of AttributeList. Most intrinsics will have at least one + // entry, for the function itself (index ~1), which is usually nounwind. + OS << " static const uint8_t IntrinsicsToAttributesMap[] = {\n"; + + for (unsigned i = 0, e = Ints.size(); i != e; ++i) { + const CodeGenIntrinsic &intrinsic = Ints[i]; + + OS << " " << UniqAttributes[&intrinsic] << ", // " + << intrinsic.Name << "\n"; + } + OS << " };\n\n"; + + OS << " AttributeList AS[" << maxArgAttrs + 1 << "];\n"; + OS << " unsigned NumAttrs = 0;\n"; + OS << " if (id != 0) {\n"; + OS << " switch(IntrinsicsToAttributesMap[id - 1]) {\n"; + OS << " default: llvm_unreachable(\"Invalid attribute number\");\n"; + for (UniqAttrMapTy::const_iterator I = UniqAttributes.begin(), + E = UniqAttributes.end(); I != E; ++I) { + OS << " case " << I->second << ": {\n"; + + const CodeGenIntrinsic &intrinsic = *(I->first); + + // Keep track of the number of attributes we're writing out. + unsigned numAttrs = 0; + + // The argument attributes are alreadys sorted by argument index. + unsigned ai = 0, ae = intrinsic.ArgumentAttributes.size(); + if (ae) { + while (ai != ae) { + unsigned attrIdx = intrinsic.ArgumentAttributes[ai].Index; + + OS << " const Attribute::AttrKind AttrParam" << attrIdx << "[]= {"; + bool addComma = false; + + bool AllValuesAreZero = true; + SmallVector<uint64_t, 8> Values; + do { + switch (intrinsic.ArgumentAttributes[ai].Kind) { + case CodeGenIntrinsic::NoCapture: + if (addComma) + OS << ","; + OS << "Attribute::NoCapture"; + addComma = true; + break; + case CodeGenIntrinsic::NoAlias: + if (addComma) + OS << ","; + OS << "Attribute::NoAlias"; + addComma = true; + break; + case CodeGenIntrinsic::Returned: + if (addComma) + OS << ","; + OS << "Attribute::Returned"; + addComma = true; + break; + case CodeGenIntrinsic::ReadOnly: + if (addComma) + OS << ","; + OS << "Attribute::ReadOnly"; + addComma = true; + break; + case CodeGenIntrinsic::WriteOnly: + if (addComma) + OS << ","; + OS << "Attribute::WriteOnly"; + addComma = true; + break; + case CodeGenIntrinsic::ReadNone: + if (addComma) + OS << ","; + OS << "Attribute::ReadNone"; + addComma = true; + break; + case CodeGenIntrinsic::ImmArg: + if (addComma) + OS << ','; + OS << "Attribute::ImmArg"; + addComma = true; + break; + case CodeGenIntrinsic::Alignment: + if (addComma) + OS << ','; + OS << "Attribute::Alignment"; + addComma = true; + break; + } + uint64_t V = intrinsic.ArgumentAttributes[ai].Value; + Values.push_back(V); + AllValuesAreZero &= (V == 0); + + ++ai; + } while (ai != ae && intrinsic.ArgumentAttributes[ai].Index == attrIdx); + OS << "};\n"; + + // Generate attribute value array if not all attribute values are zero. + if (!AllValuesAreZero) { + OS << " const uint64_t AttrValParam" << attrIdx << "[]= {"; + addComma = false; + for (const auto V : Values) { + if (addComma) + OS << ','; + OS << V; + addComma = true; + } + OS << "};\n"; + } + + OS << " AS[" << numAttrs++ << "] = AttributeList::get(C, " + << attrIdx << ", AttrParam" << attrIdx; + if (!AllValuesAreZero) + OS << ", AttrValParam" << attrIdx; + OS << ");\n"; + } + } + + if (!intrinsic.canThrow || + (intrinsic.ModRef != CodeGenIntrinsic::ReadWriteMem && + !intrinsic.hasSideEffects) || + intrinsic.isNoReturn || intrinsic.isNoSync || intrinsic.isNoFree || + intrinsic.isWillReturn || intrinsic.isCold || intrinsic.isNoDuplicate || + intrinsic.isConvergent || intrinsic.isSpeculatable) { + OS << " const Attribute::AttrKind Atts[] = {"; + bool addComma = false; + if (!intrinsic.canThrow) { + OS << "Attribute::NoUnwind"; + addComma = true; + } + if (intrinsic.isNoReturn) { + if (addComma) + OS << ","; + OS << "Attribute::NoReturn"; + addComma = true; + } + if (intrinsic.isNoSync) { + if (addComma) + OS << ","; + OS << "Attribute::NoSync"; + addComma = true; + } + if (intrinsic.isNoFree) { + if (addComma) + OS << ","; + OS << "Attribute::NoFree"; + addComma = true; + } + if (intrinsic.isWillReturn) { + if (addComma) + OS << ","; + OS << "Attribute::WillReturn"; + addComma = true; + } + if (intrinsic.isCold) { + if (addComma) + OS << ","; + OS << "Attribute::Cold"; + addComma = true; + } + if (intrinsic.isNoDuplicate) { + if (addComma) + OS << ","; + OS << "Attribute::NoDuplicate"; + addComma = true; + } + if (intrinsic.isConvergent) { + if (addComma) + OS << ","; + OS << "Attribute::Convergent"; + addComma = true; + } + if (intrinsic.isSpeculatable) { + if (addComma) + OS << ","; + OS << "Attribute::Speculatable"; + addComma = true; + } + + switch (intrinsic.ModRef) { + case CodeGenIntrinsic::NoMem: + if (intrinsic.hasSideEffects) + break; + if (addComma) + OS << ","; + OS << "Attribute::ReadNone"; + break; + case CodeGenIntrinsic::ReadArgMem: + if (addComma) + OS << ","; + OS << "Attribute::ReadOnly,"; + OS << "Attribute::ArgMemOnly"; + break; + case CodeGenIntrinsic::ReadMem: + if (addComma) + OS << ","; + OS << "Attribute::ReadOnly"; + break; + case CodeGenIntrinsic::ReadInaccessibleMem: + if (addComma) + OS << ","; + OS << "Attribute::ReadOnly,"; + OS << "Attribute::InaccessibleMemOnly"; + break; + case CodeGenIntrinsic::ReadInaccessibleMemOrArgMem: + if (addComma) + OS << ","; + OS << "Attribute::ReadOnly,"; + OS << "Attribute::InaccessibleMemOrArgMemOnly"; + break; + case CodeGenIntrinsic::WriteArgMem: + if (addComma) + OS << ","; + OS << "Attribute::WriteOnly,"; + OS << "Attribute::ArgMemOnly"; + break; + case CodeGenIntrinsic::WriteMem: + if (addComma) + OS << ","; + OS << "Attribute::WriteOnly"; + break; + case CodeGenIntrinsic::WriteInaccessibleMem: + if (addComma) + OS << ","; + OS << "Attribute::WriteOnly,"; + OS << "Attribute::InaccessibleMemOnly"; + break; + case CodeGenIntrinsic::WriteInaccessibleMemOrArgMem: + if (addComma) + OS << ","; + OS << "Attribute::WriteOnly,"; + OS << "Attribute::InaccessibleMemOrArgMemOnly"; + break; + case CodeGenIntrinsic::ReadWriteArgMem: + if (addComma) + OS << ","; + OS << "Attribute::ArgMemOnly"; + break; + case CodeGenIntrinsic::ReadWriteInaccessibleMem: + if (addComma) + OS << ","; + OS << "Attribute::InaccessibleMemOnly"; + break; + case CodeGenIntrinsic::ReadWriteInaccessibleMemOrArgMem: + if (addComma) + OS << ","; + OS << "Attribute::InaccessibleMemOrArgMemOnly"; + break; + case CodeGenIntrinsic::ReadWriteMem: + break; + } + OS << "};\n"; + OS << " AS[" << numAttrs++ << "] = AttributeList::get(C, " + << "AttributeList::FunctionIndex, Atts);\n"; + } + + if (numAttrs) { + OS << " NumAttrs = " << numAttrs << ";\n"; + OS << " break;\n"; + OS << " }\n"; + } else { + OS << " return AttributeList();\n"; + OS << " }\n"; + } + } + + OS << " }\n"; + OS << " }\n"; + OS << " return AttributeList::get(C, makeArrayRef(AS, NumAttrs));\n"; + OS << "}\n"; + OS << "#endif // GET_INTRINSIC_ATTRIBUTES\n\n"; +} + +void IntrinsicEmitter::EmitIntrinsicToBuiltinMap( + const CodeGenIntrinsicTable &Ints, bool IsGCC, raw_ostream &OS) { + StringRef CompilerName = (IsGCC ? "GCC" : "MS"); + typedef std::map<std::string, std::map<std::string, std::string>> BIMTy; + BIMTy BuiltinMap; + StringToOffsetTable Table; + for (unsigned i = 0, e = Ints.size(); i != e; ++i) { + const std::string &BuiltinName = + IsGCC ? Ints[i].GCCBuiltinName : Ints[i].MSBuiltinName; + if (!BuiltinName.empty()) { + // Get the map for this target prefix. + std::map<std::string, std::string> &BIM = + BuiltinMap[Ints[i].TargetPrefix]; + + if (!BIM.insert(std::make_pair(BuiltinName, Ints[i].EnumName)).second) + PrintFatalError(Ints[i].TheDef->getLoc(), + "Intrinsic '" + Ints[i].TheDef->getName() + + "': duplicate " + CompilerName + " builtin name!"); + Table.GetOrAddStringOffset(BuiltinName); + } + } + + OS << "// Get the LLVM intrinsic that corresponds to a builtin.\n"; + OS << "// This is used by the C front-end. The builtin name is passed\n"; + OS << "// in as BuiltinName, and a target prefix (e.g. 'ppc') is passed\n"; + OS << "// in as TargetPrefix. The result is assigned to 'IntrinsicID'.\n"; + OS << "#ifdef GET_LLVM_INTRINSIC_FOR_" << CompilerName << "_BUILTIN\n"; + + OS << "Intrinsic::ID Intrinsic::getIntrinsicFor" << CompilerName + << "Builtin(const char " + << "*TargetPrefixStr, StringRef BuiltinNameStr) {\n"; + + if (Table.Empty()) { + OS << " return Intrinsic::not_intrinsic;\n"; + OS << "}\n"; + OS << "#endif\n\n"; + return; + } + + OS << " static const char BuiltinNames[] = {\n"; + Table.EmitCharArray(OS); + OS << " };\n\n"; + + OS << " struct BuiltinEntry {\n"; + OS << " Intrinsic::ID IntrinID;\n"; + OS << " unsigned StrTabOffset;\n"; + OS << " const char *getName() const {\n"; + OS << " return &BuiltinNames[StrTabOffset];\n"; + OS << " }\n"; + OS << " bool operator<(StringRef RHS) const {\n"; + OS << " return strncmp(getName(), RHS.data(), RHS.size()) < 0;\n"; + OS << " }\n"; + OS << " };\n"; + + OS << " StringRef TargetPrefix(TargetPrefixStr);\n\n"; + + // Note: this could emit significantly better code if we cared. + for (BIMTy::iterator I = BuiltinMap.begin(), E = BuiltinMap.end();I != E;++I){ + OS << " "; + if (!I->first.empty()) + OS << "if (TargetPrefix == \"" << I->first << "\") "; + else + OS << "/* Target Independent Builtins */ "; + OS << "{\n"; + + // Emit the comparisons for this target prefix. + OS << " static const BuiltinEntry " << I->first << "Names[] = {\n"; + for (const auto &P : I->second) { + OS << " {Intrinsic::" << P.second << ", " + << Table.GetOrAddStringOffset(P.first) << "}, // " << P.first << "\n"; + } + OS << " };\n"; + OS << " auto I = std::lower_bound(std::begin(" << I->first << "Names),\n"; + OS << " std::end(" << I->first << "Names),\n"; + OS << " BuiltinNameStr);\n"; + OS << " if (I != std::end(" << I->first << "Names) &&\n"; + OS << " I->getName() == BuiltinNameStr)\n"; + OS << " return I->IntrinID;\n"; + OS << " }\n"; + } + OS << " return "; + OS << "Intrinsic::not_intrinsic;\n"; + OS << "}\n"; + OS << "#endif\n\n"; +} + +void llvm::EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS) { + IntrinsicEmitter(RK).run(OS, /*Enums=*/true); +} + +void llvm::EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS) { + IntrinsicEmitter(RK).run(OS, /*Enums=*/false); +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.cpp new file mode 100644 index 000000000000..7fcf3074e093 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.cpp @@ -0,0 +1,84 @@ +//===- OptEmitter.cpp - Helper for emitting options.----------- -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OptEmitter.h" +#include "llvm/ADT/Twine.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include <cctype> +#include <cstring> + +namespace llvm { + +// Ordering on Info. The logic should match with the consumer-side function in +// llvm/Option/OptTable.h. +// FIXME: Make this take StringRefs instead of null terminated strings to +// simplify callers. +static int StrCmpOptionName(const char *A, const char *B) { + const char *X = A, *Y = B; + char a = tolower(*A), b = tolower(*B); + while (a == b) { + if (a == '\0') + return strcmp(A, B); + + a = tolower(*++X); + b = tolower(*++Y); + } + + if (a == '\0') // A is a prefix of B. + return 1; + if (b == '\0') // B is a prefix of A. + return -1; + + // Otherwise lexicographic. + return (a < b) ? -1 : 1; +} + +int CompareOptionRecords(Record *const *Av, Record *const *Bv) { + const Record *A = *Av; + const Record *B = *Bv; + + // Sentinel options precede all others and are only ordered by precedence. + bool ASent = A->getValueAsDef("Kind")->getValueAsBit("Sentinel"); + bool BSent = B->getValueAsDef("Kind")->getValueAsBit("Sentinel"); + if (ASent != BSent) + return ASent ? -1 : 1; + + // Compare options by name, unless they are sentinels. + if (!ASent) + if (int Cmp = StrCmpOptionName(A->getValueAsString("Name").str().c_str(), + B->getValueAsString("Name").str().c_str())) + return Cmp; + + if (!ASent) { + std::vector<StringRef> APrefixes = A->getValueAsListOfStrings("Prefixes"); + std::vector<StringRef> BPrefixes = B->getValueAsListOfStrings("Prefixes"); + + for (std::vector<StringRef>::const_iterator APre = APrefixes.begin(), + AEPre = APrefixes.end(), + BPre = BPrefixes.begin(), + BEPre = BPrefixes.end(); + APre != AEPre && BPre != BEPre; ++APre, ++BPre) { + if (int Cmp = StrCmpOptionName(APre->str().c_str(), BPre->str().c_str())) + return Cmp; + } + } + + // Then by the kind precedence; + int APrec = A->getValueAsDef("Kind")->getValueAsInt("Precedence"); + int BPrec = B->getValueAsDef("Kind")->getValueAsInt("Precedence"); + if (APrec == BPrec && A->getValueAsListOfStrings("Prefixes") == + B->getValueAsListOfStrings("Prefixes")) { + PrintError(A->getLoc(), Twine("Option is equivalent to")); + PrintError(B->getLoc(), Twine("Other defined here")); + PrintFatalError("Equivalent Options found."); + } + return APrec < BPrec ? -1 : 1; +} + +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.h b/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.h new file mode 100644 index 000000000000..c8f9246ef1e6 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/OptEmitter.h @@ -0,0 +1,16 @@ +//===- OptEmitter.h - Helper for emitting options. --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_OPTEMITTER_H +#define LLVM_UTILS_TABLEGEN_OPTEMITTER_H + +namespace llvm { +class Record; +int CompareOptionRecords(Record *const *Av, Record *const *Bv); +} // namespace llvm +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/OptParserEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/OptParserEmitter.cpp new file mode 100644 index 000000000000..251533a8d154 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/OptParserEmitter.cpp @@ -0,0 +1,501 @@ +//===- OptParserEmitter.cpp - Table Driven Command Line Parsing -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OptEmitter.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cctype> +#include <cstring> +#include <map> +#include <memory> + +using namespace llvm; + +static const std::string getOptionName(const Record &R) { + // Use the record name unless EnumName is defined. + if (isa<UnsetInit>(R.getValueInit("EnumName"))) + return std::string(R.getName()); + + return std::string(R.getValueAsString("EnumName")); +} + +static raw_ostream &write_cstring(raw_ostream &OS, llvm::StringRef Str) { + OS << '"'; + OS.write_escaped(Str); + OS << '"'; + return OS; +} + +static const std::string getOptionSpelling(const Record &R, + size_t &PrefixLength) { + std::vector<StringRef> Prefixes = R.getValueAsListOfStrings("Prefixes"); + StringRef Name = R.getValueAsString("Name"); + if (Prefixes.empty()) { + PrefixLength = 0; + return Name.str(); + } + PrefixLength = Prefixes[0].size(); + return (Twine(Prefixes[0]) + Twine(Name)).str(); +} + +static const std::string getOptionSpelling(const Record &R) { + size_t PrefixLength; + return getOptionSpelling(R, PrefixLength); +} + +static void emitNameUsingSpelling(raw_ostream &OS, const Record &R) { + size_t PrefixLength; + OS << "&"; + write_cstring(OS, StringRef(getOptionSpelling(R, PrefixLength))); + OS << "[" << PrefixLength << "]"; +} + +class MarshallingKindInfo { +public: + const Record &R; + const char *MacroName; + bool ShouldAlwaysEmit; + StringRef KeyPath; + StringRef DefaultValue; + StringRef NormalizedValuesScope; + + void emit(raw_ostream &OS) const { + write_cstring(OS, StringRef(getOptionSpelling(R))); + OS << ", "; + OS << ShouldAlwaysEmit; + OS << ", "; + OS << KeyPath; + OS << ", "; + emitScopedNormalizedValue(OS, DefaultValue); + OS << ", "; + emitSpecific(OS); + } + + virtual Optional<StringRef> emitValueTable(raw_ostream &OS) const { + return None; + } + + virtual ~MarshallingKindInfo() = default; + + static std::unique_ptr<MarshallingKindInfo> create(const Record &R); + +protected: + void emitScopedNormalizedValue(raw_ostream &OS, + StringRef NormalizedValue) const { + if (!NormalizedValuesScope.empty()) + OS << NormalizedValuesScope << "::"; + OS << NormalizedValue; + } + + virtual void emitSpecific(raw_ostream &OS) const = 0; + MarshallingKindInfo(const Record &R, const char *MacroName) + : R(R), MacroName(MacroName) {} +}; + +class MarshallingFlagInfo final : public MarshallingKindInfo { +public: + bool IsPositive; + + void emitSpecific(raw_ostream &OS) const override { OS << IsPositive; } + + static std::unique_ptr<MarshallingKindInfo> create(const Record &R) { + std::unique_ptr<MarshallingFlagInfo> Ret(new MarshallingFlagInfo(R)); + Ret->IsPositive = R.getValueAsBit("IsPositive"); + return Ret; + } + +private: + MarshallingFlagInfo(const Record &R) + : MarshallingKindInfo(R, "OPTION_WITH_MARSHALLING_FLAG") {} +}; + +class MarshallingStringInfo final : public MarshallingKindInfo { +public: + StringRef NormalizerRetTy; + StringRef Normalizer; + StringRef Denormalizer; + int TableIndex = -1; + std::vector<StringRef> Values; + std::vector<StringRef> NormalizedValues; + std::string ValueTableName; + + static constexpr const char *ValueTablePreamble = R"( +struct SimpleEnumValue { + const char *Name; + unsigned Value; +}; + +struct SimpleEnumValueTable { + const SimpleEnumValue *Table; + unsigned Size; +}; +)"; + + static constexpr const char *ValueTablesDecl = + "static const SimpleEnumValueTable SimpleEnumValueTables[] = "; + + void emitSpecific(raw_ostream &OS) const override { + emitScopedNormalizedValue(OS, NormalizerRetTy); + OS << ", "; + OS << Normalizer; + OS << ", "; + OS << Denormalizer; + OS << ", "; + OS << TableIndex; + } + + Optional<StringRef> emitValueTable(raw_ostream &OS) const override { + if (TableIndex == -1) + return {}; + OS << "static const SimpleEnumValue " << ValueTableName << "[] = {\n"; + for (unsigned I = 0, E = Values.size(); I != E; ++I) { + OS << "{"; + write_cstring(OS, Values[I]); + OS << ","; + OS << "static_cast<unsigned>("; + emitScopedNormalizedValue(OS, NormalizedValues[I]); + OS << ")},"; + } + OS << "};\n"; + return StringRef(ValueTableName); + } + + static std::unique_ptr<MarshallingKindInfo> create(const Record &R) { + assert(!isa<UnsetInit>(R.getValueInit("NormalizerRetTy")) && + "String options must have a type"); + + std::unique_ptr<MarshallingStringInfo> Ret(new MarshallingStringInfo(R)); + Ret->NormalizerRetTy = R.getValueAsString("NormalizerRetTy"); + + Ret->Normalizer = R.getValueAsString("Normalizer"); + Ret->Denormalizer = R.getValueAsString("Denormalizer"); + + if (!isa<UnsetInit>(R.getValueInit("NormalizedValues"))) { + assert(!isa<UnsetInit>(R.getValueInit("Values")) && + "Cannot provide normalized values for value-less options"); + Ret->TableIndex = NextTableIndex++; + Ret->NormalizedValues = R.getValueAsListOfStrings("NormalizedValues"); + Ret->Values.reserve(Ret->NormalizedValues.size()); + Ret->ValueTableName = getOptionName(R) + "ValueTable"; + + StringRef ValuesStr = R.getValueAsString("Values"); + for (;;) { + size_t Idx = ValuesStr.find(','); + if (Idx == StringRef::npos) + break; + if (Idx > 0) + Ret->Values.push_back(ValuesStr.slice(0, Idx)); + ValuesStr = ValuesStr.slice(Idx + 1, StringRef::npos); + } + if (!ValuesStr.empty()) + Ret->Values.push_back(ValuesStr); + + assert(Ret->Values.size() == Ret->NormalizedValues.size() && + "The number of normalized values doesn't match the number of " + "values"); + } + + return Ret; + } + +private: + MarshallingStringInfo(const Record &R) + : MarshallingKindInfo(R, "OPTION_WITH_MARSHALLING_STRING") {} + + static size_t NextTableIndex; +}; + +size_t MarshallingStringInfo::NextTableIndex = 0; + +std::unique_ptr<MarshallingKindInfo> +MarshallingKindInfo::create(const Record &R) { + assert(!isa<UnsetInit>(R.getValueInit("KeyPath")) && + !isa<UnsetInit>(R.getValueInit("DefaultValue")) && + "Must provide at least a key-path and a default value for emitting " + "marshalling information"); + + std::unique_ptr<MarshallingKindInfo> Ret = nullptr; + StringRef MarshallingKindStr = R.getValueAsString("MarshallingKind"); + + if (MarshallingKindStr == "flag") + Ret = MarshallingFlagInfo::create(R); + else if (MarshallingKindStr == "string") + Ret = MarshallingStringInfo::create(R); + + Ret->ShouldAlwaysEmit = R.getValueAsBit("ShouldAlwaysEmit"); + Ret->KeyPath = R.getValueAsString("KeyPath"); + Ret->DefaultValue = R.getValueAsString("DefaultValue"); + if (!isa<UnsetInit>(R.getValueInit("NormalizedValuesScope"))) + Ret->NormalizedValuesScope = R.getValueAsString("NormalizedValuesScope"); + return Ret; +} + +/// OptParserEmitter - This tablegen backend takes an input .td file +/// describing a list of options and emits a data structure for parsing and +/// working with those options when given an input command line. +namespace llvm { +void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) { + // Get the option groups and options. + const std::vector<Record*> &Groups = + Records.getAllDerivedDefinitions("OptionGroup"); + std::vector<Record*> Opts = Records.getAllDerivedDefinitions("Option"); + + emitSourceFileHeader("Option Parsing Definitions", OS); + + array_pod_sort(Opts.begin(), Opts.end(), CompareOptionRecords); + // Generate prefix groups. + typedef SmallVector<SmallString<2>, 2> PrefixKeyT; + typedef std::map<PrefixKeyT, std::string> PrefixesT; + PrefixesT Prefixes; + Prefixes.insert(std::make_pair(PrefixKeyT(), "prefix_0")); + unsigned CurPrefix = 0; + for (unsigned i = 0, e = Opts.size(); i != e; ++i) { + const Record &R = *Opts[i]; + std::vector<StringRef> prf = R.getValueAsListOfStrings("Prefixes"); + PrefixKeyT prfkey(prf.begin(), prf.end()); + unsigned NewPrefix = CurPrefix + 1; + if (Prefixes.insert(std::make_pair(prfkey, (Twine("prefix_") + + Twine(NewPrefix)).str())).second) + CurPrefix = NewPrefix; + } + + // Dump prefixes. + + OS << "/////////\n"; + OS << "// Prefixes\n\n"; + OS << "#ifdef PREFIX\n"; + OS << "#define COMMA ,\n"; + for (PrefixesT::const_iterator I = Prefixes.begin(), E = Prefixes.end(); + I != E; ++I) { + OS << "PREFIX("; + + // Prefix name. + OS << I->second; + + // Prefix values. + OS << ", {"; + for (PrefixKeyT::const_iterator PI = I->first.begin(), + PE = I->first.end(); PI != PE; ++PI) { + OS << "\"" << *PI << "\" COMMA "; + } + OS << "nullptr})\n"; + } + OS << "#undef COMMA\n"; + OS << "#endif // PREFIX\n\n"; + + OS << "/////////\n"; + OS << "// Groups\n\n"; + OS << "#ifdef OPTION\n"; + for (unsigned i = 0, e = Groups.size(); i != e; ++i) { + const Record &R = *Groups[i]; + + // Start a single option entry. + OS << "OPTION("; + + // The option prefix; + OS << "nullptr"; + + // The option string. + OS << ", \"" << R.getValueAsString("Name") << '"'; + + // The option identifier name. + OS << ", " << getOptionName(R); + + // The option kind. + OS << ", Group"; + + // The containing option group (if any). + OS << ", "; + if (const DefInit *DI = dyn_cast<DefInit>(R.getValueInit("Group"))) + OS << getOptionName(*DI->getDef()); + else + OS << "INVALID"; + + // The other option arguments (unused for groups). + OS << ", INVALID, nullptr, 0, 0"; + + // The option help text. + if (!isa<UnsetInit>(R.getValueInit("HelpText"))) { + OS << ",\n"; + OS << " "; + write_cstring(OS, R.getValueAsString("HelpText")); + } else + OS << ", nullptr"; + + // The option meta-variable name (unused). + OS << ", nullptr"; + + // The option Values (unused for groups). + OS << ", nullptr)\n"; + } + OS << "\n"; + + OS << "//////////\n"; + OS << "// Options\n\n"; + + auto WriteOptRecordFields = [&](raw_ostream &OS, const Record &R) { + // The option prefix; + std::vector<StringRef> prf = R.getValueAsListOfStrings("Prefixes"); + OS << Prefixes[PrefixKeyT(prf.begin(), prf.end())] << ", "; + + // The option string. + emitNameUsingSpelling(OS, R); + + // The option identifier name. + OS << ", " << getOptionName(R); + + // The option kind. + OS << ", " << R.getValueAsDef("Kind")->getValueAsString("Name"); + + // The containing option group (if any). + OS << ", "; + const ListInit *GroupFlags = nullptr; + if (const DefInit *DI = dyn_cast<DefInit>(R.getValueInit("Group"))) { + GroupFlags = DI->getDef()->getValueAsListInit("Flags"); + OS << getOptionName(*DI->getDef()); + } else + OS << "INVALID"; + + // The option alias (if any). + OS << ", "; + if (const DefInit *DI = dyn_cast<DefInit>(R.getValueInit("Alias"))) + OS << getOptionName(*DI->getDef()); + else + OS << "INVALID"; + + // The option alias arguments (if any). + // Emitted as a \0 separated list in a string, e.g. ["foo", "bar"] + // would become "foo\0bar\0". Note that the compiler adds an implicit + // terminating \0 at the end. + OS << ", "; + std::vector<StringRef> AliasArgs = R.getValueAsListOfStrings("AliasArgs"); + if (AliasArgs.size() == 0) { + OS << "nullptr"; + } else { + OS << "\""; + for (size_t i = 0, e = AliasArgs.size(); i != e; ++i) + OS << AliasArgs[i] << "\\0"; + OS << "\""; + } + + // The option flags. + OS << ", "; + int NumFlags = 0; + const ListInit *LI = R.getValueAsListInit("Flags"); + for (Init *I : *LI) + OS << (NumFlags++ ? " | " : "") << cast<DefInit>(I)->getDef()->getName(); + if (GroupFlags) { + for (Init *I : *GroupFlags) + OS << (NumFlags++ ? " | " : "") + << cast<DefInit>(I)->getDef()->getName(); + } + if (NumFlags == 0) + OS << '0'; + + // The option parameter field. + OS << ", " << R.getValueAsInt("NumArgs"); + + // The option help text. + if (!isa<UnsetInit>(R.getValueInit("HelpText"))) { + OS << ",\n"; + OS << " "; + write_cstring(OS, R.getValueAsString("HelpText")); + } else + OS << ", nullptr"; + + // The option meta-variable name. + OS << ", "; + if (!isa<UnsetInit>(R.getValueInit("MetaVarName"))) + write_cstring(OS, R.getValueAsString("MetaVarName")); + else + OS << "nullptr"; + + // The option Values. Used for shell autocompletion. + OS << ", "; + if (!isa<UnsetInit>(R.getValueInit("Values"))) + write_cstring(OS, R.getValueAsString("Values")); + else + OS << "nullptr"; + }; + + std::vector<std::unique_ptr<MarshallingKindInfo>> OptsWithMarshalling; + for (unsigned I = 0, E = Opts.size(); I != E; ++I) { + const Record &R = *Opts[I]; + + // Start a single option entry. + OS << "OPTION("; + WriteOptRecordFields(OS, R); + OS << ")\n"; + if (!isa<UnsetInit>(R.getValueInit("MarshallingKind"))) + OptsWithMarshalling.push_back(MarshallingKindInfo::create(R)); + } + OS << "#endif // OPTION\n"; + + for (const auto &KindInfo : OptsWithMarshalling) { + OS << "#ifdef " << KindInfo->MacroName << "\n"; + OS << KindInfo->MacroName << "("; + WriteOptRecordFields(OS, KindInfo->R); + OS << ", "; + KindInfo->emit(OS); + OS << ")\n"; + OS << "#endif // " << KindInfo->MacroName << "\n"; + } + + OS << "\n"; + OS << "#ifdef SIMPLE_ENUM_VALUE_TABLE"; + OS << "\n"; + OS << MarshallingStringInfo::ValueTablePreamble; + std::vector<StringRef> ValueTableNames; + for (const auto &KindInfo : OptsWithMarshalling) + if (auto MaybeValueTableName = KindInfo->emitValueTable(OS)) + ValueTableNames.push_back(*MaybeValueTableName); + + OS << MarshallingStringInfo::ValueTablesDecl << "{"; + for (auto ValueTableName : ValueTableNames) + OS << "{" << ValueTableName << ", sizeof(" << ValueTableName + << ") / sizeof(SimpleEnumValue)" + << "},\n"; + OS << "};\n"; + OS << "static const unsigned SimpleEnumValueTablesSize = " + "sizeof(SimpleEnumValueTables) / sizeof(SimpleEnumValueTable);\n"; + + OS << "#endif // SIMPLE_ENUM_VALUE_TABLE\n"; + OS << "\n"; + + OS << "\n"; + OS << "#ifdef OPTTABLE_ARG_INIT\n"; + OS << "//////////\n"; + OS << "// Option Values\n\n"; + for (unsigned I = 0, E = Opts.size(); I != E; ++I) { + const Record &R = *Opts[I]; + if (isa<UnsetInit>(R.getValueInit("ValuesCode"))) + continue; + OS << "{\n"; + OS << "bool ValuesWereAdded;\n"; + OS << R.getValueAsString("ValuesCode"); + OS << "\n"; + for (StringRef Prefix : R.getValueAsListOfStrings("Prefixes")) { + OS << "ValuesWereAdded = Opt.addValues("; + std::string S(Prefix); + S += R.getValueAsString("Name"); + write_cstring(OS, S); + OS << ", Values);\n"; + OS << "(void)ValuesWereAdded;\n"; + OS << "assert(ValuesWereAdded && \"Couldn't add values to " + "OptTable!\");\n"; + } + OS << "}\n"; + } + OS << "\n"; + OS << "#endif // OPTTABLE_ARG_INIT\n"; +} +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/OptRSTEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/OptRSTEmitter.cpp new file mode 100644 index 000000000000..5e44d033109a --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/OptRSTEmitter.cpp @@ -0,0 +1,87 @@ +//===- OptParserEmitter.cpp - Table Driven Command Line Parsing -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OptEmitter.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/Twine.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <cctype> +#include <cstring> +#include <map> + +using namespace llvm; + +/// OptParserEmitter - This tablegen backend takes an input .td file +/// describing a list of options and emits a RST man page. +namespace llvm { +void EmitOptRST(RecordKeeper &Records, raw_ostream &OS) { + llvm::StringMap<std::vector<Record *>> OptionsByGroup; + std::vector<Record *> OptionsWithoutGroup; + + // Get the options. + std::vector<Record *> Opts = Records.getAllDerivedDefinitions("Option"); + array_pod_sort(Opts.begin(), Opts.end(), CompareOptionRecords); + + // Get the option groups. + const std::vector<Record *> &Groups = + Records.getAllDerivedDefinitions("OptionGroup"); + for (unsigned i = 0, e = Groups.size(); i != e; ++i) { + const Record &R = *Groups[i]; + OptionsByGroup.try_emplace(R.getValueAsString("Name")); + } + + // Map options to their group. + for (unsigned i = 0, e = Opts.size(); i != e; ++i) { + const Record &R = *Opts[i]; + if (const DefInit *DI = dyn_cast<DefInit>(R.getValueInit("Group"))) { + OptionsByGroup[DI->getDef()->getValueAsString("Name")].push_back(Opts[i]); + } else { + OptionsByGroup["options"].push_back(Opts[i]); + } + } + + // Print options under their group. + for (const auto &KV : OptionsByGroup) { + std::string GroupName = KV.getKey().upper(); + OS << GroupName << '\n'; + OS << std::string(GroupName.size(), '-') << '\n'; + OS << '\n'; + + for (Record *R : KV.getValue()) { + OS << ".. option:: "; + + // Print the prefix. + std::vector<StringRef> Prefixes = R->getValueAsListOfStrings("Prefixes"); + if (!Prefixes.empty()) + OS << Prefixes[0]; + + // Print the option name. + OS << R->getValueAsString("Name"); + + // Print the meta-variable. + if (!isa<UnsetInit>(R->getValueInit("MetaVarName"))) { + OS << '='; + OS.write_escaped(R->getValueAsString("MetaVarName")); + } + + OS << "\n\n"; + + // The option help text. + if (!isa<UnsetInit>(R->getValueInit("HelpText"))) { + OS << ' '; + OS.write_escaped(R->getValueAsString("HelpText")); + OS << "\n\n"; + } + } + } +} +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/PredicateExpander.cpp b/contrib/llvm-project/llvm/utils/TableGen/PredicateExpander.cpp new file mode 100644 index 000000000000..9f7f40db2626 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/PredicateExpander.cpp @@ -0,0 +1,529 @@ +//===--------------------- PredicateExpander.cpp --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// Functionalities used by the Tablegen backends to expand machine predicates. +// +//===----------------------------------------------------------------------===// + +#include "PredicateExpander.h" +#include "CodeGenSchedule.h" // Definition of STIPredicateFunction. + +namespace llvm { + +void PredicateExpander::expandTrue(raw_ostream &OS) { OS << "true"; } +void PredicateExpander::expandFalse(raw_ostream &OS) { OS << "false"; } + +void PredicateExpander::expandCheckImmOperand(raw_ostream &OS, int OpIndex, + int ImmVal, + StringRef FunctionMapper) { + if (!FunctionMapper.empty()) + OS << FunctionMapper << "("; + OS << "MI" << (isByRef() ? "." : "->") << "getOperand(" << OpIndex + << ").getImm()"; + if (!FunctionMapper.empty()) + OS << ")"; + OS << (shouldNegate() ? " != " : " == ") << ImmVal; +} + +void PredicateExpander::expandCheckImmOperand(raw_ostream &OS, int OpIndex, + StringRef ImmVal, + StringRef FunctionMapper) { + if (ImmVal.empty()) + expandCheckImmOperandSimple(OS, OpIndex, FunctionMapper); + + if (!FunctionMapper.empty()) + OS << FunctionMapper << "("; + OS << "MI" << (isByRef() ? "." : "->") << "getOperand(" << OpIndex + << ").getImm()"; + if (!FunctionMapper.empty()) + OS << ")"; + OS << (shouldNegate() ? " != " : " == ") << ImmVal; +} + +void PredicateExpander::expandCheckImmOperandSimple(raw_ostream &OS, + int OpIndex, + StringRef FunctionMapper) { + if (shouldNegate()) + OS << "!"; + if (!FunctionMapper.empty()) + OS << FunctionMapper << "("; + OS << "MI" << (isByRef() ? "." : "->") << "getOperand(" << OpIndex + << ").getImm()"; + if (!FunctionMapper.empty()) + OS << ")"; +} + +void PredicateExpander::expandCheckRegOperand(raw_ostream &OS, int OpIndex, + const Record *Reg, + StringRef FunctionMapper) { + assert(Reg->isSubClassOf("Register") && "Expected a register Record!"); + + if (!FunctionMapper.empty()) + OS << FunctionMapper << "("; + OS << "MI" << (isByRef() ? "." : "->") << "getOperand(" << OpIndex + << ").getReg()"; + if (!FunctionMapper.empty()) + OS << ")"; + OS << (shouldNegate() ? " != " : " == "); + const StringRef Str = Reg->getValueAsString("Namespace"); + if (!Str.empty()) + OS << Str << "::"; + OS << Reg->getName(); +} + + +void PredicateExpander::expandCheckRegOperandSimple(raw_ostream &OS, + int OpIndex, + StringRef FunctionMapper) { + if (shouldNegate()) + OS << "!"; + if (!FunctionMapper.empty()) + OS << FunctionMapper << "("; + OS << "MI" << (isByRef() ? "." : "->") << "getOperand(" << OpIndex + << ").getReg()"; + if (!FunctionMapper.empty()) + OS << ")"; +} + +void PredicateExpander::expandCheckInvalidRegOperand(raw_ostream &OS, + int OpIndex) { + OS << "MI" << (isByRef() ? "." : "->") << "getOperand(" << OpIndex + << ").getReg() " << (shouldNegate() ? "!= " : "== ") << "0"; +} + +void PredicateExpander::expandCheckSameRegOperand(raw_ostream &OS, int First, + int Second) { + OS << "MI" << (isByRef() ? "." : "->") << "getOperand(" << First + << ").getReg() " << (shouldNegate() ? "!=" : "==") << " MI" + << (isByRef() ? "." : "->") << "getOperand(" << Second << ").getReg()"; +} + +void PredicateExpander::expandCheckNumOperands(raw_ostream &OS, int NumOps) { + OS << "MI" << (isByRef() ? "." : "->") << "getNumOperands() " + << (shouldNegate() ? "!= " : "== ") << NumOps; +} + +void PredicateExpander::expandCheckOpcode(raw_ostream &OS, const Record *Inst) { + OS << "MI" << (isByRef() ? "." : "->") << "getOpcode() " + << (shouldNegate() ? "!= " : "== ") << Inst->getValueAsString("Namespace") + << "::" << Inst->getName(); +} + +void PredicateExpander::expandCheckOpcode(raw_ostream &OS, + const RecVec &Opcodes) { + assert(!Opcodes.empty() && "Expected at least one opcode to check!"); + bool First = true; + + if (Opcodes.size() == 1) { + OS << "( "; + expandCheckOpcode(OS, Opcodes[0]); + OS << " )"; + return; + } + + OS << '('; + increaseIndentLevel(); + for (const Record *Rec : Opcodes) { + OS << '\n'; + OS.indent(getIndentLevel() * 2); + if (!First) + OS << (shouldNegate() ? "&& " : "|| "); + + expandCheckOpcode(OS, Rec); + First = false; + } + + OS << '\n'; + decreaseIndentLevel(); + OS.indent(getIndentLevel() * 2); + OS << ')'; +} + +void PredicateExpander::expandCheckPseudo(raw_ostream &OS, + const RecVec &Opcodes) { + if (shouldExpandForMC()) + expandFalse(OS); + else + expandCheckOpcode(OS, Opcodes); +} + +void PredicateExpander::expandPredicateSequence(raw_ostream &OS, + const RecVec &Sequence, + bool IsCheckAll) { + assert(!Sequence.empty() && "Found an invalid empty predicate set!"); + if (Sequence.size() == 1) + return expandPredicate(OS, Sequence[0]); + + // Okay, there is more than one predicate in the set. + bool First = true; + OS << (shouldNegate() ? "!(" : "("); + increaseIndentLevel(); + + bool OldValue = shouldNegate(); + setNegatePredicate(false); + for (const Record *Rec : Sequence) { + OS << '\n'; + OS.indent(getIndentLevel() * 2); + if (!First) + OS << (IsCheckAll ? "&& " : "|| "); + expandPredicate(OS, Rec); + First = false; + } + OS << '\n'; + decreaseIndentLevel(); + OS.indent(getIndentLevel() * 2); + OS << ')'; + setNegatePredicate(OldValue); +} + +void PredicateExpander::expandTIIFunctionCall(raw_ostream &OS, + StringRef MethodName) { + OS << (shouldNegate() ? "!" : ""); + OS << TargetName << (shouldExpandForMC() ? "_MC::" : "InstrInfo::"); + OS << MethodName << (isByRef() ? "(MI)" : "(*MI)"); +} + +void PredicateExpander::expandCheckIsRegOperand(raw_ostream &OS, int OpIndex) { + OS << (shouldNegate() ? "!" : "") << "MI" << (isByRef() ? "." : "->") + << "getOperand(" << OpIndex << ").isReg() "; +} + +void PredicateExpander::expandCheckIsImmOperand(raw_ostream &OS, int OpIndex) { + OS << (shouldNegate() ? "!" : "") << "MI" << (isByRef() ? "." : "->") + << "getOperand(" << OpIndex << ").isImm() "; +} + +void PredicateExpander::expandCheckFunctionPredicate(raw_ostream &OS, + StringRef MCInstFn, + StringRef MachineInstrFn) { + OS << (shouldExpandForMC() ? MCInstFn : MachineInstrFn) + << (isByRef() ? "(MI)" : "(*MI)"); +} + +void PredicateExpander::expandCheckNonPortable(raw_ostream &OS, + StringRef Code) { + if (shouldExpandForMC()) + return expandFalse(OS); + + OS << '(' << Code << ')'; +} + +void PredicateExpander::expandReturnStatement(raw_ostream &OS, + const Record *Rec) { + std::string Buffer; + raw_string_ostream SS(Buffer); + + SS << "return "; + expandPredicate(SS, Rec); + SS << ";"; + SS.flush(); + OS << Buffer; +} + +void PredicateExpander::expandOpcodeSwitchCase(raw_ostream &OS, + const Record *Rec) { + const RecVec &Opcodes = Rec->getValueAsListOfDefs("Opcodes"); + for (const Record *Opcode : Opcodes) { + OS.indent(getIndentLevel() * 2); + OS << "case " << Opcode->getValueAsString("Namespace") + << "::" << Opcode->getName() << ":\n"; + } + + increaseIndentLevel(); + OS.indent(getIndentLevel() * 2); + expandStatement(OS, Rec->getValueAsDef("CaseStmt")); + decreaseIndentLevel(); +} + +void PredicateExpander::expandOpcodeSwitchStatement(raw_ostream &OS, + const RecVec &Cases, + const Record *Default) { + std::string Buffer; + raw_string_ostream SS(Buffer); + + SS << "switch(MI" << (isByRef() ? "." : "->") << "getOpcode()) {\n"; + for (const Record *Rec : Cases) { + expandOpcodeSwitchCase(SS, Rec); + SS << '\n'; + } + + // Expand the default case. + SS.indent(getIndentLevel() * 2); + SS << "default:\n"; + + increaseIndentLevel(); + SS.indent(getIndentLevel() * 2); + expandStatement(SS, Default); + decreaseIndentLevel(); + SS << '\n'; + + SS.indent(getIndentLevel() * 2); + SS << "} // end of switch-stmt"; + SS.flush(); + OS << Buffer; +} + +void PredicateExpander::expandStatement(raw_ostream &OS, const Record *Rec) { + // Assume that padding has been added by the caller. + if (Rec->isSubClassOf("MCOpcodeSwitchStatement")) { + expandOpcodeSwitchStatement(OS, Rec->getValueAsListOfDefs("Cases"), + Rec->getValueAsDef("DefaultCase")); + return; + } + + if (Rec->isSubClassOf("MCReturnStatement")) { + expandReturnStatement(OS, Rec->getValueAsDef("Pred")); + return; + } + + llvm_unreachable("No known rules to expand this MCStatement"); +} + +void PredicateExpander::expandPredicate(raw_ostream &OS, const Record *Rec) { + // Assume that padding has been added by the caller. + if (Rec->isSubClassOf("MCTrue")) { + if (shouldNegate()) + return expandFalse(OS); + return expandTrue(OS); + } + + if (Rec->isSubClassOf("MCFalse")) { + if (shouldNegate()) + return expandTrue(OS); + return expandFalse(OS); + } + + if (Rec->isSubClassOf("CheckNot")) { + flipNegatePredicate(); + expandPredicate(OS, Rec->getValueAsDef("Pred")); + flipNegatePredicate(); + return; + } + + if (Rec->isSubClassOf("CheckIsRegOperand")) + return expandCheckIsRegOperand(OS, Rec->getValueAsInt("OpIndex")); + + if (Rec->isSubClassOf("CheckIsImmOperand")) + return expandCheckIsImmOperand(OS, Rec->getValueAsInt("OpIndex")); + + if (Rec->isSubClassOf("CheckRegOperand")) + return expandCheckRegOperand(OS, Rec->getValueAsInt("OpIndex"), + Rec->getValueAsDef("Reg"), + Rec->getValueAsString("FunctionMapper")); + + if (Rec->isSubClassOf("CheckRegOperandSimple")) + return expandCheckRegOperandSimple(OS, Rec->getValueAsInt("OpIndex"), + Rec->getValueAsString("FunctionMapper")); + + if (Rec->isSubClassOf("CheckInvalidRegOperand")) + return expandCheckInvalidRegOperand(OS, Rec->getValueAsInt("OpIndex")); + + if (Rec->isSubClassOf("CheckImmOperand")) + return expandCheckImmOperand(OS, Rec->getValueAsInt("OpIndex"), + Rec->getValueAsInt("ImmVal"), + Rec->getValueAsString("FunctionMapper")); + + if (Rec->isSubClassOf("CheckImmOperand_s")) + return expandCheckImmOperand(OS, Rec->getValueAsInt("OpIndex"), + Rec->getValueAsString("ImmVal"), + Rec->getValueAsString("FunctionMapper")); + + if (Rec->isSubClassOf("CheckImmOperandSimple")) + return expandCheckImmOperandSimple(OS, Rec->getValueAsInt("OpIndex"), + Rec->getValueAsString("FunctionMapper")); + + if (Rec->isSubClassOf("CheckSameRegOperand")) + return expandCheckSameRegOperand(OS, Rec->getValueAsInt("FirstIndex"), + Rec->getValueAsInt("SecondIndex")); + + if (Rec->isSubClassOf("CheckNumOperands")) + return expandCheckNumOperands(OS, Rec->getValueAsInt("NumOps")); + + if (Rec->isSubClassOf("CheckPseudo")) + return expandCheckPseudo(OS, Rec->getValueAsListOfDefs("ValidOpcodes")); + + if (Rec->isSubClassOf("CheckOpcode")) + return expandCheckOpcode(OS, Rec->getValueAsListOfDefs("ValidOpcodes")); + + if (Rec->isSubClassOf("CheckAll")) + return expandPredicateSequence(OS, Rec->getValueAsListOfDefs("Predicates"), + /* AllOf */ true); + + if (Rec->isSubClassOf("CheckAny")) + return expandPredicateSequence(OS, Rec->getValueAsListOfDefs("Predicates"), + /* AllOf */ false); + + if (Rec->isSubClassOf("CheckFunctionPredicate")) + return expandCheckFunctionPredicate( + OS, Rec->getValueAsString("MCInstFnName"), + Rec->getValueAsString("MachineInstrFnName")); + + if (Rec->isSubClassOf("CheckNonPortable")) + return expandCheckNonPortable(OS, Rec->getValueAsString("CodeBlock")); + + if (Rec->isSubClassOf("TIIPredicate")) + return expandTIIFunctionCall(OS, Rec->getValueAsString("FunctionName")); + + llvm_unreachable("No known rules to expand this MCInstPredicate"); +} + +void STIPredicateExpander::expandHeader(raw_ostream &OS, + const STIPredicateFunction &Fn) { + const Record *Rec = Fn.getDeclaration(); + StringRef FunctionName = Rec->getValueAsString("Name"); + + OS.indent(getIndentLevel() * 2); + OS << "bool "; + if (shouldExpandDefinition()) + OS << getClassPrefix() << "::"; + OS << FunctionName << "("; + if (shouldExpandForMC()) + OS << "const MCInst " << (isByRef() ? "&" : "*") << "MI"; + else + OS << "const MachineInstr " << (isByRef() ? "&" : "*") << "MI"; + if (Rec->getValueAsBit("UpdatesOpcodeMask")) + OS << ", APInt &Mask"; + OS << (shouldExpandForMC() ? ", unsigned ProcessorID) const " : ") const "); + if (shouldExpandDefinition()) { + OS << "{\n"; + return; + } + + if (Rec->getValueAsBit("OverridesBaseClassMember")) + OS << "override"; + OS << ";\n"; +} + +void STIPredicateExpander::expandPrologue(raw_ostream &OS, + const STIPredicateFunction &Fn) { + RecVec Delegates = Fn.getDeclaration()->getValueAsListOfDefs("Delegates"); + bool UpdatesOpcodeMask = + Fn.getDeclaration()->getValueAsBit("UpdatesOpcodeMask"); + + increaseIndentLevel(); + unsigned IndentLevel = getIndentLevel(); + for (const Record *Delegate : Delegates) { + OS.indent(IndentLevel * 2); + OS << "if (" << Delegate->getValueAsString("Name") << "(MI"; + if (UpdatesOpcodeMask) + OS << ", Mask"; + if (shouldExpandForMC()) + OS << ", ProcessorID"; + OS << "))\n"; + OS.indent((1 + IndentLevel) * 2); + OS << "return true;\n\n"; + } + + if (shouldExpandForMC()) + return; + + OS.indent(IndentLevel * 2); + OS << "unsigned ProcessorID = getSchedModel().getProcessorID();\n"; +} + +void STIPredicateExpander::expandOpcodeGroup(raw_ostream &OS, const OpcodeGroup &Group, + bool ShouldUpdateOpcodeMask) { + const OpcodeInfo &OI = Group.getOpcodeInfo(); + for (const PredicateInfo &PI : OI.getPredicates()) { + const APInt &ProcModelMask = PI.ProcModelMask; + bool FirstProcID = true; + for (unsigned I = 0, E = ProcModelMask.getActiveBits(); I < E; ++I) { + if (!ProcModelMask[I]) + continue; + + if (FirstProcID) { + OS.indent(getIndentLevel() * 2); + OS << "if (ProcessorID == " << I; + } else { + OS << " || ProcessorID == " << I; + } + FirstProcID = false; + } + + OS << ") {\n"; + + increaseIndentLevel(); + OS.indent(getIndentLevel() * 2); + if (ShouldUpdateOpcodeMask) { + if (PI.OperandMask.isNullValue()) + OS << "Mask.clearAllBits();\n"; + else + OS << "Mask = " << PI.OperandMask << ";\n"; + OS.indent(getIndentLevel() * 2); + } + OS << "return "; + expandPredicate(OS, PI.Predicate); + OS << ";\n"; + decreaseIndentLevel(); + OS.indent(getIndentLevel() * 2); + OS << "}\n"; + } +} + +void STIPredicateExpander::expandBody(raw_ostream &OS, + const STIPredicateFunction &Fn) { + bool UpdatesOpcodeMask = + Fn.getDeclaration()->getValueAsBit("UpdatesOpcodeMask"); + + unsigned IndentLevel = getIndentLevel(); + OS.indent(IndentLevel * 2); + OS << "switch(MI" << (isByRef() ? "." : "->") << "getOpcode()) {\n"; + OS.indent(IndentLevel * 2); + OS << "default:\n"; + OS.indent(IndentLevel * 2); + OS << " break;"; + + for (const OpcodeGroup &Group : Fn.getGroups()) { + for (const Record *Opcode : Group.getOpcodes()) { + OS << '\n'; + OS.indent(IndentLevel * 2); + OS << "case " << getTargetName() << "::" << Opcode->getName() << ":"; + } + + OS << '\n'; + increaseIndentLevel(); + expandOpcodeGroup(OS, Group, UpdatesOpcodeMask); + + OS.indent(getIndentLevel() * 2); + OS << "break;\n"; + decreaseIndentLevel(); + } + + OS.indent(IndentLevel * 2); + OS << "}\n"; +} + +void STIPredicateExpander::expandEpilogue(raw_ostream &OS, + const STIPredicateFunction &Fn) { + OS << '\n'; + OS.indent(getIndentLevel() * 2); + OS << "return "; + expandPredicate(OS, Fn.getDefaultReturnPredicate()); + OS << ";\n"; + + decreaseIndentLevel(); + OS.indent(getIndentLevel() * 2); + StringRef FunctionName = Fn.getDeclaration()->getValueAsString("Name"); + OS << "} // " << ClassPrefix << "::" << FunctionName << "\n\n"; +} + +void STIPredicateExpander::expandSTIPredicate(raw_ostream &OS, + const STIPredicateFunction &Fn) { + const Record *Rec = Fn.getDeclaration(); + if (shouldExpandForMC() && !Rec->getValueAsBit("ExpandForMC")) + return; + + expandHeader(OS, Fn); + if (shouldExpandDefinition()) { + expandPrologue(OS, Fn); + expandBody(OS, Fn); + expandEpilogue(OS, Fn); + } +} + +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/PredicateExpander.h b/contrib/llvm-project/llvm/utils/TableGen/PredicateExpander.h new file mode 100644 index 000000000000..115a81cf123b --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/PredicateExpander.h @@ -0,0 +1,123 @@ +//===--------------------- PredicateExpander.h ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// Functionalities used by the Tablegen backends to expand machine predicates. +/// +/// See file llvm/Target/TargetInstrPredicate.td for a full list and description +/// of all the supported MCInstPredicate classes. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_PREDICATEEXPANDER_H +#define LLVM_UTILS_TABLEGEN_PREDICATEEXPANDER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Record.h" + +namespace llvm { + +class raw_ostream; + +class PredicateExpander { + bool EmitCallsByRef; + bool NegatePredicate; + bool ExpandForMC; + unsigned IndentLevel; + StringRef TargetName; + + PredicateExpander(const PredicateExpander &) = delete; + PredicateExpander &operator=(const PredicateExpander &) = delete; + +public: + PredicateExpander(StringRef Target) + : EmitCallsByRef(true), NegatePredicate(false), ExpandForMC(false), + IndentLevel(1U), TargetName(Target) {} + bool isByRef() const { return EmitCallsByRef; } + bool shouldNegate() const { return NegatePredicate; } + bool shouldExpandForMC() const { return ExpandForMC; } + unsigned getIndentLevel() const { return IndentLevel; } + StringRef getTargetName() const { return TargetName; } + + void setByRef(bool Value) { EmitCallsByRef = Value; } + void flipNegatePredicate() { NegatePredicate = !NegatePredicate; } + void setNegatePredicate(bool Value) { NegatePredicate = Value; } + void setExpandForMC(bool Value) { ExpandForMC = Value; } + void setIndentLevel(unsigned Level) { IndentLevel = Level; } + void increaseIndentLevel() { ++IndentLevel; } + void decreaseIndentLevel() { --IndentLevel; } + + using RecVec = std::vector<Record *>; + void expandTrue(raw_ostream &OS); + void expandFalse(raw_ostream &OS); + void expandCheckImmOperand(raw_ostream &OS, int OpIndex, int ImmVal, + StringRef FunctionMapper); + void expandCheckImmOperand(raw_ostream &OS, int OpIndex, StringRef ImmVal, + StringRef FunctionMapperer); + void expandCheckImmOperandSimple(raw_ostream &OS, int OpIndex, + StringRef FunctionMapper); + void expandCheckRegOperand(raw_ostream &OS, int OpIndex, const Record *Reg, + StringRef FunctionMapper); + void expandCheckRegOperandSimple(raw_ostream &OS, int OpIndex, + StringRef FunctionMapper); + void expandCheckSameRegOperand(raw_ostream &OS, int First, int Second); + void expandCheckNumOperands(raw_ostream &OS, int NumOps); + void expandCheckOpcode(raw_ostream &OS, const Record *Inst); + + void expandCheckPseudo(raw_ostream &OS, const RecVec &Opcodes); + void expandCheckOpcode(raw_ostream &OS, const RecVec &Opcodes); + void expandPredicateSequence(raw_ostream &OS, const RecVec &Sequence, + bool IsCheckAll); + void expandTIIFunctionCall(raw_ostream &OS, StringRef MethodName); + void expandCheckIsRegOperand(raw_ostream &OS, int OpIndex); + void expandCheckIsImmOperand(raw_ostream &OS, int OpIndex); + void expandCheckInvalidRegOperand(raw_ostream &OS, int OpIndex); + void expandCheckFunctionPredicate(raw_ostream &OS, StringRef MCInstFn, + StringRef MachineInstrFn); + void expandCheckNonPortable(raw_ostream &OS, StringRef CodeBlock); + void expandPredicate(raw_ostream &OS, const Record *Rec); + void expandReturnStatement(raw_ostream &OS, const Record *Rec); + void expandOpcodeSwitchCase(raw_ostream &OS, const Record *Rec); + void expandOpcodeSwitchStatement(raw_ostream &OS, const RecVec &Cases, + const Record *Default); + void expandStatement(raw_ostream &OS, const Record *Rec); +}; + +// Forward declarations. +class STIPredicateFunction; +class OpcodeGroup; + +class STIPredicateExpander : public PredicateExpander { + StringRef ClassPrefix; + bool ExpandDefinition; + + STIPredicateExpander(const PredicateExpander &) = delete; + STIPredicateExpander &operator=(const PredicateExpander &) = delete; + + void expandHeader(raw_ostream &OS, const STIPredicateFunction &Fn); + void expandPrologue(raw_ostream &OS, const STIPredicateFunction &Fn); + void expandOpcodeGroup(raw_ostream &OS, const OpcodeGroup &Group, + bool ShouldUpdateOpcodeMask); + void expandBody(raw_ostream &OS, const STIPredicateFunction &Fn); + void expandEpilogue(raw_ostream &OS, const STIPredicateFunction &Fn); + +public: + STIPredicateExpander(StringRef Target) + : PredicateExpander(Target), ClassPrefix(), ExpandDefinition(false) {} + + bool shouldExpandDefinition() const { return ExpandDefinition; } + StringRef getClassPrefix() const { return ClassPrefix; } + void setClassPrefix(StringRef S) { ClassPrefix = S; } + void setExpandDefinition(bool Value) { ExpandDefinition = Value; } + + void expandSTIPredicate(raw_ostream &OS, const STIPredicateFunction &Fn); +}; + +} // namespace llvm + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/PseudoLoweringEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/PseudoLoweringEmitter.cpp new file mode 100644 index 000000000000..3a80d8e5d1c4 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/PseudoLoweringEmitter.cpp @@ -0,0 +1,301 @@ +//===- PseudoLoweringEmitter.cpp - PseudoLowering Generator -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "CodeGenInstruction.h" +#include "CodeGenTarget.h" +#include "llvm/ADT/IndexedMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <vector> +using namespace llvm; + +#define DEBUG_TYPE "pseudo-lowering" + +namespace { +class PseudoLoweringEmitter { + struct OpData { + enum MapKind { Operand, Imm, Reg }; + MapKind Kind; + union { + unsigned Operand; // Operand number mapped to. + uint64_t Imm; // Integer immedate value. + Record *Reg; // Physical register. + } Data; + }; + struct PseudoExpansion { + CodeGenInstruction Source; // The source pseudo instruction definition. + CodeGenInstruction Dest; // The destination instruction to lower to. + IndexedMap<OpData> OperandMap; + + PseudoExpansion(CodeGenInstruction &s, CodeGenInstruction &d, + IndexedMap<OpData> &m) : + Source(s), Dest(d), OperandMap(m) {} + }; + + RecordKeeper &Records; + + // It's overkill to have an instance of the full CodeGenTarget object, + // but it loads everything on demand, not in the constructor, so it's + // lightweight in performance, so it works out OK. + CodeGenTarget Target; + + SmallVector<PseudoExpansion, 64> Expansions; + + unsigned addDagOperandMapping(Record *Rec, DagInit *Dag, + CodeGenInstruction &Insn, + IndexedMap<OpData> &OperandMap, + unsigned BaseIdx); + void evaluateExpansion(Record *Pseudo); + void emitLoweringEmitter(raw_ostream &o); +public: + PseudoLoweringEmitter(RecordKeeper &R) : Records(R), Target(R) {} + + /// run - Output the pseudo-lowerings. + void run(raw_ostream &o); +}; +} // End anonymous namespace + +// FIXME: This pass currently can only expand a pseudo to a single instruction. +// The pseudo expansion really should take a list of dags, not just +// a single dag, so we can do fancier things. + +unsigned PseudoLoweringEmitter:: +addDagOperandMapping(Record *Rec, DagInit *Dag, CodeGenInstruction &Insn, + IndexedMap<OpData> &OperandMap, unsigned BaseIdx) { + unsigned OpsAdded = 0; + for (unsigned i = 0, e = Dag->getNumArgs(); i != e; ++i) { + if (DefInit *DI = dyn_cast<DefInit>(Dag->getArg(i))) { + // Physical register reference. Explicit check for the special case + // "zero_reg" definition. + if (DI->getDef()->isSubClassOf("Register") || + DI->getDef()->getName() == "zero_reg") { + OperandMap[BaseIdx + i].Kind = OpData::Reg; + OperandMap[BaseIdx + i].Data.Reg = DI->getDef(); + ++OpsAdded; + continue; + } + + // Normal operands should always have the same type, or we have a + // problem. + // FIXME: We probably shouldn't ever get a non-zero BaseIdx here. + assert(BaseIdx == 0 && "Named subargument in pseudo expansion?!"); + if (DI->getDef() != Insn.Operands[BaseIdx + i].Rec) + PrintFatalError(Rec->getLoc(), + "Pseudo operand type '" + DI->getDef()->getName() + + "' does not match expansion operand type '" + + Insn.Operands[BaseIdx + i].Rec->getName() + "'"); + // Source operand maps to destination operand. The Data element + // will be filled in later, just set the Kind for now. Do it + // for each corresponding MachineInstr operand, not just the first. + for (unsigned I = 0, E = Insn.Operands[i].MINumOperands; I != E; ++I) + OperandMap[BaseIdx + i + I].Kind = OpData::Operand; + OpsAdded += Insn.Operands[i].MINumOperands; + } else if (IntInit *II = dyn_cast<IntInit>(Dag->getArg(i))) { + OperandMap[BaseIdx + i].Kind = OpData::Imm; + OperandMap[BaseIdx + i].Data.Imm = II->getValue(); + ++OpsAdded; + } else if (DagInit *SubDag = dyn_cast<DagInit>(Dag->getArg(i))) { + // Just add the operands recursively. This is almost certainly + // a constant value for a complex operand (> 1 MI operand). + unsigned NewOps = + addDagOperandMapping(Rec, SubDag, Insn, OperandMap, BaseIdx + i); + OpsAdded += NewOps; + // Since we added more than one, we also need to adjust the base. + BaseIdx += NewOps - 1; + } else + llvm_unreachable("Unhandled pseudo-expansion argument type!"); + } + return OpsAdded; +} + +void PseudoLoweringEmitter::evaluateExpansion(Record *Rec) { + LLVM_DEBUG(dbgs() << "Pseudo definition: " << Rec->getName() << "\n"); + + // Validate that the result pattern has the corrent number and types + // of arguments for the instruction it references. + DagInit *Dag = Rec->getValueAsDag("ResultInst"); + assert(Dag && "Missing result instruction in pseudo expansion!"); + LLVM_DEBUG(dbgs() << " Result: " << *Dag << "\n"); + + DefInit *OpDef = dyn_cast<DefInit>(Dag->getOperator()); + if (!OpDef) + PrintFatalError(Rec->getLoc(), Rec->getName() + + " has unexpected operator type!"); + Record *Operator = OpDef->getDef(); + if (!Operator->isSubClassOf("Instruction")) + PrintFatalError(Rec->getLoc(), "Pseudo result '" + Operator->getName() + + "' is not an instruction!"); + + CodeGenInstruction Insn(Operator); + + if (Insn.isCodeGenOnly || Insn.isPseudo) + PrintFatalError(Rec->getLoc(), "Pseudo result '" + Operator->getName() + + "' cannot be another pseudo instruction!"); + + if (Insn.Operands.size() != Dag->getNumArgs()) + PrintFatalError(Rec->getLoc(), "Pseudo result '" + Operator->getName() + + "' operand count mismatch"); + + unsigned NumMIOperands = 0; + for (unsigned i = 0, e = Insn.Operands.size(); i != e; ++i) + NumMIOperands += Insn.Operands[i].MINumOperands; + IndexedMap<OpData> OperandMap; + OperandMap.grow(NumMIOperands); + + addDagOperandMapping(Rec, Dag, Insn, OperandMap, 0); + + // If there are more operands that weren't in the DAG, they have to + // be operands that have default values, or we have an error. Currently, + // Operands that are a subclass of OperandWithDefaultOp have default values. + + // Validate that each result pattern argument has a matching (by name) + // argument in the source instruction, in either the (outs) or (ins) list. + // Also check that the type of the arguments match. + // + // Record the mapping of the source to result arguments for use by + // the lowering emitter. + CodeGenInstruction SourceInsn(Rec); + StringMap<unsigned> SourceOperands; + for (unsigned i = 0, e = SourceInsn.Operands.size(); i != e; ++i) + SourceOperands[SourceInsn.Operands[i].Name] = i; + + LLVM_DEBUG(dbgs() << " Operand mapping:\n"); + for (unsigned i = 0, e = Insn.Operands.size(); i != e; ++i) { + // We've already handled constant values. Just map instruction operands + // here. + if (OperandMap[Insn.Operands[i].MIOperandNo].Kind != OpData::Operand) + continue; + StringMap<unsigned>::iterator SourceOp = + SourceOperands.find(Dag->getArgNameStr(i)); + if (SourceOp == SourceOperands.end()) + PrintFatalError(Rec->getLoc(), + "Pseudo output operand '" + Dag->getArgNameStr(i) + + "' has no matching source operand."); + // Map the source operand to the destination operand index for each + // MachineInstr operand. + for (unsigned I = 0, E = Insn.Operands[i].MINumOperands; I != E; ++I) + OperandMap[Insn.Operands[i].MIOperandNo + I].Data.Operand = + SourceOp->getValue(); + + LLVM_DEBUG(dbgs() << " " << SourceOp->getValue() << " ==> " << i + << "\n"); + } + + Expansions.push_back(PseudoExpansion(SourceInsn, Insn, OperandMap)); +} + +void PseudoLoweringEmitter::emitLoweringEmitter(raw_ostream &o) { + // Emit file header. + emitSourceFileHeader("Pseudo-instruction MC lowering Source Fragment", o); + + o << "bool " << Target.getName() + "AsmPrinter" << "::\n" + << "emitPseudoExpansionLowering(MCStreamer &OutStreamer,\n" + << " const MachineInstr *MI) {\n"; + + if (!Expansions.empty()) { + o << " switch (MI->getOpcode()) {\n" + << " default: return false;\n"; + for (auto &Expansion : Expansions) { + CodeGenInstruction &Source = Expansion.Source; + CodeGenInstruction &Dest = Expansion.Dest; + o << " case " << Source.Namespace << "::" + << Source.TheDef->getName() << ": {\n" + << " MCInst TmpInst;\n" + << " MCOperand MCOp;\n" + << " TmpInst.setOpcode(" << Dest.Namespace << "::" + << Dest.TheDef->getName() << ");\n"; + + // Copy the operands from the source instruction. + // FIXME: Instruction operands with defaults values (predicates and cc_out + // in ARM, for example shouldn't need explicit values in the + // expansion DAG. + unsigned MIOpNo = 0; + for (const auto &DestOperand : Dest.Operands) { + o << " // Operand: " << DestOperand.Name << "\n"; + for (unsigned i = 0, e = DestOperand.MINumOperands; i != e; ++i) { + switch (Expansion.OperandMap[MIOpNo + i].Kind) { + case OpData::Operand: + o << " lowerOperand(MI->getOperand(" + << Source.Operands[Expansion.OperandMap[MIOpNo].Data + .Operand].MIOperandNo + i + << "), MCOp);\n" + << " TmpInst.addOperand(MCOp);\n"; + break; + case OpData::Imm: + o << " TmpInst.addOperand(MCOperand::createImm(" + << Expansion.OperandMap[MIOpNo + i].Data.Imm << "));\n"; + break; + case OpData::Reg: { + Record *Reg = Expansion.OperandMap[MIOpNo + i].Data.Reg; + o << " TmpInst.addOperand(MCOperand::createReg("; + // "zero_reg" is special. + if (Reg->getName() == "zero_reg") + o << "0"; + else + o << Reg->getValueAsString("Namespace") << "::" + << Reg->getName(); + o << "));\n"; + break; + } + } + } + MIOpNo += DestOperand.MINumOperands; + } + if (Dest.Operands.isVariadic) { + MIOpNo = Source.Operands.size() + 1; + o << " // variable_ops\n"; + o << " for (unsigned i = " << MIOpNo + << ", e = MI->getNumOperands(); i != e; ++i)\n" + << " if (lowerOperand(MI->getOperand(i), MCOp))\n" + << " TmpInst.addOperand(MCOp);\n"; + } + o << " EmitToStreamer(OutStreamer, TmpInst);\n" + << " break;\n" + << " }\n"; + } + o << " }\n return true;"; + } else + o << " return false;"; + + o << "\n}\n\n"; +} + +void PseudoLoweringEmitter::run(raw_ostream &o) { + Record *ExpansionClass = Records.getClass("PseudoInstExpansion"); + Record *InstructionClass = Records.getClass("Instruction"); + assert(ExpansionClass && "PseudoInstExpansion class definition missing!"); + assert(InstructionClass && "Instruction class definition missing!"); + + std::vector<Record*> Insts; + for (const auto &D : Records.getDefs()) { + if (D.second->isSubClassOf(ExpansionClass) && + D.second->isSubClassOf(InstructionClass)) + Insts.push_back(D.second.get()); + } + + // Process the pseudo expansion definitions, validating them as we do so. + for (unsigned i = 0, e = Insts.size(); i != e; ++i) + evaluateExpansion(Insts[i]); + + // Generate expansion code to lower the pseudo to an MCInst of the real + // instruction. + emitLoweringEmitter(o); +} + +namespace llvm { + +void EmitPseudoLowering(RecordKeeper &RK, raw_ostream &OS) { + PseudoLoweringEmitter(RK).run(OS); +} + +} // End llvm namespace diff --git a/contrib/llvm-project/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp new file mode 100644 index 000000000000..f298e639bf7f --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp @@ -0,0 +1,915 @@ +//===- RISCVCompressInstEmitter.cpp - Generator for RISCV Compression -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// RISCVCompressInstEmitter implements a tablegen-driven CompressPat based +// RISCV Instruction Compression mechanism. +// +//===--------------------------------------------------------------===// +// +// RISCVCompressInstEmitter implements a tablegen-driven CompressPat Instruction +// Compression mechanism for generating RISCV compressed instructions +// (C ISA Extension) from the expanded instruction form. + +// This tablegen backend processes CompressPat declarations in a +// td file and generates all the required checks to validate the pattern +// declarations; validate the input and output operands to generate the correct +// compressed instructions. The checks include validating different types of +// operands; register operands, immediate operands, fixed register and fixed +// immediate inputs. +// +// Example: +// class CompressPat<dag input, dag output> { +// dag Input = input; +// dag Output = output; +// list<Predicate> Predicates = []; +// } +// +// let Predicates = [HasStdExtC] in { +// def : CompressPat<(ADD GPRNoX0:$rs1, GPRNoX0:$rs1, GPRNoX0:$rs2), +// (C_ADD GPRNoX0:$rs1, GPRNoX0:$rs2)>; +// } +// +// The result is an auto-generated header file +// 'RISCVGenCompressInstEmitter.inc' which exports two functions for +// compressing/uncompressing MCInst instructions, plus +// some helper functions: +// +// bool compressInst(MCInst& OutInst, const MCInst &MI, +// const MCSubtargetInfo &STI, +// MCContext &Context); +// +// bool uncompressInst(MCInst& OutInst, const MCInst &MI, +// const MCRegisterInfo &MRI, +// const MCSubtargetInfo &STI); +// +// In addition, it exports a function for checking whether +// an instruction is compressable: +// +// bool isCompressibleInst(const MachineInstr& MI, +// const RISCVSubtarget *Subtarget, +// const MCRegisterInfo &MRI, +// const MCSubtargetInfo &STI); +// +// The clients that include this auto-generated header file and +// invoke these functions can compress an instruction before emitting +// it in the target-specific ASM or ELF streamer or can uncompress +// an instruction before printing it when the expanded instruction +// format aliases is favored. + +//===----------------------------------------------------------------------===// + +#include "CodeGenInstruction.h" +#include "CodeGenTarget.h" +#include "llvm/ADT/IndexedMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <set> +#include <vector> +using namespace llvm; + +#define DEBUG_TYPE "compress-inst-emitter" + +namespace { +class RISCVCompressInstEmitter { + struct OpData { + enum MapKind { Operand, Imm, Reg }; + MapKind Kind; + union { + unsigned Operand; // Operand number mapped to. + int64_t Imm; // Integer immediate value. + Record *Reg; // Physical register. + } Data; + int TiedOpIdx = -1; // Tied operand index within the instruction. + }; + struct CompressPat { + CodeGenInstruction Source; // The source instruction definition. + CodeGenInstruction Dest; // The destination instruction to transform to. + std::vector<Record *> + PatReqFeatures; // Required target features to enable pattern. + IndexedMap<OpData> + SourceOperandMap; // Maps operands in the Source Instruction to + // the corresponding Dest instruction operand. + IndexedMap<OpData> + DestOperandMap; // Maps operands in the Dest Instruction + // to the corresponding Source instruction operand. + CompressPat(CodeGenInstruction &S, CodeGenInstruction &D, + std::vector<Record *> RF, IndexedMap<OpData> &SourceMap, + IndexedMap<OpData> &DestMap) + : Source(S), Dest(D), PatReqFeatures(RF), SourceOperandMap(SourceMap), + DestOperandMap(DestMap) {} + }; + enum EmitterType { Compress, Uncompress, CheckCompress }; + RecordKeeper &Records; + CodeGenTarget Target; + SmallVector<CompressPat, 4> CompressPatterns; + + void addDagOperandMapping(Record *Rec, DagInit *Dag, CodeGenInstruction &Inst, + IndexedMap<OpData> &OperandMap, bool IsSourceInst); + void evaluateCompressPat(Record *Compress); + void emitCompressInstEmitter(raw_ostream &o, EmitterType EType); + bool validateTypes(Record *SubType, Record *Type, bool IsSourceInst); + bool validateRegister(Record *Reg, Record *RegClass); + void createDagOperandMapping(Record *Rec, StringMap<unsigned> &SourceOperands, + StringMap<unsigned> &DestOperands, + DagInit *SourceDag, DagInit *DestDag, + IndexedMap<OpData> &SourceOperandMap); + + void createInstOperandMapping(Record *Rec, DagInit *SourceDag, + DagInit *DestDag, + IndexedMap<OpData> &SourceOperandMap, + IndexedMap<OpData> &DestOperandMap, + StringMap<unsigned> &SourceOperands, + CodeGenInstruction &DestInst); + +public: + RISCVCompressInstEmitter(RecordKeeper &R) : Records(R), Target(R) {} + + void run(raw_ostream &o); +}; +} // End anonymous namespace. + +bool RISCVCompressInstEmitter::validateRegister(Record *Reg, Record *RegClass) { + assert(Reg->isSubClassOf("Register") && "Reg record should be a Register\n"); + assert(RegClass->isSubClassOf("RegisterClass") && "RegClass record should be" + " a RegisterClass\n"); + const CodeGenRegisterClass &RC = Target.getRegisterClass(RegClass); + const CodeGenRegister *R = Target.getRegisterByName(Reg->getName().lower()); + assert((R != nullptr) && + ("Register" + Reg->getName().str() + " not defined!!\n").c_str()); + return RC.contains(R); +} + +bool RISCVCompressInstEmitter::validateTypes(Record *DagOpType, + Record *InstOpType, + bool IsSourceInst) { + if (DagOpType == InstOpType) + return true; + // Only source instruction operands are allowed to not match Input Dag + // operands. + if (!IsSourceInst) + return false; + + if (DagOpType->isSubClassOf("RegisterClass") && + InstOpType->isSubClassOf("RegisterClass")) { + const CodeGenRegisterClass &RC = Target.getRegisterClass(InstOpType); + const CodeGenRegisterClass &SubRC = Target.getRegisterClass(DagOpType); + return RC.hasSubClass(&SubRC); + } + + // At this point either or both types are not registers, reject the pattern. + if (DagOpType->isSubClassOf("RegisterClass") || + InstOpType->isSubClassOf("RegisterClass")) + return false; + + // Let further validation happen when compress()/uncompress() functions are + // invoked. + LLVM_DEBUG(dbgs() << (IsSourceInst ? "Input" : "Output") + << " Dag Operand Type: '" << DagOpType->getName() + << "' and " + << "Instruction Operand Type: '" << InstOpType->getName() + << "' can't be checked at pattern validation time!\n"); + return true; +} + +/// The patterns in the Dag contain different types of operands: +/// Register operands, e.g.: GPRC:$rs1; Fixed registers, e.g: X1; Immediate +/// operands, e.g.: simm6:$imm; Fixed immediate operands, e.g.: 0. This function +/// maps Dag operands to its corresponding instruction operands. For register +/// operands and fixed registers it expects the Dag operand type to be contained +/// in the instantiated instruction operand type. For immediate operands and +/// immediates no validation checks are enforced at pattern validation time. +void RISCVCompressInstEmitter::addDagOperandMapping( + Record *Rec, DagInit *Dag, CodeGenInstruction &Inst, + IndexedMap<OpData> &OperandMap, bool IsSourceInst) { + // TiedCount keeps track of the number of operands skipped in Inst + // operands list to get to the corresponding Dag operand. This is + // necessary because the number of operands in Inst might be greater + // than number of operands in the Dag due to how tied operands + // are represented. + unsigned TiedCount = 0; + for (unsigned i = 0, e = Inst.Operands.size(); i != e; ++i) { + int TiedOpIdx = Inst.Operands[i].getTiedRegister(); + if (-1 != TiedOpIdx) { + // Set the entry in OperandMap for the tied operand we're skipping. + OperandMap[i].Kind = OperandMap[TiedOpIdx].Kind; + OperandMap[i].Data = OperandMap[TiedOpIdx].Data; + TiedCount++; + continue; + } + if (DefInit *DI = dyn_cast<DefInit>(Dag->getArg(i - TiedCount))) { + if (DI->getDef()->isSubClassOf("Register")) { + // Check if the fixed register belongs to the Register class. + if (!validateRegister(DI->getDef(), Inst.Operands[i].Rec)) + PrintFatalError(Rec->getLoc(), + "Error in Dag '" + Dag->getAsString() + + "'Register: '" + DI->getDef()->getName() + + "' is not in register class '" + + Inst.Operands[i].Rec->getName() + "'"); + OperandMap[i].Kind = OpData::Reg; + OperandMap[i].Data.Reg = DI->getDef(); + continue; + } + // Validate that Dag operand type matches the type defined in the + // corresponding instruction. Operands in the input Dag pattern are + // allowed to be a subclass of the type specified in corresponding + // instruction operand instead of being an exact match. + if (!validateTypes(DI->getDef(), Inst.Operands[i].Rec, IsSourceInst)) + PrintFatalError(Rec->getLoc(), + "Error in Dag '" + Dag->getAsString() + "'. Operand '" + + Dag->getArgNameStr(i - TiedCount) + "' has type '" + + DI->getDef()->getName() + + "' which does not match the type '" + + Inst.Operands[i].Rec->getName() + + "' in the corresponding instruction operand!"); + + OperandMap[i].Kind = OpData::Operand; + } else if (IntInit *II = dyn_cast<IntInit>(Dag->getArg(i - TiedCount))) { + // Validate that corresponding instruction operand expects an immediate. + if (Inst.Operands[i].Rec->isSubClassOf("RegisterClass")) + PrintFatalError( + Rec->getLoc(), + ("Error in Dag '" + Dag->getAsString() + "' Found immediate: '" + + II->getAsString() + + "' but corresponding instruction operand expected a register!")); + // No pattern validation check possible for values of fixed immediate. + OperandMap[i].Kind = OpData::Imm; + OperandMap[i].Data.Imm = II->getValue(); + LLVM_DEBUG( + dbgs() << " Found immediate '" << II->getValue() << "' at " + << (IsSourceInst ? "input " : "output ") + << "Dag. No validation time check possible for values of " + "fixed immediate.\n"); + } else + llvm_unreachable("Unhandled CompressPat argument type!"); + } +} + +// Verify the Dag operand count is enough to build an instruction. +static bool verifyDagOpCount(CodeGenInstruction &Inst, DagInit *Dag, + bool IsSource) { + if (Dag->getNumArgs() == Inst.Operands.size()) + return true; + // Source instructions are non compressed instructions and don't have tied + // operands. + if (IsSource) + PrintFatalError(Inst.TheDef->getLoc(), + "Input operands for Inst '" + Inst.TheDef->getName() + + "' and input Dag operand count mismatch"); + // The Dag can't have more arguments than the Instruction. + if (Dag->getNumArgs() > Inst.Operands.size()) + PrintFatalError(Inst.TheDef->getLoc(), + "Inst '" + Inst.TheDef->getName() + + "' and Dag operand count mismatch"); + + // The Instruction might have tied operands so the Dag might have + // a fewer operand count. + unsigned RealCount = Inst.Operands.size(); + for (unsigned i = 0; i < Inst.Operands.size(); i++) + if (Inst.Operands[i].getTiedRegister() != -1) + --RealCount; + + if (Dag->getNumArgs() != RealCount) + PrintFatalError(Inst.TheDef->getLoc(), + "Inst '" + Inst.TheDef->getName() + + "' and Dag operand count mismatch"); + return true; +} + +static bool validateArgsTypes(Init *Arg1, Init *Arg2) { + DefInit *Type1 = dyn_cast<DefInit>(Arg1); + DefInit *Type2 = dyn_cast<DefInit>(Arg2); + assert(Type1 && ("Arg1 type not found\n")); + assert(Type2 && ("Arg2 type not found\n")); + return Type1->getDef() == Type2->getDef(); +} + +// Creates a mapping between the operand name in the Dag (e.g. $rs1) and +// its index in the list of Dag operands and checks that operands with the same +// name have the same types. For example in 'C_ADD $rs1, $rs2' we generate the +// mapping $rs1 --> 0, $rs2 ---> 1. If the operand appears twice in the (tied) +// same Dag we use the last occurrence for indexing. +void RISCVCompressInstEmitter::createDagOperandMapping( + Record *Rec, StringMap<unsigned> &SourceOperands, + StringMap<unsigned> &DestOperands, DagInit *SourceDag, DagInit *DestDag, + IndexedMap<OpData> &SourceOperandMap) { + for (unsigned i = 0; i < DestDag->getNumArgs(); ++i) { + // Skip fixed immediates and registers, they were handled in + // addDagOperandMapping. + if ("" == DestDag->getArgNameStr(i)) + continue; + DestOperands[DestDag->getArgNameStr(i)] = i; + } + + for (unsigned i = 0; i < SourceDag->getNumArgs(); ++i) { + // Skip fixed immediates and registers, they were handled in + // addDagOperandMapping. + if ("" == SourceDag->getArgNameStr(i)) + continue; + + StringMap<unsigned>::iterator it = + SourceOperands.find(SourceDag->getArgNameStr(i)); + if (it != SourceOperands.end()) { + // Operand sharing the same name in the Dag should be mapped as tied. + SourceOperandMap[i].TiedOpIdx = it->getValue(); + if (!validateArgsTypes(SourceDag->getArg(it->getValue()), + SourceDag->getArg(i))) + PrintFatalError(Rec->getLoc(), + "Input Operand '" + SourceDag->getArgNameStr(i) + + "' has a mismatched tied operand!\n"); + } + it = DestOperands.find(SourceDag->getArgNameStr(i)); + if (it == DestOperands.end()) + PrintFatalError(Rec->getLoc(), "Operand " + SourceDag->getArgNameStr(i) + + " defined in Input Dag but not used in" + " Output Dag!\n"); + // Input Dag operand types must match output Dag operand type. + if (!validateArgsTypes(DestDag->getArg(it->getValue()), + SourceDag->getArg(i))) + PrintFatalError(Rec->getLoc(), "Type mismatch between Input and " + "Output Dag operand '" + + SourceDag->getArgNameStr(i) + "'!"); + SourceOperands[SourceDag->getArgNameStr(i)] = i; + } +} + +/// Map operand names in the Dag to their index in both corresponding input and +/// output instructions. Validate that operands defined in the input are +/// used in the output pattern while populating the maps. +void RISCVCompressInstEmitter::createInstOperandMapping( + Record *Rec, DagInit *SourceDag, DagInit *DestDag, + IndexedMap<OpData> &SourceOperandMap, IndexedMap<OpData> &DestOperandMap, + StringMap<unsigned> &SourceOperands, CodeGenInstruction &DestInst) { + // TiedCount keeps track of the number of operands skipped in Inst + // operands list to get to the corresponding Dag operand. + unsigned TiedCount = 0; + LLVM_DEBUG(dbgs() << " Operand mapping:\n Source Dest\n"); + for (unsigned i = 0, e = DestInst.Operands.size(); i != e; ++i) { + int TiedInstOpIdx = DestInst.Operands[i].getTiedRegister(); + if (TiedInstOpIdx != -1) { + ++TiedCount; + DestOperandMap[i].Data = DestOperandMap[TiedInstOpIdx].Data; + DestOperandMap[i].Kind = DestOperandMap[TiedInstOpIdx].Kind; + if (DestOperandMap[i].Kind == OpData::Operand) + // No need to fill the SourceOperandMap here since it was mapped to + // destination operand 'TiedInstOpIdx' in a previous iteration. + LLVM_DEBUG(dbgs() << " " << DestOperandMap[i].Data.Operand + << " ====> " << i + << " Dest operand tied with operand '" + << TiedInstOpIdx << "'\n"); + continue; + } + // Skip fixed immediates and registers, they were handled in + // addDagOperandMapping. + if (DestOperandMap[i].Kind != OpData::Operand) + continue; + + unsigned DagArgIdx = i - TiedCount; + StringMap<unsigned>::iterator SourceOp = + SourceOperands.find(DestDag->getArgNameStr(DagArgIdx)); + if (SourceOp == SourceOperands.end()) + PrintFatalError(Rec->getLoc(), + "Output Dag operand '" + + DestDag->getArgNameStr(DagArgIdx) + + "' has no matching input Dag operand."); + + assert(DestDag->getArgNameStr(DagArgIdx) == + SourceDag->getArgNameStr(SourceOp->getValue()) && + "Incorrect operand mapping detected!\n"); + DestOperandMap[i].Data.Operand = SourceOp->getValue(); + SourceOperandMap[SourceOp->getValue()].Data.Operand = i; + LLVM_DEBUG(dbgs() << " " << SourceOp->getValue() << " ====> " << i + << "\n"); + } +} + +/// Validates the CompressPattern and create operand mapping. +/// These are the checks to validate a CompressPat pattern declarations. +/// Error out with message under these conditions: +/// - Dag Input opcode is an expanded instruction and Dag Output opcode is a +/// compressed instruction. +/// - Operands in Dag Input must be all used in Dag Output. +/// Register Operand type in Dag Input Type must be contained in the +/// corresponding Source Instruction type. +/// - Register Operand type in Dag Input must be the same as in Dag Ouput. +/// - Register Operand type in Dag Output must be the same as the +/// corresponding Destination Inst type. +/// - Immediate Operand type in Dag Input must be the same as in Dag Ouput. +/// - Immediate Operand type in Dag Ouput must be the same as the corresponding +/// Destination Instruction type. +/// - Fixed register must be contained in the corresponding Source Instruction +/// type. +/// - Fixed register must be contained in the corresponding Destination +/// Instruction type. Warning message printed under these conditions: +/// - Fixed immediate in Dag Input or Dag Ouput cannot be checked at this time +/// and generate warning. +/// - Immediate operand type in Dag Input differs from the corresponding Source +/// Instruction type and generate a warning. +void RISCVCompressInstEmitter::evaluateCompressPat(Record *Rec) { + // Validate input Dag operands. + DagInit *SourceDag = Rec->getValueAsDag("Input"); + assert(SourceDag && "Missing 'Input' in compress pattern!"); + LLVM_DEBUG(dbgs() << "Input: " << *SourceDag << "\n"); + + // Checking we are transforming from compressed to uncompressed instructions. + Record *Operator = SourceDag->getOperatorAsDef(Rec->getLoc()); + if (!Operator->isSubClassOf("RVInst")) + PrintFatalError(Rec->getLoc(), "Input instruction '" + Operator->getName() + + "' is not a 32 bit wide instruction!"); + CodeGenInstruction SourceInst(Operator); + verifyDagOpCount(SourceInst, SourceDag, true); + + // Validate output Dag operands. + DagInit *DestDag = Rec->getValueAsDag("Output"); + assert(DestDag && "Missing 'Output' in compress pattern!"); + LLVM_DEBUG(dbgs() << "Output: " << *DestDag << "\n"); + + Record *DestOperator = DestDag->getOperatorAsDef(Rec->getLoc()); + if (!DestOperator->isSubClassOf("RVInst16")) + PrintFatalError(Rec->getLoc(), "Output instruction '" + + DestOperator->getName() + + "' is not a 16 bit wide instruction!"); + CodeGenInstruction DestInst(DestOperator); + verifyDagOpCount(DestInst, DestDag, false); + + // Fill the mapping from the source to destination instructions. + + IndexedMap<OpData> SourceOperandMap; + SourceOperandMap.grow(SourceInst.Operands.size()); + // Create a mapping between source Dag operands and source Inst operands. + addDagOperandMapping(Rec, SourceDag, SourceInst, SourceOperandMap, + /*IsSourceInst*/ true); + + IndexedMap<OpData> DestOperandMap; + DestOperandMap.grow(DestInst.Operands.size()); + // Create a mapping between destination Dag operands and destination Inst + // operands. + addDagOperandMapping(Rec, DestDag, DestInst, DestOperandMap, + /*IsSourceInst*/ false); + + StringMap<unsigned> SourceOperands; + StringMap<unsigned> DestOperands; + createDagOperandMapping(Rec, SourceOperands, DestOperands, SourceDag, DestDag, + SourceOperandMap); + // Create operand mapping between the source and destination instructions. + createInstOperandMapping(Rec, SourceDag, DestDag, SourceOperandMap, + DestOperandMap, SourceOperands, DestInst); + + // Get the target features for the CompressPat. + std::vector<Record *> PatReqFeatures; + std::vector<Record *> RF = Rec->getValueAsListOfDefs("Predicates"); + copy_if(RF, std::back_inserter(PatReqFeatures), [](Record *R) { + return R->getValueAsBit("AssemblerMatcherPredicate"); + }); + + CompressPatterns.push_back(CompressPat(SourceInst, DestInst, PatReqFeatures, + SourceOperandMap, DestOperandMap)); +} + +static void +getReqFeatures(std::set<std::pair<bool, StringRef>> &FeaturesSet, + std::set<std::set<std::pair<bool, StringRef>>> &AnyOfFeatureSets, + const std::vector<Record *> &ReqFeatures) { + for (auto &R : ReqFeatures) { + const DagInit *D = R->getValueAsDag("AssemblerCondDag"); + std::string CombineType = D->getOperator()->getAsString(); + if (CombineType != "any_of" && CombineType != "all_of") + PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!"); + if (D->getNumArgs() == 0) + PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!"); + bool IsOr = CombineType == "any_of"; + std::set<std::pair<bool, StringRef>> AnyOfSet; + + for (auto *Arg : D->getArgs()) { + bool IsNot = false; + if (auto *NotArg = dyn_cast<DagInit>(Arg)) { + if (NotArg->getOperator()->getAsString() != "not" || + NotArg->getNumArgs() != 1) + PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!"); + Arg = NotArg->getArg(0); + IsNot = true; + } + if (!isa<DefInit>(Arg) || + !cast<DefInit>(Arg)->getDef()->isSubClassOf("SubtargetFeature")) + PrintFatalError(R->getLoc(), "Invalid AssemblerCondDag!"); + if (IsOr) + AnyOfSet.insert({IsNot, cast<DefInit>(Arg)->getDef()->getName()}); + else + FeaturesSet.insert({IsNot, cast<DefInit>(Arg)->getDef()->getName()}); + } + + if (IsOr) + AnyOfFeatureSets.insert(AnyOfSet); + } +} + +static unsigned getPredicates(DenseMap<const Record *, unsigned> &PredicateMap, + std::vector<const Record *> &Predicates, + Record *Rec, StringRef Name) { + unsigned Entry = PredicateMap[Rec]; + if (Entry) + return Entry; + + if (!Rec->isValueUnset(Name)) { + Predicates.push_back(Rec); + Entry = Predicates.size(); + PredicateMap[Rec] = Entry; + return Entry; + } + + PrintFatalError(Rec->getLoc(), "No " + Name + + " predicate on this operand at all: '" + Rec->getName().str() + "'"); + return 0; +} + +static void printPredicates(std::vector<const Record *> &Predicates, + StringRef Name, raw_ostream &o) { + for (unsigned i = 0; i < Predicates.size(); ++i) { + Init *Pred = Predicates[i]->getValueInit(Name); + if (CodeInit *SI = dyn_cast<CodeInit>(Pred)) + o << " case " << i + 1 << ": {\n" + << " // " << Predicates[i]->getName().str() << "\n" + << " " << SI->getValue() << "\n" + << " }\n"; + else + llvm_unreachable("Unexpected predicate field!"); + } +} + +static std::string mergeCondAndCode(raw_string_ostream &CondStream, + raw_string_ostream &CodeStream) { + std::string S; + raw_string_ostream CombinedStream(S); + CombinedStream.indent(4) + << "if (" + << CondStream.str().substr( + 6, CondStream.str().length() - + 10) // remove first indentation and last '&&'. + << ") {\n"; + CombinedStream << CodeStream.str(); + CombinedStream.indent(4) << " return true;\n"; + CombinedStream.indent(4) << "} // if\n"; + return CombinedStream.str(); +} + +void RISCVCompressInstEmitter::emitCompressInstEmitter(raw_ostream &o, + EmitterType EType) { + Record *AsmWriter = Target.getAsmWriter(); + if (!AsmWriter->getValueAsInt("PassSubtarget")) + PrintFatalError(AsmWriter->getLoc(), + "'PassSubtarget' is false. SubTargetInfo object is needed " + "for target features.\n"); + + std::string Namespace = std::string(Target.getName()); + + // Sort entries in CompressPatterns to handle instructions that can have more + // than one candidate for compression\uncompression, e.g ADD can be + // transformed to a C_ADD or a C_MV. When emitting 'uncompress()' function the + // source and destination are flipped and the sort key needs to change + // accordingly. + llvm::stable_sort(CompressPatterns, + [EType](const CompressPat &LHS, const CompressPat &RHS) { + if (EType == EmitterType::Compress || + EType == EmitterType::CheckCompress) + return (LHS.Source.TheDef->getName().str() < + RHS.Source.TheDef->getName().str()); + else + return (LHS.Dest.TheDef->getName().str() < + RHS.Dest.TheDef->getName().str()); + }); + + // A list of MCOperandPredicates for all operands in use, and the reverse map. + std::vector<const Record *> MCOpPredicates; + DenseMap<const Record *, unsigned> MCOpPredicateMap; + // A list of ImmLeaf Predicates for all operands in use, and the reverse map. + std::vector<const Record *> ImmLeafPredicates; + DenseMap<const Record *, unsigned> ImmLeafPredicateMap; + + std::string F; + std::string FH; + raw_string_ostream Func(F); + raw_string_ostream FuncH(FH); + bool NeedMRI = false; + + if (EType == EmitterType::Compress) + o << "\n#ifdef GEN_COMPRESS_INSTR\n" + << "#undef GEN_COMPRESS_INSTR\n\n"; + else if (EType == EmitterType::Uncompress) + o << "\n#ifdef GEN_UNCOMPRESS_INSTR\n" + << "#undef GEN_UNCOMPRESS_INSTR\n\n"; + else if (EType == EmitterType::CheckCompress) + o << "\n#ifdef GEN_CHECK_COMPRESS_INSTR\n" + << "#undef GEN_CHECK_COMPRESS_INSTR\n\n"; + + if (EType == EmitterType::Compress) { + FuncH << "static bool compressInst(MCInst& OutInst,\n"; + FuncH.indent(25) << "const MCInst &MI,\n"; + FuncH.indent(25) << "const MCSubtargetInfo &STI,\n"; + FuncH.indent(25) << "MCContext &Context) {\n"; + } else if (EType == EmitterType::Uncompress){ + FuncH << "static bool uncompressInst(MCInst& OutInst,\n"; + FuncH.indent(27) << "const MCInst &MI,\n"; + FuncH.indent(27) << "const MCRegisterInfo &MRI,\n"; + FuncH.indent(27) << "const MCSubtargetInfo &STI) {\n"; + } else if (EType == EmitterType::CheckCompress) { + FuncH << "static bool isCompressibleInst(const MachineInstr& MI,\n"; + FuncH.indent(27) << "const RISCVSubtarget *Subtarget,\n"; + FuncH.indent(27) << "const MCRegisterInfo &MRI,\n"; + FuncH.indent(27) << "const MCSubtargetInfo &STI) {\n"; + } + + if (CompressPatterns.empty()) { + o << FuncH.str(); + o.indent(2) << "return false;\n}\n"; + if (EType == EmitterType::Compress) + o << "\n#endif //GEN_COMPRESS_INSTR\n"; + else if (EType == EmitterType::Uncompress) + o << "\n#endif //GEN_UNCOMPRESS_INSTR\n\n"; + else if (EType == EmitterType::CheckCompress) + o << "\n#endif //GEN_CHECK_COMPRESS_INSTR\n\n"; + return; + } + + std::string CaseString(""); + raw_string_ostream CaseStream(CaseString); + std::string PrevOp(""); + std::string CurOp(""); + CaseStream << " switch (MI.getOpcode()) {\n"; + CaseStream << " default: return false;\n"; + + bool CompressOrCheck = + EType == EmitterType::Compress || EType == EmitterType::CheckCompress; + bool CompressOrUncompress = + EType == EmitterType::Compress || EType == EmitterType::Uncompress; + + for (auto &CompressPat : CompressPatterns) { + std::string CondString; + std::string CodeString; + raw_string_ostream CondStream(CondString); + raw_string_ostream CodeStream(CodeString); + CodeGenInstruction &Source = + CompressOrCheck ? CompressPat.Source : CompressPat.Dest; + CodeGenInstruction &Dest = + CompressOrCheck ? CompressPat.Dest : CompressPat.Source; + IndexedMap<OpData> SourceOperandMap = CompressOrCheck ? + CompressPat.SourceOperandMap : CompressPat.DestOperandMap; + IndexedMap<OpData> &DestOperandMap = CompressOrCheck ? + CompressPat.DestOperandMap : CompressPat.SourceOperandMap; + + CurOp = Source.TheDef->getName().str(); + // Check current and previous opcode to decide to continue or end a case. + if (CurOp != PrevOp) { + if (PrevOp != "") + CaseStream.indent(6) << "break;\n } // case " + PrevOp + "\n"; + CaseStream.indent(4) << "case " + Namespace + "::" + CurOp + ": {\n"; + } + + std::set<std::pair<bool, StringRef>> FeaturesSet; + std::set<std::set<std::pair<bool, StringRef>>> AnyOfFeatureSets; + // Add CompressPat required features. + getReqFeatures(FeaturesSet, AnyOfFeatureSets, CompressPat.PatReqFeatures); + + // Add Dest instruction required features. + std::vector<Record *> ReqFeatures; + std::vector<Record *> RF = Dest.TheDef->getValueAsListOfDefs("Predicates"); + copy_if(RF, std::back_inserter(ReqFeatures), [](Record *R) { + return R->getValueAsBit("AssemblerMatcherPredicate"); + }); + getReqFeatures(FeaturesSet, AnyOfFeatureSets, ReqFeatures); + + // Emit checks for all required features. + for (auto &Op : FeaturesSet) { + StringRef Not = Op.first ? "!" : ""; + CondStream.indent(6) + << Not << ("STI.getFeatureBits()[" + Namespace + "::" + Op.second + "]").str() + + " &&\n"; + } + + // Emit checks for all required feature groups. + for (auto &Set : AnyOfFeatureSets) { + CondStream.indent(6) << "("; + for (auto &Op : Set) { + bool isLast = &Op == &*Set.rbegin(); + StringRef Not = Op.first ? "!" : ""; + CondStream << Not << ("STI.getFeatureBits()[" + Namespace + "::" + Op.second + + "]").str(); + if (!isLast) + CondStream << " || "; + } + CondStream << ") &&\n"; + } + + // Start Source Inst operands validation. + unsigned OpNo = 0; + for (OpNo = 0; OpNo < Source.Operands.size(); ++OpNo) { + if (SourceOperandMap[OpNo].TiedOpIdx != -1) { + if (Source.Operands[OpNo].Rec->isSubClassOf("RegisterClass")) + CondStream.indent(6) + << "(MI.getOperand(" + << std::to_string(OpNo) + ").getReg() == MI.getOperand(" + << std::to_string(SourceOperandMap[OpNo].TiedOpIdx) + << ").getReg()) &&\n"; + else + PrintFatalError("Unexpected tied operand types!\n"); + } + // Check for fixed immediates\registers in the source instruction. + switch (SourceOperandMap[OpNo].Kind) { + case OpData::Operand: + // We don't need to do anything for source instruction operand checks. + break; + case OpData::Imm: + CondStream.indent(6) + << "(MI.getOperand(" + std::to_string(OpNo) + ").isImm()) &&\n" + + " (MI.getOperand(" + std::to_string(OpNo) + + ").getImm() == " + + std::to_string(SourceOperandMap[OpNo].Data.Imm) + ") &&\n"; + break; + case OpData::Reg: { + Record *Reg = SourceOperandMap[OpNo].Data.Reg; + CondStream.indent(6) << "(MI.getOperand(" + std::to_string(OpNo) + + ").getReg() == " + Namespace + + "::" + Reg->getName().str() + ") &&\n"; + break; + } + } + } + CodeStream.indent(6) << "// " + Dest.AsmString + "\n"; + if (CompressOrUncompress) + CodeStream.indent(6) << "OutInst.setOpcode(" + Namespace + + "::" + Dest.TheDef->getName().str() + ");\n"; + OpNo = 0; + for (const auto &DestOperand : Dest.Operands) { + CodeStream.indent(6) << "// Operand: " + DestOperand.Name + "\n"; + switch (DestOperandMap[OpNo].Kind) { + case OpData::Operand: { + unsigned OpIdx = DestOperandMap[OpNo].Data.Operand; + // Check that the operand in the Source instruction fits + // the type for the Dest instruction. + if (DestOperand.Rec->isSubClassOf("RegisterClass")) { + NeedMRI = true; + // This is a register operand. Check the register class. + // Don't check register class if this is a tied operand, it was done + // for the operand its tied to. + if (DestOperand.getTiedRegister() == -1) + CondStream.indent(6) + << "(MRI.getRegClass(" + Namespace + + "::" + DestOperand.Rec->getName().str() + + "RegClassID).contains(" + "MI.getOperand(" + + std::to_string(OpIdx) + ").getReg())) &&\n"; + + if (CompressOrUncompress) + CodeStream.indent(6) << "OutInst.addOperand(MI.getOperand(" + + std::to_string(OpIdx) + "));\n"; + } else { + // Handling immediate operands. + if (CompressOrUncompress) { + unsigned Entry = getPredicates(MCOpPredicateMap, MCOpPredicates, + DestOperand.Rec, StringRef("MCOperandPredicate")); + CondStream.indent(6) << Namespace + "ValidateMCOperand(" + + "MI.getOperand(" + std::to_string(OpIdx) + + "), STI, " + std::to_string(Entry) + + ") &&\n"; + } else { + unsigned Entry = getPredicates(ImmLeafPredicateMap, ImmLeafPredicates, + DestOperand.Rec, StringRef("ImmediateCode")); + CondStream.indent(6) << "MI.getOperand(" + std::to_string(OpIdx) + + ").isImm() && \n"; + CondStream.indent(6) << Namespace + "ValidateMachineOperand(" + + "MI.getOperand(" + std::to_string(OpIdx) + + "), Subtarget, " + std::to_string(Entry) + + ") &&\n"; + } + if (CompressOrUncompress) + CodeStream.indent(6) << "OutInst.addOperand(MI.getOperand(" + + std::to_string(OpIdx) + "));\n"; + } + break; + } + case OpData::Imm: { + if (CompressOrUncompress) { + unsigned Entry = getPredicates(MCOpPredicateMap, MCOpPredicates, + DestOperand.Rec, StringRef("MCOperandPredicate")); + CondStream.indent(6) + << Namespace + "ValidateMCOperand(" + "MCOperand::createImm(" + + std::to_string(DestOperandMap[OpNo].Data.Imm) + "), STI, " + + std::to_string(Entry) + ") &&\n"; + } else { + unsigned Entry = getPredicates(ImmLeafPredicateMap, ImmLeafPredicates, + DestOperand.Rec, StringRef("ImmediateCode")); + CondStream.indent(6) + << Namespace + "ValidateMachineOperand(" + "MachineOperand::CreateImm(" + + std::to_string(DestOperandMap[OpNo].Data.Imm) + "), SubTarget, " + + std::to_string(Entry) + ") &&\n"; + } + if (CompressOrUncompress) + CodeStream.indent(6) + << "OutInst.addOperand(MCOperand::createImm(" + + std::to_string(DestOperandMap[OpNo].Data.Imm) + "));\n"; + } break; + case OpData::Reg: { + if (CompressOrUncompress) { + // Fixed register has been validated at pattern validation time. + Record *Reg = DestOperandMap[OpNo].Data.Reg; + CodeStream.indent(6) << "OutInst.addOperand(MCOperand::createReg(" + + Namespace + "::" + Reg->getName().str() + + "));\n"; + } + } break; + } + ++OpNo; + } + if (CompressOrUncompress) + CodeStream.indent(6) << "OutInst.setLoc(MI.getLoc());\n"; + CaseStream << mergeCondAndCode(CondStream, CodeStream); + PrevOp = CurOp; + } + Func << CaseStream.str() << "\n"; + // Close brace for the last case. + Func.indent(4) << "} // case " + CurOp + "\n"; + Func.indent(2) << "} // switch\n"; + Func.indent(2) << "return false;\n}\n"; + + if (!MCOpPredicates.empty()) { + o << "static bool " << Namespace + << "ValidateMCOperand(const MCOperand &MCOp,\n" + << " const MCSubtargetInfo &STI,\n" + << " unsigned PredicateIndex) {\n" + << " switch (PredicateIndex) {\n" + << " default:\n" + << " llvm_unreachable(\"Unknown MCOperandPredicate kind\");\n" + << " break;\n"; + + printPredicates(MCOpPredicates, "MCOperandPredicate", o); + + o << " }\n" + << "}\n\n"; + } + + if (!ImmLeafPredicates.empty()) { + o << "static bool " << Namespace + << "ValidateMachineOperand(const MachineOperand &MO,\n" + << " const RISCVSubtarget *Subtarget,\n" + << " unsigned PredicateIndex) {\n" + << " int64_t Imm = MO.getImm(); \n" + << " switch (PredicateIndex) {\n" + << " default:\n" + << " llvm_unreachable(\"Unknown ImmLeaf Predicate kind\");\n" + << " break;\n"; + + printPredicates(ImmLeafPredicates, "ImmediateCode", o); + + o << " }\n" + << "}\n\n"; + } + + o << FuncH.str(); + if (NeedMRI && EType == EmitterType::Compress) + o.indent(2) << "const MCRegisterInfo &MRI = *Context.getRegisterInfo();\n"; + o << Func.str(); + + if (EType == EmitterType::Compress) + o << "\n#endif //GEN_COMPRESS_INSTR\n"; + else if (EType == EmitterType::Uncompress) + o << "\n#endif //GEN_UNCOMPRESS_INSTR\n\n"; + else if (EType == EmitterType::CheckCompress) + o << "\n#endif //GEN_CHECK_COMPRESS_INSTR\n\n"; +} + +void RISCVCompressInstEmitter::run(raw_ostream &o) { + Record *CompressClass = Records.getClass("CompressPat"); + assert(CompressClass && "Compress class definition missing!"); + std::vector<Record *> Insts; + for (const auto &D : Records.getDefs()) { + if (D.second->isSubClassOf(CompressClass)) + Insts.push_back(D.second.get()); + } + + // Process the CompressPat definitions, validating them as we do so. + for (unsigned i = 0, e = Insts.size(); i != e; ++i) + evaluateCompressPat(Insts[i]); + + // Emit file header. + emitSourceFileHeader("Compress instruction Source Fragment", o); + // Generate compressInst() function. + emitCompressInstEmitter(o, EmitterType::Compress); + // Generate uncompressInst() function. + emitCompressInstEmitter(o, EmitterType::Uncompress); + // Generate isCompressibleInst() function. + emitCompressInstEmitter(o, EmitterType::CheckCompress); +} + +namespace llvm { + +void EmitCompressInst(RecordKeeper &RK, raw_ostream &OS) { + RISCVCompressInstEmitter(RK).run(OS); +} + +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/RegisterBankEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/RegisterBankEmitter.cpp new file mode 100644 index 000000000000..586f857b1fb0 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/RegisterBankEmitter.cpp @@ -0,0 +1,336 @@ +//===- RegisterBankEmitter.cpp - Generate a Register Bank Desc. -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend is responsible for emitting a description of a target +// register bank for a code generator. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/BitVector.h" +#include "llvm/Support/Debug.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" + +#include "CodeGenHwModes.h" +#include "CodeGenRegisters.h" +#include "CodeGenTarget.h" + +#define DEBUG_TYPE "register-bank-emitter" + +using namespace llvm; + +namespace { +class RegisterBank { + + /// A vector of register classes that are included in the register bank. + typedef std::vector<const CodeGenRegisterClass *> RegisterClassesTy; + +private: + const Record &TheDef; + + /// The register classes that are covered by the register bank. + RegisterClassesTy RCs; + + /// The register class with the largest register size. + const CodeGenRegisterClass *RCWithLargestRegsSize; + +public: + RegisterBank(const Record &TheDef) + : TheDef(TheDef), RCs(), RCWithLargestRegsSize(nullptr) {} + + /// Get the human-readable name for the bank. + StringRef getName() const { return TheDef.getValueAsString("Name"); } + /// Get the name of the enumerator in the ID enumeration. + std::string getEnumeratorName() const { return (TheDef.getName() + "ID").str(); } + + /// Get the name of the array holding the register class coverage data; + std::string getCoverageArrayName() const { + return (TheDef.getName() + "CoverageData").str(); + } + + /// Get the name of the global instance variable. + StringRef getInstanceVarName() const { return TheDef.getName(); } + + const Record &getDef() const { return TheDef; } + + /// Get the register classes listed in the RegisterBank.RegisterClasses field. + std::vector<const CodeGenRegisterClass *> + getExplicitlySpecifiedRegisterClasses( + const CodeGenRegBank &RegisterClassHierarchy) const { + std::vector<const CodeGenRegisterClass *> RCs; + for (const auto *RCDef : getDef().getValueAsListOfDefs("RegisterClasses")) + RCs.push_back(RegisterClassHierarchy.getRegClass(RCDef)); + return RCs; + } + + /// Add a register class to the bank without duplicates. + void addRegisterClass(const CodeGenRegisterClass *RC) { + if (std::find_if(RCs.begin(), RCs.end(), + [&RC](const CodeGenRegisterClass *X) { + return X == RC; + }) != RCs.end()) + return; + + // FIXME? We really want the register size rather than the spill size + // since the spill size may be bigger on some targets with + // limited load/store instructions. However, we don't store the + // register size anywhere (we could sum the sizes of the subregisters + // but there may be additional bits too) and we can't derive it from + // the VT's reliably due to Untyped. + if (RCWithLargestRegsSize == nullptr) + RCWithLargestRegsSize = RC; + else if (RCWithLargestRegsSize->RSI.get(DefaultMode).SpillSize < + RC->RSI.get(DefaultMode).SpillSize) + RCWithLargestRegsSize = RC; + assert(RCWithLargestRegsSize && "RC was nullptr?"); + + RCs.emplace_back(RC); + } + + const CodeGenRegisterClass *getRCWithLargestRegsSize() const { + return RCWithLargestRegsSize; + } + + iterator_range<typename RegisterClassesTy::const_iterator> + register_classes() const { + return llvm::make_range(RCs.begin(), RCs.end()); + } +}; + +class RegisterBankEmitter { +private: + CodeGenTarget Target; + RecordKeeper &Records; + + void emitHeader(raw_ostream &OS, const StringRef TargetName, + const std::vector<RegisterBank> &Banks); + void emitBaseClassDefinition(raw_ostream &OS, const StringRef TargetName, + const std::vector<RegisterBank> &Banks); + void emitBaseClassImplementation(raw_ostream &OS, const StringRef TargetName, + std::vector<RegisterBank> &Banks); + +public: + RegisterBankEmitter(RecordKeeper &R) : Target(R), Records(R) {} + + void run(raw_ostream &OS); +}; + +} // end anonymous namespace + +/// Emit code to declare the ID enumeration and external global instance +/// variables. +void RegisterBankEmitter::emitHeader(raw_ostream &OS, + const StringRef TargetName, + const std::vector<RegisterBank> &Banks) { + // <Target>RegisterBankInfo.h + OS << "namespace llvm {\n" + << "namespace " << TargetName << " {\n" + << "enum {\n"; + for (const auto &Bank : Banks) + OS << " " << Bank.getEnumeratorName() << ",\n"; + OS << " NumRegisterBanks,\n" + << "};\n" + << "} // end namespace " << TargetName << "\n" + << "} // end namespace llvm\n"; +} + +/// Emit declarations of the <Target>GenRegisterBankInfo class. +void RegisterBankEmitter::emitBaseClassDefinition( + raw_ostream &OS, const StringRef TargetName, + const std::vector<RegisterBank> &Banks) { + OS << "private:\n" + << " static RegisterBank *RegBanks[];\n\n" + << "protected:\n" + << " " << TargetName << "GenRegisterBankInfo();\n" + << "\n"; +} + +/// Visit each register class belonging to the given register bank. +/// +/// A class belongs to the bank iff any of these apply: +/// * It is explicitly specified +/// * It is a subclass of a class that is a member. +/// * It is a class containing subregisters of the registers of a class that +/// is a member. This is known as a subreg-class. +/// +/// This function must be called for each explicitly specified register class. +/// +/// \param RC The register class to search. +/// \param Kind A debug string containing the path the visitor took to reach RC. +/// \param VisitFn The action to take for each class visited. It may be called +/// multiple times for a given class if there are multiple paths +/// to the class. +static void visitRegisterBankClasses( + const CodeGenRegBank &RegisterClassHierarchy, + const CodeGenRegisterClass *RC, const Twine Kind, + std::function<void(const CodeGenRegisterClass *, StringRef)> VisitFn, + SmallPtrSetImpl<const CodeGenRegisterClass *> &VisitedRCs) { + + // Make sure we only visit each class once to avoid infinite loops. + if (VisitedRCs.count(RC)) + return; + VisitedRCs.insert(RC); + + // Visit each explicitly named class. + VisitFn(RC, Kind.str()); + + for (const auto &PossibleSubclass : RegisterClassHierarchy.getRegClasses()) { + std::string TmpKind = + (Twine(Kind) + " (" + PossibleSubclass.getName() + ")").str(); + + // Visit each subclass of an explicitly named class. + if (RC != &PossibleSubclass && RC->hasSubClass(&PossibleSubclass)) + visitRegisterBankClasses(RegisterClassHierarchy, &PossibleSubclass, + TmpKind + " " + RC->getName() + " subclass", + VisitFn, VisitedRCs); + + // Visit each class that contains only subregisters of RC with a common + // subregister-index. + // + // More precisely, PossibleSubclass is a subreg-class iff Reg:SubIdx is in + // PossibleSubclass for all registers Reg from RC using any + // subregister-index SubReg + for (const auto &SubIdx : RegisterClassHierarchy.getSubRegIndices()) { + BitVector BV(RegisterClassHierarchy.getRegClasses().size()); + PossibleSubclass.getSuperRegClasses(&SubIdx, BV); + if (BV.test(RC->EnumValue)) { + std::string TmpKind2 = (Twine(TmpKind) + " " + RC->getName() + + " class-with-subregs: " + RC->getName()) + .str(); + VisitFn(&PossibleSubclass, TmpKind2); + } + } + } +} + +void RegisterBankEmitter::emitBaseClassImplementation( + raw_ostream &OS, StringRef TargetName, + std::vector<RegisterBank> &Banks) { + const CodeGenRegBank &RegisterClassHierarchy = Target.getRegBank(); + + OS << "namespace llvm {\n" + << "namespace " << TargetName << " {\n"; + for (const auto &Bank : Banks) { + std::vector<std::vector<const CodeGenRegisterClass *>> RCsGroupedByWord( + (RegisterClassHierarchy.getRegClasses().size() + 31) / 32); + + for (const auto &RC : Bank.register_classes()) + RCsGroupedByWord[RC->EnumValue / 32].push_back(RC); + + OS << "const uint32_t " << Bank.getCoverageArrayName() << "[] = {\n"; + unsigned LowestIdxInWord = 0; + for (const auto &RCs : RCsGroupedByWord) { + OS << " // " << LowestIdxInWord << "-" << (LowestIdxInWord + 31) << "\n"; + for (const auto &RC : RCs) { + std::string QualifiedRegClassID = + (Twine(RC->Namespace) + "::" + RC->getName() + "RegClassID").str(); + OS << " (1u << (" << QualifiedRegClassID << " - " + << LowestIdxInWord << ")) |\n"; + } + OS << " 0,\n"; + LowestIdxInWord += 32; + } + OS << "};\n"; + } + OS << "\n"; + + for (const auto &Bank : Banks) { + std::string QualifiedBankID = + (TargetName + "::" + Bank.getEnumeratorName()).str(); + const CodeGenRegisterClass &RC = *Bank.getRCWithLargestRegsSize(); + unsigned Size = RC.RSI.get(DefaultMode).SpillSize; + OS << "RegisterBank " << Bank.getInstanceVarName() << "(/* ID */ " + << QualifiedBankID << ", /* Name */ \"" << Bank.getName() + << "\", /* Size */ " << Size << ", " + << "/* CoveredRegClasses */ " << Bank.getCoverageArrayName() + << ", /* NumRegClasses */ " + << RegisterClassHierarchy.getRegClasses().size() << ");\n"; + } + OS << "} // end namespace " << TargetName << "\n" + << "\n"; + + OS << "RegisterBank *" << TargetName + << "GenRegisterBankInfo::RegBanks[] = {\n"; + for (const auto &Bank : Banks) + OS << " &" << TargetName << "::" << Bank.getInstanceVarName() << ",\n"; + OS << "};\n\n"; + + OS << TargetName << "GenRegisterBankInfo::" << TargetName + << "GenRegisterBankInfo()\n" + << " : RegisterBankInfo(RegBanks, " << TargetName + << "::NumRegisterBanks) {\n" + << " // Assert that RegBank indices match their ID's\n" + << "#ifndef NDEBUG\n" + << " unsigned Index = 0;\n" + << " for (const auto &RB : RegBanks)\n" + << " assert(Index++ == RB->getID() && \"Index != ID\");\n" + << "#endif // NDEBUG\n" + << "}\n" + << "} // end namespace llvm\n"; +} + +void RegisterBankEmitter::run(raw_ostream &OS) { + StringRef TargetName = Target.getName(); + const CodeGenRegBank &RegisterClassHierarchy = Target.getRegBank(); + + std::vector<RegisterBank> Banks; + for (const auto &V : Records.getAllDerivedDefinitions("RegisterBank")) { + SmallPtrSet<const CodeGenRegisterClass *, 8> VisitedRCs; + RegisterBank Bank(*V); + + for (const CodeGenRegisterClass *RC : + Bank.getExplicitlySpecifiedRegisterClasses(RegisterClassHierarchy)) { + visitRegisterBankClasses( + RegisterClassHierarchy, RC, "explicit", + [&Bank](const CodeGenRegisterClass *RC, StringRef Kind) { + LLVM_DEBUG(dbgs() + << "Added " << RC->getName() << "(" << Kind << ")\n"); + Bank.addRegisterClass(RC); + }, + VisitedRCs); + } + + Banks.push_back(Bank); + } + + // Warn about ambiguous MIR caused by register bank/class name clashes. + for (const auto &Class : RegisterClassHierarchy.getRegClasses()) { + for (const auto &Bank : Banks) { + if (Bank.getName().lower() == StringRef(Class.getName()).lower()) { + PrintWarning(Bank.getDef().getLoc(), "Register bank names should be " + "distinct from register classes " + "to avoid ambiguous MIR"); + PrintNote(Bank.getDef().getLoc(), "RegisterBank was declared here"); + PrintNote(Class.getDef()->getLoc(), "RegisterClass was declared here"); + } + } + } + + emitSourceFileHeader("Register Bank Source Fragments", OS); + OS << "#ifdef GET_REGBANK_DECLARATIONS\n" + << "#undef GET_REGBANK_DECLARATIONS\n"; + emitHeader(OS, TargetName, Banks); + OS << "#endif // GET_REGBANK_DECLARATIONS\n\n" + << "#ifdef GET_TARGET_REGBANK_CLASS\n" + << "#undef GET_TARGET_REGBANK_CLASS\n"; + emitBaseClassDefinition(OS, TargetName, Banks); + OS << "#endif // GET_TARGET_REGBANK_CLASS\n\n" + << "#ifdef GET_TARGET_REGBANK_IMPL\n" + << "#undef GET_TARGET_REGBANK_IMPL\n"; + emitBaseClassImplementation(OS, TargetName, Banks); + OS << "#endif // GET_TARGET_REGBANK_IMPL\n"; +} + +namespace llvm { + +void EmitRegisterBank(RecordKeeper &RK, raw_ostream &OS) { + RegisterBankEmitter(RK).run(OS); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp new file mode 100644 index 000000000000..a615587efdee --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/RegisterInfoEmitter.cpp @@ -0,0 +1,1693 @@ +//===- RegisterInfoEmitter.cpp - Generate a Register File Desc. -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend is responsible for emitting a description of a target +// register file for a code generator. It uses instances of the Register, +// RegisterAliases, and RegisterClass classes to gather this information. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenRegisters.h" +#include "CodeGenTarget.h" +#include "SequenceToOffsetTable.h" +#include "Types.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/SparseBitVector.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MachineValueType.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/SetTheory.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <deque> +#include <iterator> +#include <set> +#include <string> +#include <vector> + +using namespace llvm; + +cl::OptionCategory RegisterInfoCat("Options for -gen-register-info"); + +static cl::opt<bool> + RegisterInfoDebug("register-info-debug", cl::init(false), + cl::desc("Dump register information to help debugging"), + cl::cat(RegisterInfoCat)); + +namespace { + +class RegisterInfoEmitter { + CodeGenTarget Target; + RecordKeeper &Records; + +public: + RegisterInfoEmitter(RecordKeeper &R) : Target(R), Records(R) { + CodeGenRegBank &RegBank = Target.getRegBank(); + RegBank.computeDerivedInfo(); + } + + // runEnums - Print out enum values for all of the registers. + void runEnums(raw_ostream &o, CodeGenTarget &Target, CodeGenRegBank &Bank); + + // runMCDesc - Print out MC register descriptions. + void runMCDesc(raw_ostream &o, CodeGenTarget &Target, CodeGenRegBank &Bank); + + // runTargetHeader - Emit a header fragment for the register info emitter. + void runTargetHeader(raw_ostream &o, CodeGenTarget &Target, + CodeGenRegBank &Bank); + + // runTargetDesc - Output the target register and register file descriptions. + void runTargetDesc(raw_ostream &o, CodeGenTarget &Target, + CodeGenRegBank &Bank); + + // run - Output the register file description. + void run(raw_ostream &o); + + void debugDump(raw_ostream &OS); + +private: + void EmitRegMapping(raw_ostream &o, const std::deque<CodeGenRegister> &Regs, + bool isCtor); + void EmitRegMappingTables(raw_ostream &o, + const std::deque<CodeGenRegister> &Regs, + bool isCtor); + void EmitRegUnitPressure(raw_ostream &OS, const CodeGenRegBank &RegBank, + const std::string &ClassName); + void emitComposeSubRegIndices(raw_ostream &OS, CodeGenRegBank &RegBank, + const std::string &ClassName); + void emitComposeSubRegIndexLaneMask(raw_ostream &OS, CodeGenRegBank &RegBank, + const std::string &ClassName); +}; + +} // end anonymous namespace + +// runEnums - Print out enum values for all of the registers. +void RegisterInfoEmitter::runEnums(raw_ostream &OS, + CodeGenTarget &Target, CodeGenRegBank &Bank) { + const auto &Registers = Bank.getRegisters(); + + // Register enums are stored as uint16_t in the tables. Make sure we'll fit. + assert(Registers.size() <= 0xffff && "Too many regs to fit in tables"); + + StringRef Namespace = Registers.front().TheDef->getValueAsString("Namespace"); + + emitSourceFileHeader("Target Register Enum Values", OS); + + OS << "\n#ifdef GET_REGINFO_ENUM\n"; + OS << "#undef GET_REGINFO_ENUM\n\n"; + + OS << "namespace llvm {\n\n"; + + OS << "class MCRegisterClass;\n" + << "extern const MCRegisterClass " << Target.getName() + << "MCRegisterClasses[];\n\n"; + + if (!Namespace.empty()) + OS << "namespace " << Namespace << " {\n"; + OS << "enum {\n NoRegister,\n"; + + for (const auto &Reg : Registers) + OS << " " << Reg.getName() << " = " << Reg.EnumValue << ",\n"; + assert(Registers.size() == Registers.back().EnumValue && + "Register enum value mismatch!"); + OS << " NUM_TARGET_REGS \t// " << Registers.size()+1 << "\n"; + OS << "};\n"; + if (!Namespace.empty()) + OS << "} // end namespace " << Namespace << "\n"; + + const auto &RegisterClasses = Bank.getRegClasses(); + if (!RegisterClasses.empty()) { + + // RegisterClass enums are stored as uint16_t in the tables. + assert(RegisterClasses.size() <= 0xffff && + "Too many register classes to fit in tables"); + + OS << "\n// Register classes\n\n"; + if (!Namespace.empty()) + OS << "namespace " << Namespace << " {\n"; + OS << "enum {\n"; + for (const auto &RC : RegisterClasses) + OS << " " << RC.getName() << "RegClassID" + << " = " << RC.EnumValue << ",\n"; + OS << "\n };\n"; + if (!Namespace.empty()) + OS << "} // end namespace " << Namespace << "\n\n"; + } + + const std::vector<Record*> &RegAltNameIndices = Target.getRegAltNameIndices(); + // If the only definition is the default NoRegAltName, we don't need to + // emit anything. + if (RegAltNameIndices.size() > 1) { + OS << "\n// Register alternate name indices\n\n"; + if (!Namespace.empty()) + OS << "namespace " << Namespace << " {\n"; + OS << "enum {\n"; + for (unsigned i = 0, e = RegAltNameIndices.size(); i != e; ++i) + OS << " " << RegAltNameIndices[i]->getName() << ",\t// " << i << "\n"; + OS << " NUM_TARGET_REG_ALT_NAMES = " << RegAltNameIndices.size() << "\n"; + OS << "};\n"; + if (!Namespace.empty()) + OS << "} // end namespace " << Namespace << "\n\n"; + } + + auto &SubRegIndices = Bank.getSubRegIndices(); + if (!SubRegIndices.empty()) { + OS << "\n// Subregister indices\n\n"; + std::string Namespace = SubRegIndices.front().getNamespace(); + if (!Namespace.empty()) + OS << "namespace " << Namespace << " {\n"; + OS << "enum : uint16_t {\n NoSubRegister,\n"; + unsigned i = 0; + for (const auto &Idx : SubRegIndices) + OS << " " << Idx.getName() << ",\t// " << ++i << "\n"; + OS << " NUM_TARGET_SUBREGS\n};\n"; + if (!Namespace.empty()) + OS << "} // end namespace " << Namespace << "\n\n"; + } + + OS << "// Register pressure sets enum.\n"; + if (!Namespace.empty()) + OS << "namespace " << Namespace << " {\n"; + OS << "enum RegisterPressureSets {\n"; + unsigned NumSets = Bank.getNumRegPressureSets(); + for (unsigned i = 0; i < NumSets; ++i ) { + const RegUnitSet &RegUnits = Bank.getRegSetAt(i); + OS << " " << RegUnits.Name << " = " << i << ",\n"; + } + OS << "};\n"; + if (!Namespace.empty()) + OS << "} // end namespace " << Namespace << '\n'; + OS << '\n'; + + OS << "} // end namespace llvm\n\n"; + OS << "#endif // GET_REGINFO_ENUM\n\n"; +} + +static void printInt(raw_ostream &OS, int Val) { + OS << Val; +} + +void RegisterInfoEmitter:: +EmitRegUnitPressure(raw_ostream &OS, const CodeGenRegBank &RegBank, + const std::string &ClassName) { + unsigned NumRCs = RegBank.getRegClasses().size(); + unsigned NumSets = RegBank.getNumRegPressureSets(); + + OS << "/// Get the weight in units of pressure for this register class.\n" + << "const RegClassWeight &" << ClassName << "::\n" + << "getRegClassWeight(const TargetRegisterClass *RC) const {\n" + << " static const RegClassWeight RCWeightTable[] = {\n"; + for (const auto &RC : RegBank.getRegClasses()) { + const CodeGenRegister::Vec &Regs = RC.getMembers(); + OS << " {" << RC.getWeight(RegBank) << ", "; + if (Regs.empty() || RC.Artificial) + OS << '0'; + else { + std::vector<unsigned> RegUnits; + RC.buildRegUnitSet(RegBank, RegUnits); + OS << RegBank.getRegUnitSetWeight(RegUnits); + } + OS << "}, \t// " << RC.getName() << "\n"; + } + OS << " };\n" + << " return RCWeightTable[RC->getID()];\n" + << "}\n\n"; + + // Reasonable targets (not ARMv7) have unit weight for all units, so don't + // bother generating a table. + bool RegUnitsHaveUnitWeight = true; + for (unsigned UnitIdx = 0, UnitEnd = RegBank.getNumNativeRegUnits(); + UnitIdx < UnitEnd; ++UnitIdx) { + if (RegBank.getRegUnit(UnitIdx).Weight > 1) + RegUnitsHaveUnitWeight = false; + } + OS << "/// Get the weight in units of pressure for this register unit.\n" + << "unsigned " << ClassName << "::\n" + << "getRegUnitWeight(unsigned RegUnit) const {\n" + << " assert(RegUnit < " << RegBank.getNumNativeRegUnits() + << " && \"invalid register unit\");\n"; + if (!RegUnitsHaveUnitWeight) { + OS << " static const uint8_t RUWeightTable[] = {\n "; + for (unsigned UnitIdx = 0, UnitEnd = RegBank.getNumNativeRegUnits(); + UnitIdx < UnitEnd; ++UnitIdx) { + const RegUnit &RU = RegBank.getRegUnit(UnitIdx); + assert(RU.Weight < 256 && "RegUnit too heavy"); + OS << RU.Weight << ", "; + } + OS << "};\n" + << " return RUWeightTable[RegUnit];\n"; + } + else { + OS << " // All register units have unit weight.\n" + << " return 1;\n"; + } + OS << "}\n\n"; + + OS << "\n" + << "// Get the number of dimensions of register pressure.\n" + << "unsigned " << ClassName << "::getNumRegPressureSets() const {\n" + << " return " << NumSets << ";\n}\n\n"; + + OS << "// Get the name of this register unit pressure set.\n" + << "const char *" << ClassName << "::\n" + << "getRegPressureSetName(unsigned Idx) const {\n" + << " static const char *const PressureNameTable[] = {\n"; + unsigned MaxRegUnitWeight = 0; + for (unsigned i = 0; i < NumSets; ++i ) { + const RegUnitSet &RegUnits = RegBank.getRegSetAt(i); + MaxRegUnitWeight = std::max(MaxRegUnitWeight, RegUnits.Weight); + OS << " \"" << RegUnits.Name << "\",\n"; + } + OS << " };\n" + << " return PressureNameTable[Idx];\n" + << "}\n\n"; + + OS << "// Get the register unit pressure limit for this dimension.\n" + << "// This limit must be adjusted dynamically for reserved registers.\n" + << "unsigned " << ClassName << "::\n" + << "getRegPressureSetLimit(const MachineFunction &MF, unsigned Idx) const " + "{\n" + << " static const " << getMinimalTypeForRange(MaxRegUnitWeight, 32) + << " PressureLimitTable[] = {\n"; + for (unsigned i = 0; i < NumSets; ++i ) { + const RegUnitSet &RegUnits = RegBank.getRegSetAt(i); + OS << " " << RegUnits.Weight << ", \t// " << i << ": " + << RegUnits.Name << "\n"; + } + OS << " };\n" + << " return PressureLimitTable[Idx];\n" + << "}\n\n"; + + SequenceToOffsetTable<std::vector<int>> PSetsSeqs; + + // This table may be larger than NumRCs if some register units needed a list + // of unit sets that did not correspond to a register class. + unsigned NumRCUnitSets = RegBank.getNumRegClassPressureSetLists(); + std::vector<std::vector<int>> PSets(NumRCUnitSets); + + for (unsigned i = 0, e = NumRCUnitSets; i != e; ++i) { + ArrayRef<unsigned> PSetIDs = RegBank.getRCPressureSetIDs(i); + PSets[i].reserve(PSetIDs.size()); + for (ArrayRef<unsigned>::iterator PSetI = PSetIDs.begin(), + PSetE = PSetIDs.end(); PSetI != PSetE; ++PSetI) { + PSets[i].push_back(RegBank.getRegPressureSet(*PSetI).Order); + } + llvm::sort(PSets[i]); + PSetsSeqs.add(PSets[i]); + } + + PSetsSeqs.layout(); + + OS << "/// Table of pressure sets per register class or unit.\n" + << "static const int RCSetsTable[] = {\n"; + PSetsSeqs.emit(OS, printInt, "-1"); + OS << "};\n\n"; + + OS << "/// Get the dimensions of register pressure impacted by this " + << "register class.\n" + << "/// Returns a -1 terminated array of pressure set IDs\n" + << "const int* " << ClassName << "::\n" + << "getRegClassPressureSets(const TargetRegisterClass *RC) const {\n"; + OS << " static const " << getMinimalTypeForRange(PSetsSeqs.size() - 1, 32) + << " RCSetStartTable[] = {\n "; + for (unsigned i = 0, e = NumRCs; i != e; ++i) { + OS << PSetsSeqs.get(PSets[i]) << ","; + } + OS << "};\n" + << " return &RCSetsTable[RCSetStartTable[RC->getID()]];\n" + << "}\n\n"; + + OS << "/// Get the dimensions of register pressure impacted by this " + << "register unit.\n" + << "/// Returns a -1 terminated array of pressure set IDs\n" + << "const int* " << ClassName << "::\n" + << "getRegUnitPressureSets(unsigned RegUnit) const {\n" + << " assert(RegUnit < " << RegBank.getNumNativeRegUnits() + << " && \"invalid register unit\");\n"; + OS << " static const " << getMinimalTypeForRange(PSetsSeqs.size() - 1, 32) + << " RUSetStartTable[] = {\n "; + for (unsigned UnitIdx = 0, UnitEnd = RegBank.getNumNativeRegUnits(); + UnitIdx < UnitEnd; ++UnitIdx) { + OS << PSetsSeqs.get(PSets[RegBank.getRegUnit(UnitIdx).RegClassUnitSetsIdx]) + << ","; + } + OS << "};\n" + << " return &RCSetsTable[RUSetStartTable[RegUnit]];\n" + << "}\n\n"; +} + +using DwarfRegNumsMapPair = std::pair<Record*, std::vector<int64_t>>; +using DwarfRegNumsVecTy = std::vector<DwarfRegNumsMapPair>; + +void finalizeDwarfRegNumsKeys(DwarfRegNumsVecTy &DwarfRegNums) { + // Sort and unique to get a map-like vector. We want the last assignment to + // match previous behaviour. + std::stable_sort(DwarfRegNums.begin(), DwarfRegNums.end(), + on_first<LessRecordRegister>()); + // Warn about duplicate assignments. + const Record *LastSeenReg = nullptr; + for (const auto &X : DwarfRegNums) { + const auto &Reg = X.first; + // The only way LessRecordRegister can return equal is if they're the same + // string. Use simple equality instead. + if (LastSeenReg && Reg->getName() == LastSeenReg->getName()) + PrintWarning(Reg->getLoc(), Twine("DWARF numbers for register ") + + getQualifiedName(Reg) + + "specified multiple times"); + LastSeenReg = Reg; + } + auto Last = std::unique( + DwarfRegNums.begin(), DwarfRegNums.end(), + [](const DwarfRegNumsMapPair &A, const DwarfRegNumsMapPair &B) { + return A.first->getName() == B.first->getName(); + }); + DwarfRegNums.erase(Last, DwarfRegNums.end()); +} + +void RegisterInfoEmitter::EmitRegMappingTables( + raw_ostream &OS, const std::deque<CodeGenRegister> &Regs, bool isCtor) { + // Collect all information about dwarf register numbers + DwarfRegNumsVecTy DwarfRegNums; + + // First, just pull all provided information to the map + unsigned maxLength = 0; + for (auto &RE : Regs) { + Record *Reg = RE.TheDef; + std::vector<int64_t> RegNums = Reg->getValueAsListOfInts("DwarfNumbers"); + maxLength = std::max((size_t)maxLength, RegNums.size()); + DwarfRegNums.emplace_back(Reg, std::move(RegNums)); + } + finalizeDwarfRegNumsKeys(DwarfRegNums); + + if (!maxLength) + return; + + // Now we know maximal length of number list. Append -1's, where needed + for (DwarfRegNumsVecTy::iterator I = DwarfRegNums.begin(), + E = DwarfRegNums.end(); + I != E; ++I) + for (unsigned i = I->second.size(), e = maxLength; i != e; ++i) + I->second.push_back(-1); + + StringRef Namespace = Regs.front().TheDef->getValueAsString("Namespace"); + + OS << "// " << Namespace << " Dwarf<->LLVM register mappings.\n"; + + // Emit reverse information about the dwarf register numbers. + for (unsigned j = 0; j < 2; ++j) { + for (unsigned i = 0, e = maxLength; i != e; ++i) { + OS << "extern const MCRegisterInfo::DwarfLLVMRegPair " << Namespace; + OS << (j == 0 ? "DwarfFlavour" : "EHFlavour"); + OS << i << "Dwarf2L[]"; + + if (!isCtor) { + OS << " = {\n"; + + // Store the mapping sorted by the LLVM reg num so lookup can be done + // with a binary search. + std::map<uint64_t, Record*> Dwarf2LMap; + for (DwarfRegNumsVecTy::iterator + I = DwarfRegNums.begin(), E = DwarfRegNums.end(); I != E; ++I) { + int DwarfRegNo = I->second[i]; + if (DwarfRegNo < 0) + continue; + Dwarf2LMap[DwarfRegNo] = I->first; + } + + for (std::map<uint64_t, Record*>::iterator + I = Dwarf2LMap.begin(), E = Dwarf2LMap.end(); I != E; ++I) + OS << " { " << I->first << "U, " << getQualifiedName(I->second) + << " },\n"; + + OS << "};\n"; + } else { + OS << ";\n"; + } + + // We have to store the size in a const global, it's used in multiple + // places. + OS << "extern const unsigned " << Namespace + << (j == 0 ? "DwarfFlavour" : "EHFlavour") << i << "Dwarf2LSize"; + if (!isCtor) + OS << " = array_lengthof(" << Namespace + << (j == 0 ? "DwarfFlavour" : "EHFlavour") << i + << "Dwarf2L);\n\n"; + else + OS << ";\n\n"; + } + } + + for (auto &RE : Regs) { + Record *Reg = RE.TheDef; + const RecordVal *V = Reg->getValue("DwarfAlias"); + if (!V || !V->getValue()) + continue; + + DefInit *DI = cast<DefInit>(V->getValue()); + Record *Alias = DI->getDef(); + const auto &AliasIter = + std::lower_bound(DwarfRegNums.begin(), DwarfRegNums.end(), Alias, + [](const DwarfRegNumsMapPair &A, const Record *B) { + return LessRecordRegister()(A.first, B); + }); + assert(AliasIter != DwarfRegNums.end() && AliasIter->first == Alias && + "Expected Alias to be present in map"); + const auto &RegIter = + std::lower_bound(DwarfRegNums.begin(), DwarfRegNums.end(), Reg, + [](const DwarfRegNumsMapPair &A, const Record *B) { + return LessRecordRegister()(A.first, B); + }); + assert(RegIter != DwarfRegNums.end() && RegIter->first == Reg && + "Expected Reg to be present in map"); + RegIter->second = AliasIter->second; + } + + // Emit information about the dwarf register numbers. + for (unsigned j = 0; j < 2; ++j) { + for (unsigned i = 0, e = maxLength; i != e; ++i) { + OS << "extern const MCRegisterInfo::DwarfLLVMRegPair " << Namespace; + OS << (j == 0 ? "DwarfFlavour" : "EHFlavour"); + OS << i << "L2Dwarf[]"; + if (!isCtor) { + OS << " = {\n"; + // Store the mapping sorted by the Dwarf reg num so lookup can be done + // with a binary search. + for (DwarfRegNumsVecTy::iterator + I = DwarfRegNums.begin(), E = DwarfRegNums.end(); I != E; ++I) { + int RegNo = I->second[i]; + if (RegNo == -1) // -1 is the default value, don't emit a mapping. + continue; + + OS << " { " << getQualifiedName(I->first) << ", " << RegNo + << "U },\n"; + } + OS << "};\n"; + } else { + OS << ";\n"; + } + + // We have to store the size in a const global, it's used in multiple + // places. + OS << "extern const unsigned " << Namespace + << (j == 0 ? "DwarfFlavour" : "EHFlavour") << i << "L2DwarfSize"; + if (!isCtor) + OS << " = array_lengthof(" << Namespace + << (j == 0 ? "DwarfFlavour" : "EHFlavour") << i << "L2Dwarf);\n\n"; + else + OS << ";\n\n"; + } + } +} + +void RegisterInfoEmitter::EmitRegMapping( + raw_ostream &OS, const std::deque<CodeGenRegister> &Regs, bool isCtor) { + // Emit the initializer so the tables from EmitRegMappingTables get wired up + // to the MCRegisterInfo object. + unsigned maxLength = 0; + for (auto &RE : Regs) { + Record *Reg = RE.TheDef; + maxLength = std::max((size_t)maxLength, + Reg->getValueAsListOfInts("DwarfNumbers").size()); + } + + if (!maxLength) + return; + + StringRef Namespace = Regs.front().TheDef->getValueAsString("Namespace"); + + // Emit reverse information about the dwarf register numbers. + for (unsigned j = 0; j < 2; ++j) { + OS << " switch ("; + if (j == 0) + OS << "DwarfFlavour"; + else + OS << "EHFlavour"; + OS << ") {\n" + << " default:\n" + << " llvm_unreachable(\"Unknown DWARF flavour\");\n"; + + for (unsigned i = 0, e = maxLength; i != e; ++i) { + OS << " case " << i << ":\n"; + OS << " "; + if (!isCtor) + OS << "RI->"; + std::string Tmp; + raw_string_ostream(Tmp) << Namespace + << (j == 0 ? "DwarfFlavour" : "EHFlavour") << i + << "Dwarf2L"; + OS << "mapDwarfRegsToLLVMRegs(" << Tmp << ", " << Tmp << "Size, "; + if (j == 0) + OS << "false"; + else + OS << "true"; + OS << ");\n"; + OS << " break;\n"; + } + OS << " }\n"; + } + + // Emit information about the dwarf register numbers. + for (unsigned j = 0; j < 2; ++j) { + OS << " switch ("; + if (j == 0) + OS << "DwarfFlavour"; + else + OS << "EHFlavour"; + OS << ") {\n" + << " default:\n" + << " llvm_unreachable(\"Unknown DWARF flavour\");\n"; + + for (unsigned i = 0, e = maxLength; i != e; ++i) { + OS << " case " << i << ":\n"; + OS << " "; + if (!isCtor) + OS << "RI->"; + std::string Tmp; + raw_string_ostream(Tmp) << Namespace + << (j == 0 ? "DwarfFlavour" : "EHFlavour") << i + << "L2Dwarf"; + OS << "mapLLVMRegsToDwarfRegs(" << Tmp << ", " << Tmp << "Size, "; + if (j == 0) + OS << "false"; + else + OS << "true"; + OS << ");\n"; + OS << " break;\n"; + } + OS << " }\n"; + } +} + +// Print a BitVector as a sequence of hex numbers using a little-endian mapping. +// Width is the number of bits per hex number. +static void printBitVectorAsHex(raw_ostream &OS, + const BitVector &Bits, + unsigned Width) { + assert(Width <= 32 && "Width too large"); + unsigned Digits = (Width + 3) / 4; + for (unsigned i = 0, e = Bits.size(); i < e; i += Width) { + unsigned Value = 0; + for (unsigned j = 0; j != Width && i + j != e; ++j) + Value |= Bits.test(i + j) << j; + OS << format("0x%0*x, ", Digits, Value); + } +} + +// Helper to emit a set of bits into a constant byte array. +class BitVectorEmitter { + BitVector Values; +public: + void add(unsigned v) { + if (v >= Values.size()) + Values.resize(((v/8)+1)*8); // Round up to the next byte. + Values[v] = true; + } + + void print(raw_ostream &OS) { + printBitVectorAsHex(OS, Values, 8); + } +}; + +static void printSimpleValueType(raw_ostream &OS, MVT::SimpleValueType VT) { + OS << getEnumName(VT); +} + +static void printSubRegIndex(raw_ostream &OS, const CodeGenSubRegIndex *Idx) { + OS << Idx->EnumValue; +} + +// Differentially encoded register and regunit lists allow for better +// compression on regular register banks. The sequence is computed from the +// differential list as: +// +// out[0] = InitVal; +// out[n+1] = out[n] + diff[n]; // n = 0, 1, ... +// +// The initial value depends on the specific list. The list is terminated by a +// 0 differential which means we can't encode repeated elements. + +typedef SmallVector<uint16_t, 4> DiffVec; +typedef SmallVector<LaneBitmask, 4> MaskVec; + +// Differentially encode a sequence of numbers into V. The starting value and +// terminating 0 are not added to V, so it will have the same size as List. +static +DiffVec &diffEncode(DiffVec &V, unsigned InitVal, SparseBitVector<> List) { + assert(V.empty() && "Clear DiffVec before diffEncode."); + uint16_t Val = uint16_t(InitVal); + + for (uint16_t Cur : List) { + V.push_back(Cur - Val); + Val = Cur; + } + return V; +} + +template<typename Iter> +static +DiffVec &diffEncode(DiffVec &V, unsigned InitVal, Iter Begin, Iter End) { + assert(V.empty() && "Clear DiffVec before diffEncode."); + uint16_t Val = uint16_t(InitVal); + for (Iter I = Begin; I != End; ++I) { + uint16_t Cur = (*I)->EnumValue; + V.push_back(Cur - Val); + Val = Cur; + } + return V; +} + +static void printDiff16(raw_ostream &OS, uint16_t Val) { + OS << Val; +} + +static void printMask(raw_ostream &OS, LaneBitmask Val) { + OS << "LaneBitmask(0x" << PrintLaneMask(Val) << ')'; +} + +// Try to combine Idx's compose map into Vec if it is compatible. +// Return false if it's not possible. +static bool combine(const CodeGenSubRegIndex *Idx, + SmallVectorImpl<CodeGenSubRegIndex*> &Vec) { + const CodeGenSubRegIndex::CompMap &Map = Idx->getComposites(); + for (const auto &I : Map) { + CodeGenSubRegIndex *&Entry = Vec[I.first->EnumValue - 1]; + if (Entry && Entry != I.second) + return false; + } + + // All entries are compatible. Make it so. + for (const auto &I : Map) { + auto *&Entry = Vec[I.first->EnumValue - 1]; + assert((!Entry || Entry == I.second) && + "Expected EnumValue to be unique"); + Entry = I.second; + } + return true; +} + +void +RegisterInfoEmitter::emitComposeSubRegIndices(raw_ostream &OS, + CodeGenRegBank &RegBank, + const std::string &ClName) { + const auto &SubRegIndices = RegBank.getSubRegIndices(); + OS << "unsigned " << ClName + << "::composeSubRegIndicesImpl(unsigned IdxA, unsigned IdxB) const {\n"; + + // Many sub-register indexes are composition-compatible, meaning that + // + // compose(IdxA, IdxB) == compose(IdxA', IdxB) + // + // for many IdxA, IdxA' pairs. Not all sub-register indexes can be composed. + // The illegal entries can be use as wildcards to compress the table further. + + // Map each Sub-register index to a compatible table row. + SmallVector<unsigned, 4> RowMap; + SmallVector<SmallVector<CodeGenSubRegIndex*, 4>, 4> Rows; + + auto SubRegIndicesSize = + std::distance(SubRegIndices.begin(), SubRegIndices.end()); + for (const auto &Idx : SubRegIndices) { + unsigned Found = ~0u; + for (unsigned r = 0, re = Rows.size(); r != re; ++r) { + if (combine(&Idx, Rows[r])) { + Found = r; + break; + } + } + if (Found == ~0u) { + Found = Rows.size(); + Rows.resize(Found + 1); + Rows.back().resize(SubRegIndicesSize); + combine(&Idx, Rows.back()); + } + RowMap.push_back(Found); + } + + // Output the row map if there is multiple rows. + if (Rows.size() > 1) { + OS << " static const " << getMinimalTypeForRange(Rows.size(), 32) + << " RowMap[" << SubRegIndicesSize << "] = {\n "; + for (unsigned i = 0, e = SubRegIndicesSize; i != e; ++i) + OS << RowMap[i] << ", "; + OS << "\n };\n"; + } + + // Output the rows. + OS << " static const " << getMinimalTypeForRange(SubRegIndicesSize + 1, 32) + << " Rows[" << Rows.size() << "][" << SubRegIndicesSize << "] = {\n"; + for (unsigned r = 0, re = Rows.size(); r != re; ++r) { + OS << " { "; + for (unsigned i = 0, e = SubRegIndicesSize; i != e; ++i) + if (Rows[r][i]) + OS << Rows[r][i]->getQualifiedName() << ", "; + else + OS << "0, "; + OS << "},\n"; + } + OS << " };\n\n"; + + OS << " --IdxA; assert(IdxA < " << SubRegIndicesSize << ");\n" + << " --IdxB; assert(IdxB < " << SubRegIndicesSize << ");\n"; + if (Rows.size() > 1) + OS << " return Rows[RowMap[IdxA]][IdxB];\n"; + else + OS << " return Rows[0][IdxB];\n"; + OS << "}\n\n"; +} + +void +RegisterInfoEmitter::emitComposeSubRegIndexLaneMask(raw_ostream &OS, + CodeGenRegBank &RegBank, + const std::string &ClName) { + // See the comments in computeSubRegLaneMasks() for our goal here. + const auto &SubRegIndices = RegBank.getSubRegIndices(); + + // Create a list of Mask+Rotate operations, with equivalent entries merged. + SmallVector<unsigned, 4> SubReg2SequenceIndexMap; + SmallVector<SmallVector<MaskRolPair, 1>, 4> Sequences; + for (const auto &Idx : SubRegIndices) { + const SmallVector<MaskRolPair, 1> &IdxSequence + = Idx.CompositionLaneMaskTransform; + + unsigned Found = ~0u; + unsigned SIdx = 0; + unsigned NextSIdx; + for (size_t s = 0, se = Sequences.size(); s != se; ++s, SIdx = NextSIdx) { + SmallVectorImpl<MaskRolPair> &Sequence = Sequences[s]; + NextSIdx = SIdx + Sequence.size() + 1; + if (Sequence == IdxSequence) { + Found = SIdx; + break; + } + } + if (Found == ~0u) { + Sequences.push_back(IdxSequence); + Found = SIdx; + } + SubReg2SequenceIndexMap.push_back(Found); + } + + OS << " struct MaskRolOp {\n" + " LaneBitmask Mask;\n" + " uint8_t RotateLeft;\n" + " };\n" + " static const MaskRolOp LaneMaskComposeSequences[] = {\n"; + unsigned Idx = 0; + for (size_t s = 0, se = Sequences.size(); s != se; ++s) { + OS << " "; + const SmallVectorImpl<MaskRolPair> &Sequence = Sequences[s]; + for (size_t p = 0, pe = Sequence.size(); p != pe; ++p) { + const MaskRolPair &P = Sequence[p]; + printMask(OS << "{ ", P.Mask); + OS << format(", %2u }, ", P.RotateLeft); + } + OS << "{ LaneBitmask::getNone(), 0 }"; + if (s+1 != se) + OS << ", "; + OS << " // Sequence " << Idx << "\n"; + Idx += Sequence.size() + 1; + } + OS << " };\n" + " static const MaskRolOp *const CompositeSequences[] = {\n"; + for (size_t i = 0, e = SubRegIndices.size(); i != e; ++i) { + OS << " "; + unsigned Idx = SubReg2SequenceIndexMap[i]; + OS << format("&LaneMaskComposeSequences[%u]", Idx); + if (i+1 != e) + OS << ","; + OS << " // to " << SubRegIndices[i].getName() << "\n"; + } + OS << " };\n\n"; + + OS << "LaneBitmask " << ClName + << "::composeSubRegIndexLaneMaskImpl(unsigned IdxA, LaneBitmask LaneMask)" + " const {\n" + " --IdxA; assert(IdxA < " << SubRegIndices.size() + << " && \"Subregister index out of bounds\");\n" + " LaneBitmask Result;\n" + " for (const MaskRolOp *Ops = CompositeSequences[IdxA]; Ops->Mask.any(); ++Ops) {\n" + " LaneBitmask::Type M = LaneMask.getAsInteger() & Ops->Mask.getAsInteger();\n" + " if (unsigned S = Ops->RotateLeft)\n" + " Result |= LaneBitmask((M << S) | (M >> (LaneBitmask::BitWidth - S)));\n" + " else\n" + " Result |= LaneBitmask(M);\n" + " }\n" + " return Result;\n" + "}\n\n"; + + OS << "LaneBitmask " << ClName + << "::reverseComposeSubRegIndexLaneMaskImpl(unsigned IdxA, " + " LaneBitmask LaneMask) const {\n" + " LaneMask &= getSubRegIndexLaneMask(IdxA);\n" + " --IdxA; assert(IdxA < " << SubRegIndices.size() + << " && \"Subregister index out of bounds\");\n" + " LaneBitmask Result;\n" + " for (const MaskRolOp *Ops = CompositeSequences[IdxA]; Ops->Mask.any(); ++Ops) {\n" + " LaneBitmask::Type M = LaneMask.getAsInteger();\n" + " if (unsigned S = Ops->RotateLeft)\n" + " Result |= LaneBitmask((M >> S) | (M << (LaneBitmask::BitWidth - S)));\n" + " else\n" + " Result |= LaneBitmask(M);\n" + " }\n" + " return Result;\n" + "}\n\n"; +} + +// +// runMCDesc - Print out MC register descriptions. +// +void +RegisterInfoEmitter::runMCDesc(raw_ostream &OS, CodeGenTarget &Target, + CodeGenRegBank &RegBank) { + emitSourceFileHeader("MC Register Information", OS); + + OS << "\n#ifdef GET_REGINFO_MC_DESC\n"; + OS << "#undef GET_REGINFO_MC_DESC\n\n"; + + const auto &Regs = RegBank.getRegisters(); + + auto &SubRegIndices = RegBank.getSubRegIndices(); + // The lists of sub-registers and super-registers go in the same array. That + // allows us to share suffixes. + typedef std::vector<const CodeGenRegister*> RegVec; + + // Differentially encoded lists. + SequenceToOffsetTable<DiffVec> DiffSeqs; + SmallVector<DiffVec, 4> SubRegLists(Regs.size()); + SmallVector<DiffVec, 4> SuperRegLists(Regs.size()); + SmallVector<DiffVec, 4> RegUnitLists(Regs.size()); + SmallVector<unsigned, 4> RegUnitInitScale(Regs.size()); + + // List of lane masks accompanying register unit sequences. + SequenceToOffsetTable<MaskVec> LaneMaskSeqs; + SmallVector<MaskVec, 4> RegUnitLaneMasks(Regs.size()); + + // Keep track of sub-register names as well. These are not differentially + // encoded. + typedef SmallVector<const CodeGenSubRegIndex*, 4> SubRegIdxVec; + SequenceToOffsetTable<SubRegIdxVec, deref<std::less<>>> SubRegIdxSeqs; + SmallVector<SubRegIdxVec, 4> SubRegIdxLists(Regs.size()); + + SequenceToOffsetTable<std::string> RegStrings; + + // Precompute register lists for the SequenceToOffsetTable. + unsigned i = 0; + for (auto I = Regs.begin(), E = Regs.end(); I != E; ++I, ++i) { + const auto &Reg = *I; + RegStrings.add(std::string(Reg.getName())); + + // Compute the ordered sub-register list. + SetVector<const CodeGenRegister*> SR; + Reg.addSubRegsPreOrder(SR, RegBank); + diffEncode(SubRegLists[i], Reg.EnumValue, SR.begin(), SR.end()); + DiffSeqs.add(SubRegLists[i]); + + // Compute the corresponding sub-register indexes. + SubRegIdxVec &SRIs = SubRegIdxLists[i]; + for (const CodeGenRegister *S : SR) + SRIs.push_back(Reg.getSubRegIndex(S)); + SubRegIdxSeqs.add(SRIs); + + // Super-registers are already computed. + const RegVec &SuperRegList = Reg.getSuperRegs(); + diffEncode(SuperRegLists[i], Reg.EnumValue, SuperRegList.begin(), + SuperRegList.end()); + DiffSeqs.add(SuperRegLists[i]); + + // Differentially encode the register unit list, seeded by register number. + // First compute a scale factor that allows more diff-lists to be reused: + // + // D0 -> (S0, S1) + // D1 -> (S2, S3) + // + // A scale factor of 2 allows D0 and D1 to share a diff-list. The initial + // value for the differential decoder is the register number multiplied by + // the scale. + // + // Check the neighboring registers for arithmetic progressions. + unsigned ScaleA = ~0u, ScaleB = ~0u; + SparseBitVector<> RUs = Reg.getNativeRegUnits(); + if (I != Regs.begin() && + std::prev(I)->getNativeRegUnits().count() == RUs.count()) + ScaleB = *RUs.begin() - *std::prev(I)->getNativeRegUnits().begin(); + if (std::next(I) != Regs.end() && + std::next(I)->getNativeRegUnits().count() == RUs.count()) + ScaleA = *std::next(I)->getNativeRegUnits().begin() - *RUs.begin(); + unsigned Scale = std::min(ScaleB, ScaleA); + // Default the scale to 0 if it can't be encoded in 4 bits. + if (Scale >= 16) + Scale = 0; + RegUnitInitScale[i] = Scale; + DiffSeqs.add(diffEncode(RegUnitLists[i], Scale * Reg.EnumValue, RUs)); + + const auto &RUMasks = Reg.getRegUnitLaneMasks(); + MaskVec &LaneMaskVec = RegUnitLaneMasks[i]; + assert(LaneMaskVec.empty()); + LaneMaskVec.insert(LaneMaskVec.begin(), RUMasks.begin(), RUMasks.end()); + // Terminator mask should not be used inside of the list. +#ifndef NDEBUG + for (LaneBitmask M : LaneMaskVec) { + assert(!M.all() && "terminator mask should not be part of the list"); + } +#endif + LaneMaskSeqs.add(LaneMaskVec); + } + + // Compute the final layout of the sequence table. + DiffSeqs.layout(); + LaneMaskSeqs.layout(); + SubRegIdxSeqs.layout(); + + OS << "namespace llvm {\n\n"; + + const std::string &TargetName = std::string(Target.getName()); + + // Emit the shared table of differential lists. + OS << "extern const MCPhysReg " << TargetName << "RegDiffLists[] = {\n"; + DiffSeqs.emit(OS, printDiff16); + OS << "};\n\n"; + + // Emit the shared table of regunit lane mask sequences. + OS << "extern const LaneBitmask " << TargetName << "LaneMaskLists[] = {\n"; + LaneMaskSeqs.emit(OS, printMask, "LaneBitmask::getAll()"); + OS << "};\n\n"; + + // Emit the table of sub-register indexes. + OS << "extern const uint16_t " << TargetName << "SubRegIdxLists[] = {\n"; + SubRegIdxSeqs.emit(OS, printSubRegIndex); + OS << "};\n\n"; + + // Emit the table of sub-register index sizes. + OS << "extern const MCRegisterInfo::SubRegCoveredBits " + << TargetName << "SubRegIdxRanges[] = {\n"; + OS << " { " << (uint16_t)-1 << ", " << (uint16_t)-1 << " },\n"; + for (const auto &Idx : SubRegIndices) { + OS << " { " << Idx.Offset << ", " << Idx.Size << " },\t// " + << Idx.getName() << "\n"; + } + OS << "};\n\n"; + + // Emit the string table. + RegStrings.layout(); + RegStrings.emitStringLiteralDef(OS, Twine("extern const char ") + TargetName + + "RegStrings[]"); + + OS << "extern const MCRegisterDesc " << TargetName + << "RegDesc[] = { // Descriptors\n"; + OS << " { " << RegStrings.get("") << ", 0, 0, 0, 0, 0 },\n"; + + // Emit the register descriptors now. + i = 0; + for (const auto &Reg : Regs) { + OS << " { " << RegStrings.get(std::string(Reg.getName())) << ", " + << DiffSeqs.get(SubRegLists[i]) << ", " << DiffSeqs.get(SuperRegLists[i]) + << ", " << SubRegIdxSeqs.get(SubRegIdxLists[i]) << ", " + << (DiffSeqs.get(RegUnitLists[i]) * 16 + RegUnitInitScale[i]) << ", " + << LaneMaskSeqs.get(RegUnitLaneMasks[i]) << " },\n"; + ++i; + } + OS << "};\n\n"; // End of register descriptors... + + // Emit the table of register unit roots. Each regunit has one or two root + // registers. + OS << "extern const MCPhysReg " << TargetName << "RegUnitRoots[][2] = {\n"; + for (unsigned i = 0, e = RegBank.getNumNativeRegUnits(); i != e; ++i) { + ArrayRef<const CodeGenRegister*> Roots = RegBank.getRegUnit(i).getRoots(); + assert(!Roots.empty() && "All regunits must have a root register."); + assert(Roots.size() <= 2 && "More than two roots not supported yet."); + OS << " { " << getQualifiedName(Roots.front()->TheDef); + for (unsigned r = 1; r != Roots.size(); ++r) + OS << ", " << getQualifiedName(Roots[r]->TheDef); + OS << " },\n"; + } + OS << "};\n\n"; + + const auto &RegisterClasses = RegBank.getRegClasses(); + + // Loop over all of the register classes... emitting each one. + OS << "namespace { // Register classes...\n"; + + SequenceToOffsetTable<std::string> RegClassStrings; + + // Emit the register enum value arrays for each RegisterClass + for (const auto &RC : RegisterClasses) { + ArrayRef<Record*> Order = RC.getOrder(); + + // Give the register class a legal C name if it's anonymous. + const std::string &Name = RC.getName(); + + RegClassStrings.add(Name); + + // Emit the register list now. + OS << " // " << Name << " Register Class...\n" + << " const MCPhysReg " << Name + << "[] = {\n "; + for (Record *Reg : Order) { + OS << getQualifiedName(Reg) << ", "; + } + OS << "\n };\n\n"; + + OS << " // " << Name << " Bit set.\n" + << " const uint8_t " << Name + << "Bits[] = {\n "; + BitVectorEmitter BVE; + for (Record *Reg : Order) { + BVE.add(Target.getRegBank().getReg(Reg)->EnumValue); + } + BVE.print(OS); + OS << "\n };\n\n"; + + } + OS << "} // end anonymous namespace\n\n"; + + RegClassStrings.layout(); + RegClassStrings.emitStringLiteralDef( + OS, Twine("extern const char ") + TargetName + "RegClassStrings[]"); + + OS << "extern const MCRegisterClass " << TargetName + << "MCRegisterClasses[] = {\n"; + + for (const auto &RC : RegisterClasses) { + assert(isInt<8>(RC.CopyCost) && "Copy cost too large."); + OS << " { " << RC.getName() << ", " << RC.getName() << "Bits, " + << RegClassStrings.get(RC.getName()) << ", " + << RC.getOrder().size() << ", sizeof(" << RC.getName() << "Bits), " + << RC.getQualifiedName() + "RegClassID" << ", " + << RC.CopyCost << ", " + << ( RC.Allocatable ? "true" : "false" ) << " },\n"; + } + + OS << "};\n\n"; + + EmitRegMappingTables(OS, Regs, false); + + // Emit Reg encoding table + OS << "extern const uint16_t " << TargetName; + OS << "RegEncodingTable[] = {\n"; + // Add entry for NoRegister + OS << " 0,\n"; + for (const auto &RE : Regs) { + Record *Reg = RE.TheDef; + BitsInit *BI = Reg->getValueAsBitsInit("HWEncoding"); + uint64_t Value = 0; + for (unsigned b = 0, be = BI->getNumBits(); b != be; ++b) { + if (BitInit *B = dyn_cast<BitInit>(BI->getBit(b))) + Value |= (uint64_t)B->getValue() << b; + } + OS << " " << Value << ",\n"; + } + OS << "};\n"; // End of HW encoding table + + // MCRegisterInfo initialization routine. + OS << "static inline void Init" << TargetName + << "MCRegisterInfo(MCRegisterInfo *RI, unsigned RA, " + << "unsigned DwarfFlavour = 0, unsigned EHFlavour = 0, unsigned PC = 0) " + "{\n" + << " RI->InitMCRegisterInfo(" << TargetName << "RegDesc, " + << Regs.size() + 1 << ", RA, PC, " << TargetName << "MCRegisterClasses, " + << RegisterClasses.size() << ", " << TargetName << "RegUnitRoots, " + << RegBank.getNumNativeRegUnits() << ", " << TargetName << "RegDiffLists, " + << TargetName << "LaneMaskLists, " << TargetName << "RegStrings, " + << TargetName << "RegClassStrings, " << TargetName << "SubRegIdxLists, " + << (std::distance(SubRegIndices.begin(), SubRegIndices.end()) + 1) << ",\n" + << TargetName << "SubRegIdxRanges, " << TargetName + << "RegEncodingTable);\n\n"; + + EmitRegMapping(OS, Regs, false); + + OS << "}\n\n"; + + OS << "} // end namespace llvm\n\n"; + OS << "#endif // GET_REGINFO_MC_DESC\n\n"; +} + +void +RegisterInfoEmitter::runTargetHeader(raw_ostream &OS, CodeGenTarget &Target, + CodeGenRegBank &RegBank) { + emitSourceFileHeader("Register Information Header Fragment", OS); + + OS << "\n#ifdef GET_REGINFO_HEADER\n"; + OS << "#undef GET_REGINFO_HEADER\n\n"; + + const std::string &TargetName = std::string(Target.getName()); + std::string ClassName = TargetName + "GenRegisterInfo"; + + OS << "#include \"llvm/CodeGen/TargetRegisterInfo.h\"\n\n"; + + OS << "namespace llvm {\n\n"; + + OS << "class " << TargetName << "FrameLowering;\n\n"; + + OS << "struct " << ClassName << " : public TargetRegisterInfo {\n" + << " explicit " << ClassName + << "(unsigned RA, unsigned D = 0, unsigned E = 0,\n" + << " unsigned PC = 0, unsigned HwMode = 0);\n"; + if (!RegBank.getSubRegIndices().empty()) { + OS << " unsigned composeSubRegIndicesImpl" + << "(unsigned, unsigned) const override;\n" + << " LaneBitmask composeSubRegIndexLaneMaskImpl" + << "(unsigned, LaneBitmask) const override;\n" + << " LaneBitmask reverseComposeSubRegIndexLaneMaskImpl" + << "(unsigned, LaneBitmask) const override;\n" + << " const TargetRegisterClass *getSubClassWithSubReg" + << "(const TargetRegisterClass*, unsigned) const override;\n"; + } + OS << " const RegClassWeight &getRegClassWeight(" + << "const TargetRegisterClass *RC) const override;\n" + << " unsigned getRegUnitWeight(unsigned RegUnit) const override;\n" + << " unsigned getNumRegPressureSets() const override;\n" + << " const char *getRegPressureSetName(unsigned Idx) const override;\n" + << " unsigned getRegPressureSetLimit(const MachineFunction &MF, unsigned " + "Idx) const override;\n" + << " const int *getRegClassPressureSets(" + << "const TargetRegisterClass *RC) const override;\n" + << " const int *getRegUnitPressureSets(" + << "unsigned RegUnit) const override;\n" + << " ArrayRef<const char *> getRegMaskNames() const override;\n" + << " ArrayRef<const uint32_t *> getRegMasks() const override;\n" + << " /// Devirtualized TargetFrameLowering.\n" + << " static const " << TargetName << "FrameLowering *getFrameLowering(\n" + << " const MachineFunction &MF);\n" + << "};\n\n"; + + const auto &RegisterClasses = RegBank.getRegClasses(); + + if (!RegisterClasses.empty()) { + OS << "namespace " << RegisterClasses.front().Namespace + << " { // Register classes\n"; + + for (const auto &RC : RegisterClasses) { + const std::string &Name = RC.getName(); + + // Output the extern for the instance. + OS << " extern const TargetRegisterClass " << Name << "RegClass;\n"; + } + OS << "} // end namespace " << RegisterClasses.front().Namespace << "\n\n"; + } + OS << "} // end namespace llvm\n\n"; + OS << "#endif // GET_REGINFO_HEADER\n\n"; +} + +// +// runTargetDesc - Output the target register and register file descriptions. +// +void +RegisterInfoEmitter::runTargetDesc(raw_ostream &OS, CodeGenTarget &Target, + CodeGenRegBank &RegBank){ + emitSourceFileHeader("Target Register and Register Classes Information", OS); + + OS << "\n#ifdef GET_REGINFO_TARGET_DESC\n"; + OS << "#undef GET_REGINFO_TARGET_DESC\n\n"; + + OS << "namespace llvm {\n\n"; + + // Get access to MCRegisterClass data. + OS << "extern const MCRegisterClass " << Target.getName() + << "MCRegisterClasses[];\n"; + + // Start out by emitting each of the register classes. + const auto &RegisterClasses = RegBank.getRegClasses(); + const auto &SubRegIndices = RegBank.getSubRegIndices(); + + // Collect all registers belonging to any allocatable class. + std::set<Record*> AllocatableRegs; + + // Collect allocatable registers. + for (const auto &RC : RegisterClasses) { + ArrayRef<Record*> Order = RC.getOrder(); + + if (RC.Allocatable) + AllocatableRegs.insert(Order.begin(), Order.end()); + } + + const CodeGenHwModes &CGH = Target.getHwModes(); + unsigned NumModes = CGH.getNumModeIds(); + + // Build a shared array of value types. + SequenceToOffsetTable<std::vector<MVT::SimpleValueType>> VTSeqs; + for (unsigned M = 0; M < NumModes; ++M) { + for (const auto &RC : RegisterClasses) { + std::vector<MVT::SimpleValueType> S; + for (const ValueTypeByHwMode &VVT : RC.VTs) + S.push_back(VVT.get(M).SimpleTy); + VTSeqs.add(S); + } + } + VTSeqs.layout(); + OS << "\nstatic const MVT::SimpleValueType VTLists[] = {\n"; + VTSeqs.emit(OS, printSimpleValueType, "MVT::Other"); + OS << "};\n"; + + // Emit SubRegIndex names, skipping 0. + OS << "\nstatic const char *const SubRegIndexNameTable[] = { \""; + + for (const auto &Idx : SubRegIndices) { + OS << Idx.getName(); + OS << "\", \""; + } + OS << "\" };\n\n"; + + // Emit SubRegIndex lane masks, including 0. + OS << "\nstatic const LaneBitmask SubRegIndexLaneMaskTable[] = {\n " + "LaneBitmask::getAll(),\n"; + for (const auto &Idx : SubRegIndices) { + printMask(OS << " ", Idx.LaneMask); + OS << ", // " << Idx.getName() << '\n'; + } + OS << " };\n\n"; + + OS << "\n"; + + // Now that all of the structs have been emitted, emit the instances. + if (!RegisterClasses.empty()) { + OS << "\nstatic const TargetRegisterInfo::RegClassInfo RegClassInfos[]" + << " = {\n"; + for (unsigned M = 0; M < NumModes; ++M) { + unsigned EV = 0; + OS << " // Mode = " << M << " ("; + if (M == 0) + OS << "Default"; + else + OS << CGH.getMode(M).Name; + OS << ")\n"; + for (const auto &RC : RegisterClasses) { + assert(RC.EnumValue == EV++ && "Unexpected order of register classes"); + (void)EV; + const RegSizeInfo &RI = RC.RSI.get(M); + OS << " { " << RI.RegSize << ", " << RI.SpillSize << ", " + << RI.SpillAlignment; + std::vector<MVT::SimpleValueType> VTs; + for (const ValueTypeByHwMode &VVT : RC.VTs) + VTs.push_back(VVT.get(M).SimpleTy); + OS << ", VTLists+" << VTSeqs.get(VTs) << " }, // " + << RC.getName() << '\n'; + } + } + OS << "};\n"; + + + OS << "\nstatic const TargetRegisterClass *const " + << "NullRegClasses[] = { nullptr };\n\n"; + + // Emit register class bit mask tables. The first bit mask emitted for a + // register class, RC, is the set of sub-classes, including RC itself. + // + // If RC has super-registers, also create a list of subreg indices and bit + // masks, (Idx, Mask). The bit mask has a bit for every superreg regclass, + // SuperRC, that satisfies: + // + // For all SuperReg in SuperRC: SuperReg:Idx in RC + // + // The 0-terminated list of subreg indices starts at: + // + // RC->getSuperRegIndices() = SuperRegIdxSeqs + ... + // + // The corresponding bitmasks follow the sub-class mask in memory. Each + // mask has RCMaskWords uint32_t entries. + // + // Every bit mask present in the list has at least one bit set. + + // Compress the sub-reg index lists. + typedef std::vector<const CodeGenSubRegIndex*> IdxList; + SmallVector<IdxList, 8> SuperRegIdxLists(RegisterClasses.size()); + SequenceToOffsetTable<IdxList, deref<std::less<>>> SuperRegIdxSeqs; + BitVector MaskBV(RegisterClasses.size()); + + for (const auto &RC : RegisterClasses) { + OS << "static const uint32_t " << RC.getName() + << "SubClassMask[] = {\n "; + printBitVectorAsHex(OS, RC.getSubClasses(), 32); + + // Emit super-reg class masks for any relevant SubRegIndices that can + // project into RC. + IdxList &SRIList = SuperRegIdxLists[RC.EnumValue]; + for (auto &Idx : SubRegIndices) { + MaskBV.reset(); + RC.getSuperRegClasses(&Idx, MaskBV); + if (MaskBV.none()) + continue; + SRIList.push_back(&Idx); + OS << "\n "; + printBitVectorAsHex(OS, MaskBV, 32); + OS << "// " << Idx.getName(); + } + SuperRegIdxSeqs.add(SRIList); + OS << "\n};\n\n"; + } + + OS << "static const uint16_t SuperRegIdxSeqs[] = {\n"; + SuperRegIdxSeqs.layout(); + SuperRegIdxSeqs.emit(OS, printSubRegIndex); + OS << "};\n\n"; + + // Emit NULL terminated super-class lists. + for (const auto &RC : RegisterClasses) { + ArrayRef<CodeGenRegisterClass*> Supers = RC.getSuperClasses(); + + // Skip classes without supers. We can reuse NullRegClasses. + if (Supers.empty()) + continue; + + OS << "static const TargetRegisterClass *const " + << RC.getName() << "Superclasses[] = {\n"; + for (const auto *Super : Supers) + OS << " &" << Super->getQualifiedName() << "RegClass,\n"; + OS << " nullptr\n};\n\n"; + } + + // Emit methods. + for (const auto &RC : RegisterClasses) { + if (!RC.AltOrderSelect.empty()) { + OS << "\nstatic inline unsigned " << RC.getName() + << "AltOrderSelect(const MachineFunction &MF) {" + << RC.AltOrderSelect << "}\n\n" + << "static ArrayRef<MCPhysReg> " << RC.getName() + << "GetRawAllocationOrder(const MachineFunction &MF) {\n"; + for (unsigned oi = 1 , oe = RC.getNumOrders(); oi != oe; ++oi) { + ArrayRef<Record*> Elems = RC.getOrder(oi); + if (!Elems.empty()) { + OS << " static const MCPhysReg AltOrder" << oi << "[] = {"; + for (unsigned elem = 0; elem != Elems.size(); ++elem) + OS << (elem ? ", " : " ") << getQualifiedName(Elems[elem]); + OS << " };\n"; + } + } + OS << " const MCRegisterClass &MCR = " << Target.getName() + << "MCRegisterClasses[" << RC.getQualifiedName() + "RegClassID];\n" + << " const ArrayRef<MCPhysReg> Order[] = {\n" + << " makeArrayRef(MCR.begin(), MCR.getNumRegs()"; + for (unsigned oi = 1, oe = RC.getNumOrders(); oi != oe; ++oi) + if (RC.getOrder(oi).empty()) + OS << "),\n ArrayRef<MCPhysReg>("; + else + OS << "),\n makeArrayRef(AltOrder" << oi; + OS << ")\n };\n const unsigned Select = " << RC.getName() + << "AltOrderSelect(MF);\n assert(Select < " << RC.getNumOrders() + << ");\n return Order[Select];\n}\n"; + } + } + + // Now emit the actual value-initialized register class instances. + OS << "\nnamespace " << RegisterClasses.front().Namespace + << " { // Register class instances\n"; + + for (const auto &RC : RegisterClasses) { + OS << " extern const TargetRegisterClass " << RC.getName() + << "RegClass = {\n " << '&' << Target.getName() + << "MCRegisterClasses[" << RC.getName() << "RegClassID],\n " + << RC.getName() << "SubClassMask,\n SuperRegIdxSeqs + " + << SuperRegIdxSeqs.get(SuperRegIdxLists[RC.EnumValue]) << ",\n "; + printMask(OS, RC.LaneMask); + OS << ",\n " << (unsigned)RC.AllocationPriority << ",\n " + << (RC.HasDisjunctSubRegs?"true":"false") + << ", /* HasDisjunctSubRegs */\n " + << (RC.CoveredBySubRegs?"true":"false") + << ", /* CoveredBySubRegs */\n "; + if (RC.getSuperClasses().empty()) + OS << "NullRegClasses,\n "; + else + OS << RC.getName() << "Superclasses,\n "; + if (RC.AltOrderSelect.empty()) + OS << "nullptr\n"; + else + OS << RC.getName() << "GetRawAllocationOrder\n"; + OS << " };\n\n"; + } + + OS << "} // end namespace " << RegisterClasses.front().Namespace << "\n"; + } + + OS << "\nnamespace {\n"; + OS << " const TargetRegisterClass* const RegisterClasses[] = {\n"; + for (const auto &RC : RegisterClasses) + OS << " &" << RC.getQualifiedName() << "RegClass,\n"; + OS << " };\n"; + OS << "} // end anonymous namespace\n"; + + // Emit extra information about registers. + const std::string &TargetName = std::string(Target.getName()); + OS << "\nstatic const TargetRegisterInfoDesc " + << TargetName << "RegInfoDesc[] = { // Extra Descriptors\n"; + OS << " { 0, false },\n"; + + const auto &Regs = RegBank.getRegisters(); + for (const auto &Reg : Regs) { + OS << " { "; + OS << Reg.CostPerUse << ", " + << ( AllocatableRegs.count(Reg.TheDef) != 0 ? "true" : "false" ) + << " },\n"; + } + OS << "};\n"; // End of register descriptors... + + + std::string ClassName = Target.getName().str() + "GenRegisterInfo"; + + auto SubRegIndicesSize = + std::distance(SubRegIndices.begin(), SubRegIndices.end()); + + if (!SubRegIndices.empty()) { + emitComposeSubRegIndices(OS, RegBank, ClassName); + emitComposeSubRegIndexLaneMask(OS, RegBank, ClassName); + } + + // Emit getSubClassWithSubReg. + if (!SubRegIndices.empty()) { + OS << "const TargetRegisterClass *" << ClassName + << "::getSubClassWithSubReg(const TargetRegisterClass *RC, unsigned Idx)" + << " const {\n"; + // Use the smallest type that can hold a regclass ID with room for a + // sentinel. + if (RegisterClasses.size() < UINT8_MAX) + OS << " static const uint8_t Table["; + else if (RegisterClasses.size() < UINT16_MAX) + OS << " static const uint16_t Table["; + else + PrintFatalError("Too many register classes."); + OS << RegisterClasses.size() << "][" << SubRegIndicesSize << "] = {\n"; + for (const auto &RC : RegisterClasses) { + OS << " {\t// " << RC.getName() << "\n"; + for (auto &Idx : SubRegIndices) { + if (CodeGenRegisterClass *SRC = RC.getSubClassWithSubReg(&Idx)) + OS << " " << SRC->EnumValue + 1 << ",\t// " << Idx.getName() + << " -> " << SRC->getName() << "\n"; + else + OS << " 0,\t// " << Idx.getName() << "\n"; + } + OS << " },\n"; + } + OS << " };\n assert(RC && \"Missing regclass\");\n" + << " if (!Idx) return RC;\n --Idx;\n" + << " assert(Idx < " << SubRegIndicesSize << " && \"Bad subreg\");\n" + << " unsigned TV = Table[RC->getID()][Idx];\n" + << " return TV ? getRegClass(TV - 1) : nullptr;\n}\n\n"; + } + + EmitRegUnitPressure(OS, RegBank, ClassName); + + // Emit the constructor of the class... + OS << "extern const MCRegisterDesc " << TargetName << "RegDesc[];\n"; + OS << "extern const MCPhysReg " << TargetName << "RegDiffLists[];\n"; + OS << "extern const LaneBitmask " << TargetName << "LaneMaskLists[];\n"; + OS << "extern const char " << TargetName << "RegStrings[];\n"; + OS << "extern const char " << TargetName << "RegClassStrings[];\n"; + OS << "extern const MCPhysReg " << TargetName << "RegUnitRoots[][2];\n"; + OS << "extern const uint16_t " << TargetName << "SubRegIdxLists[];\n"; + OS << "extern const MCRegisterInfo::SubRegCoveredBits " + << TargetName << "SubRegIdxRanges[];\n"; + OS << "extern const uint16_t " << TargetName << "RegEncodingTable[];\n"; + + EmitRegMappingTables(OS, Regs, true); + + OS << ClassName << "::\n" << ClassName + << "(unsigned RA, unsigned DwarfFlavour, unsigned EHFlavour,\n" + " unsigned PC, unsigned HwMode)\n" + << " : TargetRegisterInfo(" << TargetName << "RegInfoDesc" + << ", RegisterClasses, RegisterClasses+" << RegisterClasses.size() << ",\n" + << " SubRegIndexNameTable, SubRegIndexLaneMaskTable,\n" + << " "; + printMask(OS, RegBank.CoveringLanes); + OS << ", RegClassInfos, HwMode) {\n" + << " InitMCRegisterInfo(" << TargetName << "RegDesc, " << Regs.size() + 1 + << ", RA, PC,\n " << TargetName + << "MCRegisterClasses, " << RegisterClasses.size() << ",\n" + << " " << TargetName << "RegUnitRoots,\n" + << " " << RegBank.getNumNativeRegUnits() << ",\n" + << " " << TargetName << "RegDiffLists,\n" + << " " << TargetName << "LaneMaskLists,\n" + << " " << TargetName << "RegStrings,\n" + << " " << TargetName << "RegClassStrings,\n" + << " " << TargetName << "SubRegIdxLists,\n" + << " " << SubRegIndicesSize + 1 << ",\n" + << " " << TargetName << "SubRegIdxRanges,\n" + << " " << TargetName << "RegEncodingTable);\n\n"; + + EmitRegMapping(OS, Regs, true); + + OS << "}\n\n"; + + // Emit CalleeSavedRegs information. + std::vector<Record*> CSRSets = + Records.getAllDerivedDefinitions("CalleeSavedRegs"); + for (unsigned i = 0, e = CSRSets.size(); i != e; ++i) { + Record *CSRSet = CSRSets[i]; + const SetTheory::RecVec *Regs = RegBank.getSets().expand(CSRSet); + assert(Regs && "Cannot expand CalleeSavedRegs instance"); + + // Emit the *_SaveList list of callee-saved registers. + OS << "static const MCPhysReg " << CSRSet->getName() + << "_SaveList[] = { "; + for (unsigned r = 0, re = Regs->size(); r != re; ++r) + OS << getQualifiedName((*Regs)[r]) << ", "; + OS << "0 };\n"; + + // Emit the *_RegMask bit mask of call-preserved registers. + BitVector Covered = RegBank.computeCoveredRegisters(*Regs); + + // Check for an optional OtherPreserved set. + // Add those registers to RegMask, but not to SaveList. + if (DagInit *OPDag = + dyn_cast<DagInit>(CSRSet->getValueInit("OtherPreserved"))) { + SetTheory::RecSet OPSet; + RegBank.getSets().evaluate(OPDag, OPSet, CSRSet->getLoc()); + Covered |= RegBank.computeCoveredRegisters( + ArrayRef<Record*>(OPSet.begin(), OPSet.end())); + } + + OS << "static const uint32_t " << CSRSet->getName() + << "_RegMask[] = { "; + printBitVectorAsHex(OS, Covered, 32); + OS << "};\n"; + } + OS << "\n\n"; + + OS << "ArrayRef<const uint32_t *> " << ClassName + << "::getRegMasks() const {\n"; + if (!CSRSets.empty()) { + OS << " static const uint32_t *const Masks[] = {\n"; + for (Record *CSRSet : CSRSets) + OS << " " << CSRSet->getName() << "_RegMask,\n"; + OS << " };\n"; + OS << " return makeArrayRef(Masks);\n"; + } else { + OS << " return None;\n"; + } + OS << "}\n\n"; + + OS << "ArrayRef<const char *> " << ClassName + << "::getRegMaskNames() const {\n"; + if (!CSRSets.empty()) { + OS << " static const char *const Names[] = {\n"; + for (Record *CSRSet : CSRSets) + OS << " " << '"' << CSRSet->getName() << '"' << ",\n"; + OS << " };\n"; + OS << " return makeArrayRef(Names);\n"; + } else { + OS << " return None;\n"; + } + OS << "}\n\n"; + + OS << "const " << TargetName << "FrameLowering *\n" << TargetName + << "GenRegisterInfo::getFrameLowering(const MachineFunction &MF) {\n" + << " return static_cast<const " << TargetName << "FrameLowering *>(\n" + << " MF.getSubtarget().getFrameLowering());\n" + << "}\n\n"; + + OS << "} // end namespace llvm\n\n"; + OS << "#endif // GET_REGINFO_TARGET_DESC\n\n"; +} + +void RegisterInfoEmitter::run(raw_ostream &OS) { + CodeGenRegBank &RegBank = Target.getRegBank(); + runEnums(OS, Target, RegBank); + runMCDesc(OS, Target, RegBank); + runTargetHeader(OS, Target, RegBank); + runTargetDesc(OS, Target, RegBank); + + if (RegisterInfoDebug) + debugDump(errs()); +} + +void RegisterInfoEmitter::debugDump(raw_ostream &OS) { + CodeGenRegBank &RegBank = Target.getRegBank(); + const CodeGenHwModes &CGH = Target.getHwModes(); + unsigned NumModes = CGH.getNumModeIds(); + auto getModeName = [CGH] (unsigned M) -> StringRef { + if (M == 0) + return "Default"; + return CGH.getMode(M).Name; + }; + + for (const CodeGenRegisterClass &RC : RegBank.getRegClasses()) { + OS << "RegisterClass " << RC.getName() << ":\n"; + OS << "\tSpillSize: {"; + for (unsigned M = 0; M != NumModes; ++M) + OS << ' ' << getModeName(M) << ':' << RC.RSI.get(M).SpillSize; + OS << " }\n\tSpillAlignment: {"; + for (unsigned M = 0; M != NumModes; ++M) + OS << ' ' << getModeName(M) << ':' << RC.RSI.get(M).SpillAlignment; + OS << " }\n\tNumRegs: " << RC.getMembers().size() << '\n'; + OS << "\tLaneMask: " << PrintLaneMask(RC.LaneMask) << '\n'; + OS << "\tHasDisjunctSubRegs: " << RC.HasDisjunctSubRegs << '\n'; + OS << "\tCoveredBySubRegs: " << RC.CoveredBySubRegs << '\n'; + OS << "\tRegs:"; + for (const CodeGenRegister *R : RC.getMembers()) { + OS << " " << R->getName(); + } + OS << '\n'; + OS << "\tSubClasses:"; + const BitVector &SubClasses = RC.getSubClasses(); + for (const CodeGenRegisterClass &SRC : RegBank.getRegClasses()) { + if (!SubClasses.test(SRC.EnumValue)) + continue; + OS << " " << SRC.getName(); + } + OS << '\n'; + OS << "\tSuperClasses:"; + for (const CodeGenRegisterClass *SRC : RC.getSuperClasses()) { + OS << " " << SRC->getName(); + } + OS << '\n'; + } + + for (const CodeGenSubRegIndex &SRI : RegBank.getSubRegIndices()) { + OS << "SubRegIndex " << SRI.getName() << ":\n"; + OS << "\tLaneMask: " << PrintLaneMask(SRI.LaneMask) << '\n'; + OS << "\tAllSuperRegsCovered: " << SRI.AllSuperRegsCovered << '\n'; + } + + for (const CodeGenRegister &R : RegBank.getRegisters()) { + OS << "Register " << R.getName() << ":\n"; + OS << "\tCostPerUse: " << R.CostPerUse << '\n'; + OS << "\tCoveredBySubregs: " << R.CoveredBySubRegs << '\n'; + OS << "\tHasDisjunctSubRegs: " << R.HasDisjunctSubRegs << '\n'; + for (std::pair<CodeGenSubRegIndex*,CodeGenRegister*> P : R.getSubRegs()) { + OS << "\tSubReg " << P.first->getName() + << " = " << P.second->getName() << '\n'; + } + } +} + +namespace llvm { + +void EmitRegisterInfo(RecordKeeper &RK, raw_ostream &OS) { + RegisterInfoEmitter(RK).run(OS); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/SDNodeProperties.cpp b/contrib/llvm-project/llvm/utils/TableGen/SDNodeProperties.cpp new file mode 100644 index 000000000000..1843a78aa3cf --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/SDNodeProperties.cpp @@ -0,0 +1,48 @@ +//===- SDNodeProperties.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "SDNodeProperties.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" + +using namespace llvm; + +unsigned llvm::parseSDPatternOperatorProperties(Record *R) { + unsigned Properties = 0; + for (Record *Property : R->getValueAsListOfDefs("Properties")) { + if (Property->getName() == "SDNPCommutative") { + Properties |= 1 << SDNPCommutative; + } else if (Property->getName() == "SDNPAssociative") { + Properties |= 1 << SDNPAssociative; + } else if (Property->getName() == "SDNPHasChain") { + Properties |= 1 << SDNPHasChain; + } else if (Property->getName() == "SDNPOutGlue") { + Properties |= 1 << SDNPOutGlue; + } else if (Property->getName() == "SDNPInGlue") { + Properties |= 1 << SDNPInGlue; + } else if (Property->getName() == "SDNPOptInGlue") { + Properties |= 1 << SDNPOptInGlue; + } else if (Property->getName() == "SDNPMayStore") { + Properties |= 1 << SDNPMayStore; + } else if (Property->getName() == "SDNPMayLoad") { + Properties |= 1 << SDNPMayLoad; + } else if (Property->getName() == "SDNPSideEffect") { + Properties |= 1 << SDNPSideEffect; + } else if (Property->getName() == "SDNPMemOperand") { + Properties |= 1 << SDNPMemOperand; + } else if (Property->getName() == "SDNPVariadic") { + Properties |= 1 << SDNPVariadic; + } else { + PrintFatalError(R->getLoc(), "Unknown SD Node property '" + + Property->getName() + "' on node '" + + R->getName() + "'!"); + } + } + + return Properties; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/SDNodeProperties.h b/contrib/llvm-project/llvm/utils/TableGen/SDNodeProperties.h new file mode 100644 index 000000000000..66a04e63150c --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/SDNodeProperties.h @@ -0,0 +1,39 @@ +//===- SDNodeProperties.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_SDNODEPROPERTIES_H +#define LLVM_UTILS_TABLEGEN_SDNODEPROPERTIES_H + +namespace llvm { + +class Record; + +// SelectionDAG node properties. +// SDNPMemOperand: indicates that a node touches memory and therefore must +// have an associated memory operand that describes the access. +enum SDNP { + SDNPCommutative, + SDNPAssociative, + SDNPHasChain, + SDNPOutGlue, + SDNPInGlue, + SDNPOptInGlue, + SDNPMayLoad, + SDNPMayStore, + SDNPSideEffect, + SDNPMemOperand, + SDNPVariadic, + SDNPWantRoot, + SDNPWantParent +}; + +unsigned parseSDPatternOperatorProperties(Record *R); + +} + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/SearchableTableEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/SearchableTableEmitter.cpp new file mode 100644 index 000000000000..326cb4e54edc --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/SearchableTableEmitter.cpp @@ -0,0 +1,806 @@ +//===- SearchableTableEmitter.cpp - Generate efficiently searchable tables -==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend emits a generic array initialized by specified fields, +// together with companion index tables and lookup functions (binary search, +// currently). +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "CodeGenIntrinsics.h" +#include <algorithm> +#include <set> +#include <string> +#include <vector> + +using namespace llvm; + +#define DEBUG_TYPE "searchable-table-emitter" + +namespace { + +struct GenericTable; + +int getAsInt(Init *B) { + return cast<IntInit>(B->convertInitializerTo(IntRecTy::get()))->getValue(); +} +int getInt(Record *R, StringRef Field) { + return getAsInt(R->getValueInit(Field)); +} + +struct GenericEnum { + using Entry = std::pair<StringRef, int64_t>; + + std::string Name; + Record *Class = nullptr; + std::string PreprocessorGuard; + std::vector<std::unique_ptr<Entry>> Entries; + DenseMap<Record *, Entry *> EntryMap; +}; + +struct GenericField { + std::string Name; + RecTy *RecType = nullptr; + bool IsIntrinsic = false; + bool IsInstruction = false; + GenericEnum *Enum = nullptr; + + GenericField(StringRef Name) : Name(std::string(Name)) {} +}; + +struct SearchIndex { + std::string Name; + SmallVector<GenericField, 1> Fields; + bool EarlyOut = false; +}; + +struct GenericTable { + std::string Name; + std::string PreprocessorGuard; + std::string CppTypeName; + SmallVector<GenericField, 2> Fields; + std::vector<Record *> Entries; + + std::unique_ptr<SearchIndex> PrimaryKey; + SmallVector<std::unique_ptr<SearchIndex>, 2> Indices; + + const GenericField *getFieldByName(StringRef Name) const { + for (const auto &Field : Fields) { + if (Name == Field.Name) + return &Field; + } + return nullptr; + } +}; + +class SearchableTableEmitter { + RecordKeeper &Records; + DenseMap<Init *, std::unique_ptr<CodeGenIntrinsic>> Intrinsics; + std::vector<std::unique_ptr<GenericEnum>> Enums; + DenseMap<Record *, GenericEnum *> EnumMap; + std::set<std::string> PreprocessorGuards; + +public: + SearchableTableEmitter(RecordKeeper &R) : Records(R) {} + + void run(raw_ostream &OS); + +private: + typedef std::pair<Init *, int> SearchTableEntry; + + enum TypeContext { + TypeInStaticStruct, + TypeInTempStruct, + TypeInArgument, + }; + + std::string primaryRepresentation(const GenericField &Field, Init *I) { + if (StringInit *SI = dyn_cast<StringInit>(I)) + return SI->getAsString(); + else if (BitsInit *BI = dyn_cast<BitsInit>(I)) + return "0x" + utohexstr(getAsInt(BI)); + else if (BitInit *BI = dyn_cast<BitInit>(I)) + return BI->getValue() ? "true" : "false"; + else if (CodeInit *CI = dyn_cast<CodeInit>(I)) + return std::string(CI->getValue()); + else if (Field.IsIntrinsic) + return "Intrinsic::" + getIntrinsic(I).EnumName; + else if (Field.IsInstruction) + return I->getAsString(); + else if (Field.Enum) { + auto *Entry = Field.Enum->EntryMap[cast<DefInit>(I)->getDef()]; + if (!Entry) + PrintFatalError(Twine("Entry for field '") + Field.Name + "' is null"); + return std::string(Entry->first); + } + PrintFatalError(Twine("invalid field type for field '") + Field.Name + + "', expected: string, bits, bit or code"); + } + + bool isIntrinsic(Init *I) { + if (DefInit *DI = dyn_cast<DefInit>(I)) + return DI->getDef()->isSubClassOf("Intrinsic"); + return false; + } + + CodeGenIntrinsic &getIntrinsic(Init *I) { + std::unique_ptr<CodeGenIntrinsic> &Intr = Intrinsics[I]; + if (!Intr) + Intr = std::make_unique<CodeGenIntrinsic>(cast<DefInit>(I)->getDef()); + return *Intr; + } + + bool compareBy(Record *LHS, Record *RHS, const SearchIndex &Index); + + bool isIntegral(Init *I) { + return isa<BitsInit>(I) || isa<CodeInit>(I) || isIntrinsic(I); + } + + std::string searchableFieldType(const GenericField &Field, TypeContext Ctx) { + if (isa<StringRecTy>(Field.RecType)) { + if (Ctx == TypeInStaticStruct) + return "const char *"; + if (Ctx == TypeInTempStruct) + return "std::string"; + return "StringRef"; + } else if (BitsRecTy *BI = dyn_cast<BitsRecTy>(Field.RecType)) { + unsigned NumBits = BI->getNumBits(); + if (NumBits <= 8) + return "uint8_t"; + if (NumBits <= 16) + return "uint16_t"; + if (NumBits <= 32) + return "uint32_t"; + if (NumBits <= 64) + return "uint64_t"; + PrintFatalError(Twine("bitfield '") + Field.Name + + "' too large to search"); + } else if (Field.Enum || Field.IsIntrinsic || Field.IsInstruction) + return "unsigned"; + PrintFatalError(Twine("Field '") + Field.Name + "' has unknown type '" + + Field.RecType->getAsString() + "' to search by"); + } + + void emitGenericTable(const GenericTable &Table, raw_ostream &OS); + void emitGenericEnum(const GenericEnum &Enum, raw_ostream &OS); + void emitLookupDeclaration(const GenericTable &Table, + const SearchIndex &Index, raw_ostream &OS); + void emitLookupFunction(const GenericTable &Table, const SearchIndex &Index, + bool IsPrimary, raw_ostream &OS); + void emitIfdef(StringRef Guard, raw_ostream &OS); + + bool parseFieldType(GenericField &Field, Init *II); + std::unique_ptr<SearchIndex> + parseSearchIndex(GenericTable &Table, StringRef Name, + const std::vector<StringRef> &Key, bool EarlyOut); + void collectEnumEntries(GenericEnum &Enum, StringRef NameField, + StringRef ValueField, + const std::vector<Record *> &Items); + void collectTableEntries(GenericTable &Table, + const std::vector<Record *> &Items); +}; + +} // End anonymous namespace. + +// For search indices that consists of a single field whose numeric value is +// known, return that numeric value. +static int64_t getNumericKey(const SearchIndex &Index, Record *Rec) { + assert(Index.Fields.size() == 1); + + if (Index.Fields[0].Enum) { + Record *EnumEntry = Rec->getValueAsDef(Index.Fields[0].Name); + return Index.Fields[0].Enum->EntryMap[EnumEntry]->second; + } + + return getInt(Rec, Index.Fields[0].Name); +} + +/// Less-than style comparison between \p LHS and \p RHS according to the +/// key of \p Index. +bool SearchableTableEmitter::compareBy(Record *LHS, Record *RHS, + const SearchIndex &Index) { + for (const auto &Field : Index.Fields) { + Init *LHSI = LHS->getValueInit(Field.Name); + Init *RHSI = RHS->getValueInit(Field.Name); + + if (isa<BitsRecTy>(Field.RecType) || isa<IntRecTy>(Field.RecType)) { + int64_t LHSi = getAsInt(LHSI); + int64_t RHSi = getAsInt(RHSI); + if (LHSi < RHSi) + return true; + if (LHSi > RHSi) + return false; + } else if (Field.IsIntrinsic) { + CodeGenIntrinsic &LHSi = getIntrinsic(LHSI); + CodeGenIntrinsic &RHSi = getIntrinsic(RHSI); + if (std::tie(LHSi.TargetPrefix, LHSi.Name) < + std::tie(RHSi.TargetPrefix, RHSi.Name)) + return true; + if (std::tie(LHSi.TargetPrefix, LHSi.Name) > + std::tie(RHSi.TargetPrefix, RHSi.Name)) + return false; + } else if (Field.IsInstruction) { + // This does not correctly compare the predefined instructions! + Record *LHSr = cast<DefInit>(LHSI)->getDef(); + Record *RHSr = cast<DefInit>(RHSI)->getDef(); + + bool LHSpseudo = LHSr->getValueAsBit("isPseudo"); + bool RHSpseudo = RHSr->getValueAsBit("isPseudo"); + if (LHSpseudo && !RHSpseudo) + return true; + if (!LHSpseudo && RHSpseudo) + return false; + + int comp = LHSr->getName().compare(RHSr->getName()); + if (comp < 0) + return true; + if (comp > 0) + return false; + } else if (Field.Enum) { + auto LHSr = cast<DefInit>(LHSI)->getDef(); + auto RHSr = cast<DefInit>(RHSI)->getDef(); + int64_t LHSv = Field.Enum->EntryMap[LHSr]->second; + int64_t RHSv = Field.Enum->EntryMap[RHSr]->second; + if (LHSv < RHSv) + return true; + if (LHSv > RHSv) + return false; + } else { + std::string LHSs = primaryRepresentation(Field, LHSI); + std::string RHSs = primaryRepresentation(Field, RHSI); + + if (isa<StringRecTy>(Field.RecType)) { + LHSs = StringRef(LHSs).upper(); + RHSs = StringRef(RHSs).upper(); + } + + int comp = LHSs.compare(RHSs); + if (comp < 0) + return true; + if (comp > 0) + return false; + } + } + return false; +} + +void SearchableTableEmitter::emitIfdef(StringRef Guard, raw_ostream &OS) { + OS << "#ifdef " << Guard << "\n"; + PreprocessorGuards.insert(std::string(Guard)); +} + +/// Emit a generic enum. +void SearchableTableEmitter::emitGenericEnum(const GenericEnum &Enum, + raw_ostream &OS) { + emitIfdef((Twine("GET_") + Enum.PreprocessorGuard + "_DECL").str(), OS); + + OS << "enum " << Enum.Name << " {\n"; + for (const auto &Entry : Enum.Entries) + OS << " " << Entry->first << " = " << Entry->second << ",\n"; + OS << "};\n"; + + OS << "#endif\n\n"; +} + +void SearchableTableEmitter::emitLookupFunction(const GenericTable &Table, + const SearchIndex &Index, + bool IsPrimary, + raw_ostream &OS) { + OS << "\n"; + emitLookupDeclaration(Table, Index, OS); + OS << " {\n"; + + std::vector<Record *> IndexRowsStorage; + ArrayRef<Record *> IndexRows; + StringRef IndexTypeName; + StringRef IndexName; + + if (IsPrimary) { + IndexTypeName = Table.CppTypeName; + IndexName = Table.Name; + IndexRows = Table.Entries; + } else { + OS << " struct IndexType {\n"; + for (const auto &Field : Index.Fields) { + OS << " " << searchableFieldType(Field, TypeInStaticStruct) << " " + << Field.Name << ";\n"; + } + OS << " unsigned _index;\n"; + OS << " };\n"; + + OS << " static const struct IndexType Index[] = {\n"; + + std::vector<std::pair<Record *, unsigned>> Entries; + Entries.reserve(Table.Entries.size()); + for (unsigned i = 0; i < Table.Entries.size(); ++i) + Entries.emplace_back(Table.Entries[i], i); + + std::stable_sort(Entries.begin(), Entries.end(), + [&](const std::pair<Record *, unsigned> &LHS, + const std::pair<Record *, unsigned> &RHS) { + return compareBy(LHS.first, RHS.first, Index); + }); + + IndexRowsStorage.reserve(Entries.size()); + for (const auto &Entry : Entries) { + IndexRowsStorage.push_back(Entry.first); + + OS << " { "; + bool NeedComma = false; + for (const auto &Field : Index.Fields) { + if (NeedComma) + OS << ", "; + NeedComma = true; + + std::string Repr = + primaryRepresentation(Field, Entry.first->getValueInit(Field.Name)); + if (isa<StringRecTy>(Field.RecType)) + Repr = StringRef(Repr).upper(); + OS << Repr; + } + OS << ", " << Entry.second << " },\n"; + } + + OS << " };\n\n"; + + IndexTypeName = "IndexType"; + IndexName = "Index"; + IndexRows = IndexRowsStorage; + } + + bool IsContiguous = false; + + if (Index.Fields.size() == 1 && + (Index.Fields[0].Enum || isa<BitsRecTy>(Index.Fields[0].RecType))) { + IsContiguous = true; + for (unsigned i = 0; i < IndexRows.size(); ++i) { + if (getNumericKey(Index, IndexRows[i]) != i) { + IsContiguous = false; + break; + } + } + } + + if (IsContiguous) { + OS << " auto Table = makeArrayRef(" << IndexName << ");\n"; + OS << " size_t Idx = " << Index.Fields[0].Name << ";\n"; + OS << " return Idx >= Table.size() ? nullptr : "; + if (IsPrimary) + OS << "&Table[Idx]"; + else + OS << "&" << Table.Name << "[Table[Idx]._index]"; + OS << ";\n"; + OS << "}\n"; + return; + } + + if (Index.EarlyOut) { + const GenericField &Field = Index.Fields[0]; + std::string FirstRepr = + primaryRepresentation(Field, IndexRows[0]->getValueInit(Field.Name)); + std::string LastRepr = primaryRepresentation( + Field, IndexRows.back()->getValueInit(Field.Name)); + OS << " if ((" << Field.Name << " < " << FirstRepr << ") ||\n"; + OS << " (" << Field.Name << " > " << LastRepr << "))\n"; + OS << " return nullptr;\n\n"; + } + + OS << " struct KeyType {\n"; + for (const auto &Field : Index.Fields) { + OS << " " << searchableFieldType(Field, TypeInTempStruct) << " " + << Field.Name << ";\n"; + } + OS << " };\n"; + OS << " KeyType Key = { "; + bool NeedComma = false; + for (const auto &Field : Index.Fields) { + if (NeedComma) + OS << ", "; + NeedComma = true; + + OS << Field.Name; + if (isa<StringRecTy>(Field.RecType)) { + OS << ".upper()"; + if (IsPrimary) + PrintFatalError(Twine("Use a secondary index for case-insensitive " + "comparison of field '") + + Field.Name + "' in table '" + Table.Name + "'"); + } + } + OS << " };\n"; + + OS << " auto Table = makeArrayRef(" << IndexName << ");\n"; + OS << " auto Idx = std::lower_bound(Table.begin(), Table.end(), Key,\n"; + OS << " [](const " << IndexTypeName << " &LHS, const KeyType &RHS) {\n"; + + for (const auto &Field : Index.Fields) { + if (isa<StringRecTy>(Field.RecType)) { + OS << " int Cmp" << Field.Name << " = StringRef(LHS." << Field.Name + << ").compare(RHS." << Field.Name << ");\n"; + OS << " if (Cmp" << Field.Name << " < 0) return true;\n"; + OS << " if (Cmp" << Field.Name << " > 0) return false;\n"; + } else if (Field.Enum) { + // Explicitly cast to unsigned, because the signedness of enums is + // compiler-dependent. + OS << " if ((unsigned)LHS." << Field.Name << " < (unsigned)RHS." + << Field.Name << ")\n"; + OS << " return true;\n"; + OS << " if ((unsigned)LHS." << Field.Name << " > (unsigned)RHS." + << Field.Name << ")\n"; + OS << " return false;\n"; + } else { + OS << " if (LHS." << Field.Name << " < RHS." << Field.Name << ")\n"; + OS << " return true;\n"; + OS << " if (LHS." << Field.Name << " > RHS." << Field.Name << ")\n"; + OS << " return false;\n"; + } + } + + OS << " return false;\n"; + OS << " });\n\n"; + + OS << " if (Idx == Table.end()"; + + for (const auto &Field : Index.Fields) + OS << " ||\n Key." << Field.Name << " != Idx->" << Field.Name; + OS << ")\n return nullptr;\n"; + + if (IsPrimary) + OS << " return &*Idx;\n"; + else + OS << " return &" << Table.Name << "[Idx->_index];\n"; + + OS << "}\n"; +} + +void SearchableTableEmitter::emitLookupDeclaration(const GenericTable &Table, + const SearchIndex &Index, + raw_ostream &OS) { + OS << "const " << Table.CppTypeName << " *" << Index.Name << "("; + + bool NeedComma = false; + for (const auto &Field : Index.Fields) { + if (NeedComma) + OS << ", "; + NeedComma = true; + + OS << searchableFieldType(Field, TypeInArgument) << " " << Field.Name; + } + OS << ")"; +} + +void SearchableTableEmitter::emitGenericTable(const GenericTable &Table, + raw_ostream &OS) { + emitIfdef((Twine("GET_") + Table.PreprocessorGuard + "_DECL").str(), OS); + + // Emit the declarations for the functions that will perform lookup. + if (Table.PrimaryKey) { + emitLookupDeclaration(Table, *Table.PrimaryKey, OS); + OS << ";\n"; + } + for (const auto &Index : Table.Indices) { + emitLookupDeclaration(Table, *Index, OS); + OS << ";\n"; + } + + OS << "#endif\n\n"; + + emitIfdef((Twine("GET_") + Table.PreprocessorGuard + "_IMPL").str(), OS); + + // The primary data table contains all the fields defined for this map. + OS << "constexpr " << Table.CppTypeName << " " << Table.Name << "[] = {\n"; + for (unsigned i = 0; i < Table.Entries.size(); ++i) { + Record *Entry = Table.Entries[i]; + OS << " { "; + + bool NeedComma = false; + for (const auto &Field : Table.Fields) { + if (NeedComma) + OS << ", "; + NeedComma = true; + + OS << primaryRepresentation(Field, Entry->getValueInit(Field.Name)); + } + + OS << " }, // " << i << "\n"; + } + OS << " };\n"; + + // Indexes are sorted "{ Thing, PrimaryIdx }" arrays, so that a binary + // search can be performed by "Thing". + if (Table.PrimaryKey) + emitLookupFunction(Table, *Table.PrimaryKey, true, OS); + for (const auto &Index : Table.Indices) + emitLookupFunction(Table, *Index, false, OS); + + OS << "#endif\n\n"; +} + +bool SearchableTableEmitter::parseFieldType(GenericField &Field, Init *II) { + if (auto DI = dyn_cast<DefInit>(II)) { + Record *TypeRec = DI->getDef(); + if (TypeRec->isSubClassOf("GenericEnum")) { + Field.Enum = EnumMap[TypeRec]; + Field.RecType = RecordRecTy::get(Field.Enum->Class); + return true; + } + } + + return false; +} + +std::unique_ptr<SearchIndex> +SearchableTableEmitter::parseSearchIndex(GenericTable &Table, StringRef Name, + const std::vector<StringRef> &Key, + bool EarlyOut) { + auto Index = std::make_unique<SearchIndex>(); + Index->Name = std::string(Name); + Index->EarlyOut = EarlyOut; + + for (const auto &FieldName : Key) { + const GenericField *Field = Table.getFieldByName(FieldName); + if (!Field) + PrintFatalError(Twine("Search index '") + Name + + "' refers to non-existing field '" + FieldName + + "' in table '" + Table.Name + "'"); + Index->Fields.push_back(*Field); + } + + if (EarlyOut && isa<StringRecTy>(Index->Fields[0].RecType)) { + PrintFatalError( + "Early-out is not supported for string types (in search index '" + + Twine(Name) + "'"); + } + + return Index; +} + +void SearchableTableEmitter::collectEnumEntries( + GenericEnum &Enum, StringRef NameField, StringRef ValueField, + const std::vector<Record *> &Items) { + for (auto EntryRec : Items) { + StringRef Name; + if (NameField.empty()) + Name = EntryRec->getName(); + else + Name = EntryRec->getValueAsString(NameField); + + int64_t Value = 0; + if (!ValueField.empty()) + Value = getInt(EntryRec, ValueField); + + Enum.Entries.push_back(std::make_unique<GenericEnum::Entry>(Name, Value)); + Enum.EntryMap.insert(std::make_pair(EntryRec, Enum.Entries.back().get())); + } + + if (ValueField.empty()) { + std::stable_sort(Enum.Entries.begin(), Enum.Entries.end(), + [](const std::unique_ptr<GenericEnum::Entry> &LHS, + const std::unique_ptr<GenericEnum::Entry> &RHS) { + return LHS->first < RHS->first; + }); + + for (size_t i = 0; i < Enum.Entries.size(); ++i) + Enum.Entries[i]->second = i; + } +} + +void SearchableTableEmitter::collectTableEntries( + GenericTable &Table, const std::vector<Record *> &Items) { + if (Items.empty()) + PrintWarning(Twine("Table '") + Table.Name + "' has no items"); + + for (auto EntryRec : Items) { + for (auto &Field : Table.Fields) { + auto TI = dyn_cast<TypedInit>(EntryRec->getValueInit(Field.Name)); + if (!TI || !TI->isComplete()) { + PrintFatalError(EntryRec->getLoc(), + Twine("Record '") + EntryRec->getName() + + "' in table '" + Table.Name + + "' is missing field '" + Field.Name + "'"); + } + if (!Field.RecType) { + Field.RecType = TI->getType(); + } else { + RecTy *Ty = resolveTypes(Field.RecType, TI->getType()); + if (!Ty) + PrintFatalError(Twine("Field '") + Field.Name + "' of table '" + + Table.Name + "' has incompatible type: " + + Field.RecType->getAsString() + " vs. " + + TI->getType()->getAsString()); + Field.RecType = Ty; + } + } + + Table.Entries.push_back(EntryRec); + } + + Record *IntrinsicClass = Records.getClass("Intrinsic"); + Record *InstructionClass = Records.getClass("Instruction"); + for (auto &Field : Table.Fields) { + if (!Field.RecType) + PrintFatalError(Twine("Cannot determine type of field '") + Field.Name + + "' in table '" + Table.Name + "'. Maybe it is not used?"); + + if (auto RecordTy = dyn_cast<RecordRecTy>(Field.RecType)) { + if (IntrinsicClass && RecordTy->isSubClassOf(IntrinsicClass)) + Field.IsIntrinsic = true; + else if (InstructionClass && RecordTy->isSubClassOf(InstructionClass)) + Field.IsInstruction = true; + } + } +} + +void SearchableTableEmitter::run(raw_ostream &OS) { + // Emit tables in a deterministic order to avoid needless rebuilds. + SmallVector<std::unique_ptr<GenericTable>, 4> Tables; + DenseMap<Record *, GenericTable *> TableMap; + + // Collect all definitions first. + for (auto EnumRec : Records.getAllDerivedDefinitions("GenericEnum")) { + StringRef NameField; + if (!EnumRec->isValueUnset("NameField")) + NameField = EnumRec->getValueAsString("NameField"); + + StringRef ValueField; + if (!EnumRec->isValueUnset("ValueField")) + ValueField = EnumRec->getValueAsString("ValueField"); + + auto Enum = std::make_unique<GenericEnum>(); + Enum->Name = std::string(EnumRec->getName()); + Enum->PreprocessorGuard = std::string(EnumRec->getName()); + + StringRef FilterClass = EnumRec->getValueAsString("FilterClass"); + Enum->Class = Records.getClass(FilterClass); + if (!Enum->Class) + PrintFatalError(EnumRec->getLoc(), Twine("Enum FilterClass '") + + FilterClass + "' does not exist"); + + collectEnumEntries(*Enum, NameField, ValueField, + Records.getAllDerivedDefinitions(FilterClass)); + EnumMap.insert(std::make_pair(EnumRec, Enum.get())); + Enums.emplace_back(std::move(Enum)); + } + + for (auto TableRec : Records.getAllDerivedDefinitions("GenericTable")) { + auto Table = std::make_unique<GenericTable>(); + Table->Name = std::string(TableRec->getName()); + Table->PreprocessorGuard = std::string(TableRec->getName()); + Table->CppTypeName = std::string(TableRec->getValueAsString("CppTypeName")); + + std::vector<StringRef> Fields = TableRec->getValueAsListOfStrings("Fields"); + for (const auto &FieldName : Fields) { + Table->Fields.emplace_back(FieldName); + + if (auto TypeOfVal = TableRec->getValue(("TypeOf_" + FieldName).str())) { + if (!parseFieldType(Table->Fields.back(), TypeOfVal->getValue())) { + PrintFatalError(TableRec->getLoc(), + Twine("Table '") + Table->Name + + "' has bad 'TypeOf_" + FieldName + + "': " + TypeOfVal->getValue()->getAsString()); + } + } + } + + collectTableEntries(*Table, Records.getAllDerivedDefinitions( + TableRec->getValueAsString("FilterClass"))); + + if (!TableRec->isValueUnset("PrimaryKey")) { + Table->PrimaryKey = + parseSearchIndex(*Table, TableRec->getValueAsString("PrimaryKeyName"), + TableRec->getValueAsListOfStrings("PrimaryKey"), + TableRec->getValueAsBit("PrimaryKeyEarlyOut")); + + std::stable_sort(Table->Entries.begin(), Table->Entries.end(), + [&](Record *LHS, Record *RHS) { + return compareBy(LHS, RHS, *Table->PrimaryKey); + }); + } + + TableMap.insert(std::make_pair(TableRec, Table.get())); + Tables.emplace_back(std::move(Table)); + } + + for (Record *IndexRec : Records.getAllDerivedDefinitions("SearchIndex")) { + Record *TableRec = IndexRec->getValueAsDef("Table"); + auto It = TableMap.find(TableRec); + if (It == TableMap.end()) + PrintFatalError(IndexRec->getLoc(), + Twine("SearchIndex '") + IndexRec->getName() + + "' refers to non-existing table '" + + TableRec->getName()); + + GenericTable &Table = *It->second; + Table.Indices.push_back(parseSearchIndex( + Table, IndexRec->getName(), IndexRec->getValueAsListOfStrings("Key"), + IndexRec->getValueAsBit("EarlyOut"))); + } + + // Translate legacy tables. + Record *SearchableTable = Records.getClass("SearchableTable"); + for (auto &NameRec : Records.getClasses()) { + Record *Class = NameRec.second.get(); + if (Class->getSuperClasses().size() != 1 || + !Class->isSubClassOf(SearchableTable)) + continue; + + StringRef TableName = Class->getName(); + std::vector<Record *> Items = Records.getAllDerivedDefinitions(TableName); + if (!Class->isValueUnset("EnumNameField")) { + StringRef NameField = Class->getValueAsString("EnumNameField"); + StringRef ValueField; + if (!Class->isValueUnset("EnumValueField")) + ValueField = Class->getValueAsString("EnumValueField"); + + auto Enum = std::make_unique<GenericEnum>(); + Enum->Name = (Twine(Class->getName()) + "Values").str(); + Enum->PreprocessorGuard = Class->getName().upper(); + Enum->Class = Class; + + collectEnumEntries(*Enum, NameField, ValueField, Items); + + Enums.emplace_back(std::move(Enum)); + } + + auto Table = std::make_unique<GenericTable>(); + Table->Name = (Twine(Class->getName()) + "sList").str(); + Table->PreprocessorGuard = Class->getName().upper(); + Table->CppTypeName = std::string(Class->getName()); + + for (const RecordVal &Field : Class->getValues()) { + std::string FieldName = std::string(Field.getName()); + + // Skip uninteresting fields: either special to us, or injected + // template parameters (if they contain a ':'). + if (FieldName.find(':') != std::string::npos || + FieldName == "SearchableFields" || FieldName == "EnumNameField" || + FieldName == "EnumValueField") + continue; + + Table->Fields.emplace_back(FieldName); + } + + collectTableEntries(*Table, Items); + + for (const auto &Field : + Class->getValueAsListOfStrings("SearchableFields")) { + std::string Name = + (Twine("lookup") + Table->CppTypeName + "By" + Field).str(); + Table->Indices.push_back(parseSearchIndex(*Table, Name, {Field}, false)); + } + + Tables.emplace_back(std::move(Table)); + } + + // Emit everything. + for (const auto &Enum : Enums) + emitGenericEnum(*Enum, OS); + + for (const auto &Table : Tables) + emitGenericTable(*Table, OS); + + // Put all #undefs last, to allow multiple sections guarded by the same + // define. + for (const auto &Guard : PreprocessorGuards) + OS << "#undef " << Guard << "\n"; +} + +namespace llvm { + +void EmitSearchableTables(RecordKeeper &RK, raw_ostream &OS) { + SearchableTableEmitter(RK).run(OS); +} + +} // End llvm namespace. diff --git a/contrib/llvm-project/llvm/utils/TableGen/SequenceToOffsetTable.h b/contrib/llvm-project/llvm/utils/TableGen/SequenceToOffsetTable.h new file mode 100644 index 000000000000..41cdefdb1949 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/SequenceToOffsetTable.h @@ -0,0 +1,219 @@ +//===-- SequenceToOffsetTable.h - Compress similar sequences ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// SequenceToOffsetTable can be used to emit a number of null-terminated +// sequences as one big array. Use the same memory when a sequence is a suffix +// of another. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_SEQUENCETOOFFSETTABLE_H +#define LLVM_UTILS_TABLEGEN_SEQUENCETOOFFSETTABLE_H + +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <cassert> +#include <cctype> +#include <functional> +#include <map> + +namespace llvm { +extern llvm::cl::opt<bool> EmitLongStrLiterals; + +// Helper function for SequenceToOffsetTable<string>. +static inline void printStrLitEscChar(raw_ostream &OS, char C) { + const char *Escapes[] = { + "\\000", "\\001", "\\002", "\\003", "\\004", "\\005", "\\006", "\\007", + "\\010", "\\t", "\\n", "\\013", "\\014", "\\r", "\\016", "\\017", + "\\020", "\\021", "\\022", "\\023", "\\024", "\\025", "\\026", "\\027", + "\\030", "\\031", "\\032", "\\033", "\\034", "\\035", "\\036", "\\037", + " ", "!", "\\\"", "#", "$", "%", "&", "'", + "(", ")", "*", "+", ",", "-", ".", "/", + "0", "1", "2", "3", "4", "5", "6", "7", + "8", "9", ":", ";", "<", "=", ">", "?", + "@", "A", "B", "C", "D", "E", "F", "G", + "H", "I", "J", "K", "L", "M", "N", "O", + "P", "Q", "R", "S", "T", "U", "V", "W", + "X", "Y", "Z", "[", "\\\\", "]", "^", "_", + "`", "a", "b", "c", "d", "e", "f", "g", + "h", "i", "j", "k", "l", "m", "n", "o", + "p", "q", "r", "s", "t", "u", "v", "w", + "x", "y", "z", "{", "|", "}", "~", "\\177", + "\\200", "\\201", "\\202", "\\203", "\\204", "\\205", "\\206", "\\207", + "\\210", "\\211", "\\212", "\\213", "\\214", "\\215", "\\216", "\\217", + "\\220", "\\221", "\\222", "\\223", "\\224", "\\225", "\\226", "\\227", + "\\230", "\\231", "\\232", "\\233", "\\234", "\\235", "\\236", "\\237", + "\\240", "\\241", "\\242", "\\243", "\\244", "\\245", "\\246", "\\247", + "\\250", "\\251", "\\252", "\\253", "\\254", "\\255", "\\256", "\\257", + "\\260", "\\261", "\\262", "\\263", "\\264", "\\265", "\\266", "\\267", + "\\270", "\\271", "\\272", "\\273", "\\274", "\\275", "\\276", "\\277", + "\\300", "\\301", "\\302", "\\303", "\\304", "\\305", "\\306", "\\307", + "\\310", "\\311", "\\312", "\\313", "\\314", "\\315", "\\316", "\\317", + "\\320", "\\321", "\\322", "\\323", "\\324", "\\325", "\\326", "\\327", + "\\330", "\\331", "\\332", "\\333", "\\334", "\\335", "\\336", "\\337", + "\\340", "\\341", "\\342", "\\343", "\\344", "\\345", "\\346", "\\347", + "\\350", "\\351", "\\352", "\\353", "\\354", "\\355", "\\356", "\\357", + "\\360", "\\361", "\\362", "\\363", "\\364", "\\365", "\\366", "\\367", + "\\370", "\\371", "\\372", "\\373", "\\374", "\\375", "\\376", "\\377"}; + + static_assert(sizeof Escapes / sizeof Escapes[0] == + std::numeric_limits<unsigned char>::max() + 1, + "unsupported character type"); + OS << Escapes[static_cast<unsigned char>(C)]; +} + +static inline void printChar(raw_ostream &OS, char C) { + unsigned char UC(C); + if (isalnum(UC) || ispunct(UC)) { + OS << '\''; + if (C == '\\' || C == '\'') + OS << '\\'; + OS << C << '\''; + } else { + OS << unsigned(UC); + } +} + +/// SequenceToOffsetTable - Collect a number of terminated sequences of T. +/// Compute the layout of a table that contains all the sequences, possibly by +/// reusing entries. +/// +/// @tparam SeqT The sequence container. (vector or string). +/// @tparam Less A stable comparator for SeqT elements. +template<typename SeqT, typename Less = std::less<typename SeqT::value_type> > +class SequenceToOffsetTable { + typedef typename SeqT::value_type ElemT; + + // Define a comparator for SeqT that sorts a suffix immediately before a + // sequence with that suffix. + struct SeqLess { + Less L; + bool operator()(const SeqT &A, const SeqT &B) const { + return std::lexicographical_compare(A.rbegin(), A.rend(), + B.rbegin(), B.rend(), L); + } + }; + + // Keep sequences ordered according to SeqLess so suffixes are easy to find. + // Map each sequence to its offset in the table. + typedef std::map<SeqT, unsigned, SeqLess> SeqMap; + + // Sequences added so far, with suffixes removed. + SeqMap Seqs; + + // Entries in the final table, or 0 before layout was called. + unsigned Entries; + + // isSuffix - Returns true if A is a suffix of B. + static bool isSuffix(const SeqT &A, const SeqT &B) { + return A.size() <= B.size() && std::equal(A.rbegin(), A.rend(), B.rbegin()); + } + +public: + SequenceToOffsetTable() : Entries(0) {} + + /// add - Add a sequence to the table. + /// This must be called before layout(). + void add(const SeqT &Seq) { + assert(Entries == 0 && "Cannot call add() after layout()"); + typename SeqMap::iterator I = Seqs.lower_bound(Seq); + + // If SeqMap contains a sequence that has Seq as a suffix, I will be + // pointing to it. + if (I != Seqs.end() && isSuffix(Seq, I->first)) + return; + + I = Seqs.insert(I, std::make_pair(Seq, 0u)); + + // The entry before I may be a suffix of Seq that can now be erased. + if (I != Seqs.begin() && isSuffix((--I)->first, Seq)) + Seqs.erase(I); + } + + bool empty() const { return Seqs.empty(); } + + unsigned size() const { + assert((empty() || Entries) && "Call layout() before size()"); + return Entries; + } + + /// layout - Computes the final table layout. + void layout() { + assert(Entries == 0 && "Can only call layout() once"); + // Lay out the table in Seqs iteration order. + for (typename SeqMap::iterator I = Seqs.begin(), E = Seqs.end(); I != E; + ++I) { + I->second = Entries; + // Include space for a terminator. + Entries += I->first.size() + 1; + } + } + + /// get - Returns the offset of Seq in the final table. + unsigned get(const SeqT &Seq) const { + assert(Entries && "Call layout() before get()"); + typename SeqMap::const_iterator I = Seqs.lower_bound(Seq); + assert(I != Seqs.end() && isSuffix(Seq, I->first) && + "get() called with sequence that wasn't added first"); + return I->second + (I->first.size() - Seq.size()); + } + + /// `emitStringLiteralDef` - Print out the table as the body of an array + /// initializer, where each element is a C string literal terminated by + /// `\0`. Falls back to emitting a comma-separated integer list if + /// `EmitLongStrLiterals` is false + void emitStringLiteralDef(raw_ostream &OS, const llvm::Twine &Decl) const { + assert(Entries && "Call layout() before emitStringLiteralDef()"); + if (EmitLongStrLiterals) { + OS << "\n#ifdef __GNUC__\n" + << "#pragma GCC diagnostic push\n" + << "#pragma GCC diagnostic ignored \"-Woverlength-strings\"\n" + << "#endif\n" + << Decl << " = {\n"; + } else { + OS << Decl << " = {\n"; + emit(OS, printChar, "0"); + OS << "\n};\n\n"; + return; + } + for (auto I : Seqs) { + OS << " /* " << I.second << " */ \""; + for (auto C : I.first) { + printStrLitEscChar(OS, C); + } + OS << "\\0\"\n"; + } + OS << "};\n" + << "#ifdef __GNUC__\n" + << "#pragma GCC diagnostic pop\n" + << "#endif\n\n"; + } + + /// emit - Print out the table as the body of an array initializer. + /// Use the Print function to print elements. + void emit(raw_ostream &OS, + void (*Print)(raw_ostream&, ElemT), + const char *Term = "0") const { + assert((empty() || Entries) && "Call layout() before emit()"); + for (typename SeqMap::const_iterator I = Seqs.begin(), E = Seqs.end(); + I != E; ++I) { + OS << " /* " << I->second << " */ "; + for (typename SeqT::const_iterator SI = I->first.begin(), + SE = I->first.end(); SI != SE; ++SI) { + Print(OS, *SI); + OS << ", "; + } + OS << Term << ",\n"; + } + } +}; + +} // end namespace llvm + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/SubtargetEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/SubtargetEmitter.cpp new file mode 100644 index 000000000000..68ee839c43ba --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/SubtargetEmitter.cpp @@ -0,0 +1,1952 @@ +//===- SubtargetEmitter.cpp - Generate subtarget enumerations -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend emits subtarget enumerations. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "CodeGenSchedule.h" +#include "PredicateExpander.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/MC/MCInstrItineraries.h" +#include "llvm/MC/MCSchedule.h" +#include "llvm/MC/SubtargetFeature.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/TableGenBackend.h" +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <iterator> +#include <map> +#include <string> +#include <vector> + +using namespace llvm; + +#define DEBUG_TYPE "subtarget-emitter" + +namespace { + +class SubtargetEmitter { + // Each processor has a SchedClassDesc table with an entry for each SchedClass. + // The SchedClassDesc table indexes into a global write resource table, write + // latency table, and read advance table. + struct SchedClassTables { + std::vector<std::vector<MCSchedClassDesc>> ProcSchedClasses; + std::vector<MCWriteProcResEntry> WriteProcResources; + std::vector<MCWriteLatencyEntry> WriteLatencies; + std::vector<std::string> WriterNames; + std::vector<MCReadAdvanceEntry> ReadAdvanceEntries; + + // Reserve an invalid entry at index 0 + SchedClassTables() { + ProcSchedClasses.resize(1); + WriteProcResources.resize(1); + WriteLatencies.resize(1); + WriterNames.push_back("InvalidWrite"); + ReadAdvanceEntries.resize(1); + } + }; + + struct LessWriteProcResources { + bool operator()(const MCWriteProcResEntry &LHS, + const MCWriteProcResEntry &RHS) { + return LHS.ProcResourceIdx < RHS.ProcResourceIdx; + } + }; + + const CodeGenTarget &TGT; + RecordKeeper &Records; + CodeGenSchedModels &SchedModels; + std::string Target; + + void Enumeration(raw_ostream &OS, DenseMap<Record *, unsigned> &FeatureMap); + unsigned FeatureKeyValues(raw_ostream &OS, + const DenseMap<Record *, unsigned> &FeatureMap); + unsigned CPUKeyValues(raw_ostream &OS, + const DenseMap<Record *, unsigned> &FeatureMap); + void FormItineraryStageString(const std::string &Names, + Record *ItinData, std::string &ItinString, + unsigned &NStages); + void FormItineraryOperandCycleString(Record *ItinData, std::string &ItinString, + unsigned &NOperandCycles); + void FormItineraryBypassString(const std::string &Names, + Record *ItinData, + std::string &ItinString, unsigned NOperandCycles); + void EmitStageAndOperandCycleData(raw_ostream &OS, + std::vector<std::vector<InstrItinerary>> + &ProcItinLists); + void EmitItineraries(raw_ostream &OS, + std::vector<std::vector<InstrItinerary>> + &ProcItinLists); + unsigned EmitRegisterFileTables(const CodeGenProcModel &ProcModel, + raw_ostream &OS); + void EmitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel, + raw_ostream &OS); + void EmitExtraProcessorInfo(const CodeGenProcModel &ProcModel, + raw_ostream &OS); + void EmitProcessorProp(raw_ostream &OS, const Record *R, StringRef Name, + char Separator); + void EmitProcessorResourceSubUnits(const CodeGenProcModel &ProcModel, + raw_ostream &OS); + void EmitProcessorResources(const CodeGenProcModel &ProcModel, + raw_ostream &OS); + Record *FindWriteResources(const CodeGenSchedRW &SchedWrite, + const CodeGenProcModel &ProcModel); + Record *FindReadAdvance(const CodeGenSchedRW &SchedRead, + const CodeGenProcModel &ProcModel); + void ExpandProcResources(RecVec &PRVec, std::vector<int64_t> &Cycles, + const CodeGenProcModel &ProcModel); + void GenSchedClassTables(const CodeGenProcModel &ProcModel, + SchedClassTables &SchedTables); + void EmitSchedClassTables(SchedClassTables &SchedTables, raw_ostream &OS); + void EmitProcessorModels(raw_ostream &OS); + void EmitProcessorLookup(raw_ostream &OS); + void EmitSchedModelHelpers(const std::string &ClassName, raw_ostream &OS); + void emitSchedModelHelpersImpl(raw_ostream &OS, + bool OnlyExpandMCInstPredicates = false); + void emitGenMCSubtargetInfo(raw_ostream &OS); + void EmitMCInstrAnalysisPredicateFunctions(raw_ostream &OS); + + void EmitSchedModel(raw_ostream &OS); + void EmitHwModeCheck(const std::string &ClassName, raw_ostream &OS); + void ParseFeaturesFunction(raw_ostream &OS, unsigned NumFeatures, + unsigned NumProcs); + +public: + SubtargetEmitter(RecordKeeper &R, CodeGenTarget &TGT) + : TGT(TGT), Records(R), SchedModels(TGT.getSchedModels()), + Target(TGT.getName()) {} + + void run(raw_ostream &o); +}; + +} // end anonymous namespace + +// +// Enumeration - Emit the specified class as an enumeration. +// +void SubtargetEmitter::Enumeration(raw_ostream &OS, + DenseMap<Record *, unsigned> &FeatureMap) { + // Get all records of class and sort + std::vector<Record*> DefList = + Records.getAllDerivedDefinitions("SubtargetFeature"); + llvm::sort(DefList, LessRecord()); + + unsigned N = DefList.size(); + if (N == 0) + return; + if (N + 1 > MAX_SUBTARGET_FEATURES) + PrintFatalError("Too many subtarget features! Bump MAX_SUBTARGET_FEATURES."); + + OS << "namespace " << Target << " {\n"; + + // Open enumeration. + OS << "enum {\n"; + + // For each record + for (unsigned i = 0; i < N; ++i) { + // Next record + Record *Def = DefList[i]; + + // Get and emit name + OS << " " << Def->getName() << " = " << i << ",\n"; + + // Save the index for this feature. + FeatureMap[Def] = i; + } + + OS << " " + << "NumSubtargetFeatures = " << N << "\n"; + + // Close enumeration and namespace + OS << "};\n"; + OS << "} // end namespace " << Target << "\n"; +} + +static void printFeatureMask(raw_ostream &OS, RecVec &FeatureList, + const DenseMap<Record *, unsigned> &FeatureMap) { + std::array<uint64_t, MAX_SUBTARGET_WORDS> Mask = {}; + for (unsigned j = 0, M = FeatureList.size(); j < M; ++j) { + unsigned Bit = FeatureMap.lookup(FeatureList[j]); + Mask[Bit / 64] |= 1ULL << (Bit % 64); + } + + OS << "{ { { "; + for (unsigned i = 0; i != Mask.size(); ++i) { + OS << "0x"; + OS.write_hex(Mask[i]); + OS << "ULL, "; + } + OS << "} } }"; +} + +// +// FeatureKeyValues - Emit data of all the subtarget features. Used by the +// command line. +// +unsigned SubtargetEmitter::FeatureKeyValues( + raw_ostream &OS, const DenseMap<Record *, unsigned> &FeatureMap) { + // Gather and sort all the features + std::vector<Record*> FeatureList = + Records.getAllDerivedDefinitions("SubtargetFeature"); + + if (FeatureList.empty()) + return 0; + + llvm::sort(FeatureList, LessRecordFieldName()); + + // Begin feature table + OS << "// Sorted (by key) array of values for CPU features.\n" + << "extern const llvm::SubtargetFeatureKV " << Target + << "FeatureKV[] = {\n"; + + // For each feature + unsigned NumFeatures = 0; + for (unsigned i = 0, N = FeatureList.size(); i < N; ++i) { + // Next feature + Record *Feature = FeatureList[i]; + + StringRef Name = Feature->getName(); + StringRef CommandLineName = Feature->getValueAsString("Name"); + StringRef Desc = Feature->getValueAsString("Desc"); + + if (CommandLineName.empty()) continue; + + // Emit as { "feature", "description", { featureEnum }, { i1 , i2 , ... , in } } + OS << " { " + << "\"" << CommandLineName << "\", " + << "\"" << Desc << "\", " + << Target << "::" << Name << ", "; + + RecVec ImpliesList = Feature->getValueAsListOfDefs("Implies"); + + printFeatureMask(OS, ImpliesList, FeatureMap); + + OS << " },\n"; + ++NumFeatures; + } + + // End feature table + OS << "};\n"; + + return NumFeatures; +} + +// +// CPUKeyValues - Emit data of all the subtarget processors. Used by command +// line. +// +unsigned +SubtargetEmitter::CPUKeyValues(raw_ostream &OS, + const DenseMap<Record *, unsigned> &FeatureMap) { + // Gather and sort processor information + std::vector<Record*> ProcessorList = + Records.getAllDerivedDefinitions("Processor"); + llvm::sort(ProcessorList, LessRecordFieldName()); + + // Begin processor table + OS << "// Sorted (by key) array of values for CPU subtype.\n" + << "extern const llvm::SubtargetSubTypeKV " << Target + << "SubTypeKV[] = {\n"; + + // For each processor + for (Record *Processor : ProcessorList) { + StringRef Name = Processor->getValueAsString("Name"); + RecVec FeatureList = Processor->getValueAsListOfDefs("Features"); + + // Emit as { "cpu", "description", 0, { f1 , f2 , ... fn } }, + OS << " { " + << "\"" << Name << "\", "; + + printFeatureMask(OS, FeatureList, FeatureMap); + + // Emit the scheduler model pointer. + const std::string &ProcModelName = + SchedModels.getModelForProc(Processor).ModelName; + OS << ", &" << ProcModelName << " },\n"; + } + + // End processor table + OS << "};\n"; + + return ProcessorList.size(); +} + +// +// FormItineraryStageString - Compose a string containing the stage +// data initialization for the specified itinerary. N is the number +// of stages. +// +void SubtargetEmitter::FormItineraryStageString(const std::string &Name, + Record *ItinData, + std::string &ItinString, + unsigned &NStages) { + // Get states list + RecVec StageList = ItinData->getValueAsListOfDefs("Stages"); + + // For each stage + unsigned N = NStages = StageList.size(); + for (unsigned i = 0; i < N;) { + // Next stage + const Record *Stage = StageList[i]; + + // Form string as ,{ cycles, u1 | u2 | ... | un, timeinc, kind } + int Cycles = Stage->getValueAsInt("Cycles"); + ItinString += " { " + itostr(Cycles) + ", "; + + // Get unit list + RecVec UnitList = Stage->getValueAsListOfDefs("Units"); + + // For each unit + for (unsigned j = 0, M = UnitList.size(); j < M;) { + // Add name and bitwise or + ItinString += Name + "FU::" + UnitList[j]->getName().str(); + if (++j < M) ItinString += " | "; + } + + int TimeInc = Stage->getValueAsInt("TimeInc"); + ItinString += ", " + itostr(TimeInc); + + int Kind = Stage->getValueAsInt("Kind"); + ItinString += ", (llvm::InstrStage::ReservationKinds)" + itostr(Kind); + + // Close off stage + ItinString += " }"; + if (++i < N) ItinString += ", "; + } +} + +// +// FormItineraryOperandCycleString - Compose a string containing the +// operand cycle initialization for the specified itinerary. N is the +// number of operands that has cycles specified. +// +void SubtargetEmitter::FormItineraryOperandCycleString(Record *ItinData, + std::string &ItinString, unsigned &NOperandCycles) { + // Get operand cycle list + std::vector<int64_t> OperandCycleList = + ItinData->getValueAsListOfInts("OperandCycles"); + + // For each operand cycle + unsigned N = NOperandCycles = OperandCycleList.size(); + for (unsigned i = 0; i < N;) { + // Next operand cycle + const int OCycle = OperandCycleList[i]; + + ItinString += " " + itostr(OCycle); + if (++i < N) ItinString += ", "; + } +} + +void SubtargetEmitter::FormItineraryBypassString(const std::string &Name, + Record *ItinData, + std::string &ItinString, + unsigned NOperandCycles) { + RecVec BypassList = ItinData->getValueAsListOfDefs("Bypasses"); + unsigned N = BypassList.size(); + unsigned i = 0; + for (; i < N;) { + ItinString += Name + "Bypass::" + BypassList[i]->getName().str(); + if (++i < NOperandCycles) ItinString += ", "; + } + for (; i < NOperandCycles;) { + ItinString += " 0"; + if (++i < NOperandCycles) ItinString += ", "; + } +} + +// +// EmitStageAndOperandCycleData - Generate unique itinerary stages and operand +// cycle tables. Create a list of InstrItinerary objects (ProcItinLists) indexed +// by CodeGenSchedClass::Index. +// +void SubtargetEmitter:: +EmitStageAndOperandCycleData(raw_ostream &OS, + std::vector<std::vector<InstrItinerary>> + &ProcItinLists) { + // Multiple processor models may share an itinerary record. Emit it once. + SmallPtrSet<Record*, 8> ItinsDefSet; + + // Emit functional units for all the itineraries. + for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) { + + if (!ItinsDefSet.insert(ProcModel.ItinsDef).second) + continue; + + RecVec FUs = ProcModel.ItinsDef->getValueAsListOfDefs("FU"); + if (FUs.empty()) + continue; + + StringRef Name = ProcModel.ItinsDef->getName(); + OS << "\n// Functional units for \"" << Name << "\"\n" + << "namespace " << Name << "FU {\n"; + + for (unsigned j = 0, FUN = FUs.size(); j < FUN; ++j) + OS << " const InstrStage::FuncUnits " << FUs[j]->getName() + << " = 1ULL << " << j << ";\n"; + + OS << "} // end namespace " << Name << "FU\n"; + + RecVec BPs = ProcModel.ItinsDef->getValueAsListOfDefs("BP"); + if (!BPs.empty()) { + OS << "\n// Pipeline forwarding paths for itineraries \"" << Name + << "\"\n" << "namespace " << Name << "Bypass {\n"; + + OS << " const unsigned NoBypass = 0;\n"; + for (unsigned j = 0, BPN = BPs.size(); j < BPN; ++j) + OS << " const unsigned " << BPs[j]->getName() + << " = 1 << " << j << ";\n"; + + OS << "} // end namespace " << Name << "Bypass\n"; + } + } + + // Begin stages table + std::string StageTable = "\nextern const llvm::InstrStage " + Target + + "Stages[] = {\n"; + StageTable += " { 0, 0, 0, llvm::InstrStage::Required }, // No itinerary\n"; + + // Begin operand cycle table + std::string OperandCycleTable = "extern const unsigned " + Target + + "OperandCycles[] = {\n"; + OperandCycleTable += " 0, // No itinerary\n"; + + // Begin pipeline bypass table + std::string BypassTable = "extern const unsigned " + Target + + "ForwardingPaths[] = {\n"; + BypassTable += " 0, // No itinerary\n"; + + // For each Itinerary across all processors, add a unique entry to the stages, + // operand cycles, and pipeline bypass tables. Then add the new Itinerary + // object with computed offsets to the ProcItinLists result. + unsigned StageCount = 1, OperandCycleCount = 1; + std::map<std::string, unsigned> ItinStageMap, ItinOperandMap; + for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) { + // Add process itinerary to the list. + ProcItinLists.resize(ProcItinLists.size()+1); + + // If this processor defines no itineraries, then leave the itinerary list + // empty. + std::vector<InstrItinerary> &ItinList = ProcItinLists.back(); + if (!ProcModel.hasItineraries()) + continue; + + StringRef Name = ProcModel.ItinsDef->getName(); + + ItinList.resize(SchedModels.numInstrSchedClasses()); + assert(ProcModel.ItinDefList.size() == ItinList.size() && "bad Itins"); + + for (unsigned SchedClassIdx = 0, SchedClassEnd = ItinList.size(); + SchedClassIdx < SchedClassEnd; ++SchedClassIdx) { + + // Next itinerary data + Record *ItinData = ProcModel.ItinDefList[SchedClassIdx]; + + // Get string and stage count + std::string ItinStageString; + unsigned NStages = 0; + if (ItinData) + FormItineraryStageString(std::string(Name), ItinData, ItinStageString, + NStages); + + // Get string and operand cycle count + std::string ItinOperandCycleString; + unsigned NOperandCycles = 0; + std::string ItinBypassString; + if (ItinData) { + FormItineraryOperandCycleString(ItinData, ItinOperandCycleString, + NOperandCycles); + + FormItineraryBypassString(std::string(Name), ItinData, ItinBypassString, + NOperandCycles); + } + + // Check to see if stage already exists and create if it doesn't + uint16_t FindStage = 0; + if (NStages > 0) { + FindStage = ItinStageMap[ItinStageString]; + if (FindStage == 0) { + // Emit as { cycles, u1 | u2 | ... | un, timeinc }, // indices + StageTable += ItinStageString + ", // " + itostr(StageCount); + if (NStages > 1) + StageTable += "-" + itostr(StageCount + NStages - 1); + StageTable += "\n"; + // Record Itin class number. + ItinStageMap[ItinStageString] = FindStage = StageCount; + StageCount += NStages; + } + } + + // Check to see if operand cycle already exists and create if it doesn't + uint16_t FindOperandCycle = 0; + if (NOperandCycles > 0) { + std::string ItinOperandString = ItinOperandCycleString+ItinBypassString; + FindOperandCycle = ItinOperandMap[ItinOperandString]; + if (FindOperandCycle == 0) { + // Emit as cycle, // index + OperandCycleTable += ItinOperandCycleString + ", // "; + std::string OperandIdxComment = itostr(OperandCycleCount); + if (NOperandCycles > 1) + OperandIdxComment += "-" + + itostr(OperandCycleCount + NOperandCycles - 1); + OperandCycleTable += OperandIdxComment + "\n"; + // Record Itin class number. + ItinOperandMap[ItinOperandCycleString] = + FindOperandCycle = OperandCycleCount; + // Emit as bypass, // index + BypassTable += ItinBypassString + ", // " + OperandIdxComment + "\n"; + OperandCycleCount += NOperandCycles; + } + } + + // Set up itinerary as location and location + stage count + int16_t NumUOps = ItinData ? ItinData->getValueAsInt("NumMicroOps") : 0; + InstrItinerary Intinerary = { + NumUOps, + FindStage, + uint16_t(FindStage + NStages), + FindOperandCycle, + uint16_t(FindOperandCycle + NOperandCycles), + }; + + // Inject - empty slots will be 0, 0 + ItinList[SchedClassIdx] = Intinerary; + } + } + + // Closing stage + StageTable += " { 0, 0, 0, llvm::InstrStage::Required } // End stages\n"; + StageTable += "};\n"; + + // Closing operand cycles + OperandCycleTable += " 0 // End operand cycles\n"; + OperandCycleTable += "};\n"; + + BypassTable += " 0 // End bypass tables\n"; + BypassTable += "};\n"; + + // Emit tables. + OS << StageTable; + OS << OperandCycleTable; + OS << BypassTable; +} + +// +// EmitProcessorData - Generate data for processor itineraries that were +// computed during EmitStageAndOperandCycleData(). ProcItinLists lists all +// Itineraries for each processor. The Itinerary lists are indexed on +// CodeGenSchedClass::Index. +// +void SubtargetEmitter:: +EmitItineraries(raw_ostream &OS, + std::vector<std::vector<InstrItinerary>> &ProcItinLists) { + // Multiple processor models may share an itinerary record. Emit it once. + SmallPtrSet<Record*, 8> ItinsDefSet; + + // For each processor's machine model + std::vector<std::vector<InstrItinerary>>::iterator + ProcItinListsIter = ProcItinLists.begin(); + for (CodeGenSchedModels::ProcIter PI = SchedModels.procModelBegin(), + PE = SchedModels.procModelEnd(); PI != PE; ++PI, ++ProcItinListsIter) { + + Record *ItinsDef = PI->ItinsDef; + if (!ItinsDefSet.insert(ItinsDef).second) + continue; + + // Get the itinerary list for the processor. + assert(ProcItinListsIter != ProcItinLists.end() && "bad iterator"); + std::vector<InstrItinerary> &ItinList = *ProcItinListsIter; + + // Empty itineraries aren't referenced anywhere in the tablegen output + // so don't emit them. + if (ItinList.empty()) + continue; + + OS << "\n"; + OS << "static const llvm::InstrItinerary "; + + // Begin processor itinerary table + OS << ItinsDef->getName() << "[] = {\n"; + + // For each itinerary class in CodeGenSchedClass::Index order. + for (unsigned j = 0, M = ItinList.size(); j < M; ++j) { + InstrItinerary &Intinerary = ItinList[j]; + + // Emit Itinerary in the form of + // { firstStage, lastStage, firstCycle, lastCycle } // index + OS << " { " << + Intinerary.NumMicroOps << ", " << + Intinerary.FirstStage << ", " << + Intinerary.LastStage << ", " << + Intinerary.FirstOperandCycle << ", " << + Intinerary.LastOperandCycle << " }" << + ", // " << j << " " << SchedModels.getSchedClass(j).Name << "\n"; + } + // End processor itinerary table + OS << " { 0, uint16_t(~0U), uint16_t(~0U), uint16_t(~0U), uint16_t(~0U) }" + "// end marker\n"; + OS << "};\n"; + } +} + +// Emit either the value defined in the TableGen Record, or the default +// value defined in the C++ header. The Record is null if the processor does not +// define a model. +void SubtargetEmitter::EmitProcessorProp(raw_ostream &OS, const Record *R, + StringRef Name, char Separator) { + OS << " "; + int V = R ? R->getValueAsInt(Name) : -1; + if (V >= 0) + OS << V << Separator << " // " << Name; + else + OS << "MCSchedModel::Default" << Name << Separator; + OS << '\n'; +} + +void SubtargetEmitter::EmitProcessorResourceSubUnits( + const CodeGenProcModel &ProcModel, raw_ostream &OS) { + OS << "\nstatic const unsigned " << ProcModel.ModelName + << "ProcResourceSubUnits[] = {\n" + << " 0, // Invalid\n"; + + for (unsigned i = 0, e = ProcModel.ProcResourceDefs.size(); i < e; ++i) { + Record *PRDef = ProcModel.ProcResourceDefs[i]; + if (!PRDef->isSubClassOf("ProcResGroup")) + continue; + RecVec ResUnits = PRDef->getValueAsListOfDefs("Resources"); + for (Record *RUDef : ResUnits) { + Record *const RU = + SchedModels.findProcResUnits(RUDef, ProcModel, PRDef->getLoc()); + for (unsigned J = 0; J < RU->getValueAsInt("NumUnits"); ++J) { + OS << " " << ProcModel.getProcResourceIdx(RU) << ", "; + } + } + OS << " // " << PRDef->getName() << "\n"; + } + OS << "};\n"; +} + +static void EmitRetireControlUnitInfo(const CodeGenProcModel &ProcModel, + raw_ostream &OS) { + int64_t ReorderBufferSize = 0, MaxRetirePerCycle = 0; + if (Record *RCU = ProcModel.RetireControlUnit) { + ReorderBufferSize = + std::max(ReorderBufferSize, RCU->getValueAsInt("ReorderBufferSize")); + MaxRetirePerCycle = + std::max(MaxRetirePerCycle, RCU->getValueAsInt("MaxRetirePerCycle")); + } + + OS << ReorderBufferSize << ", // ReorderBufferSize\n "; + OS << MaxRetirePerCycle << ", // MaxRetirePerCycle\n "; +} + +static void EmitRegisterFileInfo(const CodeGenProcModel &ProcModel, + unsigned NumRegisterFiles, + unsigned NumCostEntries, raw_ostream &OS) { + if (NumRegisterFiles) + OS << ProcModel.ModelName << "RegisterFiles,\n " << (1 + NumRegisterFiles); + else + OS << "nullptr,\n 0"; + + OS << ", // Number of register files.\n "; + if (NumCostEntries) + OS << ProcModel.ModelName << "RegisterCosts,\n "; + else + OS << "nullptr,\n "; + OS << NumCostEntries << ", // Number of register cost entries.\n"; +} + +unsigned +SubtargetEmitter::EmitRegisterFileTables(const CodeGenProcModel &ProcModel, + raw_ostream &OS) { + if (llvm::all_of(ProcModel.RegisterFiles, [](const CodeGenRegisterFile &RF) { + return RF.hasDefaultCosts(); + })) + return 0; + + // Print the RegisterCost table first. + OS << "\n// {RegisterClassID, Register Cost, AllowMoveElimination }\n"; + OS << "static const llvm::MCRegisterCostEntry " << ProcModel.ModelName + << "RegisterCosts" + << "[] = {\n"; + + for (const CodeGenRegisterFile &RF : ProcModel.RegisterFiles) { + // Skip register files with a default cost table. + if (RF.hasDefaultCosts()) + continue; + // Add entries to the cost table. + for (const CodeGenRegisterCost &RC : RF.Costs) { + OS << " { "; + Record *Rec = RC.RCDef; + if (Rec->getValue("Namespace")) + OS << Rec->getValueAsString("Namespace") << "::"; + OS << Rec->getName() << "RegClassID, " << RC.Cost << ", " + << RC.AllowMoveElimination << "},\n"; + } + } + OS << "};\n"; + + // Now generate a table with register file info. + OS << "\n // {Name, #PhysRegs, #CostEntries, IndexToCostTbl, " + << "MaxMovesEliminatedPerCycle, AllowZeroMoveEliminationOnly }\n"; + OS << "static const llvm::MCRegisterFileDesc " << ProcModel.ModelName + << "RegisterFiles" + << "[] = {\n" + << " { \"InvalidRegisterFile\", 0, 0, 0, 0, 0 },\n"; + unsigned CostTblIndex = 0; + + for (const CodeGenRegisterFile &RD : ProcModel.RegisterFiles) { + OS << " { "; + OS << '"' << RD.Name << '"' << ", " << RD.NumPhysRegs << ", "; + unsigned NumCostEntries = RD.Costs.size(); + OS << NumCostEntries << ", " << CostTblIndex << ", " + << RD.MaxMovesEliminatedPerCycle << ", " + << RD.AllowZeroMoveEliminationOnly << "},\n"; + CostTblIndex += NumCostEntries; + } + OS << "};\n"; + + return CostTblIndex; +} + +void SubtargetEmitter::EmitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel, + raw_ostream &OS) { + unsigned QueueID = 0; + if (ProcModel.LoadQueue) { + const Record *Queue = ProcModel.LoadQueue->getValueAsDef("QueueDescriptor"); + QueueID = + 1 + std::distance(ProcModel.ProcResourceDefs.begin(), + std::find(ProcModel.ProcResourceDefs.begin(), + ProcModel.ProcResourceDefs.end(), Queue)); + } + OS << " " << QueueID << ", // Resource Descriptor for the Load Queue\n"; + + QueueID = 0; + if (ProcModel.StoreQueue) { + const Record *Queue = + ProcModel.StoreQueue->getValueAsDef("QueueDescriptor"); + QueueID = + 1 + std::distance(ProcModel.ProcResourceDefs.begin(), + std::find(ProcModel.ProcResourceDefs.begin(), + ProcModel.ProcResourceDefs.end(), Queue)); + } + OS << " " << QueueID << ", // Resource Descriptor for the Store Queue\n"; +} + +void SubtargetEmitter::EmitExtraProcessorInfo(const CodeGenProcModel &ProcModel, + raw_ostream &OS) { + // Generate a table of register file descriptors (one entry per each user + // defined register file), and a table of register costs. + unsigned NumCostEntries = EmitRegisterFileTables(ProcModel, OS); + + // Now generate a table for the extra processor info. + OS << "\nstatic const llvm::MCExtraProcessorInfo " << ProcModel.ModelName + << "ExtraInfo = {\n "; + + // Add information related to the retire control unit. + EmitRetireControlUnitInfo(ProcModel, OS); + + // Add information related to the register files (i.e. where to find register + // file descriptors and register costs). + EmitRegisterFileInfo(ProcModel, ProcModel.RegisterFiles.size(), + NumCostEntries, OS); + + // Add information about load/store queues. + EmitLoadStoreQueueInfo(ProcModel, OS); + + OS << "};\n"; +} + +void SubtargetEmitter::EmitProcessorResources(const CodeGenProcModel &ProcModel, + raw_ostream &OS) { + EmitProcessorResourceSubUnits(ProcModel, OS); + + OS << "\n// {Name, NumUnits, SuperIdx, BufferSize, SubUnitsIdxBegin}\n"; + OS << "static const llvm::MCProcResourceDesc " << ProcModel.ModelName + << "ProcResources" + << "[] = {\n" + << " {\"InvalidUnit\", 0, 0, 0, 0},\n"; + + unsigned SubUnitsOffset = 1; + for (unsigned i = 0, e = ProcModel.ProcResourceDefs.size(); i < e; ++i) { + Record *PRDef = ProcModel.ProcResourceDefs[i]; + + Record *SuperDef = nullptr; + unsigned SuperIdx = 0; + unsigned NumUnits = 0; + const unsigned SubUnitsBeginOffset = SubUnitsOffset; + int BufferSize = PRDef->getValueAsInt("BufferSize"); + if (PRDef->isSubClassOf("ProcResGroup")) { + RecVec ResUnits = PRDef->getValueAsListOfDefs("Resources"); + for (Record *RU : ResUnits) { + NumUnits += RU->getValueAsInt("NumUnits"); + SubUnitsOffset += RU->getValueAsInt("NumUnits"); + } + } + else { + // Find the SuperIdx + if (PRDef->getValueInit("Super")->isComplete()) { + SuperDef = + SchedModels.findProcResUnits(PRDef->getValueAsDef("Super"), + ProcModel, PRDef->getLoc()); + SuperIdx = ProcModel.getProcResourceIdx(SuperDef); + } + NumUnits = PRDef->getValueAsInt("NumUnits"); + } + // Emit the ProcResourceDesc + OS << " {\"" << PRDef->getName() << "\", "; + if (PRDef->getName().size() < 15) + OS.indent(15 - PRDef->getName().size()); + OS << NumUnits << ", " << SuperIdx << ", " << BufferSize << ", "; + if (SubUnitsBeginOffset != SubUnitsOffset) { + OS << ProcModel.ModelName << "ProcResourceSubUnits + " + << SubUnitsBeginOffset; + } else { + OS << "nullptr"; + } + OS << "}, // #" << i+1; + if (SuperDef) + OS << ", Super=" << SuperDef->getName(); + OS << "\n"; + } + OS << "};\n"; +} + +// Find the WriteRes Record that defines processor resources for this +// SchedWrite. +Record *SubtargetEmitter::FindWriteResources( + const CodeGenSchedRW &SchedWrite, const CodeGenProcModel &ProcModel) { + + // Check if the SchedWrite is already subtarget-specific and directly + // specifies a set of processor resources. + if (SchedWrite.TheDef->isSubClassOf("SchedWriteRes")) + return SchedWrite.TheDef; + + Record *AliasDef = nullptr; + for (Record *A : SchedWrite.Aliases) { + const CodeGenSchedRW &AliasRW = + SchedModels.getSchedRW(A->getValueAsDef("AliasRW")); + if (AliasRW.TheDef->getValueInit("SchedModel")->isComplete()) { + Record *ModelDef = AliasRW.TheDef->getValueAsDef("SchedModel"); + if (&SchedModels.getProcModel(ModelDef) != &ProcModel) + continue; + } + if (AliasDef) + PrintFatalError(AliasRW.TheDef->getLoc(), "Multiple aliases " + "defined for processor " + ProcModel.ModelName + + " Ensure only one SchedAlias exists per RW."); + AliasDef = AliasRW.TheDef; + } + if (AliasDef && AliasDef->isSubClassOf("SchedWriteRes")) + return AliasDef; + + // Check this processor's list of write resources. + Record *ResDef = nullptr; + for (Record *WR : ProcModel.WriteResDefs) { + if (!WR->isSubClassOf("WriteRes")) + continue; + if (AliasDef == WR->getValueAsDef("WriteType") + || SchedWrite.TheDef == WR->getValueAsDef("WriteType")) { + if (ResDef) { + PrintFatalError(WR->getLoc(), "Resources are defined for both " + "SchedWrite and its alias on processor " + + ProcModel.ModelName); + } + ResDef = WR; + } + } + // TODO: If ProcModel has a base model (previous generation processor), + // then call FindWriteResources recursively with that model here. + if (!ResDef) { + PrintFatalError(ProcModel.ModelDef->getLoc(), + Twine("Processor does not define resources for ") + + SchedWrite.TheDef->getName()); + } + return ResDef; +} + +/// Find the ReadAdvance record for the given SchedRead on this processor or +/// return NULL. +Record *SubtargetEmitter::FindReadAdvance(const CodeGenSchedRW &SchedRead, + const CodeGenProcModel &ProcModel) { + // Check for SchedReads that directly specify a ReadAdvance. + if (SchedRead.TheDef->isSubClassOf("SchedReadAdvance")) + return SchedRead.TheDef; + + // Check this processor's list of aliases for SchedRead. + Record *AliasDef = nullptr; + for (Record *A : SchedRead.Aliases) { + const CodeGenSchedRW &AliasRW = + SchedModels.getSchedRW(A->getValueAsDef("AliasRW")); + if (AliasRW.TheDef->getValueInit("SchedModel")->isComplete()) { + Record *ModelDef = AliasRW.TheDef->getValueAsDef("SchedModel"); + if (&SchedModels.getProcModel(ModelDef) != &ProcModel) + continue; + } + if (AliasDef) + PrintFatalError(AliasRW.TheDef->getLoc(), "Multiple aliases " + "defined for processor " + ProcModel.ModelName + + " Ensure only one SchedAlias exists per RW."); + AliasDef = AliasRW.TheDef; + } + if (AliasDef && AliasDef->isSubClassOf("SchedReadAdvance")) + return AliasDef; + + // Check this processor's ReadAdvanceList. + Record *ResDef = nullptr; + for (Record *RA : ProcModel.ReadAdvanceDefs) { + if (!RA->isSubClassOf("ReadAdvance")) + continue; + if (AliasDef == RA->getValueAsDef("ReadType") + || SchedRead.TheDef == RA->getValueAsDef("ReadType")) { + if (ResDef) { + PrintFatalError(RA->getLoc(), "Resources are defined for both " + "SchedRead and its alias on processor " + + ProcModel.ModelName); + } + ResDef = RA; + } + } + // TODO: If ProcModel has a base model (previous generation processor), + // then call FindReadAdvance recursively with that model here. + if (!ResDef && SchedRead.TheDef->getName() != "ReadDefault") { + PrintFatalError(ProcModel.ModelDef->getLoc(), + Twine("Processor does not define resources for ") + + SchedRead.TheDef->getName()); + } + return ResDef; +} + +// Expand an explicit list of processor resources into a full list of implied +// resource groups and super resources that cover them. +void SubtargetEmitter::ExpandProcResources(RecVec &PRVec, + std::vector<int64_t> &Cycles, + const CodeGenProcModel &PM) { + assert(PRVec.size() == Cycles.size() && "failed precondition"); + for (unsigned i = 0, e = PRVec.size(); i != e; ++i) { + Record *PRDef = PRVec[i]; + RecVec SubResources; + if (PRDef->isSubClassOf("ProcResGroup")) + SubResources = PRDef->getValueAsListOfDefs("Resources"); + else { + SubResources.push_back(PRDef); + PRDef = SchedModels.findProcResUnits(PRDef, PM, PRDef->getLoc()); + for (Record *SubDef = PRDef; + SubDef->getValueInit("Super")->isComplete();) { + if (SubDef->isSubClassOf("ProcResGroup")) { + // Disallow this for simplicitly. + PrintFatalError(SubDef->getLoc(), "Processor resource group " + " cannot be a super resources."); + } + Record *SuperDef = + SchedModels.findProcResUnits(SubDef->getValueAsDef("Super"), PM, + SubDef->getLoc()); + PRVec.push_back(SuperDef); + Cycles.push_back(Cycles[i]); + SubDef = SuperDef; + } + } + for (Record *PR : PM.ProcResourceDefs) { + if (PR == PRDef || !PR->isSubClassOf("ProcResGroup")) + continue; + RecVec SuperResources = PR->getValueAsListOfDefs("Resources"); + RecIter SubI = SubResources.begin(), SubE = SubResources.end(); + for( ; SubI != SubE; ++SubI) { + if (!is_contained(SuperResources, *SubI)) { + break; + } + } + if (SubI == SubE) { + PRVec.push_back(PR); + Cycles.push_back(Cycles[i]); + } + } + } +} + +// Generate the SchedClass table for this processor and update global +// tables. Must be called for each processor in order. +void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel, + SchedClassTables &SchedTables) { + SchedTables.ProcSchedClasses.resize(SchedTables.ProcSchedClasses.size() + 1); + if (!ProcModel.hasInstrSchedModel()) + return; + + std::vector<MCSchedClassDesc> &SCTab = SchedTables.ProcSchedClasses.back(); + LLVM_DEBUG(dbgs() << "\n+++ SCHED CLASSES (GenSchedClassTables) +++\n"); + for (const CodeGenSchedClass &SC : SchedModels.schedClasses()) { + LLVM_DEBUG(SC.dump(&SchedModels)); + + SCTab.resize(SCTab.size() + 1); + MCSchedClassDesc &SCDesc = SCTab.back(); + // SCDesc.Name is guarded by NDEBUG + SCDesc.NumMicroOps = 0; + SCDesc.BeginGroup = false; + SCDesc.EndGroup = false; + SCDesc.WriteProcResIdx = 0; + SCDesc.WriteLatencyIdx = 0; + SCDesc.ReadAdvanceIdx = 0; + + // A Variant SchedClass has no resources of its own. + bool HasVariants = false; + for (const CodeGenSchedTransition &CGT : + make_range(SC.Transitions.begin(), SC.Transitions.end())) { + if (CGT.ProcIndices[0] == 0 || + is_contained(CGT.ProcIndices, ProcModel.Index)) { + HasVariants = true; + break; + } + } + if (HasVariants) { + SCDesc.NumMicroOps = MCSchedClassDesc::VariantNumMicroOps; + continue; + } + + // Determine if the SchedClass is actually reachable on this processor. If + // not don't try to locate the processor resources, it will fail. + // If ProcIndices contains 0, this class applies to all processors. + assert(!SC.ProcIndices.empty() && "expect at least one procidx"); + if (SC.ProcIndices[0] != 0) { + if (!is_contained(SC.ProcIndices, ProcModel.Index)) + continue; + } + IdxVec Writes = SC.Writes; + IdxVec Reads = SC.Reads; + if (!SC.InstRWs.empty()) { + // This class has a default ReadWrite list which can be overridden by + // InstRW definitions. + Record *RWDef = nullptr; + for (Record *RW : SC.InstRWs) { + Record *RWModelDef = RW->getValueAsDef("SchedModel"); + if (&ProcModel == &SchedModels.getProcModel(RWModelDef)) { + RWDef = RW; + break; + } + } + if (RWDef) { + Writes.clear(); + Reads.clear(); + SchedModels.findRWs(RWDef->getValueAsListOfDefs("OperandReadWrites"), + Writes, Reads); + } + } + if (Writes.empty()) { + // Check this processor's itinerary class resources. + for (Record *I : ProcModel.ItinRWDefs) { + RecVec Matched = I->getValueAsListOfDefs("MatchedItinClasses"); + if (is_contained(Matched, SC.ItinClassDef)) { + SchedModels.findRWs(I->getValueAsListOfDefs("OperandReadWrites"), + Writes, Reads); + break; + } + } + if (Writes.empty()) { + LLVM_DEBUG(dbgs() << ProcModel.ModelName + << " does not have resources for class " << SC.Name + << '\n'); + SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps; + } + } + // Sum resources across all operand writes. + std::vector<MCWriteProcResEntry> WriteProcResources; + std::vector<MCWriteLatencyEntry> WriteLatencies; + std::vector<std::string> WriterNames; + std::vector<MCReadAdvanceEntry> ReadAdvanceEntries; + for (unsigned W : Writes) { + IdxVec WriteSeq; + SchedModels.expandRWSeqForProc(W, WriteSeq, /*IsRead=*/false, + ProcModel); + + // For each operand, create a latency entry. + MCWriteLatencyEntry WLEntry; + WLEntry.Cycles = 0; + unsigned WriteID = WriteSeq.back(); + WriterNames.push_back(SchedModels.getSchedWrite(WriteID).Name); + // If this Write is not referenced by a ReadAdvance, don't distinguish it + // from other WriteLatency entries. + if (!SchedModels.hasReadOfWrite( + SchedModels.getSchedWrite(WriteID).TheDef)) { + WriteID = 0; + } + WLEntry.WriteResourceID = WriteID; + + for (unsigned WS : WriteSeq) { + + Record *WriteRes = + FindWriteResources(SchedModels.getSchedWrite(WS), ProcModel); + + // Mark the parent class as invalid for unsupported write types. + if (WriteRes->getValueAsBit("Unsupported")) { + SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps; + break; + } + WLEntry.Cycles += WriteRes->getValueAsInt("Latency"); + SCDesc.NumMicroOps += WriteRes->getValueAsInt("NumMicroOps"); + SCDesc.BeginGroup |= WriteRes->getValueAsBit("BeginGroup"); + SCDesc.EndGroup |= WriteRes->getValueAsBit("EndGroup"); + SCDesc.BeginGroup |= WriteRes->getValueAsBit("SingleIssue"); + SCDesc.EndGroup |= WriteRes->getValueAsBit("SingleIssue"); + + // Create an entry for each ProcResource listed in WriteRes. + RecVec PRVec = WriteRes->getValueAsListOfDefs("ProcResources"); + std::vector<int64_t> Cycles = + WriteRes->getValueAsListOfInts("ResourceCycles"); + + if (Cycles.empty()) { + // If ResourceCycles is not provided, default to one cycle per + // resource. + Cycles.resize(PRVec.size(), 1); + } else if (Cycles.size() != PRVec.size()) { + // If ResourceCycles is provided, check consistency. + PrintFatalError( + WriteRes->getLoc(), + Twine("Inconsistent resource cycles: !size(ResourceCycles) != " + "!size(ProcResources): ") + .concat(Twine(PRVec.size())) + .concat(" vs ") + .concat(Twine(Cycles.size()))); + } + + ExpandProcResources(PRVec, Cycles, ProcModel); + + for (unsigned PRIdx = 0, PREnd = PRVec.size(); + PRIdx != PREnd; ++PRIdx) { + MCWriteProcResEntry WPREntry; + WPREntry.ProcResourceIdx = ProcModel.getProcResourceIdx(PRVec[PRIdx]); + assert(WPREntry.ProcResourceIdx && "Bad ProcResourceIdx"); + WPREntry.Cycles = Cycles[PRIdx]; + // If this resource is already used in this sequence, add the current + // entry's cycles so that the same resource appears to be used + // serially, rather than multiple parallel uses. This is important for + // in-order machine where the resource consumption is a hazard. + unsigned WPRIdx = 0, WPREnd = WriteProcResources.size(); + for( ; WPRIdx != WPREnd; ++WPRIdx) { + if (WriteProcResources[WPRIdx].ProcResourceIdx + == WPREntry.ProcResourceIdx) { + WriteProcResources[WPRIdx].Cycles += WPREntry.Cycles; + break; + } + } + if (WPRIdx == WPREnd) + WriteProcResources.push_back(WPREntry); + } + } + WriteLatencies.push_back(WLEntry); + } + // Create an entry for each operand Read in this SchedClass. + // Entries must be sorted first by UseIdx then by WriteResourceID. + for (unsigned UseIdx = 0, EndIdx = Reads.size(); + UseIdx != EndIdx; ++UseIdx) { + Record *ReadAdvance = + FindReadAdvance(SchedModels.getSchedRead(Reads[UseIdx]), ProcModel); + if (!ReadAdvance) + continue; + + // Mark the parent class as invalid for unsupported write types. + if (ReadAdvance->getValueAsBit("Unsupported")) { + SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps; + break; + } + RecVec ValidWrites = ReadAdvance->getValueAsListOfDefs("ValidWrites"); + IdxVec WriteIDs; + if (ValidWrites.empty()) + WriteIDs.push_back(0); + else { + for (Record *VW : ValidWrites) { + WriteIDs.push_back(SchedModels.getSchedRWIdx(VW, /*IsRead=*/false)); + } + } + llvm::sort(WriteIDs); + for(unsigned W : WriteIDs) { + MCReadAdvanceEntry RAEntry; + RAEntry.UseIdx = UseIdx; + RAEntry.WriteResourceID = W; + RAEntry.Cycles = ReadAdvance->getValueAsInt("Cycles"); + ReadAdvanceEntries.push_back(RAEntry); + } + } + if (SCDesc.NumMicroOps == MCSchedClassDesc::InvalidNumMicroOps) { + WriteProcResources.clear(); + WriteLatencies.clear(); + ReadAdvanceEntries.clear(); + } + // Add the information for this SchedClass to the global tables using basic + // compression. + // + // WritePrecRes entries are sorted by ProcResIdx. + llvm::sort(WriteProcResources, LessWriteProcResources()); + + SCDesc.NumWriteProcResEntries = WriteProcResources.size(); + std::vector<MCWriteProcResEntry>::iterator WPRPos = + std::search(SchedTables.WriteProcResources.begin(), + SchedTables.WriteProcResources.end(), + WriteProcResources.begin(), WriteProcResources.end()); + if (WPRPos != SchedTables.WriteProcResources.end()) + SCDesc.WriteProcResIdx = WPRPos - SchedTables.WriteProcResources.begin(); + else { + SCDesc.WriteProcResIdx = SchedTables.WriteProcResources.size(); + SchedTables.WriteProcResources.insert(WPRPos, WriteProcResources.begin(), + WriteProcResources.end()); + } + // Latency entries must remain in operand order. + SCDesc.NumWriteLatencyEntries = WriteLatencies.size(); + std::vector<MCWriteLatencyEntry>::iterator WLPos = + std::search(SchedTables.WriteLatencies.begin(), + SchedTables.WriteLatencies.end(), + WriteLatencies.begin(), WriteLatencies.end()); + if (WLPos != SchedTables.WriteLatencies.end()) { + unsigned idx = WLPos - SchedTables.WriteLatencies.begin(); + SCDesc.WriteLatencyIdx = idx; + for (unsigned i = 0, e = WriteLatencies.size(); i < e; ++i) + if (SchedTables.WriterNames[idx + i].find(WriterNames[i]) == + std::string::npos) { + SchedTables.WriterNames[idx + i] += std::string("_") + WriterNames[i]; + } + } + else { + SCDesc.WriteLatencyIdx = SchedTables.WriteLatencies.size(); + SchedTables.WriteLatencies.insert(SchedTables.WriteLatencies.end(), + WriteLatencies.begin(), + WriteLatencies.end()); + SchedTables.WriterNames.insert(SchedTables.WriterNames.end(), + WriterNames.begin(), WriterNames.end()); + } + // ReadAdvanceEntries must remain in operand order. + SCDesc.NumReadAdvanceEntries = ReadAdvanceEntries.size(); + std::vector<MCReadAdvanceEntry>::iterator RAPos = + std::search(SchedTables.ReadAdvanceEntries.begin(), + SchedTables.ReadAdvanceEntries.end(), + ReadAdvanceEntries.begin(), ReadAdvanceEntries.end()); + if (RAPos != SchedTables.ReadAdvanceEntries.end()) + SCDesc.ReadAdvanceIdx = RAPos - SchedTables.ReadAdvanceEntries.begin(); + else { + SCDesc.ReadAdvanceIdx = SchedTables.ReadAdvanceEntries.size(); + SchedTables.ReadAdvanceEntries.insert(RAPos, ReadAdvanceEntries.begin(), + ReadAdvanceEntries.end()); + } + } +} + +// Emit SchedClass tables for all processors and associated global tables. +void SubtargetEmitter::EmitSchedClassTables(SchedClassTables &SchedTables, + raw_ostream &OS) { + // Emit global WriteProcResTable. + OS << "\n// {ProcResourceIdx, Cycles}\n" + << "extern const llvm::MCWriteProcResEntry " + << Target << "WriteProcResTable[] = {\n" + << " { 0, 0}, // Invalid\n"; + for (unsigned WPRIdx = 1, WPREnd = SchedTables.WriteProcResources.size(); + WPRIdx != WPREnd; ++WPRIdx) { + MCWriteProcResEntry &WPREntry = SchedTables.WriteProcResources[WPRIdx]; + OS << " {" << format("%2d", WPREntry.ProcResourceIdx) << ", " + << format("%2d", WPREntry.Cycles) << "}"; + if (WPRIdx + 1 < WPREnd) + OS << ','; + OS << " // #" << WPRIdx << '\n'; + } + OS << "}; // " << Target << "WriteProcResTable\n"; + + // Emit global WriteLatencyTable. + OS << "\n// {Cycles, WriteResourceID}\n" + << "extern const llvm::MCWriteLatencyEntry " + << Target << "WriteLatencyTable[] = {\n" + << " { 0, 0}, // Invalid\n"; + for (unsigned WLIdx = 1, WLEnd = SchedTables.WriteLatencies.size(); + WLIdx != WLEnd; ++WLIdx) { + MCWriteLatencyEntry &WLEntry = SchedTables.WriteLatencies[WLIdx]; + OS << " {" << format("%2d", WLEntry.Cycles) << ", " + << format("%2d", WLEntry.WriteResourceID) << "}"; + if (WLIdx + 1 < WLEnd) + OS << ','; + OS << " // #" << WLIdx << " " << SchedTables.WriterNames[WLIdx] << '\n'; + } + OS << "}; // " << Target << "WriteLatencyTable\n"; + + // Emit global ReadAdvanceTable. + OS << "\n// {UseIdx, WriteResourceID, Cycles}\n" + << "extern const llvm::MCReadAdvanceEntry " + << Target << "ReadAdvanceTable[] = {\n" + << " {0, 0, 0}, // Invalid\n"; + for (unsigned RAIdx = 1, RAEnd = SchedTables.ReadAdvanceEntries.size(); + RAIdx != RAEnd; ++RAIdx) { + MCReadAdvanceEntry &RAEntry = SchedTables.ReadAdvanceEntries[RAIdx]; + OS << " {" << RAEntry.UseIdx << ", " + << format("%2d", RAEntry.WriteResourceID) << ", " + << format("%2d", RAEntry.Cycles) << "}"; + if (RAIdx + 1 < RAEnd) + OS << ','; + OS << " // #" << RAIdx << '\n'; + } + OS << "}; // " << Target << "ReadAdvanceTable\n"; + + // Emit a SchedClass table for each processor. + for (CodeGenSchedModels::ProcIter PI = SchedModels.procModelBegin(), + PE = SchedModels.procModelEnd(); PI != PE; ++PI) { + if (!PI->hasInstrSchedModel()) + continue; + + std::vector<MCSchedClassDesc> &SCTab = + SchedTables.ProcSchedClasses[1 + (PI - SchedModels.procModelBegin())]; + + OS << "\n// {Name, NumMicroOps, BeginGroup, EndGroup," + << " WriteProcResIdx,#, WriteLatencyIdx,#, ReadAdvanceIdx,#}\n"; + OS << "static const llvm::MCSchedClassDesc " + << PI->ModelName << "SchedClasses[] = {\n"; + + // The first class is always invalid. We no way to distinguish it except by + // name and position. + assert(SchedModels.getSchedClass(0).Name == "NoInstrModel" + && "invalid class not first"); + OS << " {DBGFIELD(\"InvalidSchedClass\") " + << MCSchedClassDesc::InvalidNumMicroOps + << ", false, false, 0, 0, 0, 0, 0, 0},\n"; + + for (unsigned SCIdx = 1, SCEnd = SCTab.size(); SCIdx != SCEnd; ++SCIdx) { + MCSchedClassDesc &MCDesc = SCTab[SCIdx]; + const CodeGenSchedClass &SchedClass = SchedModels.getSchedClass(SCIdx); + OS << " {DBGFIELD(\"" << SchedClass.Name << "\") "; + if (SchedClass.Name.size() < 18) + OS.indent(18 - SchedClass.Name.size()); + OS << MCDesc.NumMicroOps + << ", " << ( MCDesc.BeginGroup ? "true" : "false" ) + << ", " << ( MCDesc.EndGroup ? "true" : "false" ) + << ", " << format("%2d", MCDesc.WriteProcResIdx) + << ", " << MCDesc.NumWriteProcResEntries + << ", " << format("%2d", MCDesc.WriteLatencyIdx) + << ", " << MCDesc.NumWriteLatencyEntries + << ", " << format("%2d", MCDesc.ReadAdvanceIdx) + << ", " << MCDesc.NumReadAdvanceEntries + << "}, // #" << SCIdx << '\n'; + } + OS << "}; // " << PI->ModelName << "SchedClasses\n"; + } +} + +void SubtargetEmitter::EmitProcessorModels(raw_ostream &OS) { + // For each processor model. + for (const CodeGenProcModel &PM : SchedModels.procModels()) { + // Emit extra processor info if available. + if (PM.hasExtraProcessorInfo()) + EmitExtraProcessorInfo(PM, OS); + // Emit processor resource table. + if (PM.hasInstrSchedModel()) + EmitProcessorResources(PM, OS); + else if(!PM.ProcResourceDefs.empty()) + PrintFatalError(PM.ModelDef->getLoc(), "SchedMachineModel defines " + "ProcResources without defining WriteRes SchedWriteRes"); + + // Begin processor itinerary properties + OS << "\n"; + OS << "static const llvm::MCSchedModel " << PM.ModelName << " = {\n"; + EmitProcessorProp(OS, PM.ModelDef, "IssueWidth", ','); + EmitProcessorProp(OS, PM.ModelDef, "MicroOpBufferSize", ','); + EmitProcessorProp(OS, PM.ModelDef, "LoopMicroOpBufferSize", ','); + EmitProcessorProp(OS, PM.ModelDef, "LoadLatency", ','); + EmitProcessorProp(OS, PM.ModelDef, "HighLatency", ','); + EmitProcessorProp(OS, PM.ModelDef, "MispredictPenalty", ','); + + bool PostRAScheduler = + (PM.ModelDef ? PM.ModelDef->getValueAsBit("PostRAScheduler") : false); + + OS << " " << (PostRAScheduler ? "true" : "false") << ", // " + << "PostRAScheduler\n"; + + bool CompleteModel = + (PM.ModelDef ? PM.ModelDef->getValueAsBit("CompleteModel") : false); + + OS << " " << (CompleteModel ? "true" : "false") << ", // " + << "CompleteModel\n"; + + OS << " " << PM.Index << ", // Processor ID\n"; + if (PM.hasInstrSchedModel()) + OS << " " << PM.ModelName << "ProcResources" << ",\n" + << " " << PM.ModelName << "SchedClasses" << ",\n" + << " " << PM.ProcResourceDefs.size()+1 << ",\n" + << " " << (SchedModels.schedClassEnd() + - SchedModels.schedClassBegin()) << ",\n"; + else + OS << " nullptr, nullptr, 0, 0," + << " // No instruction-level machine model.\n"; + if (PM.hasItineraries()) + OS << " " << PM.ItinsDef->getName() << ",\n"; + else + OS << " nullptr, // No Itinerary\n"; + if (PM.hasExtraProcessorInfo()) + OS << " &" << PM.ModelName << "ExtraInfo,\n"; + else + OS << " nullptr // No extra processor descriptor\n"; + OS << "};\n"; + } +} + +// +// EmitSchedModel - Emits all scheduling model tables, folding common patterns. +// +void SubtargetEmitter::EmitSchedModel(raw_ostream &OS) { + OS << "#ifdef DBGFIELD\n" + << "#error \"<target>GenSubtargetInfo.inc requires a DBGFIELD macro\"\n" + << "#endif\n" + << "#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)\n" + << "#define DBGFIELD(x) x,\n" + << "#else\n" + << "#define DBGFIELD(x)\n" + << "#endif\n"; + + if (SchedModels.hasItineraries()) { + std::vector<std::vector<InstrItinerary>> ProcItinLists; + // Emit the stage data + EmitStageAndOperandCycleData(OS, ProcItinLists); + EmitItineraries(OS, ProcItinLists); + } + OS << "\n// ===============================================================\n" + << "// Data tables for the new per-operand machine model.\n"; + + SchedClassTables SchedTables; + for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) { + GenSchedClassTables(ProcModel, SchedTables); + } + EmitSchedClassTables(SchedTables, OS); + + OS << "\n#undef DBGFIELD\n"; + + // Emit the processor machine model + EmitProcessorModels(OS); +} + +static void emitPredicateProlog(const RecordKeeper &Records, raw_ostream &OS) { + std::string Buffer; + raw_string_ostream Stream(Buffer); + + // Collect all the PredicateProlog records and print them to the output + // stream. + std::vector<Record *> Prologs = + Records.getAllDerivedDefinitions("PredicateProlog"); + llvm::sort(Prologs, LessRecord()); + for (Record *P : Prologs) + Stream << P->getValueAsString("Code") << '\n'; + + Stream.flush(); + OS << Buffer; +} + +static void emitPredicates(const CodeGenSchedTransition &T, + const CodeGenSchedClass &SC, PredicateExpander &PE, + raw_ostream &OS) { + std::string Buffer; + raw_string_ostream SS(Buffer); + + auto IsTruePredicate = [](const Record *Rec) { + return Rec->isSubClassOf("MCSchedPredicate") && + Rec->getValueAsDef("Pred")->isSubClassOf("MCTrue"); + }; + + // If not all predicates are MCTrue, then we need an if-stmt. + unsigned NumNonTruePreds = + T.PredTerm.size() - count_if(T.PredTerm, IsTruePredicate); + + SS.indent(PE.getIndentLevel() * 2); + + if (NumNonTruePreds) { + bool FirstNonTruePredicate = true; + SS << "if ("; + + PE.setIndentLevel(PE.getIndentLevel() + 2); + + for (const Record *Rec : T.PredTerm) { + // Skip predicates that evaluate to "true". + if (IsTruePredicate(Rec)) + continue; + + if (FirstNonTruePredicate) { + FirstNonTruePredicate = false; + } else { + SS << "\n"; + SS.indent(PE.getIndentLevel() * 2); + SS << "&& "; + } + + if (Rec->isSubClassOf("MCSchedPredicate")) { + PE.expandPredicate(SS, Rec->getValueAsDef("Pred")); + continue; + } + + // Expand this legacy predicate and wrap it around braces if there is more + // than one predicate to expand. + SS << ((NumNonTruePreds > 1) ? "(" : "") + << Rec->getValueAsString("Predicate") + << ((NumNonTruePreds > 1) ? ")" : ""); + } + + SS << ")\n"; // end of if-stmt + PE.decreaseIndentLevel(); + SS.indent(PE.getIndentLevel() * 2); + PE.decreaseIndentLevel(); + } + + SS << "return " << T.ToClassIdx << "; // " << SC.Name << '\n'; + SS.flush(); + OS << Buffer; +} + +// Used by method `SubtargetEmitter::emitSchedModelHelpersImpl()` to generate +// epilogue code for the auto-generated helper. +void emitSchedModelHelperEpilogue(raw_ostream &OS, bool ShouldReturnZero) { + if (ShouldReturnZero) { + OS << " // Don't know how to resolve this scheduling class.\n" + << " return 0;\n"; + return; + } + + OS << " report_fatal_error(\"Expected a variant SchedClass\");\n"; +} + +bool hasMCSchedPredicates(const CodeGenSchedTransition &T) { + return all_of(T.PredTerm, [](const Record *Rec) { + return Rec->isSubClassOf("MCSchedPredicate"); + }); +} + +void collectVariantClasses(const CodeGenSchedModels &SchedModels, + IdxVec &VariantClasses, + bool OnlyExpandMCInstPredicates) { + for (const CodeGenSchedClass &SC : SchedModels.schedClasses()) { + // Ignore non-variant scheduling classes. + if (SC.Transitions.empty()) + continue; + + if (OnlyExpandMCInstPredicates) { + // Ignore this variant scheduling class no transitions use any meaningful + // MCSchedPredicate definitions. + if (!any_of(SC.Transitions, [](const CodeGenSchedTransition &T) { + return hasMCSchedPredicates(T); + })) + continue; + } + + VariantClasses.push_back(SC.Index); + } +} + +void collectProcessorIndices(const CodeGenSchedClass &SC, IdxVec &ProcIndices) { + // A variant scheduling class may define transitions for multiple + // processors. This function identifies wich processors are associated with + // transition rules specified by variant class `SC`. + for (const CodeGenSchedTransition &T : SC.Transitions) { + IdxVec PI; + std::set_union(T.ProcIndices.begin(), T.ProcIndices.end(), + ProcIndices.begin(), ProcIndices.end(), + std::back_inserter(PI)); + ProcIndices.swap(PI); + } +} + +void SubtargetEmitter::emitSchedModelHelpersImpl( + raw_ostream &OS, bool OnlyExpandMCInstPredicates) { + IdxVec VariantClasses; + collectVariantClasses(SchedModels, VariantClasses, + OnlyExpandMCInstPredicates); + + if (VariantClasses.empty()) { + emitSchedModelHelperEpilogue(OS, OnlyExpandMCInstPredicates); + return; + } + + // Construct a switch statement where the condition is a check on the + // scheduling class identifier. There is a `case` for every variant class + // defined by the processor models of this target. + // Each `case` implements a number of rules to resolve (i.e. to transition from) + // a variant scheduling class to another scheduling class. Rules are + // described by instances of CodeGenSchedTransition. Note that transitions may + // not be valid for all processors. + OS << " switch (SchedClass) {\n"; + for (unsigned VC : VariantClasses) { + IdxVec ProcIndices; + const CodeGenSchedClass &SC = SchedModels.getSchedClass(VC); + collectProcessorIndices(SC, ProcIndices); + + OS << " case " << VC << ": // " << SC.Name << '\n'; + + PredicateExpander PE(Target); + PE.setByRef(false); + PE.setExpandForMC(OnlyExpandMCInstPredicates); + for (unsigned PI : ProcIndices) { + OS << " "; + + // Emit a guard on the processor ID. + if (PI != 0) { + OS << (OnlyExpandMCInstPredicates + ? "if (CPUID == " + : "if (SchedModel->getProcessorID() == "); + OS << PI << ") "; + OS << "{ // " << (SchedModels.procModelBegin() + PI)->ModelName << '\n'; + } + + // Now emit transitions associated with processor PI. + for (const CodeGenSchedTransition &T : SC.Transitions) { + if (PI != 0 && !count(T.ProcIndices, PI)) + continue; + + // Emit only transitions based on MCSchedPredicate, if it's the case. + // At least the transition specified by NoSchedPred is emitted, + // which becomes the default transition for those variants otherwise + // not based on MCSchedPredicate. + // FIXME: preferably, llvm-mca should instead assume a reasonable + // default when a variant transition is not based on MCSchedPredicate + // for a given processor. + if (OnlyExpandMCInstPredicates && !hasMCSchedPredicates(T)) + continue; + + PE.setIndentLevel(3); + emitPredicates(T, SchedModels.getSchedClass(T.ToClassIdx), PE, OS); + } + + OS << " }\n"; + + if (PI == 0) + break; + } + + if (SC.isInferred()) + OS << " return " << SC.Index << ";\n"; + OS << " break;\n"; + } + + OS << " };\n"; + + emitSchedModelHelperEpilogue(OS, OnlyExpandMCInstPredicates); +} + +void SubtargetEmitter::EmitSchedModelHelpers(const std::string &ClassName, + raw_ostream &OS) { + OS << "unsigned " << ClassName + << "\n::resolveSchedClass(unsigned SchedClass, const MachineInstr *MI," + << " const TargetSchedModel *SchedModel) const {\n"; + + // Emit the predicate prolog code. + emitPredicateProlog(Records, OS); + + // Emit target predicates. + emitSchedModelHelpersImpl(OS); + + OS << "} // " << ClassName << "::resolveSchedClass\n\n"; + + OS << "unsigned " << ClassName + << "\n::resolveVariantSchedClass(unsigned SchedClass, const MCInst *MI," + << " unsigned CPUID) const {\n" + << " return " << Target << "_MC" + << "::resolveVariantSchedClassImpl(SchedClass, MI, CPUID);\n" + << "} // " << ClassName << "::resolveVariantSchedClass\n\n"; + + STIPredicateExpander PE(Target); + PE.setClassPrefix(ClassName); + PE.setExpandDefinition(true); + PE.setByRef(false); + PE.setIndentLevel(0); + + for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates()) + PE.expandSTIPredicate(OS, Fn); +} + +void SubtargetEmitter::EmitHwModeCheck(const std::string &ClassName, + raw_ostream &OS) { + const CodeGenHwModes &CGH = TGT.getHwModes(); + assert(CGH.getNumModeIds() > 0); + if (CGH.getNumModeIds() == 1) + return; + + OS << "unsigned " << ClassName << "::getHwMode() const {\n"; + for (unsigned M = 1, NumModes = CGH.getNumModeIds(); M != NumModes; ++M) { + const HwMode &HM = CGH.getMode(M); + OS << " if (checkFeatures(\"" << HM.Features + << "\")) return " << M << ";\n"; + } + OS << " return 0;\n}\n"; +} + +// +// ParseFeaturesFunction - Produces a subtarget specific function for parsing +// the subtarget features string. +// +void SubtargetEmitter::ParseFeaturesFunction(raw_ostream &OS, + unsigned NumFeatures, + unsigned NumProcs) { + std::vector<Record*> Features = + Records.getAllDerivedDefinitions("SubtargetFeature"); + llvm::sort(Features, LessRecord()); + + OS << "// ParseSubtargetFeatures - Parses features string setting specified\n" + << "// subtarget options.\n" + << "void llvm::"; + OS << Target; + OS << "Subtarget::ParseSubtargetFeatures(StringRef CPU, StringRef FS) {\n" + << " LLVM_DEBUG(dbgs() << \"\\nFeatures:\" << FS);\n" + << " LLVM_DEBUG(dbgs() << \"\\nCPU:\" << CPU << \"\\n\\n\");\n"; + + if (Features.empty()) { + OS << "}\n"; + return; + } + + OS << " InitMCProcessorInfo(CPU, FS);\n" + << " const FeatureBitset& Bits = getFeatureBits();\n"; + + for (Record *R : Features) { + // Next record + StringRef Instance = R->getName(); + StringRef Value = R->getValueAsString("Value"); + StringRef Attribute = R->getValueAsString("Attribute"); + + if (Value=="true" || Value=="false") + OS << " if (Bits[" << Target << "::" + << Instance << "]) " + << Attribute << " = " << Value << ";\n"; + else + OS << " if (Bits[" << Target << "::" + << Instance << "] && " + << Attribute << " < " << Value << ") " + << Attribute << " = " << Value << ";\n"; + } + + OS << "}\n"; +} + +void SubtargetEmitter::emitGenMCSubtargetInfo(raw_ostream &OS) { + OS << "namespace " << Target << "_MC {\n" + << "unsigned resolveVariantSchedClassImpl(unsigned SchedClass,\n" + << " const MCInst *MI, unsigned CPUID) {\n"; + emitSchedModelHelpersImpl(OS, /* OnlyExpandMCPredicates */ true); + OS << "}\n"; + OS << "} // end namespace " << Target << "_MC\n\n"; + + OS << "struct " << Target + << "GenMCSubtargetInfo : public MCSubtargetInfo {\n"; + OS << " " << Target << "GenMCSubtargetInfo(const Triple &TT, \n" + << " StringRef CPU, StringRef FS, ArrayRef<SubtargetFeatureKV> PF,\n" + << " ArrayRef<SubtargetSubTypeKV> PD,\n" + << " const MCWriteProcResEntry *WPR,\n" + << " const MCWriteLatencyEntry *WL,\n" + << " const MCReadAdvanceEntry *RA, const InstrStage *IS,\n" + << " const unsigned *OC, const unsigned *FP) :\n" + << " MCSubtargetInfo(TT, CPU, FS, PF, PD,\n" + << " WPR, WL, RA, IS, OC, FP) { }\n\n" + << " unsigned resolveVariantSchedClass(unsigned SchedClass,\n" + << " const MCInst *MI, unsigned CPUID) const override {\n" + << " return " << Target << "_MC" + << "::resolveVariantSchedClassImpl(SchedClass, MI, CPUID); \n"; + OS << " }\n"; + if (TGT.getHwModes().getNumModeIds() > 1) + OS << " unsigned getHwMode() const override;\n"; + OS << "};\n"; + EmitHwModeCheck(Target + "GenMCSubtargetInfo", OS); +} + +void SubtargetEmitter::EmitMCInstrAnalysisPredicateFunctions(raw_ostream &OS) { + OS << "\n#ifdef GET_STIPREDICATE_DECLS_FOR_MC_ANALYSIS\n"; + OS << "#undef GET_STIPREDICATE_DECLS_FOR_MC_ANALYSIS\n\n"; + + STIPredicateExpander PE(Target); + PE.setExpandForMC(true); + PE.setByRef(true); + for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates()) + PE.expandSTIPredicate(OS, Fn); + + OS << "#endif // GET_STIPREDICATE_DECLS_FOR_MC_ANALYSIS\n\n"; + + OS << "\n#ifdef GET_STIPREDICATE_DEFS_FOR_MC_ANALYSIS\n"; + OS << "#undef GET_STIPREDICATE_DEFS_FOR_MC_ANALYSIS\n\n"; + + std::string ClassPrefix = Target + "MCInstrAnalysis"; + PE.setExpandDefinition(true); + PE.setClassPrefix(ClassPrefix); + PE.setIndentLevel(0); + for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates()) + PE.expandSTIPredicate(OS, Fn); + + OS << "#endif // GET_STIPREDICATE_DEFS_FOR_MC_ANALYSIS\n\n"; +} + +// +// SubtargetEmitter::run - Main subtarget enumeration emitter. +// +void SubtargetEmitter::run(raw_ostream &OS) { + emitSourceFileHeader("Subtarget Enumeration Source Fragment", OS); + + OS << "\n#ifdef GET_SUBTARGETINFO_ENUM\n"; + OS << "#undef GET_SUBTARGETINFO_ENUM\n\n"; + + DenseMap<Record *, unsigned> FeatureMap; + + OS << "namespace llvm {\n"; + Enumeration(OS, FeatureMap); + OS << "} // end namespace llvm\n\n"; + OS << "#endif // GET_SUBTARGETINFO_ENUM\n\n"; + + OS << "\n#ifdef GET_SUBTARGETINFO_MC_DESC\n"; + OS << "#undef GET_SUBTARGETINFO_MC_DESC\n\n"; + + OS << "namespace llvm {\n"; +#if 0 + OS << "namespace {\n"; +#endif + unsigned NumFeatures = FeatureKeyValues(OS, FeatureMap); + OS << "\n"; + EmitSchedModel(OS); + OS << "\n"; + unsigned NumProcs = CPUKeyValues(OS, FeatureMap); + OS << "\n"; +#if 0 + OS << "} // end anonymous namespace\n\n"; +#endif + + // MCInstrInfo initialization routine. + emitGenMCSubtargetInfo(OS); + + OS << "\nstatic inline MCSubtargetInfo *create" << Target + << "MCSubtargetInfoImpl(" + << "const Triple &TT, StringRef CPU, StringRef FS) {\n"; + OS << " return new " << Target << "GenMCSubtargetInfo(TT, CPU, FS, "; + if (NumFeatures) + OS << Target << "FeatureKV, "; + else + OS << "None, "; + if (NumProcs) + OS << Target << "SubTypeKV, "; + else + OS << "None, "; + OS << '\n'; OS.indent(22); + OS << Target << "WriteProcResTable, " + << Target << "WriteLatencyTable, " + << Target << "ReadAdvanceTable, "; + OS << '\n'; OS.indent(22); + if (SchedModels.hasItineraries()) { + OS << Target << "Stages, " + << Target << "OperandCycles, " + << Target << "ForwardingPaths"; + } else + OS << "nullptr, nullptr, nullptr"; + OS << ");\n}\n\n"; + + OS << "} // end namespace llvm\n\n"; + + OS << "#endif // GET_SUBTARGETINFO_MC_DESC\n\n"; + + OS << "\n#ifdef GET_SUBTARGETINFO_TARGET_DESC\n"; + OS << "#undef GET_SUBTARGETINFO_TARGET_DESC\n\n"; + + OS << "#include \"llvm/Support/Debug.h\"\n"; + OS << "#include \"llvm/Support/raw_ostream.h\"\n\n"; + ParseFeaturesFunction(OS, NumFeatures, NumProcs); + + OS << "#endif // GET_SUBTARGETINFO_TARGET_DESC\n\n"; + + // Create a TargetSubtargetInfo subclass to hide the MC layer initialization. + OS << "\n#ifdef GET_SUBTARGETINFO_HEADER\n"; + OS << "#undef GET_SUBTARGETINFO_HEADER\n\n"; + + std::string ClassName = Target + "GenSubtargetInfo"; + OS << "namespace llvm {\n"; + OS << "class DFAPacketizer;\n"; + OS << "namespace " << Target << "_MC {\n" + << "unsigned resolveVariantSchedClassImpl(unsigned SchedClass," + << " const MCInst *MI, unsigned CPUID);\n" + << "} // end namespace " << Target << "_MC\n\n"; + OS << "struct " << ClassName << " : public TargetSubtargetInfo {\n" + << " explicit " << ClassName << "(const Triple &TT, StringRef CPU, " + << "StringRef FS);\n" + << "public:\n" + << " unsigned resolveSchedClass(unsigned SchedClass, " + << " const MachineInstr *DefMI," + << " const TargetSchedModel *SchedModel) const override;\n" + << " unsigned resolveVariantSchedClass(unsigned SchedClass," + << " const MCInst *MI, unsigned CPUID) const override;\n" + << " DFAPacketizer *createDFAPacketizer(const InstrItineraryData *IID)" + << " const;\n"; + if (TGT.getHwModes().getNumModeIds() > 1) + OS << " unsigned getHwMode() const override;\n"; + + STIPredicateExpander PE(Target); + PE.setByRef(false); + for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates()) + PE.expandSTIPredicate(OS, Fn); + + OS << "};\n" + << "} // end namespace llvm\n\n"; + + OS << "#endif // GET_SUBTARGETINFO_HEADER\n\n"; + + OS << "\n#ifdef GET_SUBTARGETINFO_CTOR\n"; + OS << "#undef GET_SUBTARGETINFO_CTOR\n\n"; + + OS << "#include \"llvm/CodeGen/TargetSchedule.h\"\n\n"; + OS << "namespace llvm {\n"; + OS << "extern const llvm::SubtargetFeatureKV " << Target << "FeatureKV[];\n"; + OS << "extern const llvm::SubtargetSubTypeKV " << Target << "SubTypeKV[];\n"; + OS << "extern const llvm::MCWriteProcResEntry " + << Target << "WriteProcResTable[];\n"; + OS << "extern const llvm::MCWriteLatencyEntry " + << Target << "WriteLatencyTable[];\n"; + OS << "extern const llvm::MCReadAdvanceEntry " + << Target << "ReadAdvanceTable[];\n"; + + if (SchedModels.hasItineraries()) { + OS << "extern const llvm::InstrStage " << Target << "Stages[];\n"; + OS << "extern const unsigned " << Target << "OperandCycles[];\n"; + OS << "extern const unsigned " << Target << "ForwardingPaths[];\n"; + } + + OS << ClassName << "::" << ClassName << "(const Triple &TT, StringRef CPU, " + << "StringRef FS)\n" + << " : TargetSubtargetInfo(TT, CPU, FS, "; + if (NumFeatures) + OS << "makeArrayRef(" << Target << "FeatureKV, " << NumFeatures << "), "; + else + OS << "None, "; + if (NumProcs) + OS << "makeArrayRef(" << Target << "SubTypeKV, " << NumProcs << "), "; + else + OS << "None, "; + OS << '\n'; OS.indent(24); + OS << Target << "WriteProcResTable, " + << Target << "WriteLatencyTable, " + << Target << "ReadAdvanceTable, "; + OS << '\n'; OS.indent(24); + if (SchedModels.hasItineraries()) { + OS << Target << "Stages, " + << Target << "OperandCycles, " + << Target << "ForwardingPaths"; + } else + OS << "nullptr, nullptr, nullptr"; + OS << ") {}\n\n"; + + EmitSchedModelHelpers(ClassName, OS); + EmitHwModeCheck(ClassName, OS); + + OS << "} // end namespace llvm\n\n"; + + OS << "#endif // GET_SUBTARGETINFO_CTOR\n\n"; + + EmitMCInstrAnalysisPredicateFunctions(OS); +} + +namespace llvm { + +void EmitSubtarget(RecordKeeper &RK, raw_ostream &OS) { + CodeGenTarget CGTarget(RK); + SubtargetEmitter(RK, CGTarget).run(OS); +} + +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/SubtargetFeatureInfo.cpp b/contrib/llvm-project/llvm/utils/TableGen/SubtargetFeatureInfo.cpp new file mode 100644 index 000000000000..3821f4757464 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/SubtargetFeatureInfo.cpp @@ -0,0 +1,165 @@ +//===- SubtargetFeatureInfo.cpp - Helpers for subtarget features ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "SubtargetFeatureInfo.h" + +#include "Types.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/TableGen/Record.h" + +#include <map> + +using namespace llvm; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +LLVM_DUMP_METHOD void SubtargetFeatureInfo::dump() const { + errs() << getEnumName() << " " << Index << "\n" << *TheDef; +} +#endif + +std::vector<std::pair<Record *, SubtargetFeatureInfo>> +SubtargetFeatureInfo::getAll(const RecordKeeper &Records) { + std::vector<std::pair<Record *, SubtargetFeatureInfo>> SubtargetFeatures; + std::vector<Record *> AllPredicates = + Records.getAllDerivedDefinitions("Predicate"); + for (Record *Pred : AllPredicates) { + // Ignore predicates that are not intended for the assembler. + // + // The "AssemblerMatcherPredicate" string should be promoted to an argument + // if we re-use the machinery for non-assembler purposes in future. + if (!Pred->getValueAsBit("AssemblerMatcherPredicate")) + continue; + + if (Pred->getName().empty()) + PrintFatalError(Pred->getLoc(), "Predicate has no name!"); + + // Ignore always true predicates. + if (Pred->getValueAsString("CondString").empty()) + continue; + + SubtargetFeatures.emplace_back( + Pred, SubtargetFeatureInfo(Pred, SubtargetFeatures.size())); + } + return SubtargetFeatures; +} + +void SubtargetFeatureInfo::emitSubtargetFeatureBitEnumeration( + SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS) { + OS << "// Bits for subtarget features that participate in " + << "instruction matching.\n"; + OS << "enum SubtargetFeatureBits : " + << getMinimalTypeForRange(SubtargetFeatures.size()) << " {\n"; + for (const auto &SF : SubtargetFeatures) { + const SubtargetFeatureInfo &SFI = SF.second; + OS << " " << SFI.getEnumBitName() << " = " << SFI.Index << ",\n"; + } + OS << "};\n\n"; +} + +void SubtargetFeatureInfo::emitNameTable( + SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS) { + // Need to sort the name table so that lookup by the log of the enum value + // gives the proper name. More specifically, for a feature of value 1<<n, + // SubtargetFeatureNames[n] should be the name of the feature. + uint64_t IndexUB = 0; + for (const auto &SF : SubtargetFeatures) + if (IndexUB <= SF.second.Index) + IndexUB = SF.second.Index+1; + + std::vector<std::string> Names; + if (IndexUB > 0) + Names.resize(IndexUB); + for (const auto &SF : SubtargetFeatures) + Names[SF.second.Index] = SF.second.getEnumName(); + + OS << "static const char *SubtargetFeatureNames[] = {\n"; + for (uint64_t I = 0; I < IndexUB; ++I) + OS << " \"" << Names[I] << "\",\n"; + + // A small number of targets have no predicates. Null terminate the array to + // avoid a zero-length array. + OS << " nullptr\n" + << "};\n\n"; +} + +void SubtargetFeatureInfo::emitComputeAvailableFeatures( + StringRef TargetName, StringRef ClassName, StringRef FuncName, + SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS, + StringRef ExtraParams) { + OS << "PredicateBitset " << TargetName << ClassName << "::\n" + << FuncName << "(const " << TargetName << "Subtarget *Subtarget"; + if (!ExtraParams.empty()) + OS << ", " << ExtraParams; + OS << ") const {\n"; + OS << " PredicateBitset Features;\n"; + for (const auto &SF : SubtargetFeatures) { + const SubtargetFeatureInfo &SFI = SF.second; + StringRef CondStr = SFI.TheDef->getValueAsString("CondString"); + assert(!CondStr.empty() && "true predicate should have been filtered"); + + OS << " if (" << CondStr << ")\n"; + OS << " Features.set(" << SFI.getEnumBitName() << ");\n"; + } + OS << " return Features;\n"; + OS << "}\n\n"; +} + +void SubtargetFeatureInfo::emitComputeAssemblerAvailableFeatures( + StringRef TargetName, StringRef ClassName, StringRef FuncName, + SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS) { + OS << "FeatureBitset " << TargetName << ClassName << "::\n" + << FuncName << "(const FeatureBitset& FB) const {\n"; + OS << " FeatureBitset Features;\n"; + for (const auto &SF : SubtargetFeatures) { + const SubtargetFeatureInfo &SFI = SF.second; + + OS << " if ("; + + const DagInit *D = SFI.TheDef->getValueAsDag("AssemblerCondDag"); + std::string CombineType = D->getOperator()->getAsString(); + if (CombineType != "any_of" && CombineType != "all_of") + PrintFatalError(SFI.TheDef->getLoc(), "Invalid AssemblerCondDag!"); + if (D->getNumArgs() == 0) + PrintFatalError(SFI.TheDef->getLoc(), "Invalid AssemblerCondDag!"); + bool IsOr = CombineType == "any_of"; + + if (IsOr) + OS << "("; + + bool First = true; + for (auto *Arg : D->getArgs()) { + if (!First) { + if (IsOr) + OS << " || "; + else + OS << " && "; + } + if (auto *NotArg = dyn_cast<DagInit>(Arg)) { + if (NotArg->getOperator()->getAsString() != "not" || + NotArg->getNumArgs() != 1) + PrintFatalError(SFI.TheDef->getLoc(), "Invalid AssemblerCondDag!"); + Arg = NotArg->getArg(0); + OS << "!"; + } + if (!isa<DefInit>(Arg) || + !cast<DefInit>(Arg)->getDef()->isSubClassOf("SubtargetFeature")) + PrintFatalError(SFI.TheDef->getLoc(), "Invalid AssemblerCondDag!"); + OS << "FB[" << TargetName << "::" << Arg->getAsString() << "]"; + + First = false; + } + + if (IsOr) + OS << ")"; + + OS << ")\n"; + OS << " Features.set(" << SFI.getEnumBitName() << ");\n"; + } + OS << " return Features;\n"; + OS << "}\n\n"; +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/SubtargetFeatureInfo.h b/contrib/llvm-project/llvm/utils/TableGen/SubtargetFeatureInfo.h new file mode 100644 index 000000000000..d72f8b93461f --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/SubtargetFeatureInfo.h @@ -0,0 +1,106 @@ +//===- SubtargetFeatureInfo.h - Helpers for subtarget features --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTIL_TABLEGEN_SUBTARGETFEATUREINFO_H +#define LLVM_UTIL_TABLEGEN_SUBTARGETFEATUREINFO_H + +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" + +#include <map> +#include <string> +#include <vector> + +namespace llvm { +class Record; +class RecordKeeper; + +struct SubtargetFeatureInfo; +using SubtargetFeatureInfoMap = std::map<Record *, SubtargetFeatureInfo, LessRecordByID>; + +/// Helper class for storing information on a subtarget feature which +/// participates in instruction matching. +struct SubtargetFeatureInfo { + /// The predicate record for this feature. + Record *TheDef; + + /// An unique index assigned to represent this feature. + uint64_t Index; + + SubtargetFeatureInfo(Record *D, uint64_t Idx) : TheDef(D), Index(Idx) {} + + /// The name of the enumerated constant identifying this feature. + std::string getEnumName() const { + return "Feature_" + TheDef->getName().str(); + } + + /// The name of the enumerated constant identifying the bitnumber for + /// this feature. + std::string getEnumBitName() const { + return "Feature_" + TheDef->getName().str() + "Bit"; + } + + bool mustRecomputePerFunction() const { + return TheDef->getValueAsBit("RecomputePerFunction"); + } + + void dump() const; + static std::vector<std::pair<Record *, SubtargetFeatureInfo>> + getAll(const RecordKeeper &Records); + + /// Emit the subtarget feature flag definitions. + /// + /// This version emits the bit index for the feature and can therefore support + /// more than 64 feature bits. + static void + emitSubtargetFeatureBitEnumeration(SubtargetFeatureInfoMap &SubtargetFeatures, + raw_ostream &OS); + + static void emitNameTable(SubtargetFeatureInfoMap &SubtargetFeatures, + raw_ostream &OS); + + /// Emit the function to compute the list of available features given a + /// subtarget. + /// + /// This version is used for subtarget features defined using Predicate<> + /// and supports more than 64 feature bits. + /// + /// \param TargetName The name of the target as used in class prefixes (e.g. + /// <TargetName>Subtarget) + /// \param ClassName The name of the class (without the <Target> prefix) + /// that will contain the generated functions. + /// \param FuncName The name of the function to emit. + /// \param SubtargetFeatures A map of TableGen records to the + /// SubtargetFeatureInfo equivalent. + /// \param ExtraParams Additional arguments to the generated function. + static void + emitComputeAvailableFeatures(StringRef TargetName, StringRef ClassName, + StringRef FuncName, + SubtargetFeatureInfoMap &SubtargetFeatures, + raw_ostream &OS, StringRef ExtraParams = ""); + + /// Emit the function to compute the list of available features given a + /// subtarget. + /// + /// This version is used for subtarget features defined using + /// AssemblerPredicate<> and supports up to 64 feature bits. + /// + /// \param TargetName The name of the target as used in class prefixes (e.g. + /// <TargetName>Subtarget) + /// \param ClassName The name of the class (without the <Target> prefix) + /// that will contain the generated functions. + /// \param FuncName The name of the function to emit. + /// \param SubtargetFeatures A map of TableGen records to the + /// SubtargetFeatureInfo equivalent. + static void emitComputeAssemblerAvailableFeatures( + StringRef TargetName, StringRef ClassName, StringRef FuncName, + SubtargetFeatureInfoMap &SubtargetFeatures, raw_ostream &OS); +}; +} // end namespace llvm + +#endif // LLVM_UTIL_TABLEGEN_SUBTARGETFEATUREINFO_H diff --git a/contrib/llvm-project/llvm/utils/TableGen/TableGen.cpp b/contrib/llvm-project/llvm/utils/TableGen/TableGen.cpp new file mode 100644 index 000000000000..8015a58471ca --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/TableGen.cpp @@ -0,0 +1,302 @@ +//===- TableGen.cpp - Top-Level TableGen implementation for LLVM ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the main function for LLVM's TableGen. +// +//===----------------------------------------------------------------------===// + +#include "TableGenBackends.h" // Declares all backends. +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/Signals.h" +#include "llvm/TableGen/Main.h" +#include "llvm/TableGen/Record.h" +#include "llvm/TableGen/SetTheory.h" + +using namespace llvm; + +enum ActionType { + PrintRecords, + DumpJSON, + GenEmitter, + GenRegisterInfo, + GenInstrInfo, + GenInstrDocs, + GenAsmWriter, + GenAsmMatcher, + GenDisassembler, + GenPseudoLowering, + GenCompressInst, + GenCallingConv, + GenDAGISel, + GenDFAPacketizer, + GenFastISel, + GenSubtarget, + GenIntrinsicEnums, + GenIntrinsicImpl, + PrintEnums, + PrintSets, + GenOptParserDefs, + GenOptRST, + GenCTags, + GenAttributes, + GenSearchableTables, + GenGlobalISel, + GenGICombiner, + GenX86EVEX2VEXTables, + GenX86FoldTables, + GenRegisterBank, + GenExegesis, + GenAutomata, + GenDirectivesEnumDecl, + GenDirectivesEnumImpl, + GenDirectivesEnumGen, +}; + +namespace llvm { +/// Storage for TimeRegionsOpt as a global so that backends aren't required to +/// include CommandLine.h +bool TimeRegions = false; +cl::opt<bool> EmitLongStrLiterals( + "long-string-literals", + cl::desc("when emitting large string tables, prefer string literals over " + "comma-separated char literals. This can be a readability and " + "compile-time performance win, but upsets some compilers"), + cl::Hidden, cl::init(true)); +} // end namespace llvm + +namespace { +cl::opt<ActionType> Action( + cl::desc("Action to perform:"), + cl::values( + clEnumValN(PrintRecords, "print-records", + "Print all records to stdout (default)"), + clEnumValN(DumpJSON, "dump-json", + "Dump all records as machine-readable JSON"), + clEnumValN(GenEmitter, "gen-emitter", "Generate machine code emitter"), + clEnumValN(GenRegisterInfo, "gen-register-info", + "Generate registers and register classes info"), + clEnumValN(GenInstrInfo, "gen-instr-info", + "Generate instruction descriptions"), + clEnumValN(GenInstrDocs, "gen-instr-docs", + "Generate instruction documentation"), + clEnumValN(GenCallingConv, "gen-callingconv", + "Generate calling convention descriptions"), + clEnumValN(GenAsmWriter, "gen-asm-writer", "Generate assembly writer"), + clEnumValN(GenDisassembler, "gen-disassembler", + "Generate disassembler"), + clEnumValN(GenPseudoLowering, "gen-pseudo-lowering", + "Generate pseudo instruction lowering"), + clEnumValN(GenCompressInst, "gen-compress-inst-emitter", + "Generate RISCV compressed instructions."), + clEnumValN(GenAsmMatcher, "gen-asm-matcher", + "Generate assembly instruction matcher"), + clEnumValN(GenDAGISel, "gen-dag-isel", + "Generate a DAG instruction selector"), + clEnumValN(GenDFAPacketizer, "gen-dfa-packetizer", + "Generate DFA Packetizer for VLIW targets"), + clEnumValN(GenFastISel, "gen-fast-isel", + "Generate a \"fast\" instruction selector"), + clEnumValN(GenSubtarget, "gen-subtarget", + "Generate subtarget enumerations"), + clEnumValN(GenIntrinsicEnums, "gen-intrinsic-enums", + "Generate intrinsic enums"), + clEnumValN(GenIntrinsicImpl, "gen-intrinsic-impl", + "Generate intrinsic information"), + clEnumValN(PrintEnums, "print-enums", "Print enum values for a class"), + clEnumValN(PrintSets, "print-sets", + "Print expanded sets for testing DAG exprs"), + clEnumValN(GenOptParserDefs, "gen-opt-parser-defs", + "Generate option definitions"), + clEnumValN(GenOptRST, "gen-opt-rst", "Generate option RST"), + clEnumValN(GenCTags, "gen-ctags", "Generate ctags-compatible index"), + clEnumValN(GenAttributes, "gen-attrs", "Generate attributes"), + clEnumValN(GenSearchableTables, "gen-searchable-tables", + "Generate generic binary-searchable table"), + clEnumValN(GenGlobalISel, "gen-global-isel", + "Generate GlobalISel selector"), + clEnumValN(GenGICombiner, "gen-global-isel-combiner", + "Generate GlobalISel combiner"), + clEnumValN(GenX86EVEX2VEXTables, "gen-x86-EVEX2VEX-tables", + "Generate X86 EVEX to VEX compress tables"), + clEnumValN(GenX86FoldTables, "gen-x86-fold-tables", + "Generate X86 fold tables"), + clEnumValN(GenRegisterBank, "gen-register-bank", + "Generate registers bank descriptions"), + clEnumValN(GenExegesis, "gen-exegesis", + "Generate llvm-exegesis tables"), + clEnumValN(GenAutomata, "gen-automata", "Generate generic automata"), + clEnumValN(GenDirectivesEnumDecl, "gen-directive-decl", + "Generate directive related declaration code (header file)"), + clEnumValN(GenDirectivesEnumImpl, "gen-directive-impl", + "Generate directive related implementation code"), + clEnumValN(GenDirectivesEnumGen, "gen-directive-gen", + "Generate directive related implementation code part"))); + +cl::OptionCategory PrintEnumsCat("Options for -print-enums"); +cl::opt<std::string> Class("class", cl::desc("Print Enum list for this class"), + cl::value_desc("class name"), + cl::cat(PrintEnumsCat)); + +cl::opt<bool, true> + TimeRegionsOpt("time-regions", + cl::desc("Time regions of tablegens execution"), + cl::location(TimeRegions)); + +bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) { + switch (Action) { + case PrintRecords: + OS << Records; // No argument, dump all contents + break; + case DumpJSON: + EmitJSON(Records, OS); + break; + case GenEmitter: + EmitCodeEmitter(Records, OS); + break; + case GenRegisterInfo: + EmitRegisterInfo(Records, OS); + break; + case GenInstrInfo: + EmitInstrInfo(Records, OS); + break; + case GenInstrDocs: + EmitInstrDocs(Records, OS); + break; + case GenCallingConv: + EmitCallingConv(Records, OS); + break; + case GenAsmWriter: + EmitAsmWriter(Records, OS); + break; + case GenAsmMatcher: + EmitAsmMatcher(Records, OS); + break; + case GenDisassembler: + EmitDisassembler(Records, OS); + break; + case GenPseudoLowering: + EmitPseudoLowering(Records, OS); + break; + case GenCompressInst: + EmitCompressInst(Records, OS); + break; + case GenDAGISel: + EmitDAGISel(Records, OS); + break; + case GenDFAPacketizer: + EmitDFAPacketizer(Records, OS); + break; + case GenFastISel: + EmitFastISel(Records, OS); + break; + case GenSubtarget: + EmitSubtarget(Records, OS); + break; + case GenIntrinsicEnums: + EmitIntrinsicEnums(Records, OS); + break; + case GenIntrinsicImpl: + EmitIntrinsicImpl(Records, OS); + break; + case GenOptParserDefs: + EmitOptParser(Records, OS); + break; + case GenOptRST: + EmitOptRST(Records, OS); + break; + case PrintEnums: + { + for (Record *Rec : Records.getAllDerivedDefinitions(Class)) + OS << Rec->getName() << ", "; + OS << "\n"; + break; + } + case PrintSets: + { + SetTheory Sets; + Sets.addFieldExpander("Set", "Elements"); + for (Record *Rec : Records.getAllDerivedDefinitions("Set")) { + OS << Rec->getName() << " = ["; + const std::vector<Record*> *Elts = Sets.expand(Rec); + assert(Elts && "Couldn't expand Set instance"); + for (Record *Elt : *Elts) + OS << ' ' << Elt->getName(); + OS << " ]\n"; + } + break; + } + case GenCTags: + EmitCTags(Records, OS); + break; + case GenAttributes: + EmitAttributes(Records, OS); + break; + case GenSearchableTables: + EmitSearchableTables(Records, OS); + break; + case GenGlobalISel: + EmitGlobalISel(Records, OS); + break; + case GenGICombiner: + EmitGICombiner(Records, OS); + break; + case GenRegisterBank: + EmitRegisterBank(Records, OS); + break; + case GenX86EVEX2VEXTables: + EmitX86EVEX2VEXTables(Records, OS); + break; + case GenX86FoldTables: + EmitX86FoldTables(Records, OS); + break; + case GenExegesis: + EmitExegesis(Records, OS); + break; + case GenAutomata: + EmitAutomata(Records, OS); + break; + case GenDirectivesEnumDecl: + EmitDirectivesDecl(Records, OS); + break; + case GenDirectivesEnumImpl: + EmitDirectivesImpl(Records, OS); + break; + case GenDirectivesEnumGen: + EmitDirectivesGen(Records, OS); + break; + } + + return false; +} +} + +int main(int argc, char **argv) { + sys::PrintStackTraceOnErrorSignal(argv[0]); + PrettyStackTraceProgram X(argc, argv); + cl::ParseCommandLineOptions(argc, argv); + + llvm_shutdown_obj Y; + + return TableGenMain(argv[0], &LLVMTableGenMain); +} + +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) || \ + __has_feature(leak_sanitizer) + +#include <sanitizer/lsan_interface.h> +// Disable LeakSanitizer for this binary as it has too many leaks that are not +// very interesting to fix. See compiler-rt/include/sanitizer/lsan_interface.h . +LLVM_ATTRIBUTE_USED int __lsan_is_turned_off() { return 1; } + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/TableGenBackends.h b/contrib/llvm-project/llvm/utils/TableGen/TableGenBackends.h new file mode 100644 index 000000000000..92204f39f8fa --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/TableGenBackends.h @@ -0,0 +1,99 @@ +//===- TableGenBackends.h - Declarations for LLVM TableGen Backends -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the declarations for all of the LLVM TableGen +// backends. A "TableGen backend" is just a function. See below for a +// precise description. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_TABLEGENBACKENDS_H +#define LLVM_UTILS_TABLEGEN_TABLEGENBACKENDS_H + +// A TableGen backend is a function that looks like +// +// EmitFoo(RecordKeeper &RK, raw_ostream &OS /*, anything else you need */ ) +// +// What you do inside of that function is up to you, but it will usually +// involve generating C++ code to the provided raw_ostream. +// +// The RecordKeeper is just a top-level container for an in-memory +// representation of the data encoded in the TableGen file. What a TableGen +// backend does is walk around that in-memory representation and generate +// stuff based on the information it contains. +// +// The in-memory representation is a node-graph (think of it like JSON but +// with a richer ontology of types), where the nodes are subclasses of +// Record. The methods `getClass`, `getDef` are the basic interface to +// access the node-graph. RecordKeeper also provides a handy method +// `getAllDerivedDefinitions`. Consult "include/llvm/TableGen/Record.h" for +// the exact interfaces provided by Record's and RecordKeeper. +// +// A common pattern for TableGen backends is for the EmitFoo function to +// instantiate a class which holds some context for the generation process, +// and then have most of the work happen in that class's methods. This +// pattern partly has historical roots in the previous TableGen backend API +// that involved a class and an invocation like `FooEmitter(RK).run(OS)`. +// +// Remember to wrap private things in an anonymous namespace. For most +// backends, this means that the EmitFoo function is the only thing not in +// the anonymous namespace. + + +// FIXME: Reorganize TableGen so that build dependencies can be more +// accurately expressed. Currently, touching any of the emitters (or +// anything that they transitively depend on) causes everything dependent +// on TableGen to be rebuilt (this includes all the targets!). Perhaps have +// a standalone TableGen binary and have the backends be loadable modules +// of some sort; then the dependency could be expressed as being on the +// module, and all the modules would have a common dependency on the +// TableGen binary with as few dependencies as possible on the rest of +// LLVM. + + +namespace llvm { + +class raw_ostream; +class RecordKeeper; + +void EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS); +void EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS); +void EmitAsmMatcher(RecordKeeper &RK, raw_ostream &OS); +void EmitAsmWriter(RecordKeeper &RK, raw_ostream &OS); +void EmitCallingConv(RecordKeeper &RK, raw_ostream &OS); +void EmitCodeEmitter(RecordKeeper &RK, raw_ostream &OS); +void EmitDAGISel(RecordKeeper &RK, raw_ostream &OS); +void EmitDFAPacketizer(RecordKeeper &RK, raw_ostream &OS); +void EmitDisassembler(RecordKeeper &RK, raw_ostream &OS); +void EmitFastISel(RecordKeeper &RK, raw_ostream &OS); +void EmitInstrInfo(RecordKeeper &RK, raw_ostream &OS); +void EmitInstrDocs(RecordKeeper &RK, raw_ostream &OS); +void EmitPseudoLowering(RecordKeeper &RK, raw_ostream &OS); +void EmitCompressInst(RecordKeeper &RK, raw_ostream &OS); +void EmitRegisterInfo(RecordKeeper &RK, raw_ostream &OS); +void EmitSubtarget(RecordKeeper &RK, raw_ostream &OS); +void EmitMapTable(RecordKeeper &RK, raw_ostream &OS); +void EmitOptParser(RecordKeeper &RK, raw_ostream &OS); +void EmitOptRST(RecordKeeper &RK, raw_ostream &OS); +void EmitCTags(RecordKeeper &RK, raw_ostream &OS); +void EmitAttributes(RecordKeeper &RK, raw_ostream &OS); +void EmitSearchableTables(RecordKeeper &RK, raw_ostream &OS); +void EmitGlobalISel(RecordKeeper &RK, raw_ostream &OS); +void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS); +void EmitX86EVEX2VEXTables(RecordKeeper &RK, raw_ostream &OS); +void EmitX86FoldTables(RecordKeeper &RK, raw_ostream &OS); +void EmitRegisterBank(RecordKeeper &RK, raw_ostream &OS); +void EmitExegesis(RecordKeeper &RK, raw_ostream &OS); +void EmitAutomata(RecordKeeper &RK, raw_ostream &OS); +void EmitDirectivesDecl(RecordKeeper &RK, raw_ostream &OS); +void EmitDirectivesImpl(RecordKeeper &RK, raw_ostream &OS); +void EmitDirectivesGen(RecordKeeper &RK, raw_ostream &OS); + +} // End llvm namespace + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/Types.cpp b/contrib/llvm-project/llvm/utils/TableGen/Types.cpp new file mode 100644 index 000000000000..a6682da90e6b --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/Types.cpp @@ -0,0 +1,44 @@ +//===- Types.cpp - Helper for the selection of C++ data types. ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Types.h" + +// For LLVM_ATTRIBUTE_UNUSED +#include "llvm/Support/Compiler.h" + +#include <cassert> + +using namespace llvm; + +const char *llvm::getMinimalTypeForRange(uint64_t Range, unsigned MaxSize LLVM_ATTRIBUTE_UNUSED) { + // TODO: The original callers only used 32 and 64 so these are the only + // values permitted. Rather than widen the supported values we should + // allow 64 for the callers that currently use 32 and remove the + // argument altogether. + assert((MaxSize == 32 || MaxSize == 64) && "Unexpected size"); + assert(MaxSize <= 64 && "Unexpected size"); + assert(((MaxSize > 32) ? Range <= 0xFFFFFFFFFFFFFFFFULL + : Range <= 0xFFFFFFFFULL) && + "Enum too large"); + + if (Range > 0xFFFFFFFFULL) + return "uint64_t"; + if (Range > 0xFFFF) + return "uint32_t"; + if (Range > 0xFF) + return "uint16_t"; + return "uint8_t"; +} + +const char *llvm::getMinimalTypeForEnumBitfield(uint64_t Size) { + uint64_t MaxIndex = Size; + if (MaxIndex > 0) + MaxIndex--; + assert(MaxIndex <= 64 && "Too many bits"); + return getMinimalTypeForRange(1ULL << MaxIndex); +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/Types.h b/contrib/llvm-project/llvm/utils/TableGen/Types.h new file mode 100644 index 000000000000..17c7742ccaac --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/Types.h @@ -0,0 +1,24 @@ +//===- Types.h - Helper for the selection of C++ types. ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_TYPES_H +#define LLVM_UTILS_TABLEGEN_TYPES_H + +#include <cstdint> + +namespace llvm { +/// Returns the smallest unsigned integer type that can hold the given range. +/// MaxSize indicates the largest size of integer to consider (in bits) and only +/// supports values of at least 32. +const char *getMinimalTypeForRange(uint64_t Range, unsigned MaxSize = 64); + +/// Returns the smallest unsigned integer type that can hold the given bitfield. +const char *getMinimalTypeForEnumBitfield(uint64_t Size); +} + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp new file mode 100644 index 000000000000..54aa5a8164f2 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp @@ -0,0 +1,173 @@ +//===- WebAssemblyDisassemblerEmitter.cpp - Disassembler tables -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is part of the WebAssembly Disassembler Emitter. +// It contains the implementation of the disassembler tables. +// Documentation for the disassembler emitter in general can be found in +// WebAssemblyDisassemblerEmitter.h. +// +//===----------------------------------------------------------------------===// + +#include "WebAssemblyDisassemblerEmitter.h" +#include "llvm/TableGen/Record.h" + +namespace llvm { + +static constexpr int WebAssemblyInstructionTableSize = 256; + +void emitWebAssemblyDisassemblerTables( + raw_ostream &OS, + const ArrayRef<const CodeGenInstruction *> &NumberedInstructions) { + // First lets organize all opcodes by (prefix) byte. Prefix 0 is the + // starting table. + std::map<unsigned, + std::map<unsigned, std::pair<unsigned, const CodeGenInstruction *>>> + OpcodeTable; + for (unsigned I = 0; I != NumberedInstructions.size(); ++I) { + auto &CGI = *NumberedInstructions[I]; + auto &Def = *CGI.TheDef; + if (!Def.getValue("Inst")) + continue; + auto &Inst = *Def.getValueAsBitsInit("Inst"); + auto Opc = static_cast<unsigned>( + reinterpret_cast<IntInit *>(Inst.convertInitializerTo(IntRecTy::get())) + ->getValue()); + if (Opc == 0xFFFFFFFF) + continue; // No opcode defined. + assert(Opc <= 0xFFFF); + auto Prefix = Opc >> 8; + Opc = Opc & 0xFF; + auto &CGIP = OpcodeTable[Prefix][Opc]; + // All wasm instructions have a StackBased field of type string, we only + // want the instructions for which this is "true". + auto StackString = + Def.getValue("StackBased")->getValue()->getCastTo(StringRecTy::get()); + auto IsStackBased = + StackString && + reinterpret_cast<const StringInit *>(StackString)->getValue() == "true"; + if (!IsStackBased) + continue; + if (CGIP.second) { + // We already have an instruction for this slot, so decide which one + // should be the canonical one. This determines which variant gets + // printed in a disassembly. We want e.g. "call" not "i32.call", and + // "end" when we don't know if its "end_loop" or "end_block" etc. + auto IsCanonicalExisting = CGIP.second->TheDef->getValue("IsCanonical") + ->getValue() + ->getAsString() == "1"; + // We already have one marked explicitly as canonical, so keep it. + if (IsCanonicalExisting) + continue; + auto IsCanonicalNew = + Def.getValue("IsCanonical")->getValue()->getAsString() == "1"; + // If the new one is explicitly marked as canonical, take it. + if (!IsCanonicalNew) { + // Neither the existing or new instruction is canonical. + // Pick the one with the shortest name as heuristic. + // Though ideally IsCanonical is always defined for at least one + // variant so this never has to apply. + if (CGIP.second->AsmString.size() <= CGI.AsmString.size()) + continue; + } + } + // Set this instruction as the one to use. + CGIP = std::make_pair(I, &CGI); + } + OS << "#include \"MCTargetDesc/WebAssemblyMCTargetDesc.h\"\n"; + OS << "\n"; + OS << "namespace llvm {\n\n"; + OS << "static constexpr int WebAssemblyInstructionTableSize = "; + OS << WebAssemblyInstructionTableSize << ";\n\n"; + OS << "enum EntryType : uint8_t { "; + OS << "ET_Unused, ET_Prefix, ET_Instruction };\n\n"; + OS << "struct WebAssemblyInstruction {\n"; + OS << " uint16_t Opcode;\n"; + OS << " EntryType ET;\n"; + OS << " uint8_t NumOperands;\n"; + OS << " uint16_t OperandStart;\n"; + OS << "};\n\n"; + std::vector<std::string> OperandTable, CurOperandList; + // Output one table per prefix. + for (auto &PrefixPair : OpcodeTable) { + if (PrefixPair.second.empty()) + continue; + OS << "WebAssemblyInstruction InstructionTable" << PrefixPair.first; + OS << "[] = {\n"; + for (unsigned I = 0; I < WebAssemblyInstructionTableSize; I++) { + auto InstIt = PrefixPair.second.find(I); + if (InstIt != PrefixPair.second.end()) { + // Regular instruction. + assert(InstIt->second.second); + auto &CGI = *InstIt->second.second; + OS << " // 0x"; + OS.write_hex(static_cast<unsigned long long>(I)); + OS << ": " << CGI.AsmString << "\n"; + OS << " { " << InstIt->second.first << ", ET_Instruction, "; + OS << CGI.Operands.OperandList.size() << ", "; + // Collect operand types for storage in a shared list. + CurOperandList.clear(); + for (auto &Op : CGI.Operands.OperandList) { + assert(Op.OperandType != "MCOI::OPERAND_UNKNOWN"); + CurOperandList.push_back(Op.OperandType); + } + // See if we already have stored this sequence before. This is not + // strictly necessary but makes the table really small. + size_t OperandStart = OperandTable.size(); + if (CurOperandList.size() <= OperandTable.size()) { + for (size_t J = 0; J <= OperandTable.size() - CurOperandList.size(); + ++J) { + size_t K = 0; + for (; K < CurOperandList.size(); ++K) { + if (OperandTable[J + K] != CurOperandList[K]) break; + } + if (K == CurOperandList.size()) { + OperandStart = J; + break; + } + } + } + // Store operands if no prior occurrence. + if (OperandStart == OperandTable.size()) { + OperandTable.insert(OperandTable.end(), CurOperandList.begin(), + CurOperandList.end()); + } + OS << OperandStart; + } else { + auto PrefixIt = OpcodeTable.find(I); + // If we have a non-empty table for it that's not 0, this is a prefix. + if (PrefixIt != OpcodeTable.end() && I && !PrefixPair.first) { + OS << " { 0, ET_Prefix, 0, 0"; + } else { + OS << " { 0, ET_Unused, 0, 0"; + } + } + OS << " },\n"; + } + OS << "};\n\n"; + } + // Create a table of all operands: + OS << "const uint8_t OperandTable[] = {\n"; + for (auto &Op : OperandTable) { + OS << " " << Op << ",\n"; + } + OS << "};\n\n"; + // Create a table of all extension tables: + OS << "struct { uint8_t Prefix; const WebAssemblyInstruction *Table; }\n"; + OS << "PrefixTable[] = {\n"; + for (auto &PrefixPair : OpcodeTable) { + if (PrefixPair.second.empty() || !PrefixPair.first) + continue; + OS << " { " << PrefixPair.first << ", InstructionTable" + << PrefixPair.first; + OS << " },\n"; + } + OS << " { 0, nullptr }\n};\n\n"; + OS << "} // end namespace llvm\n"; +} + +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.h b/contrib/llvm-project/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.h new file mode 100644 index 000000000000..60d3d9433eca --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/WebAssemblyDisassemblerEmitter.h @@ -0,0 +1,29 @@ +//===- WebAssemblyDisassemblerEmitter.h - Disassembler tables ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is part of the WebAssembly Disassembler Emitter. +// It contains the interface of the disassembler tables. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_WEBASSEMBLYDISASSEMBLEREMITTER_H +#define LLVM_UTILS_TABLEGEN_WEBASSEMBLYDISASSEMBLEREMITTER_H + +#include "CodeGenInstruction.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +void emitWebAssemblyDisassemblerTables( + raw_ostream &OS, + const ArrayRef<const CodeGenInstruction *> &NumberedInstructions); + +} // namespace llvm + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerShared.h b/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerShared.h new file mode 100644 index 000000000000..093f220fda5e --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerShared.h @@ -0,0 +1,57 @@ +//===- X86DisassemblerShared.h - Emitter shared header ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_X86DISASSEMBLERSHARED_H +#define LLVM_UTILS_TABLEGEN_X86DISASSEMBLERSHARED_H + +#include <cstring> +#include <string> + +#include "llvm/Support/X86DisassemblerDecoderCommon.h" + +struct InstructionSpecifier { + llvm::X86Disassembler::OperandSpecifier + operands[llvm::X86Disassembler::X86_MAX_OPERANDS]; + llvm::X86Disassembler::InstructionContext insnContext; + std::string name; + + InstructionSpecifier() { + insnContext = llvm::X86Disassembler::IC; + name = ""; + memset(operands, 0, sizeof(operands)); + } +}; + +/// Specifies whether a ModR/M byte is needed and (if so) which +/// instruction each possible value of the ModR/M byte corresponds to. Once +/// this information is known, we have narrowed down to a single instruction. +struct ModRMDecision { + uint8_t modrm_type; + llvm::X86Disassembler::InstrUID instructionIDs[256]; +}; + +/// Specifies which set of ModR/M->instruction tables to look at +/// given a particular opcode. +struct OpcodeDecision { + ModRMDecision modRMDecisions[256]; +}; + +/// Specifies which opcode->instruction tables to look at given +/// a particular context (set of attributes). Since there are many possible +/// contexts, the decoder first uses CONTEXTS_SYM to determine which context +/// applies given a specific set of attributes. Hence there are only IC_max +/// entries in this table, rather than 2^(ATTR_max). +struct ContextDecision { + OpcodeDecision opcodeDecisions[llvm::X86Disassembler::IC_max]; + + ContextDecision() { + memset(opcodeDecisions, 0, sizeof(opcodeDecisions)); + } +}; + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp b/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp new file mode 100644 index 000000000000..76e4fd9a13ee --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.cpp @@ -0,0 +1,1094 @@ +//===- X86DisassemblerTables.cpp - Disassembler tables ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is part of the X86 Disassembler Emitter. +// It contains the implementation of the disassembler tables. +// Documentation for the disassembler emitter in general can be found in +// X86DisassemblerEmitter.h. +// +//===----------------------------------------------------------------------===// + +#include "X86DisassemblerTables.h" +#include "X86DisassemblerShared.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Format.h" +#include <map> + +using namespace llvm; +using namespace X86Disassembler; + +/// stringForContext - Returns a string containing the name of a particular +/// InstructionContext, usually for diagnostic purposes. +/// +/// @param insnContext - The instruction class to transform to a string. +/// @return - A statically-allocated string constant that contains the +/// name of the instruction class. +static inline const char* stringForContext(InstructionContext insnContext) { + switch (insnContext) { + default: + llvm_unreachable("Unhandled instruction class"); +#define ENUM_ENTRY(n, r, d) case n: return #n; break; +#define ENUM_ENTRY_K_B(n, r, d) ENUM_ENTRY(n, r, d) ENUM_ENTRY(n##_K_B, r, d)\ + ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) ENUM_ENTRY(n##_B, r, d)\ + ENUM_ENTRY(n##_KZ_B, r, d) + INSTRUCTION_CONTEXTS +#undef ENUM_ENTRY +#undef ENUM_ENTRY_K_B + } +} + +/// stringForOperandType - Like stringForContext, but for OperandTypes. +static inline const char* stringForOperandType(OperandType type) { + switch (type) { + default: + llvm_unreachable("Unhandled type"); +#define ENUM_ENTRY(i, d) case i: return #i; + TYPES +#undef ENUM_ENTRY + } +} + +/// stringForOperandEncoding - like stringForContext, but for +/// OperandEncodings. +static inline const char* stringForOperandEncoding(OperandEncoding encoding) { + switch (encoding) { + default: + llvm_unreachable("Unhandled encoding"); +#define ENUM_ENTRY(i, d) case i: return #i; + ENCODINGS +#undef ENUM_ENTRY + } +} + +/// inheritsFrom - Indicates whether all instructions in one class also belong +/// to another class. +/// +/// @param child - The class that may be the subset +/// @param parent - The class that may be the superset +/// @return - True if child is a subset of parent, false otherwise. +static inline bool inheritsFrom(InstructionContext child, + InstructionContext parent, bool noPrefix = true, + bool VEX_LIG = false, bool VEX_WIG = false, + bool AdSize64 = false) { + if (child == parent) + return true; + + switch (parent) { + case IC: + return(inheritsFrom(child, IC_64BIT, AdSize64) || + (noPrefix && inheritsFrom(child, IC_OPSIZE, noPrefix)) || + inheritsFrom(child, IC_ADSIZE) || + (noPrefix && inheritsFrom(child, IC_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_XS, noPrefix))); + case IC_64BIT: + return(inheritsFrom(child, IC_64BIT_REXW) || + (noPrefix && inheritsFrom(child, IC_64BIT_OPSIZE, noPrefix)) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_ADSIZE)) || + (noPrefix && inheritsFrom(child, IC_64BIT_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_XS, noPrefix))); + case IC_OPSIZE: + return inheritsFrom(child, IC_64BIT_OPSIZE) || + inheritsFrom(child, IC_OPSIZE_ADSIZE); + case IC_ADSIZE: + return (noPrefix && inheritsFrom(child, IC_OPSIZE_ADSIZE, noPrefix)); + case IC_OPSIZE_ADSIZE: + return false; + case IC_64BIT_ADSIZE: + return (noPrefix && inheritsFrom(child, IC_64BIT_OPSIZE_ADSIZE, noPrefix)); + case IC_64BIT_OPSIZE_ADSIZE: + return false; + case IC_XD: + return inheritsFrom(child, IC_64BIT_XD); + case IC_XS: + return inheritsFrom(child, IC_64BIT_XS); + case IC_XD_OPSIZE: + return inheritsFrom(child, IC_64BIT_XD_OPSIZE); + case IC_XS_OPSIZE: + return inheritsFrom(child, IC_64BIT_XS_OPSIZE); + case IC_XD_ADSIZE: + return inheritsFrom(child, IC_64BIT_XD_ADSIZE); + case IC_XS_ADSIZE: + return inheritsFrom(child, IC_64BIT_XS_ADSIZE); + case IC_64BIT_REXW: + return((noPrefix && inheritsFrom(child, IC_64BIT_REXW_XS, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_REXW_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_REXW_OPSIZE, noPrefix)) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE))); + case IC_64BIT_OPSIZE: + return inheritsFrom(child, IC_64BIT_REXW_OPSIZE) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_OPSIZE_ADSIZE)) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE)); + case IC_64BIT_XD: + return(inheritsFrom(child, IC_64BIT_REXW_XD) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_XD_ADSIZE))); + case IC_64BIT_XS: + return(inheritsFrom(child, IC_64BIT_REXW_XS) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_XS_ADSIZE))); + case IC_64BIT_XD_OPSIZE: + case IC_64BIT_XS_OPSIZE: + return false; + case IC_64BIT_XD_ADSIZE: + case IC_64BIT_XS_ADSIZE: + return false; + case IC_64BIT_REXW_XD: + case IC_64BIT_REXW_XS: + case IC_64BIT_REXW_OPSIZE: + case IC_64BIT_REXW_ADSIZE: + return false; + case IC_VEX: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_VEX_L_W)) || + (VEX_WIG && inheritsFrom(child, IC_VEX_W)) || + (VEX_LIG && inheritsFrom(child, IC_VEX_L)); + case IC_VEX_XS: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_VEX_L_W_XS)) || + (VEX_WIG && inheritsFrom(child, IC_VEX_W_XS)) || + (VEX_LIG && inheritsFrom(child, IC_VEX_L_XS)); + case IC_VEX_XD: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_VEX_L_W_XD)) || + (VEX_WIG && inheritsFrom(child, IC_VEX_W_XD)) || + (VEX_LIG && inheritsFrom(child, IC_VEX_L_XD)); + case IC_VEX_OPSIZE: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_VEX_L_W_OPSIZE)) || + (VEX_WIG && inheritsFrom(child, IC_VEX_W_OPSIZE)) || + (VEX_LIG && inheritsFrom(child, IC_VEX_L_OPSIZE)); + case IC_VEX_W: + return VEX_LIG && inheritsFrom(child, IC_VEX_L_W); + case IC_VEX_W_XS: + return VEX_LIG && inheritsFrom(child, IC_VEX_L_W_XS); + case IC_VEX_W_XD: + return VEX_LIG && inheritsFrom(child, IC_VEX_L_W_XD); + case IC_VEX_W_OPSIZE: + return VEX_LIG && inheritsFrom(child, IC_VEX_L_W_OPSIZE); + case IC_VEX_L: + return VEX_WIG && inheritsFrom(child, IC_VEX_L_W); + case IC_VEX_L_XS: + return VEX_WIG && inheritsFrom(child, IC_VEX_L_W_XS); + case IC_VEX_L_XD: + return VEX_WIG && inheritsFrom(child, IC_VEX_L_W_XD); + case IC_VEX_L_OPSIZE: + return VEX_WIG && inheritsFrom(child, IC_VEX_L_W_OPSIZE); + case IC_VEX_L_W: + case IC_VEX_L_W_XS: + case IC_VEX_L_W_XD: + case IC_VEX_L_W_OPSIZE: + return false; + case IC_EVEX: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2)); + case IC_EVEX_XS: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS)); + case IC_EVEX_XD: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD)); + case IC_EVEX_OPSIZE: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE)); + case IC_EVEX_K: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_K)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_K)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_K)); + case IC_EVEX_XS_K: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS_K)); + case IC_EVEX_XD_K: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_K)); + case IC_EVEX_OPSIZE_K: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_K)); + case IC_EVEX_KZ: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_KZ)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_KZ)); + case IC_EVEX_XS_KZ: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS_KZ)); + case IC_EVEX_XD_KZ: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_KZ)); + case IC_EVEX_OPSIZE_KZ: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_KZ)); + case IC_EVEX_W: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W)); + case IC_EVEX_W_XS: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XS)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XS)); + case IC_EVEX_W_XD: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XD)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XD)); + case IC_EVEX_W_OPSIZE: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE)); + case IC_EVEX_W_K: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_K)); + case IC_EVEX_W_XS_K: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XS_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K)); + case IC_EVEX_W_XD_K: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XD_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K)); + case IC_EVEX_W_OPSIZE_K: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K)); + case IC_EVEX_W_KZ: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_KZ)); + case IC_EVEX_W_XS_KZ: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ)); + case IC_EVEX_W_XD_KZ: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ)); + case IC_EVEX_W_OPSIZE_KZ: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ)); + case IC_EVEX_L: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W); + case IC_EVEX_L_XS: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS); + case IC_EVEX_L_XD: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD); + case IC_EVEX_L_OPSIZE: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE); + case IC_EVEX_L_K: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_K); + case IC_EVEX_L_XS_K: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K); + case IC_EVEX_L_XD_K: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K); + case IC_EVEX_L_OPSIZE_K: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K); + case IC_EVEX_L_KZ: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_KZ); + case IC_EVEX_L_XS_KZ: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ); + case IC_EVEX_L_XD_KZ: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ); + case IC_EVEX_L_OPSIZE_KZ: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ); + case IC_EVEX_L_W: + case IC_EVEX_L_W_XS: + case IC_EVEX_L_W_XD: + case IC_EVEX_L_W_OPSIZE: + return false; + case IC_EVEX_L_W_K: + case IC_EVEX_L_W_XS_K: + case IC_EVEX_L_W_XD_K: + case IC_EVEX_L_W_OPSIZE_K: + return false; + case IC_EVEX_L_W_KZ: + case IC_EVEX_L_W_XS_KZ: + case IC_EVEX_L_W_XD_KZ: + case IC_EVEX_L_W_OPSIZE_KZ: + return false; + case IC_EVEX_L2: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W); + case IC_EVEX_L2_XS: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS); + case IC_EVEX_L2_XD: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD); + case IC_EVEX_L2_OPSIZE: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE); + case IC_EVEX_L2_K: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_K); + case IC_EVEX_L2_XS_K: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K); + case IC_EVEX_L2_XD_K: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K); + case IC_EVEX_L2_OPSIZE_K: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K); + case IC_EVEX_L2_KZ: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ); + case IC_EVEX_L2_XS_KZ: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ); + case IC_EVEX_L2_XD_KZ: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ); + case IC_EVEX_L2_OPSIZE_KZ: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ); + case IC_EVEX_L2_W: + case IC_EVEX_L2_W_XS: + case IC_EVEX_L2_W_XD: + case IC_EVEX_L2_W_OPSIZE: + return false; + case IC_EVEX_L2_W_K: + case IC_EVEX_L2_W_XS_K: + case IC_EVEX_L2_W_XD_K: + case IC_EVEX_L2_W_OPSIZE_K: + return false; + case IC_EVEX_L2_W_KZ: + case IC_EVEX_L2_W_XS_KZ: + case IC_EVEX_L2_W_XD_KZ: + case IC_EVEX_L2_W_OPSIZE_KZ: + return false; + case IC_EVEX_B: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_B)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_B)); + case IC_EVEX_XS_B: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_B)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS_B)); + case IC_EVEX_XD_B: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_B)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_B)); + case IC_EVEX_OPSIZE_B: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_B)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_B)); + case IC_EVEX_K_B: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_K_B)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_K_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_K_B)); + case IC_EVEX_XS_K_B: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K_B)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS_K_B)); + case IC_EVEX_XD_K_B: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K_B)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_K_B)); + case IC_EVEX_OPSIZE_K_B: + return (VEX_LIG && VEX_WIG && + inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B)) || + (VEX_LIG && VEX_WIG && + inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_K_B)); + case IC_EVEX_KZ_B: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_KZ_B)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_KZ_B)); + case IC_EVEX_XS_KZ_B: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ_B)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XS_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XS_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XS_KZ_B)); + case IC_EVEX_XD_KZ_B: + return (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ_B)) || + (VEX_LIG && VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_XD_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_KZ_B)); + case IC_EVEX_OPSIZE_KZ_B: + return (VEX_LIG && VEX_WIG && + inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B)) || + (VEX_LIG && VEX_WIG && + inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B)) || + (VEX_WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_KZ_B)); + case IC_EVEX_W_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_B)); + case IC_EVEX_W_XS_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XS_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XS_B)); + case IC_EVEX_W_XD_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XD_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XD_B)); + case IC_EVEX_W_OPSIZE_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_B)); + case IC_EVEX_W_K_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_K_B)); + case IC_EVEX_W_XS_K_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XS_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K_B)); + case IC_EVEX_W_XD_K_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XD_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K_B)); + case IC_EVEX_W_OPSIZE_K_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B)); + case IC_EVEX_W_KZ_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_KZ_B)); + case IC_EVEX_W_XS_KZ_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ_B)); + case IC_EVEX_W_XD_KZ_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ_B)); + case IC_EVEX_W_OPSIZE_KZ_B: + return (VEX_LIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B)) || + (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B)); + case IC_EVEX_L_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_B); + case IC_EVEX_L_XS_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_B); + case IC_EVEX_L_XD_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_B); + case IC_EVEX_L_OPSIZE_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_B); + case IC_EVEX_L_K_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_K_B); + case IC_EVEX_L_XS_K_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_K_B); + case IC_EVEX_L_XD_K_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_K_B); + case IC_EVEX_L_OPSIZE_K_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B); + case IC_EVEX_L_KZ_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_KZ_B); + case IC_EVEX_L_XS_KZ_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XS_KZ_B); + case IC_EVEX_L_XD_KZ_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_XD_KZ_B); + case IC_EVEX_L_OPSIZE_KZ_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B); + case IC_EVEX_L_W_B: + case IC_EVEX_L_W_XS_B: + case IC_EVEX_L_W_XD_B: + case IC_EVEX_L_W_OPSIZE_B: + return false; + case IC_EVEX_L_W_K_B: + case IC_EVEX_L_W_XS_K_B: + case IC_EVEX_L_W_XD_K_B: + case IC_EVEX_L_W_OPSIZE_K_B: + return false; + case IC_EVEX_L_W_KZ_B: + case IC_EVEX_L_W_XS_KZ_B: + case IC_EVEX_L_W_XD_KZ_B: + case IC_EVEX_L_W_OPSIZE_KZ_B: + return false; + case IC_EVEX_L2_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_B); + case IC_EVEX_L2_XS_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_B); + case IC_EVEX_L2_XD_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_B); + case IC_EVEX_L2_OPSIZE_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_B); + case IC_EVEX_L2_K_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_K_B); + case IC_EVEX_L2_XS_K_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_K_B); + case IC_EVEX_L2_XD_K_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_K_B); + case IC_EVEX_L2_OPSIZE_K_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B); + case IC_EVEX_L2_KZ_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_KZ_B); + case IC_EVEX_L2_XS_KZ_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XS_KZ_B); + case IC_EVEX_L2_XD_KZ_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_XD_KZ_B); + case IC_EVEX_L2_OPSIZE_KZ_B: + return VEX_WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B); + case IC_EVEX_L2_W_B: + case IC_EVEX_L2_W_XS_B: + case IC_EVEX_L2_W_XD_B: + case IC_EVEX_L2_W_OPSIZE_B: + return false; + case IC_EVEX_L2_W_K_B: + case IC_EVEX_L2_W_XS_K_B: + case IC_EVEX_L2_W_XD_K_B: + case IC_EVEX_L2_W_OPSIZE_K_B: + return false; + case IC_EVEX_L2_W_KZ_B: + case IC_EVEX_L2_W_XS_KZ_B: + case IC_EVEX_L2_W_XD_KZ_B: + case IC_EVEX_L2_W_OPSIZE_KZ_B: + return false; + default: + errs() << "Unknown instruction class: " << + stringForContext((InstructionContext)parent) << "\n"; + llvm_unreachable("Unknown instruction class"); + } +} + +/// outranks - Indicates whether, if an instruction has two different applicable +/// classes, which class should be preferred when performing decode. This +/// imposes a total ordering (ties are resolved toward "lower") +/// +/// @param upper - The class that may be preferable +/// @param lower - The class that may be less preferable +/// @return - True if upper is to be preferred, false otherwise. +static inline bool outranks(InstructionContext upper, + InstructionContext lower) { + assert(upper < IC_max); + assert(lower < IC_max); + +#define ENUM_ENTRY(n, r, d) r, +#define ENUM_ENTRY_K_B(n, r, d) ENUM_ENTRY(n, r, d) \ + ENUM_ENTRY(n##_K_B, r, d) ENUM_ENTRY(n##_KZ_B, r, d) \ + ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) ENUM_ENTRY(n##_B, r, d) + static int ranks[IC_max] = { + INSTRUCTION_CONTEXTS + }; +#undef ENUM_ENTRY +#undef ENUM_ENTRY_K_B + + return (ranks[upper] > ranks[lower]); +} + +/// getDecisionType - Determines whether a ModRM decision with 255 entries can +/// be compacted by eliminating redundant information. +/// +/// @param decision - The decision to be compacted. +/// @return - The compactest available representation for the decision. +static ModRMDecisionType getDecisionType(ModRMDecision &decision) { + bool satisfiesOneEntry = true; + bool satisfiesSplitRM = true; + bool satisfiesSplitReg = true; + bool satisfiesSplitMisc = true; + + for (unsigned index = 0; index < 256; ++index) { + if (decision.instructionIDs[index] != decision.instructionIDs[0]) + satisfiesOneEntry = false; + + if (((index & 0xc0) == 0xc0) && + (decision.instructionIDs[index] != decision.instructionIDs[0xc0])) + satisfiesSplitRM = false; + + if (((index & 0xc0) != 0xc0) && + (decision.instructionIDs[index] != decision.instructionIDs[0x00])) + satisfiesSplitRM = false; + + if (((index & 0xc0) == 0xc0) && + (decision.instructionIDs[index] != decision.instructionIDs[index&0xf8])) + satisfiesSplitReg = false; + + if (((index & 0xc0) != 0xc0) && + (decision.instructionIDs[index] != decision.instructionIDs[index&0x38])) + satisfiesSplitMisc = false; + } + + if (satisfiesOneEntry) + return MODRM_ONEENTRY; + + if (satisfiesSplitRM) + return MODRM_SPLITRM; + + if (satisfiesSplitReg && satisfiesSplitMisc) + return MODRM_SPLITREG; + + if (satisfiesSplitMisc) + return MODRM_SPLITMISC; + + return MODRM_FULL; +} + +/// stringForDecisionType - Returns a statically-allocated string corresponding +/// to a particular decision type. +/// +/// @param dt - The decision type. +/// @return - A pointer to the statically-allocated string (e.g., +/// "MODRM_ONEENTRY" for MODRM_ONEENTRY). +static const char* stringForDecisionType(ModRMDecisionType dt) { +#define ENUM_ENTRY(n) case n: return #n; + switch (dt) { + default: + llvm_unreachable("Unknown decision type"); + MODRMTYPES + }; +#undef ENUM_ENTRY +} + +DisassemblerTables::DisassemblerTables() { + for (unsigned i = 0; i < array_lengthof(Tables); i++) + Tables[i] = std::make_unique<ContextDecision>(); + + HasConflicts = false; +} + +DisassemblerTables::~DisassemblerTables() { +} + +void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, + unsigned &i1, unsigned &i2, + unsigned &ModRMTableNum, + ModRMDecision &decision) const { + static uint32_t sTableNumber = 0; + static uint32_t sEntryNumber = 1; + ModRMDecisionType dt = getDecisionType(decision); + + if (dt == MODRM_ONEENTRY && decision.instructionIDs[0] == 0) { + // Empty table. + o2 << "{" << stringForDecisionType(dt) << ", 0}"; + return; + } + + std::vector<unsigned> ModRMDecision; + + switch (dt) { + default: + llvm_unreachable("Unknown decision type"); + case MODRM_ONEENTRY: + ModRMDecision.push_back(decision.instructionIDs[0]); + break; + case MODRM_SPLITRM: + ModRMDecision.push_back(decision.instructionIDs[0x00]); + ModRMDecision.push_back(decision.instructionIDs[0xc0]); + break; + case MODRM_SPLITREG: + for (unsigned index = 0; index < 64; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + for (unsigned index = 0xc0; index < 256; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + break; + case MODRM_SPLITMISC: + for (unsigned index = 0; index < 64; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + for (unsigned index = 0xc0; index < 256; ++index) + ModRMDecision.push_back(decision.instructionIDs[index]); + break; + case MODRM_FULL: + for (unsigned index = 0; index < 256; ++index) + ModRMDecision.push_back(decision.instructionIDs[index]); + break; + } + + unsigned &EntryNumber = ModRMTable[ModRMDecision]; + if (EntryNumber == 0) { + EntryNumber = ModRMTableNum; + + ModRMTableNum += ModRMDecision.size(); + o1 << "/*Table" << EntryNumber << "*/\n"; + i1++; + for (std::vector<unsigned>::const_iterator I = ModRMDecision.begin(), + E = ModRMDecision.end(); I != E; ++I) { + o1.indent(i1 * 2) << format("0x%hx", *I) << ", /*" + << InstructionSpecifiers[*I].name << "*/\n"; + } + i1--; + } + + o2 << "{" << stringForDecisionType(dt) << ", " << EntryNumber << "}"; + + switch (dt) { + default: + llvm_unreachable("Unknown decision type"); + case MODRM_ONEENTRY: + sEntryNumber += 1; + break; + case MODRM_SPLITRM: + sEntryNumber += 2; + break; + case MODRM_SPLITREG: + sEntryNumber += 16; + break; + case MODRM_SPLITMISC: + sEntryNumber += 8 + 64; + break; + case MODRM_FULL: + sEntryNumber += 256; + break; + } + + // We assume that the index can fit into uint16_t. + assert(sEntryNumber < 65536U && + "Index into ModRMDecision is too large for uint16_t!"); + + ++sTableNumber; +} + +void DisassemblerTables::emitOpcodeDecision(raw_ostream &o1, raw_ostream &o2, + unsigned &i1, unsigned &i2, + unsigned &ModRMTableNum, + OpcodeDecision &opDecision) const { + o2 << "{"; + ++i2; + + unsigned index; + for (index = 0; index < 256; ++index) { + auto &decision = opDecision.modRMDecisions[index]; + ModRMDecisionType dt = getDecisionType(decision); + if (!(dt == MODRM_ONEENTRY && decision.instructionIDs[0] == 0)) + break; + } + if (index == 256) { + // If all 256 entries are MODRM_ONEENTRY, omit output. + assert(MODRM_ONEENTRY == 0); + --i2; + o2 << "},\n"; + } else { + o2 << " /* struct OpcodeDecision */ {\n"; + for (index = 0; index < 256; ++index) { + o2.indent(i2); + + o2 << "/*0x" << format("%02hhx", index) << "*/"; + + emitModRMDecision(o1, o2, i1, i2, ModRMTableNum, + opDecision.modRMDecisions[index]); + + if (index < 255) + o2 << ","; + + o2 << "\n"; + } + o2.indent(i2) << "}\n"; + --i2; + o2.indent(i2) << "},\n"; + } +} + +void DisassemblerTables::emitContextDecision(raw_ostream &o1, raw_ostream &o2, + unsigned &i1, unsigned &i2, + unsigned &ModRMTableNum, + ContextDecision &decision, + const char* name) const { + o2.indent(i2) << "static const struct ContextDecision " << name << " = {{/* opcodeDecisions */\n"; + i2++; + + for (unsigned index = 0; index < IC_max; ++index) { + o2.indent(i2) << "/*"; + o2 << stringForContext((InstructionContext)index); + o2 << "*/ "; + + emitOpcodeDecision(o1, o2, i1, i2, ModRMTableNum, + decision.opcodeDecisions[index]); + } + + i2--; + o2.indent(i2) << "}};" << "\n"; +} + +void DisassemblerTables::emitInstructionInfo(raw_ostream &o, + unsigned &i) const { + unsigned NumInstructions = InstructionSpecifiers.size(); + + o << "static const struct OperandSpecifier x86OperandSets[][" + << X86_MAX_OPERANDS << "] = {\n"; + + typedef SmallVector<std::pair<OperandEncoding, OperandType>, + X86_MAX_OPERANDS> OperandListTy; + std::map<OperandListTy, unsigned> OperandSets; + + unsigned OperandSetNum = 0; + for (unsigned Index = 0; Index < NumInstructions; ++Index) { + OperandListTy OperandList; + + for (unsigned OperandIndex = 0; OperandIndex < X86_MAX_OPERANDS; + ++OperandIndex) { + OperandEncoding Encoding = (OperandEncoding)InstructionSpecifiers[Index] + .operands[OperandIndex].encoding; + OperandType Type = (OperandType)InstructionSpecifiers[Index] + .operands[OperandIndex].type; + OperandList.push_back(std::make_pair(Encoding, Type)); + } + unsigned &N = OperandSets[OperandList]; + if (N != 0) continue; + + N = ++OperandSetNum; + + o << " { /* " << (OperandSetNum - 1) << " */\n"; + for (unsigned i = 0, e = OperandList.size(); i != e; ++i) { + const char *Encoding = stringForOperandEncoding(OperandList[i].first); + const char *Type = stringForOperandType(OperandList[i].second); + o << " { " << Encoding << ", " << Type << " },\n"; + } + o << " },\n"; + } + o << "};" << "\n\n"; + + o.indent(i * 2) << "static const struct InstructionSpecifier "; + o << INSTRUCTIONS_STR "[" << InstructionSpecifiers.size() << "] = {\n"; + + i++; + + for (unsigned index = 0; index < NumInstructions; ++index) { + o.indent(i * 2) << "{ /* " << index << " */\n"; + i++; + + OperandListTy OperandList; + for (unsigned OperandIndex = 0; OperandIndex < X86_MAX_OPERANDS; + ++OperandIndex) { + OperandEncoding Encoding = (OperandEncoding)InstructionSpecifiers[index] + .operands[OperandIndex].encoding; + OperandType Type = (OperandType)InstructionSpecifiers[index] + .operands[OperandIndex].type; + OperandList.push_back(std::make_pair(Encoding, Type)); + } + o.indent(i * 2) << (OperandSets[OperandList] - 1) << ",\n"; + + o.indent(i * 2) << "/* " << InstructionSpecifiers[index].name << " */\n"; + + i--; + o.indent(i * 2) << "},\n"; + } + + i--; + o.indent(i * 2) << "};" << "\n"; +} + +void DisassemblerTables::emitContextTable(raw_ostream &o, unsigned &i) const { + o.indent(i * 2) << "static const uint8_t " CONTEXTS_STR + "[" << ATTR_max << "] = {\n"; + i++; + + for (unsigned index = 0; index < ATTR_max; ++index) { + o.indent(i * 2); + + if ((index & ATTR_EVEX) || (index & ATTR_VEX) || (index & ATTR_VEXL)) { + if (index & ATTR_EVEX) + o << "IC_EVEX"; + else + o << "IC_VEX"; + + if ((index & ATTR_EVEX) && (index & ATTR_EVEXL2)) + o << "_L2"; + else if (index & ATTR_VEXL) + o << "_L"; + + if (index & ATTR_REXW) + o << "_W"; + + if (index & ATTR_OPSIZE) + o << "_OPSIZE"; + else if (index & ATTR_XD) + o << "_XD"; + else if (index & ATTR_XS) + o << "_XS"; + + if ((index & ATTR_EVEX)) { + if (index & ATTR_EVEXKZ) + o << "_KZ"; + else if (index & ATTR_EVEXK) + o << "_K"; + + if (index & ATTR_EVEXB) + o << "_B"; + } + } + else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XS)) + o << "IC_64BIT_REXW_XS"; + else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XD)) + o << "IC_64BIT_REXW_XD"; + else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && + (index & ATTR_OPSIZE)) + o << "IC_64BIT_REXW_OPSIZE"; + else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && + (index & ATTR_ADSIZE)) + o << "IC_64BIT_REXW_ADSIZE"; + else if ((index & ATTR_64BIT) && (index & ATTR_XD) && (index & ATTR_OPSIZE)) + o << "IC_64BIT_XD_OPSIZE"; + else if ((index & ATTR_64BIT) && (index & ATTR_XD) && (index & ATTR_ADSIZE)) + o << "IC_64BIT_XD_ADSIZE"; + else if ((index & ATTR_64BIT) && (index & ATTR_XS) && (index & ATTR_OPSIZE)) + o << "IC_64BIT_XS_OPSIZE"; + else if ((index & ATTR_64BIT) && (index & ATTR_XS) && (index & ATTR_ADSIZE)) + o << "IC_64BIT_XS_ADSIZE"; + else if ((index & ATTR_64BIT) && (index & ATTR_XS)) + o << "IC_64BIT_XS"; + else if ((index & ATTR_64BIT) && (index & ATTR_XD)) + o << "IC_64BIT_XD"; + else if ((index & ATTR_64BIT) && (index & ATTR_OPSIZE) && + (index & ATTR_ADSIZE)) + o << "IC_64BIT_OPSIZE_ADSIZE"; + else if ((index & ATTR_64BIT) && (index & ATTR_OPSIZE)) + o << "IC_64BIT_OPSIZE"; + else if ((index & ATTR_64BIT) && (index & ATTR_ADSIZE)) + o << "IC_64BIT_ADSIZE"; + else if ((index & ATTR_64BIT) && (index & ATTR_REXW)) + o << "IC_64BIT_REXW"; + else if ((index & ATTR_64BIT)) + o << "IC_64BIT"; + else if ((index & ATTR_XS) && (index & ATTR_OPSIZE)) + o << "IC_XS_OPSIZE"; + else if ((index & ATTR_XD) && (index & ATTR_OPSIZE)) + o << "IC_XD_OPSIZE"; + else if ((index & ATTR_XS) && (index & ATTR_ADSIZE)) + o << "IC_XS_ADSIZE"; + else if ((index & ATTR_XD) && (index & ATTR_ADSIZE)) + o << "IC_XD_ADSIZE"; + else if (index & ATTR_XS) + o << "IC_XS"; + else if (index & ATTR_XD) + o << "IC_XD"; + else if ((index & ATTR_OPSIZE) && (index & ATTR_ADSIZE)) + o << "IC_OPSIZE_ADSIZE"; + else if (index & ATTR_OPSIZE) + o << "IC_OPSIZE"; + else if (index & ATTR_ADSIZE) + o << "IC_ADSIZE"; + else + o << "IC"; + + o << ", // " << index << "\n"; + } + + i--; + o.indent(i * 2) << "};" << "\n"; +} + +void DisassemblerTables::emitContextDecisions(raw_ostream &o1, raw_ostream &o2, + unsigned &i1, unsigned &i2, + unsigned &ModRMTableNum) const { + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[0], ONEBYTE_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[1], TWOBYTE_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[2], THREEBYTE38_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[3], THREEBYTE3A_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[4], XOP8_MAP_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[5], XOP9_MAP_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[6], XOPA_MAP_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[7], THREEDNOW_MAP_STR); +} + +void DisassemblerTables::emit(raw_ostream &o) const { + unsigned i1 = 0; + unsigned i2 = 0; + + std::string s1; + std::string s2; + + raw_string_ostream o1(s1); + raw_string_ostream o2(s2); + + emitInstructionInfo(o, i2); + o << "\n"; + + emitContextTable(o, i2); + o << "\n"; + + unsigned ModRMTableNum = 0; + + o << "static const InstrUID modRMTable[] = {\n"; + i1++; + std::vector<unsigned> EmptyTable(1, 0); + ModRMTable[EmptyTable] = ModRMTableNum; + ModRMTableNum += EmptyTable.size(); + o1 << "/*EmptyTable*/\n"; + o1.indent(i1 * 2) << "0x0,\n"; + i1--; + emitContextDecisions(o1, o2, i1, i2, ModRMTableNum); + + o << o1.str(); + o << " 0x0\n"; + o << "};\n"; + o << "\n"; + o << o2.str(); + o << "\n"; + o << "\n"; +} + +void DisassemblerTables::setTableFields(ModRMDecision &decision, + const ModRMFilter &filter, + InstrUID uid, + uint8_t opcode) { + for (unsigned index = 0; index < 256; ++index) { + if (filter.accepts(index)) { + if (decision.instructionIDs[index] == uid) + continue; + + if (decision.instructionIDs[index] != 0) { + InstructionSpecifier &newInfo = + InstructionSpecifiers[uid]; + InstructionSpecifier &previousInfo = + InstructionSpecifiers[decision.instructionIDs[index]]; + + if(previousInfo.name == "NOOP" && (newInfo.name == "XCHG16ar" || + newInfo.name == "XCHG32ar" || + newInfo.name == "XCHG64ar")) + continue; // special case for XCHG*ar and NOOP + + if (outranks(previousInfo.insnContext, newInfo.insnContext)) + continue; + + if (previousInfo.insnContext == newInfo.insnContext) { + errs() << "Error: Primary decode conflict: "; + errs() << newInfo.name << " would overwrite " << previousInfo.name; + errs() << "\n"; + errs() << "ModRM " << index << "\n"; + errs() << "Opcode " << (uint16_t)opcode << "\n"; + errs() << "Context " << stringForContext(newInfo.insnContext) << "\n"; + HasConflicts = true; + } + } + + decision.instructionIDs[index] = uid; + } + } +} + +void DisassemblerTables::setTableFields(OpcodeType type, + InstructionContext insnContext, + uint8_t opcode, + const ModRMFilter &filter, + InstrUID uid, + bool is32bit, + bool noPrefix, + bool ignoresVEX_L, + bool ignoresVEX_W, + unsigned addressSize) { + ContextDecision &decision = *Tables[type]; + + for (unsigned index = 0; index < IC_max; ++index) { + if ((is32bit || addressSize == 16) && + inheritsFrom((InstructionContext)index, IC_64BIT)) + continue; + + bool adSize64 = addressSize == 64; + if (inheritsFrom((InstructionContext)index, + InstructionSpecifiers[uid].insnContext, noPrefix, + ignoresVEX_L, ignoresVEX_W, adSize64)) + setTableFields(decision.opcodeDecisions[index].modRMDecisions[opcode], + filter, + uid, + opcode); + } +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.h b/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.h new file mode 100644 index 000000000000..63af68b6fbfa --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/X86DisassemblerTables.h @@ -0,0 +1,287 @@ +//===- X86DisassemblerTables.h - Disassembler tables ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is part of the X86 Disassembler Emitter. +// It contains the interface of the disassembler tables. +// Documentation for the disassembler emitter in general can be found in +// X86DisassemblerEmitter.h. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_X86DISASSEMBLERTABLES_H +#define LLVM_UTILS_TABLEGEN_X86DISASSEMBLERTABLES_H + +#include "X86DisassemblerShared.h" +#include "X86ModRMFilters.h" +#include "llvm/Support/raw_ostream.h" +#include <map> +#include <vector> + +namespace llvm { + +namespace X86Disassembler { + +/// DisassemblerTables - Encapsulates all the decode tables being generated by +/// the table emitter. Contains functions to populate the tables as well as +/// to emit them as hierarchical C structures suitable for consumption by the +/// runtime. +class DisassemblerTables { +private: + /// The decoder tables. There is one for each opcode type: + /// [0] one-byte opcodes + /// [1] two-byte opcodes of the form 0f __ + /// [2] three-byte opcodes of the form 0f 38 __ + /// [3] three-byte opcodes of the form 0f 3a __ + /// [4] XOP8 map opcode + /// [5] XOP9 map opcode + /// [6] XOPA map opcode + /// [7] 3dnow map opcode + std::unique_ptr<ContextDecision> Tables[8]; + + // Table of ModRM encodings. + typedef std::map<std::vector<unsigned>, unsigned> ModRMMapTy; + mutable ModRMMapTy ModRMTable; + + /// The instruction information table + std::vector<InstructionSpecifier> InstructionSpecifiers; + + /// True if there are primary decode conflicts in the instruction set + bool HasConflicts; + + /// emitModRMDecision - Emits a table of entries corresponding to a single + /// ModR/M decision. Compacts the ModR/M decision if possible. ModR/M + /// decisions are printed as: + /// + /// { /* struct ModRMDecision */ + /// TYPE, + /// modRMTablennnn + /// } + /// + /// where nnnn is a unique ID for the corresponding table of IDs. + /// TYPE indicates whether the table has one entry that is the same + /// regardless of ModR/M byte, two entries - one for bytes 0x00-0xbf and one + /// for bytes 0xc0-0xff -, or 256 entries, one for each possible byte. + /// nnnn is the number of a table for looking up these values. The tables + /// are written separately so that tables consisting entirely of zeros will + /// not be duplicated. (These all have the name modRMEmptyTable.) A table + /// is printed as: + /// + /// InstrUID modRMTablennnn[k] = { + /// nnnn, /* MNEMONIC */ + /// ... + /// nnnn /* MNEMONIC */ + /// }; + /// + /// @param o1 - The output stream to print the ID table to. + /// @param o2 - The output stream to print the decision structure to. + /// @param i1 - The indentation level to use with stream o1. + /// @param i2 - The indentation level to use with stream o2. + /// @param ModRMTableNum - next table number for adding to ModRMTable. + /// @param decision - The ModR/M decision to emit. This decision has 256 + /// entries - emitModRMDecision decides how to compact it. + void emitModRMDecision(raw_ostream &o1, raw_ostream &o2, + unsigned &i1, unsigned &i2, unsigned &ModRMTableNum, + ModRMDecision &decision) const; + + /// emitOpcodeDecision - Emits an OpcodeDecision and all its subsidiary ModR/M + /// decisions. An OpcodeDecision is printed as: + /// + /// { /* struct OpcodeDecision */ + /// /* 0x00 */ + /// { /* struct ModRMDecision */ + /// ... + /// } + /// ... + /// } + /// + /// where the ModRMDecision structure is printed as described in the + /// documentation for emitModRMDecision(). emitOpcodeDecision() passes on a + /// stream and indent level for the UID tables generated by + /// emitModRMDecision(), but does not use them itself. + /// + /// @param o1 - The output stream to print the ID tables generated by + /// emitModRMDecision() to. + /// @param o2 - The output stream for the decision structure itself. + /// @param i1 - The indent level to use with stream o1. + /// @param i2 - The indent level to use with stream o2. + /// @param ModRMTableNum - next table number for adding to ModRMTable. + /// @param decision - The OpcodeDecision to emit along with its subsidiary + /// structures. + void emitOpcodeDecision(raw_ostream &o1, raw_ostream &o2, + unsigned &i1, unsigned &i2, unsigned &ModRMTableNum, + OpcodeDecision &decision) const; + + /// emitContextDecision - Emits a ContextDecision and all its subsidiary + /// Opcode and ModRMDecisions. A ContextDecision is printed as: + /// + /// struct ContextDecision NAME = { + /// { /* OpcodeDecisions */ + /// /* IC */ + /// { /* struct OpcodeDecision */ + /// ... + /// }, + /// ... + /// } + /// } + /// + /// NAME is the name of the ContextDecision (typically one of the four names + /// ONEBYTE_SYM, TWOBYTE_SYM, THREEBYTE38_SYM, THREEBYTE3A_SYM from + /// X86DisassemblerDecoderCommon.h). + /// IC is one of the contexts in InstructionContext. There is an opcode + /// decision for each possible context. + /// The OpcodeDecision structures are printed as described in the + /// documentation for emitOpcodeDecision. + /// + /// @param o1 - The output stream to print the ID tables generated by + /// emitModRMDecision() to. + /// @param o2 - The output stream to print the decision structure to. + /// @param i1 - The indent level to use with stream o1. + /// @param i2 - The indent level to use with stream o2. + /// @param ModRMTableNum - next table number for adding to ModRMTable. + /// @param decision - The ContextDecision to emit along with its subsidiary + /// structures. + /// @param name - The name for the ContextDecision. + void emitContextDecision(raw_ostream &o1, raw_ostream &o2, + unsigned &i1, unsigned &i2, unsigned &ModRMTableNum, + ContextDecision &decision, const char* name) const; + + /// emitInstructionInfo - Prints the instruction specifier table, which has + /// one entry for each instruction, and contains name and operand + /// information. This table is printed as: + /// + /// struct InstructionSpecifier CONTEXTS_SYM[k] = { + /// { + /// /* nnnn */ + /// "MNEMONIC", + /// 0xnn, + /// { + /// { + /// ENCODING, + /// TYPE + /// }, + /// ... + /// } + /// }, + /// }; + /// + /// k is the total number of instructions. + /// nnnn is the ID of the current instruction (0-based). This table + /// includes entries for non-instructions like PHINODE. + /// 0xnn is the lowest possible opcode for the current instruction, used for + /// AddRegFrm instructions to compute the operand's value. + /// ENCODING and TYPE describe the encoding and type for a single operand. + /// + /// @param o - The output stream to which the instruction table should be + /// written. + /// @param i - The indent level for use with the stream. + void emitInstructionInfo(raw_ostream &o, unsigned &i) const; + + /// emitContextTable - Prints the table that is used to translate from an + /// instruction attribute mask to an instruction context. This table is + /// printed as: + /// + /// InstructionContext CONTEXTS_STR[256] = { + /// IC, /* 0x00 */ + /// ... + /// }; + /// + /// IC is the context corresponding to the mask 0x00, and there are 256 + /// possible masks. + /// + /// @param o - The output stream to which the context table should be written. + /// @param i - The indent level for use with the stream. + void emitContextTable(raw_ostream &o, uint32_t &i) const; + + /// emitContextDecisions - Prints all four ContextDecision structures using + /// emitContextDecision(). + /// + /// @param o1 - The output stream to print the ID tables generated by + /// emitModRMDecision() to. + /// @param o2 - The output stream to print the decision structures to. + /// @param i1 - The indent level to use with stream o1. + /// @param i2 - The indent level to use with stream o2. + /// @param ModRMTableNum - next table number for adding to ModRMTable. + void emitContextDecisions(raw_ostream &o1, raw_ostream &o2, + unsigned &i1, unsigned &i2, + unsigned &ModRMTableNum) const; + + /// setTableFields - Uses a ModRMFilter to set the appropriate entries in a + /// ModRMDecision to refer to a particular instruction ID. + /// + /// @param decision - The ModRMDecision to populate. + /// @param filter - The filter to use in deciding which entries to populate. + /// @param uid - The unique ID to set matching entries to. + /// @param opcode - The opcode of the instruction, for error reporting. + void setTableFields(ModRMDecision &decision, + const ModRMFilter &filter, + InstrUID uid, + uint8_t opcode); +public: + /// Constructor - Allocates space for the class decisions and clears them. + DisassemblerTables(); + + ~DisassemblerTables(); + + /// emit - Emits the instruction table, context table, and class decisions. + /// + /// @param o - The output stream to print the tables to. + void emit(raw_ostream &o) const; + + /// setTableFields - Uses the opcode type, instruction context, opcode, and a + /// ModRMFilter as criteria to set a particular set of entries in the + /// decode tables to point to a specific uid. + /// + /// @param type - The opcode type (ONEBYTE, TWOBYTE, etc.) + /// @param insnContext - The context to use (IC, IC_64BIT, etc.) + /// @param opcode - The last byte of the opcode (not counting any escape + /// or extended opcodes). + /// @param filter - The ModRMFilter that decides which ModR/M byte values + /// correspond to the desired instruction. + /// @param uid - The unique ID of the instruction. + /// @param is32bit - Instructon is only 32-bit + /// @param noPrefix - Instruction record has no prefix. + /// @param ignoresVEX_L - Instruction ignores VEX.L + /// @param ignoresVEX_W - Instruction ignores VEX.W + /// @param AddrSize - Instructions address size 16/32/64. 0 is unspecified + void setTableFields(OpcodeType type, + InstructionContext insnContext, + uint8_t opcode, + const ModRMFilter &filter, + InstrUID uid, + bool is32bit, + bool noPrefix, + bool ignoresVEX_L, + bool ignoresVEX_W, + unsigned AddrSize); + + /// specForUID - Returns the instruction specifier for a given unique + /// instruction ID. Used when resolving collisions. + /// + /// @param uid - The unique ID of the instruction. + /// @return - A reference to the instruction specifier. + InstructionSpecifier& specForUID(InstrUID uid) { + if (uid >= InstructionSpecifiers.size()) + InstructionSpecifiers.resize(uid + 1); + + return InstructionSpecifiers[uid]; + } + + // hasConflicts - Reports whether there were primary decode conflicts + // from any instructions added to the tables. + // @return - true if there were; false otherwise. + + bool hasConflicts() { + return HasConflicts; + } +}; + +} // namespace X86Disassembler + +} // namespace llvm + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp new file mode 100644 index 000000000000..6dc7e31e0dab --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp @@ -0,0 +1,237 @@ +//===- utils/TableGen/X86EVEX2VEXTablesEmitter.cpp - X86 backend-*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// This tablegen backend is responsible for emitting the X86 backend EVEX2VEX +/// compression tables. +/// +//===----------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/TableGenBackend.h" + +using namespace llvm; + +namespace { + +class X86EVEX2VEXTablesEmitter { + RecordKeeper &Records; + CodeGenTarget Target; + + // Hold all non-masked & non-broadcasted EVEX encoded instructions + std::vector<const CodeGenInstruction *> EVEXInsts; + // Hold all VEX encoded instructions. Divided into groups with same opcodes + // to make the search more efficient + std::map<uint64_t, std::vector<const CodeGenInstruction *>> VEXInsts; + + typedef std::pair<const CodeGenInstruction *, const CodeGenInstruction *> Entry; + + // Represent both compress tables + std::vector<Entry> EVEX2VEX128; + std::vector<Entry> EVEX2VEX256; + +public: + X86EVEX2VEXTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} + + // run - Output X86 EVEX2VEX tables. + void run(raw_ostream &OS); + +private: + // Prints the given table as a C++ array of type + // X86EvexToVexCompressTableEntry + void printTable(const std::vector<Entry> &Table, raw_ostream &OS); +}; + +void X86EVEX2VEXTablesEmitter::printTable(const std::vector<Entry> &Table, + raw_ostream &OS) { + StringRef Size = (Table == EVEX2VEX128) ? "128" : "256"; + + OS << "// X86 EVEX encoded instructions that have a VEX " << Size + << " encoding\n" + << "// (table format: <EVEX opcode, VEX-" << Size << " opcode>).\n" + << "static const X86EvexToVexCompressTableEntry X86EvexToVex" << Size + << "CompressTable[] = {\n" + << " // EVEX scalar with corresponding VEX.\n"; + + // Print all entries added to the table + for (auto Pair : Table) { + OS << " { X86::" << Pair.first->TheDef->getName() + << ", X86::" << Pair.second->TheDef->getName() << " },\n"; + } + + OS << "};\n\n"; +} + +// Return true if the 2 BitsInits are equal +// Calculates the integer value residing BitsInit object +static inline uint64_t getValueFromBitsInit(const BitsInit *B) { + uint64_t Value = 0; + for (unsigned i = 0, e = B->getNumBits(); i != e; ++i) { + if (BitInit *Bit = dyn_cast<BitInit>(B->getBit(i))) + Value |= uint64_t(Bit->getValue()) << i; + else + PrintFatalError("Invalid VectSize bit"); + } + return Value; +} + +// Function object - Operator() returns true if the given VEX instruction +// matches the EVEX instruction of this object. +class IsMatch { + const CodeGenInstruction *EVEXInst; + +public: + IsMatch(const CodeGenInstruction *EVEXInst) : EVEXInst(EVEXInst) {} + + bool operator()(const CodeGenInstruction *VEXInst) { + Record *RecE = EVEXInst->TheDef; + Record *RecV = VEXInst->TheDef; + bool EVEX_W = RecE->getValueAsBit("HasVEX_W"); + bool VEX_W = RecV->getValueAsBit("HasVEX_W"); + bool VEX_WIG = RecV->getValueAsBit("IgnoresVEX_W"); + bool EVEX_WIG = RecE->getValueAsBit("IgnoresVEX_W"); + bool EVEX_W1_VEX_W0 = RecE->getValueAsBit("EVEX_W1_VEX_W0"); + + if (RecV->getValueAsDef("OpEnc")->getName().str() != "EncVEX" || + RecV->getValueAsBit("isCodeGenOnly") != RecE->getValueAsBit("isCodeGenOnly") || + // VEX/EVEX fields + RecV->getValueAsDef("OpPrefix") != RecE->getValueAsDef("OpPrefix") || + RecV->getValueAsDef("OpMap") != RecE->getValueAsDef("OpMap") || + RecV->getValueAsBit("hasVEX_4V") != RecE->getValueAsBit("hasVEX_4V") || + RecV->getValueAsBit("hasEVEX_L2") != RecE->getValueAsBit("hasEVEX_L2") || + RecV->getValueAsBit("hasVEX_L") != RecE->getValueAsBit("hasVEX_L") || + // Match is allowed if either is VEX_WIG, or they match, or EVEX + // is VEX_W1X and VEX is VEX_W0. + (!(VEX_WIG || (!EVEX_WIG && EVEX_W == VEX_W) || + (EVEX_W1_VEX_W0 && EVEX_W && !VEX_W))) || + // Instruction's format + RecV->getValueAsDef("Form") != RecE->getValueAsDef("Form")) + return false; + + // This is needed for instructions with intrinsic version (_Int). + // Where the only difference is the size of the operands. + // For example: VUCOMISDZrm and Int_VUCOMISDrm + // Also for instructions that their EVEX version was upgraded to work with + // k-registers. For example VPCMPEQBrm (xmm output register) and + // VPCMPEQBZ128rm (k register output register). + for (unsigned i = 0, e = EVEXInst->Operands.size(); i < e; i++) { + Record *OpRec1 = EVEXInst->Operands[i].Rec; + Record *OpRec2 = VEXInst->Operands[i].Rec; + + if (OpRec1 == OpRec2) + continue; + + if (isRegisterOperand(OpRec1) && isRegisterOperand(OpRec2)) { + if (getRegOperandSize(OpRec1) != getRegOperandSize(OpRec2)) + return false; + } else if (isMemoryOperand(OpRec1) && isMemoryOperand(OpRec2)) { + return false; + } else if (isImmediateOperand(OpRec1) && isImmediateOperand(OpRec2)) { + if (OpRec1->getValueAsDef("Type") != OpRec2->getValueAsDef("Type")) { + return false; + } + } else + return false; + } + + return true; + } + +private: + static inline bool isRegisterOperand(const Record *Rec) { + return Rec->isSubClassOf("RegisterClass") || + Rec->isSubClassOf("RegisterOperand"); + } + + static inline bool isMemoryOperand(const Record *Rec) { + return Rec->isSubClassOf("Operand") && + Rec->getValueAsString("OperandType") == "OPERAND_MEMORY"; + } + + static inline bool isImmediateOperand(const Record *Rec) { + return Rec->isSubClassOf("Operand") && + Rec->getValueAsString("OperandType") == "OPERAND_IMMEDIATE"; + } + + static inline unsigned int getRegOperandSize(const Record *RegRec) { + if (RegRec->isSubClassOf("RegisterClass")) + return RegRec->getValueAsInt("Alignment"); + if (RegRec->isSubClassOf("RegisterOperand")) + return RegRec->getValueAsDef("RegClass")->getValueAsInt("Alignment"); + + llvm_unreachable("Register operand's size not known!"); + } +}; + +void X86EVEX2VEXTablesEmitter::run(raw_ostream &OS) { + emitSourceFileHeader("X86 EVEX2VEX tables", OS); + + ArrayRef<const CodeGenInstruction *> NumberedInstructions = + Target.getInstructionsByEnumValue(); + + for (const CodeGenInstruction *Inst : NumberedInstructions) { + // Filter non-X86 instructions. + if (!Inst->TheDef->isSubClassOf("X86Inst")) + continue; + + // Add VEX encoded instructions to one of VEXInsts vectors according to + // it's opcode. + if (Inst->TheDef->getValueAsDef("OpEnc")->getName() == "EncVEX") { + uint64_t Opcode = getValueFromBitsInit(Inst->TheDef-> + getValueAsBitsInit("Opcode")); + VEXInsts[Opcode].push_back(Inst); + } + // Add relevant EVEX encoded instructions to EVEXInsts + else if (Inst->TheDef->getValueAsDef("OpEnc")->getName() == "EncEVEX" && + !Inst->TheDef->getValueAsBit("hasEVEX_K") && + !Inst->TheDef->getValueAsBit("hasEVEX_B") && + !Inst->TheDef->getValueAsBit("hasEVEX_L2") && + !Inst->TheDef->getValueAsBit("notEVEX2VEXConvertible")) + EVEXInsts.push_back(Inst); + } + + for (const CodeGenInstruction *EVEXInst : EVEXInsts) { + uint64_t Opcode = getValueFromBitsInit(EVEXInst->TheDef-> + getValueAsBitsInit("Opcode")); + // For each EVEX instruction look for a VEX match in the appropriate vector + // (instructions with the same opcode) using function object IsMatch. + // Allow EVEX2VEXOverride to explicitly specify a match. + const CodeGenInstruction *VEXInst = nullptr; + if (!EVEXInst->TheDef->isValueUnset("EVEX2VEXOverride")) { + StringRef AltInstStr = + EVEXInst->TheDef->getValueAsString("EVEX2VEXOverride"); + Record *AltInstRec = Records.getDef(AltInstStr); + assert(AltInstRec && "EVEX2VEXOverride instruction not found!"); + VEXInst = &Target.getInstruction(AltInstRec); + } else { + auto Match = llvm::find_if(VEXInsts[Opcode], IsMatch(EVEXInst)); + if (Match != VEXInsts[Opcode].end()) + VEXInst = *Match; + } + + if (!VEXInst) + continue; + + // In case a match is found add new entry to the appropriate table + if (EVEXInst->TheDef->getValueAsBit("hasVEX_L")) + EVEX2VEX256.push_back(std::make_pair(EVEXInst, VEXInst)); // {0,1} + else + EVEX2VEX128.push_back(std::make_pair(EVEXInst, VEXInst)); // {0,0} + } + + // Print both tables + printTable(EVEX2VEX128, OS); + printTable(EVEX2VEX256, OS); +} +} + +namespace llvm { +void EmitX86EVEX2VEXTables(RecordKeeper &RK, raw_ostream &OS) { + X86EVEX2VEXTablesEmitter(RK).run(OS); +} +} diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/contrib/llvm-project/llvm/utils/TableGen/X86FoldTablesEmitter.cpp new file mode 100644 index 000000000000..8026c324cd40 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/X86FoldTablesEmitter.cpp @@ -0,0 +1,671 @@ +//===- utils/TableGen/X86FoldTablesEmitter.cpp - X86 backend-*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This tablegen backend is responsible for emitting the memory fold tables of +// the X86 backend instructions. +// +//===----------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "X86RecognizableInstr.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/TableGenBackend.h" + +using namespace llvm; + +namespace { + +// 3 possible strategies for the unfolding flag (TB_NO_REVERSE) of the +// manual added entries. +enum UnfoldStrategy { + UNFOLD, // Allow unfolding + NO_UNFOLD, // Prevent unfolding + NO_STRATEGY // Make decision according to operands' sizes +}; + +// Represents an entry in the manual mapped instructions set. +struct ManualMapEntry { + const char *RegInstStr; + const char *MemInstStr; + UnfoldStrategy Strategy; + + ManualMapEntry(const char *RegInstStr, const char *MemInstStr, + UnfoldStrategy Strategy = NO_STRATEGY) + : RegInstStr(RegInstStr), MemInstStr(MemInstStr), Strategy(Strategy) {} +}; + +class IsMatch; + +// List of instructions requiring explicitly aligned memory. +const char *ExplicitAlign[] = {"MOVDQA", "MOVAPS", "MOVAPD", "MOVNTPS", + "MOVNTPD", "MOVNTDQ", "MOVNTDQA"}; + +// List of instructions NOT requiring explicit memory alignment. +const char *ExplicitUnalign[] = {"MOVDQU", "MOVUPS", "MOVUPD", + "PCMPESTRM", "PCMPESTRI", + "PCMPISTRM", "PCMPISTRI" }; + +// For manually mapping instructions that do not match by their encoding. +const ManualMapEntry ManualMapSet[] = { + { "ADD16ri_DB", "ADD16mi", NO_UNFOLD }, + { "ADD16ri8_DB", "ADD16mi8", NO_UNFOLD }, + { "ADD16rr_DB", "ADD16mr", NO_UNFOLD }, + { "ADD32ri_DB", "ADD32mi", NO_UNFOLD }, + { "ADD32ri8_DB", "ADD32mi8", NO_UNFOLD }, + { "ADD32rr_DB", "ADD32mr", NO_UNFOLD }, + { "ADD64ri32_DB", "ADD64mi32", NO_UNFOLD }, + { "ADD64ri8_DB", "ADD64mi8", NO_UNFOLD }, + { "ADD64rr_DB", "ADD64mr", NO_UNFOLD }, + { "ADD8ri_DB", "ADD8mi", NO_UNFOLD }, + { "ADD8rr_DB", "ADD8mr", NO_UNFOLD }, + { "ADD16rr_DB", "ADD16rm", NO_UNFOLD }, + { "ADD32rr_DB", "ADD32rm", NO_UNFOLD }, + { "ADD64rr_DB", "ADD64rm", NO_UNFOLD }, + { "ADD8rr_DB", "ADD8rm", NO_UNFOLD }, + { "PUSH16r", "PUSH16rmm", UNFOLD }, + { "PUSH32r", "PUSH32rmm", UNFOLD }, + { "PUSH64r", "PUSH64rmm", UNFOLD }, + { "TAILJMPr", "TAILJMPm", UNFOLD }, + { "TAILJMPr64", "TAILJMPm64", UNFOLD }, + { "TAILJMPr64_REX", "TAILJMPm64_REX", UNFOLD }, +}; + + +static bool isExplicitAlign(const CodeGenInstruction *Inst) { + return any_of(ExplicitAlign, [Inst](const char *InstStr) { + return Inst->TheDef->getName().find(InstStr) != StringRef::npos; + }); +} + +static bool isExplicitUnalign(const CodeGenInstruction *Inst) { + return any_of(ExplicitUnalign, [Inst](const char *InstStr) { + return Inst->TheDef->getName().find(InstStr) != StringRef::npos; + }); +} + +class X86FoldTablesEmitter { + RecordKeeper &Records; + CodeGenTarget Target; + + // Represents an entry in the folding table + class X86FoldTableEntry { + const CodeGenInstruction *RegInst; + const CodeGenInstruction *MemInst; + + public: + bool CannotUnfold = false; + bool IsLoad = false; + bool IsStore = false; + bool IsAligned = false; + unsigned int Alignment = 0; + + X86FoldTableEntry(const CodeGenInstruction *RegInst, + const CodeGenInstruction *MemInst) + : RegInst(RegInst), MemInst(MemInst) {} + + void print(formatted_raw_ostream &OS) const { + OS.indent(2); + OS << "{ X86::" << RegInst->TheDef->getName() << ","; + OS.PadToColumn(40); + OS << "X86::" << MemInst->TheDef->getName() << ","; + OS.PadToColumn(75); + + if (IsLoad) + OS << "TB_FOLDED_LOAD | "; + if (IsStore) + OS << "TB_FOLDED_STORE | "; + if (CannotUnfold) + OS << "TB_NO_REVERSE | "; + if (IsAligned) + OS << "TB_ALIGN_" << Alignment << " | "; + + OS << "0 },\n"; + } + }; + + typedef std::vector<X86FoldTableEntry> FoldTable; + // std::vector for each folding table. + // Table2Addr - Holds instructions which their memory form performs load+store + // Table#i - Holds instructions which the their memory form perform a load OR + // a store, and their #i'th operand is folded. + FoldTable Table2Addr; + FoldTable Table0; + FoldTable Table1; + FoldTable Table2; + FoldTable Table3; + FoldTable Table4; + +public: + X86FoldTablesEmitter(RecordKeeper &R) : Records(R), Target(R) {} + + // run - Generate the 6 X86 memory fold tables. + void run(formatted_raw_ostream &OS); + +private: + // Decides to which table to add the entry with the given instructions. + // S sets the strategy of adding the TB_NO_REVERSE flag. + void updateTables(const CodeGenInstruction *RegInstr, + const CodeGenInstruction *MemInstr, + const UnfoldStrategy S = NO_STRATEGY); + + // Generates X86FoldTableEntry with the given instructions and fill it with + // the appropriate flags - then adds it to Table. + void addEntryWithFlags(FoldTable &Table, const CodeGenInstruction *RegInstr, + const CodeGenInstruction *MemInstr, + const UnfoldStrategy S, const unsigned int FoldedInd); + + // Print the given table as a static const C++ array of type + // X86MemoryFoldTableEntry. + void printTable(const FoldTable &Table, StringRef TableName, + formatted_raw_ostream &OS) { + OS << "static const X86MemoryFoldTableEntry MemoryFold" << TableName + << "[] = {\n"; + + for (const X86FoldTableEntry &E : Table) + E.print(OS); + + OS << "};\n\n"; + } +}; + +// Return true if one of the instruction's operands is a RST register class +static bool hasRSTRegClass(const CodeGenInstruction *Inst) { + return any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) { + return OpIn.Rec->getName() == "RST" || OpIn.Rec->getName() == "RSTi"; + }); +} + +// Return true if one of the instruction's operands is a ptr_rc_tailcall +static bool hasPtrTailcallRegClass(const CodeGenInstruction *Inst) { + return any_of(Inst->Operands, [](const CGIOperandList::OperandInfo &OpIn) { + return OpIn.Rec->getName() == "ptr_rc_tailcall"; + }); +} + +// Calculates the integer value representing the BitsInit object +static inline uint64_t getValueFromBitsInit(const BitsInit *B) { + assert(B->getNumBits() <= sizeof(uint64_t) * 8 && "BitInits' too long!"); + + uint64_t Value = 0; + for (unsigned i = 0, e = B->getNumBits(); i != e; ++i) { + BitInit *Bit = cast<BitInit>(B->getBit(i)); + Value |= uint64_t(Bit->getValue()) << i; + } + return Value; +} + +// Returns true if the two given BitsInits represent the same integer value +static inline bool equalBitsInits(const BitsInit *B1, const BitsInit *B2) { + if (B1->getNumBits() != B2->getNumBits()) + PrintFatalError("Comparing two BitsInits with different sizes!"); + + for (unsigned i = 0, e = B1->getNumBits(); i != e; ++i) { + BitInit *Bit1 = cast<BitInit>(B1->getBit(i)); + BitInit *Bit2 = cast<BitInit>(B2->getBit(i)); + if (Bit1->getValue() != Bit2->getValue()) + return false; + } + return true; +} + +// Return the size of the register operand +static inline unsigned int getRegOperandSize(const Record *RegRec) { + if (RegRec->isSubClassOf("RegisterOperand")) + RegRec = RegRec->getValueAsDef("RegClass"); + if (RegRec->isSubClassOf("RegisterClass")) + return RegRec->getValueAsListOfDefs("RegTypes")[0]->getValueAsInt("Size"); + + llvm_unreachable("Register operand's size not known!"); +} + +// Return the size of the memory operand +static inline unsigned int +getMemOperandSize(const Record *MemRec, const bool IntrinsicSensitive = false) { + if (MemRec->isSubClassOf("Operand")) { + // Intrinsic memory instructions use ssmem/sdmem. + if (IntrinsicSensitive && + (MemRec->getName() == "sdmem" || MemRec->getName() == "ssmem")) + return 128; + + StringRef Name = + MemRec->getValueAsDef("ParserMatchClass")->getValueAsString("Name"); + if (Name == "Mem8") + return 8; + if (Name == "Mem16") + return 16; + if (Name == "Mem32") + return 32; + if (Name == "Mem64") + return 64; + if (Name == "Mem80") + return 80; + if (Name == "Mem128") + return 128; + if (Name == "Mem256") + return 256; + if (Name == "Mem512") + return 512; + } + + llvm_unreachable("Memory operand's size not known!"); +} + +// Return true if the instruction defined as a register flavor. +static inline bool hasRegisterFormat(const Record *Inst) { + const BitsInit *FormBits = Inst->getValueAsBitsInit("FormBits"); + uint64_t FormBitsNum = getValueFromBitsInit(FormBits); + + // Values from X86Local namespace defined in X86RecognizableInstr.cpp + return FormBitsNum >= X86Local::MRMDestReg && FormBitsNum <= X86Local::MRM7r; +} + +// Return true if the instruction defined as a memory flavor. +static inline bool hasMemoryFormat(const Record *Inst) { + const BitsInit *FormBits = Inst->getValueAsBitsInit("FormBits"); + uint64_t FormBitsNum = getValueFromBitsInit(FormBits); + + // Values from X86Local namespace defined in X86RecognizableInstr.cpp + return FormBitsNum >= X86Local::MRMDestMem && FormBitsNum <= X86Local::MRM7m; +} + +static inline bool isNOREXRegClass(const Record *Op) { + return Op->getName().find("_NOREX") != StringRef::npos; +} + +static inline bool isRegisterOperand(const Record *Rec) { + return Rec->isSubClassOf("RegisterClass") || + Rec->isSubClassOf("RegisterOperand") || + Rec->isSubClassOf("PointerLikeRegClass"); +} + +static inline bool isMemoryOperand(const Record *Rec) { + return Rec->isSubClassOf("Operand") && + Rec->getValueAsString("OperandType") == "OPERAND_MEMORY"; +} + +static inline bool isImmediateOperand(const Record *Rec) { + return Rec->isSubClassOf("Operand") && + Rec->getValueAsString("OperandType") == "OPERAND_IMMEDIATE"; +} + +// Get the alternative instruction pointed by "FoldGenRegForm" field. +static inline const CodeGenInstruction * +getAltRegInst(const CodeGenInstruction *I, const RecordKeeper &Records, + const CodeGenTarget &Target) { + + StringRef AltRegInstStr = I->TheDef->getValueAsString("FoldGenRegForm"); + Record *AltRegInstRec = Records.getDef(AltRegInstStr); + assert(AltRegInstRec && + "Alternative register form instruction def not found"); + CodeGenInstruction &AltRegInst = Target.getInstruction(AltRegInstRec); + return &AltRegInst; +} + +// Function object - Operator() returns true if the given VEX instruction +// matches the EVEX instruction of this object. +class IsMatch { + const CodeGenInstruction *MemInst; + +public: + IsMatch(const CodeGenInstruction *Inst, const RecordKeeper &Records) + : MemInst(Inst) {} + + bool operator()(const CodeGenInstruction *RegInst) { + Record *MemRec = MemInst->TheDef; + Record *RegRec = RegInst->TheDef; + + // Return false if one (at least) of the encoding fields of both + // instructions do not match. + if (RegRec->getValueAsDef("OpEnc") != MemRec->getValueAsDef("OpEnc") || + !equalBitsInits(RegRec->getValueAsBitsInit("Opcode"), + MemRec->getValueAsBitsInit("Opcode")) || + // VEX/EVEX fields + RegRec->getValueAsDef("OpPrefix") != + MemRec->getValueAsDef("OpPrefix") || + RegRec->getValueAsDef("OpMap") != MemRec->getValueAsDef("OpMap") || + RegRec->getValueAsDef("OpSize") != MemRec->getValueAsDef("OpSize") || + RegRec->getValueAsDef("AdSize") != MemRec->getValueAsDef("AdSize") || + RegRec->getValueAsBit("hasVEX_4V") != + MemRec->getValueAsBit("hasVEX_4V") || + RegRec->getValueAsBit("hasEVEX_K") != + MemRec->getValueAsBit("hasEVEX_K") || + RegRec->getValueAsBit("hasEVEX_Z") != + MemRec->getValueAsBit("hasEVEX_Z") || + // EVEX_B means different things for memory and register forms. + RegRec->getValueAsBit("hasEVEX_B") != 0 || + MemRec->getValueAsBit("hasEVEX_B") != 0 || + RegRec->getValueAsBit("hasEVEX_RC") != + MemRec->getValueAsBit("hasEVEX_RC") || + RegRec->getValueAsBit("hasREX_WPrefix") != + MemRec->getValueAsBit("hasREX_WPrefix") || + RegRec->getValueAsBit("hasLockPrefix") != + MemRec->getValueAsBit("hasLockPrefix") || + RegRec->getValueAsBit("hasNoTrackPrefix") != + MemRec->getValueAsBit("hasNoTrackPrefix") || + RegRec->getValueAsBit("hasVEX_L") != + MemRec->getValueAsBit("hasVEX_L") || + RegRec->getValueAsBit("hasEVEX_L2") != + MemRec->getValueAsBit("hasEVEX_L2") || + RegRec->getValueAsBit("ignoresVEX_L") != + MemRec->getValueAsBit("ignoresVEX_L") || + RegRec->getValueAsBit("HasVEX_W") != + MemRec->getValueAsBit("HasVEX_W") || + RegRec->getValueAsBit("IgnoresVEX_W") != + MemRec->getValueAsBit("IgnoresVEX_W") || + RegRec->getValueAsBit("EVEX_W1_VEX_W0") != + MemRec->getValueAsBit("EVEX_W1_VEX_W0") || + // Instruction's format - The register form's "Form" field should be + // the opposite of the memory form's "Form" field. + !areOppositeForms(RegRec->getValueAsBitsInit("FormBits"), + MemRec->getValueAsBitsInit("FormBits")) || + RegRec->getValueAsBit("isAsmParserOnly") != + MemRec->getValueAsBit("isAsmParserOnly")) + return false; + + // Make sure the sizes of the operands of both instructions suit each other. + // This is needed for instructions with intrinsic version (_Int). + // Where the only difference is the size of the operands. + // For example: VUCOMISDZrm and Int_VUCOMISDrm + // Also for instructions that their EVEX version was upgraded to work with + // k-registers. For example VPCMPEQBrm (xmm output register) and + // VPCMPEQBZ128rm (k register output register). + bool ArgFolded = false; + unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); + unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); + unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); + unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs(); + + // Instructions with one output in their memory form use the memory folded + // operand as source and destination (Read-Modify-Write). + unsigned RegStartIdx = + (MemOutSize + 1 == RegOutSize) && (MemInSize == RegInSize) ? 1 : 0; + + for (unsigned i = 0, e = MemInst->Operands.size(); i < e; i++) { + Record *MemOpRec = MemInst->Operands[i].Rec; + Record *RegOpRec = RegInst->Operands[i + RegStartIdx].Rec; + + if (MemOpRec == RegOpRec) + continue; + + if (isRegisterOperand(MemOpRec) && isRegisterOperand(RegOpRec)) { + if (getRegOperandSize(MemOpRec) != getRegOperandSize(RegOpRec) || + isNOREXRegClass(MemOpRec) != isNOREXRegClass(RegOpRec)) + return false; + } else if (isMemoryOperand(MemOpRec) && isMemoryOperand(RegOpRec)) { + if (getMemOperandSize(MemOpRec) != getMemOperandSize(RegOpRec)) + return false; + } else if (isImmediateOperand(MemOpRec) && isImmediateOperand(RegOpRec)) { + if (MemOpRec->getValueAsDef("Type") != RegOpRec->getValueAsDef("Type")) + return false; + } else { + // Only one operand can be folded. + if (ArgFolded) + return false; + + assert(isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)); + ArgFolded = true; + } + } + + return true; + } + +private: + // Return true of the 2 given forms are the opposite of each other. + bool areOppositeForms(const BitsInit *RegFormBits, + const BitsInit *MemFormBits) { + uint64_t MemFormNum = getValueFromBitsInit(MemFormBits); + uint64_t RegFormNum = getValueFromBitsInit(RegFormBits); + + if ((MemFormNum == X86Local::MRM0m && RegFormNum == X86Local::MRM0r) || + (MemFormNum == X86Local::MRM1m && RegFormNum == X86Local::MRM1r) || + (MemFormNum == X86Local::MRM2m && RegFormNum == X86Local::MRM2r) || + (MemFormNum == X86Local::MRM3m && RegFormNum == X86Local::MRM3r) || + (MemFormNum == X86Local::MRM4m && RegFormNum == X86Local::MRM4r) || + (MemFormNum == X86Local::MRM5m && RegFormNum == X86Local::MRM5r) || + (MemFormNum == X86Local::MRM6m && RegFormNum == X86Local::MRM6r) || + (MemFormNum == X86Local::MRM7m && RegFormNum == X86Local::MRM7r) || + (MemFormNum == X86Local::MRMXm && RegFormNum == X86Local::MRMXr) || + (MemFormNum == X86Local::MRMXmCC && RegFormNum == X86Local::MRMXrCC) || + (MemFormNum == X86Local::MRMDestMem && + RegFormNum == X86Local::MRMDestReg) || + (MemFormNum == X86Local::MRMSrcMem && + RegFormNum == X86Local::MRMSrcReg) || + (MemFormNum == X86Local::MRMSrcMem4VOp3 && + RegFormNum == X86Local::MRMSrcReg4VOp3) || + (MemFormNum == X86Local::MRMSrcMemOp4 && + RegFormNum == X86Local::MRMSrcRegOp4) || + (MemFormNum == X86Local::MRMSrcMemCC && + RegFormNum == X86Local::MRMSrcRegCC)) + return true; + + return false; + } +}; + +} // end anonymous namespace + +void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table, + const CodeGenInstruction *RegInstr, + const CodeGenInstruction *MemInstr, + const UnfoldStrategy S, + const unsigned int FoldedInd) { + + X86FoldTableEntry Result = X86FoldTableEntry(RegInstr, MemInstr); + Record *RegRec = RegInstr->TheDef; + Record *MemRec = MemInstr->TheDef; + + // Only table0 entries should explicitly specify a load or store flag. + if (&Table == &Table0) { + unsigned MemInOpsNum = MemRec->getValueAsDag("InOperandList")->getNumArgs(); + unsigned RegInOpsNum = RegRec->getValueAsDag("InOperandList")->getNumArgs(); + // If the instruction writes to the folded operand, it will appear as an + // output in the register form instruction and as an input in the memory + // form instruction. + // If the instruction reads from the folded operand, it well appear as in + // input in both forms. + if (MemInOpsNum == RegInOpsNum) + Result.IsLoad = true; + else + Result.IsStore = true; + } + + Record *RegOpRec = RegInstr->Operands[FoldedInd].Rec; + Record *MemOpRec = MemInstr->Operands[FoldedInd].Rec; + + // Unfolding code generates a load/store instruction according to the size of + // the register in the register form instruction. + // If the register's size is greater than the memory's operand size, do not + // allow unfolding. + if (S == UNFOLD) + Result.CannotUnfold = false; + else if (S == NO_UNFOLD) + Result.CannotUnfold = true; + else if (getRegOperandSize(RegOpRec) > getMemOperandSize(MemOpRec)) + Result.CannotUnfold = true; // S == NO_STRATEGY + + uint64_t Enc = getValueFromBitsInit(RegRec->getValueAsBitsInit("OpEncBits")); + if (isExplicitAlign(RegInstr)) { + // The instruction require explicitly aligned memory. + BitsInit *VectSize = RegRec->getValueAsBitsInit("VectSize"); + uint64_t Value = getValueFromBitsInit(VectSize); + Result.IsAligned = true; + Result.Alignment = Value; + } else if (Enc != X86Local::XOP && Enc != X86Local::VEX && + Enc != X86Local::EVEX) { + // Instructions with VEX encoding do not require alignment. + if (!isExplicitUnalign(RegInstr) && getMemOperandSize(MemOpRec) > 64) { + // SSE packed vector instructions require a 16 byte alignment. + Result.IsAligned = true; + Result.Alignment = 16; + } + } + + Table.push_back(Result); +} + +void X86FoldTablesEmitter::updateTables(const CodeGenInstruction *RegInstr, + const CodeGenInstruction *MemInstr, + const UnfoldStrategy S) { + + Record *RegRec = RegInstr->TheDef; + Record *MemRec = MemInstr->TheDef; + unsigned MemOutSize = MemRec->getValueAsDag("OutOperandList")->getNumArgs(); + unsigned RegOutSize = RegRec->getValueAsDag("OutOperandList")->getNumArgs(); + unsigned MemInSize = MemRec->getValueAsDag("InOperandList")->getNumArgs(); + unsigned RegInSize = RegRec->getValueAsDag("InOperandList")->getNumArgs(); + + // Instructions which Read-Modify-Write should be added to Table2Addr. + if (MemOutSize != RegOutSize && MemInSize == RegInSize) { + addEntryWithFlags(Table2Addr, RegInstr, MemInstr, S, 0); + return; + } + + if (MemInSize == RegInSize && MemOutSize == RegOutSize) { + // Load-Folding cases. + // If the i'th register form operand is a register and the i'th memory form + // operand is a memory operand, add instructions to Table#i. + for (unsigned i = RegOutSize, e = RegInstr->Operands.size(); i < e; i++) { + Record *RegOpRec = RegInstr->Operands[i].Rec; + Record *MemOpRec = MemInstr->Operands[i].Rec; + if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec)) { + switch (i) { + case 0: + addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0); + return; + case 1: + addEntryWithFlags(Table1, RegInstr, MemInstr, S, 1); + return; + case 2: + addEntryWithFlags(Table2, RegInstr, MemInstr, S, 2); + return; + case 3: + addEntryWithFlags(Table3, RegInstr, MemInstr, S, 3); + return; + case 4: + addEntryWithFlags(Table4, RegInstr, MemInstr, S, 4); + return; + } + } + } + } else if (MemInSize == RegInSize + 1 && MemOutSize + 1 == RegOutSize) { + // Store-Folding cases. + // If the memory form instruction performs a store, the *output* + // register of the register form instructions disappear and instead a + // memory *input* operand appears in the memory form instruction. + // For example: + // MOVAPSrr => (outs VR128:$dst), (ins VR128:$src) + // MOVAPSmr => (outs), (ins f128mem:$dst, VR128:$src) + Record *RegOpRec = RegInstr->Operands[RegOutSize - 1].Rec; + Record *MemOpRec = MemInstr->Operands[RegOutSize - 1].Rec; + if (isRegisterOperand(RegOpRec) && isMemoryOperand(MemOpRec) && + getRegOperandSize(RegOpRec) == getMemOperandSize(MemOpRec)) + addEntryWithFlags(Table0, RegInstr, MemInstr, S, 0); + } + + return; +} + +void X86FoldTablesEmitter::run(formatted_raw_ostream &OS) { + emitSourceFileHeader("X86 fold tables", OS); + + // Holds all memory instructions + std::vector<const CodeGenInstruction *> MemInsts; + // Holds all register instructions - divided according to opcode. + std::map<uint8_t, std::vector<const CodeGenInstruction *>> RegInsts; + + ArrayRef<const CodeGenInstruction *> NumberedInstructions = + Target.getInstructionsByEnumValue(); + + for (const CodeGenInstruction *Inst : NumberedInstructions) { + if (!Inst->TheDef->getNameInit() || !Inst->TheDef->isSubClassOf("X86Inst")) + continue; + + const Record *Rec = Inst->TheDef; + + // - Do not proceed if the instruction is marked as notMemoryFoldable. + // - Instructions including RST register class operands are not relevant + // for memory folding (for further details check the explanation in + // lib/Target/X86/X86InstrFPStack.td file). + // - Some instructions (listed in the manual map above) use the register + // class ptr_rc_tailcall, which can be of a size 32 or 64, to ensure + // safe mapping of these instruction we manually map them and exclude + // them from the automation. + if (Rec->getValueAsBit("isMemoryFoldable") == false || + hasRSTRegClass(Inst) || hasPtrTailcallRegClass(Inst)) + continue; + + // Add all the memory form instructions to MemInsts, and all the register + // form instructions to RegInsts[Opc], where Opc in the opcode of each + // instructions. this helps reducing the runtime of the backend. + if (hasMemoryFormat(Rec)) + MemInsts.push_back(Inst); + else if (hasRegisterFormat(Rec)) { + uint8_t Opc = getValueFromBitsInit(Rec->getValueAsBitsInit("Opcode")); + RegInsts[Opc].push_back(Inst); + } + } + + // For each memory form instruction, try to find its register form + // instruction. + for (const CodeGenInstruction *MemInst : MemInsts) { + uint8_t Opc = + getValueFromBitsInit(MemInst->TheDef->getValueAsBitsInit("Opcode")); + + auto RegInstsIt = RegInsts.find(Opc); + if (RegInstsIt == RegInsts.end()) + continue; + + // Two forms (memory & register) of the same instruction must have the same + // opcode. try matching only with register form instructions with the same + // opcode. + std::vector<const CodeGenInstruction *> &OpcRegInsts = RegInstsIt->second; + + auto Match = find_if(OpcRegInsts, IsMatch(MemInst, Records)); + if (Match != OpcRegInsts.end()) { + const CodeGenInstruction *RegInst = *Match; + // If the matched instruction has it's "FoldGenRegForm" set, map the + // memory form instruction to the register form instruction pointed by + // this field + if (RegInst->TheDef->isValueUnset("FoldGenRegForm")) { + updateTables(RegInst, MemInst); + } else { + const CodeGenInstruction *AltRegInst = + getAltRegInst(RegInst, Records, Target); + updateTables(AltRegInst, MemInst); + } + OpcRegInsts.erase(Match); + } + } + + // Add the manually mapped instructions listed above. + for (const ManualMapEntry &Entry : ManualMapSet) { + Record *RegInstIter = Records.getDef(Entry.RegInstStr); + Record *MemInstIter = Records.getDef(Entry.MemInstStr); + + updateTables(&(Target.getInstruction(RegInstIter)), + &(Target.getInstruction(MemInstIter)), Entry.Strategy); + } + + // Print all tables. + printTable(Table2Addr, "Table2Addr", OS); + printTable(Table0, "Table0", OS); + printTable(Table1, "Table1", OS); + printTable(Table2, "Table2", OS); + printTable(Table3, "Table3", OS); + printTable(Table4, "Table4", OS); +} + +namespace llvm { + +void EmitX86FoldTables(RecordKeeper &RK, raw_ostream &o) { + formatted_raw_ostream OS(o); + X86FoldTablesEmitter(RK).run(OS); +} +} // namespace llvm diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86ModRMFilters.cpp b/contrib/llvm-project/llvm/utils/TableGen/X86ModRMFilters.cpp new file mode 100644 index 000000000000..cf7507094fa7 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/X86ModRMFilters.cpp @@ -0,0 +1,23 @@ +//===- X86ModRMFilters.cpp - Disassembler ModR/M filterss -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "X86ModRMFilters.h" + +using namespace llvm::X86Disassembler; + +void ModRMFilter::anchor() { } + +void DumbFilter::anchor() { } + +void ModFilter::anchor() { } + +void ExtendedFilter::anchor() { } + +void ExtendedRMFilter::anchor() { } + +void ExactFilter::anchor() { } diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86ModRMFilters.h b/contrib/llvm-project/llvm/utils/TableGen/X86ModRMFilters.h new file mode 100644 index 000000000000..f0b8af5fb82a --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/X86ModRMFilters.h @@ -0,0 +1,157 @@ +//===- X86ModRMFilters.h - Disassembler ModR/M filterss ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is part of the X86 Disassembler Emitter. +// It contains ModR/M filters that determine which values of the ModR/M byte +// are valid for a partiuclar instruction. +// Documentation for the disassembler emitter in general can be found in +// X86DisassemblerEmitter.h. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_X86MODRMFILTERS_H +#define LLVM_UTILS_TABLEGEN_X86MODRMFILTERS_H + +#include "llvm/Support/DataTypes.h" + +namespace llvm { + +namespace X86Disassembler { + +/// ModRMFilter - Abstract base class for clases that recognize patterns in +/// ModR/M bytes. +class ModRMFilter { + virtual void anchor(); +public: + /// Destructor - Override as necessary. + virtual ~ModRMFilter() { } + + /// isDumb - Indicates whether this filter returns the same value for + /// any value of the ModR/M byte. + /// + /// @result - True if the filter returns the same value for any ModR/M + /// byte; false if not. + virtual bool isDumb() const { return false; } + + /// accepts - Indicates whether the filter accepts a particular ModR/M + /// byte value. + /// + /// @result - True if the filter accepts the ModR/M byte; false if not. + virtual bool accepts(uint8_t modRM) const = 0; +}; + +/// DumbFilter - Accepts any ModR/M byte. Used for instructions that do not +/// require a ModR/M byte or instructions where the entire ModR/M byte is used +/// for operands. +class DumbFilter : public ModRMFilter { + void anchor() override; +public: + bool isDumb() const override { + return true; + } + + bool accepts(uint8_t modRM) const override { + return true; + } +}; + +/// ModFilter - Filters based on the mod bits [bits 7-6] of the ModR/M byte. +/// Some instructions are classified based on whether they are 11 or anything +/// else. This filter performs that classification. +class ModFilter : public ModRMFilter { + void anchor() override; + bool R; +public: + /// Constructor + /// + /// \param r True if the mod bits of the ModR/M byte must be 11; false + /// otherwise. The name r derives from the fact that the mod + /// bits indicate whether the R/M bits [bits 2-0] signify a + /// register or a memory operand. + ModFilter(bool r) : + ModRMFilter(), + R(r) { + } + + bool accepts(uint8_t modRM) const override { + return (R == ((modRM & 0xc0) == 0xc0)); + } +}; + +/// ExtendedFilter - Extended opcodes are classified based on the value of the +/// mod field [bits 7-6] and the value of the nnn field [bits 5-3]. +class ExtendedFilter : public ModRMFilter { + void anchor() override; + bool R; + uint8_t NNN; +public: + /// Constructor + /// + /// \param r True if the mod field must be set to 11; false otherwise. + /// The name is explained at ModFilter. + /// \param nnn The required value of the nnn field. + ExtendedFilter(bool r, uint8_t nnn) : + ModRMFilter(), + R(r), + NNN(nnn) { + } + + bool accepts(uint8_t modRM) const override { + return (((R && ((modRM & 0xc0) == 0xc0)) || + (!R && ((modRM & 0xc0) != 0xc0))) && + (((modRM & 0x38) >> 3) == NNN)); + } +}; + +/// ExtendedRMFilter - Extended opcodes are classified based on the value of the +/// mod field [bits 7-6] and the value of the nnn field [bits 2-0]. +class ExtendedRMFilter : public ModRMFilter { + void anchor() override; + bool R; + uint8_t NNN; +public: + /// Constructor + /// + /// \param r True if the mod field must be set to 11; false otherwise. + /// The name is explained at ModFilter. + /// \param nnn The required value of the nnn field. + ExtendedRMFilter(bool r, uint8_t nnn) : + ModRMFilter(), + R(r), + NNN(nnn) { + } + + bool accepts(uint8_t modRM) const override { + return ((R && ((modRM & 0xc0) == 0xc0)) && + ((modRM & 0x7) == NNN)); + } +}; +/// ExactFilter - The occasional extended opcode (such as VMCALL or MONITOR) +/// requires the ModR/M byte to have a specific value. +class ExactFilter : public ModRMFilter { + void anchor() override; + uint8_t ModRM; +public: + /// Constructor + /// + /// \param modRM The required value of the full ModR/M byte. + ExactFilter(uint8_t modRM) : + ModRMFilter(), + ModRM(modRM) { + } + + bool accepts(uint8_t modRM) const override { + return (ModRM == modRM); + } +}; + +} // namespace X86Disassembler + +} // namespace llvm + +#endif diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp b/contrib/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp new file mode 100644 index 000000000000..84f6d5210d74 --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.cpp @@ -0,0 +1,1264 @@ +//===- X86RecognizableInstr.cpp - Disassembler instruction spec --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is part of the X86 Disassembler Emitter. +// It contains the implementation of a single recognizable instruction. +// Documentation for the disassembler emitter in general can be found in +// X86DisassemblerEmitter.h. +// +//===----------------------------------------------------------------------===// + +#include "X86RecognizableInstr.h" +#include "X86DisassemblerShared.h" +#include "X86ModRMFilters.h" +#include "llvm/Support/ErrorHandling.h" +#include <string> + +using namespace llvm; +using namespace X86Disassembler; + +/// byteFromBitsInit - Extracts a value at most 8 bits in width from a BitsInit. +/// Useful for switch statements and the like. +/// +/// @param init - A reference to the BitsInit to be decoded. +/// @return - The field, with the first bit in the BitsInit as the lowest +/// order bit. +static uint8_t byteFromBitsInit(BitsInit &init) { + int width = init.getNumBits(); + + assert(width <= 8 && "Field is too large for uint8_t!"); + + int index; + uint8_t mask = 0x01; + + uint8_t ret = 0; + + for (index = 0; index < width; index++) { + if (cast<BitInit>(init.getBit(index))->getValue()) + ret |= mask; + + mask <<= 1; + } + + return ret; +} + +/// byteFromRec - Extract a value at most 8 bits in with from a Record given the +/// name of the field. +/// +/// @param rec - The record from which to extract the value. +/// @param name - The name of the field in the record. +/// @return - The field, as translated by byteFromBitsInit(). +static uint8_t byteFromRec(const Record* rec, const std::string &name) { + BitsInit* bits = rec->getValueAsBitsInit(name); + return byteFromBitsInit(*bits); +} + +RecognizableInstr::RecognizableInstr(DisassemblerTables &tables, + const CodeGenInstruction &insn, + InstrUID uid) { + UID = uid; + + Rec = insn.TheDef; + Name = std::string(Rec->getName()); + Spec = &tables.specForUID(UID); + + if (!Rec->isSubClassOf("X86Inst")) { + ShouldBeEmitted = false; + return; + } + + OpPrefix = byteFromRec(Rec, "OpPrefixBits"); + OpMap = byteFromRec(Rec, "OpMapBits"); + Opcode = byteFromRec(Rec, "Opcode"); + Form = byteFromRec(Rec, "FormBits"); + Encoding = byteFromRec(Rec, "OpEncBits"); + + OpSize = byteFromRec(Rec, "OpSizeBits"); + AdSize = byteFromRec(Rec, "AdSizeBits"); + HasREX_WPrefix = Rec->getValueAsBit("hasREX_WPrefix"); + HasVEX_4V = Rec->getValueAsBit("hasVEX_4V"); + HasVEX_W = Rec->getValueAsBit("HasVEX_W"); + IgnoresVEX_W = Rec->getValueAsBit("IgnoresVEX_W"); + IgnoresVEX_L = Rec->getValueAsBit("ignoresVEX_L"); + HasEVEX_L2Prefix = Rec->getValueAsBit("hasEVEX_L2"); + HasEVEX_K = Rec->getValueAsBit("hasEVEX_K"); + HasEVEX_KZ = Rec->getValueAsBit("hasEVEX_Z"); + HasEVEX_B = Rec->getValueAsBit("hasEVEX_B"); + IsCodeGenOnly = Rec->getValueAsBit("isCodeGenOnly"); + ForceDisassemble = Rec->getValueAsBit("ForceDisassemble"); + CD8_Scale = byteFromRec(Rec, "CD8_Scale"); + + Name = std::string(Rec->getName()); + + Operands = &insn.Operands.OperandList; + + HasVEX_LPrefix = Rec->getValueAsBit("hasVEX_L"); + + EncodeRC = HasEVEX_B && + (Form == X86Local::MRMDestReg || Form == X86Local::MRMSrcReg); + + // Check for 64-bit inst which does not require REX + Is32Bit = false; + Is64Bit = false; + // FIXME: Is there some better way to check for In64BitMode? + std::vector<Record*> Predicates = Rec->getValueAsListOfDefs("Predicates"); + for (unsigned i = 0, e = Predicates.size(); i != e; ++i) { + if (Predicates[i]->getName().find("Not64Bit") != Name.npos || + Predicates[i]->getName().find("In32Bit") != Name.npos) { + Is32Bit = true; + break; + } + if (Predicates[i]->getName().find("In64Bit") != Name.npos) { + Is64Bit = true; + break; + } + } + + if (Form == X86Local::Pseudo || (IsCodeGenOnly && !ForceDisassemble)) { + ShouldBeEmitted = false; + return; + } + + // Special case since there is no attribute class for 64-bit and VEX + if (Name == "VMASKMOVDQU64") { + ShouldBeEmitted = false; + return; + } + + ShouldBeEmitted = true; +} + +void RecognizableInstr::processInstr(DisassemblerTables &tables, + const CodeGenInstruction &insn, + InstrUID uid) +{ + // Ignore "asm parser only" instructions. + if (insn.TheDef->getValueAsBit("isAsmParserOnly")) + return; + + RecognizableInstr recogInstr(tables, insn, uid); + + if (recogInstr.shouldBeEmitted()) { + recogInstr.emitInstructionSpecifier(); + recogInstr.emitDecodePath(tables); + } +} + +#define EVEX_KB(n) (HasEVEX_KZ && HasEVEX_B ? n##_KZ_B : \ + (HasEVEX_K && HasEVEX_B ? n##_K_B : \ + (HasEVEX_KZ ? n##_KZ : \ + (HasEVEX_K? n##_K : (HasEVEX_B ? n##_B : n))))) + +InstructionContext RecognizableInstr::insnContext() const { + InstructionContext insnContext; + + if (Encoding == X86Local::EVEX) { + if (HasVEX_LPrefix && HasEVEX_L2Prefix) { + errs() << "Don't support VEX.L if EVEX_L2 is enabled: " << Name << "\n"; + llvm_unreachable("Don't support VEX.L if EVEX_L2 is enabled"); + } + // VEX_L & VEX_W + if (!EncodeRC && HasVEX_LPrefix && HasVEX_W) { + if (OpPrefix == X86Local::PD) + insnContext = EVEX_KB(IC_EVEX_L_W_OPSIZE); + else if (OpPrefix == X86Local::XS) + insnContext = EVEX_KB(IC_EVEX_L_W_XS); + else if (OpPrefix == X86Local::XD) + insnContext = EVEX_KB(IC_EVEX_L_W_XD); + else if (OpPrefix == X86Local::PS) + insnContext = EVEX_KB(IC_EVEX_L_W); + else { + errs() << "Instruction does not use a prefix: " << Name << "\n"; + llvm_unreachable("Invalid prefix"); + } + } else if (!EncodeRC && HasVEX_LPrefix) { + // VEX_L + if (OpPrefix == X86Local::PD) + insnContext = EVEX_KB(IC_EVEX_L_OPSIZE); + else if (OpPrefix == X86Local::XS) + insnContext = EVEX_KB(IC_EVEX_L_XS); + else if (OpPrefix == X86Local::XD) + insnContext = EVEX_KB(IC_EVEX_L_XD); + else if (OpPrefix == X86Local::PS) + insnContext = EVEX_KB(IC_EVEX_L); + else { + errs() << "Instruction does not use a prefix: " << Name << "\n"; + llvm_unreachable("Invalid prefix"); + } + } else if (!EncodeRC && HasEVEX_L2Prefix && HasVEX_W) { + // EVEX_L2 & VEX_W + if (OpPrefix == X86Local::PD) + insnContext = EVEX_KB(IC_EVEX_L2_W_OPSIZE); + else if (OpPrefix == X86Local::XS) + insnContext = EVEX_KB(IC_EVEX_L2_W_XS); + else if (OpPrefix == X86Local::XD) + insnContext = EVEX_KB(IC_EVEX_L2_W_XD); + else if (OpPrefix == X86Local::PS) + insnContext = EVEX_KB(IC_EVEX_L2_W); + else { + errs() << "Instruction does not use a prefix: " << Name << "\n"; + llvm_unreachable("Invalid prefix"); + } + } else if (!EncodeRC && HasEVEX_L2Prefix) { + // EVEX_L2 + if (OpPrefix == X86Local::PD) + insnContext = EVEX_KB(IC_EVEX_L2_OPSIZE); + else if (OpPrefix == X86Local::XD) + insnContext = EVEX_KB(IC_EVEX_L2_XD); + else if (OpPrefix == X86Local::XS) + insnContext = EVEX_KB(IC_EVEX_L2_XS); + else if (OpPrefix == X86Local::PS) + insnContext = EVEX_KB(IC_EVEX_L2); + else { + errs() << "Instruction does not use a prefix: " << Name << "\n"; + llvm_unreachable("Invalid prefix"); + } + } + else if (HasVEX_W) { + // VEX_W + if (OpPrefix == X86Local::PD) + insnContext = EVEX_KB(IC_EVEX_W_OPSIZE); + else if (OpPrefix == X86Local::XS) + insnContext = EVEX_KB(IC_EVEX_W_XS); + else if (OpPrefix == X86Local::XD) + insnContext = EVEX_KB(IC_EVEX_W_XD); + else if (OpPrefix == X86Local::PS) + insnContext = EVEX_KB(IC_EVEX_W); + else { + errs() << "Instruction does not use a prefix: " << Name << "\n"; + llvm_unreachable("Invalid prefix"); + } + } + // No L, no W + else if (OpPrefix == X86Local::PD) + insnContext = EVEX_KB(IC_EVEX_OPSIZE); + else if (OpPrefix == X86Local::XD) + insnContext = EVEX_KB(IC_EVEX_XD); + else if (OpPrefix == X86Local::XS) + insnContext = EVEX_KB(IC_EVEX_XS); + else if (OpPrefix == X86Local::PS) + insnContext = EVEX_KB(IC_EVEX); + else { + errs() << "Instruction does not use a prefix: " << Name << "\n"; + llvm_unreachable("Invalid prefix"); + } + /// eof EVEX + } else if (Encoding == X86Local::VEX || Encoding == X86Local::XOP) { + if (HasVEX_LPrefix && HasVEX_W) { + if (OpPrefix == X86Local::PD) + insnContext = IC_VEX_L_W_OPSIZE; + else if (OpPrefix == X86Local::XS) + insnContext = IC_VEX_L_W_XS; + else if (OpPrefix == X86Local::XD) + insnContext = IC_VEX_L_W_XD; + else if (OpPrefix == X86Local::PS) + insnContext = IC_VEX_L_W; + else { + errs() << "Instruction does not use a prefix: " << Name << "\n"; + llvm_unreachable("Invalid prefix"); + } + } else if (OpPrefix == X86Local::PD && HasVEX_LPrefix) + insnContext = IC_VEX_L_OPSIZE; + else if (OpPrefix == X86Local::PD && HasVEX_W) + insnContext = IC_VEX_W_OPSIZE; + else if (OpPrefix == X86Local::PD) + insnContext = IC_VEX_OPSIZE; + else if (HasVEX_LPrefix && OpPrefix == X86Local::XS) + insnContext = IC_VEX_L_XS; + else if (HasVEX_LPrefix && OpPrefix == X86Local::XD) + insnContext = IC_VEX_L_XD; + else if (HasVEX_W && OpPrefix == X86Local::XS) + insnContext = IC_VEX_W_XS; + else if (HasVEX_W && OpPrefix == X86Local::XD) + insnContext = IC_VEX_W_XD; + else if (HasVEX_W && OpPrefix == X86Local::PS) + insnContext = IC_VEX_W; + else if (HasVEX_LPrefix && OpPrefix == X86Local::PS) + insnContext = IC_VEX_L; + else if (OpPrefix == X86Local::XD) + insnContext = IC_VEX_XD; + else if (OpPrefix == X86Local::XS) + insnContext = IC_VEX_XS; + else if (OpPrefix == X86Local::PS) + insnContext = IC_VEX; + else { + errs() << "Instruction does not use a prefix: " << Name << "\n"; + llvm_unreachable("Invalid prefix"); + } + } else if (Is64Bit || HasREX_WPrefix || AdSize == X86Local::AdSize64) { + if (HasREX_WPrefix && (OpSize == X86Local::OpSize16 || OpPrefix == X86Local::PD)) + insnContext = IC_64BIT_REXW_OPSIZE; + else if (HasREX_WPrefix && AdSize == X86Local::AdSize32) + insnContext = IC_64BIT_REXW_ADSIZE; + else if (OpSize == X86Local::OpSize16 && OpPrefix == X86Local::XD) + insnContext = IC_64BIT_XD_OPSIZE; + else if (OpSize == X86Local::OpSize16 && OpPrefix == X86Local::XS) + insnContext = IC_64BIT_XS_OPSIZE; + else if (AdSize == X86Local::AdSize32 && OpPrefix == X86Local::PD) + insnContext = IC_64BIT_OPSIZE_ADSIZE; + else if (OpSize == X86Local::OpSize16 && AdSize == X86Local::AdSize32) + insnContext = IC_64BIT_OPSIZE_ADSIZE; + else if (OpSize == X86Local::OpSize16 || OpPrefix == X86Local::PD) + insnContext = IC_64BIT_OPSIZE; + else if (AdSize == X86Local::AdSize32) + insnContext = IC_64BIT_ADSIZE; + else if (HasREX_WPrefix && OpPrefix == X86Local::XS) + insnContext = IC_64BIT_REXW_XS; + else if (HasREX_WPrefix && OpPrefix == X86Local::XD) + insnContext = IC_64BIT_REXW_XD; + else if (OpPrefix == X86Local::XD) + insnContext = IC_64BIT_XD; + else if (OpPrefix == X86Local::XS) + insnContext = IC_64BIT_XS; + else if (HasREX_WPrefix) + insnContext = IC_64BIT_REXW; + else + insnContext = IC_64BIT; + } else { + if (OpSize == X86Local::OpSize16 && OpPrefix == X86Local::XD) + insnContext = IC_XD_OPSIZE; + else if (OpSize == X86Local::OpSize16 && OpPrefix == X86Local::XS) + insnContext = IC_XS_OPSIZE; + else if (AdSize == X86Local::AdSize16 && OpPrefix == X86Local::XD) + insnContext = IC_XD_ADSIZE; + else if (AdSize == X86Local::AdSize16 && OpPrefix == X86Local::XS) + insnContext = IC_XS_ADSIZE; + else if (AdSize == X86Local::AdSize16 && OpPrefix == X86Local::PD) + insnContext = IC_OPSIZE_ADSIZE; + else if (OpSize == X86Local::OpSize16 && AdSize == X86Local::AdSize16) + insnContext = IC_OPSIZE_ADSIZE; + else if (OpSize == X86Local::OpSize16 || OpPrefix == X86Local::PD) + insnContext = IC_OPSIZE; + else if (AdSize == X86Local::AdSize16) + insnContext = IC_ADSIZE; + else if (OpPrefix == X86Local::XD) + insnContext = IC_XD; + else if (OpPrefix == X86Local::XS) + insnContext = IC_XS; + else + insnContext = IC; + } + + return insnContext; +} + +void RecognizableInstr::adjustOperandEncoding(OperandEncoding &encoding) { + // The scaling factor for AVX512 compressed displacement encoding is an + // instruction attribute. Adjust the ModRM encoding type to include the + // scale for compressed displacement. + if ((encoding != ENCODING_RM && + encoding != ENCODING_VSIB && + encoding != ENCODING_SIB) ||CD8_Scale == 0) + return; + encoding = (OperandEncoding)(encoding + Log2_32(CD8_Scale)); + assert(((encoding >= ENCODING_RM && encoding <= ENCODING_RM_CD64) || + (encoding == ENCODING_SIB) || + (encoding >= ENCODING_VSIB && encoding <= ENCODING_VSIB_CD64)) && + "Invalid CDisp scaling"); +} + +void RecognizableInstr::handleOperand(bool optional, unsigned &operandIndex, + unsigned &physicalOperandIndex, + unsigned numPhysicalOperands, + const unsigned *operandMapping, + OperandEncoding (*encodingFromString) + (const std::string&, + uint8_t OpSize)) { + if (optional) { + if (physicalOperandIndex >= numPhysicalOperands) + return; + } else { + assert(physicalOperandIndex < numPhysicalOperands); + } + + while (operandMapping[operandIndex] != operandIndex) { + Spec->operands[operandIndex].encoding = ENCODING_DUP; + Spec->operands[operandIndex].type = + (OperandType)(TYPE_DUP0 + operandMapping[operandIndex]); + ++operandIndex; + } + + StringRef typeName = (*Operands)[operandIndex].Rec->getName(); + + OperandEncoding encoding = encodingFromString(std::string(typeName), OpSize); + // Adjust the encoding type for an operand based on the instruction. + adjustOperandEncoding(encoding); + Spec->operands[operandIndex].encoding = encoding; + Spec->operands[operandIndex].type = + typeFromString(std::string(typeName), HasREX_WPrefix, OpSize); + + ++operandIndex; + ++physicalOperandIndex; +} + +void RecognizableInstr::emitInstructionSpecifier() { + Spec->name = Name; + + Spec->insnContext = insnContext(); + + const std::vector<CGIOperandList::OperandInfo> &OperandList = *Operands; + + unsigned numOperands = OperandList.size(); + unsigned numPhysicalOperands = 0; + + // operandMapping maps from operands in OperandList to their originals. + // If operandMapping[i] != i, then the entry is a duplicate. + unsigned operandMapping[X86_MAX_OPERANDS]; + assert(numOperands <= X86_MAX_OPERANDS && "X86_MAX_OPERANDS is not large enough"); + + for (unsigned operandIndex = 0; operandIndex < numOperands; ++operandIndex) { + if (!OperandList[operandIndex].Constraints.empty()) { + const CGIOperandList::ConstraintInfo &Constraint = + OperandList[operandIndex].Constraints[0]; + if (Constraint.isTied()) { + operandMapping[operandIndex] = operandIndex; + operandMapping[Constraint.getTiedOperand()] = operandIndex; + } else { + ++numPhysicalOperands; + operandMapping[operandIndex] = operandIndex; + } + } else { + ++numPhysicalOperands; + operandMapping[operandIndex] = operandIndex; + } + } + +#define HANDLE_OPERAND(class) \ + handleOperand(false, \ + operandIndex, \ + physicalOperandIndex, \ + numPhysicalOperands, \ + operandMapping, \ + class##EncodingFromString); + +#define HANDLE_OPTIONAL(class) \ + handleOperand(true, \ + operandIndex, \ + physicalOperandIndex, \ + numPhysicalOperands, \ + operandMapping, \ + class##EncodingFromString); + + // operandIndex should always be < numOperands + unsigned operandIndex = 0; + // physicalOperandIndex should always be < numPhysicalOperands + unsigned physicalOperandIndex = 0; + +#ifndef NDEBUG + // Given the set of prefix bits, how many additional operands does the + // instruction have? + unsigned additionalOperands = 0; + if (HasVEX_4V) + ++additionalOperands; + if (HasEVEX_K) + ++additionalOperands; +#endif + + switch (Form) { + default: llvm_unreachable("Unhandled form"); + case X86Local::PrefixByte: + return; + case X86Local::RawFrmSrc: + HANDLE_OPERAND(relocation); + return; + case X86Local::RawFrmDst: + HANDLE_OPERAND(relocation); + return; + case X86Local::RawFrmDstSrc: + HANDLE_OPERAND(relocation); + HANDLE_OPERAND(relocation); + return; + case X86Local::RawFrm: + // Operand 1 (optional) is an address or immediate. + assert(numPhysicalOperands <= 1 && + "Unexpected number of operands for RawFrm"); + HANDLE_OPTIONAL(relocation) + break; + case X86Local::RawFrmMemOffs: + // Operand 1 is an address. + HANDLE_OPERAND(relocation); + break; + case X86Local::AddRegFrm: + // Operand 1 is added to the opcode. + // Operand 2 (optional) is an address. + assert(numPhysicalOperands >= 1 && numPhysicalOperands <= 2 && + "Unexpected number of operands for AddRegFrm"); + HANDLE_OPERAND(opcodeModifier) + HANDLE_OPTIONAL(relocation) + break; + case X86Local::AddCCFrm: + // Operand 1 (optional) is an address or immediate. + assert(numPhysicalOperands == 2 && + "Unexpected number of operands for AddCCFrm"); + HANDLE_OPERAND(relocation) + HANDLE_OPERAND(opcodeModifier) + break; + case X86Local::MRMDestReg: + // Operand 1 is a register operand in the R/M field. + // - In AVX512 there may be a mask operand here - + // Operand 2 is a register operand in the Reg/Opcode field. + // - In AVX, there is a register operand in the VEX.vvvv field here - + // Operand 3 (optional) is an immediate. + assert(numPhysicalOperands >= 2 + additionalOperands && + numPhysicalOperands <= 3 + additionalOperands && + "Unexpected number of operands for MRMDestRegFrm"); + + HANDLE_OPERAND(rmRegister) + if (HasEVEX_K) + HANDLE_OPERAND(writemaskRegister) + + if (HasVEX_4V) + // FIXME: In AVX, the register below becomes the one encoded + // in ModRMVEX and the one above the one in the VEX.VVVV field + HANDLE_OPERAND(vvvvRegister) + + HANDLE_OPERAND(roRegister) + HANDLE_OPTIONAL(immediate) + break; + case X86Local::MRMDestMem: + case X86Local::MRMDestMemFSIB: + // Operand 1 is a memory operand (possibly SIB-extended) + // Operand 2 is a register operand in the Reg/Opcode field. + // - In AVX, there is a register operand in the VEX.vvvv field here - + // Operand 3 (optional) is an immediate. + assert(numPhysicalOperands >= 2 + additionalOperands && + numPhysicalOperands <= 3 + additionalOperands && + "Unexpected number of operands for MRMDestMemFrm with VEX_4V"); + + HANDLE_OPERAND(memory) + + if (HasEVEX_K) + HANDLE_OPERAND(writemaskRegister) + + if (HasVEX_4V) + // FIXME: In AVX, the register below becomes the one encoded + // in ModRMVEX and the one above the one in the VEX.VVVV field + HANDLE_OPERAND(vvvvRegister) + + HANDLE_OPERAND(roRegister) + HANDLE_OPTIONAL(immediate) + break; + case X86Local::MRMSrcReg: + // Operand 1 is a register operand in the Reg/Opcode field. + // Operand 2 is a register operand in the R/M field. + // - In AVX, there is a register operand in the VEX.vvvv field here - + // Operand 3 (optional) is an immediate. + // Operand 4 (optional) is an immediate. + + assert(numPhysicalOperands >= 2 + additionalOperands && + numPhysicalOperands <= 4 + additionalOperands && + "Unexpected number of operands for MRMSrcRegFrm"); + + HANDLE_OPERAND(roRegister) + + if (HasEVEX_K) + HANDLE_OPERAND(writemaskRegister) + + if (HasVEX_4V) + // FIXME: In AVX, the register below becomes the one encoded + // in ModRMVEX and the one above the one in the VEX.VVVV field + HANDLE_OPERAND(vvvvRegister) + + HANDLE_OPERAND(rmRegister) + HANDLE_OPTIONAL(immediate) + HANDLE_OPTIONAL(immediate) // above might be a register in 7:4 + break; + case X86Local::MRMSrcReg4VOp3: + assert(numPhysicalOperands == 3 && + "Unexpected number of operands for MRMSrcReg4VOp3Frm"); + HANDLE_OPERAND(roRegister) + HANDLE_OPERAND(rmRegister) + HANDLE_OPERAND(vvvvRegister) + break; + case X86Local::MRMSrcRegOp4: + assert(numPhysicalOperands >= 4 && numPhysicalOperands <= 5 && + "Unexpected number of operands for MRMSrcRegOp4Frm"); + HANDLE_OPERAND(roRegister) + HANDLE_OPERAND(vvvvRegister) + HANDLE_OPERAND(immediate) // Register in imm[7:4] + HANDLE_OPERAND(rmRegister) + HANDLE_OPTIONAL(immediate) + break; + case X86Local::MRMSrcRegCC: + assert(numPhysicalOperands == 3 && + "Unexpected number of operands for MRMSrcRegCC"); + HANDLE_OPERAND(roRegister) + HANDLE_OPERAND(rmRegister) + HANDLE_OPERAND(opcodeModifier) + break; + case X86Local::MRMSrcMem: + case X86Local::MRMSrcMemFSIB: + // Operand 1 is a register operand in the Reg/Opcode field. + // Operand 2 is a memory operand (possibly SIB-extended) + // - In AVX, there is a register operand in the VEX.vvvv field here - + // Operand 3 (optional) is an immediate. + + assert(numPhysicalOperands >= 2 + additionalOperands && + numPhysicalOperands <= 4 + additionalOperands && + "Unexpected number of operands for MRMSrcMemFrm"); + + HANDLE_OPERAND(roRegister) + + if (HasEVEX_K) + HANDLE_OPERAND(writemaskRegister) + + if (HasVEX_4V) + // FIXME: In AVX, the register below becomes the one encoded + // in ModRMVEX and the one above the one in the VEX.VVVV field + HANDLE_OPERAND(vvvvRegister) + + HANDLE_OPERAND(memory) + HANDLE_OPTIONAL(immediate) + HANDLE_OPTIONAL(immediate) // above might be a register in 7:4 + break; + case X86Local::MRMSrcMem4VOp3: + assert(numPhysicalOperands == 3 && + "Unexpected number of operands for MRMSrcMem4VOp3Frm"); + HANDLE_OPERAND(roRegister) + HANDLE_OPERAND(memory) + HANDLE_OPERAND(vvvvRegister) + break; + case X86Local::MRMSrcMemOp4: + assert(numPhysicalOperands >= 4 && numPhysicalOperands <= 5 && + "Unexpected number of operands for MRMSrcMemOp4Frm"); + HANDLE_OPERAND(roRegister) + HANDLE_OPERAND(vvvvRegister) + HANDLE_OPERAND(immediate) // Register in imm[7:4] + HANDLE_OPERAND(memory) + HANDLE_OPTIONAL(immediate) + break; + case X86Local::MRMSrcMemCC: + assert(numPhysicalOperands == 3 && + "Unexpected number of operands for MRMSrcMemCC"); + HANDLE_OPERAND(roRegister) + HANDLE_OPERAND(memory) + HANDLE_OPERAND(opcodeModifier) + break; + case X86Local::MRMXrCC: + assert(numPhysicalOperands == 2 && + "Unexpected number of operands for MRMXrCC"); + HANDLE_OPERAND(rmRegister) + HANDLE_OPERAND(opcodeModifier) + break; + case X86Local::MRMr0: + // Operand 1 is a register operand in the R/M field. + HANDLE_OPERAND(roRegister) + break; + case X86Local::MRMXr: + case X86Local::MRM0r: + case X86Local::MRM1r: + case X86Local::MRM2r: + case X86Local::MRM3r: + case X86Local::MRM4r: + case X86Local::MRM5r: + case X86Local::MRM6r: + case X86Local::MRM7r: + // Operand 1 is a register operand in the R/M field. + // Operand 2 (optional) is an immediate or relocation. + // Operand 3 (optional) is an immediate. + assert(numPhysicalOperands >= 0 + additionalOperands && + numPhysicalOperands <= 3 + additionalOperands && + "Unexpected number of operands for MRMnr"); + + if (HasVEX_4V) + HANDLE_OPERAND(vvvvRegister) + + if (HasEVEX_K) + HANDLE_OPERAND(writemaskRegister) + HANDLE_OPTIONAL(rmRegister) + HANDLE_OPTIONAL(relocation) + HANDLE_OPTIONAL(immediate) + break; + case X86Local::MRMXmCC: + assert(numPhysicalOperands == 2 && + "Unexpected number of operands for MRMXm"); + HANDLE_OPERAND(memory) + HANDLE_OPERAND(opcodeModifier) + break; + case X86Local::MRMXm: + case X86Local::MRM0m: + case X86Local::MRM1m: + case X86Local::MRM2m: + case X86Local::MRM3m: + case X86Local::MRM4m: + case X86Local::MRM5m: + case X86Local::MRM6m: + case X86Local::MRM7m: + // Operand 1 is a memory operand (possibly SIB-extended) + // Operand 2 (optional) is an immediate or relocation. + assert(numPhysicalOperands >= 1 + additionalOperands && + numPhysicalOperands <= 2 + additionalOperands && + "Unexpected number of operands for MRMnm"); + + if (HasVEX_4V) + HANDLE_OPERAND(vvvvRegister) + if (HasEVEX_K) + HANDLE_OPERAND(writemaskRegister) + HANDLE_OPERAND(memory) + HANDLE_OPTIONAL(relocation) + break; + case X86Local::RawFrmImm8: + // operand 1 is a 16-bit immediate + // operand 2 is an 8-bit immediate + assert(numPhysicalOperands == 2 && + "Unexpected number of operands for X86Local::RawFrmImm8"); + HANDLE_OPERAND(immediate) + HANDLE_OPERAND(immediate) + break; + case X86Local::RawFrmImm16: + // operand 1 is a 16-bit immediate + // operand 2 is a 16-bit immediate + HANDLE_OPERAND(immediate) + HANDLE_OPERAND(immediate) + break; + case X86Local::MRM0X: + case X86Local::MRM1X: + case X86Local::MRM2X: + case X86Local::MRM3X: + case X86Local::MRM4X: + case X86Local::MRM5X: + case X86Local::MRM6X: + case X86Local::MRM7X: +#define MAP(from, to) case X86Local::MRM_##from: + X86_INSTR_MRM_MAPPING +#undef MAP + HANDLE_OPTIONAL(relocation) + break; + } + +#undef HANDLE_OPERAND +#undef HANDLE_OPTIONAL +} + +void RecognizableInstr::emitDecodePath(DisassemblerTables &tables) const { + // Special cases where the LLVM tables are not complete + +#define MAP(from, to) \ + case X86Local::MRM_##from: + + llvm::Optional<OpcodeType> opcodeType; + switch (OpMap) { + default: llvm_unreachable("Invalid map!"); + case X86Local::OB: opcodeType = ONEBYTE; break; + case X86Local::TB: opcodeType = TWOBYTE; break; + case X86Local::T8: opcodeType = THREEBYTE_38; break; + case X86Local::TA: opcodeType = THREEBYTE_3A; break; + case X86Local::XOP8: opcodeType = XOP8_MAP; break; + case X86Local::XOP9: opcodeType = XOP9_MAP; break; + case X86Local::XOPA: opcodeType = XOPA_MAP; break; + case X86Local::ThreeDNow: opcodeType = THREEDNOW_MAP; break; + } + + std::unique_ptr<ModRMFilter> filter; + switch (Form) { + default: llvm_unreachable("Invalid form!"); + case X86Local::Pseudo: llvm_unreachable("Pseudo should not be emitted!"); + case X86Local::RawFrm: + case X86Local::AddRegFrm: + case X86Local::RawFrmMemOffs: + case X86Local::RawFrmSrc: + case X86Local::RawFrmDst: + case X86Local::RawFrmDstSrc: + case X86Local::RawFrmImm8: + case X86Local::RawFrmImm16: + case X86Local::AddCCFrm: + case X86Local::PrefixByte: + filter = std::make_unique<DumbFilter>(); + break; + case X86Local::MRMDestReg: + case X86Local::MRMSrcReg: + case X86Local::MRMSrcReg4VOp3: + case X86Local::MRMSrcRegOp4: + case X86Local::MRMSrcRegCC: + case X86Local::MRMXrCC: + case X86Local::MRMXr: + filter = std::make_unique<ModFilter>(true); + break; + case X86Local::MRMDestMem: + case X86Local::MRMDestMemFSIB: + case X86Local::MRMSrcMem: + case X86Local::MRMSrcMemFSIB: + case X86Local::MRMSrcMem4VOp3: + case X86Local::MRMSrcMemOp4: + case X86Local::MRMSrcMemCC: + case X86Local::MRMXmCC: + case X86Local::MRMXm: + filter = std::make_unique<ModFilter>(false); + break; + case X86Local::MRM0r: case X86Local::MRM1r: + case X86Local::MRM2r: case X86Local::MRM3r: + case X86Local::MRM4r: case X86Local::MRM5r: + case X86Local::MRM6r: case X86Local::MRM7r: + filter = std::make_unique<ExtendedFilter>(true, Form - X86Local::MRM0r); + break; + case X86Local::MRM0X: case X86Local::MRM1X: + case X86Local::MRM2X: case X86Local::MRM3X: + case X86Local::MRM4X: case X86Local::MRM5X: + case X86Local::MRM6X: case X86Local::MRM7X: + filter = std::make_unique<ExtendedFilter>(true, Form - X86Local::MRM0X); + break; + case X86Local::MRMr0: + filter = std::make_unique<ExtendedRMFilter>(true, Form - X86Local::MRMr0); + break; + case X86Local::MRM0m: case X86Local::MRM1m: + case X86Local::MRM2m: case X86Local::MRM3m: + case X86Local::MRM4m: case X86Local::MRM5m: + case X86Local::MRM6m: case X86Local::MRM7m: + filter = std::make_unique<ExtendedFilter>(false, Form - X86Local::MRM0m); + break; + X86_INSTR_MRM_MAPPING + filter = std::make_unique<ExactFilter>(0xC0 + Form - X86Local::MRM_C0); + break; + } // switch (Form) + + uint8_t opcodeToSet = Opcode; + + unsigned AddressSize = 0; + switch (AdSize) { + case X86Local::AdSize16: AddressSize = 16; break; + case X86Local::AdSize32: AddressSize = 32; break; + case X86Local::AdSize64: AddressSize = 64; break; + } + + assert(opcodeType && "Opcode type not set"); + assert(filter && "Filter not set"); + + if (Form == X86Local::AddRegFrm || Form == X86Local::MRMSrcRegCC || + Form == X86Local::MRMSrcMemCC || Form == X86Local::MRMXrCC || + Form == X86Local::MRMXmCC || Form == X86Local::AddCCFrm) { + unsigned Count = Form == X86Local::AddRegFrm ? 8 : 16; + assert(((opcodeToSet % Count) == 0) && "ADDREG_FRM opcode not aligned"); + + uint8_t currentOpcode; + + for (currentOpcode = opcodeToSet; currentOpcode < opcodeToSet + Count; + ++currentOpcode) + tables.setTableFields(*opcodeType, insnContext(), currentOpcode, *filter, + UID, Is32Bit, OpPrefix == 0, + IgnoresVEX_L || EncodeRC, + IgnoresVEX_W, AddressSize); + } else { + tables.setTableFields(*opcodeType, insnContext(), opcodeToSet, *filter, UID, + Is32Bit, OpPrefix == 0, IgnoresVEX_L || EncodeRC, + IgnoresVEX_W, AddressSize); + } + +#undef MAP +} + +#define TYPE(str, type) if (s == str) return type; +OperandType RecognizableInstr::typeFromString(const std::string &s, + bool hasREX_WPrefix, + uint8_t OpSize) { + if(hasREX_WPrefix) { + // For instructions with a REX_W prefix, a declared 32-bit register encoding + // is special. + TYPE("GR32", TYPE_R32) + } + if(OpSize == X86Local::OpSize16) { + // For OpSize16 instructions, a declared 16-bit register or + // immediate encoding is special. + TYPE("GR16", TYPE_Rv) + } else if(OpSize == X86Local::OpSize32) { + // For OpSize32 instructions, a declared 32-bit register or + // immediate encoding is special. + TYPE("GR32", TYPE_Rv) + } + TYPE("i16mem", TYPE_M) + TYPE("i16imm", TYPE_IMM) + TYPE("i16i8imm", TYPE_IMM) + TYPE("GR16", TYPE_R16) + TYPE("i32mem", TYPE_M) + TYPE("i32imm", TYPE_IMM) + TYPE("i32i8imm", TYPE_IMM) + TYPE("GR32", TYPE_R32) + TYPE("GR32orGR64", TYPE_R32) + TYPE("i64mem", TYPE_M) + TYPE("i64i32imm", TYPE_IMM) + TYPE("i64i8imm", TYPE_IMM) + TYPE("GR64", TYPE_R64) + TYPE("i8mem", TYPE_M) + TYPE("i8imm", TYPE_IMM) + TYPE("u4imm", TYPE_UIMM8) + TYPE("u8imm", TYPE_UIMM8) + TYPE("i16u8imm", TYPE_UIMM8) + TYPE("i32u8imm", TYPE_UIMM8) + TYPE("i64u8imm", TYPE_UIMM8) + TYPE("GR8", TYPE_R8) + TYPE("VR128", TYPE_XMM) + TYPE("VR128X", TYPE_XMM) + TYPE("f128mem", TYPE_M) + TYPE("f256mem", TYPE_M) + TYPE("f512mem", TYPE_M) + TYPE("FR128", TYPE_XMM) + TYPE("FR64", TYPE_XMM) + TYPE("FR64X", TYPE_XMM) + TYPE("f64mem", TYPE_M) + TYPE("sdmem", TYPE_M) + TYPE("FR32", TYPE_XMM) + TYPE("FR32X", TYPE_XMM) + TYPE("f32mem", TYPE_M) + TYPE("ssmem", TYPE_M) + TYPE("RST", TYPE_ST) + TYPE("RSTi", TYPE_ST) + TYPE("i128mem", TYPE_M) + TYPE("i256mem", TYPE_M) + TYPE("i512mem", TYPE_M) + TYPE("i64i32imm_brtarget", TYPE_REL) + TYPE("i16imm_brtarget", TYPE_REL) + TYPE("i32imm_brtarget", TYPE_REL) + TYPE("ccode", TYPE_IMM) + TYPE("AVX512RC", TYPE_IMM) + TYPE("brtarget32", TYPE_REL) + TYPE("brtarget16", TYPE_REL) + TYPE("brtarget8", TYPE_REL) + TYPE("f80mem", TYPE_M) + TYPE("lea64_32mem", TYPE_M) + TYPE("lea64mem", TYPE_M) + TYPE("VR64", TYPE_MM64) + TYPE("i64imm", TYPE_IMM) + TYPE("anymem", TYPE_M) + TYPE("opaquemem", TYPE_M) + TYPE("sibmem", TYPE_MSIB) + TYPE("SEGMENT_REG", TYPE_SEGMENTREG) + TYPE("DEBUG_REG", TYPE_DEBUGREG) + TYPE("CONTROL_REG", TYPE_CONTROLREG) + TYPE("srcidx8", TYPE_SRCIDX) + TYPE("srcidx16", TYPE_SRCIDX) + TYPE("srcidx32", TYPE_SRCIDX) + TYPE("srcidx64", TYPE_SRCIDX) + TYPE("dstidx8", TYPE_DSTIDX) + TYPE("dstidx16", TYPE_DSTIDX) + TYPE("dstidx32", TYPE_DSTIDX) + TYPE("dstidx64", TYPE_DSTIDX) + TYPE("offset16_8", TYPE_MOFFS) + TYPE("offset16_16", TYPE_MOFFS) + TYPE("offset16_32", TYPE_MOFFS) + TYPE("offset32_8", TYPE_MOFFS) + TYPE("offset32_16", TYPE_MOFFS) + TYPE("offset32_32", TYPE_MOFFS) + TYPE("offset32_64", TYPE_MOFFS) + TYPE("offset64_8", TYPE_MOFFS) + TYPE("offset64_16", TYPE_MOFFS) + TYPE("offset64_32", TYPE_MOFFS) + TYPE("offset64_64", TYPE_MOFFS) + TYPE("VR256", TYPE_YMM) + TYPE("VR256X", TYPE_YMM) + TYPE("VR512", TYPE_ZMM) + TYPE("VK1", TYPE_VK) + TYPE("VK1WM", TYPE_VK) + TYPE("VK2", TYPE_VK) + TYPE("VK2WM", TYPE_VK) + TYPE("VK4", TYPE_VK) + TYPE("VK4WM", TYPE_VK) + TYPE("VK8", TYPE_VK) + TYPE("VK8WM", TYPE_VK) + TYPE("VK16", TYPE_VK) + TYPE("VK16WM", TYPE_VK) + TYPE("VK32", TYPE_VK) + TYPE("VK32WM", TYPE_VK) + TYPE("VK64", TYPE_VK) + TYPE("VK64WM", TYPE_VK) + TYPE("VK1Pair", TYPE_VK_PAIR) + TYPE("VK2Pair", TYPE_VK_PAIR) + TYPE("VK4Pair", TYPE_VK_PAIR) + TYPE("VK8Pair", TYPE_VK_PAIR) + TYPE("VK16Pair", TYPE_VK_PAIR) + TYPE("vx64mem", TYPE_MVSIBX) + TYPE("vx128mem", TYPE_MVSIBX) + TYPE("vx256mem", TYPE_MVSIBX) + TYPE("vy128mem", TYPE_MVSIBY) + TYPE("vy256mem", TYPE_MVSIBY) + TYPE("vx64xmem", TYPE_MVSIBX) + TYPE("vx128xmem", TYPE_MVSIBX) + TYPE("vx256xmem", TYPE_MVSIBX) + TYPE("vy128xmem", TYPE_MVSIBY) + TYPE("vy256xmem", TYPE_MVSIBY) + TYPE("vy512xmem", TYPE_MVSIBY) + TYPE("vz256mem", TYPE_MVSIBZ) + TYPE("vz512mem", TYPE_MVSIBZ) + TYPE("BNDR", TYPE_BNDR) + TYPE("TILE", TYPE_TMM) + errs() << "Unhandled type string " << s << "\n"; + llvm_unreachable("Unhandled type string"); +} +#undef TYPE + +#define ENCODING(str, encoding) if (s == str) return encoding; +OperandEncoding +RecognizableInstr::immediateEncodingFromString(const std::string &s, + uint8_t OpSize) { + if(OpSize != X86Local::OpSize16) { + // For instructions without an OpSize prefix, a declared 16-bit register or + // immediate encoding is special. + ENCODING("i16imm", ENCODING_IW) + } + ENCODING("i32i8imm", ENCODING_IB) + ENCODING("AVX512RC", ENCODING_IRC) + ENCODING("i16imm", ENCODING_Iv) + ENCODING("i16i8imm", ENCODING_IB) + ENCODING("i32imm", ENCODING_Iv) + ENCODING("i64i32imm", ENCODING_ID) + ENCODING("i64i8imm", ENCODING_IB) + ENCODING("i8imm", ENCODING_IB) + ENCODING("u4imm", ENCODING_IB) + ENCODING("u8imm", ENCODING_IB) + ENCODING("i16u8imm", ENCODING_IB) + ENCODING("i32u8imm", ENCODING_IB) + ENCODING("i64u8imm", ENCODING_IB) + // This is not a typo. Instructions like BLENDVPD put + // register IDs in 8-bit immediates nowadays. + ENCODING("FR32", ENCODING_IB) + ENCODING("FR64", ENCODING_IB) + ENCODING("FR128", ENCODING_IB) + ENCODING("VR128", ENCODING_IB) + ENCODING("VR256", ENCODING_IB) + ENCODING("FR32X", ENCODING_IB) + ENCODING("FR64X", ENCODING_IB) + ENCODING("VR128X", ENCODING_IB) + ENCODING("VR256X", ENCODING_IB) + ENCODING("VR512", ENCODING_IB) + ENCODING("TILE", ENCODING_IB) + errs() << "Unhandled immediate encoding " << s << "\n"; + llvm_unreachable("Unhandled immediate encoding"); +} + +OperandEncoding +RecognizableInstr::rmRegisterEncodingFromString(const std::string &s, + uint8_t OpSize) { + ENCODING("RST", ENCODING_FP) + ENCODING("RSTi", ENCODING_FP) + ENCODING("GR16", ENCODING_RM) + ENCODING("GR32", ENCODING_RM) + ENCODING("GR32orGR64", ENCODING_RM) + ENCODING("GR64", ENCODING_RM) + ENCODING("GR8", ENCODING_RM) + ENCODING("VR128", ENCODING_RM) + ENCODING("VR128X", ENCODING_RM) + ENCODING("FR128", ENCODING_RM) + ENCODING("FR64", ENCODING_RM) + ENCODING("FR32", ENCODING_RM) + ENCODING("FR64X", ENCODING_RM) + ENCODING("FR32X", ENCODING_RM) + ENCODING("VR64", ENCODING_RM) + ENCODING("VR256", ENCODING_RM) + ENCODING("VR256X", ENCODING_RM) + ENCODING("VR512", ENCODING_RM) + ENCODING("VK1", ENCODING_RM) + ENCODING("VK2", ENCODING_RM) + ENCODING("VK4", ENCODING_RM) + ENCODING("VK8", ENCODING_RM) + ENCODING("VK16", ENCODING_RM) + ENCODING("VK32", ENCODING_RM) + ENCODING("VK64", ENCODING_RM) + ENCODING("VK1PAIR", ENCODING_RM) + ENCODING("VK2PAIR", ENCODING_RM) + ENCODING("VK4PAIR", ENCODING_RM) + ENCODING("VK8PAIR", ENCODING_RM) + ENCODING("VK16PAIR", ENCODING_RM) + ENCODING("BNDR", ENCODING_RM) + ENCODING("TILE", ENCODING_RM) + errs() << "Unhandled R/M register encoding " << s << "\n"; + llvm_unreachable("Unhandled R/M register encoding"); +} + +OperandEncoding +RecognizableInstr::roRegisterEncodingFromString(const std::string &s, + uint8_t OpSize) { + ENCODING("GR16", ENCODING_REG) + ENCODING("GR32", ENCODING_REG) + ENCODING("GR32orGR64", ENCODING_REG) + ENCODING("GR64", ENCODING_REG) + ENCODING("GR8", ENCODING_REG) + ENCODING("VR128", ENCODING_REG) + ENCODING("FR128", ENCODING_REG) + ENCODING("FR64", ENCODING_REG) + ENCODING("FR32", ENCODING_REG) + ENCODING("VR64", ENCODING_REG) + ENCODING("SEGMENT_REG", ENCODING_REG) + ENCODING("DEBUG_REG", ENCODING_REG) + ENCODING("CONTROL_REG", ENCODING_REG) + ENCODING("VR256", ENCODING_REG) + ENCODING("VR256X", ENCODING_REG) + ENCODING("VR128X", ENCODING_REG) + ENCODING("FR64X", ENCODING_REG) + ENCODING("FR32X", ENCODING_REG) + ENCODING("VR512", ENCODING_REG) + ENCODING("VK1", ENCODING_REG) + ENCODING("VK2", ENCODING_REG) + ENCODING("VK4", ENCODING_REG) + ENCODING("VK8", ENCODING_REG) + ENCODING("VK16", ENCODING_REG) + ENCODING("VK32", ENCODING_REG) + ENCODING("VK64", ENCODING_REG) + ENCODING("VK1Pair", ENCODING_REG) + ENCODING("VK2Pair", ENCODING_REG) + ENCODING("VK4Pair", ENCODING_REG) + ENCODING("VK8Pair", ENCODING_REG) + ENCODING("VK16Pair", ENCODING_REG) + ENCODING("VK1WM", ENCODING_REG) + ENCODING("VK2WM", ENCODING_REG) + ENCODING("VK4WM", ENCODING_REG) + ENCODING("VK8WM", ENCODING_REG) + ENCODING("VK16WM", ENCODING_REG) + ENCODING("VK32WM", ENCODING_REG) + ENCODING("VK64WM", ENCODING_REG) + ENCODING("BNDR", ENCODING_REG) + ENCODING("TILE", ENCODING_REG) + errs() << "Unhandled reg/opcode register encoding " << s << "\n"; + llvm_unreachable("Unhandled reg/opcode register encoding"); +} + +OperandEncoding +RecognizableInstr::vvvvRegisterEncodingFromString(const std::string &s, + uint8_t OpSize) { + ENCODING("GR32", ENCODING_VVVV) + ENCODING("GR64", ENCODING_VVVV) + ENCODING("FR32", ENCODING_VVVV) + ENCODING("FR128", ENCODING_VVVV) + ENCODING("FR64", ENCODING_VVVV) + ENCODING("VR128", ENCODING_VVVV) + ENCODING("VR256", ENCODING_VVVV) + ENCODING("FR32X", ENCODING_VVVV) + ENCODING("FR64X", ENCODING_VVVV) + ENCODING("VR128X", ENCODING_VVVV) + ENCODING("VR256X", ENCODING_VVVV) + ENCODING("VR512", ENCODING_VVVV) + ENCODING("VK1", ENCODING_VVVV) + ENCODING("VK2", ENCODING_VVVV) + ENCODING("VK4", ENCODING_VVVV) + ENCODING("VK8", ENCODING_VVVV) + ENCODING("VK16", ENCODING_VVVV) + ENCODING("VK32", ENCODING_VVVV) + ENCODING("VK64", ENCODING_VVVV) + ENCODING("VK1PAIR", ENCODING_VVVV) + ENCODING("VK2PAIR", ENCODING_VVVV) + ENCODING("VK4PAIR", ENCODING_VVVV) + ENCODING("VK8PAIR", ENCODING_VVVV) + ENCODING("VK16PAIR", ENCODING_VVVV) + ENCODING("TILE", ENCODING_VVVV) + errs() << "Unhandled VEX.vvvv register encoding " << s << "\n"; + llvm_unreachable("Unhandled VEX.vvvv register encoding"); +} + +OperandEncoding +RecognizableInstr::writemaskRegisterEncodingFromString(const std::string &s, + uint8_t OpSize) { + ENCODING("VK1WM", ENCODING_WRITEMASK) + ENCODING("VK2WM", ENCODING_WRITEMASK) + ENCODING("VK4WM", ENCODING_WRITEMASK) + ENCODING("VK8WM", ENCODING_WRITEMASK) + ENCODING("VK16WM", ENCODING_WRITEMASK) + ENCODING("VK32WM", ENCODING_WRITEMASK) + ENCODING("VK64WM", ENCODING_WRITEMASK) + errs() << "Unhandled mask register encoding " << s << "\n"; + llvm_unreachable("Unhandled mask register encoding"); +} + +OperandEncoding +RecognizableInstr::memoryEncodingFromString(const std::string &s, + uint8_t OpSize) { + ENCODING("i16mem", ENCODING_RM) + ENCODING("i32mem", ENCODING_RM) + ENCODING("i64mem", ENCODING_RM) + ENCODING("i8mem", ENCODING_RM) + ENCODING("ssmem", ENCODING_RM) + ENCODING("sdmem", ENCODING_RM) + ENCODING("f128mem", ENCODING_RM) + ENCODING("f256mem", ENCODING_RM) + ENCODING("f512mem", ENCODING_RM) + ENCODING("f64mem", ENCODING_RM) + ENCODING("f32mem", ENCODING_RM) + ENCODING("i128mem", ENCODING_RM) + ENCODING("i256mem", ENCODING_RM) + ENCODING("i512mem", ENCODING_RM) + ENCODING("f80mem", ENCODING_RM) + ENCODING("lea64_32mem", ENCODING_RM) + ENCODING("lea64mem", ENCODING_RM) + ENCODING("anymem", ENCODING_RM) + ENCODING("opaquemem", ENCODING_RM) + ENCODING("sibmem", ENCODING_SIB) + ENCODING("vx64mem", ENCODING_VSIB) + ENCODING("vx128mem", ENCODING_VSIB) + ENCODING("vx256mem", ENCODING_VSIB) + ENCODING("vy128mem", ENCODING_VSIB) + ENCODING("vy256mem", ENCODING_VSIB) + ENCODING("vx64xmem", ENCODING_VSIB) + ENCODING("vx128xmem", ENCODING_VSIB) + ENCODING("vx256xmem", ENCODING_VSIB) + ENCODING("vy128xmem", ENCODING_VSIB) + ENCODING("vy256xmem", ENCODING_VSIB) + ENCODING("vy512xmem", ENCODING_VSIB) + ENCODING("vz256mem", ENCODING_VSIB) + ENCODING("vz512mem", ENCODING_VSIB) + errs() << "Unhandled memory encoding " << s << "\n"; + llvm_unreachable("Unhandled memory encoding"); +} + +OperandEncoding +RecognizableInstr::relocationEncodingFromString(const std::string &s, + uint8_t OpSize) { + if(OpSize != X86Local::OpSize16) { + // For instructions without an OpSize prefix, a declared 16-bit register or + // immediate encoding is special. + ENCODING("i16imm", ENCODING_IW) + } + ENCODING("i16imm", ENCODING_Iv) + ENCODING("i16i8imm", ENCODING_IB) + ENCODING("i32imm", ENCODING_Iv) + ENCODING("i32i8imm", ENCODING_IB) + ENCODING("i64i32imm", ENCODING_ID) + ENCODING("i64i8imm", ENCODING_IB) + ENCODING("i8imm", ENCODING_IB) + ENCODING("u8imm", ENCODING_IB) + ENCODING("i16u8imm", ENCODING_IB) + ENCODING("i32u8imm", ENCODING_IB) + ENCODING("i64u8imm", ENCODING_IB) + ENCODING("i64i32imm_brtarget", ENCODING_ID) + ENCODING("i16imm_brtarget", ENCODING_IW) + ENCODING("i32imm_brtarget", ENCODING_ID) + ENCODING("brtarget32", ENCODING_ID) + ENCODING("brtarget16", ENCODING_IW) + ENCODING("brtarget8", ENCODING_IB) + ENCODING("i64imm", ENCODING_IO) + ENCODING("offset16_8", ENCODING_Ia) + ENCODING("offset16_16", ENCODING_Ia) + ENCODING("offset16_32", ENCODING_Ia) + ENCODING("offset32_8", ENCODING_Ia) + ENCODING("offset32_16", ENCODING_Ia) + ENCODING("offset32_32", ENCODING_Ia) + ENCODING("offset32_64", ENCODING_Ia) + ENCODING("offset64_8", ENCODING_Ia) + ENCODING("offset64_16", ENCODING_Ia) + ENCODING("offset64_32", ENCODING_Ia) + ENCODING("offset64_64", ENCODING_Ia) + ENCODING("srcidx8", ENCODING_SI) + ENCODING("srcidx16", ENCODING_SI) + ENCODING("srcidx32", ENCODING_SI) + ENCODING("srcidx64", ENCODING_SI) + ENCODING("dstidx8", ENCODING_DI) + ENCODING("dstidx16", ENCODING_DI) + ENCODING("dstidx32", ENCODING_DI) + ENCODING("dstidx64", ENCODING_DI) + errs() << "Unhandled relocation encoding " << s << "\n"; + llvm_unreachable("Unhandled relocation encoding"); +} + +OperandEncoding +RecognizableInstr::opcodeModifierEncodingFromString(const std::string &s, + uint8_t OpSize) { + ENCODING("GR32", ENCODING_Rv) + ENCODING("GR64", ENCODING_RO) + ENCODING("GR16", ENCODING_Rv) + ENCODING("GR8", ENCODING_RB) + ENCODING("ccode", ENCODING_CC) + errs() << "Unhandled opcode modifier encoding " << s << "\n"; + llvm_unreachable("Unhandled opcode modifier encoding"); +} +#undef ENCODING diff --git a/contrib/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.h b/contrib/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.h new file mode 100644 index 000000000000..a7b88b4d12ed --- /dev/null +++ b/contrib/llvm-project/llvm/utils/TableGen/X86RecognizableInstr.h @@ -0,0 +1,363 @@ +//===- X86RecognizableInstr.h - Disassembler instruction spec ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is part of the X86 Disassembler Emitter. +// It contains the interface of a single recognizable instruction. +// Documentation for the disassembler emitter in general can be found in +// X86DisassemblerEmitter.h. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H +#define LLVM_UTILS_TABLEGEN_X86RECOGNIZABLEINSTR_H + +#include "CodeGenTarget.h" +#include "X86DisassemblerTables.h" +#include "llvm/Support/DataTypes.h" +#include "llvm/TableGen/Record.h" + +namespace llvm { + +#define X86_INSTR_MRM_MAPPING \ + MAP(C0, 64) \ + MAP(C1, 65) \ + MAP(C2, 66) \ + MAP(C3, 67) \ + MAP(C4, 68) \ + MAP(C5, 69) \ + MAP(C6, 70) \ + MAP(C7, 71) \ + MAP(C8, 72) \ + MAP(C9, 73) \ + MAP(CA, 74) \ + MAP(CB, 75) \ + MAP(CC, 76) \ + MAP(CD, 77) \ + MAP(CE, 78) \ + MAP(CF, 79) \ + MAP(D0, 80) \ + MAP(D1, 81) \ + MAP(D2, 82) \ + MAP(D3, 83) \ + MAP(D4, 84) \ + MAP(D5, 85) \ + MAP(D6, 86) \ + MAP(D7, 87) \ + MAP(D8, 88) \ + MAP(D9, 89) \ + MAP(DA, 90) \ + MAP(DB, 91) \ + MAP(DC, 92) \ + MAP(DD, 93) \ + MAP(DE, 94) \ + MAP(DF, 95) \ + MAP(E0, 96) \ + MAP(E1, 97) \ + MAP(E2, 98) \ + MAP(E3, 99) \ + MAP(E4, 100) \ + MAP(E5, 101) \ + MAP(E6, 102) \ + MAP(E7, 103) \ + MAP(E8, 104) \ + MAP(E9, 105) \ + MAP(EA, 106) \ + MAP(EB, 107) \ + MAP(EC, 108) \ + MAP(ED, 109) \ + MAP(EE, 110) \ + MAP(EF, 111) \ + MAP(F0, 112) \ + MAP(F1, 113) \ + MAP(F2, 114) \ + MAP(F3, 115) \ + MAP(F4, 116) \ + MAP(F5, 117) \ + MAP(F6, 118) \ + MAP(F7, 119) \ + MAP(F8, 120) \ + MAP(F9, 121) \ + MAP(FA, 122) \ + MAP(FB, 123) \ + MAP(FC, 124) \ + MAP(FD, 125) \ + MAP(FE, 126) \ + MAP(FF, 127) + +// A clone of X86 since we can't depend on something that is generated. +namespace X86Local { + enum { + Pseudo = 0, + RawFrm = 1, + AddRegFrm = 2, + RawFrmMemOffs = 3, + RawFrmSrc = 4, + RawFrmDst = 5, + RawFrmDstSrc = 6, + RawFrmImm8 = 7, + RawFrmImm16 = 8, + AddCCFrm = 9, + PrefixByte = 10, + MRMr0 = 21, + MRMSrcMemFSIB = 22, + MRMDestMemFSIB = 23, + MRMDestMem = 24, + MRMSrcMem = 25, + MRMSrcMem4VOp3 = 26, + MRMSrcMemOp4 = 27, + MRMSrcMemCC = 28, + MRMXmCC = 30, MRMXm = 31, + MRM0m = 32, MRM1m = 33, MRM2m = 34, MRM3m = 35, + MRM4m = 36, MRM5m = 37, MRM6m = 38, MRM7m = 39, + MRMDestReg = 40, + MRMSrcReg = 41, + MRMSrcReg4VOp3 = 42, + MRMSrcRegOp4 = 43, + MRMSrcRegCC = 44, + MRMXrCC = 46, MRMXr = 47, + MRM0r = 48, MRM1r = 49, MRM2r = 50, MRM3r = 51, + MRM4r = 52, MRM5r = 53, MRM6r = 54, MRM7r = 55, + MRM0X = 56, MRM1X = 57, MRM2X = 58, MRM3X = 59, + MRM4X = 60, MRM5X = 61, MRM6X = 62, MRM7X = 63, +#define MAP(from, to) MRM_##from = to, + X86_INSTR_MRM_MAPPING +#undef MAP + }; + + enum { + OB = 0, TB = 1, T8 = 2, TA = 3, XOP8 = 4, XOP9 = 5, XOPA = 6, ThreeDNow = 7 + }; + + enum { + PD = 1, XS = 2, XD = 3, PS = 4 + }; + + enum { + VEX = 1, XOP = 2, EVEX = 3 + }; + + enum { + OpSize16 = 1, OpSize32 = 2 + }; + + enum { + AdSize16 = 1, AdSize32 = 2, AdSize64 = 3 + }; +} + +namespace X86Disassembler { + +/// RecognizableInstr - Encapsulates all information required to decode a single +/// instruction, as extracted from the LLVM instruction tables. Has methods +/// to interpret the information available in the LLVM tables, and to emit the +/// instruction into DisassemblerTables. +class RecognizableInstr { +private: + /// The opcode of the instruction, as used in an MCInst + InstrUID UID; + /// The record from the .td files corresponding to this instruction + const Record* Rec; + /// The OpPrefix field from the record + uint8_t OpPrefix; + /// The OpMap field from the record + uint8_t OpMap; + /// The opcode field from the record; this is the opcode used in the Intel + /// encoding and therefore distinct from the UID + uint8_t Opcode; + /// The form field from the record + uint8_t Form; + // The encoding field from the record + uint8_t Encoding; + /// The OpSize field from the record + uint8_t OpSize; + /// The AdSize field from the record + uint8_t AdSize; + /// The hasREX_WPrefix field from the record + bool HasREX_WPrefix; + /// The hasVEX_4V field from the record + bool HasVEX_4V; + /// The HasVEX_WPrefix field from the record + bool HasVEX_W; + /// The IgnoresVEX_W field from the record + bool IgnoresVEX_W; + /// Inferred from the operands; indicates whether the L bit in the VEX prefix is set + bool HasVEX_LPrefix; + /// The ignoreVEX_L field from the record + bool IgnoresVEX_L; + /// The hasEVEX_L2Prefix field from the record + bool HasEVEX_L2Prefix; + /// The hasEVEX_K field from the record + bool HasEVEX_K; + /// The hasEVEX_KZ field from the record + bool HasEVEX_KZ; + /// The hasEVEX_B field from the record + bool HasEVEX_B; + /// Indicates that the instruction uses the L and L' fields for RC. + bool EncodeRC; + /// The isCodeGenOnly field from the record + bool IsCodeGenOnly; + /// The ForceDisassemble field from the record + bool ForceDisassemble; + // The CD8_Scale field from the record + uint8_t CD8_Scale; + // Whether the instruction has the predicate "In64BitMode" + bool Is64Bit; + // Whether the instruction has the predicate "In32BitMode" + bool Is32Bit; + + /// The instruction name as listed in the tables + std::string Name; + + /// Indicates whether the instruction should be emitted into the decode + /// tables; regardless, it will be emitted into the instruction info table + bool ShouldBeEmitted; + + /// The operands of the instruction, as listed in the CodeGenInstruction. + /// They are not one-to-one with operands listed in the MCInst; for example, + /// memory operands expand to 5 operands in the MCInst + const std::vector<CGIOperandList::OperandInfo>* Operands; + + /// The description of the instruction that is emitted into the instruction + /// info table + InstructionSpecifier* Spec; + + /// insnContext - Returns the primary context in which the instruction is + /// valid. + /// + /// @return - The context in which the instruction is valid. + InstructionContext insnContext() const; + + /// typeFromString - Translates an operand type from the string provided in + /// the LLVM tables to an OperandType for use in the operand specifier. + /// + /// @param s - The string, as extracted by calling Rec->getName() + /// on a CodeGenInstruction::OperandInfo. + /// @param hasREX_WPrefix - Indicates whether the instruction has a REX.W + /// prefix. If it does, 32-bit register operands stay + /// 32-bit regardless of the operand size. + /// @param OpSize Indicates the operand size of the instruction. + /// If register size does not match OpSize, then + /// register sizes keep their size. + /// @return - The operand's type. + static OperandType typeFromString(const std::string& s, + bool hasREX_WPrefix, uint8_t OpSize); + + /// immediateEncodingFromString - Translates an immediate encoding from the + /// string provided in the LLVM tables to an OperandEncoding for use in + /// the operand specifier. + /// + /// @param s - See typeFromString(). + /// @param OpSize - Indicates whether this is an OpSize16 instruction. + /// If it is not, then 16-bit immediate operands stay 16-bit. + /// @return - The operand's encoding. + static OperandEncoding immediateEncodingFromString(const std::string &s, + uint8_t OpSize); + + /// rmRegisterEncodingFromString - Like immediateEncodingFromString, but + /// handles operands that are in the REG field of the ModR/M byte. + static OperandEncoding rmRegisterEncodingFromString(const std::string &s, + uint8_t OpSize); + + /// rmRegisterEncodingFromString - Like immediateEncodingFromString, but + /// handles operands that are in the REG field of the ModR/M byte. + static OperandEncoding roRegisterEncodingFromString(const std::string &s, + uint8_t OpSize); + static OperandEncoding memoryEncodingFromString(const std::string &s, + uint8_t OpSize); + static OperandEncoding relocationEncodingFromString(const std::string &s, + uint8_t OpSize); + static OperandEncoding opcodeModifierEncodingFromString(const std::string &s, + uint8_t OpSize); + static OperandEncoding vvvvRegisterEncodingFromString(const std::string &s, + uint8_t OpSize); + static OperandEncoding writemaskRegisterEncodingFromString(const std::string &s, + uint8_t OpSize); + + /// Adjust the encoding type for an operand based on the instruction. + void adjustOperandEncoding(OperandEncoding &encoding); + + /// handleOperand - Converts a single operand from the LLVM table format to + /// the emitted table format, handling any duplicate operands it encounters + /// and then one non-duplicate. + /// + /// @param optional - Determines whether to assert that the + /// operand exists. + /// @param operandIndex - The index into the generated operand table. + /// Incremented by this function one or more + /// times to reflect possible duplicate + /// operands). + /// @param physicalOperandIndex - The index of the current operand into the + /// set of non-duplicate ('physical') operands. + /// Incremented by this function once. + /// @param numPhysicalOperands - The number of non-duplicate operands in the + /// instructions. + /// @param operandMapping - The operand mapping, which has an entry for + /// each operand that indicates whether it is a + /// duplicate, and of what. + void handleOperand(bool optional, + unsigned &operandIndex, + unsigned &physicalOperandIndex, + unsigned numPhysicalOperands, + const unsigned *operandMapping, + OperandEncoding (*encodingFromString) + (const std::string&, + uint8_t OpSize)); + + /// shouldBeEmitted - Returns the shouldBeEmitted field. Although filter() + /// filters out many instructions, at various points in decoding we + /// determine that the instruction should not actually be decodable. In + /// particular, MMX MOV instructions aren't emitted, but they're only + /// identified during operand parsing. + /// + /// @return - true if at this point we believe the instruction should be + /// emitted; false if not. This will return false if filter() returns false + /// once emitInstructionSpecifier() has been called. + bool shouldBeEmitted() const { + return ShouldBeEmitted; + } + + /// emitInstructionSpecifier - Loads the instruction specifier for the current + /// instruction into a DisassemblerTables. + /// + void emitInstructionSpecifier(); + + /// emitDecodePath - Populates the proper fields in the decode tables + /// corresponding to the decode paths for this instruction. + /// + /// \param tables The DisassemblerTables to populate with the decode + /// decode information for the current instruction. + void emitDecodePath(DisassemblerTables &tables) const; + + /// Constructor - Initializes a RecognizableInstr with the appropriate fields + /// from a CodeGenInstruction. + /// + /// \param tables The DisassemblerTables that the specifier will be added to. + /// \param insn The CodeGenInstruction to extract information from. + /// \param uid The unique ID of the current instruction. + RecognizableInstr(DisassemblerTables &tables, + const CodeGenInstruction &insn, + InstrUID uid); +public: + /// processInstr - Accepts a CodeGenInstruction and loads decode information + /// for it into a DisassemblerTables if appropriate. + /// + /// \param tables The DiassemblerTables to be populated with decode + /// information. + /// \param insn The CodeGenInstruction to be used as a source for this + /// information. + /// \param uid The unique ID of the instruction. + static void processInstr(DisassemblerTables &tables, + const CodeGenInstruction &insn, + InstrUID uid); +}; + +} // namespace X86Disassembler + +} // namespace llvm + +#endif |