diff options
Diffstat (limited to 'include/llvm/CodeGen/GlobalISel')
18 files changed, 2322 insertions, 284 deletions
diff --git a/include/llvm/CodeGen/GlobalISel/CallLowering.h b/include/llvm/CodeGen/GlobalISel/CallLowering.h index ba84d76de164..58eb412d8c24 100644 --- a/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -17,11 +17,11 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/CodeGen/CallingConvLower.h" -#include "llvm/CodeGen/MachineValueType.h" #include "llvm/CodeGen/TargetCallingConv.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MachineValueType.h" #include <cstdint> #include <functional> @@ -123,7 +123,7 @@ protected: } template <typename FuncInfoTy> - void setArgFlags(ArgInfo &Arg, unsigned OpNum, const DataLayout &DL, + void setArgFlags(ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const; /// Invoke Handler::assignArg on each of the given \p Args and then use @@ -131,7 +131,7 @@ protected: /// /// \return True if everything has succeeded, false otherwise. bool handleAssignments(MachineIRBuilder &MIRBuilder, ArrayRef<ArgInfo> Args, - ValueHandler &Callback) const; + ValueHandler &Handler) const; public: CallLowering(const TargetLowering *TLI) : TLI(TLI) {} diff --git a/include/llvm/CodeGen/GlobalISel/Combiner.h b/include/llvm/CodeGen/GlobalISel/Combiner.h new file mode 100644 index 000000000000..36a33deb4a64 --- /dev/null +++ b/include/llvm/CodeGen/GlobalISel/Combiner.h @@ -0,0 +1,43 @@ +//== ----- llvm/CodeGen/GlobalISel/Combiner.h --------------------- == // +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// This contains common code to drive combines. Combiner Passes will need to +/// setup a CombinerInfo and call combineMachineFunction. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_H +#define LLVM_CODEGEN_GLOBALISEL_COMBINER_H + +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/MachineFunctionPass.h" + +namespace llvm { +class MachineRegisterInfo; +class CombinerInfo; +class TargetPassConfig; +class MachineFunction; + +class Combiner { +public: + Combiner(CombinerInfo &CombinerInfo, const TargetPassConfig *TPC); + + bool combineMachineInstrs(MachineFunction &MF); + +protected: + CombinerInfo &CInfo; + + MachineRegisterInfo *MRI = nullptr; + const TargetPassConfig *TPC; + MachineIRBuilder Builder; +}; + +} // End namespace llvm. + +#endif // LLVM_CODEGEN_GLOBALISEL_GICOMBINER_H diff --git a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h new file mode 100644 index 000000000000..5d5b8398452c --- /dev/null +++ b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -0,0 +1,44 @@ +//== llvm/CodeGen/GlobalISel/CombinerHelper.h -------------- -*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--------------------------------------------------------------------===// +// +/// This contains common combine transformations that may be used in a combine +/// pass,or by the target elsewhere. +/// Targets can pick individual opcode transformations from the helper or use +/// tryCombine which invokes all transformations. All of the transformations +/// return true if the MachineInstruction changed and false otherwise. +// +//===--------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_HELPER_H +#define LLVM_CODEGEN_GLOBALISEL_COMBINER_HELPER_H + +namespace llvm { + +class MachineIRBuilder; +class MachineRegisterInfo; +class MachineInstr; + +class CombinerHelper { + MachineIRBuilder &Builder; + MachineRegisterInfo &MRI; + +public: + CombinerHelper(MachineIRBuilder &B); + + /// If \p MI is COPY, try to combine it. + /// Returns true if MI changed. + bool tryCombineCopy(MachineInstr &MI); + + /// Try to transform \p MI by using all of the above + /// combine functions. Returns true if changed. + bool tryCombine(MachineInstr &MI); +}; +} // namespace llvm + +#endif diff --git a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h new file mode 100644 index 000000000000..1d248547adbf --- /dev/null +++ b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h @@ -0,0 +1,48 @@ +//===- llvm/CodeGen/GlobalISel/CombinerInfo.h ------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// Interface for Targets to specify which operations are combined how and when. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_INFO_H +#define LLVM_CODEGEN_GLOBALISEL_COMBINER_INFO_H + +#include <cassert> +namespace llvm { + +class LegalizerInfo; +class MachineInstr; +class MachineIRBuilder; +class MachineRegisterInfo; +// Contains information relevant to enabling/disabling various combines for a +// pass. +class CombinerInfo { +public: + CombinerInfo(bool AllowIllegalOps, bool ShouldLegalizeIllegal, + LegalizerInfo *LInfo) + : IllegalOpsAllowed(AllowIllegalOps), + LegalizeIllegalOps(ShouldLegalizeIllegal), LInfo(LInfo) { + assert(((AllowIllegalOps || !LegalizeIllegalOps) || LInfo) && + "Expecting legalizerInfo when illegalops not allowed"); + } + virtual ~CombinerInfo() = default; + /// If \p IllegalOpsAllowed is false, the CombinerHelper will make use of + /// the legalizerInfo to check for legality before each transformation. + bool IllegalOpsAllowed; // TODO: Make use of this. + + /// If \p LegalizeIllegalOps is true, the Combiner will also legalize the + /// illegal ops that are created. + bool LegalizeIllegalOps; // TODO: Make use of this. + const LegalizerInfo *LInfo; + virtual bool combine(MachineInstr &MI, MachineIRBuilder &B) const = 0; +}; +} // namespace llvm + +#endif diff --git a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h new file mode 100644 index 000000000000..8d61f9a68279 --- /dev/null +++ b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h @@ -0,0 +1,134 @@ +//===-- llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h --*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements a version of MachineIRBuilder which does trivial +/// constant folding. +//===----------------------------------------------------------------------===// +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" + +namespace llvm { + +static Optional<APInt> ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, + const unsigned Op2, + const MachineRegisterInfo &MRI) { + auto MaybeOp1Cst = getConstantVRegVal(Op1, MRI); + auto MaybeOp2Cst = getConstantVRegVal(Op2, MRI); + if (MaybeOp1Cst && MaybeOp2Cst) { + LLT Ty = MRI.getType(Op1); + APInt C1(Ty.getSizeInBits(), *MaybeOp1Cst, true); + APInt C2(Ty.getSizeInBits(), *MaybeOp2Cst, true); + switch (Opcode) { + default: + break; + case TargetOpcode::G_ADD: + return C1 + C2; + case TargetOpcode::G_AND: + return C1 & C2; + case TargetOpcode::G_ASHR: + return C1.ashr(C2); + case TargetOpcode::G_LSHR: + return C1.lshr(C2); + case TargetOpcode::G_MUL: + return C1 * C2; + case TargetOpcode::G_OR: + return C1 | C2; + case TargetOpcode::G_SHL: + return C1 << C2; + case TargetOpcode::G_SUB: + return C1 - C2; + case TargetOpcode::G_XOR: + return C1 ^ C2; + case TargetOpcode::G_UDIV: + if (!C2.getBoolValue()) + break; + return C1.udiv(C2); + case TargetOpcode::G_SDIV: + if (!C2.getBoolValue()) + break; + return C1.sdiv(C2); + case TargetOpcode::G_UREM: + if (!C2.getBoolValue()) + break; + return C1.urem(C2); + case TargetOpcode::G_SREM: + if (!C2.getBoolValue()) + break; + return C1.srem(C2); + } + } + return None; +} + +/// An MIRBuilder which does trivial constant folding of binary ops. +/// Calls to buildInstr will also try to constant fold binary ops. +class ConstantFoldingMIRBuilder + : public FoldableInstructionsBuilder<ConstantFoldingMIRBuilder> { +public: + // Pull in base class constructors. + using FoldableInstructionsBuilder< + ConstantFoldingMIRBuilder>::FoldableInstructionsBuilder; + // Unhide buildInstr + using FoldableInstructionsBuilder<ConstantFoldingMIRBuilder>::buildInstr; + + // Implement buildBinaryOp required by FoldableInstructionsBuilder which + // tries to constant fold. + MachineInstrBuilder buildBinaryOp(unsigned Opcode, unsigned Dst, + unsigned Src0, unsigned Src1) { + validateBinaryOp(Dst, Src0, Src1); + auto MaybeCst = ConstantFoldBinOp(Opcode, Src0, Src1, getMF().getRegInfo()); + if (MaybeCst) + return buildConstant(Dst, MaybeCst->getSExtValue()); + return buildInstr(Opcode).addDef(Dst).addUse(Src0).addUse(Src1); + } + + template <typename DstTy, typename UseArg1Ty, typename UseArg2Ty> + MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, UseArg1Ty &&Arg1, + UseArg2Ty &&Arg2) { + unsigned Dst = getDestFromArg(Ty); + return buildInstr(Opc, Dst, getRegFromArg(std::forward<UseArg1Ty>(Arg1)), + getRegFromArg(std::forward<UseArg2Ty>(Arg2))); + } + + // Try to provide an overload for buildInstr for binary ops in order to + // constant fold. + MachineInstrBuilder buildInstr(unsigned Opc, unsigned Dst, unsigned Src0, + unsigned Src1) { + switch (Opc) { + default: + break; + case TargetOpcode::G_ADD: + case TargetOpcode::G_AND: + case TargetOpcode::G_ASHR: + case TargetOpcode::G_LSHR: + case TargetOpcode::G_MUL: + case TargetOpcode::G_OR: + case TargetOpcode::G_SHL: + case TargetOpcode::G_SUB: + case TargetOpcode::G_XOR: + case TargetOpcode::G_UDIV: + case TargetOpcode::G_SDIV: + case TargetOpcode::G_UREM: + case TargetOpcode::G_SREM: { + return buildBinaryOp(Opc, Dst, Src0, Src1); + } + } + return buildInstr(Opc).addDef(Dst).addUse(Src0).addUse(Src1); + } + + // Fallback implementation of buildInstr. + template <typename DstTy, typename... UseArgsTy> + MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, + UseArgsTy &&... Args) { + auto MIB = buildInstr(Opc).addDef(getDestFromArg(Ty)); + addUsesFromArgs(MIB, std::forward<UseArgsTy>(Args)...); + return MIB; + } +}; +} // namespace llvm diff --git a/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/include/llvm/CodeGen/GlobalISel/IRTranslator.h index 7061c014d9b7..f3553966fcdf 100644 --- a/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -24,6 +24,7 @@ #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Types.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/Support/Allocator.h" #include "llvm/IR/Intrinsics.h" #include <memory> #include <utility> @@ -63,9 +64,83 @@ private: /// Interface used to lower the everything related to calls. const CallLowering *CLI; - /// Mapping of the values of the current LLVM IR function - /// to the related virtual registers. - ValueToVReg ValToVReg; + /// This class contains the mapping between the Values to vreg related data. + class ValueToVRegInfo { + public: + ValueToVRegInfo() = default; + + using VRegListT = SmallVector<unsigned, 1>; + using OffsetListT = SmallVector<uint64_t, 1>; + + using const_vreg_iterator = + DenseMap<const Value *, VRegListT *>::const_iterator; + using const_offset_iterator = + DenseMap<const Value *, OffsetListT *>::const_iterator; + + inline const_vreg_iterator vregs_end() const { return ValToVRegs.end(); } + + VRegListT *getVRegs(const Value &V) { + auto It = ValToVRegs.find(&V); + if (It != ValToVRegs.end()) + return It->second; + + return insertVRegs(V); + } + + OffsetListT *getOffsets(const Value &V) { + auto It = TypeToOffsets.find(V.getType()); + if (It != TypeToOffsets.end()) + return It->second; + + return insertOffsets(V); + } + + const_vreg_iterator findVRegs(const Value &V) const { + return ValToVRegs.find(&V); + } + + bool contains(const Value &V) const { + return ValToVRegs.find(&V) != ValToVRegs.end(); + } + + void reset() { + ValToVRegs.clear(); + TypeToOffsets.clear(); + VRegAlloc.DestroyAll(); + OffsetAlloc.DestroyAll(); + } + + private: + VRegListT *insertVRegs(const Value &V) { + assert(ValToVRegs.find(&V) == ValToVRegs.end() && "Value already exists"); + + // We placement new using our fast allocator since we never try to free + // the vectors until translation is finished. + auto *VRegList = new (VRegAlloc.Allocate()) VRegListT(); + ValToVRegs[&V] = VRegList; + return VRegList; + } + + OffsetListT *insertOffsets(const Value &V) { + assert(TypeToOffsets.find(V.getType()) == TypeToOffsets.end() && + "Type already exists"); + + auto *OffsetList = new (OffsetAlloc.Allocate()) OffsetListT(); + TypeToOffsets[V.getType()] = OffsetList; + return OffsetList; + } + SpecificBumpPtrAllocator<VRegListT> VRegAlloc; + SpecificBumpPtrAllocator<OffsetListT> OffsetAlloc; + + // We store pointers to vectors here since references may be invalidated + // while we hold them if we stored the vectors directly. + DenseMap<const Value *, VRegListT*> ValToVRegs; + DenseMap<const Type *, OffsetListT*> TypeToOffsets; + }; + + /// Mapping of the values of the current LLVM IR function to the related + /// virtual registers and offsets. + ValueToVRegInfo VMap; // N.b. it's not completely obvious that this will be sufficient for every // LLVM IR construct (with "invoke" being the obvious candidate to mess up our @@ -82,7 +157,8 @@ private: // List of stubbed PHI instructions, for values and basic blocks to be filled // in once all MachineBasicBlocks have been created. - SmallVector<std::pair<const PHINode *, MachineInstr *>, 4> PendingPHIs; + SmallVector<std::pair<const PHINode *, SmallVector<MachineInstr *, 1>>, 4> + PendingPHIs; /// Record of what frame index has been allocated to specified allocas for /// this function. @@ -99,7 +175,7 @@ private: /// The general algorithm is: /// 1. Look for a virtual register for each operand or /// create one. - /// 2 Update the ValToVReg accordingly. + /// 2 Update the VMap accordingly. /// 2.alt. For constant arguments, if they are compile time constants, /// produce an immediate in the right operand and do not touch /// ValToReg. Actually we will go with a virtual register for each @@ -134,7 +210,7 @@ private: /// Translate an LLVM string intrinsic (memcpy, memset, ...). bool translateMemfunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, - unsigned Intrinsic); + unsigned ID); void getStackGuard(unsigned DstReg, MachineIRBuilder &MIRBuilder); @@ -146,6 +222,19 @@ private: bool translateInlineAsm(const CallInst &CI, MachineIRBuilder &MIRBuilder); + // FIXME: temporary function to expose previous interface to call lowering + // until it is refactored. + /// Combines all component registers of \p V into a single scalar with size + /// "max(Offsets) + last size". + unsigned packRegs(const Value &V, MachineIRBuilder &MIRBuilder); + + void unpackRegs(const Value &V, unsigned Src, MachineIRBuilder &MIRBuilder); + + /// Returns true if the value should be split into multiple LLTs. + /// If \p Offsets is given then the split type's offsets will be stored in it. + bool valueIsSplit(const Value &V, + SmallVectorImpl<uint64_t> *Offsets = nullptr); + /// Translate call instruction. /// \pre \p U is a call instruction. bool translateCall(const User &U, MachineIRBuilder &MIRBuilder); @@ -310,6 +399,9 @@ private: bool translateShuffleVector(const User &U, MachineIRBuilder &MIRBuilder); + bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder); + bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder); + // Stubs to keep the compiler happy while we implement the rest of the // translation. bool translateResume(const User &U, MachineIRBuilder &MIRBuilder) { @@ -327,14 +419,8 @@ private: bool translateFence(const User &U, MachineIRBuilder &MIRBuilder) { return false; } - bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder) { - return false; - } - bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder) { - return false; - } bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) { - return false; + return translateCast(TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder); } bool translateCleanupPad(const User &U, MachineIRBuilder &MIRBuilder) { return false; @@ -381,9 +467,24 @@ private: // * Clear the different maps. void finalizeFunction(); - /// Get the VReg that represents \p Val. - /// If such VReg does not exist, it is created. - unsigned getOrCreateVReg(const Value &Val); + /// Get the VRegs that represent \p Val. + /// Non-aggregate types have just one corresponding VReg and the list can be + /// used as a single "unsigned". Aggregates get flattened. If such VRegs do + /// not exist, they are created. + ArrayRef<unsigned> getOrCreateVRegs(const Value &Val); + + unsigned getOrCreateVReg(const Value &Val) { + auto Regs = getOrCreateVRegs(Val); + if (Regs.empty()) + return 0; + assert(Regs.size() == 1 && + "attempt to get single VReg for aggregate or void"); + return Regs[0]; + } + + /// Allocate some vregs and offsets in the VMap. Then populate just the + /// offsets while leaving the vregs empty. + ValueToVRegInfo::VRegListT &allocateVRegs(const Value &Val); /// Get the frame index that represents \p Val. /// If such VReg does not exist, it is created. diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index 4264a866b6c0..471def7f45a3 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -20,6 +20,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/CodeGenCoverage.h" +#include "llvm/Support/LowLevelTypeImpl.h" #include <bitset> #include <cstddef> #include <cstdint> @@ -31,7 +32,6 @@ namespace llvm { class APInt; class APFloat; -class LLT; class MachineInstr; class MachineInstrBuilder; class MachineFunction; @@ -81,6 +81,23 @@ enum { /// failed match. GIM_Try, + /// Switch over the opcode on the specified instruction + /// - InsnID - Instruction ID + /// - LowerBound - numerically minimum opcode supported + /// - UpperBound - numerically maximum + 1 opcode supported + /// - Default - failure jump target + /// - JumpTable... - (UpperBound - LowerBound) (at least 2) jump targets + GIM_SwitchOpcode, + + /// Switch over the LLT on the specified instruction operand + /// - InsnID - Instruction ID + /// - OpIdx - Operand index + /// - LowerBound - numerically minimum Type ID supported + /// - UpperBound - numerically maximum + 1 Type ID supported + /// - Default - failure jump target + /// - JumpTable... - (UpperBound - LowerBound) (at least 2) jump targets + GIM_SwitchType, + /// Record the specified instruction /// - NewInsnID - Instruction ID to define /// - InsnID - Instruction ID @@ -117,6 +134,23 @@ enum { GIM_CheckAtomicOrdering, GIM_CheckAtomicOrderingOrStrongerThan, GIM_CheckAtomicOrderingWeakerThan, + /// Check the size of the memory access for the given machine memory operand. + /// - InsnID - Instruction ID + /// - MMOIdx - MMO index + /// - Size - The size in bytes of the memory access + GIM_CheckMemorySizeEqualTo, + /// Check the size of the memory access for the given machine memory operand + /// against the size of an operand. + /// - InsnID - Instruction ID + /// - MMOIdx - MMO index + /// - OpIdx - The operand index to compare the MMO against + GIM_CheckMemorySizeEqualToLLT, + GIM_CheckMemorySizeLessThanLLT, + GIM_CheckMemorySizeGreaterThanLLT, + /// Check a generic C++ instruction predicate + /// - InsnID - Instruction ID + /// - PredicateID - The ID of the predicate function to call + GIM_CheckCxxInsnPredicate, /// Check the type for the specified operand /// - InsnID - Instruction ID @@ -133,12 +167,14 @@ enum { /// - OpIdx - Operand index /// - Expected register bank (specified as a register class) GIM_CheckRegBankForClass, + /// Check the operand matches a complex predicate /// - InsnID - Instruction ID /// - OpIdx - Operand index /// - RendererID - The renderer to hold the result /// - Complex predicate ID GIM_CheckComplexPattern, + /// Check the operand is a specific integer /// - InsnID - Instruction ID /// - OpIdx - Operand index @@ -155,6 +191,7 @@ enum { /// - OpIdx - Operand index /// - Expected Intrinsic ID GIM_CheckIntrinsicID, + /// Check the specified operand is an MBB /// - InsnID - Instruction ID /// - OpIdx - Operand index @@ -183,6 +220,7 @@ enum { /// - OldInsnID - Instruction ID to mutate /// - NewOpcode - The new opcode to use GIR_MutateOpcode, + /// Build a new instruction /// - InsnID - Instruction ID to define /// - Opcode - The new opcode to use @@ -193,6 +231,7 @@ enum { /// - OldInsnID - Instruction ID to copy from /// - OpIdx - The operand to copy GIR_Copy, + /// Copy an operand to the specified instruction or add a zero register if the /// operand is a zero immediate. /// - NewInsnID - Instruction ID to modify @@ -206,6 +245,7 @@ enum { /// - OpIdx - The operand to copy /// - SubRegIdx - The subregister to copy GIR_CopySubReg, + /// Add an implicit register def to the specified instruction /// - InsnID - Instruction ID to modify /// - RegNum - The register to add @@ -218,10 +258,13 @@ enum { /// - InsnID - Instruction ID to modify /// - RegNum - The register to add GIR_AddRegister, - /// Add a a temporary register to the specified instruction + + /// Add a temporary register to the specified instruction /// - InsnID - Instruction ID to modify /// - TempRegID - The temporary register ID to add + /// - TempRegFlags - The register flags to set GIR_AddTempRegister, + /// Add an immediate to the specified instruction /// - InsnID - Instruction ID to modify /// - Imm - The immediate to add @@ -230,11 +273,17 @@ enum { /// - InsnID - Instruction ID to modify /// - RendererID - The renderer to call GIR_ComplexRenderer, + /// Render sub-operands of complex operands to the specified instruction /// - InsnID - Instruction ID to modify /// - RendererID - The renderer to call /// - RenderOpID - The suboperand to render. GIR_ComplexSubOperandRenderer, + /// Render operands to the specified instruction using a custom function + /// - InsnID - Instruction ID to modify + /// - OldInsnID - Instruction ID to get the matched operand from + /// - RendererFnID - Custom renderer function to call + GIR_CustomRenderer, /// Render a G_CONSTANT operator as a sign-extended immediate. /// - NewInsnID - Instruction ID to modify @@ -242,24 +291,34 @@ enum { /// The operand index is implicitly 1. GIR_CopyConstantAsSImm, + /// Render a G_FCONSTANT operator as a sign-extended immediate. + /// - NewInsnID - Instruction ID to modify + /// - OldInsnID - Instruction ID to copy from + /// The operand index is implicitly 1. + GIR_CopyFConstantAsFPImm, + /// Constrain an instruction operand to a register class. /// - InsnID - Instruction ID to modify /// - OpIdx - Operand index /// - RCEnum - Register class enumeration value GIR_ConstrainOperandRC, + /// Constrain an instructions operands according to the instruction /// description. /// - InsnID - Instruction ID to modify GIR_ConstrainSelectedInstOperands, + /// Merge all memory operands into instruction. /// - InsnID - Instruction ID to modify /// - MergeInsnID... - One or more Instruction ID to merge into the result. /// - GIU_MergeMemOperands_EndOfList - Terminates the list of instructions to /// merge. GIR_MergeMemOperands, + /// Erase from parent. /// - InsnID - Instruction ID to erase GIR_EraseFromParent, + /// Create a new temporary register that's not constrained. /// - TempRegID - The temporary register ID to initialize. /// - Expected type @@ -271,6 +330,9 @@ enum { /// Increment the rule coverage counter. /// - RuleID - The ID of the rule that was covered. GIR_Coverage, + + /// Keeping track of the number of the GI opcodes. Must be the last entry. + GIU_NumOpcodes, }; enum { @@ -311,11 +373,27 @@ protected: }; public: - template <class PredicateBitset, class ComplexMatcherMemFn> - struct MatcherInfoTy { + template <class PredicateBitset, class ComplexMatcherMemFn, + class CustomRendererFn> + struct ISelInfoTy { + ISelInfoTy(const LLT *TypeObjects, size_t NumTypeObjects, + const PredicateBitset *FeatureBitsets, + const ComplexMatcherMemFn *ComplexPredicates, + const CustomRendererFn *CustomRenderers) + : TypeObjects(TypeObjects), + FeatureBitsets(FeatureBitsets), + ComplexPredicates(ComplexPredicates), + CustomRenderers(CustomRenderers) { + + for (size_t I = 0; I < NumTypeObjects; ++I) + TypeIDMap[TypeObjects[I]] = I; + } const LLT *TypeObjects; const PredicateBitset *FeatureBitsets; const ComplexMatcherMemFn *ComplexPredicates; + const CustomRendererFn *CustomRenderers; + + SmallDenseMap<LLT, unsigned, 64> TypeIDMap; }; protected: @@ -324,23 +402,35 @@ protected: /// Execute a given matcher table and return true if the match was successful /// and false otherwise. template <class TgtInstructionSelector, class PredicateBitset, - class ComplexMatcherMemFn> + class ComplexMatcherMemFn, class CustomRendererFn> bool executeMatchTable( TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State, - const MatcherInfoTy<PredicateBitset, ComplexMatcherMemFn> &MatcherInfo, + const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> + &ISelInfo, const int64_t *MatchTable, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, CodeGenCoverage &CoverageInfo) const; + virtual const int64_t *getMatchTable() const { + llvm_unreachable("Should have been overridden by tablegen if used"); + } + virtual bool testImmPredicate_I64(unsigned, int64_t) const { - llvm_unreachable("Subclasses must override this to use tablegen"); + llvm_unreachable( + "Subclasses must override this with a tablegen-erated function"); } virtual bool testImmPredicate_APInt(unsigned, const APInt &) const { - llvm_unreachable("Subclasses must override this to use tablegen"); + llvm_unreachable( + "Subclasses must override this with a tablegen-erated function"); } virtual bool testImmPredicate_APFloat(unsigned, const APFloat &) const { - llvm_unreachable("Subclasses must override this to use tablegen"); + llvm_unreachable( + "Subclasses must override this with a tablegen-erated function"); + } + virtual bool testMIPredicate_MI(unsigned, const MachineInstr &) const { + llvm_unreachable( + "Subclasses must override this with a tablegen-erated function"); } /// Constrain a register operand of an instruction \p I to a specified @@ -353,20 +443,6 @@ protected: const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const; - /// Mutate the newly-selected instruction \p I to constrain its (possibly - /// generic) virtual register operands to the instruction's register class. - /// This could involve inserting COPYs before (for uses) or after (for defs). - /// This requires the number of operands to match the instruction description. - /// \returns whether operand regclass constraining succeeded. - /// - // FIXME: Not all instructions have the same number of operands. We should - // probably expose a constrain helper per operand and let the target selector - // constrain individual registers, like fast-isel. - bool constrainSelectedInstRegOperands(MachineInstr &I, - const TargetInstrInfo &TII, - const TargetRegisterInfo &TRI, - const RegisterBankInfo &RBI) const; - bool isOperandImmEqual(const MachineOperand &MO, int64_t Value, const MachineRegisterInfo &MRI) const; diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index bf834cf8f5e3..2003a79f6b20 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -19,6 +19,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -40,19 +41,22 @@ enum { GIPFP_I64_Invalid = 0, GIPFP_APInt_Invalid = 0, GIPFP_APFloat_Invalid = 0, + GIPFP_MI_Invalid = 0, }; template <class TgtInstructionSelector, class PredicateBitset, - class ComplexMatcherMemFn> + class ComplexMatcherMemFn, class CustomRendererFn> bool InstructionSelector::executeMatchTable( TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State, - const MatcherInfoTy<PredicateBitset, ComplexMatcherMemFn> &MatcherInfo, + const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> + &ISelInfo, const int64_t *MatchTable, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, CodeGenCoverage &CoverageInfo) const { + uint64_t CurrentIdx = 0; - SmallVector<uint64_t, 8> OnFailResumeAt; + SmallVector<uint64_t, 4> OnFailResumeAt; enum RejectAction { RejectAndGiveUp, RejectAndResume }; auto handleReject = [&]() -> RejectAction { @@ -60,8 +64,7 @@ bool InstructionSelector::executeMatchTable( dbgs() << CurrentIdx << ": Rejected\n"); if (OnFailResumeAt.empty()) return RejectAndGiveUp; - CurrentIdx = OnFailResumeAt.back(); - OnFailResumeAt.pop_back(); + CurrentIdx = OnFailResumeAt.pop_back_val(); DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" << OnFailResumeAt.size() << " try-blocks remain)\n"); @@ -70,7 +73,8 @@ bool InstructionSelector::executeMatchTable( while (true) { assert(CurrentIdx != ~0u && "Invalid MatchTable index"); - switch (MatchTable[CurrentIdx++]) { + int64_t MatcherOpcode = MatchTable[CurrentIdx++]; + switch (MatcherOpcode) { case GIM_Try: { DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << CurrentIdx << ": Begin try-block\n"); @@ -124,8 +128,8 @@ bool InstructionSelector::executeMatchTable( dbgs() << CurrentIdx << ": GIM_CheckFeatures(ExpectedBitsetID=" << ExpectedBitsetID << ")\n"); - if ((AvailableFeatures & MatcherInfo.FeatureBitsets[ExpectedBitsetID]) != - MatcherInfo.FeatureBitsets[ExpectedBitsetID]) { + if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) != + ISelInfo.FeatureBitsets[ExpectedBitsetID]) { if (handleReject() == RejectAndGiveUp) return false; } @@ -136,12 +140,13 @@ bool InstructionSelector::executeMatchTable( int64_t InsnID = MatchTable[CurrentIdx++]; int64_t Expected = MatchTable[CurrentIdx++]; + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); unsigned Opcode = State.MIs[InsnID]->getOpcode(); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID << "], ExpectedOpcode=" << Expected << ") // Got=" << Opcode << "\n"); - assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); if (Opcode != Expected) { if (handleReject() == RejectAndGiveUp) return false; @@ -149,6 +154,77 @@ bool InstructionSelector::executeMatchTable( break; } + case GIM_SwitchOpcode: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t LowerBound = MatchTable[CurrentIdx++]; + int64_t UpperBound = MatchTable[CurrentIdx++]; + int64_t Default = MatchTable[CurrentIdx++]; + + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + const int64_t Opcode = State.MIs[InsnID]->getOpcode(); + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { + dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" + << LowerBound << ", " << UpperBound << "), Default=" << Default + << ", JumpTable...) // Got=" << Opcode << "\n"; + }); + if (Opcode < LowerBound || UpperBound <= Opcode) { + CurrentIdx = Default; + break; + } + CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; + if (!CurrentIdx) { + CurrentIdx = Default; + break; + } + OnFailResumeAt.push_back(Default); + break; + } + + case GIM_SwitchType: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + int64_t LowerBound = MatchTable[CurrentIdx++]; + int64_t UpperBound = MatchTable[CurrentIdx++]; + int64_t Default = MatchTable[CurrentIdx++]; + + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { + dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID + << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " + << UpperBound << "), Default=" << Default + << ", JumpTable...) // Got="; + if (!MO.isReg()) + dbgs() << "Not a VReg\n"; + else + dbgs() << MRI.getType(MO.getReg()) << "\n"; + }); + if (!MO.isReg()) { + CurrentIdx = Default; + break; + } + const LLT Ty = MRI.getType(MO.getReg()); + const auto TyI = ISelInfo.TypeIDMap.find(Ty); + if (TyI == ISelInfo.TypeIDMap.end()) { + CurrentIdx = Default; + break; + } + const int64_t TypeID = TyI->second; + if (TypeID < LowerBound || UpperBound <= TypeID) { + CurrentIdx = Default; + break; + } + CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; + if (!CurrentIdx) { + CurrentIdx = Default; + break; + } + OnFailResumeAt.push_back(Default); + break; + } + case GIM_CheckNumOperands: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t Expected = MatchTable[CurrentIdx++]; @@ -194,7 +270,8 @@ bool InstructionSelector::executeMatchTable( << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" << InsnID << "], Predicate=" << Predicate << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - assert(State.MIs[InsnID]->getOpcode() && "Expected G_CONSTANT"); + assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && + "Expected G_CONSTANT"); assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); APInt Value; if (State.MIs[InsnID]->getOperand(1).isCImm()) @@ -226,6 +303,21 @@ bool InstructionSelector::executeMatchTable( return false; break; } + case GIM_CheckCxxInsnPredicate: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t Predicate = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() + << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" + << InsnID << "], Predicate=" << Predicate << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate"); + + if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID])) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } case GIM_CheckAtomicOrdering: { int64_t InsnID = MatchTable[CurrentIdx++]; AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; @@ -233,7 +325,6 @@ bool InstructionSelector::executeMatchTable( dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" << InsnID << "], " << (uint64_t)Ordering << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (!State.MIs[InsnID]->hasOneMemOperand()) if (handleReject() == RejectAndGiveUp) return false; @@ -252,7 +343,6 @@ bool InstructionSelector::executeMatchTable( << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" << InsnID << "], " << (uint64_t)Ordering << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (!State.MIs[InsnID]->hasOneMemOperand()) if (handleReject() == RejectAndGiveUp) return false; @@ -271,7 +361,6 @@ bool InstructionSelector::executeMatchTable( << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" << InsnID << "], " << (uint64_t)Ordering << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (!State.MIs[InsnID]->hasOneMemOperand()) if (handleReject() == RejectAndGiveUp) return false; @@ -282,6 +371,87 @@ bool InstructionSelector::executeMatchTable( return false; break; } + case GIM_CheckMemorySizeEqualTo: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t MMOIdx = MatchTable[CurrentIdx++]; + uint64_t Size = MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx + << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID + << "]->memoperands() + " << MMOIdx + << ", Size=" << Size << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + + if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + + MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); + + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << MMO->getSize() << " bytes vs " << Size + << " bytes\n"); + if (MMO->getSize() != Size) + if (handleReject() == RejectAndGiveUp) + return false; + + break; + } + case GIM_CheckMemorySizeEqualToLLT: + case GIM_CheckMemorySizeLessThanLLT: + case GIM_CheckMemorySizeGreaterThanLLT: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t MMOIdx = MatchTable[CurrentIdx++]; + int64_t OpIdx = MatchTable[CurrentIdx++]; + + DEBUG_WITH_TYPE( + TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIM_CheckMemorySize" + << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT + ? "EqualTo" + : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT + ? "GreaterThan" + : "LessThan") + << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx + << ", OpIdx=" << OpIdx << ")\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (!MO.isReg()) { + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": Not a register\n"); + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + + if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + + MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); + + unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); + if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && + MMO->getSize() * 8 != Size) { + if (handleReject() == RejectAndGiveUp) + return false; + } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && + MMO->getSize() * 8 >= Size) { + if (handleReject() == RejectAndGiveUp) + return false; + } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && + MMO->getSize() * 8 <= Size) + if (handleReject() == RejectAndGiveUp) + return false; + + break; + } case GIM_CheckType: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t OpIdx = MatchTable[CurrentIdx++]; @@ -291,8 +461,9 @@ bool InstructionSelector::executeMatchTable( << "]->getOperand(" << OpIdx << "), TypeID=" << TypeID << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()) != - MatcherInfo.TypeObjects[TypeID]) { + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (!MO.isReg() || + MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) { if (handleReject() == RejectAndGiveUp) return false; } @@ -308,7 +479,6 @@ bool InstructionSelector::executeMatchTable( << InsnID << "]->getOperand(" << OpIdx << "), SizeInBits=" << SizeInBits << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - // iPTR must be looked up in the target. if (SizeInBits == 0) { MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); @@ -317,11 +487,15 @@ bool InstructionSelector::executeMatchTable( assert(SizeInBits != 0 && "Pointer size must be known"); - const LLT &Ty = MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()); - if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) { - if (handleReject() == RejectAndGiveUp) - return false; - } + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (MO.isReg()) { + const LLT &Ty = MRI.getType(MO.getReg()); + if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) + if (handleReject() == RejectAndGiveUp) + return false; + } else if (handleReject() == RejectAndGiveUp) + return false; + break; } case GIM_CheckRegBankForClass: { @@ -333,9 +507,10 @@ bool InstructionSelector::executeMatchTable( << InsnID << "]->getOperand(" << OpIdx << "), RCEnum=" << RCEnum << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); - if (&RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != - RBI.getRegBank(State.MIs[InsnID]->getOperand(OpIdx).getReg(), MRI, - TRI)) { + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (!MO.isReg() || + &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != + RBI.getRegBank(MO.getReg(), MRI, TRI)) { if (handleReject() == RejectAndGiveUp) return false; } @@ -356,7 +531,7 @@ bool InstructionSelector::executeMatchTable( assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); // FIXME: Use std::invoke() when it's available. ComplexRendererFns Renderer = - (ISel.*MatcherInfo.ComplexPredicates[ComplexPredicateID])( + (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])( State.MIs[InsnID]->getOperand(OpIdx)); if (Renderer.hasValue()) State.Renderers[RendererID] = Renderer.getValue(); @@ -375,16 +550,19 @@ bool InstructionSelector::executeMatchTable( << InsnID << "]->getOperand(" << OpIdx << "), Value=" << Value << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + if (MO.isReg()) { + // isOperandImmEqual() will sign-extend to 64-bits, so should we. + LLT Ty = MRI.getType(MO.getReg()); + Value = SignExtend64(Value, Ty.getSizeInBits()); - // isOperandImmEqual() will sign-extend to 64-bits, so should we. - LLT Ty = MRI.getType(State.MIs[InsnID]->getOperand(OpIdx).getReg()); - Value = SignExtend64(Value, Ty.getSizeInBits()); + if (!isOperandImmEqual(MO, Value, MRI)) { + if (handleReject() == RejectAndGiveUp) + return false; + } + } else if (handleReject() == RejectAndGiveUp) + return false; - if (!isOperandImmEqual(State.MIs[InsnID]->getOperand(OpIdx), Value, - MRI)) { - if (handleReject() == RejectAndGiveUp) - return false; - } break; } @@ -467,7 +645,7 @@ bool InstructionSelector::executeMatchTable( } case GIM_Reject: DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), - dbgs() << CurrentIdx << ": GIM_Reject"); + dbgs() << CurrentIdx << ": GIM_Reject\n"); if (handleReject() == RejectAndGiveUp) return false; break; @@ -649,6 +827,36 @@ bool InstructionSelector::executeMatchTable( break; } + // TODO: Needs a test case once we have a pattern that uses this. + case GIR_CopyFConstantAsFPImm: { + int64_t NewInsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); + assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT"); + if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) + OutMIs[NewInsnID].addFPImm( + State.MIs[OldInsnID]->getOperand(1).getFPImm()); + else + llvm_unreachable("Expected FPImm operand"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" + << NewInsnID << "], MIs[" << OldInsnID << "])\n"); + break; + } + + case GIR_CustomRenderer: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t OldInsnID = MatchTable[CurrentIdx++]; + int64_t RendererFnID = MatchTable[CurrentIdx++]; + assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); + DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), + dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" + << InsnID << "], MIs[" << OldInsnID << "], " + << RendererFnID << ")\n"); + (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], + *State.MIs[OldInsnID]); + break; + } case GIR_ConstrainOperandRC: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t OpIdx = MatchTable[CurrentIdx++]; @@ -710,7 +918,7 @@ bool InstructionSelector::executeMatchTable( int64_t TypeID = MatchTable[CurrentIdx++]; State.TempRegisters[TempRegID] = - MRI.createGenericVirtualRegister(MatcherInfo.TypeObjects[TypeID]); + MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]); DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << CurrentIdx << ": TempRegs[" << TempRegID << "] = GIR_MakeTempReg(" << TypeID << ")\n"); @@ -729,7 +937,7 @@ bool InstructionSelector::executeMatchTable( case GIR_Done: DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), - dbgs() << CurrentIdx << ": GIR_Done"); + dbgs() << CurrentIdx << ": GIR_Done\n"); return true; default: diff --git a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index e7945ff5bf4f..873587651efd 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -38,7 +38,7 @@ public: return false; if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_TRUNC, MI.getOperand(1).getReg(), MRI)) { - DEBUG(dbgs() << ".. Combine MI: " << MI;); + LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); unsigned DstReg = MI.getOperand(0).getReg(); unsigned SrcReg = DefMI->getOperand(1).getReg(); Builder.setInstr(MI); @@ -59,10 +59,10 @@ public: MI.getOperand(1).getReg(), MRI)) { unsigned DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); - if (isInstUnsupported(TargetOpcode::G_AND, DstTy) || - isInstUnsupported(TargetOpcode::G_CONSTANT, DstTy)) + if (isInstUnsupported({TargetOpcode::G_AND, {DstTy}}) || + isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) return false; - DEBUG(dbgs() << ".. Combine MI: " << MI;); + LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); Builder.setInstr(MI); unsigned ZExtSrc = MI.getOperand(1).getReg(); LLT ZExtSrcTy = MRI.getType(ZExtSrc); @@ -87,11 +87,11 @@ public: MI.getOperand(1).getReg(), MRI)) { unsigned DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); - if (isInstUnsupported(TargetOpcode::G_SHL, DstTy) || - isInstUnsupported(TargetOpcode::G_ASHR, DstTy) || - isInstUnsupported(TargetOpcode::G_CONSTANT, DstTy)) + if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy}}) || + isInstUnsupported({TargetOpcode::G_ASHR, {DstTy}}) || + isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) return false; - DEBUG(dbgs() << ".. Combine MI: " << MI;); + LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); Builder.setInstr(MI); unsigned SExtSrc = MI.getOperand(1).getReg(); LLT SExtSrcTy = MRI.getType(SExtSrc); @@ -121,9 +121,9 @@ public: MI.getOperand(1).getReg(), MRI)) { unsigned DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); - if (isInstUnsupported(TargetOpcode::G_IMPLICIT_DEF, DstTy)) + if (isInstUnsupported({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) return false; - DEBUG(dbgs() << ".. Combine EXT(IMPLICIT_DEF) " << MI;); + LLVM_DEBUG(dbgs() << ".. Combine EXT(IMPLICIT_DEF) " << MI;); Builder.setInstr(MI); Builder.buildInstr(TargetOpcode::G_IMPLICIT_DEF, DstReg); markInstAndDefDead(MI, *DefMI, DeadInsts); @@ -139,9 +139,9 @@ public: return false; unsigned NumDefs = MI.getNumOperands() - 1; - unsigned SrcReg = MI.getOperand(NumDefs).getReg(); - MachineInstr *MergeI = MRI.getVRegDef(SrcReg); - if (!MergeI || (MergeI->getOpcode() != TargetOpcode::G_MERGE_VALUES)) + MachineInstr *MergeI = getOpcodeDef(TargetOpcode::G_MERGE_VALUES, + MI.getOperand(NumDefs).getReg(), MRI); + if (!MergeI) return false; const unsigned NumMergeRegs = MergeI->getNumOperands() - 1; @@ -253,11 +253,8 @@ private: // and as a result, %3, %2, %1 are dead. MachineInstr *PrevMI = &MI; while (PrevMI != &DefMI) { - // If we're dealing with G_UNMERGE_VALUES, tryCombineMerges doesn't really try - // to fold copies in between and we can ignore them here. - if (PrevMI->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) - break; - unsigned PrevRegSrc = PrevMI->getOperand(1).getReg(); + unsigned PrevRegSrc = + PrevMI->getOperand(PrevMI->getNumOperands() - 1).getReg(); MachineInstr *TmpDef = MRI.getVRegDef(PrevRegSrc); if (MRI.hasOneUse(PrevRegSrc)) { if (TmpDef != &DefMI) { @@ -269,18 +266,16 @@ private: break; PrevMI = TmpDef; } - if ((PrevMI == &DefMI || - DefMI.getOpcode() == TargetOpcode::G_MERGE_VALUES) && - MRI.hasOneUse(DefMI.getOperand(0).getReg())) + if (PrevMI == &DefMI && MRI.hasOneUse(DefMI.getOperand(0).getReg())) DeadInsts.push_back(&DefMI); } /// Checks if the target legalizer info has specified anything about the /// instruction, or if unsupported. - bool isInstUnsupported(unsigned Opcode, const LLT &DstTy) const { - auto Action = LI.getAction({Opcode, 0, DstTy}); - return Action.first == LegalizerInfo::LegalizeAction::Unsupported || - Action.first == LegalizerInfo::LegalizeAction::NotFound; + bool isInstUnsupported(const LegalityQuery &Query) const { + using namespace LegalizeActions; + auto Step = LI.getAction(Query); + return Step.Action == Unsupported || Step.Action == NotFound; } }; diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 8bd8a9dcd0e2..d122e67b87b8 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -93,12 +93,24 @@ public: const LegalizerInfo &getLegalizerInfo() const { return LI; } private: + /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a + /// Use by extending the operand's type to \p WideTy using the specified \p + /// ExtOpcode for the extension instruction, and replacing the vreg of the + /// operand in place. + void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, + unsigned ExtOpcode); + + /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a + /// Def by extending the operand's type to \p WideTy and truncating it back + /// with the \p TruncOpcode, and replacing the vreg of the operand in place. + void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0, + unsigned TruncOpcode = TargetOpcode::G_TRUNC); /// Helper function to split a wide generic register into bitwise blocks with /// the given Type (which implies the number of blocks needed). The generic /// registers created are appended to Ops, starting at bit 0 of Reg. void extractParts(unsigned Reg, LLT Ty, int NumParts, - SmallVectorImpl<unsigned> &Ops); + SmallVectorImpl<unsigned> &VRegs); MachineRegisterInfo &MRI; const LegalizerInfo &LI; diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index b6735d538b37..713d72eb4c9b 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -19,8 +19,11 @@ #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Support/LowLevelTypeImpl.h" #include <cassert> #include <cstdint> @@ -30,9 +33,67 @@ namespace llvm { +extern cl::opt<bool> DisableGISelLegalityCheck; + class MachineInstr; class MachineIRBuilder; class MachineRegisterInfo; +class MCInstrInfo; + +namespace LegalizeActions { +enum LegalizeAction : std::uint8_t { + /// The operation is expected to be selectable directly by the target, and + /// no transformation is necessary. + Legal, + + /// The operation should be synthesized from multiple instructions acting on + /// a narrower scalar base-type. For example a 64-bit add might be + /// implemented in terms of 32-bit add-with-carry. + NarrowScalar, + + /// The operation should be implemented in terms of a wider scalar + /// base-type. For example a <2 x s8> add could be implemented as a <2 + /// x s32> add (ignoring the high bits). + WidenScalar, + + /// The (vector) operation should be implemented by splitting it into + /// sub-vectors where the operation is legal. For example a <8 x s64> add + /// might be implemented as 4 separate <2 x s64> adds. + FewerElements, + + /// The (vector) operation should be implemented by widening the input + /// vector and ignoring the lanes added by doing so. For example <2 x i8> is + /// rarely legal, but you might perform an <8 x i8> and then only look at + /// the first two results. + MoreElements, + + /// The operation itself must be expressed in terms of simpler actions on + /// this target. E.g. a SREM replaced by an SDIV and subtraction. + Lower, + + /// The operation should be implemented as a call to some kind of runtime + /// support library. For example this usually happens on machines that don't + /// support floating-point operations natively. + Libcall, + + /// The target wants to do something special with this combination of + /// operand and type. A callback will be issued when it is needed. + Custom, + + /// This operation is completely unsupported on the target. A programming + /// error has occurred. + Unsupported, + + /// Sentinel value for when no action was found in the specified table. + NotFound, + + /// Fall back onto the old rules. + /// TODO: Remove this once we've migrated + UseLegacyRules, +}; +} // end namespace LegalizeActions + +using LegalizeActions::LegalizeAction; /// Legalization is decided based on an instruction's opcode, which type slot /// we're considering, and what the existing type is. These aspects are gathered @@ -51,64 +112,642 @@ struct InstrAspect { } }; -class LegalizerInfo { -public: - enum LegalizeAction : std::uint8_t { - /// The operation is expected to be selectable directly by the target, and - /// no transformation is necessary. - Legal, - - /// The operation should be synthesized from multiple instructions acting on - /// a narrower scalar base-type. For example a 64-bit add might be - /// implemented in terms of 32-bit add-with-carry. - NarrowScalar, - - /// The operation should be implemented in terms of a wider scalar - /// base-type. For example a <2 x s8> add could be implemented as a <2 - /// x s32> add (ignoring the high bits). - WidenScalar, - - /// The (vector) operation should be implemented by splitting it into - /// sub-vectors where the operation is legal. For example a <8 x s64> add - /// might be implemented as 4 separate <2 x s64> adds. - FewerElements, - - /// The (vector) operation should be implemented by widening the input - /// vector and ignoring the lanes added by doing so. For example <2 x i8> is - /// rarely legal, but you might perform an <8 x i8> and then only look at - /// the first two results. - MoreElements, - - /// The operation itself must be expressed in terms of simpler actions on - /// this target. E.g. a SREM replaced by an SDIV and subtraction. - Lower, - - /// The operation should be implemented as a call to some kind of runtime - /// support library. For example this usually happens on machines that don't - /// support floating-point operations natively. - Libcall, - - /// The target wants to do something special with this combination of - /// operand and type. A callback will be issued when it is needed. - Custom, - - /// This operation is completely unsupported on the target. A programming - /// error has occurred. - Unsupported, - - /// Sentinel value for when no action was found in the specified table. - NotFound, +/// The LegalityQuery object bundles together all the information that's needed +/// to decide whether a given operation is legal or not. +/// For efficiency, it doesn't make a copy of Types so care must be taken not +/// to free it before using the query. +struct LegalityQuery { + unsigned Opcode; + ArrayRef<LLT> Types; + + struct MemDesc { + uint64_t Size; + AtomicOrdering Ordering; }; + /// Operations which require memory can use this to place requirements on the + /// memory type for each MMO. + ArrayRef<MemDesc> MMODescrs; + + constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types, + const ArrayRef<MemDesc> MMODescrs) + : Opcode(Opcode), Types(Types), MMODescrs(MMODescrs) {} + constexpr LegalityQuery(unsigned Opcode, const ArrayRef<LLT> Types) + : LegalityQuery(Opcode, Types, {}) {} + + raw_ostream &print(raw_ostream &OS) const; +}; + +/// The result of a query. It either indicates a final answer of Legal or +/// Unsupported or describes an action that must be taken to make an operation +/// more legal. +struct LegalizeActionStep { + /// The action to take or the final answer. + LegalizeAction Action; + /// If describing an action, the type index to change. Otherwise zero. + unsigned TypeIdx; + /// If describing an action, the new type for TypeIdx. Otherwise LLT{}. + LLT NewType; + + LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx, + const LLT &NewType) + : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {} + + bool operator==(const LegalizeActionStep &RHS) const { + return std::tie(Action, TypeIdx, NewType) == + std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType); + } +}; + +using LegalityPredicate = std::function<bool (const LegalityQuery &)>; +using LegalizeMutation = + std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>; + +namespace LegalityPredicates { +struct TypePairAndMemSize { + LLT Type0; + LLT Type1; + uint64_t MemSize; + + bool operator==(const TypePairAndMemSize &Other) const { + return Type0 == Other.Type0 && Type1 == Other.Type1 && + MemSize == Other.MemSize; + } +}; + +/// True iff P0 and P1 are true. +template<typename Predicate> +Predicate all(Predicate P0, Predicate P1) { + return [=](const LegalityQuery &Query) { + return P0(Query) && P1(Query); + }; +} +/// True iff all given predicates are true. +template<typename Predicate, typename... Args> +Predicate all(Predicate P0, Predicate P1, Args... args) { + return all(all(P0, P1), args...); +} +/// True iff the given type index is the specified types. +LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit); +/// True iff the given type index is one of the specified types. +LegalityPredicate typeInSet(unsigned TypeIdx, + std::initializer_list<LLT> TypesInit); +/// True iff the given types for the given pair of type indexes is one of the +/// specified type pairs. +LegalityPredicate +typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, + std::initializer_list<std::pair<LLT, LLT>> TypesInit); +/// True iff the given types for the given pair of type indexes is one of the +/// specified type pairs. +LegalityPredicate typePairAndMemSizeInSet( + unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, + std::initializer_list<TypePairAndMemSize> TypesAndMemSizeInit); +/// True iff the specified type index is a scalar. +LegalityPredicate isScalar(unsigned TypeIdx); +/// True iff the specified type index is a scalar that's narrower than the given +/// size. +LegalityPredicate narrowerThan(unsigned TypeIdx, unsigned Size); +/// True iff the specified type index is a scalar that's wider than the given +/// size. +LegalityPredicate widerThan(unsigned TypeIdx, unsigned Size); +/// True iff the specified type index is a scalar whose size is not a power of +/// 2. +LegalityPredicate sizeNotPow2(unsigned TypeIdx); +/// True iff the specified MMO index has a size that is not a power of 2 +LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx); +/// True iff the specified type index is a vector whose element count is not a +/// power of 2. +LegalityPredicate numElementsNotPow2(unsigned TypeIdx); +/// True iff the specified MMO index has at an atomic ordering of at Ordering or +/// stronger. +LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, + AtomicOrdering Ordering); +} // end namespace LegalityPredicates + +namespace LegalizeMutations { +/// Select this specific type for the given type index. +LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty); +/// Keep the same type as the given type index. +LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx); +/// Widen the type for the given type index to the next power of 2. +LegalizeMutation widenScalarToNextPow2(unsigned TypeIdx, unsigned Min = 0); +/// Add more elements to the type for the given type index to the next power of +/// 2. +LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0); +} // end namespace LegalizeMutations + +/// A single rule in a legalizer info ruleset. +/// The specified action is chosen when the predicate is true. Where appropriate +/// for the action (e.g. for WidenScalar) the new type is selected using the +/// given mutator. +class LegalizeRule { + LegalityPredicate Predicate; + LegalizeAction Action; + LegalizeMutation Mutation; + +public: + LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action, + LegalizeMutation Mutation = nullptr) + : Predicate(Predicate), Action(Action), Mutation(Mutation) {} + + /// Test whether the LegalityQuery matches. + bool match(const LegalityQuery &Query) const { + return Predicate(Query); + } + + LegalizeAction getAction() const { return Action; } + + /// Determine the change to make. + std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const { + if (Mutation) + return Mutation(Query); + return std::make_pair(0, LLT{}); + } +}; + +class LegalizeRuleSet { + /// When non-zero, the opcode we are an alias of + unsigned AliasOf; + /// If true, there is another opcode that aliases this one + bool IsAliasedByAnother; + SmallVector<LegalizeRule, 2> Rules; + +#ifndef NDEBUG + /// If bit I is set, this rule set contains a rule that may handle (predicate + /// or perform an action upon (or both)) the type index I. The uncertainty + /// comes from free-form rules executing user-provided lambda functions. We + /// conservatively assume such rules do the right thing and cover all type + /// indices. The bitset is intentionally 1 bit wider than it absolutely needs + /// to be to distinguish such cases from the cases where all type indices are + /// individually handled. + SmallBitVector TypeIdxsCovered{MCOI::OPERAND_LAST_GENERIC - + MCOI::OPERAND_FIRST_GENERIC + 2}; +#endif + + unsigned typeIdx(unsigned TypeIdx) { + assert(TypeIdx <= + (MCOI::OPERAND_LAST_GENERIC - MCOI::OPERAND_FIRST_GENERIC) && + "Type Index is out of bounds"); +#ifndef NDEBUG + TypeIdxsCovered.set(TypeIdx); +#endif + return TypeIdx; + } + void markAllTypeIdxsAsCovered() { +#ifndef NDEBUG + TypeIdxsCovered.set(); +#endif + } + + void add(const LegalizeRule &Rule) { + assert(AliasOf == 0 && + "RuleSet is aliased, change the representative opcode instead"); + Rules.push_back(Rule); + } + + static bool always(const LegalityQuery &) { return true; } + + /// Use the given action when the predicate is true. + /// Action should not be an action that requires mutation. + LegalizeRuleSet &actionIf(LegalizeAction Action, + LegalityPredicate Predicate) { + add({Predicate, Action}); + return *this; + } + /// Use the given action when the predicate is true. + /// Action should be an action that requires mutation. + LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate, + LegalizeMutation Mutation) { + add({Predicate, Action, Mutation}); + return *this; + } + /// Use the given action when type index 0 is any type in the given list. + /// Action should not be an action that requires mutation. + LegalizeRuleSet &actionFor(LegalizeAction Action, + std::initializer_list<LLT> Types) { + using namespace LegalityPredicates; + return actionIf(Action, typeInSet(typeIdx(0), Types)); + } + /// Use the given action when type index 0 is any type in the given list. + /// Action should be an action that requires mutation. + LegalizeRuleSet &actionFor(LegalizeAction Action, + std::initializer_list<LLT> Types, + LegalizeMutation Mutation) { + using namespace LegalityPredicates; + return actionIf(Action, typeInSet(typeIdx(0), Types), Mutation); + } + /// Use the given action when type indexes 0 and 1 is any type pair in the + /// given list. + /// Action should not be an action that requires mutation. + LegalizeRuleSet &actionFor(LegalizeAction Action, + std::initializer_list<std::pair<LLT, LLT>> Types) { + using namespace LegalityPredicates; + return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types)); + } + /// Use the given action when type indexes 0 and 1 is any type pair in the + /// given list. + /// Action should be an action that requires mutation. + LegalizeRuleSet &actionFor(LegalizeAction Action, + std::initializer_list<std::pair<LLT, LLT>> Types, + LegalizeMutation Mutation) { + using namespace LegalityPredicates; + return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types), + Mutation); + } + /// Use the given action when type indexes 0 and 1 are both in the given list. + /// That is, the type pair is in the cartesian product of the list. + /// Action should not be an action that requires mutation. + LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action, + std::initializer_list<LLT> Types) { + using namespace LegalityPredicates; + return actionIf(Action, all(typeInSet(typeIdx(0), Types), + typeInSet(typeIdx(1), Types))); + } + /// Use the given action when type indexes 0 and 1 are both in their + /// respective lists. + /// That is, the type pair is in the cartesian product of the lists + /// Action should not be an action that requires mutation. + LegalizeRuleSet & + actionForCartesianProduct(LegalizeAction Action, + std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1) { + using namespace LegalityPredicates; + return actionIf(Action, all(typeInSet(typeIdx(0), Types0), + typeInSet(typeIdx(1), Types1))); + } + /// Use the given action when type indexes 0, 1, and 2 are all in their + /// respective lists. + /// That is, the type triple is in the cartesian product of the lists + /// Action should not be an action that requires mutation. + LegalizeRuleSet &actionForCartesianProduct( + LegalizeAction Action, std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1, std::initializer_list<LLT> Types2) { + using namespace LegalityPredicates; + return actionIf(Action, all(typeInSet(typeIdx(0), Types0), + all(typeInSet(typeIdx(1), Types1), + typeInSet(typeIdx(2), Types2)))); + } + +public: + LegalizeRuleSet() : AliasOf(0), IsAliasedByAnother(false), Rules() {} + + bool isAliasedByAnother() { return IsAliasedByAnother; } + void setIsAliasedByAnother() { IsAliasedByAnother = true; } + void aliasTo(unsigned Opcode) { + assert((AliasOf == 0 || AliasOf == Opcode) && + "Opcode is already aliased to another opcode"); + assert(Rules.empty() && "Aliasing will discard rules"); + AliasOf = Opcode; + } + unsigned getAlias() const { return AliasOf; } + + /// The instruction is legal if predicate is true. + LegalizeRuleSet &legalIf(LegalityPredicate Predicate) { + // We have no choice but conservatively assume that the free-form + // user-provided Predicate properly handles all type indices: + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::Legal, Predicate); + } + /// The instruction is legal when type index 0 is any type in the given list. + LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) { + return actionFor(LegalizeAction::Legal, Types); + } + /// The instruction is legal when type indexes 0 and 1 is any type pair in the + /// given list. + LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) { + return actionFor(LegalizeAction::Legal, Types); + } + /// The instruction is legal when type indexes 0 and 1 along with the memory + /// size is any type and size tuple in the given list. + LegalizeRuleSet &legalForTypesWithMemSize( + std::initializer_list<LegalityPredicates::TypePairAndMemSize> + TypesAndMemSize) { + return actionIf(LegalizeAction::Legal, + LegalityPredicates::typePairAndMemSizeInSet( + typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemSize)); + } + /// The instruction is legal when type indexes 0 and 1 are both in the given + /// list. That is, the type pair is in the cartesian product of the list. + LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) { + return actionForCartesianProduct(LegalizeAction::Legal, Types); + } + /// The instruction is legal when type indexes 0 and 1 are both their + /// respective lists. + LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1) { + return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1); + } + + /// The instruction is lowered. + LegalizeRuleSet &lower() { + using namespace LegalizeMutations; + // We have no choice but conservatively assume that predicate-less lowering + // properly handles all type indices by design: + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::Lower, always); + } + /// The instruction is lowered if predicate is true. Keep type index 0 as the + /// same type. + LegalizeRuleSet &lowerIf(LegalityPredicate Predicate) { + using namespace LegalizeMutations; + // We have no choice but conservatively assume that lowering with a + // free-form user provided Predicate properly handles all type indices: + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::Lower, Predicate); + } + /// The instruction is lowered if predicate is true. + LegalizeRuleSet &lowerIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + // We have no choice but conservatively assume that lowering with a + // free-form user provided Predicate properly handles all type indices: + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::Lower, Predicate, Mutation); + } + /// The instruction is lowered when type index 0 is any type in the given + /// list. Keep type index 0 as the same type. + LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types) { + return actionFor(LegalizeAction::Lower, Types, + LegalizeMutations::changeTo(0, 0)); + } + /// The instruction is lowered when type index 0 is any type in the given + /// list. + LegalizeRuleSet &lowerFor(std::initializer_list<LLT> Types, + LegalizeMutation Mutation) { + return actionFor(LegalizeAction::Lower, Types, Mutation); + } + /// The instruction is lowered when type indexes 0 and 1 is any type pair in + /// the given list. Keep type index 0 as the same type. + LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types) { + return actionFor(LegalizeAction::Lower, Types, + LegalizeMutations::changeTo(0, 0)); + } + /// The instruction is lowered when type indexes 0 and 1 is any type pair in + /// the given list. + LegalizeRuleSet &lowerFor(std::initializer_list<std::pair<LLT, LLT>> Types, + LegalizeMutation Mutation) { + return actionFor(LegalizeAction::Lower, Types, Mutation); + } + /// The instruction is lowered when type indexes 0 and 1 are both in their + /// respective lists. + LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1) { + using namespace LegalityPredicates; + return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1); + } + /// The instruction is lowered when when type indexes 0, 1, and 2 are all in + /// their respective lists. + LegalizeRuleSet &lowerForCartesianProduct(std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1, + std::initializer_list<LLT> Types2) { + using namespace LegalityPredicates; + return actionForCartesianProduct(LegalizeAction::Lower, Types0, Types1, + Types2); + } + + /// Like legalIf, but for the Libcall action. + LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) { + // We have no choice but conservatively assume that a libcall with a + // free-form user provided Predicate properly handles all type indices: + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::Libcall, Predicate); + } + LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) { + return actionFor(LegalizeAction::Libcall, Types); + } + LegalizeRuleSet & + libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) { + return actionFor(LegalizeAction::Libcall, Types); + } + LegalizeRuleSet & + libcallForCartesianProduct(std::initializer_list<LLT> Types) { + return actionForCartesianProduct(LegalizeAction::Libcall, Types); + } + LegalizeRuleSet & + libcallForCartesianProduct(std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1) { + return actionForCartesianProduct(LegalizeAction::Libcall, Types0, Types1); + } + + /// Widen the scalar to the one selected by the mutation if the predicate is + /// true. + LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + // We have no choice but conservatively assume that an action with a + // free-form user provided Predicate properly handles all type indices: + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::WidenScalar, Predicate, Mutation); + } + /// Narrow the scalar to the one selected by the mutation if the predicate is + /// true. + LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + // We have no choice but conservatively assume that an action with a + // free-form user provided Predicate properly handles all type indices: + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation); + } + + /// Add more elements to reach the type selected by the mutation if the + /// predicate is true. + LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + // We have no choice but conservatively assume that an action with a + // free-form user provided Predicate properly handles all type indices: + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::MoreElements, Predicate, Mutation); + } + /// Remove elements to reach the type selected by the mutation if the + /// predicate is true. + LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + // We have no choice but conservatively assume that an action with a + // free-form user provided Predicate properly handles all type indices: + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::FewerElements, Predicate, Mutation); + } + + /// The instruction is unsupported. + LegalizeRuleSet &unsupported() { + return actionIf(LegalizeAction::Unsupported, always); + } + LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) { + return actionIf(LegalizeAction::Unsupported, Predicate); + } + LegalizeRuleSet &unsupportedIfMemSizeNotPow2() { + return actionIf(LegalizeAction::Unsupported, + LegalityPredicates::memSizeInBytesNotPow2(0)); + } + + LegalizeRuleSet &customIf(LegalityPredicate Predicate) { + // We have no choice but conservatively assume that a custom action with a + // free-form user provided Predicate properly handles all type indices: + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::Custom, Predicate); + } + LegalizeRuleSet &customFor(std::initializer_list<LLT> Types) { + return actionFor(LegalizeAction::Custom, Types); + } + LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) { + return actionForCartesianProduct(LegalizeAction::Custom, Types); + } + LegalizeRuleSet & + customForCartesianProduct(std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1) { + return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1); + } + + /// Widen the scalar to the next power of two that is at least MinSize. + /// No effect if the type is not a scalar or is a power of two. + LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, + unsigned MinSize = 0) { + using namespace LegalityPredicates; + return actionIf(LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), + LegalizeMutations::widenScalarToNextPow2(TypeIdx, MinSize)); + } + + LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) { + using namespace LegalityPredicates; + return actionIf(LegalizeAction::NarrowScalar, isScalar(typeIdx(TypeIdx)), + Mutation); + } + + /// Ensure the scalar is at least as wide as Ty. + LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::WidenScalar, + narrowerThan(TypeIdx, Ty.getSizeInBits()), + changeTo(typeIdx(TypeIdx), Ty)); + } + + /// Ensure the scalar is at most as wide as Ty. + LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::NarrowScalar, + widerThan(TypeIdx, Ty.getSizeInBits()), + changeTo(typeIdx(TypeIdx), Ty)); + } + + /// Conditionally limit the maximum size of the scalar. + /// For example, when the maximum size of one type depends on the size of + /// another such as extracting N bits from an M bit container. + LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, + const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::NarrowScalar, + [=](const LegalityQuery &Query) { + return widerThan(TypeIdx, Ty.getSizeInBits()) && + Predicate(Query); + }, + changeTo(typeIdx(TypeIdx), Ty)); + } + + /// Limit the range of scalar sizes to MinTy and MaxTy. + LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT &MinTy, + const LLT &MaxTy) { + assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types"); + return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy); + } + + /// Add more elements to the vector to reach the next power of two. + /// No effect if the type is not a vector or the element count is a power of + /// two. + LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) { + using namespace LegalityPredicates; + return actionIf(LegalizeAction::MoreElements, + numElementsNotPow2(typeIdx(TypeIdx)), + LegalizeMutations::moreElementsToNextPow2(TypeIdx)); + } + + /// Limit the number of elements in EltTy vectors to at least MinElements. + LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT &EltTy, + unsigned MinElements) { + // Mark the type index as covered: + typeIdx(TypeIdx); + return actionIf( + LegalizeAction::MoreElements, + [=](const LegalityQuery &Query) { + LLT VecTy = Query.Types[TypeIdx]; + return VecTy.isVector() && VecTy.getElementType() == EltTy && + VecTy.getNumElements() < MinElements; + }, + [=](const LegalityQuery &Query) { + LLT VecTy = Query.Types[TypeIdx]; + return std::make_pair( + TypeIdx, LLT::vector(MinElements, VecTy.getScalarSizeInBits())); + }); + } + /// Limit the number of elements in EltTy vectors to at most MaxElements. + LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT &EltTy, + unsigned MaxElements) { + // Mark the type index as covered: + typeIdx(TypeIdx); + return actionIf( + LegalizeAction::FewerElements, + [=](const LegalityQuery &Query) { + LLT VecTy = Query.Types[TypeIdx]; + return VecTy.isVector() && VecTy.getElementType() == EltTy && + VecTy.getNumElements() > MaxElements; + }, + [=](const LegalityQuery &Query) { + LLT VecTy = Query.Types[TypeIdx]; + return std::make_pair( + TypeIdx, LLT::vector(MaxElements, VecTy.getScalarSizeInBits())); + }); + } + /// Limit the number of elements for the given vectors to at least MinTy's + /// number of elements and at most MaxTy's number of elements. + /// + /// No effect if the type is not a vector or does not have the same element + /// type as the constraints. + /// The element type of MinTy and MaxTy must match. + LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT &MinTy, + const LLT &MaxTy) { + assert(MinTy.getElementType() == MaxTy.getElementType() && + "Expected element types to agree"); + + const LLT &EltTy = MinTy.getElementType(); + return clampMinNumElements(TypeIdx, EltTy, MinTy.getNumElements()) + .clampMaxNumElements(TypeIdx, EltTy, MaxTy.getNumElements()); + } + + /// Fallback on the previous implementation. This should only be used while + /// porting a rule. + LegalizeRuleSet &fallback() { + add({always, LegalizeAction::UseLegacyRules}); + return *this; + } + + /// Check if there is no type index which is obviously not handled by the + /// LegalizeRuleSet in any way at all. + /// \pre Type indices of the opcode form a dense [0, \p NumTypeIdxs) set. + bool verifyTypeIdxsCoverage(unsigned NumTypeIdxs) const; + + /// Apply the ruleset to the given LegalityQuery. + LegalizeActionStep apply(const LegalityQuery &Query) const; +}; + +class LegalizerInfo { +public: LegalizerInfo(); virtual ~LegalizerInfo() = default; + unsigned getOpcodeIdxForOpcode(unsigned Opcode) const; + unsigned getActionDefinitionsIdx(unsigned Opcode) const; + /// Compute any ancillary tables needed to quickly decide how an operation /// should be handled. This must be called after all "set*Action"methods but /// before any query is made or incorrect results may be returned. void computeTables(); + /// Perform simple self-diagnostic and assert if there is anything obviously + /// wrong with the actions set up. + void verify(const MCInstrInfo &MII) const; + static bool needsLegalizingToDifferentSize(const LegalizeAction Action) { + using namespace LegalizeActions; switch (Action) { case NarrowScalar: case WidenScalar: @@ -121,8 +760,8 @@ public: } } - typedef std::pair<uint16_t, LegalizeAction> SizeAndAction; - typedef std::vector<SizeAndAction> SizeAndActionsVec; + using SizeAndAction = std::pair<uint16_t, LegalizeAction>; + using SizeAndActionsVec = std::vector<SizeAndAction>; using SizeChangeStrategy = std::function<SizeAndActionsVec(const SizeAndActionsVec &v)>; @@ -186,8 +825,9 @@ public: /// and Unsupported for all other scalar types T. static SizeAndActionsVec unsupportedForDifferentSizes(const SizeAndActionsVec &v) { + using namespace LegalizeActions; return increaseToLargerTypesAndDecreaseToLargest(v, Unsupported, - Unsupported); + Unsupported); } /// A SizeChangeStrategy for the common case where legalization for a @@ -196,32 +836,36 @@ public: /// largest legal type. static SizeAndActionsVec widenToLargerTypesAndNarrowToLargest(const SizeAndActionsVec &v) { + using namespace LegalizeActions; assert(v.size() > 0 && "At least one size that can be legalized towards is needed" " for this SizeChangeStrategy"); return increaseToLargerTypesAndDecreaseToLargest(v, WidenScalar, - NarrowScalar); + NarrowScalar); } static SizeAndActionsVec widenToLargerTypesUnsupportedOtherwise(const SizeAndActionsVec &v) { + using namespace LegalizeActions; return increaseToLargerTypesAndDecreaseToLargest(v, WidenScalar, - Unsupported); + Unsupported); } static SizeAndActionsVec narrowToSmallerAndUnsupportedIfTooSmall(const SizeAndActionsVec &v) { + using namespace LegalizeActions; return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar, - Unsupported); + Unsupported); } static SizeAndActionsVec narrowToSmallerAndWidenToSmallest(const SizeAndActionsVec &v) { + using namespace LegalizeActions; assert(v.size() > 0 && "At least one size that can be legalized towards is needed" " for this SizeChangeStrategy"); return decreaseToSmallerTypesAndIncreaseToSmallest(v, NarrowScalar, - WidenScalar); + WidenScalar); } /// A SizeChangeStrategy for the common case where legalization for a @@ -244,8 +888,9 @@ public: /// (FewerElements, vector(4,32)). static SizeAndActionsVec moreToWiderTypesAndLessToWidest(const SizeAndActionsVec &v) { + using namespace LegalizeActions; return increaseToLargerTypesAndDecreaseToLargest(v, MoreElements, - FewerElements); + FewerElements); } /// Helper function to implement many typical SizeChangeStrategy functions. @@ -259,22 +904,46 @@ public: LegalizeAction DecreaseAction, LegalizeAction IncreaseAction); - /// Determine what action should be taken to legalize the given generic - /// instruction opcode, type-index and type. Requires computeTables to have - /// been called. + /// Get the action definitions for the given opcode. Use this to run a + /// LegalityQuery through the definitions. + const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const; + + /// Get the action definition builder for the given opcode. Use this to define + /// the action definitions. /// - /// \returns a pair consisting of the kind of legalization that should be - /// performed and the destination type. - std::pair<LegalizeAction, LLT> getAction(const InstrAspect &Aspect) const; + /// It is an error to request an opcode that has already been requested by the + /// multiple-opcode variant. + LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode); + + /// Get the action definition builder for the given set of opcodes. Use this + /// to define the action definitions for multiple opcodes at once. The first + /// opcode given will be considered the representative opcode and will hold + /// the definitions whereas the other opcodes will be configured to refer to + /// the representative opcode. This lowers memory requirements and very + /// slightly improves performance. + /// + /// It would be very easy to introduce unexpected side-effects as a result of + /// this aliasing if it were permitted to request different but intersecting + /// sets of opcodes but that is difficult to keep track of. It is therefore an + /// error to request the same opcode twice using this API, to request an + /// opcode that already has definitions, or to use the single-opcode API on an + /// opcode that has already been requested by this API. + LegalizeRuleSet & + getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes); + void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom); + + /// Determine what action should be taken to legalize the described + /// instruction. Requires computeTables to have been called. + /// + /// \returns a description of the next legalization step to perform. + LegalizeActionStep getAction(const LegalityQuery &Query) const; /// Determine what action should be taken to legalize the given generic /// instruction. /// - /// \returns a tuple consisting of the LegalizeAction that should be - /// performed, the type-index it should be performed on and the destination - /// type. - std::tuple<LegalizeAction, unsigned, LLT> - getAction(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; + /// \returns a description of the next legalization step to perform. + LegalizeActionStep getAction(const MachineInstr &MI, + const MachineRegisterInfo &MRI) const; bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; @@ -283,6 +952,15 @@ public: MachineIRBuilder &MIRBuilder) const; private: + /// Determine what action should be taken to legalize the given generic + /// instruction opcode, type-index and type. Requires computeTables to have + /// been called. + /// + /// \returns a pair consisting of the kind of legalization that should be + /// performed and the destination type. + std::pair<LegalizeAction, LLT> + getAspectAction(const InstrAspect &Aspect) const; + /// The SizeAndActionsVec is a representation mapping between all natural /// numbers and an Action. The natural number represents the bit size of /// the InstrAspect. For example, for a target with native support for 32-bit @@ -350,6 +1028,7 @@ private: /// A partial SizeAndActionsVec potentially doesn't cover all bit sizes, /// i.e. it's OK if it doesn't start from size 1. static void checkPartialSizeAndActionsVector(const SizeAndActionsVec& v) { + using namespace LegalizeActions; #ifndef NDEBUG // The sizes should be in increasing order int prev_size = -1; @@ -441,7 +1120,7 @@ private: static const int LastOp = TargetOpcode::PRE_ISEL_GENERIC_OPCODE_END; // Data structures used temporarily during construction of legality data: - typedef DenseMap<LLT, LegalizeAction> TypeMap; + using TypeMap = DenseMap<LLT, LegalizeAction>; SmallVector<TypeMap, 1> SpecifiedActions[LastOp - FirstOp + 1]; SmallVector<SizeChangeStrategy, 1> ScalarSizeChangeStrategies[LastOp - FirstOp + 1]; @@ -456,8 +1135,16 @@ private: AddrSpace2PointerActions[LastOp - FirstOp + 1]; std::unordered_map<uint16_t, SmallVector<SizeAndActionsVec, 1>> NumElements2Actions[LastOp - FirstOp + 1]; + + LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1]; }; +#ifndef NDEBUG +/// Checks that MIR is fully legal, returns an illegal instruction if it's not, +/// nullptr otherwise +const MachineInstr *machineFunctionIsIllegal(const MachineFunction &MF); +#endif + } // end namespace llvm. #endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZERINFO_H diff --git a/include/llvm/CodeGen/GlobalISel/Localizer.h b/include/llvm/CodeGen/GlobalISel/Localizer.h index 0a46eb9e7840..1e2d4763e5e1 100644 --- a/include/llvm/CodeGen/GlobalISel/Localizer.h +++ b/include/llvm/CodeGen/GlobalISel/Localizer.h @@ -70,6 +70,8 @@ public: .set(MachineFunctionProperties::Property::RegBankSelected); } + void getAnalysisUsage(AnalysisUsage &AU) const override; + bool runOnMachineFunction(MachineFunction &MF) override; }; diff --git a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h new file mode 100644 index 000000000000..f77f9a8df7ee --- /dev/null +++ b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h @@ -0,0 +1,338 @@ +//== ----- llvm/CodeGen/GlobalISel/MIPatternMatch.h --------------------- == // +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// Contains matchers for matching SSA Machine Instructions. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_GMIR_PATTERNMATCH_H +#define LLVM_GMIR_PATTERNMATCH_H + +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/APInt.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" + +namespace llvm { +namespace MIPatternMatch { + +template <typename Reg, typename Pattern> +bool mi_match(Reg R, MachineRegisterInfo &MRI, Pattern &&P) { + return P.match(MRI, R); +} + +// TODO: Extend for N use. +template <typename SubPatternT> struct OneUse_match { + SubPatternT SubPat; + OneUse_match(const SubPatternT &SP) : SubPat(SP) {} + + template <typename OpTy> + bool match(const MachineRegisterInfo &MRI, unsigned Reg) { + return MRI.hasOneUse(Reg) && SubPat.match(MRI, Reg); + } +}; + +template <typename SubPat> +inline OneUse_match<SubPat> m_OneUse(const SubPat &SP) { + return SP; +} + +struct ConstantMatch { + int64_t &CR; + ConstantMatch(int64_t &C) : CR(C) {} + bool match(const MachineRegisterInfo &MRI, unsigned Reg) { + if (auto MaybeCst = getConstantVRegVal(Reg, MRI)) { + CR = *MaybeCst; + return true; + } + return false; + } +}; + +inline ConstantMatch m_ICst(int64_t &Cst) { return ConstantMatch(Cst); } + +// TODO: Rework this for different kinds of MachineOperand. +// Currently assumes the Src for a match is a register. +// We might want to support taking in some MachineOperands and call getReg on +// that. + +struct operand_type_match { + bool match(const MachineRegisterInfo &MRI, unsigned Reg) { return true; } + bool match(const MachineRegisterInfo &MRI, MachineOperand *MO) { + return MO->isReg(); + } +}; + +inline operand_type_match m_Reg() { return operand_type_match(); } + +/// Matching combinators. +template <typename... Preds> struct And { + template <typename MatchSrc> + bool match(MachineRegisterInfo &MRI, MatchSrc &&src) { + return true; + } +}; + +template <typename Pred, typename... Preds> +struct And<Pred, Preds...> : And<Preds...> { + Pred P; + And(Pred &&p, Preds &&... preds) + : And<Preds...>(std::forward<Preds>(preds)...), P(std::forward<Pred>(p)) { + } + template <typename MatchSrc> + bool match(MachineRegisterInfo &MRI, MatchSrc &&src) { + return P.match(MRI, src) && And<Preds...>::match(MRI, src); + } +}; + +template <typename... Preds> struct Or { + template <typename MatchSrc> + bool match(MachineRegisterInfo &MRI, MatchSrc &&src) { + return false; + } +}; + +template <typename Pred, typename... Preds> +struct Or<Pred, Preds...> : Or<Preds...> { + Pred P; + Or(Pred &&p, Preds &&... preds) + : Or<Preds...>(std::forward<Preds>(preds)...), P(std::forward<Pred>(p)) {} + template <typename MatchSrc> + bool match(MachineRegisterInfo &MRI, MatchSrc &&src) { + return P.match(MRI, src) || Or<Preds...>::match(MRI, src); + } +}; + +template <typename... Preds> And<Preds...> m_all_of(Preds &&... preds) { + return And<Preds...>(std::forward<Preds>(preds)...); +} + +template <typename... Preds> Or<Preds...> m_any_of(Preds &&... preds) { + return Or<Preds...>(std::forward<Preds>(preds)...); +} + +template <typename BindTy> struct bind_helper { + static bool bind(const MachineRegisterInfo &MRI, BindTy &VR, BindTy &V) { + VR = V; + return true; + } +}; + +template <> struct bind_helper<MachineInstr *> { + static bool bind(const MachineRegisterInfo &MRI, MachineInstr *&MI, + unsigned Reg) { + MI = MRI.getVRegDef(Reg); + if (MI) + return true; + return false; + } +}; + +template <> struct bind_helper<LLT> { + static bool bind(const MachineRegisterInfo &MRI, LLT &Ty, unsigned Reg) { + Ty = MRI.getType(Reg); + if (Ty.isValid()) + return true; + return false; + } +}; + +template <> struct bind_helper<const ConstantFP *> { + static bool bind(const MachineRegisterInfo &MRI, const ConstantFP *&F, + unsigned Reg) { + F = getConstantFPVRegVal(Reg, MRI); + if (F) + return true; + return false; + } +}; + +template <typename Class> struct bind_ty { + Class &VR; + + bind_ty(Class &V) : VR(V) {} + + template <typename ITy> bool match(const MachineRegisterInfo &MRI, ITy &&V) { + return bind_helper<Class>::bind(MRI, VR, V); + } +}; + +inline bind_ty<unsigned> m_Reg(unsigned &R) { return R; } +inline bind_ty<MachineInstr *> m_MInstr(MachineInstr *&MI) { return MI; } +inline bind_ty<LLT> m_Type(LLT &Ty) { return Ty; } + +// Helper for matching G_FCONSTANT +inline bind_ty<const ConstantFP *> m_GFCst(const ConstantFP *&C) { return C; } + +// General helper for all the binary generic MI such as G_ADD/G_SUB etc +template <typename LHS_P, typename RHS_P, unsigned Opcode, + bool Commutable = false> +struct BinaryOp_match { + LHS_P L; + RHS_P R; + + BinaryOp_match(const LHS_P &LHS, const RHS_P &RHS) : L(LHS), R(RHS) {} + template <typename OpTy> bool match(MachineRegisterInfo &MRI, OpTy &&Op) { + MachineInstr *TmpMI; + if (mi_match(Op, MRI, m_MInstr(TmpMI))) { + if (TmpMI->getOpcode() == Opcode && TmpMI->getNumOperands() == 3) { + return (L.match(MRI, TmpMI->getOperand(1).getReg()) && + R.match(MRI, TmpMI->getOperand(2).getReg())) || + (Commutable && (R.match(MRI, TmpMI->getOperand(1).getReg()) && + L.match(MRI, TmpMI->getOperand(2).getReg()))); + } + } + return false; + } +}; + +template <typename LHS, typename RHS> +inline BinaryOp_match<LHS, RHS, TargetOpcode::G_ADD, true> +m_GAdd(const LHS &L, const RHS &R) { + return BinaryOp_match<LHS, RHS, TargetOpcode::G_ADD, true>(L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOp_match<LHS, RHS, TargetOpcode::G_SUB> m_GSub(const LHS &L, + const RHS &R) { + return BinaryOp_match<LHS, RHS, TargetOpcode::G_SUB>(L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOp_match<LHS, RHS, TargetOpcode::G_MUL, true> +m_GMul(const LHS &L, const RHS &R) { + return BinaryOp_match<LHS, RHS, TargetOpcode::G_MUL, true>(L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOp_match<LHS, RHS, TargetOpcode::G_FADD, true> +m_GFAdd(const LHS &L, const RHS &R) { + return BinaryOp_match<LHS, RHS, TargetOpcode::G_FADD, true>(L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOp_match<LHS, RHS, TargetOpcode::G_FMUL, true> +m_GFMul(const LHS &L, const RHS &R) { + return BinaryOp_match<LHS, RHS, TargetOpcode::G_FMUL, true>(L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOp_match<LHS, RHS, TargetOpcode::G_FSUB, false> +m_GFSub(const LHS &L, const RHS &R) { + return BinaryOp_match<LHS, RHS, TargetOpcode::G_FSUB, false>(L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOp_match<LHS, RHS, TargetOpcode::G_AND, true> +m_GAnd(const LHS &L, const RHS &R) { + return BinaryOp_match<LHS, RHS, TargetOpcode::G_AND, true>(L, R); +} + +template <typename LHS, typename RHS> +inline BinaryOp_match<LHS, RHS, TargetOpcode::G_OR, true> m_GOr(const LHS &L, + const RHS &R) { + return BinaryOp_match<LHS, RHS, TargetOpcode::G_OR, true>(L, R); +} + +// Helper for unary instructions (G_[ZSA]EXT/G_TRUNC) etc +template <typename SrcTy, unsigned Opcode> struct UnaryOp_match { + SrcTy L; + + UnaryOp_match(const SrcTy &LHS) : L(LHS) {} + template <typename OpTy> bool match(MachineRegisterInfo &MRI, OpTy &&Op) { + MachineInstr *TmpMI; + if (mi_match(Op, MRI, m_MInstr(TmpMI))) { + if (TmpMI->getOpcode() == Opcode && TmpMI->getNumOperands() == 2) { + return L.match(MRI, TmpMI->getOperand(1).getReg()); + } + } + return false; + } +}; + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_ANYEXT> +m_GAnyExt(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_ANYEXT>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_SEXT> m_GSExt(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_SEXT>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_ZEXT> m_GZExt(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_ZEXT>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_FPEXT> m_GFPExt(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_FPEXT>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_TRUNC> m_GTrunc(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_TRUNC>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_BITCAST> +m_GBitcast(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_BITCAST>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_PTRTOINT> +m_GPtrToInt(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_PTRTOINT>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_INTTOPTR> +m_GIntToPtr(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_INTTOPTR>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_FPTRUNC> +m_GFPTrunc(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_FPTRUNC>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_FABS> m_GFabs(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_FABS>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::G_FNEG> m_GFNeg(const SrcTy &Src) { + return UnaryOp_match<SrcTy, TargetOpcode::G_FNEG>(Src); +} + +template <typename SrcTy> +inline UnaryOp_match<SrcTy, TargetOpcode::COPY> m_Copy(SrcTy &&Src) { + return UnaryOp_match<SrcTy, TargetOpcode::COPY>(std::forward<SrcTy>(Src)); +} + +// Helper for checking if a Reg is of specific type. +struct CheckType { + LLT Ty; + CheckType(const LLT &Ty) : Ty(Ty) {} + + bool match(MachineRegisterInfo &MRI, unsigned Reg) { + return MRI.getType(Reg) == Ty; + } +}; + +inline CheckType m_SpecificType(LLT Ty) { return Ty; } + +} // namespace GMIPatternMatch +} // namespace llvm + +#endif diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index aa875c11d86f..983a4e680d5c 100644 --- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -23,7 +23,6 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/DebugLoc.h" -#include <queue> namespace llvm { @@ -32,11 +31,10 @@ class MachineFunction; class MachineInstr; class TargetInstrInfo; -/// Helper class to build MachineInstr. -/// It keeps internally the insertion point and debug location for all -/// the new instructions we want to create. -/// This information can be modify via the related setters. -class MachineIRBuilder { +/// Class which stores all the state required in a MachineIRBuilder. +/// Since MachineIRBuilders will only store state in this object, it allows +/// to transfer BuilderState between different kinds of MachineIRBuilders. +struct MachineIRBuilderState { /// MachineFunction under construction. MachineFunction *MF; /// Information used to access the description of the opcodes. @@ -53,15 +51,23 @@ class MachineIRBuilder { /// @} std::function<void(MachineInstr *)> InsertedInstr; +}; + +/// Helper class to build MachineInstr. +/// It keeps internally the insertion point and debug location for all +/// the new instructions we want to create. +/// This information can be modify via the related setters. +class MachineIRBuilderBase { + MachineIRBuilderState State; const TargetInstrInfo &getTII() { - assert(TII && "TargetInstrInfo is not set"); - return *TII; + assert(State.TII && "TargetInstrInfo is not set"); + return *State.TII; } void validateTruncExt(unsigned Dst, unsigned Src, bool IsExtend); - MachineInstrBuilder buildBinaryOp(unsigned Opcode, unsigned Res, unsigned Op0, unsigned Op1); +protected: unsigned getDestFromArg(unsigned Reg) { return Reg; } unsigned getDestFromArg(LLT Ty) { return getMF().getRegInfo().createGenericVirtualRegister(Ty); @@ -89,30 +95,41 @@ class MachineIRBuilder { return MIB->getOperand(0).getReg(); } + void validateBinaryOp(unsigned Res, unsigned Op0, unsigned Op1); + public: /// Some constructors for easy use. - MachineIRBuilder() = default; - MachineIRBuilder(MachineFunction &MF) { setMF(MF); } - MachineIRBuilder(MachineInstr &MI) : MachineIRBuilder(*MI.getMF()) { + MachineIRBuilderBase() = default; + MachineIRBuilderBase(MachineFunction &MF) { setMF(MF); } + MachineIRBuilderBase(MachineInstr &MI) : MachineIRBuilderBase(*MI.getMF()) { setInstr(MI); } + MachineIRBuilderBase(const MachineIRBuilderState &BState) : State(BState) {} + /// Getter for the function we currently build. MachineFunction &getMF() { - assert(MF && "MachineFunction is not set"); - return *MF; + assert(State.MF && "MachineFunction is not set"); + return *State.MF; } + /// Getter for DebugLoc + const DebugLoc &getDL() { return State.DL; } + + /// Getter for MRI + MachineRegisterInfo *getMRI() { return State.MRI; } + + /// Getter for the State + MachineIRBuilderState &getState() { return State; } + /// Getter for the basic block we currently build. MachineBasicBlock &getMBB() { - assert(MBB && "MachineBasicBlock is not set"); - return *MBB; + assert(State.MBB && "MachineBasicBlock is not set"); + return *State.MBB; } /// Current insertion point for new instructions. - MachineBasicBlock::iterator getInsertPt() { - return II; - } + MachineBasicBlock::iterator getInsertPt() { return State.II; } /// Set the insertion point before the specified position. /// \pre MBB must be in getMF(). @@ -137,15 +154,16 @@ public: /// \name Control where instructions we create are recorded (typically for /// visiting again later during legalization). /// @{ + void recordInsertion(MachineInstr *InsertedInstr) const; void recordInsertions(std::function<void(MachineInstr *)> InsertedInstr); void stopRecordingInsertions(); /// @} /// Set the debug location to \p DL for all the next build instructions. - void setDebugLoc(const DebugLoc &DL) { this->DL = DL; } + void setDebugLoc(const DebugLoc &DL) { this->State.DL = DL; } /// Get the current instruction's debug location. - DebugLoc getDebugLoc() { return DL; } + DebugLoc getDebugLoc() { return State.DL; } /// Build and insert <empty> = \p Opcode <empty>. /// The insertion point is the one set by the last call of either @@ -156,20 +174,6 @@ public: /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildInstr(unsigned Opcode); - /// DAG like Generic method for building arbitrary instructions as above. - /// \Opc opcode for the instruction. - /// \Ty Either LLT/TargetRegisterClass/unsigned types for Dst - /// \Args Variadic list of uses of types(unsigned/MachineInstrBuilder) - /// Uses of type MachineInstrBuilder will perform - /// getOperand(0).getReg() to convert to register. - template <typename DstTy, typename... UseArgsTy> - MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, - UseArgsTy &&... Args) { - auto MIB = buildInstr(Opc).addDef(getDestFromArg(Ty)); - addUsesFromArgs(MIB, std::forward<UseArgsTy>(Args)...); - return MIB; - } - /// Build but don't insert <empty> = \p Opcode <empty>. /// /// \pre setMF, setBasicBlock or setMI must have been called. @@ -227,49 +231,6 @@ public: /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildGlobalValue(unsigned Res, const GlobalValue *GV); - /// Build and insert \p Res = G_ADD \p Op0, \p Op1 - /// - /// G_ADD sets \p Res to the sum of integer parameters \p Op0 and \p Op1, - /// truncated to their width. - /// - /// \pre setBasicBlock or setMI must have been called. - /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers - /// with the same (scalar or vector) type). - /// - /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAdd(unsigned Res, unsigned Op0, - unsigned Op1); - template <typename DstTy, typename... UseArgsTy> - MachineInstrBuilder buildAdd(DstTy &&Ty, UseArgsTy &&... UseArgs) { - unsigned Res = getDestFromArg(Ty); - return buildAdd(Res, (getRegFromArg(UseArgs))...); - } - - /// Build and insert \p Res = G_SUB \p Op0, \p Op1 - /// - /// G_SUB sets \p Res to the sum of integer parameters \p Op0 and \p Op1, - /// truncated to their width. - /// - /// \pre setBasicBlock or setMI must have been called. - /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers - /// with the same (scalar or vector) type). - /// - /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildSub(unsigned Res, unsigned Op0, - unsigned Op1); - - /// Build and insert \p Res = G_MUL \p Op0, \p Op1 - /// - /// G_MUL sets \p Res to the sum of integer parameters \p Op0 and \p Op1, - /// truncated to their width. - /// - /// \pre setBasicBlock or setMI must have been called. - /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers - /// with the same (scalar or vector) type). - /// - /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildMul(unsigned Res, unsigned Op0, - unsigned Op1); /// Build and insert \p Res = G_GEP \p Op0, \p Op1 /// @@ -338,34 +299,6 @@ public: MachineInstrBuilder buildUAdde(unsigned Res, unsigned CarryOut, unsigned Op0, unsigned Op1, unsigned CarryIn); - /// Build and insert \p Res = G_AND \p Op0, \p Op1 - /// - /// G_AND sets \p Res to the bitwise and of integer parameters \p Op0 and \p - /// Op1. - /// - /// \pre setBasicBlock or setMI must have been called. - /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers - /// with the same (scalar or vector) type). - /// - /// \return a MachineInstrBuilder for the newly created instruction. - template <typename DstTy, typename... UseArgsTy> - MachineInstrBuilder buildAnd(DstTy &&Dst, UseArgsTy &&... UseArgs) { - return buildAnd(getDestFromArg(Dst), getRegFromArg(UseArgs)...); - } - MachineInstrBuilder buildAnd(unsigned Res, unsigned Op0, - unsigned Op1); - - /// Build and insert \p Res = G_OR \p Op0, \p Op1 - /// - /// G_OR sets \p Res to the bitwise or of integer parameters \p Op0 and \p - /// Op1. - /// - /// \pre setBasicBlock or setMI must have been called. - /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers - /// with the same (scalar or vector) type). - /// - /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildOr(unsigned Res, unsigned Op0, unsigned Op1); /// Build and insert \p Res = G_ANYEXT \p Op0 /// @@ -399,6 +332,10 @@ public: /// \pre \p Op must be smaller than \p Res /// /// \return The newly created instruction. + template <typename DstType, typename ArgType> + MachineInstrBuilder buildSExt(DstType &&Res, ArgType &&Arg) { + return buildSExt(getDestFromArg(Res), getRegFromArg(Arg)); + } MachineInstrBuilder buildSExt(unsigned Res, unsigned Op); /// Build and insert \p Res = G_ZEXT \p Op @@ -413,6 +350,10 @@ public: /// \pre \p Op must be smaller than \p Res /// /// \return The newly created instruction. + template <typename DstType, typename ArgType> + MachineInstrBuilder buildZExt(DstType &&Res, ArgType &&Arg) { + return buildZExt(getDestFromArg(Res), getRegFromArg(Arg)); + } MachineInstrBuilder buildZExt(unsigned Res, unsigned Op); /// Build and insert \p Res = G_SEXT \p Op, \p Res = G_TRUNC \p Op, or @@ -423,6 +364,10 @@ public: /// \pre \p Op must be a generic virtual register with scalar or vector type. /// /// \return The newly created instruction. + template <typename DstTy, typename UseArgTy> + MachineInstrBuilder buildSExtOrTrunc(DstTy &&Dst, UseArgTy &&Use) { + return buildSExtOrTrunc(getDestFromArg(Dst), getRegFromArg(Use)); + } MachineInstrBuilder buildSExtOrTrunc(unsigned Res, unsigned Op); /// Build and insert \p Res = G_ZEXT \p Op, \p Res = G_TRUNC \p Op, or @@ -433,6 +378,10 @@ public: /// \pre \p Op must be a generic virtual register with scalar or vector type. /// /// \return The newly created instruction. + template <typename DstTy, typename UseArgTy> + MachineInstrBuilder buildZExtOrTrunc(DstTy &&Dst, UseArgTy &&Use) { + return buildZExtOrTrunc(getDestFromArg(Dst), getRegFromArg(Use)); + } MachineInstrBuilder buildZExtOrTrunc(unsigned Res, unsigned Op); // Build and insert \p Res = G_ANYEXT \p Op, \p Res = G_TRUNC \p Op, or @@ -462,6 +411,10 @@ public: unsigned Op); /// Build and insert an appropriate cast between two registers of equal size. + template <typename DstType, typename ArgType> + MachineInstrBuilder buildCast(DstType &&Res, ArgType &&Arg) { + return buildCast(getDestFromArg(Res), getRegFromArg(Arg)); + } MachineInstrBuilder buildCast(unsigned Dst, unsigned Src); /// Build and insert G_BR \p Dest @@ -471,7 +424,7 @@ public: /// \pre setBasicBlock or setMI must have been called. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildBr(MachineBasicBlock &BB); + MachineInstrBuilder buildBr(MachineBasicBlock &Dest); /// Build and insert G_BRCOND \p Tst, \p Dest /// @@ -485,7 +438,7 @@ public: /// depend on bit 0 (for now). /// /// \return The newly created instruction. - MachineInstrBuilder buildBrCond(unsigned Tst, MachineBasicBlock &BB); + MachineInstrBuilder buildBrCond(unsigned Tst, MachineBasicBlock &Dest); /// Build and insert G_BRINDIRECT \p Tgt /// @@ -532,8 +485,18 @@ public: /// \pre \p Res must be a generic virtual register with scalar type. /// /// \return The newly created instruction. + template <typename DstType> + MachineInstrBuilder buildFConstant(DstType &&Res, const ConstantFP &Val) { + return buildFConstant(getDestFromArg(Res), Val); + } MachineInstrBuilder buildFConstant(unsigned Res, const ConstantFP &Val); + template <typename DstType> + MachineInstrBuilder buildFConstant(DstType &&Res, double Val) { + return buildFConstant(getDestFromArg(Res), Val); + } + MachineInstrBuilder buildFConstant(unsigned Res, double Val); + /// Build and insert \p Res = COPY Op /// /// Register-to-register COPY sets \p Res to \p Op. @@ -559,6 +522,18 @@ public: MachineInstrBuilder buildLoad(unsigned Res, unsigned Addr, MachineMemOperand &MMO); + /// Build and insert `Res = <opcode> Addr, MMO`. + /// + /// Loads the value stored at \p Addr. Puts the result in \p Res. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildLoadInstr(unsigned Opcode, unsigned Res, + unsigned Addr, MachineMemOperand &MMO); + /// Build and insert `G_STORE Val, Addr, MMO`. /// /// Stores the value \p Val to \p Addr. @@ -580,7 +555,10 @@ public: MachineInstrBuilder buildExtract(unsigned Res, unsigned Src, uint64_t Index); /// Build and insert \p Res = IMPLICIT_DEF. - MachineInstrBuilder buildUndef(unsigned Dst); + template <typename DstType> MachineInstrBuilder buildUndef(DstType &&Res) { + return buildUndef(getDestFromArg(Res)); + } + MachineInstrBuilder buildUndef(unsigned Res); /// Build and insert instructions to put \p Ops together at the specified p /// Indices to form a larger register. @@ -649,6 +627,10 @@ public: /// \pre \p Res must be smaller than \p Op /// /// \return The newly created instruction. + template <typename DstType, typename SrcType> + MachineInstrBuilder buildFPTrunc(DstType &&Res, SrcType &&Src) { + return buildFPTrunc(getDestFromArg(Res), getRegFromArg(Src)); + } MachineInstrBuilder buildFPTrunc(unsigned Res, unsigned Op); /// Build and insert \p Res = G_TRUNC \p Op @@ -735,7 +717,28 @@ public: MachineInstrBuilder buildExtractVectorElement(unsigned Res, unsigned Val, unsigned Idx); - /// Build and insert `OldValRes = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, + /// Build and insert `OldValRes<def>, SuccessRes<def> = + /// G_ATOMIC_CMPXCHG_WITH_SUCCESS Addr, CmpVal, NewVal, MMO`. + /// + /// Atomically replace the value at \p Addr with \p NewVal if it is currently + /// \p CmpVal otherwise leaves it unchanged. Puts the original value from \p + /// Addr in \p Res, along with an s1 indicating whether it was replaced. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register of scalar type. + /// \pre \p SuccessRes must be a generic virtual register of scalar type. It + /// will be assigned 0 on failure and 1 on success. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, \p CmpVal, and \p NewVal must be generic virtual + /// registers of the same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder + buildAtomicCmpXchgWithSuccess(unsigned OldValRes, unsigned SuccessRes, + unsigned Addr, unsigned CmpVal, unsigned NewVal, + MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, /// MMO`. /// /// Atomically replace the value at \p Addr with \p NewVal if it is currently @@ -752,6 +755,328 @@ public: MachineInstrBuilder buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr, unsigned CmpVal, unsigned NewVal, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_<Opcode> Addr, Val, MMO`. + /// + /// Atomically read-modify-update the value at \p Addr with \p Val. Puts the + /// original value from \p Addr in \p OldValRes. The modification is + /// determined by the opcode. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMW(unsigned Opcode, unsigned OldValRes, + unsigned Addr, unsigned Val, + MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_XCHG Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with \p Val. Puts the original + /// value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_ADD Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the addition of \p Val and + /// the original value. Puts the original value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_SUB Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the subtraction of \p Val and + /// the original value. Puts the original value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWSub(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_AND Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the bitwise and of \p Val and + /// the original value. Puts the original value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_NAND Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the bitwise nand of \p Val + /// and the original value. Puts the original value from \p Addr in \p + /// OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWNand(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_OR Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the bitwise or of \p Val and + /// the original value. Puts the original value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWOr(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_XOR Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the bitwise xor of \p Val and + /// the original value. Puts the original value from \p Addr in \p OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWXor(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_MAX Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the signed maximum of \p + /// Val and the original value. Puts the original value from \p Addr in \p + /// OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWMax(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_MIN Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the signed minimum of \p + /// Val and the original value. Puts the original value from \p Addr in \p + /// OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWMin(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_UMAX Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the unsigned maximum of \p + /// Val and the original value. Puts the original value from \p Addr in \p + /// OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); + + /// Build and insert `OldValRes<def> = G_ATOMICRMW_UMIN Addr, Val, MMO`. + /// + /// Atomically replace the value at \p Addr with the unsigned minimum of \p + /// Val and the original value. Puts the original value from \p Addr in \p + /// OldValRes. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p OldValRes must be a generic virtual register. + /// \pre \p Addr must be a generic virtual register with pointer type. + /// \pre \p OldValRes, and \p Val must be generic virtual registers of the + /// same type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr, + unsigned Val, MachineMemOperand &MMO); +}; + +/// A CRTP class that contains methods for building instructions that can +/// be constant folded. MachineIRBuilders that want to inherit from this will +/// need to implement buildBinaryOp (for constant folding binary ops). +/// Alternatively, they can implement buildInstr(Opc, Dst, Uses...) to perform +/// additional folding for Opc. +template <typename Base> +class FoldableInstructionsBuilder : public MachineIRBuilderBase { + Base &base() { return static_cast<Base &>(*this); } + +public: + using MachineIRBuilderBase::MachineIRBuilderBase; + /// Build and insert \p Res = G_ADD \p Op0, \p Op1 + /// + /// G_ADD sets \p Res to the sum of integer parameters \p Op0 and \p Op1, + /// truncated to their width. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers + /// with the same (scalar or vector) type). + /// + /// \return a MachineInstrBuilder for the newly created instruction. + + MachineInstrBuilder buildAdd(unsigned Dst, unsigned Src0, unsigned Src1) { + return base().buildBinaryOp(TargetOpcode::G_ADD, Dst, Src0, Src1); + } + template <typename DstTy, typename... UseArgsTy> + MachineInstrBuilder buildAdd(DstTy &&Ty, UseArgsTy &&... UseArgs) { + unsigned Res = base().getDestFromArg(Ty); + return base().buildAdd(Res, (base().getRegFromArg(UseArgs))...); + } + + /// Build and insert \p Res = G_SUB \p Op0, \p Op1 + /// + /// G_SUB sets \p Res to the sum of integer parameters \p Op0 and \p Op1, + /// truncated to their width. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers + /// with the same (scalar or vector) type). + /// + /// \return a MachineInstrBuilder for the newly created instruction. + + MachineInstrBuilder buildSub(unsigned Dst, unsigned Src0, unsigned Src1) { + return base().buildBinaryOp(TargetOpcode::G_SUB, Dst, Src0, Src1); + } + template <typename DstTy, typename... UseArgsTy> + MachineInstrBuilder buildSub(DstTy &&Ty, UseArgsTy &&... UseArgs) { + unsigned Res = base().getDestFromArg(Ty); + return base().buildSub(Res, (base().getRegFromArg(UseArgs))...); + } + + /// Build and insert \p Res = G_MUL \p Op0, \p Op1 + /// + /// G_MUL sets \p Res to the sum of integer parameters \p Op0 and \p Op1, + /// truncated to their width. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers + /// with the same (scalar or vector) type). + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildMul(unsigned Dst, unsigned Src0, unsigned Src1) { + return base().buildBinaryOp(TargetOpcode::G_MUL, Dst, Src0, Src1); + } + template <typename DstTy, typename... UseArgsTy> + MachineInstrBuilder buildMul(DstTy &&Ty, UseArgsTy &&... UseArgs) { + unsigned Res = base().getDestFromArg(Ty); + return base().buildMul(Res, (base().getRegFromArg(UseArgs))...); + } + + /// Build and insert \p Res = G_AND \p Op0, \p Op1 + /// + /// G_AND sets \p Res to the bitwise and of integer parameters \p Op0 and \p + /// Op1. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers + /// with the same (scalar or vector) type). + /// + /// \return a MachineInstrBuilder for the newly created instruction. + + MachineInstrBuilder buildAnd(unsigned Dst, unsigned Src0, unsigned Src1) { + return base().buildBinaryOp(TargetOpcode::G_AND, Dst, Src0, Src1); + } + template <typename DstTy, typename... UseArgsTy> + MachineInstrBuilder buildAnd(DstTy &&Ty, UseArgsTy &&... UseArgs) { + unsigned Res = base().getDestFromArg(Ty); + return base().buildAnd(Res, (base().getRegFromArg(UseArgs))...); + } + + /// Build and insert \p Res = G_OR \p Op0, \p Op1 + /// + /// G_OR sets \p Res to the bitwise or of integer parameters \p Op0 and \p + /// Op1. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers + /// with the same (scalar or vector) type). + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildOr(unsigned Dst, unsigned Src0, unsigned Src1) { + return base().buildBinaryOp(TargetOpcode::G_OR, Dst, Src0, Src1); + } + template <typename DstTy, typename... UseArgsTy> + MachineInstrBuilder buildOr(DstTy &&Ty, UseArgsTy &&... UseArgs) { + unsigned Res = base().getDestFromArg(Ty); + return base().buildOr(Res, (base().getRegFromArg(UseArgs))...); + } +}; + +class MachineIRBuilder : public FoldableInstructionsBuilder<MachineIRBuilder> { +public: + using FoldableInstructionsBuilder< + MachineIRBuilder>::FoldableInstructionsBuilder; + MachineInstrBuilder buildBinaryOp(unsigned Opcode, unsigned Dst, + unsigned Src0, unsigned Src1) { + validateBinaryOp(Dst, Src0, Src1); + return buildInstr(Opcode).addDef(Dst).addUse(Src0).addUse(Src1); + } + using FoldableInstructionsBuilder<MachineIRBuilder>::buildInstr; + /// DAG like Generic method for building arbitrary instructions as above. + /// \Opc opcode for the instruction. + /// \Ty Either LLT/TargetRegisterClass/unsigned types for Dst + /// \Args Variadic list of uses of types(unsigned/MachineInstrBuilder) + /// Uses of type MachineInstrBuilder will perform + /// getOperand(0).getReg() to convert to register. + template <typename DstTy, typename... UseArgsTy> + MachineInstrBuilder buildInstr(unsigned Opc, DstTy &&Ty, + UseArgsTy &&... Args) { + auto MIB = buildInstr(Opc).addDef(getDestFromArg(Ty)); + addUsesFromArgs(MIB, std::forward<UseArgsTy>(Args)...); + return MIB; + } }; } // End namespace llvm. diff --git a/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/include/llvm/CodeGen/GlobalISel/RegBankSelect.h index 676955c33fe9..c53ae416e60b 100644 --- a/include/llvm/CodeGen/GlobalISel/RegBankSelect.h +++ b/include/llvm/CodeGen/GlobalISel/RegBankSelect.h @@ -22,7 +22,7 @@ /// of an instruction should live. It asks the target which banks may be /// used for each operand of the instruction and what is the cost. Then, /// it chooses the solution which minimize the cost of the instruction plus -/// the cost of any move that may be needed to to the values into the right +/// the cost of any move that may be needed to the values into the right /// register bank. /// In other words, the cost for an instruction on a register bank RegBank /// is: Cost of I on RegBank plus the sum of the cost for bringing the diff --git a/include/llvm/CodeGen/GlobalISel/RegisterBank.h b/include/llvm/CodeGen/GlobalISel/RegisterBank.h index 5d758423f4e7..d5612e17393c 100644 --- a/include/llvm/CodeGen/GlobalISel/RegisterBank.h +++ b/include/llvm/CodeGen/GlobalISel/RegisterBank.h @@ -42,7 +42,7 @@ private: public: RegisterBank(unsigned ID, const char *Name, unsigned Size, - const uint32_t *ContainedRegClasses, unsigned NumRegClasses); + const uint32_t *CoveredClasses, unsigned NumRegClasses); /// Get the identifier of this register bank. unsigned getID() const { return ID; } diff --git a/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h b/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h index 02868b220984..82fd7eddb68a 100644 --- a/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h +++ b/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h @@ -622,6 +622,8 @@ public: /// \pre \p Reg is a virtual register that either has a bank or a class. /// \returns The constrained register class, or nullptr if there is none. /// \note This is a generic variant of MachineRegisterInfo::constrainRegClass + /// \note Use MachineRegisterInfo::constrainRegAttrs instead for any non-isel + /// purpose, including non-select passes of GlobalISel static const TargetRegisterClass * constrainGenericRegister(unsigned Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI); diff --git a/include/llvm/CodeGen/GlobalISel/Utils.h b/include/llvm/CodeGen/GlobalISel/Utils.h index 5864c15cc8eb..51e3a2732972 100644 --- a/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/include/llvm/CodeGen/GlobalISel/Utils.h @@ -19,8 +19,10 @@ namespace llvm { +class AnalysisUsage; class MachineFunction; class MachineInstr; +class MachineOperand; class MachineOptimizationRemarkEmitter; class MachineOptimizationRemarkMissed; class MachineRegisterInfo; @@ -32,6 +34,7 @@ class TargetRegisterInfo; class TargetRegisterClass; class Twine; class ConstantFP; +class APFloat; /// Try to constrain Reg to the specified register class. If this fails, /// create a new virtual register in the correct class and insert a COPY before @@ -57,8 +60,21 @@ unsigned constrainOperandRegClass(const MachineFunction &MF, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const MCInstrDesc &II, - unsigned Reg, unsigned OpIdx); + const MachineOperand &RegMO, unsigned OpIdx); +/// Mutate the newly-selected instruction \p I to constrain its (possibly +/// generic) virtual register operands to the instruction's register class. +/// This could involve inserting COPYs before (for uses) or after (for defs). +/// This requires the number of operands to match the instruction description. +/// \returns whether operand regclass constraining succeeded. +/// +// FIXME: Not all instructions have the same number of operands. We should +// probably expose a constrain helper per operand and let the target selector +// constrain individual registers, like fast-isel. +bool constrainSelectedInstRegOperands(MachineInstr &I, + const TargetInstrInfo &TII, + const TargetRegisterInfo &TRI, + const RegisterBankInfo &RBI); /// Check whether an instruction \p MI is dead: it only defines dead virtual /// registers, and doesn't have other side effects. bool isTriviallyDead(const MachineInstr &MI, const MachineRegisterInfo &MRI); @@ -85,5 +101,12 @@ const ConstantFP* getConstantFPVRegVal(unsigned VReg, MachineInstr *getOpcodeDef(unsigned Opcode, unsigned Reg, const MachineRegisterInfo &MRI); +/// Returns an APFloat from Val converted to the appropriate size. +APFloat getAPFloatFromSize(double Val, unsigned Size); + +/// Modify analysis usage so it preserves passes required for the SelectionDAG +/// fallback. +void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU); + } // End namespace llvm. #endif |