diff options
Diffstat (limited to 'llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp')
| -rw-r--r-- | llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 439 |
1 files changed, 434 insertions, 5 deletions
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp index 3f829cc2e677..8f03a7ac41d3 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp @@ -11,23 +11,452 @@ //===----------------------------------------------------------------------===// #include "RISCVLegalizerInfo.h" +#include "RISCVMachineFunctionInfo.h" #include "RISCVSubtarget.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" +#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Type.h" using namespace llvm; +using namespace LegalityPredicates; +using namespace LegalizeMutations; -RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) { - const unsigned XLen = ST.getXLen(); - const LLT XLenLLT = LLT::scalar(XLen); +// Is this type supported by scalar FP arithmetic operations given the current +// subtarget. +static LegalityPredicate typeIsScalarFPArith(unsigned TypeIdx, + const RISCVSubtarget &ST) { + return [=, &ST](const LegalityQuery &Query) { + return Query.Types[TypeIdx].isScalar() && + ((ST.hasStdExtF() && Query.Types[TypeIdx].getSizeInBits() == 32) || + (ST.hasStdExtD() && Query.Types[TypeIdx].getSizeInBits() == 64)); + }; +} + +RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) + : STI(ST), XLen(STI.getXLen()), sXLen(LLT::scalar(XLen)) { + const LLT sDoubleXLen = LLT::scalar(2 * XLen); + const LLT p0 = LLT::pointer(0, XLen); + const LLT s1 = LLT::scalar(1); + const LLT s8 = LLT::scalar(8); + const LLT s16 = LLT::scalar(16); + const LLT s32 = LLT::scalar(32); + const LLT s64 = LLT::scalar(64); using namespace TargetOpcode; getActionDefinitionsBuilder({G_ADD, G_SUB, G_AND, G_OR, G_XOR}) - .legalFor({XLenLLT}) - .clampScalar(0, XLenLLT, XLenLLT); + .legalFor({s32, sXLen}) + .widenScalarToNextPow2(0) + .clampScalar(0, s32, sXLen); + + getActionDefinitionsBuilder( + {G_UADDE, G_UADDO, G_USUBE, G_USUBO}).lower(); + + getActionDefinitionsBuilder({G_SADDO, G_SSUBO}).minScalar(0, sXLen).lower(); + + auto &ShiftActions = getActionDefinitionsBuilder({G_ASHR, G_LSHR, G_SHL}); + if (ST.is64Bit()) + ShiftActions.customFor({{s32, s32}}); + ShiftActions.legalFor({{s32, s32}, {s32, sXLen}, {sXLen, sXLen}}) + .widenScalarToNextPow2(0) + .clampScalar(1, s32, sXLen) + .clampScalar(0, s32, sXLen) + .minScalarSameAs(1, 0); + + if (ST.is64Bit()) { + getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT}) + .legalFor({{sXLen, s32}}) + .maxScalar(0, sXLen); + + getActionDefinitionsBuilder(G_SEXT_INREG) + .customFor({sXLen}) + .maxScalar(0, sXLen) + .lower(); + } else { + getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT}).maxScalar(0, sXLen); + + getActionDefinitionsBuilder(G_SEXT_INREG).maxScalar(0, sXLen).lower(); + } + + // Merge/Unmerge + for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) { + unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1; + unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0; + auto &MergeUnmergeActions = getActionDefinitionsBuilder(Op); + if (XLen == 32 && ST.hasStdExtD()) { + LLT IdxZeroTy = G_MERGE_VALUES ? s64 : s32; + LLT IdxOneTy = G_MERGE_VALUES ? s32 : s64; + MergeUnmergeActions.legalFor({IdxZeroTy, IdxOneTy}); + } + MergeUnmergeActions.widenScalarToNextPow2(LitTyIdx, XLen) + .widenScalarToNextPow2(BigTyIdx, XLen) + .clampScalar(LitTyIdx, sXLen, sXLen) + .clampScalar(BigTyIdx, sXLen, sXLen); + } + + getActionDefinitionsBuilder({G_FSHL, G_FSHR}).lower(); + + auto &RotateActions = getActionDefinitionsBuilder({G_ROTL, G_ROTR}); + if (ST.hasStdExtZbb()) { + RotateActions.legalFor({{s32, sXLen}, {sXLen, sXLen}}); + // Widen s32 rotate amount to s64 so SDAG patterns will match. + if (ST.is64Bit()) + RotateActions.widenScalarIf(all(typeIs(0, s32), typeIs(1, s32)), + changeTo(1, sXLen)); + } + RotateActions.lower(); + + getActionDefinitionsBuilder(G_BITREVERSE).maxScalar(0, sXLen).lower(); + + auto &BSWAPActions = getActionDefinitionsBuilder(G_BSWAP); + if (ST.hasStdExtZbb()) + BSWAPActions.legalFor({sXLen}).clampScalar(0, sXLen, sXLen); + else + BSWAPActions.maxScalar(0, sXLen).lower(); + + auto &CountZerosActions = getActionDefinitionsBuilder({G_CTLZ, G_CTTZ}); + auto &CountZerosUndefActions = + getActionDefinitionsBuilder({G_CTLZ_ZERO_UNDEF, G_CTTZ_ZERO_UNDEF}); + if (ST.hasStdExtZbb()) { + CountZerosActions.legalFor({{s32, s32}, {sXLen, sXLen}}) + .clampScalar(0, s32, sXLen) + .widenScalarToNextPow2(0) + .scalarSameSizeAs(1, 0); + } else { + CountZerosActions.maxScalar(0, sXLen).scalarSameSizeAs(1, 0).lower(); + CountZerosUndefActions.maxScalar(0, sXLen).scalarSameSizeAs(1, 0); + } + CountZerosUndefActions.lower(); + + auto &CTPOPActions = getActionDefinitionsBuilder(G_CTPOP); + if (ST.hasStdExtZbb()) { + CTPOPActions.legalFor({{s32, s32}, {sXLen, sXLen}}) + .clampScalar(0, s32, sXLen) + .widenScalarToNextPow2(0) + .scalarSameSizeAs(1, 0); + } else { + CTPOPActions.maxScalar(0, sXLen).scalarSameSizeAs(1, 0).lower(); + } + + getActionDefinitionsBuilder({G_CONSTANT, G_IMPLICIT_DEF}) + .legalFor({s32, sXLen, p0}) + .widenScalarToNextPow2(0) + .clampScalar(0, s32, sXLen); + + getActionDefinitionsBuilder(G_ICMP) + .legalFor({{sXLen, sXLen}, {sXLen, p0}}) + .widenScalarToNextPow2(1) + .clampScalar(1, sXLen, sXLen) + .clampScalar(0, sXLen, sXLen); + + auto &SelectActions = getActionDefinitionsBuilder(G_SELECT).legalFor( + {{s32, sXLen}, {p0, sXLen}}); + if (XLen == 64 || ST.hasStdExtD()) + SelectActions.legalFor({{s64, sXLen}}); + SelectActions.widenScalarToNextPow2(0) + .clampScalar(0, s32, (XLen == 64 || ST.hasStdExtD()) ? s64 : s32) + .clampScalar(1, sXLen, sXLen); + + auto &LoadStoreActions = + getActionDefinitionsBuilder({G_LOAD, G_STORE}) + .legalForTypesWithMemDesc({{s32, p0, s8, 8}, + {s32, p0, s16, 16}, + {s32, p0, s32, 32}, + {p0, p0, sXLen, XLen}}); + auto &ExtLoadActions = + getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD}) + .legalForTypesWithMemDesc({{s32, p0, s8, 8}, {s32, p0, s16, 16}}); + if (XLen == 64) { + LoadStoreActions.legalForTypesWithMemDesc({{s64, p0, s8, 8}, + {s64, p0, s16, 16}, + {s64, p0, s32, 32}, + {s64, p0, s64, 64}}); + ExtLoadActions.legalForTypesWithMemDesc( + {{s64, p0, s8, 8}, {s64, p0, s16, 16}, {s64, p0, s32, 32}}); + } else if (ST.hasStdExtD()) { + LoadStoreActions.legalForTypesWithMemDesc({{s64, p0, s64, 64}}); + } + LoadStoreActions.clampScalar(0, s32, sXLen).lower(); + ExtLoadActions.widenScalarToNextPow2(0).clampScalar(0, s32, sXLen).lower(); + + getActionDefinitionsBuilder({G_PTR_ADD, G_PTRMASK}).legalFor({{p0, sXLen}}); + + getActionDefinitionsBuilder(G_PTRTOINT) + .legalFor({{sXLen, p0}}) + .clampScalar(0, sXLen, sXLen); + + getActionDefinitionsBuilder(G_INTTOPTR) + .legalFor({{p0, sXLen}}) + .clampScalar(1, sXLen, sXLen); + + getActionDefinitionsBuilder(G_BRCOND).legalFor({sXLen}).minScalar(0, sXLen); + + getActionDefinitionsBuilder(G_BRJT).legalFor({{p0, sXLen}}); + + getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0}); + + getActionDefinitionsBuilder(G_PHI) + .legalFor({p0, sXLen}) + .widenScalarToNextPow2(0) + .clampScalar(0, sXLen, sXLen); + + getActionDefinitionsBuilder({G_GLOBAL_VALUE, G_JUMP_TABLE, G_CONSTANT_POOL}) + .legalFor({p0}); + + if (ST.hasStdExtM() || ST.hasStdExtZmmul()) { + getActionDefinitionsBuilder(G_MUL) + .legalFor({s32, sXLen}) + .widenScalarToNextPow2(0) + .clampScalar(0, s32, sXLen); + + // clang-format off + getActionDefinitionsBuilder({G_SMULH, G_UMULH}) + .legalFor({sXLen}) + .lower(); + // clang-format on + + getActionDefinitionsBuilder({G_SMULO, G_UMULO}).minScalar(0, sXLen).lower(); + } else { + getActionDefinitionsBuilder(G_MUL) + .libcallFor({sXLen, sDoubleXLen}) + .widenScalarToNextPow2(0) + .clampScalar(0, sXLen, sDoubleXLen); + + getActionDefinitionsBuilder({G_SMULH, G_UMULH}).lowerFor({sXLen}); + + getActionDefinitionsBuilder({G_SMULO, G_UMULO}) + .minScalar(0, sXLen) + // Widen sXLen to sDoubleXLen so we can use a single libcall to get + // the low bits for the mul result and high bits to do the overflow + // check. + .widenScalarIf(typeIs(0, sXLen), + LegalizeMutations::changeTo(0, sDoubleXLen)) + .lower(); + } + + if (ST.hasStdExtM()) { + getActionDefinitionsBuilder({G_UDIV, G_SDIV, G_UREM, G_SREM}) + .legalFor({s32, sXLen}) + .libcallFor({sDoubleXLen}) + .clampScalar(0, s32, sDoubleXLen) + .widenScalarToNextPow2(0); + } else { + getActionDefinitionsBuilder({G_UDIV, G_SDIV, G_UREM, G_SREM}) + .libcallFor({sXLen, sDoubleXLen}) + .clampScalar(0, sXLen, sDoubleXLen) + .widenScalarToNextPow2(0); + } + + auto &AbsActions = getActionDefinitionsBuilder(G_ABS); + if (ST.hasStdExtZbb()) + AbsActions.customFor({s32, sXLen}).minScalar(0, sXLen); + AbsActions.lower(); + + auto &MinMaxActions = + getActionDefinitionsBuilder({G_UMAX, G_UMIN, G_SMAX, G_SMIN}); + if (ST.hasStdExtZbb()) + MinMaxActions.legalFor({sXLen}).minScalar(0, sXLen); + MinMaxActions.lower(); + + getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0}); + + getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall(); + + getActionDefinitionsBuilder(G_DYN_STACKALLOC).lower(); + + // FP Operations + + getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FNEG, + G_FABS, G_FSQRT, G_FMAXNUM, G_FMINNUM}) + .legalIf(typeIsScalarFPArith(0, ST)); + + getActionDefinitionsBuilder(G_FCOPYSIGN) + .legalIf(all(typeIsScalarFPArith(0, ST), typeIsScalarFPArith(1, ST))); + + getActionDefinitionsBuilder(G_FPTRUNC).legalIf( + [=, &ST](const LegalityQuery &Query) -> bool { + return (ST.hasStdExtD() && typeIs(0, s32)(Query) && + typeIs(1, s64)(Query)); + }); + getActionDefinitionsBuilder(G_FPEXT).legalIf( + [=, &ST](const LegalityQuery &Query) -> bool { + return (ST.hasStdExtD() && typeIs(0, s64)(Query) && + typeIs(1, s32)(Query)); + }); + + getActionDefinitionsBuilder(G_FCMP) + .legalIf(all(typeIs(0, sXLen), typeIsScalarFPArith(1, ST))) + .clampScalar(0, sXLen, sXLen); + + // TODO: Support vector version of G_IS_FPCLASS. + getActionDefinitionsBuilder(G_IS_FPCLASS) + .customIf(all(typeIs(0, s1), typeIsScalarFPArith(1, ST))); + + getActionDefinitionsBuilder(G_FCONSTANT) + .legalIf(typeIsScalarFPArith(0, ST)) + .lowerFor({s32, s64}); + + getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI}) + .legalIf(all(typeInSet(0, {s32, sXLen}), typeIsScalarFPArith(1, ST))) + .widenScalarToNextPow2(0) + .clampScalar(0, s32, sXLen); + + getActionDefinitionsBuilder({G_SITOFP, G_UITOFP}) + .legalIf(all(typeIsScalarFPArith(0, ST), typeInSet(1, {s32, sXLen}))) + .widenScalarToNextPow2(1) + .clampScalar(1, s32, sXLen); + + // FIXME: We can do custom inline expansion like SelectionDAG. + // FIXME: Legal with Zfa. + getActionDefinitionsBuilder({G_FCEIL, G_FFLOOR}) + .libcallFor({s32, s64}); + + getActionDefinitionsBuilder(G_VASTART).customFor({p0}); + + // va_list must be a pointer, but most sized types are pretty easy to handle + // as the destination. + getActionDefinitionsBuilder(G_VAARG) + // TODO: Implement narrowScalar and widenScalar for G_VAARG for types + // outside the [s32, sXLen] range. + .clampScalar(0, s32, sXLen) + .lowerForCartesianProduct({s32, sXLen, p0}, {p0}); getLegacyLegalizerInfo().computeTables(); } + +static Type *getTypeForLLT(LLT Ty, LLVMContext &C) { + if (Ty.isVector()) + return FixedVectorType::get(IntegerType::get(C, Ty.getScalarSizeInBits()), + Ty.getNumElements()); + return IntegerType::get(C, Ty.getSizeInBits()); +} + +bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, + MachineInstr &MI) const { + Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID(); + switch (IntrinsicID) { + default: + return false; + case Intrinsic::vacopy: { + // vacopy arguments must be legal because of the intrinsic signature. + // No need to check here. + + MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; + MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); + MachineFunction &MF = *MI.getMF(); + const DataLayout &DL = MIRBuilder.getDataLayout(); + LLVMContext &Ctx = MF.getFunction().getContext(); + + Register DstLst = MI.getOperand(1).getReg(); + LLT PtrTy = MRI.getType(DstLst); + + // Load the source va_list + Align Alignment = DL.getABITypeAlign(getTypeForLLT(PtrTy, Ctx)); + MachineMemOperand *LoadMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOLoad, PtrTy, Alignment); + auto Tmp = MIRBuilder.buildLoad(PtrTy, MI.getOperand(2), *LoadMMO); + + // Store the result in the destination va_list + MachineMemOperand *StoreMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOStore, PtrTy, Alignment); + MIRBuilder.buildStore(DstLst, Tmp, *StoreMMO); + + MI.eraseFromParent(); + return true; + } + } +} + +bool RISCVLegalizerInfo::legalizeShlAshrLshr( + MachineInstr &MI, MachineIRBuilder &MIRBuilder, + GISelChangeObserver &Observer) const { + assert(MI.getOpcode() == TargetOpcode::G_ASHR || + MI.getOpcode() == TargetOpcode::G_LSHR || + MI.getOpcode() == TargetOpcode::G_SHL); + MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); + // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the + // imported patterns can select it later. Either way, it will be legal. + Register AmtReg = MI.getOperand(2).getReg(); + auto VRegAndVal = getIConstantVRegValWithLookThrough(AmtReg, MRI); + if (!VRegAndVal) + return true; + // Check the shift amount is in range for an immediate form. + uint64_t Amount = VRegAndVal->Value.getZExtValue(); + if (Amount > 31) + return true; // This will have to remain a register variant. + auto ExtCst = MIRBuilder.buildConstant(LLT::scalar(64), Amount); + Observer.changingInstr(MI); + MI.getOperand(2).setReg(ExtCst.getReg(0)); + Observer.changedInstr(MI); + return true; +} + +bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI, + MachineIRBuilder &MIRBuilder) const { + // Stores the address of the VarArgsFrameIndex slot into the memory location + assert(MI.getOpcode() == TargetOpcode::G_VASTART); + MachineFunction *MF = MI.getParent()->getParent(); + RISCVMachineFunctionInfo *FuncInfo = MF->getInfo<RISCVMachineFunctionInfo>(); + int FI = FuncInfo->getVarArgsFrameIndex(); + LLT AddrTy = MIRBuilder.getMRI()->getType(MI.getOperand(0).getReg()); + auto FINAddr = MIRBuilder.buildFrameIndex(AddrTy, FI); + assert(MI.hasOneMemOperand()); + MIRBuilder.buildStore(FINAddr, MI.getOperand(0).getReg(), + *MI.memoperands()[0]); + MI.eraseFromParent(); + return true; +} + +bool RISCVLegalizerInfo::legalizeCustom(LegalizerHelper &Helper, + MachineInstr &MI) const { + MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; + GISelChangeObserver &Observer = Helper.Observer; + switch (MI.getOpcode()) { + default: + // No idea what to do. + return false; + case TargetOpcode::G_ABS: + return Helper.lowerAbsToMaxNeg(MI); + case TargetOpcode::G_SHL: + case TargetOpcode::G_ASHR: + case TargetOpcode::G_LSHR: + return legalizeShlAshrLshr(MI, MIRBuilder, Observer); + case TargetOpcode::G_SEXT_INREG: { + // Source size of 32 is sext.w. + int64_t SizeInBits = MI.getOperand(2).getImm(); + if (SizeInBits == 32) + return true; + + return Helper.lower(MI, 0, /* Unused hint type */ LLT()) == + LegalizerHelper::Legalized; + } + case TargetOpcode::G_IS_FPCLASS: { + Register GISFPCLASS = MI.getOperand(0).getReg(); + Register Src = MI.getOperand(1).getReg(); + const MachineOperand &ImmOp = MI.getOperand(2); + MachineIRBuilder MIB(MI); + + // Turn LLVM IR's floating point classes to that in RISC-V, + // by simply rotating the 10-bit immediate right by two bits. + APInt GFpClassImm(10, static_cast<uint64_t>(ImmOp.getImm())); + auto FClassMask = MIB.buildConstant(sXLen, GFpClassImm.rotr(2).zext(XLen)); + auto ConstZero = MIB.buildConstant(sXLen, 0); + + auto GFClass = MIB.buildInstr(RISCV::G_FCLASS, {sXLen}, {Src}); + auto And = MIB.buildAnd(sXLen, GFClass, FClassMask); + MIB.buildICmp(CmpInst::ICMP_NE, GISFPCLASS, And, ConstZero); + + MI.eraseFromParent(); + return true; + } + case TargetOpcode::G_VASTART: + return legalizeVAStart(MI, MIRBuilder); + } + + llvm_unreachable("expected switch to return"); +} |