diff options
Diffstat (limited to 'llvm/lib/CodeGen/GlobalISel')
23 files changed, 698 insertions, 1144 deletions
diff --git a/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp b/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp index f9bfe8518083..ac140e745600 100644 --- a/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp @@ -67,7 +67,8 @@ bool CSEConfigFull::shouldCSEOpc(unsigned Opc) { } bool CSEConfigConstantOnly::shouldCSEOpc(unsigned Opc) { - return Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::G_IMPLICIT_DEF; + return Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::G_FCONSTANT || + Opc == TargetOpcode::G_IMPLICIT_DEF; } std::unique_ptr<CSEConfigBase> @@ -88,7 +89,7 @@ void GISelCSEInfo::setMF(MachineFunction &MF) { this->MRI = &MF.getRegInfo(); } -GISelCSEInfo::~GISelCSEInfo() {} +GISelCSEInfo::~GISelCSEInfo() = default; bool GISelCSEInfo::isUniqueMachineInstValid( const UniqueMachineInstr &UMI) const { diff --git a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp index 1a642e233a6a..a432e4ed7fb7 100644 --- a/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp @@ -12,6 +12,7 @@ // #include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineInstrBuilder.h" @@ -174,6 +175,7 @@ MachineInstrBuilder CSEMIRBuilder::buildInstr(unsigned Opc, default: break; case TargetOpcode::G_ADD: + case TargetOpcode::G_PTR_ADD: case TargetOpcode::G_AND: case TargetOpcode::G_ASHR: case TargetOpcode::G_LSHR: @@ -185,23 +187,54 @@ MachineInstrBuilder CSEMIRBuilder::buildInstr(unsigned Opc, case TargetOpcode::G_UDIV: case TargetOpcode::G_SDIV: case TargetOpcode::G_UREM: - case TargetOpcode::G_SREM: { + case TargetOpcode::G_SREM: + case TargetOpcode::G_SMIN: + case TargetOpcode::G_SMAX: + case TargetOpcode::G_UMIN: + case TargetOpcode::G_UMAX: { // Try to constant fold these. assert(SrcOps.size() == 2 && "Invalid sources"); assert(DstOps.size() == 1 && "Invalid dsts"); - if (SrcOps[0].getLLTTy(*getMRI()).isVector()) { + LLT SrcTy = SrcOps[0].getLLTTy(*getMRI()); + + if (Opc == TargetOpcode::G_PTR_ADD && + getDataLayout().isNonIntegralAddressSpace(SrcTy.getAddressSpace())) + break; + + if (SrcTy.isVector()) { // Try to constant fold vector constants. - Register VecCst = ConstantFoldVectorBinop( - Opc, SrcOps[0].getReg(), SrcOps[1].getReg(), *getMRI(), *this); - if (VecCst) - return buildCopy(DstOps[0], VecCst); + SmallVector<APInt> VecCst = ConstantFoldVectorBinop( + Opc, SrcOps[0].getReg(), SrcOps[1].getReg(), *getMRI()); + if (!VecCst.empty()) + return buildBuildVectorConstant(DstOps[0], VecCst); break; } + if (Optional<APInt> Cst = ConstantFoldBinOp(Opc, SrcOps[0].getReg(), SrcOps[1].getReg(), *getMRI())) return buildConstant(DstOps[0], *Cst); break; } + case TargetOpcode::G_FADD: + case TargetOpcode::G_FSUB: + case TargetOpcode::G_FMUL: + case TargetOpcode::G_FDIV: + case TargetOpcode::G_FREM: + case TargetOpcode::G_FMINNUM: + case TargetOpcode::G_FMAXNUM: + case TargetOpcode::G_FMINNUM_IEEE: + case TargetOpcode::G_FMAXNUM_IEEE: + case TargetOpcode::G_FMINIMUM: + case TargetOpcode::G_FMAXIMUM: + case TargetOpcode::G_FCOPYSIGN: { + // Try to constant fold these. + assert(SrcOps.size() == 2 && "Invalid sources"); + assert(DstOps.size() == 1 && "Invalid dsts"); + if (Optional<APFloat> Cst = ConstantFoldFPBinOp( + Opc, SrcOps[0].getReg(), SrcOps[1].getReg(), *getMRI())) + return buildFConstant(DstOps[0], *Cst); + break; + } case TargetOpcode::G_SEXT_INREG: { assert(DstOps.size() == 1 && "Invalid dst ops"); assert(SrcOps.size() == 2 && "Invalid src ops"); diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index 1ec7868f2234..081c8b125f17 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -11,16 +11,16 @@ /// //===----------------------------------------------------------------------===// +#include "llvm/CodeGen/GlobalISel/CallLowering.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/CallingConvLower.h" -#include "llvm/CodeGen/GlobalISel/CallLowering.h" -#include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/Target/TargetMachine.h" @@ -698,10 +698,12 @@ bool CallLowering::handleAssignments(ValueHandler &Handler, ValTy, extendOpFromFlags(Args[i].Flags[0])); } + bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(OrigVT, DL); for (unsigned Part = 0; Part < NumParts; ++Part) { Register ArgReg = Args[i].Regs[Part]; // There should be Regs.size() ArgLocs per argument. - VA = ArgLocs[j + Part]; + unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part; + CCValAssign &VA = ArgLocs[j + Idx]; const ISD::ArgFlagsTy Flags = Args[i].Flags[Part]; if (VA.isMemLoc() && !Flags.isByVal()) { diff --git a/llvm/lib/CodeGen/GlobalISel/Combiner.cpp b/llvm/lib/CodeGen/GlobalISel/Combiner.cpp index 30f8838805b5..1a5fe3e84c17 100644 --- a/llvm/lib/CodeGen/GlobalISel/Combiner.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Combiner.cpp @@ -13,14 +13,13 @@ #include "llvm/CodeGen/GlobalISel/Combiner.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/CodeGen/GlobalISel/CSEInfo.h" -#include "llvm/CodeGen/GlobalISel/CombinerInfo.h" #include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/CombinerInfo.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/GISelWorkList.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" #define DEBUG_TYPE "gi-combiner" @@ -57,8 +56,7 @@ class WorkListMaintainer : public GISelChangeObserver { public: WorkListMaintainer(WorkListTy &WorkList) : WorkList(WorkList) {} - virtual ~WorkListMaintainer() { - } + virtual ~WorkListMaintainer() = default; void erasingInstr(MachineInstr &MI) override { LLVM_DEBUG(dbgs() << "Erasing: " << MI << "\n"); @@ -115,7 +113,7 @@ bool Combiner::combineMachineInstrs(MachineFunction &MF, bool MFChanged = false; bool Changed; - MachineIRBuilder &B = *Builder.get(); + MachineIRBuilder &B = *Builder; do { // Collect all instructions. Do a post order traversal for basic blocks and diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index d6a009744161..2c94f87804ac 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -8,7 +8,6 @@ #include "llvm/CodeGen/GlobalISel/CombinerHelper.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallBitVector.h" -#include "llvm/CodeGen/GlobalISel/Combiner.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" @@ -16,23 +15,22 @@ #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/LowLevelType.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/RegisterBankInfo.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/IR/DataLayout.h" #include "llvm/Support/Casting.h" #include "llvm/Support/DivisionByConstantInfo.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Target/TargetMachine.h" #include <tuple> #define DEBUG_TYPE "gi-combiner" @@ -131,9 +129,27 @@ isBigEndian(const SmallDenseMap<int64_t, int64_t, 8> &MemOffset2Idx, return BigEndian; } +bool CombinerHelper::isPreLegalize() const { return !LI; } + +bool CombinerHelper::isLegal(const LegalityQuery &Query) const { + assert(LI && "Must have LegalizerInfo to query isLegal!"); + return LI->getAction(Query).Action == LegalizeActions::Legal; +} + bool CombinerHelper::isLegalOrBeforeLegalizer( const LegalityQuery &Query) const { - return !LI || LI->getAction(Query).Action == LegalizeActions::Legal; + return isPreLegalize() || isLegal(Query); +} + +bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const { + if (!Ty.isVector()) + return isLegalOrBeforeLegalizer({TargetOpcode::G_CONSTANT, {Ty}}); + // Vector constants are represented as a G_BUILD_VECTOR of scalar G_CONSTANTs. + if (isPreLegalize()) + return true; + LLT EltTy = Ty.getElementType(); + return isLegal({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) && + isLegal({TargetOpcode::G_CONSTANT, {EltTy}}); } void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, @@ -1275,12 +1291,12 @@ bool CombinerHelper::matchCombineConstantFoldFpUnary(MachineInstr &MI, Register SrcReg = MI.getOperand(1).getReg(); LLT DstTy = MRI.getType(DstReg); Cst = constantFoldFpUnary(MI.getOpcode(), DstTy, SrcReg, MRI); - return Cst.hasValue(); + return Cst.has_value(); } void CombinerHelper::applyCombineConstantFoldFpUnary(MachineInstr &MI, Optional<APFloat> &Cst) { - assert(Cst.hasValue() && "Optional is unexpectedly empty!"); + assert(Cst && "Optional is unexpectedly empty!"); Builder.setInstrAndDebugLoc(MI); MachineFunction &MF = Builder.getMF(); auto *FPVal = ConstantFP::get(MF.getFunction().getContext(), *Cst); @@ -2350,6 +2366,19 @@ bool CombinerHelper::matchEqualDefs(const MachineOperand &MOP1, if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad(nullptr)) return false; + // If both instructions are loads or stores, they are equal only if both + // are dereferenceable invariant loads with the same number of bits. + if (I1->mayLoadOrStore() && I2->mayLoadOrStore()) { + GLoadStore *LS1 = dyn_cast<GLoadStore>(I1); + GLoadStore *LS2 = dyn_cast<GLoadStore>(I2); + if (!LS1 || !LS2) + return false; + + if (!I2->isDereferenceableInvariantLoad(nullptr) || + (LS1->getMemSizeInBits() != LS2->getMemSizeInBits())) + return false; + } + // Check for physical registers on the instructions first to avoid cases // like this: // @@ -2397,7 +2426,7 @@ bool CombinerHelper::matchConstantOp(const MachineOperand &MOP, int64_t C) { return false; auto *MI = MRI.getVRegDef(MOP.getReg()); auto MaybeCst = isConstantOrConstantSplatVector(*MI, MRI); - return MaybeCst.hasValue() && MaybeCst->getBitWidth() <= 64 && + return MaybeCst && MaybeCst->getBitWidth() <= 64 && MaybeCst->getSExtValue() == C; } @@ -2916,7 +2945,7 @@ bool CombinerHelper::matchNotCmp(MachineInstr &MI, int64_t Cst; if (Ty.isVector()) { MachineInstr *CstDef = MRI.getVRegDef(CstReg); - auto MaybeCst = getBuildVectorConstantSplat(*CstDef, MRI); + auto MaybeCst = getIConstantSplatSExtVal(*CstDef, MRI); if (!MaybeCst) return false; if (!isConstValidTrue(TLI, Ty.getScalarSizeInBits(), *MaybeCst, true, IsFP)) @@ -3049,6 +3078,102 @@ void CombinerHelper::applySimplifyURemByPow2(MachineInstr &MI) { MI.eraseFromParent(); } +bool CombinerHelper::matchFoldBinOpIntoSelect(MachineInstr &MI, + unsigned &SelectOpNo) { + Register LHS = MI.getOperand(1).getReg(); + Register RHS = MI.getOperand(2).getReg(); + + Register OtherOperandReg = RHS; + SelectOpNo = 1; + MachineInstr *Select = MRI.getVRegDef(LHS); + + // Don't do this unless the old select is going away. We want to eliminate the + // binary operator, not replace a binop with a select. + if (Select->getOpcode() != TargetOpcode::G_SELECT || + !MRI.hasOneNonDBGUse(LHS)) { + OtherOperandReg = LHS; + SelectOpNo = 2; + Select = MRI.getVRegDef(RHS); + if (Select->getOpcode() != TargetOpcode::G_SELECT || + !MRI.hasOneNonDBGUse(RHS)) + return false; + } + + MachineInstr *SelectLHS = MRI.getVRegDef(Select->getOperand(2).getReg()); + MachineInstr *SelectRHS = MRI.getVRegDef(Select->getOperand(3).getReg()); + + if (!isConstantOrConstantVector(*SelectLHS, MRI, + /*AllowFP*/ true, + /*AllowOpaqueConstants*/ false)) + return false; + if (!isConstantOrConstantVector(*SelectRHS, MRI, + /*AllowFP*/ true, + /*AllowOpaqueConstants*/ false)) + return false; + + unsigned BinOpcode = MI.getOpcode(); + + // We know know one of the operands is a select of constants. Now verify that + // the other binary operator operand is either a constant, or we can handle a + // variable. + bool CanFoldNonConst = + (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) && + (isNullOrNullSplat(*SelectLHS, MRI) || + isAllOnesOrAllOnesSplat(*SelectLHS, MRI)) && + (isNullOrNullSplat(*SelectRHS, MRI) || + isAllOnesOrAllOnesSplat(*SelectRHS, MRI)); + if (CanFoldNonConst) + return true; + + return isConstantOrConstantVector(*MRI.getVRegDef(OtherOperandReg), MRI, + /*AllowFP*/ true, + /*AllowOpaqueConstants*/ false); +} + +/// \p SelectOperand is the operand in binary operator \p MI that is the select +/// to fold. +bool CombinerHelper::applyFoldBinOpIntoSelect(MachineInstr &MI, + const unsigned &SelectOperand) { + Builder.setInstrAndDebugLoc(MI); + + Register Dst = MI.getOperand(0).getReg(); + Register LHS = MI.getOperand(1).getReg(); + Register RHS = MI.getOperand(2).getReg(); + MachineInstr *Select = MRI.getVRegDef(MI.getOperand(SelectOperand).getReg()); + + Register SelectCond = Select->getOperand(1).getReg(); + Register SelectTrue = Select->getOperand(2).getReg(); + Register SelectFalse = Select->getOperand(3).getReg(); + + LLT Ty = MRI.getType(Dst); + unsigned BinOpcode = MI.getOpcode(); + + Register FoldTrue, FoldFalse; + + // We have a select-of-constants followed by a binary operator with a + // constant. Eliminate the binop by pulling the constant math into the select. + // Example: add (select Cond, CT, CF), CBO --> select Cond, CT + CBO, CF + CBO + if (SelectOperand == 1) { + // TODO: SelectionDAG verifies this actually constant folds before + // committing to the combine. + + FoldTrue = Builder.buildInstr(BinOpcode, {Ty}, {SelectTrue, RHS}).getReg(0); + FoldFalse = + Builder.buildInstr(BinOpcode, {Ty}, {SelectFalse, RHS}).getReg(0); + } else { + FoldTrue = Builder.buildInstr(BinOpcode, {Ty}, {LHS, SelectTrue}).getReg(0); + FoldFalse = + Builder.buildInstr(BinOpcode, {Ty}, {LHS, SelectFalse}).getReg(0); + } + + Builder.buildSelect(Dst, SelectCond, FoldTrue, FoldFalse, MI.getFlags()); + Observer.erasingInstr(*Select); + Select->eraseFromParent(); + MI.eraseFromParent(); + + return true; +} + Optional<SmallVector<Register, 8>> CombinerHelper::findCandidatesForLoadOrCombine(const MachineInstr *Root) const { assert(Root->getOpcode() == TargetOpcode::G_OR && "Expected G_OR only!"); @@ -3340,7 +3465,7 @@ bool CombinerHelper::matchLoadOrCombine( // BSWAP. bool IsBigEndianTarget = MF.getDataLayout().isBigEndian(); Optional<bool> IsBigEndian = isBigEndian(MemOffset2Idx, LowestIdx); - if (!IsBigEndian.hasValue()) + if (!IsBigEndian) return false; bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian; if (NeedsBSwap && !isLegalOrBeforeLegalizer({TargetOpcode::G_BSWAP, {Ty}})) @@ -3848,7 +3973,7 @@ bool CombinerHelper::matchExtractAllEltsFromBuildVector( auto Cst = getIConstantVRegVal(II.getOperand(2).getReg(), MRI); if (!Cst) return false; - unsigned Idx = Cst.getValue().getZExtValue(); + unsigned Idx = Cst->getZExtValue(); if (Idx >= NumElts) return false; // Out of range. ExtractedElts.set(Idx); @@ -3904,10 +4029,9 @@ bool CombinerHelper::matchOrShiftToFunnelShift(MachineInstr &MI, // Given constants C0 and C1 such that C0 + C1 is bit-width: // (or (shl x, C0), (lshr y, C1)) -> (fshl x, y, C0) or (fshr x, y, C1) - // TODO: Match constant splat. int64_t CstShlAmt, CstLShrAmt; - if (mi_match(ShlAmt, MRI, m_ICst(CstShlAmt)) && - mi_match(LShrAmt, MRI, m_ICst(CstLShrAmt)) && + if (mi_match(ShlAmt, MRI, m_ICstOrSplat(CstShlAmt)) && + mi_match(LShrAmt, MRI, m_ICstOrSplat(CstLShrAmt)) && CstShlAmt + CstLShrAmt == BitWidth) { FshOpc = TargetOpcode::G_FSHR; Amt = LShrAmt; @@ -3958,7 +4082,7 @@ void CombinerHelper::applyFunnelShiftToRotate(MachineInstr &MI) { Observer.changingInstr(MI); MI.setDesc(Builder.getTII().get(IsFSHL ? TargetOpcode::G_ROTL : TargetOpcode::G_ROTR)); - MI.RemoveOperand(2); + MI.removeOperand(2); Observer.changedInstr(MI); } @@ -4100,18 +4224,23 @@ bool CombinerHelper::matchAndOrDisjointMask( return false; Register Src; - int64_t MaskAnd; - int64_t MaskOr; + Register AndMaskReg; + int64_t AndMaskBits; + int64_t OrMaskBits; if (!mi_match(MI, MRI, - m_GAnd(m_GOr(m_Reg(Src), m_ICst(MaskOr)), m_ICst(MaskAnd)))) + m_GAnd(m_GOr(m_Reg(Src), m_ICst(OrMaskBits)), + m_all_of(m_ICst(AndMaskBits), m_Reg(AndMaskReg))))) return false; - // Check if MaskOr could turn on any bits in Src. - if (MaskAnd & MaskOr) + // Check if OrMask could turn on any bits in Src. + if (AndMaskBits & OrMaskBits) return false; MatchInfo = [=, &MI](MachineIRBuilder &B) { Observer.changingInstr(MI); + // Canonicalize the result to have the constant on the RHS. + if (MI.getOperand(1).getReg() == AndMaskReg) + MI.getOperand(2).setReg(AndMaskReg); MI.getOperand(1).setReg(Src); Observer.changedInstr(MI); }; @@ -4259,6 +4388,14 @@ bool CombinerHelper::matchBitfieldExtractFromShrAnd( if (ShrAmt < 0 || ShrAmt >= Size) return false; + // If the shift subsumes the mask, emit the 0 directly. + if (0 == (SMask >> ShrAmt)) { + MatchInfo = [=](MachineIRBuilder &B) { + B.buildConstant(Dst, 0); + }; + return true; + } + // Check that ubfx can do the extraction, with no holes in the mask. uint64_t UMask = SMask; UMask |= maskTrailingOnes<uint64_t>(ShrAmt); @@ -4585,6 +4722,42 @@ bool CombinerHelper::matchMulOBy2(MachineInstr &MI, BuildFnTy &MatchInfo) { return true; } +bool CombinerHelper::matchMulOBy0(MachineInstr &MI, BuildFnTy &MatchInfo) { + // (G_*MULO x, 0) -> 0 + no carry out + assert(MI.getOpcode() == TargetOpcode::G_UMULO || + MI.getOpcode() == TargetOpcode::G_SMULO); + if (!mi_match(MI.getOperand(3).getReg(), MRI, m_SpecificICstOrSplat(0))) + return false; + Register Dst = MI.getOperand(0).getReg(); + Register Carry = MI.getOperand(1).getReg(); + if (!isConstantLegalOrBeforeLegalizer(MRI.getType(Dst)) || + !isConstantLegalOrBeforeLegalizer(MRI.getType(Carry))) + return false; + MatchInfo = [=](MachineIRBuilder &B) { + B.buildConstant(Dst, 0); + B.buildConstant(Carry, 0); + }; + return true; +} + +bool CombinerHelper::matchAddOBy0(MachineInstr &MI, BuildFnTy &MatchInfo) { + // (G_*ADDO x, 0) -> x + no carry out + assert(MI.getOpcode() == TargetOpcode::G_UADDO || + MI.getOpcode() == TargetOpcode::G_SADDO); + if (!mi_match(MI.getOperand(3).getReg(), MRI, m_SpecificICstOrSplat(0))) + return false; + Register Carry = MI.getOperand(1).getReg(); + if (!isConstantLegalOrBeforeLegalizer(MRI.getType(Carry))) + return false; + Register Dst = MI.getOperand(0).getReg(); + Register LHS = MI.getOperand(2).getReg(); + MatchInfo = [=](MachineIRBuilder &B) { + B.buildCopy(Dst, LHS); + B.buildConstant(Carry, 0); + }; + return true; +} + MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) { assert(MI.getOpcode() == TargetOpcode::G_UDIV); auto &UDiv = cast<GenericMachineInstr>(MI); @@ -5376,6 +5549,106 @@ bool CombinerHelper::matchCombineFSubFpExtFNegFMulToFMadOrFMA( return false; } +bool CombinerHelper::matchSelectToLogical(MachineInstr &MI, + BuildFnTy &MatchInfo) { + GSelect &Sel = cast<GSelect>(MI); + Register DstReg = Sel.getReg(0); + Register Cond = Sel.getCondReg(); + Register TrueReg = Sel.getTrueReg(); + Register FalseReg = Sel.getFalseReg(); + + auto *TrueDef = getDefIgnoringCopies(TrueReg, MRI); + auto *FalseDef = getDefIgnoringCopies(FalseReg, MRI); + + const LLT CondTy = MRI.getType(Cond); + const LLT OpTy = MRI.getType(TrueReg); + if (CondTy != OpTy || OpTy.getScalarSizeInBits() != 1) + return false; + + // We have a boolean select. + + // select Cond, Cond, F --> or Cond, F + // select Cond, 1, F --> or Cond, F + auto MaybeCstTrue = isConstantOrConstantSplatVector(*TrueDef, MRI); + if (Cond == TrueReg || (MaybeCstTrue && MaybeCstTrue->isOne())) { + MatchInfo = [=](MachineIRBuilder &MIB) { + MIB.buildOr(DstReg, Cond, FalseReg); + }; + return true; + } + + // select Cond, T, Cond --> and Cond, T + // select Cond, T, 0 --> and Cond, T + auto MaybeCstFalse = isConstantOrConstantSplatVector(*FalseDef, MRI); + if (Cond == FalseReg || (MaybeCstFalse && MaybeCstFalse->isZero())) { + MatchInfo = [=](MachineIRBuilder &MIB) { + MIB.buildAnd(DstReg, Cond, TrueReg); + }; + return true; + } + + // select Cond, T, 1 --> or (not Cond), T + if (MaybeCstFalse && MaybeCstFalse->isOne()) { + MatchInfo = [=](MachineIRBuilder &MIB) { + MIB.buildOr(DstReg, MIB.buildNot(OpTy, Cond), TrueReg); + }; + return true; + } + + // select Cond, 0, F --> and (not Cond), F + if (MaybeCstTrue && MaybeCstTrue->isZero()) { + MatchInfo = [=](MachineIRBuilder &MIB) { + MIB.buildAnd(DstReg, MIB.buildNot(OpTy, Cond), FalseReg); + }; + return true; + } + return false; +} + +bool CombinerHelper::matchCombineFMinMaxNaN(MachineInstr &MI, + unsigned &IdxToPropagate) { + bool PropagateNaN; + switch (MI.getOpcode()) { + default: + return false; + case TargetOpcode::G_FMINNUM: + case TargetOpcode::G_FMAXNUM: + PropagateNaN = false; + break; + case TargetOpcode::G_FMINIMUM: + case TargetOpcode::G_FMAXIMUM: + PropagateNaN = true; + break; + } + + auto MatchNaN = [&](unsigned Idx) { + Register MaybeNaNReg = MI.getOperand(Idx).getReg(); + const ConstantFP *MaybeCst = getConstantFPVRegVal(MaybeNaNReg, MRI); + if (!MaybeCst || !MaybeCst->getValueAPF().isNaN()) + return false; + IdxToPropagate = PropagateNaN ? Idx : (Idx == 1 ? 2 : 1); + return true; + }; + + return MatchNaN(1) || MatchNaN(2); +} + +bool CombinerHelper::matchAddSubSameReg(MachineInstr &MI, Register &Src) { + assert(MI.getOpcode() == TargetOpcode::G_ADD && "Expected a G_ADD"); + Register LHS = MI.getOperand(1).getReg(); + Register RHS = MI.getOperand(2).getReg(); + + // Helper lambda to check for opportunities for + // A + (B - A) -> B + // (B - A) + A -> B + auto CheckFold = [&](Register MaybeSub, Register MaybeSameReg) { + Register Reg; + return mi_match(MaybeSub, MRI, m_GSub(m_Reg(Src), m_Reg(Reg))) && + Reg == MaybeSameReg; + }; + return CheckFold(LHS, RHS) || CheckFold(RHS, LHS); +} + bool CombinerHelper::tryCombine(MachineInstr &MI) { if (tryCombineCopy(MI)) return true; diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp index 64c2f0d5f8e4..4f03af0fce82 100644 --- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp +++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp @@ -567,6 +567,26 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known, Known = KnownBits::ashr(KnownBits::shl(Known, ShiftKnown), ShiftKnown); break; } + case TargetOpcode::G_UADDO: + case TargetOpcode::G_UADDE: + case TargetOpcode::G_SADDO: + case TargetOpcode::G_SADDE: + case TargetOpcode::G_USUBO: + case TargetOpcode::G_USUBE: + case TargetOpcode::G_SSUBO: + case TargetOpcode::G_SSUBE: + case TargetOpcode::G_UMULO: + case TargetOpcode::G_SMULO: { + if (MI.getOperand(1).getReg() == R) { + // If we know the result of a compare has the top bits zero, use this + // info. + if (TL.getBooleanContents(DstTy.isVector(), false) == + TargetLowering::ZeroOrOneBooleanContent && + BitWidth > 1) + Known.Zero.setBitsFrom(1); + } + break; + } } assert(!Known.hasConflict() && "Bits known to be one AND zero?"); @@ -673,6 +693,27 @@ unsigned GISelKnownBits::computeNumSignBits(Register R, MI.getOperand(3).getReg(), DemandedElts, Depth + 1); } + case TargetOpcode::G_SADDO: + case TargetOpcode::G_SADDE: + case TargetOpcode::G_UADDO: + case TargetOpcode::G_UADDE: + case TargetOpcode::G_SSUBO: + case TargetOpcode::G_SSUBE: + case TargetOpcode::G_USUBO: + case TargetOpcode::G_USUBE: + case TargetOpcode::G_SMULO: + case TargetOpcode::G_UMULO: { + // If compares returns 0/-1, all bits are sign bits. + // We know that we have an integer-based boolean since these operations + // are only available for integer. + if (MI.getOperand(1).getReg() == R) { + if (TL.getBooleanContents(DstTy.isVector(), false) == + TargetLowering::ZeroOrNegativeOneBooleanContent) + return TyBits; + } + + break; + } case TargetOpcode::G_INTRINSIC: case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: default: { diff --git a/llvm/lib/CodeGen/GlobalISel/GlobalISel.cpp b/llvm/lib/CodeGen/GlobalISel/GlobalISel.cpp index 252b931602c6..efcc40641ea8 100644 --- a/llvm/lib/CodeGen/GlobalISel/GlobalISel.cpp +++ b/llvm/lib/CodeGen/GlobalISel/GlobalISel.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "llvm/InitializePasses.h" -#include "llvm/PassRegistry.h" using namespace llvm; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 6d415c9c7f90..a2af66d28f4a 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -16,10 +16,11 @@ #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/BranchProbabilityInfo.h" -#include "llvm/Analysis/Loads.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/Analysis.h" +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" +#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h" @@ -47,7 +48,6 @@ #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" @@ -78,7 +78,6 @@ #include "llvm/Transforms/Utils/MemoryOpRemark.h" #include <algorithm> #include <cassert> -#include <cstddef> #include <cstdint> #include <iterator> #include <string> @@ -1818,7 +1817,7 @@ static unsigned getConstrainedOpcode(Intrinsic::ID ID) { bool IRTranslator::translateConstrainedFPIntrinsic( const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) { - fp::ExceptionBehavior EB = FPI.getExceptionBehavior().getValue(); + fp::ExceptionBehavior EB = *FPI.getExceptionBehavior(); unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID()); if (!Opcode) @@ -2252,6 +2251,23 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0}; return CLI->lowerCall(MIRBuilder, Info); } + case Intrinsic::fptrunc_round: { + unsigned Flags = MachineInstr::copyFlagsFromInstruction(CI); + + // Convert the metadata argument to a constant integer + Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata(); + Optional<RoundingMode> RoundMode = + convertStrToRoundingMode(cast<MDString>(MD)->getString()); + + // Add the Rounding mode as an integer + MIRBuilder + .buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND, + {getOrCreateVReg(CI)}, + {getOrCreateVReg(*CI.getArgOperand(0))}, Flags) + .addImm((int)*RoundMode); + + return true; + } #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ case Intrinsic::INTRINSIC: #include "llvm/IR/ConstrainedOps.def" @@ -2409,7 +2425,7 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { TargetLowering::IntrinsicInfo Info; // TODO: Add a GlobalISel version of getTgtMemIntrinsic. if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) { - Align Alignment = Info.align.getValueOr( + Align Alignment = Info.align.value_or( DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext()))); LLT MemTy = Info.memVT.isSimple() ? getLLTForMVT(Info.memVT.getSimpleVT()) @@ -2934,15 +2950,6 @@ void IRTranslator::finishPendingPhis() { } } -bool IRTranslator::valueIsSplit(const Value &V, - SmallVectorImpl<uint64_t> *Offsets) { - SmallVector<LLT, 4> SplitTys; - if (Offsets && !Offsets->empty()) - Offsets->clear(); - computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets); - return SplitTys.size() > 1; -} - bool IRTranslator::translate(const Instruction &Inst) { CurBuilder->setDebugLoc(Inst.getDebugLoc()); @@ -2984,7 +2991,7 @@ bool IRTranslator::translate(const Constant &C, Register Reg) { // Return the scalar if it is a <1 x Ty> vector. unsigned NumElts = CAZ->getElementCount().getFixedValue(); if (NumElts == 1) - return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder.get()); + return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder); SmallVector<Register, 4> Ops; for (unsigned I = 0; I < NumElts; ++I) { Constant &Elt = *CAZ->getElementValue(I); @@ -2994,8 +3001,7 @@ bool IRTranslator::translate(const Constant &C, Register Reg) { } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) { // Return the scalar if it is a <1 x Ty> vector. if (CV->getNumElements() == 1) - return translateCopy(C, *CV->getElementAsConstant(0), - *EntryBuilder.get()); + return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder); SmallVector<Register, 4> Ops; for (unsigned i = 0; i < CV->getNumElements(); ++i) { Constant &Elt = *CV->getElementAsConstant(i); @@ -3013,7 +3019,7 @@ bool IRTranslator::translate(const Constant &C, Register Reg) { } } else if (auto CV = dyn_cast<ConstantVector>(&C)) { if (CV->getNumOperands() == 1) - return translateCopy(C, *CV->getOperand(0), *EntryBuilder.get()); + return translateCopy(C, *CV->getOperand(0), *EntryBuilder); SmallVector<Register, 4> Ops; for (unsigned i = 0; i < CV->getNumOperands(); ++i) { Ops.push_back(getOrCreateVReg(*CV->getOperand(i))); @@ -3255,14 +3261,13 @@ bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD, return false; } - // On PS4, the "return address" must still be within the calling function, - // even if it's at the very end, so emit an explicit TRAP here. - // Passing 'true' for doesNotReturn above won't generate the trap for us. + // On PS4/PS5, the "return address" must still be within the calling + // function, even if it's at the very end, so emit an explicit TRAP here. // WebAssembly needs an unreachable instruction after a non-returning call, // because the function return type can be different from __stack_chk_fail's // return type (void). const TargetMachine &TM = MF->getTarget(); - if (TM.getTargetTriple().isPS4CPU() || TM.getTargetTriple().isWasm()) { + if (TM.getTargetTriple().isPS() || TM.getTargetTriple().isWasm()) { LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n"); return false; } @@ -3413,7 +3418,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { } } - if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs, FuncInfo)) { + if (!CLI->lowerFormalArguments(*EntryBuilder, F, VRegArgs, FuncInfo)) { OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure", F.getSubprogram(), &F.getEntryBlock()); R << "unable to lower arguments: " << ore::NV("Prototype", F.getType()); @@ -3469,8 +3474,13 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { return false; } - if (!finalizeBasicBlock(*BB, MBB)) + if (!finalizeBasicBlock(*BB, MBB)) { + OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure", + BB->getTerminator()->getDebugLoc(), BB); + R << "unable to translate basic block"; + reportTranslationError(*MF, *TPC, *ORE, R); return false; + } } #ifndef NDEBUG WrapperObserver.removeObserver(&Verifier); diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp index e5f95ca5aa73..95ae8383b6fa 100644 --- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp @@ -12,15 +12,10 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h" -#include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetLowering.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #define DEBUG_TYPE "inline-asm-lowering" @@ -150,6 +145,7 @@ static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) { case TargetLowering::C_RegisterClass: return 2; case TargetLowering::C_Memory: + case TargetLowering::C_Address: return 3; } llvm_unreachable("Invalid constraint type"); @@ -310,7 +306,7 @@ bool InlineAsmLowering::lowerInlineAsm( // If this is an indirect operand, the operand is a pointer to the // accessed type. if (OpInfo.isIndirect) { - OpTy = Call.getAttributes().getParamElementType(ArgNo); + OpTy = Call.getParamElementType(ArgNo); assert(OpTy && "Indirect operand must have elementtype attribute"); } @@ -649,6 +645,8 @@ bool InlineAsmLowering::lowerInlineAsm( return false; case TargetLowering::C_Memory: break; // Already handled. + case TargetLowering::C_Address: + break; // Silence warning. case TargetLowering::C_Unknown: LLVM_DEBUG(dbgs() << "Unexpected unknown constraint\n"); return false; diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp index 2bb5addefe48..28f3b425c67d 100644 --- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp +++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp @@ -12,8 +12,6 @@ #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/ScopeExit.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/LazyBlockFrequencyInfo.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" @@ -23,14 +21,13 @@ #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/Config/config.h" -#include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/MC/TargetRegistry.h" +#include "llvm/Support/CodeGenCoverage.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Target/TargetMachine.h" diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp index 1d0c106fd5db..8959d215ecd1 100644 --- a/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp +++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelector.cpp @@ -13,16 +13,9 @@ #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" #include "llvm/CodeGen/GlobalISel/Utils.h" -#include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/TargetRegisterInfo.h" -#include "llvm/MC/MCInstrDesc.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -#include <cassert> #define DEBUG_TYPE "instructionselector" @@ -66,6 +59,10 @@ bool InstructionSelector::isObviouslySafeToFold(MachineInstr &MI, std::next(MI.getIterator()) == IntoMI.getIterator()) return true; + // Convergent instructions cannot be moved in the CFG. + if (MI.isConvergent() && MI.getParent() != IntoMI.getParent()) + return false; + return !MI.mayLoadOrStore() && !MI.mayRaiseFPException() && !MI.hasUnmodeledSideEffects() && MI.implicit_operands().empty(); } diff --git a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp index 1f0738a8d9d2..54a82cac95d5 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp @@ -188,6 +188,13 @@ LegalityPredicate LegalityPredicates::memSizeInBytesNotPow2(unsigned MMOIdx) { }; } +LegalityPredicate LegalityPredicates::memSizeNotByteSizePow2(unsigned MMOIdx) { + return [=](const LegalityQuery &Query) { + const LLT MemTy = Query.MMODescrs[MMOIdx].MemoryTy; + return !MemTy.isByteSized() || !isPowerOf2_32(MemTy.getSizeInBytes()); + }; +} + LegalityPredicate LegalityPredicates::numElementsNotPow2(unsigned TypeIdx) { return [=](const LegalityQuery &Query) { const LLT QueryTy = Query.Types[TypeIdx]; diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp index 75b7fcb5663a..25c1db91b05d 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp @@ -43,6 +43,27 @@ LegalizeMutation LegalizeMutations::changeElementTo(unsigned TypeIdx, }; } +LegalizeMutation LegalizeMutations::changeElementCountTo(unsigned TypeIdx, + unsigned FromTypeIdx) { + return [=](const LegalityQuery &Query) { + const LLT OldTy = Query.Types[TypeIdx]; + const LLT NewTy = Query.Types[FromTypeIdx]; + ElementCount NewEltCount = + NewTy.isVector() ? NewTy.getElementCount() : ElementCount::getFixed(1); + return std::make_pair(TypeIdx, OldTy.changeElementCount(NewEltCount)); + }; +} + +LegalizeMutation LegalizeMutations::changeElementCountTo(unsigned TypeIdx, + LLT NewEltTy) { + return [=](const LegalityQuery &Query) { + const LLT OldTy = Query.Types[TypeIdx]; + ElementCount NewEltCount = NewEltTy.isVector() ? NewEltTy.getElementCount() + : ElementCount::getFixed(1); + return std::make_pair(TypeIdx, OldTy.changeElementCount(NewEltCount)); + }; +} + LegalizeMutation LegalizeMutations::changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx) { return [=](const LegalityQuery &Query) { diff --git a/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp b/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp index 0ab4a7f64840..f09e5b7ce783 100644 --- a/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Legalizer.cpp @@ -14,7 +14,7 @@ #include "llvm/CodeGen/GlobalISel/Legalizer.h" #include "llvm/ADT/PostOrderIterator.h" -#include "llvm/ADT/SetVector.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/CodeGen/GlobalISel/CSEInfo.h" #include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" @@ -24,15 +24,11 @@ #include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/InitializePasses.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" -#include "llvm/Target/TargetMachine.h" - -#include <iterator> #define DEBUG_TYPE "legalizer" diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 37bc8a65dc7c..fb046d519ac8 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -15,10 +15,13 @@ #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" #include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/CodeGen/TargetInstrInfo.h" @@ -1611,40 +1614,6 @@ LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, return Legalized; } -Register LegalizerHelper::widenWithUnmerge(LLT WideTy, Register OrigReg) { - Register WideReg = MRI.createGenericVirtualRegister(WideTy); - LLT OrigTy = MRI.getType(OrigReg); - LLT LCMTy = getLCMType(WideTy, OrigTy); - - const int NumMergeParts = LCMTy.getSizeInBits() / WideTy.getSizeInBits(); - const int NumUnmergeParts = LCMTy.getSizeInBits() / OrigTy.getSizeInBits(); - - Register UnmergeSrc = WideReg; - - // Create a merge to the LCM type, padding with undef - // %0:_(<3 x s32>) = G_FOO => <4 x s32> - // => - // %1:_(<4 x s32>) = G_FOO - // %2:_(<4 x s32>) = G_IMPLICIT_DEF - // %3:_(<12 x s32>) = G_CONCAT_VECTORS %1, %2, %2 - // %0:_(<3 x s32>), %4:_, %5:_, %6:_ = G_UNMERGE_VALUES %3 - if (NumMergeParts > 1) { - Register Undef = MIRBuilder.buildUndef(WideTy).getReg(0); - SmallVector<Register, 8> MergeParts(NumMergeParts, Undef); - MergeParts[0] = WideReg; - UnmergeSrc = MIRBuilder.buildMerge(LCMTy, MergeParts).getReg(0); - } - - // Unmerge to the original register and pad with dead defs. - SmallVector<Register, 8> UnmergeResults(NumUnmergeParts); - UnmergeResults[0] = OrigReg; - for (int I = 1; I != NumUnmergeParts; ++I) - UnmergeResults[I] = MRI.createGenericVirtualRegister(OrigTy); - - MIRBuilder.buildUnmerge(UnmergeResults, UnmergeSrc); - return WideReg; -} - LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { @@ -1867,9 +1836,6 @@ LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { - if (TypeIdx == 1) - return UnableToLegalize; // TODO - unsigned Opcode; unsigned ExtOpcode; Optional<Register> CarryIn = None; @@ -1914,6 +1880,18 @@ LegalizerHelper::widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx, break; } + if (TypeIdx == 1) { + unsigned BoolExtOp = MIRBuilder.getBoolExtOp(WideTy.isVector(), false); + + Observer.changingInstr(MI); + widenScalarDst(MI, WideTy, 1); + if (CarryIn) + widenScalarSrc(MI, WideTy, 4, BoolExtOp); + + Observer.changedInstr(MI); + return Legalized; + } + auto LHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(2)}); auto RHSExt = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MI.getOperand(3)}); // Do the arithmetic in the larger type. @@ -1985,8 +1963,12 @@ LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, LegalizerHelper::LegalizeResult LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { - if (TypeIdx == 1) - return UnableToLegalize; + if (TypeIdx == 1) { + Observer.changingInstr(MI); + widenScalarDst(MI, WideTy, 1); + Observer.changedInstr(MI); + return Legalized; + } bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULO; Register Result = MI.getOperand(0).getReg(); @@ -2992,7 +2974,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerLoad(GAnyLoad &LoadMI) { if (isa<GSExtLoad>(LoadMI)) { auto NewLoad = MIRBuilder.buildLoad(LoadTy, PtrReg, *NewMMO); MIRBuilder.buildSExtInReg(LoadReg, NewLoad, MemSizeInBits); - } else if (isa<GZExtLoad>(LoadMI) || WideMemTy == DstTy) { + } else if (isa<GZExtLoad>(LoadMI) || WideMemTy == LoadTy) { auto NewLoad = MIRBuilder.buildLoad(LoadTy, PtrReg, *NewMMO); // The extra bits are guaranteed to be zero, since we stored them that // way. A zext load from Wide thus automatically gives zext from MemVT. @@ -3314,7 +3296,7 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { Observer.changingInstr(MI); const auto &TII = MIRBuilder.getTII(); MI.setDesc(TII.get(TargetOpcode::G_MUL)); - MI.RemoveOperand(1); + MI.removeOperand(1); Observer.changedInstr(MI); auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS}); @@ -4096,13 +4078,14 @@ LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx, // is a load, return the new registers in ValRegs. For a store, each elements // of ValRegs should be PartTy. Returns the next offset that needs to be // handled. + bool isBigEndian = MIRBuilder.getDataLayout().isBigEndian(); auto MMO = LdStMI.getMMO(); auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs, - unsigned Offset) -> unsigned { + unsigned NumParts, unsigned Offset) -> unsigned { MachineFunction &MF = MIRBuilder.getMF(); unsigned PartSize = PartTy.getSizeInBits(); for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize; - Offset += PartSize, ++Idx) { + ++Idx) { unsigned ByteOffset = Offset / 8; Register NewAddrReg; @@ -4118,16 +4101,19 @@ LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx, } else { MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO); } + Offset = isBigEndian ? Offset - PartSize : Offset + PartSize; } return Offset; }; - unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0); + unsigned Offset = isBigEndian ? TotalSize - NarrowTy.getSizeInBits() : 0; + unsigned HandledOffset = + splitTypePieces(NarrowTy, NarrowRegs, NumParts, Offset); // Handle the rest of the register if this isn't an even type breakdown. if (LeftoverTy.isValid()) - splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset); + splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset); if (IsLoad) { insertParts(ValReg, ValTy, NarrowTy, NarrowRegs, @@ -4236,6 +4222,14 @@ LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, case G_INTTOPTR: case G_PTRTOINT: case G_ADDRSPACE_CAST: + case G_UADDO: + case G_USUBO: + case G_UADDE: + case G_USUBE: + case G_SADDO: + case G_SSUBO: + case G_SADDE: + case G_SSUBE: return fewerElementsVectorMultiEltType(GMI, NumElts); case G_ICMP: case G_FCMP: @@ -4882,10 +4876,26 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx, moreElementsVectorDst(MI, MoreTy, 0); Observer.changedInstr(MI); return Legalized; - case TargetOpcode::G_SELECT: - if (TypeIdx != 0) - return UnableToLegalize; - if (MRI.getType(MI.getOperand(1).getReg()).isVector()) + case TargetOpcode::G_SELECT: { + Register DstReg = MI.getOperand(0).getReg(); + Register CondReg = MI.getOperand(1).getReg(); + LLT DstTy = MRI.getType(DstReg); + LLT CondTy = MRI.getType(CondReg); + if (TypeIdx == 1) { + if (!CondTy.isScalar() || + DstTy.getElementCount() != MoreTy.getElementCount()) + return UnableToLegalize; + + // This is turning a scalar select of vectors into a vector + // select. Broadcast the select condition. + auto ShufSplat = MIRBuilder.buildShuffleSplat(MoreTy, CondReg); + Observer.changingInstr(MI); + MI.getOperand(1).setReg(ShufSplat.getReg(0)); + Observer.changedInstr(MI); + return Legalized; + } + + if (CondTy.isVector()) return UnableToLegalize; Observer.changingInstr(MI); @@ -4894,6 +4904,7 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx, moreElementsVectorDst(MI, MoreTy, 0); Observer.changedInstr(MI); return Legalized; + } case TargetOpcode::G_UNMERGE_VALUES: return UnableToLegalize; case TargetOpcode::G_PHI: @@ -7229,25 +7240,32 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) { Register Op2Reg = MI.getOperand(3).getReg(); LLT DstTy = MRI.getType(DstReg); LLT MaskTy = MRI.getType(MaskReg); - LLT Op1Ty = MRI.getType(Op1Reg); if (!DstTy.isVector()) return UnableToLegalize; - // Vector selects can have a scalar predicate. If so, splat into a vector and - // finish for later legalization attempts to try again. if (MaskTy.isScalar()) { + // Turn the scalar condition into a vector condition mask. + Register MaskElt = MaskReg; - if (MaskTy.getSizeInBits() < DstTy.getScalarSizeInBits()) - MaskElt = MIRBuilder.buildSExt(DstTy.getElementType(), MaskElt).getReg(0); - // Generate a vector splat idiom to be pattern matched later. + + // The condition was potentially zero extended before, but we want a sign + // extended boolean. + if (MaskTy.getSizeInBits() <= DstTy.getScalarSizeInBits() && + MaskTy != LLT::scalar(1)) { + MaskElt = MIRBuilder.buildSExtInReg(MaskTy, MaskElt, 1).getReg(0); + } + + // Continue the sign extension (or truncate) to match the data type. + MaskElt = MIRBuilder.buildSExtOrTrunc(DstTy.getElementType(), + MaskElt).getReg(0); + + // Generate a vector splat idiom. auto ShufSplat = MIRBuilder.buildShuffleSplat(DstTy, MaskElt); - Observer.changingInstr(MI); - MI.getOperand(1).setReg(ShufSplat.getReg(0)); - Observer.changedInstr(MI); - return Legalized; + MaskReg = ShufSplat.getReg(0); + MaskTy = DstTy; } - if (MaskTy.getSizeInBits() != Op1Ty.getSizeInBits()) { + if (MaskTy.getSizeInBits() != DstTy.getSizeInBits()) { return UnableToLegalize; } @@ -7414,7 +7432,7 @@ static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) { unsigned NumBits = Ty.getScalarSizeInBits(); auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Val, MRI); if (!Ty.isVector() && ValVRegAndVal) { - APInt Scalar = ValVRegAndVal->Value.truncOrSelf(8); + APInt Scalar = ValVRegAndVal->Value.trunc(8); APInt SplatVal = APInt::getSplat(NumBits, Scalar); return MIB.buildConstant(Ty, SplatVal).getReg(0); } @@ -7569,7 +7587,7 @@ LegalizerHelper::lowerMemcpyInline(MachineInstr &MI) { // See if this is a constant length copy auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI); // FIXME: support dynamically sized G_MEMCPY_INLINE - assert(LenVRegAndVal.hasValue() && + assert(LenVRegAndVal && "inline memcpy with dynamic size is not yet supported"); uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue(); if (KnownLen == 0) { @@ -7609,7 +7627,7 @@ LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src, bool DstAlignCanChange = false; MachineFrameInfo &MFI = MF.getFrameInfo(); - Align Alignment = commonAlignment(DstAlign, SrcAlign); + Align Alignment = std::min(DstAlign, SrcAlign); MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex())) @@ -7644,7 +7662,7 @@ LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src, const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); if (!TRI->hasStackRealignment(MF)) while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign)) - NewAlign = NewAlign / 2; + NewAlign = NewAlign.previous(); if (NewAlign > Alignment) { Alignment = NewAlign; @@ -7717,7 +7735,7 @@ LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src, bool DstAlignCanChange = false; MachineFrameInfo &MFI = MF.getFrameInfo(); bool OptSize = shouldLowerMemFuncForSize(MF); - Align Alignment = commonAlignment(DstAlign, SrcAlign); + Align Alignment = std::min(DstAlign, SrcAlign); MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex())) @@ -7752,7 +7770,7 @@ LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src, const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); if (!TRI->hasStackRealignment(MF)) while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign)) - NewAlign = NewAlign / 2; + NewAlign = NewAlign.previous(); if (NewAlign > Alignment) { Alignment = NewAlign; diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp index 30697913a6a4..6adb7ddb5b66 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp @@ -13,7 +13,6 @@ #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" #include "llvm/ADT/SmallBitVector.h" -#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -23,9 +22,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/LowLevelTypeImpl.h" -#include "llvm/Support/MathExtras.h" #include <algorithm> -#include <map> using namespace llvm; using namespace LegalizeActions; @@ -132,15 +129,16 @@ static bool mutationIsSane(const LegalizeRule &Rule, LLVM_FALLTHROUGH; case MoreElements: { // MoreElements can go from scalar to vector. - const unsigned OldElts = OldTy.isVector() ? OldTy.getNumElements() : 1; + const ElementCount OldElts = OldTy.isVector() ? + OldTy.getElementCount() : ElementCount::getFixed(1); if (NewTy.isVector()) { if (Rule.getAction() == FewerElements) { // Make sure the element count really decreased. - if (NewTy.getNumElements() >= OldElts) + if (ElementCount::isKnownGE(NewTy.getElementCount(), OldElts)) return false; } else { // Make sure the element count really increased. - if (NewTy.getNumElements() <= OldElts) + if (ElementCount::isKnownLE(NewTy.getElementCount(), OldElts)) return false; } } else if (Rule.getAction() == MoreElements) diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp index de8dbd456901..d4fbf7d15089 100644 --- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp @@ -73,6 +73,7 @@ void LoadStoreOpt::init(MachineFunction &MF) { void LoadStoreOpt::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired<AAResultsWrapperPass>(); + AU.setPreservesAll(); getSelectionDAGFallbackAnalysisUsage(AU); MachineFunctionPass::getAnalysisUsage(AU); } @@ -508,6 +509,12 @@ bool LoadStoreOpt::addStoreToCandidate(GStore &StoreMI, if (StoreMI.getMemSizeInBits() != ValueTy.getSizeInBits()) return false; + // Avoid adding volatile or ordered stores to the candidate. We already have a + // check for this in instMayAlias() but that only get's called later between + // potential aliasing hazards. + if (!StoreMI.isSimple()) + return false; + Register StoreAddr = StoreMI.getPointerReg(); auto BIO = getPointerInfo(StoreAddr, *MRI); Register StoreBase = BIO.BaseReg; diff --git a/llvm/lib/CodeGen/GlobalISel/Localizer.cpp b/llvm/lib/CodeGen/GlobalISel/Localizer.cpp index 328a278f3d68..c1287693e74d 100644 --- a/llvm/lib/CodeGen/GlobalISel/Localizer.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Localizer.cpp @@ -13,6 +13,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/InitializePasses.h" diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index c6720568b362..19ebf46191a9 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -9,8 +9,6 @@ /// This file implements the MachineIRBuidler class. //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/Analysis/MemoryLocation.h" -#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" @@ -19,7 +17,7 @@ #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/DebugInfoMetadata.h" using namespace llvm; @@ -568,47 +566,6 @@ MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst, return Extract; } -void MachineIRBuilder::buildSequence(Register Res, ArrayRef<Register> Ops, - ArrayRef<uint64_t> Indices) { -#ifndef NDEBUG - assert(Ops.size() == Indices.size() && "incompatible args"); - assert(!Ops.empty() && "invalid trivial sequence"); - assert(llvm::is_sorted(Indices) && - "sequence offsets must be in ascending order"); - - assert(getMRI()->getType(Res).isValid() && "invalid operand type"); - for (auto Op : Ops) - assert(getMRI()->getType(Op).isValid() && "invalid operand type"); -#endif - - LLT ResTy = getMRI()->getType(Res); - LLT OpTy = getMRI()->getType(Ops[0]); - unsigned OpSize = OpTy.getSizeInBits(); - bool MaybeMerge = true; - for (unsigned i = 0; i < Ops.size(); ++i) { - if (getMRI()->getType(Ops[i]) != OpTy || Indices[i] != i * OpSize) { - MaybeMerge = false; - break; - } - } - - if (MaybeMerge && Ops.size() * OpSize == ResTy.getSizeInBits()) { - buildMerge(Res, Ops); - return; - } - - Register ResIn = getMRI()->createGenericVirtualRegister(ResTy); - buildUndef(ResIn); - - for (unsigned i = 0; i < Ops.size(); ++i) { - Register ResOut = i + 1 == Ops.size() - ? Res - : getMRI()->createGenericVirtualRegister(ResTy); - buildInsert(ResOut, ResIn, Ops[i], Indices[i]); - ResIn = ResOut; - } -} - MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) { return buildInstr(TargetOpcode::G_IMPLICIT_DEF, {Res}, {}); } @@ -666,6 +623,17 @@ MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res, return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); } +MachineInstrBuilder +MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res, + ArrayRef<APInt> Ops) { + SmallVector<SrcOp> TmpVec; + TmpVec.reserve(Ops.size()); + LLT EltTy = Res.getLLTTy(*getMRI()).getElementType(); + for (auto &Op : Ops) + TmpVec.push_back(buildConstant(EltTy, Op)); + return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); +} + MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res, const SrcOp &Src) { SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src); diff --git a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp index 01af6bb51bb7..bce850ee212c 100644 --- a/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp +++ b/llvm/lib/CodeGen/GlobalISel/RegBankSelect.cpp @@ -14,8 +14,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" -#include "llvm/CodeGen/GlobalISel/RegisterBank.h" -#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" #include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" @@ -25,12 +23,13 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/RegisterBank.h" +#include "llvm/CodeGen/RegisterBankInfo.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/Config/llvm-config.h" -#include "llvm/IR/Attributes.h" #include "llvm/IR/Function.h" #include "llvm/InitializePasses.h" #include "llvm/Pass.h" @@ -631,7 +630,8 @@ bool RegBankSelect::assignInstr(MachineInstr &MI) { "Unexpected hint opcode!"); // The only correct mapping for these is to always use the source register // bank. - const RegisterBank *RB = MRI->getRegBankOrNull(MI.getOperand(1).getReg()); + const RegisterBank *RB = + RBI->getRegBank(MI.getOperand(1).getReg(), *MRI, *TRI); // We can assume every instruction above this one has a selected register // bank. assert(RB && "Expected source register to have a register bank?"); diff --git a/llvm/lib/CodeGen/GlobalISel/RegisterBank.cpp b/llvm/lib/CodeGen/GlobalISel/RegisterBank.cpp deleted file mode 100644 index 5c4d18ad79c5..000000000000 --- a/llvm/lib/CodeGen/GlobalISel/RegisterBank.cpp +++ /dev/null @@ -1,110 +0,0 @@ -//===- llvm/CodeGen/GlobalISel/RegisterBank.cpp - Register Bank --*- C++ -*-==// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// \file -/// This file implements the RegisterBank class. -//===----------------------------------------------------------------------===// - -#include "llvm/CodeGen/GlobalISel/RegisterBank.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/CodeGen/TargetRegisterInfo.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/Support/Debug.h" - -#define DEBUG_TYPE "registerbank" - -using namespace llvm; - -const unsigned RegisterBank::InvalidID = UINT_MAX; - -RegisterBank::RegisterBank( - unsigned ID, const char *Name, unsigned Size, - const uint32_t *CoveredClasses, unsigned NumRegClasses) - : ID(ID), Name(Name), Size(Size) { - ContainedRegClasses.resize(NumRegClasses); - ContainedRegClasses.setBitsInMask(CoveredClasses); -} - -bool RegisterBank::verify(const TargetRegisterInfo &TRI) const { - assert(isValid() && "Invalid register bank"); - for (unsigned RCId = 0, End = TRI.getNumRegClasses(); RCId != End; ++RCId) { - const TargetRegisterClass &RC = *TRI.getRegClass(RCId); - - if (!covers(RC)) - continue; - // Verify that the register bank covers all the sub classes of the - // classes it covers. - - // Use a different (slow in that case) method than - // RegisterBankInfo to find the subclasses of RC, to make sure - // both agree on the covers. - for (unsigned SubRCId = 0; SubRCId != End; ++SubRCId) { - const TargetRegisterClass &SubRC = *TRI.getRegClass(RCId); - - if (!RC.hasSubClassEq(&SubRC)) - continue; - - // Verify that the Size of the register bank is big enough to cover - // all the register classes it covers. - assert(getSize() >= TRI.getRegSizeInBits(SubRC) && - "Size is not big enough for all the subclasses!"); - assert(covers(SubRC) && "Not all subclasses are covered"); - } - } - return true; -} - -bool RegisterBank::covers(const TargetRegisterClass &RC) const { - assert(isValid() && "RB hasn't been initialized yet"); - return ContainedRegClasses.test(RC.getID()); -} - -bool RegisterBank::isValid() const { - return ID != InvalidID && Name != nullptr && Size != 0 && - // A register bank that does not cover anything is useless. - !ContainedRegClasses.empty(); -} - -bool RegisterBank::operator==(const RegisterBank &OtherRB) const { - // There must be only one instance of a given register bank alive - // for the whole compilation. - // The RegisterBankInfo is supposed to enforce that. - assert((OtherRB.getID() != getID() || &OtherRB == this) && - "ID does not uniquely identify a RegisterBank"); - return &OtherRB == this; -} - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -LLVM_DUMP_METHOD void RegisterBank::dump(const TargetRegisterInfo *TRI) const { - print(dbgs(), /* IsForDebug */ true, TRI); -} -#endif - -void RegisterBank::print(raw_ostream &OS, bool IsForDebug, - const TargetRegisterInfo *TRI) const { - OS << getName(); - if (!IsForDebug) - return; - OS << "(ID:" << getID() << ", Size:" << getSize() << ")\n" - << "isValid:" << isValid() << '\n' - << "Number of Covered register classes: " << ContainedRegClasses.count() - << '\n'; - // Print all the subclasses if we can. - // This register classes may not be properly initialized yet. - if (!TRI || ContainedRegClasses.empty()) - return; - assert(ContainedRegClasses.size() == TRI->getNumRegClasses() && - "TRI does not match the initialization process?"); - OS << "Covered register classes:\n"; - ListSeparator LS; - for (unsigned RCId = 0, End = TRI->getNumRegClasses(); RCId != End; ++RCId) { - const TargetRegisterClass &RC = *TRI->getRegClass(RCId); - - if (covers(RC)) - OS << LS << TRI->getRegClassName(&RC); - } -} diff --git a/llvm/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp b/llvm/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp deleted file mode 100644 index 650500c7eb31..000000000000 --- a/llvm/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp +++ /dev/null @@ -1,805 +0,0 @@ -//===- llvm/CodeGen/GlobalISel/RegisterBankInfo.cpp --------------*- C++ -*-==// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -/// \file -/// This file implements the RegisterBankInfo class. -//===----------------------------------------------------------------------===// - -#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/iterator_range.h" -#include "llvm/CodeGen/GlobalISel/RegisterBank.h" -#include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/TargetOpcodes.h" -#include "llvm/CodeGen/TargetRegisterInfo.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/Config/llvm-config.h" -#include "llvm/IR/Type.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" - -#include <algorithm> // For std::max. - -#define DEBUG_TYPE "registerbankinfo" - -using namespace llvm; - -STATISTIC(NumPartialMappingsCreated, - "Number of partial mappings dynamically created"); -STATISTIC(NumPartialMappingsAccessed, - "Number of partial mappings dynamically accessed"); -STATISTIC(NumValueMappingsCreated, - "Number of value mappings dynamically created"); -STATISTIC(NumValueMappingsAccessed, - "Number of value mappings dynamically accessed"); -STATISTIC(NumOperandsMappingsCreated, - "Number of operands mappings dynamically created"); -STATISTIC(NumOperandsMappingsAccessed, - "Number of operands mappings dynamically accessed"); -STATISTIC(NumInstructionMappingsCreated, - "Number of instruction mappings dynamically created"); -STATISTIC(NumInstructionMappingsAccessed, - "Number of instruction mappings dynamically accessed"); - -const unsigned RegisterBankInfo::DefaultMappingID = UINT_MAX; -const unsigned RegisterBankInfo::InvalidMappingID = UINT_MAX - 1; - -//------------------------------------------------------------------------------ -// RegisterBankInfo implementation. -//------------------------------------------------------------------------------ -RegisterBankInfo::RegisterBankInfo(RegisterBank **RegBanks, - unsigned NumRegBanks) - : RegBanks(RegBanks), NumRegBanks(NumRegBanks) { -#ifndef NDEBUG - for (unsigned Idx = 0, End = getNumRegBanks(); Idx != End; ++Idx) { - assert(RegBanks[Idx] != nullptr && "Invalid RegisterBank"); - assert(RegBanks[Idx]->isValid() && "RegisterBank should be valid"); - } -#endif // NDEBUG -} - -bool RegisterBankInfo::verify(const TargetRegisterInfo &TRI) const { -#ifndef NDEBUG - for (unsigned Idx = 0, End = getNumRegBanks(); Idx != End; ++Idx) { - const RegisterBank &RegBank = getRegBank(Idx); - assert(Idx == RegBank.getID() && - "ID does not match the index in the array"); - LLVM_DEBUG(dbgs() << "Verify " << RegBank << '\n'); - assert(RegBank.verify(TRI) && "RegBank is invalid"); - } -#endif // NDEBUG - return true; -} - -const RegisterBank * -RegisterBankInfo::getRegBank(Register Reg, const MachineRegisterInfo &MRI, - const TargetRegisterInfo &TRI) const { - if (Register::isPhysicalRegister(Reg)) { - // FIXME: This was probably a copy to a virtual register that does have a - // type we could use. - return &getRegBankFromRegClass(getMinimalPhysRegClass(Reg, TRI), LLT()); - } - - assert(Reg && "NoRegister does not have a register bank"); - const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); - if (auto *RB = RegClassOrBank.dyn_cast<const RegisterBank *>()) - return RB; - if (auto *RC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>()) - return &getRegBankFromRegClass(*RC, MRI.getType(Reg)); - return nullptr; -} - -const TargetRegisterClass & -RegisterBankInfo::getMinimalPhysRegClass(Register Reg, - const TargetRegisterInfo &TRI) const { - assert(Register::isPhysicalRegister(Reg) && "Reg must be a physreg"); - const auto &RegRCIt = PhysRegMinimalRCs.find(Reg); - if (RegRCIt != PhysRegMinimalRCs.end()) - return *RegRCIt->second; - const TargetRegisterClass *PhysRC = TRI.getMinimalPhysRegClass(Reg); - PhysRegMinimalRCs[Reg] = PhysRC; - return *PhysRC; -} - -const RegisterBank *RegisterBankInfo::getRegBankFromConstraints( - const MachineInstr &MI, unsigned OpIdx, const TargetInstrInfo &TII, - const MachineRegisterInfo &MRI) const { - const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo(); - - // The mapping of the registers may be available via the - // register class constraints. - const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx, &TII, TRI); - - if (!RC) - return nullptr; - - Register Reg = MI.getOperand(OpIdx).getReg(); - const RegisterBank &RegBank = getRegBankFromRegClass(*RC, MRI.getType(Reg)); - // Check that the target properly implemented getRegBankFromRegClass. - assert(RegBank.covers(*RC) && - "The mapping of the register bank does not make sense"); - return &RegBank; -} - -const TargetRegisterClass *RegisterBankInfo::constrainGenericRegister( - Register Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI) { - - // If the register already has a class, fallback to MRI::constrainRegClass. - auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); - if (RegClassOrBank.is<const TargetRegisterClass *>()) - return MRI.constrainRegClass(Reg, &RC); - - const RegisterBank *RB = RegClassOrBank.get<const RegisterBank *>(); - // Otherwise, all we can do is ensure the bank covers the class, and set it. - if (RB && !RB->covers(RC)) - return nullptr; - - // If nothing was set or the class is simply compatible, set it. - MRI.setRegClass(Reg, &RC); - return &RC; -} - -/// Check whether or not \p MI should be treated like a copy -/// for the mappings. -/// Copy like instruction are special for mapping because -/// they don't have actual register constraints. Moreover, -/// they sometimes have register classes assigned and we can -/// just use that instead of failing to provide a generic mapping. -static bool isCopyLike(const MachineInstr &MI) { - return MI.isCopy() || MI.isPHI() || - MI.getOpcode() == TargetOpcode::REG_SEQUENCE; -} - -const RegisterBankInfo::InstructionMapping & -RegisterBankInfo::getInstrMappingImpl(const MachineInstr &MI) const { - // For copies we want to walk over the operands and try to find one - // that has a register bank since the instruction itself will not get - // us any constraint. - bool IsCopyLike = isCopyLike(MI); - // For copy like instruction, only the mapping of the definition - // is important. The rest is not constrained. - unsigned NumOperandsForMapping = IsCopyLike ? 1 : MI.getNumOperands(); - - const MachineFunction &MF = *MI.getMF(); - const TargetSubtargetInfo &STI = MF.getSubtarget(); - const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); - const MachineRegisterInfo &MRI = MF.getRegInfo(); - // We may need to query the instruction encoding to guess the mapping. - const TargetInstrInfo &TII = *STI.getInstrInfo(); - - // Before doing anything complicated check if the mapping is not - // directly available. - bool CompleteMapping = true; - - SmallVector<const ValueMapping *, 8> OperandsMapping(NumOperandsForMapping); - for (unsigned OpIdx = 0, EndIdx = MI.getNumOperands(); OpIdx != EndIdx; - ++OpIdx) { - const MachineOperand &MO = MI.getOperand(OpIdx); - if (!MO.isReg()) - continue; - Register Reg = MO.getReg(); - if (!Reg) - continue; - // The register bank of Reg is just a side effect of the current - // excution and in particular, there is no reason to believe this - // is the best default mapping for the current instruction. Keep - // it as an alternative register bank if we cannot figure out - // something. - const RegisterBank *AltRegBank = getRegBank(Reg, MRI, TRI); - // For copy-like instruction, we want to reuse the register bank - // that is already set on Reg, if any, since those instructions do - // not have any constraints. - const RegisterBank *CurRegBank = IsCopyLike ? AltRegBank : nullptr; - if (!CurRegBank) { - // If this is a target specific instruction, we can deduce - // the register bank from the encoding constraints. - CurRegBank = getRegBankFromConstraints(MI, OpIdx, TII, MRI); - if (!CurRegBank) { - // All our attempts failed, give up. - CompleteMapping = false; - - if (!IsCopyLike) - // MI does not carry enough information to guess the mapping. - return getInvalidInstructionMapping(); - continue; - } - } - - unsigned Size = getSizeInBits(Reg, MRI, TRI); - const ValueMapping *ValMapping = &getValueMapping(0, Size, *CurRegBank); - if (IsCopyLike) { - if (!OperandsMapping[0]) { - if (MI.isRegSequence()) { - // For reg_sequence, the result size does not match the input. - unsigned ResultSize = getSizeInBits(MI.getOperand(0).getReg(), - MRI, TRI); - OperandsMapping[0] = &getValueMapping(0, ResultSize, *CurRegBank); - } else { - OperandsMapping[0] = ValMapping; - } - } - - // The default handling assumes any register bank can be copied to any - // other. If this isn't the case, the target should specially deal with - // reg_sequence/phi. There may also be unsatisfiable copies. - for (; OpIdx != EndIdx; ++OpIdx) { - const MachineOperand &MO = MI.getOperand(OpIdx); - if (!MO.isReg()) - continue; - Register Reg = MO.getReg(); - if (!Reg) - continue; - - const RegisterBank *AltRegBank = getRegBank(Reg, MRI, TRI); - if (AltRegBank && - cannotCopy(*CurRegBank, *AltRegBank, getSizeInBits(Reg, MRI, TRI))) - return getInvalidInstructionMapping(); - } - - CompleteMapping = true; - break; - } - - OperandsMapping[OpIdx] = ValMapping; - } - - if (IsCopyLike && !CompleteMapping) { - // No way to deduce the type from what we have. - return getInvalidInstructionMapping(); - } - - assert(CompleteMapping && "Setting an uncomplete mapping"); - return getInstructionMapping( - DefaultMappingID, /*Cost*/ 1, - /*OperandsMapping*/ getOperandsMapping(OperandsMapping), - NumOperandsForMapping); -} - -/// Hashing function for PartialMapping. -static hash_code hashPartialMapping(unsigned StartIdx, unsigned Length, - const RegisterBank *RegBank) { - return hash_combine(StartIdx, Length, RegBank ? RegBank->getID() : 0); -} - -/// Overloaded version of hash_value for a PartialMapping. -hash_code -llvm::hash_value(const RegisterBankInfo::PartialMapping &PartMapping) { - return hashPartialMapping(PartMapping.StartIdx, PartMapping.Length, - PartMapping.RegBank); -} - -const RegisterBankInfo::PartialMapping & -RegisterBankInfo::getPartialMapping(unsigned StartIdx, unsigned Length, - const RegisterBank &RegBank) const { - ++NumPartialMappingsAccessed; - - hash_code Hash = hashPartialMapping(StartIdx, Length, &RegBank); - const auto &It = MapOfPartialMappings.find(Hash); - if (It != MapOfPartialMappings.end()) - return *It->second; - - ++NumPartialMappingsCreated; - - auto &PartMapping = MapOfPartialMappings[Hash]; - PartMapping = std::make_unique<PartialMapping>(StartIdx, Length, RegBank); - return *PartMapping; -} - -const RegisterBankInfo::ValueMapping & -RegisterBankInfo::getValueMapping(unsigned StartIdx, unsigned Length, - const RegisterBank &RegBank) const { - return getValueMapping(&getPartialMapping(StartIdx, Length, RegBank), 1); -} - -static hash_code -hashValueMapping(const RegisterBankInfo::PartialMapping *BreakDown, - unsigned NumBreakDowns) { - if (LLVM_LIKELY(NumBreakDowns == 1)) - return hash_value(*BreakDown); - SmallVector<size_t, 8> Hashes(NumBreakDowns); - for (unsigned Idx = 0; Idx != NumBreakDowns; ++Idx) - Hashes.push_back(hash_value(BreakDown[Idx])); - return hash_combine_range(Hashes.begin(), Hashes.end()); -} - -const RegisterBankInfo::ValueMapping & -RegisterBankInfo::getValueMapping(const PartialMapping *BreakDown, - unsigned NumBreakDowns) const { - ++NumValueMappingsAccessed; - - hash_code Hash = hashValueMapping(BreakDown, NumBreakDowns); - const auto &It = MapOfValueMappings.find(Hash); - if (It != MapOfValueMappings.end()) - return *It->second; - - ++NumValueMappingsCreated; - - auto &ValMapping = MapOfValueMappings[Hash]; - ValMapping = std::make_unique<ValueMapping>(BreakDown, NumBreakDowns); - return *ValMapping; -} - -template <typename Iterator> -const RegisterBankInfo::ValueMapping * -RegisterBankInfo::getOperandsMapping(Iterator Begin, Iterator End) const { - - ++NumOperandsMappingsAccessed; - - // The addresses of the value mapping are unique. - // Therefore, we can use them directly to hash the operand mapping. - hash_code Hash = hash_combine_range(Begin, End); - auto &Res = MapOfOperandsMappings[Hash]; - if (Res) - return Res.get(); - - ++NumOperandsMappingsCreated; - - // Create the array of ValueMapping. - // Note: this array will not hash to this instance of operands - // mapping, because we use the pointer of the ValueMapping - // to hash and we expect them to uniquely identify an instance - // of value mapping. - Res = std::make_unique<ValueMapping[]>(std::distance(Begin, End)); - unsigned Idx = 0; - for (Iterator It = Begin; It != End; ++It, ++Idx) { - const ValueMapping *ValMap = *It; - if (!ValMap) - continue; - Res[Idx] = *ValMap; - } - return Res.get(); -} - -const RegisterBankInfo::ValueMapping *RegisterBankInfo::getOperandsMapping( - const SmallVectorImpl<const RegisterBankInfo::ValueMapping *> &OpdsMapping) - const { - return getOperandsMapping(OpdsMapping.begin(), OpdsMapping.end()); -} - -const RegisterBankInfo::ValueMapping *RegisterBankInfo::getOperandsMapping( - std::initializer_list<const RegisterBankInfo::ValueMapping *> OpdsMapping) - const { - return getOperandsMapping(OpdsMapping.begin(), OpdsMapping.end()); -} - -static hash_code -hashInstructionMapping(unsigned ID, unsigned Cost, - const RegisterBankInfo::ValueMapping *OperandsMapping, - unsigned NumOperands) { - return hash_combine(ID, Cost, OperandsMapping, NumOperands); -} - -const RegisterBankInfo::InstructionMapping & -RegisterBankInfo::getInstructionMappingImpl( - bool IsInvalid, unsigned ID, unsigned Cost, - const RegisterBankInfo::ValueMapping *OperandsMapping, - unsigned NumOperands) const { - assert(((IsInvalid && ID == InvalidMappingID && Cost == 0 && - OperandsMapping == nullptr && NumOperands == 0) || - !IsInvalid) && - "Mismatch argument for invalid input"); - ++NumInstructionMappingsAccessed; - - hash_code Hash = - hashInstructionMapping(ID, Cost, OperandsMapping, NumOperands); - const auto &It = MapOfInstructionMappings.find(Hash); - if (It != MapOfInstructionMappings.end()) - return *It->second; - - ++NumInstructionMappingsCreated; - - auto &InstrMapping = MapOfInstructionMappings[Hash]; - InstrMapping = std::make_unique<InstructionMapping>( - ID, Cost, OperandsMapping, NumOperands); - return *InstrMapping; -} - -const RegisterBankInfo::InstructionMapping & -RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { - const RegisterBankInfo::InstructionMapping &Mapping = getInstrMappingImpl(MI); - if (Mapping.isValid()) - return Mapping; - llvm_unreachable("The target must implement this"); -} - -RegisterBankInfo::InstructionMappings -RegisterBankInfo::getInstrPossibleMappings(const MachineInstr &MI) const { - InstructionMappings PossibleMappings; - const auto &Mapping = getInstrMapping(MI); - if (Mapping.isValid()) { - // Put the default mapping first. - PossibleMappings.push_back(&Mapping); - } - - // Then the alternative mapping, if any. - InstructionMappings AltMappings = getInstrAlternativeMappings(MI); - append_range(PossibleMappings, AltMappings); -#ifndef NDEBUG - for (const InstructionMapping *Mapping : PossibleMappings) - assert(Mapping->verify(MI) && "Mapping is invalid"); -#endif - return PossibleMappings; -} - -RegisterBankInfo::InstructionMappings -RegisterBankInfo::getInstrAlternativeMappings(const MachineInstr &MI) const { - // No alternative for MI. - return InstructionMappings(); -} - -void RegisterBankInfo::applyDefaultMapping(const OperandsMapper &OpdMapper) { - MachineInstr &MI = OpdMapper.getMI(); - MachineRegisterInfo &MRI = OpdMapper.getMRI(); - LLVM_DEBUG(dbgs() << "Applying default-like mapping\n"); - for (unsigned OpIdx = 0, - EndIdx = OpdMapper.getInstrMapping().getNumOperands(); - OpIdx != EndIdx; ++OpIdx) { - LLVM_DEBUG(dbgs() << "OpIdx " << OpIdx); - MachineOperand &MO = MI.getOperand(OpIdx); - if (!MO.isReg()) { - LLVM_DEBUG(dbgs() << " is not a register, nothing to be done\n"); - continue; - } - if (!MO.getReg()) { - LLVM_DEBUG(dbgs() << " is $noreg, nothing to be done\n"); - continue; - } - assert(OpdMapper.getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns != - 0 && - "Invalid mapping"); - assert(OpdMapper.getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns == - 1 && - "This mapping is too complex for this function"); - iterator_range<SmallVectorImpl<Register>::const_iterator> NewRegs = - OpdMapper.getVRegs(OpIdx); - if (NewRegs.empty()) { - LLVM_DEBUG(dbgs() << " has not been repaired, nothing to be done\n"); - continue; - } - Register OrigReg = MO.getReg(); - Register NewReg = *NewRegs.begin(); - LLVM_DEBUG(dbgs() << " changed, replace " << printReg(OrigReg, nullptr)); - MO.setReg(NewReg); - LLVM_DEBUG(dbgs() << " with " << printReg(NewReg, nullptr)); - - // The OperandsMapper creates plain scalar, we may have to fix that. - // Check if the types match and if not, fix that. - LLT OrigTy = MRI.getType(OrigReg); - LLT NewTy = MRI.getType(NewReg); - if (OrigTy != NewTy) { - // The default mapping is not supposed to change the size of - // the storage. However, right now we don't necessarily bump all - // the types to storage size. For instance, we can consider - // s16 G_AND legal whereas the storage size is going to be 32. - assert(OrigTy.getSizeInBits() <= NewTy.getSizeInBits() && - "Types with difference size cannot be handled by the default " - "mapping"); - LLVM_DEBUG(dbgs() << "\nChange type of new opd from " << NewTy << " to " - << OrigTy); - MRI.setType(NewReg, OrigTy); - } - LLVM_DEBUG(dbgs() << '\n'); - } -} - -unsigned RegisterBankInfo::getSizeInBits(Register Reg, - const MachineRegisterInfo &MRI, - const TargetRegisterInfo &TRI) const { - if (Register::isPhysicalRegister(Reg)) { - // The size is not directly available for physical registers. - // Instead, we need to access a register class that contains Reg and - // get the size of that register class. - // Because this is expensive, we'll cache the register class by calling - auto *RC = &getMinimalPhysRegClass(Reg, TRI); - assert(RC && "Expecting Register class"); - return TRI.getRegSizeInBits(*RC); - } - return TRI.getRegSizeInBits(Reg, MRI); -} - -//------------------------------------------------------------------------------ -// Helper classes implementation. -//------------------------------------------------------------------------------ -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -LLVM_DUMP_METHOD void RegisterBankInfo::PartialMapping::dump() const { - print(dbgs()); - dbgs() << '\n'; -} -#endif - -bool RegisterBankInfo::PartialMapping::verify() const { - assert(RegBank && "Register bank not set"); - assert(Length && "Empty mapping"); - assert((StartIdx <= getHighBitIdx()) && "Overflow, switch to APInt?"); - // Check if the minimum width fits into RegBank. - assert(RegBank->getSize() >= Length && "Register bank too small for Mask"); - return true; -} - -void RegisterBankInfo::PartialMapping::print(raw_ostream &OS) const { - OS << "[" << StartIdx << ", " << getHighBitIdx() << "], RegBank = "; - if (RegBank) - OS << *RegBank; - else - OS << "nullptr"; -} - -bool RegisterBankInfo::ValueMapping::partsAllUniform() const { - if (NumBreakDowns < 2) - return true; - - const PartialMapping *First = begin(); - for (const PartialMapping *Part = First + 1; Part != end(); ++Part) { - if (Part->Length != First->Length || Part->RegBank != First->RegBank) - return false; - } - - return true; -} - -bool RegisterBankInfo::ValueMapping::verify(unsigned MeaningfulBitWidth) const { - assert(NumBreakDowns && "Value mapped nowhere?!"); - unsigned OrigValueBitWidth = 0; - for (const RegisterBankInfo::PartialMapping &PartMap : *this) { - // Check that each register bank is big enough to hold the partial value: - // this check is done by PartialMapping::verify - assert(PartMap.verify() && "Partial mapping is invalid"); - // The original value should completely be mapped. - // Thus the maximum accessed index + 1 is the size of the original value. - OrigValueBitWidth = - std::max(OrigValueBitWidth, PartMap.getHighBitIdx() + 1); - } - assert(OrigValueBitWidth >= MeaningfulBitWidth && - "Meaningful bits not covered by the mapping"); - APInt ValueMask(OrigValueBitWidth, 0); - for (const RegisterBankInfo::PartialMapping &PartMap : *this) { - // Check that the union of the partial mappings covers the whole value, - // without overlaps. - // The high bit is exclusive in the APInt API, thus getHighBitIdx + 1. - APInt PartMapMask = APInt::getBitsSet(OrigValueBitWidth, PartMap.StartIdx, - PartMap.getHighBitIdx() + 1); - ValueMask ^= PartMapMask; - assert((ValueMask & PartMapMask) == PartMapMask && - "Some partial mappings overlap"); - } - assert(ValueMask.isAllOnes() && "Value is not fully mapped"); - return true; -} - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -LLVM_DUMP_METHOD void RegisterBankInfo::ValueMapping::dump() const { - print(dbgs()); - dbgs() << '\n'; -} -#endif - -void RegisterBankInfo::ValueMapping::print(raw_ostream &OS) const { - OS << "#BreakDown: " << NumBreakDowns << " "; - bool IsFirst = true; - for (const PartialMapping &PartMap : *this) { - if (!IsFirst) - OS << ", "; - OS << '[' << PartMap << ']'; - IsFirst = false; - } -} - -bool RegisterBankInfo::InstructionMapping::verify( - const MachineInstr &MI) const { - // Check that all the register operands are properly mapped. - // Check the constructor invariant. - // For PHI, we only care about mapping the definition. - assert(NumOperands == (isCopyLike(MI) ? 1 : MI.getNumOperands()) && - "NumOperands must match, see constructor"); - assert(MI.getParent() && MI.getMF() && - "MI must be connected to a MachineFunction"); - const MachineFunction &MF = *MI.getMF(); - const RegisterBankInfo *RBI = MF.getSubtarget().getRegBankInfo(); - (void)RBI; - - for (unsigned Idx = 0; Idx < NumOperands; ++Idx) { - const MachineOperand &MO = MI.getOperand(Idx); - if (!MO.isReg()) { - assert(!getOperandMapping(Idx).isValid() && - "We should not care about non-reg mapping"); - continue; - } - Register Reg = MO.getReg(); - if (!Reg) - continue; - assert(getOperandMapping(Idx).isValid() && - "We must have a mapping for reg operands"); - const RegisterBankInfo::ValueMapping &MOMapping = getOperandMapping(Idx); - (void)MOMapping; - // Register size in bits. - // This size must match what the mapping expects. - assert(MOMapping.verify(RBI->getSizeInBits( - Reg, MF.getRegInfo(), *MF.getSubtarget().getRegisterInfo())) && - "Value mapping is invalid"); - } - return true; -} - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -LLVM_DUMP_METHOD void RegisterBankInfo::InstructionMapping::dump() const { - print(dbgs()); - dbgs() << '\n'; -} -#endif - -void RegisterBankInfo::InstructionMapping::print(raw_ostream &OS) const { - OS << "ID: " << getID() << " Cost: " << getCost() << " Mapping: "; - - for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) { - const ValueMapping &ValMapping = getOperandMapping(OpIdx); - if (OpIdx) - OS << ", "; - OS << "{ Idx: " << OpIdx << " Map: " << ValMapping << '}'; - } -} - -const int RegisterBankInfo::OperandsMapper::DontKnowIdx = -1; - -RegisterBankInfo::OperandsMapper::OperandsMapper( - MachineInstr &MI, const InstructionMapping &InstrMapping, - MachineRegisterInfo &MRI) - : MRI(MRI), MI(MI), InstrMapping(InstrMapping) { - unsigned NumOpds = InstrMapping.getNumOperands(); - OpToNewVRegIdx.resize(NumOpds, OperandsMapper::DontKnowIdx); - assert(InstrMapping.verify(MI) && "Invalid mapping for MI"); -} - -iterator_range<SmallVectorImpl<Register>::iterator> -RegisterBankInfo::OperandsMapper::getVRegsMem(unsigned OpIdx) { - assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access"); - unsigned NumPartialVal = - getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns; - int StartIdx = OpToNewVRegIdx[OpIdx]; - - if (StartIdx == OperandsMapper::DontKnowIdx) { - // This is the first time we try to access OpIdx. - // Create the cells that will hold all the partial values at the - // end of the list of NewVReg. - StartIdx = NewVRegs.size(); - OpToNewVRegIdx[OpIdx] = StartIdx; - for (unsigned i = 0; i < NumPartialVal; ++i) - NewVRegs.push_back(0); - } - SmallVectorImpl<Register>::iterator End = - getNewVRegsEnd(StartIdx, NumPartialVal); - - return make_range(&NewVRegs[StartIdx], End); -} - -SmallVectorImpl<Register>::const_iterator -RegisterBankInfo::OperandsMapper::getNewVRegsEnd(unsigned StartIdx, - unsigned NumVal) const { - return const_cast<OperandsMapper *>(this)->getNewVRegsEnd(StartIdx, NumVal); -} -SmallVectorImpl<Register>::iterator -RegisterBankInfo::OperandsMapper::getNewVRegsEnd(unsigned StartIdx, - unsigned NumVal) { - assert((NewVRegs.size() == StartIdx + NumVal || - NewVRegs.size() > StartIdx + NumVal) && - "NewVRegs too small to contain all the partial mapping"); - return NewVRegs.size() <= StartIdx + NumVal ? NewVRegs.end() - : &NewVRegs[StartIdx + NumVal]; -} - -void RegisterBankInfo::OperandsMapper::createVRegs(unsigned OpIdx) { - assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access"); - iterator_range<SmallVectorImpl<Register>::iterator> NewVRegsForOpIdx = - getVRegsMem(OpIdx); - const ValueMapping &ValMapping = getInstrMapping().getOperandMapping(OpIdx); - const PartialMapping *PartMap = ValMapping.begin(); - for (Register &NewVReg : NewVRegsForOpIdx) { - assert(PartMap != ValMapping.end() && "Out-of-bound access"); - assert(NewVReg == 0 && "Register has already been created"); - // The new registers are always bound to scalar with the right size. - // The actual type has to be set when the target does the mapping - // of the instruction. - // The rationale is that this generic code cannot guess how the - // target plans to split the input type. - NewVReg = MRI.createGenericVirtualRegister(LLT::scalar(PartMap->Length)); - MRI.setRegBank(NewVReg, *PartMap->RegBank); - ++PartMap; - } -} - -void RegisterBankInfo::OperandsMapper::setVRegs(unsigned OpIdx, - unsigned PartialMapIdx, - Register NewVReg) { - assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access"); - assert(getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns > - PartialMapIdx && - "Out-of-bound access for partial mapping"); - // Make sure the memory is initialized for that operand. - (void)getVRegsMem(OpIdx); - assert(NewVRegs[OpToNewVRegIdx[OpIdx] + PartialMapIdx] == 0 && - "This value is already set"); - NewVRegs[OpToNewVRegIdx[OpIdx] + PartialMapIdx] = NewVReg; -} - -iterator_range<SmallVectorImpl<Register>::const_iterator> -RegisterBankInfo::OperandsMapper::getVRegs(unsigned OpIdx, - bool ForDebug) const { - (void)ForDebug; - assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access"); - int StartIdx = OpToNewVRegIdx[OpIdx]; - - if (StartIdx == OperandsMapper::DontKnowIdx) - return make_range(NewVRegs.end(), NewVRegs.end()); - - unsigned PartMapSize = - getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns; - SmallVectorImpl<Register>::const_iterator End = - getNewVRegsEnd(StartIdx, PartMapSize); - iterator_range<SmallVectorImpl<Register>::const_iterator> Res = - make_range(&NewVRegs[StartIdx], End); -#ifndef NDEBUG - for (Register VReg : Res) - assert((VReg || ForDebug) && "Some registers are uninitialized"); -#endif - return Res; -} - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -LLVM_DUMP_METHOD void RegisterBankInfo::OperandsMapper::dump() const { - print(dbgs(), true); - dbgs() << '\n'; -} -#endif - -void RegisterBankInfo::OperandsMapper::print(raw_ostream &OS, - bool ForDebug) const { - unsigned NumOpds = getInstrMapping().getNumOperands(); - if (ForDebug) { - OS << "Mapping for " << getMI() << "\nwith " << getInstrMapping() << '\n'; - // Print out the internal state of the index table. - OS << "Populated indices (CellNumber, IndexInNewVRegs): "; - bool IsFirst = true; - for (unsigned Idx = 0; Idx != NumOpds; ++Idx) { - if (OpToNewVRegIdx[Idx] != DontKnowIdx) { - if (!IsFirst) - OS << ", "; - OS << '(' << Idx << ", " << OpToNewVRegIdx[Idx] << ')'; - IsFirst = false; - } - } - OS << '\n'; - } else - OS << "Mapping ID: " << getInstrMapping().getID() << ' '; - - OS << "Operand Mapping: "; - // If we have a function, we can pretty print the name of the registers. - // Otherwise we will print the raw numbers. - const TargetRegisterInfo *TRI = - getMI().getParent() && getMI().getMF() - ? getMI().getMF()->getSubtarget().getRegisterInfo() - : nullptr; - bool IsFirst = true; - for (unsigned Idx = 0; Idx != NumOpds; ++Idx) { - if (OpToNewVRegIdx[Idx] == DontKnowIdx) - continue; - if (!IsFirst) - OS << ", "; - IsFirst = false; - OS << '(' << printReg(getMI().getOperand(Idx).getReg(), TRI) << ", ["; - bool IsFirstNewVReg = true; - for (Register VReg : getVRegs(Idx)) { - if (!IsFirstNewVReg) - OS << ", "; - IsFirstNewVReg = false; - OS << printReg(VReg, TRI); - } - OS << "])"; - } -} diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp index 544af9a2954f..7781761bc131 100644 --- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp @@ -16,14 +16,14 @@ #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" +#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" -#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" -#include "llvm/CodeGen/MachineSizeOpts.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/MachineSizeOpts.h" +#include "llvm/CodeGen/RegisterBankInfo.h" #include "llvm/CodeGen/StackProtector.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" @@ -31,6 +31,7 @@ #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/IR/Constants.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/Utils/SizeOpts.h" #define DEBUG_TYPE "globalisel-utils" @@ -56,6 +57,11 @@ Register llvm::constrainOperandRegClass( // Assume physical registers are properly constrained. assert(Register::isVirtualRegister(Reg) && "PhysReg not implemented"); + // Save the old register class to check whether + // the change notifications will be required. + // TODO: A better approach would be to pass + // the observers to constrainRegToClass(). + auto *OldRegClass = MRI.getRegClassOrNull(Reg); Register ConstrainedReg = constrainRegToClass(MRI, TII, RBI, Reg, RegClass); // If we created a new virtual register because the class is not compatible // then create a copy between the new and the old register. @@ -81,7 +87,7 @@ Register llvm::constrainOperandRegClass( if (GISelChangeObserver *Observer = MF.getObserver()) { Observer->changedInstr(*RegMO.getParent()); } - } else { + } else if (OldRegClass != MRI.getRegClassOrNull(Reg)) { if (GISelChangeObserver *Observer = MF.getObserver()) { if (!RegMO.isDef()) { MachineInstr *RegDef = MRI.getVRegDef(Reg); @@ -500,6 +506,7 @@ Optional<APInt> llvm::ConstantFoldBinOp(unsigned Opcode, const Register Op1, default: break; case TargetOpcode::G_ADD: + case TargetOpcode::G_PTR_ADD: return C1 + C2; case TargetOpcode::G_AND: return C1 & C2; @@ -533,6 +540,14 @@ Optional<APInt> llvm::ConstantFoldBinOp(unsigned Opcode, const Register Op1, if (!C2.getBoolValue()) break; return C1.srem(C2); + case TargetOpcode::G_SMIN: + return APIntOps::smin(C1, C2); + case TargetOpcode::G_SMAX: + return APIntOps::smax(C1, C2); + case TargetOpcode::G_UMIN: + return APIntOps::umin(C1, C2); + case TargetOpcode::G_UMAX: + return APIntOps::umax(C1, C2); } return None; @@ -592,33 +607,27 @@ Optional<APFloat> llvm::ConstantFoldFPBinOp(unsigned Opcode, const Register Op1, return None; } -Register llvm::ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, - const Register Op2, - const MachineRegisterInfo &MRI, - MachineIRBuilder &MIB) { +SmallVector<APInt> +llvm::ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, + const Register Op2, + const MachineRegisterInfo &MRI) { auto *SrcVec2 = getOpcodeDef<GBuildVector>(Op2, MRI); if (!SrcVec2) - return Register(); + return SmallVector<APInt>(); auto *SrcVec1 = getOpcodeDef<GBuildVector>(Op1, MRI); if (!SrcVec1) - return Register(); + return SmallVector<APInt>(); - const LLT EltTy = MRI.getType(SrcVec1->getSourceReg(0)); - - SmallVector<Register, 16> FoldedElements; + SmallVector<APInt> FoldedElements; for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) { auto MaybeCst = ConstantFoldBinOp(Opcode, SrcVec1->getSourceReg(Idx), SrcVec2->getSourceReg(Idx), MRI); if (!MaybeCst) - return Register(); - auto FoldedCstReg = MIB.buildConstant(EltTy, *MaybeCst).getReg(0); - FoldedElements.emplace_back(FoldedCstReg); + return SmallVector<APInt>(); + FoldedElements.push_back(*MaybeCst); } - // Create the new vector constant. - auto CstVec = - MIB.buildBuildVector(MRI.getType(SrcVec1->getReg(0)), FoldedElements); - return CstVec.getReg(0); + return FoldedElements; } bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI, @@ -1061,15 +1070,38 @@ bool llvm::isBuildVectorConstantSplat(const MachineInstr &MI, AllowUndef); } +Optional<APInt> llvm::getIConstantSplatVal(const Register Reg, + const MachineRegisterInfo &MRI) { + if (auto SplatValAndReg = + getAnyConstantSplat(Reg, MRI, /* AllowUndef */ false)) { + Optional<ValueAndVReg> ValAndVReg = + getIConstantVRegValWithLookThrough(SplatValAndReg->VReg, MRI); + return ValAndVReg->Value; + } + + return None; +} + +Optional<APInt> getIConstantSplatVal(const MachineInstr &MI, + const MachineRegisterInfo &MRI) { + return getIConstantSplatVal(MI.getOperand(0).getReg(), MRI); +} + Optional<int64_t> -llvm::getBuildVectorConstantSplat(const MachineInstr &MI, - const MachineRegisterInfo &MRI) { +llvm::getIConstantSplatSExtVal(const Register Reg, + const MachineRegisterInfo &MRI) { if (auto SplatValAndReg = - getAnyConstantSplat(MI.getOperand(0).getReg(), MRI, false)) + getAnyConstantSplat(Reg, MRI, /* AllowUndef */ false)) return getIConstantVRegSExtVal(SplatValAndReg->VReg, MRI); return None; } +Optional<int64_t> +llvm::getIConstantSplatSExtVal(const MachineInstr &MI, + const MachineRegisterInfo &MRI) { + return getIConstantSplatSExtVal(MI.getOperand(0).getReg(), MRI); +} + Optional<FPValueAndVReg> llvm::getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI, bool AllowUndef) { @@ -1095,7 +1127,7 @@ Optional<RegOrConstant> llvm::getVectorSplat(const MachineInstr &MI, unsigned Opc = MI.getOpcode(); if (!isBuildVectorOp(Opc)) return None; - if (auto Splat = getBuildVectorConstantSplat(MI, MRI)) + if (auto Splat = getIConstantSplatSExtVal(MI, MRI)) return RegOrConstant(*Splat); auto Reg = MI.getOperand(1).getReg(); if (any_of(make_range(MI.operands_begin() + 2, MI.operands_end()), @@ -1104,6 +1136,26 @@ Optional<RegOrConstant> llvm::getVectorSplat(const MachineInstr &MI, return RegOrConstant(Reg); } +static bool isConstantScalar(const MachineInstr &MI, + const MachineRegisterInfo &MRI, + bool AllowFP = true, + bool AllowOpaqueConstants = true) { + switch (MI.getOpcode()) { + case TargetOpcode::G_CONSTANT: + case TargetOpcode::G_IMPLICIT_DEF: + return true; + case TargetOpcode::G_FCONSTANT: + return AllowFP; + case TargetOpcode::G_GLOBAL_VALUE: + case TargetOpcode::G_FRAME_INDEX: + case TargetOpcode::G_BLOCK_ADDR: + case TargetOpcode::G_JUMP_TABLE: + return AllowOpaqueConstants; + default: + return false; + } +} + bool llvm::isConstantOrConstantVector(MachineInstr &MI, const MachineRegisterInfo &MRI) { Register Def = MI.getOperand(0).getReg(); @@ -1121,19 +1173,71 @@ bool llvm::isConstantOrConstantVector(MachineInstr &MI, return true; } +bool llvm::isConstantOrConstantVector(const MachineInstr &MI, + const MachineRegisterInfo &MRI, + bool AllowFP, bool AllowOpaqueConstants) { + if (isConstantScalar(MI, MRI, AllowFP, AllowOpaqueConstants)) + return true; + + if (!isBuildVectorOp(MI.getOpcode())) + return false; + + const unsigned NumOps = MI.getNumOperands(); + for (unsigned I = 1; I != NumOps; ++I) { + const MachineInstr *ElementDef = MRI.getVRegDef(MI.getOperand(I).getReg()); + if (!isConstantScalar(*ElementDef, MRI, AllowFP, AllowOpaqueConstants)) + return false; + } + + return true; +} + Optional<APInt> llvm::isConstantOrConstantSplatVector(MachineInstr &MI, const MachineRegisterInfo &MRI) { Register Def = MI.getOperand(0).getReg(); if (auto C = getIConstantVRegValWithLookThrough(Def, MRI)) return C->Value; - auto MaybeCst = getBuildVectorConstantSplat(MI, MRI); + auto MaybeCst = getIConstantSplatSExtVal(MI, MRI); if (!MaybeCst) return None; const unsigned ScalarSize = MRI.getType(Def).getScalarSizeInBits(); return APInt(ScalarSize, *MaybeCst, true); } +bool llvm::isNullOrNullSplat(const MachineInstr &MI, + const MachineRegisterInfo &MRI, bool AllowUndefs) { + switch (MI.getOpcode()) { + case TargetOpcode::G_IMPLICIT_DEF: + return AllowUndefs; + case TargetOpcode::G_CONSTANT: + return MI.getOperand(1).getCImm()->isNullValue(); + case TargetOpcode::G_FCONSTANT: { + const ConstantFP *FPImm = MI.getOperand(1).getFPImm(); + return FPImm->isZero() && !FPImm->isNegative(); + } + default: + if (!AllowUndefs) // TODO: isBuildVectorAllZeros assumes undef is OK already + return false; + return isBuildVectorAllZeros(MI, MRI); + } +} + +bool llvm::isAllOnesOrAllOnesSplat(const MachineInstr &MI, + const MachineRegisterInfo &MRI, + bool AllowUndefs) { + switch (MI.getOpcode()) { + case TargetOpcode::G_IMPLICIT_DEF: + return AllowUndefs; + case TargetOpcode::G_CONSTANT: + return MI.getOperand(1).getCImm()->isAllOnesValue(); + default: + if (!AllowUndefs) // TODO: isBuildVectorAllOnes assumes undef is OK already + return false; + return isBuildVectorAllOnes(MI, MRI); + } +} + bool llvm::matchUnaryPredicate( const MachineRegisterInfo &MRI, Register Reg, std::function<bool(const Constant *ConstVal)> Match, bool AllowUndefs) { |