diff options
Diffstat (limited to 'llvm/lib/Target')
297 files changed, 12339 insertions, 5635 deletions
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td index 548e4e0c9389..cb17fd94c335 100644 --- a/llvm/lib/Target/AArch64/AArch64.td +++ b/llvm/lib/Target/AArch64/AArch64.td @@ -455,6 +455,9 @@ def FeatureEL2VMSA : SubtargetFeature<"el2vmsa", "HasEL2VMSA", "true", def FeatureEL3 : SubtargetFeature<"el3", "HasEL3", "true", "Enable Exception Level 3">; +def FeatureFixCortexA53_835769 : SubtargetFeature<"fix-cortex-a53-835769", + "FixCortexA53_835769", "true", "Mitigate Cortex-A53 Erratum 835769">; + //===----------------------------------------------------------------------===// // Architectures. // diff --git a/llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp b/llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp index 7fd51a98ad94..4cdf5f144437 100644 --- a/llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp +++ b/llvm/lib/Target/AArch64/AArch64A53Fix835769.cpp @@ -15,6 +15,7 @@ //===----------------------------------------------------------------------===// #include "AArch64.h" +#include "AArch64Subtarget.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -116,8 +117,13 @@ INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass", bool AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) { LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n"); + auto &STI = F.getSubtarget<AArch64Subtarget>(); + // Fix not requested, skip pass. + if (!STI.fixCortexA53_835769()) + return false; + bool Changed = false; - TII = F.getSubtarget().getInstrInfo(); + TII = STI.getInstrInfo(); for (auto &MBB : F) { Changed |= runOnBasicBlock(MBB); diff --git a/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp b/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp index cd67e058a9c1..9e31243cd696 100644 --- a/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp +++ b/llvm/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp @@ -398,8 +398,8 @@ bool AArch64AdvSIMDScalar::runOnMachineFunction(MachineFunction &mf) { TII = mf.getSubtarget().getInstrInfo(); // Just check things on a one-block-at-a-time basis. - for (MachineFunction::iterator I = mf.begin(), E = mf.end(); I != E; ++I) - if (processMachineBasicBlock(&*I)) + for (MachineBasicBlock &MBB : mf) + if (processMachineBasicBlock(&MBB)) Changed = true; return Changed; } diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index aeebb49675b2..85a9c04a3fef 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -73,6 +73,7 @@ class AArch64AsmPrinter : public AsmPrinter { StackMaps SM; FaultMaps FM; const AArch64Subtarget *STI; + bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false; public: AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer) @@ -186,6 +187,10 @@ private: using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>; MInstToMCSymbol LOHInstToLabel; + + bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override { + return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags; + } }; } // end anonymous namespace @@ -1132,6 +1137,15 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { if (emitPseudoExpansionLowering(*OutStreamer, MI)) return; + if (MI->getOpcode() == AArch64::ADRP) { + for (auto &Opd : MI->operands()) { + if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) == + "swift_async_extendedFramePointerFlags") { + ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true; + } + } + } + if (AArch64FI->getLOHRelated().count(MI)) { // Generate a label for LOH related instruction MCSymbol *LOHLabel = createTempSymbol("loh"); diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td index d2097f7e6ee3..1994e0eb7fb9 100644 --- a/llvm/lib/Target/AArch64/AArch64Combine.td +++ b/llvm/lib/Target/AArch64/AArch64Combine.td @@ -196,6 +196,13 @@ def mutate_anyext_to_zext : GICombineRule< (apply [{ applyMutateAnyExtToZExt(*${d}, MRI, B, Observer); }]) >; +def split_store_zero_128 : GICombineRule< + (defs root:$d), + (match (wip_match_opcode G_STORE):$d, + [{ return matchSplitStoreZero128(*${d}, MRI); }]), + (apply [{ applySplitStoreZero128(*${d}, MRI, B, Observer); }]) +>; + // Post-legalization combines which should happen at all optimization levels. // (E.g. ones that facilitate matching for the selector) For example, matching // pseudos. @@ -220,6 +227,7 @@ def AArch64PostLegalizerCombinerHelper icmp_to_true_false_known_bits, merge_unmerge, select_combines, fold_merge_to_zext, constant_fold, identity_combines, - ptr_add_immed_chain, overlapping_and]> { + ptr_add_immed_chain, overlapping_and, + split_store_zero_128]> { let DisableRuleOption = "aarch64postlegalizercombiner-disable-rule"; } diff --git a/llvm/lib/Target/AArch64/AArch64ExpandImm.cpp b/llvm/lib/Target/AArch64/AArch64ExpandImm.cpp index d98a5cfd4f50..4f324198f3dc 100644 --- a/llvm/lib/Target/AArch64/AArch64ExpandImm.cpp +++ b/llvm/lib/Target/AArch64/AArch64ExpandImm.cpp @@ -51,10 +51,9 @@ static bool tryToreplicateChunks(uint64_t UImm, ++Counts[getChunk(UImm, Idx)]; // Traverse the chunks to find one which occurs more than once. - for (CountMap::const_iterator Chunk = Counts.begin(), End = Counts.end(); - Chunk != End; ++Chunk) { - const uint64_t ChunkVal = Chunk->first; - const unsigned Count = Chunk->second; + for (const auto &Chunk : Counts) { + const uint64_t ChunkVal = Chunk.first; + const unsigned Count = Chunk.second; uint64_t Encoding = 0; diff --git a/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp b/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp index 209f9f7255a5..793663ef97d7 100644 --- a/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp +++ b/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp @@ -138,8 +138,8 @@ bool FalkorMarkStridedAccesses::run() { bool MadeChange = false; for (Loop *L : LI) - for (auto LIt = df_begin(L), LE = df_end(L); LIt != LE; ++LIt) - MadeChange |= runOnLoop(**LIt); + for (Loop *LIt : depth_first(L)) + MadeChange |= runOnLoop(*LIt); return MadeChange; } @@ -828,10 +828,10 @@ bool FalkorHWPFFix::runOnMachineFunction(MachineFunction &Fn) { Modified = false; for (MachineLoop *I : LI) - for (auto L = df_begin(I), LE = df_end(I); L != LE; ++L) + for (MachineLoop *L : depth_first(I)) // Only process inner-loops if (L->isInnermost()) - runOnLoop(**L, Fn); + runOnLoop(*L, Fn); return Modified; } diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp index b630f4f0df5f..638e45b30d99 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -3041,10 +3041,21 @@ static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI, // Create a buffer of SVE objects to allocate and sort it. SmallVector<int, 8> ObjectsToAllocate; + // If we have a stack protector, and we've previously decided that we have SVE + // objects on the stack and thus need it to go in the SVE stack area, then it + // needs to go first. + int StackProtectorFI = -1; + if (MFI.hasStackProtectorIndex()) { + StackProtectorFI = MFI.getStackProtectorIndex(); + if (MFI.getStackID(StackProtectorFI) == TargetStackID::ScalableVector) + ObjectsToAllocate.push_back(StackProtectorFI); + } for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) { unsigned StackID = MFI.getStackID(I); if (StackID != TargetStackID::ScalableVector) continue; + if (I == StackProtectorFI) + continue; if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex) continue; if (MFI.isDeadObjectIndex(I)) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 72461aa1f772..e141179fb5c8 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -33,6 +33,7 @@ #include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/CallingConvLower.h" +#include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -204,6 +205,8 @@ static bool isMergePassthruOpcode(unsigned Opc) { return false; case AArch64ISD::BITREVERSE_MERGE_PASSTHRU: case AArch64ISD::BSWAP_MERGE_PASSTHRU: + case AArch64ISD::REVH_MERGE_PASSTHRU: + case AArch64ISD::REVW_MERGE_PASSTHRU: case AArch64ISD::CTLZ_MERGE_PASSTHRU: case AArch64ISD::CTPOP_MERGE_PASSTHRU: case AArch64ISD::DUP_MERGE_PASSTHRU: @@ -2227,6 +2230,8 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const { MAKE_CASE(AArch64ISD::STNP) MAKE_CASE(AArch64ISD::BITREVERSE_MERGE_PASSTHRU) MAKE_CASE(AArch64ISD::BSWAP_MERGE_PASSTHRU) + MAKE_CASE(AArch64ISD::REVH_MERGE_PASSTHRU) + MAKE_CASE(AArch64ISD::REVW_MERGE_PASSTHRU) MAKE_CASE(AArch64ISD::CTLZ_MERGE_PASSTHRU) MAKE_CASE(AArch64ISD::CTPOP_MERGE_PASSTHRU) MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU) @@ -4213,6 +4218,12 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, case Intrinsic::aarch64_sve_revb: return DAG.getNode(AArch64ISD::BSWAP_MERGE_PASSTHRU, dl, Op.getValueType(), Op.getOperand(2), Op.getOperand(3), Op.getOperand(1)); + case Intrinsic::aarch64_sve_revh: + return DAG.getNode(AArch64ISD::REVH_MERGE_PASSTHRU, dl, Op.getValueType(), + Op.getOperand(2), Op.getOperand(3), Op.getOperand(1)); + case Intrinsic::aarch64_sve_revw: + return DAG.getNode(AArch64ISD::REVW_MERGE_PASSTHRU, dl, Op.getValueType(), + Op.getOperand(2), Op.getOperand(3), Op.getOperand(1)); case Intrinsic::aarch64_sve_sxtb: return DAG.getNode( AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(), @@ -10958,16 +10969,15 @@ SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op, EVT InVT = Op.getOperand(1).getValueType(); unsigned Idx = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue(); - if (InVT.isScalableVector()) { - SDLoc DL(Op); - EVT VT = Op.getValueType(); + SDValue Vec0 = Op.getOperand(0); + SDValue Vec1 = Op.getOperand(1); + SDLoc DL(Op); + EVT VT = Op.getValueType(); + if (InVT.isScalableVector()) { if (!isTypeLegal(VT)) return SDValue(); - SDValue Vec0 = Op.getOperand(0); - SDValue Vec1 = Op.getOperand(1); - // Ensure the subvector is half the size of the main vector. if (VT.getVectorElementCount() != (InVT.getVectorElementCount() * 2)) return SDValue(); @@ -10997,9 +11007,18 @@ SDValue AArch64TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op, return SDValue(); } - // This will be matched by custom code during ISelDAGToDAG. - if (Idx == 0 && isPackedVectorType(InVT, DAG) && Op.getOperand(0).isUndef()) - return Op; + if (Idx == 0 && isPackedVectorType(VT, DAG)) { + // This will be matched by custom code during ISelDAGToDAG. + if (Vec0.isUndef()) + return Op; + + unsigned int PredPattern = + getSVEPredPatternFromNumElements(InVT.getVectorNumElements()); + auto PredTy = VT.changeVectorElementType(MVT::i1); + SDValue PTrue = getPTrue(DAG, DL, PredTy, PredPattern); + SDValue ScalableVec1 = convertToScalableVector(DAG, VT, Vec1); + return DAG.getNode(ISD::VSELECT, DL, VT, PTrue, ScalableVec1, Vec0); + } return SDValue(); } @@ -11794,6 +11813,9 @@ bool AArch64TargetLowering::shouldReduceLoadWidth(SDNode *Load, Base.getOperand(1).getOpcode() == ISD::SHL && Base.getOperand(1).hasOneUse() && Base.getOperand(1).getOperand(1).getOpcode() == ISD::Constant) { + // It's unknown whether a scalable vector has a power-of-2 bitwidth. + if (Mem->getMemoryVT().isScalableVector()) + return false; // The shift can be combined if it matches the size of the value being // loaded (and so reducing the width would make it not match). uint64_t ShiftAmount = Base.getOperand(1).getConstantOperandVal(1); @@ -15820,6 +15842,23 @@ static SDValue performVectorShiftCombine(SDNode *N, return SDValue(); } +static SDValue performSunpkloCombine(SDNode *N, SelectionDAG &DAG) { + // sunpklo(sext(pred)) -> sext(extract_low_half(pred)) + // This transform works in partnership with performSetCCPunpkCombine to + // remove unnecessary transfer of predicates into standard registers and back + if (N->getOperand(0).getOpcode() == ISD::SIGN_EXTEND && + N->getOperand(0)->getOperand(0)->getValueType(0).getScalarType() == + MVT::i1) { + SDValue CC = N->getOperand(0)->getOperand(0); + auto VT = CC->getValueType(0).getHalfNumVectorElementsVT(*DAG.getContext()); + SDValue Unpk = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, CC, + DAG.getVectorIdxConstant(0, SDLoc(N))); + return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), N->getValueType(0), Unpk); + } + + return SDValue(); +} + /// Target-specific DAG combine function for post-increment LD1 (lane) and /// post-increment LD1R. static SDValue performPostLD1Combine(SDNode *N, @@ -15982,7 +16021,9 @@ static SDValue performSTORECombine(SDNode *N, if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND && Value.getNode()->hasOneUse() && ST->isUnindexed() && Subtarget->useSVEForFixedLengthVectors() && - Value.getValueType().isFixedLengthVector()) + Value.getValueType().isFixedLengthVector() && + Value.getValueType().getFixedSizeInBits() >= + Subtarget->getMinSVEVectorSizeInBits()) return DAG.getTruncStore(Chain, SDLoc(N), Value.getOperand(0), Ptr, ST->getMemoryVT(), ST->getMemOperand()); @@ -16495,6 +16536,44 @@ static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG) { return SDValue(); } +static SDValue performSetCCPunpkCombine(SDNode *N, SelectionDAG &DAG) { + // setcc_merge_zero pred + // (sign_extend (extract_subvector (setcc_merge_zero ... pred ...))), 0, ne + // => extract_subvector (inner setcc_merge_zero) + SDValue Pred = N->getOperand(0); + SDValue LHS = N->getOperand(1); + SDValue RHS = N->getOperand(2); + ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(3))->get(); + + if (Cond != ISD::SETNE || !isZerosVector(RHS.getNode()) || + LHS->getOpcode() != ISD::SIGN_EXTEND) + return SDValue(); + + SDValue Extract = LHS->getOperand(0); + if (Extract->getOpcode() != ISD::EXTRACT_SUBVECTOR || + Extract->getValueType(0) != N->getValueType(0) || + Extract->getConstantOperandVal(1) != 0) + return SDValue(); + + SDValue InnerSetCC = Extract->getOperand(0); + if (InnerSetCC->getOpcode() != AArch64ISD::SETCC_MERGE_ZERO) + return SDValue(); + + // By this point we've effectively got + // zero_inactive_lanes_and_trunc_i1(sext_i1(A)). If we can prove A's inactive + // lanes are already zero then the trunc(sext()) sequence is redundant and we + // can operate on A directly. + SDValue InnerPred = InnerSetCC.getOperand(0); + if (Pred.getOpcode() == AArch64ISD::PTRUE && + InnerPred.getOpcode() == AArch64ISD::PTRUE && + Pred.getConstantOperandVal(0) == InnerPred.getConstantOperandVal(0) && + Pred->getConstantOperandVal(0) >= AArch64SVEPredPattern::vl1 && + Pred->getConstantOperandVal(0) <= AArch64SVEPredPattern::vl256) + return Extract; + + return SDValue(); +} + static SDValue performSetccMergeZeroCombine(SDNode *N, SelectionDAG &DAG) { assert(N->getOpcode() == AArch64ISD::SETCC_MERGE_ZERO && "Unexpected opcode!"); @@ -16513,6 +16592,9 @@ static SDValue performSetccMergeZeroCombine(SDNode *N, SelectionDAG &DAG) { LHS->getOperand(0)->getOperand(0) == Pred) return LHS->getOperand(0); + if (SDValue V = performSetCCPunpkCombine(N, DAG)) + return V; + return SDValue(); } @@ -17343,7 +17425,8 @@ SDValue performFPExtendCombine(SDNode *N, SelectionDAG &DAG, // they can be split down into something legal. if (DCI.isBeforeLegalizeOps() && ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && Subtarget->useSVEForFixedLengthVectors() && - VT.isFixedLengthVector()) { + VT.isFixedLengthVector() && + VT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits()) { LoadSDNode *LN0 = cast<LoadSDNode>(N0); SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, LN0->getChain(), LN0->getBasePtr(), @@ -17455,6 +17538,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, case AArch64ISD::VASHR: case AArch64ISD::VLSHR: return performVectorShiftCombine(N, *this, DCI); + case AArch64ISD::SUNPKLO: + return performSunpkloCombine(N, DAG); case ISD::INSERT_VECTOR_ELT: return performInsertVectorEltCombine(N, DCI); case ISD::EXTRACT_VECTOR_ELT: @@ -18570,7 +18655,25 @@ AArch64TargetLowering::getVaListSizeInBits(const DataLayout &DL) const { } void AArch64TargetLowering::finalizeLowering(MachineFunction &MF) const { - MF.getFrameInfo().computeMaxCallFrameSize(MF); + MachineFrameInfo &MFI = MF.getFrameInfo(); + // If we have any vulnerable SVE stack objects then the stack protector + // needs to be placed at the top of the SVE stack area, as the SVE locals + // are placed above the other locals, so we allocate it as if it were a + // scalable vector. + // FIXME: It may be worthwhile having a specific interface for this rather + // than doing it here in finalizeLowering. + if (MFI.hasStackProtectorIndex()) { + for (unsigned int i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) { + if (MFI.getStackID(i) == TargetStackID::ScalableVector && + MFI.getObjectSSPLayout(i) != MachineFrameInfo::SSPLK_None) { + MFI.setStackID(MFI.getStackProtectorIndex(), + TargetStackID::ScalableVector); + MFI.setObjectAlignment(MFI.getStackProtectorIndex(), Align(16)); + break; + } + } + } + MFI.computeMaxCallFrameSize(MF); TargetLoweringBase::finalizeLowering(MF); } @@ -18855,10 +18958,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorStoreToSVE( SDValue AArch64TargetLowering::LowerFixedLengthVectorMStoreToSVE( SDValue Op, SelectionDAG &DAG) const { - auto Store = cast<MaskedStoreSDNode>(Op); - - if (Store->isTruncatingStore()) - return SDValue(); + auto *Store = cast<MaskedStoreSDNode>(Op); SDLoc DL(Op); EVT VT = Store->getValue().getValueType(); @@ -19103,7 +19203,7 @@ SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op, if (isMergePassthruOpcode(NewOp)) Operands.push_back(DAG.getUNDEF(VT)); - return DAG.getNode(NewOp, DL, VT, Operands); + return DAG.getNode(NewOp, DL, VT, Operands, Op->getFlags()); } // If a fixed length vector operation has no side effects when applied to @@ -19498,6 +19598,94 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE( return convertFromScalableVector(DAG, VT, Op); } + for (unsigned LaneSize : {64U, 32U, 16U}) { + if (isREVMask(ShuffleMask, VT, LaneSize)) { + EVT NewVT = + getPackedSVEVectorVT(EVT::getIntegerVT(*DAG.getContext(), LaneSize)); + unsigned RevOp; + unsigned EltSz = VT.getScalarSizeInBits(); + if (EltSz == 8) + RevOp = AArch64ISD::BSWAP_MERGE_PASSTHRU; + else if (EltSz == 16) + RevOp = AArch64ISD::REVH_MERGE_PASSTHRU; + else + RevOp = AArch64ISD::REVW_MERGE_PASSTHRU; + + Op = DAG.getNode(ISD::BITCAST, DL, NewVT, Op1); + Op = LowerToPredicatedOp(Op, DAG, RevOp); + Op = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Op); + return convertFromScalableVector(DAG, VT, Op); + } + } + + unsigned WhichResult; + if (isZIPMask(ShuffleMask, VT, WhichResult) && WhichResult == 0) + return convertFromScalableVector( + DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op2)); + + if (isTRNMask(ShuffleMask, VT, WhichResult)) { + unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2; + return convertFromScalableVector( + DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2)); + } + + if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult == 0) + return convertFromScalableVector( + DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op1)); + + if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) { + unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2; + return convertFromScalableVector( + DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1)); + } + + // Functions like isZIPMask return true when a ISD::VECTOR_SHUFFLE's mask + // represents the same logical operation as performed by a ZIP instruction. In + // isolation these functions do not mean the ISD::VECTOR_SHUFFLE is exactly + // equivalent to an AArch64 instruction. There's the extra component of + // ISD::VECTOR_SHUFFLE's value type to consider. Prior to SVE these functions + // only operated on 64/128bit vector types that have a direct mapping to a + // target register and so an exact mapping is implied. + // However, when using SVE for fixed length vectors, most legal vector types + // are actually sub-vectors of a larger SVE register. When mapping + // ISD::VECTOR_SHUFFLE to an SVE instruction care must be taken to consider + // how the mask's indices translate. Specifically, when the mapping requires + // an exact meaning for a specific vector index (e.g. Index X is the last + // vector element in the register) then such mappings are often only safe when + // the exact SVE register size is know. The main exception to this is when + // indices are logically relative to the first element of either + // ISD::VECTOR_SHUFFLE operand because these relative indices don't change + // when converting from fixed-length to scalable vector types (i.e. the start + // of a fixed length vector is always the start of a scalable vector). + unsigned MinSVESize = Subtarget->getMinSVEVectorSizeInBits(); + unsigned MaxSVESize = Subtarget->getMaxSVEVectorSizeInBits(); + if (MinSVESize == MaxSVESize && MaxSVESize == VT.getSizeInBits()) { + if (ShuffleVectorInst::isReverseMask(ShuffleMask) && Op2.isUndef()) { + Op = DAG.getNode(ISD::VECTOR_REVERSE, DL, ContainerVT, Op1); + return convertFromScalableVector(DAG, VT, Op); + } + + if (isZIPMask(ShuffleMask, VT, WhichResult) && WhichResult != 0) + return convertFromScalableVector( + DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op2)); + + if (isUZPMask(ShuffleMask, VT, WhichResult)) { + unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2; + return convertFromScalableVector( + DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op2)); + } + + if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult) && WhichResult != 0) + return convertFromScalableVector( + DAG, VT, DAG.getNode(AArch64ISD::ZIP2, DL, ContainerVT, Op1, Op1)); + + if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) { + unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2; + return convertFromScalableVector( + DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1)); + } + } + return SDValue(); } diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index ea884cdccd28..367ba3039a0c 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -324,6 +324,8 @@ enum NodeType : unsigned { BITREVERSE_MERGE_PASSTHRU, BSWAP_MERGE_PASSTHRU, + REVH_MERGE_PASSTHRU, + REVW_MERGE_PASSTHRU, CTLZ_MERGE_PASSTHRU, CTPOP_MERGE_PASSTHRU, DUP_MERGE_PASSTHRU, diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td index cd4bc8a61a8a..f8d492188744 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -387,16 +387,16 @@ def simm7s16 : Operand<i32> { let PrintMethod = "printImmScale<16>"; } -def am_sve_fi : ComplexPattern<i64, 2, "SelectAddrModeFrameIndexSVE", []>; +def am_sve_fi : ComplexPattern<iPTR, 2, "SelectAddrModeFrameIndexSVE", []>; -def am_indexed7s8 : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S8", []>; -def am_indexed7s16 : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S16", []>; -def am_indexed7s32 : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S32", []>; -def am_indexed7s64 : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S64", []>; -def am_indexed7s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexed7S128", []>; +def am_indexed7s8 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S8", []>; +def am_indexed7s16 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S16", []>; +def am_indexed7s32 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S32", []>; +def am_indexed7s64 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S64", []>; +def am_indexed7s128 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed7S128", []>; -def am_indexedu6s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexedU6S128", []>; -def am_indexeds9s128 : ComplexPattern<i64, 2, "SelectAddrModeIndexedS9S128", []>; +def am_indexedu6s128 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedU6S128", []>; +def am_indexeds9s128 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedS9S128", []>; def UImmS1XForm : SDNodeXForm<imm, [{ return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i64); @@ -3177,18 +3177,18 @@ def maski16_or_more : Operand<i32>, // (unsigned immediate) // Indexed for 8-bit registers. offset is in range [0,4095]. -def am_indexed8 : ComplexPattern<i64, 2, "SelectAddrModeIndexed8", []>; -def am_indexed16 : ComplexPattern<i64, 2, "SelectAddrModeIndexed16", []>; -def am_indexed32 : ComplexPattern<i64, 2, "SelectAddrModeIndexed32", []>; -def am_indexed64 : ComplexPattern<i64, 2, "SelectAddrModeIndexed64", []>; -def am_indexed128 : ComplexPattern<i64, 2, "SelectAddrModeIndexed128", []>; +def am_indexed8 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed8", []>; +def am_indexed16 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed16", []>; +def am_indexed32 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed32", []>; +def am_indexed64 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed64", []>; +def am_indexed128 : ComplexPattern<iPTR, 2, "SelectAddrModeIndexed128", []>; // (unsigned immediate) // Indexed for 8-bit registers. offset is in range [0,63]. -def am_indexed8_6b : ComplexPattern<i64, 2, "SelectAddrModeIndexedUImm<1,63>", []>; -def am_indexed16_6b : ComplexPattern<i64, 2, "SelectAddrModeIndexedUImm<2,63>", []>; -def am_indexed32_6b : ComplexPattern<i64, 2, "SelectAddrModeIndexedUImm<4,63>", []>; -def am_indexed64_6b : ComplexPattern<i64, 2, "SelectAddrModeIndexedUImm<8,63>", []>; +def am_indexed8_6b : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedUImm<1,63>", []>; +def am_indexed16_6b : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedUImm<2,63>", []>; +def am_indexed32_6b : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedUImm<4,63>", []>; +def am_indexed64_6b : ComplexPattern<iPTR, 2, "SelectAddrModeIndexedUImm<8,63>", []>; def gi_am_indexed8 : GIComplexOperandMatcher<s64, "selectAddrModeIndexed<8>">, @@ -3358,11 +3358,11 @@ class PrefetchLiteral<bits<2> opc, bit V, string asm, list<dag> pat> // Load/store register offset //--- -def ro_Xindexed8 : ComplexPattern<i64, 4, "SelectAddrModeXRO<8>", []>; -def ro_Xindexed16 : ComplexPattern<i64, 4, "SelectAddrModeXRO<16>", []>; -def ro_Xindexed32 : ComplexPattern<i64, 4, "SelectAddrModeXRO<32>", []>; -def ro_Xindexed64 : ComplexPattern<i64, 4, "SelectAddrModeXRO<64>", []>; -def ro_Xindexed128 : ComplexPattern<i64, 4, "SelectAddrModeXRO<128>", []>; +def ro_Xindexed8 : ComplexPattern<iPTR, 4, "SelectAddrModeXRO<8>", []>; +def ro_Xindexed16 : ComplexPattern<iPTR, 4, "SelectAddrModeXRO<16>", []>; +def ro_Xindexed32 : ComplexPattern<iPTR, 4, "SelectAddrModeXRO<32>", []>; +def ro_Xindexed64 : ComplexPattern<iPTR, 4, "SelectAddrModeXRO<64>", []>; +def ro_Xindexed128 : ComplexPattern<iPTR, 4, "SelectAddrModeXRO<128>", []>; def gi_ro_Xindexed8 : GIComplexOperandMatcher<s64, "selectAddrModeXRO<8>">, @@ -3380,11 +3380,11 @@ def gi_ro_Xindexed128 : GIComplexOperandMatcher<s64, "selectAddrModeXRO<128>">, GIComplexPatternEquiv<ro_Xindexed128>; -def ro_Windexed8 : ComplexPattern<i64, 4, "SelectAddrModeWRO<8>", []>; -def ro_Windexed16 : ComplexPattern<i64, 4, "SelectAddrModeWRO<16>", []>; -def ro_Windexed32 : ComplexPattern<i64, 4, "SelectAddrModeWRO<32>", []>; -def ro_Windexed64 : ComplexPattern<i64, 4, "SelectAddrModeWRO<64>", []>; -def ro_Windexed128 : ComplexPattern<i64, 4, "SelectAddrModeWRO<128>", []>; +def ro_Windexed8 : ComplexPattern<iPTR, 4, "SelectAddrModeWRO<8>", []>; +def ro_Windexed16 : ComplexPattern<iPTR, 4, "SelectAddrModeWRO<16>", []>; +def ro_Windexed32 : ComplexPattern<iPTR, 4, "SelectAddrModeWRO<32>", []>; +def ro_Windexed64 : ComplexPattern<iPTR, 4, "SelectAddrModeWRO<64>", []>; +def ro_Windexed128 : ComplexPattern<iPTR, 4, "SelectAddrModeWRO<128>", []>; def gi_ro_Windexed8 : GIComplexOperandMatcher<s64, "selectAddrModeWRO<8>">, @@ -3880,11 +3880,11 @@ multiclass PrefetchRO<bits<2> sz, bit V, bits<2> opc, string asm> { // Load/store unscaled immediate //--- -def am_unscaled8 : ComplexPattern<i64, 2, "SelectAddrModeUnscaled8", []>; -def am_unscaled16 : ComplexPattern<i64, 2, "SelectAddrModeUnscaled16", []>; -def am_unscaled32 : ComplexPattern<i64, 2, "SelectAddrModeUnscaled32", []>; -def am_unscaled64 : ComplexPattern<i64, 2, "SelectAddrModeUnscaled64", []>; -def am_unscaled128 :ComplexPattern<i64, 2, "SelectAddrModeUnscaled128", []>; +def am_unscaled8 : ComplexPattern<iPTR, 2, "SelectAddrModeUnscaled8", []>; +def am_unscaled16 : ComplexPattern<iPTR, 2, "SelectAddrModeUnscaled16", []>; +def am_unscaled32 : ComplexPattern<iPTR, 2, "SelectAddrModeUnscaled32", []>; +def am_unscaled64 : ComplexPattern<iPTR, 2, "SelectAddrModeUnscaled64", []>; +def am_unscaled128 :ComplexPattern<iPTR, 2, "SelectAddrModeUnscaled128", []>; def gi_am_unscaled8 : GIComplexOperandMatcher<s64, "selectAddrModeUnscaled8">, diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index f8f8ee3f1e6c..5fc5e4e5eb35 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -7055,6 +7055,8 @@ bool AArch64InstrInfo::isFunctionSafeToOutlineFrom( bool AArch64InstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const { + if (!TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags)) + return false; // Check if LR is available through all of the MBB. If it's not, then set // a flag. assert(MBB.getParent()->getRegInfo().tracksLiveness() && diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index decee117d2d5..ebccc07edc7a 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -4174,6 +4174,21 @@ defm CMLT : SIMDCmpTwoVector<0, 0b01010, "cmlt", AArch64cmltz>; defm CNT : SIMDTwoVectorB<0, 0b00, 0b00101, "cnt", ctpop>; defm FABS : SIMDTwoVectorFP<0, 1, 0b01111, "fabs", fabs>; +def : Pat<(v8i8 (AArch64vashr (v8i8 V64:$Rn), (i32 7))), + (CMLTv8i8rz V64:$Rn)>; +def : Pat<(v4i16 (AArch64vashr (v4i16 V64:$Rn), (i32 15))), + (CMLTv4i16rz V64:$Rn)>; +def : Pat<(v2i32 (AArch64vashr (v2i32 V64:$Rn), (i32 31))), + (CMLTv2i32rz V64:$Rn)>; +def : Pat<(v16i8 (AArch64vashr (v16i8 V128:$Rn), (i32 7))), + (CMLTv16i8rz V128:$Rn)>; +def : Pat<(v8i16 (AArch64vashr (v8i16 V128:$Rn), (i32 15))), + (CMLTv8i16rz V128:$Rn)>; +def : Pat<(v4i32 (AArch64vashr (v4i32 V128:$Rn), (i32 31))), + (CMLTv4i32rz V128:$Rn)>; +def : Pat<(v2i64 (AArch64vashr (v2i64 V128:$Rn), (i32 63))), + (CMLTv2i64rz V128:$Rn)>; + defm FCMEQ : SIMDFPCmpTwoVector<0, 1, 0b01101, "fcmeq", AArch64fcmeqz>; defm FCMGE : SIMDFPCmpTwoVector<1, 1, 0b01100, "fcmge", AArch64fcmgez>; defm FCMGT : SIMDFPCmpTwoVector<0, 1, 0b01100, "fcmgt", AArch64fcmgtz>; @@ -4363,6 +4378,32 @@ def : Pat<(v4i16 (trunc (smax (smin (v4i32 V128:$Vn), (v4i32 VImm7FFF)), (v4i32 VImm8000)))), (SQXTNv4i16 V128:$Vn)>; +// concat_vectors(Vd, trunc(smin(smax Vm, -128), 127) ~> SQXTN2(Vd, Vn) +// with reversed min/max +def : Pat<(v16i8 (concat_vectors + (v8i8 V64:$Vd), + (v8i8 (trunc (smin (smax (v8i16 V128:$Vn), (v8i16 VImm80)), + (v8i16 VImm7F)))))), + (SQXTNv16i8 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>; +def : Pat<(v16i8 (concat_vectors + (v8i8 V64:$Vd), + (v8i8 (trunc (smax (smin (v8i16 V128:$Vn), (v8i16 VImm7F)), + (v8i16 VImm80)))))), + (SQXTNv16i8 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>; + +// concat_vectors(Vd, trunc(smin(smax Vm, -32768), 32767) ~> SQXTN2(Vd, Vn) +// with reversed min/max +def : Pat<(v8i16 (concat_vectors + (v4i16 V64:$Vd), + (v4i16 (trunc (smin (smax (v4i32 V128:$Vn), (v4i32 VImm8000)), + (v4i32 VImm7FFF)))))), + (SQXTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>; +def : Pat<(v8i16 (concat_vectors + (v4i16 V64:$Vd), + (v4i16 (trunc (smax (smin (v4i32 V128:$Vn), (v4i32 VImm7FFF)), + (v4i32 VImm8000)))))), + (SQXTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>; + //===----------------------------------------------------------------------===// // Advanced SIMD three vector instructions. //===----------------------------------------------------------------------===// @@ -4825,6 +4866,9 @@ defm UQXTN : SIMDTwoScalarMixedBHS<1, 0b10100, "uqxtn", int_aarch64_neon_scalar defm USQADD : SIMDTwoScalarBHSDTied< 1, 0b00011, "usqadd", int_aarch64_neon_usqadd>; +def : Pat<(v1i64 (AArch64vashr (v1i64 V64:$Rn), (i32 63))), + (CMLTv1i64rz V64:$Rn)>; + def : Pat<(v1i64 (int_aarch64_neon_fcvtas (v1f64 FPR64:$Rn))), (FCVTASv1i64 FPR64:$Rn)>; def : Pat<(v1i64 (int_aarch64_neon_fcvtau (v1f64 FPR64:$Rn))), @@ -5288,6 +5332,29 @@ defm UZP2 : SIMDZipVector<0b101, "uzp2", AArch64uzp2>; defm ZIP1 : SIMDZipVector<0b011, "zip1", AArch64zip1>; defm ZIP2 : SIMDZipVector<0b111, "zip2", AArch64zip2>; +def : Pat<(v16i8 (concat_vectors (v8i8 (trunc (v8i16 V128:$Vn))), + (v8i8 (trunc (v8i16 V128:$Vm))))), + (UZP1v16i8 V128:$Vn, V128:$Vm)>; +def : Pat<(v8i16 (concat_vectors (v4i16 (trunc (v4i32 V128:$Vn))), + (v4i16 (trunc (v4i32 V128:$Vm))))), + (UZP1v8i16 V128:$Vn, V128:$Vm)>; +def : Pat<(v4i32 (concat_vectors (v2i32 (trunc (v2i64 V128:$Vn))), + (v2i32 (trunc (v2i64 V128:$Vm))))), + (UZP1v4i32 V128:$Vn, V128:$Vm)>; + +def : Pat<(v16i8 (concat_vectors + (v8i8 (trunc (AArch64vlshr (v8i16 V128:$Vn), (i32 8)))), + (v8i8 (trunc (AArch64vlshr (v8i16 V128:$Vm), (i32 8)))))), + (UZP2v16i8 V128:$Vn, V128:$Vm)>; +def : Pat<(v8i16 (concat_vectors + (v4i16 (trunc (AArch64vlshr (v4i32 V128:$Vn), (i32 16)))), + (v4i16 (trunc (AArch64vlshr (v4i32 V128:$Vm), (i32 16)))))), + (UZP2v8i16 V128:$Vn, V128:$Vm)>; +def : Pat<(v4i32 (concat_vectors + (v2i32 (trunc (AArch64vlshr (v2i64 V128:$Vn), (i32 32)))), + (v2i32 (trunc (AArch64vlshr (v2i64 V128:$Vm), (i32 32)))))), + (UZP2v4i32 V128:$Vn, V128:$Vm)>; + //---------------------------------------------------------------------------- // AdvSIMD TBL/TBX instructions //---------------------------------------------------------------------------- @@ -6536,6 +6603,34 @@ defm USHR : SIMDVectorRShiftBHSD<1, 0b00000, "ushr", AArch64vlshr>; defm USRA : SIMDVectorRShiftBHSDTied<1, 0b00010, "usra", TriOpFrag<(add node:$LHS, (AArch64vlshr node:$MHS, node:$RHS))> >; +// RADDHN patterns for when RSHRN shifts by half the size of the vector element +def : Pat<(v8i8 (int_aarch64_neon_rshrn (v8i16 V128:$Vn), (i32 8))), + (RADDHNv8i16_v8i8 V128:$Vn, (v8i16 (MOVIv2d_ns (i32 0))))>; +def : Pat<(v4i16 (int_aarch64_neon_rshrn (v4i32 V128:$Vn), (i32 16))), + (RADDHNv4i32_v4i16 V128:$Vn, (v4i32 (MOVIv2d_ns (i32 0))))>; +def : Pat<(v2i32 (int_aarch64_neon_rshrn (v2i64 V128:$Vn), (i32 32))), + (RADDHNv2i64_v2i32 V128:$Vn, (v2i64 (MOVIv2d_ns (i32 0))))>; + +// RADDHN2 patterns for when RSHRN shifts by half the size of the vector element +def : Pat<(v16i8 (concat_vectors + (v8i8 V64:$Vd), + (v8i8 (int_aarch64_neon_rshrn (v8i16 V128:$Vn), (i32 8))))), + (RADDHNv8i16_v16i8 + (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn, + (v8i16 (MOVIv2d_ns (i32 0))))>; +def : Pat<(v8i16 (concat_vectors + (v4i16 V64:$Vd), + (v4i16 (int_aarch64_neon_rshrn (v4i32 V128:$Vn), (i32 16))))), + (RADDHNv4i32_v8i16 + (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn, + (v4i32 (MOVIv2d_ns (i32 0))))>; +def : Pat<(v4i32 (concat_vectors + (v2i32 V64:$Vd), + (v2i32 (int_aarch64_neon_rshrn (v2i64 V128:$Vn), (i32 32))))), + (RADDHNv2i64_v4i32 + (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn, + (v2i64 (MOVIv2d_ns (i32 0))))>; + // SHRN patterns for when a logical right shift was used instead of arithmetic // (the immediate guarantees no sign bits actually end up in the result so it // doesn't matter). diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 25d53f4ab065..eb55a472a69a 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -136,15 +136,15 @@ def AArch64stnt1_scatter : SDNode<"AArch64ISD::SSTNT1_PRED", SDT_AArch64_SCATTER // // SVE CNT/INC/RDVL -def sve_rdvl_imm : ComplexPattern<i32, 1, "SelectRDVLImm<-32, 31, 16>">; -def sve_cnth_imm : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, 8>">; -def sve_cntw_imm : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, 4>">; -def sve_cntd_imm : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, 2>">; +def sve_rdvl_imm : ComplexPattern<i64, 1, "SelectRDVLImm<-32, 31, 16>">; +def sve_cnth_imm : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, 8>">; +def sve_cntw_imm : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, 4>">; +def sve_cntd_imm : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, 2>">; // SVE DEC -def sve_cnth_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -8>">; -def sve_cntw_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -4>">; -def sve_cntd_imm_neg : ComplexPattern<i32, 1, "SelectRDVLImm<1, 16, -2>">; +def sve_cnth_imm_neg : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, -8>">; +def sve_cntw_imm_neg : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, -4>">; +def sve_cntd_imm_neg : ComplexPattern<i64, 1, "SelectRDVLImm<1, 16, -2>">; def SDT_AArch64Reduce : SDTypeProfile<1, 2, [SDTCisVec<1>, SDTCisVec<2>]>; def AArch64faddv_p : SDNode<"AArch64ISD::FADDV_PRED", SDT_AArch64Reduce>; @@ -231,6 +231,8 @@ def AArch64fsqrt_mt : SDNode<"AArch64ISD::FSQRT_MERGE_PASSTHRU", SDT_AArch64Ari def AArch64frecpx_mt : SDNode<"AArch64ISD::FRECPX_MERGE_PASSTHRU", SDT_AArch64Arith>; def AArch64rbit_mt : SDNode<"AArch64ISD::BITREVERSE_MERGE_PASSTHRU", SDT_AArch64Arith>; def AArch64revb_mt : SDNode<"AArch64ISD::BSWAP_MERGE_PASSTHRU", SDT_AArch64Arith>; +def AArch64revh_mt : SDNode<"AArch64ISD::REVH_MERGE_PASSTHRU", SDT_AArch64Arith>; +def AArch64revw_mt : SDNode<"AArch64ISD::REVW_MERGE_PASSTHRU", SDT_AArch64Arith>; // These are like the above but we don't yet have need for ISD nodes. They allow // a single pattern to match intrinsic and ISD operand layouts. @@ -275,6 +277,11 @@ def AArch64mul_p_oneuse : PatFrag<(ops node:$pred, node:$src1, node:$src2), return N->hasOneUse(); }]>; +def AArch64fneg_mt_nsz : PatFrag<(ops node:$pred, node:$op, node:$pt), + (AArch64fneg_mt node:$pred, node:$op, node:$pt), [{ + return N->getFlags().hasNoSignedZeros(); +}]>; + def SDT_AArch64Arith_Unpred : SDTypeProfile<1, 2, [ SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisSameAs<0,1>, SDTCisSameAs<1,2> @@ -536,7 +543,8 @@ let Predicates = [HasSVEorStreamingSVE] in { (!cast<Instruction>("FNMLA_ZPZZZ_UNDEF_"#Suffix) $P, ZPR:$Za, ZPR:$Zn, ZPR:$Zm)>; // Zd = -(Za + Zn * Zm) - def : Pat<(AArch64fneg_mt PredTy:$P, (AArch64fma_p PredTy:$P, Ty:$Zn, Ty:$Zm, Ty:$Za), (Ty (undef))), + // (with nsz neg.) + def : Pat<(AArch64fneg_mt_nsz PredTy:$P, (AArch64fma_p PredTy:$P, Ty:$Zn, Ty:$Zm, Ty:$Za), (Ty (undef))), (!cast<Instruction>("FNMLA_ZPZZZ_UNDEF_"#Suffix) $P, ZPR:$Za, ZPR:$Zn, ZPR:$Zm)>; // Zda = Zda + Zn * Zm @@ -624,13 +632,13 @@ let Predicates = [HasSVEorStreamingSVE] in { def : Pat<(nxv8bf16 (AArch64dup (bf16 fpimm0))), (DUP_ZI_H 0, 0)>; // Duplicate Int immediate into all vector elements - def : Pat<(nxv16i8 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv16i8 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))), (DUP_ZI_B $a, $b)>; - def : Pat<(nxv8i16 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv8i16 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))), (DUP_ZI_H $a, $b)>; - def : Pat<(nxv4i32 (AArch64dup (i32 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv4i32 (AArch64dup (i32 (SVE8BitLslImm32 i32:$a, i32:$b)))), (DUP_ZI_S $a, $b)>; - def : Pat<(nxv2i64 (AArch64dup (i64 (SVE8BitLslImm i32:$a, i32:$b)))), + def : Pat<(nxv2i64 (AArch64dup (i64 (SVE8BitLslImm64 i32:$a, i32:$b)))), (DUP_ZI_D $a, $b)>; // Duplicate immediate FP into all vector elements. @@ -674,8 +682,8 @@ let Predicates = [HasSVEorStreamingSVE] in { defm RBIT_ZPmZ : sve_int_perm_rev_rbit<"rbit", AArch64rbit_mt>; defm REVB_ZPmZ : sve_int_perm_rev_revb<"revb", AArch64revb_mt>; - defm REVH_ZPmZ : sve_int_perm_rev_revh<"revh", int_aarch64_sve_revh>; - defm REVW_ZPmZ : sve_int_perm_rev_revw<"revw", int_aarch64_sve_revw>; + defm REVH_ZPmZ : sve_int_perm_rev_revh<"revh", AArch64revh_mt>; + defm REVW_ZPmZ : sve_int_perm_rev_revw<"revw", AArch64revw_mt>; defm REV_PP : sve_int_perm_reverse_p<"rev", vector_reverse>; defm REV_ZZ : sve_int_perm_reverse_z<"rev", vector_reverse>; @@ -2686,13 +2694,13 @@ let Predicates = [HasSVEorStreamingSVE] in { // Splice with lane bigger or equal to 0 def : Pat<(nxv16i8 (vector_splice (nxv16i8 ZPR:$Z1), (nxv16i8 ZPR:$Z2), (i64 (sve_ext_imm_0_255 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_255:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; def : Pat<(nxv8i16 (vector_splice (nxv8i16 ZPR:$Z1), (nxv8i16 ZPR:$Z2), (i64 (sve_ext_imm_0_127 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_127:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; def : Pat<(nxv4i32 (vector_splice (nxv4i32 ZPR:$Z1), (nxv4i32 ZPR:$Z2), (i64 (sve_ext_imm_0_63 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_63:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; def : Pat<(nxv2i64 (vector_splice (nxv2i64 ZPR:$Z1), (nxv2i64 ZPR:$Z2), (i64 (sve_ext_imm_0_31 i32:$index)))), - (EXT_ZZI ZPR:$Z1, ZPR:$Z2, sve_ext_imm_0_31:$index)>; + (EXT_ZZI ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>; } // End HasSVEorStreamingSVE diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp index 5cec4cb66339..566c7a16db23 100644 --- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp +++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp @@ -488,7 +488,7 @@ Instruction *AArch64StackTagging::insertBaseTaggedPointer( void AArch64StackTagging::alignAndPadAlloca(AllocaInfo &Info) { const Align NewAlignment = - max(MaybeAlign(Info.AI->getAlignment()), kTagGranuleSize); + max(MaybeAlign(Info.AI->getAlign()), kTagGranuleSize); Info.AI->setAlignment(NewAlignment); uint64_t Size = Info.AI->getAllocationSizeInBits(*DL).getValue() / 8; @@ -537,15 +537,14 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) { SmallVector<Instruction *, 4> UnrecognizedLifetimes; for (auto &BB : *F) { - for (BasicBlock::iterator IT = BB.begin(); IT != BB.end(); ++IT) { - Instruction *I = &*IT; - if (auto *AI = dyn_cast<AllocaInst>(I)) { + for (Instruction &I : BB) { + if (auto *AI = dyn_cast<AllocaInst>(&I)) { Allocas[AI].AI = AI; Allocas[AI].OldAI = AI; continue; } - if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(I)) { + if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I)) { for (Value *V : DVI->location_ops()) if (auto *AI = dyn_cast_or_null<AllocaInst>(V)) if (Allocas[AI].DbgVariableIntrinsics.empty() || @@ -554,12 +553,12 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) { continue; } - auto *II = dyn_cast<IntrinsicInst>(I); + auto *II = dyn_cast<IntrinsicInst>(&I); if (II && (II->getIntrinsicID() == Intrinsic::lifetime_start || II->getIntrinsicID() == Intrinsic::lifetime_end)) { AllocaInst *AI = findAllocaForValue(II->getArgOperand(1)); if (!AI) { - UnrecognizedLifetimes.push_back(I); + UnrecognizedLifetimes.push_back(&I); continue; } if (II->getIntrinsicID() == Intrinsic::lifetime_start) @@ -568,8 +567,8 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) { Allocas[AI].LifetimeEnd.push_back(II); } - if (isa<ReturnInst>(I) || isa<ResumeInst>(I) || isa<CleanupReturnInst>(I)) - RetVec.push_back(I); + if (isa<ReturnInst, ResumeInst, CleanupReturnInst>(&I)) + RetVec.push_back(&I); } } diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp index d782d6352cbe..f7d3dd0bc222 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -346,9 +346,7 @@ bool AArch64Subtarget::supportsAddressTopByteIgnored() const { return false; if (TargetTriple.isiOS()) { - unsigned Major, Minor, Micro; - TargetTriple.getiOSVersion(Major, Minor, Micro); - return Major >= 8; + return TargetTriple.getiOSVersion() >= VersionTuple(8); } return false; diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h index 19db774ccd7b..b3cd5ebd5f65 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.h +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -116,6 +116,8 @@ protected: bool HasFP16FML = false; bool HasSPE = false; + bool FixCortexA53_835769 = false; + // ARMv8.1 extensions bool HasVH = false; bool HasPAN = false; @@ -571,6 +573,8 @@ public: bool hasEL2VMSA() const { return HasEL2VMSA; } bool hasEL3() const { return HasEL3; } + bool fixCortexA53_835769() const { return FixCortexA53_835769; } + bool addrSinkUsingGEPs() const override { // Keeping GEPs inbounds is important for exploiting AArch64 // addressing-modes in ILP32 mode. @@ -632,8 +636,7 @@ public: // extended frames should be flagged as present. const Triple &TT = getTargetTriple(); - unsigned Major, Minor, Micro; - TT.getOSVersion(Major, Minor, Micro); + unsigned Major = TT.getOSVersion().getMajor(); switch(TT.getOS()) { default: return false; diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp index ce26c62af61a..4af28fc070dd 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -117,11 +117,6 @@ static cl::opt<bool> cl::init(true), cl::Hidden); static cl::opt<bool> -EnableA53Fix835769("aarch64-fix-cortex-a53-835769", cl::Hidden, - cl::desc("Work around Cortex-A53 erratum 835769"), - cl::init(false)); - -static cl::opt<bool> EnableGEPOpt("aarch64-enable-gep-opt", cl::Hidden, cl::desc("Enable optimizations on complex GEPs"), cl::init(false)); @@ -382,10 +377,9 @@ AArch64TargetMachine::getSubtargetImpl(const Function &F) const { unsigned MaxSVEVectorSize = 0; Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange); if (VScaleRangeAttr.isValid()) { - std::tie(MinSVEVectorSize, MaxSVEVectorSize) = - VScaleRangeAttr.getVScaleRangeArgs(); - MinSVEVectorSize *= 128; - MaxSVEVectorSize *= 128; + Optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax(); + MinSVEVectorSize = VScaleRangeAttr.getVScaleRangeMin() * 128; + MaxSVEVectorSize = VScaleMax ? VScaleMax.getValue() * 128 : 0; } else { MinSVEVectorSize = SVEVectorBitsMinOpt; MaxSVEVectorSize = SVEVectorBitsMaxOpt; @@ -765,8 +759,7 @@ void AArch64PassConfig::addPreEmitPass() { if (TM->getOptLevel() >= CodeGenOpt::Aggressive && EnableLoadStoreOpt) addPass(createAArch64LoadStoreOptimizationPass()); - if (EnableA53Fix835769) - addPass(createAArch64A53Fix835769()); + addPass(createAArch64A53Fix835769()); if (EnableBranchTargets) addPass(createAArch64BranchTargetsPass()); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 34015d2dbd49..d21854e38f5a 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -30,6 +30,12 @@ using namespace llvm::PatternMatch; static cl::opt<bool> EnableFalkorHWPFUnrollFix("enable-falkor-hwpf-unroll-fix", cl::init(true), cl::Hidden); +static cl::opt<unsigned> SVEGatherOverhead("sve-gather-overhead", cl::init(10), + cl::Hidden); + +static cl::opt<unsigned> SVEScatterOverhead("sve-scatter-overhead", + cl::init(10), cl::Hidden); + bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, const Function *Callee) const { const TargetMachine &TM = getTLI()->getTargetMachine(); @@ -725,6 +731,22 @@ static Optional<Instruction *> instCombineSVEVectorFMLA(InstCombiner &IC, return IC.replaceInstUsesWith(II, FMLA); } +static bool isAllActivePredicate(Value *Pred) { + // Look through convert.from.svbool(convert.to.svbool(...) chain. + Value *UncastedPred; + if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_convert_from_svbool>( + m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>( + m_Value(UncastedPred))))) + // If the predicate has the same or less lanes than the uncasted + // predicate then we know the casting has no effect. + if (cast<ScalableVectorType>(Pred->getType())->getMinNumElements() <= + cast<ScalableVectorType>(UncastedPred->getType())->getMinNumElements()) + Pred = UncastedPred; + + return match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>( + m_ConstantInt<AArch64SVEPredPattern::all>())); +} + static Optional<Instruction *> instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) { IRBuilder<> Builder(II.getContext()); @@ -735,8 +757,7 @@ instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) { Type *VecTy = II.getType(); Value *VecPtr = Builder.CreateBitCast(PtrOp, VecTy->getPointerTo()); - if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>( - m_ConstantInt<AArch64SVEPredPattern::all>()))) { + if (isAllActivePredicate(Pred)) { LoadInst *Load = Builder.CreateLoad(VecTy, VecPtr); return IC.replaceInstUsesWith(II, Load); } @@ -758,8 +779,7 @@ instCombineSVEST1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) { Value *VecPtr = Builder.CreateBitCast(PtrOp, VecOp->getType()->getPointerTo()); - if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>( - m_ConstantInt<AArch64SVEPredPattern::all>()))) { + if (isAllActivePredicate(Pred)) { Builder.CreateStore(VecOp, VecPtr); return IC.eraseInstFromFunction(II); } @@ -1008,6 +1028,40 @@ static Optional<Instruction *> instCombineST1ScatterIndex(InstCombiner &IC, return None; } +static Optional<Instruction *> instCombineSVESDIV(InstCombiner &IC, + IntrinsicInst &II) { + IRBuilder<> Builder(II.getContext()); + Builder.SetInsertPoint(&II); + Type *Int32Ty = Builder.getInt32Ty(); + Value *Pred = II.getOperand(0); + Value *Vec = II.getOperand(1); + Value *DivVec = II.getOperand(2); + + Value *SplatValue = getSplatValue(DivVec); + ConstantInt *SplatConstantInt = dyn_cast_or_null<ConstantInt>(SplatValue); + if (!SplatConstantInt) + return None; + APInt Divisor = SplatConstantInt->getValue(); + + if (Divisor.isPowerOf2()) { + Constant *DivisorLog2 = ConstantInt::get(Int32Ty, Divisor.logBase2()); + auto ASRD = Builder.CreateIntrinsic( + Intrinsic::aarch64_sve_asrd, {II.getType()}, {Pred, Vec, DivisorLog2}); + return IC.replaceInstUsesWith(II, ASRD); + } + if (Divisor.isNegatedPowerOf2()) { + Divisor.negate(); + Constant *DivisorLog2 = ConstantInt::get(Int32Ty, Divisor.logBase2()); + auto ASRD = Builder.CreateIntrinsic( + Intrinsic::aarch64_sve_asrd, {II.getType()}, {Pred, Vec, DivisorLog2}); + auto NEG = Builder.CreateIntrinsic(Intrinsic::aarch64_sve_neg, + {ASRD->getType()}, {ASRD, Pred, ASRD}); + return IC.replaceInstUsesWith(II, NEG); + } + + return None; +} + Optional<Instruction *> AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { @@ -1068,6 +1122,8 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC, return instCombineSVELD1(IC, II, DL); case Intrinsic::aarch64_sve_st1: return instCombineSVEST1(IC, II, DL); + case Intrinsic::aarch64_sve_sdiv: + return instCombineSVESDIV(IC, II); } return None; @@ -1746,7 +1802,7 @@ InstructionCost AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind) { - if (!isa<ScalableVectorType>(Src)) + if (useNeonVector(Src)) return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); auto LT = TLI->getTypeLegalizationCost(DL, Src); @@ -1763,6 +1819,10 @@ AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, return LT.first * 2; } +static unsigned getSVEGatherScatterOverhead(unsigned Opcode) { + return Opcode == Instruction::Load ? SVEGatherOverhead : SVEScatterOverhead; +} + InstructionCost AArch64TTIImpl::getGatherScatterOpCost( unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) { @@ -1785,6 +1845,10 @@ InstructionCost AArch64TTIImpl::getGatherScatterOpCost( ElementCount LegalVF = LT.second.getVectorElementCount(); InstructionCost MemOpCost = getMemoryOpCost(Opcode, VT->getElementType(), Alignment, 0, CostKind, I); + // Add on an overhead cost for using gathers/scatters. + // TODO: At the moment this is applied unilaterally for all CPUs, but at some + // point we may want a per-CPU overhead. + MemOpCost *= getSVEGatherScatterOverhead(Opcode); return LT.first * MemOpCost * getMaxNumElements(LegalVF); } diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index d1e8cd204b3a..c3e1735cd4cd 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -309,6 +309,8 @@ public: bool supportsScalableVectors() const { return ST->hasSVE(); } + bool enableScalableVectorization() const { return ST->hasSVE(); } + bool isLegalToVectorizeReduction(const RecurrenceDescriptor &RdxDesc, ElementCount VF) const; diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index 6d3aea2721de..62038b10fccd 100644 --- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -1031,12 +1031,7 @@ public: if (DarwinRefKind != MCSymbolRefExpr::VK_None) return false; - for (unsigned i = 0; i != AllowedModifiers.size(); ++i) { - if (ELFRefKind == AllowedModifiers[i]) - return true; - } - - return false; + return llvm::is_contained(AllowedModifiers, ELFRefKind); } bool isMovWSymbolG3() const { diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 1524aa5eb0ec..e8894e7933d6 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -785,6 +785,11 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) .libcallFor({s128}) .minScalar(0, MinFPScalar); + // TODO: Vector types. + getActionDefinitionsBuilder({G_FMAXIMUM, G_FMINIMUM}) + .legalFor({MinFPScalar, s32, s64}) + .minScalar(0, MinFPScalar); + // TODO: Libcall support for s128. // TODO: s16 should be legal with full FP16 support. getActionDefinitionsBuilder({G_LROUND, G_LLROUND}) diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp index a9b3792e0118..3dec980a819a 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp @@ -289,6 +289,44 @@ static void applyMutateAnyExtToZExt(MachineInstr &MI, MachineRegisterInfo &MRI, Observer.changedInstr(MI); } +/// Match a 128b store of zero and split it into two 64 bit stores, for +/// size/performance reasons. +static bool matchSplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI) { + GStore &Store = cast<GStore>(MI); + if (!Store.isSimple()) + return false; + LLT ValTy = MRI.getType(Store.getValueReg()); + if (!ValTy.isVector() || ValTy.getSizeInBits() != 128) + return false; + if (ValTy.getSizeInBits() != Store.getMemSizeInBits()) + return false; // Don't split truncating stores. + if (!MRI.hasOneNonDBGUse(Store.getValueReg())) + return false; + auto MaybeCst = isConstantOrConstantSplatVector( + *MRI.getVRegDef(Store.getValueReg()), MRI); + return MaybeCst && MaybeCst->isZero(); +} + +static void applySplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B, + GISelChangeObserver &Observer) { + B.setInstrAndDebugLoc(MI); + GStore &Store = cast<GStore>(MI); + assert(MRI.getType(Store.getValueReg()).isVector() && + "Expected a vector store value"); + LLT NewTy = LLT::scalar(64); + Register PtrReg = Store.getPointerReg(); + auto Zero = B.buildConstant(NewTy, 0); + auto HighPtr = B.buildPtrAdd(MRI.getType(PtrReg), PtrReg, + B.buildConstant(LLT::scalar(64), 8)); + auto &MF = *MI.getMF(); + auto *LowMMO = MF.getMachineMemOperand(&Store.getMMO(), 0, NewTy); + auto *HighMMO = MF.getMachineMemOperand(&Store.getMMO(), 8, NewTy); + B.buildStore(Zero, PtrReg, *LowMMO); + B.buildStore(Zero, HighPtr, *HighMMO); + Store.eraseFromParent(); +} + #define AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS #include "AArch64GenPostLegalizeGICombiner.inc" #undef AARCH64POSTLEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp index 40ddf6a94f73..515a5c63a559 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp @@ -430,6 +430,8 @@ static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) { case TargetOpcode::G_INTRINSIC_ROUND: case TargetOpcode::G_FMAXNUM: case TargetOpcode::G_FMINNUM: + case TargetOpcode::G_FMAXIMUM: + case TargetOpcode::G_FMINIMUM: return true; } return false; @@ -600,6 +602,8 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case TargetOpcode::G_FSUB: case TargetOpcode::G_FMUL: case TargetOpcode::G_FDIV: + case TargetOpcode::G_FMAXIMUM: + case TargetOpcode::G_FMINIMUM: return getSameKindOfOperandsMapping(MI); case TargetOpcode::G_FPEXT: { LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp index 90688f1a3e83..c1186ae804d2 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp @@ -239,8 +239,8 @@ void AArch64_MC::initLLVMToCVRegMapping(MCRegisterInfo *MRI) { {codeview::RegisterId::ARM64_Q31, AArch64::Q31}, }; - for (unsigned I = 0; I < array_lengthof(RegMap); ++I) - MRI->mapLLVMRegToCVReg(RegMap[I].Reg, static_cast<int>(RegMap[I].CVReg)); + for (const auto &I : RegMap) + MRI->mapLLVMRegToCVReg(I.Reg, static_cast<int>(I.CVReg)); } static MCRegisterInfo *createAArch64MCRegisterInfo(const Triple &Triple) { diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td index 010ffa1502de..bb488cd7da32 100644 --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -197,34 +197,42 @@ def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64 def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>; def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>; def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>; -def SVEAddSubImm64Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i64>", []>; +def SVEAddSubImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64>", []>; -def SVELogicalImm8Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8>", []>; -def SVELogicalImm16Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16>", []>; -def SVELogicalImm32Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32>", []>; +def SVELogicalImm8Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8>", []>; +def SVELogicalImm16Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16>", []>; +def SVELogicalImm32Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32>", []>; def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>; -def SVELogicalImm8NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8, true>", []>; -def SVELogicalImm16NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16, true>", []>; -def SVELogicalImm32NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32, true>", []>; +def SVELogicalImm8NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8, true>", []>; +def SVELogicalImm16NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16, true>", []>; +def SVELogicalImm32NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32, true>", []>; def SVELogicalImm64NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64, true>", []>; -def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>; +def SVE8BitLslImm32 : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>; +def SVE8BitLslImm64 : ComplexPattern<i64, 2, "SelectSVE8BitLslImm", [imm]>; +class SVE8BitLslImm<ValueType ty> { + ComplexPattern Pat = !cond( + !eq(ty, i32): SVE8BitLslImm32, + !eq(ty, i64): SVE8BitLslImm64); +} def SVEArithUImm8Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>; def SVEArithUImm16Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>; def SVEArithUImm32Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>; -def SVEArithUImm64Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i64>", []>; -def SVEArithSImmPat : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>; +def SVEArithUImm64Pat : ComplexPattern<i64, 1, "SelectSVEArithImm<MVT::i64>", []>; + +def SVEArithSImmPat32 : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>; +def SVEArithSImmPat64 : ComplexPattern<i64, 1, "SelectSVESignedArithImm", []>; def SVEShiftImmL8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>", []>; def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>; def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>; -def SVEShiftImmL64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 63>", []>; +def SVEShiftImmL64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<0, 63>", []>; def SVEShiftImmR8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8, true>", []>; def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>; def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>; -def SVEShiftImmR64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 64, true>", []>; +def SVEShiftImmR64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<1, 64, true>", []>; def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>; @@ -260,14 +268,14 @@ def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{ } // This allows i32 immediate extraction from i64 based arithmetic. -def sve_cnt_mul_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">; -def sve_cnt_shl_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, true>">; - +def sve_cnt_mul_imm_i32 : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">; +def sve_cnt_mul_imm_i64 : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, false>">; +def sve_cnt_shl_imm : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, true>">; -def sve_ext_imm_0_31 : ComplexPattern<i32, 1, "SelectEXTImm<31, 8>">; -def sve_ext_imm_0_63 : ComplexPattern<i32, 1, "SelectEXTImm<63, 4>">; -def sve_ext_imm_0_127 : ComplexPattern<i32, 1, "SelectEXTImm<127, 2>">; -def sve_ext_imm_0_255 : ComplexPattern<i32, 1, "SelectEXTImm<255, 1>">; +def sve_ext_imm_0_31 : ComplexPattern<i64, 1, "SelectEXTImm<31, 8>">; +def sve_ext_imm_0_63 : ComplexPattern<i64, 1, "SelectEXTImm<63, 4>">; +def sve_ext_imm_0_127 : ComplexPattern<i64, 1, "SelectEXTImm<127, 2>">; +def sve_ext_imm_0_255 : ComplexPattern<i64, 1, "SelectEXTImm<255, 1>">; def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2), (int_aarch64_sve_cntp node:$pred, node:$src2), [{ @@ -435,8 +443,8 @@ class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, : Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; -def SVEDup0 : ComplexPattern<i64, 0, "SelectDupZero", []>; -def SVEDup0Undef : ComplexPattern<i64, 0, "SelectDupZeroOrUndef", []>; +def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>; +def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>; let AddedComplexity = 1 in { class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1, @@ -868,10 +876,10 @@ multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> { def : InstAlias<asm # "\t$Rd", (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>; - def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm))), + def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm))), (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; - def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm)))), + def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm))), (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; def : Pat<(i64 (op sve_pred_enum:$pattern)), @@ -951,10 +959,10 @@ multiclass sve_int_pred_pattern_a<bits<3> opc, string asm, def : Pat<(i64 (op GPR64:$Rdn, (opcnt sve_pred_enum:$pattern))), (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1)>; - def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm)))), + def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm)))), (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; - def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm))))), + def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm)))), (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; def : Pat<(i32 (op GPR32:$Rdn, (i32 (trunc (opcnt (sve_pred_enum:$pattern)))))), @@ -962,12 +970,12 @@ multiclass sve_int_pred_pattern_a<bits<3> opc, string asm, GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, 1), sub_32))>; - def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm i32:$imm)))), + def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm_i32 i32:$imm)))), (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), sub_32))>; - def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (i64 (sve_cnt_shl_imm i32:$imm))))), + def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_shl_imm i32:$imm)))), (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), sub_32))>; @@ -4324,10 +4332,10 @@ multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> { def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>; def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>; - def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>; - def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>; - def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>; - def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>; + def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>; + def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>; + def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>; + def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>; } multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> { @@ -4348,10 +4356,10 @@ multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> { def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>; def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>; - def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>; - def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>; - def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>; - def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>; + def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>; + def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>; + def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>; + def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>; } //===----------------------------------------------------------------------===// @@ -4542,7 +4550,7 @@ multiclass sve_int_dup_imm_pred_merge_inst< (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; def : Pat<(intty (vselect predty:$Pg, - (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), + (intty (AArch64dup (scalarty (SVE8BitLslImm<scalarty>.Pat i32:$imm, i32:$shift)))), intty:$Zd)), (!cast<Instruction>(NAME) zprty:$Zd, $Pg, i32:$imm, i32:$shift)>; } @@ -4580,7 +4588,7 @@ multiclass sve_int_dup_imm_pred_zero_inst< (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; def : Pat<(intty (vselect predty:$Pg, - (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), + (intty (AArch64dup (scalarty (SVE8BitLslImm<scalarty>.Pat i32:$imm, i32:$shift)))), (intty (AArch64dup (scalarty 0))))), (!cast<Instruction>(NAME) $Pg, i32:$imm, i32:$shift)>; } @@ -6476,14 +6484,14 @@ multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> { def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>; def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>; - def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; - def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; + def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; + def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; } multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> { def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>; - def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; + def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; } class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, @@ -8377,13 +8385,13 @@ multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatter } /// Addressing modes -def am_sve_indexed_s4 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>; -def am_sve_indexed_s6 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>; +def am_sve_indexed_s4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>; +def am_sve_indexed_s6 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>; -def am_sve_regreg_lsl0 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<0>", []>; -def am_sve_regreg_lsl1 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<1>", []>; -def am_sve_regreg_lsl2 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<2>", []>; -def am_sve_regreg_lsl3 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<3>", []>; +def am_sve_regreg_lsl0 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<0>", []>; +def am_sve_regreg_lsl1 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<1>", []>; +def am_sve_regreg_lsl2 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<2>", []>; +def am_sve_regreg_lsl3 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<3>", []>; // Predicated pseudo floating point two operand instructions. multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> { diff --git a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp index e72dccdc4b78..642080a0d40d 100644 --- a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp +++ b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp @@ -152,7 +152,7 @@ bool SVEIntrinsicOpts::coalescePTrueIntrinsicCalls( // Remove the most encompassing ptrue, as well as any promoted ptrues, leaving // behind only the ptrues to be coalesced. PTrues.remove(MostEncompassingPTrue); - PTrues.remove_if([](auto *PTrue) { return isPTruePromoted(PTrue); }); + PTrues.remove_if(isPTruePromoted); // Hoist MostEncompassingPTrue to the start of the basic block. It is always // safe to do this, since ptrue intrinsic calls are guaranteed to have no @@ -287,10 +287,10 @@ bool SVEIntrinsicOpts::optimizePredicateStore(Instruction *I) { if (!Attr.isValid()) return false; - unsigned MinVScale, MaxVScale; - std::tie(MinVScale, MaxVScale) = Attr.getVScaleRangeArgs(); + unsigned MinVScale = Attr.getVScaleRangeMin(); + Optional<unsigned> MaxVScale = Attr.getVScaleRangeMax(); // The transform needs to know the exact runtime length of scalable vectors - if (MinVScale != MaxVScale || MinVScale == 0) + if (!MaxVScale || MinVScale != MaxVScale) return false; auto *PredType = @@ -351,10 +351,10 @@ bool SVEIntrinsicOpts::optimizePredicateLoad(Instruction *I) { if (!Attr.isValid()) return false; - unsigned MinVScale, MaxVScale; - std::tie(MinVScale, MaxVScale) = Attr.getVScaleRangeArgs(); + unsigned MinVScale = Attr.getVScaleRangeMin(); + Optional<unsigned> MaxVScale = Attr.getVScaleRangeMax(); // The transform needs to know the exact runtime length of scalable vectors - if (MinVScale != MaxVScale || MinVScale == 0) + if (!MaxVScale || MinVScale != MaxVScale) return false; auto *PredType = diff --git a/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp index aab76d27ef11..d28f38e42430 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp @@ -173,14 +173,7 @@ constexpr AMDGPUFunctionArgInfo AMDGPUFunctionArgInfo::fixedABILayout() { const AMDGPUFunctionArgInfo & AMDGPUArgumentUsageInfo::lookupFuncArgInfo(const Function &F) const { auto I = ArgInfoMap.find(&F); - if (I == ArgInfoMap.end()) { - if (AMDGPUTargetMachine::EnableFixedFunctionABI) - return FixedABIFunctionInfo; - - // Without the fixed ABI, we assume no function has special inputs. - assert(F.isDeclaration()); - return ExternFunctionInfo; - } - + if (I == ArgInfoMap.end()) + return FixedABIFunctionInfo; return I->second; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp index f0aadab3302f..b4ebc7d7d75f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAttributor.cpp @@ -112,6 +112,17 @@ static bool isDSAddress(const Constant *C) { return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS; } +/// Returns true if the function requires the implicit argument be passed +/// regardless of the function contents. +static bool funcRequiresImplicitArgPtr(const Function &F) { + // Sanitizers require the hostcall buffer passed in the implicit arguments. + return F.hasFnAttribute(Attribute::SanitizeAddress) || + F.hasFnAttribute(Attribute::SanitizeThread) || + F.hasFnAttribute(Attribute::SanitizeMemory) || + F.hasFnAttribute(Attribute::SanitizeHWAddress) || + F.hasFnAttribute(Attribute::SanitizeMemTag); +} + namespace { class AMDGPUInformationCache : public InformationCache { public: @@ -296,7 +307,7 @@ struct AAUniformWorkGroupSizeFunction : public AAUniformWorkGroupSize { bool AllCallSitesKnown = true; if (!A.checkForAllCallSites(CheckCallSite, *this, true, AllCallSitesKnown)) - indicatePessimisticFixpoint(); + return indicatePessimisticFixpoint(); return Change; } @@ -339,7 +350,17 @@ struct AAAMDAttributesFunction : public AAAMDAttributes { void initialize(Attributor &A) override { Function *F = getAssociatedFunction(); + + // If the function requires the implicit arg pointer due to sanitizers, + // assume it's needed even if explicitly marked as not requiring it. + const bool NeedsImplicit = funcRequiresImplicitArgPtr(*F); + if (NeedsImplicit) + removeAssumedBits(IMPLICIT_ARG_PTR); + for (auto Attr : ImplicitAttrs) { + if (NeedsImplicit && Attr.first == IMPLICIT_ARG_PTR) + continue; + if (F->hasFnAttribute(Attr.second)) addKnownBits(Attr.first); } @@ -500,6 +521,9 @@ struct AAAMDFlatWorkGroupSize std::tie(MinGroupSize, MaxGroupSize) = InfoCache.getFlatWorkGroupSizes(*F); intersectKnown( ConstantRange(APInt(32, MinGroupSize), APInt(32, MaxGroupSize + 1))); + + if (AMDGPU::isEntryFunctionCC(F->getCallingConv())) + indicatePessimisticFixpoint(); } ChangeStatus updateImpl(Attributor &A) override { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp index 43928d7c2a09..2f1e7823f65c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp @@ -652,8 +652,8 @@ bool AMDGPUCallLowering::lowerFormalArguments( ++PSInputNum; if (SkipArg) { - for (int I = 0, E = VRegs[Idx].size(); I != E; ++I) - B.buildUndef(VRegs[Idx][I]); + for (Register R : VRegs[Idx]) + B.buildUndef(R); ++Idx; continue; @@ -715,10 +715,9 @@ bool AMDGPUCallLowering::lowerFormalArguments( if (!MBB.empty()) B.setInstr(*MBB.begin()); - if (!IsEntryFunc) { + if (!IsEntryFunc && !IsGraphics) { // For the fixed ABI, pass workitem IDs in the last argument register. - if (AMDGPUTargetMachine::EnableFixedFunctionABI) - TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info); + TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info); } IncomingValueAssigner Assigner(AssignFn); @@ -731,11 +730,6 @@ bool AMDGPUCallLowering::lowerFormalArguments( uint64_t StackOffset = Assigner.StackOffset; - if (!IsEntryFunc && !AMDGPUTargetMachine::EnableFixedFunctionABI) { - // Special inputs come after user arguments. - TLI.allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info); - } - // Start adding system SGPRs. if (IsEntryFunc) { TLI.allocateSystemSGPRs(CCInfo, MF, *Info, CC, IsGraphics); @@ -829,9 +823,12 @@ bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder, if (IncomingArg) { LI->loadInputValue(InputReg, MIRBuilder, IncomingArg, ArgRC, ArgTy); - } else { - assert(InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR); + } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) { LI->getImplicitArgPtr(InputReg, MRI, MIRBuilder); + } else { + // We may have proven the input wasn't needed, although the ABI is + // requiring it. We just need to allocate the register appropriately. + MIRBuilder.buildUndef(InputReg); } if (OutgoingArg->isRegister()) { @@ -1233,8 +1230,7 @@ bool AMDGPUCallLowering::lowerTailCall( // after the ordinary user argument registers. SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs; - if (AMDGPUTargetMachine::EnableFixedFunctionABI && - Info.CallConv != CallingConv::AMDGPU_Gfx) { + if (Info.CallConv != CallingConv::AMDGPU_Gfx) { // With a fixed ABI, allocate fixed registers before user arguments. if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info)) return false; @@ -1300,12 +1296,6 @@ bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const SITargetLowering &TLI = *getTLI<SITargetLowering>(); const DataLayout &DL = F.getParent()->getDataLayout(); - if (!AMDGPUTargetMachine::EnableFixedFunctionABI && - Info.CallConv != CallingConv::AMDGPU_Gfx) { - LLVM_DEBUG(dbgs() << "Variable function ABI not implemented\n"); - return false; - } - SmallVector<ArgInfo, 8> OutArgs; for (auto &OrigArg : Info.OrigArgs) splitToValueTypes(OrigArg, OutArgs, DL, Info.CallConv); @@ -1359,8 +1349,7 @@ bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, // after the ordinary user argument registers. SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs; - if (AMDGPUTargetMachine::EnableFixedFunctionABI && - Info.CallConv != CallingConv::AMDGPU_Gfx) { + if (Info.CallConv != CallingConv::AMDGPU_Gfx) { // With a fixed ABI, allocate fixed registers before user arguments. if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info)) return false; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td index c7c5ff7bcbe7..2415fdfecaae 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td @@ -64,6 +64,30 @@ def int_minmax_to_med3 : GICombineRule< [{ return RegBankHelper.matchIntMinMaxToMed3(*${min_or_max}, ${matchinfo}); }]), (apply [{ RegBankHelper.applyMed3(*${min_or_max}, ${matchinfo}); }])>; +def fp_minmax_to_med3 : GICombineRule< + (defs root:$min_or_max, med3_matchdata:$matchinfo), + (match (wip_match_opcode G_FMAXNUM, + G_FMINNUM, + G_FMAXNUM_IEEE, + G_FMINNUM_IEEE):$min_or_max, + [{ return RegBankHelper.matchFPMinMaxToMed3(*${min_or_max}, ${matchinfo}); }]), + (apply [{ RegBankHelper.applyMed3(*${min_or_max}, ${matchinfo}); }])>; + +def fp_minmax_to_clamp : GICombineRule< + (defs root:$min_or_max, register_matchinfo:$matchinfo), + (match (wip_match_opcode G_FMAXNUM, + G_FMINNUM, + G_FMAXNUM_IEEE, + G_FMINNUM_IEEE):$min_or_max, + [{ return RegBankHelper.matchFPMinMaxToClamp(*${min_or_max}, ${matchinfo}); }]), + (apply [{ RegBankHelper.applyClamp(*${min_or_max}, ${matchinfo}); }])>; + +def fmed3_intrinsic_to_clamp : GICombineRule< + (defs root:$fmed3, register_matchinfo:$matchinfo), + (match (wip_match_opcode G_INTRINSIC):$fmed3, + [{ return RegBankHelper.matchFPMed3ToClamp(*${fmed3}, ${matchinfo}); }]), + (apply [{ RegBankHelper.applyClamp(*${fmed3}, ${matchinfo}); }])>; + def remove_fcanonicalize_matchinfo : GIDefMatchData<"Register">; def remove_fcanonicalize : GICombineRule< @@ -102,7 +126,9 @@ def AMDGPUPostLegalizerCombinerHelper: GICombinerHelper< } def AMDGPURegBankCombinerHelper : GICombinerHelper< - "AMDGPUGenRegBankCombinerHelper", [zext_trunc_fold, int_minmax_to_med3, ptr_add_immed_chain]> { + "AMDGPUGenRegBankCombinerHelper", + [zext_trunc_fold, int_minmax_to_med3, ptr_add_immed_chain, + fp_minmax_to_clamp, fp_minmax_to_med3, fmed3_intrinsic_to_clamp]> { let DisableRuleOption = "amdgpuregbankcombiner-disable-rule"; let StateClass = "AMDGPURegBankCombinerHelperState"; let AdditionalArguments = []; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp index 301e6f6d6f42..e79ff9b597c9 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp @@ -378,5 +378,4 @@ void AMDGPUCombinerHelper::applyFoldableFneg(MachineInstr &MI, } MI.eraseFromParent(); - return; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td index 12cef2774aaf..7fd94a977be7 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td @@ -172,6 +172,8 @@ def : GINodeEquiv<G_AMDGPU_CVT_F32_UBYTE3, AMDGPUcvt_f32_ubyte3>; def : GINodeEquiv<G_AMDGPU_CVT_PK_I16_I32, AMDGPUpk_i16_i32_impl>; def : GINodeEquiv<G_AMDGPU_SMED3, AMDGPUsmed3>; def : GINodeEquiv<G_AMDGPU_UMED3, AMDGPUumed3>; +def : GINodeEquiv<G_AMDGPU_FMED3, AMDGPUfmed3_impl>; +def : GINodeEquiv<G_AMDGPU_CLAMP, AMDGPUclamp>; def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>; def : GINodeEquiv<G_AMDGPU_BUFFER_LOAD, SIbuffer_load>; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp index b9c59f4c615a..699c6c479455 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp @@ -280,11 +280,12 @@ void MetadataStreamerV2::emitKernelAttrs(const Function &Func) { } } -void MetadataStreamerV2::emitKernelArgs(const Function &Func) { +void MetadataStreamerV2::emitKernelArgs(const Function &Func, + const GCNSubtarget &ST) { for (auto &Arg : Func.args()) emitKernelArg(Arg); - emitHiddenKernelArgs(Func); + emitHiddenKernelArgs(Func, ST); } void MetadataStreamerV2::emitKernelArg(const Argument &Arg) { @@ -381,10 +382,9 @@ void MetadataStreamerV2::emitKernelArg(const DataLayout &DL, Type *Ty, } } -void MetadataStreamerV2::emitHiddenKernelArgs(const Function &Func) { - int HiddenArgNumBytes = - getIntegerAttribute(Func, "amdgpu-implicitarg-num-bytes", 0); - +void MetadataStreamerV2::emitHiddenKernelArgs(const Function &Func, + const GCNSubtarget &ST) { + unsigned HiddenArgNumBytes = ST.getImplicitArgNumBytes(Func); if (!HiddenArgNumBytes) return; @@ -465,11 +465,12 @@ void MetadataStreamerV2::emitKernel(const MachineFunction &MF, HSAMetadata.mKernels.push_back(Kernel::Metadata()); auto &Kernel = HSAMetadata.mKernels.back(); + const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); Kernel.mName = std::string(Func.getName()); Kernel.mSymbolName = (Twine(Func.getName()) + Twine("@kd")).str(); emitKernelLanguage(Func); emitKernelAttrs(Func); - emitKernelArgs(Func); + emitKernelArgs(Func, ST); HSAMetadata.mKernels.back().mCodeProps = CodeProps; HSAMetadata.mKernels.back().mDebugProps = DebugProps; } @@ -673,13 +674,14 @@ void MetadataStreamerV3::emitKernelAttrs(const Function &Func, } void MetadataStreamerV3::emitKernelArgs(const Function &Func, + const GCNSubtarget &ST, msgpack::MapDocNode Kern) { unsigned Offset = 0; auto Args = HSAMetadataDoc->getArrayNode(); for (auto &Arg : Func.args()) emitKernelArg(Arg, Offset, Args); - emitHiddenKernelArgs(Func, Offset, Args); + emitHiddenKernelArgs(Func, ST, Offset, Args); Kern[".args"] = Args; } @@ -791,11 +793,10 @@ void MetadataStreamerV3::emitKernelArg( } void MetadataStreamerV3::emitHiddenKernelArgs(const Function &Func, + const GCNSubtarget &ST, unsigned &Offset, msgpack::ArrayDocNode Args) { - int HiddenArgNumBytes = - getIntegerAttribute(Func, "amdgpu-implicitarg-num-bytes", 0); - + unsigned HiddenArgNumBytes = ST.getImplicitArgNumBytes(Func); if (!HiddenArgNumBytes) return; @@ -912,6 +913,7 @@ void MetadataStreamerV3::emitKernel(const MachineFunction &MF, const SIProgramInfo &ProgramInfo) { auto &Func = MF.getFunction(); auto Kern = getHSAKernelProps(MF, ProgramInfo); + const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); assert(Func.getCallingConv() == CallingConv::AMDGPU_KERNEL || Func.getCallingConv() == CallingConv::SPIR_KERNEL); @@ -925,7 +927,7 @@ void MetadataStreamerV3::emitKernel(const MachineFunction &MF, (Twine(Func.getName()) + Twine(".kd")).str(), /*Copy=*/true); emitKernelLanguage(Func, Kern); emitKernelAttrs(Func, Kern); - emitKernelArgs(Func, Kern); + emitKernelArgs(Func, ST, Kern); } Kernels.push_back(Kern); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h index af5dae1cd8c0..54ed0afbba6d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h @@ -30,6 +30,7 @@ class MDNode; class Module; struct SIProgramInfo; class Type; +class GCNSubtarget; namespace AMDGPU { @@ -86,7 +87,8 @@ protected: void emitKernelAttrs(const Function &Func, msgpack::MapDocNode Kern); - void emitKernelArgs(const Function &Func, msgpack::MapDocNode Kern); + void emitKernelArgs(const Function &Func, const GCNSubtarget &ST, + msgpack::MapDocNode Kern); void emitKernelArg(const Argument &Arg, unsigned &Offset, msgpack::ArrayDocNode Args); @@ -98,8 +100,8 @@ protected: StringRef BaseTypeName = "", StringRef AccQual = "", StringRef TypeQual = ""); - void emitHiddenKernelArgs(const Function &Func, unsigned &Offset, - msgpack::ArrayDocNode Args); + void emitHiddenKernelArgs(const Function &Func, const GCNSubtarget &ST, + unsigned &Offset, msgpack::ArrayDocNode Args); msgpack::DocNode &getRootMetadata(StringRef Key) { return HSAMetadataDoc->getRoot().getMap(/*Convert=*/true)[Key]; @@ -173,7 +175,7 @@ private: void emitKernelAttrs(const Function &Func); - void emitKernelArgs(const Function &Func); + void emitKernelArgs(const Function &Func, const GCNSubtarget &ST); void emitKernelArg(const Argument &Arg); @@ -183,7 +185,7 @@ private: StringRef BaseTypeName = "", StringRef AccQual = "", StringRef TypeQual = ""); - void emitHiddenKernelArgs(const Function &Func); + void emitHiddenKernelArgs(const Function &Func, const GCNSubtarget &ST); const Metadata &getHSAMetadata() const { return HSAMetadata; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp index 88b4ec53a2a0..db84b8766924 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp @@ -892,6 +892,15 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { } break; } + case Intrinsic::amdgcn_is_shared: + case Intrinsic::amdgcn_is_private: { + if (isa<UndefValue>(II.getArgOperand(0))) + return IC.replaceInstUsesWith(II, UndefValue::get(II.getType())); + + if (isa<ConstantPointerNull>(II.getArgOperand(0))) + return IC.replaceInstUsesWith(II, ConstantInt::getFalse(II.getType())); + break; + } default: { if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr = AMDGPU::getImageDimIntrinsicInfo(II.getIntrinsicID())) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 1f898f2ba8b3..5046daaed977 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -533,7 +533,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL}) .legalFor({S32, S16, V2S16}) .minScalar(0, S16) - .clampMaxNumElements(0, S16, 2) + .clampMaxNumElementsStrict(0, S16, 2) .widenScalarToNextMultipleOf(0, 32) .maxScalar(0, S32) .scalarize(0); @@ -541,7 +541,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT, G_SADDSAT, G_SSUBSAT}) .legalFor({S32, S16, V2S16}) // Clamp modifier .minScalarOrElt(0, S16) - .clampMaxNumElements(0, S16, 2) + .clampMaxNumElementsStrict(0, S16, 2) .scalarize(0) .widenScalarToNextPow2(0, 32) .lower(); @@ -712,7 +712,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, } if (ST.hasVOP3PInsts()) - FPOpActions.clampMaxNumElements(0, S16, 2); + FPOpActions.clampMaxNumElementsStrict(0, S16, 2); FPOpActions .scalarize(0) @@ -728,7 +728,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder({G_FNEG, G_FABS}) .legalFor(FPTypesPK16) - .clampMaxNumElements(0, S16, 2) + .clampMaxNumElementsStrict(0, S16, 2) .scalarize(0) .clampScalar(0, S16, S64); @@ -965,7 +965,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, if (ST.has16BitInsts()) { getActionDefinitionsBuilder(G_BSWAP) .legalFor({S16, S32, V2S16}) - .clampMaxNumElements(0, S16, 2) + .clampMaxNumElementsStrict(0, S16, 2) // FIXME: Fixing non-power-of-2 before clamp is workaround for // narrowScalar limitation. .widenScalarToNextPow2(0) @@ -1052,10 +1052,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, // Split vector extloads. unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits(); - unsigned AlignBits = Query.MMODescrs[0].AlignInBits; - - if (MemSize < DstTy.getSizeInBits()) - MemSize = std::max(MemSize, AlignBits); if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize) return true; @@ -1077,12 +1073,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, return true; } - if (AlignBits < MemSize) { - const SITargetLowering *TLI = ST.getTargetLowering(); - return !TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS, - Align(AlignBits / 8)); - } - return false; }; @@ -1176,20 +1166,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, if (DstSize > MemSize) return std::make_pair(0, LLT::scalar(MemSize)); - if (!isPowerOf2_32(DstSize)) { - // We're probably decomposing an odd sized store. Try to split - // to the widest type. TODO: Account for alignment. As-is it - // should be OK, since the new parts will be further legalized. - unsigned FloorSize = PowerOf2Floor(DstSize); - return std::make_pair(0, LLT::scalar(FloorSize)); - } - - if (DstSize > 32 && (DstSize % 32 != 0)) { - // FIXME: Need a way to specify non-extload of larger size if - // suitably aligned. - return std::make_pair(0, LLT::scalar(32 * (DstSize / 32))); - } - unsigned MaxSize = maxSizeForAddrSpace(ST, PtrTy.getAddressSpace(), Op == G_LOAD); @@ -1257,14 +1233,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, ElementCount::getFixed(FloorSize / EltSize), EltTy)); } - // Need to split because of alignment. - unsigned Align = Query.MMODescrs[0].AlignInBits; - if (EltSize > Align && - (EltSize / Align < DstTy.getNumElements())) { - return std::make_pair( - 0, LLT::fixed_vector(EltSize / Align, EltTy)); - } - // May need relegalization for the scalars. return std::make_pair(0, EltTy); }) @@ -1457,6 +1425,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, // FIXME: Doesn't handle extract of illegal sizes. getActionDefinitionsBuilder(Op) .lowerIf(all(typeIs(LitTyIdx, S16), sizeIs(BigTyIdx, 32))) + .lowerIf([=](const LegalityQuery &Query) { + // Sub-vector(or single element) insert and extract. + // TODO: verify immediate offset here since lower only works with + // whole elements. + const LLT BigTy = Query.Types[BigTyIdx]; + return BigTy.isVector(); + }) // FIXME: Multiples of 16 should not be legal. .legalIf([=](const LegalityQuery &Query) { const LLT BigTy = Query.Types[BigTyIdx]; @@ -1615,7 +1590,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, // Prefer to reduce vector widths for 16-bit vectors before lowering, to // get more vector shift opportunities, since we'll get those when // expanded. - .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16)); + .clampMaxNumElementsStrict(0, S16, 2); } else if (ST.has16BitInsts()) { SextInReg.lowerFor({{S32}, {S64}, {S16}}); } else { @@ -1637,14 +1612,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder(G_FSHR) .legalFor({{S32, S32}}) .lowerFor({{V2S16, V2S16}}) - .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16)) + .clampMaxNumElementsStrict(0, S16, 2) .scalarize(0) .lower(); if (ST.hasVOP3PInsts()) { getActionDefinitionsBuilder(G_FSHL) .lowerFor({{V2S16, V2S16}}) - .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16)) + .clampMaxNumElementsStrict(0, S16, 2) .scalarize(0) .lower(); } else { @@ -2567,10 +2542,8 @@ bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper, } else { // For cases where the widened type isn't a nice register value, unmerge // from a widened register (e.g. <3 x s16> -> <4 x s16>) - B.setInsertPt(B.getMBB(), ++B.getInsertPt()); - WideLoad = Helper.widenWithUnmerge(WideTy, ValReg); - B.setInsertPt(B.getMBB(), MI.getIterator()); - B.buildLoadFromOffset(WideLoad, PtrReg, *MMO, 0); + WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0); + B.buildDeleteTrailingVectorElements(ValReg, WideLoad); } } @@ -3843,6 +3816,10 @@ Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B, llvm_unreachable("invalid data type"); } + if (StoreVT == LLT::fixed_vector(3, S16)) { + Reg = B.buildPadVectorWithUndefElements(LLT::fixed_vector(4, S16), Reg) + .getReg(0); + } return Reg; } @@ -4237,8 +4214,17 @@ static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI, (I >= Intr->GradientStart && I < Intr->CoordStart && !IsG16) || (I >= Intr->CoordStart && !IsA16)) { // Handle any gradient or coordinate operands that should not be packed - AddrReg = B.buildBitcast(V2S16, AddrReg).getReg(0); - PackedAddrs.push_back(AddrReg); + if ((I < Intr->GradientStart) && IsA16 && + (B.getMRI()->getType(AddrReg) == S16)) { + // Special handling of bias when A16 is on. Bias is of type half but + // occupies full 32-bit. + PackedAddrs.push_back( + B.buildBuildVector(V2S16, {AddrReg, B.buildUndef(S16).getReg(0)}) + .getReg(0)); + } else { + AddrReg = B.buildBitcast(V2S16, AddrReg).getReg(0); + PackedAddrs.push_back(AddrReg); + } } else { // Dz/dh, dz/dv and the last odd coord are packed with undef. Also, in 1D, // derivatives dx/dh and dx/dv are packed with undef. @@ -4676,9 +4662,23 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic( // Deal with the one annoying legal case. const LLT V3S16 = LLT::fixed_vector(3, 16); if (Ty == V3S16) { - padWithUndef(ResTy, RegsToCover - ResultRegs.size() + 1); - auto Concat = B.buildConcatVectors(LLT::fixed_vector(6, 16), ResultRegs); - B.buildUnmerge({DstReg, MRI->createGenericVirtualRegister(V3S16)}, Concat); + if (IsTFE) { + if (ResultRegs.size() == 1) { + NewResultReg = ResultRegs[0]; + } else if (ResultRegs.size() == 2) { + LLT V4S16 = LLT::fixed_vector(4, 16); + NewResultReg = B.buildConcatVectors(V4S16, ResultRegs).getReg(0); + } else { + return false; + } + } + + if (MRI->getType(DstReg).getNumElements() < + MRI->getType(NewResultReg).getNumElements()) { + B.buildDeleteTrailingVectorElements(DstReg, NewResultReg); + } else { + B.buildPadVectorWithUndefElements(DstReg, NewResultReg); + } return true; } @@ -4869,8 +4869,8 @@ bool AMDGPULegalizerInfo::legalizeBVHIntrinsic(MachineInstr &MI, } Ops.push_back(RayExtent); - auto packLanes = [&Ops, &S32, &B] (Register Src) { - auto Unmerge = B.buildUnmerge({S32, S32, S32, S32}, Src); + auto packLanes = [&Ops, &S32, &B](Register Src) { + auto Unmerge = B.buildUnmerge({S32, S32, S32}, Src); Ops.push_back(Unmerge.getReg(0)); Ops.push_back(Unmerge.getReg(1)); Ops.push_back(Unmerge.getReg(2)); @@ -4878,8 +4878,8 @@ bool AMDGPULegalizerInfo::legalizeBVHIntrinsic(MachineInstr &MI, packLanes(RayOrigin); if (IsA16) { - auto UnmergeRayDir = B.buildUnmerge({S16, S16, S16, S16}, RayDir); - auto UnmergeRayInvDir = B.buildUnmerge({S16, S16, S16, S16}, RayInvDir); + auto UnmergeRayDir = B.buildUnmerge({S16, S16, S16}, RayDir); + auto UnmergeRayInvDir = B.buildUnmerge({S16, S16, S16}, RayInvDir); Register R1 = MRI.createGenericVirtualRegister(S32); Register R2 = MRI.createGenericVirtualRegister(S32); Register R3 = MRI.createGenericVirtualRegister(S32); diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp index 12d6d35a6917..6e2b5dc471bc 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp @@ -24,13 +24,6 @@ // A possible future refinement is to specialise the structure per-kernel, so // that fields can be elided based on more expensive analysis. // -// NOTE: Since this pass will directly pack LDS (assume large LDS) into a struct -// type which would cause allocating huge memory for struct instance within -// every kernel. Hence, before running this pass, it is advisable to run the -// pass "amdgpu-replace-lds-use-with-pointer" which will replace LDS uses within -// non-kernel functions by pointers and thereby minimizes the unnecessary per -// kernel allocation of LDS memory. -// //===----------------------------------------------------------------------===// #include "AMDGPU.h" @@ -62,6 +55,20 @@ static cl::opt<bool> SuperAlignLDSGlobals( namespace { +SmallPtrSet<GlobalValue *, 32> getUsedList(Module &M) { + SmallPtrSet<GlobalValue *, 32> UsedList; + + SmallVector<GlobalValue *, 32> TmpVec; + collectUsedGlobalVariables(M, TmpVec, true); + UsedList.insert(TmpVec.begin(), TmpVec.end()); + + TmpVec.clear(); + collectUsedGlobalVariables(M, TmpVec, false); + UsedList.insert(TmpVec.begin(), TmpVec.end()); + + return UsedList; +} + class AMDGPULowerModuleLDS : public ModulePass { static void removeFromUsedList(Module &M, StringRef Name, @@ -105,11 +112,9 @@ class AMDGPULowerModuleLDS : public ModulePass { removeFromUsedLists(Module &M, const std::vector<GlobalVariable *> &LocalVars) { SmallPtrSet<Constant *, 32> LocalVarsSet; - for (size_t I = 0; I < LocalVars.size(); I++) { - if (Constant *C = dyn_cast<Constant>(LocalVars[I]->stripPointerCasts())) { + for (GlobalVariable *LocalVar : LocalVars) + if (Constant *C = dyn_cast<Constant>(LocalVar->stripPointerCasts())) LocalVarsSet.insert(C); - } - } removeFromUsedList(M, "llvm.used", LocalVarsSet); removeFromUsedList(M, "llvm.compiler.used", LocalVarsSet); } @@ -158,9 +163,9 @@ public: } bool runOnModule(Module &M) override { - UsedList = AMDGPU::getUsedList(M); - - bool Changed = processUsedLDS(M); + UsedList = getUsedList(M); + bool Changed = superAlignLDSGlobals(M); + Changed |= processUsedLDS(M); for (Function &F : M.functions()) { if (F.isDeclaration()) @@ -177,6 +182,50 @@ public: } private: + // Increase the alignment of LDS globals if necessary to maximise the chance + // that we can use aligned LDS instructions to access them. + static bool superAlignLDSGlobals(Module &M) { + const DataLayout &DL = M.getDataLayout(); + bool Changed = false; + if (!SuperAlignLDSGlobals) { + return Changed; + } + + for (auto &GV : M.globals()) { + if (GV.getType()->getPointerAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) { + // Only changing alignment of LDS variables + continue; + } + if (!GV.hasInitializer()) { + // cuda/hip extern __shared__ variable, leave alignment alone + continue; + } + + Align Alignment = AMDGPU::getAlign(DL, &GV); + TypeSize GVSize = DL.getTypeAllocSize(GV.getValueType()); + + if (GVSize > 8) { + // We might want to use a b96 or b128 load/store + Alignment = std::max(Alignment, Align(16)); + } else if (GVSize > 4) { + // We might want to use a b64 load/store + Alignment = std::max(Alignment, Align(8)); + } else if (GVSize > 2) { + // We might want to use a b32 load/store + Alignment = std::max(Alignment, Align(4)); + } else if (GVSize > 1) { + // We might want to use a b16 load/store + Alignment = std::max(Alignment, Align(2)); + } + + if (Alignment != AMDGPU::getAlign(DL, &GV)) { + Changed = true; + GV.setAlignment(Alignment); + } + } + return Changed; + } + bool processUsedLDS(Module &M, Function *F = nullptr) { LLVMContext &Ctx = M.getContext(); const DataLayout &DL = M.getDataLayout(); @@ -190,31 +239,6 @@ private: return false; } - // Increase the alignment of LDS globals if necessary to maximise the chance - // that we can use aligned LDS instructions to access them. - if (SuperAlignLDSGlobals) { - for (auto *GV : FoundLocalVars) { - Align Alignment = AMDGPU::getAlign(DL, GV); - TypeSize GVSize = DL.getTypeAllocSize(GV->getValueType()); - - if (GVSize > 8) { - // We might want to use a b96 or b128 load/store - Alignment = std::max(Alignment, Align(16)); - } else if (GVSize > 4) { - // We might want to use a b64 load/store - Alignment = std::max(Alignment, Align(8)); - } else if (GVSize > 2) { - // We might want to use a b32 load/store - Alignment = std::max(Alignment, Align(4)); - } else if (GVSize > 1) { - // We might want to use a b16 load/store - Alignment = std::max(Alignment, Align(2)); - } - - GV->setAlignment(Alignment); - } - } - SmallVector<OptimizedStructLayoutField, 8> LayoutFields; LayoutFields.reserve(FoundLocalVars.size()); for (GlobalVariable *GV : FoundLocalVars) { @@ -343,20 +367,14 @@ private: refineUsesAlignmentAndAA(GEP, A, DL, AliasScope, NoAlias); } - // Mark kernels with asm that reads the address of the allocated structure - // This is not necessary for lowering. This lets other passes, specifically - // PromoteAlloca, accurately calculate how much LDS will be used by the - // kernel after lowering. + // This ensures the variable is allocated when called functions access it. + // It also lets other passes, specifically PromoteAlloca, accurately + // calculate how much LDS will be used by the kernel after lowering. if (!F) { IRBuilder<> Builder(Ctx); - SmallPtrSet<Function *, 32> Kernels; for (Function &Func : M.functions()) { - if (Func.isDeclaration()) - continue; - - if (AMDGPU::isKernelCC(&Func) && !Kernels.contains(&Func)) { + if (!Func.isDeclaration() && AMDGPU::isKernelCC(&Func)) { markUsedByKernel(Builder, &Func, SGV); - Kernels.insert(&Func); } } } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp index 5d4b007f11e6..4e2f98d2a5db 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp @@ -2786,12 +2786,8 @@ AMDGPUMachineCFGStructurizer::initializeSelectRegisters(MRT *MRT, unsigned Selec // Fixme: Move linearization creation to the original spot createLinearizedRegion(Region, SelectOut); - for (auto CI = Region->getChildren()->begin(), - CE = Region->getChildren()->end(); - CI != CE; ++CI) { - InnerSelectOut = - initializeSelectRegisters((*CI), InnerSelectOut, MRI, TII); - } + for (auto *CI : *Region->getChildren()) + InnerSelectOut = initializeSelectRegisters(CI, InnerSelectOut, MRI, TII); MRT->setBBSelectRegIn(InnerSelectOut); return InnerSelectOut; } else { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp index 2aa02299ecdc..8ad344816ad2 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp @@ -119,31 +119,27 @@ private: bool isConstantAddr(const Value *V) const; }; -static const Value *getMemoryInstrPtr(const Instruction *Inst) { - if (auto LI = dyn_cast<LoadInst>(Inst)) { - return LI->getPointerOperand(); - } - if (auto SI = dyn_cast<StoreInst>(Inst)) { - return SI->getPointerOperand(); - } - if (auto AI = dyn_cast<AtomicCmpXchgInst>(Inst)) { - return AI->getPointerOperand(); - } - if (auto AI = dyn_cast<AtomicRMWInst>(Inst)) { - return AI->getPointerOperand(); - } - if (auto MI = dyn_cast<AnyMemIntrinsic>(Inst)) { - return MI->getRawDest(); - } +static std::pair<const Value *, const Type *> getMemoryInstrPtrAndType( + const Instruction *Inst) { + if (auto LI = dyn_cast<LoadInst>(Inst)) + return {LI->getPointerOperand(), LI->getType()}; + if (auto SI = dyn_cast<StoreInst>(Inst)) + return {SI->getPointerOperand(), SI->getValueOperand()->getType()}; + if (auto AI = dyn_cast<AtomicCmpXchgInst>(Inst)) + return {AI->getPointerOperand(), AI->getCompareOperand()->getType()}; + if (auto AI = dyn_cast<AtomicRMWInst>(Inst)) + return {AI->getPointerOperand(), AI->getValOperand()->getType()}; + if (auto MI = dyn_cast<AnyMemIntrinsic>(Inst)) + return {MI->getRawDest(), Type::getInt8Ty(MI->getContext())}; - return nullptr; + return {nullptr, nullptr}; } bool AMDGPUPerfHint::isIndirectAccess(const Instruction *Inst) const { LLVM_DEBUG(dbgs() << "[isIndirectAccess] " << *Inst << '\n'); SmallSet<const Value *, 32> WorkSet; SmallSet<const Value *, 32> Visited; - if (const Value *MO = getMemoryInstrPtr(Inst)) { + if (const Value *MO = getMemoryInstrPtrAndType(Inst).first) { if (isGlobalAddr(MO)) WorkSet.insert(MO); } @@ -209,10 +205,8 @@ AMDGPUPerfHintAnalysis::FuncInfo *AMDGPUPerfHint::visit(const Function &F) { for (auto &B : F) { LastAccess = MemAccessInfo(); for (auto &I : B) { - if (const Value *Ptr = getMemoryInstrPtr(&I)) { - unsigned Size = divideCeil( - Ptr->getType()->getPointerElementType()->getPrimitiveSizeInBits(), - 32); + if (const Type *Ty = getMemoryInstrPtrAndType(&I).second) { + unsigned Size = divideCeil(Ty->getPrimitiveSizeInBits(), 32); if (isIndirectAccess(&I)) FI.IAMInstCost += Size; if (isLargeStride(&I)) @@ -326,7 +320,7 @@ bool AMDGPUPerfHint::isLargeStride(const Instruction *Inst) { AMDGPUPerfHint::MemAccessInfo AMDGPUPerfHint::makeMemAccessInfo(Instruction *Inst) const { MemAccessInfo MAI; - const Value *MO = getMemoryInstrPtr(Inst); + const Value *MO = getMemoryInstrPtrAndType(Inst).first; LLVM_DEBUG(dbgs() << "[isLargeStride] MO: " << *MO << '\n'); // Do not treat local-addr memory access as large stride. diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp index 3ec5dd7e0eff..f9a9fe403ff6 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp @@ -939,7 +939,7 @@ bool AMDGPUPromoteAllocaImpl::handleAlloca(AllocaInst &I, bool SufficientLDS) { GlobalVariable::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS); GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); - GV->setAlignment(MaybeAlign(I.getAlignment())); + GV->setAlignment(I.getAlign()); Value *TCntY, *TCntZ; diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp index 12b5830ef930..3ce67a733c10 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp @@ -16,6 +16,7 @@ #include "AMDGPURegisterBankInfo.h" #include "GCNSubtarget.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" +#include "SIMachineFunctionInfo.h" #include "llvm/CodeGen/GlobalISel/Combiner.h" #include "llvm/CodeGen/GlobalISel/CombinerHelper.h" #include "llvm/CodeGen/GlobalISel/CombinerInfo.h" @@ -23,6 +24,7 @@ #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/Target/TargetMachine.h" #define DEBUG_TYPE "amdgpu-regbank-combiner" @@ -36,13 +38,15 @@ protected: MachineRegisterInfo &MRI; const RegisterBankInfo &RBI; const TargetRegisterInfo &TRI; + const SIInstrInfo &TII; CombinerHelper &Helper; public: AMDGPURegBankCombinerHelper(MachineIRBuilder &B, CombinerHelper &Helper) : B(B), MF(B.getMF()), MRI(*B.getMRI()), RBI(*MF.getSubtarget().getRegBankInfo()), - TRI(*MF.getSubtarget().getRegisterInfo()), Helper(Helper){}; + TRI(*MF.getSubtarget().getRegisterInfo()), + TII(*MF.getSubtarget<GCNSubtarget>().getInstrInfo()), Helper(Helper){}; bool isVgprRegBank(Register Reg); Register getAsVgpr(Register Reg); @@ -63,7 +67,19 @@ public: Register &Val, CstTy &K0, CstTy &K1); bool matchIntMinMaxToMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo); + bool matchFPMinMaxToMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo); + bool matchFPMinMaxToClamp(MachineInstr &MI, Register &Reg); + bool matchFPMed3ToClamp(MachineInstr &MI, Register &Reg); void applyMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo); + void applyClamp(MachineInstr &MI, Register &Reg); + +private: + AMDGPU::SIModeRegisterDefaults getMode(); + bool getIEEE(); + bool getDX10Clamp(); + bool isFminnumIeee(const MachineInstr &MI); + bool isFCst(MachineInstr *MI); + bool isClampZeroToOne(MachineInstr *K0, MachineInstr *K1); }; bool AMDGPURegBankCombinerHelper::isVgprRegBank(Register Reg) { @@ -98,6 +114,13 @@ AMDGPURegBankCombinerHelper::getMinMaxPair(unsigned Opc) { case AMDGPU::G_UMAX: case AMDGPU::G_UMIN: return {AMDGPU::G_UMIN, AMDGPU::G_UMAX, AMDGPU::G_AMDGPU_UMED3}; + case AMDGPU::G_FMAXNUM: + case AMDGPU::G_FMINNUM: + return {AMDGPU::G_FMINNUM, AMDGPU::G_FMAXNUM, AMDGPU::G_AMDGPU_FMED3}; + case AMDGPU::G_FMAXNUM_IEEE: + case AMDGPU::G_FMINNUM_IEEE: + return {AMDGPU::G_FMINNUM_IEEE, AMDGPU::G_FMAXNUM_IEEE, + AMDGPU::G_AMDGPU_FMED3}; } } @@ -148,6 +171,146 @@ bool AMDGPURegBankCombinerHelper::matchIntMinMaxToMed3( return true; } +// fmed3(NaN, K0, K1) = min(min(NaN, K0), K1) +// ieee = true : min/max(SNaN, K) = QNaN, min/max(QNaN, K) = K +// ieee = false : min/max(NaN, K) = K +// clamp(NaN) = dx10_clamp ? 0.0 : NaN +// Consider values of min(max(Val, K0), K1) and max(min(Val, K1), K0) as input. +// Other operand commutes (see matchMed) give same result since min and max are +// commutative. + +// Try to replace fp min(max(Val, K0), K1) or max(min(Val, K1), K0), KO<=K1 +// with fmed3(Val, K0, K1) or clamp(Val). Clamp requires K0 = 0.0 and K1 = 1.0. +// Val = SNaN only for ieee = true +// fmed3(SNaN, K0, K1) = min(min(SNaN, K0), K1) = min(QNaN, K1) = K1 +// min(max(SNaN, K0), K1) = min(QNaN, K1) = K1 +// max(min(SNaN, K1), K0) = max(K1, K0) = K1 +// Val = NaN,ieee = false or Val = QNaN,ieee = true +// fmed3(NaN, K0, K1) = min(min(NaN, K0), K1) = min(K0, K1) = K0 +// min(max(NaN, K0), K1) = min(K0, K1) = K0 (can clamp when dx10_clamp = true) +// max(min(NaN, K1), K0) = max(K1, K0) = K1 != K0 +bool AMDGPURegBankCombinerHelper::matchFPMinMaxToMed3( + MachineInstr &MI, Med3MatchInfo &MatchInfo) { + Register Dst = MI.getOperand(0).getReg(); + LLT Ty = MRI.getType(Dst); + if (Ty != LLT::scalar(16) && Ty != LLT::scalar(32)) + return false; + + auto OpcodeTriple = getMinMaxPair(MI.getOpcode()); + + Register Val; + Optional<FPValueAndVReg> K0, K1; + // Match min(max(Val, K0), K1) or max(min(Val, K1), K0). Then see if K0 <= K1. + if (!matchMed<GFCstAndRegMatch>(MI, MRI, OpcodeTriple, Val, K0, K1)) + return false; + + if (K0->Value > K1->Value) + return false; + + // For IEEE=false perform combine only when it's safe to assume that there are + // no NaN inputs. Most often MI is marked with nnan fast math flag. + // For IEEE=true consider NaN inputs. fmed3(NaN, K0, K1) is equivalent to + // min(min(NaN, K0), K1). Safe to fold for min(max(Val, K0), K1) since inner + // nodes(max/min) have same behavior when one input is NaN and other isn't. + // Don't consider max(min(SNaN, K1), K0) since there is no isKnownNeverQNaN, + // also post-legalizer inputs to min/max are fcanonicalized (never SNaN). + if ((getIEEE() && isFminnumIeee(MI)) || isKnownNeverNaN(Dst, MRI)) { + // Don't fold single use constant that can't be inlined. + if ((!MRI.hasOneNonDBGUse(K0->VReg) || TII.isInlineConstant(K0->Value)) && + (!MRI.hasOneNonDBGUse(K1->VReg) || TII.isInlineConstant(K1->Value))) { + MatchInfo = {OpcodeTriple.Med, Val, K0->VReg, K1->VReg}; + return true; + } + } + + return false; +} + +bool AMDGPURegBankCombinerHelper::matchFPMinMaxToClamp(MachineInstr &MI, + Register &Reg) { + // Clamp is available on all types after regbankselect (f16, f32, f64, v2f16). + auto OpcodeTriple = getMinMaxPair(MI.getOpcode()); + Register Val; + Optional<FPValueAndVReg> K0, K1; + // Match min(max(Val, K0), K1) or max(min(Val, K1), K0). + if (!matchMed<GFCstOrSplatGFCstMatch>(MI, MRI, OpcodeTriple, Val, K0, K1)) + return false; + + if (!K0->Value.isExactlyValue(0.0) || !K1->Value.isExactlyValue(1.0)) + return false; + + // For IEEE=false perform combine only when it's safe to assume that there are + // no NaN inputs. Most often MI is marked with nnan fast math flag. + // For IEEE=true consider NaN inputs. Only min(max(QNaN, 0.0), 1.0) evaluates + // to 0.0 requires dx10_clamp = true. + if ((getIEEE() && getDX10Clamp() && isFminnumIeee(MI) && + isKnownNeverSNaN(Val, MRI)) || + isKnownNeverNaN(MI.getOperand(0).getReg(), MRI)) { + Reg = Val; + return true; + } + + return false; +} + +// Replacing fmed3(NaN, 0.0, 1.0) with clamp. Requires dx10_clamp = true. +// Val = SNaN only for ieee = true. It is important which operand is NaN. +// min(min(SNaN, 0.0), 1.0) = min(QNaN, 1.0) = 1.0 +// min(min(SNaN, 1.0), 0.0) = min(QNaN, 0.0) = 0.0 +// min(min(0.0, 1.0), SNaN) = min(0.0, SNaN) = QNaN +// Val = NaN,ieee = false or Val = QNaN,ieee = true +// min(min(NaN, 0.0), 1.0) = min(0.0, 1.0) = 0.0 +// min(min(NaN, 1.0), 0.0) = min(1.0, 0.0) = 0.0 +// min(min(0.0, 1.0), NaN) = min(0.0, NaN) = 0.0 +bool AMDGPURegBankCombinerHelper::matchFPMed3ToClamp(MachineInstr &MI, + Register &Reg) { + if (MI.getIntrinsicID() != Intrinsic::amdgcn_fmed3) + return false; + + // In llvm-ir, clamp is often represented as an intrinsic call to + // @llvm.amdgcn.fmed3.f32(%Val, 0.0, 1.0). Check for other operand orders. + MachineInstr *Src0 = getDefIgnoringCopies(MI.getOperand(2).getReg(), MRI); + MachineInstr *Src1 = getDefIgnoringCopies(MI.getOperand(3).getReg(), MRI); + MachineInstr *Src2 = getDefIgnoringCopies(MI.getOperand(4).getReg(), MRI); + + if (isFCst(Src0) && !isFCst(Src1)) + std::swap(Src0, Src1); + if (isFCst(Src1) && !isFCst(Src2)) + std::swap(Src1, Src2); + if (isFCst(Src0) && !isFCst(Src1)) + std::swap(Src0, Src1); + if (!isClampZeroToOne(Src1, Src2)) + return false; + + Register Val = Src0->getOperand(0).getReg(); + + auto isOp3Zero = [&]() { + MachineInstr *Op3 = getDefIgnoringCopies(MI.getOperand(4).getReg(), MRI); + if (Op3->getOpcode() == TargetOpcode::G_FCONSTANT) + return Op3->getOperand(1).getFPImm()->isExactlyValue(0.0); + return false; + }; + // For IEEE=false perform combine only when it's safe to assume that there are + // no NaN inputs. Most often MI is marked with nnan fast math flag. + // For IEEE=true consider NaN inputs. Requires dx10_clamp = true. Safe to fold + // when Val could be QNaN. If Val can also be SNaN third input should be 0.0. + if (isKnownNeverNaN(MI.getOperand(0).getReg(), MRI) || + (getIEEE() && getDX10Clamp() && + (isKnownNeverSNaN(Val, MRI) || isOp3Zero()))) { + Reg = Val; + return true; + } + + return false; +} + +void AMDGPURegBankCombinerHelper::applyClamp(MachineInstr &MI, Register &Reg) { + B.setInstrAndDebugLoc(MI); + B.buildInstr(AMDGPU::G_AMDGPU_CLAMP, {MI.getOperand(0)}, {Reg}, + MI.getFlags()); + MI.eraseFromParent(); +} + void AMDGPURegBankCombinerHelper::applyMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo) { B.setInstrAndDebugLoc(MI); @@ -158,6 +321,33 @@ void AMDGPURegBankCombinerHelper::applyMed3(MachineInstr &MI, MI.eraseFromParent(); } +AMDGPU::SIModeRegisterDefaults AMDGPURegBankCombinerHelper::getMode() { + return MF.getInfo<SIMachineFunctionInfo>()->getMode(); +} + +bool AMDGPURegBankCombinerHelper::getIEEE() { return getMode().IEEE; } + +bool AMDGPURegBankCombinerHelper::getDX10Clamp() { return getMode().DX10Clamp; } + +bool AMDGPURegBankCombinerHelper::isFminnumIeee(const MachineInstr &MI) { + return MI.getOpcode() == AMDGPU::G_FMINNUM_IEEE; +} + +bool AMDGPURegBankCombinerHelper::isFCst(MachineInstr *MI) { + return MI->getOpcode() == AMDGPU::G_FCONSTANT; +} + +bool AMDGPURegBankCombinerHelper::isClampZeroToOne(MachineInstr *K0, + MachineInstr *K1) { + if (isFCst(K0) && isFCst(K1)) { + const ConstantFP *KO_FPImm = K0->getOperand(1).getFPImm(); + const ConstantFP *K1_FPImm = K1->getOperand(1).getFPImm(); + return (KO_FPImm->isExactlyValue(0.0) && K1_FPImm->isExactlyValue(1.0)) || + (KO_FPImm->isExactlyValue(1.0) && K1_FPImm->isExactlyValue(0.0)); + } + return false; +} + class AMDGPURegBankCombinerHelperState { protected: CombinerHelper &Helper; diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp index 5988403c0a29..c60012bcfe2e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp @@ -707,9 +707,6 @@ bool AMDGPURegisterBankInfo::executeInWaterfallLoop( iterator_range<MachineBasicBlock::iterator> Range, SmallSet<Register, 4> &SGPROperandRegs, MachineRegisterInfo &MRI) const { - SmallVector<Register, 4> ResultRegs; - SmallVector<Register, 4> InitResultRegs; - SmallVector<Register, 4> PhiRegs; // Track use registers which have already been expanded with a readfirstlane // sequence. This may have multiple uses if moving a sequence. @@ -774,15 +771,6 @@ bool AMDGPURegisterBankInfo::executeInWaterfallLoop( .addReg(NewExec) .addMBB(LoopBB); - for (auto Result : zip(InitResultRegs, ResultRegs, PhiRegs)) { - B.buildInstr(TargetOpcode::G_PHI) - .addDef(std::get<2>(Result)) - .addReg(std::get<0>(Result)) // Initial value / implicit_def - .addMBB(&MBB) - .addReg(std::get<1>(Result)) // Mid-loop value. - .addMBB(LoopBB); - } - const DebugLoc &DL = B.getDL(); MachineInstr &FirstInst = *Range.begin(); @@ -1174,18 +1162,25 @@ bool AMDGPURegisterBankInfo::applyMappingLoad(MachineInstr &MI, // 96-bit loads are only available for vector loads. We need to split this // into a 64-bit part, and 32 (unless we can widen to a 128-bit load). if (MMO->getAlign() < Align(16)) { + MachineFunction *MF = MI.getParent()->getParent(); + ApplyRegBankMapping ApplyBank(*this, MRI, DstBank); + MachineIRBuilder B(MI, ApplyBank); + LegalizerHelper Helper(*MF, ApplyBank, B); LLT Part64, Part32; std::tie(Part64, Part32) = splitUnequalType(LoadTy, 64); - auto Load0 = B.buildLoadFromOffset(Part64, PtrReg, *MMO, 0); - auto Load1 = B.buildLoadFromOffset(Part32, PtrReg, *MMO, 8); - - auto Undef = B.buildUndef(LoadTy); - auto Ins0 = B.buildInsert(LoadTy, Undef, Load0, 0); - B.buildInsert(MI.getOperand(0), Ins0, Load1, 64); + if (Helper.reduceLoadStoreWidth(cast<GAnyLoad>(MI), 0, Part64) != + LegalizerHelper::Legalized) + return false; + return true; } else { LLT WiderTy = widen96To128(LoadTy); auto WideLoad = B.buildLoadFromOffset(WiderTy, PtrReg, *MMO, 0); - B.buildExtract(MI.getOperand(0), WideLoad, 0); + if (WiderTy.isScalar()) + B.buildTrunc(MI.getOperand(0), WideLoad); + else { + B.buildDeleteTrailingVectorElements(MI.getOperand(0).getReg(), + WideLoad); + } } } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUReplaceLDSUseWithPointer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUReplaceLDSUseWithPointer.cpp index d55bf3917e9c..2475b44b42a3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUReplaceLDSUseWithPointer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUReplaceLDSUseWithPointer.cpp @@ -87,6 +87,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetOperations.h" +#include "llvm/Analysis/CallGraph.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" @@ -110,6 +111,18 @@ using namespace llvm; namespace { +namespace AMDGPU { +/// Collect all the instructions where user \p U belongs to. \p U could be +/// instruction itself or it could be a constant expression which is used within +/// an instruction. If \p CollectKernelInsts is true, collect instructions only +/// from kernels, otherwise collect instructions only from non-kernel functions. +DenseMap<Function *, SmallPtrSet<Instruction *, 8>> +getFunctionToInstsMap(User *U, bool CollectKernelInsts); + +SmallPtrSet<Function *, 8> collectNonKernelAccessorsOfLDS(GlobalVariable *GV); + +} // namespace AMDGPU + class ReplaceLDSUseImpl { Module &M; LLVMContext &Ctx; @@ -127,7 +140,8 @@ class ReplaceLDSUseImpl { // Collect LDS which requires their uses to be replaced by pointer. std::vector<GlobalVariable *> collectLDSRequiringPointerReplace() { // Collect LDS which requires module lowering. - std::vector<GlobalVariable *> LDSGlobals = AMDGPU::findVariablesToLower(M); + std::vector<GlobalVariable *> LDSGlobals = + llvm::AMDGPU::findVariablesToLower(M); // Remove LDS which don't qualify for replacement. llvm::erase_if(LDSGlobals, [&](GlobalVariable *GV) { @@ -172,7 +186,7 @@ class ReplaceLDSUseImpl { AMDGPUAS::LOCAL_ADDRESS); LDSPointer->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); - LDSPointer->setAlignment(AMDGPU::getAlign(DL, LDSPointer)); + LDSPointer->setAlignment(llvm::AMDGPU::getAlign(DL, LDSPointer)); // Mark that an associated LDS pointer is created for LDS. LDSToPointer[GV] = LDSPointer; @@ -245,10 +259,9 @@ class ReplaceLDSUseImpl { auto FunctionToInsts = AMDGPU::getFunctionToInstsMap(U, false /*=CollectKernelInsts*/); - for (auto FI = FunctionToInsts.begin(), FE = FunctionToInsts.end(); - FI != FE; ++FI) { - Function *F = FI->first; - auto &Insts = FI->second; + for (const auto &FunctionToInst : FunctionToInsts) { + Function *F = FunctionToInst.first; + auto &Insts = FunctionToInst.second; for (auto *I : Insts) { // If `U` is a constant expression, then we need to break the // associated instruction into a set of separate instructions by @@ -341,10 +354,9 @@ bool ReplaceLDSUseImpl::replaceLDSUse(GlobalVariable *GV) { // Traverse through each kernel K, check and if required, initialize the // LDS pointer to point to LDS within K. - for (auto KI = KernelToCallees.begin(), KE = KernelToCallees.end(); KI != KE; - ++KI) { - Function *K = KI->first; - SmallPtrSet<Function *, 8> Callees = KI->second; + for (const auto &KernelToCallee : KernelToCallees) { + Function *K = KernelToCallee.first; + SmallPtrSet<Function *, 8> Callees = KernelToCallee.second; // Compute reachable and LDS used callees for kernel K. set_intersect(Callees, LDSAccessors); @@ -378,6 +390,184 @@ bool ReplaceLDSUseImpl::replaceLDSUse(GlobalVariable *GV) { return true; } +namespace AMDGPU { + +// An helper class for collecting all reachable callees for each kernel defined +// within the module. +class CollectReachableCallees { + Module &M; + CallGraph CG; + SmallPtrSet<CallGraphNode *, 8> AddressTakenFunctions; + + // Collect all address taken functions within the module. + void collectAddressTakenFunctions() { + auto *ECNode = CG.getExternalCallingNode(); + + for (const auto &GI : *ECNode) { + auto *CGN = GI.second; + auto *F = CGN->getFunction(); + if (!F || F->isDeclaration() || llvm::AMDGPU::isKernelCC(F)) + continue; + AddressTakenFunctions.insert(CGN); + } + } + + // For given kernel, collect all its reachable non-kernel functions. + SmallPtrSet<Function *, 8> collectReachableCallees(Function *K) { + SmallPtrSet<Function *, 8> ReachableCallees; + + // Call graph node which represents this kernel. + auto *KCGN = CG[K]; + + // Go through all call graph nodes reachable from the node representing this + // kernel, visit all their call sites, if the call site is direct, add + // corresponding callee to reachable callee set, if it is indirect, resolve + // the indirect call site to potential reachable callees, add them to + // reachable callee set, and repeat the process for the newly added + // potential callee nodes. + // + // FIXME: Need to handle bit-casted function pointers. + // + SmallVector<CallGraphNode *, 8> CGNStack(depth_first(KCGN)); + SmallPtrSet<CallGraphNode *, 8> VisitedCGNodes; + while (!CGNStack.empty()) { + auto *CGN = CGNStack.pop_back_val(); + + if (!VisitedCGNodes.insert(CGN).second) + continue; + + // Ignore call graph node which does not have associated function or + // associated function is not a definition. + if (!CGN->getFunction() || CGN->getFunction()->isDeclaration()) + continue; + + for (const auto &GI : *CGN) { + auto *RCB = cast<CallBase>(GI.first.getValue()); + auto *RCGN = GI.second; + + if (auto *DCallee = RCGN->getFunction()) { + ReachableCallees.insert(DCallee); + } else if (RCB->isIndirectCall()) { + auto *RCBFTy = RCB->getFunctionType(); + for (auto *ACGN : AddressTakenFunctions) { + auto *ACallee = ACGN->getFunction(); + if (ACallee->getFunctionType() == RCBFTy) { + ReachableCallees.insert(ACallee); + CGNStack.append(df_begin(ACGN), df_end(ACGN)); + } + } + } + } + } + + return ReachableCallees; + } + +public: + explicit CollectReachableCallees(Module &M) : M(M), CG(CallGraph(M)) { + // Collect address taken functions. + collectAddressTakenFunctions(); + } + + void collectReachableCallees( + DenseMap<Function *, SmallPtrSet<Function *, 8>> &KernelToCallees) { + // Collect reachable callee set for each kernel defined in the module. + for (Function &F : M.functions()) { + if (!llvm::AMDGPU::isKernelCC(&F)) + continue; + Function *K = &F; + KernelToCallees[K] = collectReachableCallees(K); + } + } +}; + +/// Collect reachable callees for each kernel defined in the module \p M and +/// return collected callees at \p KernelToCallees. +void collectReachableCallees( + Module &M, + DenseMap<Function *, SmallPtrSet<Function *, 8>> &KernelToCallees) { + CollectReachableCallees CRC{M}; + CRC.collectReachableCallees(KernelToCallees); +} + +/// For the given LDS global \p GV, visit all its users and collect all +/// non-kernel functions within which \p GV is used and return collected list of +/// such non-kernel functions. +SmallPtrSet<Function *, 8> collectNonKernelAccessorsOfLDS(GlobalVariable *GV) { + SmallPtrSet<Function *, 8> LDSAccessors; + SmallVector<User *, 8> UserStack(GV->users()); + SmallPtrSet<User *, 8> VisitedUsers; + + while (!UserStack.empty()) { + auto *U = UserStack.pop_back_val(); + + // `U` is already visited? continue to next one. + if (!VisitedUsers.insert(U).second) + continue; + + // `U` is a global variable which is initialized with LDS. Ignore LDS. + if (isa<GlobalValue>(U)) + return SmallPtrSet<Function *, 8>(); + + // Recursively explore constant users. + if (isa<Constant>(U)) { + append_range(UserStack, U->users()); + continue; + } + + // `U` should be an instruction, if it belongs to a non-kernel function F, + // then collect F. + Function *F = cast<Instruction>(U)->getFunction(); + if (!llvm::AMDGPU::isKernelCC(F)) + LDSAccessors.insert(F); + } + + return LDSAccessors; +} + +DenseMap<Function *, SmallPtrSet<Instruction *, 8>> +getFunctionToInstsMap(User *U, bool CollectKernelInsts) { + DenseMap<Function *, SmallPtrSet<Instruction *, 8>> FunctionToInsts; + SmallVector<User *, 8> UserStack; + SmallPtrSet<User *, 8> VisitedUsers; + + UserStack.push_back(U); + + while (!UserStack.empty()) { + auto *UU = UserStack.pop_back_val(); + + if (!VisitedUsers.insert(UU).second) + continue; + + if (isa<GlobalValue>(UU)) + continue; + + if (isa<Constant>(UU)) { + append_range(UserStack, UU->users()); + continue; + } + + auto *I = cast<Instruction>(UU); + Function *F = I->getFunction(); + if (CollectKernelInsts) { + if (!llvm::AMDGPU::isKernelCC(F)) { + continue; + } + } else { + if (llvm::AMDGPU::isKernelCC(F)) { + continue; + } + } + + FunctionToInsts.insert(std::make_pair(F, SmallPtrSet<Instruction *, 8>())); + FunctionToInsts[F].insert(I); + } + + return FunctionToInsts; +} + +} // namespace AMDGPU + // Entry-point function which interface ReplaceLDSUseImpl with outside of the // class. bool ReplaceLDSUseImpl::replaceLDSUse() { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp index 0655b4342ba1..cd05797fdbdb 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp @@ -413,21 +413,21 @@ bool GCNSubtarget::zeroesHigh16BitsOfDest(unsigned Opcode) const { case AMDGPU::V_MAX_I16_e32: case AMDGPU::V_MIN_I16_e64: case AMDGPU::V_MIN_I16_e32: + case AMDGPU::V_MAD_F16_e64: + case AMDGPU::V_MAD_U16_e64: + case AMDGPU::V_MAD_I16_e64: + case AMDGPU::V_FMA_F16_e64: + case AMDGPU::V_DIV_FIXUP_F16_e64: // On gfx10, all 16-bit instructions preserve the high bits. return getGeneration() <= AMDGPUSubtarget::GFX9; - case AMDGPU::V_MAD_F16_e64: case AMDGPU::V_MADAK_F16: case AMDGPU::V_MADMK_F16: case AMDGPU::V_MAC_F16_e64: case AMDGPU::V_MAC_F16_e32: case AMDGPU::V_FMAMK_F16: case AMDGPU::V_FMAAK_F16: - case AMDGPU::V_MAD_U16_e64: - case AMDGPU::V_MAD_I16_e64: - case AMDGPU::V_FMA_F16_e64: case AMDGPU::V_FMAC_F16_e64: case AMDGPU::V_FMAC_F16_e32: - case AMDGPU::V_DIV_FIXUP_F16_e64: // In gfx9, the preferred handling of the unused high 16-bits changed. Most // instructions maintain the legacy behavior of 0ing. Some instructions // changed to preserving the high bits. @@ -648,9 +648,18 @@ bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const { } unsigned AMDGPUSubtarget::getImplicitArgNumBytes(const Function &F) const { + assert(AMDGPU::isKernel(F.getCallingConv())); + + // We don't allocate the segment if we know the implicit arguments weren't + // used, even if the ABI implies we need them. + if (F.hasFnAttribute("amdgpu-no-implicitarg-ptr")) + return 0; + if (isMesaKernel(F)) return 16; - return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 0); + + // Assume all implicit inputs are used by default + return AMDGPU::getIntegerAttribute(F, "amdgpu-implicitarg-num-bytes", 56); } uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index de11676279f2..a2c61f9da8da 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -231,13 +231,6 @@ static cl::opt<bool, true> LateCFGStructurize( cl::location(AMDGPUTargetMachine::EnableLateStructurizeCFG), cl::Hidden); -static cl::opt<bool, true> EnableAMDGPUFixedFunctionABIOpt( - "amdgpu-fixed-function-abi", - cl::desc("Enable all implicit function arguments"), - cl::location(AMDGPUTargetMachine::EnableFixedFunctionABI), - cl::init(false), - cl::Hidden); - // Enable lib calls simplifications static cl::opt<bool> EnableLibCallSimplify( "amdgpu-simplify-libcall", @@ -505,7 +498,6 @@ AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT, bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false; bool AMDGPUTargetMachine::EnableFunctionCalls = false; -bool AMDGPUTargetMachine::EnableFixedFunctionABI = false; bool AMDGPUTargetMachine::EnableLowerModuleLDS = true; AMDGPUTargetMachine::~AMDGPUTargetMachine() = default; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h index 0ff2db2a52d9..226646a96953 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h @@ -37,7 +37,6 @@ protected: public: static bool EnableLateStructurizeCFG; static bool EnableFunctionCalls; - static bool EnableFixedFunctionABI; static bool EnableLowerModuleLDS; AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index ecdbdf613a53..09c5eb192e1f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -519,57 +519,6 @@ InstructionCost GCNTTIImpl::getArithmeticInstrCost( TTI::OperandValueProperties Opd1PropInfo, TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args, const Instruction *CxtI) { - EVT OrigTy = TLI->getValueType(DL, Ty); - if (!OrigTy.isSimple()) { - // FIXME: We're having to query the throughput cost so that the basic - // implementation tries to generate legalize and scalarization costs. Maybe - // we could hoist the scalarization code here? - if (CostKind != TTI::TCK_CodeSize) - return BaseT::getArithmeticInstrCost(Opcode, Ty, TTI::TCK_RecipThroughput, - Opd1Info, Opd2Info, Opd1PropInfo, - Opd2PropInfo, Args, CxtI); - // Scalarization - - // Check if any of the operands are vector operands. - int ISD = TLI->InstructionOpcodeToISD(Opcode); - assert(ISD && "Invalid opcode"); - - std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty); - - bool IsFloat = Ty->isFPOrFPVectorTy(); - // Assume that floating point arithmetic operations cost twice as much as - // integer operations. - unsigned OpCost = (IsFloat ? 2 : 1); - - if (TLI->isOperationLegalOrPromote(ISD, LT.second)) { - // The operation is legal. Assume it costs 1. - // TODO: Once we have extract/insert subvector cost we need to use them. - return LT.first * OpCost; - } - - if (!TLI->isOperationExpand(ISD, LT.second)) { - // If the operation is custom lowered, then assume that the code is twice - // as expensive. - return LT.first * 2 * OpCost; - } - - // Else, assume that we need to scalarize this op. - // TODO: If one of the types get legalized by splitting, handle this - // similarly to what getCastInstrCost() does. - if (auto *VTy = dyn_cast<VectorType>(Ty)) { - unsigned Num = cast<FixedVectorType>(VTy)->getNumElements(); - InstructionCost Cost = getArithmeticInstrCost( - Opcode, VTy->getScalarType(), CostKind, Opd1Info, Opd2Info, - Opd1PropInfo, Opd2PropInfo, Args, CxtI); - // Return the cost of multiple scalar invocation plus the cost of - // inserting and extracting the values. - SmallVector<Type *> Tys(Args.size(), Ty); - return getScalarizationOverhead(VTy, Args, Tys) + Num * Cost; - } - - // We don't know anything about this scalar instruction. - return OpCost; - } // Legalize the type. std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty); @@ -742,40 +691,6 @@ GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, return BaseT::getIntrinsicInstrCost(ICA, CostKind); Type *RetTy = ICA.getReturnType(); - EVT OrigTy = TLI->getValueType(DL, RetTy); - if (!OrigTy.isSimple()) { - if (CostKind != TTI::TCK_CodeSize) - return BaseT::getIntrinsicInstrCost(ICA, CostKind); - - // TODO: Combine these two logic paths. - if (ICA.isTypeBasedOnly()) - return getTypeBasedIntrinsicInstrCost(ICA, CostKind); - - unsigned RetVF = - (RetTy->isVectorTy() ? cast<FixedVectorType>(RetTy)->getNumElements() - : 1); - const IntrinsicInst *I = ICA.getInst(); - const SmallVectorImpl<const Value *> &Args = ICA.getArgs(); - FastMathFlags FMF = ICA.getFlags(); - // Assume that we need to scalarize this intrinsic. - - // Compute the scalarization overhead based on Args for a vector - // intrinsic. A vectorizer will pass a scalar RetTy and VF > 1, while - // CostModel will pass a vector RetTy and VF is 1. - InstructionCost ScalarizationCost = InstructionCost::getInvalid(); - if (RetVF > 1) { - ScalarizationCost = 0; - if (!RetTy->isVoidTy()) - ScalarizationCost += - getScalarizationOverhead(cast<VectorType>(RetTy), true, false); - ScalarizationCost += - getOperandsScalarizationOverhead(Args, ICA.getArgTypes()); - } - - IntrinsicCostAttributes Attrs(ICA.getID(), RetTy, ICA.getArgTypes(), FMF, I, - ScalarizationCost); - return getIntrinsicInstrCost(Attrs, CostKind); - } // Legalize the type. std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, RetTy); diff --git a/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp b/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp index 712f6dece911..1736c078eb83 100644 --- a/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp @@ -173,10 +173,8 @@ protected: } static void PrintLoopinfo(const MachineLoopInfo &LoopInfo) { - for (MachineLoop::iterator iter = LoopInfo.begin(), - iterEnd = LoopInfo.end(); iter != iterEnd; ++iter) { - (*iter)->print(dbgs()); - } + for (const MachineLoop *L : LoopInfo) + L->print(dbgs()); } // UTILITY FUNCTIONS @@ -691,9 +689,7 @@ bool AMDGPUCFGStructurizer::prepare() { SmallVector<MachineBasicBlock *, DEFAULT_VEC_SLOTS> RetBlks; // Add an ExitBlk to loop that don't have one - for (MachineLoopInfo::iterator It = MLI->begin(), - E = MLI->end(); It != E; ++It) { - MachineLoop *LoopRep = (*It); + for (MachineLoop *LoopRep : *MLI) { MBBVector ExitingMBBs; LoopRep->getExitingBlocks(ExitingMBBs); @@ -827,14 +823,13 @@ bool AMDGPUCFGStructurizer::run() { wrapup(*GraphTraits<MachineFunction *>::nodes_begin(FuncRep)); // Detach retired Block, release memory. - for (MBBInfoMap::iterator It = BlockInfoMap.begin(), E = BlockInfoMap.end(); - It != E; ++It) { - if ((*It).second && (*It).second->IsRetired) { - assert(((*It).first)->getNumber() != -1); - LLVM_DEBUG(dbgs() << "Erase BB" << ((*It).first)->getNumber() << "\n";); - (*It).first->eraseFromParent(); //Remove from the parent Function. + for (auto &It : BlockInfoMap) { + if (It.second && It.second->IsRetired) { + assert((It.first)->getNumber() != -1); + LLVM_DEBUG(dbgs() << "Erase BB" << (It.first)->getNumber() << "\n";); + It.first->eraseFromParent(); // Remove from the parent Function. } - delete (*It).second; + delete It.second; } BlockInfoMap.clear(); LLInfoMap.clear(); diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 4acd77a9d5d2..2bb59086f391 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -246,8 +246,12 @@ public: return isRegKind() && !hasModifiers(); } + bool isRegOrInline(unsigned RCID, MVT type) const { + return isRegClass(RCID) || isInlinableImm(type); + } + bool isRegOrImmWithInputMods(unsigned RCID, MVT type) const { - return isRegClass(RCID) || isInlinableImm(type) || isLiteralImm(type); + return isRegOrInline(RCID, type) || isLiteralImm(type); } bool isRegOrImmWithInt16InputMods() const { @@ -372,7 +376,7 @@ public: bool isInlineValue() const; bool isRegOrInlineNoMods(unsigned RCID, MVT type) const { - return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers(); + return isRegOrInline(RCID, type) && !hasModifiers(); } bool isSCSrcB16() const { diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index d3644db7cf8b..a535c8cc0918 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -6,11 +6,11 @@ // //===----------------------------------------------------------------------===// -def MUBUFAddr64 : ComplexPattern<i64, 4, "SelectMUBUFAddr64">; -def MUBUFOffset : ComplexPattern<i64, 3, "SelectMUBUFOffset">; +def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; +def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; -def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; -def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; +def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; +def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; def BUFAddrKind { int Offset = 0; diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index bb0aa648ff90..c7ec5308e6d0 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -6,12 +6,12 @@ // //===----------------------------------------------------------------------===// -def FlatOffset : ComplexPattern<i64, 2, "SelectFlatOffset", [], [SDNPWantRoot], -10>; -def GlobalOffset : ComplexPattern<i64, 2, "SelectGlobalOffset", [], [SDNPWantRoot], -10>; -def ScratchOffset : ComplexPattern<i32, 2, "SelectScratchOffset", [], [SDNPWantRoot], -10>; +def FlatOffset : ComplexPattern<iPTR, 2, "SelectFlatOffset", [], [SDNPWantRoot], -10>; +def GlobalOffset : ComplexPattern<iPTR, 2, "SelectGlobalOffset", [], [SDNPWantRoot], -10>; +def ScratchOffset : ComplexPattern<iPTR, 2, "SelectScratchOffset", [], [SDNPWantRoot], -10>; -def GlobalSAddr : ComplexPattern<i64, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>; -def ScratchSAddr : ComplexPattern<i32, 2, "SelectScratchSAddr", [], [SDNPWantRoot], -10>; +def GlobalSAddr : ComplexPattern<iPTR, 3, "SelectGlobalSAddr", [], [SDNPWantRoot], -10>; +def ScratchSAddr : ComplexPattern<iPTR, 2, "SelectScratchSAddr", [], [SDNPWantRoot], -10>; //===----------------------------------------------------------------------===// // FLAT classes diff --git a/llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp b/llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp index f3f664f7972a..912bcc792e4d 100644 --- a/llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp +++ b/llvm/lib/Target/AMDGPU/MCA/AMDGPUCustomBehaviour.cpp @@ -120,8 +120,7 @@ unsigned AMDGPUCustomBehaviour::handleWaitCnt(ArrayRef<InstRef> IssuedInst, // We will now look at each of the currently executing instructions // to find out if this wait instruction still needs to wait. - for (auto I = IssuedInst.begin(), E = IssuedInst.end(); I != E; I++) { - const InstRef &PrevIR = *I; + for (const InstRef &PrevIR : IssuedInst) { const Instruction &PrevInst = *PrevIR.getInstruction(); const unsigned PrevInstIndex = PrevIR.getSourceIndex() % SrcMgr.size(); const WaitCntInfo &PrevInstWaitInfo = InstrWaitCntInfo[PrevInstIndex]; diff --git a/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp b/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp index 29c37c706138..8a48a67b829c 100644 --- a/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp +++ b/llvm/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp @@ -440,9 +440,8 @@ private: CounterPropagateAddr(*Clause.first, CfCount); MachineBasicBlock *BB = Clause.first->getParent(); BuildMI(BB, DL, TII->get(R600::FETCH_CLAUSE)).addImm(CfCount); - for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) { - BB->splice(InsertPos, BB, Clause.second[i]); - } + for (MachineInstr *MI : Clause.second) + BB->splice(InsertPos, BB, MI); CfCount += 2 * Clause.second.size(); } @@ -452,9 +451,8 @@ private: CounterPropagateAddr(*Clause.first, CfCount); MachineBasicBlock *BB = Clause.first->getParent(); BuildMI(BB, DL, TII->get(R600::ALU_CLAUSE)).addImm(CfCount); - for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) { - BB->splice(InsertPos, BB, Clause.second[i]); - } + for (MachineInstr *MI : Clause.second) + BB->splice(InsertPos, BB, MI); CfCount += Clause.second.size(); } @@ -635,10 +633,10 @@ public: CfCount++; } MI->eraseFromParent(); - for (unsigned i = 0, e = FetchClauses.size(); i < e; i++) - EmitFetchClause(I, DL, FetchClauses[i], CfCount); - for (unsigned i = 0, e = AluClauses.size(); i < e; i++) - EmitALUClause(I, DL, AluClauses[i], CfCount); + for (ClauseFile &CF : FetchClauses) + EmitFetchClause(I, DL, CF, CfCount); + for (ClauseFile &CF : AluClauses) + EmitALUClause(I, DL, CF, CfCount); break; } default: @@ -649,8 +647,7 @@ public: break; } } - for (unsigned i = 0, e = ToPopAfter.size(); i < e; ++i) { - MachineInstr *Alu = ToPopAfter[i]; + for (MachineInstr *Alu : ToPopAfter) { BuildMI(MBB, Alu, MBB.findDebugLoc((MachineBasicBlock::iterator)Alu), TII->get(R600::CF_ALU_POP_AFTER)) .addImm(Alu->getOperand(0).getImm()) diff --git a/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp b/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp index a7ebf72315cb..aec8b1ae4837 100644 --- a/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp @@ -268,17 +268,15 @@ R600InstrInfo::getSrcs(MachineInstr &MI) const { {R600::OpName::src1_W, R600::OpName::src1_sel_W}, }; - for (unsigned j = 0; j < 8; j++) { - MachineOperand &MO = - MI.getOperand(getOperandIdx(MI.getOpcode(), OpTable[j][0])); + for (const auto &Op : OpTable) { + MachineOperand &MO = MI.getOperand(getOperandIdx(MI.getOpcode(), Op[0])); Register Reg = MO.getReg(); if (Reg == R600::ALU_CONST) { MachineOperand &Sel = - MI.getOperand(getOperandIdx(MI.getOpcode(), OpTable[j][1])); + MI.getOperand(getOperandIdx(MI.getOpcode(), Op[1])); Result.push_back(std::make_pair(&MO, Sel.getImm())); continue; } - } return Result; } @@ -289,15 +287,14 @@ R600InstrInfo::getSrcs(MachineInstr &MI) const { {R600::OpName::src2, R600::OpName::src2_sel}, }; - for (unsigned j = 0; j < 3; j++) { - int SrcIdx = getOperandIdx(MI.getOpcode(), OpTable[j][0]); + for (const auto &Op : OpTable) { + int SrcIdx = getOperandIdx(MI.getOpcode(), Op[0]); if (SrcIdx < 0) break; MachineOperand &MO = MI.getOperand(SrcIdx); Register Reg = MO.getReg(); if (Reg == R600::ALU_CONST) { - MachineOperand &Sel = - MI.getOperand(getOperandIdx(MI.getOpcode(), OpTable[j][1])); + MachineOperand &Sel = MI.getOperand(getOperandIdx(MI.getOpcode(), Op[1])); Result.push_back(std::make_pair(&MO, Sel.getImm())); continue; } @@ -521,12 +518,11 @@ R600InstrInfo::fitsReadPortLimitations(const std::vector<MachineInstr *> &IG, ValidSwizzle.clear(); unsigned ConstCount; BankSwizzle TransBS = ALU_VEC_012_SCL_210; - for (unsigned i = 0, e = IG.size(); i < e; ++i) { - IGSrcs.push_back(ExtractSrcs(*IG[i], PV, ConstCount)); - unsigned Op = getOperandIdx(IG[i]->getOpcode(), - R600::OpName::bank_swizzle); - ValidSwizzle.push_back( (R600InstrInfo::BankSwizzle) - IG[i]->getOperand(Op).getImm()); + for (MachineInstr *MI : IG) { + IGSrcs.push_back(ExtractSrcs(*MI, PV, ConstCount)); + unsigned Op = getOperandIdx(MI->getOpcode(), R600::OpName::bank_swizzle); + ValidSwizzle.push_back( + (R600InstrInfo::BankSwizzle)MI->getOperand(Op).getImm()); } std::vector<std::pair<int, unsigned>> TransOps; if (!isLastAluTrans) @@ -542,8 +538,7 @@ R600InstrInfo::fitsReadPortLimitations(const std::vector<MachineInstr *> &IG, ALU_VEC_120_SCL_212, ALU_VEC_102_SCL_221 }; - for (unsigned i = 0; i < 4; i++) { - TransBS = TransSwz[i]; + for (R600InstrInfo::BankSwizzle TransBS : TransSwz) { if (!isConstCompatible(TransBS, TransOps, ConstCount)) continue; bool Result = FindSwizzleForVectorSlot(IGSrcs, ValidSwizzle, TransOps, @@ -562,9 +557,9 @@ R600InstrInfo::fitsConstReadLimitations(const std::vector<unsigned> &Consts) const { assert (Consts.size() <= 12 && "Too many operands in instructions group"); unsigned Pair1 = 0, Pair2 = 0; - for (unsigned i = 0, n = Consts.size(); i < n; ++i) { - unsigned ReadConstHalf = Consts[i] & 2; - unsigned ReadConstIndex = Consts[i] & (~3); + for (unsigned Const : Consts) { + unsigned ReadConstHalf = Const & 2; + unsigned ReadConstIndex = Const & (~3); unsigned ReadHalfConst = ReadConstIndex | ReadConstHalf; if (!Pair1) { Pair1 = ReadHalfConst; @@ -587,12 +582,11 @@ R600InstrInfo::fitsConstReadLimitations(const std::vector<MachineInstr *> &MIs) const { std::vector<unsigned> Consts; SmallSet<int64_t, 4> Literals; - for (unsigned i = 0, n = MIs.size(); i < n; i++) { - MachineInstr &MI = *MIs[i]; - if (!isALUInstr(MI.getOpcode())) + for (MachineInstr *MI : MIs) { + if (!isALUInstr(MI->getOpcode())) continue; - for (const auto &Src : getSrcs(MI)) { + for (const auto &Src : getSrcs(*MI)) { if (Src.first->getReg() == R600::ALU_LITERAL_X) Literals.insert(Src.second); if (Literals.size() > 4) @@ -1330,11 +1324,11 @@ MachineInstr *R600InstrInfo::buildSlotOfVectorInstruction( MIB->getOperand(getOperandIdx(Opcode, R600::OpName::pred_sel)) .setReg(MO.getReg()); - for (unsigned i = 0; i < 14; i++) { + for (unsigned Operand : Operands) { MachineOperand &MO = MI->getOperand( - getOperandIdx(MI->getOpcode(), getSlotedOps(Operands[i], Slot))); + getOperandIdx(MI->getOpcode(), getSlotedOps(Operand, Slot))); assert (MO.isImm()); - setImmOperand(*MIB, Operands[i], MO.getImm()); + setImmOperand(*MIB, Operand, MO.getImm()); } MIB->getOperand(20).setImm(0); return MIB; diff --git a/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp b/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp index 6aee2f591b56..d26879ed8d60 100644 --- a/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp +++ b/llvm/lib/Target/AMDGPU/R600MachineScheduler.cpp @@ -328,9 +328,9 @@ SUnit *R600SchedStrategy::PopInst(std::vector<SUnit *> &Q, bool AnyALU) { void R600SchedStrategy::LoadAlu() { std::vector<SUnit *> &QSrc = Pending[IDAlu]; - for (unsigned i = 0, e = QSrc.size(); i < e; ++i) { - AluKind AK = getAluKind(QSrc[i]); - AvailableAlus[AK].push_back(QSrc[i]); + for (SUnit *SU : QSrc) { + AluKind AK = getAluKind(SU); + AvailableAlus[AK].push_back(SU); } QSrc.clear(); } diff --git a/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp b/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp index ac6a3581e255..aa156190b7ae 100644 --- a/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp +++ b/llvm/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp @@ -307,8 +307,8 @@ class R600OpenCLImageTypeLoweringPass : public ModulePass { // Build new MDNode. SmallVector<Metadata *, 6> KernelMDArgs; KernelMDArgs.push_back(ConstantAsMetadata::get(NewF)); - for (unsigned i = 0; i < NumKernelArgMDNodes; ++i) - KernelMDArgs.push_back(MDNode::get(*Context, NewArgMDs.ArgVector[i])); + for (const MDVector &MDV : NewArgMDs.ArgVector) + KernelMDArgs.push_back(MDNode::get(*Context, MDV)); MDNode *NewMDNode = MDNode::get(*Context, KernelMDArgs); return std::make_tuple(NewF, NewMDNode); diff --git a/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp b/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp index 72cf48c04e7f..795bc898a7bf 100644 --- a/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp +++ b/llvm/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp @@ -150,19 +150,18 @@ bool R600VectorRegMerger::tryMergeVector(const RegSeqInfo *Untouched, RegSeqInfo *ToMerge, std::vector< std::pair<unsigned, unsigned>> &Remap) const { unsigned CurrentUndexIdx = 0; - for (DenseMap<Register, unsigned>::iterator It = ToMerge->RegToChan.begin(), - E = ToMerge->RegToChan.end(); It != E; ++It) { + for (auto &It : ToMerge->RegToChan) { DenseMap<Register, unsigned>::const_iterator PosInUntouched = - Untouched->RegToChan.find((*It).first); + Untouched->RegToChan.find(It.first); if (PosInUntouched != Untouched->RegToChan.end()) { - Remap.push_back(std::pair<unsigned, unsigned> - ((*It).second, (*PosInUntouched).second)); + Remap.push_back( + std::pair<unsigned, unsigned>(It.second, (*PosInUntouched).second)); continue; } if (CurrentUndexIdx >= Untouched->UndefReg.size()) return false; - Remap.push_back(std::pair<unsigned, unsigned> - ((*It).second, Untouched->UndefReg[CurrentUndexIdx++])); + Remap.push_back(std::pair<unsigned, unsigned>( + It.second, Untouched->UndefReg[CurrentUndexIdx++])); } return true; @@ -172,9 +171,9 @@ static unsigned getReassignedChan( const std::vector<std::pair<unsigned, unsigned>> &RemapChan, unsigned Chan) { - for (unsigned j = 0, je = RemapChan.size(); j < je; j++) { - if (RemapChan[j].first == Chan) - return RemapChan[j].second; + for (const auto &J : RemapChan) { + if (J.first == Chan) + return J.second; } llvm_unreachable("Chan wasn't reassigned"); } @@ -190,11 +189,10 @@ MachineInstr *R600VectorRegMerger::RebuildVector( Register SrcVec = BaseRSI->Instr->getOperand(0).getReg(); DenseMap<Register, unsigned> UpdatedRegToChan = BaseRSI->RegToChan; std::vector<Register> UpdatedUndef = BaseRSI->UndefReg; - for (DenseMap<Register, unsigned>::iterator It = RSI->RegToChan.begin(), - E = RSI->RegToChan.end(); It != E; ++It) { + for (const auto &It : RSI->RegToChan) { Register DstReg = MRI->createVirtualRegister(&R600::R600_Reg128RegClass); - unsigned SubReg = (*It).first; - unsigned Swizzle = (*It).second; + unsigned SubReg = It.first; + unsigned Swizzle = It.second; unsigned Chan = getReassignedChan(RemapChan, Swizzle); MachineInstr *Tmp = BuildMI(MBB, Pos, DL, TII->get(R600::INSERT_SUBREG), @@ -234,14 +232,12 @@ MachineInstr *R600VectorRegMerger::RebuildVector( } void R600VectorRegMerger::RemoveMI(MachineInstr *MI) { - for (InstructionSetMap::iterator It = PreviousRegSeqByReg.begin(), - E = PreviousRegSeqByReg.end(); It != E; ++It) { - std::vector<MachineInstr *> &MIs = (*It).second; + for (auto &It : PreviousRegSeqByReg) { + std::vector<MachineInstr *> &MIs = It.second; MIs.erase(llvm::find(MIs, MI), MIs.end()); } - for (InstructionSetMap::iterator It = PreviousRegSeqByUndefCount.begin(), - E = PreviousRegSeqByUndefCount.end(); It != E; ++It) { - std::vector<MachineInstr *> &MIs = (*It).second; + for (auto &It : PreviousRegSeqByUndefCount) { + std::vector<MachineInstr *> &MIs = It.second; MIs.erase(llvm::find(MIs, MI), MIs.end()); } } @@ -255,9 +251,9 @@ void R600VectorRegMerger::SwizzleInput(MachineInstr &MI, Offset = 3; for (unsigned i = 0; i < 4; i++) { unsigned Swizzle = MI.getOperand(i + Offset).getImm() + 1; - for (unsigned j = 0, e = RemapChan.size(); j < e; j++) { - if (RemapChan[j].first == Swizzle) { - MI.getOperand(i + Offset).setImm(RemapChan[j].second - 1); + for (const auto &J : RemapChan) { + if (J.first == Swizzle) { + MI.getOperand(i + Offset).setImm(J.second - 1); break; } } diff --git a/llvm/lib/Target/AMDGPU/R600Packetizer.cpp b/llvm/lib/Target/AMDGPU/R600Packetizer.cpp index beb0aad86e89..fbe2a1cd9fba 100644 --- a/llvm/lib/Target/AMDGPU/R600Packetizer.cpp +++ b/llvm/lib/Target/AMDGPU/R600Packetizer.cpp @@ -127,8 +127,8 @@ private: R600::OpName::src1, R600::OpName::src2 }; - for (unsigned i = 0; i < 3; i++) { - int OperandIdx = TII->getOperandIdx(MI.getOpcode(), Ops[i]); + for (unsigned Op : Ops) { + int OperandIdx = TII->getOperandIdx(MI.getOpcode(), Op); if (OperandIdx < 0) continue; Register Src = MI.getOperand(OperandIdx).getReg(); diff --git a/llvm/lib/Target/AMDGPU/R600RegisterInfo.cpp b/llvm/lib/Target/AMDGPU/R600RegisterInfo.cpp index 99a1a8e9871a..c329bae50f92 100644 --- a/llvm/lib/Target/AMDGPU/R600RegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/R600RegisterInfo.cpp @@ -54,10 +54,8 @@ BitVector R600RegisterInfo::getReservedRegs(const MachineFunction &MF) const { reserveRegisterTuples(Reserved, R600::PRED_SEL_ONE); reserveRegisterTuples(Reserved, R600::INDIRECT_BASE_ADDR); - for (TargetRegisterClass::iterator I = R600::R600_AddrRegClass.begin(), - E = R600::R600_AddrRegClass.end(); I != E; ++I) { - reserveRegisterTuples(Reserved, *I); - } + for (MCPhysReg R : R600::R600_AddrRegClass) + reserveRegisterTuples(Reserved, R); TII->reserveIndirectRegisters(Reserved, MF, *this); diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp index 200e00ee5521..1f93284fc7ee 100644 --- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp +++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp @@ -1620,7 +1620,7 @@ bool SIFoldOperands::tryFoldRegSequence(MachineInstr &MI) { // Erase the REG_SEQUENCE eagerly, unless we followed a chain of COPY users, // in which case we can erase them all later in runOnMachineFunction. if (MRI->use_nodbg_empty(MI.getOperand(0).getReg())) - MI.eraseFromParentAndMarkDBGValuesForRemoval(); + MI.eraseFromParent(); return true; } @@ -1821,7 +1821,7 @@ bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) { while (MRI->use_nodbg_empty(InstToErase->getOperand(0).getReg())) { auto &SrcOp = InstToErase->getOperand(1); auto SrcReg = SrcOp.isReg() ? SrcOp.getReg() : Register(); - InstToErase->eraseFromParentAndMarkDBGValuesForRemoval(); + InstToErase->eraseFromParent(); InstToErase = nullptr; if (!SrcReg || SrcReg.isPhysical()) break; @@ -1831,7 +1831,7 @@ bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) { } if (InstToErase && InstToErase->isRegSequence() && MRI->use_nodbg_empty(InstToErase->getOperand(0).getReg())) - InstToErase->eraseFromParentAndMarkDBGValuesForRemoval(); + InstToErase->eraseFromParent(); } } return true; diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp index 4706c74be721..d4fe74ecb96e 100644 --- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp @@ -1167,11 +1167,13 @@ void SIFrameLowering::processFunctionBeforeFrameFinalized( if (SpillVGPRToAGPR) { // To track the spill frame indices handled in this pass. BitVector SpillFIs(MFI.getObjectIndexEnd(), false); + BitVector NonVGPRSpillFIs(MFI.getObjectIndexEnd(), false); bool SeenDbgInstr = false; for (MachineBasicBlock &MBB : MF) { for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) { + int FrameIndex; if (MI.isDebugInstr()) SeenDbgInstr = true; @@ -1191,10 +1193,18 @@ void SIFrameLowering::processFunctionBeforeFrameFinalized( SpillFIs.set(FI); continue; } - } + } else if (TII->isStoreToStackSlot(MI, FrameIndex) || + TII->isLoadFromStackSlot(MI, FrameIndex)) + NonVGPRSpillFIs.set(FrameIndex); } } + // Stack slot coloring may assign different objets to the same stack slot. + // If not, then the VGPR to AGPR spill slot is dead. + for (unsigned FI : SpillFIs.set_bits()) + if (!NonVGPRSpillFIs.test(FI)) + FuncInfo->setVGPRToAGPRSpillDead(FI); + for (MachineBasicBlock &MBB : MF) { for (MCPhysReg Reg : FuncInfo->getVGPRSpillAGPRs()) MBB.addLiveIn(Reg); diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 35b72f5d201b..9f138136e6e9 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -24,6 +24,7 @@ #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" +#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/IR/DiagnosticInfo.h" @@ -2062,33 +2063,30 @@ void SITargetLowering::allocateSpecialInputSGPRs( SIMachineFunctionInfo &Info) const { auto &ArgInfo = Info.getArgInfo(); - // We need to allocate these in place regardless of their use. - const bool IsFixed = AMDGPUTargetMachine::EnableFixedFunctionABI; - // TODO: Unify handling with private memory pointers. - if (IsFixed || Info.hasDispatchPtr()) + if (Info.hasDispatchPtr()) allocateSGPR64Input(CCInfo, ArgInfo.DispatchPtr); - if (IsFixed || Info.hasQueuePtr()) + if (Info.hasQueuePtr()) allocateSGPR64Input(CCInfo, ArgInfo.QueuePtr); // Implicit arg ptr takes the place of the kernarg segment pointer. This is a // constant offset from the kernarg segment. - if (IsFixed || Info.hasImplicitArgPtr()) + if (Info.hasImplicitArgPtr()) allocateSGPR64Input(CCInfo, ArgInfo.ImplicitArgPtr); - if (IsFixed || Info.hasDispatchID()) + if (Info.hasDispatchID()) allocateSGPR64Input(CCInfo, ArgInfo.DispatchID); // flat_scratch_init is not applicable for non-kernel functions. - if (IsFixed || Info.hasWorkGroupIDX()) + if (Info.hasWorkGroupIDX()) allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDX); - if (IsFixed || Info.hasWorkGroupIDY()) + if (Info.hasWorkGroupIDY()) allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDY); - if (IsFixed || Info.hasWorkGroupIDZ()) + if (Info.hasWorkGroupIDZ()) allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDZ); } @@ -2419,10 +2417,9 @@ SDValue SITargetLowering::LowerFormalArguments( if (IsEntryFunc) { allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info); allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info); - } else { + } else if (!IsGraphics) { // For the fixed ABI, pass workitem IDs in the last argument register. - if (AMDGPUTargetMachine::EnableFixedFunctionABI) - allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info); + allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info); } if (IsKernel) { @@ -2549,17 +2546,13 @@ SDValue SITargetLowering::LowerFormalArguments( InVals.push_back(Val); } - if (!IsEntryFunc && !AMDGPUTargetMachine::EnableFixedFunctionABI) { - // Special inputs come after user arguments. - allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info); - } - // Start adding system SGPRs. if (IsEntryFunc) { allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsGraphics); } else { CCInfo.AllocateReg(Info->getScratchRSrcReg()); - allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); + if (!IsGraphics) + allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info); } auto &ArgUsageInfo = @@ -3123,8 +3116,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg); - if (AMDGPUTargetMachine::EnableFixedFunctionABI && - CallConv != CallingConv::AMDGPU_Gfx) { + if (CallConv != CallingConv::AMDGPU_Gfx) { // With a fixed ABI, allocate fixed registers before user arguments. passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain); } @@ -3263,12 +3255,6 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, } } - if (!AMDGPUTargetMachine::EnableFixedFunctionABI && - CallConv != CallingConv::AMDGPU_Gfx) { - // Copy special input registers after user input arguments. - passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain); - } - if (!MemOpChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); @@ -6282,10 +6268,6 @@ SDValue SITargetLowering::lowerImage(SDValue Op, } } - // Push back extra arguments. - for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) - VAddrs.push_back(Op.getOperand(ArgOffset + I)); - // Check for 16 bit addresses or derivatives and pack if true. MVT VAddrVT = Op.getOperand(ArgOffset + Intr->GradientStart).getSimpleValueType(); @@ -6298,6 +6280,17 @@ SDValue SITargetLowering::lowerImage(SDValue Op, MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; + // Push back extra arguments. + for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) { + if (IsA16 && (Op.getOperand(ArgOffset + I).getValueType() == MVT::f16)) { + // Special handling of bias when A16 is on. Bias is of type half but + // occupies full 32-bit. + SDValue bias = DAG.getBuildVector( MVT::v2f16, DL, {Op.getOperand(ArgOffset + I), DAG.getUNDEF(MVT::f16)}); + VAddrs.push_back(bias); + } else + VAddrs.push_back(Op.getOperand(ArgOffset + I)); + } + if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) { // 16 bit gradients are supported, but are tied to the A16 control // so both gradients and addresses must be 16 bit @@ -7502,8 +7495,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, assert(NodePtr.getValueType() == MVT::i32 || NodePtr.getValueType() == MVT::i64); - assert(RayDir.getValueType() == MVT::v4f16 || - RayDir.getValueType() == MVT::v4f32); + assert(RayDir.getValueType() == MVT::v3f16 || + RayDir.getValueType() == MVT::v3f32); if (!Subtarget->hasGFX10_AEncoding()) { emitRemovedIntrinsicError(DAG, DL, Op.getValueType()); @@ -9837,11 +9830,13 @@ bool SITargetLowering::isCanonicalized(Register Reg, MachineFunction &MF, if (Opcode == AMDGPU::G_FCANONICALIZE) return true; - if (Opcode == AMDGPU::G_FCONSTANT) { - auto F = MI->getOperand(1).getFPImm()->getValueAPF(); - if (F.isNaN() && F.isSignaling()) + Optional<FPValueAndVReg> FCR; + // Constant splat (can be padded with undef) or scalar constant. + if (mi_match(Reg, MRI, MIPatternMatch::m_GFCstOrSplat(FCR))) { + if (FCR->Value.isSignaling()) return false; - return !F.isDenormal() || denormalsEnabledForType(MRI.getType(Reg), MF); + return !FCR->Value.isDenormal() || + denormalsEnabledForType(MRI.getType(FCR->VReg), MF); } if (MaxDepth == 0) @@ -11514,7 +11509,7 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, // Prefer VGPRs over AGPRs in mAI instructions where possible. // This saves a chain-copy of registers and better ballance register // use between vgpr and agpr as agpr tuples tend to be big. - if (const MCOperandInfo *OpInfo = MI.getDesc().OpInfo) { + if (MI.getDesc().OpInfo) { unsigned Opc = MI.getOpcode(); const SIRegisterInfo *TRI = Subtarget->getRegisterInfo(); for (auto I : { AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), @@ -12477,6 +12472,6 @@ SITargetLowering::getTypeLegalizationCost(const DataLayout &DL, if (Size <= 256) return Cost; - Cost.first = (Size + 255) / 256; + Cost.first += (Size + 255) / 256; return Cost; } diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp index c9d9dd1fb82c..6fbe5d45ce0a 100644 --- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp +++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp @@ -30,6 +30,7 @@ #include "Utils/AMDGPUBaseInfo.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/PostOrderIterator.h" +#include "llvm/ADT/Sequence.h" #include "llvm/CodeGen/MachinePostDominators.h" #include "llvm/InitializePasses.h" #include "llvm/Support/DebugCounter.h" @@ -51,26 +52,6 @@ static cl::opt<bool> ForceEmitZeroFlag( cl::init(false), cl::Hidden); namespace { - -template <typename EnumT> -class enum_iterator - : public iterator_facade_base<enum_iterator<EnumT>, - std::forward_iterator_tag, const EnumT> { - EnumT Value; -public: - enum_iterator() = default; - enum_iterator(EnumT Value) : Value(Value) {} - - enum_iterator &operator++() { - Value = static_cast<EnumT>(Value + 1); - return *this; - } - - bool operator==(const enum_iterator &RHS) const { return Value == RHS.Value; } - - EnumT operator*() const { return Value; } -}; - // Class of object that encapsulates latest instruction counter score // associated with the operand. Used for determining whether // s_waitcnt instruction needs to be emitted. @@ -78,27 +59,32 @@ public: #define CNT_MASK(t) (1u << (t)) enum InstCounterType { VM_CNT = 0, LGKM_CNT, EXP_CNT, VS_CNT, NUM_INST_CNTS }; +} // namespace -iterator_range<enum_iterator<InstCounterType>> inst_counter_types() { - return make_range(enum_iterator<InstCounterType>(VM_CNT), - enum_iterator<InstCounterType>(NUM_INST_CNTS)); -} +namespace llvm { +template <> struct enum_iteration_traits<InstCounterType> { + static constexpr bool is_iterable = true; +}; +} // namespace llvm + +namespace { +auto inst_counter_types() { return enum_seq(VM_CNT, NUM_INST_CNTS); } using RegInterval = std::pair<int, int>; -struct { +struct HardwareLimits { unsigned VmcntMax; unsigned ExpcntMax; unsigned LgkmcntMax; unsigned VscntMax; -} HardwareLimits; +}; -struct { +struct RegisterEncoding { unsigned VGPR0; unsigned VGPRL; unsigned SGPR0; unsigned SGPRL; -} RegisterEncoding; +}; enum WaitEventType { VMEM_ACCESS, // vector-memory read & write @@ -194,18 +180,20 @@ void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) { // "s_waitcnt 0" before use. class WaitcntBrackets { public: - WaitcntBrackets(const GCNSubtarget *SubTarget) : ST(SubTarget) {} + WaitcntBrackets(const GCNSubtarget *SubTarget, HardwareLimits Limits, + RegisterEncoding Encoding) + : ST(SubTarget), Limits(Limits), Encoding(Encoding) {} - static unsigned getWaitCountMax(InstCounterType T) { + unsigned getWaitCountMax(InstCounterType T) const { switch (T) { case VM_CNT: - return HardwareLimits.VmcntMax; + return Limits.VmcntMax; case LGKM_CNT: - return HardwareLimits.LgkmcntMax; + return Limits.LgkmcntMax; case EXP_CNT: - return HardwareLimits.ExpcntMax; + return Limits.ExpcntMax; case VS_CNT: - return HardwareLimits.VscntMax; + return Limits.VscntMax; default: break; } @@ -338,6 +326,8 @@ private: unsigned OpNo, unsigned Val); const GCNSubtarget *ST = nullptr; + HardwareLimits Limits = {}; + RegisterEncoding Encoding = {}; unsigned ScoreLBs[NUM_INST_CNTS] = {0}; unsigned ScoreUBs[NUM_INST_CNTS] = {0}; unsigned PendingEvents = 0; @@ -471,14 +461,14 @@ RegInterval WaitcntBrackets::getRegInterval(const MachineInstr *MI, unsigned Reg = TRI->getEncodingValue(AMDGPU::getMCReg(Op.getReg(), *ST)); if (TRI->isVectorRegister(*MRI, Op.getReg())) { - assert(Reg >= RegisterEncoding.VGPR0 && Reg <= RegisterEncoding.VGPRL); - Result.first = Reg - RegisterEncoding.VGPR0; + assert(Reg >= Encoding.VGPR0 && Reg <= Encoding.VGPRL); + Result.first = Reg - Encoding.VGPR0; if (TRI->isAGPR(*MRI, Op.getReg())) Result.first += AGPR_OFFSET; assert(Result.first >= 0 && Result.first < SQ_MAX_PGM_VGPRS); } else if (TRI->isSGPRReg(*MRI, Op.getReg())) { - assert(Reg >= RegisterEncoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS); - Result.first = Reg - RegisterEncoding.SGPR0 + NUM_ALL_VGPRS; + assert(Reg >= Encoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS); + Result.first = Reg - Encoding.SGPR0 + NUM_ALL_VGPRS; assert(Result.first >= NUM_ALL_VGPRS && Result.first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS); } @@ -1589,20 +1579,22 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) { for (auto T : inst_counter_types()) ForceEmitWaitcnt[T] = false; - HardwareLimits.VmcntMax = AMDGPU::getVmcntBitMask(IV); - HardwareLimits.ExpcntMax = AMDGPU::getExpcntBitMask(IV); - HardwareLimits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV); - HardwareLimits.VscntMax = ST->hasVscnt() ? 63 : 0; + HardwareLimits Limits = {}; + Limits.VmcntMax = AMDGPU::getVmcntBitMask(IV); + Limits.ExpcntMax = AMDGPU::getExpcntBitMask(IV); + Limits.LgkmcntMax = AMDGPU::getLgkmcntBitMask(IV); + Limits.VscntMax = ST->hasVscnt() ? 63 : 0; unsigned NumVGPRsMax = ST->getAddressableNumVGPRs(); unsigned NumSGPRsMax = ST->getAddressableNumSGPRs(); assert(NumVGPRsMax <= SQ_MAX_PGM_VGPRS); assert(NumSGPRsMax <= SQ_MAX_PGM_SGPRS); - RegisterEncoding.VGPR0 = TRI->getEncodingValue(AMDGPU::VGPR0); - RegisterEncoding.VGPRL = RegisterEncoding.VGPR0 + NumVGPRsMax - 1; - RegisterEncoding.SGPR0 = TRI->getEncodingValue(AMDGPU::SGPR0); - RegisterEncoding.SGPRL = RegisterEncoding.SGPR0 + NumSGPRsMax - 1; + RegisterEncoding Encoding = {}; + Encoding.VGPR0 = TRI->getEncodingValue(AMDGPU::VGPR0); + Encoding.VGPRL = Encoding.VGPR0 + NumVGPRsMax - 1; + Encoding.SGPR0 = TRI->getEncodingValue(AMDGPU::SGPR0); + Encoding.SGPRL = Encoding.SGPR0 + NumSGPRsMax - 1; TrackedWaitcntSet.clear(); BlockInfos.clear(); @@ -1652,9 +1644,9 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) { *Brackets = *BI.Incoming; } else { if (!Brackets) - Brackets = std::make_unique<WaitcntBrackets>(ST); + Brackets = std::make_unique<WaitcntBrackets>(ST, Limits, Encoding); else - *Brackets = WaitcntBrackets(ST); + *Brackets = WaitcntBrackets(ST, Limits, Encoding); } Modified |= insertWaitcntInBlock(MF, *BI.MBB, *Brackets); @@ -1686,45 +1678,47 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) { } } while (Repeat); - SmallVector<MachineBasicBlock *, 4> EndPgmBlocks; - - bool HaveScalarStores = false; + if (ST->hasScalarStores()) { + SmallVector<MachineBasicBlock *, 4> EndPgmBlocks; + bool HaveScalarStores = false; - for (MachineBasicBlock &MBB : MF) { - for (MachineInstr &MI : MBB) { - if (!HaveScalarStores && TII->isScalarStore(MI)) - HaveScalarStores = true; + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &MI : MBB) { + if (!HaveScalarStores && TII->isScalarStore(MI)) + HaveScalarStores = true; - if (MI.getOpcode() == AMDGPU::S_ENDPGM || - MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) - EndPgmBlocks.push_back(&MBB); + if (MI.getOpcode() == AMDGPU::S_ENDPGM || + MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) + EndPgmBlocks.push_back(&MBB); + } } - } - if (HaveScalarStores) { - // If scalar writes are used, the cache must be flushed or else the next - // wave to reuse the same scratch memory can be clobbered. - // - // Insert s_dcache_wb at wave termination points if there were any scalar - // stores, and only if the cache hasn't already been flushed. This could be - // improved by looking across blocks for flushes in postdominating blocks - // from the stores but an explicitly requested flush is probably very rare. - for (MachineBasicBlock *MBB : EndPgmBlocks) { - bool SeenDCacheWB = false; + if (HaveScalarStores) { + // If scalar writes are used, the cache must be flushed or else the next + // wave to reuse the same scratch memory can be clobbered. + // + // Insert s_dcache_wb at wave termination points if there were any scalar + // stores, and only if the cache hasn't already been flushed. This could + // be improved by looking across blocks for flushes in postdominating + // blocks from the stores but an explicitly requested flush is probably + // very rare. + for (MachineBasicBlock *MBB : EndPgmBlocks) { + bool SeenDCacheWB = false; - for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E; - ++I) { - if (I->getOpcode() == AMDGPU::S_DCACHE_WB) - SeenDCacheWB = true; - else if (TII->isScalarStore(*I)) - SeenDCacheWB = false; + for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); + I != E; ++I) { + if (I->getOpcode() == AMDGPU::S_DCACHE_WB) + SeenDCacheWB = true; + else if (TII->isScalarStore(*I)) + SeenDCacheWB = false; - // FIXME: It would be better to insert this before a waitcnt if any. - if ((I->getOpcode() == AMDGPU::S_ENDPGM || - I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) && - !SeenDCacheWB) { - Modified = true; - BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB)); + // FIXME: It would be better to insert this before a waitcnt if any. + if ((I->getOpcode() == AMDGPU::S_ENDPGM || + I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) && + !SeenDCacheWB) { + Modified = true; + BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB)); + } } } } diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 92f5322b8ad2..1755b93538ce 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -899,8 +899,12 @@ void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, unsigned EltSize = 4; unsigned Opcode = AMDGPU::V_MOV_B32_e32; if (RI.isAGPRClass(RC)) { - Opcode = (RI.hasVGPRs(SrcRC)) ? - AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; + if (ST.hasGFX90AInsts() && RI.isAGPRClass(SrcRC)) + Opcode = AMDGPU::V_ACCVGPR_MOV_B32; + else if (RI.hasVGPRs(SrcRC)) + Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64; + else + Opcode = AMDGPU::INSTRUCTION_LIST_END; } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(SrcRC)) { Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64; } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) && @@ -1417,6 +1421,33 @@ static unsigned getAGPRSpillSaveOpcode(unsigned Size) { } } +static unsigned getAVSpillSaveOpcode(unsigned Size) { + switch (Size) { + case 4: + return AMDGPU::SI_SPILL_AV32_SAVE; + case 8: + return AMDGPU::SI_SPILL_AV64_SAVE; + case 12: + return AMDGPU::SI_SPILL_AV96_SAVE; + case 16: + return AMDGPU::SI_SPILL_AV128_SAVE; + case 20: + return AMDGPU::SI_SPILL_AV160_SAVE; + case 24: + return AMDGPU::SI_SPILL_AV192_SAVE; + case 28: + return AMDGPU::SI_SPILL_AV224_SAVE; + case 32: + return AMDGPU::SI_SPILL_AV256_SAVE; + case 64: + return AMDGPU::SI_SPILL_AV512_SAVE; + case 128: + return AMDGPU::SI_SPILL_AV1024_SAVE; + default: + llvm_unreachable("unknown register size"); + } +} + void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, @@ -1463,21 +1494,11 @@ void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, return; } - unsigned Opcode = RI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(SpillSize) - : getVGPRSpillSaveOpcode(SpillSize); + unsigned Opcode = RI.isVectorSuperClass(RC) ? getAVSpillSaveOpcode(SpillSize) + : RI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(SpillSize) + : getVGPRSpillSaveOpcode(SpillSize); MFI->setHasSpilledVGPRs(); - if (RI.isVectorSuperClass(RC)) { - // Convert an AV spill into a VGPR spill. Introduce a copy from AV to an - // equivalent VGPR register beforehand. Regalloc might want to introduce - // AV spills only to be relevant until rewriter at which they become - // either spills of VGPRs or AGPRs. - Register TmpVReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass(RC)); - BuildMI(MBB, MI, DL, get(TargetOpcode::COPY), TmpVReg) - .addReg(SrcReg, RegState::Kill); - SrcReg = TmpVReg; - } - BuildMI(MBB, MI, DL, get(Opcode)) .addReg(SrcReg, getKillRegState(isKill)) // data .addFrameIndex(FrameIndex) // addr @@ -1567,6 +1588,33 @@ static unsigned getAGPRSpillRestoreOpcode(unsigned Size) { } } +static unsigned getAVSpillRestoreOpcode(unsigned Size) { + switch (Size) { + case 4: + return AMDGPU::SI_SPILL_AV32_RESTORE; + case 8: + return AMDGPU::SI_SPILL_AV64_RESTORE; + case 12: + return AMDGPU::SI_SPILL_AV96_RESTORE; + case 16: + return AMDGPU::SI_SPILL_AV128_RESTORE; + case 20: + return AMDGPU::SI_SPILL_AV160_RESTORE; + case 24: + return AMDGPU::SI_SPILL_AV192_RESTORE; + case 28: + return AMDGPU::SI_SPILL_AV224_RESTORE; + case 32: + return AMDGPU::SI_SPILL_AV256_RESTORE; + case 64: + return AMDGPU::SI_SPILL_AV512_RESTORE; + case 128: + return AMDGPU::SI_SPILL_AV1024_RESTORE; + default: + llvm_unreachable("unknown register size"); + } +} + void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg, int FrameIndex, @@ -1609,26 +1657,15 @@ void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, return; } - unsigned Opcode = RI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(SpillSize) - : getVGPRSpillRestoreOpcode(SpillSize); - - bool IsVectorSuperClass = RI.isVectorSuperClass(RC); - Register TmpReg = DestReg; - if (IsVectorSuperClass) { - // For AV classes, insert the spill restore to a VGPR followed by a copy - // into an equivalent AV register. - MachineRegisterInfo &MRI = MF->getRegInfo(); - DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass(RC)); - } + unsigned Opcode = RI.isVectorSuperClass(RC) + ? getAVSpillRestoreOpcode(SpillSize) + : RI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(SpillSize) + : getVGPRSpillRestoreOpcode(SpillSize); BuildMI(MBB, MI, DL, get(Opcode), DestReg) - .addFrameIndex(FrameIndex) // vaddr - .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset - .addImm(0) // offset - .addMemOperand(MMO); - - if (IsVectorSuperClass) - BuildMI(MBB, MI, DL, get(TargetOpcode::COPY), TmpReg) - .addReg(DestReg, RegState::Kill); + .addFrameIndex(FrameIndex) // vaddr + .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset + .addImm(0) // offset + .addMemOperand(MMO); } void SIInstrInfo::insertNoop(MachineBasicBlock &MBB, @@ -2358,8 +2395,6 @@ void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, OffsetLo->setVariableValue(MCBinaryExpr::createAnd(Offset, Mask, MCCtx)); auto *ShAmt = MCConstantExpr::create(32, MCCtx); OffsetHi->setVariableValue(MCBinaryExpr::createAShr(Offset, ShAmt, MCCtx)); - - return; } unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) { @@ -3106,23 +3141,26 @@ bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, } static bool getFoldableImm(Register Reg, const MachineRegisterInfo &MRI, - int64_t &Imm) { + int64_t &Imm, MachineInstr **DefMI = nullptr) { if (Reg.isPhysical()) return false; auto *Def = MRI.getUniqueVRegDef(Reg); if (Def && SIInstrInfo::isFoldableCopy(*Def) && Def->getOperand(1).isImm()) { Imm = Def->getOperand(1).getImm(); + if (DefMI) + *DefMI = Def; return true; } return false; } -static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm) { +static bool getFoldableImm(const MachineOperand *MO, int64_t &Imm, + MachineInstr **DefMI = nullptr) { if (!MO->isReg()) return false; const MachineFunction *MF = MO->getParent()->getParent()->getParent(); const MachineRegisterInfo &MRI = MF->getRegInfo(); - return getFoldableImm(MO->getReg(), MRI, Imm); + return getFoldableImm(MO->getReg(), MRI, Imm, DefMI); } static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI, @@ -3195,8 +3233,20 @@ MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI, // If we have an SGPR input, we will violate the constant bus restriction. (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() || !RI.isSGPRReg(MBB.getParent()->getRegInfo(), Src0->getReg()))) { + MachineInstr *DefMI; + const auto killDef = [&DefMI, &MBB, this]() -> void { + const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); + // The only user is the instruction which will be killed. + if (!MRI.hasOneNonDBGUse(DefMI->getOperand(0).getReg())) + return; + // We cannot just remove the DefMI here, calling pass will crash. + DefMI->setDesc(get(AMDGPU::IMPLICIT_DEF)); + for (unsigned I = DefMI->getNumOperands() - 1; I != 0; --I) + DefMI->RemoveOperand(I); + }; + int64_t Imm; - if (getFoldableImm(Src2, Imm)) { + if (getFoldableImm(Src2, Imm, &DefMI)) { unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32) : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); @@ -3209,13 +3259,14 @@ MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI, updateLiveVariables(LV, MI, *MIB); if (LIS) LIS->ReplaceMachineInstrInMaps(MI, *MIB); + killDef(); return MIB; } } unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32) : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); - if (getFoldableImm(Src1, Imm)) { + if (getFoldableImm(Src1, Imm, &DefMI)) { if (pseudoToMCOpcode(NewOpc) != -1) { MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc)) .add(*Dst) @@ -3225,10 +3276,11 @@ MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI, updateLiveVariables(LV, MI, *MIB); if (LIS) LIS->ReplaceMachineInstrInMaps(MI, *MIB); + killDef(); return MIB; } } - if (getFoldableImm(Src0, Imm)) { + if (getFoldableImm(Src0, Imm, &DefMI)) { if (pseudoToMCOpcode(NewOpc) != -1 && isOperandLegal( MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), @@ -3241,12 +3293,13 @@ MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI, updateLiveVariables(LV, MI, *MIB); if (LIS) LIS->ReplaceMachineInstrInMaps(MI, *MIB); + killDef(); return MIB; } } } - unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16_e64 + unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16_gfx9_e64 : IsF64 ? AMDGPU::V_FMA_F64_e64 : AMDGPU::V_FMA_F32_e64) : (IsF16 ? AMDGPU::V_MAD_F16_e64 : AMDGPU::V_MAD_F32_e64); @@ -3605,12 +3658,6 @@ bool SIInstrInfo::canShrink(const MachineInstr &MI, const MachineRegisterInfo &MRI) const { const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); // Can't shrink instruction with three operands. - // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add - // a special case for it. It can only be shrunk if the third operand - // is vcc, and src0_modifiers and src1_modifiers are not set. - // We should handle this the same way we handle vopc, by addding - // a register allocation hint pre-regalloc and then do the shrinking - // post-regalloc. if (Src2) { switch (MI.getOpcode()) { default: return false; @@ -4563,8 +4610,9 @@ static unsigned adjustAllocatableRegClass(const GCNSubtarget &ST, unsigned RCID, bool IsAllocatable) { if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && - (TID.mayLoad() || TID.mayStore() || - (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) { + (((TID.mayLoad() || TID.mayStore()) && + !(TID.TSFlags & SIInstrFlags::VGPRSpill)) || + (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) { switch (RCID) { case AMDGPU::AV_32RegClassID: return AMDGPU::VGPR_32RegClassID; case AMDGPU::AV_64RegClassID: return AMDGPU::VReg_64RegClassID; @@ -5001,8 +5049,7 @@ void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI, --ConstantBusLimit; } - for (unsigned i = 0; i < 3; ++i) { - int Idx = VOP3Idx[i]; + for (int Idx : VOP3Idx) { if (Idx == -1) break; MachineOperand &MO = MI.getOperand(Idx); diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index 47ee83eb9351..dda92d3d25ff 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -1350,11 +1350,11 @@ def PackedI16InputMods : PackedIntInputMods<PackedI16InputModsMatchClass>; // Complex patterns //===----------------------------------------------------------------------===// -def DS1Addr1Offset : ComplexPattern<i32, 2, "SelectDS1Addr1Offset">; -def DS64Bit4ByteAligned : ComplexPattern<i32, 3, "SelectDS64Bit4ByteAligned">; -def DS128Bit8ByteAligned : ComplexPattern<i64, 3, "SelectDS128Bit8ByteAligned">; +def DS1Addr1Offset : ComplexPattern<iPTR, 2, "SelectDS1Addr1Offset">; +def DS64Bit4ByteAligned : ComplexPattern<iPTR, 3, "SelectDS64Bit4ByteAligned">; +def DS128Bit8ByteAligned : ComplexPattern<iPTR, 3, "SelectDS128Bit8ByteAligned">; -def MOVRELOffset : ComplexPattern<i32, 2, "SelectMOVRELOffset">; +def MOVRELOffset : ComplexPattern<iPTR, 2, "SelectMOVRELOffset">; def VOP3Mods0 : ComplexPattern<untyped, 4, "SelectVOP3Mods0">; def VOP3Mods : ComplexPattern<untyped, 2, "SelectVOP3Mods">; diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index d55d8da8699a..636337ede000 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -761,6 +761,17 @@ defm SI_SPILL_A256 : SI_SPILL_VGPR <AReg_256, 1>; defm SI_SPILL_A512 : SI_SPILL_VGPR <AReg_512, 1>; defm SI_SPILL_A1024 : SI_SPILL_VGPR <AReg_1024, 1>; +defm SI_SPILL_AV32 : SI_SPILL_VGPR <AV_32, 1>; +defm SI_SPILL_AV64 : SI_SPILL_VGPR <AV_64, 1>; +defm SI_SPILL_AV96 : SI_SPILL_VGPR <AV_96, 1>; +defm SI_SPILL_AV128 : SI_SPILL_VGPR <AV_128, 1>; +defm SI_SPILL_AV160 : SI_SPILL_VGPR <AV_160, 1>; +defm SI_SPILL_AV192 : SI_SPILL_VGPR <AV_192, 1>; +defm SI_SPILL_AV224 : SI_SPILL_VGPR <AV_224, 1>; +defm SI_SPILL_AV256 : SI_SPILL_VGPR <AV_256, 1>; +defm SI_SPILL_AV512 : SI_SPILL_VGPR <AV_512, 1>; +defm SI_SPILL_AV1024 : SI_SPILL_VGPR <AV_1024, 1>; + def SI_PC_ADD_REL_OFFSET : SPseudoInstSI < (outs SReg_64:$dst), (ins si_ga:$ptr_lo, si_ga:$ptr_hi), @@ -2106,6 +2117,19 @@ def : GCNPat < } // end isWave32 def : GCNPat < + (i32 (DivergentBinFrag<xor> i32:$src0, (i32 -1))), + (V_NOT_B32_e32 $src0) +>; + +def : GCNPat < + (i64 (DivergentBinFrag<xor> i64:$src0, (i64 -1))), + (REG_SEQUENCE VReg_64, + (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub0))), sub0, + (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub1))), sub1 + ) +>; + +def : GCNPat < (f16 (sint_to_fp i1:$src)), (V_CVT_F16_F32_e32 ( V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0), @@ -2188,18 +2212,18 @@ def : GCNPat < >; def : GCNPat < - (i1 (trunc i32:$a)), - (V_CMP_EQ_U32_e64 (S_AND_B32 (i32 1), $a), (i32 1)) + (i1 (DivergentUnaryFrag<trunc> i32:$a)), + (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1)) >; def : GCNPat < - (i1 (trunc i16:$a)), - (V_CMP_EQ_U32_e64 (S_AND_B32 (i32 1), $a), (i32 1)) + (i1 (DivergentUnaryFrag<trunc> i16:$a)), + (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), $a), (i32 1)) >; def : GCNPat < - (i1 (trunc i64:$a)), - (V_CMP_EQ_U32_e64 (S_AND_B32 (i32 1), + (i1 (DivergentUnaryFrag<trunc> i64:$a)), + (V_CMP_EQ_U32_e64 (V_AND_B32_e64 (i32 1), (i32 (EXTRACT_SUBREG $a, sub0))), (i32 1)) >; @@ -2405,21 +2429,37 @@ def : GCNPat < // COPY is workaround tablegen bug from multiple outputs // from S_LSHL_B32's multiple outputs from implicit scc def. def : GCNPat < - (v2i16 (build_vector (i16 0), (i16 SReg_32:$src1))), + (v2i16 (UniformBinFrag<build_vector> (i16 0), (i16 SReg_32:$src1))), (S_LSHL_B32 SReg_32:$src1, (i16 16)) >; def : GCNPat < - (v2i16 (build_vector (i16 SReg_32:$src1), (i16 0))), + (v2i16 (DivergentBinFrag<build_vector> (i16 0), (i16 SReg_32:$src1))), + (v2i16 (V_LSHLREV_B32_e64 (i16 16), SReg_32:$src1)) +>; + + +def : GCNPat < + (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src1), (i16 0))), (S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1) >; def : GCNPat < - (v2f16 (build_vector (f16 SReg_32:$src1), (f16 FP_ZERO))), + (v2i16 (DivergentBinFrag<build_vector> (i16 SReg_32:$src1), (i16 0))), + (v2i16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), SReg_32:$src1)) +>; + +def : GCNPat < + (v2f16 (UniformBinFrag<build_vector> (f16 SReg_32:$src1), (f16 FP_ZERO))), (S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1) >; def : GCNPat < + (v2f16 (DivergentBinFrag<build_vector> (f16 SReg_32:$src1), (f16 FP_ZERO))), + (v2f16 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), SReg_32:$src1)) +>; + +def : GCNPat < (v2i16 (build_vector (i16 SReg_32:$src0), (i16 undef))), (COPY_TO_REGCLASS SReg_32:$src0, SReg_32) >; @@ -2435,42 +2475,74 @@ def : GCNPat < >; def : GCNPat < - (v2i16 (build_vector (i16 undef), (i16 SReg_32:$src1))), + (v2i16 (UniformBinFrag<build_vector> (i16 undef), (i16 SReg_32:$src1))), (S_LSHL_B32 SReg_32:$src1, (i32 16)) >; def : GCNPat < - (v2f16 (build_vector (f16 undef), (f16 SReg_32:$src1))), + (v2i16 (DivergentBinFrag<build_vector> (i16 undef), (i16 SReg_32:$src1))), + (v2i16 (V_LSHLREV_B32_e64 (i32 16), SReg_32:$src1)) +>; + + +def : GCNPat < + (v2f16 (UniformBinFrag<build_vector> (f16 undef), (f16 SReg_32:$src1))), (S_LSHL_B32 SReg_32:$src1, (i32 16)) >; +def : GCNPat < + (v2f16 (DivergentBinFrag<build_vector> (f16 undef), (f16 SReg_32:$src1))), + (v2f16 (V_LSHLREV_B32_e64 (i32 16), SReg_32:$src1)) +>; + let SubtargetPredicate = HasVOP3PInsts in { def : GCNPat < - (v2i16 (build_vector (i16 SReg_32:$src0), (i16 SReg_32:$src1))), + (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 SReg_32:$src1))), (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1) >; +def : GCNPat < + (v2i16 (DivergentBinFrag<build_vector> (i16 SReg_32:$src0), (i16 SReg_32:$src1))), + (v2i16 (V_LSHL_OR_B32_e64 $src1, (i32 16), (i32 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), $src0)))) +>; + // With multiple uses of the shift, this will duplicate the shift and // increase register pressure. def : GCNPat < - (v2i16 (build_vector (i16 SReg_32:$src0), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), + (v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src0), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), (v2i16 (S_PACK_LH_B32_B16 SReg_32:$src0, SReg_32:$src1)) >; +def : GCNPat < + (v2i16 (DivergentBinFrag<build_vector> (i16 SReg_32:$src0), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), + (v2i16 (V_BFI_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), SReg_32:$src0, SReg_32:$src1)) +>; + def : GCNPat < - (v2i16 (build_vector (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))), + (v2i16 (UniformBinFrag<build_vector> (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), (S_PACK_HH_B32_B16 SReg_32:$src0, SReg_32:$src1) >; -// TODO: Should source modifiers be matched to v_pack_b32_f16? def : GCNPat < - (v2f16 (build_vector (f16 SReg_32:$src0), (f16 SReg_32:$src1))), + (v2i16 (DivergentBinFrag<build_vector> (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))), + (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), + (v2i16 (V_AND_OR_B32_e64 SReg_32:$src1, (i32 (V_MOV_B32_e32 (i32 0xffff0000))), (i32 (V_LSHRREV_B32_e64 (i32 16), SReg_32:$src0)))) +>; + +def : GCNPat < + (v2f16 (UniformBinFrag<build_vector> (f16 SReg_32:$src0), (f16 SReg_32:$src1))), (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1) >; def : GCNPat < + (v2f16 (DivergentBinFrag<build_vector> (f16 SReg_32:$src0), (f16 SReg_32:$src1))), + (v2f16 (V_LSHL_OR_B32_e64 SReg_32:$src1, (i32 16), (i32 (V_AND_B32_e64 (i32 (V_MOV_B32_e32 (i32 0xffff))), SReg_32:$src0)))) +>; + + +def : GCNPat < (v2f16 (is_canonicalized<build_vector> (f16 (VOP3Mods (f16 VGPR_32:$src0), i32:$src0_mods)), (f16 (VOP3Mods (f16 VGPR_32:$src1), i32:$src1_mods)))), (V_PACK_B32_F16_e64 $src0_mods, VGPR_32:$src0, $src1_mods, VGPR_32:$src1) @@ -2866,6 +2938,18 @@ def G_AMDGPU_UMED3 : AMDGPUGenericInstruction { let hasSideEffects = 0; } +def G_AMDGPU_FMED3 : AMDGPUGenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src0, type0:$src1, type0:$src2); + let hasSideEffects = 0; +} + +def G_AMDGPU_CLAMP : AMDGPUGenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src); + let hasSideEffects = 0; +} + // Atomic cmpxchg. $cmpval ad $newval are packed in a single vector // operand Expects a MachineMemOperand in addition to explicit // operands. diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp index c4007f56f350..3ce368ef4db9 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -62,11 +62,6 @@ SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF) // calls. const bool HasCalls = F.hasFnAttribute("amdgpu-calls"); - // Enable all kernel inputs if we have the fixed ABI. Don't bother if we don't - // have any calls. - const bool UseFixedABI = AMDGPUTargetMachine::EnableFixedFunctionABI && - CC != CallingConv::AMDGPU_Gfx && - (!isEntryFunction() || HasCalls); const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL; @@ -80,7 +75,7 @@ SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF) } if (!isEntryFunction()) { - if (UseFixedABI) + if (CC != CallingConv::AMDGPU_Gfx) ArgInfo = AMDGPUArgumentUsageInfo::FixedABIFunctionInfo; // TODO: Pick a high register, and shift down, similar to a kernel. @@ -110,20 +105,7 @@ SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF) else if (ST.isMesaGfxShader(F)) ImplicitBufferPtr = true; - if (UseFixedABI) { - DispatchPtr = true; - QueuePtr = true; - ImplicitArgPtr = true; - WorkGroupIDX = true; - WorkGroupIDY = true; - WorkGroupIDZ = true; - WorkItemIDX = true; - WorkItemIDY = true; - WorkItemIDZ = true; - - // FIXME: We don't need this? - DispatchID = true; - } else if (!AMDGPU::isGraphics(CC)) { + if (!AMDGPU::isGraphics(CC)) { if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x")) WorkGroupIDX = true; @@ -462,7 +444,7 @@ void SIMachineFunctionInfo::removeDeadFrameIndices(MachineFrameInfo &MFI) { MFI.setStackID(i, TargetStackID::Default); for (auto &R : VGPRToAGPRSpills) { - if (R.second.FullyAllocated) + if (R.second.IsDead) MFI.RemoveStackObject(R.first); } } diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h index c305bc20e40d..8accbf611c5f 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h @@ -465,6 +465,7 @@ public: struct VGPRSpillToAGPR { SmallVector<MCPhysReg, 32> Lanes; bool FullyAllocated = false; + bool IsDead = false; }; // Map WWM VGPR to a stack slot that is used to save/restore it in the @@ -546,6 +547,12 @@ public: : I->second.Lanes[Lane]; } + void setVGPRToAGPRSpillDead(int FrameIndex) { + auto I = VGPRToAGPRSpills.find(FrameIndex); + if (I != VGPRToAGPRSpills.end()) + I->second.IsDead = true; + } + bool haveFreeLanesForSGPRSpill(const MachineFunction &MF, unsigned NumLane) const; bool allocateSGPRSpillToVGPR(MachineFunction &MF, int FI); diff --git a/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp b/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp index 5590d84cc3ab..81db66a98ddf 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp @@ -869,29 +869,27 @@ void SIScheduleBlockCreator::colorComputeReservedDependencies() { } void SIScheduleBlockCreator::colorAccordingToReservedDependencies() { - unsigned DAGSize = DAG->SUnits.size(); std::map<std::pair<unsigned, unsigned>, unsigned> ColorCombinations; // Every combination of colors given by the top down // and bottom up Reserved node dependency - for (unsigned i = 0, e = DAGSize; i != e; ++i) { - SUnit *SU = &DAG->SUnits[i]; + for (const SUnit &SU : DAG->SUnits) { std::pair<unsigned, unsigned> SUColors; // High latency instructions: already given. - if (CurrentColoring[SU->NodeNum]) + if (CurrentColoring[SU.NodeNum]) continue; - SUColors.first = CurrentTopDownReservedDependencyColoring[SU->NodeNum]; - SUColors.second = CurrentBottomUpReservedDependencyColoring[SU->NodeNum]; + SUColors.first = CurrentTopDownReservedDependencyColoring[SU.NodeNum]; + SUColors.second = CurrentBottomUpReservedDependencyColoring[SU.NodeNum]; std::map<std::pair<unsigned, unsigned>, unsigned>::iterator Pos = ColorCombinations.find(SUColors); if (Pos != ColorCombinations.end()) { - CurrentColoring[SU->NodeNum] = Pos->second; + CurrentColoring[SU.NodeNum] = Pos->second; } else { - CurrentColoring[SU->NodeNum] = NextNonReservedID; + CurrentColoring[SU.NodeNum] = NextNonReservedID; ColorCombinations[SUColors] = NextNonReservedID++; } } @@ -1232,15 +1230,13 @@ void SIScheduleBlockCreator::createBlocksForVariant(SISchedulerBlockCreatorVaria } // Free root and leafs of all blocks to enable scheduling inside them. - for (unsigned i = 0, e = CurrentBlocks.size(); i != e; ++i) { - SIScheduleBlock *Block = CurrentBlocks[i]; + for (SIScheduleBlock *Block : CurrentBlocks) Block->finalizeUnits(); - } - LLVM_DEBUG(dbgs() << "Blocks created:\n\n"; - for (unsigned i = 0, e = CurrentBlocks.size(); i != e; ++i) { - SIScheduleBlock *Block = CurrentBlocks[i]; - Block->printDebug(true); - }); + LLVM_DEBUG({ + dbgs() << "Blocks created:\n\n"; + for (SIScheduleBlock *Block : CurrentBlocks) + Block->printDebug(true); + }); } // Two functions taken from Codegen/MachineScheduler.cpp @@ -1379,9 +1375,9 @@ void SIScheduleBlockCreator::scheduleInsideBlocks() { } } - LLVM_DEBUG(for (unsigned i = 0, e = CurrentBlocks.size(); i != e; ++i) { - SIScheduleBlock *Block = CurrentBlocks[i]; - Block->printDebug(true); + LLVM_DEBUG({ + for (SIScheduleBlock *Block : CurrentBlocks) + Block->printDebug(true); }); } @@ -1437,8 +1433,7 @@ SIScheduleBlockScheduler::SIScheduleBlockScheduler(SIScheduleDAGMI *DAG, // found for several parents, we increment the usage of the one with the // highest topological index. LiveOutRegsNumUsages.resize(Blocks.size()); - for (unsigned i = 0, e = Blocks.size(); i != e; ++i) { - SIScheduleBlock *Block = Blocks[i]; + for (SIScheduleBlock *Block : Blocks) { for (unsigned Reg : Block->getInRegs()) { bool Found = false; int topoInd = -1; @@ -1502,8 +1497,7 @@ SIScheduleBlockScheduler::SIScheduleBlockScheduler(SIScheduleDAGMI *DAG, // Fill LiveRegsConsumers for regs that were already // defined before scheduling. - for (unsigned i = 0, e = Blocks.size(); i != e; ++i) { - SIScheduleBlock *Block = Blocks[i]; + for (SIScheduleBlock *Block : Blocks) { for (unsigned Reg : Block->getInRegs()) { bool Found = false; for (SIScheduleBlock* Pred: Block->getPreds()) { @@ -1700,10 +1694,7 @@ void SIScheduleBlockScheduler::blockScheduled(SIScheduleBlock *Block) { decreaseLiveRegs(Block, Block->getInRegs()); addLiveRegs(Block->getOutRegs()); releaseBlockSuccs(Block); - for (std::map<unsigned, unsigned>::iterator RegI = - LiveOutRegsNumUsages[Block->getID()].begin(), - E = LiveOutRegsNumUsages[Block->getID()].end(); RegI != E; ++RegI) { - std::pair<unsigned, unsigned> RegP = *RegI; + for (const auto &RegP : LiveOutRegsNumUsages[Block->getID()]) { // We produce this register, thus it must not be previously alive. assert(LiveRegsConsumers.find(RegP.first) == LiveRegsConsumers.end() || LiveRegsConsumers[RegP.first] == 0); @@ -1759,8 +1750,7 @@ SIScheduler::scheduleVariant(SISchedulerBlockCreatorVariant BlockVariant, ScheduledBlocks = Scheduler.getBlocks(); - for (unsigned b = 0; b < ScheduledBlocks.size(); ++b) { - SIScheduleBlock *Block = ScheduledBlocks[b]; + for (SIScheduleBlock *Block : ScheduledBlocks) { std::vector<SUnit*> SUs = Block->getScheduledUnits(); for (SUnit* SU : SUs) @@ -2000,9 +1990,8 @@ void SIScheduleDAGMI::schedule() assert(TopRPTracker.getPos() == RegionBegin && "bad initial Top tracker"); TopRPTracker.setPos(CurrentTop); - for (std::vector<unsigned>::iterator I = ScheduledSUnits.begin(), - E = ScheduledSUnits.end(); I != E; ++I) { - SUnit *SU = &SUnits[*I]; + for (unsigned I : ScheduledSUnits) { + SUnit *SU = &SUnits[I]; scheduleMI(SU, true); diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index a1d9a23a5084..21aed4ececb5 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -210,6 +210,7 @@ struct SGPRSpillBuilder { auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); if (!TmpVGPRLive) I.addReg(TmpVGPR, RegState::ImplicitDefine); + I->getOperand(2).setIsDead(true); // Mark SCC as dead. TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ false); } } @@ -242,9 +243,10 @@ struct SGPRSpillBuilder { TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true, /*IsKill*/ false); auto I = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); - if (!TmpVGPRLive) { + if (!TmpVGPRLive) I.addReg(TmpVGPR, RegState::ImplicitKill); - } + I->getOperand(2).setIsDead(true); // Mark SCC as dead. + // Restore active lanes if (TmpVGPRLive) TRI.buildVGPRSpillLoadStore(*this, TmpVGPRIndex, 0, /*IsLoad*/ true); @@ -267,9 +269,11 @@ struct SGPRSpillBuilder { TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad, /*IsKill*/ false); // Spill inactive lanes - BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); + auto Not0 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); + Not0->getOperand(2).setIsDead(); // Mark SCC as dead. TRI.buildVGPRSpillLoadStore(*this, Index, Offset, IsLoad); - BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); + auto Not1 = BuildMI(*MBB, MI, DL, TII.get(NotOpc), ExecReg).addReg(ExecReg); + Not1->getOperand(2).setIsDead(); // Mark SCC as dead. } } @@ -908,6 +912,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) { case AMDGPU::SI_SPILL_V1024_RESTORE: case AMDGPU::SI_SPILL_A1024_SAVE: case AMDGPU::SI_SPILL_A1024_RESTORE: + case AMDGPU::SI_SPILL_AV1024_SAVE: + case AMDGPU::SI_SPILL_AV1024_RESTORE: return 32; case AMDGPU::SI_SPILL_S512_SAVE: case AMDGPU::SI_SPILL_S512_RESTORE: @@ -915,6 +921,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) { case AMDGPU::SI_SPILL_V512_RESTORE: case AMDGPU::SI_SPILL_A512_SAVE: case AMDGPU::SI_SPILL_A512_RESTORE: + case AMDGPU::SI_SPILL_AV512_SAVE: + case AMDGPU::SI_SPILL_AV512_RESTORE: return 16; case AMDGPU::SI_SPILL_S256_SAVE: case AMDGPU::SI_SPILL_S256_RESTORE: @@ -922,6 +930,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) { case AMDGPU::SI_SPILL_V256_RESTORE: case AMDGPU::SI_SPILL_A256_SAVE: case AMDGPU::SI_SPILL_A256_RESTORE: + case AMDGPU::SI_SPILL_AV256_SAVE: + case AMDGPU::SI_SPILL_AV256_RESTORE: return 8; case AMDGPU::SI_SPILL_S224_SAVE: case AMDGPU::SI_SPILL_S224_RESTORE: @@ -929,6 +939,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) { case AMDGPU::SI_SPILL_V224_RESTORE: case AMDGPU::SI_SPILL_A224_SAVE: case AMDGPU::SI_SPILL_A224_RESTORE: + case AMDGPU::SI_SPILL_AV224_SAVE: + case AMDGPU::SI_SPILL_AV224_RESTORE: return 7; case AMDGPU::SI_SPILL_S192_SAVE: case AMDGPU::SI_SPILL_S192_RESTORE: @@ -936,6 +948,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) { case AMDGPU::SI_SPILL_V192_RESTORE: case AMDGPU::SI_SPILL_A192_SAVE: case AMDGPU::SI_SPILL_A192_RESTORE: + case AMDGPU::SI_SPILL_AV192_SAVE: + case AMDGPU::SI_SPILL_AV192_RESTORE: return 6; case AMDGPU::SI_SPILL_S160_SAVE: case AMDGPU::SI_SPILL_S160_RESTORE: @@ -943,6 +957,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) { case AMDGPU::SI_SPILL_V160_RESTORE: case AMDGPU::SI_SPILL_A160_SAVE: case AMDGPU::SI_SPILL_A160_RESTORE: + case AMDGPU::SI_SPILL_AV160_SAVE: + case AMDGPU::SI_SPILL_AV160_RESTORE: return 5; case AMDGPU::SI_SPILL_S128_SAVE: case AMDGPU::SI_SPILL_S128_RESTORE: @@ -950,6 +966,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) { case AMDGPU::SI_SPILL_V128_RESTORE: case AMDGPU::SI_SPILL_A128_SAVE: case AMDGPU::SI_SPILL_A128_RESTORE: + case AMDGPU::SI_SPILL_AV128_SAVE: + case AMDGPU::SI_SPILL_AV128_RESTORE: return 4; case AMDGPU::SI_SPILL_S96_SAVE: case AMDGPU::SI_SPILL_S96_RESTORE: @@ -957,6 +975,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) { case AMDGPU::SI_SPILL_V96_RESTORE: case AMDGPU::SI_SPILL_A96_SAVE: case AMDGPU::SI_SPILL_A96_RESTORE: + case AMDGPU::SI_SPILL_AV96_SAVE: + case AMDGPU::SI_SPILL_AV96_RESTORE: return 3; case AMDGPU::SI_SPILL_S64_SAVE: case AMDGPU::SI_SPILL_S64_RESTORE: @@ -964,6 +984,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) { case AMDGPU::SI_SPILL_V64_RESTORE: case AMDGPU::SI_SPILL_A64_SAVE: case AMDGPU::SI_SPILL_A64_RESTORE: + case AMDGPU::SI_SPILL_AV64_SAVE: + case AMDGPU::SI_SPILL_AV64_RESTORE: return 2; case AMDGPU::SI_SPILL_S32_SAVE: case AMDGPU::SI_SPILL_S32_RESTORE: @@ -971,6 +993,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) { case AMDGPU::SI_SPILL_V32_RESTORE: case AMDGPU::SI_SPILL_A32_SAVE: case AMDGPU::SI_SPILL_A32_RESTORE: + case AMDGPU::SI_SPILL_AV32_SAVE: + case AMDGPU::SI_SPILL_AV32_RESTORE: return 1; default: llvm_unreachable("Invalid spill opcode"); } @@ -1240,9 +1264,10 @@ void SIRegisterInfo::buildSpillLoadStore( if (ScratchOffsetReg == AMDGPU::NoRegister) { BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), SOffset).addImm(Offset); } else { - BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset) + auto Add = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_ADD_I32), SOffset) .addReg(ScratchOffsetReg) .addImm(Offset); + Add->getOperand(3).setIsDead(); // Mark SCC as dead. } Offset = 0; @@ -1810,7 +1835,17 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, case AMDGPU::SI_SPILL_A128_SAVE: case AMDGPU::SI_SPILL_A96_SAVE: case AMDGPU::SI_SPILL_A64_SAVE: - case AMDGPU::SI_SPILL_A32_SAVE: { + case AMDGPU::SI_SPILL_A32_SAVE: + case AMDGPU::SI_SPILL_AV1024_SAVE: + case AMDGPU::SI_SPILL_AV512_SAVE: + case AMDGPU::SI_SPILL_AV256_SAVE: + case AMDGPU::SI_SPILL_AV224_SAVE: + case AMDGPU::SI_SPILL_AV192_SAVE: + case AMDGPU::SI_SPILL_AV160_SAVE: + case AMDGPU::SI_SPILL_AV128_SAVE: + case AMDGPU::SI_SPILL_AV96_SAVE: + case AMDGPU::SI_SPILL_AV64_SAVE: + case AMDGPU::SI_SPILL_AV32_SAVE: { const MachineOperand *VData = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == @@ -1846,7 +1881,17 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, case AMDGPU::SI_SPILL_A224_RESTORE: case AMDGPU::SI_SPILL_A256_RESTORE: case AMDGPU::SI_SPILL_A512_RESTORE: - case AMDGPU::SI_SPILL_A1024_RESTORE: { + case AMDGPU::SI_SPILL_A1024_RESTORE: + case AMDGPU::SI_SPILL_AV32_RESTORE: + case AMDGPU::SI_SPILL_AV64_RESTORE: + case AMDGPU::SI_SPILL_AV96_RESTORE: + case AMDGPU::SI_SPILL_AV128_RESTORE: + case AMDGPU::SI_SPILL_AV160_RESTORE: + case AMDGPU::SI_SPILL_AV192_RESTORE: + case AMDGPU::SI_SPILL_AV224_RESTORE: + case AMDGPU::SI_SPILL_AV256_RESTORE: + case AMDGPU::SI_SPILL_AV512_RESTORE: + case AMDGPU::SI_SPILL_AV1024_RESTORE: { const MachineOperand *VData = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp index 3a372d4519fb..c8f1daf26de9 100644 --- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp +++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp @@ -731,11 +731,6 @@ bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) { continue; } - // getVOPe32 could be -1 here if we started with an instruction that had - // a 32-bit encoding and then commuted it to an instruction that did not. - if (!TII->hasVALU32BitEncoding(MI.getOpcode())) - continue; - int Op32 = AMDGPU::getVOPe32(MI.getOpcode()); if (TII->isVOPC(Op32)) { @@ -776,10 +771,6 @@ bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) { const MachineOperand *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst); - // Check the carry-in operand for v_addc_u32_e64. - const MachineOperand *Src2 = TII->getNamedOperand(MI, - AMDGPU::OpName::src2); - if (SDst) { bool Next = false; @@ -791,6 +782,8 @@ bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) { // All of the instructions with carry outs also have an SGPR input in // src2. + const MachineOperand *Src2 = TII->getNamedOperand(MI, + AMDGPU::OpName::src2); if (Src2 && Src2->getReg() != VCCReg) { if (Src2->getReg().isVirtual()) MRI.setRegAllocationHint(Src2->getReg(), 0, VCCReg); diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp index 46012e5d7d97..77ee3c0ff0e4 100644 --- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp +++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp @@ -495,11 +495,10 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF, // instruction as needing e.g. WQM before visiting it and realizing it needs // WQM disabled. ReversePostOrderTraversal<MachineFunction *> RPOT(&MF); - for (auto BI = RPOT.begin(), BE = RPOT.end(); BI != BE; ++BI) { - MachineBasicBlock &MBB = **BI; - BlockInfo &BBI = Blocks[&MBB]; + for (MachineBasicBlock *MBB : RPOT) { + BlockInfo &BBI = Blocks[MBB]; - for (MachineInstr &MI : MBB) { + for (MachineInstr &MI : *MBB) { InstrInfo &III = Instructions[&MI]; unsigned Opcode = MI.getOpcode(); char Flags = 0; @@ -561,7 +560,7 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF, BBI.Needs |= StateExact; if (!(BBI.InNeeds & StateExact)) { BBI.InNeeds |= StateExact; - Worklist.push_back(&MBB); + Worklist.push_back(MBB); } GlobalFlags |= StateExact; III.Disabled = StateWQM | StateStrict; diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td index 8502ed61b366..184c871db775 100644 --- a/llvm/lib/Target/AMDGPU/SMInstructions.td +++ b/llvm/lib/Target/AMDGPU/SMInstructions.td @@ -181,15 +181,8 @@ class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pse " $sdst", [(set i64:$sdst, (node))]> { let hasSideEffects = 1; - // FIXME: This should be definitively mayStore = 0. TableGen - // brokenly tries to infer these based on the intrinsic properties - // corresponding to the IR attributes. The target intrinsics are - // considered as writing to memory for IR dependency purposes, but - // those can be modeled with hasSideEffects here. These also end up - // inferring differently for llvm.readcyclecounter and the amdgcn - // intrinsics. - let mayStore = ?; - let mayLoad = 1; + let mayStore = 0; + let mayLoad = 0; let has_sbase = 0; let has_offset = 0; } @@ -765,11 +758,11 @@ def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformL }]; } -def SMRDImm : ComplexPattern<i64, 2, "SelectSMRDImm">; -def SMRDImm32 : ComplexPattern<i64, 2, "SelectSMRDImm32">; -def SMRDSgpr : ComplexPattern<i64, 2, "SelectSMRDSgpr">; -def SMRDBufferImm : ComplexPattern<i32, 1, "SelectSMRDBufferImm">; -def SMRDBufferImm32 : ComplexPattern<i32, 1, "SelectSMRDBufferImm32">; +def SMRDImm : ComplexPattern<iPTR, 2, "SelectSMRDImm">; +def SMRDImm32 : ComplexPattern<iPTR, 2, "SelectSMRDImm32">; +def SMRDSgpr : ComplexPattern<iPTR, 2, "SelectSMRDSgpr">; +def SMRDBufferImm : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">; +def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">; multiclass SMRD_Pattern <string Instr, ValueType vt> { diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td index 61ecc13620a1..1713586dcf5b 100644 --- a/llvm/lib/Target/AMDGPU/SOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td @@ -157,6 +157,42 @@ class SOP1_1 <string opName, RegisterClass rc = SReg_64, list<dag> pattern=[]> : let has_sdst = 0; } +class UniformUnaryFrag<SDPatternOperator Op> : PatFrag < + (ops node:$src0), + (Op $src0), + [{ return !N->isDivergent(); }]> { + // This check is unnecessary as it's captured by the result register + // bank constraint. + // + // FIXME: Should add a way for the emitter to recognize this is a + // trivially true predicate to eliminate the check. + let GISelPredicateCode = [{return true;}]; +} + +class UniformBinFrag<SDPatternOperator Op> : PatFrag < + (ops node:$src0, node:$src1), + (Op $src0, $src1), + [{ return !N->isDivergent(); }]> { + // This check is unnecessary as it's captured by the result register + // bank constraint. + // + // FIXME: Should add a way for the emitter to recognize this is a + // trivially true predicate to eliminate the check. + let GISelPredicateCode = [{return true;}]; +} + +class DivergentBinFrag<SDPatternOperator Op> : PatFrag < + (ops node:$src0, node:$src1), + (Op $src0, $src1), + [{ return N->isDivergent(); }]> { + // This check is unnecessary as it's captured by the result register + // bank constraint. + // + // FIXME: Should add a way for the emitter to recognize this is a + // trivially true predicate to eliminate the check. + let GISelPredicateCode = [{return true;}]; +} + let isMoveImm = 1 in { let isReMaterializable = 1, isAsCheapAsAMove = 1 in { @@ -172,11 +208,11 @@ let isMoveImm = 1 in { let Defs = [SCC] in { def S_NOT_B32 : SOP1_32 <"s_not_b32", - [(set i32:$sdst, (not i32:$src0))] + [(set i32:$sdst, (UniformUnaryFrag<not> i32:$src0))] >; def S_NOT_B64 : SOP1_64 <"s_not_b64", - [(set i64:$sdst, (not i64:$src0))] + [(set i64:$sdst, (UniformUnaryFrag<not> i64:$src0))] >; def S_WQM_B32 : SOP1_32 <"s_wqm_b32">; def S_WQM_B64 : SOP1_64 <"s_wqm_b64">; @@ -221,22 +257,22 @@ let isReMaterializable = 1 in { def S_FF0_I32_B32 : SOP1_32 <"s_ff0_i32_b32">; def S_FF0_I32_B64 : SOP1_32_64 <"s_ff0_i32_b64">; def S_FF1_I32_B64 : SOP1_32_64 <"s_ff1_i32_b64", - [(set i32:$sdst, (AMDGPUffbl_b32 i64:$src0))] + [(set i32:$sdst, (UniformUnaryFrag<AMDGPUffbl_b32> i64:$src0))] >; def S_FF1_I32_B32 : SOP1_32 <"s_ff1_i32_b32", - [(set i32:$sdst, (AMDGPUffbl_b32 i32:$src0))] + [(set i32:$sdst, (UniformUnaryFrag<AMDGPUffbl_b32> i32:$src0))] >; def S_FLBIT_I32_B32 : SOP1_32 <"s_flbit_i32_b32", - [(set i32:$sdst, (AMDGPUffbh_u32 i32:$src0))] + [(set i32:$sdst, (UniformUnaryFrag<AMDGPUffbh_u32> i32:$src0))] >; def S_FLBIT_I32_B64 : SOP1_32_64 <"s_flbit_i32_b64", - [(set i32:$sdst, (AMDGPUffbh_u32 i64:$src0))] + [(set i32:$sdst, (UniformUnaryFrag<AMDGPUffbh_u32> i64:$src0))] >; def S_FLBIT_I32 : SOP1_32 <"s_flbit_i32", - [(set i32:$sdst, (AMDGPUffbh_i32 i32:$src0))] + [(set i32:$sdst, (UniformUnaryFrag<AMDGPUffbh_i32> i32:$src0))] >; def S_FLBIT_I32_I64 : SOP1_32_64 <"s_flbit_i32_i64">; def S_SEXT_I32_I8 : SOP1_32 <"s_sext_i32_i8", @@ -426,41 +462,6 @@ class SOP2_64_32_32 <string opName, list<dag> pattern=[]> : SOP2_Pseudo < "$sdst, $src0, $src1", pattern >; -class UniformUnaryFrag<SDPatternOperator Op> : PatFrag < - (ops node:$src0), - (Op $src0), - [{ return !N->isDivergent(); }]> { - // This check is unnecessary as it's captured by the result register - // bank constraint. - // - // FIXME: Should add a way for the emitter to recognize this is a - // trivially true predicate to eliminate the check. - let GISelPredicateCode = [{return true;}]; -} - -class UniformBinFrag<SDPatternOperator Op> : PatFrag < - (ops node:$src0, node:$src1), - (Op $src0, $src1), - [{ return !N->isDivergent(); }]> { - // This check is unnecessary as it's captured by the result register - // bank constraint. - // - // FIXME: Should add a way for the emitter to recognize this is a - // trivially true predicate to eliminate the check. - let GISelPredicateCode = [{return true;}]; -} - -class DivergentBinFrag<SDPatternOperator Op> : PatFrag < - (ops node:$src0, node:$src1), - (Op $src0, $src1), - [{ return N->isDivergent(); }]> { - // This check is unnecessary as it's captured by the result register - // bank constraint. - // - // FIXME: Should add a way for the emitter to recognize this is a - // trivially true predicate to eliminate the check. - let GISelPredicateCode = [{return true;}]; -} let Defs = [SCC] in { // Carry out goes to SCC let isCommutable = 1 in { @@ -485,19 +486,18 @@ def S_SUBB_U32 : SOP2_32 <"s_subb_u32", [(set i32:$sdst, (UniformBinFrag<sube> (i32 SSrc_b32:$src0), (i32 SSrc_b32:$src1)))]>; } // End Uses = [SCC] - let isCommutable = 1 in { def S_MIN_I32 : SOP2_32 <"s_min_i32", - [(set i32:$sdst, (smin i32:$src0, i32:$src1))] + [(set i32:$sdst, (UniformBinFrag<smin> i32:$src0, i32:$src1))] >; def S_MIN_U32 : SOP2_32 <"s_min_u32", - [(set i32:$sdst, (umin i32:$src0, i32:$src1))] + [(set i32:$sdst, (UniformBinFrag<umin> i32:$src0, i32:$src1))] >; def S_MAX_I32 : SOP2_32 <"s_max_i32", - [(set i32:$sdst, (smax i32:$src0, i32:$src1))] + [(set i32:$sdst, (UniformBinFrag<smax> i32:$src0, i32:$src1))] >; def S_MAX_U32 : SOP2_32 <"s_max_u32", - [(set i32:$sdst, (umax i32:$src0, i32:$src1))] + [(set i32:$sdst, (UniformBinFrag<umax> i32:$src0, i32:$src1))] >; } // End isCommutable = 1 } // End Defs = [SCC] @@ -870,7 +870,7 @@ def S_GETREG_B32 : SOPK_Pseudo < } } // End mayLoad = 1 -let mayLoad = 0, mayStore = 0, Defs = [MODE], Uses = [MODE] in { +let Defs = [MODE], Uses = [MODE] in { // FIXME: Need to truncate immediate to 16-bits. class S_SETREG_B32_Pseudo <list<dag> pattern=[]> : SOPK_Pseudo < @@ -914,7 +914,7 @@ def S_SETREG_IMM32_B32_mode : S_SETREG_IMM32_B32_Pseudo { let hasSideEffects = 0; } -} // End mayLoad = 0, mayStore = 0, Defs = [MODE], Uses = [MODE] +} // End Defs = [MODE], Uses = [MODE] class SOPK_WAITCNT<string opName, list<dag> pat=[]> : SOPK_Pseudo< @@ -1264,7 +1264,7 @@ def S_WAKEUP : SOPP_Pseudo <"s_wakeup", (ins) > { let mayStore = 1; } -let mayLoad = 0, mayStore = 0, hasSideEffects = 1 in +let hasSideEffects = 1 in def S_WAITCNT : SOPP_Pseudo <"s_waitcnt" , (ins WAIT_FLAG:$simm16), "$simm16", [(int_amdgcn_s_waitcnt timm:$simm16)]>; def S_SETHALT : SOPP_Pseudo <"s_sethalt" , (ins i32imm:$simm16), "$simm16", @@ -1278,8 +1278,6 @@ def S_SETKILL : SOPP_Pseudo <"s_setkill" , (ins i16imm:$simm16), "$simm16">; def S_SLEEP : SOPP_Pseudo <"s_sleep", (ins i32imm:$simm16), "$simm16", [(int_amdgcn_s_sleep timm:$simm16)]> { let hasSideEffects = 1; - let mayLoad = 0; - let mayStore = 0; } def S_SETPRIO : SOPP_Pseudo <"s_setprio" , (ins i16imm:$simm16), "$simm16">; @@ -1305,14 +1303,10 @@ def S_ICACHE_INV : SOPP_Pseudo <"s_icache_inv", (ins)> { def S_INCPERFLEVEL : SOPP_Pseudo <"s_incperflevel", (ins i32imm:$simm16), "$simm16", [(int_amdgcn_s_incperflevel timm:$simm16)]> { let hasSideEffects = 1; - let mayLoad = 0; - let mayStore = 0; } def S_DECPERFLEVEL : SOPP_Pseudo <"s_decperflevel", (ins i32imm:$simm16), "$simm16", [(int_amdgcn_s_decperflevel timm:$simm16)]> { let hasSideEffects = 1; - let mayLoad = 0; - let mayStore = 0; } def S_TTRACEDATA : SOPP_Pseudo <"s_ttracedata", (ins)> { let simm16 = 0; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp index 2e4d83fbbc39..a83ff6667956 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.cpp @@ -15,7 +15,6 @@ #include "Utils/AMDGPUBaseInfo.h" #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/SetVector.h" -#include "llvm/Analysis/CallGraph.h" #include "llvm/IR/Constants.h" #include "llvm/IR/ReplaceConstant.h" @@ -25,175 +24,6 @@ namespace llvm { namespace AMDGPU { -// An helper class for collecting all reachable callees for each kernel defined -// within the module. -class CollectReachableCallees { - Module &M; - CallGraph CG; - SmallPtrSet<CallGraphNode *, 8> AddressTakenFunctions; - - // Collect all address taken functions within the module. - void collectAddressTakenFunctions() { - auto *ECNode = CG.getExternalCallingNode(); - - for (auto GI = ECNode->begin(), GE = ECNode->end(); GI != GE; ++GI) { - auto *CGN = GI->second; - auto *F = CGN->getFunction(); - if (!F || F->isDeclaration() || AMDGPU::isKernelCC(F)) - continue; - AddressTakenFunctions.insert(CGN); - } - } - - // For given kernel, collect all its reachable non-kernel functions. - SmallPtrSet<Function *, 8> collectReachableCallees(Function *K) { - SmallPtrSet<Function *, 8> ReachableCallees; - - // Call graph node which represents this kernel. - auto *KCGN = CG[K]; - - // Go through all call graph nodes reachable from the node representing this - // kernel, visit all their call sites, if the call site is direct, add - // corresponding callee to reachable callee set, if it is indirect, resolve - // the indirect call site to potential reachable callees, add them to - // reachable callee set, and repeat the process for the newly added - // potential callee nodes. - // - // FIXME: Need to handle bit-casted function pointers. - // - SmallVector<CallGraphNode *, 8> CGNStack(df_begin(KCGN), df_end(KCGN)); - SmallPtrSet<CallGraphNode *, 8> VisitedCGNodes; - while (!CGNStack.empty()) { - auto *CGN = CGNStack.pop_back_val(); - - if (!VisitedCGNodes.insert(CGN).second) - continue; - - // Ignore call graph node which does not have associated function or - // associated function is not a definition. - if (!CGN->getFunction() || CGN->getFunction()->isDeclaration()) - continue; - - for (auto GI = CGN->begin(), GE = CGN->end(); GI != GE; ++GI) { - auto *RCB = cast<CallBase>(GI->first.getValue()); - auto *RCGN = GI->second; - - if (auto *DCallee = RCGN->getFunction()) { - ReachableCallees.insert(DCallee); - } else if (RCB->isIndirectCall()) { - auto *RCBFTy = RCB->getFunctionType(); - for (auto *ACGN : AddressTakenFunctions) { - auto *ACallee = ACGN->getFunction(); - if (ACallee->getFunctionType() == RCBFTy) { - ReachableCallees.insert(ACallee); - CGNStack.append(df_begin(ACGN), df_end(ACGN)); - } - } - } - } - } - - return ReachableCallees; - } - -public: - explicit CollectReachableCallees(Module &M) : M(M), CG(CallGraph(M)) { - // Collect address taken functions. - collectAddressTakenFunctions(); - } - - void collectReachableCallees( - DenseMap<Function *, SmallPtrSet<Function *, 8>> &KernelToCallees) { - // Collect reachable callee set for each kernel defined in the module. - for (Function &F : M.functions()) { - if (!AMDGPU::isKernelCC(&F)) - continue; - Function *K = &F; - KernelToCallees[K] = collectReachableCallees(K); - } - } -}; - -void collectReachableCallees( - Module &M, - DenseMap<Function *, SmallPtrSet<Function *, 8>> &KernelToCallees) { - CollectReachableCallees CRC{M}; - CRC.collectReachableCallees(KernelToCallees); -} - -SmallPtrSet<Function *, 8> collectNonKernelAccessorsOfLDS(GlobalVariable *GV) { - SmallPtrSet<Function *, 8> LDSAccessors; - SmallVector<User *, 8> UserStack(GV->users()); - SmallPtrSet<User *, 8> VisitedUsers; - - while (!UserStack.empty()) { - auto *U = UserStack.pop_back_val(); - - // `U` is already visited? continue to next one. - if (!VisitedUsers.insert(U).second) - continue; - - // `U` is a global variable which is initialized with LDS. Ignore LDS. - if (isa<GlobalValue>(U)) - return SmallPtrSet<Function *, 8>(); - - // Recursively explore constant users. - if (isa<Constant>(U)) { - append_range(UserStack, U->users()); - continue; - } - - // `U` should be an instruction, if it belongs to a non-kernel function F, - // then collect F. - Function *F = cast<Instruction>(U)->getFunction(); - if (!AMDGPU::isKernelCC(F)) - LDSAccessors.insert(F); - } - - return LDSAccessors; -} - -DenseMap<Function *, SmallPtrSet<Instruction *, 8>> -getFunctionToInstsMap(User *U, bool CollectKernelInsts) { - DenseMap<Function *, SmallPtrSet<Instruction *, 8>> FunctionToInsts; - SmallVector<User *, 8> UserStack; - SmallPtrSet<User *, 8> VisitedUsers; - - UserStack.push_back(U); - - while (!UserStack.empty()) { - auto *UU = UserStack.pop_back_val(); - - if (!VisitedUsers.insert(UU).second) - continue; - - if (isa<GlobalValue>(UU)) - continue; - - if (isa<Constant>(UU)) { - append_range(UserStack, UU->users()); - continue; - } - - auto *I = cast<Instruction>(UU); - Function *F = I->getFunction(); - if (CollectKernelInsts) { - if (!AMDGPU::isKernelCC(F)) { - continue; - } - } else { - if (AMDGPU::isKernelCC(F)) { - continue; - } - } - - FunctionToInsts.insert(std::make_pair(F, SmallPtrSet<Instruction *, 8>())); - FunctionToInsts[F].insert(I); - } - - return FunctionToInsts; -} - bool isKernelCC(const Function *Func) { return AMDGPU::isModuleEntryFunctionCC(Func->getCallingConv()); } @@ -232,26 +62,8 @@ void replaceConstantUsesInFunction(ConstantExpr *C, const Function *F) { } } -bool hasUserInstruction(const GlobalValue *GV) { - SmallPtrSet<const User *, 8> Visited; - SmallVector<const User *, 16> Stack(GV->users()); - - while (!Stack.empty()) { - const User *U = Stack.pop_back_val(); - - if (!Visited.insert(U).second) - continue; - - if (isa<Instruction>(U)) - return true; - - append_range(Stack, U->users()); - } - - return false; -} - -bool shouldLowerLDSToStruct(const GlobalVariable &GV, const Function *F) { +static bool shouldLowerLDSToStruct(const GlobalVariable &GV, + const Function *F) { // We are not interested in kernel LDS lowering for module LDS itself. if (F && GV.getName() == "llvm.amdgcn.module.lds") return false; @@ -259,7 +71,6 @@ bool shouldLowerLDSToStruct(const GlobalVariable &GV, const Function *F) { bool Ret = false; SmallPtrSet<const User *, 8> Visited; SmallVector<const User *, 16> Stack(GV.users()); - SmallPtrSet<const GlobalValue *, 8> GlobalUsers; assert(!F || isKernelCC(F)); @@ -267,15 +78,10 @@ bool shouldLowerLDSToStruct(const GlobalVariable &GV, const Function *F) { const User *V = Stack.pop_back_val(); Visited.insert(V); - if (auto *G = dyn_cast<GlobalValue>(V)) { - StringRef GName = G->getName(); - if (F && GName != "llvm.used" && GName != "llvm.compiler.used") { - // For kernel LDS lowering, if G is not a compiler.used list, then we - // cannot lower the lds GV since we cannot replace the use of GV within - // G. - return false; - } - GlobalUsers.insert(G); + if (isa<GlobalValue>(V)) { + // This use of the LDS variable is the initializer of a global variable. + // This is ill formed. The address of an LDS variable is kernel dependent + // and unknown until runtime. It can't be written to a global variable. continue; } @@ -297,15 +103,6 @@ bool shouldLowerLDSToStruct(const GlobalVariable &GV, const Function *F) { append_range(Stack, V->users()); } - if (!F && !Ret) { - // For module LDS lowering, we have not yet decided if we should lower GV or - // not. Explore all global users of GV, and check if atleast one of these - // global users appear as an use within an instruction (possibly nested use - // via constant expression), if so, then conservately lower LDS. - for (auto *G : GlobalUsers) - Ret |= hasUserInstruction(G); - } - return Ret; } @@ -324,7 +121,7 @@ std::vector<GlobalVariable *> findVariablesToLower(Module &M, continue; } if (!isa<UndefValue>(GV.getInitializer())) { - // Initializers are unimplemented for local address space. + // Initializers are unimplemented for LDS address space. // Leave such variables in place for consistent error reporting. continue; } @@ -342,20 +139,6 @@ std::vector<GlobalVariable *> findVariablesToLower(Module &M, return LocalVars; } -SmallPtrSet<GlobalValue *, 32> getUsedList(Module &M) { - SmallPtrSet<GlobalValue *, 32> UsedList; - - SmallVector<GlobalValue *, 32> TmpVec; - collectUsedGlobalVariables(M, TmpVec, true); - UsedList.insert(TmpVec.begin(), TmpVec.end()); - - TmpVec.clear(); - collectUsedGlobalVariables(M, TmpVec, false); - UsedList.insert(TmpVec.begin(), TmpVec.end()); - - return UsedList; -} - } // end namespace AMDGPU } // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h index d1c9229bc336..83ef68cc3f60 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPULDSUtils.h @@ -22,44 +22,13 @@ class ConstantExpr; namespace AMDGPU { -/// Collect reachable callees for each kernel defined in the module \p M and -/// return collected callees at \p KernelToCallees. -void collectReachableCallees( - Module &M, - DenseMap<Function *, SmallPtrSet<Function *, 8>> &KernelToCallees); - -/// For the given LDS global \p GV, visit all its users and collect all -/// non-kernel functions within which \p GV is used and return collected list of -/// such non-kernel functions. -SmallPtrSet<Function *, 8> collectNonKernelAccessorsOfLDS(GlobalVariable *GV); - -/// Collect all the instructions where user \p U belongs to. \p U could be -/// instruction itself or it could be a constant expression which is used within -/// an instruction. If \p CollectKernelInsts is true, collect instructions only -/// from kernels, otherwise collect instructions only from non-kernel functions. -DenseMap<Function *, SmallPtrSet<Instruction *, 8>> -getFunctionToInstsMap(User *U, bool CollectKernelInsts); - bool isKernelCC(const Function *Func); Align getAlign(DataLayout const &DL, const GlobalVariable *GV); -/// \returns true if a given global variable \p GV (or its global users) appear -/// as an use within some instruction (either from kernel or from non-kernel). -bool hasUserInstruction(const GlobalValue *GV); - -/// \returns true if an LDS global requires lowering to a module LDS structure -/// if \p F is not given. If \p F is given it must be a kernel and function -/// \returns true if an LDS global is directly used from that kernel and it -/// is safe to replace its uses with a kernel LDS structure member. -bool shouldLowerLDSToStruct(const GlobalVariable &GV, - const Function *F = nullptr); - std::vector<GlobalVariable *> findVariablesToLower(Module &M, const Function *F = nullptr); -SmallPtrSet<GlobalValue *, 32> getUsedList(Module &M); - /// Replace all uses of constant \p C with instructions in \p F. void replaceConstantUsesInFunction(ConstantExpr *C, const Function *F); } // end namespace AMDGPU diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td index a3eccf13cd71..a8368892c565 100644 --- a/llvm/lib/Target/AMDGPU/VOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td @@ -794,6 +794,18 @@ class VOPPatGen<SDPatternOperator Op, VOPProfile P> { list<dag> ret = [!con(Outs, (set Ins))]; } +class DivergentUnaryFrag<SDPatternOperator Op> : PatFrag < + (ops node:$src0), + (Op $src0), + [{ return N->isDivergent(); }]> { + // This check is unnecessary as it's captured by the result register + // bank constraint. + // + // FIXME: Should add a way for the emitter to recognize this is a + // trivially true predicate to eliminate the check. + let GISelPredicateCode = [{return true;}]; +} + class VOPPatOrNull<SDPatternOperator Op, VOPProfile P> { list<dag> ret = !if(!ne(P.NeedPatGen,PatGenMode.NoPattern), VOPPatGen<Op, P>.ret, []); } diff --git a/llvm/lib/Target/ARM/A15SDOptimizer.cpp b/llvm/lib/Target/ARM/A15SDOptimizer.cpp index f4d0f4a6d6b0..d0efecad63bc 100644 --- a/llvm/lib/Target/ARM/A15SDOptimizer.cpp +++ b/llvm/lib/Target/ARM/A15SDOptimizer.cpp @@ -592,16 +592,15 @@ bool A15SDOptimizer::runOnInstruction(MachineInstr *MI) { SmallVector<unsigned, 8> Defs = getReadDPRs(MI); bool Modified = false; - for (SmallVectorImpl<unsigned>::iterator I = Defs.begin(), E = Defs.end(); - I != E; ++I) { + for (unsigned I : Defs) { // Follow the def-use chain for this DPR through COPYs, and also through // PHIs (which are essentially multi-way COPYs). It is because of PHIs that // we can end up with multiple defs of this DPR. SmallVector<MachineInstr *, 8> DefSrcs; - if (!Register::isVirtualRegister(*I)) + if (!Register::isVirtualRegister(I)) continue; - MachineInstr *Def = MRI->getVRegDef(*I); + MachineInstr *Def = MRI->getVRegDef(I); if (!Def) continue; @@ -628,18 +627,17 @@ bool A15SDOptimizer::runOnInstruction(MachineInstr *MI) { if (NewReg != 0) { Modified = true; - for (SmallVectorImpl<MachineOperand *>::const_iterator I = Uses.begin(), - E = Uses.end(); I != E; ++I) { + for (MachineOperand *Use : Uses) { // Make sure to constrain the register class of the new register to // match what we're replacing. Otherwise we can optimize a DPR_VFP2 // reference into a plain DPR, and that will end poorly. NewReg is // always virtual here, so there will always be a matching subclass // to find. - MRI->constrainRegClass(NewReg, MRI->getRegClass((*I)->getReg())); + MRI->constrainRegClass(NewReg, MRI->getRegClass(Use->getReg())); - LLVM_DEBUG(dbgs() << "Replacing operand " << **I << " with " + LLVM_DEBUG(dbgs() << "Replacing operand " << *Use << " with " << printReg(NewReg) << "\n"); - (*I)->substVirtReg(NewReg, 0, *TRI); + Use->substVirtReg(NewReg, 0, *TRI); } } Replacements[MI] = NewReg; diff --git a/llvm/lib/Target/ARM/ARM.td b/llvm/lib/Target/ARM/ARM.td index e03dd597eb65..8173fe4036a8 100644 --- a/llvm/lib/Target/ARM/ARM.td +++ b/llvm/lib/Target/ARM/ARM.td @@ -446,6 +446,11 @@ def FeaturePACBTI : SubtargetFeature<"pacbti", "HasPACBTI", "true", "Enable Pointer Authentication and Branch " "Target Identification">; +def FeatureNoBTIAtReturnTwice : SubtargetFeature<"no-bti-at-return-twice", + "NoBTIAtReturnTwice", "true", + "Don't place a BTI instruction " + "after a return-twice">; + //===----------------------------------------------------------------------===// // ARM architecture class // diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp index 6a88ac485e69..fa09b2567aa9 100644 --- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp +++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp @@ -1153,8 +1153,12 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) { unsigned StartOp = 2 + 2; // Use all the operands. unsigned NumOffset = 0; - // Amount of SP adjustment folded into a push. - unsigned Pad = 0; + // Amount of SP adjustment folded into a push, before the + // registers are stored (pad at higher addresses). + unsigned PadBefore = 0; + // Amount of SP adjustment folded into a push, after the + // registers are stored (pad at lower addresses). + unsigned PadAfter = 0; switch (Opc) { default: @@ -1185,7 +1189,7 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) { "Pad registers must come before restored ones"); unsigned Width = TargetRegInfo->getRegSizeInBits(MO.getReg(), MachineRegInfo) / 8; - Pad += Width; + PadAfter += Width; continue; } // Check for registers that are remapped (for a Thumb1 prologue that @@ -1201,14 +1205,32 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) { case ARM::t2STR_PRE: assert(MI->getOperand(2).getReg() == ARM::SP && "Only stack pointer as a source reg is supported"); + if (unsigned RemappedReg = AFI->EHPrologueRemappedRegs.lookup(SrcReg)) + SrcReg = RemappedReg; + + RegList.push_back(SrcReg); + break; + case ARM::t2STRD_PRE: + assert(MI->getOperand(3).getReg() == ARM::SP && + "Only stack pointer as a source reg is supported"); + SrcReg = MI->getOperand(1).getReg(); + if (unsigned RemappedReg = AFI->EHPrologueRemappedRegs.lookup(SrcReg)) + SrcReg = RemappedReg; + RegList.push_back(SrcReg); + SrcReg = MI->getOperand(2).getReg(); + if (unsigned RemappedReg = AFI->EHPrologueRemappedRegs.lookup(SrcReg)) + SrcReg = RemappedReg; RegList.push_back(SrcReg); + PadBefore = -MI->getOperand(4).getImm() - 8; break; } if (MAI->getExceptionHandlingType() == ExceptionHandling::ARM) { + if (PadBefore) + ATS.emitPad(PadBefore); ATS.emitRegSave(RegList, Opc == ARM::VSTMDDB_UPD); // Account for the SP adjustment, folded into the push. - if (Pad) - ATS.emitPad(Pad); + if (PadAfter) + ATS.emitPad(PadAfter); } } else { // Changes of stack / frame pointer. @@ -1300,6 +1322,10 @@ void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) { Offset = MI->getOperand(2).getImm(); AFI->EHPrologueOffsetInRegs[DstReg] |= (Offset << 16); break; + case ARM::t2PAC: + case ARM::t2PACBTI: + AFI->EHPrologueRemappedRegs[ARM::R12] = ARM::RA_AUTH_CODE; + break; default: MI->print(errs()); llvm_unreachable("Unsupported opcode for unwinding information"); diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp index 2a12947d24a8..884f38ff6c58 100644 --- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp +++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp @@ -2629,8 +2629,8 @@ bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget, // Add the complete list back in. MachineInstrBuilder MIB(MF, &*MI); - for (int i = RegList.size() - 1; i >= 0; --i) - MIB.add(RegList[i]); + for (const MachineOperand &MO : llvm::reverse(RegList)) + MIB.add(MO); return true; } @@ -5678,7 +5678,7 @@ bool llvm::HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2, /// | | Thumb2 | ARM | /// +-------------------------+--------+-----+ /// | Call overhead in Bytes | 4 | 4 | -/// | Frame overhead in Bytes | 4 | 4 | +/// | Frame overhead in Bytes | 2 | 4 | /// | Stack fixup required | No | No | /// +-------------------------+--------+-----+ /// @@ -5755,7 +5755,7 @@ struct OutlinerCosts { CallThunk(target.isThumb() ? 4 : 4), FrameThunk(target.isThumb() ? 0 : 0), CallNoLRSave(target.isThumb() ? 4 : 4), - FrameNoLRSave(target.isThumb() ? 4 : 4), + FrameNoLRSave(target.isThumb() ? 2 : 4), CallRegSave(target.isThumb() ? 8 : 12), FrameRegSave(target.isThumb() ? 2 : 4), CallDefault(target.isThumb() ? 8 : 12), @@ -5868,11 +5868,17 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( return outliner::OutlinedFunction(); } + // We expect the majority of the outlining candidates to be in consensus with + // regard to return address sign and authentication, and branch target + // enforcement, in other words, partitioning according to all the four + // possible combinations of PAC-RET and BTI is going to yield one big subset + // and three small (likely empty) subsets. That allows us to cull incompatible + // candidates separately for PAC-RET and BTI. + // Partition the candidates in two sets: one with BTI enabled and one with BTI - // disabled. Remove the candidates from the smaller set. We expect the - // majority of the candidates to be in consensus with regard to branch target - // enforcement with just a few oddballs, but if they are the same number - // prefer the non-BTI ones for outlining, since they have less overhead. + // disabled. Remove the candidates from the smaller set. If they are the same + // number prefer the non-BTI ones for outlining, since they have less + // overhead. auto NoBTI = llvm::partition(RepeatedSequenceLocs, [](const outliner::Candidate &C) { const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>(); @@ -5883,6 +5889,24 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( RepeatedSequenceLocs.erase(NoBTI, RepeatedSequenceLocs.end()); else RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoBTI); + + if (RepeatedSequenceLocs.size() < 2) + return outliner::OutlinedFunction(); + + // Likewise, partition the candidates according to PAC-RET enablement. + auto NoPAC = + llvm::partition(RepeatedSequenceLocs, [](const outliner::Candidate &C) { + const ARMFunctionInfo &AFI = *C.getMF()->getInfo<ARMFunctionInfo>(); + // If the function happens to not spill the LR, do not disqualify it + // from the outlining. + return AFI.shouldSignReturnAddress(true); + }); + if (std::distance(RepeatedSequenceLocs.begin(), NoPAC) > + std::distance(NoPAC, RepeatedSequenceLocs.end())) + RepeatedSequenceLocs.erase(NoPAC, RepeatedSequenceLocs.end()); + else + RepeatedSequenceLocs.erase(RepeatedSequenceLocs.begin(), NoPAC); + if (RepeatedSequenceLocs.size() < 2) return outliner::OutlinedFunction(); @@ -5899,6 +5923,7 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( }; OutlinerCosts Costs(Subtarget); + const auto &SomeMFI = *RepeatedSequenceLocs.front().getMF()->getInfo<ARMFunctionInfo>(); // Adjust costs to account for the BTI instructions. @@ -5909,6 +5934,13 @@ outliner::OutlinedFunction ARMBaseInstrInfo::getOutliningCandidateInfo( Costs.FrameTailCall += 4; Costs.FrameThunk += 4; } + + // Adjust costs to account for sign and authentication instructions. + if (SomeMFI.shouldSignReturnAddress(true)) { + Costs.CallDefault += 8; // +PAC instr, +AUT instr + Costs.SaveRestoreLROnStack += 8; // +PAC instr, +AUT instr + } + unsigned FrameID = MachineOutlinerDefault; unsigned NumBytesToCreateFrame = Costs.FrameDefault; @@ -6325,6 +6357,11 @@ ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT, // * LR is available in the range (No save/restore around call) // * The range doesn't include calls (No save/restore in outlined frame) // are true. + // These conditions also ensure correctness of the return address + // authentication - we insert sign and authentication instructions only if + // we save/restore LR on stack, but then this condition ensures that the + // outlined range does not modify the SP, therefore the SP value used for + // signing is the same as the one used for authentication. // FIXME: This is very restrictive; the flags check the whole block, // not just the bit we will try to outline. bool MightNeedStackFixUp = @@ -6369,23 +6406,39 @@ void ARMBaseInstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const { } void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const { - unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM; - int Align = -Subtarget.getStackAlignment().value(); - BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP) - .addReg(ARM::LR, RegState::Kill) - .addReg(ARM::SP) - .addImm(Align) - .add(predOps(ARMCC::AL)); -} + MachineBasicBlock::iterator It, bool CFI, + bool Auth) const { + int Align = std::max(Subtarget.getStackAlignment().value(), uint64_t(8)); + assert(Align >= 8 && Align <= 256); + if (Auth) { + assert(Subtarget.isThumb2()); + // Compute PAC in R12. Outlining ensures R12 is dead across the outlined + // sequence. + BuildMI(MBB, It, DebugLoc(), get(ARM::t2PAC)) + .setMIFlags(MachineInstr::FrameSetup); + BuildMI(MBB, It, DebugLoc(), get(ARM::t2STRD_PRE), ARM::SP) + .addReg(ARM::R12, RegState::Kill) + .addReg(ARM::LR, RegState::Kill) + .addReg(ARM::SP) + .addImm(-Align) + .add(predOps(ARMCC::AL)) + .setMIFlags(MachineInstr::FrameSetup); + } else { + unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM; + BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP) + .addReg(ARM::LR, RegState::Kill) + .addReg(ARM::SP) + .addImm(-Align) + .add(predOps(ARMCC::AL)) + .setMIFlags(MachineInstr::FrameSetup); + } + + if (!CFI) + return; -void ARMBaseInstrInfo::emitCFIForLRSaveOnStack( - MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { MachineFunction &MF = *MBB.getParent(); - const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); - unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); - int Align = Subtarget.getStackAlignment().value(); - // Add a CFI saying the stack was moved down. + + // Add a CFI, saying CFA is offset by Align bytes from SP. int64_t StackPosEntry = MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Align)); BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) @@ -6394,11 +6447,23 @@ void ARMBaseInstrInfo::emitCFIForLRSaveOnStack( // Add a CFI saying that the LR that we want to find is now higher than // before. - int64_t LRPosEntry = - MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, DwarfLR, -Align)); + int LROffset = Auth ? Align - 4 : Align; + const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); + unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); + int64_t LRPosEntry = MF.addFrameInst( + MCCFIInstruction::createOffset(nullptr, DwarfLR, -LROffset)); BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) .addCFIIndex(LRPosEntry) .setMIFlags(MachineInstr::FrameSetup); + if (Auth) { + // Add a CFI for the location of the return adddress PAC. + unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true); + int64_t RACPosEntry = MF.addFrameInst( + MCCFIInstruction::createOffset(nullptr, DwarfRAC, -Align)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(RACPosEntry) + .setMIFlags(MachineInstr::FrameSetup); + } } void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB, @@ -6416,35 +6481,64 @@ void ARMBaseInstrInfo::emitCFIForLRSaveToReg(MachineBasicBlock &MBB, .setMIFlags(MachineInstr::FrameSetup); } -void ARMBaseInstrInfo::restoreLRFromStack( - MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { - unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM; - MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR) - .addReg(ARM::SP, RegState::Define) - .addReg(ARM::SP); - if (!Subtarget.isThumb()) - MIB.addReg(0); - MIB.addImm(Subtarget.getStackAlignment().value()).add(predOps(ARMCC::AL)); -} +void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB, + MachineBasicBlock::iterator It, + bool CFI, bool Auth) const { + int Align = Subtarget.getStackAlignment().value(); + if (Auth) { + assert(Subtarget.isThumb2()); + // Restore return address PAC and LR. + BuildMI(MBB, It, DebugLoc(), get(ARM::t2LDRD_POST)) + .addReg(ARM::R12, RegState::Define) + .addReg(ARM::LR, RegState::Define) + .addReg(ARM::SP, RegState::Define) + .addReg(ARM::SP) + .addImm(Align) + .add(predOps(ARMCC::AL)) + .setMIFlags(MachineInstr::FrameDestroy); + // LR authentication is after the CFI instructions, below. + } else { + unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM; + MachineInstrBuilder MIB = BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::LR) + .addReg(ARM::SP, RegState::Define) + .addReg(ARM::SP); + if (!Subtarget.isThumb()) + MIB.addReg(0); + MIB.addImm(Subtarget.getStackAlignment().value()) + .add(predOps(ARMCC::AL)) + .setMIFlags(MachineInstr::FrameDestroy); + } -void ARMBaseInstrInfo::emitCFIForLRRestoreFromStack( - MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const { - // Now stack has moved back up... - MachineFunction &MF = *MBB.getParent(); - const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); - unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); - int64_t StackPosEntry = - MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0)); - BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) - .addCFIIndex(StackPosEntry) - .setMIFlags(MachineInstr::FrameDestroy); + if (CFI) { + // Now stack has moved back up... + MachineFunction &MF = *MBB.getParent(); + const MCRegisterInfo *MRI = Subtarget.getRegisterInfo(); + unsigned DwarfLR = MRI->getDwarfRegNum(ARM::LR, true); + int64_t StackPosEntry = + MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, 0)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(StackPosEntry) + .setMIFlags(MachineInstr::FrameDestroy); - // ... and we have restored LR. - int64_t LRPosEntry = - MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR)); - BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) - .addCFIIndex(LRPosEntry) - .setMIFlags(MachineInstr::FrameDestroy); + // ... and we have restored LR. + int64_t LRPosEntry = + MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, DwarfLR)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(LRPosEntry) + .setMIFlags(MachineInstr::FrameDestroy); + + if (Auth) { + unsigned DwarfRAC = MRI->getDwarfRegNum(ARM::RA_AUTH_CODE, true); + int64_t Entry = + MF.addFrameInst(MCCFIInstruction::createUndefined(nullptr, DwarfRAC)); + BuildMI(MBB, It, DebugLoc(), get(ARM::CFI_INSTRUCTION)) + .addCFIIndex(Entry) + .setMIFlags(MachineInstr::FrameDestroy); + } + } + + if (Auth) + BuildMI(MBB, It, DebugLoc(), get(ARM::t2AUT)); } void ARMBaseInstrInfo::emitCFIForLRRestoreFromReg( @@ -6500,8 +6594,11 @@ void ARMBaseInstrInfo::buildOutlinedFrame( MBB.addLiveIn(ARM::LR); // Insert a save before the outlined region - saveLROnStack(MBB, It); - emitCFIForLRSaveOnStack(MBB, It); + bool Auth = OF.Candidates.front() + .getMF() + ->getInfo<ARMFunctionInfo>() + ->shouldSignReturnAddress(true); + saveLROnStack(MBB, It, true, Auth); // Fix up the instructions in the range, since we're going to modify the // stack. @@ -6510,8 +6607,7 @@ void ARMBaseInstrInfo::buildOutlinedFrame( fixupPostOutline(MBB); // Insert a restore before the terminator for the function. Restore LR. - restoreLRFromStack(MBB, Et); - emitCFIForLRRestoreFromStack(MBB, Et); + restoreLRFromStack(MBB, Et, true, Auth); } // If this is a tail call outlined function, then there's already a return. @@ -6590,13 +6686,10 @@ MachineBasicBlock::iterator ARMBaseInstrInfo::insertOutlinedCall( // We have the default case. Save and restore from SP. if (!MBB.isLiveIn(ARM::LR)) MBB.addLiveIn(ARM::LR); - saveLROnStack(MBB, It); - if (!AFI.isLRSpilled()) - emitCFIForLRSaveOnStack(MBB, It); + bool Auth = !AFI.isLRSpilled() && AFI.shouldSignReturnAddress(true); + saveLROnStack(MBB, It, !AFI.isLRSpilled(), Auth); CallPt = MBB.insert(It, CallMIB); - restoreLRFromStack(MBB, It); - if (!AFI.isLRSpilled()) - emitCFIForLRRestoreFromStack(MBB, It); + restoreLRFromStack(MBB, It, !AFI.isLRSpilled(), Auth); It--; return CallPt; } diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h index 5fa912ae35d7..defce07dd862 100644 --- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h +++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h @@ -377,20 +377,20 @@ private: /// constructing an outlined call if one exists. Returns 0 otherwise. unsigned findRegisterToSaveLRTo(const outliner::Candidate &C) const; - // Adds an instruction which saves the link register on top of the stack into - /// the MachineBasicBlock \p MBB at position \p It. - void saveLROnStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const; + /// Adds an instruction which saves the link register on top of the stack into + /// the MachineBasicBlock \p MBB at position \p It. If \p Auth is true, + /// compute and store an authentication code alongiside the link register. + /// If \p CFI is true, emit CFI instructions. + void saveLROnStack(MachineBasicBlock &MBB, MachineBasicBlock::iterator It, + bool CFI, bool Auth) const; /// Adds an instruction which restores the link register from the top the - /// stack into the MachineBasicBlock \p MBB at position \p It. + /// stack into the MachineBasicBlock \p MBB at position \p It. If \p Auth is + /// true, restore an authentication code and authenticate LR. + /// If \p CFI is true, emit CFI instructions. void restoreLRFromStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const; - - /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It, - /// for the case when the LR is saved on the stack. - void emitCFIForLRSaveOnStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const; + MachineBasicBlock::iterator It, bool CFI, + bool Auth) const; /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It, /// for the case when the LR is saved in the register \p Reg. @@ -399,11 +399,6 @@ private: Register Reg) const; /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It, - /// after the LR is was restored from the stack. - void emitCFIForLRRestoreFromStack(MachineBasicBlock &MBB, - MachineBasicBlock::iterator It) const; - - /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It, /// after the LR is was restored from a register. void emitCFIForLRRestoreFromReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const; diff --git a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp index b53efe58e8de..c543d02ff75a 100644 --- a/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp +++ b/llvm/lib/Target/ARM/ARMBaseRegisterInfo.cpp @@ -530,6 +530,8 @@ getFrameIndexInstrOffset(const MachineInstr *MI, int Idx) const { unsigned ImmIdx = 0; switch (AddrMode) { case ARMII::AddrModeT2_i8: + case ARMII::AddrModeT2_i8neg: + case ARMII::AddrModeT2_i8pos: case ARMII::AddrModeT2_i12: case ARMII::AddrMode_i12: InstrOffs = MI->getOperand(Idx+1).getImm(); @@ -728,6 +730,8 @@ bool ARMBaseRegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, bool isSigned = true; switch (AddrMode) { case ARMII::AddrModeT2_i8: + case ARMII::AddrModeT2_i8pos: + case ARMII::AddrModeT2_i8neg: case ARMII::AddrModeT2_i12: // i8 supports only negative, and i12 supports only positive, so // based on Offset sign, consider the appropriate instruction diff --git a/llvm/lib/Target/ARM/ARMBranchTargets.cpp b/llvm/lib/Target/ARM/ARMBranchTargets.cpp index 1091c1f970fa..8ba3e627c039 100644 --- a/llvm/lib/Target/ARM/ARMBranchTargets.cpp +++ b/llvm/lib/Target/ARM/ARMBranchTargets.cpp @@ -108,6 +108,7 @@ void ARMBranchTargets::addBTI(const ARMInstrInfo &TII, MachineBasicBlock &MBB, bool IsFirstBB) { // Which instruction to insert: BTI or PACBTI unsigned OpCode = ARM::t2BTI; + unsigned MIFlags = 0; // Skip meta instructions, including EH labels auto MBBI = llvm::find_if_not(MBB.instrs(), [](const MachineInstr &MI) { @@ -121,6 +122,7 @@ void ARMBranchTargets::addBTI(const ARMInstrInfo &TII, MachineBasicBlock &MBB, LLVM_DEBUG(dbgs() << "Removing a 'PAC' instr from BB '" << MBB.getName() << "' to replace with PACBTI\n"); OpCode = ARM::t2PACBTI; + MIFlags = MachineInstr::FrameSetup; auto NextMBBI = std::next(MBBI); MBBI->eraseFromParent(); MBBI = NextMBBI; @@ -131,5 +133,6 @@ void ARMBranchTargets::addBTI(const ARMInstrInfo &TII, MachineBasicBlock &MBB, << (OpCode == ARM::t2BTI ? "BTI" : "PACBTI") << "' instr into BB '" << MBB.getName() << "'\n"); // Finally, insert a new instruction (either PAC or PACBTI) - BuildMI(MBB, MBBI, MBB.findDebugLoc(MBBI), TII.get(OpCode)); + BuildMI(MBB, MBBI, MBB.findDebugLoc(MBBI), TII.get(OpCode)) + .setMIFlags(MIFlags); } diff --git a/llvm/lib/Target/ARM/ARMCallingConv.cpp b/llvm/lib/Target/ARM/ARMCallingConv.cpp index d8d9ca3b912f..32f3a4a632f5 100644 --- a/llvm/lib/Target/ARM/ARMCallingConv.cpp +++ b/llvm/lib/Target/ARM/ARMCallingConv.cpp @@ -230,10 +230,9 @@ static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned ValNo, MVT ValVT, unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size()); if (RegResult) { - for (SmallVectorImpl<CCValAssign>::iterator It = PendingMembers.begin(); - It != PendingMembers.end(); ++It) { - It->convertToReg(RegResult); - State.addLoc(*It); + for (CCValAssign &PendingMember : PendingMembers) { + PendingMember.convertToReg(RegResult); + State.addLoc(PendingMember); ++RegResult; } PendingMembers.clear(); diff --git a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp index c2ca4708c208..a2a4f1f3bdfd 100644 --- a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp +++ b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp @@ -310,8 +310,7 @@ void ARMConstantIslands::verify() { BBInfo[RHS.getNumber()].postOffset(); })); LLVM_DEBUG(dbgs() << "Verifying " << CPUsers.size() << " CP users.\n"); - for (unsigned i = 0, e = CPUsers.size(); i != e; ++i) { - CPUser &U = CPUsers[i]; + for (CPUser &U : CPUsers) { unsigned UserOffset = getUserOffset(U); // Verify offset using the real max displacement without the safety // adjustment. @@ -697,10 +696,9 @@ ARMConstantIslands::findConstPoolEntry(unsigned CPI, std::vector<CPEntry> &CPEs = CPEntries[CPI]; // Number of entries per constpool index should be small, just do a // linear search. - for (unsigned i = 0, e = CPEs.size(); i != e; ++i) { - if (CPEs[i].CPEMI == CPEMI) - return &CPEs[i]; - } + for (CPEntry &CPE : CPEs) + if (CPE.CPEMI == CPEMI) + return &CPE; return nullptr; } @@ -1234,27 +1232,27 @@ int ARMConstantIslands::findInRangeCPEntry(CPUser& U, unsigned UserOffset) { // No. Look for previously created clones of the CPE that are in range. unsigned CPI = getCombinedIndex(CPEMI); std::vector<CPEntry> &CPEs = CPEntries[CPI]; - for (unsigned i = 0, e = CPEs.size(); i != e; ++i) { + for (CPEntry &CPE : CPEs) { // We already tried this one - if (CPEs[i].CPEMI == CPEMI) + if (CPE.CPEMI == CPEMI) continue; // Removing CPEs can leave empty entries, skip - if (CPEs[i].CPEMI == nullptr) + if (CPE.CPEMI == nullptr) continue; - if (isCPEntryInRange(UserMI, UserOffset, CPEs[i].CPEMI, U.getMaxDisp(), - U.NegOk)) { - LLVM_DEBUG(dbgs() << "Replacing CPE#" << CPI << " with CPE#" - << CPEs[i].CPI << "\n"); + if (isCPEntryInRange(UserMI, UserOffset, CPE.CPEMI, U.getMaxDisp(), + U.NegOk)) { + LLVM_DEBUG(dbgs() << "Replacing CPE#" << CPI << " with CPE#" << CPE.CPI + << "\n"); // Point the CPUser node to the replacement - U.CPEMI = CPEs[i].CPEMI; + U.CPEMI = CPE.CPEMI; // Change the CPI in the instruction operand to refer to the clone. for (MachineOperand &MO : UserMI->operands()) if (MO.isCPI()) { - MO.setIndex(CPEs[i].CPI); + MO.setIndex(CPE.CPI); break; } // Adjust the refcount of the clone... - CPEs[i].RefCount++; + CPE.RefCount++; // ...and the original. If we didn't remove the old entry, none of the // addresses changed, so we don't need another pass. return decrementCPEReferenceCount(CPI, CPEMI) ? 2 : 1; @@ -1675,15 +1673,14 @@ void ARMConstantIslands::removeDeadCPEMI(MachineInstr *CPEMI) { /// are zero. bool ARMConstantIslands::removeUnusedCPEntries() { unsigned MadeChange = false; - for (unsigned i = 0, e = CPEntries.size(); i != e; ++i) { - std::vector<CPEntry> &CPEs = CPEntries[i]; - for (unsigned j = 0, ee = CPEs.size(); j != ee; ++j) { - if (CPEs[j].RefCount == 0 && CPEs[j].CPEMI) { - removeDeadCPEMI(CPEs[j].CPEMI); - CPEs[j].CPEMI = nullptr; - MadeChange = true; - } + for (std::vector<CPEntry> &CPEs : CPEntries) { + for (CPEntry &CPE : CPEs) { + if (CPE.RefCount == 0 && CPE.CPEMI) { + removeDeadCPEMI(CPE.CPEMI); + CPE.CPEMI = nullptr; + MadeChange = true; } + } } return MadeChange; } @@ -1829,8 +1826,7 @@ bool ARMConstantIslands::optimizeThumb2Instructions() { bool MadeChange = false; // Shrink ADR and LDR from constantpool. - for (unsigned i = 0, e = CPUsers.size(); i != e; ++i) { - CPUser &U = CPUsers[i]; + for (CPUser &U : CPUsers) { unsigned Opcode = U.MI->getOpcode(); unsigned NewOpc = 0; unsigned Scale = 1; diff --git a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp index 7a35f252b22a..fa244786a80d 100644 --- a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp +++ b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp @@ -2160,6 +2160,11 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB, return true; } case ARM::tBXNS_RET: { + // For v8.0-M.Main we need to authenticate LR before clearing FPRs, which + // uses R12 as a scratch register. + if (!STI->hasV8_1MMainlineOps() && AFI->shouldSignReturnAddress()) + BuildMI(MBB, MBBI, DebugLoc(), TII->get(ARM::t2AUT)); + MachineBasicBlock &AfterBB = CMSEClearFPRegs(MBB, MBBI); if (STI->hasV8_1MMainlineOps()) { @@ -2169,6 +2174,9 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB, .addReg(ARM::SP) .addImm(4) .add(predOps(ARMCC::AL)); + + if (AFI->shouldSignReturnAddress()) + BuildMI(AfterBB, AfterBB.end(), DebugLoc(), TII->get(ARM::t2AUT)); } // Clear all GPR that are not a use of the return instruction. @@ -3073,6 +3081,22 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB, MI.eraseFromParent(); return true; } + case ARM::t2CALL_BTI: { + MachineFunction &MF = *MI.getMF(); + MachineInstrBuilder MIB = + BuildMI(MF, MI.getDebugLoc(), TII->get(ARM::tBL)); + MIB.cloneMemRefs(MI); + for (unsigned i = 0; i < MI.getNumOperands(); ++i) + MIB.add(MI.getOperand(i)); + if (MI.isCandidateForCallSiteEntry()) + MF.moveCallSiteInfo(&MI, MIB.getInstr()); + MIBundleBuilder Bundler(MBB, MI); + Bundler.append(MIB); + Bundler.append(BuildMI(MF, MI.getDebugLoc(), TII->get(ARM::t2BTI))); + finalizeBundle(MBB, Bundler.begin(), Bundler.end()); + MI.eraseFromParent(); + return true; + } case ARM::LOADDUAL: case ARM::STOREDUAL: { Register PairReg = MI.getOperand(0).getReg(); diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp index b866cf952ff1..4b59f9cb94ce 100644 --- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp +++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp @@ -503,20 +503,12 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, StackAdjustingInsts DefCFAOffsetCandidates; bool HasFP = hasFP(MF); - // Allocate the vararg register save area. - if (ArgRegsSaveSize) { - emitSPUpdate(isARM, MBB, MBBI, dl, TII, -ArgRegsSaveSize, - MachineInstr::FrameSetup); - DefCFAOffsetCandidates.addInst(std::prev(MBBI), ArgRegsSaveSize, true); - } - if (!AFI->hasStackFrame() && (!STI.isTargetWindows() || !WindowsRequiresStackProbe(MF, NumBytes))) { - if (NumBytes - ArgRegsSaveSize != 0) { - emitSPUpdate(isARM, MBB, MBBI, dl, TII, -(NumBytes - ArgRegsSaveSize), + if (NumBytes != 0) { + emitSPUpdate(isARM, MBB, MBBI, dl, TII, -NumBytes, MachineInstr::FrameSetup); - DefCFAOffsetCandidates.addInst(std::prev(MBBI), - NumBytes - ArgRegsSaveSize, true); + DefCFAOffsetCandidates.addInst(std::prev(MBBI), NumBytes, true); } DefCFAOffsetCandidates.emitDefCFAOffsets(MBB, dl, TII, HasFP); return; @@ -562,13 +554,26 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, } } - // Move past FPCXT area. MachineBasicBlock::iterator LastPush = MBB.end(), GPRCS1Push, GPRCS2Push; + + // Move past the PAC computation. + if (AFI->shouldSignReturnAddress()) + LastPush = MBBI++; + + // Move past FPCXT area. if (FPCXTSaveSize > 0) { LastPush = MBBI++; DefCFAOffsetCandidates.addInst(LastPush, FPCXTSaveSize, true); } + // Allocate the vararg register save area. + if (ArgRegsSaveSize) { + emitSPUpdate(isARM, MBB, MBBI, dl, TII, -ArgRegsSaveSize, + MachineInstr::FrameSetup); + LastPush = std::prev(MBBI); + DefCFAOffsetCandidates.addInst(LastPush, ArgRegsSaveSize, true); + } + // Move past area 1. if (GPRCS1Size > 0) { GPRCS1Push = LastPush = MBBI++; @@ -788,7 +793,8 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF, case ARM::R11: case ARM::R12: if (STI.splitFramePushPop(MF)) { - unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); + unsigned DwarfReg = MRI->getDwarfRegNum( + Reg == ARM::R12 ? (unsigned)ARM::RA_AUTH_CODE : Reg, true); unsigned Offset = MFI.getObjectOffset(FI); unsigned CFIIndex = MF.addFrameInst( MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset)); @@ -923,8 +929,9 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF, DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); if (!AFI->hasStackFrame()) { - if (NumBytes - ReservedArgStack != 0) - emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes - ReservedArgStack, + if (NumBytes + IncomingArgStackToRestore != 0) + emitSPUpdate(isARM, MBB, MBBI, dl, TII, + NumBytes + IncomingArgStackToRestore, MachineInstr::FrameDestroy); } else { // Unwind MBBI to point to first LDR / VLDRD. @@ -1007,15 +1014,21 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF, if (AFI->getGPRCalleeSavedArea2Size()) MBBI++; if (AFI->getGPRCalleeSavedArea1Size()) MBBI++; - if (AFI->getFPCXTSaveAreaSize()) MBBI++; - } - if (ReservedArgStack || IncomingArgStackToRestore) { - assert((int)ReservedArgStack + IncomingArgStackToRestore >= 0 && - "attempting to restore negative stack amount"); - emitSPUpdate(isARM, MBB, MBBI, dl, TII, - ReservedArgStack + IncomingArgStackToRestore, - MachineInstr::FrameDestroy); + if (ReservedArgStack || IncomingArgStackToRestore) { + assert((int)ReservedArgStack + IncomingArgStackToRestore >= 0 && + "attempting to restore negative stack amount"); + emitSPUpdate(isARM, MBB, MBBI, dl, TII, + ReservedArgStack + IncomingArgStackToRestore, + MachineInstr::FrameDestroy); + } + + // Validate PAC, It should have been already popped into R12. For CMSE entry + // function, the validation instruction is emitted during expansion of the + // tBXNS_RET, since the validation must use the value of SP at function + // entry, before saving, resp. after restoring, FPCXTNS. + if (AFI->shouldSignReturnAddress() && !AFI->isCmseNSEntryFunction()) + BuildMI(MBB, MBBI, DebugLoc(), STI.getInstrInfo()->get(ARM::t2AUT)); } } @@ -1199,6 +1212,7 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB, const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); + bool hasPAC = AFI->shouldSignReturnAddress(); DebugLoc DL; bool isTailCall = false; bool isInterrupt = false; @@ -1231,7 +1245,7 @@ void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB, continue; if (Reg == ARM::LR && !isTailCall && !isVarArg && !isInterrupt && !isCmseEntry && !isTrap && AFI->getArgumentStackToRestore() == 0 && - STI.hasV5TOps() && MBB.succ_empty()) { + STI.hasV5TOps() && MBB.succ_empty() && !hasPAC) { Reg = ARM::PC; // Fold the return instruction into the LDM. DeleteRet = true; @@ -1580,6 +1594,11 @@ bool ARMFrameLowering::spillCalleeSavedRegisters( ARM::t2STR_PRE : ARM::STR_PRE_IMM; unsigned FltOpc = ARM::VSTMDDB_UPD; unsigned NumAlignedDPRCS2Regs = AFI->getNumAlignedDPRCS2Regs(); + // Compute PAC in R12. + if (AFI->shouldSignReturnAddress()) { + BuildMI(MBB, MI, DebugLoc(), STI.getInstrInfo()->get(ARM::t2PAC)) + .setMIFlags(MachineInstr::FrameSetup); + } // Save the non-secure floating point context. if (llvm::any_of(CSI, [](const CalleeSavedInfo &C) { return C.getReg() == ARM::FPCXTNS; @@ -1789,6 +1808,13 @@ bool ARMFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const { MF.getInfo<ARMFunctionInfo>()->isCmseNSEntryFunction()) return false; + // We are disabling shrinkwrapping for now when PAC is enabled, as + // shrinkwrapping can cause clobbering of r12 when the PAC code is + // generated. A follow-up patch will fix this in a more performant manner. + if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress( + false /*SpillsLR */)) + return false; + return true; } @@ -2315,6 +2341,26 @@ bool ARMFrameLowering::assignCalleeSavedSpillSlots( CSI.back().setRestored(false); } + // For functions, which sign their return address, upon function entry, the + // return address PAC is computed in R12. Treat R12 as a callee-saved register + // in this case. + const auto &AFI = *MF.getInfo<ARMFunctionInfo>(); + if (AFI.shouldSignReturnAddress()) { + // The order of register must match the order we push them, because the + // PEI assigns frame indices in that order. When compiling for return + // address sign and authenication, we use split push, therefore the orders + // we want are: + // LR, R7, R6, R5, R4, <R12>, R11, R10, R9, R8, D15-D8 + CSI.insert(find_if(CSI, + [=](const auto &CS) { + unsigned Reg = CS.getReg(); + return Reg == ARM::R10 || Reg == ARM::R11 || + Reg == ARM::R8 || Reg == ARM::R9 || + ARM::DPRRegClass.contains(Reg); + }), + CalleeSavedInfo(ARM::R12)); + } + return false; } diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 33d115945614..3d45db349644 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -391,6 +391,7 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) { setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); setOperationAction(ISD::BUILD_VECTOR, VT, Custom); + setOperationAction(ISD::VSELECT, VT, Legal); } setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); @@ -428,7 +429,7 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) { } // Predicate types - const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1}; + const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1}; for (auto VT : pTypes) { addRegisterClass(VT, &ARM::VCCRRegClass); setOperationAction(ISD::BUILD_VECTOR, VT, Custom); @@ -445,6 +446,16 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) { setOperationAction(ISD::VSELECT, VT, Expand); setOperationAction(ISD::SELECT, VT, Expand); } + setOperationAction(ISD::SETCC, MVT::v2i1, Expand); + setOperationAction(ISD::TRUNCATE, MVT::v2i1, Expand); + setOperationAction(ISD::AND, MVT::v2i1, Expand); + setOperationAction(ISD::OR, MVT::v2i1, Expand); + setOperationAction(ISD::XOR, MVT::v2i1, Expand); + setOperationAction(ISD::SINT_TO_FP, MVT::v2i1, Expand); + setOperationAction(ISD::UINT_TO_FP, MVT::v2i1, Expand); + setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Expand); + setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Expand); + setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom); setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); @@ -1647,6 +1658,7 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const { MAKE_CASE(ARMISD::CALL_PRED) MAKE_CASE(ARMISD::CALL_NOLINK) MAKE_CASE(ARMISD::tSECALL) + MAKE_CASE(ARMISD::t2CALL_BTI) MAKE_CASE(ARMISD::BRCOND) MAKE_CASE(ARMISD::BR_JT) MAKE_CASE(ARMISD::BR2_JT) @@ -1853,8 +1865,10 @@ EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, // MVE has a predicate register. if ((Subtarget->hasMVEIntegerOps() && - (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8)) || - (Subtarget->hasMVEFloatOps() && (VT == MVT::v4f32 || VT == MVT::v8f16))) + (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || + VT == MVT::v16i8)) || + (Subtarget->hasMVEFloatOps() && + (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16))) return MVT::getVectorVT(MVT::i1, VT.getVectorElementCount()); return VT.changeVectorElementTypeToInteger(); } @@ -2308,6 +2322,12 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, bool isCmseNSCall = false; bool isSibCall = false; bool PreferIndirect = false; + bool GuardWithBTI = false; + + // Lower 'returns_twice' calls to a pseudo-instruction. + if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Attribute::ReturnsTwice) && + !Subtarget->getNoBTIAtReturnTwice()) + GuardWithBTI = AFI->branchTargetEnforcement(); // Determine whether this is a non-secure function call. if (CLI.CB && CLI.CB->getAttributes().hasFnAttr("cmse_nonsecure_call")) @@ -2713,7 +2733,9 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // FIXME: handle tail calls differently. unsigned CallOpc; if (Subtarget->isThumb()) { - if (isCmseNSCall) + if (GuardWithBTI) + CallOpc = ARMISD::t2CALL_BTI; + else if (isCmseNSCall) CallOpc = ARMISD::tSECALL; else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps()) CallOpc = ARMISD::CALL_NOLINK; @@ -2930,9 +2952,17 @@ bool ARMTargetLowering::IsEligibleForTailCallOptimization( // Indirect tail calls cannot be optimized for Thumb1 if the args // to the call take up r0-r3. The reason is that there are no legal registers // left to hold the pointer to the function to be called. - if (Subtarget->isThumb1Only() && Outs.size() >= 4 && - (!isa<GlobalAddressSDNode>(Callee.getNode()) || isIndirect)) - return false; + // Similarly, if the function uses return address sign and authentication, + // r12 is needed to hold the PAC and is not available to hold the callee + // address. + if (Outs.size() >= 4 && + (!isa<GlobalAddressSDNode>(Callee.getNode()) || isIndirect)) { + if (Subtarget->isThumb1Only()) + return false; + // Conservatively assume the function spills LR. + if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(true)) + return false; + } // Look for obvious safe cases to perform tail call optimization that do not // require ABI changes. This is what gcc calls sibcall. @@ -7616,7 +7646,10 @@ static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG, unsigned NumElts = VT.getVectorNumElements(); unsigned BoolMask; unsigned BitsPerBool; - if (NumElts == 4) { + if (NumElts == 2) { + BitsPerBool = 8; + BoolMask = 0xff; + } else if (NumElts == 4) { BitsPerBool = 4; BoolMask = 0xf; } else if (NumElts == 8) { @@ -7699,6 +7732,46 @@ static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG, DAG.getConstant(N, DL, MVT::i32)); } +// Returns true if the operation N can be treated as qr instruction variant at +// operand Op. +static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op) { + switch (N->getOpcode()) { + case ISD::ADD: + case ISD::MUL: + case ISD::SADDSAT: + case ISD::UADDSAT: + return true; + case ISD::SUB: + case ISD::SSUBSAT: + case ISD::USUBSAT: + return N->getOperand(1).getNode() == Op; + case ISD::INTRINSIC_WO_CHAIN: + switch (N->getConstantOperandVal(0)) { + case Intrinsic::arm_mve_add_predicated: + case Intrinsic::arm_mve_mul_predicated: + case Intrinsic::arm_mve_qadd_predicated: + case Intrinsic::arm_mve_vhadd: + case Intrinsic::arm_mve_hadd_predicated: + case Intrinsic::arm_mve_vqdmulh: + case Intrinsic::arm_mve_qdmulh_predicated: + case Intrinsic::arm_mve_vqrdmulh: + case Intrinsic::arm_mve_qrdmulh_predicated: + case Intrinsic::arm_mve_vqdmull: + case Intrinsic::arm_mve_vqdmull_predicated: + return true; + case Intrinsic::arm_mve_sub_predicated: + case Intrinsic::arm_mve_qsub_predicated: + case Intrinsic::arm_mve_vhsub: + case Intrinsic::arm_mve_hsub_predicated: + return N->getOperand(2).getNode() == Op; + default: + return false; + } + default: + return false; + } +} + // If this is a case we can't handle, return null and let the default // expansion code take care of it. SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, @@ -7720,6 +7793,20 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, if (SplatUndef.isAllOnes()) return DAG.getUNDEF(VT); + // If all the users of this constant splat are qr instruction variants, + // generate a vdup of the constant. + if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize && + (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) && + all_of(BVN->uses(), + [BVN](const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) { + EVT DupVT = SplatBitSize == 32 ? MVT::v4i32 + : SplatBitSize == 16 ? MVT::v8i16 + : MVT::v16i8; + SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32); + SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const); + return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup); + } + if ((ST->hasNEON() && SplatBitSize <= 64) || (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) { // Check if an immediate VMOV works. @@ -8313,9 +8400,8 @@ static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op, SDLoc DL(Op); SmallVector<SDValue, 8> VTBLMask; - for (ArrayRef<int>::iterator - I = ShuffleMask.begin(), E = ShuffleMask.end(); I != E; ++I) - VTBLMask.push_back(DAG.getConstant(*I, DL, MVT::i32)); + for (int I : ShuffleMask) + VTBLMask.push_back(DAG.getConstant(I, DL, MVT::i32)); if (V2.getNode()->isUndef()) return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1, @@ -8346,6 +8432,8 @@ static SDValue LowerReverse_VECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) { static EVT getVectorTyFromPredicateVector(EVT VT) { switch (VT.getSimpleVT().SimpleTy) { + case MVT::v2i1: + return MVT::v2f64; case MVT::v4i1: return MVT::v4i32; case MVT::v8i1: @@ -8427,7 +8515,14 @@ static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG, DAG.getUNDEF(NewVT), ShuffleMask); // Now return the result of comparing the shuffled vector with zero, - // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. + // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1 + // we convert to a v4i1 compare to fill in the two halves of the i64 as i32s. + if (VT == MVT::v2i1) { + SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Shuffled); + SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC, + DAG.getConstant(ARMCC::NE, dl, MVT::i32)); + return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp); + } return DAG.getNode(ARMISD::VCMPZ, dl, VT, Shuffled, DAG.getConstant(ARMCC::NE, dl, MVT::i32)); } @@ -8927,8 +9022,15 @@ static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG, ConVec = ExtractInto(NewV1, ConVec, j); ConVec = ExtractInto(NewV2, ConVec, j); - // Now return the result of comparing the subvector with zero, - // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1. + // Now return the result of comparing the subvector with zero, which will + // generate a real predicate, i.e. v4i1, v8i1 or v16i1. For a v2i1 we + // convert to a v4i1 compare to fill in the two halves of the i64 as i32s. + if (VT == MVT::v2i1) { + SDValue BC = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, ConVec); + SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, BC, + DAG.getConstant(ARMCC::NE, dl, MVT::i32)); + return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp); + } return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec, DAG.getConstant(ARMCC::NE, dl, MVT::i32)); }; @@ -8993,6 +9095,22 @@ static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, MVT ElType = getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT(); + if (NumElts == 2) { + EVT SubVT = MVT::v4i32; + SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT); + for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j += 2) { + SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1, + DAG.getIntPtrConstant(i, dl)); + SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt, + DAG.getConstant(j, dl, MVT::i32)); + SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt, + DAG.getConstant(j + 1, dl, MVT::i32)); + } + SDValue Cmp = DAG.getNode(ARMISD::VCMPZ, dl, MVT::v4i1, SubVec, + DAG.getConstant(ARMCC::NE, dl, MVT::i32)); + return DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v2i1, Cmp); + } + EVT SubVT = MVT::getVectorVT(ElType, NumElts); SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT); for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) { @@ -9839,16 +9957,17 @@ void ARMTargetLowering::ExpandDIV_Windows( static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) { LoadSDNode *LD = cast<LoadSDNode>(Op.getNode()); EVT MemVT = LD->getMemoryVT(); - assert((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) && + assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || + MemVT == MVT::v16i1) && "Expected a predicate type!"); assert(MemVT == Op.getValueType()); assert(LD->getExtensionType() == ISD::NON_EXTLOAD && "Expected a non-extending load"); assert(LD->isUnindexed() && "Expected a unindexed load"); - // The basic MVE VLDR on a v4i1/v8i1 actually loads the entire 16bit + // The basic MVE VLDR on a v2i1/v4i1/v8i1 actually loads the entire 16bit // predicate, with the "v4i1" bits spread out over the 16 bits loaded. We - // need to make sure that 8/4 bits are actually loaded into the correct + // need to make sure that 8/4/2 bits are actually loaded into the correct // place, which means loading the value and then shuffling the values into // the bottom bits of the predicate. // Equally, VLDR for an v16i1 will actually load 32bits (so will be incorrect @@ -9895,14 +10014,15 @@ void ARMTargetLowering::LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results, static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) { StoreSDNode *ST = cast<StoreSDNode>(Op.getNode()); EVT MemVT = ST->getMemoryVT(); - assert((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) && + assert((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || + MemVT == MVT::v16i1) && "Expected a predicate type!"); assert(MemVT == ST->getValue().getValueType()); assert(!ST->isTruncatingStore() && "Expected a non-extending store"); assert(ST->isUnindexed() && "Expected a unindexed store"); - // Only store the v4i1 or v8i1 worth of bits, via a buildvector with top bits - // unset and a scalar store. + // Only store the v2i1 or v4i1 or v8i1 worth of bits, via a buildvector with + // top bits unset and a scalar store. SDLoc dl(Op); SDValue Build = ST->getValue(); if (MemVT != MVT::v16i1) { @@ -9953,7 +10073,7 @@ static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, {ST->getChain(), Lo, Hi, ST->getBasePtr()}, MemVT, ST->getMemOperand()); } else if (Subtarget->hasMVEIntegerOps() && - ((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || + ((MemVT == MVT::v2i1 || MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1))) { return LowerPredicateStore(Op, DAG); } @@ -10561,25 +10681,23 @@ void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI, // associated with. DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad; unsigned MaxCSNum = 0; - for (MachineFunction::iterator BB = MF->begin(), E = MF->end(); BB != E; - ++BB) { - if (!BB->isEHPad()) continue; + for (MachineBasicBlock &BB : *MF) { + if (!BB.isEHPad()) + continue; // FIXME: We should assert that the EH_LABEL is the first MI in the landing // pad. - for (MachineBasicBlock::iterator - II = BB->begin(), IE = BB->end(); II != IE; ++II) { - if (!II->isEHLabel()) continue; + for (MachineInstr &II : BB) { + if (!II.isEHLabel()) + continue; - MCSymbol *Sym = II->getOperand(0).getMCSymbol(); + MCSymbol *Sym = II.getOperand(0).getMCSymbol(); if (!MF->hasCallSiteLandingPad(Sym)) continue; SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym); - for (SmallVectorImpl<unsigned>::iterator - CSI = CallSiteIdxs.begin(), CSE = CallSiteIdxs.end(); - CSI != CSE; ++CSI) { - CallSiteNumToLPad[*CSI].push_back(&*BB); - MaxCSNum = std::max(MaxCSNum, *CSI); + for (unsigned Idx : CallSiteIdxs) { + CallSiteNumToLPad[Idx].push_back(&BB); + MaxCSNum = std::max(MaxCSNum, Idx); } break; } @@ -14002,8 +14120,8 @@ static SDValue PerformANDCombine(SDNode *N, EVT VT = N->getValueType(0); SelectionDAG &DAG = DCI.DAG; - if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v4i1 || - VT == MVT::v8i1 || VT == MVT::v16i1) + if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v2i1 || + VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1) return SDValue(); APInt SplatBits, SplatUndef; @@ -14298,8 +14416,8 @@ static SDValue PerformORCombine(SDNode *N, if(!DAG.getTargetLoweringInfo().isTypeLegal(VT)) return SDValue(); - if (Subtarget->hasMVEIntegerOps() && - (VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1)) + if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 || + VT == MVT::v8i1 || VT == MVT::v16i1)) return PerformORCombine_i1(N, DAG, Subtarget); APInt SplatBits, SplatUndef; @@ -14569,6 +14687,15 @@ static SDValue IsCMPZCSINC(SDNode *Cmp, ARMCC::CondCodes &CC) { if (Cmp->getOpcode() != ARMISD::CMPZ || !isNullConstant(Cmp->getOperand(1))) return SDValue(); SDValue CSInc = Cmp->getOperand(0); + + // Ignore any `And 1` nodes that may not yet have been removed. We are + // looking for a value that produces 1/0, so these have no effect on the + // code. + while (CSInc.getOpcode() == ISD::AND && + isa<ConstantSDNode>(CSInc.getOperand(1)) && + CSInc.getConstantOperandVal(1) == 1 && CSInc->hasOneUse()) + CSInc = CSInc.getOperand(0); + if (CSInc.getOpcode() != ARMISD::CSINC || !isNullConstant(CSInc.getOperand(0)) || !isNullConstant(CSInc.getOperand(1)) || !CSInc->hasOneUse()) @@ -17897,6 +18024,23 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const { if (!VT.isInteger()) return SDValue(); + // Fold away an unneccessary CMPZ/CMOV + // CMOV A, B, C1, $cpsr, (CMPZ (CMOV 1, 0, C2, D), 0) -> + // if C1==EQ -> CMOV A, B, C2, $cpsr, D + // if C1==NE -> CMOV A, B, NOT(C2), $cpsr, D + if (N->getConstantOperandVal(2) == ARMCC::EQ || + N->getConstantOperandVal(2) == ARMCC::NE) { + ARMCC::CondCodes Cond; + if (SDValue C = IsCMPZCSINC(N->getOperand(4).getNode(), Cond)) { + if (N->getConstantOperandVal(2) == ARMCC::NE) + Cond = ARMCC::getOppositeCondition(Cond); + return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::i32, N->getOperand(0), + N->getOperand(1), + DAG.getTargetConstant(Cond, SDLoc(N), MVT::i32), + N->getOperand(3), C); + } + } + // Materialize a boolean comparison for integers so we can avoid branching. if (isNullConstant(FalseVal)) { if (CC == ARMCC::EQ && isOneConstant(TrueVal)) { @@ -18564,7 +18708,8 @@ bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, return false; // These are for predicates - if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1)) { + if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1 || + Ty == MVT::v2i1)) { if (Fast) *Fast = true; return true; diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h index e3b422358cae..1c5f8389f57c 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.h +++ b/llvm/lib/Target/ARM/ARMISelLowering.h @@ -69,6 +69,7 @@ class VectorType; CALL_PRED, // Function call that's predicable. CALL_NOLINK, // Function call with branch not branch-and-link. tSECALL, // CMSE non-secure function call. + t2CALL_BTI, // Thumb function call followed by BTI instruction. BRCOND, // Conditional branch. BR_JT, // Jumptable branch. BR2_JT, // Jumptable branch (2 level - jumptable entry is a jump). diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td index f53814a80e01..1ae0354ffc37 100644 --- a/llvm/lib/Target/ARM/ARMInstrMVE.td +++ b/llvm/lib/Target/ARM/ARMInstrMVE.td @@ -254,13 +254,6 @@ class MVEVectorVTInfo<ValueType vec, ValueType dblvec, // An LLVM ValueType representing a corresponding vector of // predicate bits, for use in ISel patterns that handle an IR // intrinsic describing the predicated form of the instruction. - // - // Usually, for a vector of N things, this will be vNi1. But for - // vectors of 2 values, we make an exception, and use v4i1 instead - // of v2i1. Rationale: MVE codegen doesn't support doing all the - // auxiliary operations on v2i1 (vector shuffles etc), and also, - // there's no MVE compare instruction that will _generate_ v2i1 - // directly. ValueType Pred = pred; // Same as Pred but for DblVec rather than Vec. @@ -294,25 +287,25 @@ class MVEVectorVTInfo<ValueType vec, ValueType dblvec, // Integer vector types that don't treat signed and unsigned differently. def MVE_v16i8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "i", ?>; def MVE_v8i16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "i", ?>; -def MVE_v4i32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v4i1, 0b10, "i", ?>; -def MVE_v2i64 : MVEVectorVTInfo<v2i64, ?, v4i1, ?, 0b11, "i", ?>; +def MVE_v4i32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v2i1, 0b10, "i", ?>; +def MVE_v2i64 : MVEVectorVTInfo<v2i64, ?, v2i1, ?, 0b11, "i", ?>; // Explicitly signed and unsigned integer vectors. They map to the // same set of LLVM ValueTypes as above, but are represented // differently in assembly and instruction encodings. def MVE_v16s8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "s", 0b0>; def MVE_v8s16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "s", 0b0>; -def MVE_v4s32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v4i1, 0b10, "s", 0b0>; -def MVE_v2s64 : MVEVectorVTInfo<v2i64, ?, v4i1, ?, 0b11, "s", 0b0>; +def MVE_v4s32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v2i1, 0b10, "s", 0b0>; +def MVE_v2s64 : MVEVectorVTInfo<v2i64, ?, v2i1, ?, 0b11, "s", 0b0>; def MVE_v16u8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "u", 0b1>; def MVE_v8u16 : MVEVectorVTInfo<v8i16, v4i32, v8i1, v4i1, 0b01, "u", 0b1>; -def MVE_v4u32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v4i1, 0b10, "u", 0b1>; -def MVE_v2u64 : MVEVectorVTInfo<v2i64, ?, v4i1, ?, 0b11, "u", 0b1>; +def MVE_v4u32 : MVEVectorVTInfo<v4i32, v2i64, v4i1, v2i1, 0b10, "u", 0b1>; +def MVE_v2u64 : MVEVectorVTInfo<v2i64, ?, v2i1, ?, 0b11, "u", 0b1>; // FP vector types. def MVE_v8f16 : MVEVectorVTInfo<v8f16, v4f32, v8i1, v4i1, 0b01, "f", ?>; -def MVE_v4f32 : MVEVectorVTInfo<v4f32, v2f64, v4i1, v4i1, 0b10, "f", ?>; -def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?, v4i1, ?, 0b11, "f", ?>; +def MVE_v4f32 : MVEVectorVTInfo<v4f32, v2f64, v4i1, v2i1, 0b10, "f", ?>; +def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?, v2i1, ?, 0b11, "f", ?>; // Polynomial vector types. def MVE_v16p8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b11, "p", 0b0>; @@ -2260,6 +2253,31 @@ let Predicates = [HasMVEInt] in { (v4i32 (ARMvmovImm (i32 1)))), (i32 1))), (MVE_VRHADDu32 MQPR:$Qm, MQPR:$Qn)>; + + def : Pat<(v16i8 (ARMvshrsImm (addnsw (addnsw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)), + (v16i8 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDs8 MQPR:$Qm, MQPR:$Qn)>; + def : Pat<(v8i16 (ARMvshrsImm (addnsw (addnsw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)), + (v8i16 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDs16 MQPR:$Qm, MQPR:$Qn)>; + def : Pat<(v4i32 (ARMvshrsImm (addnsw (addnsw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)), + (v4i32 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDs32 MQPR:$Qm, MQPR:$Qn)>; + def : Pat<(v16i8 (ARMvshruImm (addnuw (addnuw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)), + (v16i8 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDu8 MQPR:$Qm, MQPR:$Qn)>; + def : Pat<(v8i16 (ARMvshruImm (addnuw (addnuw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)), + (v8i16 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDu16 MQPR:$Qm, MQPR:$Qn)>; + def : Pat<(v4i32 (ARMvshruImm (addnuw (addnuw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)), + (v4i32 (ARMvdup (i32 1)))), + (i32 1))), + (MVE_VRHADDu32 MQPR:$Qm, MQPR:$Qn)>; } @@ -4450,6 +4468,11 @@ multiclass two_predops<SDPatternOperator opnode, Instruction insn> { (insn (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p1), rGPR)), (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p2), rGPR))), VCCR))>; + def v2i1 : Pat<(v2i1 (opnode (v2i1 VCCR:$p1), (v2i1 VCCR:$p2))), + (v2i1 (COPY_TO_REGCLASS + (insn (i32 (COPY_TO_REGCLASS (v2i1 VCCR:$p1), rGPR)), + (i32 (COPY_TO_REGCLASS (v2i1 VCCR:$p2), rGPR))), + VCCR))>; } let Predicates = [HasMVEInt] in { @@ -4469,20 +4492,20 @@ def load_align4 : PatFrag<(ops node:$ptr), (load node:$ptr), [{ }]>; let Predicates = [HasMVEInt] in { - foreach VT = [ v4i1, v8i1, v16i1 ] in { + foreach VT = [ v2i1, v4i1, v8i1, v16i1 ] in { def : Pat<(i32 (predicate_cast (VT VCCR:$src))), (i32 (COPY_TO_REGCLASS (VT VCCR:$src), VCCR))>; def : Pat<(VT (predicate_cast (i32 VCCR:$src))), (VT (COPY_TO_REGCLASS (i32 VCCR:$src), VCCR))>; - foreach VT2 = [ v4i1, v8i1, v16i1 ] in + foreach VT2 = [ v2i1, v4i1, v8i1, v16i1 ] in def : Pat<(VT (predicate_cast (VT2 VCCR:$src))), (VT (COPY_TO_REGCLASS (VT2 VCCR:$src), VCCR))>; } // If we happen to be casting from a load we can convert that straight // into a predicate load, so long as the load is of the correct type. - foreach VT = [ v4i1, v8i1, v16i1 ] in { + foreach VT = [ v2i1, v4i1, v8i1, v16i1 ] in { def : Pat<(VT (predicate_cast (i32 (load_align4 taddrmode_imm7<2>:$addr)))), (VT (VLDR_P0_off taddrmode_imm7<2>:$addr))>; } @@ -5350,33 +5373,40 @@ class MVE_VxADDSUB_qr<string iname, string suffix, } multiclass MVE_VHADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract, - Intrinsic unpred_int, Intrinsic pred_int> { + Intrinsic unpred_int, Intrinsic pred_int, PatFrag add_op, + SDNode shift_op> { def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, subtract, VTI.Size>; defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI, unpred_int, pred_int, 1, 1>; + defvar Inst = !cast<Instruction>(NAME); + + let Predicates = [HasMVEInt] in { + def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))), (i32 1))), + (Inst MQPR:$Qm, rGPR:$Rn)>; + } } -multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI> : - MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, int_arm_mve_vhadd, - int_arm_mve_hadd_predicated>; +multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> : + MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, int_arm_mve_vhadd, int_arm_mve_hadd_predicated, + add_op, shift_op>; -multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI> : - MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, int_arm_mve_vhsub, - int_arm_mve_hsub_predicated>; +multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> : + MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, int_arm_mve_vhsub, int_arm_mve_hsub_predicated, + add_op, shift_op>; -defm MVE_VHADD_qr_s8 : MVE_VHADD_qr_m<MVE_v16s8>; -defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16>; -defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32>; -defm MVE_VHADD_qr_u8 : MVE_VHADD_qr_m<MVE_v16u8>; -defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16>; -defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32>; +defm MVE_VHADD_qr_s8 : MVE_VHADD_qr_m<MVE_v16s8, addnsw, ARMvshrsImm>; +defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16, addnsw, ARMvshrsImm>; +defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32, addnsw, ARMvshrsImm>; +defm MVE_VHADD_qr_u8 : MVE_VHADD_qr_m<MVE_v16u8, addnuw, ARMvshruImm>; +defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16, addnuw, ARMvshruImm>; +defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32, addnuw, ARMvshruImm>; -defm MVE_VHSUB_qr_s8 : MVE_VHSUB_qr_m<MVE_v16s8>; -defm MVE_VHSUB_qr_s16 : MVE_VHSUB_qr_m<MVE_v8s16>; -defm MVE_VHSUB_qr_s32 : MVE_VHSUB_qr_m<MVE_v4s32>; -defm MVE_VHSUB_qr_u8 : MVE_VHSUB_qr_m<MVE_v16u8>; -defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16>; -defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32>; +defm MVE_VHSUB_qr_s8 : MVE_VHSUB_qr_m<MVE_v16s8, subnsw, ARMvshrsImm>; +defm MVE_VHSUB_qr_s16 : MVE_VHSUB_qr_m<MVE_v8s16, subnsw, ARMvshrsImm>; +defm MVE_VHSUB_qr_s32 : MVE_VHSUB_qr_m<MVE_v4s32, subnsw, ARMvshrsImm>; +defm MVE_VHSUB_qr_u8 : MVE_VHSUB_qr_m<MVE_v16u8, subnuw, ARMvshruImm>; +defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16, subnuw, ARMvshruImm>; +defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32, subnuw, ARMvshruImm>; multiclass MVE_VADDSUB_qr_f<string iname, MVEVectorVTInfo VTI, bit subtract, SDNode Op, Intrinsic PredInt, SDPatternOperator IdentityVec> { @@ -6778,11 +6808,15 @@ let Predicates = [HasMVEInt] in { (v8i16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred), (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))), (v4i32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; + def : Pat<(v2i64 (vselect (v2i1 VCCR:$pred), (v2i64 MQPR:$v1), (v2i64 MQPR:$v2))), + (v2i64 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v8f16 (vselect (v8i1 VCCR:$pred), (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))), (v8f16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v4f32 (vselect (v4i1 VCCR:$pred), (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))), (v4f32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; + def : Pat<(v2f64 (vselect (v2i1 VCCR:$pred), (v2f64 MQPR:$v1), (v2f64 MQPR:$v2))), + (v2f64 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v16i8 (vselect (v16i8 MQPR:$pred), (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))), (v16i8 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, @@ -6808,6 +6842,8 @@ let Predicates = [HasMVEInt] in { (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v4i32 (zext (v4i1 VCCR:$pred))), (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; + def : Pat<(v2i64 (zext (v2i1 VCCR:$pred))), + (v2i64 (MVE_VPSEL (MVE_VMOVimmi64 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v16i8 (sext (v16i1 VCCR:$pred))), (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>; @@ -6815,6 +6851,8 @@ let Predicates = [HasMVEInt] in { (v8i16 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v4i32 (sext (v4i1 VCCR:$pred))), (v4i32 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; + def : Pat<(v2i64 (sext (v2i1 VCCR:$pred))), + (v2i64 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v16i8 (anyext (v16i1 VCCR:$pred))), (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 1), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>; @@ -6822,6 +6860,8 @@ let Predicates = [HasMVEInt] in { (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>; def : Pat<(v4i32 (anyext (v4i1 VCCR:$pred))), (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; + def : Pat<(v2i64 (anyext (v2i1 VCCR:$pred))), + (v2i64 (MVE_VPSEL (MVE_VMOVimmi64 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>; } let Predicates = [HasMVEFloat] in { @@ -6862,6 +6902,8 @@ def MVE_VPNOT : MVE_p<(outs VCCR:$P0), (ins VCCR:$P0_in), NoItinerary, } let Predicates = [HasMVEInt] in { + def : Pat<(v2i1 (xor (v2i1 VCCR:$pred), (v2i1 (predicate_cast (i32 65535))))), + (v2i1 (MVE_VPNOT (v2i1 VCCR:$pred)))>; def : Pat<(v4i1 (xor (v4i1 VCCR:$pred), (v4i1 (predicate_cast (i32 65535))))), (v4i1 (MVE_VPNOT (v4i1 VCCR:$pred)))>; def : Pat<(v8i1 (xor (v8i1 VCCR:$pred), (v8i1 (predicate_cast (i32 65535))))), diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td index 4471317f4ea4..6e8e61ca2b8e 100644 --- a/llvm/lib/Target/ARM/ARMInstrThumb2.td +++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td @@ -5736,3 +5736,10 @@ def t2BTI : PACBTIHintSpaceNoOpsInst<"bti", 0b00001111>; def t2AUT : PACBTIHintSpaceUseInst<"aut", 0b00101101> { let hasSideEffects = 1; } + +def ARMt2CallBTI : SDNode<"ARMISD::t2CALL_BTI", SDT_ARMcall, + [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, SDNPVariadic]>; + +def t2CALL_BTI : PseudoInst<(outs), (ins pred:$p, thumb_bl_target:$func), + IIC_Br, [(ARMt2CallBTI tglobaladdr:$func)]>, + Requires<[IsThumb2]>, Sched<[WriteBrL]>; diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td index 9d1bfa414dff..dc5f1b92a6c2 100644 --- a/llvm/lib/Target/ARM/ARMInstrVFP.td +++ b/llvm/lib/Target/ARM/ARMInstrVFP.td @@ -1076,6 +1076,9 @@ multiclass vrint_inst_anpm<string opc, bits<2> rm, } } + def : InstAlias<!strconcat("vrint", opc, ".f16.f16\t$Sd, $Sm"), + (!cast<Instruction>(NAME#"H") HPR:$Sd, HPR:$Sm), 0>, + Requires<[HasFullFP16]>; def : InstAlias<!strconcat("vrint", opc, ".f32.f32\t$Sd, $Sm"), (!cast<Instruction>(NAME#"S") SPR:$Sd, SPR:$Sm), 0>, Requires<[HasFPARMv8]>; diff --git a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp index 3b10c60a0654..ef5fc12feb54 100644 --- a/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp +++ b/llvm/lib/Target/ARM/ARMLoadStoreOptimizer.cpp @@ -2121,7 +2121,7 @@ bool ARMLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) { bool Modified = false; for (MachineBasicBlock &MBB : Fn) { Modified |= LoadStoreMultipleOpti(MBB); - if (STI->hasV5TOps()) + if (STI->hasV5TOps() && !AFI->shouldSignReturnAddress()) Modified |= MergeReturnIntoLDM(MBB); if (isThumb1) Modified |= CombineMovBx(MBB); @@ -2349,9 +2349,8 @@ bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB, unsigned LastOpcode = 0; unsigned LastBytes = 0; unsigned NumMove = 0; - for (int i = Ops.size() - 1; i >= 0; --i) { + for (MachineInstr *Op : llvm::reverse(Ops)) { // Make sure each operation has the same kind. - MachineInstr *Op = Ops[i]; unsigned LSMOpcode = getLoadStoreMultipleOpcode(Op->getOpcode(), ARM_AM::ia); if (LastOpcode && LSMOpcode != LastOpcode) diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp index 3874db5792d6..f822672c4477 100644 --- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp +++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp @@ -251,10 +251,7 @@ namespace { SetVector<MachineInstr *> &Predicates = PredicatedInsts[MI]->Predicates; if (Exclusive && Predicates.size() != 1) return false; - for (auto *PredMI : Predicates) - if (isVCTP(PredMI)) - return true; - return false; + return llvm::any_of(Predicates, isVCTP); } // Is the VPST, controlling the block entry, predicated upon a VCTP. @@ -351,10 +348,7 @@ namespace { } bool containsVCTP() const { - for (auto *MI : Insts) - if (isVCTP(MI)) - return true; - return false; + return llvm::any_of(Insts, isVCTP); } unsigned size() const { return Insts.size(); } @@ -1334,8 +1328,8 @@ bool ARMLowOverheadLoops::ProcessLoop(MachineLoop *ML) { bool Changed = false; // Process inner loops first. - for (auto I = ML->begin(), E = ML->end(); I != E; ++I) - Changed |= ProcessLoop(*I); + for (MachineLoop *L : *ML) + Changed |= ProcessLoop(L); LLVM_DEBUG({ dbgs() << "ARM Loops: Processing loop containing:\n"; @@ -1699,7 +1693,7 @@ void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) { // If any of the instructions between the VCMP and VPST are predicated // then a different code path is expected to have merged the VCMP and // VPST already. - if (!std::any_of(++MachineBasicBlock::iterator(VCMP), + if (std::none_of(++MachineBasicBlock::iterator(VCMP), MachineBasicBlock::iterator(VPST), hasVPRUse) && RDA->hasSameReachingDef(VCMP, VPST, VCMP->getOperand(1).getReg()) && RDA->hasSameReachingDef(VCMP, VPST, VCMP->getOperand(2).getReg())) { diff --git a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h index 4077fc058217..d8d937055d23 100644 --- a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h +++ b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h @@ -289,7 +289,7 @@ public: return false; if (SignReturnAddressAll) return true; - return LRSpilled; + return SpillsLR; } bool branchTargetEnforcement() const { return BranchTargetEnforcement; } diff --git a/llvm/lib/Target/ARM/ARMRegisterInfo.td b/llvm/lib/Target/ARM/ARMRegisterInfo.td index 760a5a5a20cf..194d65cad8d1 100644 --- a/llvm/lib/Target/ARM/ARMRegisterInfo.td +++ b/llvm/lib/Target/ARM/ARMRegisterInfo.td @@ -211,6 +211,8 @@ def FPCXTS : ARMReg<15, "fpcxts">; def ZR : ARMReg<15, "zr">, DwarfRegNum<[15]>; +def RA_AUTH_CODE : ARMReg<12, "ra_auth_code">, DwarfRegNum<[143]>; + // Register classes. // // pc == Program Counter @@ -395,7 +397,7 @@ def CCR : RegisterClass<"ARM", [i32], 32, (add CPSR)> { } // MVE Condition code register. -def VCCR : RegisterClass<"ARM", [i32, v16i1, v8i1, v4i1], 32, (add VPR)> { +def VCCR : RegisterClass<"ARM", [i32, v16i1, v8i1, v4i1, v2i1], 32, (add VPR)> { // let CopyCost = -1; // Don't allow copying of status registers. } diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h index d51a888c951f..e61b90af31b0 100644 --- a/llvm/lib/Target/ARM/ARMSubtarget.h +++ b/llvm/lib/Target/ARM/ARMSubtarget.h @@ -18,6 +18,7 @@ #include "ARMConstantPoolValue.h" #include "ARMFrameLowering.h" #include "ARMISelLowering.h" +#include "ARMMachineFunctionInfo.h" #include "ARMSelectionDAGInfo.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetTransformInfo.h" @@ -534,6 +535,10 @@ protected: /// Selected instruction itineraries (one entry per itinerary class.) InstrItineraryData InstrItins; + /// NoBTIAtReturnTwice - Don't place a BTI instruction after + /// return-twice constructs (setjmp) + bool NoBTIAtReturnTwice = false; + /// Options passed via command line that could influence the target const TargetOptions &Options; @@ -840,6 +845,8 @@ public: /// to lr. This is always required on Thumb1-only targets, as the push and /// pop instructions can't access the high registers. bool splitFramePushPop(const MachineFunction &MF) const { + if (MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress()) + return true; return (getFramePointerReg() == ARM::R7 && MF.getTarget().Options.DisableFramePointerElim(MF)) || isThumb1Only(); @@ -948,6 +955,8 @@ public: bool hardenSlsRetBr() const { return HardenSlsRetBr; } bool hardenSlsBlr() const { return HardenSlsBlr; } bool hardenSlsNoComdat() const { return HardenSlsNoComdat; } + + bool getNoBTIAtReturnTwice() const { return NoBTIAtReturnTwice; } }; } // end namespace llvm diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp index 39f407ba7149..bfe078b06861 100644 --- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp +++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp @@ -137,21 +137,18 @@ public: int getFPReg() const { return FPReg; } void emitFnStartLocNotes() const { - for (Locs::const_iterator FI = FnStartLocs.begin(), FE = FnStartLocs.end(); - FI != FE; ++FI) - Parser.Note(*FI, ".fnstart was specified here"); + for (const SMLoc &Loc : FnStartLocs) + Parser.Note(Loc, ".fnstart was specified here"); } void emitCantUnwindLocNotes() const { - for (Locs::const_iterator UI = CantUnwindLocs.begin(), - UE = CantUnwindLocs.end(); UI != UE; ++UI) - Parser.Note(*UI, ".cantunwind was specified here"); + for (const SMLoc &Loc : CantUnwindLocs) + Parser.Note(Loc, ".cantunwind was specified here"); } void emitHandlerDataLocNotes() const { - for (Locs::const_iterator HI = HandlerDataLocs.begin(), - HE = HandlerDataLocs.end(); HI != HE; ++HI) - Parser.Note(*HI, ".handlerdata was specified here"); + for (const SMLoc &Loc : HandlerDataLocs) + Parser.Note(Loc, ".handlerdata was specified here"); } void emitPersonalityLocNotes() const { @@ -452,7 +449,8 @@ class ARMAsmParser : public MCTargetAsmParser { int tryParseRegister(); bool tryParseRegisterWithWriteBack(OperandVector &); int tryParseShiftRegister(OperandVector &); - bool parseRegisterList(OperandVector &, bool EnforceOrder = true); + bool parseRegisterList(OperandVector &, bool EnforceOrder = true, + bool AllowRAAC = false); bool parseMemory(OperandVector &); bool parseOperand(OperandVector &, StringRef Mnemonic); bool parsePrefix(ARMMCExpr::VariantKind &RefKind); @@ -2572,17 +2570,15 @@ public: void addRegListOperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); const SmallVectorImpl<unsigned> &RegList = getRegList(); - for (SmallVectorImpl<unsigned>::const_iterator - I = RegList.begin(), E = RegList.end(); I != E; ++I) - Inst.addOperand(MCOperand::createReg(*I)); + for (unsigned Reg : RegList) + Inst.addOperand(MCOperand::createReg(Reg)); } void addRegListWithAPSROperands(MCInst &Inst, unsigned N) const { assert(N == 1 && "Invalid number of operands!"); const SmallVectorImpl<unsigned> &RegList = getRegList(); - for (SmallVectorImpl<unsigned>::const_iterator - I = RegList.begin(), E = RegList.end(); I != E; ++I) - Inst.addOperand(MCOperand::createReg(*I)); + for (unsigned Reg : RegList) + Inst.addOperand(MCOperand::createReg(Reg)); } void addDPRRegListOperands(MCInst &Inst, unsigned N) const { @@ -4464,8 +4460,8 @@ insertNoDuplicates(SmallVectorImpl<std::pair<unsigned, unsigned>> &Regs, } /// Parse a register list. -bool ARMAsmParser::parseRegisterList(OperandVector &Operands, - bool EnforceOrder) { +bool ARMAsmParser::parseRegisterList(OperandVector &Operands, bool EnforceOrder, + bool AllowRAAC) { MCAsmParser &Parser = getParser(); if (Parser.getTok().isNot(AsmToken::LCurly)) return TokError("Token is not a Left Curly Brace"); @@ -4478,7 +4474,8 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, int Reg = tryParseRegister(); if (Reg == -1) return Error(RegLoc, "register expected"); - + if (!AllowRAAC && Reg == ARM::RA_AUTH_CODE) + return Error(RegLoc, "pseudo-register not allowed"); // The reglist instructions have at most 16 registers, so reserve // space for that many. int EReg = 0; @@ -4492,7 +4489,8 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, ++Reg; } const MCRegisterClass *RC; - if (ARMMCRegisterClasses[ARM::GPRRegClassID].contains(Reg)) + if (Reg == ARM::RA_AUTH_CODE || + ARMMCRegisterClasses[ARM::GPRRegClassID].contains(Reg)) RC = &ARMMCRegisterClasses[ARM::GPRRegClassID]; else if (ARMMCRegisterClasses[ARM::DPRRegClassID].contains(Reg)) RC = &ARMMCRegisterClasses[ARM::DPRRegClassID]; @@ -4513,11 +4511,15 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, while (Parser.getTok().is(AsmToken::Comma) || Parser.getTok().is(AsmToken::Minus)) { if (Parser.getTok().is(AsmToken::Minus)) { + if (Reg == ARM::RA_AUTH_CODE) + return Error(RegLoc, "pseudo-register not allowed"); Parser.Lex(); // Eat the minus. SMLoc AfterMinusLoc = Parser.getTok().getLoc(); int EndReg = tryParseRegister(); if (EndReg == -1) return Error(AfterMinusLoc, "register expected"); + if (EndReg == ARM::RA_AUTH_CODE) + return Error(AfterMinusLoc, "pseudo-register not allowed"); // Allow Q regs and just interpret them as the two D sub-registers. if (ARMMCRegisterClasses[ARM::QPRRegClassID].contains(EndReg)) EndReg = getDRegFromQReg(EndReg) + 1; @@ -4526,7 +4528,9 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, if (Reg == EndReg) continue; // The register must be in the same register class as the first. - if (!RC->contains(EndReg)) + if ((Reg == ARM::RA_AUTH_CODE && + RC != &ARMMCRegisterClasses[ARM::GPRRegClassID]) || + (Reg != ARM::RA_AUTH_CODE && !RC->contains(Reg))) return Error(AfterMinusLoc, "invalid register in register list"); // Ranges must go from low to high. if (MRI->getEncodingValue(Reg) > MRI->getEncodingValue(EndReg)) @@ -4551,13 +4555,15 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, Reg = tryParseRegister(); if (Reg == -1) return Error(RegLoc, "register expected"); + if (!AllowRAAC && Reg == ARM::RA_AUTH_CODE) + return Error(RegLoc, "pseudo-register not allowed"); // Allow Q regs and just interpret them as the two D sub-registers. bool isQReg = false; if (ARMMCRegisterClasses[ARM::QPRRegClassID].contains(Reg)) { Reg = getDRegFromQReg(Reg); isQReg = true; } - if (!RC->contains(Reg) && + if (Reg != ARM::RA_AUTH_CODE && !RC->contains(Reg) && RC->getID() == ARMMCRegisterClasses[ARM::GPRRegClassID].getID() && ARMMCRegisterClasses[ARM::GPRwithAPSRnospRegClassID].contains(Reg)) { // switch the register classes, as GPRwithAPSRnospRegClassID is a partial @@ -4577,7 +4583,9 @@ bool ARMAsmParser::parseRegisterList(OperandVector &Operands, continue; } // The register must be in the same register class as the first. - if (!RC->contains(Reg)) + if ((Reg == ARM::RA_AUTH_CODE && + RC != &ARMMCRegisterClasses[ARM::GPRRegClassID]) || + (Reg != ARM::RA_AUTH_CODE && !RC->contains(Reg))) return Error(RegLoc, "invalid register in register list"); // In most cases, the list must be monotonically increasing. An // exception is CLRM, which is order-independent anyway, so @@ -7106,13 +7114,12 @@ bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, return Error(Loc, "too many conditions on VPT instruction"); } unsigned Mask = 8; - for (unsigned i = ITMask.size(); i != 0; --i) { - char pos = ITMask[i - 1]; - if (pos != 't' && pos != 'e') { + for (char Pos : llvm::reverse(ITMask)) { + if (Pos != 't' && Pos != 'e') { return Error(Loc, "illegal IT block condition mask '" + ITMask + "'"); } Mask >>= 1; - if (ITMask[i - 1] == 'e') + if (Pos == 'e') Mask |= 8; } Operands.push_back(ARMOperand::CreateITMask(Mask, Loc)); @@ -11685,7 +11692,7 @@ bool ARMAsmParser::parseDirectiveRegSave(SMLoc L, bool IsVector) { SmallVector<std::unique_ptr<MCParsedAsmOperand>, 1> Operands; // Parse the register list - if (parseRegisterList(Operands) || + if (parseRegisterList(Operands, true, true) || parseToken(AsmToken::EndOfStatement, "unexpected token in directive")) return true; ARMOperand &Op = (ARMOperand &)*Operands[0]; diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp index 896b104e8d97..e060e59e3759 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp @@ -1289,34 +1289,65 @@ void ARMELFStreamer::emitPad(int64_t Offset) { PendingOffset -= Offset; } -void ARMELFStreamer::emitRegSave(const SmallVectorImpl<unsigned> &RegList, - bool IsVector) { - // Collect the registers in the register list - unsigned Count = 0; +static std::pair<unsigned, unsigned> +collectHWRegs(const MCRegisterInfo &MRI, unsigned Idx, + const SmallVectorImpl<unsigned> &RegList, bool IsVector, + uint32_t &Mask_) { uint32_t Mask = 0; - const MCRegisterInfo *MRI = getContext().getRegisterInfo(); - for (size_t i = 0; i < RegList.size(); ++i) { - unsigned Reg = MRI->getEncodingValue(RegList[i]); + unsigned Count = 0; + while (Idx > 0) { + unsigned Reg = RegList[Idx - 1]; + if (Reg == ARM::RA_AUTH_CODE) + break; + Reg = MRI.getEncodingValue(Reg); assert(Reg < (IsVector ? 32U : 16U) && "Register out of range"); unsigned Bit = (1u << Reg); if ((Mask & Bit) == 0) { Mask |= Bit; ++Count; } + --Idx; } - // Track the change the $sp offset: For the .save directive, the - // corresponding push instruction will decrease the $sp by (4 * Count). - // For the .vsave directive, the corresponding vpush instruction will - // decrease $sp by (8 * Count). - SPOffset -= Count * (IsVector ? 8 : 4); + Mask_ = Mask; + return {Idx, Count}; +} - // Emit the opcode - FlushPendingOffset(); - if (IsVector) - UnwindOpAsm.EmitVFPRegSave(Mask); - else - UnwindOpAsm.EmitRegSave(Mask); +void ARMELFStreamer::emitRegSave(const SmallVectorImpl<unsigned> &RegList, + bool IsVector) { + uint32_t Mask; + unsigned Idx, Count; + const MCRegisterInfo &MRI = *getContext().getRegisterInfo(); + + // Collect the registers in the register list. Issue unwinding instructions in + // three parts: ordinary hardware registers, return address authentication + // code pseudo register, the rest of the registers. The RA PAC is kept in an + // architectural register (usually r12), but we treat it as a special case in + // order to distinguish between that register containing RA PAC or a general + // value. + Idx = RegList.size(); + while (Idx > 0) { + std::tie(Idx, Count) = collectHWRegs(MRI, Idx, RegList, IsVector, Mask); + if (Count) { + // Track the change the $sp offset: For the .save directive, the + // corresponding push instruction will decrease the $sp by (4 * Count). + // For the .vsave directive, the corresponding vpush instruction will + // decrease $sp by (8 * Count). + SPOffset -= Count * (IsVector ? 8 : 4); + + // Emit the opcode + FlushPendingOffset(); + if (IsVector) + UnwindOpAsm.EmitVFPRegSave(Mask); + else + UnwindOpAsm.EmitRegSave(Mask); + } else if (Idx > 0 && RegList[Idx - 1] == ARM::RA_AUTH_CODE) { + --Idx; + SPOffset -= 4; + FlushPendingOffset(); + UnwindOpAsm.EmitRegSave(0); + } + } } void ARMELFStreamer::emitUnwindRaw(int64_t Offset, diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp index 781627c3c425..50f416b23db2 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp @@ -64,8 +64,11 @@ namespace { } // end anonymous namespace void UnwindOpcodeAssembler::EmitRegSave(uint32_t RegSave) { - if (RegSave == 0u) + if (RegSave == 0u) { + // That's the special case for RA PAC. + EmitInt8(ARM::EHABI::UNWIND_OPCODE_POP_RA_AUTH_CODE); return; + } // One byte opcode to save register r14 and r11-r4 if (RegSave & (1u << 4)) { diff --git a/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp b/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp index dc58b5427425..7e31ea77f4f5 100644 --- a/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp +++ b/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp @@ -366,7 +366,7 @@ bool MVETPAndVPTOptimisations::MergeLoopEnd(MachineLoop *ML) { while (!Worklist.empty()) { Register Reg = Worklist.pop_back_val(); for (MachineInstr &MI : MRI->use_nodbg_instructions(Reg)) { - if (count(ExpectedUsers, &MI)) + if (llvm::is_contained(ExpectedUsers, &MI)) continue; if (MI.getOpcode() != TargetOpcode::COPY || !MI.getOperand(0).getReg().isVirtual()) { diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp index 6a5bc9284266..0e6960bce32b 100644 --- a/llvm/lib/Target/ARM/MVETailPredication.cpp +++ b/llvm/lib/Target/ARM/MVETailPredication.cpp @@ -213,7 +213,8 @@ bool MVETailPredication::IsSafeActiveMask(IntrinsicInst *ActiveLaneMask, auto *TC = SE->getSCEV(TripCount); int VectorWidth = cast<FixedVectorType>(ActiveLaneMask->getType())->getNumElements(); - if (VectorWidth != 4 && VectorWidth != 8 && VectorWidth != 16) + if (VectorWidth != 2 && VectorWidth != 4 && VectorWidth != 8 && + VectorWidth != 16) return false; ConstantInt *ConstElemCount = nullptr; @@ -371,15 +372,10 @@ void MVETailPredication::InsertVCTPIntrinsic(IntrinsicInst *ActiveLaneMask, switch (VectorWidth) { default: llvm_unreachable("unexpected number of lanes"); + case 2: VCTPID = Intrinsic::arm_mve_vctp64; break; case 4: VCTPID = Intrinsic::arm_mve_vctp32; break; case 8: VCTPID = Intrinsic::arm_mve_vctp16; break; case 16: VCTPID = Intrinsic::arm_mve_vctp8; break; - - // FIXME: vctp64 currently not supported because the predicate - // vector wants to be <2 x i1>, but v2i1 is not a legal MVE - // type, so problems happen at isel time. - // Intrinsic::arm_mve_vctp64 exists for ACLE intrinsics - // purposes, but takes a v4i1 instead of a v2i1. } Function *VCTP = Intrinsic::getDeclaration(M, VCTPID); Value *VCTPCall = Builder.CreateCall(VCTP, Processed); diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp index 224c61b9f065..54e80a095dd4 100644 --- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp +++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp @@ -824,8 +824,8 @@ bool Thumb1FrameLowering::spillCalleeSavedRegisters( ARMRegSet CopyRegs; // Registers which can be used after pushing // LoRegs for saving HiRegs. - for (unsigned i = CSI.size(); i != 0; --i) { - unsigned Reg = CSI[i-1].getReg(); + for (const CalleeSavedInfo &I : llvm::reverse(CSI)) { + unsigned Reg = I.getReg(); if (ARM::tGPRRegClass.contains(Reg) || Reg == ARM::LR) { LoRegsToSave[Reg] = true; @@ -1021,8 +1021,7 @@ bool Thumb1FrameLowering::restoreCalleeSavedRegisters( BuildMI(MF, DL, TII.get(ARM::tPOP)).add(predOps(ARMCC::AL)); bool NeedsPop = false; - for (unsigned i = CSI.size(); i != 0; --i) { - CalleeSavedInfo &Info = CSI[i-1]; + for (CalleeSavedInfo &Info : llvm::reverse(CSI)) { unsigned Reg = Info.getReg(); // High registers (excluding lr) have already been dealt with @@ -1067,7 +1066,7 @@ bool Thumb1FrameLowering::restoreCalleeSavedRegisters( if (NeedsPop) MBB.insert(MI, &*MIB); else - MF.DeleteMachineInstr(MIB); + MF.deleteMachineInstr(MIB); return true; } diff --git a/llvm/lib/Target/AVR/AVRFrameLowering.cpp b/llvm/lib/Target/AVR/AVRFrameLowering.cpp index 672611ea2234..543d94875037 100644 --- a/llvm/lib/Target/AVR/AVRFrameLowering.cpp +++ b/llvm/lib/Target/AVR/AVRFrameLowering.cpp @@ -247,8 +247,8 @@ bool AVRFrameLowering::spillCalleeSavedRegisters( const TargetInstrInfo &TII = *STI.getInstrInfo(); AVRMachineFunctionInfo *AVRFI = MF.getInfo<AVRMachineFunctionInfo>(); - for (unsigned i = CSI.size(); i != 0; --i) { - unsigned Reg = CSI[i - 1].getReg(); + for (const CalleeSavedInfo &I : llvm::reverse(CSI)) { + unsigned Reg = I.getReg(); bool IsNotLiveIn = !MBB.isLiveIn(Reg); assert(TRI->getRegSizeInBits(*TRI->getMinimalPhysRegClass(Reg)) == 8 && diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.cpp b/llvm/lib/Target/AVR/AVRInstrInfo.cpp index 798d08393eae..51060018a5ca 100644 --- a/llvm/lib/Target/AVR/AVRInstrInfo.cpp +++ b/llvm/lib/Target/AVR/AVRInstrInfo.cpp @@ -571,8 +571,6 @@ void AVRInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, // See lib/CodeGen/RegisterRelaxation.cpp for details. // We end up here when a jump is too long for a RJMP instruction. BuildMI(&MBB, DL, get(AVR::JMPk)).addMBB(&NewDestBB); - - return; } } // end of namespace llvm diff --git a/llvm/lib/Target/BPF/BPFPreserveDIType.cpp b/llvm/lib/Target/BPF/BPFPreserveDIType.cpp index 0348e2200acb..36237b2fc4fd 100644 --- a/llvm/lib/Target/BPF/BPFPreserveDIType.cpp +++ b/llvm/lib/Target/BPF/BPFPreserveDIType.cpp @@ -93,8 +93,13 @@ static bool BPFPreserveDITypeImpl(Function &F) { Ty = DTy->getBaseType(); } - if (Ty->getName().empty()) - report_fatal_error("Empty type name for BTF_TYPE_ID_REMOTE reloc"); + if (Ty->getName().empty()) { + if (isa<DISubroutineType>(Ty)) + report_fatal_error( + "SubroutineType not supported for BTF_TYPE_ID_REMOTE reloc"); + else + report_fatal_error("Empty type name for BTF_TYPE_ID_REMOTE reloc"); + } MD = Ty; } diff --git a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp index ebc04b40d428..29b99a84a6cd 100644 --- a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp +++ b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp @@ -11,6 +11,7 @@ #include "MCTargetDesc/CSKYMCTargetDesc.h" #include "TargetInfo/CSKYTargetInfo.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/CodeGen/Register.h" #include "llvm/MC/MCContext.h" @@ -25,11 +26,24 @@ #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +using namespace llvm; + #define DEBUG_TYPE "csky-asm-parser" -using namespace llvm; +// Include the auto-generated portion of the compress emitter. +#define GEN_COMPRESS_INSTR +#include "CSKYGenCompressInstEmitter.inc" + +STATISTIC(CSKYNumInstrsCompressed, + "Number of C-SKY Compressed instructions emitted"); + +static cl::opt<bool> + EnableCompressedInst("enable-csky-asm-compressed-inst", cl::Hidden, + cl::init(false), + cl::desc("Enable C-SKY asm compressed instruction")); namespace { struct CSKYOperand; @@ -55,6 +69,10 @@ class CSKYAsmParser : public MCTargetAsmParser { bool ParseDirective(AsmToken DirectiveID) override; + // Helper to actually emit an instruction to the MCStreamer. Also, when + // possible, compression of the instruction is performed. + void emitToStreamer(MCStreamer &S, const MCInst &Inst); + OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override; @@ -264,12 +282,6 @@ public: bool isConstpool() const { return isConstPoolOp(); } bool isDataSymbol() const { return isConstPoolOp(); } - bool isSPOperand() const { - if (!isReg()) - return false; - return getReg() == CSKY::R14; - } - bool isPSRFlag() const { int64_t Imm; // Must be of 'immediate' type and a constant. @@ -755,10 +767,6 @@ bool CSKYAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, SMLoc ErrorLoc = ((CSKYOperand &)*Operands[ErrorInfo]).getStartLoc(); return Error(ErrorLoc, "register is out of range"); } - case Match_InvalidSPOperand: { - SMLoc ErrorLoc = ((CSKYOperand &)*Operands[ErrorInfo]).getStartLoc(); - return Error(ErrorLoc, "operand must be sp register"); - } case Match_RequiresSameSrcAndDst: { SMLoc ErrorLoc = ((CSKYOperand &)*Operands[ErrorInfo]).getStartLoc(); return Error(ErrorLoc, "src and dst operand must be same"); @@ -776,27 +784,62 @@ bool CSKYAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc, OperandVector &Operands, MCStreamer &Out) { - if (Inst.getOpcode() == CSKY::LDQ32 || Inst.getOpcode() == CSKY::STQ32) { + switch (Inst.getOpcode()) { + default: + break; + case CSKY::LDQ32: + case CSKY::STQ32: if (Inst.getOperand(1).getReg() != CSKY::R4 || Inst.getOperand(2).getReg() != CSKY::R7) { return Error(IDLoc, "Register sequence is not valid. 'r4-r7' expected"); } Inst.setOpcode(Inst.getOpcode() == CSKY::LDQ32 ? CSKY::LDM32 : CSKY::STM32); - Out.emitInstruction(Inst, getSTI()); - return false; - } else if (Inst.getOpcode() == CSKY::SEXT32 || - Inst.getOpcode() == CSKY::ZEXT32) { + break; + case CSKY::SEXT32: + case CSKY::ZEXT32: if (Inst.getOperand(2).getImm() < Inst.getOperand(3).getImm()) return Error(IDLoc, "msb must be greater or equal to lsb"); - } else if (Inst.getOpcode() == CSKY::INS32) { + break; + case CSKY::INS32: if (Inst.getOperand(3).getImm() < Inst.getOperand(4).getImm()) return Error(IDLoc, "msb must be greater or equal to lsb"); - } else if (Inst.getOpcode() == CSKY::IDLY32) { + break; + case CSKY::IDLY32: if (Inst.getOperand(0).getImm() > 32 || Inst.getOperand(0).getImm() < 0) return Error(IDLoc, "n must be in range [0,32]"); + break; + case CSKY::ADDC32: + case CSKY::SUBC32: + case CSKY::ADDC16: + case CSKY::SUBC16: + Inst.erase(std::next(Inst.begin())); + Inst.erase(std::prev(Inst.end())); + Inst.insert(std::next(Inst.begin()), MCOperand::createReg(CSKY::C)); + Inst.insert(Inst.end(), MCOperand::createReg(CSKY::C)); + break; + case CSKY::CMPNEI32: + case CSKY::CMPNEI16: + case CSKY::CMPNE32: + case CSKY::CMPNE16: + case CSKY::CMPHSI32: + case CSKY::CMPHSI16: + case CSKY::CMPHS32: + case CSKY::CMPHS16: + case CSKY::CMPLTI32: + case CSKY::CMPLTI16: + case CSKY::CMPLT32: + case CSKY::CMPLT16: + case CSKY::BTSTI32: + Inst.erase(Inst.begin()); + Inst.insert(Inst.begin(), MCOperand::createReg(CSKY::C)); + break; + case CSKY::MVCV32: + Inst.erase(std::next(Inst.begin())); + Inst.insert(Inst.end(), MCOperand::createReg(CSKY::C)); + break; } - Out.emitInstruction(Inst, getSTI()); + emitToStreamer(Out, Inst); return false; } @@ -1422,6 +1465,16 @@ OperandMatchResultTy CSKYAsmParser::tryParseRegister(unsigned &RegNo, bool CSKYAsmParser::ParseDirective(AsmToken DirectiveID) { return true; } +void CSKYAsmParser::emitToStreamer(MCStreamer &S, const MCInst &Inst) { + MCInst CInst; + bool Res = false; + if (EnableCompressedInst) + Res = compressInst(CInst, Inst, getSTI(), S.getContext()); + if (Res) + ++CSKYNumInstrsCompressed; + S.emitInstruction((Res ? CInst : Inst), getSTI()); +} + extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeCSKYAsmParser() { RegisterMCAsmParser<CSKYAsmParser> X(getTheCSKYTarget()); } diff --git a/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp b/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp index 1c38c5d1fde6..85129f78e726 100644 --- a/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp +++ b/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp @@ -30,6 +30,9 @@ using namespace llvm; #define DEBUG_TYPE "csky-asm-printer" +STATISTIC(CSKYNumInstrsCompressed, + "Number of C-SKY Compressed instructions emitted"); + CSKYAsmPrinter::CSKYAsmPrinter(llvm::TargetMachine &TM, std::unique_ptr<llvm::MCStreamer> Streamer) : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this) {} @@ -39,6 +42,16 @@ bool CSKYAsmPrinter::runOnMachineFunction(MachineFunction &MF) { return AsmPrinter::runOnMachineFunction(MF); } +#define GEN_COMPRESS_INSTR +#include "CSKYGenCompressInstEmitter.inc" +void CSKYAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) { + MCInst CInst; + bool Res = compressInst(CInst, Inst, *Subtarget, OutStreamer->getContext()); + if (Res) + ++CSKYNumInstrsCompressed; + AsmPrinter::EmitToStreamer(*OutStreamer, Res ? CInst : Inst); +} + // Simple pseudo-instructions have their lowering (with expansion to real // instructions) auto-generated. #include "CSKYGenMCPseudoLowering.inc" diff --git a/llvm/lib/Target/CSKY/CSKYAsmPrinter.h b/llvm/lib/Target/CSKY/CSKYAsmPrinter.h index f0f5d8657c04..b30311e0ca64 100644 --- a/llvm/lib/Target/CSKY/CSKYAsmPrinter.h +++ b/llvm/lib/Target/CSKY/CSKYAsmPrinter.h @@ -26,6 +26,8 @@ public: StringRef getPassName() const override { return "CSKY Assembly Printer"; } + void EmitToStreamer(MCStreamer &S, const MCInst &Inst); + /// tblgen'erated driver function for lowering simple MI->MC /// pseudo instructions. bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, diff --git a/llvm/lib/Target/CSKY/CSKYCallingConv.td b/llvm/lib/Target/CSKY/CSKYCallingConv.td index 87e2e6b9dc31..91102e3714df 100644 --- a/llvm/lib/Target/CSKY/CSKYCallingConv.td +++ b/llvm/lib/Target/CSKY/CSKYCallingConv.td @@ -79,4 +79,4 @@ def RetCC_CSKY_ABIV2_FP : CallingConv<[ CCIfType<[i32], CCAssignToReg<[R0, R1]>>, CCIfType<[f32], CCAssignToReg<[F0_32]>>, CCIfType<[f64], CCAssignToReg<[F0_64]>> -]>;
\ No newline at end of file +]>; diff --git a/llvm/lib/Target/CSKY/CSKYFrameLowering.cpp b/llvm/lib/Target/CSKY/CSKYFrameLowering.cpp index 9b22c95cfe21..3a8ee5713584 100644 --- a/llvm/lib/Target/CSKY/CSKYFrameLowering.cpp +++ b/llvm/lib/Target/CSKY/CSKYFrameLowering.cpp @@ -54,4 +54,4 @@ void CSKYFrameLowering::emitPrologue(MachineFunction &MF, void CSKYFrameLowering::emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const { // FIXME: Implement this when we have function calls -}
\ No newline at end of file +} diff --git a/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp b/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp index fc9ef8bfd9d9..8dc91904b8cc 100644 --- a/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp +++ b/llvm/lib/Target/CSKY/CSKYISelDAGToDAG.cpp @@ -40,6 +40,8 @@ public: } void Select(SDNode *N) override; + bool selectAddCarry(SDNode *N); + bool selectSubCarry(SDNode *N); #include "CSKYGenDAGISel.inc" }; @@ -60,7 +62,12 @@ void CSKYDAGToDAGISel::Select(SDNode *N) { switch (Opcode) { default: break; - // FIXME: Add selection nodes needed later. + case ISD::ADDCARRY: + IsSelected = selectAddCarry(N); + break; + case ISD::SUBCARRY: + IsSelected = selectSubCarry(N); + break; } if (IsSelected) @@ -70,6 +77,86 @@ void CSKYDAGToDAGISel::Select(SDNode *N) { SelectCode(N); } +bool CSKYDAGToDAGISel::selectAddCarry(SDNode *N) { + MachineSDNode *NewNode = nullptr; + auto Type0 = N->getValueType(0); + auto Type1 = N->getValueType(1); + auto Op0 = N->getOperand(0); + auto Op1 = N->getOperand(1); + auto Op2 = N->getOperand(2); + + SDLoc Dl(N); + + if (isNullConstant(Op2)) { + auto *CA = CurDAG->getMachineNode( + Subtarget->has2E3() ? CSKY::CLRC32 : CSKY::CLRC16, Dl, Type1); + NewNode = CurDAG->getMachineNode( + Subtarget->has2E3() ? CSKY::ADDC32 : CSKY::ADDC16, Dl, {Type0, Type1}, + {Op0, Op1, SDValue(CA, 0)}); + } else if (isOneConstant(Op2)) { + auto *CA = CurDAG->getMachineNode( + Subtarget->has2E3() ? CSKY::SETC32 : CSKY::SETC16, Dl, Type1); + NewNode = CurDAG->getMachineNode( + Subtarget->has2E3() ? CSKY::ADDC32 : CSKY::ADDC16, Dl, {Type0, Type1}, + {Op0, Op1, SDValue(CA, 0)}); + } else { + NewNode = CurDAG->getMachineNode(Subtarget->has2E3() ? CSKY::ADDC32 + : CSKY::ADDC16, + Dl, {Type0, Type1}, {Op0, Op1, Op2}); + } + ReplaceNode(N, NewNode); + return true; +} + +static SDValue InvertCarryFlag(const CSKYSubtarget *Subtarget, + SelectionDAG *DAG, SDLoc Dl, SDValue OldCarry) { + auto NewCarryReg = + DAG->getMachineNode(Subtarget->has2E3() ? CSKY::MVCV32 : CSKY::MVCV16, Dl, + MVT::i32, OldCarry); + auto NewCarry = + DAG->getMachineNode(Subtarget->hasE2() ? CSKY::BTSTI32 : CSKY::BTSTI16, + Dl, OldCarry.getValueType(), SDValue(NewCarryReg, 0), + DAG->getTargetConstant(0, Dl, MVT::i32)); + return SDValue(NewCarry, 0); +} + +bool CSKYDAGToDAGISel::selectSubCarry(SDNode *N) { + MachineSDNode *NewNode = nullptr; + auto Type0 = N->getValueType(0); + auto Type1 = N->getValueType(1); + auto Op0 = N->getOperand(0); + auto Op1 = N->getOperand(1); + auto Op2 = N->getOperand(2); + + SDLoc Dl(N); + + if (isNullConstant(Op2)) { + auto *CA = CurDAG->getMachineNode( + Subtarget->has2E3() ? CSKY::SETC32 : CSKY::SETC16, Dl, Type1); + NewNode = CurDAG->getMachineNode( + Subtarget->has2E3() ? CSKY::SUBC32 : CSKY::SUBC16, Dl, {Type0, Type1}, + {Op0, Op1, SDValue(CA, 0)}); + } else if (isOneConstant(Op2)) { + auto *CA = CurDAG->getMachineNode( + Subtarget->has2E3() ? CSKY::CLRC32 : CSKY::CLRC16, Dl, Type1); + NewNode = CurDAG->getMachineNode( + Subtarget->has2E3() ? CSKY::SUBC32 : CSKY::SUBC16, Dl, {Type0, Type1}, + {Op0, Op1, SDValue(CA, 0)}); + } else { + auto CarryIn = InvertCarryFlag(Subtarget, CurDAG, Dl, Op2); + NewNode = CurDAG->getMachineNode(Subtarget->has2E3() ? CSKY::SUBC32 + : CSKY::SUBC16, + Dl, {Type0, Type1}, {Op0, Op1, CarryIn}); + } + auto CarryOut = InvertCarryFlag(Subtarget, CurDAG, Dl, SDValue(NewNode, 1)); + + ReplaceUses(SDValue(N, 0), SDValue(NewNode, 0)); + ReplaceUses(SDValue(N, 1), CarryOut); + CurDAG->RemoveDeadNode(N); + + return true; +} + FunctionPass *llvm::createCSKYISelDag(CSKYTargetMachine &TM) { return new CSKYDAGToDAGISel(TM); } diff --git a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp index ac6d069e592c..a1f7cc685d4c 100644 --- a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp +++ b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp @@ -37,6 +37,46 @@ CSKYTargetLowering::CSKYTargetLowering(const TargetMachine &TM, // Register Class addRegisterClass(MVT::i32, &CSKY::GPRRegClass); + setOperationAction(ISD::ADDCARRY, MVT::i32, Legal); + setOperationAction(ISD::SUBCARRY, MVT::i32, Legal); + setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); + + setOperationAction(ISD::SREM, MVT::i32, Expand); + setOperationAction(ISD::UREM, MVT::i32, Expand); + setOperationAction(ISD::UDIVREM, MVT::i32, Expand); + setOperationAction(ISD::SDIVREM, MVT::i32, Expand); + setOperationAction(ISD::CTTZ, MVT::i32, Expand); + setOperationAction(ISD::CTPOP, MVT::i32, Expand); + setOperationAction(ISD::ROTR, MVT::i32, Expand); + setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand); + setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand); + setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand); + setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); + setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); + setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand); + setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); + setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); + setOperationAction(ISD::MULHS, MVT::i32, Expand); + setOperationAction(ISD::MULHU, MVT::i32, Expand); + + setLoadExtAction(ISD::EXTLOAD, MVT::i32, MVT::i1, Promote); + setLoadExtAction(ISD::SEXTLOAD, MVT::i32, MVT::i1, Promote); + setLoadExtAction(ISD::ZEXTLOAD, MVT::i32, MVT::i1, Promote); + + if (!Subtarget.hasE2()) { + setLoadExtAction(ISD::SEXTLOAD, MVT::i32, MVT::i8, Expand); + setLoadExtAction(ISD::SEXTLOAD, MVT::i32, MVT::i16, Expand); + setOperationAction(ISD::CTLZ, MVT::i32, Expand); + setOperationAction(ISD::BSWAP, MVT::i32, Expand); + } + + if (!Subtarget.has2E3()) { + setOperationAction(ISD::ABS, MVT::i32, Expand); + setOperationAction(ISD::BITREVERSE, MVT::i32, Expand); + setOperationAction(ISD::SDIV, MVT::i32, Expand); + setOperationAction(ISD::UDIV, MVT::i32, Expand); + } + // Compute derived properties from the register classes. computeRegisterProperties(STI.getRegisterInfo()); diff --git a/llvm/lib/Target/CSKY/CSKYInstrFormats16Instr.td b/llvm/lib/Target/CSKY/CSKYInstrFormats16Instr.td index 6d42bddcdd78..ea0761d97545 100644 --- a/llvm/lib/Target/CSKY/CSKYInstrFormats16Instr.td +++ b/llvm/lib/Target/CSKY/CSKYInstrFormats16Instr.td @@ -88,6 +88,19 @@ class R16_XZ_UNOP<bits<4> op, bits<2> sop, string opstr> : CSKY16Inst< let Inst{1, 0} = sop; } +class R16_Z_UNOP<bits<4> op, bits<2> sop, string opstr> : CSKY16Inst< + AddrModeNone, (outs sGPR:$rz), (ins sGPR:$rx), !strconcat(opstr, "\t$rz"), + []> { + bits<4> rz; + bits<4> rx; + let Inst{15, 14} = 0b01; + let Inst{13 - 10} = op; + let Inst{9 - 6} = rz; + let Inst{5 - 2} = rx; + let Inst{1, 0} = sop; + let Constraints = "$rz = $rx"; +} + class R16_XY_CMP<bits<2> sop, string opstr> : CSKY16Inst< AddrModeNone, (outs CARRY:$ca), (ins sGPR:$rx, sGPR:$ry), !strconcat(opstr, "\t$rx, $ry"), []> { @@ -146,7 +159,7 @@ class I16_X_CMP<bits<3> sop, string opstr, Operand Immoperand> : CSKY16Inst< } class I16_SP_IMM7<bits<3> sop, string opstr> : CSKY16Inst< - AddrModeNone, (outs SPOp:$sp2), (ins SPOp:$sp1, uimm7_2:$imm7), + AddrModeNone, (outs GPRSP:$sp2), (ins GPRSP:$sp1, uimm7_2:$imm7), !strconcat(opstr, "\t$sp2, $sp1, $imm7"), []> { bits<7> imm7; let Inst{15, 14} = 0b00; diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp b/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp index e12235cf9478..6fcb136cd99b 100644 --- a/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp +++ b/llvm/lib/Target/CSKY/CSKYInstrInfo.cpp @@ -11,6 +11,8 @@ //===----------------------------------------------------------------------===// #include "CSKYInstrInfo.h" +#include "CSKYMachineFunctionInfo.h" +#include "CSKYTargetMachine.h" #include "llvm/MC/MCContext.h" #define DEBUG_TYPE "csky-instr-info" @@ -23,3 +25,289 @@ using namespace llvm; CSKYInstrInfo::CSKYInstrInfo(CSKYSubtarget &STI) : CSKYGenInstrInfo(CSKY::ADJCALLSTACKDOWN, CSKY::ADJCALLSTACKUP), STI(STI) { } + +Register CSKYInstrInfo::movImm(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBI, + const DebugLoc &DL, int64_t Val, + MachineInstr::MIFlag Flag) const { + assert(isUInt<32>(Val) && "should be uint32"); + + MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); + + Register DstReg; + if (STI.hasE2()) { + DstReg = MRI.createVirtualRegister(&CSKY::GPRRegClass); + + if (isUInt<16>(Val)) { + BuildMI(MBB, MBBI, DL, get(CSKY::MOVI32), DstReg) + .addImm(Val & 0xFFFF) + .setMIFlags(Flag); + } else if (isShiftedUInt<16, 16>(Val)) { + BuildMI(MBB, MBBI, DL, get(CSKY::MOVIH32), DstReg) + .addImm((Val >> 16) & 0xFFFF) + .setMIFlags(Flag); + } else { + BuildMI(MBB, MBBI, DL, get(CSKY::MOVIH32), DstReg) + .addImm((Val >> 16) & 0xFFFF) + .setMIFlags(Flag); + BuildMI(MBB, MBBI, DL, get(CSKY::ORI32), DstReg) + .addReg(DstReg) + .addImm(Val & 0xFFFF) + .setMIFlags(Flag); + } + + } else { + DstReg = MRI.createVirtualRegister(&CSKY::mGPRRegClass); + if (isUInt<8>(Val)) { + BuildMI(MBB, MBBI, DL, get(CSKY::MOVI16), DstReg) + .addImm(Val & 0xFF) + .setMIFlags(Flag); + } else if (isUInt<16>(Val)) { + BuildMI(MBB, MBBI, DL, get(CSKY::MOVI16), DstReg) + .addImm((Val >> 8) & 0xFF) + .setMIFlags(Flag); + BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg) + .addReg(DstReg) + .addImm(8) + .setMIFlags(Flag); + if ((Val & 0xFF) != 0) + BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg) + .addReg(DstReg) + .addImm(Val & 0xFF) + .setMIFlags(Flag); + } else if (isUInt<24>(Val)) { + BuildMI(MBB, MBBI, DL, get(CSKY::MOVI16), DstReg) + .addImm((Val >> 16) & 0xFF) + .setMIFlags(Flag); + BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg) + .addReg(DstReg) + .addImm(8) + .setMIFlags(Flag); + if (((Val >> 8) & 0xFF) != 0) + BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg) + .addReg(DstReg) + .addImm((Val >> 8) & 0xFF) + .setMIFlags(Flag); + BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg) + .addReg(DstReg) + .addImm(8) + .setMIFlags(Flag); + if ((Val & 0xFF) != 0) + BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg) + .addReg(DstReg) + .addImm(Val & 0xFF) + .setMIFlags(Flag); + } else { + BuildMI(MBB, MBBI, DL, get(CSKY::MOVI16), DstReg) + .addImm((Val >> 24) & 0xFF) + .setMIFlags(Flag); + BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg) + .addReg(DstReg) + .addImm(8) + .setMIFlags(Flag); + if (((Val >> 16) & 0xFF) != 0) + BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg) + .addReg(DstReg) + .addImm((Val >> 16) & 0xFF) + .setMIFlags(Flag); + BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg) + .addReg(DstReg) + .addImm(8) + .setMIFlags(Flag); + if (((Val >> 8) & 0xFF) != 0) + BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg) + .addReg(DstReg) + .addImm((Val >> 8) & 0xFF) + .setMIFlags(Flag); + BuildMI(MBB, MBBI, DL, get(CSKY::LSLI16), DstReg) + .addReg(DstReg) + .addImm(8) + .setMIFlags(Flag); + if ((Val & 0xFF) != 0) + BuildMI(MBB, MBBI, DL, get(CSKY::ADDI16), DstReg) + .addReg(DstReg) + .addImm(Val & 0xFF) + .setMIFlags(Flag); + } + } + + return DstReg; +} + +unsigned CSKYInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, + int &FrameIndex) const { + switch (MI.getOpcode()) { + default: + return 0; + case CSKY::LD16B: + case CSKY::LD16H: + case CSKY::LD16W: + case CSKY::LD32B: + case CSKY::LD32BS: + case CSKY::LD32H: + case CSKY::LD32HS: + case CSKY::LD32W: + case CSKY::RESTORE_CARRY: + break; + } + + if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() && + MI.getOperand(2).getImm() == 0) { + FrameIndex = MI.getOperand(1).getIndex(); + return MI.getOperand(0).getReg(); + } + + return 0; +} + +unsigned CSKYInstrInfo::isStoreToStackSlot(const MachineInstr &MI, + int &FrameIndex) const { + switch (MI.getOpcode()) { + default: + return 0; + case CSKY::ST16B: + case CSKY::ST16H: + case CSKY::ST16W: + case CSKY::ST32B: + case CSKY::ST32H: + case CSKY::ST32W: + case CSKY::SPILL_CARRY: + break; + } + + if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() && + MI.getOperand(2).getImm() == 0) { + FrameIndex = MI.getOperand(1).getIndex(); + return MI.getOperand(0).getReg(); + } + + return 0; +} + +void CSKYInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, + MachineBasicBlock::iterator I, + Register SrcReg, bool IsKill, int FI, + const TargetRegisterClass *RC, + const TargetRegisterInfo *TRI) const { + DebugLoc DL; + if (I != MBB.end()) + DL = I->getDebugLoc(); + + MachineFunction &MF = *MBB.getParent(); + CSKYMachineFunctionInfo *CFI = MF.getInfo<CSKYMachineFunctionInfo>(); + MachineFrameInfo &MFI = MF.getFrameInfo(); + + unsigned Opcode = 0; + + if (CSKY::GPRRegClass.hasSubClassEq(RC)) { + Opcode = CSKY::ST32W; // Optimize for 16bit + } else if (CSKY::CARRYRegClass.hasSubClassEq(RC)) { + Opcode = CSKY::SPILL_CARRY; + CFI->setSpillsCR(); + } else { + llvm_unreachable("Unknown RegisterClass"); + } + + MachineMemOperand *MMO = MF.getMachineMemOperand( + MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOStore, + MFI.getObjectSize(FI), MFI.getObjectAlign(FI)); + + BuildMI(MBB, I, DL, get(Opcode)) + .addReg(SrcReg, getKillRegState(IsKill)) + .addFrameIndex(FI) + .addImm(0) + .addMemOperand(MMO); +} + +void CSKYInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, + MachineBasicBlock::iterator I, + Register DestReg, int FI, + const TargetRegisterClass *RC, + const TargetRegisterInfo *TRI) const { + DebugLoc DL; + if (I != MBB.end()) + DL = I->getDebugLoc(); + + MachineFunction &MF = *MBB.getParent(); + CSKYMachineFunctionInfo *CFI = MF.getInfo<CSKYMachineFunctionInfo>(); + MachineFrameInfo &MFI = MF.getFrameInfo(); + + unsigned Opcode = 0; + + if (CSKY::GPRRegClass.hasSubClassEq(RC)) { + Opcode = CSKY::LD32W; + } else if (CSKY::CARRYRegClass.hasSubClassEq(RC)) { + Opcode = CSKY::RESTORE_CARRY; + CFI->setSpillsCR(); + } else { + llvm_unreachable("Unknown RegisterClass"); + } + + MachineMemOperand *MMO = MF.getMachineMemOperand( + MachinePointerInfo::getFixedStack(MF, FI), MachineMemOperand::MOLoad, + MFI.getObjectSize(FI), MFI.getObjectAlign(FI)); + + BuildMI(MBB, I, DL, get(Opcode), DestReg) + .addFrameIndex(FI) + .addImm(0) + .addMemOperand(MMO); +} + +void CSKYInstrInfo::copyPhysReg(MachineBasicBlock &MBB, + MachineBasicBlock::iterator I, + const DebugLoc &DL, MCRegister DestReg, + MCRegister SrcReg, bool KillSrc) const { + + MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); + + if (CSKY::GPRRegClass.contains(SrcReg) && + CSKY::CARRYRegClass.contains(DestReg)) { + if (STI.hasE2()) { + BuildMI(MBB, I, DL, get(CSKY::BTSTI32), DestReg) + .addReg(SrcReg, getKillRegState(KillSrc)) + .addImm(0); + } else { + assert(SrcReg < CSKY::R8); + BuildMI(MBB, I, DL, get(CSKY::BTSTI16), DestReg) + .addReg(SrcReg, getKillRegState(KillSrc)) + .addImm(0); + } + return; + } + + if (CSKY::CARRYRegClass.contains(SrcReg) && + CSKY::GPRRegClass.contains(DestReg)) { + + if (STI.hasE2()) { + BuildMI(MBB, I, DL, get(CSKY::MVC32), DestReg) + .addReg(SrcReg, getKillRegState(KillSrc)); + } else { + assert(DestReg < CSKY::R16); + assert(DestReg < CSKY::R8); + BuildMI(MBB, I, DL, get(CSKY::MOVI16), DestReg).addImm(0); + BuildMI(MBB, I, DL, get(CSKY::ADDC16)) + .addReg(DestReg, RegState::Define) + .addReg(SrcReg, RegState::Define) + .addReg(DestReg, getKillRegState(true)) + .addReg(DestReg, getKillRegState(true)) + .addReg(SrcReg, getKillRegState(true)); + BuildMI(MBB, I, DL, get(CSKY::BTSTI16)) + .addReg(SrcReg, RegState::Define | getDeadRegState(KillSrc)) + .addReg(DestReg) + .addImm(0); + } + return; + } + + unsigned Opcode = 0; + if (CSKY::GPRRegClass.contains(DestReg, SrcReg)) + Opcode = CSKY::MOV32; + else { + LLVM_DEBUG(dbgs() << "src = " << SrcReg << ", dst = " << DestReg); + LLVM_DEBUG(I->dump()); + llvm_unreachable("Unknown RegisterClass"); + } + + BuildMI(MBB, I, DL, get(Opcode), DestReg) + .addReg(SrcReg, getKillRegState(KillSrc)); +} diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo.h b/llvm/lib/Target/CSKY/CSKYInstrInfo.h index 04be9da27b57..450641d96b74 100644 --- a/llvm/lib/Target/CSKY/CSKYInstrInfo.h +++ b/llvm/lib/Target/CSKY/CSKYInstrInfo.h @@ -29,6 +29,31 @@ protected: public: explicit CSKYInstrInfo(CSKYSubtarget &STI); + + unsigned isLoadFromStackSlot(const MachineInstr &MI, + int &FrameIndex) const override; + unsigned isStoreToStackSlot(const MachineInstr &MI, + int &FrameIndex) const override; + + void storeRegToStackSlot(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI, Register SrcReg, + bool IsKill, int FrameIndex, + const TargetRegisterClass *RC, + const TargetRegisterInfo *TRI) const override; + + void loadRegFromStackSlot(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MI, Register DestReg, + int FrameIndex, const TargetRegisterClass *RC, + const TargetRegisterInfo *TRI) const override; + + void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, + const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, + bool KillSrc) const override; + + // Materializes the given integer Val into DstReg. + Register movImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + const DebugLoc &DL, int64_t Val, + MachineInstr::MIFlag Flag = MachineInstr::NoFlags) const; }; } // namespace llvm diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo.td b/llvm/lib/Target/CSKY/CSKYInstrInfo.td index 9dda3159e446..30d9206eec68 100644 --- a/llvm/lib/Target/CSKY/CSKYInstrInfo.td +++ b/llvm/lib/Target/CSKY/CSKYInstrInfo.td @@ -52,6 +52,11 @@ class OImmAsmOperand<int width, string suffix = ""> : ImmAsmOperand<"O", width, suffix> { } +def to_tframeindex : SDNodeXForm<frameindex, [{ + auto FI = cast<FrameIndexSDNode>(N); + return CurDAG->getTargetFrameIndex(FI->getIndex(), TLI->getPointerTy(CurDAG->getDataLayout())); +}]>; + class oimm<int num> : Operand<i32>, ImmLeaf<i32, "return isUInt<"#num#">(Imm - 1);"> { let EncoderMethod = "getOImmOpValue"; @@ -166,9 +171,23 @@ def bare_symbol : Operand<iPTR> { let OperandType = "OPERAND_PCREL"; } -def oimm3 : oimm<3>; +def oimm3 : oimm<3> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isUInt<3>(Imm - 1); + return MCOp.isBareSymbolRef(); + }]; +} def oimm4 : oimm<4>; -def oimm5 : oimm<5>; +def oimm5 : oimm<5> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isUInt<5>(Imm - 1); + return MCOp.isBareSymbolRef(); + }]; +} def oimm6 : oimm<6>; def imm5_idly : Operand<i32>, ImmLeaf<i32, @@ -177,9 +196,30 @@ def imm5_idly : Operand<i32>, ImmLeaf<i32, let DecoderMethod = "decodeOImmOperand<5>"; } -def oimm8 : oimm<8>; -def oimm12 : oimm<12>; -def oimm16 : oimm<16>; +def oimm8 : oimm<8> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isUInt<8>(Imm - 1); + return MCOp.isBareSymbolRef(); + }]; +} +def oimm12 : oimm<12> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isUInt<12>(Imm - 1); + return MCOp.isBareSymbolRef(); + }]; +} +def oimm16 : oimm<16> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isUInt<16>(Imm - 1); + return MCOp.isBareSymbolRef(); + }]; +} def nimm12 : nimm<12>; @@ -195,28 +235,98 @@ def uimm2_jmpix : Operand<i32>, def uimm3 : uimm<3>; def uimm4 : uimm<4>; -def uimm5 : uimm<5>; +def uimm5 : uimm<5> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isShiftedUInt<5, 0>(Imm); + return MCOp.isBareSymbolRef(); + }]; +} def uimm5_msb_size : uimm<5> { let EncoderMethod = "getImmOpValueMSBSize"; } -def uimm5_1 : uimm<5, 1>; -def uimm5_2 : uimm<5, 2>; +def uimm5_1 : uimm<5, 1> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isShiftedUInt<5, 1>(Imm); + return MCOp.isBareSymbolRef(); + }]; +} +def uimm5_2 : uimm<5, 2> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isShiftedUInt<5, 2>(Imm); + return MCOp.isBareSymbolRef(); + }]; +} def uimm6 : uimm<6>; def uimm7 : uimm<7>; def uimm7_1 : uimm<7, 1>; -def uimm7_2 : uimm<7, 2>; +def uimm7_2 : uimm<7, 2>{ + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isShiftedUInt<7, 2>(Imm); + return MCOp.isBareSymbolRef(); + }]; +} def uimm7_3 : uimm<7, 3>; -def uimm8 : uimm<8>; -def uimm8_2 : uimm<8, 2>; +def uimm8 : uimm<8> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isShiftedUInt<8, 0>(Imm); + return MCOp.isBareSymbolRef(); + }]; +} +def uimm8_2 : uimm<8, 2> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isShiftedUInt<8, 2>(Imm); + return MCOp.isBareSymbolRef(); + }]; +} def uimm8_3 : uimm<8, 3>; def uimm8_8 : uimm<8, 8>; def uimm8_16 : uimm<8, 16>; def uimm8_24 : uimm<8, 24>; -def uimm12 : uimm<12>; -def uimm12_1 : uimm<12, 1>; -def uimm12_2 : uimm<12, 2>; -def uimm16 : uimm<16>; +def uimm12 : uimm<12> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isShiftedUInt<12, 0>(Imm); + return MCOp.isBareSymbolRef(); + }]; +} +def uimm12_1 : uimm<12, 1> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isShiftedUInt<12, 1>(Imm); + return MCOp.isBareSymbolRef(); + }]; +} +def uimm12_2 : uimm<12, 2> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isShiftedUInt<12, 2>(Imm); + return MCOp.isBareSymbolRef(); + }]; +} +def uimm16 : uimm<16> { + let MCOperandPredicate = [{ + int64_t Imm; + if (MCOp.evaluateAsConstantImm(Imm)) + return isShiftedUInt<16, 0>(Imm); + return MCOp.isBareSymbolRef(); + }]; +} def uimm16_8 : uimm<16, 8>; def uimm16_16 : uimm<16, 16>; def uimm20 : uimm<20>; @@ -642,11 +752,6 @@ def BSR32_BR : J<0x38, (outs), (ins call_symbol:$offset), "bsr32", []>{ let Defs = [ R15 ]; } -let Predicates = [iHasE2], isCodeGenOnly = 1 in { - def RTS32 : I_16_RET<0x6, 0xF, "rts32", [(CSKY_RET)]>; -} - - //===----------------------------------------------------------------------===// // Symbol address instructions. //===----------------------------------------------------------------------===// @@ -872,6 +977,102 @@ def TRAP32 : CSKY32Inst<AddrModeNone, 0x30, (outs), (ins uimm2:$imm2), "trap32 $ } +//===----------------------------------------------------------------------===// +// Instruction Patterns. +//===----------------------------------------------------------------------===// + +// Load & Store Patterns +multiclass LdPat<PatFrag LoadOp, ImmLeaf imm_type, Instruction Inst, ValueType Type> { + def : Pat<(Type (LoadOp GPR:$rs1)), (Inst GPR:$rs1, 0)>; + def : Pat<(Type (LoadOp (i32 frameindex:$rs1))), (Inst (i32 (to_tframeindex tframeindex:$rs1)), 0)>; + def : Pat<(Type (LoadOp (add GPR:$rs1, imm_type:$uimm))), + (Inst GPR:$rs1, imm_type:$uimm)>; + def : Pat<(Type (LoadOp (add frameindex:$rs1, imm_type:$uimm))), + (Inst (i32 (to_tframeindex tframeindex:$rs1)), imm_type:$uimm)>; + def : Pat<(Type (LoadOp (eqToAdd frameindex:$rs1, imm_type:$uimm))), + (Inst (i32 (to_tframeindex tframeindex:$rs1)), imm_type:$uimm)>; + def : Pat<(Type (LoadOp (add GPR:$rs1, tglobaladdr:$gd))), + (Inst GPR:$rs1, tglobaladdr:$gd)>; +} + +defm : LdPat<extloadi8, uimm12, LD32B, i32>; +defm : LdPat<zextloadi8, uimm12, LD32B, i32>; +let Predicates = [iHasE2] in { + defm : LdPat<sextloadi8, uimm12, LD32BS, i32>; +} +defm : LdPat<extloadi16, uimm12_1, LD32H, i32>; +defm : LdPat<zextloadi16, uimm12_1, LD32H, i32>; +let Predicates = [iHasE2] in { +defm : LdPat<sextloadi16, uimm12_1, LD32HS, i32>; +} +defm : LdPat<load, uimm12_2, LD32W, i32>; + +multiclass LdrPat<PatFrag LoadOp, Instruction Inst, ValueType Type> { + def : Pat<(Type (LoadOp (add GPR:$rs1, GPR:$rs2))), (Inst GPR:$rs1, GPR:$rs2, 0)>; + def : Pat<(Type (LoadOp (add GPR:$rs1, (shl GPR:$rs2, (i32 1))))), (Inst GPR:$rs1, GPR:$rs2, 1)>; + def : Pat<(Type (LoadOp (add GPR:$rs1, (shl GPR:$rs2, (i32 2))))), (Inst GPR:$rs1, GPR:$rs2, 2)>; + def : Pat<(Type (LoadOp (add GPR:$rs1, (shl GPR:$rs2, (i32 3))))), (Inst GPR:$rs1, GPR:$rs2, 3)>; +} + +let Predicates = [iHas2E3] in { + defm : LdrPat<zextloadi8, LDR32B, i32>; + defm : LdrPat<sextloadi8, LDR32BS, i32>; + defm : LdrPat<extloadi8, LDR32BS, i32>; + defm : LdrPat<zextloadi16, LDR32H, i32>; + defm : LdrPat<sextloadi16, LDR32HS, i32>; + defm : LdrPat<extloadi16, LDR32HS, i32>; + defm : LdrPat<load, LDR32W, i32>; +} + +multiclass StPat<PatFrag StoreOp, ValueType Type, ImmLeaf imm_type, Instruction Inst> { + def : Pat<(StoreOp Type:$rs2, GPR:$rs1), (Inst Type:$rs2, GPR:$rs1, 0)>; + def : Pat<(StoreOp Type:$rs2, frameindex:$rs1), (Inst Type:$rs2, (i32 (to_tframeindex tframeindex:$rs1)), 0)>; + def : Pat<(StoreOp Type:$rs2, (add GPR:$rs1, imm_type:$uimm12)), + (Inst Type:$rs2, GPR:$rs1, imm_type:$uimm12)>; + def : Pat<(StoreOp Type:$rs2, (add frameindex:$rs1, imm_type:$uimm12)), + (Inst Type:$rs2, (i32 (to_tframeindex tframeindex:$rs1)), imm_type:$uimm12)>; + def : Pat<(StoreOp Type:$rs2, (eqToAdd frameindex:$rs1, imm_type:$uimm12)), + (Inst Type:$rs2, (i32 (to_tframeindex tframeindex:$rs1)), imm_type:$uimm12)>; +} + +defm : StPat<truncstorei8, i32, uimm12, ST32B>; +defm : StPat<truncstorei16, i32, uimm12_1, ST32H>; +defm : StPat<store, i32, uimm12_2, ST32W>; + +multiclass StrPat<PatFrag StoreOp, ValueType Type, Instruction Inst> { + def : Pat<(StoreOp Type:$rz, (add GPR:$rs1, GPR:$rs2)), (Inst Type:$rz, GPR:$rs1, GPR:$rs2, 0)>; + def : Pat<(StoreOp Type:$rz, (add GPR:$rs1, (shl GPR:$rs2, (i32 1)))), (Inst Type:$rz, GPR:$rs1, GPR:$rs2, 1)>; + def : Pat<(StoreOp Type:$rz, (add GPR:$rs1, (shl GPR:$rs2, (i32 2)))), (Inst Type:$rz, GPR:$rs1, GPR:$rs2, 2)>; + def : Pat<(StoreOp Type:$rz, (add GPR:$rs1, (shl GPR:$rs2, (i32 3)))), (Inst Type:$rz, GPR:$rs1, GPR:$rs2, 3)>; +} + +let Predicates = [iHas2E3] in { + defm : StrPat<truncstorei8, i32, STR32B>; + defm : StrPat<truncstorei16, i32, STR32H>; + defm : StrPat<store, i32, STR32W>; + + // Sext & Zext Patterns + def : Pat<(sext_inreg GPR:$src, i1), (SEXT32 GPR:$src, 0, 0)>; + def : Pat<(and GPR:$src, 255), (ZEXT32 GPR:$src, 7, 0)>; + def : Pat<(and GPR:$src, 65535), (ZEXT32 GPR:$src, 15, 0)>; +} + +// Constant materialize patterns. +let Predicates = [iHasE2] in + def : Pat<(i32 imm:$imm), + (ORI32 (MOVIH32 (uimm32_hi16 imm:$imm)), (uimm32_lo16 imm:$imm))>; + + +// Other operations. +let Predicates = [iHasE2] in { + def : Pat<(rotl GPR:$rs1, GPR:$rs2), + (ROTL32 GPR:$rs1, (ANDI32 GPR:$rs2, 0x1f))>; + let Predicates = [iHas2E3] in { + def : Pat<(bitreverse GPR:$rx), (BREV32 GPR:$rx)>; + def : Pat<(bswap GPR:$rx), (REVB32 GPR:$rx)>; + } + def : Pat<(i32 (ctlz GPR:$rx)), (FF1 GPR:$rx)>; +} //===----------------------------------------------------------------------===// // Pseudo for assembly diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo16Instr.td b/llvm/lib/Target/CSKY/CSKYInstrInfo16Instr.td index c98f43622155..6a9dd03dfa1d 100644 --- a/llvm/lib/Target/CSKY/CSKYInstrInfo16Instr.td +++ b/llvm/lib/Target/CSKY/CSKYInstrInfo16Instr.td @@ -33,16 +33,6 @@ def br_symbol_16bit : Operand<iPTR> { let OperandType = "OPERAND_PCREL"; } -def SPOperand : AsmOperandClass { - let Name = "SPOperand"; - let RenderMethod = "addRegOperands"; - let DiagnosticType = !strconcat("Invalid", Name); -} - -def SPOp : RegisterOperand<GPR> { - let ParserMatchClass = SPOperand; -} - def constpool_symbol_16bit : Operand<iPTR> { let ParserMatchClass = Constpool; let EncoderMethod = @@ -83,7 +73,7 @@ let isCommutable = 1 in { def XOR16 : R16_XZ_BINOP<0b1011, 0b01, "xor16", BinOpFrag<(xor node:$LHS, node:$RHS)>>; def NOR16 : R16_XZ_BINOP<0b1011, 0b10, "nor16", BinOpFrag<(not (or node:$LHS, node:$RHS))>>; let isCodeGenOnly = 1 in - def NOT16 : R16_XZ_UNOP<0b1011, 0b10, "not16">; + def NOT16 : R16_Z_UNOP<0b1011, 0b10, "not16">; def MULT16 : R16_XZ_BINOP<0b1111, 0b00, "mult16", BinOpFrag<(mul node:$LHS, node:$RHS)>>; } def SUBU16XZ : R16_XZ_BINOP<0b1000, 0b10, "subu16", BinOpFrag<(sub node:$LHS, node:$RHS)>>; @@ -108,7 +98,7 @@ let Constraints = "$rZ = $rz", isReMaterializable = 1, isAsCheapAsAMove = 1 in { } let isAdd = 1 in -def ADDI16ZSP : I16_Z_8<0b011, (ins SPOp:$sp, uimm8_2:$imm8), +def ADDI16ZSP : I16_Z_8<0b011, (ins GPRSP:$sp, uimm8_2:$imm8), "addi16\t$rz, $sp, $imm8">; let isAdd = 1 in @@ -150,9 +140,9 @@ def ST16W : I16_XZ_LDST<AddrMode16W, 0b110, "st16.w", (outs), (ins mGPR:$rz, mGPR:$rx, uimm5_2:$imm)>; def LD16WSP : I16_ZSP_LDST<AddrMode16W, 0b011, "ld16.w", - (outs mGPR:$rz), (ins SPOp:$sp, uimm8_2:$addr)>; + (outs mGPR:$rz), (ins GPRSP:$sp, uimm8_2:$addr)>; def ST16WSP : I16_ZSP_LDST<AddrMode16W, 0b111, "st16.w", - (outs), (ins mGPR:$rz, SPOp:$sp, uimm8_2:$addr)>; + (outs), (ins mGPR:$rz, GPRSP:$sp, uimm8_2:$addr)>; //===----------------------------------------------------------------------===// // Compare instructions. @@ -450,3 +440,150 @@ def JBF16 : JBranchPseudo<(outs), let mayLoad = 1, Size = 2, isCodeGenOnly = 0 in def PseudoLRW16 : CSKYPseudo<(outs mGPR:$rz), (ins bare_symbol:$src), "lrw16 $rz, $src", []>; + + +//===----------------------------------------------------------------------===// +// Compress Instruction tablegen backend. +//===----------------------------------------------------------------------===// + +def : CompressPat<(ADDU32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (ADDU16XZ sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(ADDU32 sGPR:$rd, sGPR:$rs1, sGPR:$rd), + (ADDU16XZ sGPR:$rd, sGPR:$rs1)>; +def : CompressPat<(ADDU32 mGPR:$rd, mGPR:$rs1, mGPR:$rs2), + (ADDU16 mGPR:$rd, mGPR:$rs1, mGPR:$rs2)>; +def : CompressPat<(SUBU32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (SUBU16XZ sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(SUBU32 mGPR:$rd, mGPR:$rs1, mGPR:$rs2), + (SUBU16 mGPR:$rd, mGPR:$rs1, mGPR:$rs2)>; + +def : CompressPat< + (ADDC32 sGPR:$rd, CARRY:$cout, sGPR:$rd, sGPR:$rs2, CARRY:$cout), + (ADDC16 sGPR:$rd, CARRY:$cout, sGPR:$rs2, CARRY:$cout) + >; +def : CompressPat< + (SUBC32 sGPR:$rd, CARRY:$cout, sGPR:$rd, sGPR:$rs2, CARRY:$cout), + (SUBC16 sGPR:$rd, CARRY:$cout, sGPR:$rs2, CARRY:$cout) + >; + +def : CompressPat<(ADDI32 mGPR:$rd, mGPR:$rs, oimm3:$imm), + (ADDI16XZ mGPR:$rd, mGPR:$rs, oimm3:$imm)>; +def : CompressPat<(SUBI32 mGPR:$rd, mGPR:$rs, oimm3:$imm), + (SUBI16XZ mGPR:$rd, mGPR:$rs, oimm3:$imm)>; + +def : CompressPat<(ADDI32 mGPR:$rd, mGPR:$rd, oimm8:$imm), + (ADDI16 mGPR:$rd, oimm8:$imm)>; +def : CompressPat<(SUBI32 mGPR:$rd, mGPR:$rd, oimm8:$imm), + (SUBI16 mGPR:$rd, oimm8:$imm)>; + +def : CompressPat<(ADDI32 GPRSP:$sp, GPRSP:$sp, uimm7_2:$imm), + (ADDI16SPSP GPRSP:$sp, GPRSP:$sp, uimm7_2:$imm)>; +def : CompressPat<(SUBI32 GPRSP:$sp, GPRSP:$sp, uimm7_2:$imm), + (SUBI16SPSP GPRSP:$sp, GPRSP:$sp, uimm7_2:$imm)>; + +def : CompressPat<(ADDI32 mGPR:$rd, GPRSP:$sp, uimm8_2:$imm), + (ADDI16ZSP mGPR:$rd, GPRSP:$sp, uimm8_2:$imm)>; + +def : CompressPat<(MULT32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (MULT16 sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(MULT32 sGPR:$rd, sGPR:$rs1, sGPR:$rd), + (MULT16 sGPR:$rd, sGPR:$rs1)>; +def : CompressPat<(AND32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (AND16 sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(AND32 sGPR:$rd, sGPR:$rs1, sGPR:$rd), + (AND16 sGPR:$rd, sGPR:$rs1)>; +def : CompressPat<(OR32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (OR16 sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(OR32 sGPR:$rd, sGPR:$rs1, sGPR:$rd), + (OR16 sGPR:$rd, sGPR:$rs1)>; +def : CompressPat<(XOR32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (XOR16 sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(XOR32 sGPR:$rd, sGPR:$rs1, sGPR:$rd), + (XOR16 sGPR:$rd, sGPR:$rs1)>; + +def : CompressPat<(ANDN32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (ANDN16 sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(NOR32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (NOR16 sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(LSL32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (LSL16 sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(LSR32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (LSR16 sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(ASR32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (ASR16 sGPR:$rd, sGPR:$rs2)>; +def : CompressPat<(ROTL32 sGPR:$rd, sGPR:$rd, sGPR:$rs2), + (ROTL16 sGPR:$rd, sGPR:$rs2)>; + +def : CompressPat<(NOT32 sGPR:$rd, sGPR:$rd), + (NOT16 sGPR:$rd)>; + +let Predicates = [iHas2E3] in +def : CompressPat<(REVB32 sGPR:$rd, sGPR:$rs), + (REVB16 sGPR:$rd, sGPR:$rs)>; + +def : CompressPat<(LSLI32 mGPR:$rd, mGPR:$rs, uimm5:$imm), + (LSLI16 mGPR:$rd, mGPR:$rs, uimm5:$imm)>; +def : CompressPat<(LSRI32 mGPR:$rd, mGPR:$rs, uimm5:$imm), + (LSRI16 mGPR:$rd, mGPR:$rs, uimm5:$imm)>; +def : CompressPat<(ASRI32 mGPR:$rd, mGPR:$rs, uimm5:$imm), + (ASRI16 mGPR:$rd, mGPR:$rs, uimm5:$imm)>; + +def : CompressPat<(CMPHS32 CARRY:$ca, sGPR:$rs1, sGPR:$rs2), + (CMPHS16 CARRY:$ca, sGPR:$rs1, sGPR:$rs2)>; +def : CompressPat<(CMPLT32 CARRY:$ca, sGPR:$rs1, sGPR:$rs2), + (CMPLT16 CARRY:$ca, sGPR:$rs1, sGPR:$rs2)>; +def : CompressPat<(CMPNE32 CARRY:$ca, sGPR:$rs1, sGPR:$rs2), + (CMPNE16 CARRY:$ca, sGPR:$rs1, sGPR:$rs2)>; + +def : CompressPat<(CMPHSI32 CARRY:$ca, mGPR:$rs, oimm5:$imm), + (CMPHSI16 CARRY:$ca, mGPR:$rs, oimm5:$imm)>; +def : CompressPat<(CMPLTI32 CARRY:$ca, mGPR:$rs, oimm5:$imm), + (CMPLTI16 CARRY:$ca, mGPR:$rs, oimm5:$imm)>; +def : CompressPat<(CMPNEI32 CARRY:$ca, mGPR:$rs, uimm5:$imm), + (CMPNEI16 CARRY:$ca, mGPR:$rs, uimm5:$imm)>; + +def : CompressPat<(JSR32 sGPR:$rd), + (JSR16 sGPR:$rd)>; + + +def : CompressPat<(MVCV32 sGPR:$rd, CARRY:$ca), + (MVCV16 sGPR:$rd, CARRY:$ca)>; +def : CompressPat<(MOV32 sGPR:$rd, sGPR:$ca), + (MOV16 sGPR:$rd, sGPR:$ca)>; +def : CompressPat<(MOVI32 mGPR:$rd, uimm8:$imm), + (MOVI16 mGPR:$rd, uimm8:$imm)>; + +def : CompressPat<(LD32B mGPR:$rd, mGPR:$rs, uimm5:$imm), + (LD16B mGPR:$rd, mGPR:$rs, uimm5:$imm)>; +def : CompressPat<(LD32H mGPR:$rd, mGPR:$rs, uimm5_1:$imm), + (LD16H mGPR:$rd, mGPR:$rs, uimm5_1:$imm)>; +def : CompressPat<(LD32W mGPR:$rd, mGPR:$rs, uimm5_2:$imm), + (LD16W mGPR:$rd, mGPR:$rs, uimm5_2:$imm)>; +def : CompressPat<(LD32W mGPR:$rd, GPRSP:$sp, uimm8_2:$imm), + (LD16WSP mGPR:$rd, GPRSP:$sp, uimm8_2:$imm)>; + +def : CompressPat<(ST32B mGPR:$rd, mGPR:$rs, uimm5:$imm), + (ST16B mGPR:$rd, mGPR:$rs, uimm5:$imm)>; +def : CompressPat<(ST32H mGPR:$rd, mGPR:$rs, uimm5_1:$imm), + (ST16H mGPR:$rd, mGPR:$rs, uimm5_1:$imm)>; +def : CompressPat<(ST32W mGPR:$rd, mGPR:$rs, uimm5_2:$imm), + (ST16W mGPR:$rd, mGPR:$rs, uimm5_2:$imm)>; +def : CompressPat<(ST32W mGPR:$rd, GPRSP:$sp, uimm8_2:$imm), + (ST16WSP mGPR:$rd, GPRSP:$sp, uimm8_2:$imm)>; + +let Predicates = [HasBTST16] in +def : CompressPat<(BTSTI32 CARRY:$ca, mGPR:$rs, uimm5:$imm), + (BTSTI16 CARRY:$ca, mGPR:$rs, uimm5:$imm)>; +def : CompressPat<(BCLRI32 mGPR:$rd, mGPR:$rd, uimm5:$imm), + (BCLRI16 mGPR:$rd, uimm5:$imm)>; +def : CompressPat<(BSETI32 mGPR:$rd, mGPR:$rd, uimm5:$imm), + (BSETI16 mGPR:$rd, uimm5:$imm)>; + +def : CompressPat<(ZEXTB32 sGPR:$rd, sGPR:$rs), + (ZEXTB16 sGPR:$rd, sGPR:$rs)>; +def : CompressPat<(ZEXTH32 sGPR:$rd, sGPR:$rs), + (ZEXTH16 sGPR:$rd, sGPR:$rs)>; +def : CompressPat<(SEXTB32 sGPR:$rd, sGPR:$rs), + (SEXTB16 sGPR:$rd, sGPR:$rs)>; +def : CompressPat<(SEXTH32 sGPR:$rd, sGPR:$rs), + (SEXTH16 sGPR:$rd, sGPR:$rs)>; diff --git a/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp b/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp index c42a56bfb04e..7e0b9bcd7549 100644 --- a/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp +++ b/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp @@ -114,4 +114,4 @@ bool CSKYMCInstLower::lowerOperand(const MachineOperand &MO, break; } return true; -}
\ No newline at end of file +} diff --git a/llvm/lib/Target/CSKY/CSKYRegisterInfo.cpp b/llvm/lib/Target/CSKY/CSKYRegisterInfo.cpp index a1d45fea534b..57b6ae3c27b5 100644 --- a/llvm/lib/Target/CSKY/CSKYRegisterInfo.cpp +++ b/llvm/lib/Target/CSKY/CSKYRegisterInfo.cpp @@ -88,8 +88,187 @@ CSKYRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { return CSR_I32_SaveList; } +static bool IsLegalOffset(const CSKYInstrInfo *TII, MachineInstr *MI, + int &Offset) { + const MCInstrDesc &Desc = MI->getDesc(); + unsigned AddrMode = (Desc.TSFlags & CSKYII::AddrModeMask); + unsigned i = 0; + for (; !MI->getOperand(i).isFI(); ++i) { + assert(i + 1 < MI->getNumOperands() && + "Instr doesn't have FrameIndex operand!"); + } + + if (MI->getOpcode() == CSKY::ADDI32) { + if (!isUInt<12>(std::abs(Offset) - 1)) + return false; + if (Offset < 0) { + MI->setDesc(TII->get(CSKY::SUBI32)); + Offset = -Offset; + } + + return true; + } + + if (MI->getOpcode() == CSKY::ADDI16XZ) + return false; + + if (Offset < 0) + return false; + + unsigned NumBits = 0; + unsigned Scale = 1; + switch (AddrMode) { + case CSKYII::AddrMode32B: + Scale = 1; + NumBits = 12; + break; + case CSKYII::AddrMode32H: + Scale = 2; + NumBits = 12; + break; + case CSKYII::AddrMode32WD: + Scale = 4; + NumBits = 12; + break; + case CSKYII::AddrMode16B: + Scale = 1; + NumBits = 5; + break; + case CSKYII::AddrMode16H: + Scale = 2; + NumBits = 5; + break; + case CSKYII::AddrMode16W: + Scale = 4; + NumBits = 5; + break; + case CSKYII::AddrMode32SDF: + Scale = 4; + NumBits = 8; + break; + default: + llvm_unreachable("Unsupported addressing mode!"); + } + + // Cannot encode offset. + if ((Offset & (Scale - 1)) != 0) + return false; + + unsigned Mask = (1 << NumBits) - 1; + if ((unsigned)Offset <= Mask * Scale) + return true; + + // Offset out of range. + return false; +} + void CSKYRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, int SPAdj, unsigned FIOperandNum, RegScavenger *RS) const { assert(SPAdj == 0 && "Unexpected non-zero SPAdj value"); -}
\ No newline at end of file + + MachineInstr *MI = &*II; + MachineBasicBlock &MBB = *MI->getParent(); + MachineFunction &MF = *MI->getParent()->getParent(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + const CSKYInstrInfo *TII = MF.getSubtarget<CSKYSubtarget>().getInstrInfo(); + DebugLoc DL = MI->getDebugLoc(); + const CSKYSubtarget &STI = MF.getSubtarget<CSKYSubtarget>(); + + switch (MI->getOpcode()) { + default: + break; + case CSKY::RESTORE_CARRY: { + Register NewReg = STI.hasE2() + ? MRI.createVirtualRegister(&CSKY::GPRRegClass) + : MRI.createVirtualRegister(&CSKY::mGPRRegClass); + + auto *Temp = BuildMI(MBB, II, DL, TII->get(CSKY::LD32W), NewReg) + .add(MI->getOperand(1)) + .add(MI->getOperand(2)) + .getInstr(); + + BuildMI(MBB, II, DL, TII->get(STI.hasE2() ? CSKY::BTSTI32 : CSKY::BTSTI16), + MI->getOperand(0).getReg()) + .addReg(NewReg, getKillRegState(true)) + .addImm(0); + + MI = Temp; + + MBB.erase(II); + break; + } + case CSKY::SPILL_CARRY: { + Register NewReg; + if (STI.hasE2()) { + NewReg = MRI.createVirtualRegister(&CSKY::GPRRegClass); + BuildMI(MBB, II, DL, TII->get(CSKY::MVC32), NewReg) + .add(MI->getOperand(0)); + } else { + NewReg = MRI.createVirtualRegister(&CSKY::mGPRRegClass); + BuildMI(MBB, II, DL, TII->get(CSKY::MOVI16), NewReg).addImm(0); + BuildMI(MBB, II, DL, TII->get(CSKY::ADDC16)) + .addReg(NewReg, RegState::Define) + .addReg(MI->getOperand(0).getReg(), RegState::Define) + .addReg(NewReg, getKillRegState(true)) + .addReg(NewReg, getKillRegState(true)) + .addReg(MI->getOperand(0).getReg()); + + BuildMI(MBB, II, DL, TII->get(CSKY::BTSTI16), MI->getOperand(0).getReg()) + .addReg(NewReg) + .addImm(0); + } + + MI = BuildMI(MBB, II, DL, TII->get(CSKY::ST32W)) + .addReg(NewReg, getKillRegState(true)) + .add(MI->getOperand(1)) + .add(MI->getOperand(2)) + .getInstr(); + + MBB.erase(II); + + break; + } + } + + int FrameIndex = MI->getOperand(FIOperandNum).getIndex(); + Register FrameReg; + int Offset = getFrameLowering(MF) + ->getFrameIndexReference(MF, FrameIndex, FrameReg) + .getFixed() + + MI->getOperand(FIOperandNum + 1).getImm(); + + if (!isInt<32>(Offset)) + report_fatal_error( + "Frame offsets outside of the signed 32-bit range not supported"); + + bool FrameRegIsKill = false; + MachineBasicBlock::iterator NewII(MI); + if (!IsLegalOffset(TII, MI, Offset)) { + assert(isInt<32>(Offset) && "Int32 expected"); + // The offset won't fit in an immediate, so use a scratch register instead + // Modify Offset and FrameReg appropriately + assert(Offset >= 0); + Register ScratchReg = TII->movImm(MBB, NewII, DL, Offset); + BuildMI(MBB, NewII, DL, + TII->get(STI.hasE2() ? CSKY::ADDU32 : CSKY::ADDU16XZ), ScratchReg) + .addReg(ScratchReg, RegState::Kill) + .addReg(FrameReg); + + Offset = 0; + FrameReg = ScratchReg; + FrameRegIsKill = true; + } + + if (Offset == 0 && + (MI->getOpcode() == CSKY::ADDI32 || MI->getOpcode() == CSKY::ADDI16XZ)) { + MI->setDesc(TII->get(TargetOpcode::COPY)); + MI->getOperand(FIOperandNum) + .ChangeToRegister(FrameReg, false, false, FrameRegIsKill); + MI->RemoveOperand(FIOperandNum + 1); + } else { + MI->getOperand(FIOperandNum) + .ChangeToRegister(FrameReg, false, false, FrameRegIsKill); + MI->getOperand(FIOperandNum + 1).ChangeToImmediate(Offset); + } +} diff --git a/llvm/lib/Target/CSKY/CSKYRegisterInfo.h b/llvm/lib/Target/CSKY/CSKYRegisterInfo.h index 779ea6493c7e..5b3b62ec0db2 100644 --- a/llvm/lib/Target/CSKY/CSKYRegisterInfo.h +++ b/llvm/lib/Target/CSKY/CSKYRegisterInfo.h @@ -38,6 +38,18 @@ public: void eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj, unsigned FIOperandNum, RegScavenger *RS) const override; + + bool requiresFrameIndexScavenging(const MachineFunction &MF) const override { + return true; + } + + bool requiresRegisterScavenging(const MachineFunction &MF) const override { + return true; + } + + bool useFPForScavengingIndex(const MachineFunction &MF) const override { + return false; + } }; } // namespace llvm diff --git a/llvm/lib/Target/CSKY/CSKYRegisterInfo.td b/llvm/lib/Target/CSKY/CSKYRegisterInfo.td index 7548c22bb2c5..ade5c7f795af 100644 --- a/llvm/lib/Target/CSKY/CSKYRegisterInfo.td +++ b/llvm/lib/Target/CSKY/CSKYRegisterInfo.td @@ -168,6 +168,11 @@ def mGPR : RegisterClass<"CSKY", [i32], 32, let Size = 32; } +// Register class for SP only. +def GPRSP : RegisterClass<"CSKY", [i32], 32, (add R14)> { + let Size = 32; +} + def GPRPair : RegisterClass<"CSKY", [untyped], 32, (add GPRTuple)> { let Size = 64; } diff --git a/llvm/lib/Target/Hexagon/Hexagon.td b/llvm/lib/Target/Hexagon/Hexagon.td index 7518fd774a48..ae811b30434d 100644 --- a/llvm/lib/Target/Hexagon/Hexagon.td +++ b/llvm/lib/Target/Hexagon/Hexagon.td @@ -29,6 +29,8 @@ def ProcTinyCore: SubtargetFeature<"tinycore", "HexagonProcFamily", // Hexagon ISA Extensions def ExtensionZReg: SubtargetFeature<"zreg", "UseZRegOps", "true", "Hexagon ZReg extension instructions">; +def ExtensionHVXQFloat: SubtargetFeature<"hvx-qfloat", "UseHVXQFloatOps", + "true", "Hexagon HVX QFloating point instructions">; def ExtensionHVX: SubtargetFeature<"hvx", "HexagonHVXVersion", "Hexagon::ArchEnum::V60", "Hexagon HVX instructions">; @@ -52,6 +54,10 @@ def ExtensionHVXV68: SubtargetFeature<"hvxv68", "HexagonHVXVersion", "Hexagon::ArchEnum::V68", "Hexagon HVX instructions", [ExtensionHVXV60, ExtensionHVXV62, ExtensionHVXV65, ExtensionHVXV66, ExtensionHVXV67]>; +def ExtensionHVXV69: SubtargetFeature<"hvxv69", "HexagonHVXVersion", + "Hexagon::ArchEnum::V69", "Hexagon HVX instructions", + [ExtensionHVXV60, ExtensionHVXV62, ExtensionHVXV65, ExtensionHVXV66, + ExtensionHVXV67, ExtensionHVXV68]>; def ExtensionHVX64B: SubtargetFeature<"hvx-length64b", "UseHVX64BOps", "true", "Hexagon HVX 64B instructions", [ExtensionHVX]>; @@ -61,6 +67,9 @@ def ExtensionHVX128B: SubtargetFeature<"hvx-length128b", "UseHVX128BOps", def ExtensionAudio: SubtargetFeature<"audio", "UseAudioOps", "true", "Hexagon Audio extension instructions">; +def ExtensionHVXIEEEFP: SubtargetFeature<"hvx-ieee-fp", "UseHVXIEEEFPOps", + "true", "Hexagon HVX IEEE floating point instructions">; + def FeatureCompound: SubtargetFeature<"compound", "UseCompound", "true", "Use compound instructions">; def FeaturePackets: SubtargetFeature<"packets", "UsePackets", "true", @@ -88,6 +97,8 @@ def FeatureReservedR19: SubtargetFeature<"reserved-r19", "ReservedR19", def FeatureNoreturnStackElim: SubtargetFeature<"noreturn-stack-elim", "NoreturnStackElim", "true", "Eliminate stack allocation in a noreturn function when possible">; +def FeatureCabac: SubtargetFeature<"cabac", "UseCabac", "false", + "Emit the CABAC instruction">; //===----------------------------------------------------------------------===// // Hexagon Instruction Predicate Definitions. @@ -112,6 +123,8 @@ def UseHVXV67 : Predicate<"HST->useHVXV67Ops()">, AssemblerPredicate<(all_of ExtensionHVXV67)>; def UseHVXV68 : Predicate<"HST->useHVXV68Ops()">, AssemblerPredicate<(all_of ExtensionHVXV68)>; +def UseHVXV69 : Predicate<"HST->useHVXV69Ops()">, + AssemblerPredicate<(all_of ExtensionHVXV69)>; def UseAudio : Predicate<"HST->useAudioOps()">, AssemblerPredicate<(all_of ExtensionAudio)>; def UseZReg : Predicate<"HST->useZRegOps()">, @@ -119,6 +132,11 @@ def UseZReg : Predicate<"HST->useZRegOps()">, def UseCompound : Predicate<"HST->useCompound()">; def HasPreV65 : Predicate<"HST->hasPreV65()">, AssemblerPredicate<(all_of FeaturePreV65)>; +def UseHVXIEEEFP : Predicate<"HST->useHVXIEEEFPOps()">, + AssemblerPredicate<(all_of ExtensionHVXIEEEFP)>; +def UseHVXQFloat : Predicate<"HST->useHVXQFloatOps()">, + AssemblerPredicate<(all_of ExtensionHVXQFloat)>; +def UseHVXFloatingPoint: Predicate<"HST->useHVXFloatingPoint()">; def HasMemNoShuf : Predicate<"HST->hasMemNoShuf()">, AssemblerPredicate<(all_of FeatureMemNoShuf)>; def UseUnsafeMath : Predicate<"HST->useUnsafeMath()">; @@ -127,6 +145,8 @@ def NotOptTinyCore : Predicate<"!HST->isTinyCore() ||" let RecomputePerFunction = 1; } def UseSmallData : Predicate<"HST->useSmallData()">; +def UseCabac : Predicate<"HST->useCabac()">, + AssemblerPredicate<(any_of FeatureCabac)>; def Hvx64: HwMode<"+hvx-length64b">; def Hvx128: HwMode<"+hvx-length128b">; @@ -299,7 +319,7 @@ def changeAddrMode_rr_ur: InstrMapping { let ValueCols = [["BaseLongOffset"]]; } -def changeAddrMode_ur_rr : InstrMapping { +def changeAddrMode_ur_rr: InstrMapping { let FilterClass = "ImmRegShl"; let RowFields = ["CextOpcode", "PredSense", "PNewValue", "isNVStore"]; let ColFields = ["addrMode"]; @@ -370,40 +390,55 @@ class Proc<string Name, SchedMachineModel Model, def : Proc<"generic", HexagonModelV60, [ArchV5, ArchV55, ArchV60, FeatureCompound, FeatureDuplex, FeaturePreV65, FeatureMemops, - FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>; + FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData, + FeatureCabac]>; def : Proc<"hexagonv5", HexagonModelV5, [ArchV5, FeatureCompound, FeatureDuplex, FeaturePreV65, FeatureMemops, - FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>; + FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData, + FeatureCabac]>; def : Proc<"hexagonv55", HexagonModelV55, [ArchV5, ArchV55, FeatureCompound, FeatureDuplex, FeaturePreV65, FeatureMemops, - FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>; + FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData, + FeatureCabac]>; def : Proc<"hexagonv60", HexagonModelV60, [ArchV5, ArchV55, ArchV60, FeatureCompound, FeatureDuplex, FeaturePreV65, FeatureMemops, - FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>; + FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData, + FeatureCabac]>; def : Proc<"hexagonv62", HexagonModelV62, [ArchV5, ArchV55, ArchV60, ArchV62, FeatureCompound, FeatureDuplex, FeaturePreV65, FeatureMemops, - FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>; + FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData, + FeatureCabac]>; def : Proc<"hexagonv65", HexagonModelV65, [ArchV5, ArchV55, ArchV60, ArchV62, ArchV65, FeatureCompound, FeatureDuplex, FeatureMemNoShuf, FeatureMemops, - FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>; + FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData, + FeatureCabac]>; def : Proc<"hexagonv66", HexagonModelV66, [ArchV5, ArchV55, ArchV60, ArchV62, ArchV65, ArchV66, FeatureCompound, FeatureDuplex, FeatureMemNoShuf, FeatureMemops, - FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>; + FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData, + FeatureCabac]>; def : Proc<"hexagonv67", HexagonModelV67, [ArchV5, ArchV55, ArchV60, ArchV62, ArchV65, ArchV66, ArchV67, FeatureCompound, FeatureDuplex, FeatureMemNoShuf, FeatureMemops, - FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>; + FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData, + FeatureCabac]>; def : Proc<"hexagonv68", HexagonModelV68, [ArchV5, ArchV55, ArchV60, ArchV62, ArchV65, ArchV66, ArchV67, ArchV68, FeatureCompound, FeatureDuplex, FeatureMemNoShuf, FeatureMemops, - FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData]>; + FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData, + FeatureCabac]>; +def : Proc<"hexagonv69", HexagonModelV69, + [ArchV5, ArchV55, ArchV60, ArchV62, ArchV65, ArchV66, ArchV67, + ArchV68, ArchV69, + FeatureCompound, FeatureDuplex, FeatureMemNoShuf, FeatureMemops, + FeatureNVJ, FeatureNVS, FeaturePackets, FeatureSmallData, + FeatureCabac]>; // Need to update the correct features for tiny core. // Disable NewValueJumps since the packetizer is unable to handle a packet with // a new value jump and another SLOT0 instruction. diff --git a/llvm/lib/Target/Hexagon/HexagonAsmPrinter.cpp b/llvm/lib/Target/Hexagon/HexagonAsmPrinter.cpp index 8e6a01e3a186..411078052e0f 100644 --- a/llvm/lib/Target/Hexagon/HexagonAsmPrinter.cpp +++ b/llvm/lib/Target/Hexagon/HexagonAsmPrinter.cpp @@ -773,6 +773,67 @@ void HexagonAsmPrinter::emitInstruction(const MachineInstr *MI) { OutStreamer->emitInstruction(MCB, getSubtargetInfo()); } +void HexagonAsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind) { + static const int8_t NoopsInSledCount = 4; + // We want to emit the following pattern: + // + // .L_xray_sled_N: + // <xray_sled_base>: + // { jump .Ltmp0 } + // { nop + // nop + // nop + // nop } + // .Ltmp0: + // + // We need the 4 nop words because at runtime, we'd be patching over the + // full 5 words with the following pattern: + // + // <xray_sled_n>: + // { immext(#...) // upper 26-bits of trampoline + // r6 = ##... // lower 6-bits of trampoline + // immext(#...) // upper 26-bits of func id + // r7 = ##... } // lower 6 bits of func id + // { callr r6 } + // + // + auto CurSled = OutContext.createTempSymbol("xray_sled_", true); + OutStreamer->emitLabel(CurSled); + + MCInst *SledJump = new (OutContext) MCInst(); + SledJump->setOpcode(Hexagon::J2_jump); + auto PostSled = OutContext.createTempSymbol(); + SledJump->addOperand(MCOperand::createExpr(HexagonMCExpr::create( + MCSymbolRefExpr::create(PostSled, OutContext), OutContext))); + + // Emit "jump PostSled" instruction, which jumps over the nop series. + MCInst SledJumpPacket; + SledJumpPacket.setOpcode(Hexagon::BUNDLE); + SledJumpPacket.addOperand(MCOperand::createImm(0)); + SledJumpPacket.addOperand(MCOperand::createInst(SledJump)); + + EmitToStreamer(*OutStreamer, SledJumpPacket); + + // FIXME: this will emit individual packets, we should + // special-case this and combine them into a single packet. + emitNops(NoopsInSledCount); + + OutStreamer->emitLabel(PostSled); + recordSled(CurSled, MI, Kind, 0); +} + +void HexagonAsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI) { + EmitSled(MI, SledKind::FUNCTION_ENTER); +} + +void HexagonAsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) { + EmitSled(MI, SledKind::FUNCTION_EXIT); +} + +void HexagonAsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) { + EmitSled(MI, SledKind::TAIL_CALL); +} + extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeHexagonAsmPrinter() { RegisterAsmPrinter<HexagonAsmPrinter> X(getTheHexagonTarget()); } diff --git a/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h b/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h index 3932def87854..93d5f1dce7af 100644 --- a/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h +++ b/llvm/lib/Target/Hexagon/HexagonAsmPrinter.h @@ -36,7 +36,11 @@ class TargetMachine; bool runOnMachineFunction(MachineFunction &Fn) override { Subtarget = &Fn.getSubtarget<HexagonSubtarget>(); - return AsmPrinter::runOnMachineFunction(Fn); + const bool Modified = AsmPrinter::runOnMachineFunction(Fn); + // Emit the XRay table for this function. + emitXRayTable(); + + return Modified; } StringRef getPassName() const override { @@ -47,6 +51,16 @@ class TargetMachine; const override; void emitInstruction(const MachineInstr *MI) override; + + //===------------------------------------------------------------------===// + // XRay implementation + //===------------------------------------------------------------------===// + // XRay-specific lowering for Hexagon. + void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI); + void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); + void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); + void EmitSled(const MachineInstr &MI, SledKind Kind); + void HexagonProcessInstruction(MCInst &Inst, const MachineInstr &MBB); void printOperand(const MachineInstr *MI, unsigned OpNo, raw_ostream &O); diff --git a/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp b/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp index 2c5ad3b589d2..428d25da6dbc 100644 --- a/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp +++ b/llvm/lib/Target/Hexagon/HexagonBitSimplify.cpp @@ -995,8 +995,8 @@ bool DeadCodeElimination::runOnNode(MachineDomTreeNode *N) { MachineBasicBlock *B = N->getBlock(); std::vector<MachineInstr*> Instrs; - for (auto I = B->rbegin(), E = B->rend(); I != E; ++I) - Instrs.push_back(&*I); + for (MachineInstr &MI : llvm::reverse(*B)) + Instrs.push_back(&MI); for (auto MI : Instrs) { unsigned Opc = MI->getOpcode(); @@ -3084,8 +3084,7 @@ void HexagonLoopRescheduling::moveGroup(InstrGroup &G, MachineBasicBlock &LB, .addMBB(&LB); RegMap.insert(std::make_pair(G.Inp.Reg, PhiR)); - for (unsigned i = G.Ins.size(); i > 0; --i) { - const MachineInstr *SI = G.Ins[i-1]; + for (const MachineInstr *SI : llvm::reverse(G.Ins)) { unsigned DR = getDefReg(SI); const TargetRegisterClass *RC = MRI->getRegClass(DR); Register NewDR = MRI->createVirtualRegister(RC); @@ -3156,20 +3155,20 @@ bool HexagonLoopRescheduling::processLoop(LoopCand &C) { // if that instruction could potentially be moved to the front of the loop: // the output of the loop cannot be used in a non-shuffling instruction // in this loop. - for (auto I = C.LB->rbegin(), E = C.LB->rend(); I != E; ++I) { - if (I->isTerminator()) + for (MachineInstr &MI : llvm::reverse(*C.LB)) { + if (MI.isTerminator()) continue; - if (I->isPHI()) + if (MI.isPHI()) break; RegisterSet Defs; - HBS::getInstrDefs(*I, Defs); + HBS::getInstrDefs(MI, Defs); if (Defs.count() != 1) continue; Register DefR = Defs.find_first(); if (!DefR.isVirtual()) continue; - if (!isBitShuffle(&*I, DefR)) + if (!isBitShuffle(&MI, DefR)) continue; bool BadUse = false; @@ -3183,8 +3182,7 @@ bool HexagonLoopRescheduling::processLoop(LoopCand &C) { if (UseI->getOperand(Idx+1).getMBB() != C.LB) BadUse = true; } else { - auto F = find(ShufIns, UseI); - if (F == ShufIns.end()) + if (!llvm::is_contained(ShufIns, UseI)) BadUse = true; } } else { @@ -3199,7 +3197,7 @@ bool HexagonLoopRescheduling::processLoop(LoopCand &C) { if (BadUse) continue; - ShufIns.push_back(&*I); + ShufIns.push_back(&MI); } // Partition the list of shuffling instructions into instruction groups, diff --git a/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp b/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp index 8c3b9572201e..a53efeb96961 100644 --- a/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp +++ b/llvm/lib/Target/Hexagon/HexagonCommonGEP.cpp @@ -1256,15 +1256,11 @@ void HexagonCommonGEP::removeDeadCode() { BO.push_back(DTN->getBlock()); } - for (unsigned i = BO.size(); i > 0; --i) { - BasicBlock *B = cast<BasicBlock>(BO[i-1]); - BasicBlock::InstListType &IL = B->getInstList(); - - using reverse_iterator = BasicBlock::InstListType::reverse_iterator; - + for (Value *V : llvm::reverse(BO)) { + BasicBlock *B = cast<BasicBlock>(V); ValueVect Ins; - for (reverse_iterator I = IL.rbegin(), E = IL.rend(); I != E; ++I) - Ins.push_back(&*I); + for (Instruction &I : llvm::reverse(*B)) + Ins.push_back(&I); for (ValueVect::iterator I = Ins.begin(), E = Ins.end(); I != E; ++I) { Instruction *In = cast<Instruction>(*I); if (isInstructionTriviallyDead(In)) diff --git a/llvm/lib/Target/Hexagon/HexagonDepArch.h b/llvm/lib/Target/Hexagon/HexagonDepArch.h index 7a43a4440b2d..56174dc7e136 100644 --- a/llvm/lib/Target/Hexagon/HexagonDepArch.h +++ b/llvm/lib/Target/Hexagon/HexagonDepArch.h @@ -21,31 +21,32 @@ namespace llvm { namespace Hexagon { -enum class ArchEnum { NoArch, Generic, V5, V55, V60, V62, V65, V66, V67, V68 }; +enum class ArchEnum { NoArch, Generic, V5, V55, V60, V62, V65, V66, V67, V68, V69 }; -static constexpr unsigned ArchValsNumArray[] = {5, 55, 60, 62, 65, 66, 67, 68}; +static constexpr unsigned ArchValsNumArray[] = {5, 55, 60, 62, 65, 66, 67, 68, 69}; static constexpr ArrayRef<unsigned> ArchValsNum(ArchValsNumArray); -static constexpr StringLiteral ArchValsTextArray[] = { "v5", "v55", "v60", "v62", "v65", "v66", "v67", "v68" }; +static constexpr StringLiteral ArchValsTextArray[] = { "v5", "v55", "v60", "v62", "v65", "v66", "v67", "v68", "v69" }; static constexpr ArrayRef<StringLiteral> ArchValsText(ArchValsTextArray); -static constexpr StringLiteral CpuValsTextArray[] = { "hexagonv5", "hexagonv55", "hexagonv60", "hexagonv62", "hexagonv65", "hexagonv66", "hexagonv67", "hexagonv67t", "hexagonv68" }; +static constexpr StringLiteral CpuValsTextArray[] = { "hexagonv5", "hexagonv55", "hexagonv60", "hexagonv62", "hexagonv65", "hexagonv66", "hexagonv67", "hexagonv67t", "hexagonv68", "hexagonv69" }; static constexpr ArrayRef<StringLiteral> CpuValsText(CpuValsTextArray); -static constexpr StringLiteral CpuNickTextArray[] = { "v5", "v55", "v60", "v62", "v65", "v66", "v67", "v67t", "v68" }; +static constexpr StringLiteral CpuNickTextArray[] = { "v5", "v55", "v60", "v62", "v65", "v66", "v67", "v67t", "v68", "v69" }; static constexpr ArrayRef<StringLiteral> CpuNickText(CpuNickTextArray); static const std::map<std::string, ArchEnum> CpuTable{ - {"generic", Hexagon::ArchEnum::V5}, - {"hexagonv5", Hexagon::ArchEnum::V5}, - {"hexagonv55", Hexagon::ArchEnum::V55}, - {"hexagonv60", Hexagon::ArchEnum::V60}, - {"hexagonv62", Hexagon::ArchEnum::V62}, - {"hexagonv65", Hexagon::ArchEnum::V65}, - {"hexagonv66", Hexagon::ArchEnum::V66}, - {"hexagonv67", Hexagon::ArchEnum::V67}, - {"hexagonv67t", Hexagon::ArchEnum::V67}, - {"hexagonv68", Hexagon::ArchEnum::V68}, + {"generic", Hexagon::ArchEnum::V5}, + {"hexagonv5", Hexagon::ArchEnum::V5}, + {"hexagonv55", Hexagon::ArchEnum::V55}, + {"hexagonv60", Hexagon::ArchEnum::V60}, + {"hexagonv62", Hexagon::ArchEnum::V62}, + {"hexagonv65", Hexagon::ArchEnum::V65}, + {"hexagonv66", Hexagon::ArchEnum::V66}, + {"hexagonv67", Hexagon::ArchEnum::V67}, + {"hexagonv67t", Hexagon::ArchEnum::V67}, + {"hexagonv68", Hexagon::ArchEnum::V68}, + {"hexagonv69", Hexagon::ArchEnum::V69}, }; static const std::map<std::string, unsigned> ElfFlagsByCpuStr = { @@ -59,6 +60,7 @@ static const std::map<std::string, unsigned> ElfFlagsByCpuStr = { {"hexagonv67", llvm::ELF::EF_HEXAGON_MACH_V67}, {"hexagonv67t", llvm::ELF::EF_HEXAGON_MACH_V67T}, {"hexagonv68", llvm::ELF::EF_HEXAGON_MACH_V68}, + {"hexagonv69", llvm::ELF::EF_HEXAGON_MACH_V69}, }; static const std::map<unsigned, std::string> ElfArchByMachFlags = { {llvm::ELF::EF_HEXAGON_MACH_V5, "V5"}, @@ -70,6 +72,7 @@ static const std::map<unsigned, std::string> ElfArchByMachFlags = { {llvm::ELF::EF_HEXAGON_MACH_V67, "V67"}, {llvm::ELF::EF_HEXAGON_MACH_V67T, "V67T"}, {llvm::ELF::EF_HEXAGON_MACH_V68, "V68"}, + {llvm::ELF::EF_HEXAGON_MACH_V69, "V69"}, }; static const std::map<unsigned, std::string> ElfCpuByMachFlags = { {llvm::ELF::EF_HEXAGON_MACH_V5, "hexagonv5"}, @@ -81,6 +84,7 @@ static const std::map<unsigned, std::string> ElfCpuByMachFlags = { {llvm::ELF::EF_HEXAGON_MACH_V67, "hexagonv67"}, {llvm::ELF::EF_HEXAGON_MACH_V67T, "hexagonv67t"}, {llvm::ELF::EF_HEXAGON_MACH_V68, "hexagonv68"}, + {llvm::ELF::EF_HEXAGON_MACH_V69, "hexagonv69"}, }; } // namespace Hexagon diff --git a/llvm/lib/Target/Hexagon/HexagonDepArch.td b/llvm/lib/Target/Hexagon/HexagonDepArch.td index e743a291f1e5..e4f24e3c2e66 100644 --- a/llvm/lib/Target/Hexagon/HexagonDepArch.td +++ b/llvm/lib/Target/Hexagon/HexagonDepArch.td @@ -24,3 +24,5 @@ def ArchV67: SubtargetFeature<"v67", "HexagonArchVersion", "Hexagon::ArchEnum::V def HasV67 : Predicate<"HST->hasV67Ops()">, AssemblerPredicate<(all_of ArchV67)>; def ArchV68: SubtargetFeature<"v68", "HexagonArchVersion", "Hexagon::ArchEnum::V68", "Enable Hexagon V68 architecture">; def HasV68 : Predicate<"HST->hasV68Ops()">, AssemblerPredicate<(all_of ArchV68)>; +def ArchV69: SubtargetFeature<"v69", "HexagonArchVersion", "Hexagon::ArchEnum::V69", "Enable Hexagon V69 architecture">; +def HasV69 : Predicate<"HST->hasV69Ops()">, AssemblerPredicate<(all_of ArchV69)>; diff --git a/llvm/lib/Target/Hexagon/HexagonDepDecoders.inc b/llvm/lib/Target/Hexagon/HexagonDepDecoders.inc index 40f6e14aed13..7164af3ad5c6 100644 --- a/llvm/lib/Target/Hexagon/HexagonDepDecoders.inc +++ b/llvm/lib/Target/Hexagon/HexagonDepDecoders.inc @@ -8,6 +8,7 @@ // Automatically generated file, do not edit! //===----------------------------------------------------------------------===// + #if defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wunused-function" diff --git a/llvm/lib/Target/Hexagon/HexagonDepIICHVX.td b/llvm/lib/Target/Hexagon/HexagonDepIICHVX.td index a1db3ae7239d..d195df918293 100644 --- a/llvm/lib/Target/Hexagon/HexagonDepIICHVX.td +++ b/llvm/lib/Target/Hexagon/HexagonDepIICHVX.td @@ -11,6 +11,7 @@ def tc_04da405a : InstrItinClass; def tc_05ca8cfd : InstrItinClass; def tc_08a4f1b6 : InstrItinClass; +def tc_0afc8be9 : InstrItinClass; def tc_0b04c6c7 : InstrItinClass; def tc_0ec46cf9 : InstrItinClass; def tc_131f1c81 : InstrItinClass; @@ -21,6 +22,7 @@ def tc_191381c1 : InstrItinClass; def tc_1ad8a370 : InstrItinClass; def tc_1ba8a0cd : InstrItinClass; def tc_20a4bbec : InstrItinClass; +def tc_2120355e : InstrItinClass; def tc_257f6f7c : InstrItinClass; def tc_26a377fe : InstrItinClass; def tc_2b4c548e : InstrItinClass; @@ -28,15 +30,18 @@ def tc_2c745bb8 : InstrItinClass; def tc_2d4051cd : InstrItinClass; def tc_2e8f5f6e : InstrItinClass; def tc_309dbb4f : InstrItinClass; +def tc_37820f4c : InstrItinClass; def tc_3904b926 : InstrItinClass; def tc_3aacf4a8 : InstrItinClass; def tc_3ad719fb : InstrItinClass; def tc_3c56e5ce : InstrItinClass; +def tc_3c8c15d0 : InstrItinClass; def tc_3ce09744 : InstrItinClass; def tc_3e2aaafc : InstrItinClass; def tc_447d9895 : InstrItinClass; def tc_453fe68d : InstrItinClass; def tc_46d6c3e0 : InstrItinClass; +def tc_4942646a : InstrItinClass; def tc_51d0ecc3 : InstrItinClass; def tc_52447ecc : InstrItinClass; def tc_540c3da3 : InstrItinClass; @@ -46,6 +51,7 @@ def tc_56c4f9fe : InstrItinClass; def tc_56e64202 : InstrItinClass; def tc_58d21193 : InstrItinClass; def tc_5bf8afbb : InstrItinClass; +def tc_5cdf8c84 : InstrItinClass; def tc_61bf7c03 : InstrItinClass; def tc_649072c2 : InstrItinClass; def tc_660769f1 : InstrItinClass; @@ -57,6 +63,8 @@ def tc_71646d06 : InstrItinClass; def tc_7177e272 : InstrItinClass; def tc_718b5c53 : InstrItinClass; def tc_7273323b : InstrItinClass; +def tc_72e2b393 : InstrItinClass; +def tc_73efe966 : InstrItinClass; def tc_7417e785 : InstrItinClass; def tc_767c4e9d : InstrItinClass; def tc_7d68d5c2 : InstrItinClass; @@ -71,9 +79,11 @@ def tc_9d1dc972 : InstrItinClass; def tc_9f363d21 : InstrItinClass; def tc_a02a10a8 : InstrItinClass; def tc_a0dbea28 : InstrItinClass; +def tc_a19b9305 : InstrItinClass; def tc_a28f32b5 : InstrItinClass; def tc_a69eeee1 : InstrItinClass; def tc_a7e6707d : InstrItinClass; +def tc_aa047364 : InstrItinClass; def tc_ab23f776 : InstrItinClass; def tc_abe8c3b2 : InstrItinClass; def tc_ac4046bc : InstrItinClass; @@ -89,8 +99,10 @@ def tc_c4edf264 : InstrItinClass; def tc_c5dba46e : InstrItinClass; def tc_c7039829 : InstrItinClass; def tc_cd94bfe0 : InstrItinClass; +def tc_cda936da : InstrItinClass; def tc_d8287c14 : InstrItinClass; def tc_db5555f3 : InstrItinClass; +def tc_dcca380f : InstrItinClass; def tc_dd5b0695 : InstrItinClass; def tc_df80eeb0 : InstrItinClass; def tc_e2d2e9e5 : InstrItinClass; @@ -99,6 +111,7 @@ def tc_e3f68a46 : InstrItinClass; def tc_e675c45a : InstrItinClass; def tc_e699ae41 : InstrItinClass; def tc_e99d4c2e : InstrItinClass; +def tc_f175e046 : InstrItinClass; def tc_f1de44ef : InstrItinClass; def tc_f21e8abb : InstrItinClass; @@ -119,6 +132,11 @@ class DepHVXItinV55 { InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -174,6 +192,10 @@ class DepHVXItinV55 { InstrStage<1, [CVI_ST]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_2120355e, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7], @@ -209,6 +231,11 @@ class DepHVXItinV55 { InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2], [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3904b926, /*SLOT01,LOAD*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_LD]>], [9, 2, 1, 2], @@ -231,6 +258,11 @@ class DepHVXItinV55 { InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3ce09744, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2], @@ -259,6 +291,11 @@ class DepHVXItinV55 { InstrStage<1, [CVI_XLANE]>], [9, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_4942646a, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5], @@ -306,6 +343,11 @@ class DepHVXItinV55 { InstrStage<1, [CVI_XLANE]>], [9, 2], [HVX_FWD, Hex_FWD]>, + InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2], @@ -363,6 +405,16 @@ class DepHVXItinV55 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7], [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_7417e785, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5, 2], @@ -437,6 +489,11 @@ class DepHVXItinV55 { InstrStage<1, [CVI_ZW]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_a19b9305, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/ [InstrStage<1, [SLOT1], 0>, InstrStage<1, [CVI_LD], 0>, @@ -456,6 +513,10 @@ class DepHVXItinV55 { InstrStage<1, [CVI_XLANE]>], [9, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_aa047364, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_ab23f776, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2, 5], @@ -537,6 +598,11 @@ class DepHVXItinV55 { InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2], [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_cda936da, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 5], @@ -547,6 +613,11 @@ class DepHVXItinV55 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_ZW]>], [2, 1, 2], @@ -589,6 +660,11 @@ class DepHVXItinV55 { InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5], [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + InstrItinData <tc_f175e046, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/ [InstrStage<1, [SLOT2], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -620,6 +696,11 @@ class DepHVXItinV60 { InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -675,6 +756,10 @@ class DepHVXItinV60 { InstrStage<1, [CVI_ST]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_2120355e, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7], @@ -710,6 +795,11 @@ class DepHVXItinV60 { InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2], [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3904b926, /*SLOT01,LOAD*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_LD]>], [9, 2, 1, 2], @@ -732,6 +822,11 @@ class DepHVXItinV60 { InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3ce09744, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2], @@ -760,6 +855,11 @@ class DepHVXItinV60 { InstrStage<1, [CVI_XLANE]>], [9, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_4942646a, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5], @@ -807,6 +907,11 @@ class DepHVXItinV60 { InstrStage<1, [CVI_XLANE]>], [9, 2], [HVX_FWD, Hex_FWD]>, + InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2], @@ -864,6 +969,16 @@ class DepHVXItinV60 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7], [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_7417e785, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5, 2], @@ -938,6 +1053,11 @@ class DepHVXItinV60 { InstrStage<1, [CVI_ZW]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_a19b9305, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/ [InstrStage<1, [SLOT1], 0>, InstrStage<1, [CVI_LD], 0>, @@ -957,6 +1077,10 @@ class DepHVXItinV60 { InstrStage<1, [CVI_XLANE]>], [9, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_aa047364, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_ab23f776, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2, 5], @@ -1038,6 +1162,11 @@ class DepHVXItinV60 { InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2], [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_cda936da, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 5], @@ -1048,6 +1177,11 @@ class DepHVXItinV60 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_ZW]>], [2, 1, 2], @@ -1090,6 +1224,11 @@ class DepHVXItinV60 { InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5], [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + InstrItinData <tc_f175e046, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/ [InstrStage<1, [SLOT2], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -1121,6 +1260,11 @@ class DepHVXItinV62 { InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -1176,6 +1320,10 @@ class DepHVXItinV62 { InstrStage<1, [CVI_ST]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_2120355e, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7], @@ -1211,6 +1359,11 @@ class DepHVXItinV62 { InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2], [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3904b926, /*SLOT01,LOAD*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_LD]>], [9, 2, 1, 2], @@ -1233,6 +1386,11 @@ class DepHVXItinV62 { InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3ce09744, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2], @@ -1261,6 +1419,11 @@ class DepHVXItinV62 { InstrStage<1, [CVI_XLANE]>], [9, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_4942646a, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5], @@ -1308,6 +1471,11 @@ class DepHVXItinV62 { InstrStage<1, [CVI_XLANE]>], [9, 2], [HVX_FWD, Hex_FWD]>, + InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2], @@ -1365,6 +1533,16 @@ class DepHVXItinV62 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7], [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_7417e785, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5, 2], @@ -1439,6 +1617,11 @@ class DepHVXItinV62 { InstrStage<1, [CVI_ZW]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_a19b9305, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/ [InstrStage<1, [SLOT1], 0>, InstrStage<1, [CVI_LD], 0>, @@ -1458,6 +1641,10 @@ class DepHVXItinV62 { InstrStage<1, [CVI_XLANE]>], [9, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_aa047364, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_ab23f776, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2, 5], @@ -1539,6 +1726,11 @@ class DepHVXItinV62 { InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2], [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_cda936da, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 5], @@ -1549,6 +1741,11 @@ class DepHVXItinV62 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_ZW]>], [2, 1, 2], @@ -1591,6 +1788,11 @@ class DepHVXItinV62 { InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5], [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + InstrItinData <tc_f175e046, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/ [InstrStage<1, [SLOT2], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -1622,6 +1824,11 @@ class DepHVXItinV65 { InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -1677,6 +1884,10 @@ class DepHVXItinV65 { InstrStage<1, [CVI_ST]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_2120355e, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7], @@ -1712,6 +1923,11 @@ class DepHVXItinV65 { InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2], [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3904b926, /*SLOT01,LOAD*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_LD]>], [9, 2, 1, 2], @@ -1734,6 +1950,11 @@ class DepHVXItinV65 { InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3ce09744, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2], @@ -1762,6 +1983,11 @@ class DepHVXItinV65 { InstrStage<1, [CVI_XLANE]>], [9, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_4942646a, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5], @@ -1809,6 +2035,11 @@ class DepHVXItinV65 { InstrStage<1, [CVI_XLANE]>], [9, 2], [HVX_FWD, Hex_FWD]>, + InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2], @@ -1866,6 +2097,16 @@ class DepHVXItinV65 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7], [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_7417e785, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5, 2], @@ -1940,6 +2181,11 @@ class DepHVXItinV65 { InstrStage<1, [CVI_ZW]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_a19b9305, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/ [InstrStage<1, [SLOT1], 0>, InstrStage<1, [CVI_LD], 0>, @@ -1959,6 +2205,10 @@ class DepHVXItinV65 { InstrStage<1, [CVI_XLANE]>], [9, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_aa047364, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_ab23f776, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2, 5], @@ -2040,6 +2290,11 @@ class DepHVXItinV65 { InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2], [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_cda936da, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 5], @@ -2050,6 +2305,11 @@ class DepHVXItinV65 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_ZW]>], [2, 1, 2], @@ -2092,6 +2352,11 @@ class DepHVXItinV65 { InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5], [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + InstrItinData <tc_f175e046, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/ [InstrStage<1, [SLOT2], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -2123,6 +2388,11 @@ class DepHVXItinV66 { InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -2178,6 +2448,10 @@ class DepHVXItinV66 { InstrStage<1, [CVI_ST]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_2120355e, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7], @@ -2213,6 +2487,11 @@ class DepHVXItinV66 { InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2], [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3904b926, /*SLOT01,LOAD*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_LD]>], [9, 2, 1, 2], @@ -2235,6 +2514,11 @@ class DepHVXItinV66 { InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3ce09744, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2], @@ -2263,6 +2547,11 @@ class DepHVXItinV66 { InstrStage<1, [CVI_XLANE]>], [9, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_4942646a, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5], @@ -2310,6 +2599,11 @@ class DepHVXItinV66 { InstrStage<1, [CVI_XLANE]>], [9, 2], [HVX_FWD, Hex_FWD]>, + InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2], @@ -2367,6 +2661,16 @@ class DepHVXItinV66 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7], [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_7417e785, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5, 2], @@ -2441,6 +2745,11 @@ class DepHVXItinV66 { InstrStage<1, [CVI_ZW]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_a19b9305, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/ [InstrStage<1, [SLOT1], 0>, InstrStage<1, [CVI_LD], 0>, @@ -2460,6 +2769,10 @@ class DepHVXItinV66 { InstrStage<1, [CVI_XLANE]>], [9, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_aa047364, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_ab23f776, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2, 5], @@ -2541,6 +2854,11 @@ class DepHVXItinV66 { InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2], [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_cda936da, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 5], @@ -2551,6 +2869,11 @@ class DepHVXItinV66 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_ZW]>], [2, 1, 2], @@ -2593,6 +2916,11 @@ class DepHVXItinV66 { InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5], [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + InstrItinData <tc_f175e046, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/ [InstrStage<1, [SLOT2], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -2624,6 +2952,11 @@ class DepHVXItinV67 { InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -2679,6 +3012,10 @@ class DepHVXItinV67 { InstrStage<1, [CVI_ST]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_2120355e, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7], @@ -2714,6 +3051,11 @@ class DepHVXItinV67 { InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2], [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3904b926, /*SLOT01,LOAD*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_LD]>], [9, 2, 1, 2], @@ -2736,6 +3078,11 @@ class DepHVXItinV67 { InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3ce09744, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2], @@ -2764,6 +3111,11 @@ class DepHVXItinV67 { InstrStage<1, [CVI_XLANE]>], [9, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_4942646a, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5], @@ -2811,6 +3163,11 @@ class DepHVXItinV67 { InstrStage<1, [CVI_XLANE]>], [9, 2], [HVX_FWD, Hex_FWD]>, + InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2], @@ -2868,6 +3225,16 @@ class DepHVXItinV67 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7], [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_7417e785, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5, 2], @@ -2942,6 +3309,11 @@ class DepHVXItinV67 { InstrStage<1, [CVI_ZW]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_a19b9305, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_a28f32b5, /*SLOT1,LOAD,VA*/ [InstrStage<1, [SLOT1], 0>, InstrStage<1, [CVI_LD], 0>, @@ -2961,6 +3333,10 @@ class DepHVXItinV67 { InstrStage<1, [CVI_XLANE]>], [9, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_aa047364, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_ab23f776, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2, 5], @@ -3042,6 +3418,11 @@ class DepHVXItinV67 { InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2], [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_cda936da, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 5], @@ -3052,6 +3433,11 @@ class DepHVXItinV67 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_ZW]>], [2, 1, 2], @@ -3094,6 +3480,11 @@ class DepHVXItinV67 { InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5], [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + InstrItinData <tc_f175e046, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/ [InstrStage<1, [SLOT2], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -3125,6 +3516,575 @@ class DepHVXItinV68 { InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_0ec46cf9, /*SLOT0123,VA*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_131f1c81, /*SLOT0,NOSLOT1,STORE,VP*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [SLOT1], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_XLANE]>], [2, 1, 2, 5], + [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_1381a97c, /*SLOT0123,4SLOT*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_ALL]>], [], + []>, + + InstrItinData <tc_15fdf750, /*SLOT23,VS_VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1], 0>, + InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_16ff9ef8, /*SLOT0123,VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_SHIFT]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_191381c1, /*SLOT0,STORE,VA*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [3, 7, 1, 2, 7], + [Hex_FWD, HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_1ad8a370, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 2, 2], + [HVX_FWD, HVX_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_1ba8a0cd, /*SLOT01,LOAD,VA*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 3, 1, 2], + [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_20a4bbec, /*SLOT0,STORE*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST]>], [3, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_2120355e, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_26a377fe, /*SLOT23,4SLOT_MPY*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_ALL_NOMEM]>], [9, 3, 5, 2], + [HVX_FWD, Hex_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_2b4c548e, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_2c745bb8, /*SLOT0123,VP_VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLSHF]>], [9, 7, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_2d4051cd, /*SLOT23,4SLOT_MPY*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_ALL_NOMEM]>], [9, 3, 7, 5, 2], + [HVX_FWD, Hex_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_2e8f5f6e, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_309dbb4f, /*SLOT0123,VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_37820f4c, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_3904b926, /*SLOT01,LOAD*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD]>], [9, 2, 1, 2], + [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_3aacf4a8, /*SLOT0123,VA*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 2, 7], + [HVX_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_3ad719fb, /*SLOT01,ZW*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_ZW]>], [3, 2, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_3c56e5ce, /*SLOT0,NOSLOT1,LOAD,VP*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [SLOT1], 0>, + InstrStage<1, [CVI_LD], 0>, + InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2], + [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_3ce09744, /*SLOT0,STORE*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST]>], [1, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_3e2aaafc, /*SLOT0,STORE,VA*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [3, 1, 2, 7], + [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_447d9895, /*SLOT0,STORE,VA*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [7, 1, 2, 7], + [HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_453fe68d, /*SLOT01,LOAD,VA*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 3, 2, 1, 2], + [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_46d6c3e0, /*SLOT0123,VP*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLANE]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_4942646a, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_SHIFT]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_52447ecc, /*SLOT01,LOAD*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD]>], [9, 1, 2], + [HVX_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_540c3da3, /*SLOT0,VA*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [4, 7, 1], + [Hex_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_54a0dc47, /*SLOT0,STORE,VA*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [3, 2, 1, 2, 7], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_561aaa58, /*SLOT0123,VP_VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLSHF]>], [9, 9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_56c4f9fe, /*SLOT0123,VA*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_56e64202, /*SLOT0123,VP*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLANE]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_58d21193, /*SLOT0,STORE,VA_DV*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [7, 1, 2, 7, 7], + [HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_5bf8afbb, /*SLOT0123,VP*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLANE]>], [9, 2], + [HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_649072c2, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_660769f1, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_663c80a7, /*SLOT01,LOAD*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD]>], [9, 3, 1, 2], + [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_6942b6e0, /*SLOT0,STORE*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST]>], [3, 1, 2, 5], + [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_6e7fa133, /*SLOT0123,VP*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLANE]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_7095ecba, /*SLOT01,LOAD,VA_DV*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD], 0>, + InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7], + [Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_71646d06, /*SLOT0123,VA_DV*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_7177e272, /*SLOT0,STORE*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST]>], [2, 1, 2, 5], + [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_718b5c53, /*SLOT0123,VA_DV*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9], + [HVX_FWD]>, + + InstrItinData <tc_7273323b, /*SLOT0,STORE,VA_DV*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7], + [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_72e2b393, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_73efe966, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_7417e785, /*SLOT0123,VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_SHIFT]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_767c4e9d, /*SLOT0123,4SLOT*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_ALL]>], [3, 2], + [HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_7d68d5c2, /*SLOT01,LOAD,VA*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [7, 1, 2, 7], + [HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_7e6a3e89, /*SLOT0123,VA*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 9, 7, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_8772086c, /*SLOT0123,VA*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_87adc037, /*SLOT0123,VP_VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLSHF]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_8e420e4d, /*SLOT0,STORE,VA*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [7, 1, 2, 7, 7], + [HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_90bcc1db, /*SLOT2,VX_DV*/ + [InstrStage<1, [SLOT2], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_933f2b39, /*SLOT23,4SLOT_MPY*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_ALL_NOMEM]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_946013d8, /*SLOT0123,VP*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLANE]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_9d1dc972, /*SLOT0123,VP_VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLSHF]>], [9, 7, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_9f363d21, /*SLOT0,STORE,VA*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [1, 2, 7, 7], + [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_a02a10a8, /*SLOT0,STORE,VA*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [2, 1, 2, 7], + [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_a0dbea28, /*SLOT01,ZW*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_ZW]>], [3, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a19b9305, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_a28f32b5, /*SLOT01,LOAD,VA*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [1, 2, 7], + [Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_a69eeee1, /*SLOT01,LOAD,VA_DV*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD], 0>, + InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [7, 1, 2, 7], + [HVX_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_a7e6707d, /*SLOT0,NOSLOT1,LOAD,VP*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [SLOT1], 0>, + InstrStage<1, [CVI_LD], 0>, + InstrStage<1, [CVI_XLANE]>], [9, 1, 2], + [HVX_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_aa047364, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_ab23f776, /*SLOT0,STORE*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST]>], [1, 2, 5], + [Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_abe8c3b2, /*SLOT01,LOAD,VA*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 2, 1, 2], + [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_ac4046bc, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_af25efd9, /*SLOT0123,VA_DV*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 2, 7, 7], + [HVX_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_b091f1c6, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_b28e51aa, /*SLOT0123,4SLOT*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_ALL]>], [2], + [Hex_FWD]>, + + InstrItinData <tc_b4416217, /*SLOT0123,VA_DV*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_b9db8205, /*SLOT01,LOAD*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD]>], [9, 3, 2, 1, 2], + [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_bb599486, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_c0749f3c, /*SLOT01,LOAD,VA*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_LD], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 1, 2], + [HVX_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_c127de3a, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_c4edf264, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 2], + [HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_c5dba46e, /*SLOT0,STORE,VA*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [1, 2, 7], + [Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_c7039829, /*SLOT0,NOSLOT1,STORE,VP*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [SLOT1], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_XLANE]>], [3, 2, 1, 2, 5], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_cd94bfe0, /*SLOT23,VS_VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1], 0>, + InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_cda936da, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_db5555f3, /*SLOT0123,VA_DV*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_dcca380f, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_ZW]>], [2, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_df80eeb0, /*SLOT0123,VP_VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLSHF]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_e2d2e9e5, /*SLOT0,NOSLOT1,STORE,VP*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [SLOT1], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_XLANE]>], [3, 1, 2, 5], + [Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_e35c1e93, /*SLOT0123,VA*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_e3f68a46, /*SLOT0123,4SLOT*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_ALL]>], [3], + [HVX_FWD]>, + + InstrItinData <tc_e675c45a, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 2, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_e699ae41, /*SLOT01,ZW*/ + [InstrStage<1, [SLOT0, SLOT1], 0>, + InstrStage<1, [CVI_ZW]>], [1, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_e99d4c2e, /*SLOT0,STORE*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + + InstrItinData <tc_f175e046, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/ + [InstrStage<1, [SLOT2], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_f21e8abb, /*SLOT0,NOSLOT1,STORE,VP*/ + [InstrStage<1, [SLOT0], 0>, + InstrStage<1, [SLOT1], 0>, + InstrStage<1, [CVI_ST], 0>, + InstrStage<1, [CVI_XLANE]>], [1, 2, 5], + [Hex_FWD, Hex_FWD, HVX_FWD]> + ]; +} + +class DepHVXItinV69 { + list<InstrItinData> DepHVXItinV69_list = [ + InstrItinData <tc_04da405a, /*SLOT0123,VP_VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_XLSHF]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_05ca8cfd, /*SLOT0123,VS*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_SHIFT]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_08a4f1b6, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + + InstrItinData <tc_0afc8be9, /*SLOT23,VX_DV*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY01]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_0b04c6c7, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], @@ -3180,6 +4140,10 @@ class DepHVXItinV68 { InstrStage<1, [CVI_ST]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_2120355e, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_257f6f7c, /*SLOT0123,VA*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY0, CVI_MPY1, CVI_SHIFT, CVI_XLANE]>], [9, 7, 7, 7], @@ -3215,6 +4179,11 @@ class DepHVXItinV68 { InstrStage<1, [CVI_SHIFT]>], [9, 7, 5, 2], [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_37820f4c, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3904b926, /*SLOT01,LOAD*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_LD]>], [9, 2, 1, 2], @@ -3237,6 +4206,11 @@ class DepHVXItinV68 { InstrStage<1, [CVI_XLANE]>], [9, 3, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_3c8c15d0, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_3ce09744, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2], @@ -3265,6 +4239,11 @@ class DepHVXItinV68 { InstrStage<1, [CVI_XLANE]>], [9, 5, 5], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_4942646a, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_51d0ecc3, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5], @@ -3312,6 +4291,11 @@ class DepHVXItinV68 { InstrStage<1, [CVI_XLANE]>], [9, 2], [HVX_FWD, Hex_FWD]>, + InstrItinData <tc_5cdf8c84, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7], + [HVX_FWD, HVX_FWD]>, + InstrItinData <tc_61bf7c03, /*SLOT23,4SLOT_MPY*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_ALL_NOMEM]>], [9, 5, 2], @@ -3369,6 +4353,16 @@ class DepHVXItinV68 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [1, 2, 7, 7], [Hex_FWD, Hex_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_72e2b393, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + + InstrItinData <tc_73efe966, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_7417e785, /*SLOT0123,VS*/ [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3], 0>, InstrStage<1, [CVI_SHIFT]>], [9, 5, 2], @@ -3443,6 +4437,11 @@ class DepHVXItinV68 { InstrStage<1, [CVI_ZW]>], [3, 1, 2], [Hex_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_a19b9305, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 5, 5], + [HVX_FWD, HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_a28f32b5, /*SLOT01,LOAD,VA*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_LD], 0>, @@ -3462,6 +4461,10 @@ class DepHVXItinV68 { InstrStage<1, [CVI_XLANE]>], [9, 1, 2], [HVX_FWD, Hex_FWD, Hex_FWD]>, + InstrItinData <tc_aa047364, /*SLOT0123*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_ab23f776, /*SLOT0,STORE*/ [InstrStage<1, [SLOT0], 0>, InstrStage<1, [CVI_ST]>], [1, 2, 5], @@ -3543,6 +4546,11 @@ class DepHVXItinV68 { InstrStage<1, [CVI_SHIFT, CVI_XLANE]>], [9, 5, 2], [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_cda936da, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 7, 7], + [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_d8287c14, /*SLOT23,VX_DV*/ [InstrStage<1, [SLOT2, SLOT3], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 5], @@ -3553,6 +4561,11 @@ class DepHVXItinV68 { InstrStage<1, [CVI_MPY01, CVI_XLSHF]>], [9, 7, 7], [HVX_FWD, HVX_FWD, HVX_FWD]>, + InstrItinData <tc_dcca380f, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 2], + [HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_dd5b0695, /*SLOT01,ZW*/ [InstrStage<1, [SLOT0, SLOT1], 0>, InstrStage<1, [CVI_ZW]>], [2, 1, 2], @@ -3595,6 +4608,11 @@ class DepHVXItinV68 { InstrStage<1, [CVI_ST]>], [3, 2, 1, 2, 5], [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, HVX_FWD]>, + InstrItinData <tc_f175e046, /*SLOT23,VX*/ + [InstrStage<1, [SLOT2, SLOT3], 0>, + InstrStage<1, [CVI_MPY0, CVI_MPY1]>], [9, 5, 5, 2], + [HVX_FWD, HVX_FWD, HVX_FWD, Hex_FWD]>, + InstrItinData <tc_f1de44ef, /*SLOT2,VX_DV*/ [InstrStage<1, [SLOT2], 0>, InstrStage<1, [CVI_MPY01]>], [9, 5, 2], diff --git a/llvm/lib/Target/Hexagon/HexagonDepIICScalar.td b/llvm/lib/Target/Hexagon/HexagonDepIICScalar.td index a3766652794b..a979bafe8e33 100644 --- a/llvm/lib/Target/Hexagon/HexagonDepIICScalar.td +++ b/llvm/lib/Target/Hexagon/HexagonDepIICScalar.td @@ -7338,3 +7338,771 @@ class DepScalarItinV68 { [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]> ]; } + +class DepScalarItinV69 { + list<InstrItinData> DepScalarItinV69_list = [ + InstrItinData <tc_011e0e9d, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [2, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_01d44cb2, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_01e1be3b, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_02fe1c65, /*tc_4x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [5, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_0655b949, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [2, 3], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_075c8dd8, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_0a195f2c, /*tc_4x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [5, 2, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_0a6c20ae, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [2, 1, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_0ba0d5da, /*tc_3stall*/ + [InstrStage<1, [SLOT2]>], [1], + [Hex_FWD]>, + + InstrItinData <tc_0dfac0a7, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_0fac1eb8, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [3, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_112d30d6, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2], + [Hex_FWD]>, + + InstrItinData <tc_1242dc2a, /*tc_ld*/ + [InstrStage<1, [SLOT0]>], [2], + [Hex_FWD]>, + + InstrItinData <tc_1248597c, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [2, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_14ab4f41, /*tc_newvjump*/ + [InstrStage<1, [SLOT0]>], [3, 3, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_151bf368, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_158aa3f7, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [1, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_197dce51, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [4, 2, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_1981450d, /*tc_newvjump*/ + [InstrStage<1, [SLOT0]>], [3], + [Hex_FWD]>, + + InstrItinData <tc_1c2c7a4a, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_1c7522a8, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 2, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_1d41f8b7, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 4, 2, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_1fcb8495, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_1fe4ab69, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [2, 1, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_20131976, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_2237d952, /*tc_ld*/ + [InstrStage<1, [SLOT0]>], [1, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_23708a21, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [], + []>, + + InstrItinData <tc_2471c1c8, /*tc_ld*/ + [InstrStage<1, [SLOT0]>], [4, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_24e109c7, /*tc_newvjump*/ + [InstrStage<1, [SLOT0]>], [3, 3, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_24f426ab, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_280f7fe1, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [1, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_28e55c6f, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [1, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_2c13e7f5, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_2c3e17fc, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [1], + [Hex_FWD]>, + + InstrItinData <tc_2f573607, /*tc_1*/ + [InstrStage<1, [SLOT2]>], [2, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_362b0be2, /*tc_3*/ + [InstrStage<1, [SLOT2]>], [1], + [Hex_FWD]>, + + InstrItinData <tc_38382228, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_388f9897, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_38e0bae9, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 4, 2, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_3d14a17b, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1]>], [3, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_3edca78f, /*tc_2*/ + [InstrStage<1, [SLOT3]>], [4, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_3fbf1042, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1]>], [3], + [Hex_FWD]>, + + InstrItinData <tc_407e96f9, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_40d64c94, /*tc_newvjump*/ + [InstrStage<1, [SLOT0]>], [3, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_4222e6bf, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_42ff66ba, /*tc_1*/ + [InstrStage<1, [SLOT2]>], [2, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_442395f3, /*tc_2latepred*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_449acf79, /*tc_latepredstaia*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 1, 2, 1], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_44d5a428, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [1, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_44fffc58, /*tc_3*/ + [InstrStage<1, [SLOT2, SLOT3]>], [2], + [Hex_FWD]>, + + InstrItinData <tc_45791fb8, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 2, 1, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_45f9d1be, /*tc_2early*/ + [InstrStage<1, [SLOT2]>], [2], + [Hex_FWD]>, + + InstrItinData <tc_49fdfd4b, /*tc_3stall*/ + [InstrStage<1, [SLOT3]>], [4, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_4a55d03c, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_4abdbdc6, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [2, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_4ac61d92, /*tc_2latepred*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 3, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_4bf903b0, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [3], + [Hex_FWD]>, + + InstrItinData <tc_503ce0f3, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2, 1], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_53c851ab, /*tc_3stall*/ + [InstrStage<1, [SLOT2]>], [4, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_5502c366, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_55255f2b, /*tc_3stall*/ + [InstrStage<1, [SLOT3]>], [], + []>, + + InstrItinData <tc_556f6577, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_55a9a350, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [1, 2, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_55b33fda, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_56a124a7, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_57a55b54, /*tc_1*/ + [InstrStage<1, [SLOT3]>], [2, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_5944960d, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [1, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_59a7822c, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1]>], [2, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_5a4b5e58, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [4, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_5b347363, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_5ceb2f9e, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 1, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_5da50c4b, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_5deb5e47, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_5e4cf0e8, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_5f2afaf7, /*tc_latepredldaia*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 4, 3, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_60e324ff, /*tc_1*/ + [InstrStage<1, [SLOT2]>], [2], + [Hex_FWD]>, + + InstrItinData <tc_63567288, /*tc_2latepred*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4], + [Hex_FWD]>, + + InstrItinData <tc_64b00d8a, /*tc_ld*/ + [InstrStage<1, [SLOT0]>], [4, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_651cbe02, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_65279839, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_65cbd974, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [3, 1, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_69bfb303, /*tc_3*/ + [InstrStage<1, [SLOT2, SLOT3]>], [2, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_6ae3426b, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [4, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_6d861a95, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [2, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_6e20402a, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [2, 3], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_6f42bc60, /*tc_3stall*/ + [InstrStage<1, [SLOT0]>], [4, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_6fc5dbea, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_711c805f, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_713b66bf, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_7401744f, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_7476d766, /*tc_3stall*/ + [InstrStage<1, [SLOT3]>], [4, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_74a42bda, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [3, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_76bb5435, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 2, 1, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_77f94a5e, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [], + []>, + + InstrItinData <tc_788b1d09, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_7af3a37e, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [1, 3], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_7b9187d3, /*tc_newvjump*/ + [InstrStage<1, [SLOT0]>], [3, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_7c31e19a, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [1, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_7c6d32e4, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_7f7f45f5, /*tc_4x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [5, 5, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_7f8ae742, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_8035e91f, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [2, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_822c3c68, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_829d8a86, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [3, 1, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_838c4d7a, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [1, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_84a7500d, /*tc_2*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_86173609, /*tc_2latepred*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 3, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_887d1bb7, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [1, 2, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_8a6d0d94, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_8a825db2, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_8b5bd4f5, /*tc_2*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_8e82e8ca, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [3, 1, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_9124c04f, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_92240447, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [3, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_934753bb, /*tc_ld*/ + [InstrStage<1, [SLOT0]>], [3, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_937dd41c, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [], + []>, + + InstrItinData <tc_9406230a, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [2, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_95a33176, /*tc_2*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_96ef76ef, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [1, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_975a4e54, /*tc_newvjump*/ + [InstrStage<1, [SLOT0]>], [3, 3, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_9783714b, /*tc_4x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [5, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_9b34f5e0, /*tc_3stall*/ + [InstrStage<1, [SLOT2]>], [], + []>, + + InstrItinData <tc_9b3c0462, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_9bcfb2ee, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_9c52f549, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_9e27f2f9, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_9e72dc89, /*tc_4x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [5, 2, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_9edb7c77, /*tc_4x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [5, 2, 1, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_9edefe01, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [3, 2, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_9f6cd987, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a08b630b, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a1297125, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a154b476, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a2b365d2, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [3, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a3070909, /*tc_3stall*/ + [InstrStage<1, [SLOT0]>], [1, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a32e03e7, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 2, 1, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a38c45dc, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 1, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a4e22bbd, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a4ee89db, /*tc_2early*/ + [InstrStage<1, [SLOT0]>], [], + []>, + + InstrItinData <tc_a7a13fac, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a7bdb22c, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_a9edeffa, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_abfd9a6d, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 1, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_ac65613f, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_addc37a8, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [3, 1, 2, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_ae5babd7, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_aee6250c, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_b1ae5f67, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [1], + [Hex_FWD]>, + + InstrItinData <tc_b4dc7630, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [3, 1, 2, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_b7c4062a, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 3, 1, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_b837298f, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [], + []>, + + InstrItinData <tc_ba9255a6, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [2, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_bb07f2c5, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [3, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_bb831a7c, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_bf2ffc0f, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 1, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_c20701f0, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_c21d7447, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_c57d9f39, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_c818ff7f, /*tc_newvjump*/ + [InstrStage<1, [SLOT0]>], [], + []>, + + InstrItinData <tc_ce59038e, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [3, 2, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_cfa0e29b, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [2, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_d03278fd, /*tc_st*/ + [InstrStage<1, [SLOT0, SLOT1]>], [2, 1, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_d33e5eee, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_d3632d88, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_d45ba9cd, /*tc_ld*/ + [InstrStage<1, [SLOT0]>], [1], + [Hex_FWD]>, + + InstrItinData <tc_d57d649c, /*tc_3stall*/ + [InstrStage<1, [SLOT2]>], [2], + [Hex_FWD]>, + + InstrItinData <tc_d61dfdc3, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_d68dca5c, /*tc_3stall*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_d7718fbe, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [1], + [Hex_FWD]>, + + InstrItinData <tc_db596beb, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_db96aa6b, /*tc_st*/ + [InstrStage<1, [SLOT0]>], [1], + [Hex_FWD]>, + + InstrItinData <tc_dc51281d, /*tc_3*/ + [InstrStage<1, [SLOT2]>], [2, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_decdde8a, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2], + [Hex_FWD]>, + + InstrItinData <tc_df5d53f9, /*tc_newvjump*/ + [InstrStage<1, [SLOT0]>], [3, 2, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_e3d699e3, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_e9170fb7, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 1], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_ed03645c, /*tc_1*/ + [InstrStage<1, [SLOT2]>], [3, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_eed07714, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_eeda4109, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [3, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_ef921005, /*tc_1*/ + [InstrStage<1, [SLOT2, SLOT3]>], [3, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_f098b237, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_f0cdeccf, /*tc_3x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 1, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_f0e8e832, /*tc_4x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [5, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_f34c1c21, /*tc_2*/ + [InstrStage<1, [SLOT2, SLOT3]>], [4, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_f38f92e1, /*tc_newvjump*/ + [InstrStage<1, [SLOT0]>], [2], + [Hex_FWD]>, + + InstrItinData <tc_f529831b, /*tc_latepredstaia*/ + [InstrStage<1, [SLOT0]>], [4, 3, 1, 2, 3], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_f6e2aff9, /*tc_newvjump*/ + [InstrStage<1, [SLOT0]>], [3, 2, 2], + [Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_f7569068, /*tc_4x*/ + [InstrStage<1, [SLOT2, SLOT3]>], [5, 5, 1, 1], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_f999c66e, /*tc_1*/ + [InstrStage<1, [SLOT0, SLOT1, SLOT2, SLOT3]>], [2, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_fae9dfa5, /*tc_3x*/ + [InstrStage<1, [SLOT3]>], [4, 2], + [Hex_FWD, Hex_FWD]>, + + InstrItinData <tc_fedb7e19, /*tc_ld*/ + [InstrStage<1, [SLOT0, SLOT1]>], [4, 2, 1, 2], + [Hex_FWD, Hex_FWD, Hex_FWD, Hex_FWD]> + ]; +} diff --git a/llvm/lib/Target/Hexagon/HexagonDepInstrFormats.td b/llvm/lib/Target/Hexagon/HexagonDepInstrFormats.td index b3f1b6638193..65d36924ba48 100644 --- a/llvm/lib/Target/Hexagon/HexagonDepInstrFormats.td +++ b/llvm/lib/Target/Hexagon/HexagonDepInstrFormats.td @@ -2288,6 +2288,12 @@ class Enc_a30110 : OpcodeHexagon { bits <5> Vd32; let Inst{4-0} = Vd32{4-0}; } +class Enc_a33d04 : OpcodeHexagon { + bits <5> Vuu32; + let Inst{12-8} = Vuu32{4-0}; + bits <5> Vd32; + let Inst{4-0} = Vd32{4-0}; +} class Enc_a42857 : OpcodeHexagon { bits <11> Ii; let Inst{21-20} = Ii{10-9}; @@ -3109,6 +3115,14 @@ class Enc_de0214 : OpcodeHexagon { bits <5> Rd32; let Inst{4-0} = Rd32{4-0}; } +class Enc_de5ea0 : OpcodeHexagon { + bits <5> Vuu32; + let Inst{12-8} = Vuu32{4-0}; + bits <5> Vv32; + let Inst{20-16} = Vv32{4-0}; + bits <5> Vd32; + let Inst{4-0} = Vd32{4-0}; +} class Enc_e07374 : OpcodeHexagon { bits <5> Rs32; let Inst{20-16} = Rs32{4-0}; diff --git a/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td b/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td index 4f00409c336c..c02988266584 100644 --- a/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td +++ b/llvm/lib/Target/Hexagon/HexagonDepInstrInfo.td @@ -5824,8 +5824,8 @@ let Inst{31-21} = 0b01010100100; let hasNewValue = 1; let opNewValue = 0; let isSolo = 1; -let Uses = [GOSP]; -let Defs = [GOSP, PC]; +let Uses = [CCR, GOSP]; +let Defs = [CCR, GOSP, PC]; let hasSideEffects = 1; let Constraints = "$Rx32 = $Rx32in"; } @@ -8500,6 +8500,8 @@ let Inst{31-21} = 0b01010010101; let isTerminator = 1; let isIndirectBranch = 1; let isBranch = 1; +let cofRelax1 = 1; +let cofRelax2 = 1; let cofMax1 = 1; } def J4_jumpseti : HInst< @@ -18210,16 +18212,6 @@ let opExtentBits = 18; let opExtentAlign = 2; let opNewValue = 1; } -def PS_trap1 : HInst< -(outs), -(ins u8_0Imm:$Ii), -"trap1(#$Ii)", -tc_53c851ab, TypeJ>, Enc_a51a9a, Requires<[HasPreV65]> { -let Inst{1-0} = 0b00; -let Inst{7-5} = 0b000; -let Inst{13-13} = 0b0; -let Inst{31-16} = 0b0101010010000000; -} def R6_release_at_vi : HInst< (outs), (ins IntRegs:$Rs32), @@ -18964,7 +18956,7 @@ def S2_cabacdecbin : HInst< (outs DoubleRegs:$Rdd32), (ins DoubleRegs:$Rss32, DoubleRegs:$Rtt32), "$Rdd32 = decbin($Rss32,$Rtt32)", -tc_db596beb, TypeS_3op>, Enc_a56825 { +tc_db596beb, TypeS_3op>, Enc_a56825, Requires<[UseCabac]> { let Inst{7-5} = 0b110; let Inst{13-13} = 0b0; let Inst{31-21} = 0b11000001110; @@ -26883,17 +26875,6 @@ let isPseudo = 1; let isCodeGenOnly = 1; let DecoderNamespace = "EXT_mmvec"; } -def V6_ldntnt0 : HInst< -(outs HvxVR:$Vd32), -(ins IntRegs:$Rt32), -"$Vd32 = vmem($Rt32):nt", -PSEUDO, TypeMAPPING>, Requires<[HasV62]> { -let hasNewValue = 1; -let opNewValue = 0; -let isPseudo = 1; -let isCodeGenOnly = 1; -let DecoderNamespace = "EXT_mmvec"; -} def V6_ldp0 : HInst< (outs HvxVR:$Vd32), (ins PredRegs:$Pv4, IntRegs:$Rt32), @@ -27312,6 +27293,30 @@ let isPseudo = 1; let isCodeGenOnly = 1; let DecoderNamespace = "EXT_mmvec"; } +def V6_v10mpyubs10 : HInst< +(outs HvxWR:$Vdd32), +(ins HvxWR:$Vuu32, HvxWR:$Vvv32, u1_0Imm:$Ii), +"$Vdd32.w = v10mpy($Vuu32.ub,$Vvv32.b,#$Ii)", +tc_f175e046, TypeCVI_VX>, Requires<[UseHVXV69]> { +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let isPseudo = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_v10mpyubs10_vxx : HInst< +(outs HvxWR:$Vxx32), +(ins HvxWR:$Vxx32in, HvxWR:$Vuu32, HvxWR:$Vvv32, u1_0Imm:$Ii), +"$Vxx32.w += v10mpy($Vuu32.ub,$Vvv32.b,#$Ii)", +tc_4942646a, TypeCVI_VX>, Requires<[UseHVXV69]> { +let hasNewValue = 1; +let opNewValue = 0; +let isAccumulator = 1; +let isCVI = 1; +let isPseudo = 1; +let DecoderNamespace = "EXT_mmvec"; +let Constraints = "$Vxx32 = $Vxx32in"; +} def V6_v6mpyhubs10 : HInst< (outs HvxWR:$Vdd32), (ins HvxWR:$Vuu32, HvxWR:$Vvv32, u2_0Imm:$Ii), @@ -27396,7 +27401,7 @@ def V6_vL32Ub_ai : HInst< (outs HvxVR:$Vd32), (ins IntRegs:$Rt32, s4_0Imm:$Ii), "$Vd32 = vmemu($Rt32+#$Ii)", -tc_a7e6707d, TypeCVI_VM_VP_LDU>, Enc_f3f408, Requires<[UseHVXV60]> { +tc_a7e6707d, TypeCVI_VM_VP_LDU>, Enc_f3f408, Requires<[UseHVXV60]>, PostInc_BaseImm { let Inst{7-5} = 0b111; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000000; @@ -27408,13 +27413,15 @@ let isCVLoad = 1; let isCVI = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; +let BaseOpcode = "V6_vL32Ub_ai"; +let CextOpcode = "V6_vL32Ub"; let DecoderNamespace = "EXT_mmvec"; } def V6_vL32Ub_pi : HInst< (outs HvxVR:$Vd32, IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii), "$Vd32 = vmemu($Rx32++#$Ii)", -tc_3c56e5ce, TypeCVI_VM_VP_LDU>, Enc_a255dc, Requires<[UseHVXV60]> { +tc_3c56e5ce, TypeCVI_VM_VP_LDU>, Enc_a255dc, Requires<[UseHVXV60]>, PostInc_BaseImm { let Inst{7-5} = 0b111; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001000; @@ -27427,6 +27434,7 @@ let isCVI = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_pi"; +let CextOpcode = "V6_vL32Ub"; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Rx32 = $Rx32in"; } @@ -27452,7 +27460,7 @@ def V6_vL32b_ai : HInst< (outs HvxVR:$Vd32), (ins IntRegs:$Rt32, s4_0Imm:$Ii), "$Vd32 = vmem($Rt32+#$Ii)", -tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel { +tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b000; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000000; @@ -27465,6 +27473,7 @@ let isCVI = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_ai"; +let CextOpcode = "V6_vL32b"; let isCVLoadable = 1; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; @@ -27473,7 +27482,7 @@ def V6_vL32b_cur_ai : HInst< (outs HvxVR:$Vd32), (ins IntRegs:$Rt32, s4_0Imm:$Ii), "$Vd32.cur = vmem($Rt32+#$Ii)", -tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel { +tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b001; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000000; @@ -27487,6 +27496,7 @@ let CVINew = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_cur_ai"; +let CextOpcode = "V6_vL32b_cur"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; } @@ -27560,7 +27570,7 @@ def V6_vL32b_cur_pi : HInst< (outs HvxVR:$Vd32, IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii), "$Vd32.cur = vmem($Rx32++#$Ii)", -tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel { +tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b001; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001000; @@ -27574,6 +27584,7 @@ let CVINew = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_cur_pi"; +let CextOpcode = "V6_vL32b_cur"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Rx32 = $Rx32in"; @@ -27729,7 +27740,7 @@ def V6_vL32b_nt_ai : HInst< (outs HvxVR:$Vd32), (ins IntRegs:$Rt32, s4_0Imm:$Ii), "$Vd32 = vmem($Rt32+#$Ii):nt", -tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel { +tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b000; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000010; @@ -27743,6 +27754,7 @@ let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_nt_ai"; +let CextOpcode = "V6_vL32b_nt"; let isCVLoadable = 1; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; @@ -27751,7 +27763,7 @@ def V6_vL32b_nt_cur_ai : HInst< (outs HvxVR:$Vd32), (ins IntRegs:$Rt32, s4_0Imm:$Ii), "$Vd32.cur = vmem($Rt32+#$Ii):nt", -tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel { +tc_c0749f3c, TypeCVI_VM_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b001; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000010; @@ -27766,6 +27778,7 @@ let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_nt_cur_ai"; +let CextOpcode = "V6_vL32b_nt_cur"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; } @@ -27842,7 +27855,7 @@ def V6_vL32b_nt_cur_pi : HInst< (outs HvxVR:$Vd32, IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii), "$Vd32.cur = vmem($Rx32++#$Ii):nt", -tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel { +tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b001; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001010; @@ -27857,6 +27870,7 @@ let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_nt_cur_pi"; +let CextOpcode = "V6_vL32b_nt_cur"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Rx32 = $Rx32in"; @@ -28019,7 +28033,7 @@ def V6_vL32b_nt_pi : HInst< (outs HvxVR:$Vd32, IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii), "$Vd32 = vmem($Rx32++#$Ii):nt", -tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel { +tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b000; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001010; @@ -28033,6 +28047,7 @@ let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_nt_pi"; +let CextOpcode = "V6_vL32b_nt"; let isCVLoadable = 1; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; @@ -28127,7 +28142,7 @@ def V6_vL32b_nt_tmp_ai : HInst< (outs HvxVR:$Vd32), (ins IntRegs:$Rt32, s4_0Imm:$Ii), "$Vd32.tmp = vmem($Rt32+#$Ii):nt", -tc_52447ecc, TypeCVI_VM_TMP_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel { +tc_52447ecc, TypeCVI_VM_TMP_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b010; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000010; @@ -28137,11 +28152,12 @@ let addrMode = BaseImmOffset; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_nt_tmp_ai"; +let CextOpcode = "V6_vL32b_nt_tmp"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; } @@ -28160,7 +28176,7 @@ let addrMode = BaseImmOffset; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; @@ -28183,7 +28199,7 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; @@ -28206,7 +28222,7 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; @@ -28218,7 +28234,7 @@ def V6_vL32b_nt_tmp_pi : HInst< (outs HvxVR:$Vd32, IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii), "$Vd32.tmp = vmem($Rx32++#$Ii):nt", -tc_663c80a7, TypeCVI_VM_TMP_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel { +tc_663c80a7, TypeCVI_VM_TMP_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b010; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001010; @@ -28228,11 +28244,12 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_nt_tmp_pi"; +let CextOpcode = "V6_vL32b_nt_tmp"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Rx32 = $Rx32in"; @@ -28250,7 +28267,7 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; @@ -28273,7 +28290,7 @@ let addrMode = BaseImmOffset; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; @@ -28295,7 +28312,7 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; @@ -28317,7 +28334,7 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isNonTemporal = 1; let isRestrictNoSlot1Store = 1; @@ -28329,7 +28346,7 @@ def V6_vL32b_pi : HInst< (outs HvxVR:$Vd32, IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii), "$Vd32 = vmem($Rx32++#$Ii)", -tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel { +tc_1ba8a0cd, TypeCVI_VM_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b000; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001000; @@ -28342,6 +28359,7 @@ let isCVI = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_pi"; +let CextOpcode = "V6_vL32b"; let isCVLoadable = 1; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; @@ -28432,7 +28450,7 @@ def V6_vL32b_tmp_ai : HInst< (outs HvxVR:$Vd32), (ins IntRegs:$Rt32, s4_0Imm:$Ii), "$Vd32.tmp = vmem($Rt32+#$Ii)", -tc_52447ecc, TypeCVI_VM_TMP_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel { +tc_52447ecc, TypeCVI_VM_TMP_LD>, Enc_f3f408, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b010; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000000; @@ -28442,10 +28460,11 @@ let addrMode = BaseImmOffset; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_tmp_ai"; +let CextOpcode = "V6_vL32b_tmp"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; } @@ -28464,7 +28483,7 @@ let addrMode = BaseImmOffset; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_tmp_ai"; @@ -28486,7 +28505,7 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_tmp_pi"; @@ -28508,7 +28527,7 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_tmp_ppu"; @@ -28519,7 +28538,7 @@ def V6_vL32b_tmp_pi : HInst< (outs HvxVR:$Vd32, IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii), "$Vd32.tmp = vmem($Rx32++#$Ii)", -tc_663c80a7, TypeCVI_VM_TMP_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel { +tc_663c80a7, TypeCVI_VM_TMP_LD>, Enc_a255dc, Requires<[UseHVXV60]>, PredRel, PostInc_BaseImm { let Inst{7-5} = 0b010; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001000; @@ -28529,10 +28548,11 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_tmp_pi"; +let CextOpcode = "V6_vL32b_tmp"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Rx32 = $Rx32in"; @@ -28550,7 +28570,7 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_tmp_ppu"; @@ -28572,7 +28592,7 @@ let addrMode = BaseImmOffset; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_tmp_ai"; @@ -28593,7 +28613,7 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_tmp_pi"; @@ -28614,7 +28634,7 @@ let addrMode = PostInc; let accessSize = HVXVectorAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; +let hasHvxTmp = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; let BaseOpcode = "V6_vL32b_tmp_ppu"; @@ -28625,7 +28645,7 @@ def V6_vS32Ub_ai : HInst< (outs), (ins IntRegs:$Rt32, s4_0Imm:$Ii, HvxVR:$Vs32), "vmemu($Rt32+#$Ii) = $Vs32", -tc_f21e8abb, TypeCVI_VM_STU>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel { +tc_f21e8abb, TypeCVI_VM_STU>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm { let Inst{7-5} = 0b111; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000001; @@ -28634,6 +28654,7 @@ let accessSize = HVXVectorAccess; let isCVI = 1; let mayStore = 1; let BaseOpcode = "V6_vS32Ub_ai"; +let CextOpcode = "V6_vS32Ub"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; } @@ -28692,7 +28713,7 @@ def V6_vS32Ub_pi : HInst< (outs IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii, HvxVR:$Vs32), "vmemu($Rx32++#$Ii) = $Vs32", -tc_e2d2e9e5, TypeCVI_VM_STU>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel { +tc_e2d2e9e5, TypeCVI_VM_STU>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm { let Inst{7-5} = 0b111; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001001; @@ -28701,6 +28722,7 @@ let accessSize = HVXVectorAccess; let isCVI = 1; let mayStore = 1; let BaseOpcode = "V6_vS32Ub_pi"; +let CextOpcode = "V6_vS32Ub"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Rx32 = $Rx32in"; @@ -28773,7 +28795,7 @@ def V6_vS32b_ai : HInst< (outs), (ins IntRegs:$Rt32, s4_0Imm:$Ii, HvxVR:$Vs32), "vmem($Rt32+#$Ii) = $Vs32", -tc_c5dba46e, TypeCVI_VM_ST>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel { +tc_c5dba46e, TypeCVI_VM_ST>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm { let Inst{7-5} = 0b000; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000001; @@ -28782,6 +28804,7 @@ let accessSize = HVXVectorAccess; let isCVI = 1; let mayStore = 1; let BaseOpcode = "V6_vS32b_ai"; +let CextOpcode = "V6_vS32b"; let isNVStorable = 1; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; @@ -28790,7 +28813,7 @@ def V6_vS32b_new_ai : HInst< (outs), (ins IntRegs:$Rt32, s4_0Imm:$Ii, HvxVR:$Os8), "vmem($Rt32+#$Ii) = $Os8.new", -tc_ab23f776, TypeCVI_VM_NEW_ST>, Enc_f77fbc, Requires<[UseHVXV60]>, NewValueRel { +tc_ab23f776, TypeCVI_VM_NEW_ST>, Enc_f77fbc, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm { let Inst{7-3} = 0b00100; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000001; @@ -28802,6 +28825,7 @@ let CVINew = 1; let isNewValue = 1; let mayStore = 1; let BaseOpcode = "V6_vS32b_ai"; +let CextOpcode = "V6_vS32b_new"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; let opNewValue = 2; @@ -28873,7 +28897,7 @@ def V6_vS32b_new_pi : HInst< (outs IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii, HvxVR:$Os8), "vmem($Rx32++#$Ii) = $Os8.new", -tc_6942b6e0, TypeCVI_VM_NEW_ST>, Enc_1aaec1, Requires<[UseHVXV60]>, NewValueRel { +tc_6942b6e0, TypeCVI_VM_NEW_ST>, Enc_1aaec1, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm { let Inst{7-3} = 0b00100; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001001; @@ -28885,6 +28909,7 @@ let CVINew = 1; let isNewValue = 1; let mayStore = 1; let BaseOpcode = "V6_vS32b_pi"; +let CextOpcode = "V6_vS32b_new"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; let opNewValue = 3; @@ -29070,7 +29095,7 @@ def V6_vS32b_nt_ai : HInst< (outs), (ins IntRegs:$Rt32, s4_0Imm:$Ii, HvxVR:$Vs32), "vmem($Rt32+#$Ii):nt = $Vs32", -tc_c5dba46e, TypeCVI_VM_ST>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel { +tc_c5dba46e, TypeCVI_VM_ST>, Enc_c9e3bc, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm { let Inst{7-5} = 0b000; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000011; @@ -29080,6 +29105,7 @@ let isCVI = 1; let isNonTemporal = 1; let mayStore = 1; let BaseOpcode = "V6_vS32b_ai"; +let CextOpcode = "V6_vS32b_nt"; let isNVStorable = 1; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; @@ -29088,7 +29114,7 @@ def V6_vS32b_nt_new_ai : HInst< (outs), (ins IntRegs:$Rt32, s4_0Imm:$Ii, HvxVR:$Os8), "vmem($Rt32+#$Ii):nt = $Os8.new", -tc_ab23f776, TypeCVI_VM_NEW_ST>, Enc_f77fbc, Requires<[UseHVXV60]>, NewValueRel { +tc_ab23f776, TypeCVI_VM_NEW_ST>, Enc_f77fbc, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm { let Inst{7-3} = 0b00100; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101000011; @@ -29101,6 +29127,7 @@ let isNewValue = 1; let isNonTemporal = 1; let mayStore = 1; let BaseOpcode = "V6_vS32b_ai"; +let CextOpcode = "V6_vS32b_nt_new"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; let opNewValue = 2; @@ -29175,7 +29202,7 @@ def V6_vS32b_nt_new_pi : HInst< (outs IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii, HvxVR:$Os8), "vmem($Rx32++#$Ii):nt = $Os8.new", -tc_6942b6e0, TypeCVI_VM_NEW_ST>, Enc_1aaec1, Requires<[UseHVXV60]>, NewValueRel { +tc_6942b6e0, TypeCVI_VM_NEW_ST>, Enc_1aaec1, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm { let Inst{7-3} = 0b00100; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001011; @@ -29188,6 +29215,7 @@ let isNewValue = 1; let isNonTemporal = 1; let mayStore = 1; let BaseOpcode = "V6_vS32b_pi"; +let CextOpcode = "V6_vS32b_nt_new"; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; let opNewValue = 3; @@ -29383,7 +29411,7 @@ def V6_vS32b_nt_pi : HInst< (outs IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii, HvxVR:$Vs32), "vmem($Rx32++#$Ii):nt = $Vs32", -tc_3e2aaafc, TypeCVI_VM_ST>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel { +tc_3e2aaafc, TypeCVI_VM_ST>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm { let Inst{7-5} = 0b000; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001011; @@ -29393,6 +29421,7 @@ let isCVI = 1; let isNonTemporal = 1; let mayStore = 1; let BaseOpcode = "V6_vS32b_pi"; +let CextOpcode = "V6_vS32b_nt"; let isNVStorable = 1; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; @@ -29519,7 +29548,7 @@ def V6_vS32b_pi : HInst< (outs IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii, HvxVR:$Vs32), "vmem($Rx32++#$Ii) = $Vs32", -tc_3e2aaafc, TypeCVI_VM_ST>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel { +tc_3e2aaafc, TypeCVI_VM_ST>, Enc_b62ef7, Requires<[UseHVXV60]>, NewValueRel, PostInc_BaseImm { let Inst{7-5} = 0b000; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101001001; @@ -29528,6 +29557,7 @@ let accessSize = HVXVectorAccess; let isCVI = 1; let mayStore = 1; let BaseOpcode = "V6_vS32b_pi"; +let CextOpcode = "V6_vS32b"; let isNVStorable = 1; let isPredicable = 1; let DecoderNamespace = "EXT_mmvec"; @@ -29689,6 +29719,32 @@ let mayStore = 1; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Rx32 = $Rx32in"; } +def V6_vabs_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.hf = vabs($Vu32.hf)", +tc_5cdf8c84, TypeCVI_VX_LATE>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b100; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vabs_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.sf = vabs($Vu32.sf)", +tc_5cdf8c84, TypeCVI_VX_LATE>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b101; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vabsb : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32), @@ -29975,6 +30031,123 @@ let isPseudo = 1; let isCodeGenOnly = 1; let DecoderNamespace = "EXT_mmvec"; } +def V6_vadd_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf16 = vadd($Vu32.hf,$Vv32.hf)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b011; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vadd_hf_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.hf = vadd($Vu32.hf,$Vv32.hf)", +tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b111; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111101; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vadd_qf16 : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf16 = vadd($Vu32.qf16,$Vv32.qf16)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b010; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vadd_qf16_mix : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf16 = vadd($Vu32.qf16,$Vv32.hf)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b100; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vadd_qf32 : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf32 = vadd($Vu32.qf32,$Vv32.qf32)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b000; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111101; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vadd_qf32_mix : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf32 = vadd($Vu32.qf32,$Vv32.sf)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b010; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111101; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vadd_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf32 = vadd($Vu32.sf,$Vv32.sf)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b001; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111101; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vadd_sf_hf : HInst< +(outs HvxWR:$Vdd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vdd32.sf = vadd($Vu32.hf,$Vv32.hf)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b100; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vadd_sf_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.sf = vadd($Vu32.sf,$Vv32.sf)", +tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b110; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vaddb : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, HvxVR:$Vv32), @@ -31440,6 +31613,58 @@ let opNewValue = 0; let isCVI = 1; let DecoderNamespace = "EXT_mmvec"; } +def V6_vasrvuhubrndsat : HInst< +(outs HvxVR:$Vd32), +(ins HvxWR:$Vuu32, HvxVR:$Vv32), +"$Vd32.ub = vasr($Vuu32.uh,$Vv32.ub):rnd:sat", +tc_05ca8cfd, TypeCVI_VS>, Enc_de5ea0, Requires<[UseHVXV69]> { +let Inst{7-5} = 0b011; +let Inst{13-13} = 0b0; +let Inst{31-21} = 0b00011101000; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vasrvuhubsat : HInst< +(outs HvxVR:$Vd32), +(ins HvxWR:$Vuu32, HvxVR:$Vv32), +"$Vd32.ub = vasr($Vuu32.uh,$Vv32.ub):sat", +tc_05ca8cfd, TypeCVI_VS>, Enc_de5ea0, Requires<[UseHVXV69]> { +let Inst{7-5} = 0b010; +let Inst{13-13} = 0b0; +let Inst{31-21} = 0b00011101000; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vasrvwuhrndsat : HInst< +(outs HvxVR:$Vd32), +(ins HvxWR:$Vuu32, HvxVR:$Vv32), +"$Vd32.uh = vasr($Vuu32.w,$Vv32.uh):rnd:sat", +tc_05ca8cfd, TypeCVI_VS>, Enc_de5ea0, Requires<[UseHVXV69]> { +let Inst{7-5} = 0b001; +let Inst{13-13} = 0b0; +let Inst{31-21} = 0b00011101000; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vasrvwuhsat : HInst< +(outs HvxVR:$Vd32), +(ins HvxWR:$Vuu32, HvxVR:$Vv32), +"$Vd32.uh = vasr($Vuu32.w,$Vv32.uh):sat", +tc_05ca8cfd, TypeCVI_VS>, Enc_de5ea0, Requires<[UseHVXV69]> { +let Inst{7-5} = 0b000; +let Inst{13-13} = 0b0; +let Inst{31-21} = 0b00011101000; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vasrw : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, IntRegs:$Rt32), @@ -31597,6 +31822,33 @@ let opNewValue = 0; let isCVI = 1; let DecoderNamespace = "EXT_mmvec"; } +def V6_vassign_fp : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.w = vfmv($Vu32.w)", +tc_5cdf8c84, TypeCVI_VX_LATE>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b001; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vassign_tmp : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.tmp = $Vu32", +tc_2120355e, TypeCVI_VX>, Enc_e7581c, Requires<[UseHVXV69]> { +let Inst{7-5} = 0b110; +let Inst{13-13} = 0b0; +let Inst{31-16} = 0b0001111000000001; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let hasHvxTmp = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vassignp : HInst< (outs HvxWR:$Vdd32), (ins HvxWR:$Vuu32), @@ -32000,6 +32252,189 @@ let isCVI = 1; let isRegSequence = 1; let DecoderNamespace = "EXT_mmvec"; } +def V6_vcombine_tmp : HInst< +(outs HvxWR:$Vdd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vdd32.tmp = vcombine($Vu32,$Vv32)", +tc_aa047364, TypeCVI_VX>, Enc_71bb9b, Requires<[UseHVXV69]> { +let Inst{7-5} = 0b111; +let Inst{13-13} = 0b0; +let Inst{31-21} = 0b00011110101; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let hasHvxTmp = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vconv_hf_qf16 : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.hf = $Vu32.qf16", +tc_51d0ecc3, TypeCVI_VS>, Enc_e7581c, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b011; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vconv_hf_qf32 : HInst< +(outs HvxVR:$Vd32), +(ins HvxWR:$Vuu32), +"$Vd32.hf = $Vuu32.qf32", +tc_51d0ecc3, TypeCVI_VS>, Enc_a33d04, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b110; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vconv_sf_qf32 : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.sf = $Vu32.qf32", +tc_51d0ecc3, TypeCVI_VS>, Enc_e7581c, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b000; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vcvt_b_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.b = vcvt($Vu32.hf,$Vv32.hf)", +tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b110; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vcvt_h_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.h = vcvt($Vu32.hf)", +tc_3c8c15d0, TypeCVI_VX>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b000; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vcvt_hf_b : HInst< +(outs HvxWR:$Vdd32), +(ins HvxVR:$Vu32), +"$Vdd32.hf = vcvt($Vu32.b)", +tc_0afc8be9, TypeCVI_VX_DV>, Enc_dd766a, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b010; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vcvt_hf_h : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.hf = vcvt($Vu32.h)", +tc_3c8c15d0, TypeCVI_VX>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b111; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vcvt_hf_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.hf = vcvt($Vu32.sf,$Vv32.sf)", +tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b001; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vcvt_hf_ub : HInst< +(outs HvxWR:$Vdd32), +(ins HvxVR:$Vu32), +"$Vdd32.hf = vcvt($Vu32.ub)", +tc_0afc8be9, TypeCVI_VX_DV>, Enc_dd766a, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b001; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vcvt_hf_uh : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.hf = vcvt($Vu32.uh)", +tc_3c8c15d0, TypeCVI_VX>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b101; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vcvt_sf_hf : HInst< +(outs HvxWR:$Vdd32), +(ins HvxVR:$Vu32), +"$Vdd32.sf = vcvt($Vu32.hf)", +tc_0afc8be9, TypeCVI_VX_DV>, Enc_dd766a, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b100; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vcvt_ub_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.ub = vcvt($Vu32.hf,$Vv32.hf)", +tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b101; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vcvt_uh_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.uh = vcvt($Vu32.hf)", +tc_3c8c15d0, TypeCVI_VX>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b000; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000101; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vd0 : HInst< (outs HvxVR:$Vd32), (ins), @@ -32141,6 +32576,34 @@ let opNewValue = 0; let isCVI = 1; let DecoderNamespace = "EXT_mmvec"; } +def V6_vdmpy_sf_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.sf = vdmpy($Vu32.hf,$Vv32.hf)", +tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b110; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111101; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vdmpy_sf_hf_acc : HInst< +(outs HvxVR:$Vx32), +(ins HvxVR:$Vx32in, HvxVR:$Vu32, HvxVR:$Vv32), +"$Vx32.sf += vdmpy($Vu32.hf,$Vv32.hf)", +tc_a19b9305, TypeCVI_VX>, Enc_a7341a, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b011; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100010; +let hasNewValue = 1; +let opNewValue = 0; +let isAccumulator = 1; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +let Constraints = "$Vx32 = $Vx32in"; +} def V6_vdmpybus : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, IntRegs:$Rt32), @@ -32415,7 +32878,7 @@ def V6_vdmpyhsat : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, IntRegs:$Rt32), "$Vd32.w = vdmpy($Vu32.h,$Rt32.h):sat", -tc_0b04c6c7, TypeCVI_VX_DV>, Enc_b087ac, Requires<[UseHVXV60]> { +tc_dcca380f, TypeCVI_VX>, Enc_b087ac, Requires<[UseHVXV60]> { let Inst{7-5} = 0b010; let Inst{13-13} = 0b0; let Inst{31-21} = 0b00011001001; @@ -32428,7 +32891,7 @@ def V6_vdmpyhsat_acc : HInst< (outs HvxVR:$Vx32), (ins HvxVR:$Vx32in, HvxVR:$Vu32, IntRegs:$Rt32), "$Vx32.w += vdmpy($Vu32.h,$Rt32.h):sat", -tc_660769f1, TypeCVI_VX_DV>, Enc_5138b3, Requires<[UseHVXV60]> { +tc_72e2b393, TypeCVI_VX>, Enc_5138b3, Requires<[UseHVXV60]> { let Inst{7-5} = 0b011; let Inst{13-13} = 0b1; let Inst{31-21} = 0b00011001001; @@ -32523,7 +32986,7 @@ def V6_vdmpyhsusat : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, IntRegs:$Rt32), "$Vd32.w = vdmpy($Vu32.h,$Rt32.uh):sat", -tc_0b04c6c7, TypeCVI_VX_DV>, Enc_b087ac, Requires<[UseHVXV60]> { +tc_dcca380f, TypeCVI_VX>, Enc_b087ac, Requires<[UseHVXV60]> { let Inst{7-5} = 0b000; let Inst{13-13} = 0b0; let Inst{31-21} = 0b00011001001; @@ -32536,7 +32999,7 @@ def V6_vdmpyhsusat_acc : HInst< (outs HvxVR:$Vx32), (ins HvxVR:$Vx32in, HvxVR:$Vu32, IntRegs:$Rt32), "$Vx32.w += vdmpy($Vu32.h,$Rt32.uh):sat", -tc_660769f1, TypeCVI_VX_DV>, Enc_5138b3, Requires<[UseHVXV60]> { +tc_72e2b393, TypeCVI_VX>, Enc_5138b3, Requires<[UseHVXV60]> { let Inst{7-5} = 0b000; let Inst{13-13} = 0b1; let Inst{31-21} = 0b00011001001; @@ -32577,7 +33040,7 @@ def V6_vdmpyhvsat : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, HvxVR:$Vv32), "$Vd32.w = vdmpy($Vu32.h,$Vv32.h):sat", -tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV60]> { +tc_73efe966, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV60]> { let Inst{7-5} = 0b011; let Inst{13-13} = 0b0; let Inst{31-21} = 0b00011100000; @@ -32831,6 +33294,84 @@ let isCVI = 1; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Qx4 = $Qx4in"; } +def V6_vfmax_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.hf = vfmax($Vu32.hf,$Vv32.hf)", +tc_cda936da, TypeCVI_VX_LATE>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b010; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vfmax_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.sf = vfmax($Vu32.sf,$Vv32.sf)", +tc_cda936da, TypeCVI_VX_LATE>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b011; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vfmin_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.hf = vfmin($Vu32.hf,$Vv32.hf)", +tc_cda936da, TypeCVI_VX_LATE>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b000; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vfmin_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.sf = vfmin($Vu32.sf,$Vv32.sf)", +tc_cda936da, TypeCVI_VX_LATE>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b001; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vfneg_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.hf = vfneg($Vu32.hf)", +tc_5cdf8c84, TypeCVI_VX_LATE>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b010; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vfneg_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32), +"$Vd32.sf = vfneg($Vu32.sf)", +tc_5cdf8c84, TypeCVI_VX_LATE>, Enc_e7581c, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b011; +let Inst{13-13} = 0b1; +let Inst{31-16} = 0b0001111000000110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vgathermh : HInst< (outs), (ins IntRegs:$Rt32, ModRegs:$Mu2, HvxVR:$Vv32), @@ -32843,7 +33384,6 @@ let opNewValue = 0; let accessSize = HalfWordAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; let mayLoad = 1; let Defs = [VTMP]; let DecoderNamespace = "EXT_mmvec"; @@ -32860,7 +33400,6 @@ let opNewValue = 0; let accessSize = HalfWordAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; let mayLoad = 1; let Defs = [VTMP]; let DecoderNamespace = "EXT_mmvec"; @@ -32877,7 +33416,6 @@ let opNewValue = 0; let accessSize = HalfWordAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; let mayLoad = 1; let Defs = [VTMP]; let DecoderNamespace = "EXT_mmvec"; @@ -32894,7 +33432,6 @@ let opNewValue = 0; let accessSize = HalfWordAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; let mayLoad = 1; let Defs = [VTMP]; let DecoderNamespace = "EXT_mmvec"; @@ -32911,7 +33448,6 @@ let opNewValue = 0; let accessSize = WordAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; let mayLoad = 1; let Defs = [VTMP]; let DecoderNamespace = "EXT_mmvec"; @@ -32928,7 +33464,6 @@ let opNewValue = 0; let accessSize = WordAccess; let isCVLoad = 1; let isCVI = 1; -let hasTmpDst = 1; let mayLoad = 1; let Defs = [VTMP]; let DecoderNamespace = "EXT_mmvec"; @@ -33033,6 +33568,106 @@ let isCVI = 1; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Qx4 = $Qx4in"; } +def V6_vgthf : HInst< +(outs HvxQR:$Qd4), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Qd4 = vcmp.gt($Vu32.hf,$Vv32.hf)", +tc_56c4f9fe, TypeCVI_VA>, Enc_95441f, Requires<[UseHVXV68,UseHVXFloatingPoint]> { +let Inst{7-2} = 0b011101; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vgthf_and : HInst< +(outs HvxQR:$Qx4), +(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32), +"$Qx4 &= vcmp.gt($Vu32.hf,$Vv32.hf)", +tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> { +let Inst{7-2} = 0b110011; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100100; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +let Constraints = "$Qx4 = $Qx4in"; +} +def V6_vgthf_or : HInst< +(outs HvxQR:$Qx4), +(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32), +"$Qx4 |= vcmp.gt($Vu32.hf,$Vv32.hf)", +tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> { +let Inst{7-2} = 0b001101; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100100; +let isAccumulator = 1; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +let Constraints = "$Qx4 = $Qx4in"; +} +def V6_vgthf_xor : HInst< +(outs HvxQR:$Qx4), +(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32), +"$Qx4 ^= vcmp.gt($Vu32.hf,$Vv32.hf)", +tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> { +let Inst{7-2} = 0b111011; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100100; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +let Constraints = "$Qx4 = $Qx4in"; +} +def V6_vgtsf : HInst< +(outs HvxQR:$Qd4), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Qd4 = vcmp.gt($Vu32.sf,$Vv32.sf)", +tc_56c4f9fe, TypeCVI_VA>, Enc_95441f, Requires<[UseHVXV68,UseHVXFloatingPoint]> { +let Inst{7-2} = 0b011100; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vgtsf_and : HInst< +(outs HvxQR:$Qx4), +(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32), +"$Qx4 &= vcmp.gt($Vu32.sf,$Vv32.sf)", +tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> { +let Inst{7-2} = 0b110010; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100100; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +let Constraints = "$Qx4 = $Qx4in"; +} +def V6_vgtsf_or : HInst< +(outs HvxQR:$Qx4), +(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32), +"$Qx4 |= vcmp.gt($Vu32.sf,$Vv32.sf)", +tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> { +let Inst{7-2} = 0b001100; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100100; +let isAccumulator = 1; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +let Constraints = "$Qx4 = $Qx4in"; +} +def V6_vgtsf_xor : HInst< +(outs HvxQR:$Qx4), +(ins HvxQR:$Qx4in, HvxVR:$Vu32, HvxVR:$Vv32), +"$Qx4 ^= vcmp.gt($Vu32.sf,$Vv32.sf)", +tc_257f6f7c, TypeCVI_VA>, Enc_eaa9f8, Requires<[UseHVXV68,UseHVXFloatingPoint]> { +let Inst{7-2} = 0b111010; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100100; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +let Constraints = "$Qx4 = $Qx4in"; +} def V6_vgtub : HInst< (outs HvxQR:$Qd4), (ins HvxVR:$Vu32, HvxVR:$Vv32), @@ -33552,6 +34187,32 @@ let opNewValue = 0; let isCVI = 1; let DecoderNamespace = "EXT_mmvec"; } +def V6_vmax_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.hf = vmax($Vu32.hf,$Vv32.hf)", +tc_56c4f9fe, TypeCVI_VA>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b011; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmax_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.sf = vmax($Vu32.sf,$Vv32.sf)", +tc_56c4f9fe, TypeCVI_VA>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b001; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vmaxb : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, HvxVR:$Vv32), @@ -33677,6 +34338,32 @@ let isPseudo = 1; let isCodeGenOnly = 1; let DecoderNamespace = "EXT_mmvec"; } +def V6_vmin_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.hf = vmin($Vu32.hf,$Vv32.hf)", +tc_56c4f9fe, TypeCVI_VA>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b100; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmin_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.sf = vmin($Vu32.sf,$Vv32.sf)", +tc_56c4f9fe, TypeCVI_VA>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b010; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vminb : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, HvxVR:$Vv32), @@ -34110,6 +34797,179 @@ let isCVI = 1; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Vx32 = $Vx32in"; } +def V6_vmpy_hf_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.hf = vmpy($Vu32.hf,$Vv32.hf)", +tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b011; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmpy_hf_hf_acc : HInst< +(outs HvxVR:$Vx32), +(ins HvxVR:$Vx32in, HvxVR:$Vu32, HvxVR:$Vv32), +"$Vx32.hf += vmpy($Vu32.hf,$Vv32.hf)", +tc_a19b9305, TypeCVI_VX>, Enc_a7341a, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b010; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100010; +let hasNewValue = 1; +let opNewValue = 0; +let isAccumulator = 1; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +let Constraints = "$Vx32 = $Vx32in"; +} +def V6_vmpy_qf16 : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf16 = vmpy($Vu32.qf16,$Vv32.qf16)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b011; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111111; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmpy_qf16_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf16 = vmpy($Vu32.hf,$Vv32.hf)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b100; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111111; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmpy_qf16_mix_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf16 = vmpy($Vu32.qf16,$Vv32.hf)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b101; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111111; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmpy_qf32 : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf32 = vmpy($Vu32.qf32,$Vv32.qf32)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b000; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111111; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmpy_qf32_hf : HInst< +(outs HvxWR:$Vdd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vdd32.qf32 = vmpy($Vu32.hf,$Vv32.hf)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b111; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111111; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmpy_qf32_mix_hf : HInst< +(outs HvxWR:$Vdd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vdd32.qf32 = vmpy($Vu32.qf16,$Vv32.hf)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b000; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmpy_qf32_qf16 : HInst< +(outs HvxWR:$Vdd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vdd32.qf32 = vmpy($Vu32.qf16,$Vv32.qf16)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b110; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111111; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmpy_qf32_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf32 = vmpy($Vu32.sf,$Vv32.sf)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b001; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111111; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmpy_sf_hf : HInst< +(outs HvxWR:$Vdd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vdd32.sf = vmpy($Vu32.hf,$Vv32.hf)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b010; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vmpy_sf_hf_acc : HInst< +(outs HvxWR:$Vxx32), +(ins HvxWR:$Vxx32in, HvxVR:$Vu32, HvxVR:$Vv32), +"$Vxx32.sf += vmpy($Vu32.hf,$Vv32.hf)", +tc_08a4f1b6, TypeCVI_VX_DV>, Enc_3fc427, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b001; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011100010; +let hasNewValue = 1; +let opNewValue = 0; +let isAccumulator = 1; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +let Constraints = "$Vxx32 = $Vxx32in"; +} +def V6_vmpy_sf_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.sf = vmpy($Vu32.sf,$Vv32.sf)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b001; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vmpybus : HInst< (outs HvxWR:$Vdd32), (ins HvxVR:$Vu32, IntRegs:$Rt32), @@ -34397,7 +35257,7 @@ def V6_vmpyhsrs : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, IntRegs:$Rt32), "$Vd32.h = vmpy($Vu32.h,$Rt32.h):<<1:rnd:sat", -tc_0b04c6c7, TypeCVI_VX_DV>, Enc_b087ac, Requires<[UseHVXV60]> { +tc_dcca380f, TypeCVI_VX>, Enc_b087ac, Requires<[UseHVXV60]> { let Inst{7-5} = 0b010; let Inst{13-13} = 0b0; let Inst{31-21} = 0b00011001010; @@ -34422,7 +35282,7 @@ def V6_vmpyhss : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, IntRegs:$Rt32), "$Vd32.h = vmpy($Vu32.h,$Rt32.h):<<1:sat", -tc_0b04c6c7, TypeCVI_VX_DV>, Enc_b087ac, Requires<[UseHVXV60]> { +tc_dcca380f, TypeCVI_VX>, Enc_b087ac, Requires<[UseHVXV60]> { let Inst{7-5} = 0b001; let Inst{13-13} = 0b0; let Inst{31-21} = 0b00011001010; @@ -34555,7 +35415,7 @@ def V6_vmpyhvsrs : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, HvxVR:$Vv32), "$Vd32.h = vmpy($Vu32.h,$Vv32.h):<<1:rnd:sat", -tc_d8287c14, TypeCVI_VX_DV>, Enc_45364e, Requires<[UseHVXV60]> { +tc_73efe966, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV60]> { let Inst{7-5} = 0b001; let Inst{13-13} = 0b0; let Inst{31-21} = 0b00011100001; @@ -35332,6 +36192,19 @@ let isPseudo = 1; let isCodeGenOnly = 1; let DecoderNamespace = "EXT_mmvec"; } +def V6_vmpyuhvs : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.uh = vmpy($Vu32.uh,$Vv32.uh):>>16", +tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV69]> { +let Inst{7-5} = 0b111; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111110; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vmux : HInst< (outs HvxVR:$Vd32), (ins HvxQR:$Qt4, HvxVR:$Vu32, HvxVR:$Vv32), @@ -36007,7 +36880,7 @@ def V6_vrmpybusv_acc : HInst< (outs HvxVR:$Vx32), (ins HvxVR:$Vx32in, HvxVR:$Vu32, HvxVR:$Vv32), "$Vx32.w += vrmpy($Vu32.ub,$Vv32.b)", -tc_08a4f1b6, TypeCVI_VX_DV>, Enc_a7341a, Requires<[UseHVXV60]> { +tc_37820f4c, TypeCVI_VX>, Enc_a7341a, Requires<[UseHVXV60]> { let Inst{7-5} = 0b010; let Inst{13-13} = 0b1; let Inst{31-21} = 0b00011100000; @@ -36061,7 +36934,7 @@ def V6_vrmpybv_acc : HInst< (outs HvxVR:$Vx32), (ins HvxVR:$Vx32in, HvxVR:$Vu32, HvxVR:$Vv32), "$Vx32.w += vrmpy($Vu32.b,$Vv32.b)", -tc_08a4f1b6, TypeCVI_VX_DV>, Enc_a7341a, Requires<[UseHVXV60]> { +tc_37820f4c, TypeCVI_VX>, Enc_a7341a, Requires<[UseHVXV60]> { let Inst{7-5} = 0b001; let Inst{13-13} = 0b1; let Inst{31-21} = 0b00011100000; @@ -36277,7 +37150,7 @@ def V6_vrmpyubv_acc : HInst< (outs HvxVR:$Vx32), (ins HvxVR:$Vx32in, HvxVR:$Vu32, HvxVR:$Vv32), "$Vx32.uw += vrmpy($Vu32.ub,$Vv32.ub)", -tc_08a4f1b6, TypeCVI_VX_DV>, Enc_a7341a, Requires<[UseHVXV60]> { +tc_37820f4c, TypeCVI_VX>, Enc_a7341a, Requires<[UseHVXV60]> { let Inst{7-5} = 0b000; let Inst{13-13} = 0b1; let Inst{31-21} = 0b00011100000; @@ -37412,6 +38285,123 @@ let isPseudo = 1; let isCodeGenOnly = 1; let DecoderNamespace = "EXT_mmvec"; } +def V6_vsub_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf16 = vsub($Vu32.hf,$Vv32.hf)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b110; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vsub_hf_hf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.hf = vsub($Vu32.hf,$Vv32.hf)", +tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b000; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vsub_qf16 : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf16 = vsub($Vu32.qf16,$Vv32.qf16)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b101; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vsub_qf16_mix : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf16 = vsub($Vu32.qf16,$Vv32.hf)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b111; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111011; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vsub_qf32 : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf32 = vsub($Vu32.qf32,$Vv32.qf32)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b011; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111101; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vsub_qf32_mix : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf32 = vsub($Vu32.qf32,$Vv32.sf)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b101; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111101; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vsub_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.qf32 = vsub($Vu32.sf,$Vv32.sf)", +tc_05ca8cfd, TypeCVI_VS>, Enc_45364e, Requires<[UseHVXV68,UseHVXQFloat]> { +let Inst{7-5} = 0b100; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111101; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vsub_sf_hf : HInst< +(outs HvxWR:$Vdd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vdd32.sf = vsub($Vu32.hf,$Vv32.hf)", +tc_d8287c14, TypeCVI_VX_DV>, Enc_71bb9b, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b101; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} +def V6_vsub_sf_sf : HInst< +(outs HvxVR:$Vd32), +(ins HvxVR:$Vu32, HvxVR:$Vv32), +"$Vd32.sf = vsub($Vu32.sf,$Vv32.sf)", +tc_c127de3a, TypeCVI_VX>, Enc_45364e, Requires<[UseHVXV68,UseHVXIEEEFP]> { +let Inst{7-5} = 0b111; +let Inst{13-13} = 0b1; +let Inst{31-21} = 0b00011111100; +let hasNewValue = 1; +let opNewValue = 0; +let isCVI = 1; +let DecoderNamespace = "EXT_mmvec"; +} def V6_vsubb : HInst< (outs HvxVR:$Vd32), (ins HvxVR:$Vu32, HvxVR:$Vv32), @@ -38647,7 +39637,7 @@ def V6_zLd_ai : HInst< (outs), (ins IntRegs:$Rt32, s4_0Imm:$Ii), "z = vmem($Rt32+#$Ii)", -tc_e699ae41, TypeCVI_ZW>, Enc_ff3442, Requires<[UseHVXV66,UseZReg]> { +tc_e699ae41, TypeCVI_ZW>, Enc_ff3442, Requires<[UseHVXV66,UseZReg]>, PostInc_BaseImm { let Inst{7-0} = 0b00000000; let Inst{12-11} = 0b00; let Inst{31-21} = 0b00101100000; @@ -38655,13 +39645,14 @@ let addrMode = BaseImmOffset; let isCVI = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; +let CextOpcode = "V6_zLd"; let DecoderNamespace = "EXT_mmvec"; } def V6_zLd_pi : HInst< (outs IntRegs:$Rx32), (ins IntRegs:$Rx32in, s3_0Imm:$Ii), "z = vmem($Rx32++#$Ii)", -tc_a0dbea28, TypeCVI_ZW>, Enc_6c9ee0, Requires<[UseHVXV66,UseZReg]> { +tc_a0dbea28, TypeCVI_ZW>, Enc_6c9ee0, Requires<[UseHVXV66,UseZReg]>, PostInc_BaseImm { let Inst{7-0} = 0b00000000; let Inst{13-11} = 0b000; let Inst{31-21} = 0b00101101000; @@ -38669,6 +39660,7 @@ let addrMode = PostInc; let isCVI = 1; let mayLoad = 1; let isRestrictNoSlot1Store = 1; +let CextOpcode = "V6_zLd"; let DecoderNamespace = "EXT_mmvec"; let Constraints = "$Rx32 = $Rx32in"; } @@ -38782,6 +39774,17 @@ let Inst{13-0} = 0b00000000000000; let Inst{31-16} = 0b0110110000100000; let isSolo = 1; } +def Y2_crswap_old : HInst< +(outs IntRegs:$Rx32), +(ins IntRegs:$Rx32in), +"crswap($Rx32,sgp)", +PSEUDO, TypeMAPPING> { +let hasNewValue = 1; +let opNewValue = 0; +let isPseudo = 1; +let isCodeGenOnly = 1; +let Constraints = "$Rx32 = $Rx32in"; +} def Y2_dccleana : HInst< (outs), (ins IntRegs:$Rs32), @@ -38861,6 +39864,22 @@ let Inst{13-0} = 0b00000000000010; let Inst{31-16} = 0b0101011111000000; let isSolo = 1; } +def Y2_k1lock_map : HInst< +(outs), +(ins), +"k1lock", +PSEUDO, TypeMAPPING>, Requires<[HasV65]> { +let isPseudo = 1; +let isCodeGenOnly = 1; +} +def Y2_k1unlock_map : HInst< +(outs), +(ins), +"k1unlock", +PSEUDO, TypeMAPPING>, Requires<[HasV65]> { +let isPseudo = 1; +let isCodeGenOnly = 1; +} def Y2_syncht : HInst< (outs), (ins), @@ -39083,7 +40102,7 @@ def dup_A2_add : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32, IntRegs:$Rt32), "$Rd32 = add($Rs32,$Rt32)", -tc_388f9897, TypeALU32_3op>, Requires<[HasV68]> { +tc_388f9897, TypeALU32_3op>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let AsmVariantName = "NonParsable"; @@ -39093,7 +40112,7 @@ def dup_A2_addi : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32, s32_0Imm:$Ii), "$Rd32 = add($Rs32,#$Ii)", -tc_388f9897, TypeALU32_ADDI>, Requires<[HasV68]> { +tc_388f9897, TypeALU32_ADDI>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let AsmVariantName = "NonParsable"; @@ -39108,7 +40127,7 @@ def dup_A2_andir : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32, s32_0Imm:$Ii), "$Rd32 = and($Rs32,#$Ii)", -tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> { +tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let AsmVariantName = "NonParsable"; @@ -39123,7 +40142,7 @@ def dup_A2_combineii : HInst< (outs DoubleRegs:$Rdd32), (ins s32_0Imm:$Ii, s8_0Imm:$II), "$Rdd32 = combine(#$Ii,#$II)", -tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> { +tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> { let AsmVariantName = "NonParsable"; let isPseudo = 1; let isExtendable = 1; @@ -39136,7 +40155,7 @@ def dup_A2_sxtb : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32), "$Rd32 = sxtb($Rs32)", -tc_9124c04f, TypeALU32_2op>, Requires<[HasV68]> { +tc_9124c04f, TypeALU32_2op>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let AsmVariantName = "NonParsable"; @@ -39146,7 +40165,7 @@ def dup_A2_sxth : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32), "$Rd32 = sxth($Rs32)", -tc_9124c04f, TypeALU32_2op>, Requires<[HasV68]> { +tc_9124c04f, TypeALU32_2op>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let AsmVariantName = "NonParsable"; @@ -39156,7 +40175,7 @@ def dup_A2_tfr : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32), "$Rd32 = $Rs32", -tc_9124c04f, TypeALU32_2op>, Requires<[HasV68]> { +tc_9124c04f, TypeALU32_2op>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let AsmVariantName = "NonParsable"; @@ -39166,7 +40185,7 @@ def dup_A2_tfrsi : HInst< (outs IntRegs:$Rd32), (ins s32_0Imm:$Ii), "$Rd32 = #$Ii", -tc_9124c04f, TypeALU32_2op>, Requires<[HasV68]> { +tc_9124c04f, TypeALU32_2op>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let AsmVariantName = "NonParsable"; @@ -39181,7 +40200,7 @@ def dup_A2_zxtb : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32), "$Rd32 = zxtb($Rs32)", -PSEUDO, TypeMAPPING>, Requires<[HasV68]> { +PSEUDO, TypeMAPPING>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let AsmVariantName = "NonParsable"; @@ -39191,7 +40210,7 @@ def dup_A2_zxth : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32), "$Rd32 = zxth($Rs32)", -tc_9124c04f, TypeALU32_2op>, Requires<[HasV68]> { +tc_9124c04f, TypeALU32_2op>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let AsmVariantName = "NonParsable"; @@ -39201,7 +40220,7 @@ def dup_A4_combineii : HInst< (outs DoubleRegs:$Rdd32), (ins s8_0Imm:$Ii, u32_0Imm:$II), "$Rdd32 = combine(#$Ii,#$II)", -tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> { +tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> { let AsmVariantName = "NonParsable"; let isPseudo = 1; let isExtendable = 1; @@ -39214,7 +40233,7 @@ def dup_A4_combineir : HInst< (outs DoubleRegs:$Rdd32), (ins s32_0Imm:$Ii, IntRegs:$Rs32), "$Rdd32 = combine(#$Ii,$Rs32)", -tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> { +tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> { let AsmVariantName = "NonParsable"; let isPseudo = 1; let isExtendable = 1; @@ -39227,7 +40246,7 @@ def dup_A4_combineri : HInst< (outs DoubleRegs:$Rdd32), (ins IntRegs:$Rs32, s32_0Imm:$Ii), "$Rdd32 = combine($Rs32,#$Ii)", -tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> { +tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> { let AsmVariantName = "NonParsable"; let isPseudo = 1; let isExtendable = 1; @@ -39240,7 +40259,7 @@ def dup_C2_cmoveif : HInst< (outs IntRegs:$Rd32), (ins PredRegs:$Pu4, s32_0Imm:$Ii), "if (!$Pu4) $Rd32 = #$Ii", -tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> { +tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> { let isPredicated = 1; let isPredicatedFalse = 1; let hasNewValue = 1; @@ -39257,7 +40276,7 @@ def dup_C2_cmoveit : HInst< (outs IntRegs:$Rd32), (ins PredRegs:$Pu4, s32_0Imm:$Ii), "if ($Pu4) $Rd32 = #$Ii", -tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> { +tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> { let isPredicated = 1; let hasNewValue = 1; let opNewValue = 0; @@ -39273,7 +40292,7 @@ def dup_C2_cmovenewif : HInst< (outs IntRegs:$Rd32), (ins PredRegs:$Pu4, s32_0Imm:$Ii), "if (!$Pu4.new) $Rd32 = #$Ii", -tc_4ac61d92, TypeALU32_2op>, Requires<[HasV68]> { +tc_4ac61d92, TypeALU32_2op>, Requires<[HasV69]> { let isPredicated = 1; let isPredicatedFalse = 1; let hasNewValue = 1; @@ -39291,7 +40310,7 @@ def dup_C2_cmovenewit : HInst< (outs IntRegs:$Rd32), (ins PredRegs:$Pu4, s32_0Imm:$Ii), "if ($Pu4.new) $Rd32 = #$Ii", -tc_4ac61d92, TypeALU32_2op>, Requires<[HasV68]> { +tc_4ac61d92, TypeALU32_2op>, Requires<[HasV69]> { let isPredicated = 1; let hasNewValue = 1; let opNewValue = 0; @@ -39308,7 +40327,7 @@ def dup_C2_cmpeqi : HInst< (outs PredRegs:$Pd4), (ins IntRegs:$Rs32, s32_0Imm:$Ii), "$Pd4 = cmp.eq($Rs32,#$Ii)", -tc_388f9897, TypeALU32_2op>, Requires<[HasV68]> { +tc_388f9897, TypeALU32_2op>, Requires<[HasV69]> { let AsmVariantName = "NonParsable"; let isPseudo = 1; let isExtendable = 1; @@ -39321,7 +40340,7 @@ def dup_L2_deallocframe : HInst< (outs DoubleRegs:$Rdd32), (ins IntRegs:$Rs32), "$Rdd32 = deallocframe($Rs32):raw", -tc_aee6250c, TypeLD>, Requires<[HasV68]> { +tc_aee6250c, TypeLD>, Requires<[HasV69]> { let accessSize = DoubleWordAccess; let AsmVariantName = "NonParsable"; let mayLoad = 1; @@ -39333,7 +40352,7 @@ def dup_L2_loadrb_io : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32, s32_0Imm:$Ii), "$Rd32 = memb($Rs32+#$Ii)", -tc_eed07714, TypeLD>, Requires<[HasV68]> { +tc_eed07714, TypeLD>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let addrMode = BaseImmOffset; @@ -39351,7 +40370,7 @@ def dup_L2_loadrd_io : HInst< (outs DoubleRegs:$Rdd32), (ins IntRegs:$Rs32, s29_3Imm:$Ii), "$Rdd32 = memd($Rs32+#$Ii)", -tc_eed07714, TypeLD>, Requires<[HasV68]> { +tc_eed07714, TypeLD>, Requires<[HasV69]> { let addrMode = BaseImmOffset; let accessSize = DoubleWordAccess; let AsmVariantName = "NonParsable"; @@ -39367,7 +40386,7 @@ def dup_L2_loadrh_io : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32, s31_1Imm:$Ii), "$Rd32 = memh($Rs32+#$Ii)", -tc_eed07714, TypeLD>, Requires<[HasV68]> { +tc_eed07714, TypeLD>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let addrMode = BaseImmOffset; @@ -39385,7 +40404,7 @@ def dup_L2_loadri_io : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32, s30_2Imm:$Ii), "$Rd32 = memw($Rs32+#$Ii)", -tc_eed07714, TypeLD>, Requires<[HasV68]> { +tc_eed07714, TypeLD>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let addrMode = BaseImmOffset; @@ -39403,7 +40422,7 @@ def dup_L2_loadrub_io : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32, s32_0Imm:$Ii), "$Rd32 = memub($Rs32+#$Ii)", -tc_eed07714, TypeLD>, Requires<[HasV68]> { +tc_eed07714, TypeLD>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let addrMode = BaseImmOffset; @@ -39421,7 +40440,7 @@ def dup_L2_loadruh_io : HInst< (outs IntRegs:$Rd32), (ins IntRegs:$Rs32, s31_1Imm:$Ii), "$Rd32 = memuh($Rs32+#$Ii)", -tc_eed07714, TypeLD>, Requires<[HasV68]> { +tc_eed07714, TypeLD>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let addrMode = BaseImmOffset; @@ -39439,7 +40458,7 @@ def dup_S2_allocframe : HInst< (outs IntRegs:$Rx32), (ins IntRegs:$Rx32in, u11_3Imm:$Ii), "allocframe($Rx32,#$Ii):raw", -tc_74a42bda, TypeST>, Requires<[HasV68]> { +tc_74a42bda, TypeST>, Requires<[HasV69]> { let hasNewValue = 1; let opNewValue = 0; let addrMode = BaseImmOffset; @@ -39455,7 +40474,7 @@ def dup_S2_storerb_io : HInst< (outs), (ins IntRegs:$Rs32, s32_0Imm:$Ii, IntRegs:$Rt32), "memb($Rs32+#$Ii) = $Rt32", -tc_a9edeffa, TypeST>, Requires<[HasV68]> { +tc_a9edeffa, TypeST>, Requires<[HasV69]> { let addrMode = BaseImmOffset; let accessSize = ByteAccess; let AsmVariantName = "NonParsable"; @@ -39471,7 +40490,7 @@ def dup_S2_storerd_io : HInst< (outs), (ins IntRegs:$Rs32, s29_3Imm:$Ii, DoubleRegs:$Rtt32), "memd($Rs32+#$Ii) = $Rtt32", -tc_a9edeffa, TypeST>, Requires<[HasV68]> { +tc_a9edeffa, TypeST>, Requires<[HasV69]> { let addrMode = BaseImmOffset; let accessSize = DoubleWordAccess; let AsmVariantName = "NonParsable"; @@ -39487,7 +40506,7 @@ def dup_S2_storerh_io : HInst< (outs), (ins IntRegs:$Rs32, s31_1Imm:$Ii, IntRegs:$Rt32), "memh($Rs32+#$Ii) = $Rt32", -tc_a9edeffa, TypeST>, Requires<[HasV68]> { +tc_a9edeffa, TypeST>, Requires<[HasV69]> { let addrMode = BaseImmOffset; let accessSize = HalfWordAccess; let AsmVariantName = "NonParsable"; @@ -39503,7 +40522,7 @@ def dup_S2_storeri_io : HInst< (outs), (ins IntRegs:$Rs32, s30_2Imm:$Ii, IntRegs:$Rt32), "memw($Rs32+#$Ii) = $Rt32", -tc_a9edeffa, TypeST>, Requires<[HasV68]> { +tc_a9edeffa, TypeST>, Requires<[HasV69]> { let addrMode = BaseImmOffset; let accessSize = WordAccess; let AsmVariantName = "NonParsable"; @@ -39519,7 +40538,7 @@ def dup_S4_storeirb_io : HInst< (outs), (ins IntRegs:$Rs32, u6_0Imm:$Ii, s32_0Imm:$II), "memb($Rs32+#$Ii) = #$II", -tc_838c4d7a, TypeV4LDST>, Requires<[HasV68]> { +tc_838c4d7a, TypeV4LDST>, Requires<[HasV69]> { let addrMode = BaseImmOffset; let accessSize = ByteAccess; let AsmVariantName = "NonParsable"; @@ -39535,7 +40554,7 @@ def dup_S4_storeiri_io : HInst< (outs), (ins IntRegs:$Rs32, u6_2Imm:$Ii, s32_0Imm:$II), "memw($Rs32+#$Ii) = #$II", -tc_838c4d7a, TypeV4LDST>, Requires<[HasV68]> { +tc_838c4d7a, TypeV4LDST>, Requires<[HasV69]> { let addrMode = BaseImmOffset; let accessSize = WordAccess; let AsmVariantName = "NonParsable"; diff --git a/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td b/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td index e5c78d122c9e..64bc5091d1d1 100644 --- a/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td +++ b/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td @@ -1661,8 +1661,6 @@ def: Pat<(int_hexagon_Y2_dccleana IntRegs:$src1), (Y2_dccleana IntRegs:$src1)>, Requires<[HasV5]>; def: Pat<(int_hexagon_Y2_dccleaninva IntRegs:$src1), (Y2_dccleaninva IntRegs:$src1)>, Requires<[HasV5]>; -def: Pat<(int_hexagon_Y2_dcfetch IntRegs:$src1), - (Y2_dcfetch IntRegs:$src1)>, Requires<[HasV5]>; def: Pat<(int_hexagon_Y2_dcinva IntRegs:$src1), (Y2_dcinva IntRegs:$src1)>, Requires<[HasV5]>; def: Pat<(int_hexagon_Y2_dczeroa IntRegs:$src1), @@ -3380,3 +3378,294 @@ def: Pat<(int_hexagon_V6_v6mpyvubs10_vxx HvxWR:$src1, HvxWR:$src2, HvxWR:$src3, (V6_v6mpyvubs10_vxx HvxWR:$src1, HvxWR:$src2, HvxWR:$src3, u2_0ImmPred_timm:$src4)>, Requires<[HasV68, UseHVX64B]>; def: Pat<(int_hexagon_V6_v6mpyvubs10_vxx_128B HvxWR:$src1, HvxWR:$src2, HvxWR:$src3, u2_0ImmPred_timm:$src4), (V6_v6mpyvubs10_vxx HvxWR:$src1, HvxWR:$src2, HvxWR:$src3, u2_0ImmPred_timm:$src4)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vabs_hf HvxVR:$src1), + (V6_vabs_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vabs_hf_128B HvxVR:$src1), + (V6_vabs_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vabs_sf HvxVR:$src1), + (V6_vabs_sf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vabs_sf_128B HvxVR:$src1), + (V6_vabs_sf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vadd_hf HvxVR:$src1, HvxVR:$src2), + (V6_vadd_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vadd_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_hf_hf HvxVR:$src1, HvxVR:$src2), + (V6_vadd_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vadd_hf_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vadd_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vadd_qf16 HvxVR:$src1, HvxVR:$src2), + (V6_vadd_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_qf16_128B HvxVR:$src1, HvxVR:$src2), + (V6_vadd_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_qf16_mix HvxVR:$src1, HvxVR:$src2), + (V6_vadd_qf16_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_qf16_mix_128B HvxVR:$src1, HvxVR:$src2), + (V6_vadd_qf16_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_qf32 HvxVR:$src1, HvxVR:$src2), + (V6_vadd_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_qf32_128B HvxVR:$src1, HvxVR:$src2), + (V6_vadd_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_qf32_mix HvxVR:$src1, HvxVR:$src2), + (V6_vadd_qf32_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_qf32_mix_128B HvxVR:$src1, HvxVR:$src2), + (V6_vadd_qf32_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_sf HvxVR:$src1, HvxVR:$src2), + (V6_vadd_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vadd_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vadd_sf_hf HvxVR:$src1, HvxVR:$src2), + (V6_vadd_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vadd_sf_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vadd_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vadd_sf_sf HvxVR:$src1, HvxVR:$src2), + (V6_vadd_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vadd_sf_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vadd_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vassign_fp HvxVR:$src1), + (V6_vassign_fp HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vassign_fp_128B HvxVR:$src1), + (V6_vassign_fp HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vconv_hf_qf16 HvxVR:$src1), + (V6_vconv_hf_qf16 HvxVR:$src1)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vconv_hf_qf16_128B HvxVR:$src1), + (V6_vconv_hf_qf16 HvxVR:$src1)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vconv_hf_qf32 HvxWR:$src1), + (V6_vconv_hf_qf32 HvxWR:$src1)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vconv_hf_qf32_128B HvxWR:$src1), + (V6_vconv_hf_qf32 HvxWR:$src1)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vconv_sf_qf32 HvxVR:$src1), + (V6_vconv_sf_qf32 HvxVR:$src1)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vconv_sf_qf32_128B HvxVR:$src1), + (V6_vconv_sf_qf32 HvxVR:$src1)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vcvt_b_hf HvxVR:$src1, HvxVR:$src2), + (V6_vcvt_b_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vcvt_b_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vcvt_b_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vcvt_h_hf HvxVR:$src1), + (V6_vcvt_h_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vcvt_h_hf_128B HvxVR:$src1), + (V6_vcvt_h_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vcvt_hf_b HvxVR:$src1), + (V6_vcvt_hf_b HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vcvt_hf_b_128B HvxVR:$src1), + (V6_vcvt_hf_b HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vcvt_hf_h HvxVR:$src1), + (V6_vcvt_hf_h HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vcvt_hf_h_128B HvxVR:$src1), + (V6_vcvt_hf_h HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vcvt_hf_sf HvxVR:$src1, HvxVR:$src2), + (V6_vcvt_hf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vcvt_hf_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vcvt_hf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vcvt_hf_ub HvxVR:$src1), + (V6_vcvt_hf_ub HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vcvt_hf_ub_128B HvxVR:$src1), + (V6_vcvt_hf_ub HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vcvt_hf_uh HvxVR:$src1), + (V6_vcvt_hf_uh HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vcvt_hf_uh_128B HvxVR:$src1), + (V6_vcvt_hf_uh HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vcvt_sf_hf HvxVR:$src1), + (V6_vcvt_sf_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vcvt_sf_hf_128B HvxVR:$src1), + (V6_vcvt_sf_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vcvt_ub_hf HvxVR:$src1, HvxVR:$src2), + (V6_vcvt_ub_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vcvt_ub_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vcvt_ub_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vcvt_uh_hf HvxVR:$src1), + (V6_vcvt_uh_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vcvt_uh_hf_128B HvxVR:$src1), + (V6_vcvt_uh_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vdmpy_sf_hf HvxVR:$src1, HvxVR:$src2), + (V6_vdmpy_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vdmpy_sf_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vdmpy_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vdmpy_sf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vdmpy_sf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vdmpy_sf_hf_acc_128B HvxVR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vdmpy_sf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vfmax_hf HvxVR:$src1, HvxVR:$src2), + (V6_vfmax_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vfmax_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vfmax_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vfmax_sf HvxVR:$src1, HvxVR:$src2), + (V6_vfmax_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vfmax_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vfmax_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vfmin_hf HvxVR:$src1, HvxVR:$src2), + (V6_vfmin_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vfmin_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vfmin_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vfmin_sf HvxVR:$src1, HvxVR:$src2), + (V6_vfmin_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vfmin_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vfmin_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vfneg_hf HvxVR:$src1), + (V6_vfneg_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vfneg_hf_128B HvxVR:$src1), + (V6_vfneg_hf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vfneg_sf HvxVR:$src1), + (V6_vfneg_sf HvxVR:$src1)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vfneg_sf_128B HvxVR:$src1), + (V6_vfneg_sf HvxVR:$src1)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vgthf HvxVR:$src1, HvxVR:$src2), + (V6_vgthf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgthf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vgthf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgthf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgthf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgthf_and_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgthf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgthf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgthf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgthf_or_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgthf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgthf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgthf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgthf_xor_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgthf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgtsf HvxVR:$src1, HvxVR:$src2), + (V6_vgtsf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgtsf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vgtsf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgtsf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgtsf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgtsf_and_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgtsf_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgtsf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgtsf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgtsf_or_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgtsf_or HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgtsf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgtsf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vgtsf_xor_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vgtsf_xor HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmax_hf HvxVR:$src1, HvxVR:$src2), + (V6_vmax_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmax_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmax_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmax_sf HvxVR:$src1, HvxVR:$src2), + (V6_vmax_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmax_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmax_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmin_hf HvxVR:$src1, HvxVR:$src2), + (V6_vmin_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmin_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmin_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmin_sf HvxVR:$src1, HvxVR:$src2), + (V6_vmin_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmin_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmin_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_hf_hf HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vmpy_hf_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vmpy_hf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vmpy_hf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vmpy_hf_hf_acc_128B HvxVR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vmpy_hf_hf_acc HvxVR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vmpy_qf16 HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf16_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf16_hf HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf16_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf16_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf16_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf16_mix_hf HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf16_mix_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf16_mix_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf16_mix_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf32 HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf32_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf32_hf HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf32_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf32_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf32_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf32_mix_hf HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf32_mix_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf32_mix_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf32_mix_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf32_qf16 HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf32_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf32_qf16_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf32_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf32_sf HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf32_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_qf32_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_qf32_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vmpy_sf_hf HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vmpy_sf_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vmpy_sf_hf_acc HvxWR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vmpy_sf_hf_acc HvxWR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vmpy_sf_hf_acc_128B HvxWR:$src1, HvxVR:$src2, HvxVR:$src3), + (V6_vmpy_sf_hf_acc HvxWR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vmpy_sf_sf HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vmpy_sf_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpy_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vsub_hf HvxVR:$src1, HvxVR:$src2), + (V6_vsub_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vsub_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_hf_hf HvxVR:$src1, HvxVR:$src2), + (V6_vsub_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vsub_hf_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vsub_hf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vsub_qf16 HvxVR:$src1, HvxVR:$src2), + (V6_vsub_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_qf16_128B HvxVR:$src1, HvxVR:$src2), + (V6_vsub_qf16 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_qf16_mix HvxVR:$src1, HvxVR:$src2), + (V6_vsub_qf16_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_qf16_mix_128B HvxVR:$src1, HvxVR:$src2), + (V6_vsub_qf16_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_qf32 HvxVR:$src1, HvxVR:$src2), + (V6_vsub_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_qf32_128B HvxVR:$src1, HvxVR:$src2), + (V6_vsub_qf32 HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_qf32_mix HvxVR:$src1, HvxVR:$src2), + (V6_vsub_qf32_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_qf32_mix_128B HvxVR:$src1, HvxVR:$src2), + (V6_vsub_qf32_mix HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_sf HvxVR:$src1, HvxVR:$src2), + (V6_vsub_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vsub_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B, UseHVXQFloat]>; +def: Pat<(int_hexagon_V6_vsub_sf_hf HvxVR:$src1, HvxVR:$src2), + (V6_vsub_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vsub_sf_hf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vsub_sf_hf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vsub_sf_sf HvxVR:$src1, HvxVR:$src2), + (V6_vsub_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vsub_sf_sf_128B HvxVR:$src1, HvxVR:$src2), + (V6_vsub_sf_sf HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV68, UseHVX128B]>; + +// V69 HVX Instructions. + +def: Pat<(int_hexagon_V6_vasrvuhubrndsat HvxWR:$src1, HvxVR:$src2), + (V6_vasrvuhubrndsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vasrvuhubrndsat_128B HvxWR:$src1, HvxVR:$src2), + (V6_vasrvuhubrndsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vasrvuhubsat HvxWR:$src1, HvxVR:$src2), + (V6_vasrvuhubsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vasrvuhubsat_128B HvxWR:$src1, HvxVR:$src2), + (V6_vasrvuhubsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vasrvwuhrndsat HvxWR:$src1, HvxVR:$src2), + (V6_vasrvwuhrndsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vasrvwuhrndsat_128B HvxWR:$src1, HvxVR:$src2), + (V6_vasrvwuhrndsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vasrvwuhsat HvxWR:$src1, HvxVR:$src2), + (V6_vasrvwuhsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vasrvwuhsat_128B HvxWR:$src1, HvxVR:$src2), + (V6_vasrvwuhsat HvxWR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX128B]>; +def: Pat<(int_hexagon_V6_vmpyuhvs HvxVR:$src1, HvxVR:$src2), + (V6_vmpyuhvs HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX64B]>; +def: Pat<(int_hexagon_V6_vmpyuhvs_128B HvxVR:$src1, HvxVR:$src2), + (V6_vmpyuhvs HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV69, UseHVX128B]>; diff --git a/llvm/lib/Target/Hexagon/HexagonDepMappings.td b/llvm/lib/Target/Hexagon/HexagonDepMappings.td index 919cb996ad15..2f7b76b893a9 100644 --- a/llvm/lib/Target/Hexagon/HexagonDepMappings.td +++ b/llvm/lib/Target/Hexagon/HexagonDepMappings.td @@ -174,7 +174,6 @@ def V6_ldcpnt0Alias : InstAlias<"if ($Pv4) $Vd32.cur = vmem($Rt32):nt", (V6_vL32 def V6_ldnp0Alias : InstAlias<"if (!$Pv4) $Vd32 = vmem($Rt32)", (V6_vL32b_npred_pi HvxVR:$Vd32, IntRegs:$Rt32, PredRegs:$Pv4, 0)>, Requires<[UseHVX]>; def V6_ldnpnt0Alias : InstAlias<"if (!$Pv4) $Vd32 = vmem($Rt32):nt", (V6_vL32b_nt_npred_pi HvxVR:$Vd32, IntRegs:$Rt32, PredRegs:$Pv4, 0)>, Requires<[UseHVX]>; def V6_ldnt0Alias : InstAlias<"$Vd32 = vmem($Rt32):nt", (V6_vL32b_nt_ai HvxVR:$Vd32, IntRegs:$Rt32, 0)>, Requires<[UseHVX]>; -def V6_ldntnt0Alias : InstAlias<"$Vd32 = vmem($Rt32):nt", (V6_vL32b_nt_ai HvxVR:$Vd32, IntRegs:$Rt32, 0)>; def V6_ldp0Alias : InstAlias<"if ($Pv4) $Vd32 = vmem($Rt32)", (V6_vL32b_pred_ai HvxVR:$Vd32, PredRegs:$Pv4, IntRegs:$Rt32, 0)>, Requires<[UseHVX]>; def V6_ldpnt0Alias : InstAlias<"if ($Pv4) $Vd32 = vmem($Rt32):nt", (V6_vL32b_nt_pred_ai HvxVR:$Vd32, PredRegs:$Pv4, IntRegs:$Rt32, 0)>, Requires<[UseHVX]>; def V6_ldtnp0Alias : InstAlias<"if (!$Pv4) $Vd32.tmp = vmem($Rt32)", (V6_vL32b_npred_ai HvxVR:$Vd32, PredRegs:$Pv4, IntRegs:$Rt32, 0)>, Requires<[UseHVX]>; diff --git a/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp b/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp index 46c1fbc6eeb2..85230cac9d7c 100644 --- a/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp +++ b/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp @@ -1445,8 +1445,8 @@ bool HexagonGenInsert::removeDeadCode(MachineDomTreeNode *N) { MachineBasicBlock *B = N->getBlock(); std::vector<MachineInstr*> Instrs; - for (auto I = B->rbegin(), E = B->rend(); I != E; ++I) - Instrs.push_back(&*I); + for (MachineInstr &MI : llvm::reverse(*B)) + Instrs.push_back(&MI); for (MachineInstr *MI : Instrs) { unsigned Opc = MI->getOpcode(); diff --git a/llvm/lib/Target/Hexagon/HexagonHazardRecognizer.cpp b/llvm/lib/Target/Hexagon/HexagonHazardRecognizer.cpp index e45126bec6ef..44679d429de5 100644 --- a/llvm/lib/Target/Hexagon/HexagonHazardRecognizer.cpp +++ b/llvm/lib/Target/Hexagon/HexagonHazardRecognizer.cpp @@ -60,7 +60,7 @@ HexagonHazardRecognizer::getHazardType(SUnit *SU, int stalls) { RetVal = NoHazard; LLVM_DEBUG(dbgs() << "*** Try .new version? " << (RetVal == NoHazard) << "\n"); - MF->DeleteMachineInstr(NewMI); + MF->deleteMachineInstr(NewMI); } return RetVal; } @@ -129,7 +129,7 @@ void HexagonHazardRecognizer::EmitInstruction(SUnit *SU) { MI->getDebugLoc()); assert(Resources->canReserveResources(*NewMI)); Resources->reserveResources(*NewMI); - MF->DeleteMachineInstr(NewMI); + MF->deleteMachineInstr(NewMI); } else Resources->reserveResources(*MI); diff --git a/llvm/lib/Target/Hexagon/HexagonInstrFormats.td b/llvm/lib/Target/Hexagon/HexagonInstrFormats.td index 45adaf50774f..898ef51bd48f 100644 --- a/llvm/lib/Target/Hexagon/HexagonInstrFormats.td +++ b/llvm/lib/Target/Hexagon/HexagonInstrFormats.td @@ -146,9 +146,6 @@ class InstHexagon<dag outs, dag ins, string asmstr, list<dag> pattern, bits<1> isFP = 0; let TSFlags {50} = isFP; // Floating-point. - bits<1> isSomeOK = 0; - let TSFlags {51} = isSomeOK; // Relax some grouping constraints. - bits<1> hasNewValue2 = 0; let TSFlags{52} = hasNewValue2; // Second New-value producer insn. bits<3> opNewValue2 = 0; @@ -160,8 +157,8 @@ class InstHexagon<dag outs, dag ins, string asmstr, list<dag> pattern, bits<1> prefersSlot3 = 0; let TSFlags{57} = prefersSlot3; // Complex XU - bits<1> hasTmpDst = 0; - let TSFlags{60} = hasTmpDst; // v65 : 'fake" register VTMP is set + bits<1> hasHvxTmp = 0; + let TSFlags{60} = hasHvxTmp; // vector register vX.tmp false-write bit CVINew = 0; let TSFlags{62} = CVINew; diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp index b6984d40f78e..931b0c0e0090 100644 --- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp @@ -40,6 +40,7 @@ #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DebugLoc.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCInstrItineraries.h" #include "llvm/MC/MCRegisterInfo.h" @@ -4655,3 +4656,11 @@ short HexagonInstrInfo::changeAddrMode_rr_ur(short Opc) const { short HexagonInstrInfo::changeAddrMode_ur_rr(short Opc) const { return Opc >= 0 ? Hexagon::changeAddrMode_ur_rr(Opc) : Opc; } + +MCInst HexagonInstrInfo::getNop() const { + static const MCInst Nop = MCInstBuilder(Hexagon::A2_nop); + + return MCInstBuilder(Hexagon::BUNDLE) + .addImm(0) + .addInst(&Nop); +} diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h index eaaf9f7046c7..830f04d9eac3 100644 --- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h @@ -524,6 +524,8 @@ public: short changeAddrMode_ur_rr(const MachineInstr &MI) const { return changeAddrMode_ur_rr(MI.getOpcode()); } + + MCInst getNop() const override; }; } // end namespace llvm diff --git a/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp b/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp index 987c4a5fa6c4..d5c34ac467c3 100644 --- a/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp +++ b/llvm/lib/Target/Hexagon/HexagonMCInstLower.cpp @@ -104,6 +104,19 @@ void llvm::HexagonLowerToMC(const MCInstrInfo &MCII, const MachineInstr *MI, HexagonMCInstrInfo::setOuterLoop(MCB); return; } + if (MI->getOpcode() == Hexagon::PATCHABLE_FUNCTION_ENTER) { + AP.EmitSled(*MI, HexagonAsmPrinter::SledKind::FUNCTION_ENTER); + return; + } + if (MI->getOpcode() == Hexagon::PATCHABLE_FUNCTION_EXIT) { + AP.EmitSled(*MI, HexagonAsmPrinter::SledKind::FUNCTION_EXIT); + return; + } + if (MI->getOpcode() == Hexagon::PATCHABLE_TAIL_CALL) { + AP.EmitSled(*MI, HexagonAsmPrinter::SledKind::TAIL_CALL); + return; + } + MCInst *MCI = AP.OutContext.createMCInst(); MCI->setOpcode(MI->getOpcode()); assert(MCI->getOpcode() == static_cast<unsigned>(MI->getOpcode()) && diff --git a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp index 60d58f421bbb..53e82ac66b85 100644 --- a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp +++ b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp @@ -14,676 +14,44 @@ #include "HexagonMachineScheduler.h" #include "HexagonInstrInfo.h" #include "HexagonSubtarget.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/DFAPacketizer.h" -#include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/RegisterClassInfo.h" -#include "llvm/CodeGen/RegisterPressure.h" +#include "llvm/CodeGen/MachineScheduler.h" #include "llvm/CodeGen/ScheduleDAG.h" -#include "llvm/CodeGen/ScheduleHazardRecognizer.h" -#include "llvm/CodeGen/TargetInstrInfo.h" -#include "llvm/CodeGen/TargetOpcodes.h" -#include "llvm/CodeGen/TargetRegisterInfo.h" -#include "llvm/CodeGen/TargetSchedule.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/IR/Function.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -#include <algorithm> -#include <cassert> -#include <iomanip> -#include <limits> -#include <memory> -#include <sstream> +#include "llvm/CodeGen/VLIWMachineScheduler.h" using namespace llvm; #define DEBUG_TYPE "machine-scheduler" -static cl::opt<bool> IgnoreBBRegPressure("ignore-bb-reg-pressure", - cl::Hidden, cl::ZeroOrMore, cl::init(false)); - -static cl::opt<bool> UseNewerCandidate("use-newer-candidate", - cl::Hidden, cl::ZeroOrMore, cl::init(true)); - -static cl::opt<unsigned> SchedDebugVerboseLevel("misched-verbose-level", - cl::Hidden, cl::ZeroOrMore, cl::init(1)); - -// Check if the scheduler should penalize instructions that are available to -// early due to a zero-latency dependence. -static cl::opt<bool> CheckEarlyAvail("check-early-avail", cl::Hidden, - cl::ZeroOrMore, cl::init(true)); - -// This value is used to determine if a register class is a high pressure set. -// We compute the maximum number of registers needed and divided by the total -// available. Then, we compare the result to this value. -static cl::opt<float> RPThreshold("hexagon-reg-pressure", cl::Hidden, - cl::init(0.75f), cl::desc("High register pressure threhold.")); - /// Return true if there is a dependence between SUd and SUu. -static bool hasDependence(const SUnit *SUd, const SUnit *SUu, - const HexagonInstrInfo &QII) { - if (SUd->Succs.size() == 0) - return false; +bool HexagonVLIWResourceModel::hasDependence(const SUnit *SUd, + const SUnit *SUu) { + const auto *QII = static_cast<const HexagonInstrInfo *>(TII); // Enable .cur formation. - if (QII.mayBeCurLoad(*SUd->getInstr())) + if (QII->mayBeCurLoad(*SUd->getInstr())) return false; - if (QII.canExecuteInBundle(*SUd->getInstr(), *SUu->getInstr())) - return false; - - for (const auto &S : SUd->Succs) { - // Since we do not add pseudos to packets, might as well - // ignore order dependencies. - if (S.isCtrl()) - continue; - - if (S.getSUnit() == SUu && S.getLatency() > 0) - return true; - } - return false; -} - -/// Check if scheduling of this SU is possible -/// in the current packet. -/// It is _not_ precise (statefull), it is more like -/// another heuristic. Many corner cases are figured -/// empirically. -bool VLIWResourceModel::isResourceAvailable(SUnit *SU, bool IsTop) { - if (!SU || !SU->getInstr()) + if (QII->canExecuteInBundle(*SUd->getInstr(), *SUu->getInstr())) return false; - // First see if the pipeline could receive this instruction - // in the current cycle. - switch (SU->getInstr()->getOpcode()) { - default: - if (!ResourcesModel->canReserveResources(*SU->getInstr())) - return false; - break; - case TargetOpcode::EXTRACT_SUBREG: - case TargetOpcode::INSERT_SUBREG: - case TargetOpcode::SUBREG_TO_REG: - case TargetOpcode::REG_SEQUENCE: - case TargetOpcode::IMPLICIT_DEF: - case TargetOpcode::COPY: - case TargetOpcode::INLINEASM: - case TargetOpcode::INLINEASM_BR: - break; - } - - MachineBasicBlock *MBB = SU->getInstr()->getParent(); - auto &QST = MBB->getParent()->getSubtarget<HexagonSubtarget>(); - const auto &QII = *QST.getInstrInfo(); - - // Now see if there are no other dependencies to instructions already - // in the packet. - if (IsTop) { - for (unsigned i = 0, e = Packet.size(); i != e; ++i) - if (hasDependence(Packet[i], SU, QII)) - return false; - } else { - for (unsigned i = 0, e = Packet.size(); i != e; ++i) - if (hasDependence(SU, Packet[i], QII)) - return false; - } - return true; -} - -/// Keep track of available resources. -bool VLIWResourceModel::reserveResources(SUnit *SU, bool IsTop) { - bool startNewCycle = false; - // Artificially reset state. - if (!SU) { - ResourcesModel->clearResources(); - Packet.clear(); - TotalPackets++; - return false; - } - // If this SU does not fit in the packet or the packet is now full - // start a new one. - if (!isResourceAvailable(SU, IsTop) || - Packet.size() >= SchedModel->getIssueWidth()) { - ResourcesModel->clearResources(); - Packet.clear(); - TotalPackets++; - startNewCycle = true; - } - - switch (SU->getInstr()->getOpcode()) { - default: - ResourcesModel->reserveResources(*SU->getInstr()); - break; - case TargetOpcode::EXTRACT_SUBREG: - case TargetOpcode::INSERT_SUBREG: - case TargetOpcode::SUBREG_TO_REG: - case TargetOpcode::REG_SEQUENCE: - case TargetOpcode::IMPLICIT_DEF: - case TargetOpcode::KILL: - case TargetOpcode::CFI_INSTRUCTION: - case TargetOpcode::EH_LABEL: - case TargetOpcode::COPY: - case TargetOpcode::INLINEASM: - case TargetOpcode::INLINEASM_BR: - break; - } - Packet.push_back(SU); - -#ifndef NDEBUG - LLVM_DEBUG(dbgs() << "Packet[" << TotalPackets << "]:\n"); - for (unsigned i = 0, e = Packet.size(); i != e; ++i) { - LLVM_DEBUG(dbgs() << "\t[" << i << "] SU("); - LLVM_DEBUG(dbgs() << Packet[i]->NodeNum << ")\t"); - LLVM_DEBUG(Packet[i]->getInstr()->dump()); - } -#endif - - return startNewCycle; + return VLIWResourceModel::hasDependence(SUd, SUu); } -/// schedule - Called back from MachineScheduler::runOnMachineFunction -/// after setting up the current scheduling region. [RegionBegin, RegionEnd) -/// only includes instructions that have DAG nodes, not scheduling boundaries. -void VLIWMachineScheduler::schedule() { - LLVM_DEBUG(dbgs() << "********** MI Converging Scheduling VLIW " - << printMBBReference(*BB) << " " << BB->getName() - << " in_func " << BB->getParent()->getName() - << " at loop depth " << MLI->getLoopDepth(BB) << " \n"); - - buildDAGWithRegPressure(); - - Topo.InitDAGTopologicalSorting(); - - // Postprocess the DAG to add platform-specific artificial dependencies. - postprocessDAG(); - - SmallVector<SUnit*, 8> TopRoots, BotRoots; - findRootsAndBiasEdges(TopRoots, BotRoots); - - // Initialize the strategy before modifying the DAG. - SchedImpl->initialize(this); - - LLVM_DEBUG(unsigned maxH = 0; - for (unsigned su = 0, e = SUnits.size(); su != e; - ++su) if (SUnits[su].getHeight() > maxH) maxH = - SUnits[su].getHeight(); - dbgs() << "Max Height " << maxH << "\n";); - LLVM_DEBUG(unsigned maxD = 0; - for (unsigned su = 0, e = SUnits.size(); su != e; - ++su) if (SUnits[su].getDepth() > maxD) maxD = - SUnits[su].getDepth(); - dbgs() << "Max Depth " << maxD << "\n";); - LLVM_DEBUG(dump()); - - initQueues(TopRoots, BotRoots); - - bool IsTopNode = false; - while (true) { - LLVM_DEBUG( - dbgs() << "** VLIWMachineScheduler::schedule picking next node\n"); - SUnit *SU = SchedImpl->pickNode(IsTopNode); - if (!SU) break; - - if (!checkSchedLimit()) - break; - - scheduleMI(SU, IsTopNode); - - // Notify the scheduling strategy after updating the DAG. - SchedImpl->schedNode(SU, IsTopNode); - - updateQueues(SU, IsTopNode); - } - assert(CurrentTop == CurrentBottom && "Nonempty unscheduled zone."); - - placeDebugValues(); - - LLVM_DEBUG({ - dbgs() << "*** Final schedule for " - << printMBBReference(*begin()->getParent()) << " ***\n"; - dumpSchedule(); - dbgs() << '\n'; - }); +VLIWResourceModel *HexagonConvergingVLIWScheduler::createVLIWResourceModel( + const TargetSubtargetInfo &STI, const TargetSchedModel *SchedModel) const { + return new HexagonVLIWResourceModel(STI, SchedModel); } -void ConvergingVLIWScheduler::initialize(ScheduleDAGMI *dag) { - DAG = static_cast<VLIWMachineScheduler*>(dag); - SchedModel = DAG->getSchedModel(); - - Top.init(DAG, SchedModel); - Bot.init(DAG, SchedModel); - - // Initialize the HazardRecognizers. If itineraries don't exist, are empty, or - // are disabled, then these HazardRecs will be disabled. - const InstrItineraryData *Itin = DAG->getSchedModel()->getInstrItineraries(); - const TargetSubtargetInfo &STI = DAG->MF.getSubtarget(); - const TargetInstrInfo *TII = STI.getInstrInfo(); - delete Top.HazardRec; - delete Bot.HazardRec; - Top.HazardRec = TII->CreateTargetMIHazardRecognizer(Itin, DAG); - Bot.HazardRec = TII->CreateTargetMIHazardRecognizer(Itin, DAG); - - delete Top.ResourceModel; - delete Bot.ResourceModel; - Top.ResourceModel = new VLIWResourceModel(STI, DAG->getSchedModel()); - Bot.ResourceModel = new VLIWResourceModel(STI, DAG->getSchedModel()); - - const std::vector<unsigned> &MaxPressure = - DAG->getRegPressure().MaxSetPressure; - HighPressureSets.assign(MaxPressure.size(), 0); - for (unsigned i = 0, e = MaxPressure.size(); i < e; ++i) { - unsigned Limit = DAG->getRegClassInfo()->getRegPressureSetLimit(i); - HighPressureSets[i] = - ((float) MaxPressure[i] > ((float) Limit * RPThreshold)); - } - - assert((!ForceTopDown || !ForceBottomUp) && - "-misched-topdown incompatible with -misched-bottomup"); -} - -void ConvergingVLIWScheduler::releaseTopNode(SUnit *SU) { - for (const SDep &PI : SU->Preds) { - unsigned PredReadyCycle = PI.getSUnit()->TopReadyCycle; - unsigned MinLatency = PI.getLatency(); -#ifndef NDEBUG - Top.MaxMinLatency = std::max(MinLatency, Top.MaxMinLatency); -#endif - if (SU->TopReadyCycle < PredReadyCycle + MinLatency) - SU->TopReadyCycle = PredReadyCycle + MinLatency; - } - - if (!SU->isScheduled) - Top.releaseNode(SU, SU->TopReadyCycle); -} - -void ConvergingVLIWScheduler::releaseBottomNode(SUnit *SU) { - assert(SU->getInstr() && "Scheduled SUnit must have instr"); - - for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); - I != E; ++I) { - unsigned SuccReadyCycle = I->getSUnit()->BotReadyCycle; - unsigned MinLatency = I->getLatency(); -#ifndef NDEBUG - Bot.MaxMinLatency = std::max(MinLatency, Bot.MaxMinLatency); -#endif - if (SU->BotReadyCycle < SuccReadyCycle + MinLatency) - SU->BotReadyCycle = SuccReadyCycle + MinLatency; - } - - if (!SU->isScheduled) - Bot.releaseNode(SU, SU->BotReadyCycle); -} - -/// Does this SU have a hazard within the current instruction group. -/// -/// The scheduler supports two modes of hazard recognition. The first is the -/// ScheduleHazardRecognizer API. It is a fully general hazard recognizer that -/// supports highly complicated in-order reservation tables -/// (ScoreboardHazardRecognizer) and arbitrary target-specific logic. -/// -/// The second is a streamlined mechanism that checks for hazards based on -/// simple counters that the scheduler itself maintains. It explicitly checks -/// for instruction dispatch limitations, including the number of micro-ops that -/// can dispatch per cycle. -/// -/// TODO: Also check whether the SU must start a new group. -bool ConvergingVLIWScheduler::VLIWSchedBoundary::checkHazard(SUnit *SU) { - if (HazardRec->isEnabled()) - return HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard; - - unsigned uops = SchedModel->getNumMicroOps(SU->getInstr()); - if (IssueCount + uops > SchedModel->getIssueWidth()) - return true; - - return false; -} - -void ConvergingVLIWScheduler::VLIWSchedBoundary::releaseNode(SUnit *SU, - unsigned ReadyCycle) { - if (ReadyCycle < MinReadyCycle) - MinReadyCycle = ReadyCycle; - - // Check for interlocks first. For the purpose of other heuristics, an - // instruction that cannot issue appears as if it's not in the ReadyQueue. - if (ReadyCycle > CurrCycle || checkHazard(SU)) - - Pending.push(SU); - else - Available.push(SU); -} - -/// Move the boundary of scheduled code by one cycle. -void ConvergingVLIWScheduler::VLIWSchedBoundary::bumpCycle() { - unsigned Width = SchedModel->getIssueWidth(); - IssueCount = (IssueCount <= Width) ? 0 : IssueCount - Width; - - assert(MinReadyCycle < std::numeric_limits<unsigned>::max() && - "MinReadyCycle uninitialized"); - unsigned NextCycle = std::max(CurrCycle + 1, MinReadyCycle); - - if (!HazardRec->isEnabled()) { - // Bypass HazardRec virtual calls. - CurrCycle = NextCycle; - } else { - // Bypass getHazardType calls in case of long latency. - for (; CurrCycle != NextCycle; ++CurrCycle) { - if (isTop()) - HazardRec->AdvanceCycle(); - else - HazardRec->RecedeCycle(); - } - } - CheckPending = true; - - LLVM_DEBUG(dbgs() << "*** Next cycle " << Available.getName() << " cycle " - << CurrCycle << '\n'); -} - -/// Move the boundary of scheduled code by one SUnit. -void ConvergingVLIWScheduler::VLIWSchedBoundary::bumpNode(SUnit *SU) { - bool startNewCycle = false; - - // Update the reservation table. - if (HazardRec->isEnabled()) { - if (!isTop() && SU->isCall) { - // Calls are scheduled with their preceding instructions. For bottom-up - // scheduling, clear the pipeline state before emitting. - HazardRec->Reset(); - } - HazardRec->EmitInstruction(SU); - } - - // Update DFA model. - startNewCycle = ResourceModel->reserveResources(SU, isTop()); - - // Check the instruction group dispatch limit. - // TODO: Check if this SU must end a dispatch group. - IssueCount += SchedModel->getNumMicroOps(SU->getInstr()); - if (startNewCycle) { - LLVM_DEBUG(dbgs() << "*** Max instrs at cycle " << CurrCycle << '\n'); - bumpCycle(); - } - else - LLVM_DEBUG(dbgs() << "*** IssueCount " << IssueCount << " at cycle " - << CurrCycle << '\n'); -} - -/// Release pending ready nodes in to the available queue. This makes them -/// visible to heuristics. -void ConvergingVLIWScheduler::VLIWSchedBoundary::releasePending() { - // If the available queue is empty, it is safe to reset MinReadyCycle. - if (Available.empty()) - MinReadyCycle = std::numeric_limits<unsigned>::max(); - - // Check to see if any of the pending instructions are ready to issue. If - // so, add them to the available queue. - for (unsigned i = 0, e = Pending.size(); i != e; ++i) { - SUnit *SU = *(Pending.begin()+i); - unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle; - - if (ReadyCycle < MinReadyCycle) - MinReadyCycle = ReadyCycle; - - if (ReadyCycle > CurrCycle) - continue; - - if (checkHazard(SU)) - continue; - - Available.push(SU); - Pending.remove(Pending.begin()+i); - --i; --e; - } - CheckPending = false; -} - -/// Remove SU from the ready set for this boundary. -void ConvergingVLIWScheduler::VLIWSchedBoundary::removeReady(SUnit *SU) { - if (Available.isInQueue(SU)) - Available.remove(Available.find(SU)); - else { - assert(Pending.isInQueue(SU) && "bad ready count"); - Pending.remove(Pending.find(SU)); - } -} +int HexagonConvergingVLIWScheduler::SchedulingCost(ReadyQueue &Q, SUnit *SU, + SchedCandidate &Candidate, + RegPressureDelta &Delta, + bool verbose) { + int ResCount = + ConvergingVLIWScheduler::SchedulingCost(Q, SU, Candidate, Delta, verbose); -/// If this queue only has one ready candidate, return it. As a side effect, -/// advance the cycle until at least one node is ready. If multiple instructions -/// are ready, return NULL. -SUnit *ConvergingVLIWScheduler::VLIWSchedBoundary::pickOnlyChoice() { - if (CheckPending) - releasePending(); - - auto AdvanceCycle = [this]() { - if (Available.empty()) - return true; - if (Available.size() == 1 && Pending.size() > 0) - return !ResourceModel->isResourceAvailable(*Available.begin(), isTop()) || - getWeakLeft(*Available.begin(), isTop()) != 0; - return false; - }; - for (unsigned i = 0; AdvanceCycle(); ++i) { - assert(i <= (HazardRec->getMaxLookAhead() + MaxMinLatency) && - "permanent hazard"); (void)i; - ResourceModel->reserveResources(nullptr, isTop()); - bumpCycle(); - releasePending(); - } - if (Available.size() == 1) - return *Available.begin(); - return nullptr; -} - -#ifndef NDEBUG -void ConvergingVLIWScheduler::traceCandidate(const char *Label, - const ReadyQueue &Q, SUnit *SU, int Cost, PressureChange P) { - dbgs() << Label << " " << Q.getName() << " "; - if (P.isValid()) - dbgs() << DAG->TRI->getRegPressureSetName(P.getPSet()) << ":" - << P.getUnitInc() << " "; - else - dbgs() << " "; - dbgs() << "cost(" << Cost << ")\t"; - DAG->dumpNode(*SU); -} - -// Very detailed queue dump, to be used with higher verbosity levels. -void ConvergingVLIWScheduler::readyQueueVerboseDump( - const RegPressureTracker &RPTracker, SchedCandidate &Candidate, - ReadyQueue &Q) { - RegPressureTracker &TempTracker = const_cast<RegPressureTracker &>(RPTracker); - - dbgs() << ">>> " << Q.getName() << "\n"; - for (ReadyQueue::iterator I = Q.begin(), E = Q.end(); I != E; ++I) { - RegPressureDelta RPDelta; - TempTracker.getMaxPressureDelta((*I)->getInstr(), RPDelta, - DAG->getRegionCriticalPSets(), - DAG->getRegPressure().MaxSetPressure); - std::stringstream dbgstr; - dbgstr << "SU(" << std::setw(3) << (*I)->NodeNum << ")"; - dbgs() << dbgstr.str(); - SchedulingCost(Q, *I, Candidate, RPDelta, true); - dbgs() << "\t"; - (*I)->getInstr()->dump(); - } - dbgs() << "\n"; -} -#endif - -/// isSingleUnscheduledPred - If SU2 is the only unscheduled predecessor -/// of SU, return true (we may have duplicates) -static inline bool isSingleUnscheduledPred(SUnit *SU, SUnit *SU2) { - if (SU->NumPredsLeft == 0) - return false; - - for (auto &Pred : SU->Preds) { - // We found an available, but not scheduled, predecessor. - if (!Pred.getSUnit()->isScheduled && (Pred.getSUnit() != SU2)) - return false; - } - - return true; -} - -/// isSingleUnscheduledSucc - If SU2 is the only unscheduled successor -/// of SU, return true (we may have duplicates) -static inline bool isSingleUnscheduledSucc(SUnit *SU, SUnit *SU2) { - if (SU->NumSuccsLeft == 0) - return false; - - for (auto &Succ : SU->Succs) { - // We found an available, but not scheduled, successor. - if (!Succ.getSUnit()->isScheduled && (Succ.getSUnit() != SU2)) - return false; - } - return true; -} - -/// Check if the instruction changes the register pressure of a register in the -/// high pressure set. The function returns a negative value if the pressure -/// decreases and a positive value is the pressure increases. If the instruction -/// doesn't use a high pressure register or doesn't change the register -/// pressure, then return 0. -int ConvergingVLIWScheduler::pressureChange(const SUnit *SU, bool isBotUp) { - PressureDiff &PD = DAG->getPressureDiff(SU); - for (auto &P : PD) { - if (!P.isValid()) - continue; - // The pressure differences are computed bottom-up, so the comparision for - // an increase is positive in the bottom direction, but negative in the - // top-down direction. - if (HighPressureSets[P.getPSet()]) - return (isBotUp ? P.getUnitInc() : -P.getUnitInc()); - } - return 0; -} - -// Constants used to denote relative importance of -// heuristic components for cost computation. -static const unsigned PriorityOne = 200; -static const unsigned PriorityTwo = 50; -static const unsigned PriorityThree = 75; -static const unsigned ScaleTwo = 10; - -/// Single point to compute overall scheduling cost. -/// TODO: More heuristics will be used soon. -int ConvergingVLIWScheduler::SchedulingCost(ReadyQueue &Q, SUnit *SU, - SchedCandidate &Candidate, - RegPressureDelta &Delta, - bool verbose) { - // Initial trivial priority. - int ResCount = 1; - - // Do not waste time on a node that is already scheduled. if (!SU || SU->isScheduled) return ResCount; - LLVM_DEBUG(if (verbose) dbgs() - << ((Q.getID() == TopQID) ? "(top|" : "(bot|")); - // Forced priority is high. - if (SU->isScheduleHigh) { - ResCount += PriorityOne; - LLVM_DEBUG(dbgs() << "H|"); - } - - unsigned IsAvailableAmt = 0; - // Critical path first. - if (Q.getID() == TopQID) { - if (Top.isLatencyBound(SU)) { - LLVM_DEBUG(if (verbose) dbgs() << "LB|"); - ResCount += (SU->getHeight() * ScaleTwo); - } - - LLVM_DEBUG(if (verbose) { - std::stringstream dbgstr; - dbgstr << "h" << std::setw(3) << SU->getHeight() << "|"; - dbgs() << dbgstr.str(); - }); - - // If resources are available for it, multiply the - // chance of scheduling. - if (Top.ResourceModel->isResourceAvailable(SU, true)) { - IsAvailableAmt = (PriorityTwo + PriorityThree); - ResCount += IsAvailableAmt; - LLVM_DEBUG(if (verbose) dbgs() << "A|"); - } else - LLVM_DEBUG(if (verbose) dbgs() << " |"); - } else { - if (Bot.isLatencyBound(SU)) { - LLVM_DEBUG(if (verbose) dbgs() << "LB|"); - ResCount += (SU->getDepth() * ScaleTwo); - } - - LLVM_DEBUG(if (verbose) { - std::stringstream dbgstr; - dbgstr << "d" << std::setw(3) << SU->getDepth() << "|"; - dbgs() << dbgstr.str(); - }); - - // If resources are available for it, multiply the - // chance of scheduling. - if (Bot.ResourceModel->isResourceAvailable(SU, false)) { - IsAvailableAmt = (PriorityTwo + PriorityThree); - ResCount += IsAvailableAmt; - LLVM_DEBUG(if (verbose) dbgs() << "A|"); - } else - LLVM_DEBUG(if (verbose) dbgs() << " |"); - } - - unsigned NumNodesBlocking = 0; - if (Q.getID() == TopQID) { - // How many SUs does it block from scheduling? - // Look at all of the successors of this node. - // Count the number of nodes that - // this node is the sole unscheduled node for. - if (Top.isLatencyBound(SU)) - for (const SDep &SI : SU->Succs) - if (isSingleUnscheduledPred(SI.getSUnit(), SU)) - ++NumNodesBlocking; - } else { - // How many unscheduled predecessors block this node? - if (Bot.isLatencyBound(SU)) - for (const SDep &PI : SU->Preds) - if (isSingleUnscheduledSucc(PI.getSUnit(), SU)) - ++NumNodesBlocking; - } - ResCount += (NumNodesBlocking * ScaleTwo); - - LLVM_DEBUG(if (verbose) { - std::stringstream dbgstr; - dbgstr << "blk " << std::setw(2) << NumNodesBlocking << ")|"; - dbgs() << dbgstr.str(); - }); - - // Factor in reg pressure as a heuristic. - if (!IgnoreBBRegPressure) { - // Decrease priority by the amount that register pressure exceeds the limit. - ResCount -= (Delta.Excess.getUnitInc()*PriorityOne); - // Decrease priority if register pressure exceeds the limit. - ResCount -= (Delta.CriticalMax.getUnitInc()*PriorityOne); - // Decrease priority slightly if register pressure would increase over the - // current maximum. - ResCount -= (Delta.CurrentMax.getUnitInc()*PriorityTwo); - // If there are register pressure issues, then we remove the value added for - // the instruction being available. The rationale is that we really don't - // want to schedule an instruction that causes a spill. - if (IsAvailableAmt && pressureChange(SU, Q.getID() != TopQID) > 0 && - (Delta.Excess.getUnitInc() || Delta.CriticalMax.getUnitInc() || - Delta.CurrentMax.getUnitInc())) - ResCount -= IsAvailableAmt; - LLVM_DEBUG(if (verbose) { - dbgs() << "RP " << Delta.Excess.getUnitInc() << "/" - << Delta.CriticalMax.getUnitInc() << "/" - << Delta.CurrentMax.getUnitInc() << ")|"; - }); - } - - // Give a little extra priority to a .cur instruction if there is a resource - // available for it. auto &QST = DAG->MF.getSubtarget<HexagonSubtarget>(); auto &QII = *QST.getInstrInfo(); if (SU->isInstr() && QII.mayBeCurLoad(*SU->getInstr())) { @@ -698,303 +66,5 @@ int ConvergingVLIWScheduler::SchedulingCost(ReadyQueue &Q, SUnit *SU, } } - // Give preference to a zero latency instruction if the dependent - // instruction is in the current packet. - if (Q.getID() == TopQID && getWeakLeft(SU, true) == 0) { - for (const SDep &PI : SU->Preds) { - if (!PI.getSUnit()->getInstr()->isPseudo() && PI.isAssignedRegDep() && - PI.getLatency() == 0 && - Top.ResourceModel->isInPacket(PI.getSUnit())) { - ResCount += PriorityThree; - LLVM_DEBUG(if (verbose) dbgs() << "Z|"); - } - } - } else if (Q.getID() == BotQID && getWeakLeft(SU, false) == 0) { - for (const SDep &SI : SU->Succs) { - if (!SI.getSUnit()->getInstr()->isPseudo() && SI.isAssignedRegDep() && - SI.getLatency() == 0 && - Bot.ResourceModel->isInPacket(SI.getSUnit())) { - ResCount += PriorityThree; - LLVM_DEBUG(if (verbose) dbgs() << "Z|"); - } - } - } - - // If the instruction has a non-zero latency dependence with an instruction in - // the current packet, then it should not be scheduled yet. The case occurs - // when the dependent instruction is scheduled in a new packet, so the - // scheduler updates the current cycle and pending instructions become - // available. - if (CheckEarlyAvail) { - if (Q.getID() == TopQID) { - for (const auto &PI : SU->Preds) { - if (PI.getLatency() > 0 && - Top.ResourceModel->isInPacket(PI.getSUnit())) { - ResCount -= PriorityOne; - LLVM_DEBUG(if (verbose) dbgs() << "D|"); - } - } - } else { - for (const auto &SI : SU->Succs) { - if (SI.getLatency() > 0 && - Bot.ResourceModel->isInPacket(SI.getSUnit())) { - ResCount -= PriorityOne; - LLVM_DEBUG(if (verbose) dbgs() << "D|"); - } - } - } - } - - LLVM_DEBUG(if (verbose) { - std::stringstream dbgstr; - dbgstr << "Total " << std::setw(4) << ResCount << ")"; - dbgs() << dbgstr.str(); - }); - return ResCount; } - -/// Pick the best candidate from the top queue. -/// -/// TODO: getMaxPressureDelta results can be mostly cached for each SUnit during -/// DAG building. To adjust for the current scheduling location we need to -/// maintain the number of vreg uses remaining to be top-scheduled. -ConvergingVLIWScheduler::CandResult ConvergingVLIWScheduler:: -pickNodeFromQueue(VLIWSchedBoundary &Zone, const RegPressureTracker &RPTracker, - SchedCandidate &Candidate) { - ReadyQueue &Q = Zone.Available; - LLVM_DEBUG(if (SchedDebugVerboseLevel > 1) - readyQueueVerboseDump(RPTracker, Candidate, Q); - else Q.dump();); - - // getMaxPressureDelta temporarily modifies the tracker. - RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker); - - // BestSU remains NULL if no top candidates beat the best existing candidate. - CandResult FoundCandidate = NoCand; - for (ReadyQueue::iterator I = Q.begin(), E = Q.end(); I != E; ++I) { - RegPressureDelta RPDelta; - TempTracker.getMaxPressureDelta((*I)->getInstr(), RPDelta, - DAG->getRegionCriticalPSets(), - DAG->getRegPressure().MaxSetPressure); - - int CurrentCost = SchedulingCost(Q, *I, Candidate, RPDelta, false); - - // Initialize the candidate if needed. - if (!Candidate.SU) { - LLVM_DEBUG(traceCandidate("DCAND", Q, *I, CurrentCost)); - Candidate.SU = *I; - Candidate.RPDelta = RPDelta; - Candidate.SCost = CurrentCost; - FoundCandidate = NodeOrder; - continue; - } - - // Choose node order for negative cost candidates. There is no good - // candidate in this case. - if (CurrentCost < 0 && Candidate.SCost < 0) { - if ((Q.getID() == TopQID && (*I)->NodeNum < Candidate.SU->NodeNum) - || (Q.getID() == BotQID && (*I)->NodeNum > Candidate.SU->NodeNum)) { - LLVM_DEBUG(traceCandidate("NCAND", Q, *I, CurrentCost)); - Candidate.SU = *I; - Candidate.RPDelta = RPDelta; - Candidate.SCost = CurrentCost; - FoundCandidate = NodeOrder; - } - continue; - } - - // Best cost. - if (CurrentCost > Candidate.SCost) { - LLVM_DEBUG(traceCandidate("CCAND", Q, *I, CurrentCost)); - Candidate.SU = *I; - Candidate.RPDelta = RPDelta; - Candidate.SCost = CurrentCost; - FoundCandidate = BestCost; - continue; - } - - // Choose an instruction that does not depend on an artificial edge. - unsigned CurrWeak = getWeakLeft(*I, (Q.getID() == TopQID)); - unsigned CandWeak = getWeakLeft(Candidate.SU, (Q.getID() == TopQID)); - if (CurrWeak != CandWeak) { - if (CurrWeak < CandWeak) { - LLVM_DEBUG(traceCandidate("WCAND", Q, *I, CurrentCost)); - Candidate.SU = *I; - Candidate.RPDelta = RPDelta; - Candidate.SCost = CurrentCost; - FoundCandidate = Weak; - } - continue; - } - - if (CurrentCost == Candidate.SCost && Zone.isLatencyBound(*I)) { - unsigned CurrSize, CandSize; - if (Q.getID() == TopQID) { - CurrSize = (*I)->Succs.size(); - CandSize = Candidate.SU->Succs.size(); - } else { - CurrSize = (*I)->Preds.size(); - CandSize = Candidate.SU->Preds.size(); - } - if (CurrSize > CandSize) { - LLVM_DEBUG(traceCandidate("SPCAND", Q, *I, CurrentCost)); - Candidate.SU = *I; - Candidate.RPDelta = RPDelta; - Candidate.SCost = CurrentCost; - FoundCandidate = BestCost; - } - // Keep the old candidate if it's a better candidate. That is, don't use - // the subsequent tie breaker. - if (CurrSize != CandSize) - continue; - } - - // Tie breaker. - // To avoid scheduling indeterminism, we need a tie breaker - // for the case when cost is identical for two nodes. - if (UseNewerCandidate && CurrentCost == Candidate.SCost) { - if ((Q.getID() == TopQID && (*I)->NodeNum < Candidate.SU->NodeNum) - || (Q.getID() == BotQID && (*I)->NodeNum > Candidate.SU->NodeNum)) { - LLVM_DEBUG(traceCandidate("TCAND", Q, *I, CurrentCost)); - Candidate.SU = *I; - Candidate.RPDelta = RPDelta; - Candidate.SCost = CurrentCost; - FoundCandidate = NodeOrder; - continue; - } - } - - // Fall through to original instruction order. - // Only consider node order if Candidate was chosen from this Q. - if (FoundCandidate == NoCand) - continue; - } - return FoundCandidate; -} - -/// Pick the best candidate node from either the top or bottom queue. -SUnit *ConvergingVLIWScheduler::pickNodeBidrectional(bool &IsTopNode) { - // Schedule as far as possible in the direction of no choice. This is most - // efficient, but also provides the best heuristics for CriticalPSets. - if (SUnit *SU = Bot.pickOnlyChoice()) { - LLVM_DEBUG(dbgs() << "Picked only Bottom\n"); - IsTopNode = false; - return SU; - } - if (SUnit *SU = Top.pickOnlyChoice()) { - LLVM_DEBUG(dbgs() << "Picked only Top\n"); - IsTopNode = true; - return SU; - } - SchedCandidate BotCand; - // Prefer bottom scheduling when heuristics are silent. - CandResult BotResult = pickNodeFromQueue(Bot, - DAG->getBotRPTracker(), BotCand); - assert(BotResult != NoCand && "failed to find the first candidate"); - - // If either Q has a single candidate that provides the least increase in - // Excess pressure, we can immediately schedule from that Q. - // - // RegionCriticalPSets summarizes the pressure within the scheduled region and - // affects picking from either Q. If scheduling in one direction must - // increase pressure for one of the excess PSets, then schedule in that - // direction first to provide more freedom in the other direction. - if (BotResult == SingleExcess || BotResult == SingleCritical) { - LLVM_DEBUG(dbgs() << "Prefered Bottom Node\n"); - IsTopNode = false; - return BotCand.SU; - } - // Check if the top Q has a better candidate. - SchedCandidate TopCand; - CandResult TopResult = pickNodeFromQueue(Top, - DAG->getTopRPTracker(), TopCand); - assert(TopResult != NoCand && "failed to find the first candidate"); - - if (TopResult == SingleExcess || TopResult == SingleCritical) { - LLVM_DEBUG(dbgs() << "Prefered Top Node\n"); - IsTopNode = true; - return TopCand.SU; - } - // If either Q has a single candidate that minimizes pressure above the - // original region's pressure pick it. - if (BotResult == SingleMax) { - LLVM_DEBUG(dbgs() << "Prefered Bottom Node SingleMax\n"); - IsTopNode = false; - return BotCand.SU; - } - if (TopResult == SingleMax) { - LLVM_DEBUG(dbgs() << "Prefered Top Node SingleMax\n"); - IsTopNode = true; - return TopCand.SU; - } - if (TopCand.SCost > BotCand.SCost) { - LLVM_DEBUG(dbgs() << "Prefered Top Node Cost\n"); - IsTopNode = true; - return TopCand.SU; - } - // Otherwise prefer the bottom candidate in node order. - LLVM_DEBUG(dbgs() << "Prefered Bottom in Node order\n"); - IsTopNode = false; - return BotCand.SU; -} - -/// Pick the best node to balance the schedule. Implements MachineSchedStrategy. -SUnit *ConvergingVLIWScheduler::pickNode(bool &IsTopNode) { - if (DAG->top() == DAG->bottom()) { - assert(Top.Available.empty() && Top.Pending.empty() && - Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage"); - return nullptr; - } - SUnit *SU; - if (ForceTopDown) { - SU = Top.pickOnlyChoice(); - if (!SU) { - SchedCandidate TopCand; - CandResult TopResult = - pickNodeFromQueue(Top, DAG->getTopRPTracker(), TopCand); - assert(TopResult != NoCand && "failed to find the first candidate"); - (void)TopResult; - SU = TopCand.SU; - } - IsTopNode = true; - } else if (ForceBottomUp) { - SU = Bot.pickOnlyChoice(); - if (!SU) { - SchedCandidate BotCand; - CandResult BotResult = - pickNodeFromQueue(Bot, DAG->getBotRPTracker(), BotCand); - assert(BotResult != NoCand && "failed to find the first candidate"); - (void)BotResult; - SU = BotCand.SU; - } - IsTopNode = false; - } else { - SU = pickNodeBidrectional(IsTopNode); - } - if (SU->isTopReady()) - Top.removeReady(SU); - if (SU->isBottomReady()) - Bot.removeReady(SU); - - LLVM_DEBUG(dbgs() << "*** " << (IsTopNode ? "Top" : "Bottom") - << " Scheduling instruction in cycle " - << (IsTopNode ? Top.CurrCycle : Bot.CurrCycle) << " (" - << reportPackets() << ")\n"; - DAG->dumpNode(*SU)); - return SU; -} - -/// Update the scheduler's state after scheduling a node. This is the same node -/// that was just returned by pickNode(). However, VLIWMachineScheduler needs -/// to update it's state based on the current cycle before MachineSchedStrategy -/// does. -void ConvergingVLIWScheduler::schedNode(SUnit *SU, bool IsTopNode) { - if (IsTopNode) { - Top.bumpNode(SU); - SU->TopReadyCycle = Top.CurrCycle; - } else { - Bot.bumpNode(SU); - SU->BotReadyCycle = Bot.CurrCycle; - } -} diff --git a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h index fb0a7abd339b..3d8f557dc787 100644 --- a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h +++ b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h @@ -13,261 +13,28 @@ #ifndef LLVM_LIB_TARGET_HEXAGON_HEXAGONMACHINESCHEDULER_H #define LLVM_LIB_TARGET_HEXAGON_HEXAGONMACHINESCHEDULER_H -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/Twine.h" -#include "llvm/CodeGen/DFAPacketizer.h" #include "llvm/CodeGen/MachineScheduler.h" #include "llvm/CodeGen/RegisterPressure.h" -#include "llvm/CodeGen/ScheduleHazardRecognizer.h" -#include "llvm/CodeGen/TargetInstrInfo.h" -#include "llvm/CodeGen/TargetSchedule.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" -#include <algorithm> -#include <cassert> -#include <limits> -#include <memory> -#include <vector> +#include "llvm/CodeGen/VLIWMachineScheduler.h" namespace llvm { class SUnit; -class VLIWResourceModel { - /// ResourcesModel - Represents VLIW state. - /// Not limited to VLIW targets per se, but assumes - /// definition of DFA by a target. - DFAPacketizer *ResourcesModel; - - const TargetSchedModel *SchedModel; - - /// Local packet/bundle model. Purely - /// internal to the MI schedulre at the time. - std::vector<SUnit *> Packet; - - /// Total packets created. - unsigned TotalPackets = 0; - +class HexagonVLIWResourceModel : public VLIWResourceModel { public: - VLIWResourceModel(const TargetSubtargetInfo &STI, const TargetSchedModel *SM) - : SchedModel(SM) { - ResourcesModel = STI.getInstrInfo()->CreateTargetScheduleState(STI); - - // This hard requirement could be relaxed, - // but for now do not let it proceed. - assert(ResourcesModel && "Unimplemented CreateTargetScheduleState."); - - Packet.resize(SchedModel->getIssueWidth()); - Packet.clear(); - ResourcesModel->clearResources(); - } - - ~VLIWResourceModel() { - delete ResourcesModel; - } - - void resetPacketState() { - Packet.clear(); - } - - void resetDFA() { - ResourcesModel->clearResources(); - } - - void reset() { - Packet.clear(); - ResourcesModel->clearResources(); - } - - bool isResourceAvailable(SUnit *SU, bool IsTop); - bool reserveResources(SUnit *SU, bool IsTop); - unsigned getTotalPackets() const { return TotalPackets; } - bool isInPacket(SUnit *SU) const { return is_contained(Packet, SU); } + using VLIWResourceModel::VLIWResourceModel; + bool hasDependence(const SUnit *SUd, const SUnit *SUu) override; }; -/// Extend the standard ScheduleDAGMI to provide more context and override the -/// top-level schedule() driver. -class VLIWMachineScheduler : public ScheduleDAGMILive { -public: - VLIWMachineScheduler(MachineSchedContext *C, - std::unique_ptr<MachineSchedStrategy> S) - : ScheduleDAGMILive(C, std::move(S)) {} - - /// Schedule - This is called back from ScheduleDAGInstrs::Run() when it's - /// time to do some work. - void schedule() override; - - RegisterClassInfo *getRegClassInfo() { return RegClassInfo; } - int getBBSize() { return BB->size(); } -}; - -//===----------------------------------------------------------------------===// -// ConvergingVLIWScheduler - Implementation of the standard -// MachineSchedStrategy. -//===----------------------------------------------------------------------===// - -/// ConvergingVLIWScheduler shrinks the unscheduled zone using heuristics -/// to balance the schedule. -class ConvergingVLIWScheduler : public MachineSchedStrategy { - /// Store the state used by ConvergingVLIWScheduler heuristics, required - /// for the lifetime of one invocation of pickNode(). - struct SchedCandidate { - // The best SUnit candidate. - SUnit *SU = nullptr; - - // Register pressure values for the best candidate. - RegPressureDelta RPDelta; - - // Best scheduling cost. - int SCost = 0; - - SchedCandidate() = default; - }; - /// Represent the type of SchedCandidate found within a single queue. - enum CandResult { - NoCand, NodeOrder, SingleExcess, SingleCritical, SingleMax, MultiPressure, - BestCost, Weak}; - - /// Each Scheduling boundary is associated with ready queues. It tracks the - /// current cycle in whichever direction at has moved, and maintains the state - /// of "hazards" and other interlocks at the current cycle. - struct VLIWSchedBoundary { - VLIWMachineScheduler *DAG = nullptr; - const TargetSchedModel *SchedModel = nullptr; - - ReadyQueue Available; - ReadyQueue Pending; - bool CheckPending = false; - - ScheduleHazardRecognizer *HazardRec = nullptr; - VLIWResourceModel *ResourceModel = nullptr; - - unsigned CurrCycle = 0; - unsigned IssueCount = 0; - unsigned CriticalPathLength = 0; - - /// MinReadyCycle - Cycle of the soonest available instruction. - unsigned MinReadyCycle = std::numeric_limits<unsigned>::max(); - - // Remember the greatest min operand latency. - unsigned MaxMinLatency = 0; - - /// Pending queues extend the ready queues with the same ID and the - /// PendingFlag set. - VLIWSchedBoundary(unsigned ID, const Twine &Name) - : Available(ID, Name+".A"), - Pending(ID << ConvergingVLIWScheduler::LogMaxQID, Name+".P") {} - - ~VLIWSchedBoundary() { - delete ResourceModel; - delete HazardRec; - } - - void init(VLIWMachineScheduler *dag, const TargetSchedModel *smodel) { - DAG = dag; - SchedModel = smodel; - CurrCycle = 0; - IssueCount = 0; - // Initialize the critical path length limit, which used by the scheduling - // cost model to determine the value for scheduling an instruction. We use - // a slightly different heuristic for small and large functions. For small - // functions, it's important to use the height/depth of the instruction. - // For large functions, prioritizing by height or depth increases spills. - CriticalPathLength = DAG->getBBSize() / SchedModel->getIssueWidth(); - if (DAG->getBBSize() < 50) - // We divide by two as a cheap and simple heuristic to reduce the - // critcal path length, which increases the priority of using the graph - // height/depth in the scheduler's cost computation. - CriticalPathLength >>= 1; - else { - // For large basic blocks, we prefer a larger critical path length to - // decrease the priority of using the graph height/depth. - unsigned MaxPath = 0; - for (auto &SU : DAG->SUnits) - MaxPath = std::max(MaxPath, isTop() ? SU.getHeight() : SU.getDepth()); - CriticalPathLength = std::max(CriticalPathLength, MaxPath) + 1; - } - } - - bool isTop() const { - return Available.getID() == ConvergingVLIWScheduler::TopQID; - } - - bool checkHazard(SUnit *SU); - - void releaseNode(SUnit *SU, unsigned ReadyCycle); - - void bumpCycle(); - - void bumpNode(SUnit *SU); - - void releasePending(); - - void removeReady(SUnit *SU); - - SUnit *pickOnlyChoice(); - - bool isLatencyBound(SUnit *SU) { - if (CurrCycle >= CriticalPathLength) - return true; - unsigned PathLength = isTop() ? SU->getHeight() : SU->getDepth(); - return CriticalPathLength - CurrCycle <= PathLength; - } - }; - - VLIWMachineScheduler *DAG = nullptr; - const TargetSchedModel *SchedModel = nullptr; - - // State of the top and bottom scheduled instruction boundaries. - VLIWSchedBoundary Top; - VLIWSchedBoundary Bot; - - /// List of pressure sets that have a high pressure level in the region. - std::vector<bool> HighPressureSets; - -public: - /// SUnit::NodeQueueId: 0 (none), 1 (top), 2 (bot), 3 (both) - enum { - TopQID = 1, - BotQID = 2, - LogMaxQID = 2 - }; - - ConvergingVLIWScheduler() : Top(TopQID, "TopQ"), Bot(BotQID, "BotQ") {} - - void initialize(ScheduleDAGMI *dag) override; - - SUnit *pickNode(bool &IsTopNode) override; - - void schedNode(SUnit *SU, bool IsTopNode) override; - - void releaseTopNode(SUnit *SU) override; - - void releaseBottomNode(SUnit *SU) override; - - unsigned reportPackets() { - return Top.ResourceModel->getTotalPackets() + - Bot.ResourceModel->getTotalPackets(); - } - +class HexagonConvergingVLIWScheduler : public ConvergingVLIWScheduler { protected: - SUnit *pickNodeBidrectional(bool &IsTopNode); - - int pressureChange(const SUnit *SU, bool isBotUp); - - int SchedulingCost(ReadyQueue &Q, - SUnit *SU, SchedCandidate &Candidate, - RegPressureDelta &Delta, bool verbose); - - CandResult pickNodeFromQueue(VLIWSchedBoundary &Zone, - const RegPressureTracker &RPTracker, - SchedCandidate &Candidate); -#ifndef NDEBUG - void traceCandidate(const char *Label, const ReadyQueue &Q, SUnit *SU, - int Cost, PressureChange P = PressureChange()); - - void readyQueueVerboseDump(const RegPressureTracker &RPTracker, - SchedCandidate &Candidate, ReadyQueue &Q); -#endif + VLIWResourceModel * + createVLIWResourceModel(const TargetSubtargetInfo &STI, + const TargetSchedModel *SchedModel) const override; + int SchedulingCost(ReadyQueue &Q, SUnit *SU, SchedCandidate &Candidate, + RegPressureDelta &Delta, bool verbose) override; }; } // end namespace llvm diff --git a/llvm/lib/Target/Hexagon/HexagonPseudo.td b/llvm/lib/Target/Hexagon/HexagonPseudo.td index 11f8af7c41a0..afd63d6d4aa7 100644 --- a/llvm/lib/Target/Hexagon/HexagonPseudo.td +++ b/llvm/lib/Target/Hexagon/HexagonPseudo.td @@ -572,3 +572,14 @@ defm PS_storerd : NewCircularStore<DoubleRegs, WordAccess>; // __builtin_trap. let hasSideEffects = 1, isPseudo = 1, isCodeGenOnly = 1, isSolo = 1 in def PS_crash: InstHexagon<(outs), (ins), "", [], "", PSEUDO, TypePSEUDO>; + +// This is actual trap1 instruction from before v65. It's here since it is +// no longer included in DepInstrInfo.td. +def PS_trap1 : HInst<(outs), (ins u8_0Imm:$Ii), "trap1(#$Ii)", tc_53c851ab, + TypeJ>, Enc_a51a9a, Requires<[HasPreV65]> { + let Inst{1-0} = 0b00; + let Inst{7-5} = 0b000; + let Inst{13-13} = 0b0; + let Inst{31-16} = 0b0101010010000000; +} + diff --git a/llvm/lib/Target/Hexagon/HexagonSchedule.td b/llvm/lib/Target/Hexagon/HexagonSchedule.td index 88d775f16a7f..931578c9e78d 100644 --- a/llvm/lib/Target/Hexagon/HexagonSchedule.td +++ b/llvm/lib/Target/Hexagon/HexagonSchedule.td @@ -69,3 +69,4 @@ include "HexagonScheduleV66.td" include "HexagonScheduleV67.td" include "HexagonScheduleV67T.td" include "HexagonScheduleV68.td" +include "HexagonScheduleV69.td" diff --git a/llvm/lib/Target/Hexagon/HexagonScheduleV69.td b/llvm/lib/Target/Hexagon/HexagonScheduleV69.td new file mode 100644 index 000000000000..ddd246866e20 --- /dev/null +++ b/llvm/lib/Target/Hexagon/HexagonScheduleV69.td @@ -0,0 +1,40 @@ +//=-HexagonScheduleV69.td - HexagonV69 Scheduling Definitions *- tablegen -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +// +// ScalarItin and HVXItin contain some old itineraries +// still used by a handful of instructions. Hopefully, we will be able +// to get rid of them soon. +def HexagonV69ItinList : DepScalarItinV69, ScalarItin, + DepHVXItinV69, HVXItin, PseudoItin { + list<InstrItinData> ItinList = + !listconcat(DepScalarItinV69_list, ScalarItin_list, + DepHVXItinV69_list, HVXItin_list, PseudoItin_list); +} + +def HexagonItinerariesV69 : + ProcessorItineraries<[SLOT0, SLOT1, SLOT2, SLOT3, SLOT_ENDLOOP, + CVI_ST, CVI_XLANE, CVI_SHIFT, CVI_MPY0, CVI_MPY1, + CVI_LD, CVI_XLSHF, CVI_MPY01, CVI_ALL, + CVI_ALL_NOMEM, CVI_ZW], + [Hex_FWD, HVX_FWD], + HexagonV69ItinList.ItinList>; + +def HexagonModelV69 : SchedMachineModel { + // Max issue per cycle == bundle width. + let IssueWidth = 4; + let Itineraries = HexagonItinerariesV69; + let LoadLatency = 1; + let CompleteModel = 0; +} + +//===----------------------------------------------------------------------===// +// Hexagon V69 Resource Definitions - +//===----------------------------------------------------------------------===// + diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp index ecb2f88d8096..08bb4580b585 100644 --- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp +++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp @@ -75,6 +75,10 @@ static cl::opt<bool> EnableCheckBankConflict("hexagon-check-bank-conflict", cl::Hidden, cl::ZeroOrMore, cl::init(true), cl::desc("Enable checking for cache bank conflicts")); +static cl::opt<bool> EnableV68FloatCodeGen( + "force-hvx-float", cl::Hidden, cl::ZeroOrMore, cl::init(false), + cl::desc("Enable the code-generation for vector float instructions on v68.")); + HexagonSubtarget::HexagonSubtarget(const Triple &TT, StringRef CPU, StringRef FS, const TargetMachine &TM) : HexagonGenSubtargetInfo(TT, CPU, /*TuneCPU*/ CPU, FS), @@ -103,13 +107,71 @@ HexagonSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) { UseAudioOps = false; UseLongCalls = false; - UseBSBScheduling = hasV60Ops() && EnableBSBSched; + SubtargetFeatures Features(FS); + + // Turn on QFloat if the HVX version is v68+. + // The function ParseSubtargetFeatures will set feature bits and initialize + // subtarget's variables all in one, so there isn't a good way to preprocess + // the feature string, other than by tinkering with it directly. + auto IsQFloatFS = [](StringRef F) { + return F == "+hvx-qfloat" || F == "-hvx-qfloat"; + }; + if (!llvm::count_if(Features.getFeatures(), IsQFloatFS)) { + auto getHvxVersion = [&Features](StringRef FS) -> StringRef { + for (StringRef F : llvm::reverse(Features.getFeatures())) { + if (F.startswith("+hvxv")) + return F; + } + for (StringRef F : llvm::reverse(Features.getFeatures())) { + if (F == "-hvx") + return StringRef(); + if (F.startswith("+hvx") || F == "-hvx") + return F.take_front(4); // Return "+hvx" or "-hvx". + } + return StringRef(); + }; + + bool AddQFloat = false; + StringRef HvxVer = getHvxVersion(FS); + if (HvxVer.startswith("+hvxv")) { + int Ver = 0; + if (!HvxVer.drop_front(5).consumeInteger(10, Ver) && Ver >= 68) + AddQFloat = true; + } else if (HvxVer == "+hvx") { + if (hasV68Ops()) + AddQFloat = true; + } - ParseSubtargetFeatures(CPUString, /*TuneCPU*/ CPUString, FS); + if (AddQFloat) + Features.AddFeature("+hvx-qfloat"); + } + + std::string FeatureString = Features.getString(); + ParseSubtargetFeatures(CPUString, /*TuneCPU*/ CPUString, FeatureString); + + // Enable float code generation only if the flag(s) are set and + // the feature is enabled. v68 is guarded by additional flags. + bool GreaterThanV68 = false; + if (useHVXV69Ops()) + GreaterThanV68 = true; + + // Support for deprecated qfloat/ieee codegen flags + if (!GreaterThanV68) { + if (EnableV68FloatCodeGen) + UseHVXFloatingPoint = true; + } else { + UseHVXFloatingPoint = true; + } + + if (UseHVXQFloatOps && UseHVXIEEEFPOps && UseHVXFloatingPoint) + LLVM_DEBUG( + dbgs() << "Behavior is undefined for simultaneous qfloat and ieee hvx codegen..."); if (OverrideLongCalls.getPosition()) UseLongCalls = OverrideLongCalls; + UseBSBScheduling = hasV60Ops() && EnableBSBSched; + if (isTinyCore()) { // Tiny core has a single thread, so back-to-back scheduling is enabled by // default. @@ -117,10 +179,10 @@ HexagonSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) { UseBSBScheduling = false; } - FeatureBitset Features = getFeatureBits(); + FeatureBitset FeatureBits = getFeatureBits(); if (HexagonDisableDuplex) - setFeatureBits(Features.reset(Hexagon::FeatureDuplex)); - setFeatureBits(Hexagon_MC::completeHVXFeatures(Features)); + setFeatureBits(FeatureBits.reset(Hexagon::FeatureDuplex)); + setFeatureBits(Hexagon_MC::completeHVXFeatures(FeatureBits)); return *this; } diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.h b/llvm/lib/Target/Hexagon/HexagonSubtarget.h index a4f2e159bf4b..e4f375440be1 100644 --- a/llvm/lib/Target/Hexagon/HexagonSubtarget.h +++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.h @@ -56,6 +56,10 @@ class HexagonSubtarget : public HexagonGenSubtargetInfo { bool UseSmallData = false; bool UseUnsafeMath = false; bool UseZRegOps = false; + bool UseHVXIEEEFPOps = false; + bool UseHVXQFloatOps = false; + bool UseHVXFloatingPoint = false; + bool UseCabac = false; bool HasPreV65 = false; bool HasMemNoShuf = false; @@ -138,6 +142,8 @@ public: /// subtarget options. Definition of function is auto generated by tblgen. void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS); + bool isXRaySupported() const override { return true; } + bool hasV5Ops() const { return getHexagonArchVersion() >= Hexagon::ArchEnum::V5; } @@ -186,6 +192,12 @@ public: bool hasV68OpsOnly() const { return getHexagonArchVersion() == Hexagon::ArchEnum::V68; } + bool hasV69Ops() const { + return getHexagonArchVersion() >= Hexagon::ArchEnum::V69; + } + bool hasV69OpsOnly() const { + return getHexagonArchVersion() == Hexagon::ArchEnum::V69; + } bool useAudioOps() const { return UseAudioOps; } bool useCompound() const { return UseCompound; } @@ -197,10 +209,16 @@ public: bool useSmallData() const { return UseSmallData; } bool useUnsafeMath() const { return UseUnsafeMath; } bool useZRegOps() const { return UseZRegOps; } + bool useCabac() const { return UseCabac; } bool isTinyCore() const { return HexagonProcFamily == TinyCore; } bool isTinyCoreWithDuplex() const { return isTinyCore() && EnableDuplex; } + bool useHVXIEEEFPOps() const { return UseHVXIEEEFPOps && useHVXOps(); } + bool useHVXQFloatOps() const { + return UseHVXQFloatOps && HexagonHVXVersion >= Hexagon::ArchEnum::V68; + } + bool useHVXFloatingPoint() const { return UseHVXFloatingPoint; } bool useHVXOps() const { return HexagonHVXVersion > Hexagon::ArchEnum::NoArch; } @@ -222,6 +240,9 @@ public: bool useHVXV68Ops() const { return HexagonHVXVersion >= Hexagon::ArchEnum::V68; } + bool useHVXV69Ops() const { + return HexagonHVXVersion >= Hexagon::ArchEnum::V69; + } bool useHVX128BOps() const { return useHVXOps() && UseHVX128BOps; } bool useHVX64BOps() const { return useHVXOps() && UseHVX64BOps; } @@ -281,7 +302,11 @@ public: } ArrayRef<MVT> getHVXElementTypes() const { - static MVT Types[] = { MVT::i8, MVT::i16, MVT::i32 }; + static MVT Types[] = {MVT::i8, MVT::i16, MVT::i32}; + static MVT TypesV68[] = {MVT::i8, MVT::i16, MVT::i32, MVT::f16, MVT::f32}; + + if (useHVXV68Ops() && useHVXFloatingPoint()) + return makeArrayRef(TypesV68); return makeArrayRef(Types); } diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp index 66de698182d7..fcf829b522cc 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp @@ -21,6 +21,7 @@ #include "TargetInfo/HexagonTargetInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/VLIWMachineScheduler.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/MC/TargetRegistry.h" @@ -120,8 +121,8 @@ extern "C" int HexagonTargetMachineModule; int HexagonTargetMachineModule = 0; static ScheduleDAGInstrs *createVLIWMachineSched(MachineSchedContext *C) { - ScheduleDAGMILive *DAG = - new VLIWMachineScheduler(C, std::make_unique<ConvergingVLIWScheduler>()); + ScheduleDAGMILive *DAG = new VLIWMachineScheduler( + C, std::make_unique<HexagonConvergingVLIWScheduler>()); DAG->addMutation(std::make_unique<HexagonSubtarget::UsrOverflowMutation>()); DAG->addMutation(std::make_unique<HexagonSubtarget::HVXMemLatencyMutation>()); DAG->addMutation(std::make_unique<HexagonSubtarget::CallMutation>()); diff --git a/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp b/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp index 1d325553f45a..85ec0cdcd8f0 100644 --- a/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp +++ b/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp @@ -294,7 +294,7 @@ bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) { bool Avail = ResourceTracker->canReserveResources(*ExtMI); if (Reserve && Avail) ResourceTracker->reserveResources(*ExtMI); - MF.DeleteMachineInstr(ExtMI); + MF.deleteMachineInstr(ExtMI); return Avail; } @@ -890,7 +890,7 @@ bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI, const MCInstrDesc &D = HII->get(NewOpcode); MachineInstr *NewMI = MF.CreateMachineInstr(D, DebugLoc()); bool ResourcesAvailable = ResourceTracker->canReserveResources(*NewMI); - MF.DeleteMachineInstr(NewMI); + MF.deleteMachineInstr(NewMI); if (!ResourcesAvailable) return false; @@ -1082,6 +1082,11 @@ bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) { if (HII->isSolo(MI)) return true; + if (MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_ENTER || + MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_EXIT || + MI.getOpcode() == Hexagon::PATCHABLE_TAIL_CALL) + return true; + if (MI.getOpcode() == Hexagon::A2_nop) return true; diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp index ea2798a3b44e..21386a91c7b3 100644 --- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp +++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp @@ -536,7 +536,7 @@ auto AlignVectors::createAddressGroups() -> bool { erase_if(AddrGroups, [](auto &G) { return G.second.size() == 1; }); // Remove groups that don't use HVX types. erase_if(AddrGroups, [&](auto &G) { - return !llvm::any_of( + return llvm::none_of( G.second, [&](auto &I) { return HVC.HST.isTypeForHVX(I.ValTy); }); }); diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h index 4125566bc58a..c9a1781a4543 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h @@ -154,9 +154,8 @@ namespace HexagonII { PrefersSlot3Pos = 57, PrefersSlot3Mask = 0x1, - // v65 - HasTmpDstPos = 60, - HasTmpDstMask = 0x1, + HasHvxTmpPos = 60, + HasHvxTmpMask = 0x1, CVINewPos = 62, CVINewMask = 0x1, diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp index fee1acdbbe8a..96c2965296ca 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp @@ -98,6 +98,10 @@ void HexagonMCChecker::init(MCInst const &MCI) { for (unsigned i = 0; i < MCID.getNumImplicitUses(); ++i) initReg(MCI, MCID.getImplicitUses()[i], PredReg, isTrue); + const bool IgnoreTmpDst = (HexagonMCInstrInfo::hasTmpDst(MCII, MCI) || + HexagonMCInstrInfo::hasHvxTmp(MCII, MCI)) && + STI.getFeatureBits()[Hexagon::ArchV69]; + // Get implicit register definitions. if (const MCPhysReg *ImpDef = MCID.getImplicitDefs()) for (; *ImpDef; ++ImpDef) { @@ -123,7 +127,7 @@ void HexagonMCChecker::init(MCInst const &MCI) { HexagonMCInstrInfo::isPredicateLate(MCII, MCI)) // Include implicit late predicates. LatePreds.insert(R); - else + else if (!IgnoreTmpDst) Defs[R].insert(PredSense(PredReg, isTrue)); } @@ -178,7 +182,7 @@ void HexagonMCChecker::init(MCInst const &MCI) { // vshuff(Vx, Vy, Rx) <- Vx(0) and Vy(1) are both source and // destination registers with this instruction. same for vdeal(Vx,Vy,Rx) Uses.insert(*SRI); - else + else if (!IgnoreTmpDst) Defs[*SRI].insert(PredSense(PredReg, isTrue)); } } @@ -227,9 +231,11 @@ bool HexagonMCChecker::check(bool FullCheck) { bool chkAXOK = checkAXOK(); bool chkCofMax1 = checkCOFMax1(); bool chkHWLoop = checkHWLoop(); + bool chkValidTmpDst = FullCheck ? checkValidTmpDst() : true; bool chkLegalVecRegPair = checkLegalVecRegPair(); bool chk = chkP && chkNV && chkR && chkRRO && chkS && chkSh && chkSl && - chkAXOK && chkCofMax1 && chkHWLoop && chkLegalVecRegPair; + chkAXOK && chkCofMax1 && chkHWLoop && chkValidTmpDst && + chkLegalVecRegPair; return chk; } @@ -676,6 +682,32 @@ bool HexagonMCChecker::checkShuffle() { return MCSDX.check(); } +bool HexagonMCChecker::checkValidTmpDst() { + if (!STI.getFeatureBits()[Hexagon::ArchV69]) { + return true; + } + auto HasTmp = [&](MCInst const &I) { + return HexagonMCInstrInfo::hasTmpDst(MCII, I) || + HexagonMCInstrInfo::hasHvxTmp(MCII, I); + }; + unsigned HasTmpCount = + llvm::count_if(HexagonMCInstrInfo::bundleInstructions(MCII, MCB), HasTmp); + + if (HasTmpCount > 1) { + reportError( + MCB.getLoc(), + "this packet has more than one HVX vtmp/.tmp destination instruction"); + + for (auto const &I : HexagonMCInstrInfo::bundleInstructions(MCII, MCB)) + if (HasTmp(I)) + reportNote(I.getLoc(), + "this is an HVX vtmp/.tmp destination instruction"); + + return false; + } + return true; +} + void HexagonMCChecker::compoundRegisterMap(unsigned &Register) { switch (Register) { default: diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h index 00afdb664ba5..dbd3d8ae45e6 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h @@ -99,6 +99,7 @@ class HexagonMCChecker { bool checkHWLoop(); bool checkCOFMax1(); bool checkLegalVecRegPair(); + bool checkValidTmpDst(); static void compoundRegisterMap(unsigned &); diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp index fa12fe1da448..68ccb20f4f15 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp @@ -939,10 +939,24 @@ bool HexagonMCInstrInfo::prefersSlot3(MCInstrInfo const &MCII, return (F >> HexagonII::PrefersSlot3Pos) & HexagonII::PrefersSlot3Mask; } -/// return true if instruction has hasTmpDst attribute. bool HexagonMCInstrInfo::hasTmpDst(MCInstrInfo const &MCII, MCInst const &MCI) { + switch (MCI.getOpcode()) { + default: + return false; + case Hexagon::V6_vgathermh: + case Hexagon::V6_vgathermhq: + case Hexagon::V6_vgathermhw: + case Hexagon::V6_vgathermhwq: + case Hexagon::V6_vgathermw: + case Hexagon::V6_vgathermwq: + return true; + } + return false; +} + +bool HexagonMCInstrInfo::hasHvxTmp(MCInstrInfo const &MCII, MCInst const &MCI) { const uint64_t F = HexagonMCInstrInfo::getDesc(MCII, MCI).TSFlags; - return (F >> HexagonII::HasTmpDstPos) & HexagonII::HasTmpDstMask; + return (F >> HexagonII::HasHvxTmpPos) & HexagonII::HasHvxTmpMask; } bool HexagonMCInstrInfo::requiresSlot(MCSubtargetInfo const &STI, diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h index 7b3c079880f8..5c56db14798f 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h @@ -41,7 +41,8 @@ public: namespace Hexagon { -class PacketIterator { +class PacketIterator : public std::iterator<std::forward_iterator_tag, + PacketIterator> { MCInstrInfo const &MCII; MCInst::const_iterator BundleCurrent; MCInst::const_iterator BundleEnd; @@ -188,6 +189,7 @@ bool hasImmExt(MCInst const &MCI); bool hasNewValue(MCInstrInfo const &MCII, MCInst const &MCI); bool hasNewValue2(MCInstrInfo const &MCII, MCInst const &MCI); bool hasTmpDst(MCInstrInfo const &MCII, MCInst const &MCI); +bool hasHvxTmp(MCInstrInfo const &MCII, MCInst const &MCI); unsigned iClassOfDuplexPair(unsigned Ga, unsigned Gb); int64_t minConstant(MCInst const &MCI, size_t Index); diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp index d832a756cb92..dfdddb50657c 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp @@ -80,6 +80,8 @@ cl::opt<bool> MV67T("mv67t", cl::Hidden, cl::desc("Build for Hexagon V67T"), cl::init(false)); cl::opt<bool> MV68("mv68", cl::Hidden, cl::desc("Build for Hexagon V68"), cl::init(false)); +cl::opt<bool> MV69("mv69", cl::Hidden, cl::desc("Build for Hexagon V69"), + cl::init(false)); cl::opt<Hexagon::ArchEnum> EnableHVX("mhvx", @@ -91,6 +93,7 @@ cl::opt<Hexagon::ArchEnum> clEnumValN(Hexagon::ArchEnum::V66, "v66", "Build for HVX v66"), clEnumValN(Hexagon::ArchEnum::V67, "v67", "Build for HVX v67"), clEnumValN(Hexagon::ArchEnum::V68, "v68", "Build for HVX v68"), + clEnumValN(Hexagon::ArchEnum::V69, "v69", "Build for HVX v69"), // Sentinel for no value specified. clEnumValN(Hexagon::ArchEnum::Generic, "", "")), // Sentinel for flag not present. @@ -101,6 +104,11 @@ static cl::opt<bool> DisableHVX("mno-hvx", cl::Hidden, cl::desc("Disable Hexagon Vector eXtensions")); +static cl::opt<bool> + EnableHvxIeeeFp("mhvx-ieee-fp", cl::Hidden, + cl::desc("Enable HVX IEEE floating point extensions")); +static cl::opt<bool> EnableHexagonCabac + ("mcabac", cl::desc("tbd"), cl::init(false)); static StringRef DefaultArch = "hexagonv60"; @@ -123,6 +131,8 @@ static StringRef HexagonGetArchVariant() { return "hexagonv67t"; if (MV68) return "hexagonv68"; + if (MV69) + return "hexagonv69"; return ""; } @@ -371,6 +381,9 @@ std::string selectHexagonFS(StringRef CPU, StringRef FS) { case Hexagon::ArchEnum::V68: Result.push_back("+hvxv68"); break; + case Hexagon::ArchEnum::V69: + Result.push_back("+hvxv69"); + break; case Hexagon::ArchEnum::Generic:{ Result.push_back(StringSwitch<StringRef>(CPU) .Case("hexagonv60", "+hvxv60") @@ -379,13 +392,19 @@ std::string selectHexagonFS(StringRef CPU, StringRef FS) { .Case("hexagonv66", "+hvxv66") .Case("hexagonv67", "+hvxv67") .Case("hexagonv67t", "+hvxv67") - .Case("hexagonv68", "+hvxv68")); + .Case("hexagonv68", "+hvxv68") + .Case("hexagonv69", "+hvxv69")); break; } case Hexagon::ArchEnum::NoArch: // Sentinel if -mhvx isn't specified break; } + if (EnableHvxIeeeFp) + Result.push_back("+hvx-ieee-fp"); + if (EnableHexagonCabac) + Result.push_back("+cabac"); + return join(Result.begin(), Result.end(), ","); } } @@ -422,8 +441,8 @@ FeatureBitset Hexagon_MC::completeHVXFeatures(const FeatureBitset &S) { // turns on hvxvNN, corresponding to the existing ArchVNN. FeatureBitset FB = S; unsigned CpuArch = ArchV5; - for (unsigned F : {ArchV68, ArchV67, ArchV66, ArchV65, ArchV62, ArchV60, - ArchV55, ArchV5}) { + for (unsigned F : {ArchV69, ArchV68, ArchV67, ArchV66, ArchV65, ArchV62, + ArchV60, ArchV55, ArchV5}) { if (!FB.test(F)) continue; CpuArch = F; @@ -438,7 +457,8 @@ FeatureBitset Hexagon_MC::completeHVXFeatures(const FeatureBitset &S) { } bool HasHvxVer = false; for (unsigned F : {ExtensionHVXV60, ExtensionHVXV62, ExtensionHVXV65, - ExtensionHVXV66, ExtensionHVXV67, ExtensionHVXV68}) { + ExtensionHVXV66, ExtensionHVXV67, ExtensionHVXV68, + ExtensionHVXV69}) { if (!FB.test(F)) continue; HasHvxVer = true; @@ -451,6 +471,9 @@ FeatureBitset Hexagon_MC::completeHVXFeatures(const FeatureBitset &S) { // HasHvxVer is false, and UseHvx is true. switch (CpuArch) { + case ArchV69: + FB.set(ExtensionHVXV69); + LLVM_FALLTHROUGH; case ArchV68: FB.set(ExtensionHVXV68); LLVM_FALLTHROUGH; @@ -538,6 +561,7 @@ unsigned Hexagon_MC::GetELFFlags(const MCSubtargetInfo &STI) { {"hexagonv67", ELF::EF_HEXAGON_MACH_V67}, {"hexagonv67t", ELF::EF_HEXAGON_MACH_V67T}, {"hexagonv68", ELF::EF_HEXAGON_MACH_V68}, + {"hexagonv69", ELF::EF_HEXAGON_MACH_V69}, }; auto F = ElfFlags.find(STI.getCPU()); diff --git a/llvm/lib/Target/M68k/M68kInstrControl.td b/llvm/lib/Target/M68k/M68kInstrControl.td index 708474726861..9f87833ab0e2 100644 --- a/llvm/lib/Target/M68k/M68kInstrControl.td +++ b/llvm/lib/Target/M68k/M68kInstrControl.td @@ -118,13 +118,13 @@ def SET#"p8"#cc : MxSccM<cc, MxType8.POp, MxType8.PPat, MxEncEAp_0, MxExtI16_0>; /// 0 1 0 0 1 1 1 0 1 1 | MODE | REG ///------------------------------+---------+--------- let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in -class MxJMP<MxOperand LOCOp, ComplexPattern LOCPat, MxEncEA EA, MxEncExt EXT> +class MxJMP<MxOperand LOCOp, MxEncEA EA, MxEncExt EXT> : MxInst<(outs), (ins LOCOp:$dst), "jmp\t$dst", [(brind iPTR:$dst)], MxEncoding<EA.Reg, EA.DA, EA.Mode, MxBead2Bits<0b11>, MxBead4Bits<0b1110>, MxBead4Bits<0b0100>, EXT.Imm, EXT.B8, EXT.Scale, EXT.WL, EXT.DAReg>>; -def JMP32j : MxJMP<MxARI32, MxCP_ARI, MxEncEAj_0, MxExtEmpty>; +def JMP32j : MxJMP<MxARI32, MxEncEAj_0, MxExtEmpty>; // FIXME Support 16 bit indirect jump. @@ -147,17 +147,17 @@ def JMP32j : MxJMP<MxARI32, MxCP_ARI, MxEncEAj_0, MxExtEmpty>; /// 32-BIT DISPLACEMENT IF 8-BIT DISPLACEMENT = $FF /// -------------------------------------------------- let isBranch = 1, isTerminator = 1, Uses = [CCR] in -class MxBcc<string cc, Operand TARGET, MxType TYPE, MxEncoding ENC = MxEncEmpty> +class MxBcc<string cc, Operand TARGET, MxEncoding ENC = MxEncEmpty> : MxInst<(outs), (ins TARGET:$dst), "b"#cc#"\t$dst", [], ENC>; foreach cc = [ "cc", "ls", "lt", "eq", "mi", "ne", "ge", "cs", "pl", "gt", "hi", "vc", "le", "vs"] in { def B#cc#"8" - : MxBcc<cc, MxBrTarget8, MxType8, + : MxBcc<cc, MxBrTarget8, MxEncoding<MxBead8Disp<0>, !cast<MxBead4Bits>("MxCC"#cc), MxBead4Bits<0x6>>>; def B#cc#"16" - : MxBcc<cc, MxBrTarget16, MxType16, + : MxBcc<cc, MxBrTarget16, MxEncoding<MxBead4Bits<0x0>, MxBead4Bits<0x0>, !cast<MxBead4Bits>("MxCC"#cc), MxBead4Bits<0x6>, MxBead16Imm<0>>>; @@ -179,13 +179,13 @@ def : Pat<(MxBrCond bb:$target, !cast<PatLeaf>("MxCOND"#cc), CCR), /// 32-BIT DISPLACEMENT IF 8-BIT DISPLACEMENT = $FF /// ------------------------------------------------- let isBranch = 1, isTerminator = 1, isBarrier=1 in -class MxBra<Operand TARGET, MxType TYPE, MxEncoding ENC = MxEncEmpty> +class MxBra<Operand TARGET, MxEncoding ENC = MxEncEmpty> : MxInst<(outs), (ins TARGET:$dst), "bra\t$dst", [], ENC>; -def BRA8 : MxBra<MxBrTarget8, MxType8, +def BRA8 : MxBra<MxBrTarget8, MxEncoding<MxBead8Disp<0>, MxBead4Bits<0x0>, MxBead4Bits<0x6>>>; -def BRA16 : MxBra<MxBrTarget16, MxType16, +def BRA16 : MxBra<MxBrTarget16, MxEncoding<MxBead4Bits<0x0>, MxBead4Bits<0x0>, MxBead4Bits<0x0>, MxBead4Bits<0x6>, MxBead16Imm<0>>>; diff --git a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp index 2a77a150f9aa..4ef9a567d453 100644 --- a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp +++ b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp @@ -189,8 +189,8 @@ bool MSP430FrameLowering::spillCalleeSavedRegisters( MSP430MachineFunctionInfo *MFI = MF.getInfo<MSP430MachineFunctionInfo>(); MFI->setCalleeSavedFrameSize(CSI.size() * 2); - for (unsigned i = CSI.size(); i != 0; --i) { - unsigned Reg = CSI[i-1].getReg(); + for (const CalleeSavedInfo &I : llvm::reverse(CSI)) { + unsigned Reg = I.getReg(); // Add the callee-saved register as live-in. It's killed at the spill. MBB.addLiveIn(Reg); BuildMI(MBB, MI, DL, TII.get(MSP430::PUSH16r)) diff --git a/llvm/lib/Target/Mips/Mips16HardFloat.cpp b/llvm/lib/Target/Mips/Mips16HardFloat.cpp index 203e05dde7ad..419f0ac1a8a7 100644 --- a/llvm/lib/Target/Mips/Mips16HardFloat.cpp +++ b/llvm/lib/Target/Mips/Mips16HardFloat.cpp @@ -479,14 +479,12 @@ static void createFPFnStub(Function *F, Module *M, FPParamVariant PV, // remove the use-soft-float attribute static void removeUseSoftFloat(Function &F) { - AttrBuilder B; LLVM_DEBUG(errs() << "removing -use-soft-float\n"); - B.addAttribute("use-soft-float", "false"); - F.removeFnAttrs(B); + F.removeFnAttr("use-soft-float"); if (F.hasFnAttribute("use-soft-float")) { LLVM_DEBUG(errs() << "still has -use-soft-float\n"); } - F.addFnAttrs(B); + F.addFnAttr("use-soft-float", "false"); } // This pass only makes sense when the underlying chip has floating point but diff --git a/llvm/lib/Target/Mips/MipsBranchExpansion.cpp b/llvm/lib/Target/Mips/MipsBranchExpansion.cpp index aa8e298fa759..4e9a23d077da 100644 --- a/llvm/lib/Target/Mips/MipsBranchExpansion.cpp +++ b/llvm/lib/Target/Mips/MipsBranchExpansion.cpp @@ -36,7 +36,7 @@ /// /// Regarding compact branch hazard prevention: /// -/// Hazards handled: forbidden slots for MIPSR6. +/// Hazards handled: forbidden slots for MIPSR6, FPU slots for MIPS3 and below. /// /// A forbidden slot hazard occurs when a compact branch instruction is executed /// and the adjacent instruction in memory is a control transfer instruction @@ -160,7 +160,10 @@ private: bool buildProperJumpMI(MachineBasicBlock *MBB, MachineBasicBlock::iterator Pos, DebugLoc DL); void expandToLongBranch(MBBInfo &Info); + template <typename Pred, typename Safe> + bool handleSlot(Pred Predicate, Safe SafeInSlot); bool handleForbiddenSlot(); + bool handleFPUDelaySlot(); bool handlePossibleLongBranch(); const MipsSubtarget *STI; @@ -738,30 +741,27 @@ static void emitGPDisp(MachineFunction &F, const MipsInstrInfo *TII) { MBB.removeLiveIn(Mips::V0); } -bool MipsBranchExpansion::handleForbiddenSlot() { - // Forbidden slot hazards are only defined for MIPSR6 but not microMIPSR6. - if (!STI->hasMips32r6() || STI->inMicroMipsMode()) - return false; - +template <typename Pred, typename Safe> +bool MipsBranchExpansion::handleSlot(Pred Predicate, Safe SafeInSlot) { bool Changed = false; for (MachineFunction::iterator FI = MFp->begin(); FI != MFp->end(); ++FI) { for (Iter I = FI->begin(); I != FI->end(); ++I) { - // Forbidden slot hazard handling. Use lookahead over state. - if (!TII->HasForbiddenSlot(*I)) + // Delay slot hazard handling. Use lookahead over state. + if (!Predicate(*I)) continue; - Iter Inst; + Iter IInSlot; bool LastInstInFunction = std::next(I) == FI->end() && std::next(FI) == MFp->end(); if (!LastInstInFunction) { std::pair<Iter, bool> Res = getNextMachineInstr(std::next(I), &*FI); LastInstInFunction |= Res.second; - Inst = Res.first; + IInSlot = Res.first; } - if (LastInstInFunction || !TII->SafeInForbiddenSlot(*Inst)) { + if (LastInstInFunction || !SafeInSlot(*IInSlot, *I)) { MachineBasicBlock::instr_iterator Iit = I->getIterator(); if (std::next(Iit) == FI->end() || @@ -778,6 +778,29 @@ bool MipsBranchExpansion::handleForbiddenSlot() { return Changed; } +bool MipsBranchExpansion::handleForbiddenSlot() { + // Forbidden slot hazards are only defined for MIPSR6 but not microMIPSR6. + if (!STI->hasMips32r6() || STI->inMicroMipsMode()) + return false; + + return handleSlot( + [this](auto &I) -> bool { return TII->HasForbiddenSlot(I); }, + [this](auto &IInSlot, auto &I) -> bool { + return TII->SafeInForbiddenSlot(IInSlot); + }); +} + +bool MipsBranchExpansion::handleFPUDelaySlot() { + // FPU delay slots are only defined for MIPS3 and below. + if (STI->hasMips32() || STI->hasMips4()) + return false; + + return handleSlot([this](auto &I) -> bool { return TII->HasFPUDelaySlot(I); }, + [this](auto &IInSlot, auto &I) -> bool { + return TII->SafeInFPUDelaySlot(IInSlot, I); + }); +} + bool MipsBranchExpansion::handlePossibleLongBranch() { if (STI->inMips16Mode() || !STI->enableLongBranchPass()) return false; @@ -857,13 +880,16 @@ bool MipsBranchExpansion::runOnMachineFunction(MachineFunction &MF) { // Run these two at least once bool longBranchChanged = handlePossibleLongBranch(); bool forbiddenSlotChanged = handleForbiddenSlot(); + bool fpuDelaySlotChanged = handleFPUDelaySlot(); - bool Changed = longBranchChanged || forbiddenSlotChanged; + bool Changed = + longBranchChanged || forbiddenSlotChanged || fpuDelaySlotChanged; // Then run them alternatively while there are changes while (forbiddenSlotChanged) { longBranchChanged = handlePossibleLongBranch(); - if (!longBranchChanged) + fpuDelaySlotChanged = handleFPUDelaySlot(); + if (!longBranchChanged && !fpuDelaySlotChanged) break; forbiddenSlotChanged = handleForbiddenSlot(); } diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp index 4f364ef6afc7..9377e83524e1 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.cpp +++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp @@ -4121,7 +4121,7 @@ MipsTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, case 'd': // Address register. Same as 'r' unless generating MIPS16 code. case 'y': // Same as 'r'. Exists for compatibility. case 'r': - if (VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8) { + if (VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8 || VT == MVT::i1) { if (Subtarget.inMips16Mode()) return std::make_pair(0U, &Mips::CPU16RegsRegClass); return std::make_pair(0U, &Mips::GPR32RegClass); diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.cpp b/llvm/lib/Target/Mips/MipsInstrInfo.cpp index 94828a976695..2bf8562895d7 100644 --- a/llvm/lib/Target/Mips/MipsInstrInfo.cpp +++ b/llvm/lib/Target/Mips/MipsInstrInfo.cpp @@ -568,11 +568,60 @@ bool MipsInstrInfo::SafeInForbiddenSlot(const MachineInstr &MI) const { return (MI.getDesc().TSFlags & MipsII::IsCTI) == 0; } +bool MipsInstrInfo::SafeInFPUDelaySlot(const MachineInstr &MIInSlot, + const MachineInstr &FPUMI) const { + if (MIInSlot.isInlineAsm()) + return false; + + if (HasFPUDelaySlot(MIInSlot)) + return false; + + switch (MIInSlot.getOpcode()) { + case Mips::BC1F: + case Mips::BC1FL: + case Mips::BC1T: + case Mips::BC1TL: + return false; + } + + for (const MachineOperand &Op : FPUMI.defs()) { + if (!Op.isReg()) + continue; + + bool Reads, Writes; + std::tie(Reads, Writes) = MIInSlot.readsWritesVirtualRegister(Op.getReg()); + + if (Reads || Writes) + return false; + } + + return true; +} + /// Predicate for distingushing instructions that have forbidden slots. bool MipsInstrInfo::HasForbiddenSlot(const MachineInstr &MI) const { return (MI.getDesc().TSFlags & MipsII::HasForbiddenSlot) != 0; } +/// Predicate for distingushing instructions that have FPU delay slots. +bool MipsInstrInfo::HasFPUDelaySlot(const MachineInstr &MI) const { + switch (MI.getOpcode()) { + case Mips::MTC1: + case Mips::MFC1: + case Mips::MTC1_D64: + case Mips::MFC1_D64: + case Mips::DMTC1: + case Mips::DMFC1: + case Mips::FCMP_S32: + case Mips::FCMP_D32: + case Mips::FCMP_D64: + return true; + + default: + return false; + } +} + /// Return the number of bytes of code the specified instruction may be. unsigned MipsInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { switch (MI.getOpcode()) { diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.h b/llvm/lib/Target/Mips/MipsInstrInfo.h index c96ed202df30..46c1b73d512f 100644 --- a/llvm/lib/Target/Mips/MipsInstrInfo.h +++ b/llvm/lib/Target/Mips/MipsInstrInfo.h @@ -92,9 +92,16 @@ public: /// Predicate to determine if an instruction can go in a forbidden slot. bool SafeInForbiddenSlot(const MachineInstr &MI) const; + /// Predicate to determine if an instruction can go in an FPU delay slot. + bool SafeInFPUDelaySlot(const MachineInstr &MIInSlot, + const MachineInstr &FPUMI) const; + /// Predicate to determine if an instruction has a forbidden slot. bool HasForbiddenSlot(const MachineInstr &MI) const; + /// Predicate to determine if an instruction has an FPU delay slot. + bool HasFPUDelaySlot(const MachineInstr &MI) const; + /// Insert nop instruction when hazard condition is found void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override; diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp index c35e67d6726f..16add48d4602 100644 --- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp @@ -1098,10 +1098,10 @@ void NVPTXAsmPrinter::printModuleLevelGV(const GlobalVariable *GVar, O << " .attribute(.managed)"; } - if (GVar->getAlignment() == 0) - O << " .align " << (int)DL.getPrefTypeAlignment(ETy); + if (MaybeAlign A = GVar->getAlign()) + O << " .align " << A->value(); else - O << " .align " << GVar->getAlignment(); + O << " .align " << (int)DL.getPrefTypeAlignment(ETy); if (ETy->isFloatingPointTy() || ETy->isPointerTy() || (ETy->isIntegerTy() && ETy->getScalarSizeInBits() <= 64)) { @@ -1290,10 +1290,10 @@ void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable *GVar, O << "."; emitPTXAddressSpace(GVar->getType()->getAddressSpace(), O); - if (GVar->getAlignment() == 0) - O << " .align " << (int)DL.getPrefTypeAlignment(ETy); + if (MaybeAlign A = GVar->getAlign()) + O << " .align " << A->value(); else - O << " .align " << GVar->getAlignment(); + O << " .align " << (int)DL.getPrefTypeAlignment(ETy); // Special case for i128 if (ETy->isIntegerTy(128)) { diff --git a/llvm/lib/Target/NVPTX/NVPTXPeephole.cpp b/llvm/lib/Target/NVPTX/NVPTXPeephole.cpp index 1f3b4c9440d8..bf3c87df2e08 100644 --- a/llvm/lib/Target/NVPTX/NVPTXPeephole.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXPeephole.cpp @@ -126,9 +126,9 @@ static void CombineCVTAToLocal(MachineInstr &Root) { // Check if MRI has only one non dbg use, which is Root if (MRI.hasOneNonDBGUse(Prev.getOperand(0).getReg())) { - Prev.eraseFromParentAndMarkDBGValuesForRemoval(); + Prev.eraseFromParent(); } - Root.eraseFromParentAndMarkDBGValuesForRemoval(); + Root.eraseFromParent(); } bool NVPTXPeephole::runOnMachineFunction(MachineFunction &MF) { @@ -157,7 +157,7 @@ bool NVPTXPeephole::runOnMachineFunction(MachineFunction &MF) { const auto &MRI = MF.getRegInfo(); if (MRI.use_empty(NRI->getFrameRegister(MF))) { if (auto MI = MRI.getUniqueVRegDef(NRI->getFrameRegister(MF))) { - MI->eraseFromParentAndMarkDBGValuesForRemoval(); + MI->eraseFromParent(); } } diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp index 9e181d4052d6..ded922329ebf 100644 --- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp +++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp @@ -1576,6 +1576,16 @@ bool PPCAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, std::swap(Operands[2], Operands[1]); } + // Handle base mnemonic for atomic loads where the EH bit is zero. + if (Name == "lqarx" || Name == "ldarx" || Name == "lwarx" || + Name == "lharx" || Name == "lbarx") { + if (Operands.size() != 5) + return false; + PPCOperand &EHOp = (PPCOperand &)*Operands[4]; + if (EHOp.isU1Imm() && EHOp.getImm() == 0) + Operands.pop_back(); + } + return false; } @@ -1745,7 +1755,7 @@ unsigned PPCAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp, } PPCOperand &Op = static_cast<PPCOperand &>(AsmOp); - if (Op.isImm() && Op.getImm() == ImmVal) + if (Op.isU3Imm() && Op.getImm() == ImmVal) return Match_Success; return Match_InvalidOperand; diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp index 22b948a83c34..d6e02d0d0862 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp @@ -28,6 +28,7 @@ #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCELFStreamer.h" #include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCInstrAnalysis.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCRegisterInfo.h" @@ -368,6 +369,31 @@ static MCInstPrinter *createPPCMCInstPrinter(const Triple &T, return new PPCInstPrinter(MAI, MII, MRI, T); } +namespace { + +class PPCMCInstrAnalysis : public MCInstrAnalysis { +public: + explicit PPCMCInstrAnalysis(const MCInstrInfo *Info) + : MCInstrAnalysis(Info) {} + + bool evaluateBranch(const MCInst &Inst, uint64_t Addr, uint64_t Size, + uint64_t &Target) const override { + unsigned NumOps = Inst.getNumOperands(); + if (NumOps == 0 || + Info->get(Inst.getOpcode()).OpInfo[NumOps - 1].OperandType != + MCOI::OPERAND_PCREL) + return false; + Target = Addr + Inst.getOperand(NumOps - 1).getImm() * Size; + return true; + } +}; + +} // end anonymous namespace + +static MCInstrAnalysis *createPPCMCInstrAnalysis(const MCInstrInfo *Info) { + return new PPCMCInstrAnalysis(Info); +} + extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCTargetMC() { for (Target *T : {&getThePPC32Target(), &getThePPC32LETarget(), &getThePPC64Target(), &getThePPC64LETarget()}) { @@ -383,6 +409,9 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCTargetMC() { // Register the MC subtarget info. TargetRegistry::RegisterMCSubtargetInfo(*T, createPPCMCSubtargetInfo); + // Register the MC instruction analyzer. + TargetRegistry::RegisterMCInstrAnalysis(*T, createPPCMCInstrAnalysis); + // Register the MC Code Emitter TargetRegistry::RegisterMCCodeEmitter(*T, createPPCMCCodeEmitter); diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td index 422bd11dca52..bbd5f5fd1941 100644 --- a/llvm/lib/Target/PowerPC/PPC.td +++ b/llvm/lib/Target/PowerPC/PPC.td @@ -219,6 +219,10 @@ def FeatureZeroMoveFusion: SubtargetFeature<"fuse-zeromove", "HasZeroMoveFusion", "true", "Target supports move to SPR with branch fusion", [FeatureFusion]>; +def FeatureBack2BackFusion: + SubtargetFeature<"fuse-back2back", "HasBack2BackFusion", "true", + "Target supports general back to back fusion", + [FeatureFusion]>; def FeatureUnalignedFloats : SubtargetFeature<"allow-unaligned-fp-access", "AllowsUnalignedFPAccess", "true", "CPU does not trap on unaligned FP access">; diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 16e3b2b85c2e..f26c15667a0b 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -347,7 +347,6 @@ bool PPCAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, // At the moment, all inline asm memory operands are a single register. // In any case, the output of this routine should always be just one // assembler operand. - bool PPCAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &O) { diff --git a/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def b/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def new file mode 100644 index 000000000000..38ed5f2e78e3 --- /dev/null +++ b/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def @@ -0,0 +1,1042 @@ +// Automatically generated file, do not edit! +// +// This file defines instruction list for general back2back fusion. +//===----------------------------------------------------------------------===// +FUSION_FEATURE(GeneralBack2Back, hasBack2BackFusion, -1, + FUSION_OP_SET(ADD4, + ADD4O, + ADD4TLS, + ADD4_rec, + ADD8, + ADD8O, + ADD8TLS, + ADD8TLS_, + ADD8_rec, + ADDE, + ADDE8, + ADDE8O, + ADDEO, + ADDEX, + ADDEX8, + ADDI, + ADDI8, + ADDIC, + ADDIC8, + ADDIS, + ADDIS8, + ADDISdtprelHA32, + ADDIStocHA, + ADDIStocHA8, + ADDIdtprelL32, + ADDItlsldLADDR32, + ADDItocL, + ADDME, + ADDME8, + ADDME8O, + ADDMEO, + ADDZE, + ADDZE8, + ADDZE8O, + ADDZEO, + AND, + AND8, + AND8_rec, + ANDC, + ANDC8, + ANDC8_rec, + ANDC_rec, + ANDI8_rec, + ANDIS8_rec, + ANDIS_rec, + ANDI_rec, + AND_rec, + CMPB, + CMPB8, + CNTLZD, + CNTLZD_rec, + CNTLZW, + CNTLZW8, + CNTLZW8_rec, + CNTLZW_rec, + CNTTZD, + CNTTZD_rec, + CNTTZW, + CNTTZW8, + CNTTZW8_rec, + CNTTZW_rec, + EQV, + EQV8, + EQV8_rec, + EQV_rec, + EXTSB, + EXTSB8, + EXTSB8_32_64, + EXTSB8_rec, + EXTSB_rec, + EXTSH, + EXTSH8, + EXTSH8_32_64, + EXTSH8_rec, + EXTSH_rec, + EXTSW, + EXTSWSLI, + EXTSWSLI_32_64, + EXTSWSLI_32_64_rec, + EXTSWSLI_rec, + EXTSW_32, + EXTSW_32_64, + EXTSW_32_64_rec, + EXTSW_rec, + FABSD, + FABSS, + FCPSGND, + FCPSGNS, + FMR, + FNABSD, + FNABSS, + FNEGD, + FNEGS, + ISEL, + ISEL8, + LI, + LI8, + LIS, + LIS8, + MFCTR, + MFCTR8, + MFLR, + MFLR8, + MFOCRF, + MFOCRF8, + MFVRD, + MFVRWZ, + MFVSRD, + MFVSRWZ, + MTVRD, + MTVRWA, + MTVRWZ, + MTVSRBM, + MTVSRBMI, + MTVSRD, + MTVSRDM, + MTVSRHM, + MTVSRQM, + MTVSRWA, + MTVSRWM, + MTVSRWZ, + NAND, + NAND8, + NAND8_rec, + NAND_rec, + NEG, + NEG8, + NEG8O, + NEG8_rec, + NEGO, + NEG_rec, + NOP, + NOP_GT_PWR6, + NOP_GT_PWR7, + NOR, + NOR8, + NOR8_rec, + NOR_rec, + OR, + OR8, + OR8_rec, + ORC, + ORC8, + ORC8_rec, + ORC_rec, + ORI, + ORI8, + ORIS, + ORIS8, + OR_rec, + POPCNTB, + POPCNTB8, + POPCNTD, + POPCNTW, + RLDCL, + RLDCL_rec, + RLDCR, + RLDCR_rec, + RLDIC, + RLDICL, + RLDICL_32, + RLDICL_32_64, + RLDICL_32_rec, + RLDICL_rec, + RLDICR, + RLDICR_32, + RLDICR_rec, + RLDIC_rec, + RLDIMI, + RLDIMI_rec, + RLWIMI, + RLWIMI8, + RLWIMI8_rec, + RLWIMI_rec, + RLWINM, + RLWINM8, + RLWINM8_rec, + RLWINM_rec, + RLWNM, + RLWNM8, + RLWNM8_rec, + RLWNM_rec, + SETB, + SETB8, + SETBC, + SETBC8, + SETBCR, + SETBCR8, + SETNBC, + SETNBC8, + SETNBCR, + SETNBCR8, + SLD, + SLD_rec, + SLW, + SLW8, + SLW8_rec, + SLW_rec, + SRAD, + SRADI, + SRADI_32, + SRAW, + SRAWI, + SRD, + SRD_rec, + SRW, + SRW8, + SRW8_rec, + SRW_rec, + SUBF, + SUBF8, + SUBF8O, + SUBF8_rec, + SUBFE, + SUBFE8, + SUBFE8O, + SUBFEO, + SUBFIC, + SUBFIC8, + SUBFME, + SUBFME8, + SUBFME8O, + SUBFMEO, + SUBFO, + SUBFZE, + SUBFZE8, + SUBFZE8O, + SUBFZEO, + SUBF_rec, + VABSDUB, + VABSDUH, + VABSDUW, + VADDCUW, + VADDSBS, + VADDSHS, + VADDSWS, + VADDUBM, + VADDUBS, + VADDUDM, + VADDUHM, + VADDUHS, + VADDUWM, + VADDUWS, + VAND, + VANDC, + VAVGSB, + VAVGSH, + VAVGSW, + VAVGUB, + VAVGUH, + VAVGUW, + VCLZB, + VCLZD, + VCLZH, + VCLZW, + VCMPBFP, + VCMPBFP_rec, + VCMPEQFP, + VCMPEQFP_rec, + VCMPEQUB, + VCMPEQUB_rec, + VCMPEQUD, + VCMPEQUD_rec, + VCMPEQUH, + VCMPEQUH_rec, + VCMPEQUQ, + VCMPEQUQ_rec, + VCMPEQUW, + VCMPEQUW_rec, + VCMPGEFP, + VCMPGEFP_rec, + VCMPGTFP, + VCMPGTFP_rec, + VCMPGTSB, + VCMPGTSB_rec, + VCMPGTSD, + VCMPGTSD_rec, + VCMPGTSH, + VCMPGTSH_rec, + VCMPGTSQ, + VCMPGTSQ_rec, + VCMPGTSW, + VCMPGTSW_rec, + VCMPGTUB, + VCMPGTUB_rec, + VCMPGTUD, + VCMPGTUD_rec, + VCMPGTUH, + VCMPGTUH_rec, + VCMPGTUQ, + VCMPGTUQ_rec, + VCMPGTUW, + VCMPGTUW_rec, + VCMPNEB, + VCMPNEB_rec, + VCMPNEH, + VCMPNEH_rec, + VCMPNEW, + VCMPNEW_rec, + VCMPNEZB, + VCMPNEZB_rec, + VCMPNEZH, + VCMPNEZH_rec, + VCMPNEZW, + VCMPNEZW_rec, + VCNTMBB, + VCNTMBD, + VCNTMBH, + VCNTMBW, + VCTZB, + VCTZD, + VCTZH, + VCTZW, + VEQV, + VEXPANDBM, + VEXPANDDM, + VEXPANDHM, + VEXPANDQM, + VEXPANDWM, + VEXTRACTBM, + VEXTRACTDM, + VEXTRACTHM, + VEXTRACTQM, + VEXTRACTWM, + VEXTSB2D, + VEXTSB2Ds, + VEXTSB2W, + VEXTSB2Ws, + VEXTSD2Q, + VEXTSH2D, + VEXTSH2Ds, + VEXTSH2W, + VEXTSH2Ws, + VEXTSW2D, + VEXTSW2Ds, + VMAXFP, + VMAXSB, + VMAXSD, + VMAXSH, + VMAXSW, + VMAXUB, + VMAXUD, + VMAXUH, + VMAXUW, + VMINFP, + VMINSB, + VMINSD, + VMINSH, + VMINSW, + VMINUB, + VMINUD, + VMINUH, + VMINUW, + VMRGEW, + VMRGOW, + VNAND, + VNEGD, + VNEGW, + VNOR, + VOR, + VORC, + VPOPCNTB, + VPOPCNTD, + VPOPCNTH, + VPOPCNTW, + VPRTYBD, + VPRTYBW, + VRLB, + VRLD, + VRLDMI, + VRLDNM, + VRLH, + VRLW, + VRLWMI, + VRLWNM, + VSEL, + VSHASIGMAD, + VSHASIGMAW, + VSLB, + VSLD, + VSLH, + VSLW, + VSRAB, + VSRAD, + VSRAH, + VSRAW, + VSRB, + VSRD, + VSRH, + VSRW, + VSUBCUW, + VSUBSBS, + VSUBSHS, + VSUBSWS, + VSUBUBM, + VSUBUBS, + VSUBUDM, + VSUBUHM, + VSUBUHS, + VSUBUWM, + VSUBUWS, + VXOR, + V_SET0, + V_SET0B, + V_SET0H, + XOR, + XOR8, + XOR8_rec, + XORI, + XORI8, + XORIS, + XORIS8, + XOR_rec, + XSABSDP, + XSABSQP, + XSCMPEQDP, + XSCMPGEDP, + XSCMPGTDP, + XSCPSGNDP, + XSCPSGNQP, + XSCVHPDP, + XSCVSPDPN, + XSIEXPDP, + XSIEXPQP, + XSMAXCDP, + XSMAXDP, + XSMAXJDP, + XSMINCDP, + XSMINDP, + XSMINJDP, + XSNABSDP, + XSNABSQP, + XSNEGDP, + XSNEGQP, + XSXEXPDP, + XSXEXPQP, + XSXSIGDP, + XVABSDP, + XVABSSP, + XVCMPEQDP, + XVCMPEQDP_rec, + XVCMPEQSP, + XVCMPEQSP_rec, + XVCMPGEDP, + XVCMPGEDP_rec, + XVCMPGESP, + XVCMPGESP_rec, + XVCMPGTDP, + XVCMPGTDP_rec, + XVCMPGTSP, + XVCMPGTSP_rec, + XVCPSGNDP, + XVCPSGNSP, + XVCVHPSP, + XVIEXPDP, + XVIEXPSP, + XVMAXDP, + XVMAXSP, + XVMINDP, + XVMINSP, + XVNABSDP, + XVNABSSP, + XVNEGDP, + XVNEGSP, + XVTSTDCDP, + XVTSTDCSP, + XVXEXPDP, + XVXEXPSP, + XVXSIGDP, + XVXSIGSP, + XXLAND, + XXLANDC, + XXLEQV, + XXLEQVOnes, + XXLNAND, + XXLNOR, + XXLOR, + XXLORC, + XXLORf, + XXLXOR, + XXLXORdpz, + XXLXORspz, + XXLXORz, + XXSEL), + FUSION_OP_SET(ADD4, + ADD4O, + ADD4TLS, + ADD4_rec, + ADD8, + ADD8O, + ADD8TLS, + ADD8TLS_, + ADD8_rec, + ADDE, + ADDE8, + ADDE8O, + ADDEO, + ADDEX, + ADDEX8, + ADDI, + ADDI8, + ADDIC, + ADDIC8, + ADDIS, + ADDIS8, + ADDISdtprelHA32, + ADDIStocHA, + ADDIStocHA8, + ADDIdtprelL32, + ADDItlsldLADDR32, + ADDItocL, + ADDME, + ADDME8, + ADDME8O, + ADDMEO, + ADDZE, + ADDZE8, + ADDZE8O, + ADDZEO, + AND, + AND8, + AND8_rec, + ANDC, + ANDC8, + ANDC8_rec, + ANDC_rec, + ANDI8_rec, + ANDIS8_rec, + ANDIS_rec, + ANDI_rec, + AND_rec, + CMPB, + CMPB8, + CMPD, + CMPDI, + CMPEQB, + CMPLD, + CMPLDI, + CMPLW, + CMPLWI, + CMPRB, + CMPRB8, + CMPW, + CMPWI, + CNTLZD, + CNTLZD_rec, + CNTLZW, + CNTLZW8, + CNTLZW8_rec, + CNTLZW_rec, + CNTTZD, + CNTTZD_rec, + CNTTZW, + CNTTZW8, + CNTTZW8_rec, + CNTTZW_rec, + CR6SET, + CR6UNSET, + CRAND, + CRANDC, + CREQV, + CRNAND, + CRNOR, + CROR, + CRORC, + CRSET, + CRUNSET, + CRXOR, + DSS, + DSSALL, + DST, + DST64, + DSTST, + DSTST64, + DSTSTT, + DSTSTT64, + DSTT, + DSTT64, + EQV, + EQV8, + EQV8_rec, + EQV_rec, + EXTSB, + EXTSB8, + EXTSB8_32_64, + EXTSB8_rec, + EXTSB_rec, + EXTSH, + EXTSH8, + EXTSH8_32_64, + EXTSH8_rec, + EXTSH_rec, + EXTSW, + EXTSWSLI, + EXTSWSLI_32_64, + EXTSWSLI_32_64_rec, + EXTSWSLI_rec, + EXTSW_32, + EXTSW_32_64, + EXTSW_32_64_rec, + EXTSW_rec, + FABSD, + FABSS, + FCMPOD, + FCMPOS, + FCMPUD, + FCMPUS, + FCPSGND, + FCPSGNS, + FMR, + FNABSD, + FNABSS, + FNEGD, + FNEGS, + FTDIV, + FTSQRT, + ISEL, + ISEL8, + LI, + LI8, + LIS, + LIS8, + MCRF, + MCRXRX, + MFCTR, + MFCTR8, + MFLR, + MFLR8, + MFOCRF, + MFOCRF8, + MFVRD, + MFVRWZ, + MFVSRD, + MFVSRWZ, + MTCTR, + MTCTR8, + MTCTR8loop, + MTCTRloop, + MTLR, + MTLR8, + MTOCRF, + MTOCRF8, + MTVRD, + MTVRWA, + MTVRWZ, + MTVSRBM, + MTVSRBMI, + MTVSRD, + MTVSRDM, + MTVSRHM, + MTVSRQM, + MTVSRWA, + MTVSRWM, + MTVSRWZ, + NAND, + NAND8, + NAND8_rec, + NAND_rec, + NEG, + NEG8, + NEG8O, + NEG8_rec, + NEGO, + NEG_rec, + NOP, + NOP_GT_PWR6, + NOP_GT_PWR7, + NOR, + NOR8, + NOR8_rec, + NOR_rec, + OR, + OR8, + OR8_rec, + ORC, + ORC8, + ORC8_rec, + ORC_rec, + ORI, + ORI8, + ORIS, + ORIS8, + OR_rec, + POPCNTB, + POPCNTB8, + POPCNTD, + POPCNTW, + RLDCL, + RLDCL_rec, + RLDCR, + RLDCR_rec, + RLDIC, + RLDICL, + RLDICL_32, + RLDICL_32_64, + RLDICL_32_rec, + RLDICL_rec, + RLDICR, + RLDICR_32, + RLDICR_rec, + RLDIC_rec, + RLDIMI, + RLDIMI_rec, + RLWIMI, + RLWIMI8, + RLWIMI8_rec, + RLWIMI_rec, + RLWINM, + RLWINM8, + RLWINM8_rec, + RLWINM_rec, + RLWNM, + RLWNM8, + RLWNM8_rec, + RLWNM_rec, + SETB, + SETB8, + SETBC, + SETBC8, + SETBCR, + SETBCR8, + SETNBC, + SETNBC8, + SETNBCR, + SETNBCR8, + SLD, + SLD_rec, + SLW, + SLW8, + SLW8_rec, + SLW_rec, + SRAD, + SRADI, + SRADI_32, + SRAW, + SRAWI, + SRD, + SRD_rec, + SRW, + SRW8, + SRW8_rec, + SRW_rec, + SUBF, + SUBF8, + SUBF8O, + SUBF8_rec, + SUBFE, + SUBFE8, + SUBFE8O, + SUBFEO, + SUBFIC, + SUBFIC8, + SUBFME, + SUBFME8, + SUBFME8O, + SUBFMEO, + SUBFO, + SUBFZE, + SUBFZE8, + SUBFZE8O, + SUBFZEO, + SUBF_rec, + TD, + TDI, + TRAP, + TW, + TWI, + VABSDUB, + VABSDUH, + VABSDUW, + VADDCUW, + VADDSBS, + VADDSHS, + VADDSWS, + VADDUBM, + VADDUBS, + VADDUDM, + VADDUHM, + VADDUHS, + VADDUWM, + VADDUWS, + VAND, + VANDC, + VAVGSB, + VAVGSH, + VAVGSW, + VAVGUB, + VAVGUH, + VAVGUW, + VCLZB, + VCLZD, + VCLZH, + VCLZW, + VCMPBFP, + VCMPBFP_rec, + VCMPEQFP, + VCMPEQFP_rec, + VCMPEQUB, + VCMPEQUB_rec, + VCMPEQUD, + VCMPEQUD_rec, + VCMPEQUH, + VCMPEQUH_rec, + VCMPEQUQ, + VCMPEQUQ_rec, + VCMPEQUW, + VCMPEQUW_rec, + VCMPGEFP, + VCMPGEFP_rec, + VCMPGTFP, + VCMPGTFP_rec, + VCMPGTSB, + VCMPGTSB_rec, + VCMPGTSD, + VCMPGTSD_rec, + VCMPGTSH, + VCMPGTSH_rec, + VCMPGTSQ, + VCMPGTSQ_rec, + VCMPGTSW, + VCMPGTSW_rec, + VCMPGTUB, + VCMPGTUB_rec, + VCMPGTUD, + VCMPGTUD_rec, + VCMPGTUH, + VCMPGTUH_rec, + VCMPGTUQ, + VCMPGTUQ_rec, + VCMPGTUW, + VCMPGTUW_rec, + VCMPNEB, + VCMPNEB_rec, + VCMPNEH, + VCMPNEH_rec, + VCMPNEW, + VCMPNEW_rec, + VCMPNEZB, + VCMPNEZB_rec, + VCMPNEZH, + VCMPNEZH_rec, + VCMPNEZW, + VCMPNEZW_rec, + VCMPSQ, + VCMPUQ, + VCNTMBB, + VCNTMBD, + VCNTMBH, + VCNTMBW, + VCTZB, + VCTZD, + VCTZH, + VCTZW, + VEQV, + VEXPANDBM, + VEXPANDDM, + VEXPANDHM, + VEXPANDQM, + VEXPANDWM, + VEXTRACTBM, + VEXTRACTDM, + VEXTRACTHM, + VEXTRACTQM, + VEXTRACTWM, + VEXTSB2D, + VEXTSB2Ds, + VEXTSB2W, + VEXTSB2Ws, + VEXTSD2Q, + VEXTSH2D, + VEXTSH2Ds, + VEXTSH2W, + VEXTSH2Ws, + VEXTSW2D, + VEXTSW2Ds, + VMAXFP, + VMAXSB, + VMAXSD, + VMAXSH, + VMAXSW, + VMAXUB, + VMAXUD, + VMAXUH, + VMAXUW, + VMINFP, + VMINSB, + VMINSD, + VMINSH, + VMINSW, + VMINUB, + VMINUD, + VMINUH, + VMINUW, + VMRGEW, + VMRGOW, + VNAND, + VNEGD, + VNEGW, + VNOR, + VOR, + VORC, + VPOPCNTB, + VPOPCNTD, + VPOPCNTH, + VPOPCNTW, + VPRTYBD, + VPRTYBW, + VRLB, + VRLD, + VRLDMI, + VRLDNM, + VRLH, + VRLW, + VRLWMI, + VRLWNM, + VSEL, + VSHASIGMAD, + VSHASIGMAW, + VSLB, + VSLD, + VSLH, + VSLW, + VSRAB, + VSRAD, + VSRAH, + VSRAW, + VSRB, + VSRD, + VSRH, + VSRW, + VSUBCUW, + VSUBSBS, + VSUBSHS, + VSUBSWS, + VSUBUBM, + VSUBUBS, + VSUBUDM, + VSUBUHM, + VSUBUHS, + VSUBUWM, + VSUBUWS, + VXOR, + V_SET0, + V_SET0B, + V_SET0H, + WAIT, + XOR, + XOR8, + XOR8_rec, + XORI, + XORI8, + XORIS, + XORIS8, + XOR_rec, + XSABSDP, + XSABSQP, + XSCMPEQDP, + XSCMPEXPDP, + XSCMPGEDP, + XSCMPGTDP, + XSCMPODP, + XSCMPUDP, + XSCPSGNDP, + XSCPSGNQP, + XSCVHPDP, + XSCVSPDPN, + XSIEXPDP, + XSIEXPQP, + XSMAXCDP, + XSMAXDP, + XSMAXJDP, + XSMINCDP, + XSMINDP, + XSMINJDP, + XSNABSDP, + XSNABSQP, + XSNEGDP, + XSNEGQP, + XSTDIVDP, + XSTSQRTDP, + XSTSTDCDP, + XSTSTDCSP, + XSXEXPDP, + XSXEXPQP, + XSXSIGDP, + XVABSDP, + XVABSSP, + XVCMPEQDP, + XVCMPEQDP_rec, + XVCMPEQSP, + XVCMPEQSP_rec, + XVCMPGEDP, + XVCMPGEDP_rec, + XVCMPGESP, + XVCMPGESP_rec, + XVCMPGTDP, + XVCMPGTDP_rec, + XVCMPGTSP, + XVCMPGTSP_rec, + XVCPSGNDP, + XVCPSGNSP, + XVCVHPSP, + XVIEXPDP, + XVIEXPSP, + XVMAXDP, + XVMAXSP, + XVMINDP, + XVMINSP, + XVNABSDP, + XVNABSSP, + XVNEGDP, + XVNEGSP, + XVTDIVDP, + XVTDIVSP, + XVTLSBB, + XVTSQRTDP, + XVTSQRTSP, + XVTSTDCDP, + XVTSTDCSP, + XVXEXPDP, + XVXEXPSP, + XVXSIGDP, + XVXSIGSP, + XXLAND, + XXLANDC, + XXLEQV, + XXLEQVOnes, + XXLNAND, + XXLNOR, + XXLOR, + XXLORC, + XXLORf, + XXLXOR, + XXLXORdpz, + XXLXORspz, + XXLXORz, + XXSEL))
\ No newline at end of file diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp index a2664bcff4ab..ba74af5ef5f7 100644 --- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp @@ -4464,9 +4464,10 @@ bool PPCDAGToDAGISel::trySETCC(SDNode *N) { bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const { LoadSDNode *LDN = dyn_cast<LoadSDNode>(N); StoreSDNode *STN = dyn_cast<StoreSDNode>(N); + MemIntrinsicSDNode *MIN = dyn_cast<MemIntrinsicSDNode>(N); SDValue AddrOp; - if (LDN) - AddrOp = LDN->getOperand(1); + if (LDN || (MIN && MIN->getOpcode() == PPCISD::LD_SPLAT)) + AddrOp = N->getOperand(1); else if (STN) AddrOp = STN->getOperand(2); @@ -5973,6 +5974,15 @@ void PPCDAGToDAGISel::Select(SDNode *N) { if (Type != MVT::v16i8 && Type != MVT::v8i16) break; + // If the alignment for the load is 16 or bigger, we don't need the + // permutated mask to get the required value. The value must be the 0 + // element in big endian target or 7/15 in little endian target in the + // result vsx register of lvx instruction. + // Select the instruction in the .td file. + if (cast<MemIntrinsicSDNode>(N)->getAlign() >= Align(16) && + isOffsetMultipleOf(N, 16)) + break; + SDValue ZeroReg = CurDAG->getRegister(Subtarget->isPPC64() ? PPC::ZERO8 : PPC::ZERO, Subtarget->isPPC64() ? MVT::i64 : MVT::i32); diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index ec7e30d7e362..8d6edf07bc53 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -3500,15 +3500,16 @@ SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { if (LHS.getValueType() == MVT::v2i64) { // Equality can be handled by casting to the legal type for Altivec // comparisons, everything else needs to be expanded. - if (CC == ISD::SETEQ || CC == ISD::SETNE) { - return DAG.getNode( - ISD::BITCAST, dl, MVT::v2i64, - DAG.getSetCC(dl, MVT::v4i32, - DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, LHS), - DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, RHS), CC)); - } - - return SDValue(); + if (CC != ISD::SETEQ && CC != ISD::SETNE) + return SDValue(); + SDValue SetCC32 = DAG.getSetCC( + dl, MVT::v4i32, DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, LHS), + DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, RHS), CC); + int ShuffV[] = {1, 0, 3, 2}; + SDValue Shuff = + DAG.getVectorShuffle(MVT::v4i32, dl, SetCC32, SetCC32, ShuffV); + return DAG.getBitcast( + MVT::v2i64, DAG.getNode(ISD::AND, dl, MVT::v4i32, Shuff, SetCC32)); } // We handle most of these in the usual way. @@ -6206,20 +6207,13 @@ SDValue PPCTargetLowering::LowerCall_64SVR4( ArgOffset += PtrByteSize; continue; } - // Copy entire object into memory. There are cases where gcc-generated - // code assumes it is there, even if it could be put entirely into - // registers. (This is not what the doc says.) - - // FIXME: The above statement is likely due to a misunderstanding of the - // documents. All arguments must be copied into the parameter area BY - // THE CALLEE in the event that the callee takes the address of any - // formal argument. That has not yet been implemented. However, it is - // reasonable to use the stack area as a staging area for the register - // load. - - // Skip this for small aggregates, as we will use the same slot for a - // right-justified copy, below. - if (Size >= 8) + // Copy the object to parameter save area if it can not be entirely passed + // by registers. + // FIXME: we only need to copy the parts which need to be passed in + // parameter save area. For the parts passed by registers, we don't need + // to copy them to the stack although we need to allocate space for them + // in parameter save area. + if ((NumGPRs - GPR_idx) * PtrByteSize < Size) Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, CallSeqStart, Flags, DAG, dl); @@ -17548,14 +17542,14 @@ unsigned PPCTargetLowering::computeMOFlags(const SDNode *Parent, SDValue N, if (Subtarget.isISA3_1() && ((ParentOp == ISD::INTRINSIC_W_CHAIN) || (ParentOp == ISD::INTRINSIC_VOID))) { unsigned ID = cast<ConstantSDNode>(Parent->getOperand(1))->getZExtValue(); - assert( - ((ID == Intrinsic::ppc_vsx_lxvp) || (ID == Intrinsic::ppc_vsx_stxvp)) && - "Only the paired load and store (lxvp/stxvp) intrinsics are valid."); - SDValue IntrinOp = (ID == Intrinsic::ppc_vsx_lxvp) ? Parent->getOperand(2) - : Parent->getOperand(3); - computeFlagsForAddressComputation(IntrinOp, FlagSet, DAG); - FlagSet |= PPC::MOF_Vector; - return FlagSet; + if ((ID == Intrinsic::ppc_vsx_lxvp) || (ID == Intrinsic::ppc_vsx_stxvp)) { + SDValue IntrinOp = (ID == Intrinsic::ppc_vsx_lxvp) + ? Parent->getOperand(2) + : Parent->getOperand(3); + computeFlagsForAddressComputation(IntrinOp, FlagSet, DAG); + FlagSet |= PPC::MOF_Vector; + return FlagSet; + } } // Mark this as something we don't want to handle here if it is atomic diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h index 2cfd53de3290..c16e146da247 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h @@ -393,7 +393,9 @@ public: MachineInstr &NewMI1, MachineInstr &NewMI2) const override; - void setSpecialOperandAttr(MachineInstr &MI, uint16_t Flags) const override; + // PowerPC specific version of setSpecialOperandAttr that copies Flags to MI + // and clears nuw, nsw, and exact flags. + void setSpecialOperandAttr(MachineInstr &MI, uint16_t Flags) const; bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg, Register &DstReg, diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index d83ecc699b19..2340be5b5915 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -4780,6 +4780,7 @@ class PPCAsmPseudo<string asm, dag iops> def : InstAlias<"sc", (SC 0)>; def : InstAlias<"sync", (SYNC 0)>, Requires<[HasSYNC]>; +def : InstAlias<"hwsync", (SYNC 0), 0>, Requires<[HasSYNC]>; def : InstAlias<"msync", (SYNC 0), 0>, Requires<[HasSYNC]>; def : InstAlias<"lwsync", (SYNC 1)>, Requires<[HasSYNC]>; def : InstAlias<"ptesync", (SYNC 2)>, Requires<[HasSYNC]>; diff --git a/llvm/lib/Target/PowerPC/PPCInstrVSX.td b/llvm/lib/Target/PowerPC/PPCInstrVSX.td index d92a10c5b208..110f7d79fbc5 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrVSX.td +++ b/llvm/lib/Target/PowerPC/PPCInstrVSX.td @@ -158,6 +158,11 @@ def HasP9Vector : Predicate<"Subtarget->hasP9Vector()">; def NoP9Altivec : Predicate<"!Subtarget->hasP9Altivec()">; def NoP10Vector: Predicate<"!Subtarget->hasP10Vector()">; +def PPCldsplatAlign16 : PatFrag<(ops node:$ptr), (PPCldsplat node:$ptr), [{ + return cast<MemIntrinsicSDNode>(N)->getAlign() >= Align(16) && + isOffsetMultipleOf(N, 16); +}]>; + //--------------------- VSX-specific instruction formats ---------------------// // By default, all VSX instructions are to be selected over their Altivec // counter parts and they do not have unmodeled sideeffects. @@ -3180,6 +3185,12 @@ defm : ScalToVecWPermute< v2f64, (f64 (load ForceXForm:$src)), (XXPERMDIs (XFLOADf64 ForceXForm:$src), 2), (SUBREG_TO_REG (i64 1), (XFLOADf64 ForceXForm:$src), sub_64)>; + +// Splat loads. +def : Pat<(v8i16 (PPCldsplatAlign16 ForceXForm:$A)), + (v8i16 (VSPLTH 7, (LVX ForceXForm:$A)))>; +def : Pat<(v16i8 (PPCldsplatAlign16 ForceXForm:$A)), + (v16i8 (VSPLTB 15, (LVX ForceXForm:$A)))>; } // HasVSX, NoP9Vector, IsLittleEndian let Predicates = [HasVSX, NoP9Vector, IsBigEndian] in { @@ -3187,6 +3198,12 @@ let Predicates = [HasVSX, NoP9Vector, IsBigEndian] in { (LXVD2X ForceXForm:$src)>; def : Pat<(int_ppc_vsx_stxvd2x v2f64:$rS, ForceXForm:$dst), (STXVD2X $rS, ForceXForm:$dst)>; + + // Splat loads. + def : Pat<(v8i16 (PPCldsplatAlign16 ForceXForm:$A)), + (v8i16 (VSPLTH 0, (LVX ForceXForm:$A)))>; + def : Pat<(v16i8 (PPCldsplatAlign16 ForceXForm:$A)), + (v16i8 (VSPLTB 0, (LVX ForceXForm:$A)))>; } // HasVSX, NoP9Vector, IsBigEndian // Any VSX subtarget that only has loads and stores that load in big endian diff --git a/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp b/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp index 7f63827afbd6..0c7be96a0595 100644 --- a/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp +++ b/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp @@ -413,9 +413,9 @@ bool PPCLoopInstrFormPrep::runOnFunction(Function &F) { bool MadeChange = false; - for (auto I = LI->begin(), IE = LI->end(); I != IE; ++I) - for (auto L = df_begin(*I), LE = df_end(*I); L != LE; ++L) - MadeChange |= runOnLoop(*L); + for (Loop *I : *LI) + for (Loop *L : depth_first(I)) + MadeChange |= runOnLoop(L); return MadeChange; } diff --git a/llvm/lib/Target/PowerPC/PPCMacroFusion.def b/llvm/lib/Target/PowerPC/PPCMacroFusion.def index e4954b722fd0..6b8ad22639c8 100644 --- a/llvm/lib/Target/PowerPC/PPCMacroFusion.def +++ b/llvm/lib/Target/PowerPC/PPCMacroFusion.def @@ -153,5 +153,7 @@ FUSION_FEATURE(ZeroMoveLR, hasZeroMoveFusion, -1, FUSION_OP_SET(MTLR8, MTLR, MTSPR8, MTSPR), FUSION_OP_SET(BCLR, BCLRn, gBCLR, BCLRL, BCLRLn, gBCLRL)) +#include "PPCBack2BackFusion.def" + #undef FUSION_FEATURE #undef FUSION_OP_SET diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp index 1258a1281597..f11b4e14073e 100644 --- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp +++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp @@ -135,6 +135,7 @@ void PPCSubtarget::initializeEnvironment() { HasCompareFusion = false; HasWideImmFusion = false; HasZeroMoveFusion = false; + HasBack2BackFusion = false; IsISA2_06 = false; IsISA2_07 = false; IsISA3_0 = false; diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h index d52833cb1465..1300b62b623a 100644 --- a/llvm/lib/Target/PowerPC/PPCSubtarget.h +++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h @@ -155,6 +155,7 @@ protected: bool HasCompareFusion; bool HasWideImmFusion; bool HasZeroMoveFusion; + bool HasBack2BackFusion; bool IsISA2_06; bool IsISA2_07; bool IsISA3_0; @@ -348,6 +349,7 @@ public: bool hasWideImmFusion() const { return HasWideImmFusion; } bool hasSha3Fusion() const { return HasSha3Fusion; } bool hasZeroMoveFusion() const { return HasZeroMoveFusion; } + bool hasBack2BackFusion() const { return HasBack2BackFusion; } bool needsSwapsForVSXMemOps() const { return hasVSX() && isLittleEndian() && !hasP9Vector(); } diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp index 5d6f58a77a39..ed28731b8ef2 100644 --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp @@ -328,10 +328,6 @@ static bool isMMAType(Type *Ty) { InstructionCost PPCTTIImpl::getUserCost(const User *U, ArrayRef<const Value *> Operands, TTI::TargetCostKind CostKind) { - // Set the max cost if an MMA type is present (v256i1, v512i1). - if (isMMAType(U->getType())) - return InstructionCost::getMax(); - // We already implement getCastInstrCost and getMemoryOpCost where we perform // the vector adjustment there. if (isa<CastInst>(U) || isa<LoadInst>(U) || isa<StoreInst>(U)) @@ -1276,23 +1272,21 @@ PPCTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, return BaseT::getIntrinsicInstrCost(ICA, CostKind); } -bool PPCTTIImpl::areFunctionArgsABICompatible( - const Function *Caller, const Function *Callee, - SmallPtrSetImpl<Argument *> &Args) const { +bool PPCTTIImpl::areTypesABICompatible(const Function *Caller, + const Function *Callee, + const ArrayRef<Type *> &Types) const { // We need to ensure that argument promotion does not // attempt to promote pointers to MMA types (__vector_pair // and __vector_quad) since these types explicitly cannot be // passed as arguments. Both of these types are larger than // the 128-bit Altivec vectors and have a scalar size of 1 bit. - if (!BaseT::areFunctionArgsABICompatible(Caller, Callee, Args)) + if (!BaseT::areTypesABICompatible(Caller, Callee, Types)) return false; - return llvm::none_of(Args, [](Argument *A) { - auto *EltTy = cast<PointerType>(A->getType())->getElementType(); - if (EltTy->isSized()) - return (EltTy->isIntOrIntVectorTy(1) && - EltTy->getPrimitiveSizeInBits() > 128); + return llvm::none_of(Types, [](Type *Ty) { + if (Ty->isSized()) + return Ty->isIntOrIntVectorTy(1) && Ty->getPrimitiveSizeInBits() > 128; return false; }); } @@ -1388,3 +1382,86 @@ bool PPCTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst, return false; } + +bool PPCTTIImpl::hasActiveVectorLength(unsigned Opcode, Type *DataType, + Align Alignment) const { + // Only load and stores instructions can have variable vector length on Power. + if (Opcode != Instruction::Load && Opcode != Instruction::Store) + return false; + // Loads/stores with length instructions use bits 0-7 of the GPR operand and + // therefore cannot be used in 32-bit mode. + if ((!ST->hasP9Vector() && !ST->hasP10Vector()) || !ST->isPPC64()) + return false; + if (isa<FixedVectorType>(DataType)) { + unsigned VecWidth = DataType->getPrimitiveSizeInBits(); + return VecWidth == 128; + } + Type *ScalarTy = DataType->getScalarType(); + + if (ScalarTy->isPointerTy()) + return true; + + if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy()) + return true; + + if (!ScalarTy->isIntegerTy()) + return false; + + unsigned IntWidth = ScalarTy->getIntegerBitWidth(); + return IntWidth == 8 || IntWidth == 16 || IntWidth == 32 || IntWidth == 64; +} + +InstructionCost PPCTTIImpl::getVPMemoryOpCost(unsigned Opcode, Type *Src, + Align Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I) { + InstructionCost Cost = BaseT::getVPMemoryOpCost(Opcode, Src, Alignment, + AddressSpace, CostKind, I); + if (TLI->getValueType(DL, Src, true) == MVT::Other) + return Cost; + // TODO: Handle other cost kinds. + if (CostKind != TTI::TCK_RecipThroughput) + return Cost; + + assert((Opcode == Instruction::Load || Opcode == Instruction::Store) && + "Invalid Opcode"); + + auto *SrcVTy = dyn_cast<FixedVectorType>(Src); + assert(SrcVTy && "Expected a vector type for VP memory operations"); + + if (hasActiveVectorLength(Opcode, Src, Alignment)) { + std::pair<InstructionCost, MVT> LT = + TLI->getTypeLegalizationCost(DL, SrcVTy); + + InstructionCost CostFactor = + vectorCostAdjustmentFactor(Opcode, Src, nullptr); + if (!CostFactor.isValid()) + return InstructionCost::getMax(); + + InstructionCost Cost = LT.first * CostFactor; + assert(Cost.isValid() && "Expected valid cost"); + + // On P9 but not on P10, if the op is misaligned then it will cause a + // pipeline flush. Otherwise the VSX masked memops cost the same as unmasked + // ones. + const Align DesiredAlignment(16); + if (Alignment >= DesiredAlignment || ST->getCPUDirective() != PPC::DIR_PWR9) + return Cost; + + // Since alignment may be under estimated, we try to compute the probability + // that the actual address is aligned to the desired boundary. For example + // an 8-byte aligned load is assumed to be actually 16-byte aligned half the + // time, while a 4-byte aligned load has a 25% chance of being 16-byte + // aligned. + float AlignmentProb = ((float)Alignment.value()) / DesiredAlignment.value(); + float MisalignmentProb = 1.0 - AlignmentProb; + return (MisalignmentProb * P9PipelineFlushEstimate) + + (AlignmentProb * *Cost.getValue()); + } + + // Usually we should not get to this point, but the following is an attempt to + // model the cost of legalization. Currently we can only lower intrinsics with + // evl but no mask, on Power 9/10. Otherwise, we must scalarize. + return getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); +} diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h index 7aeb0c59d503..0af6f2a308d9 100644 --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h @@ -134,9 +134,19 @@ public: bool UseMaskForCond = false, bool UseMaskForGaps = false); InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind); - bool areFunctionArgsABICompatible(const Function *Caller, - const Function *Callee, - SmallPtrSetImpl<Argument *> &Args) const; + bool areTypesABICompatible(const Function *Caller, const Function *Callee, + const ArrayRef<Type *> &Types) const; + bool hasActiveVectorLength(unsigned Opcode, Type *DataType, + Align Alignment) const; + InstructionCost getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, + unsigned AddressSpace, + TTI::TargetCostKind CostKind, + const Instruction *I = nullptr); + +private: + // The following constant is used for estimating costs on power9. + static const InstructionCost::CostType P9PipelineFlushEstimate = 80; + /// @} }; diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp index f00813f1301a..75592dd4c6f5 100644 --- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp +++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp @@ -169,6 +169,7 @@ class RISCVAsmParser : public MCTargetAsmParser { OperandMatchResultTy parseJALOffset(OperandVector &Operands); OperandMatchResultTy parseVTypeI(OperandVector &Operands); OperandMatchResultTy parseMaskReg(OperandVector &Operands); + OperandMatchResultTy parseInsnDirectiveOpcode(OperandVector &Operands); bool parseOperand(OperandVector &Operands, StringRef Mnemonic); @@ -827,6 +828,7 @@ public: Op->SysReg.Length = Str.size(); Op->SysReg.Encoding = Encoding; Op->StartLoc = S; + Op->EndLoc = S; Op->IsRV64 = IsRV64; return Op; } @@ -836,6 +838,7 @@ public: auto Op = std::make_unique<RISCVOperand>(KindTy::VType); Op->VType.Val = VTypeI; Op->StartLoc = S; + Op->EndLoc = S; Op->IsRV64 = IsRV64; return Op; } @@ -1291,7 +1294,7 @@ OperandMatchResultTy RISCVAsmParser::parseRegister(OperandVector &Operands, if (HadParens) Operands.push_back(RISCVOperand::createToken("(", FirstS, isRV64())); SMLoc S = getLoc(); - SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1); + SMLoc E = SMLoc::getFromPointer(S.getPointer() + Name.size()); getLexer().Lex(); Operands.push_back(RISCVOperand::createReg(RegNo, S, E, isRV64())); } @@ -1305,6 +1308,67 @@ OperandMatchResultTy RISCVAsmParser::parseRegister(OperandVector &Operands, } OperandMatchResultTy +RISCVAsmParser::parseInsnDirectiveOpcode(OperandVector &Operands) { + SMLoc S = getLoc(); + SMLoc E; + const MCExpr *Res; + + switch (getLexer().getKind()) { + default: + return MatchOperand_NoMatch; + case AsmToken::LParen: + case AsmToken::Minus: + case AsmToken::Plus: + case AsmToken::Exclaim: + case AsmToken::Tilde: + case AsmToken::Integer: + case AsmToken::String: { + if (getParser().parseExpression(Res, E)) + return MatchOperand_ParseFail; + + auto *CE = dyn_cast<MCConstantExpr>(Res); + if (CE) { + int64_t Imm = CE->getValue(); + if (isUInt<7>(Imm)) { + Operands.push_back(RISCVOperand::createImm(Res, S, E, isRV64())); + return MatchOperand_Success; + } + } + + Twine Msg = "immediate must be an integer in the range"; + Error(S, Msg + " [" + Twine(0) + ", " + Twine((1 << 7) - 1) + "]"); + return MatchOperand_ParseFail; + } + case AsmToken::Identifier: { + StringRef Identifier; + if (getParser().parseIdentifier(Identifier)) + return MatchOperand_ParseFail; + + auto Opcode = RISCVInsnOpcode::lookupRISCVOpcodeByName(Identifier); + if (Opcode) { + Res = MCConstantExpr::create(Opcode->Value, getContext()); + E = SMLoc::getFromPointer(S.getPointer() + Identifier.size()); + Operands.push_back(RISCVOperand::createImm(Res, S, E, isRV64())); + return MatchOperand_Success; + } + + Twine Msg = "operand must be a valid opcode name or an " + "integer in the range"; + Error(S, Msg + " [" + Twine(0) + ", " + Twine((1 << 7) - 1) + "]"); + return MatchOperand_ParseFail; + } + case AsmToken::Percent: { + // Discard operand with modifier. + Twine Msg = "immediate must be an integer in the range"; + Error(S, Msg + " [" + Twine(0) + ", " + Twine((1 << 7) - 1) + "]"); + return MatchOperand_ParseFail; + } + } + + return MatchOperand_NoMatch; +} + +OperandMatchResultTy RISCVAsmParser::parseCSRSystemRegister(OperandVector &Operands) { SMLoc S = getLoc(); const MCExpr *Res; @@ -1381,7 +1445,7 @@ RISCVAsmParser::parseCSRSystemRegister(OperandVector &Operands) { OperandMatchResultTy RISCVAsmParser::parseImmediate(OperandVector &Operands) { SMLoc S = getLoc(); - SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1); + SMLoc E; const MCExpr *Res; switch (getLexer().getKind()) { @@ -1396,7 +1460,7 @@ OperandMatchResultTy RISCVAsmParser::parseImmediate(OperandVector &Operands) { case AsmToken::Integer: case AsmToken::String: case AsmToken::Identifier: - if (getParser().parseExpression(Res)) + if (getParser().parseExpression(Res, E)) return MatchOperand_ParseFail; break; case AsmToken::Percent: @@ -1410,7 +1474,7 @@ OperandMatchResultTy RISCVAsmParser::parseImmediate(OperandVector &Operands) { OperandMatchResultTy RISCVAsmParser::parseOperandWithModifier(OperandVector &Operands) { SMLoc S = getLoc(); - SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1); + SMLoc E; if (getLexer().getKind() != AsmToken::Percent) { Error(getLoc(), "expected '%' for operand modifier"); @@ -1449,7 +1513,6 @@ RISCVAsmParser::parseOperandWithModifier(OperandVector &Operands) { OperandMatchResultTy RISCVAsmParser::parseBareSymbol(OperandVector &Operands) { SMLoc S = getLoc(); - SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1); const MCExpr *Res; if (getLexer().getKind() != AsmToken::Identifier) @@ -1461,6 +1524,8 @@ OperandMatchResultTy RISCVAsmParser::parseBareSymbol(OperandVector &Operands) { if (getParser().parseIdentifier(Identifier)) return MatchOperand_ParseFail; + SMLoc E = SMLoc::getFromPointer(S.getPointer() + Identifier.size()); + if (Identifier.consume_back("@plt")) { Error(getLoc(), "'@plt' operand not valid for instruction"); return MatchOperand_ParseFail; @@ -1492,7 +1557,7 @@ OperandMatchResultTy RISCVAsmParser::parseBareSymbol(OperandVector &Operands) { } const MCExpr *Expr; - if (getParser().parseExpression(Expr)) + if (getParser().parseExpression(Expr, E)) return MatchOperand_ParseFail; Res = MCBinaryExpr::create(Opcode, Res, Expr, getContext()); Operands.push_back(RISCVOperand::createImm(Res, S, E, isRV64())); @@ -1501,7 +1566,6 @@ OperandMatchResultTy RISCVAsmParser::parseBareSymbol(OperandVector &Operands) { OperandMatchResultTy RISCVAsmParser::parseCallSymbol(OperandVector &Operands) { SMLoc S = getLoc(); - SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1); const MCExpr *Res; if (getLexer().getKind() != AsmToken::Identifier) @@ -1515,6 +1579,8 @@ OperandMatchResultTy RISCVAsmParser::parseCallSymbol(OperandVector &Operands) { if (getParser().parseIdentifier(Identifier)) return MatchOperand_ParseFail; + SMLoc E = SMLoc::getFromPointer(S.getPointer() + Identifier.size()); + RISCVMCExpr::VariantKind Kind = RISCVMCExpr::VK_RISCV_CALL; if (Identifier.consume_back("@plt")) Kind = RISCVMCExpr::VK_RISCV_CALL_PLT; @@ -1529,10 +1595,10 @@ OperandMatchResultTy RISCVAsmParser::parseCallSymbol(OperandVector &Operands) { OperandMatchResultTy RISCVAsmParser::parsePseudoJumpSymbol(OperandVector &Operands) { SMLoc S = getLoc(); - SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1); + SMLoc E; const MCExpr *Res; - if (getParser().parseExpression(Res)) + if (getParser().parseExpression(Res, E)) return MatchOperand_ParseFail; if (Res->getKind() != MCExpr::ExprKind::SymbolRef || @@ -1662,7 +1728,7 @@ OperandMatchResultTy RISCVAsmParser::parseMaskReg(OperandVector &Operands) { if (RegNo != RISCV::V0) return MatchOperand_NoMatch; SMLoc S = getLoc(); - SMLoc E = SMLoc::getFromPointer(S.getPointer() - 1); + SMLoc E = SMLoc::getFromPointer(S.getPointer() + Name.size()); getLexer().Lex(); Operands.push_back(RISCVOperand::createReg(RegNo, S, E, isRV64())); } @@ -2062,7 +2128,11 @@ bool RISCVAsmParser::parseDirectiveAttribute() { "unexpected token in '.attribute' directive")) return true; - if (Tag == RISCVAttrs::ARCH) { + if (IsIntegerValue) + getTargetStreamer().emitAttribute(Tag, IntegerValue); + else if (Tag != RISCVAttrs::ARCH) + getTargetStreamer().emitTextAttribute(Tag, StringValue); + else { StringRef Arch = StringValue; for (auto Feature : RISCVFeatureKV) if (llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key)) @@ -2070,7 +2140,7 @@ bool RISCVAsmParser::parseDirectiveAttribute() { auto ParseResult = llvm::RISCVISAInfo::parseArchString( StringValue, /*EnableExperimentalExtension=*/true, - /*ExperimentalExtensionVersionCheck=*/false); + /*ExperimentalExtensionVersionCheck=*/true); if (!ParseResult) { std::string Buffer; raw_string_ostream OutputErrMsg(Buffer); @@ -2093,35 +2163,9 @@ bool RISCVAsmParser::parseDirectiveAttribute() { setFeatureBits(RISCV::Feature64Bit, "64bit"); else return Error(ValueExprLoc, "bad arch string " + Arch); - } - - if (IsIntegerValue) - getTargetStreamer().emitAttribute(Tag, IntegerValue); - else { - if (Tag != RISCVAttrs::ARCH) { - getTargetStreamer().emitTextAttribute(Tag, StringValue); - } else { - std::vector<std::string> FeatureVector; - RISCVFeatures::toFeatureVector(FeatureVector, getSTI().getFeatureBits()); - // Parse that by RISCVISAInfo-> - unsigned XLen = getFeatureBits(RISCV::Feature64Bit) ? 64 : 32; - auto ParseResult = llvm::RISCVISAInfo::parseFeatures(XLen, FeatureVector); - if (!ParseResult) { - std::string Buffer; - raw_string_ostream OutputErrMsg(Buffer); - handleAllErrors(ParseResult.takeError(), - [&](llvm::StringError &ErrMsg) { - OutputErrMsg << ErrMsg.getMessage(); - }); - - return Error(ValueExprLoc, OutputErrMsg.str()); - } - auto &ISAInfo = *ParseResult; - - // Then emit the arch string. - getTargetStreamer().emitTextAttribute(Tag, ISAInfo->toString()); - } + // Then emit the arch string. + getTargetStreamer().emitTextAttribute(Tag, ISAInfo->toString()); } return false; diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp index 0aba18b20f0d..144e761f002d 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.cpp @@ -27,6 +27,11 @@ namespace RISCVSysReg { #include "RISCVGenSearchableTables.inc" } // namespace RISCVSysReg +namespace RISCVInsnOpcode { +#define GET_RISCVOpcodesList_IMPL +#include "RISCVGenSearchableTables.inc" +} // namespace RISCVInsnOpcode + namespace RISCVABI { ABI computeTargetABI(const Triple &TT, FeatureBitset FeatureBits, StringRef ABIName) { diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h index d8f4403c824f..9cfd36745f46 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h @@ -299,6 +299,16 @@ struct SysReg { #include "RISCVGenSearchableTables.inc" } // end namespace RISCVSysReg +namespace RISCVInsnOpcode { +struct RISCVOpcode { + const char *Name; + unsigned Value; +}; + +#define GET_RISCVOpcodesList_DECL +#include "RISCVGenSearchableTables.inc" +} // end namespace RISCVInsnOpcode + namespace RISCVABI { enum ABI { diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp index f1c3810f4ee5..89a7d54f60f8 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp @@ -171,9 +171,9 @@ void RISCVInstPrinter::printVTypeI(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI, raw_ostream &O) { unsigned Imm = MI->getOperand(OpNo).getImm(); // Print the raw immediate for reserved values: vlmul[2:0]=4, vsew[2:0]=0b1xx, - // or non-zero bits 8/9/10. + // or non-zero in bits 8 and above. if (RISCVVType::getVLMUL(Imm) == RISCVII::VLMUL::LMUL_RESERVED || - RISCVVType::getSEW(Imm) > 64 || (Imm & 0x700) != 0) { + RISCVVType::getSEW(Imm) > 64 || (Imm >> 8) != 0) { O << Imm; return; } diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td index 772a4f8ecd53..6aa915c01929 100644 --- a/llvm/lib/Target/RISCV/RISCV.td +++ b/llvm/lib/Target/RISCV/RISCV.td @@ -168,14 +168,6 @@ def HasStdExtZvlsseg : Predicate<"Subtarget->hasStdExtZvlsseg()">, AssemblerPredicate<(all_of FeatureStdExtZvlsseg), "'Zvlsseg' (Vector segment load/store instructions)">; -def FeatureStdExtZvamo - : SubtargetFeature<"experimental-zvamo", "HasStdExtZvamo", "true", - "'Zvamo' (Vector AMO Operations)", - [FeatureStdExtV]>; -def HasStdExtZvamo : Predicate<"Subtarget->hasStdExtZvamo()">, - AssemblerPredicate<(all_of FeatureStdExtZvamo), - "'Zvamo' (Vector AMO Operations)">; - def Feature64Bit : SubtargetFeature<"64bit", "HasRV64", "true", "Implements RV64">; def IsRV64 : Predicate<"Subtarget->is64Bit()">, diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index 66a34d73dd37..b24eb5f7bbf4 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -718,6 +718,71 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { break; } + case ISD::MUL: { + // Special case for calculating (mul (and X, C2), C1) where the full product + // fits in XLen bits. We can shift X left by the number of leading zeros in + // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final + // product has XLen trailing zeros, putting it in the output of MULHU. This + // can avoid materializing a constant in a register for C2. + + // RHS should be a constant. + auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1)); + if (!N1C || !N1C->hasOneUse()) + break; + + // LHS should be an AND with constant. + SDValue N0 = Node->getOperand(0); + if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1))) + break; + + uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue(); + + // Constant should be a mask. + if (!isMask_64(C2)) + break; + + // This should be the only use of the AND unless we will use + // (SRLI (SLLI X, 32), 32). We don't use a shift pair for other AND + // constants. + if (!N0.hasOneUse() && C2 != UINT64_C(0xFFFFFFFF)) + break; + + // If this can be an ANDI, ZEXT.H or ZEXT.W we don't need to do this + // optimization. + if (isInt<12>(C2) || + (C2 == UINT64_C(0xFFFF) && + (Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp())) || + (C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba())) + break; + + // We need to shift left the AND input and C1 by a total of XLen bits. + + // How far left do we need to shift the AND input? + unsigned XLen = Subtarget->getXLen(); + unsigned LeadingZeros = XLen - (64 - countLeadingZeros(C2)); + + // The constant gets shifted by the remaining amount unless that would + // shift bits out. + uint64_t C1 = N1C->getZExtValue(); + unsigned ConstantShift = XLen - LeadingZeros; + if (ConstantShift > (XLen - (64 - countLeadingZeros(C1)))) + break; + + uint64_t ShiftedC1 = C1 << ConstantShift; + // If this RV32, we need to sign extend the constant. + if (XLen == 32) + ShiftedC1 = SignExtend64(ShiftedC1, 32); + + // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))). + SDNode *Imm = selectImm(CurDAG, DL, ShiftedC1, *Subtarget); + SDNode *SLLI = + CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0), + CurDAG->getTargetConstant(LeadingZeros, DL, VT)); + SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT, + SDValue(SLLI, 0), SDValue(Imm, 0)); + ReplaceNode(Node, MULHU); + return; + } case ISD::INTRINSIC_WO_CHAIN: { unsigned IntNo = Node->getConstantOperandVal(0); switch (IntNo) { @@ -1450,6 +1515,7 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { ReplaceNode(Node, Extract.getNode()); return; } + case ISD::SPLAT_VECTOR: case RISCVISD::VMV_V_X_VL: case RISCVISD::VFMV_V_F_VL: { // Try to match splat of a scalar load to a strided load with stride of x0. @@ -1466,7 +1532,10 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { break; SDValue VL; - selectVLOp(Node->getOperand(1), VL); + if (Node->getOpcode() == ISD::SPLAT_VECTOR) + VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, DL, XLenVT); + else + selectVLOp(Node->getOperand(1), VL); unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits()); SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index f3331571fc55..4f5512e6fb37 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -330,6 +330,14 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::LLRINT, MVT::f16, Legal); setOperationAction(ISD::LROUND, MVT::f16, Legal); setOperationAction(ISD::LLROUND, MVT::f16, Legal); + setOperationAction(ISD::STRICT_FADD, MVT::f16, Legal); + setOperationAction(ISD::STRICT_FMA, MVT::f16, Legal); + setOperationAction(ISD::STRICT_FSUB, MVT::f16, Legal); + setOperationAction(ISD::STRICT_FMUL, MVT::f16, Legal); + setOperationAction(ISD::STRICT_FDIV, MVT::f16, Legal); + setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal); + setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Legal); for (auto CC : FPCCToExpand) setCondCodeAction(CC, MVT::f16, Expand); setOperationAction(ISD::SELECT_CC, MVT::f16, Expand); @@ -367,6 +375,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::LLRINT, MVT::f32, Legal); setOperationAction(ISD::LROUND, MVT::f32, Legal); setOperationAction(ISD::LLROUND, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal); for (auto CC : FPCCToExpand) setCondCodeAction(CC, MVT::f32, Expand); setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); @@ -388,6 +402,14 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::LLRINT, MVT::f64, Legal); setOperationAction(ISD::LROUND, MVT::f64, Legal); setOperationAction(ISD::LLROUND, MVT::f64, Legal); + setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal); + setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal); + setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal); + setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal); + setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal); + setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal); + setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal); + setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal); for (auto CC : FPCCToExpand) setCondCodeAction(CC, MVT::f64, Expand); setOperationAction(ISD::SELECT_CC, MVT::f64, Expand); @@ -412,6 +434,11 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::FP_TO_UINT_SAT, XLenVT, Custom); setOperationAction(ISD::FP_TO_SINT_SAT, XLenVT, Custom); + setOperationAction(ISD::STRICT_FP_TO_UINT, XLenVT, Legal); + setOperationAction(ISD::STRICT_FP_TO_SINT, XLenVT, Legal); + setOperationAction(ISD::STRICT_UINT_TO_FP, XLenVT, Legal); + setOperationAction(ISD::STRICT_SINT_TO_FP, XLenVT, Legal); + setOperationAction(ISD::FLT_ROUNDS_, XLenVT, Custom); setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom); } @@ -471,12 +498,13 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR, ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX, - ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN}; + ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN, + ISD::VP_SELECT}; static const unsigned FloatingPointVPOps[] = { ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL, ISD::VP_FDIV, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD, - ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX}; + ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SELECT}; if (!Subtarget.is64Bit()) { // We must custom-lower certain vXi64 operations on RV32 due to the vector @@ -519,6 +547,10 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::SELECT_CC, VT, Expand); setOperationAction(ISD::VSELECT, VT, Expand); + setOperationAction(ISD::VP_AND, VT, Custom); + setOperationAction(ISD::VP_OR, VT, Custom); + setOperationAction(ISD::VP_XOR, VT, Custom); + setOperationAction(ISD::VECREDUCE_AND, VT, Custom); setOperationAction(ISD::VECREDUCE_OR, VT, Custom); setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); @@ -803,6 +835,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, // Operations below are different for between masks and other vectors. if (VT.getVectorElementType() == MVT::i1) { + setOperationAction(ISD::VP_AND, VT, Custom); + setOperationAction(ISD::VP_OR, VT, Custom); + setOperationAction(ISD::VP_XOR, VT, Custom); setOperationAction(ISD::AND, VT, Custom); setOperationAction(ISD::OR, VT, Custom); setOperationAction(ISD::XOR, VT, Custom); @@ -1147,7 +1182,7 @@ bool RISCVTargetLowering::isCheapToSpeculateCtlz() const { return Subtarget.hasStdExtZbb(); } -bool RISCVTargetLowering::hasAndNot(SDValue Y) const { +bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const { EVT VT = Y.getValueType(); // FIXME: Support vectors once we have tests. @@ -1235,7 +1270,8 @@ bool RISCVTargetLowering::shouldSinkOperands( bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const { - if (VT == MVT::f16 && !Subtarget.hasStdExtZfhmin()) + // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin. + if (VT == MVT::f16 && !Subtarget.hasStdExtZfh()) return false; if (VT == MVT::f32 && !Subtarget.hasStdExtF()) return false; @@ -1255,9 +1291,10 @@ bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const { MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const { - // Use f32 to pass f16 if it is legal and Zfhmin/Zfh is not enabled. + // Use f32 to pass f16 if it is legal and Zfh is not enabled. // We might still end up using a GPR but that will be decided based on ABI. - if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfhmin()) + // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin. + if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh()) return MVT::f32; return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT); @@ -1266,9 +1303,10 @@ MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const { - // Use f32 to pass f16 if it is legal and Zfhmin/Zfh is not enabled. + // Use f32 to pass f16 if it is legal and Zfh is not enabled. // We might still end up using a GPR but that will be decided based on ABI. - if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfhmin()) + // FIXME: Change to Zfhmin once f16 becomes a legal type with Zfhmin. + if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh()) return 1; return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT); @@ -1959,29 +1997,37 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, int64_t StepNumerator = SimpleVID->StepNumerator; unsigned StepDenominator = SimpleVID->StepDenominator; int64_t Addend = SimpleVID->Addend; + + assert(StepNumerator != 0 && "Invalid step"); + bool Negate = false; + int64_t SplatStepVal = StepNumerator; + unsigned StepOpcode = ISD::MUL; + if (StepNumerator != 1) { + if (isPowerOf2_64(std::abs(StepNumerator))) { + Negate = StepNumerator < 0; + StepOpcode = ISD::SHL; + SplatStepVal = Log2_64(std::abs(StepNumerator)); + } + } + // Only emit VIDs with suitably-small steps/addends. We use imm5 is a // threshold since it's the immediate value many RVV instructions accept. - if (isInt<5>(StepNumerator) && isPowerOf2_32(StepDenominator) && - isInt<5>(Addend)) { + // There is no vmul.vi instruction so ensure multiply constant can fit in + // a single addi instruction. + if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) || + (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) && + isPowerOf2_32(StepDenominator) && isInt<5>(Addend)) { SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL); // Convert right out of the scalable type so we can use standard ISD // nodes for the rest of the computation. If we used scalable types with // these, we'd lose the fixed-length vector info and generate worse // vsetvli code. VID = convertFromScalableVector(VT, VID, DAG, Subtarget); - assert(StepNumerator != 0 && "Invalid step"); - bool Negate = false; - if (StepNumerator != 1) { - int64_t SplatStepVal = StepNumerator; - unsigned Opcode = ISD::MUL; - if (isPowerOf2_64(std::abs(StepNumerator))) { - Negate = StepNumerator < 0; - Opcode = ISD::SHL; - SplatStepVal = Log2_64(std::abs(StepNumerator)); - } + if ((StepOpcode == ISD::MUL && SplatStepVal != 1) || + (StepOpcode == ISD::SHL && SplatStepVal != 0)) { SDValue SplatStep = DAG.getSplatVector( VT, DL, DAG.getConstant(SplatStepVal, DL, XLenVT)); - VID = DAG.getNode(Opcode, DL, VT, VID, SplatStep); + VID = DAG.getNode(StepOpcode, DL, VT, VID, SplatStep); } if (StepDenominator != 1) { SDValue SplatStep = DAG.getSplatVector( @@ -3133,6 +3179,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, return lowerGET_ROUNDING(Op, DAG); case ISD::SET_ROUNDING: return lowerSET_ROUNDING(Op, DAG); + case ISD::VP_SELECT: + return lowerVPOp(Op, DAG, RISCVISD::VSELECT_VL); case ISD::VP_ADD: return lowerVPOp(Op, DAG, RISCVISD::ADD_VL); case ISD::VP_SUB: @@ -3148,11 +3196,11 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, case ISD::VP_UREM: return lowerVPOp(Op, DAG, RISCVISD::UREM_VL); case ISD::VP_AND: - return lowerVPOp(Op, DAG, RISCVISD::AND_VL); + return lowerLogicVPOp(Op, DAG, RISCVISD::VMAND_VL, RISCVISD::AND_VL); case ISD::VP_OR: - return lowerVPOp(Op, DAG, RISCVISD::OR_VL); + return lowerLogicVPOp(Op, DAG, RISCVISD::VMOR_VL, RISCVISD::OR_VL); case ISD::VP_XOR: - return lowerVPOp(Op, DAG, RISCVISD::XOR_VL); + return lowerLogicVPOp(Op, DAG, RISCVISD::VMXOR_VL, RISCVISD::XOR_VL); case ISD::VP_ASHR: return lowerVPOp(Op, DAG, RISCVISD::SRA_VL); case ISD::VP_LSHR: @@ -4469,19 +4517,19 @@ SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op, } MVT M1VT = getLMUL1VT(ContainerVT); + MVT XLenVT = Subtarget.getXLenVT(); SDValue Mask, VL; std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); - // FIXME: This is a VLMAX splat which might be too large and can prevent - // vsetvli removal. SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags()); - SDValue IdentitySplat = DAG.getSplatVector(M1VT, DL, NeutralElem); + SDValue IdentitySplat = lowerScalarSplat( + NeutralElem, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget); SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT), Vec, IdentitySplat, Mask, VL); SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction, - DAG.getConstant(0, DL, Subtarget.getXLenVT())); + DAG.getConstant(0, DL, XLenVT)); return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType()); } @@ -4497,9 +4545,12 @@ getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) { switch (Opcode) { default: llvm_unreachable("Unhandled reduction"); - case ISD::VECREDUCE_FADD: - return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), - DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags)); + case ISD::VECREDUCE_FADD: { + // Use positive zero if we can. It is cheaper to materialize. + SDValue Zero = + DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT); + return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero); + } case ISD::VECREDUCE_SEQ_FADD: return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1), Op.getOperand(0)); @@ -4530,17 +4581,17 @@ SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op, } MVT M1VT = getLMUL1VT(VectorVal.getSimpleValueType()); + MVT XLenVT = Subtarget.getXLenVT(); SDValue Mask, VL; std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget); - // FIXME: This is a VLMAX splat which might be too large and can prevent - // vsetvli removal. - SDValue ScalarSplat = DAG.getSplatVector(M1VT, DL, ScalarVal); + SDValue ScalarSplat = lowerScalarSplat( + ScalarVal, DAG.getConstant(1, DL, XLenVT), M1VT, DL, DAG, Subtarget); SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, DAG.getUNDEF(M1VT), VectorVal, ScalarSplat, Mask, VL); return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction, - DAG.getConstant(0, DL, Subtarget.getXLenVT())); + DAG.getConstant(0, DL, XLenVT)); } static unsigned getRVVVPReductionOp(unsigned ISDOpcode) { @@ -4602,13 +4653,13 @@ SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op, MVT XLenVT = Subtarget.getXLenVT(); MVT ResVT = !VecVT.isInteger() || VecEltVT.bitsGE(XLenVT) ? VecEltVT : XLenVT; - // FIXME: This is a VLMAX splat which might be too large and can prevent - // vsetvli removal. - SDValue StartSplat = DAG.getSplatVector(M1VT, DL, Op.getOperand(0)); + SDValue StartSplat = + lowerScalarSplat(Op.getOperand(0), DAG.getConstant(1, DL, XLenVT), M1VT, + DL, DAG, Subtarget); SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, StartSplat, Vec, StartSplat, Mask, VL); SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction, - DAG.getConstant(0, DL, Subtarget.getXLenVT())); + DAG.getConstant(0, DL, XLenVT)); if (!VecVT.isInteger()) return Elt0; return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType()); @@ -5365,6 +5416,33 @@ SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG, return convertFromScalableVector(VT, VPOp, DAG, Subtarget); } +SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op, SelectionDAG &DAG, + unsigned MaskOpc, + unsigned VecOpc) const { + MVT VT = Op.getSimpleValueType(); + if (VT.getVectorElementType() != MVT::i1) + return lowerVPOp(Op, DAG, VecOpc); + + // It is safe to drop mask parameter as masked-off elements are undef. + SDValue Op1 = Op->getOperand(0); + SDValue Op2 = Op->getOperand(1); + SDValue VL = Op->getOperand(3); + + MVT ContainerVT = VT; + const bool IsFixed = VT.isFixedLengthVector(); + if (IsFixed) { + ContainerVT = getContainerForFixedLengthVector(VT); + Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget); + Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget); + } + + SDLoc DL(Op); + SDValue Val = DAG.getNode(MaskOpc, DL, ContainerVT, Op1, Op2, VL); + if (!IsFixed) + return Val; + return convertFromScalableVector(VT, Val, DAG, Subtarget); +} + // Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be // matched to a RVV indexed load. The RVV indexed load instructions only // support the "unsigned unscaled" addressing mode; indices are implicitly @@ -5695,11 +5773,17 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0); if (getTypeAction(*DAG.getContext(), Op0.getValueType()) != TargetLowering::TypeSoftenFloat) { - // FIXME: Support strict FP. - if (IsStrict) - return; if (!isTypeLegal(Op0.getValueType())) return; + if (IsStrict) { + unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RTZ_RV64 + : RISCVISD::STRICT_FCVT_WU_RTZ_RV64; + SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other); + SDValue Res = DAG.getNode(Opc, DL, VTs, N->getOperand(0), Op0); + Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); + Results.push_back(Res.getValue(1)); + return; + } unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RTZ_RV64 : RISCVISD::FCVT_WU_RTZ_RV64; SDValue Res = DAG.getNode(Opc, DL, MVT::i64, Op0); @@ -7026,7 +7110,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N, if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth))) return SDValue(N, 0); - return combineGREVI_GORCI(N, DCI.DAG); + return combineGREVI_GORCI(N, DAG); } case RISCVISD::GREVW: case RISCVISD::GORCW: { @@ -7035,7 +7119,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N, SimplifyDemandedLowBitsHelper(1, 5)) return SDValue(N, 0); - return combineGREVI_GORCI(N, DCI.DAG); + return combineGREVI_GORCI(N, DAG); } case RISCVISD::SHFL: case RISCVISD::UNSHFL: { @@ -7120,11 +7204,23 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N, // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during // type legalization. This is safe because fp_to_uint produces poison if // it overflows. - if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit() && - N->getOperand(0).getOpcode() == ISD::FP_TO_UINT && - isTypeLegal(N->getOperand(0).getOperand(0).getValueType())) - return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64, - N->getOperand(0).getOperand(0)); + if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit()) { + SDValue Src = N->getOperand(0); + if (Src.getOpcode() == ISD::FP_TO_UINT && + isTypeLegal(Src.getOperand(0).getValueType())) + return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64, + Src.getOperand(0)); + if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() && + isTypeLegal(Src.getOperand(1).getValueType())) { + SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other); + SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs, + Src.getOperand(0), Src.getOperand(1)); + DCI.CombineTo(N, Res); + DAG.ReplaceAllUsesOfValueWith(Src.getValue(1), Res.getValue(1)); + DCI.recursivelyDeleteUnusedNodes(Src.getNode()); + return SDValue(N, 0); // Return N so it doesn't get rechecked. + } + } return SDValue(); case RISCVISD::SELECT_CC: { // Transform @@ -7685,6 +7781,8 @@ unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode( case RISCVISD::BDECOMPRESSW: case RISCVISD::FCVT_W_RTZ_RV64: case RISCVISD::FCVT_WU_RTZ_RV64: + case RISCVISD::STRICT_FCVT_W_RTZ_RV64: + case RISCVISD::STRICT_FCVT_WU_RTZ_RV64: // TODO: As the result is sign-extended, this is conservatively correct. A // more precise answer could be calculated for SRAW depending on known // bits in the shift amount. @@ -8004,6 +8102,22 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, } } +void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, + SDNode *Node) const { + // Add FRM dependency to any instructions with dynamic rounding mode. + unsigned Opc = MI.getOpcode(); + auto Idx = RISCV::getNamedOperandIdx(Opc, RISCV::OpName::frm); + if (Idx < 0) + return; + if (MI.getOperand(Idx).getImm() != RISCVFPRndMode::DYN) + return; + // If the instruction already reads FRM, don't add another read. + if (MI.readsRegister(RISCV::FRM)) + return; + MI.addOperand( + MachineOperand::CreateReg(RISCV::FRM, /*isDef*/ false, /*isImp*/ true)); +} + // Calling Convention Implementation. // The expectations for frontend ABI lowering vary from target to target. // Ideally, an LLVM frontend would be able to avoid worrying about many ABI @@ -9400,6 +9514,8 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(FCVT_XU_RTZ) NODE_NAME_CASE(FCVT_W_RTZ_RV64) NODE_NAME_CASE(FCVT_WU_RTZ_RV64) + NODE_NAME_CASE(STRICT_FCVT_W_RTZ_RV64) + NODE_NAME_CASE(STRICT_FCVT_WU_RTZ_RV64) NODE_NAME_CASE(READ_CYCLE_WIDE) NODE_NAME_CASE(GREV) NODE_NAME_CASE(GREVW) @@ -9541,6 +9657,9 @@ RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, if (Constraint.size() == 1) { switch (Constraint[0]) { case 'r': + // TODO: Support fixed vectors up to XLen for P extension? + if (VT.isVector()) + break; return std::make_pair(0U, &RISCV::GPRRegClass); case 'f': if (Subtarget.hasStdExtZfh() && VT == MVT::f16) @@ -9553,17 +9672,15 @@ RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, default: break; } - } else { - if (Constraint == "vr") { - for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass, - &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) { - if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) - return std::make_pair(0U, RC); - } - } else if (Constraint == "vm") { - if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy)) - return std::make_pair(0U, &RISCV::VMRegClass); + } else if (Constraint == "vr") { + for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass, + &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) { + if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) + return std::make_pair(0U, RC); } + } else if (Constraint == "vm") { + if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy)) + return std::make_pair(0U, &RISCV::VMV0RegClass); } // Clang will correctly decode the usage of register name aliases into their @@ -10101,17 +10218,29 @@ bool RISCVTargetLowering::splitValueIntoRegisterParts( unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize(); unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize(); if (PartVTBitSize % ValueVTBitSize == 0) { + assert(PartVTBitSize >= ValueVTBitSize); // If the element types are different, bitcast to the same element type of // PartVT first. + // Give an example here, we want copy a <vscale x 1 x i8> value to + // <vscale x 4 x i16>. + // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert + // subvector, then we can bitcast to <vscale x 4 x i16>. if (ValueEltVT != PartEltVT) { - unsigned Count = ValueVTBitSize / PartEltVT.getSizeInBits(); - assert(Count != 0 && "The number of element should not be zero."); - EVT SameEltTypeVT = - EVT::getVectorVT(Context, PartEltVT, Count, /*IsScalable=*/true); - Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val); + if (PartVTBitSize > ValueVTBitSize) { + unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits(); + assert(Count != 0 && "The number of element should not be zero."); + EVT SameEltTypeVT = + EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true); + Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT, + DAG.getUNDEF(SameEltTypeVT), Val, + DAG.getVectorIdxConstant(0, DL)); + } + Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); + } else { + Val = + DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT), + Val, DAG.getVectorIdxConstant(0, DL)); } - Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT), - Val, DAG.getConstant(0, DL, Subtarget.getXLenVT())); Parts[0] = Val; return true; } @@ -10141,19 +10270,23 @@ SDValue RISCVTargetLowering::joinRegisterPartsIntoValue( unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize(); unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize(); if (PartVTBitSize % ValueVTBitSize == 0) { + assert(PartVTBitSize >= ValueVTBitSize); EVT SameEltTypeVT = ValueVT; // If the element types are different, convert it to the same element type // of PartVT. + // Give an example here, we want copy a <vscale x 1 x i8> value from + // <vscale x 4 x i16>. + // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first, + // then we can extract <vscale x 1 x i8>. if (ValueEltVT != PartEltVT) { - unsigned Count = ValueVTBitSize / PartEltVT.getSizeInBits(); + unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits(); assert(Count != 0 && "The number of element should not be zero."); SameEltTypeVT = - EVT::getVectorVT(Context, PartEltVT, Count, /*IsScalable=*/true); + EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true); + Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val); } - Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SameEltTypeVT, Val, - DAG.getConstant(0, DL, Subtarget.getXLenVT())); - if (ValueEltVT != PartEltVT) - Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); + Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, + DAG.getVectorIdxConstant(0, DL)); return Val; } } diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index 849928eb46ae..48c5ce730933 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -282,6 +282,11 @@ enum NodeType : unsigned { // the value read before the modification and the new chain pointer. SWAP_CSR, + // FP to 32 bit int conversions for RV64. These are used to keep track of the + // result being sign extended to 64 bit. These saturate out of range inputs. + STRICT_FCVT_W_RTZ_RV64 = ISD::FIRST_TARGET_STRICTFP_OPCODE, + STRICT_FCVT_WU_RTZ_RV64, + // Memory opcodes start here. VLE_VL = ISD::FIRST_TARGET_MEMORY_OPCODE, VSE_VL, @@ -315,7 +320,7 @@ public: bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override; bool isCheapToSpeculateCttz() const override; bool isCheapToSpeculateCtlz() const override; - bool hasAndNot(SDValue Y) const override; + bool hasAndNotCompare(SDValue Y) const override; bool shouldSinkOperands(Instruction *I, SmallVectorImpl<Use *> &Ops) const override; bool isFPImmLegal(const APFloat &Imm, EVT VT, @@ -383,6 +388,9 @@ public: EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override; + void AdjustInstrPostInstrSelection(MachineInstr &MI, + SDNode *Node) const override; + EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override; @@ -593,6 +601,8 @@ private: SDValue lowerToScalableOp(SDValue Op, SelectionDAG &DAG, unsigned NewOpc, bool HasMask = true) const; SDValue lowerVPOp(SDValue Op, SelectionDAG &DAG, unsigned RISCVISDOpc) const; + SDValue lowerLogicVPOp(SDValue Op, SelectionDAG &DAG, unsigned MaskOpc, + unsigned VecOpc) const; SDValue lowerFixedLengthVectorExtendToRVV(SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const; SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const; diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td index cfad4cdb9364..6a16b6354f95 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td +++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td @@ -107,31 +107,44 @@ def Vcompress : RISCVVConstraint<!or(VS2Constraint.Value, // The following opcode names match those given in Table 19.1 in the // RISC-V User-level ISA specification ("RISC-V base opcode map"). -class RISCVOpcode<bits<7> val> { +class RISCVOpcode<string name, bits<7> val> { + string Name = name; bits<7> Value = val; } -def OPC_LOAD : RISCVOpcode<0b0000011>; -def OPC_LOAD_FP : RISCVOpcode<0b0000111>; -def OPC_MISC_MEM : RISCVOpcode<0b0001111>; -def OPC_OP_IMM : RISCVOpcode<0b0010011>; -def OPC_AUIPC : RISCVOpcode<0b0010111>; -def OPC_OP_IMM_32 : RISCVOpcode<0b0011011>; -def OPC_STORE : RISCVOpcode<0b0100011>; -def OPC_STORE_FP : RISCVOpcode<0b0100111>; -def OPC_AMO : RISCVOpcode<0b0101111>; -def OPC_OP : RISCVOpcode<0b0110011>; -def OPC_LUI : RISCVOpcode<0b0110111>; -def OPC_OP_32 : RISCVOpcode<0b0111011>; -def OPC_MADD : RISCVOpcode<0b1000011>; -def OPC_MSUB : RISCVOpcode<0b1000111>; -def OPC_NMSUB : RISCVOpcode<0b1001011>; -def OPC_NMADD : RISCVOpcode<0b1001111>; -def OPC_OP_FP : RISCVOpcode<0b1010011>; -def OPC_OP_V : RISCVOpcode<0b1010111>; -def OPC_BRANCH : RISCVOpcode<0b1100011>; -def OPC_JALR : RISCVOpcode<0b1100111>; -def OPC_JAL : RISCVOpcode<0b1101111>; -def OPC_SYSTEM : RISCVOpcode<0b1110011>; +def RISCVOpcodesList : GenericTable { + let FilterClass = "RISCVOpcode"; + let Fields = [ + "Name", "Value" + ]; + let PrimaryKey = [ "Value" ]; + let PrimaryKeyName = "lookupRISCVOpcodeByValue"; +} +def lookupRISCVOpcodeByName : SearchIndex { + let Table = RISCVOpcodesList; + let Key = [ "Name" ]; +} +def OPC_LOAD : RISCVOpcode<"LOAD", 0b0000011>; +def OPC_LOAD_FP : RISCVOpcode<"LOAD_FP", 0b0000111>; +def OPC_MISC_MEM : RISCVOpcode<"MISC_MEM", 0b0001111>; +def OPC_OP_IMM : RISCVOpcode<"OP_IMM", 0b0010011>; +def OPC_AUIPC : RISCVOpcode<"AUIPC", 0b0010111>; +def OPC_OP_IMM_32 : RISCVOpcode<"OP_IMM_32", 0b0011011>; +def OPC_STORE : RISCVOpcode<"STORE", 0b0100011>; +def OPC_STORE_FP : RISCVOpcode<"STORE_FP", 0b0100111>; +def OPC_AMO : RISCVOpcode<"AMO", 0b0101111>; +def OPC_OP : RISCVOpcode<"OP", 0b0110011>; +def OPC_LUI : RISCVOpcode<"LUI", 0b0110111>; +def OPC_OP_32 : RISCVOpcode<"OP_32", 0b0111011>; +def OPC_MADD : RISCVOpcode<"MADD", 0b1000011>; +def OPC_MSUB : RISCVOpcode<"MSUB", 0b1000111>; +def OPC_NMSUB : RISCVOpcode<"NMSUB", 0b1001011>; +def OPC_NMADD : RISCVOpcode<"NMADD", 0b1001111>; +def OPC_OP_FP : RISCVOpcode<"OP_FP", 0b1010011>; +def OPC_OP_V : RISCVOpcode<"OP_V", 0b1010111>; +def OPC_BRANCH : RISCVOpcode<"BRANCH", 0b1100011>; +def OPC_JALR : RISCVOpcode<"JALR", 0b1100111>; +def OPC_JAL : RISCVOpcode<"JAL", 0b1101111>; +def OPC_SYSTEM : RISCVOpcode<"SYSTEM", 0b1110011>; class RVInst<dag outs, dag ins, string opcodestr, string argstr, list<dag> pattern, InstFormat format> @@ -188,8 +201,7 @@ class RVInst<dag outs, dag ins, string opcodestr, string argstr, // Pseudo instructions class Pseudo<dag outs, dag ins, list<dag> pattern, string opcodestr = "", string argstr = ""> - : RVInst<outs, ins, opcodestr, argstr, pattern, InstFormatPseudo>, - Sched<[]> { + : RVInst<outs, ins, opcodestr, argstr, pattern, InstFormatPseudo> { let isPseudo = 1; let isCodeGenOnly = 1; } @@ -265,14 +277,14 @@ class RVInstR4Frm<bits<2> funct2, RISCVOpcode opcode, dag outs, dag ins, bits<5> rs3; bits<5> rs2; bits<5> rs1; - bits<3> funct3; + bits<3> frm; bits<5> rd; let Inst{31-27} = rs3; let Inst{26-25} = funct2; let Inst{24-20} = rs2; let Inst{19-15} = rs1; - let Inst{14-12} = funct3; + let Inst{14-12} = frm; let Inst{11-7} = rd; let Opcode = opcode.Value; } @@ -300,13 +312,13 @@ class RVInstRFrm<bits<7> funct7, RISCVOpcode opcode, dag outs, dag ins, : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> { bits<5> rs2; bits<5> rs1; - bits<3> funct3; + bits<3> frm; bits<5> rd; let Inst{31-25} = funct7; let Inst{24-20} = rs2; let Inst{19-15} = rs1; - let Inst{14-12} = funct3; + let Inst{14-12} = frm; let Inst{11-7} = rd; let Opcode = opcode.Value; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td b/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td index 80f46b73bfd7..69e9d3553b30 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td +++ b/llvm/lib/Target/RISCV/RISCVInstrFormatsV.td @@ -45,19 +45,6 @@ def SUMOPUnitStride : RISCVLSUMOP<0b00000>; def SUMOPUnitStrideMask : RISCVLSUMOP<0b01011>; def SUMOPUnitStrideWholeReg : RISCVLSUMOP<0b01000>; -class RISCVAMOOP<bits<5> val> { - bits<5> Value = val; -} -def AMOOPVamoSwap : RISCVAMOOP<0b00001>; -def AMOOPVamoAdd : RISCVAMOOP<0b00000>; -def AMOOPVamoXor : RISCVAMOOP<0b00100>; -def AMOOPVamoAnd : RISCVAMOOP<0b01100>; -def AMOOPVamoOr : RISCVAMOOP<0b01000>; -def AMOOPVamoMin : RISCVAMOOP<0b10000>; -def AMOOPVamoMax : RISCVAMOOP<0b10100>; -def AMOOPVamoMinu : RISCVAMOOP<0b11000>; -def AMOOPVamoMaxu : RISCVAMOOP<0b11100>; - class RISCVWidth<bits<4> val> { bits<4> Value = val; } @@ -342,22 +329,3 @@ class RVInstVSX<bits<3> nf, bit mew, RISCVMOP mop, bits<3> width, let Uses = [VTYPE, VL]; } - -class RVInstVAMO<RISCVAMOOP amoop, bits<3> width, dag outs, - dag ins, string opcodestr, string argstr> - : RVInst<outs, ins, opcodestr, argstr, [], InstFormatR> { - bits<5> vs2; - bits<5> rs1; - bit wd; - bit vm; - - let Inst{31-27} = amoop.Value; - let Inst{26} = wd; - let Inst{25} = vm; - let Inst{24-20} = vs2; - let Inst{19-15} = rs1; - let Inst{14-12} = width; - let Opcode = OPC_AMO.Value; - - let Uses = [VTYPE, VL]; -} diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index 547d82550cac..2e2e00886d57 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -35,6 +35,7 @@ using namespace llvm; #include "RISCVGenCompressInstEmitter.inc" #define GET_INSTRINFO_CTOR_DTOR +#define GET_INSTRINFO_NAMED_OPS #include "RISCVGenInstrInfo.inc" static cl::opt<bool> PreferWholeRegisterMove( @@ -1059,6 +1060,7 @@ bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const { break; case RISCV::FSGNJ_D: case RISCV::FSGNJ_S: + case RISCV::FSGNJ_H: // The canonical floating-point move is fsgnj rd, rs, rs. return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() && MI.getOperand(1).getReg() == MI.getOperand(2).getReg(); @@ -1087,6 +1089,7 @@ RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const { break; case RISCV::FSGNJ_D: case RISCV::FSGNJ_S: + case RISCV::FSGNJ_H: // The canonical floating-point move is fsgnj rd, rs, rs. if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() && MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) @@ -1254,7 +1257,7 @@ bool RISCVInstrInfo::isFunctionSafeToOutlineFrom( bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const { // More accurate safety checking is done in getOutliningCandidateInfo. - return true; + return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags); } // Enum values indicating how an outlined call should be constructed. diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h index 2bfad7844c43..da0877c4299a 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h @@ -18,6 +18,7 @@ #include "llvm/IR/DiagnosticInfo.h" #define GET_INSTRINFO_HEADER +#define GET_INSTRINFO_OPERAND_ENUM #include "RISCVGenInstrInfo.inc" namespace llvm { @@ -181,6 +182,10 @@ protected: }; namespace RISCV { + +// Implemented in RISCVGenInstrInfo.inc +int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex); + // Special immediate for AVL operand of V pseudo instructions to indicate VLMax. static constexpr int64_t VLMaxSentinel = -1LL; } // namespace RISCV diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td index 6f9cde966132..71eb6f01a4f4 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td @@ -174,6 +174,20 @@ def uimm5 : Operand<XLenVT>, ImmLeaf<XLenVT, [{return isUInt<5>(Imm);}]> { let OperandNamespace = "RISCVOp"; } +def InsnDirectiveOpcode : AsmOperandClass { + let Name = "InsnDirectiveOpcode"; + let ParserMethod = "parseInsnDirectiveOpcode"; + let RenderMethod = "addImmOperands"; + let PredicateMethod = "isImm"; +} + +def uimm7_opcode : Operand<XLenVT> { + let ParserMatchClass = InsnDirectiveOpcode; + let DecoderMethod = "decodeUImmOperand<7>"; + let OperandType = "OPERAND_UIMM7"; + let OperandNamespace = "RISCVOp"; +} + def uimm7 : Operand<XLenVT> { let ParserMatchClass = UImmAsmOperand<7>; let DecoderMethod = "decodeUImmOperand<7>"; @@ -878,35 +892,35 @@ def : InstAlias<"zext.b $rd, $rs", (ANDI GPR:$rd, GPR:$rs, 0xFF), 0>; // isCodeGenOnly = 1 to hide them from the tablegened assembly parser. let isCodeGenOnly = 1, hasSideEffects = 1, mayLoad = 1, mayStore = 1, hasNoSchedulingInfo = 1 in { -def InsnR : DirectiveInsnR<(outs AnyReg:$rd), (ins uimm7:$opcode, uimm3:$funct3, +def InsnR : DirectiveInsnR<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode, uimm3:$funct3, uimm7:$funct7, AnyReg:$rs1, AnyReg:$rs2), "$opcode, $funct3, $funct7, $rd, $rs1, $rs2">; -def InsnR4 : DirectiveInsnR4<(outs AnyReg:$rd), (ins uimm7:$opcode, +def InsnR4 : DirectiveInsnR4<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode, uimm3:$funct3, uimm2:$funct2, AnyReg:$rs1, AnyReg:$rs2, AnyReg:$rs3), "$opcode, $funct3, $funct2, $rd, $rs1, $rs2, $rs3">; -def InsnI : DirectiveInsnI<(outs AnyReg:$rd), (ins uimm7:$opcode, uimm3:$funct3, +def InsnI : DirectiveInsnI<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1, simm12:$imm12), "$opcode, $funct3, $rd, $rs1, $imm12">; -def InsnI_Mem : DirectiveInsnI<(outs AnyReg:$rd), (ins uimm7:$opcode, +def InsnI_Mem : DirectiveInsnI<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1, simm12:$imm12), "$opcode, $funct3, $rd, ${imm12}(${rs1})">; -def InsnB : DirectiveInsnB<(outs), (ins uimm7:$opcode, uimm3:$funct3, +def InsnB : DirectiveInsnB<(outs), (ins uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1, AnyReg:$rs2, simm13_lsb0:$imm12), "$opcode, $funct3, $rs1, $rs2, $imm12">; -def InsnU : DirectiveInsnU<(outs AnyReg:$rd), (ins uimm7:$opcode, +def InsnU : DirectiveInsnU<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode, uimm20_lui:$imm20), "$opcode, $rd, $imm20">; -def InsnJ : DirectiveInsnJ<(outs AnyReg:$rd), (ins uimm7:$opcode, +def InsnJ : DirectiveInsnJ<(outs AnyReg:$rd), (ins uimm7_opcode:$opcode, simm21_lsb0_jal:$imm20), "$opcode, $rd, $imm20">; -def InsnS : DirectiveInsnS<(outs), (ins uimm7:$opcode, uimm3:$funct3, +def InsnS : DirectiveInsnS<(outs), (ins uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs2, AnyReg:$rs1, simm12:$imm12), "$opcode, $funct3, $rs2, ${imm12}(${rs1})">; @@ -918,37 +932,37 @@ def InsnS : DirectiveInsnS<(outs), (ins uimm7:$opcode, uimm3:$funct3, // for known formats. let EmitPriority = 0 in { def : InstAlias<".insn_r $opcode, $funct3, $funct7, $rd, $rs1, $rs2", - (InsnR AnyReg:$rd, uimm7:$opcode, uimm3:$funct3, uimm7:$funct7, + (InsnR AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3, uimm7:$funct7, AnyReg:$rs1, AnyReg:$rs2)>; // Accept 4 register form of ".insn r" as alias for ".insn r4". def : InstAlias<".insn_r $opcode, $funct3, $funct2, $rd, $rs1, $rs2, $rs3", - (InsnR4 AnyReg:$rd, uimm7:$opcode, uimm3:$funct3, uimm2:$funct2, + (InsnR4 AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3, uimm2:$funct2, AnyReg:$rs1, AnyReg:$rs2, AnyReg:$rs3)>; def : InstAlias<".insn_r4 $opcode, $funct3, $funct2, $rd, $rs1, $rs2, $rs3", - (InsnR4 AnyReg:$rd, uimm7:$opcode, uimm3:$funct3, uimm2:$funct2, + (InsnR4 AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3, uimm2:$funct2, AnyReg:$rs1, AnyReg:$rs2, AnyReg:$rs3)>; def : InstAlias<".insn_i $opcode, $funct3, $rd, $rs1, $imm12", - (InsnI AnyReg:$rd, uimm7:$opcode, uimm3:$funct3, AnyReg:$rs1, + (InsnI AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1, simm12:$imm12)>; def : InstAlias<".insn_i $opcode, $funct3, $rd, ${imm12}(${rs1})", - (InsnI_Mem AnyReg:$rd, uimm7:$opcode, uimm3:$funct3, + (InsnI_Mem AnyReg:$rd, uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1, simm12:$imm12)>; def : InstAlias<".insn_b $opcode, $funct3, $rs1, $rs2, $imm12", - (InsnB uimm7:$opcode, uimm3:$funct3, AnyReg:$rs1, + (InsnB uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1, AnyReg:$rs2, simm13_lsb0:$imm12)>; // Accept sb as an alias for b. def : InstAlias<".insn_sb $opcode, $funct3, $rs1, $rs2, $imm12", - (InsnB uimm7:$opcode, uimm3:$funct3, AnyReg:$rs1, + (InsnB uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs1, AnyReg:$rs2, simm13_lsb0:$imm12)>; def : InstAlias<".insn_u $opcode, $rd, $imm20", - (InsnU AnyReg:$rd, uimm7:$opcode, uimm20_lui:$imm20)>; + (InsnU AnyReg:$rd, uimm7_opcode:$opcode, uimm20_lui:$imm20)>; def : InstAlias<".insn_j $opcode, $rd, $imm20", - (InsnJ AnyReg:$rd, uimm7:$opcode, simm21_lsb0_jal:$imm20)>; + (InsnJ AnyReg:$rd, uimm7_opcode:$opcode, simm21_lsb0_jal:$imm20)>; // Accept uj as an alias for j. def : InstAlias<".insn_uj $opcode, $rd, $imm20", - (InsnJ AnyReg:$rd, uimm7:$opcode, simm21_lsb0_jal:$imm20)>; + (InsnJ AnyReg:$rd, uimm7_opcode:$opcode, simm21_lsb0_jal:$imm20)>; def : InstAlias<".insn_s $opcode, $funct3, $rs2, ${imm12}(${rs1})", - (InsnS uimm7:$opcode, uimm3:$funct3, AnyReg:$rs2, + (InsnS uimm7_opcode:$opcode, uimm3:$funct3, AnyReg:$rs2, AnyReg:$rs1, simm12:$imm12)>; } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td index 2cd011a02345..d6c31c4804db 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td @@ -26,41 +26,6 @@ def RISCVBuildPairF64 : SDNode<"RISCVISD::BuildPairF64", SDT_RISCVBuildPairF64>; def RISCVSplitF64 : SDNode<"RISCVISD::SplitF64", SDT_RISCVSplitF64>; //===----------------------------------------------------------------------===// -// Instruction Class Templates -//===----------------------------------------------------------------------===// - -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPFMAD_rrr_frm<RISCVOpcode opcode, string opcodestr> - : RVInstR4Frm<0b01, opcode, (outs FPR64:$rd), - (ins FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, frmarg:$funct3), - opcodestr, "$rd, $rs1, $rs2, $rs3, $funct3">; - -class FPFMADDynFrmAlias<FPFMAD_rrr_frm Inst, string OpcodeStr> - : InstAlias<OpcodeStr#" $rd, $rs1, $rs2, $rs3", - (Inst FPR64:$rd, FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, 0b111)>; - -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPALUD_rr<bits<7> funct7, bits<3> funct3, string opcodestr> - : RVInstR<funct7, funct3, OPC_OP_FP, (outs FPR64:$rd), - (ins FPR64:$rs1, FPR64:$rs2), opcodestr, "$rd, $rs1, $rs2">; - -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPALUD_rr_frm<bits<7> funct7, string opcodestr> - : RVInstRFrm<funct7, OPC_OP_FP, (outs FPR64:$rd), - (ins FPR64:$rs1, FPR64:$rs2, frmarg:$funct3), opcodestr, - "$rd, $rs1, $rs2, $funct3">; - -class FPALUDDynFrmAlias<FPALUD_rr_frm Inst, string OpcodeStr> - : InstAlias<OpcodeStr#" $rd, $rs1, $rs2", - (Inst FPR64:$rd, FPR64:$rs1, FPR64:$rs2, 0b111)>; - -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPCmpD_rr<bits<3> funct3, string opcodestr> - : RVInstR<0b1010001, funct3, OPC_OP_FP, (outs GPR:$rd), - (ins FPR64:$rs1, FPR64:$rs2), opcodestr, "$rd, $rs1, $rs2">, - Sched<[WriteFCmp64, ReadFCmp64, ReadFCmp64]>; - -//===----------------------------------------------------------------------===// // Instructions //===----------------------------------------------------------------------===// @@ -81,126 +46,104 @@ def FSD : RVInstS<0b011, OPC_STORE_FP, (outs), "fsd", "$rs2, ${imm12}(${rs1})">, Sched<[WriteFST64, ReadStoreData, ReadFMemBase]>; -def FMADD_D : FPFMAD_rrr_frm<OPC_MADD, "fmadd.d">, - Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>; -def : FPFMADDynFrmAlias<FMADD_D, "fmadd.d">; -def FMSUB_D : FPFMAD_rrr_frm<OPC_MSUB, "fmsub.d">, - Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>; -def : FPFMADDynFrmAlias<FMSUB_D, "fmsub.d">; -def FNMSUB_D : FPFMAD_rrr_frm<OPC_NMSUB, "fnmsub.d">, - Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>; -def : FPFMADDynFrmAlias<FNMSUB_D, "fnmsub.d">; -def FNMADD_D : FPFMAD_rrr_frm<OPC_NMADD, "fnmadd.d">, - Sched<[WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64]>; -def : FPFMADDynFrmAlias<FNMADD_D, "fnmadd.d">; +let SchedRW = [WriteFMA64, ReadFMA64, ReadFMA64, ReadFMA64] in { +def FMADD_D : FPFMA_rrr_frm<OPC_MADD, 0b01, "fmadd.d", FPR64>; +def FMSUB_D : FPFMA_rrr_frm<OPC_MSUB, 0b01, "fmsub.d", FPR64>; +def FNMSUB_D : FPFMA_rrr_frm<OPC_NMSUB, 0b01, "fnmsub.d", FPR64>; +def FNMADD_D : FPFMA_rrr_frm<OPC_NMADD, 0b01, "fnmadd.d", FPR64>; +} + +def : FPFMADynFrmAlias<FMADD_D, "fmadd.d", FPR64>; +def : FPFMADynFrmAlias<FMSUB_D, "fmsub.d", FPR64>; +def : FPFMADynFrmAlias<FNMSUB_D, "fnmsub.d", FPR64>; +def : FPFMADynFrmAlias<FNMADD_D, "fnmadd.d", FPR64>; -def FADD_D : FPALUD_rr_frm<0b0000001, "fadd.d">, +def FADD_D : FPALU_rr_frm<0b0000001, "fadd.d", FPR64>, Sched<[WriteFALU64, ReadFALU64, ReadFALU64]>; -def : FPALUDDynFrmAlias<FADD_D, "fadd.d">; -def FSUB_D : FPALUD_rr_frm<0b0000101, "fsub.d">, +def FSUB_D : FPALU_rr_frm<0b0000101, "fsub.d", FPR64>, Sched<[WriteFALU64, ReadFALU64, ReadFALU64]>; -def : FPALUDDynFrmAlias<FSUB_D, "fsub.d">; -def FMUL_D : FPALUD_rr_frm<0b0001001, "fmul.d">, +def FMUL_D : FPALU_rr_frm<0b0001001, "fmul.d", FPR64>, Sched<[WriteFMul64, ReadFMul64, ReadFMul64]>; -def : FPALUDDynFrmAlias<FMUL_D, "fmul.d">; -def FDIV_D : FPALUD_rr_frm<0b0001101, "fdiv.d">, +def FDIV_D : FPALU_rr_frm<0b0001101, "fdiv.d", FPR64>, Sched<[WriteFDiv64, ReadFDiv64, ReadFDiv64]>; -def : FPALUDDynFrmAlias<FDIV_D, "fdiv.d">; -def FSQRT_D : FPUnaryOp_r_frm<0b0101101, FPR64, FPR64, "fsqrt.d">, - Sched<[WriteFSqrt64, ReadFSqrt64]> { - let rs2 = 0b00000; -} -def : FPUnaryOpDynFrmAlias<FSQRT_D, "fsqrt.d", FPR64, FPR64>; +def : FPALUDynFrmAlias<FADD_D, "fadd.d", FPR64>; +def : FPALUDynFrmAlias<FSUB_D, "fsub.d", FPR64>; +def : FPALUDynFrmAlias<FMUL_D, "fmul.d", FPR64>; +def : FPALUDynFrmAlias<FDIV_D, "fdiv.d", FPR64>; -def FSGNJ_D : FPALUD_rr<0b0010001, 0b000, "fsgnj.d">, - Sched<[WriteFSGNJ64, ReadFSGNJ64, ReadFSGNJ64]>; -def FSGNJN_D : FPALUD_rr<0b0010001, 0b001, "fsgnjn.d">, - Sched<[WriteFSGNJ64, ReadFSGNJ64, ReadFSGNJ64]>; -def FSGNJX_D : FPALUD_rr<0b0010001, 0b010, "fsgnjx.d">, - Sched<[WriteFSGNJ64, ReadFSGNJ64, ReadFSGNJ64]>; -def FMIN_D : FPALUD_rr<0b0010101, 0b000, "fmin.d">, - Sched<[WriteFMinMax64, ReadFMinMax64, ReadFMinMax64]>; -def FMAX_D : FPALUD_rr<0b0010101, 0b001, "fmax.d">, - Sched<[WriteFMinMax64, ReadFMinMax64, ReadFMinMax64]>; +def FSQRT_D : FPUnaryOp_r_frm<0b0101101, 0b00000, FPR64, FPR64, "fsqrt.d">, + Sched<[WriteFSqrt64, ReadFSqrt64]>; +def : FPUnaryOpDynFrmAlias<FSQRT_D, "fsqrt.d", FPR64, FPR64>; -def FCVT_S_D : FPUnaryOp_r_frm<0b0100000, FPR32, FPR64, "fcvt.s.d">, - Sched<[WriteFCvtF64ToF32, ReadFCvtF64ToF32]> { - let rs2 = 0b00001; +let SchedRW = [WriteFSGNJ64, ReadFSGNJ64, ReadFSGNJ64], + mayRaiseFPException = 0 in { +def FSGNJ_D : FPALU_rr<0b0010001, 0b000, "fsgnj.d", FPR64>; +def FSGNJN_D : FPALU_rr<0b0010001, 0b001, "fsgnjn.d", FPR64>; +def FSGNJX_D : FPALU_rr<0b0010001, 0b010, "fsgnjx.d", FPR64>; } -def : FPUnaryOpDynFrmAlias<FCVT_S_D, "fcvt.s.d", FPR32, FPR64>; -def FCVT_D_S : FPUnaryOp_r<0b0100001, 0b000, FPR64, FPR32, "fcvt.d.s">, - Sched<[WriteFCvtF32ToF64, ReadFCvtF32ToF64]> { - let rs2 = 0b00000; +let SchedRW = [WriteFMinMax64, ReadFMinMax64, ReadFMinMax64] in { +def FMIN_D : FPALU_rr<0b0010101, 0b000, "fmin.d", FPR64>; +def FMAX_D : FPALU_rr<0b0010101, 0b001, "fmax.d", FPR64>; } -def FEQ_D : FPCmpD_rr<0b010, "feq.d">; -def FLT_D : FPCmpD_rr<0b001, "flt.d">; -def FLE_D : FPCmpD_rr<0b000, "fle.d">; +def FCVT_S_D : FPUnaryOp_r_frm<0b0100000, 0b00001, FPR32, FPR64, "fcvt.s.d">, + Sched<[WriteFCvtF64ToF32, ReadFCvtF64ToF32]>; +def : FPUnaryOpDynFrmAlias<FCVT_S_D, "fcvt.s.d", FPR32, FPR64>; + +def FCVT_D_S : FPUnaryOp_r<0b0100001, 0b00000, 0b000, FPR64, FPR32, "fcvt.d.s">, + Sched<[WriteFCvtF32ToF64, ReadFCvtF32ToF64]>; -def FCLASS_D : FPUnaryOp_r<0b1110001, 0b001, GPR, FPR64, "fclass.d">, - Sched<[WriteFClass64, ReadFClass64]> { - let rs2 = 0b00000; +let SchedRW = [WriteFCmp64, ReadFCmp64, ReadFCmp64] in { +def FEQ_D : FPCmp_rr<0b1010001, 0b010, "feq.d", FPR64>; +def FLT_D : FPCmp_rr<0b1010001, 0b001, "flt.d", FPR64>; +def FLE_D : FPCmp_rr<0b1010001, 0b000, "fle.d", FPR64>; } -def FCVT_W_D : FPUnaryOp_r_frm<0b1100001, GPR, FPR64, "fcvt.w.d">, - Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]> { - let rs2 = 0b00000; -} +let mayRaiseFPException = 0 in +def FCLASS_D : FPUnaryOp_r<0b1110001, 0b00000, 0b001, GPR, FPR64, "fclass.d">, + Sched<[WriteFClass64, ReadFClass64]>; + +def FCVT_W_D : FPUnaryOp_r_frm<0b1100001, 0b00000, GPR, FPR64, "fcvt.w.d">, + Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]>; def : FPUnaryOpDynFrmAlias<FCVT_W_D, "fcvt.w.d", GPR, FPR64>; -def FCVT_WU_D : FPUnaryOp_r_frm<0b1100001, GPR, FPR64, "fcvt.wu.d">, - Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]> { - let rs2 = 0b00001; -} +def FCVT_WU_D : FPUnaryOp_r_frm<0b1100001, 0b00001, GPR, FPR64, "fcvt.wu.d">, + Sched<[WriteFCvtF64ToI32, ReadFCvtF64ToI32]>; def : FPUnaryOpDynFrmAlias<FCVT_WU_D, "fcvt.wu.d", GPR, FPR64>; -def FCVT_D_W : FPUnaryOp_r<0b1101001, 0b000, FPR64, GPR, "fcvt.d.w">, - Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]> { - let rs2 = 0b00000; -} +def FCVT_D_W : FPUnaryOp_r<0b1101001, 0b00000, 0b000, FPR64, GPR, "fcvt.d.w">, + Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]>; -def FCVT_D_WU : FPUnaryOp_r<0b1101001, 0b000, FPR64, GPR, "fcvt.d.wu">, - Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]> { - let rs2 = 0b00001; -} +def FCVT_D_WU : FPUnaryOp_r<0b1101001, 0b00001, 0b000, FPR64, GPR, "fcvt.d.wu">, + Sched<[WriteFCvtI32ToF64, ReadFCvtI32ToF64]>; } // Predicates = [HasStdExtD] let Predicates = [HasStdExtD, IsRV64] in { -def FCVT_L_D : FPUnaryOp_r_frm<0b1100001, GPR, FPR64, "fcvt.l.d">, - Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]> { - let rs2 = 0b00010; -} +def FCVT_L_D : FPUnaryOp_r_frm<0b1100001, 0b00010, GPR, FPR64, "fcvt.l.d">, + Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]>; def : FPUnaryOpDynFrmAlias<FCVT_L_D, "fcvt.l.d", GPR, FPR64>; -def FCVT_LU_D : FPUnaryOp_r_frm<0b1100001, GPR, FPR64, "fcvt.lu.d">, - Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]> { - let rs2 = 0b00011; -} +def FCVT_LU_D : FPUnaryOp_r_frm<0b1100001, 0b00011, GPR, FPR64, "fcvt.lu.d">, + Sched<[WriteFCvtF64ToI64, ReadFCvtF64ToI64]>; def : FPUnaryOpDynFrmAlias<FCVT_LU_D, "fcvt.lu.d", GPR, FPR64>; -def FMV_X_D : FPUnaryOp_r<0b1110001, 0b000, GPR, FPR64, "fmv.x.d">, - Sched<[WriteFMovF64ToI64, ReadFMovF64ToI64]> { - let rs2 = 0b00000; -} +let mayRaiseFPException = 0 in +def FMV_X_D : FPUnaryOp_r<0b1110001, 0b00000, 0b000, GPR, FPR64, "fmv.x.d">, + Sched<[WriteFMovF64ToI64, ReadFMovF64ToI64]>; -def FCVT_D_L : FPUnaryOp_r_frm<0b1101001, FPR64, GPR, "fcvt.d.l">, - Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]> { - let rs2 = 0b00010; -} +def FCVT_D_L : FPUnaryOp_r_frm<0b1101001, 0b00010, FPR64, GPR, "fcvt.d.l">, + Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]>; def : FPUnaryOpDynFrmAlias<FCVT_D_L, "fcvt.d.l", FPR64, GPR>; -def FCVT_D_LU : FPUnaryOp_r_frm<0b1101001, FPR64, GPR, "fcvt.d.lu">, - Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]> { - let rs2 = 0b00011; -} +def FCVT_D_LU : FPUnaryOp_r_frm<0b1101001, 0b00011, FPR64, GPR, "fcvt.d.lu">, + Sched<[WriteFCvtI64ToF64, ReadFCvtI64ToF64]>; def : FPUnaryOpDynFrmAlias<FCVT_D_LU, "fcvt.d.lu", FPR64, GPR>; -def FMV_D_X : FPUnaryOp_r<0b1111001, 0b000, FPR64, GPR, "fmv.d.x">, - Sched<[WriteFMovI64ToF64, ReadFMovI64ToF64]> { - let rs2 = 0b00000; -} +let mayRaiseFPException = 0 in +def FMV_D_X : FPUnaryOp_r<0b1111001, 0b00000, 0b000, FPR64, GPR, "fmv.d.x">, + Sched<[WriteFMovI64ToF64, ReadFMovI64ToF64]>; } // Predicates = [HasStdExtD, IsRV64] //===----------------------------------------------------------------------===// @@ -241,20 +184,20 @@ let Predicates = [HasStdExtD] in { /// Float conversion operations // f64 -> f32, f32 -> f64 -def : Pat<(fpround FPR64:$rs1), (FCVT_S_D FPR64:$rs1, 0b111)>; -def : Pat<(fpextend FPR32:$rs1), (FCVT_D_S FPR32:$rs1)>; +def : Pat<(any_fpround FPR64:$rs1), (FCVT_S_D FPR64:$rs1, 0b111)>; +def : Pat<(any_fpextend FPR32:$rs1), (FCVT_D_S FPR32:$rs1)>; // [u]int<->double conversion patterns must be gated on IsRV32 or IsRV64, so // are defined later. /// Float arithmetic operations -def : PatFpr64Fpr64DynFrm<fadd, FADD_D>; -def : PatFpr64Fpr64DynFrm<fsub, FSUB_D>; -def : PatFpr64Fpr64DynFrm<fmul, FMUL_D>; -def : PatFpr64Fpr64DynFrm<fdiv, FDIV_D>; +def : PatFpr64Fpr64DynFrm<any_fadd, FADD_D>; +def : PatFpr64Fpr64DynFrm<any_fsub, FSUB_D>; +def : PatFpr64Fpr64DynFrm<any_fmul, FMUL_D>; +def : PatFpr64Fpr64DynFrm<any_fdiv, FDIV_D>; -def : Pat<(fsqrt FPR64:$rs1), (FSQRT_D FPR64:$rs1, 0b111)>; +def : Pat<(any_fsqrt FPR64:$rs1), (FSQRT_D FPR64:$rs1, 0b111)>; def : Pat<(fneg FPR64:$rs1), (FSGNJN_D $rs1, $rs1)>; def : Pat<(fabs FPR64:$rs1), (FSGNJX_D $rs1, $rs1)>; @@ -266,19 +209,19 @@ def : Pat<(fcopysign FPR32:$rs1, FPR64:$rs2), (FSGNJ_S $rs1, (FCVT_S_D $rs2, 0b111))>; // fmadd: rs1 * rs2 + rs3 -def : Pat<(fma FPR64:$rs1, FPR64:$rs2, FPR64:$rs3), +def : Pat<(any_fma FPR64:$rs1, FPR64:$rs2, FPR64:$rs3), (FMADD_D $rs1, $rs2, $rs3, 0b111)>; // fmsub: rs1 * rs2 - rs3 -def : Pat<(fma FPR64:$rs1, FPR64:$rs2, (fneg FPR64:$rs3)), +def : Pat<(any_fma FPR64:$rs1, FPR64:$rs2, (fneg FPR64:$rs3)), (FMSUB_D FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, 0b111)>; // fnmsub: -rs1 * rs2 + rs3 -def : Pat<(fma (fneg FPR64:$rs1), FPR64:$rs2, FPR64:$rs3), +def : Pat<(any_fma (fneg FPR64:$rs1), FPR64:$rs2, FPR64:$rs3), (FNMSUB_D FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, 0b111)>; // fnmadd: -rs1 * rs2 - rs3 -def : Pat<(fma (fneg FPR64:$rs1), FPR64:$rs2, (fneg FPR64:$rs3)), +def : Pat<(any_fma (fneg FPR64:$rs1), FPR64:$rs2, (fneg FPR64:$rs3)), (FNMADD_D FPR64:$rs1, FPR64:$rs2, FPR64:$rs3, 0b111)>; // The ratified 20191213 ISA spec defines fmin and fmax in a way that matches @@ -328,8 +271,8 @@ let Predicates = [HasStdExtD, IsRV32] in { def : Pat<(f64 (fpimm0)), (FCVT_D_W (i32 X0))>; // double->[u]int. Round-to-zero must be used. -def : Pat<(i32 (fp_to_sint FPR64:$rs1)), (FCVT_W_D FPR64:$rs1, 0b001)>; -def : Pat<(i32 (fp_to_uint FPR64:$rs1)), (FCVT_WU_D FPR64:$rs1, 0b001)>; +def : Pat<(i32 (any_fp_to_sint FPR64:$rs1)), (FCVT_W_D FPR64:$rs1, 0b001)>; +def : Pat<(i32 (any_fp_to_uint FPR64:$rs1)), (FCVT_WU_D FPR64:$rs1, 0b001)>; // Saturating double->[u]int32. def : Pat<(i32 (riscv_fcvt_x_rtz FPR64:$rs1)), (FCVT_W_D $rs1, 0b001)>; @@ -342,8 +285,8 @@ def : Pat<(i32 (lrint FPR64:$rs1)), (FCVT_W_D $rs1, 0b111)>; def : Pat<(i32 (lround FPR64:$rs1)), (FCVT_W_D $rs1, 0b100)>; // [u]int->double. -def : Pat<(sint_to_fp (i32 GPR:$rs1)), (FCVT_D_W GPR:$rs1)>; -def : Pat<(uint_to_fp (i32 GPR:$rs1)), (FCVT_D_WU GPR:$rs1)>; +def : Pat<(any_sint_to_fp (i32 GPR:$rs1)), (FCVT_D_W GPR:$rs1)>; +def : Pat<(any_uint_to_fp (i32 GPR:$rs1)), (FCVT_D_WU GPR:$rs1)>; } // Predicates = [HasStdExtD, IsRV32] let Predicates = [HasStdExtD, IsRV64] in { @@ -358,20 +301,20 @@ def : Pat<(i64 (bitconvert FPR64:$rs1)), (FMV_X_D FPR64:$rs1)>; // Use target specific isd nodes to help us remember the result is sign // extended. Matching sext_inreg+fptoui/fptosi may cause the conversion to be // duplicated if it has another user that didn't need the sign_extend. -def : Pat<(riscv_fcvt_w_rtz_rv64 FPR64:$rs1), (FCVT_W_D $rs1, 0b001)>; -def : Pat<(riscv_fcvt_wu_rtz_rv64 FPR64:$rs1), (FCVT_WU_D $rs1, 0b001)>; +def : Pat<(riscv_any_fcvt_w_rtz_rv64 FPR64:$rs1), (FCVT_W_D $rs1, 0b001)>; +def : Pat<(riscv_any_fcvt_wu_rtz_rv64 FPR64:$rs1), (FCVT_WU_D $rs1, 0b001)>; // [u]int32->fp -def : Pat<(sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_D_W $rs1)>; -def : Pat<(uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_D_WU $rs1)>; +def : Pat<(any_sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_D_W $rs1)>; +def : Pat<(any_uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_D_WU $rs1)>; // Saturating double->[u]int64. def : Pat<(i64 (riscv_fcvt_x_rtz FPR64:$rs1)), (FCVT_L_D $rs1, 0b001)>; def : Pat<(i64 (riscv_fcvt_xu_rtz FPR64:$rs1)), (FCVT_LU_D $rs1, 0b001)>; // double->[u]int64. Round-to-zero must be used. -def : Pat<(i64 (fp_to_sint FPR64:$rs1)), (FCVT_L_D FPR64:$rs1, 0b001)>; -def : Pat<(i64 (fp_to_uint FPR64:$rs1)), (FCVT_LU_D FPR64:$rs1, 0b001)>; +def : Pat<(i64 (any_fp_to_sint FPR64:$rs1)), (FCVT_L_D FPR64:$rs1, 0b001)>; +def : Pat<(i64 (any_fp_to_uint FPR64:$rs1)), (FCVT_LU_D FPR64:$rs1, 0b001)>; // double->int64 with current rounding mode. def : Pat<(i64 (lrint FPR64:$rs1)), (FCVT_L_D $rs1, 0b111)>; @@ -382,6 +325,6 @@ def : Pat<(i64 (lround FPR64:$rs1)), (FCVT_L_D $rs1, 0b100)>; def : Pat<(i64 (llround FPR64:$rs1)), (FCVT_L_D $rs1, 0b100)>; // [u]int64->fp. Match GCC and default to using dynamic rounding mode. -def : Pat<(sint_to_fp (i64 GPR:$rs1)), (FCVT_D_L GPR:$rs1, 0b111)>; -def : Pat<(uint_to_fp (i64 GPR:$rs1)), (FCVT_D_LU GPR:$rs1, 0b111)>; +def : Pat<(any_sint_to_fp (i64 GPR:$rs1)), (FCVT_D_L GPR:$rs1, 0b111)>; +def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_D_LU GPR:$rs1, 0b111)>; } // Predicates = [HasStdExtD, IsRV64] diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td index 3400c3be52bf..bb45ed859442 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td @@ -19,9 +19,9 @@ def SDT_RISCVFMV_W_X_RV64 : SDTypeProfile<1, 1, [SDTCisVT<0, f32>, SDTCisVT<1, i64>]>; def SDT_RISCVFMV_X_ANYEXTW_RV64 : SDTypeProfile<1, 1, [SDTCisVT<0, i64>, SDTCisVT<1, f32>]>; -def STD_RISCVFCVT_W_RV64 +def SDT_RISCVFCVT_W_RV64 : SDTypeProfile<1, 1, [SDTCisVT<0, i64>, SDTCisFP<1>]>; -def STD_RISCVFCVT_X +def SDT_RISCVFCVT_X : SDTypeProfile<1, 1, [SDTCisVT<0, XLenVT>, SDTCisFP<1>]>; def riscv_fmv_w_x_rv64 @@ -29,13 +29,27 @@ def riscv_fmv_w_x_rv64 def riscv_fmv_x_anyextw_rv64 : SDNode<"RISCVISD::FMV_X_ANYEXTW_RV64", SDT_RISCVFMV_X_ANYEXTW_RV64>; def riscv_fcvt_w_rtz_rv64 - : SDNode<"RISCVISD::FCVT_W_RTZ_RV64", STD_RISCVFCVT_W_RV64>; + : SDNode<"RISCVISD::FCVT_W_RTZ_RV64", SDT_RISCVFCVT_W_RV64>; def riscv_fcvt_wu_rtz_rv64 - : SDNode<"RISCVISD::FCVT_WU_RTZ_RV64", STD_RISCVFCVT_W_RV64>; + : SDNode<"RISCVISD::FCVT_WU_RTZ_RV64", SDT_RISCVFCVT_W_RV64>; def riscv_fcvt_x_rtz - : SDNode<"RISCVISD::FCVT_X_RTZ", STD_RISCVFCVT_X>; + : SDNode<"RISCVISD::FCVT_X_RTZ", SDT_RISCVFCVT_X>; def riscv_fcvt_xu_rtz - : SDNode<"RISCVISD::FCVT_XU_RTZ", STD_RISCVFCVT_X>; + : SDNode<"RISCVISD::FCVT_XU_RTZ", SDT_RISCVFCVT_X>; + +def riscv_strict_fcvt_w_rtz_rv64 + : SDNode<"RISCVISD::STRICT_FCVT_W_RTZ_RV64", SDT_RISCVFCVT_W_RV64, + [SDNPHasChain]>; +def riscv_strict_fcvt_wu_rtz_rv64 + : SDNode<"RISCVISD::STRICT_FCVT_WU_RTZ_RV64", SDT_RISCVFCVT_W_RV64, + [SDNPHasChain]>; + +def riscv_any_fcvt_w_rtz_rv64 : PatFrags<(ops node:$src), + [(riscv_strict_fcvt_w_rtz_rv64 node:$src), + (riscv_fcvt_w_rtz_rv64 node:$src)]>; +def riscv_any_fcvt_wu_rtz_rv64 : PatFrags<(ops node:$src), + [(riscv_strict_fcvt_wu_rtz_rv64 node:$src), + (riscv_fcvt_wu_rtz_rv64 node:$src)]>; //===----------------------------------------------------------------------===// // Operand and SDNode transformation definitions. @@ -59,54 +73,65 @@ def frmarg : Operand<XLenVT> { // Instruction class templates //===----------------------------------------------------------------------===// -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPFMAS_rrr_frm<RISCVOpcode opcode, string opcodestr> - : RVInstR4Frm<0b00, opcode, (outs FPR32:$rd), - (ins FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, frmarg:$funct3), - opcodestr, "$rd, $rs1, $rs2, $rs3, $funct3">; +let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1, + UseNamedOperandTable = 1, hasPostISelHook = 1 in +class FPFMA_rrr_frm<RISCVOpcode opcode, bits<2> funct2, string opcodestr, + RegisterClass rty> + : RVInstR4Frm<funct2, opcode, (outs rty:$rd), + (ins rty:$rs1, rty:$rs2, rty:$rs3, frmarg:$frm), + opcodestr, "$rd, $rs1, $rs2, $rs3, $frm">; -class FPFMASDynFrmAlias<FPFMAS_rrr_frm Inst, string OpcodeStr> +class FPFMADynFrmAlias<FPFMA_rrr_frm Inst, string OpcodeStr, + RegisterClass rty> : InstAlias<OpcodeStr#" $rd, $rs1, $rs2, $rs3", - (Inst FPR32:$rd, FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, 0b111)>; + (Inst rty:$rd, rty:$rs1, rty:$rs2, rty:$rs3, 0b111)>; -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPALUS_rr<bits<7> funct7, bits<3> funct3, string opcodestr> - : RVInstR<funct7, funct3, OPC_OP_FP, (outs FPR32:$rd), - (ins FPR32:$rs1, FPR32:$rs2), opcodestr, "$rd, $rs1, $rs2">; +let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1 in +class FPALU_rr<bits<7> funct7, bits<3> funct3, string opcodestr, + RegisterClass rty> + : RVInstR<funct7, funct3, OPC_OP_FP, (outs rty:$rd), + (ins rty:$rs1, rty:$rs2), opcodestr, "$rd, $rs1, $rs2">; -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPALUS_rr_frm<bits<7> funct7, string opcodestr> - : RVInstRFrm<funct7, OPC_OP_FP, (outs FPR32:$rd), - (ins FPR32:$rs1, FPR32:$rs2, frmarg:$funct3), opcodestr, - "$rd, $rs1, $rs2, $funct3">; +let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1, + UseNamedOperandTable = 1, hasPostISelHook = 1 in +class FPALU_rr_frm<bits<7> funct7, string opcodestr, RegisterClass rty> + : RVInstRFrm<funct7, OPC_OP_FP, (outs rty:$rd), + (ins rty:$rs1, rty:$rs2, frmarg:$frm), opcodestr, + "$rd, $rs1, $rs2, $frm">; -class FPALUSDynFrmAlias<FPALUS_rr_frm Inst, string OpcodeStr> +class FPALUDynFrmAlias<FPALU_rr_frm Inst, string OpcodeStr, + RegisterClass rty> : InstAlias<OpcodeStr#" $rd, $rs1, $rs2", - (Inst FPR32:$rd, FPR32:$rs1, FPR32:$rs2, 0b111)>; + (Inst rty:$rd, rty:$rs1, rty:$rs2, 0b111)>; -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPUnaryOp_r<bits<7> funct7, bits<3> funct3, RegisterClass rdty, - RegisterClass rs1ty, string opcodestr> +let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1 in +class FPUnaryOp_r<bits<7> funct7, bits<5> rs2val, bits<3> funct3, + RegisterClass rdty, RegisterClass rs1ty, string opcodestr> : RVInstR<funct7, funct3, OPC_OP_FP, (outs rdty:$rd), (ins rs1ty:$rs1), - opcodestr, "$rd, $rs1">; + opcodestr, "$rd, $rs1"> { + let rs2 = rs2val; +} -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPUnaryOp_r_frm<bits<7> funct7, RegisterClass rdty, RegisterClass rs1ty, - string opcodestr> +let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1, + UseNamedOperandTable = 1, hasPostISelHook = 1 in +class FPUnaryOp_r_frm<bits<7> funct7, bits<5> rs2val, RegisterClass rdty, + RegisterClass rs1ty, string opcodestr> : RVInstRFrm<funct7, OPC_OP_FP, (outs rdty:$rd), - (ins rs1ty:$rs1, frmarg:$funct3), opcodestr, - "$rd, $rs1, $funct3">; + (ins rs1ty:$rs1, frmarg:$frm), opcodestr, + "$rd, $rs1, $frm"> { + let rs2 = rs2val; +} class FPUnaryOpDynFrmAlias<FPUnaryOp_r_frm Inst, string OpcodeStr, RegisterClass rdty, RegisterClass rs1ty> : InstAlias<OpcodeStr#" $rd, $rs1", (Inst rdty:$rd, rs1ty:$rs1, 0b111)>; -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPCmpS_rr<bits<3> funct3, string opcodestr> - : RVInstR<0b1010000, funct3, OPC_OP_FP, (outs GPR:$rd), - (ins FPR32:$rs1, FPR32:$rs2), opcodestr, "$rd, $rs1, $rs2">, - Sched<[WriteFCmp32, ReadFCmp32, ReadFCmp32]>; +let hasSideEffects = 0, mayLoad = 0, mayStore = 0, mayRaiseFPException = 1 in +class FPCmp_rr<bits<7> funct7, bits<3> funct3, string opcodestr, + RegisterClass rty> + : RVInstR<funct7, funct3, OPC_OP_FP, (outs GPR:$rd), + (ins rty:$rs1, rty:$rs2), opcodestr, "$rd, $rs1, $rs2">; //===----------------------------------------------------------------------===// // Instructions @@ -128,116 +153,98 @@ def FSW : RVInstS<0b010, OPC_STORE_FP, (outs), "fsw", "$rs2, ${imm12}(${rs1})">, Sched<[WriteFST32, ReadStoreData, ReadFMemBase]>; -def FMADD_S : FPFMAS_rrr_frm<OPC_MADD, "fmadd.s">, - Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>; -def : FPFMASDynFrmAlias<FMADD_S, "fmadd.s">; -def FMSUB_S : FPFMAS_rrr_frm<OPC_MSUB, "fmsub.s">, - Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>; -def : FPFMASDynFrmAlias<FMSUB_S, "fmsub.s">; -def FNMSUB_S : FPFMAS_rrr_frm<OPC_NMSUB, "fnmsub.s">, - Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>; -def : FPFMASDynFrmAlias<FNMSUB_S, "fnmsub.s">; -def FNMADD_S : FPFMAS_rrr_frm<OPC_NMADD, "fnmadd.s">, - Sched<[WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32]>; -def : FPFMASDynFrmAlias<FNMADD_S, "fnmadd.s">; +let SchedRW = [WriteFMA32, ReadFMA32, ReadFMA32, ReadFMA32] in { +def FMADD_S : FPFMA_rrr_frm<OPC_MADD, 0b00, "fmadd.s", FPR32>; +def FMSUB_S : FPFMA_rrr_frm<OPC_MSUB, 0b00, "fmsub.s", FPR32>; +def FNMSUB_S : FPFMA_rrr_frm<OPC_NMSUB, 0b00, "fnmsub.s", FPR32>; +def FNMADD_S : FPFMA_rrr_frm<OPC_NMADD, 0b00, "fnmadd.s", FPR32>; +} + +def : FPFMADynFrmAlias<FMADD_S, "fmadd.s", FPR32>; +def : FPFMADynFrmAlias<FMSUB_S, "fmsub.s", FPR32>; +def : FPFMADynFrmAlias<FNMSUB_S, "fnmsub.s", FPR32>; +def : FPFMADynFrmAlias<FNMADD_S, "fnmadd.s", FPR32>; -def FADD_S : FPALUS_rr_frm<0b0000000, "fadd.s">, +def FADD_S : FPALU_rr_frm<0b0000000, "fadd.s", FPR32>, Sched<[WriteFALU32, ReadFALU32, ReadFALU32]>; -def : FPALUSDynFrmAlias<FADD_S, "fadd.s">; -def FSUB_S : FPALUS_rr_frm<0b0000100, "fsub.s">, +def FSUB_S : FPALU_rr_frm<0b0000100, "fsub.s", FPR32>, Sched<[WriteFALU32, ReadFALU32, ReadFALU32]>; -def : FPALUSDynFrmAlias<FSUB_S, "fsub.s">; -def FMUL_S : FPALUS_rr_frm<0b0001000, "fmul.s">, +def FMUL_S : FPALU_rr_frm<0b0001000, "fmul.s", FPR32>, Sched<[WriteFMul32, ReadFMul32, ReadFMul32]>; -def : FPALUSDynFrmAlias<FMUL_S, "fmul.s">; -def FDIV_S : FPALUS_rr_frm<0b0001100, "fdiv.s">, +def FDIV_S : FPALU_rr_frm<0b0001100, "fdiv.s", FPR32>, Sched<[WriteFDiv32, ReadFDiv32, ReadFDiv32]>; -def : FPALUSDynFrmAlias<FDIV_S, "fdiv.s">; -def FSQRT_S : FPUnaryOp_r_frm<0b0101100, FPR32, FPR32, "fsqrt.s">, - Sched<[WriteFSqrt32, ReadFSqrt32]> { - let rs2 = 0b00000; -} +def : FPALUDynFrmAlias<FADD_S, "fadd.s", FPR32>; +def : FPALUDynFrmAlias<FSUB_S, "fsub.s", FPR32>; +def : FPALUDynFrmAlias<FMUL_S, "fmul.s", FPR32>; +def : FPALUDynFrmAlias<FDIV_S, "fdiv.s", FPR32>; + +def FSQRT_S : FPUnaryOp_r_frm<0b0101100, 0b00000, FPR32, FPR32, "fsqrt.s">, + Sched<[WriteFSqrt32, ReadFSqrt32]>; def : FPUnaryOpDynFrmAlias<FSQRT_S, "fsqrt.s", FPR32, FPR32>; -def FSGNJ_S : FPALUS_rr<0b0010000, 0b000, "fsgnj.s">, - Sched<[WriteFSGNJ32, ReadFSGNJ32, ReadFSGNJ32]>; -def FSGNJN_S : FPALUS_rr<0b0010000, 0b001, "fsgnjn.s">, - Sched<[WriteFSGNJ32, ReadFSGNJ32, ReadFSGNJ32]>; -def FSGNJX_S : FPALUS_rr<0b0010000, 0b010, "fsgnjx.s">, - Sched<[WriteFSGNJ32, ReadFSGNJ32, ReadFSGNJ32]>; -def FMIN_S : FPALUS_rr<0b0010100, 0b000, "fmin.s">, - Sched<[WriteFMinMax32, ReadFMinMax32, ReadFMinMax32]>; -def FMAX_S : FPALUS_rr<0b0010100, 0b001, "fmax.s">, - Sched<[WriteFMinMax32, ReadFMinMax32, ReadFMinMax32]>; +let SchedRW = [WriteFSGNJ32, ReadFSGNJ32, ReadFSGNJ32], + mayRaiseFPException = 0 in { +def FSGNJ_S : FPALU_rr<0b0010000, 0b000, "fsgnj.s", FPR32>; +def FSGNJN_S : FPALU_rr<0b0010000, 0b001, "fsgnjn.s", FPR32>; +def FSGNJX_S : FPALU_rr<0b0010000, 0b010, "fsgnjx.s", FPR32>; +} -def FCVT_W_S : FPUnaryOp_r_frm<0b1100000, GPR, FPR32, "fcvt.w.s">, - Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]> { - let rs2 = 0b00000; +let SchedRW = [WriteFMinMax32, ReadFMinMax32, ReadFMinMax32] in { +def FMIN_S : FPALU_rr<0b0010100, 0b000, "fmin.s", FPR32>; +def FMAX_S : FPALU_rr<0b0010100, 0b001, "fmax.s", FPR32>; } + +def FCVT_W_S : FPUnaryOp_r_frm<0b1100000, 0b00000, GPR, FPR32, "fcvt.w.s">, + Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]>; def : FPUnaryOpDynFrmAlias<FCVT_W_S, "fcvt.w.s", GPR, FPR32>; -def FCVT_WU_S : FPUnaryOp_r_frm<0b1100000, GPR, FPR32, "fcvt.wu.s">, - Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]> { - let rs2 = 0b00001; -} +def FCVT_WU_S : FPUnaryOp_r_frm<0b1100000, 0b00001, GPR, FPR32, "fcvt.wu.s">, + Sched<[WriteFCvtF32ToI32, ReadFCvtF32ToI32]>; def : FPUnaryOpDynFrmAlias<FCVT_WU_S, "fcvt.wu.s", GPR, FPR32>; -def FMV_X_W : FPUnaryOp_r<0b1110000, 0b000, GPR, FPR32, "fmv.x.w">, - Sched<[WriteFMovF32ToI32, ReadFMovF32ToI32]> { - let rs2 = 0b00000; -} - -def FEQ_S : FPCmpS_rr<0b010, "feq.s">; -def FLT_S : FPCmpS_rr<0b001, "flt.s">; -def FLE_S : FPCmpS_rr<0b000, "fle.s">; +let mayRaiseFPException = 0 in +def FMV_X_W : FPUnaryOp_r<0b1110000, 0b00000, 0b000, GPR, FPR32, "fmv.x.w">, + Sched<[WriteFMovF32ToI32, ReadFMovF32ToI32]>; -def FCLASS_S : FPUnaryOp_r<0b1110000, 0b001, GPR, FPR32, "fclass.s">, - Sched<[WriteFClass32, ReadFClass32]> { - let rs2 = 0b00000; +let SchedRW = [WriteFCmp32, ReadFCmp32, ReadFCmp32] in { +def FEQ_S : FPCmp_rr<0b1010000, 0b010, "feq.s", FPR32>; +def FLT_S : FPCmp_rr<0b1010000, 0b001, "flt.s", FPR32>; +def FLE_S : FPCmp_rr<0b1010000, 0b000, "fle.s", FPR32>; } -def FCVT_S_W : FPUnaryOp_r_frm<0b1101000, FPR32, GPR, "fcvt.s.w">, - Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]> { - let rs2 = 0b00000; -} +let mayRaiseFPException = 0 in +def FCLASS_S : FPUnaryOp_r<0b1110000, 0b00000, 0b001, GPR, FPR32, "fclass.s">, + Sched<[WriteFClass32, ReadFClass32]>; + +def FCVT_S_W : FPUnaryOp_r_frm<0b1101000, 0b00000, FPR32, GPR, "fcvt.s.w">, + Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]>; def : FPUnaryOpDynFrmAlias<FCVT_S_W, "fcvt.s.w", FPR32, GPR>; -def FCVT_S_WU : FPUnaryOp_r_frm<0b1101000, FPR32, GPR, "fcvt.s.wu">, - Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]> { - let rs2 = 0b00001; -} +def FCVT_S_WU : FPUnaryOp_r_frm<0b1101000, 0b00001, FPR32, GPR, "fcvt.s.wu">, + Sched<[WriteFCvtI32ToF32, ReadFCvtI32ToF32]>; def : FPUnaryOpDynFrmAlias<FCVT_S_WU, "fcvt.s.wu", FPR32, GPR>; -def FMV_W_X : FPUnaryOp_r<0b1111000, 0b000, FPR32, GPR, "fmv.w.x">, - Sched<[WriteFMovI32ToF32, ReadFMovI32ToF32]> { - let rs2 = 0b00000; -} +let mayRaiseFPException = 0 in +def FMV_W_X : FPUnaryOp_r<0b1111000, 0b00000, 0b000, FPR32, GPR, "fmv.w.x">, + Sched<[WriteFMovI32ToF32, ReadFMovI32ToF32]>; } // Predicates = [HasStdExtF] let Predicates = [HasStdExtF, IsRV64] in { -def FCVT_L_S : FPUnaryOp_r_frm<0b1100000, GPR, FPR32, "fcvt.l.s">, - Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]> { - let rs2 = 0b00010; -} +def FCVT_L_S : FPUnaryOp_r_frm<0b1100000, 0b00010, GPR, FPR32, "fcvt.l.s">, + Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]>; def : FPUnaryOpDynFrmAlias<FCVT_L_S, "fcvt.l.s", GPR, FPR32>; -def FCVT_LU_S : FPUnaryOp_r_frm<0b1100000, GPR, FPR32, "fcvt.lu.s">, - Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]> { - let rs2 = 0b00011; -} +def FCVT_LU_S : FPUnaryOp_r_frm<0b1100000, 0b00011, GPR, FPR32, "fcvt.lu.s">, + Sched<[WriteFCvtF32ToI64, ReadFCvtF32ToI64]>; def : FPUnaryOpDynFrmAlias<FCVT_LU_S, "fcvt.lu.s", GPR, FPR32>; -def FCVT_S_L : FPUnaryOp_r_frm<0b1101000, FPR32, GPR, "fcvt.s.l">, - Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]> { - let rs2 = 0b00010; -} +def FCVT_S_L : FPUnaryOp_r_frm<0b1101000, 0b00010, FPR32, GPR, "fcvt.s.l">, + Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]>; def : FPUnaryOpDynFrmAlias<FCVT_S_L, "fcvt.s.l", FPR32, GPR>; -def FCVT_S_LU : FPUnaryOp_r_frm<0b1101000, FPR32, GPR, "fcvt.s.lu">, - Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]> { - let rs2 = 0b00011; -} +def FCVT_S_LU : FPUnaryOp_r_frm<0b1101000, 0b00011, FPR32, GPR, "fcvt.s.lu">, + Sched<[WriteFCvtI64ToF32, ReadFCvtI64ToF32]>; def : FPUnaryOpDynFrmAlias<FCVT_S_LU, "fcvt.s.lu", FPR32, GPR>; } // Predicates = [HasStdExtF, IsRV64] @@ -320,12 +327,12 @@ def : Pat<(f32 (fpimm0)), (FMV_W_X X0)>; /// Float arithmetic operations -def : PatFpr32Fpr32DynFrm<fadd, FADD_S>; -def : PatFpr32Fpr32DynFrm<fsub, FSUB_S>; -def : PatFpr32Fpr32DynFrm<fmul, FMUL_S>; -def : PatFpr32Fpr32DynFrm<fdiv, FDIV_S>; +def : PatFpr32Fpr32DynFrm<any_fadd, FADD_S>; +def : PatFpr32Fpr32DynFrm<any_fsub, FSUB_S>; +def : PatFpr32Fpr32DynFrm<any_fmul, FMUL_S>; +def : PatFpr32Fpr32DynFrm<any_fdiv, FDIV_S>; -def : Pat<(fsqrt FPR32:$rs1), (FSQRT_S FPR32:$rs1, 0b111)>; +def : Pat<(any_fsqrt FPR32:$rs1), (FSQRT_S FPR32:$rs1, 0b111)>; def : Pat<(fneg FPR32:$rs1), (FSGNJN_S $rs1, $rs1)>; def : Pat<(fabs FPR32:$rs1), (FSGNJX_S $rs1, $rs1)>; @@ -334,19 +341,19 @@ def : PatFpr32Fpr32<fcopysign, FSGNJ_S>; def : Pat<(fcopysign FPR32:$rs1, (fneg FPR32:$rs2)), (FSGNJN_S $rs1, $rs2)>; // fmadd: rs1 * rs2 + rs3 -def : Pat<(fma FPR32:$rs1, FPR32:$rs2, FPR32:$rs3), +def : Pat<(any_fma FPR32:$rs1, FPR32:$rs2, FPR32:$rs3), (FMADD_S $rs1, $rs2, $rs3, 0b111)>; // fmsub: rs1 * rs2 - rs3 -def : Pat<(fma FPR32:$rs1, FPR32:$rs2, (fneg FPR32:$rs3)), +def : Pat<(any_fma FPR32:$rs1, FPR32:$rs2, (fneg FPR32:$rs3)), (FMSUB_S FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, 0b111)>; // fnmsub: -rs1 * rs2 + rs3 -def : Pat<(fma (fneg FPR32:$rs1), FPR32:$rs2, FPR32:$rs3), +def : Pat<(any_fma (fneg FPR32:$rs1), FPR32:$rs2, FPR32:$rs3), (FNMSUB_S FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, 0b111)>; // fnmadd: -rs1 * rs2 - rs3 -def : Pat<(fma (fneg FPR32:$rs1), FPR32:$rs2, (fneg FPR32:$rs3)), +def : Pat<(any_fma (fneg FPR32:$rs1), FPR32:$rs2, (fneg FPR32:$rs3)), (FNMADD_S FPR32:$rs1, FPR32:$rs2, FPR32:$rs3, 0b111)>; // The ratified 20191213 ISA spec defines fmin and fmax in a way that matches @@ -382,8 +389,8 @@ def : Pat<(bitconvert (i32 GPR:$rs1)), (FMV_W_X GPR:$rs1)>; def : Pat<(i32 (bitconvert FPR32:$rs1)), (FMV_X_W FPR32:$rs1)>; // float->[u]int. Round-to-zero must be used. -def : Pat<(i32 (fp_to_sint FPR32:$rs1)), (FCVT_W_S $rs1, 0b001)>; -def : Pat<(i32 (fp_to_uint FPR32:$rs1)), (FCVT_WU_S $rs1, 0b001)>; +def : Pat<(i32 (any_fp_to_sint FPR32:$rs1)), (FCVT_W_S $rs1, 0b001)>; +def : Pat<(i32 (any_fp_to_uint FPR32:$rs1)), (FCVT_WU_S $rs1, 0b001)>; // Saturating float->[u]int32. def : Pat<(i32 (riscv_fcvt_x_rtz FPR32:$rs1)), (FCVT_W_S $rs1, 0b001)>; @@ -396,8 +403,8 @@ def : Pat<(i32 (lrint FPR32:$rs1)), (FCVT_W_S $rs1, 0b111)>; def : Pat<(i32 (lround FPR32:$rs1)), (FCVT_W_S $rs1, 0b100)>; // [u]int->float. Match GCC and default to using dynamic rounding mode. -def : Pat<(sint_to_fp (i32 GPR:$rs1)), (FCVT_S_W $rs1, 0b111)>; -def : Pat<(uint_to_fp (i32 GPR:$rs1)), (FCVT_S_WU $rs1, 0b111)>; +def : Pat<(any_sint_to_fp (i32 GPR:$rs1)), (FCVT_S_W $rs1, 0b111)>; +def : Pat<(any_uint_to_fp (i32 GPR:$rs1)), (FCVT_S_WU $rs1, 0b111)>; } // Predicates = [HasStdExtF, IsRV32] let Predicates = [HasStdExtF, IsRV64] in { @@ -410,12 +417,12 @@ def : Pat<(sext_inreg (riscv_fmv_x_anyextw_rv64 FPR32:$src), i32), // Use target specific isd nodes to help us remember the result is sign // extended. Matching sext_inreg+fptoui/fptosi may cause the conversion to be // duplicated if it has another user that didn't need the sign_extend. -def : Pat<(riscv_fcvt_w_rtz_rv64 FPR32:$rs1), (FCVT_W_S $rs1, 0b001)>; -def : Pat<(riscv_fcvt_wu_rtz_rv64 FPR32:$rs1), (FCVT_WU_S $rs1, 0b001)>; +def : Pat<(riscv_any_fcvt_w_rtz_rv64 FPR32:$rs1), (FCVT_W_S $rs1, 0b001)>; +def : Pat<(riscv_any_fcvt_wu_rtz_rv64 FPR32:$rs1), (FCVT_WU_S $rs1, 0b001)>; // float->[u]int64. Round-to-zero must be used. -def : Pat<(i64 (fp_to_sint FPR32:$rs1)), (FCVT_L_S $rs1, 0b001)>; -def : Pat<(i64 (fp_to_uint FPR32:$rs1)), (FCVT_LU_S $rs1, 0b001)>; +def : Pat<(i64 (any_fp_to_sint FPR32:$rs1)), (FCVT_L_S $rs1, 0b001)>; +def : Pat<(i64 (any_fp_to_uint FPR32:$rs1)), (FCVT_LU_S $rs1, 0b001)>; // Saturating float->[u]int64. def : Pat<(i64 (riscv_fcvt_x_rtz FPR32:$rs1)), (FCVT_L_S $rs1, 0b001)>; @@ -430,8 +437,8 @@ def : Pat<(i64 (lround FPR32:$rs1)), (FCVT_L_S $rs1, 0b100)>; def : Pat<(i64 (llround FPR32:$rs1)), (FCVT_L_S $rs1, 0b100)>; // [u]int->fp. Match GCC and default to using dynamic rounding mode. -def : Pat<(sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_S_W $rs1, 0b111)>; -def : Pat<(uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_S_WU $rs1, 0b111)>; -def : Pat<(sint_to_fp (i64 GPR:$rs1)), (FCVT_S_L $rs1, 0b111)>; -def : Pat<(uint_to_fp (i64 GPR:$rs1)), (FCVT_S_LU $rs1, 0b111)>; +def : Pat<(any_sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_S_W $rs1, 0b111)>; +def : Pat<(any_uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_S_WU $rs1, 0b111)>; +def : Pat<(any_sint_to_fp (i64 GPR:$rs1)), (FCVT_S_L $rs1, 0b111)>; +def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_S_LU $rs1, 0b111)>; } // Predicates = [HasStdExtF, IsRV64] diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td index a037dbf585ce..b62e23d3b0fa 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td @@ -96,14 +96,6 @@ def : Pat<(srem (sexti32 (i64 GPR:$rs1)), (sexti32 (i64 GPR:$rs2))), (REMW GPR:$rs1, GPR:$rs2)>; } // Predicates = [HasStdExtM, IsRV64] -// Pattern to detect constants with no more than 32 active bits that can't -// be materialized with lui+addiw. -def uimm32_not_simm32 : PatLeaf<(XLenVT GPR:$a), [{ - auto *C = dyn_cast<ConstantSDNode>(N); - return C && C->hasOneUse() && isUInt<32>(C->getZExtValue()) && - !isInt<32>(C->getSExtValue()); -}]>; - let Predicates = [HasStdExtM, IsRV64, NotHasStdExtZba] in { // Special case for calculating the full 64-bit product of a 32x32 unsigned // multiply where the inputs aren't known to be zero extended. We can shift the @@ -111,9 +103,4 @@ let Predicates = [HasStdExtM, IsRV64, NotHasStdExtZba] in { // zeroing the upper 32 bits. def : Pat<(i64 (mul (and GPR:$rs1, 0xffffffff), (and GPR:$rs2, 0xffffffff))), (MULHU (SLLI GPR:$rs1, 32), (SLLI GPR:$rs2, 32))>; -// The RHS could also be a constant that is hard to materialize. By shifting -// left we can allow constant materialization to use LUI+ADDIW via -// hasAllWUsers. -def : Pat<(i64 (mul (and GPR:$rs1, 0xffffffff), uimm32_not_simm32:$rs2)), - (MULHU (SLLI GPR:$rs1, 32), (SLLI GPR:$rs2, 32))>; } // Predicates = [HasStdExtM, IsRV64, NotHasStdExtZba] diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td index 3d5f9bc54731..173ae43a08d6 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td @@ -338,29 +338,6 @@ class VALUVs2<bits<6> funct6, bits<5> vs1, RISCVVFormat opv, string opcodestr> opcodestr, "$vd, $vs2$vm">; } // hasSideEffects = 0, mayLoad = 0, mayStore = 0 -let hasSideEffects = 0, mayLoad = 1, mayStore = 1 in { -// vamo vd, (rs1), vs2, vd, vm -class VAMOWd<RISCVAMOOP amoop, RISCVWidth width, string opcodestr> - : RVInstVAMO<amoop, width.Value{2-0}, (outs VR:$vd_wd), - (ins GPR:$rs1, VR:$vs2, VR:$vd, VMaskOp:$vm), - opcodestr, "$vd_wd, (${rs1}), $vs2, $vd$vm"> { - let Constraints = "$vd_wd = $vd"; - let wd = 1; - bits<5> vd; - let Inst{11-7} = vd; -} - -// vamo x0, (rs1), vs2, vs3, vm -class VAMONoWd<RISCVAMOOP amoop, RISCVWidth width, string opcodestr> - : RVInstVAMO<amoop, width.Value{2-0}, (outs), - (ins GPR:$rs1, VR:$vs2, VR:$vs3, VMaskOp:$vm), - opcodestr, "x0, (${rs1}), $vs2, $vs3$vm"> { - bits<5> vs3; - let Inst{11-7} = vs3; -} - -} // hasSideEffects = 0, mayLoad = 1, mayStore = 1 - //===----------------------------------------------------------------------===// // Combination of instruction classes. // Use these multiclasses to define instructions more easily. @@ -779,11 +756,6 @@ multiclass VCPR_MV_Mask<string opcodestr, bits<6> funct6, string vm = "v"> { Sched<[WriteVCompressV, ReadVCompressV, ReadVCompressV]>; } -multiclass VAMO<RISCVAMOOP amoop, RISCVWidth width, string opcodestr> { - def _WD : VAMOWd<amoop, width, opcodestr>; - def _UNWD : VAMONoWd<amoop, width, opcodestr>; -} - multiclass VWholeLoadN<bits<3> nf, string opcodestr, RegisterClass VRC> { foreach l = [8, 16, 32, 64] in { defvar w = !cast<RISCVWidth>("LSWidth" # l); @@ -822,7 +794,7 @@ foreach eew = [8, 16, 32, 64] in { // Vector Strided Instructions def VLSE#eew#_V : VStridedLoad<w, "vlse"#eew#".v">, VLSSched<eew>; def VSSE#eew#_V : VStridedStore<w, "vsse"#eew#".v">, VSSSched<eew>; - + // Vector Indexed Instructions def VLUXEI#eew#_V : VIndexedLoad<MOPLDIndexedUnord, w, "vluxei"#eew#".v">, VLXSched<eew, "U">; @@ -1416,13 +1388,20 @@ defm VCOMPRESS_V : VCPR_MV_Mask<"vcompress", 0b010111>; let hasSideEffects = 0, mayLoad = 0, mayStore = 0, RVVConstraint = NoConstraint in { -foreach n = [1, 2, 4, 8] in { - def VMV#n#R_V : RVInstV<0b100111, !add(n, -1), OPIVI, (outs VR:$vd), - (ins VR:$vs2), "vmv" # n # "r.v", "$vd, $vs2">, - VMVRSched<n> { +def VMV1R_V : RVInstV<0b100111, 0, OPIVI, (outs VR:$vd), (ins VR:$vs2), + "vmv1r.v", "$vd, $vs2">, VMVRSched<1> { let Uses = []; let vm = 1; } +// A future extension may relax the vector register alignment restrictions. +foreach n = [2, 4, 8] in { + defvar vrc = !cast<VReg>("VRM"#n); + def VMV#n#R_V : RVInstV<0b100111, !add(n, -1), OPIVI, (outs vrc:$vd), + (ins vrc:$vs2), "vmv" # n # "r.v", "$vd, $vs2">, + VMVRSched<n> { + let Uses = []; + let vm = 1; + } } } // hasSideEffects = 0, mayLoad = 0, mayStore = 0 } // Predicates = [HasStdExtV] @@ -1462,31 +1441,4 @@ let Predicates = [HasStdExtZvlsseg] in { } } // Predicates = [HasStdExtZvlsseg] -let Predicates = [HasStdExtZvamo, HasStdExtA] in { - foreach eew = [8, 16, 32] in { - defvar w = !cast<RISCVWidth>("LSWidth"#eew); - defm VAMOSWAPEI#eew : VAMO<AMOOPVamoSwap, w, "vamoswapei"#eew#".v">; - defm VAMOADDEI#eew : VAMO<AMOOPVamoAdd, w, "vamoaddei"#eew#".v">; - defm VAMOXOREI#eew : VAMO<AMOOPVamoXor, w, "vamoxorei"#eew#".v">; - defm VAMOANDEI#eew : VAMO<AMOOPVamoAnd, w, "vamoandei"#eew#".v">; - defm VAMOOREI#eew : VAMO<AMOOPVamoOr, w, "vamoorei"#eew#".v">; - defm VAMOMINEI#eew : VAMO<AMOOPVamoMin, w, "vamominei"#eew#".v">; - defm VAMOMAXEI#eew : VAMO<AMOOPVamoMax, w, "vamomaxei"#eew#".v">; - defm VAMOMINUEI#eew : VAMO<AMOOPVamoMinu, w, "vamominuei"#eew#".v">; - defm VAMOMAXUEI#eew : VAMO<AMOOPVamoMaxu, w, "vamomaxuei"#eew#".v">; - } -} // Predicates = [HasStdExtZvamo, HasStdExtA] - -let Predicates = [HasStdExtZvamo, HasStdExtA, IsRV64] in { - defm VAMOSWAPEI64 : VAMO<AMOOPVamoSwap, LSWidth64, "vamoswapei64.v">; - defm VAMOADDEI64 : VAMO<AMOOPVamoAdd, LSWidth64, "vamoaddei64.v">; - defm VAMOXOREI64 : VAMO<AMOOPVamoXor, LSWidth64, "vamoxorei64.v">; - defm VAMOANDEI64 : VAMO<AMOOPVamoAnd, LSWidth64, "vamoandei64.v">; - defm VAMOOREI64 : VAMO<AMOOPVamoOr, LSWidth64, "vamoorei64.v">; - defm VAMOMINEI64 : VAMO<AMOOPVamoMin, LSWidth64, "vamominei64.v">; - defm VAMOMAXEI64 : VAMO<AMOOPVamoMax, LSWidth64, "vamomaxei64.v">; - defm VAMOMINUEI64 : VAMO<AMOOPVamoMinu, LSWidth64, "vamominuei64.v">; - defm VAMOMAXUEI64 : VAMO<AMOOPVamoMaxu, LSWidth64, "vamomaxuei64.v">; -} // Predicates = [HasStdExtZvamo, HasStdExtA, IsRV64] - include "RISCVInstrInfoVPseudos.td" diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index a82e333e6bab..073fa605e0fb 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -1124,68 +1124,6 @@ class VPseudoTernaryNoMaskWithPolicy<VReg RetClass, let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -class VPseudoAMOWDNoMask<VReg RetClass, - VReg Op1Class> : - Pseudo<(outs GetVRegNoV0<RetClass>.R:$vd_wd), - (ins GPR:$rs1, - Op1Class:$vs2, - GetVRegNoV0<RetClass>.R:$vd, - AVL:$vl, ixlenimm:$sew), []>, - RISCVVPseudo { - let mayLoad = 1; - let mayStore = 1; - let hasSideEffects = 1; - let Constraints = "$vd_wd = $vd"; - let HasVLOp = 1; - let HasSEWOp = 1; - let HasDummyMask = 1; - let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); -} - -class VPseudoAMOWDMask<VReg RetClass, - VReg Op1Class> : - Pseudo<(outs GetVRegNoV0<RetClass>.R:$vd_wd), - (ins GPR:$rs1, - Op1Class:$vs2, - GetVRegNoV0<RetClass>.R:$vd, - VMaskOp:$vm, AVL:$vl, ixlenimm:$sew), []>, - RISCVVPseudo { - let mayLoad = 1; - let mayStore = 1; - let hasSideEffects = 1; - let Constraints = "$vd_wd = $vd"; - let HasVLOp = 1; - let HasSEWOp = 1; - let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); -} - -multiclass VPseudoAMOEI<int eew> { - // Standard scalar AMO supports 32, 64, and 128 Mem data bits, - // and in the base vector "V" extension, only SEW up to ELEN = max(XLEN, FLEN) - // are required to be supported. - // therefore only [32, 64] is allowed here. - foreach sew = [32, 64] in { - foreach lmul = MxSet<sew>.m in { - defvar octuple_lmul = lmul.octuple; - // Calculate emul = eew * lmul / sew - defvar octuple_emul = !srl(!mul(eew, octuple_lmul), log2<sew>.val); - if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { - defvar emulMX = octuple_to_str<octuple_emul>.ret; - defvar emul= !cast<LMULInfo>("V_" # emulMX); - let VLMul = lmul.value in { - def "_WD_" # lmul.MX # "_" # emulMX : VPseudoAMOWDNoMask<lmul.vrclass, emul.vrclass>; - def "_WD_" # lmul.MX # "_" # emulMX # "_MASK" : VPseudoAMOWDMask<lmul.vrclass, emul.vrclass>; - } - } - } - } -} - -multiclass VPseudoAMO { - foreach eew = EEWList in - defm "EI" # eew : VPseudoAMOEI<eew>; -} - class VPseudoUSSegLoadNoMask<VReg RetClass, int EEW, bits<4> NF, bit isFF>: Pseudo<(outs RetClass:$rd), (ins GPR:$rs1, AVL:$vl, ixlenimm:$sew),[]>, @@ -1376,17 +1314,35 @@ class VPseudoISegStoreMask<VReg ValClass, VReg IdxClass, int EEW, bits<3> LMUL, let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst); } -multiclass VPseudoUSLoad<bit isFF> { +multiclass VPseudoUSLoad { foreach eew = EEWList in { foreach lmul = MxSet<eew>.m in { defvar LInfo = lmul.MX; defvar vreg = lmul.vrclass; - defvar FFStr = !if(isFF, "FF", ""); let VLMul = lmul.value in { - def "E" # eew # FFStr # "_V_" # LInfo : - VPseudoUSLoadNoMask<vreg, eew, isFF>; - def "E" # eew # FFStr # "_V_" # LInfo # "_MASK" : - VPseudoUSLoadMask<vreg, eew, isFF>; + def "E" # eew # "_V_" # LInfo : + VPseudoUSLoadNoMask<vreg, eew, false>, + VLESched<eew>; + def "E" # eew # "_V_" # LInfo # "_MASK" : + VPseudoUSLoadMask<vreg, eew, false>, + VLESched<eew>; + } + } + } +} + +multiclass VPseudoFFLoad { + foreach eew = EEWList in { + foreach lmul = MxSet<eew>.m in { + defvar LInfo = lmul.MX; + defvar vreg = lmul.vrclass; + let VLMul = lmul.value in { + def "E" # eew # "FF_V_" # LInfo : + VPseudoUSLoadNoMask<vreg, eew, true>, + VLFSched<eew>; + def "E" # eew # "FF_V_" # LInfo # "_MASK" : + VPseudoUSLoadMask<vreg, eew, true>, + VLFSched<eew>; } } } @@ -1406,8 +1362,10 @@ multiclass VPseudoSLoad { defvar LInfo = lmul.MX; defvar vreg = lmul.vrclass; let VLMul = lmul.value in { - def "E" # eew # "_V_" # LInfo : VPseudoSLoadNoMask<vreg, eew>; - def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSLoadMask<vreg, eew>; + def "E" # eew # "_V_" # LInfo : VPseudoSLoadNoMask<vreg, eew>, + VLSSched<eew>; + def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSLoadMask<vreg, eew>, + VLSSched<eew>; } } } @@ -1427,11 +1385,14 @@ multiclass VPseudoILoad<bit Ordered> { defvar Vreg = lmul.vrclass; defvar IdxVreg = idx_lmul.vrclass; defvar HasConstraint = !ne(sew, eew); + defvar Order = !if(Ordered, "O", "U"); let VLMul = lmul.value in { def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo : - VPseudoILoadNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>; + VPseudoILoadNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>, + VLXSched<eew, Order>; def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo # "_MASK" : - VPseudoILoadMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>; + VPseudoILoadMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered, HasConstraint>, + VLXSched<eew, Order>; } } } @@ -1445,8 +1406,10 @@ multiclass VPseudoUSStore { defvar LInfo = lmul.MX; defvar vreg = lmul.vrclass; let VLMul = lmul.value in { - def "E" # eew # "_V_" # LInfo : VPseudoUSStoreNoMask<vreg, eew>; - def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoUSStoreMask<vreg, eew>; + def "E" # eew # "_V_" # LInfo : VPseudoUSStoreNoMask<vreg, eew>, + VSESched<eew>; + def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoUSStoreMask<vreg, eew>, + VSESched<eew>; } } } @@ -1466,8 +1429,10 @@ multiclass VPseudoSStore { defvar LInfo = lmul.MX; defvar vreg = lmul.vrclass; let VLMul = lmul.value in { - def "E" # eew # "_V_" # LInfo : VPseudoSStoreNoMask<vreg, eew>; - def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSStoreMask<vreg, eew>; + def "E" # eew # "_V_" # LInfo : VPseudoSStoreNoMask<vreg, eew>, + VSSSched<eew>; + def "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoSStoreMask<vreg, eew>, + VSSSched<eew>; } } } @@ -1486,11 +1451,14 @@ multiclass VPseudoIStore<bit Ordered> { defvar idx_lmul = !cast<LMULInfo>("V_" # IdxLInfo); defvar Vreg = lmul.vrclass; defvar IdxVreg = idx_lmul.vrclass; + defvar Order = !if(Ordered, "O", "U"); let VLMul = lmul.value in { def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo : - VPseudoIStoreNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>; + VPseudoIStoreNoMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>, + VSXSched<eew, Order>; def "EI" # eew # "_V_" # IdxLInfo # "_" # LInfo # "_MASK" : - VPseudoIStoreMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>; + VPseudoIStoreMask<Vreg, IdxVreg, eew, idx_lmul.value, Ordered>, + VSXSched<eew, Order>; } } } @@ -1498,32 +1466,50 @@ multiclass VPseudoIStore<bit Ordered> { } } -multiclass VPseudoUnaryS_M { +multiclass VPseudoVPOP_M { foreach mti = AllMasks in { let VLMul = mti.LMul.value in { - def "_M_" # mti.BX : VPseudoUnaryNoMask<GPR, VR>; - def "_M_" # mti.BX # "_MASK" : VPseudoMaskUnarySOutMask; + def "_M_" # mti.BX : VPseudoUnaryNoMask<GPR, VR>, + Sched<[WriteVMPopV, ReadVMPopV, ReadVMPopV]>; + def "_M_" # mti.BX # "_MASK" : VPseudoMaskUnarySOutMask, + Sched<[WriteVMPopV, ReadVMPopV, ReadVMPopV]>; } } } -multiclass VPseudoUnaryM_M { +multiclass VPseudoV1ST_M { + foreach mti = AllMasks in + { + let VLMul = mti.LMul.value in { + def "_M_" # mti.BX : VPseudoUnaryNoMask<GPR, VR>, + Sched<[WriteVMFFSV, ReadVMFFSV, ReadVMFFSV]>; + def "_M_" # mti.BX # "_MASK" : VPseudoMaskUnarySOutMask, + Sched<[WriteVMFFSV, ReadVMFFSV, ReadVMFFSV]>; + } + } +} + +multiclass VPseudoVSFS_M { defvar constraint = "@earlyclobber $rd"; foreach mti = AllMasks in { let VLMul = mti.LMul.value in { - def "_M_" # mti.BX : VPseudoUnaryNoMask<VR, VR, constraint>; - def "_M_" # mti.BX # "_MASK" : VPseudoUnaryMask<VR, VR, constraint>; + def "_M_" # mti.BX : VPseudoUnaryNoMask<VR, VR, constraint>, + Sched<[WriteVMSFSV, ReadVMSFSV, ReadVMask]>; + def "_M_" # mti.BX # "_MASK" : VPseudoUnaryMask<VR, VR, constraint>, + Sched<[WriteVMSFSV, ReadVMSFSV, ReadVMask]>; } } } -multiclass VPseudoMaskNullaryV { +multiclass VPseudoVID_V { foreach m = MxList.m in { let VLMul = m.value in { - def "_V_" # m.MX : VPseudoNullaryNoMask<m.vrclass>; - def "_V_" # m.MX # "_MASK" : VPseudoNullaryMask<m.vrclass>; + def "_V_" # m.MX : VPseudoNullaryNoMask<m.vrclass>, + Sched<[WriteVMIdxV, ReadVMask]>; + def "_V_" # m.MX # "_MASK" : VPseudoNullaryMask<m.vrclass>, + Sched<[WriteVMIdxV, ReadVMask]>; } } } @@ -1536,20 +1522,23 @@ multiclass VPseudoNullaryPseudoM <string BaseInst> { } } -multiclass VPseudoUnaryV_M { +multiclass VPseudoVIOT_M { defvar constraint = "@earlyclobber $rd"; foreach m = MxList.m in { let VLMul = m.value in { - def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, VR, constraint>; - def "_" # m.MX # "_MASK" : VPseudoUnaryMask<m.vrclass, VR, constraint>; + def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, VR, constraint>, + Sched<[WriteVMIotV, ReadVMIotV, ReadVMask]>; + def "_" # m.MX # "_MASK" : VPseudoUnaryMask<m.vrclass, VR, constraint>, + Sched<[WriteVMIotV, ReadVMIotV, ReadVMask]>; } } } -multiclass VPseudoUnaryV_V_AnyMask { +multiclass VPseudoVCPR_V { foreach m = MxList.m in { let VLMul = m.value in - def _VM # "_" # m.MX : VPseudoUnaryAnyMask<m.vrclass, m.vrclass>; + def _VM # "_" # m.MX : VPseudoUnaryAnyMask<m.vrclass, m.vrclass>, + Sched<[WriteVCompressV, ReadVCompressV, ReadVCompressV]>; } } @@ -1611,7 +1600,7 @@ multiclass VPseudoBinaryV_VV<string Constraint = ""> { defm _VV : VPseudoBinary<m.vrclass, m.vrclass, m.vrclass, m, Constraint>; } -multiclass VPseudoBinaryV_VV_EEW<int eew, string Constraint = ""> { +multiclass VPseudoVGTR_VV_EEW<int eew, string Constraint = ""> { foreach m = MxList.m in { foreach sew = EEWList in { defvar octuple_lmul = m.octuple; @@ -1620,7 +1609,8 @@ multiclass VPseudoBinaryV_VV_EEW<int eew, string Constraint = ""> { if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { defvar emulMX = octuple_to_str<octuple_emul>.ret; defvar emul = !cast<LMULInfo>("V_" # emulMX); - defm _VV : VPseudoBinaryEmul<m.vrclass, m.vrclass, emul.vrclass, m, emul, Constraint>; + defm _VV : VPseudoBinaryEmul<m.vrclass, m.vrclass, emul.vrclass, m, emul, Constraint>, + Sched<[WriteVGatherV, ReadVGatherV, ReadVGatherV]>; } } } @@ -1631,6 +1621,12 @@ multiclass VPseudoBinaryV_VX<string Constraint = ""> { defm "_VX" : VPseudoBinary<m.vrclass, m.vrclass, GPR, m, Constraint>; } +multiclass VPseudoVSLD1_VX<string Constraint = ""> { + foreach m = MxList.m in + defm "_VX" : VPseudoBinary<m.vrclass, m.vrclass, GPR, m, Constraint>, + Sched<[WriteVISlide1X, ReadVISlideV, ReadVISlideX, ReadVMask]>; +} + multiclass VPseudoBinaryV_VF<string Constraint = ""> { foreach m = MxList.m in foreach f = FPList.fpinfo in @@ -1638,15 +1634,24 @@ multiclass VPseudoBinaryV_VF<string Constraint = ""> { f.fprclass, m, Constraint>; } +multiclass VPseudoVSLD1_VF<string Constraint = ""> { + foreach m = MxList.m in + foreach f = FPList.fpinfo in + defm "_V" # f.FX : + VPseudoBinary<m.vrclass, m.vrclass, f.fprclass, m, Constraint>, + Sched<[WriteVFSlide1F, ReadVFSlideV, ReadVFSlideF, ReadVMask]>; +} + multiclass VPseudoBinaryV_VI<Operand ImmType = simm5, string Constraint = ""> { foreach m = MxList.m in defm _VI : VPseudoBinary<m.vrclass, m.vrclass, ImmType, m, Constraint>; } -multiclass VPseudoBinaryM_MM { +multiclass VPseudoVALU_MM { foreach m = MxList.m in let VLMul = m.value in { - def "_MM_" # m.MX : VPseudoBinaryNoMask<VR, VR, VR, "">; + def "_MM_" # m.MX : VPseudoBinaryNoMask<VR, VR, VR, "">, + Sched<[WriteVMALUV, ReadVMALUV, ReadVMALUV]>; } } @@ -1744,12 +1749,13 @@ multiclass VPseudoBinaryV_XM<bit CarryOut = 0, bit CarryIn = 1, m.vrclass, GPR, m, CarryIn, Constraint>; } -multiclass VPseudoBinaryV_FM { +multiclass VPseudoVMRG_FM { foreach m = MxList.m in foreach f = FPList.fpinfo in def "_V" # f.FX # "M_" # m.MX : VPseudoBinaryCarryIn<GetVRegNoV0<m.vrclass>.R, - m.vrclass, f.fprclass, m, /*CarryIn=*/1, "">; + m.vrclass, f.fprclass, m, /*CarryIn=*/1, "">, + Sched<[WriteVFMergeV, ReadVFMergeV, ReadVFMergeF, ReadVMask]>; } multiclass VPseudoBinaryV_IM<bit CarryOut = 0, bit CarryIn = 1, @@ -1762,76 +1768,102 @@ multiclass VPseudoBinaryV_IM<bit CarryOut = 0, bit CarryIn = 1, m.vrclass, simm5, m, CarryIn, Constraint>; } -multiclass VPseudoUnaryV_V_X_I_NoDummyMask { +multiclass VPseudoUnaryVMV_V_X_I { foreach m = MxList.m in { let VLMul = m.value in { - def "_V_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, m.vrclass>; - def "_X_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, GPR>; - def "_I_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, simm5>; + def "_V_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, m.vrclass>, + Sched<[WriteVIMovV, ReadVIMovV]>; + def "_X_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, GPR>, + Sched<[WriteVIMovX, ReadVIMovX]>; + def "_I_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, simm5>, + Sched<[WriteVIMovI]>; } } } -multiclass VPseudoUnaryV_F_NoDummyMask { +multiclass VPseudoVMV_F { foreach m = MxList.m in { foreach f = FPList.fpinfo in { let VLMul = m.value in { - def "_" # f.FX # "_" # m.MX : VPseudoUnaryNoDummyMask<m.vrclass, f.fprclass>; + def "_" # f.FX # "_" # m.MX : + VPseudoUnaryNoDummyMask<m.vrclass, f.fprclass>, + Sched<[WriteVFMovV, ReadVFMovF]>; } } } } -multiclass VPseudoUnaryTAV_V { +multiclass VPseudoVCLS_V { foreach m = MxList.m in { let VLMul = m.value in { - def "_V_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.vrclass>; - def "_V_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.vrclass>; + def "_V_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.vrclass>, + Sched<[WriteVFClassV, ReadVFClassV, ReadVMask]>; + def "_V_" # m.MX # "_MASK" : VPseudoUnaryMask<m.vrclass, m.vrclass>, + Sched<[WriteVFClassV, ReadVFClassV, ReadVMask]>; } } } -multiclass VPseudoUnaryV_V { +multiclass VPseudoVSQR_V { foreach m = MxList.m in { let VLMul = m.value in { - def "_V_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.vrclass>; - def "_V_" # m.MX # "_MASK" : VPseudoUnaryMask<m.vrclass, m.vrclass>; + def "_V_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.vrclass>, + Sched<[WriteVFSqrtV, ReadVFSqrtV, ReadVMask]>; + def "_V_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.vrclass>, + Sched<[WriteVFSqrtV, ReadVFSqrtV, ReadVMask]>; } } } -multiclass PseudoUnaryV_VF2 { +multiclass VPseudoVRCP_V { + foreach m = MxList.m in { + let VLMul = m.value in { + def "_V_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.vrclass>, + Sched<[WriteVFRecpV, ReadVFRecpV, ReadVMask]>; + def "_V_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.vrclass>, + Sched<[WriteVFRecpV, ReadVFRecpV, ReadVMask]>; + } + } +} + +multiclass PseudoVEXT_VF2 { defvar constraints = "@earlyclobber $rd"; foreach m = MxListVF2.m in { let VLMul = m.value in { - def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f2vrclass, constraints>; - def "_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.f2vrclass, - constraints>; + def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f2vrclass, constraints>, + Sched<[WriteVExtV, ReadVExtV, ReadVMask]>; + def "_" # m.MX # "_MASK" : + VPseudoUnaryMaskTA<m.vrclass, m.f2vrclass, constraints>, + Sched<[WriteVExtV, ReadVExtV, ReadVMask]>; } } } -multiclass PseudoUnaryV_VF4 { +multiclass PseudoVEXT_VF4 { defvar constraints = "@earlyclobber $rd"; foreach m = MxListVF4.m in { let VLMul = m.value in { - def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f4vrclass, constraints>; - def "_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.f4vrclass, - constraints>; + def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f4vrclass, constraints>, + Sched<[WriteVExtV, ReadVExtV, ReadVMask]>; + def "_" # m.MX # "_MASK" : + VPseudoUnaryMaskTA<m.vrclass, m.f4vrclass, constraints>, + Sched<[WriteVExtV, ReadVExtV, ReadVMask]>; } } } -multiclass PseudoUnaryV_VF8 { +multiclass PseudoVEXT_VF8 { defvar constraints = "@earlyclobber $rd"; foreach m = MxListVF8.m in { let VLMul = m.value in { - def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f8vrclass, constraints>; - def "_" # m.MX # "_MASK" : VPseudoUnaryMaskTA<m.vrclass, m.f8vrclass, - constraints>; + def "_" # m.MX : VPseudoUnaryNoMask<m.vrclass, m.f8vrclass, constraints>, + Sched<[WriteVExtV, ReadVExtV, ReadVMask]>; + def "_" # m.MX # "_MASK" : + VPseudoUnaryMaskTA<m.vrclass, m.f8vrclass, constraints>, + Sched<[WriteVExtV, ReadVExtV, ReadVMask]>; } } } @@ -1874,30 +1906,172 @@ multiclass VPseudoBinaryM_VI { !if(!ge(m.octuple, 16), "@earlyclobber $rd", "")>; } -multiclass VPseudoBinaryV_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> { - defm "" : VPseudoBinaryV_VV<Constraint>; - defm "" : VPseudoBinaryV_VX<Constraint>; - defm "" : VPseudoBinaryV_VI<ImmType, Constraint>; +multiclass VPseudoVGTR_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> { + defm "" : VPseudoBinaryV_VV<Constraint>, + Sched<[WriteVGatherV, ReadVGatherV, ReadVGatherV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX<Constraint>, + Sched<[WriteVGatherX, ReadVGatherV, ReadVGatherX, ReadVMask]>; + defm "" : VPseudoBinaryV_VI<ImmType, Constraint>, + Sched<[WriteVGatherI, ReadVGatherV, ReadVMask]>; } -multiclass VPseudoBinaryV_VV_VX { - defm "" : VPseudoBinaryV_VV; - defm "" : VPseudoBinaryV_VX; +multiclass VPseudoVSALU_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> { + defm "" : VPseudoBinaryV_VV<Constraint>, + Sched<[WriteVSALUV, ReadVSALUV, ReadVSALUV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX<Constraint>, + Sched<[WriteVSALUX, ReadVSALUV, ReadVSALUX, ReadVMask]>; + defm "" : VPseudoBinaryV_VI<ImmType, Constraint>, + Sched<[WriteVSALUI, ReadVSALUV, ReadVMask]>; } -multiclass VPseudoBinaryV_VV_VF { - defm "" : VPseudoBinaryV_VV; - defm "" : VPseudoBinaryV_VF; + +multiclass VPseudoVSHT_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> { + defm "" : VPseudoBinaryV_VV<Constraint>, + Sched<[WriteVShiftV, ReadVShiftV, ReadVShiftV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX<Constraint>, + Sched<[WriteVShiftX, ReadVShiftV, ReadVShiftX, ReadVMask]>; + defm "" : VPseudoBinaryV_VI<ImmType, Constraint>, + Sched<[WriteVShiftI, ReadVShiftV, ReadVMask]>; } -multiclass VPseudoBinaryV_VX_VI<Operand ImmType = simm5> { - defm "" : VPseudoBinaryV_VX; - defm "" : VPseudoBinaryV_VI<ImmType>; +multiclass VPseudoVSSHT_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> { + defm "" : VPseudoBinaryV_VV<Constraint>, + Sched<[WriteVSShiftV, ReadVSShiftV, ReadVSShiftV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX<Constraint>, + Sched<[WriteVSShiftX, ReadVSShiftV, ReadVSShiftX, ReadVMask]>; + defm "" : VPseudoBinaryV_VI<ImmType, Constraint>, + Sched<[WriteVSShiftI, ReadVSShiftV, ReadVMask]>; } -multiclass VPseudoBinaryW_VV_VX { - defm "" : VPseudoBinaryW_VV; - defm "" : VPseudoBinaryW_VX; +multiclass VPseudoVALU_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> { + defm "" : VPseudoBinaryV_VV<Constraint>, + Sched<[WriteVIALUV, ReadVIALUV, ReadVIALUV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX<Constraint>, + Sched<[WriteVIALUX, ReadVIALUV, ReadVIALUX, ReadVMask]>; + defm "" : VPseudoBinaryV_VI<ImmType, Constraint>, + Sched<[WriteVIALUI, ReadVIALUV, ReadVMask]>; +} + +multiclass VPseudoVSALU_VV_VX { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVSALUV, ReadVSALUV, ReadVSALUV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX, + Sched<[WriteVSALUX, ReadVSALUV, ReadVSALUX, ReadVMask]>; +} + +multiclass VPseudoVSMUL_VV_VX { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVSMulV, ReadVSMulV, ReadVSMulV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX, + Sched<[WriteVSMulX, ReadVSMulV, ReadVSMulX, ReadVMask]>; +} + +multiclass VPseudoVAALU_VV_VX { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVAALUV, ReadVAALUV, ReadVAALUV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX, + Sched<[WriteVAALUX, ReadVAALUV, ReadVAALUX, ReadVMask]>; +} + +multiclass VPseudoVMINMAX_VV_VX { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVICmpV, ReadVICmpV, ReadVICmpV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX, + Sched<[WriteVICmpX, ReadVICmpV, ReadVICmpX, ReadVMask]>; +} + +multiclass VPseudoVMUL_VV_VX { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVIMulV, ReadVIMulV, ReadVIMulV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX, + Sched<[WriteVIMulX, ReadVIMulV, ReadVIMulX, ReadVMask]>; +} + +multiclass VPseudoVDIV_VV_VX { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVIDivV, ReadVIDivV, ReadVIDivV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX, + Sched<[WriteVIDivX, ReadVIDivV, ReadVIDivX, ReadVMask]>; +} + +multiclass VPseudoVFMUL_VV_VF { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVFMulV, ReadVFMulV, ReadVFMulV, ReadVMask]>; + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFMulF, ReadVFMulV, ReadVFMulF, ReadVMask]>; +} + +multiclass VPseudoVFDIV_VV_VF { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVFDivV, ReadVFDivV, ReadVFDivV, ReadVMask]>; + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFDivF, ReadVFDivV, ReadVFDivF, ReadVMask]>; +} + +multiclass VPseudoVFRDIV_VF { + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFDivF, ReadVFDivV, ReadVFDivF, ReadVMask]>; +} + +multiclass VPseudoVALU_VV_VX { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVIALUV, ReadVIALUV, ReadVIALUV, ReadVMask]>; + defm "" : VPseudoBinaryV_VX, + Sched<[WriteVIALUX, ReadVIALUV, ReadVIALUX, ReadVMask]>; +} + +multiclass VPseudoVSGNJ_VV_VF { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVFSgnjV, ReadVFSgnjV, ReadVFSgnjV, ReadVMask]>; + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFSgnjF, ReadVFSgnjV, ReadVFSgnjF, ReadVMask]>; +} + +multiclass VPseudoVMAX_VV_VF { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVFCmpV, ReadVFCmpV, ReadVFCmpV, ReadVMask]>; + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>; +} + +multiclass VPseudoVALU_VV_VF { + defm "" : VPseudoBinaryV_VV, + Sched<[WriteVFALUV, ReadVFALUV, ReadVFALUV, ReadVMask]>; + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFALUF, ReadVFALUV, ReadVFALUF, ReadVMask]>; +} + +multiclass VPseudoVALU_VF { + defm "" : VPseudoBinaryV_VF, + Sched<[WriteVFALUF, ReadVFALUV, ReadVFALUF, ReadVMask]>; +} + +multiclass VPseudoVALU_VX_VI<Operand ImmType = simm5> { + defm "" : VPseudoBinaryV_VX, + Sched<[WriteVIALUX, ReadVIALUV, ReadVIALUX, ReadVMask]>; + defm "" : VPseudoBinaryV_VI<ImmType>, + Sched<[WriteVIALUI, ReadVIALUV, ReadVMask]>; +} + +multiclass VPseudoVWALU_VV_VX { + defm "" : VPseudoBinaryW_VV, + Sched<[WriteVIWALUV, ReadVIWALUV, ReadVIWALUV, ReadVMask]>; + defm "" : VPseudoBinaryW_VX, + Sched<[WriteVIWALUX, ReadVIWALUV, ReadVIWALUX, ReadVMask]>; +} + +multiclass VPseudoVWMUL_VV_VX { + defm "" : VPseudoBinaryW_VV, + Sched<[WriteVIWMulV, ReadVIWMulV, ReadVIWMulV, ReadVMask]>; + defm "" : VPseudoBinaryW_VX, + Sched<[WriteVIWMulX, ReadVIWMulV, ReadVIWMulX, ReadVMask]>; +} + +multiclass VPseudoVWMUL_VV_VF { + defm "" : VPseudoBinaryW_VV, + Sched<[WriteVFWMulV, ReadVFWMulV, ReadVFWMulV, ReadVMask]>; + defm "" : VPseudoBinaryW_VF, + Sched<[WriteVFWMulF, ReadVFWMulV, ReadVFWMulF, ReadVMask]>; } multiclass VPseudoBinaryW_VV_VF { @@ -1905,53 +2079,100 @@ multiclass VPseudoBinaryW_VV_VF { defm "" : VPseudoBinaryW_VF; } -multiclass VPseudoBinaryW_WV_WX { - defm "" : VPseudoBinaryW_WV; - defm "" : VPseudoBinaryW_WX; +multiclass VPseudoVWALU_WV_WX { + defm "" : VPseudoBinaryW_WV, + Sched<[WriteVIWALUV, ReadVIWALUV, ReadVIWALUV, ReadVMask]>; + defm "" : VPseudoBinaryW_WX, + Sched<[WriteVIWALUX, ReadVIWALUV, ReadVIWALUX, ReadVMask]>; +} + +multiclass VPseudoVFWALU_VV_VF { + defm "" : VPseudoBinaryW_VV, + Sched<[WriteVFWALUV, ReadVFWALUV, ReadVFWALUV, ReadVMask]>; + defm "" : VPseudoBinaryW_VF, + Sched<[WriteVFWALUF, ReadVFWALUV, ReadVFWALUF, ReadVMask]>; +} + +multiclass VPseudoVFWALU_WV_WF { + defm "" : VPseudoBinaryW_WV, + Sched<[WriteVFWALUV, ReadVFWALUV, ReadVFWALUV, ReadVMask]>; + defm "" : VPseudoBinaryW_WF, + Sched<[WriteVFWALUF, ReadVFWALUV, ReadVFWALUF, ReadVMask]>; +} + +multiclass VPseudoVMRG_VM_XM_IM { + defm "" : VPseudoBinaryV_VM, + Sched<[WriteVIMergeV, ReadVIMergeV, ReadVIMergeV, ReadVMask]>; + defm "" : VPseudoBinaryV_XM, + Sched<[WriteVIMergeX, ReadVIMergeV, ReadVIMergeX, ReadVMask]>; + defm "" : VPseudoBinaryV_IM, + Sched<[WriteVIMergeI, ReadVIMergeV, ReadVMask]>; } -multiclass VPseudoBinaryW_WV_WF { - defm "" : VPseudoBinaryW_WV; - defm "" : VPseudoBinaryW_WF; +multiclass VPseudoVCALU_VM_XM_IM { + defm "" : VPseudoBinaryV_VM, + Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV, ReadVMask]>; + defm "" : VPseudoBinaryV_XM, + Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX, ReadVMask]>; + defm "" : VPseudoBinaryV_IM, + Sched<[WriteVICALUI, ReadVIALUCV, ReadVMask]>; } -multiclass VPseudoBinaryV_VM_XM_IM { - defm "" : VPseudoBinaryV_VM; - defm "" : VPseudoBinaryV_XM; - defm "" : VPseudoBinaryV_IM; +multiclass VPseudoVCALU_VM_XM { + defm "" : VPseudoBinaryV_VM, + Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV, ReadVMask]>; + defm "" : VPseudoBinaryV_XM, + Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX, ReadVMask]>; } -multiclass VPseudoBinaryV_VM_XM { - defm "" : VPseudoBinaryV_VM; - defm "" : VPseudoBinaryV_XM; +multiclass VPseudoVCALUM_VM_XM_IM<string Constraint> { + defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>, + Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV, ReadVMask]>; + defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>, + Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX, ReadVMask]>; + defm "" : VPseudoBinaryV_IM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>, + Sched<[WriteVICALUI, ReadVIALUCV, ReadVMask]>; } -multiclass VPseudoBinaryM_VM_XM_IM<string Constraint> { - defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>; - defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>; - defm "" : VPseudoBinaryV_IM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>; +multiclass VPseudoVCALUM_VM_XM<string Constraint> { + defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>, + Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV, ReadVMask]>; + defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>, + Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX, ReadVMask]>; } -multiclass VPseudoBinaryM_VM_XM<string Constraint> { - defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>; - defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/1, Constraint>; +multiclass VPseudoVCALUM_V_X_I<string Constraint> { + defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>, + Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV]>; + defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>, + Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX]>; + defm "" : VPseudoBinaryV_IM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>, + Sched<[WriteVICALUI, ReadVIALUCV]>; } -multiclass VPseudoBinaryM_V_X_I<string Constraint> { - defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>; - defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>; - defm "" : VPseudoBinaryV_IM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>; +multiclass VPseudoVCALUM_V_X<string Constraint> { + defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>, + Sched<[WriteVICALUV, ReadVIALUCV, ReadVIALUCV]>; + defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>, + Sched<[WriteVICALUX, ReadVIALUCV, ReadVIALUCX]>; } -multiclass VPseudoBinaryM_V_X<string Constraint> { - defm "" : VPseudoBinaryV_VM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>; - defm "" : VPseudoBinaryV_XM</*CarryOut=*/1, /*CarryIn=*/0, Constraint>; +multiclass VPseudoVNCLP_WV_WX_WI { + defm "" : VPseudoBinaryV_WV, + Sched<[WriteVNClipV, ReadVNClipV, ReadVNClipV, ReadVMask]>; + defm "" : VPseudoBinaryV_WX, + Sched<[WriteVNClipX, ReadVNClipV, ReadVNClipX, ReadVMask]>; + defm "" : VPseudoBinaryV_WI, + Sched<[WriteVNClipI, ReadVNClipV, ReadVMask]>; } -multiclass VPseudoBinaryV_WV_WX_WI { - defm "" : VPseudoBinaryV_WV; - defm "" : VPseudoBinaryV_WX; - defm "" : VPseudoBinaryV_WI; +multiclass VPseudoVNSHT_WV_WX_WI { + defm "" : VPseudoBinaryV_WV, + Sched<[WriteVNShiftV, ReadVNShiftV, ReadVNShiftV, ReadVMask]>; + defm "" : VPseudoBinaryV_WX, + Sched<[WriteVNShiftX, ReadVNShiftV, ReadVNShiftX, ReadVMask]>; + defm "" : VPseudoBinaryV_WI, + Sched<[WriteVNShiftI, ReadVNShiftV, ReadVMask]>; } multiclass VPseudoTernary<VReg RetClass, @@ -2031,55 +2252,113 @@ multiclass VPseudoTernaryV_VI<Operand ImmType = simm5, string Constraint = ""> { defm _VI : VPseudoTernary<m.vrclass, m.vrclass, ImmType, m, Constraint>; } -multiclass VPseudoTernaryV_VV_VX_AAXA<string Constraint = ""> { - defm "" : VPseudoTernaryV_VV_AAXA<Constraint>; - defm "" : VPseudoTernaryV_VX_AAXA<Constraint>; +multiclass VPseudoVMAC_VV_VX_AAXA<string Constraint = ""> { + defm "" : VPseudoTernaryV_VV_AAXA<Constraint>, + Sched<[WriteVIMulAddV, ReadVIMulAddV, ReadVIMulAddV, ReadVIMulAddV, ReadVMask]>; + defm "" : VPseudoTernaryV_VX_AAXA<Constraint>, + Sched<[WriteVIMulAddX, ReadVIMulAddV, ReadVIMulAddV, ReadVIMulAddX, ReadVMask]>; } -multiclass VPseudoTernaryV_VV_VF_AAXA<string Constraint = ""> { - defm "" : VPseudoTernaryV_VV_AAXA<Constraint>; - defm "" : VPseudoTernaryV_VF_AAXA<Constraint>; +multiclass VPseudoVMAC_VV_VF_AAXA<string Constraint = ""> { + defm "" : VPseudoTernaryV_VV_AAXA<Constraint>, + Sched<[WriteVFMulAddV, ReadVFMulAddV, ReadVFMulAddV, ReadVFMulAddV, ReadVMask]>; + defm "" : VPseudoTernaryV_VF_AAXA<Constraint>, + Sched<[WriteVFMulAddF, ReadVFMulAddV, ReadVFMulAddV, ReadVFMulAddF, ReadVMask]>; } -multiclass VPseudoTernaryV_VX_VI<Operand ImmType = simm5, string Constraint = ""> { - defm "" : VPseudoTernaryV_VX<Constraint>; - defm "" : VPseudoTernaryV_VI<ImmType, Constraint>; +multiclass VPseudoVSLD_VX_VI<Operand ImmType = simm5, string Constraint = ""> { + defm "" : VPseudoTernaryV_VX<Constraint>, + Sched<[WriteVISlideX, ReadVISlideV, ReadVISlideV, ReadVISlideX, ReadVMask]>; + defm "" : VPseudoTernaryV_VI<ImmType, Constraint>, + Sched<[WriteVISlideI, ReadVISlideV, ReadVISlideV, ReadVMask]>; } -multiclass VPseudoTernaryW_VV_VX { - defm "" : VPseudoTernaryW_VV; - defm "" : VPseudoTernaryW_VX; +multiclass VPseudoVWMAC_VV_VX { + defm "" : VPseudoTernaryW_VV, + Sched<[WriteVIWMulAddV, ReadVIWMulAddV, ReadVIWMulAddV, ReadVIWMulAddV, ReadVMask]>; + defm "" : VPseudoTernaryW_VX, + Sched<[WriteVIWMulAddX, ReadVIWMulAddV, ReadVIWMulAddV, ReadVIWMulAddX, ReadVMask]>; } -multiclass VPseudoTernaryW_VV_VF { - defm "" : VPseudoTernaryW_VV; - defm "" : VPseudoTernaryW_VF; +multiclass VPseudoVWMAC_VX { + defm "" : VPseudoTernaryW_VX, + Sched<[WriteVIWMulAddX, ReadVIWMulAddV, ReadVIWMulAddV, ReadVIWMulAddX, ReadVMask]>; } -multiclass VPseudoBinaryM_VV_VX_VI { - defm "" : VPseudoBinaryM_VV; - defm "" : VPseudoBinaryM_VX; - defm "" : VPseudoBinaryM_VI; +multiclass VPseudoVWMAC_VV_VF { + defm "" : VPseudoTernaryW_VV, + Sched<[WriteVFWMulAddV, ReadVFWMulAddV, ReadVFWMulAddV, ReadVFWMulAddV, ReadVMask]>; + defm "" : VPseudoTernaryW_VF, + Sched<[WriteVFWMulAddF, ReadVFWMulAddV, ReadVFWMulAddV, ReadVFWMulAddF, ReadVMask]>; } -multiclass VPseudoBinaryM_VV_VX { - defm "" : VPseudoBinaryM_VV; - defm "" : VPseudoBinaryM_VX; +multiclass VPseudoVCMPM_VV_VX_VI { + defm "" : VPseudoBinaryM_VV, + Sched<[WriteVICmpV, ReadVICmpV, ReadVICmpV, ReadVMask]>; + defm "" : VPseudoBinaryM_VX, + Sched<[WriteVICmpX, ReadVICmpV, ReadVICmpX, ReadVMask]>; + defm "" : VPseudoBinaryM_VI, + Sched<[WriteVICmpI, ReadVICmpV, ReadVMask]>; } -multiclass VPseudoBinaryM_VV_VF { - defm "" : VPseudoBinaryM_VV; - defm "" : VPseudoBinaryM_VF; +multiclass VPseudoVCMPM_VV_VX { + defm "" : VPseudoBinaryM_VV, + Sched<[WriteVICmpV, ReadVICmpV, ReadVICmpV, ReadVMask]>; + defm "" : VPseudoBinaryM_VX, + Sched<[WriteVICmpX, ReadVICmpV, ReadVICmpX, ReadVMask]>; } -multiclass VPseudoBinaryM_VX_VI { - defm "" : VPseudoBinaryM_VX; - defm "" : VPseudoBinaryM_VI; +multiclass VPseudoVCMPM_VV_VF { + defm "" : VPseudoBinaryM_VV, + Sched<[WriteVFCmpV, ReadVFCmpV, ReadVFCmpV, ReadVMask]>; + defm "" : VPseudoBinaryM_VF, + Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>; } -multiclass VPseudoReductionV_VS { +multiclass VPseudoVCMPM_VF { + defm "" : VPseudoBinaryM_VF, + Sched<[WriteVFCmpF, ReadVFCmpV, ReadVFCmpF, ReadVMask]>; +} + +multiclass VPseudoVCMPM_VX_VI { + defm "" : VPseudoBinaryM_VX, + Sched<[WriteVICmpX, ReadVICmpV, ReadVICmpX, ReadVMask]>; + defm "" : VPseudoBinaryM_VI, + Sched<[WriteVICmpI, ReadVICmpV, ReadVMask]>; +} + +multiclass VPseudoVRED_VS { foreach m = MxList.m in { - defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>; + defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>, + Sched<[WriteVIRedV, ReadVIRedV, ReadVIRedV, ReadVIRedV, ReadVMask]>; + } +} + +multiclass VPseudoVWRED_VS { + foreach m = MxList.m in { + defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>, + Sched<[WriteVIWRedV, ReadVIWRedV, ReadVIWRedV, ReadVIWRedV, ReadVMask]>; + } +} + +multiclass VPseudoVFRED_VS { + foreach m = MxList.m in { + defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>, + Sched<[WriteVFRedV, ReadVFRedV, ReadVFRedV, ReadVFRedV, ReadVMask]>; + } +} + +multiclass VPseudoVFREDO_VS { + foreach m = MxList.m in { + defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>, + Sched<[WriteVFRedOV, ReadVFRedOV, ReadVFRedOV, ReadVFRedOV, ReadVMask]>; + } +} + +multiclass VPseudoVFWRED_VS { + foreach m = MxList.m in { + defm _VS : VPseudoTernary<V_M1.vrclass, m.vrclass, V_M1.vrclass, m>, + Sched<[WriteVFWRedV, ReadVFWRedV, ReadVFWRedV, ReadVFWRedV, ReadVMask]>; } } @@ -2094,9 +2373,16 @@ multiclass VPseudoConversion<VReg RetClass, } } -multiclass VPseudoConversionV_V { +multiclass VPseudoVCVTI_V { + foreach m = MxList.m in + defm _V : VPseudoConversion<m.vrclass, m.vrclass, m>, + Sched<[WriteVFCvtFToIV, ReadVFCvtFToIV, ReadVMask]>; +} + +multiclass VPseudoVCVTF_V { foreach m = MxList.m in - defm _V : VPseudoConversion<m.vrclass, m.vrclass, m>; + defm _V : VPseudoConversion<m.vrclass, m.vrclass, m>, + Sched<[WriteVFCvtIToFV, ReadVFCvtIToFV, ReadVMask]>; } multiclass VPseudoConversionW_V { @@ -2105,10 +2391,46 @@ multiclass VPseudoConversionW_V { defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint>; } -multiclass VPseudoConversionV_W { +multiclass VPseudoVWCVTI_V { + defvar constraint = "@earlyclobber $rd"; + foreach m = MxList.m[0-5] in + defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint>, + Sched<[WriteVFWCvtFToIV, ReadVFWCvtFToIV, ReadVMask]>; +} + +multiclass VPseudoVWCVTF_V { + defvar constraint = "@earlyclobber $rd"; + foreach m = MxList.m[0-5] in + defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint>, + Sched<[WriteVFWCvtIToFV, ReadVFWCvtIToFV, ReadVMask]>; +} + +multiclass VPseudoVWCVTD_V { + defvar constraint = "@earlyclobber $rd"; + foreach m = MxList.m[0-5] in + defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint>, + Sched<[WriteVFWCvtFToFV, ReadVFWCvtFToFV, ReadVMask]>; +} + +multiclass VPseudoVNCVTI_W { + defvar constraint = "@earlyclobber $rd"; + foreach m = MxList.m[0-5] in + defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint>, + Sched<[WriteVFNCvtFToIV, ReadVFNCvtFToIV, ReadVMask]>; +} + +multiclass VPseudoVNCVTF_W { + defvar constraint = "@earlyclobber $rd"; + foreach m = MxList.m[0-5] in + defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint>, + Sched<[WriteVFNCvtIToFV, ReadVFNCvtIToFV, ReadVMask]>; +} + +multiclass VPseudoVNCVTD_W { defvar constraint = "@earlyclobber $rd"; foreach m = MxListW.m in - defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint>; + defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint>, + Sched<[WriteVFNCvtFToFV, ReadVFNCvtFToFV, ReadVMask]>; } multiclass VPseudoUSSegLoad<bit isFF> { @@ -2543,42 +2865,6 @@ class VPatTernaryMask<string intrinsic, (mask_type V0), GPR:$vl, sew)>; -class VPatAMOWDNoMask<string intrinsic_name, - string inst, - ValueType result_type, - ValueType op1_type, - int sew, - LMULInfo vlmul, - LMULInfo emul, - VReg op1_reg_class> : - Pat<(result_type (!cast<Intrinsic>(intrinsic_name) - GPR:$rs1, - (op1_type op1_reg_class:$vs2), - (result_type vlmul.vrclass:$vd), - VLOpFrag)), - (!cast<Instruction>(inst # "_WD_" # vlmul.MX # "_" # emul.MX) - $rs1, $vs2, $vd, - GPR:$vl, sew)>; - -class VPatAMOWDMask<string intrinsic_name, - string inst, - ValueType result_type, - ValueType op1_type, - ValueType mask_type, - int sew, - LMULInfo vlmul, - LMULInfo emul, - VReg op1_reg_class> : - Pat<(result_type (!cast<Intrinsic>(intrinsic_name # "_mask") - GPR:$rs1, - (op1_type op1_reg_class:$vs2), - (result_type vlmul.vrclass:$vd), - (mask_type V0), - VLOpFrag)), - (!cast<Instruction>(inst # "_WD_" # vlmul.MX # "_" # emul.MX # "_MASK") - $rs1, $vs2, $vd, - (mask_type V0), GPR:$vl, sew)>; - multiclass VPatUnaryS_M<string intrinsic_name, string inst> { @@ -3416,44 +3702,6 @@ multiclass VPatConversionVF_WF <string intrinsic, string instruction> { } } -multiclass VPatAMOWD<string intrinsic, - string inst, - ValueType result_type, - ValueType offset_type, - ValueType mask_type, - int sew, - LMULInfo vlmul, - LMULInfo emul, - VReg op1_reg_class> -{ - def : VPatAMOWDNoMask<intrinsic, inst, result_type, offset_type, - sew, vlmul, emul, op1_reg_class>; - def : VPatAMOWDMask<intrinsic, inst, result_type, offset_type, - mask_type, sew, vlmul, emul, op1_reg_class>; -} - -multiclass VPatAMOV_WD<string intrinsic, - string inst, - list<VTypeInfo> vtilist> { - foreach eew = EEWList in { - foreach vti = vtilist in { - if !or(!eq(vti.SEW, 32), !eq(vti.SEW, 64)) then { - defvar octuple_lmul = vti.LMul.octuple; - // Calculate emul = eew * lmul / sew - defvar octuple_emul = !srl(!mul(eew, octuple_lmul), vti.Log2SEW); - if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then { - defvar emulMX = octuple_to_str<octuple_emul>.ret; - defvar offsetVti = !cast<VTypeInfo>("VI" # eew # emulMX); - defvar inst_ei = inst # "EI" # eew; - defm : VPatAMOWD<intrinsic, inst_ei, - vti.Vector, offsetVti.Vector, - vti.Mask, vti.Log2SEW, vti.LMul, offsetVti.LMul, offsetVti.RegClass>; - } - } - } - } -} - //===----------------------------------------------------------------------===// // Pseudo instructions //===----------------------------------------------------------------------===// @@ -3531,11 +3779,13 @@ def PseudoVSETIVLI : Pseudo<(outs GPR:$rd), (ins uimm5:$rs1, VTypeIOp:$vtypei), //===----------------------------------------------------------------------===// // Pseudos Unit-Stride Loads and Stores -defm PseudoVL : VPseudoUSLoad</*isFF=*/false>; +defm PseudoVL : VPseudoUSLoad; defm PseudoVS : VPseudoUSStore; -defm PseudoVLM : VPseudoLoadMask; -defm PseudoVSM : VPseudoStoreMask; +defm PseudoVLM : VPseudoLoadMask, + Sched<[WriteVLDM, ReadVLDX]>; +defm PseudoVSM : VPseudoStoreMask, + Sched<[WriteVSTM, ReadVSTX]>; //===----------------------------------------------------------------------===// // 7.5 Vector Strided Instructions @@ -3561,7 +3811,7 @@ defm PseudoVSUX : VPseudoIStore</*Ordered=*/false>; // vleff may update VL register let hasSideEffects = 1, Defs = [VL] in -defm PseudoVL : VPseudoUSLoad</*isFF=*/true>; +defm PseudoVL : VPseudoFFLoad; //===----------------------------------------------------------------------===// // 7.8. Vector Load/Store Segment Instructions @@ -3580,28 +3830,15 @@ let hasSideEffects = 1, Defs = [VL] in defm PseudoVLSEG : VPseudoUSSegLoad</*isFF=*/true>; //===----------------------------------------------------------------------===// -// 8. Vector AMO Operations -//===----------------------------------------------------------------------===// -defm PseudoVAMOSWAP : VPseudoAMO; -defm PseudoVAMOADD : VPseudoAMO; -defm PseudoVAMOXOR : VPseudoAMO; -defm PseudoVAMOAND : VPseudoAMO; -defm PseudoVAMOOR : VPseudoAMO; -defm PseudoVAMOMIN : VPseudoAMO; -defm PseudoVAMOMAX : VPseudoAMO; -defm PseudoVAMOMINU : VPseudoAMO; -defm PseudoVAMOMAXU : VPseudoAMO; - -//===----------------------------------------------------------------------===// // 12. Vector Integer Arithmetic Instructions //===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===// // 12.1. Vector Single-Width Integer Add and Subtract //===----------------------------------------------------------------------===// -defm PseudoVADD : VPseudoBinaryV_VV_VX_VI; -defm PseudoVSUB : VPseudoBinaryV_VV_VX; -defm PseudoVRSUB : VPseudoBinaryV_VX_VI; +defm PseudoVADD : VPseudoVALU_VV_VX_VI; +defm PseudoVSUB : VPseudoVALU_VV_VX; +defm PseudoVRSUB : VPseudoVALU_VX_VI; foreach vti = AllIntegerVectors in { // Match vrsub with 2 vector operands to vsub.vv by swapping operands. This @@ -3657,166 +3894,166 @@ foreach vti = AllIntegerVectors in { //===----------------------------------------------------------------------===// // 12.2. Vector Widening Integer Add/Subtract //===----------------------------------------------------------------------===// -defm PseudoVWADDU : VPseudoBinaryW_VV_VX; -defm PseudoVWSUBU : VPseudoBinaryW_VV_VX; -defm PseudoVWADD : VPseudoBinaryW_VV_VX; -defm PseudoVWSUB : VPseudoBinaryW_VV_VX; -defm PseudoVWADDU : VPseudoBinaryW_WV_WX; -defm PseudoVWSUBU : VPseudoBinaryW_WV_WX; -defm PseudoVWADD : VPseudoBinaryW_WV_WX; -defm PseudoVWSUB : VPseudoBinaryW_WV_WX; +defm PseudoVWADDU : VPseudoVWALU_VV_VX; +defm PseudoVWSUBU : VPseudoVWALU_VV_VX; +defm PseudoVWADD : VPseudoVWALU_VV_VX; +defm PseudoVWSUB : VPseudoVWALU_VV_VX; +defm PseudoVWADDU : VPseudoVWALU_WV_WX; +defm PseudoVWSUBU : VPseudoVWALU_WV_WX; +defm PseudoVWADD : VPseudoVWALU_WV_WX; +defm PseudoVWSUB : VPseudoVWALU_WV_WX; //===----------------------------------------------------------------------===// // 12.3. Vector Integer Extension //===----------------------------------------------------------------------===// -defm PseudoVZEXT_VF2 : PseudoUnaryV_VF2; -defm PseudoVZEXT_VF4 : PseudoUnaryV_VF4; -defm PseudoVZEXT_VF8 : PseudoUnaryV_VF8; -defm PseudoVSEXT_VF2 : PseudoUnaryV_VF2; -defm PseudoVSEXT_VF4 : PseudoUnaryV_VF4; -defm PseudoVSEXT_VF8 : PseudoUnaryV_VF8; +defm PseudoVZEXT_VF2 : PseudoVEXT_VF2; +defm PseudoVZEXT_VF4 : PseudoVEXT_VF4; +defm PseudoVZEXT_VF8 : PseudoVEXT_VF8; +defm PseudoVSEXT_VF2 : PseudoVEXT_VF2; +defm PseudoVSEXT_VF4 : PseudoVEXT_VF4; +defm PseudoVSEXT_VF8 : PseudoVEXT_VF8; //===----------------------------------------------------------------------===// // 12.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions //===----------------------------------------------------------------------===// -defm PseudoVADC : VPseudoBinaryV_VM_XM_IM; -defm PseudoVMADC : VPseudoBinaryM_VM_XM_IM<"@earlyclobber $rd">; -defm PseudoVMADC : VPseudoBinaryM_V_X_I<"@earlyclobber $rd">; +defm PseudoVADC : VPseudoVCALU_VM_XM_IM; +defm PseudoVMADC : VPseudoVCALUM_VM_XM_IM<"@earlyclobber $rd">; +defm PseudoVMADC : VPseudoVCALUM_V_X_I<"@earlyclobber $rd">; -defm PseudoVSBC : VPseudoBinaryV_VM_XM; -defm PseudoVMSBC : VPseudoBinaryM_VM_XM<"@earlyclobber $rd">; -defm PseudoVMSBC : VPseudoBinaryM_V_X<"@earlyclobber $rd">; +defm PseudoVSBC : VPseudoVCALU_VM_XM; +defm PseudoVMSBC : VPseudoVCALUM_VM_XM<"@earlyclobber $rd">; +defm PseudoVMSBC : VPseudoVCALUM_V_X<"@earlyclobber $rd">; //===----------------------------------------------------------------------===// // 12.5. Vector Bitwise Logical Instructions //===----------------------------------------------------------------------===// -defm PseudoVAND : VPseudoBinaryV_VV_VX_VI; -defm PseudoVOR : VPseudoBinaryV_VV_VX_VI; -defm PseudoVXOR : VPseudoBinaryV_VV_VX_VI; +defm PseudoVAND : VPseudoVALU_VV_VX_VI; +defm PseudoVOR : VPseudoVALU_VV_VX_VI; +defm PseudoVXOR : VPseudoVALU_VV_VX_VI; //===----------------------------------------------------------------------===// // 12.6. Vector Single-Width Bit Shift Instructions //===----------------------------------------------------------------------===// -defm PseudoVSLL : VPseudoBinaryV_VV_VX_VI<uimm5>; -defm PseudoVSRL : VPseudoBinaryV_VV_VX_VI<uimm5>; -defm PseudoVSRA : VPseudoBinaryV_VV_VX_VI<uimm5>; +defm PseudoVSLL : VPseudoVSHT_VV_VX_VI<uimm5>; +defm PseudoVSRL : VPseudoVSHT_VV_VX_VI<uimm5>; +defm PseudoVSRA : VPseudoVSHT_VV_VX_VI<uimm5>; //===----------------------------------------------------------------------===// // 12.7. Vector Narrowing Integer Right Shift Instructions //===----------------------------------------------------------------------===// -defm PseudoVNSRL : VPseudoBinaryV_WV_WX_WI; -defm PseudoVNSRA : VPseudoBinaryV_WV_WX_WI; +defm PseudoVNSRL : VPseudoVNSHT_WV_WX_WI; +defm PseudoVNSRA : VPseudoVNSHT_WV_WX_WI; //===----------------------------------------------------------------------===// // 12.8. Vector Integer Comparison Instructions //===----------------------------------------------------------------------===// -defm PseudoVMSEQ : VPseudoBinaryM_VV_VX_VI; -defm PseudoVMSNE : VPseudoBinaryM_VV_VX_VI; -defm PseudoVMSLTU : VPseudoBinaryM_VV_VX; -defm PseudoVMSLT : VPseudoBinaryM_VV_VX; -defm PseudoVMSLEU : VPseudoBinaryM_VV_VX_VI; -defm PseudoVMSLE : VPseudoBinaryM_VV_VX_VI; -defm PseudoVMSGTU : VPseudoBinaryM_VX_VI; -defm PseudoVMSGT : VPseudoBinaryM_VX_VI; +defm PseudoVMSEQ : VPseudoVCMPM_VV_VX_VI; +defm PseudoVMSNE : VPseudoVCMPM_VV_VX_VI; +defm PseudoVMSLTU : VPseudoVCMPM_VV_VX; +defm PseudoVMSLT : VPseudoVCMPM_VV_VX; +defm PseudoVMSLEU : VPseudoVCMPM_VV_VX_VI; +defm PseudoVMSLE : VPseudoVCMPM_VV_VX_VI; +defm PseudoVMSGTU : VPseudoVCMPM_VX_VI; +defm PseudoVMSGT : VPseudoVCMPM_VX_VI; //===----------------------------------------------------------------------===// // 12.9. Vector Integer Min/Max Instructions //===----------------------------------------------------------------------===// -defm PseudoVMINU : VPseudoBinaryV_VV_VX; -defm PseudoVMIN : VPseudoBinaryV_VV_VX; -defm PseudoVMAXU : VPseudoBinaryV_VV_VX; -defm PseudoVMAX : VPseudoBinaryV_VV_VX; +defm PseudoVMINU : VPseudoVMINMAX_VV_VX; +defm PseudoVMIN : VPseudoVMINMAX_VV_VX; +defm PseudoVMAXU : VPseudoVMINMAX_VV_VX; +defm PseudoVMAX : VPseudoVMINMAX_VV_VX; //===----------------------------------------------------------------------===// // 12.10. Vector Single-Width Integer Multiply Instructions //===----------------------------------------------------------------------===// -defm PseudoVMUL : VPseudoBinaryV_VV_VX; -defm PseudoVMULH : VPseudoBinaryV_VV_VX; -defm PseudoVMULHU : VPseudoBinaryV_VV_VX; -defm PseudoVMULHSU : VPseudoBinaryV_VV_VX; +defm PseudoVMUL : VPseudoVMUL_VV_VX; +defm PseudoVMULH : VPseudoVMUL_VV_VX; +defm PseudoVMULHU : VPseudoVMUL_VV_VX; +defm PseudoVMULHSU : VPseudoVMUL_VV_VX; //===----------------------------------------------------------------------===// // 12.11. Vector Integer Divide Instructions //===----------------------------------------------------------------------===// -defm PseudoVDIVU : VPseudoBinaryV_VV_VX; -defm PseudoVDIV : VPseudoBinaryV_VV_VX; -defm PseudoVREMU : VPseudoBinaryV_VV_VX; -defm PseudoVREM : VPseudoBinaryV_VV_VX; +defm PseudoVDIVU : VPseudoVDIV_VV_VX; +defm PseudoVDIV : VPseudoVDIV_VV_VX; +defm PseudoVREMU : VPseudoVDIV_VV_VX; +defm PseudoVREM : VPseudoVDIV_VV_VX; //===----------------------------------------------------------------------===// // 12.12. Vector Widening Integer Multiply Instructions //===----------------------------------------------------------------------===// -defm PseudoVWMUL : VPseudoBinaryW_VV_VX; -defm PseudoVWMULU : VPseudoBinaryW_VV_VX; -defm PseudoVWMULSU : VPseudoBinaryW_VV_VX; +defm PseudoVWMUL : VPseudoVWMUL_VV_VX; +defm PseudoVWMULU : VPseudoVWMUL_VV_VX; +defm PseudoVWMULSU : VPseudoVWMUL_VV_VX; //===----------------------------------------------------------------------===// // 12.13. Vector Single-Width Integer Multiply-Add Instructions //===----------------------------------------------------------------------===// -defm PseudoVMACC : VPseudoTernaryV_VV_VX_AAXA; -defm PseudoVNMSAC : VPseudoTernaryV_VV_VX_AAXA; -defm PseudoVMADD : VPseudoTernaryV_VV_VX_AAXA; -defm PseudoVNMSUB : VPseudoTernaryV_VV_VX_AAXA; +defm PseudoVMACC : VPseudoVMAC_VV_VX_AAXA; +defm PseudoVNMSAC : VPseudoVMAC_VV_VX_AAXA; +defm PseudoVMADD : VPseudoVMAC_VV_VX_AAXA; +defm PseudoVNMSUB : VPseudoVMAC_VV_VX_AAXA; //===----------------------------------------------------------------------===// // 12.14. Vector Widening Integer Multiply-Add Instructions //===----------------------------------------------------------------------===// -defm PseudoVWMACCU : VPseudoTernaryW_VV_VX; -defm PseudoVWMACC : VPseudoTernaryW_VV_VX; -defm PseudoVWMACCSU : VPseudoTernaryW_VV_VX; -defm PseudoVWMACCUS : VPseudoTernaryW_VX; +defm PseudoVWMACCU : VPseudoVWMAC_VV_VX; +defm PseudoVWMACC : VPseudoVWMAC_VV_VX; +defm PseudoVWMACCSU : VPseudoVWMAC_VV_VX; +defm PseudoVWMACCUS : VPseudoVWMAC_VX; //===----------------------------------------------------------------------===// // 12.15. Vector Integer Merge Instructions //===----------------------------------------------------------------------===// -defm PseudoVMERGE : VPseudoBinaryV_VM_XM_IM; +defm PseudoVMERGE : VPseudoVMRG_VM_XM_IM; //===----------------------------------------------------------------------===// // 12.16. Vector Integer Move Instructions //===----------------------------------------------------------------------===// -defm PseudoVMV_V : VPseudoUnaryV_V_X_I_NoDummyMask; +defm PseudoVMV_V : VPseudoUnaryVMV_V_X_I; //===----------------------------------------------------------------------===// // 13.1. Vector Single-Width Saturating Add and Subtract //===----------------------------------------------------------------------===// let Defs = [VXSAT], hasSideEffects = 1 in { - defm PseudoVSADDU : VPseudoBinaryV_VV_VX_VI; - defm PseudoVSADD : VPseudoBinaryV_VV_VX_VI; - defm PseudoVSSUBU : VPseudoBinaryV_VV_VX; - defm PseudoVSSUB : VPseudoBinaryV_VV_VX; + defm PseudoVSADDU : VPseudoVSALU_VV_VX_VI; + defm PseudoVSADD : VPseudoVSALU_VV_VX_VI; + defm PseudoVSSUBU : VPseudoVSALU_VV_VX; + defm PseudoVSSUB : VPseudoVSALU_VV_VX; } //===----------------------------------------------------------------------===// // 13.2. Vector Single-Width Averaging Add and Subtract //===----------------------------------------------------------------------===// let Uses = [VXRM], hasSideEffects = 1 in { - defm PseudoVAADDU : VPseudoBinaryV_VV_VX; - defm PseudoVAADD : VPseudoBinaryV_VV_VX; - defm PseudoVASUBU : VPseudoBinaryV_VV_VX; - defm PseudoVASUB : VPseudoBinaryV_VV_VX; + defm PseudoVAADDU : VPseudoVAALU_VV_VX; + defm PseudoVAADD : VPseudoVAALU_VV_VX; + defm PseudoVASUBU : VPseudoVAALU_VV_VX; + defm PseudoVASUB : VPseudoVAALU_VV_VX; } //===----------------------------------------------------------------------===// // 13.3. Vector Single-Width Fractional Multiply with Rounding and Saturation //===----------------------------------------------------------------------===// let Uses = [VXRM], Defs = [VXSAT], hasSideEffects = 1 in { - defm PseudoVSMUL : VPseudoBinaryV_VV_VX; + defm PseudoVSMUL : VPseudoVSMUL_VV_VX; } //===----------------------------------------------------------------------===// // 13.4. Vector Single-Width Scaling Shift Instructions //===----------------------------------------------------------------------===// let Uses = [VXRM], hasSideEffects = 1 in { - defm PseudoVSSRL : VPseudoBinaryV_VV_VX_VI<uimm5>; - defm PseudoVSSRA : VPseudoBinaryV_VV_VX_VI<uimm5>; + defm PseudoVSSRL : VPseudoVSSHT_VV_VX_VI<uimm5>; + defm PseudoVSSRA : VPseudoVSSHT_VV_VX_VI<uimm5>; } //===----------------------------------------------------------------------===// // 13.5. Vector Narrowing Fixed-Point Clip Instructions //===----------------------------------------------------------------------===// let Uses = [VXRM], Defs = [VXSAT], hasSideEffects = 1 in { - defm PseudoVNCLIP : VPseudoBinaryV_WV_WX_WI; - defm PseudoVNCLIPU : VPseudoBinaryV_WV_WX_WI; + defm PseudoVNCLIP : VPseudoVNCLP_WV_WX_WI; + defm PseudoVNCLIPU : VPseudoVNCLP_WV_WX_WI; } } // Predicates = [HasVInstructions] @@ -3825,156 +4062,156 @@ let Predicates = [HasVInstructionsAnyF] in { //===----------------------------------------------------------------------===// // 14.2. Vector Single-Width Floating-Point Add/Subtract Instructions //===----------------------------------------------------------------------===// -defm PseudoVFADD : VPseudoBinaryV_VV_VF; -defm PseudoVFSUB : VPseudoBinaryV_VV_VF; -defm PseudoVFRSUB : VPseudoBinaryV_VF; +defm PseudoVFADD : VPseudoVALU_VV_VF; +defm PseudoVFSUB : VPseudoVALU_VV_VF; +defm PseudoVFRSUB : VPseudoVALU_VF; //===----------------------------------------------------------------------===// // 14.3. Vector Widening Floating-Point Add/Subtract Instructions //===----------------------------------------------------------------------===// -defm PseudoVFWADD : VPseudoBinaryW_VV_VF; -defm PseudoVFWSUB : VPseudoBinaryW_VV_VF; -defm PseudoVFWADD : VPseudoBinaryW_WV_WF; -defm PseudoVFWSUB : VPseudoBinaryW_WV_WF; +defm PseudoVFWADD : VPseudoVFWALU_VV_VF; +defm PseudoVFWSUB : VPseudoVFWALU_VV_VF; +defm PseudoVFWADD : VPseudoVFWALU_WV_WF; +defm PseudoVFWSUB : VPseudoVFWALU_WV_WF; //===----------------------------------------------------------------------===// // 14.4. Vector Single-Width Floating-Point Multiply/Divide Instructions //===----------------------------------------------------------------------===// -defm PseudoVFMUL : VPseudoBinaryV_VV_VF; -defm PseudoVFDIV : VPseudoBinaryV_VV_VF; -defm PseudoVFRDIV : VPseudoBinaryV_VF; +defm PseudoVFMUL : VPseudoVFMUL_VV_VF; +defm PseudoVFDIV : VPseudoVFDIV_VV_VF; +defm PseudoVFRDIV : VPseudoVFRDIV_VF; //===----------------------------------------------------------------------===// // 14.5. Vector Widening Floating-Point Multiply //===----------------------------------------------------------------------===// -defm PseudoVFWMUL : VPseudoBinaryW_VV_VF; +defm PseudoVFWMUL : VPseudoVWMUL_VV_VF; //===----------------------------------------------------------------------===// // 14.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions //===----------------------------------------------------------------------===// -defm PseudoVFMACC : VPseudoTernaryV_VV_VF_AAXA; -defm PseudoVFNMACC : VPseudoTernaryV_VV_VF_AAXA; -defm PseudoVFMSAC : VPseudoTernaryV_VV_VF_AAXA; -defm PseudoVFNMSAC : VPseudoTernaryV_VV_VF_AAXA; -defm PseudoVFMADD : VPseudoTernaryV_VV_VF_AAXA; -defm PseudoVFNMADD : VPseudoTernaryV_VV_VF_AAXA; -defm PseudoVFMSUB : VPseudoTernaryV_VV_VF_AAXA; -defm PseudoVFNMSUB : VPseudoTernaryV_VV_VF_AAXA; +defm PseudoVFMACC : VPseudoVMAC_VV_VF_AAXA; +defm PseudoVFNMACC : VPseudoVMAC_VV_VF_AAXA; +defm PseudoVFMSAC : VPseudoVMAC_VV_VF_AAXA; +defm PseudoVFNMSAC : VPseudoVMAC_VV_VF_AAXA; +defm PseudoVFMADD : VPseudoVMAC_VV_VF_AAXA; +defm PseudoVFNMADD : VPseudoVMAC_VV_VF_AAXA; +defm PseudoVFMSUB : VPseudoVMAC_VV_VF_AAXA; +defm PseudoVFNMSUB : VPseudoVMAC_VV_VF_AAXA; //===----------------------------------------------------------------------===// // 14.7. Vector Widening Floating-Point Fused Multiply-Add Instructions //===----------------------------------------------------------------------===// -defm PseudoVFWMACC : VPseudoTernaryW_VV_VF; -defm PseudoVFWNMACC : VPseudoTernaryW_VV_VF; -defm PseudoVFWMSAC : VPseudoTernaryW_VV_VF; -defm PseudoVFWNMSAC : VPseudoTernaryW_VV_VF; +defm PseudoVFWMACC : VPseudoVWMAC_VV_VF; +defm PseudoVFWNMACC : VPseudoVWMAC_VV_VF; +defm PseudoVFWMSAC : VPseudoVWMAC_VV_VF; +defm PseudoVFWNMSAC : VPseudoVWMAC_VV_VF; //===----------------------------------------------------------------------===// // 14.8. Vector Floating-Point Square-Root Instruction //===----------------------------------------------------------------------===// -defm PseudoVFSQRT : VPseudoUnaryTAV_V; +defm PseudoVFSQRT : VPseudoVSQR_V; //===----------------------------------------------------------------------===// // 14.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction //===----------------------------------------------------------------------===// -defm PseudoVFRSQRT7 : VPseudoUnaryTAV_V; +defm PseudoVFRSQRT7 : VPseudoVRCP_V; //===----------------------------------------------------------------------===// // 14.10. Vector Floating-Point Reciprocal Estimate Instruction //===----------------------------------------------------------------------===// -defm PseudoVFREC7 : VPseudoUnaryTAV_V; +defm PseudoVFREC7 : VPseudoVRCP_V; //===----------------------------------------------------------------------===// // 14.11. Vector Floating-Point Min/Max Instructions //===----------------------------------------------------------------------===// -defm PseudoVFMIN : VPseudoBinaryV_VV_VF; -defm PseudoVFMAX : VPseudoBinaryV_VV_VF; +defm PseudoVFMIN : VPseudoVMAX_VV_VF; +defm PseudoVFMAX : VPseudoVMAX_VV_VF; //===----------------------------------------------------------------------===// // 14.12. Vector Floating-Point Sign-Injection Instructions //===----------------------------------------------------------------------===// -defm PseudoVFSGNJ : VPseudoBinaryV_VV_VF; -defm PseudoVFSGNJN : VPseudoBinaryV_VV_VF; -defm PseudoVFSGNJX : VPseudoBinaryV_VV_VF; +defm PseudoVFSGNJ : VPseudoVSGNJ_VV_VF; +defm PseudoVFSGNJN : VPseudoVSGNJ_VV_VF; +defm PseudoVFSGNJX : VPseudoVSGNJ_VV_VF; //===----------------------------------------------------------------------===// // 14.13. Vector Floating-Point Compare Instructions //===----------------------------------------------------------------------===// -defm PseudoVMFEQ : VPseudoBinaryM_VV_VF; -defm PseudoVMFNE : VPseudoBinaryM_VV_VF; -defm PseudoVMFLT : VPseudoBinaryM_VV_VF; -defm PseudoVMFLE : VPseudoBinaryM_VV_VF; -defm PseudoVMFGT : VPseudoBinaryM_VF; -defm PseudoVMFGE : VPseudoBinaryM_VF; +defm PseudoVMFEQ : VPseudoVCMPM_VV_VF; +defm PseudoVMFNE : VPseudoVCMPM_VV_VF; +defm PseudoVMFLT : VPseudoVCMPM_VV_VF; +defm PseudoVMFLE : VPseudoVCMPM_VV_VF; +defm PseudoVMFGT : VPseudoVCMPM_VF; +defm PseudoVMFGE : VPseudoVCMPM_VF; //===----------------------------------------------------------------------===// // 14.14. Vector Floating-Point Classify Instruction //===----------------------------------------------------------------------===// -defm PseudoVFCLASS : VPseudoUnaryV_V; +defm PseudoVFCLASS : VPseudoVCLS_V; //===----------------------------------------------------------------------===// // 14.15. Vector Floating-Point Merge Instruction //===----------------------------------------------------------------------===// -defm PseudoVFMERGE : VPseudoBinaryV_FM; +defm PseudoVFMERGE : VPseudoVMRG_FM; //===----------------------------------------------------------------------===// // 14.16. Vector Floating-Point Move Instruction //===----------------------------------------------------------------------===// -defm PseudoVFMV_V : VPseudoUnaryV_F_NoDummyMask; +defm PseudoVFMV_V : VPseudoVMV_F; //===----------------------------------------------------------------------===// // 14.17. Single-Width Floating-Point/Integer Type-Convert Instructions //===----------------------------------------------------------------------===// -defm PseudoVFCVT_XU_F : VPseudoConversionV_V; -defm PseudoVFCVT_X_F : VPseudoConversionV_V; -defm PseudoVFCVT_RTZ_XU_F : VPseudoConversionV_V; -defm PseudoVFCVT_RTZ_X_F : VPseudoConversionV_V; -defm PseudoVFCVT_F_XU : VPseudoConversionV_V; -defm PseudoVFCVT_F_X : VPseudoConversionV_V; +defm PseudoVFCVT_XU_F : VPseudoVCVTI_V; +defm PseudoVFCVT_X_F : VPseudoVCVTI_V; +defm PseudoVFCVT_RTZ_XU_F : VPseudoVCVTI_V; +defm PseudoVFCVT_RTZ_X_F : VPseudoVCVTI_V; +defm PseudoVFCVT_F_XU : VPseudoVCVTF_V; +defm PseudoVFCVT_F_X : VPseudoVCVTF_V; //===----------------------------------------------------------------------===// // 14.18. Widening Floating-Point/Integer Type-Convert Instructions //===----------------------------------------------------------------------===// -defm PseudoVFWCVT_XU_F : VPseudoConversionW_V; -defm PseudoVFWCVT_X_F : VPseudoConversionW_V; -defm PseudoVFWCVT_RTZ_XU_F : VPseudoConversionW_V; -defm PseudoVFWCVT_RTZ_X_F : VPseudoConversionW_V; -defm PseudoVFWCVT_F_XU : VPseudoConversionW_V; -defm PseudoVFWCVT_F_X : VPseudoConversionW_V; -defm PseudoVFWCVT_F_F : VPseudoConversionW_V; +defm PseudoVFWCVT_XU_F : VPseudoVWCVTI_V; +defm PseudoVFWCVT_X_F : VPseudoVWCVTI_V; +defm PseudoVFWCVT_RTZ_XU_F : VPseudoVWCVTI_V; +defm PseudoVFWCVT_RTZ_X_F : VPseudoVWCVTI_V; +defm PseudoVFWCVT_F_XU : VPseudoVWCVTF_V; +defm PseudoVFWCVT_F_X : VPseudoVWCVTF_V; +defm PseudoVFWCVT_F_F : VPseudoVWCVTD_V; //===----------------------------------------------------------------------===// // 14.19. Narrowing Floating-Point/Integer Type-Convert Instructions //===----------------------------------------------------------------------===// -defm PseudoVFNCVT_XU_F : VPseudoConversionV_W; -defm PseudoVFNCVT_X_F : VPseudoConversionV_W; -defm PseudoVFNCVT_RTZ_XU_F : VPseudoConversionV_W; -defm PseudoVFNCVT_RTZ_X_F : VPseudoConversionV_W; -defm PseudoVFNCVT_F_XU : VPseudoConversionV_W; -defm PseudoVFNCVT_F_X : VPseudoConversionV_W; -defm PseudoVFNCVT_F_F : VPseudoConversionV_W; -defm PseudoVFNCVT_ROD_F_F : VPseudoConversionV_W; +defm PseudoVFNCVT_XU_F : VPseudoVNCVTI_W; +defm PseudoVFNCVT_X_F : VPseudoVNCVTI_W; +defm PseudoVFNCVT_RTZ_XU_F : VPseudoVNCVTI_W; +defm PseudoVFNCVT_RTZ_X_F : VPseudoVNCVTI_W; +defm PseudoVFNCVT_F_XU : VPseudoVNCVTF_W; +defm PseudoVFNCVT_F_X : VPseudoVNCVTF_W; +defm PseudoVFNCVT_F_F : VPseudoVNCVTD_W; +defm PseudoVFNCVT_ROD_F_F : VPseudoVNCVTD_W; } // Predicates = [HasVInstructionsAnyF] let Predicates = [HasVInstructions] in { //===----------------------------------------------------------------------===// // 15.1. Vector Single-Width Integer Reduction Instructions //===----------------------------------------------------------------------===// -defm PseudoVREDSUM : VPseudoReductionV_VS; -defm PseudoVREDAND : VPseudoReductionV_VS; -defm PseudoVREDOR : VPseudoReductionV_VS; -defm PseudoVREDXOR : VPseudoReductionV_VS; -defm PseudoVREDMINU : VPseudoReductionV_VS; -defm PseudoVREDMIN : VPseudoReductionV_VS; -defm PseudoVREDMAXU : VPseudoReductionV_VS; -defm PseudoVREDMAX : VPseudoReductionV_VS; +defm PseudoVREDSUM : VPseudoVRED_VS; +defm PseudoVREDAND : VPseudoVRED_VS; +defm PseudoVREDOR : VPseudoVRED_VS; +defm PseudoVREDXOR : VPseudoVRED_VS; +defm PseudoVREDMINU : VPseudoVRED_VS; +defm PseudoVREDMIN : VPseudoVRED_VS; +defm PseudoVREDMAXU : VPseudoVRED_VS; +defm PseudoVREDMAX : VPseudoVRED_VS; //===----------------------------------------------------------------------===// // 15.2. Vector Widening Integer Reduction Instructions //===----------------------------------------------------------------------===// let IsRVVWideningReduction = 1 in { -defm PseudoVWREDSUMU : VPseudoReductionV_VS; -defm PseudoVWREDSUM : VPseudoReductionV_VS; +defm PseudoVWREDSUMU : VPseudoVWRED_VS; +defm PseudoVWREDSUM : VPseudoVWRED_VS; } } // Predicates = [HasVInstructions] @@ -3982,17 +4219,17 @@ let Predicates = [HasVInstructionsAnyF] in { //===----------------------------------------------------------------------===// // 15.3. Vector Single-Width Floating-Point Reduction Instructions //===----------------------------------------------------------------------===// -defm PseudoVFREDOSUM : VPseudoReductionV_VS; -defm PseudoVFREDUSUM : VPseudoReductionV_VS; -defm PseudoVFREDMIN : VPseudoReductionV_VS; -defm PseudoVFREDMAX : VPseudoReductionV_VS; +defm PseudoVFREDOSUM : VPseudoVFREDO_VS; +defm PseudoVFREDUSUM : VPseudoVFRED_VS; +defm PseudoVFREDMIN : VPseudoVFRED_VS; +defm PseudoVFREDMAX : VPseudoVFRED_VS; //===----------------------------------------------------------------------===// // 15.4. Vector Widening Floating-Point Reduction Instructions //===----------------------------------------------------------------------===// let IsRVVWideningReduction = 1 in { -defm PseudoVFWREDUSUM : VPseudoReductionV_VS; -defm PseudoVFWREDOSUM : VPseudoReductionV_VS; +defm PseudoVFWREDUSUM : VPseudoVFWRED_VS; +defm PseudoVFWREDOSUM : VPseudoVFWRED_VS; } } // Predicates = [HasVInstructionsAnyF] @@ -4005,55 +4242,57 @@ defm PseudoVFWREDOSUM : VPseudoReductionV_VS; // 16.1 Vector Mask-Register Logical Instructions //===----------------------------------------------------------------------===// -defm PseudoVMAND: VPseudoBinaryM_MM; -defm PseudoVMNAND: VPseudoBinaryM_MM; -defm PseudoVMANDN: VPseudoBinaryM_MM; -defm PseudoVMXOR: VPseudoBinaryM_MM; -defm PseudoVMOR: VPseudoBinaryM_MM; -defm PseudoVMNOR: VPseudoBinaryM_MM; -defm PseudoVMORN: VPseudoBinaryM_MM; -defm PseudoVMXNOR: VPseudoBinaryM_MM; +defm PseudoVMAND: VPseudoVALU_MM; +defm PseudoVMNAND: VPseudoVALU_MM; +defm PseudoVMANDN: VPseudoVALU_MM; +defm PseudoVMXOR: VPseudoVALU_MM; +defm PseudoVMOR: VPseudoVALU_MM; +defm PseudoVMNOR: VPseudoVALU_MM; +defm PseudoVMORN: VPseudoVALU_MM; +defm PseudoVMXNOR: VPseudoVALU_MM; // Pseudo instructions -defm PseudoVMCLR : VPseudoNullaryPseudoM<"VMXOR">; -defm PseudoVMSET : VPseudoNullaryPseudoM<"VMXNOR">; +defm PseudoVMCLR : VPseudoNullaryPseudoM<"VMXOR">, + Sched<[WriteVMALUV, ReadVMALUV, ReadVMALUV]>; +defm PseudoVMSET : VPseudoNullaryPseudoM<"VMXNOR">, + Sched<[WriteVMALUV, ReadVMALUV, ReadVMALUV]>; //===----------------------------------------------------------------------===// // 16.2. Vector mask population count vcpop //===----------------------------------------------------------------------===// -defm PseudoVCPOP: VPseudoUnaryS_M; +defm PseudoVCPOP: VPseudoVPOP_M; //===----------------------------------------------------------------------===// // 16.3. vfirst find-first-set mask bit //===----------------------------------------------------------------------===// -defm PseudoVFIRST: VPseudoUnaryS_M; +defm PseudoVFIRST: VPseudoV1ST_M; //===----------------------------------------------------------------------===// // 16.4. vmsbf.m set-before-first mask bit //===----------------------------------------------------------------------===// -defm PseudoVMSBF: VPseudoUnaryM_M; +defm PseudoVMSBF: VPseudoVSFS_M; //===----------------------------------------------------------------------===// // 16.5. vmsif.m set-including-first mask bit //===----------------------------------------------------------------------===// -defm PseudoVMSIF: VPseudoUnaryM_M; +defm PseudoVMSIF: VPseudoVSFS_M; //===----------------------------------------------------------------------===// // 16.6. vmsof.m set-only-first mask bit //===----------------------------------------------------------------------===// -defm PseudoVMSOF: VPseudoUnaryM_M; +defm PseudoVMSOF: VPseudoVSFS_M; //===----------------------------------------------------------------------===// // 16.8. Vector Iota Instruction //===----------------------------------------------------------------------===// -defm PseudoVIOTA_M: VPseudoUnaryV_M; +defm PseudoVIOTA_M: VPseudoVIOT_M; //===----------------------------------------------------------------------===// // 16.9. Vector Element Index Instruction //===----------------------------------------------------------------------===// -defm PseudoVID : VPseudoMaskNullaryV; +defm PseudoVID : VPseudoVID_V; //===----------------------------------------------------------------------===// // 17. Vector Permutation Instructions @@ -4068,15 +4307,18 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in { foreach m = MxList.m in { let VLMul = m.value in { let HasSEWOp = 1, BaseInstr = VMV_X_S in - def PseudoVMV_X_S # "_" # m.MX: Pseudo<(outs GPR:$rd), - (ins m.vrclass:$rs2, ixlenimm:$sew), - []>, RISCVVPseudo; + def PseudoVMV_X_S # "_" # m.MX: + Pseudo<(outs GPR:$rd), (ins m.vrclass:$rs2, ixlenimm:$sew), []>, + Sched<[WriteVIMovVX, ReadVIMovVX]>, + RISCVVPseudo; let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VMV_S_X, Constraints = "$rd = $rs1" in def PseudoVMV_S_X # "_" # m.MX: Pseudo<(outs m.vrclass:$rd), (ins m.vrclass:$rs1, GPR:$rs2, AVL:$vl, ixlenimm:$sew), - []>, RISCVVPseudo; + []>, + Sched<[WriteVIMovXV, ReadVIMovXV, ReadVIMovXX]>, + RISCVVPseudo; } } } @@ -4093,17 +4335,19 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in { let VLMul = m.value in { let HasSEWOp = 1, BaseInstr = VFMV_F_S in def "PseudoVFMV_" # f.FX # "_S_" # m.MX : - Pseudo<(outs f.fprclass:$rd), - (ins m.vrclass:$rs2, - ixlenimm:$sew), - []>, RISCVVPseudo; + Pseudo<(outs f.fprclass:$rd), + (ins m.vrclass:$rs2, ixlenimm:$sew), []>, + Sched<[WriteVFMovVF, ReadVFMovVF]>, + RISCVVPseudo; let HasVLOp = 1, HasSEWOp = 1, BaseInstr = VFMV_S_F, Constraints = "$rd = $rs1" in def "PseudoVFMV_S_" # f.FX # "_" # m.MX : Pseudo<(outs m.vrclass:$rd), (ins m.vrclass:$rs1, f.fprclass:$rs2, AVL:$vl, ixlenimm:$sew), - []>, RISCVVPseudo; + []>, + Sched<[WriteVFMovFV, ReadVFMovFV, ReadVFMovFX]>, + RISCVVPseudo; } } } @@ -4114,52 +4358,33 @@ let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in { // 17.3. Vector Slide Instructions //===----------------------------------------------------------------------===// let Predicates = [HasVInstructions] in { - defm PseudoVSLIDEUP : VPseudoTernaryV_VX_VI<uimm5, "@earlyclobber $rd">; - defm PseudoVSLIDEDOWN : VPseudoTernaryV_VX_VI<uimm5>; - defm PseudoVSLIDE1UP : VPseudoBinaryV_VX<"@earlyclobber $rd">; - defm PseudoVSLIDE1DOWN : VPseudoBinaryV_VX; + defm PseudoVSLIDEUP : VPseudoVSLD_VX_VI<uimm5, "@earlyclobber $rd">; + defm PseudoVSLIDEDOWN : VPseudoVSLD_VX_VI<uimm5>; + defm PseudoVSLIDE1UP : VPseudoVSLD1_VX<"@earlyclobber $rd">; + defm PseudoVSLIDE1DOWN : VPseudoVSLD1_VX; } // Predicates = [HasVInstructions] let Predicates = [HasVInstructionsAnyF] in { - defm PseudoVFSLIDE1UP : VPseudoBinaryV_VF<"@earlyclobber $rd">; - defm PseudoVFSLIDE1DOWN : VPseudoBinaryV_VF; + defm PseudoVFSLIDE1UP : VPseudoVSLD1_VF<"@earlyclobber $rd">; + defm PseudoVFSLIDE1DOWN : VPseudoVSLD1_VF; } // Predicates = [HasVInstructionsAnyF] //===----------------------------------------------------------------------===// // 17.4. Vector Register Gather Instructions //===----------------------------------------------------------------------===// -defm PseudoVRGATHER : VPseudoBinaryV_VV_VX_VI<uimm5, "@earlyclobber $rd">; -defm PseudoVRGATHEREI16 : VPseudoBinaryV_VV_EEW</* eew */ 16, "@earlyclobber $rd">; +defm PseudoVRGATHER : VPseudoVGTR_VV_VX_VI<uimm5, "@earlyclobber $rd">; +defm PseudoVRGATHEREI16 : VPseudoVGTR_VV_EEW</* eew */ 16, "@earlyclobber $rd">; //===----------------------------------------------------------------------===// // 17.5. Vector Compress Instruction //===----------------------------------------------------------------------===// -defm PseudoVCOMPRESS : VPseudoUnaryV_V_AnyMask; +defm PseudoVCOMPRESS : VPseudoVCPR_V; //===----------------------------------------------------------------------===// // Patterns. //===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===// -// 8. Vector AMO Operations -//===----------------------------------------------------------------------===// -let Predicates = [HasStdExtZvamo] in { - defm : VPatAMOV_WD<"int_riscv_vamoswap", "PseudoVAMOSWAP", AllIntegerVectors>; - defm : VPatAMOV_WD<"int_riscv_vamoadd", "PseudoVAMOADD", AllIntegerVectors>; - defm : VPatAMOV_WD<"int_riscv_vamoxor", "PseudoVAMOXOR", AllIntegerVectors>; - defm : VPatAMOV_WD<"int_riscv_vamoand", "PseudoVAMOAND", AllIntegerVectors>; - defm : VPatAMOV_WD<"int_riscv_vamoor", "PseudoVAMOOR", AllIntegerVectors>; - defm : VPatAMOV_WD<"int_riscv_vamomin", "PseudoVAMOMIN", AllIntegerVectors>; - defm : VPatAMOV_WD<"int_riscv_vamomax", "PseudoVAMOMAX", AllIntegerVectors>; - defm : VPatAMOV_WD<"int_riscv_vamominu", "PseudoVAMOMINU", AllIntegerVectors>; - defm : VPatAMOV_WD<"int_riscv_vamomaxu", "PseudoVAMOMAXU", AllIntegerVectors>; -} // Predicates = [HasStdExtZvamo] - -let Predicates = [HasStdExtZvamo, HasVInstructionsAnyF] in { - defm : VPatAMOV_WD<"int_riscv_vamoswap", "PseudoVAMOSWAP", AllFloatVectors>; -} // Predicates = [HasStdExtZvamo, HasVInstructionsAnyF] - -//===----------------------------------------------------------------------===// // 12. Vector Integer Arithmetic Instructions //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td index 461bdd348934..7eb8ae7d4193 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td @@ -382,50 +382,50 @@ def FSRI : RVBTernaryImm6<0b101, OPC_OP_IMM, "fsri", } // Predicates = [HasStdExtZbt] let Predicates = [HasStdExtZbb] in { -def CLZ : RVBUnary<0b0110000, 0b00000, 0b001, RISCVOpcode<0b0010011>, "clz">, +def CLZ : RVBUnary<0b0110000, 0b00000, 0b001, OPC_OP_IMM, "clz">, Sched<[WriteCLZ, ReadCLZ]>; -def CTZ : RVBUnary<0b0110000, 0b00001, 0b001, RISCVOpcode<0b0010011>, "ctz">, +def CTZ : RVBUnary<0b0110000, 0b00001, 0b001, OPC_OP_IMM, "ctz">, Sched<[WriteCTZ, ReadCTZ]>; -def CPOP : RVBUnary<0b0110000, 0b00010, 0b001, RISCVOpcode<0b0010011>, "cpop">, +def CPOP : RVBUnary<0b0110000, 0b00010, 0b001, OPC_OP_IMM, "cpop">, Sched<[WriteCPOP, ReadCPOP]>; } // Predicates = [HasStdExtZbb] let Predicates = [HasStdExtZbm, IsRV64] in -def BMATFLIP : RVBUnary<0b0110000, 0b00011, 0b001, RISCVOpcode<0b0010011>, - "bmatflip">, Sched<[]>; +def BMATFLIP : RVBUnary<0b0110000, 0b00011, 0b001, OPC_OP_IMM, "bmatflip">, + Sched<[]>; let Predicates = [HasStdExtZbb] in { -def SEXTB : RVBUnary<0b0110000, 0b00100, 0b001, RISCVOpcode<0b0010011>, - "sext.b">, Sched<[WriteIALU, ReadIALU]>; -def SEXTH : RVBUnary<0b0110000, 0b00101, 0b001, RISCVOpcode<0b0010011>, - "sext.h">, Sched<[WriteIALU, ReadIALU]>; +def SEXTB : RVBUnary<0b0110000, 0b00100, 0b001, OPC_OP_IMM, "sext.b">, + Sched<[WriteIALU, ReadIALU]>; +def SEXTH : RVBUnary<0b0110000, 0b00101, 0b001, OPC_OP_IMM, "sext.h">, + Sched<[WriteIALU, ReadIALU]>; } // Predicates = [HasStdExtZbb] let Predicates = [HasStdExtZbr] in { -def CRC32B : RVBUnary<0b0110000, 0b10000, 0b001, RISCVOpcode<0b0010011>, - "crc32.b">, Sched<[]>; -def CRC32H : RVBUnary<0b0110000, 0b10001, 0b001, RISCVOpcode<0b0010011>, - "crc32.h">, Sched<[]>; -def CRC32W : RVBUnary<0b0110000, 0b10010, 0b001, RISCVOpcode<0b0010011>, - "crc32.w">, Sched<[]>; +def CRC32B : RVBUnary<0b0110000, 0b10000, 0b001, OPC_OP_IMM, "crc32.b">, + Sched<[]>; +def CRC32H : RVBUnary<0b0110000, 0b10001, 0b001, OPC_OP_IMM, "crc32.h">, + Sched<[]>; +def CRC32W : RVBUnary<0b0110000, 0b10010, 0b001, OPC_OP_IMM, "crc32.w">, + Sched<[]>; } // Predicates = [HasStdExtZbr] let Predicates = [HasStdExtZbr, IsRV64] in -def CRC32D : RVBUnary<0b0110000, 0b10011, 0b001, RISCVOpcode<0b0010011>, - "crc32.d">, Sched<[]>; +def CRC32D : RVBUnary<0b0110000, 0b10011, 0b001, OPC_OP_IMM, "crc32.d">, + Sched<[]>; let Predicates = [HasStdExtZbr] in { -def CRC32CB : RVBUnary<0b0110000, 0b11000, 0b001, RISCVOpcode<0b0010011>, - "crc32c.b">, Sched<[]>; -def CRC32CH : RVBUnary<0b0110000, 0b11001, 0b001, RISCVOpcode<0b0010011>, - "crc32c.h">, Sched<[]>; -def CRC32CW : RVBUnary<0b0110000, 0b11010, 0b001, RISCVOpcode<0b0010011>, - "crc32c.w">, Sched<[]>; +def CRC32CB : RVBUnary<0b0110000, 0b11000, 0b001, OPC_OP_IMM, "crc32c.b">, + Sched<[]>; +def CRC32CH : RVBUnary<0b0110000, 0b11001, 0b001, OPC_OP_IMM, "crc32c.h">, + Sched<[]>; +def CRC32CW : RVBUnary<0b0110000, 0b11010, 0b001, OPC_OP_IMM, "crc32c.w">, + Sched<[]>; } // Predicates = [HasStdExtZbr] let Predicates = [HasStdExtZbr, IsRV64] in -def CRC32CD : RVBUnary<0b0110000, 0b11011, 0b001, RISCVOpcode<0b0010011>, - "crc32c.d">, Sched<[]>; +def CRC32CD : RVBUnary<0b0110000, 0b11011, 0b001, OPC_OP_IMM, "crc32c.d">, + Sched<[]>; let Predicates = [HasStdExtZbc] in { def CLMUL : ALU_rr<0b0000101, 0b001, "clmul">, Sched<[]>; @@ -523,12 +523,12 @@ def FSRIW : RVBTernaryImm5<0b10, 0b101, OPC_OP_IMM_32, } // Predicates = [HasStdExtZbt, IsRV64] let Predicates = [HasStdExtZbb, IsRV64] in { -def CLZW : RVBUnary<0b0110000, 0b00000, 0b001, RISCVOpcode<0b0011011>, - "clzw">, Sched<[WriteCLZ32, ReadCLZ32]>; -def CTZW : RVBUnary<0b0110000, 0b00001, 0b001, RISCVOpcode<0b0011011>, - "ctzw">, Sched<[WriteCTZ32, ReadCTZ32]>; -def CPOPW : RVBUnary<0b0110000, 0b00010, 0b001, RISCVOpcode<0b0011011>, - "cpopw">, Sched<[WriteCPOP32, ReadCPOP32]>; +def CLZW : RVBUnary<0b0110000, 0b00000, 0b001, OPC_OP_IMM_32, "clzw">, + Sched<[WriteCLZ32, ReadCLZ32]>; +def CTZW : RVBUnary<0b0110000, 0b00001, 0b001, OPC_OP_IMM_32, "ctzw">, + Sched<[WriteCTZ32, ReadCTZ32]>; +def CPOPW : RVBUnary<0b0110000, 0b00010, 0b001, OPC_OP_IMM_32, "cpopw">, + Sched<[WriteCPOP32, ReadCPOP32]>; } // Predicates = [HasStdExtZbb, IsRV64] let Predicates = [HasStdExtZbp, IsRV64] in { @@ -791,6 +791,9 @@ def : Pat<(xor GPR:$rs1, BSETINVMask:$mask), def : Pat<(and (srl GPR:$rs1, uimmlog2xlen:$shamt), (XLenVT 1)), (BEXTI GPR:$rs1, uimmlog2xlen:$shamt)>; +def : Pat<(and (not (srl GPR:$rs1, uimmlog2xlen:$shamt)), (XLenVT 1)), + (XORI (BEXTI GPR:$rs1, uimmlog2xlen:$shamt), (XLenVT 1))>; + def : Pat<(or GPR:$r, BSETINVTwoBitsMask:$i), (BSETI (BSETI GPR:$r, (TrailingZerosXForm BSETINVTwoBitsMask:$i)), (BSETINVTwoBitsMaskHigh BSETINVTwoBitsMask:$i))>; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td index a33494461869..663e44813899 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td @@ -28,41 +28,6 @@ def riscv_fmv_x_anyexth : SDNode<"RISCVISD::FMV_X_ANYEXTH", SDT_RISCVFMV_X_ANYEXTH>; //===----------------------------------------------------------------------===// -// Instruction class templates -//===----------------------------------------------------------------------===// - -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPFMAH_rrr_frm<RISCVOpcode opcode, string opcodestr> - : RVInstR4Frm<0b10, opcode, (outs FPR16:$rd), - (ins FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, frmarg:$funct3), - opcodestr, "$rd, $rs1, $rs2, $rs3, $funct3">; - -class FPFMAHDynFrmAlias<FPFMAH_rrr_frm Inst, string OpcodeStr> - : InstAlias<OpcodeStr#" $rd, $rs1, $rs2, $rs3", - (Inst FPR16:$rd, FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>; - -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPALUH_rr<bits<7> funct7, bits<3> funct3, string opcodestr> - : RVInstR<funct7, funct3, OPC_OP_FP, (outs FPR16:$rd), - (ins FPR16:$rs1, FPR16:$rs2), opcodestr, "$rd, $rs1, $rs2">; - -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPALUH_rr_frm<bits<7> funct7, string opcodestr> - : RVInstRFrm<funct7, OPC_OP_FP, (outs FPR16:$rd), - (ins FPR16:$rs1, FPR16:$rs2, frmarg:$funct3), opcodestr, - "$rd, $rs1, $rs2, $funct3">; - -class FPALUHDynFrmAlias<FPALUH_rr_frm Inst, string OpcodeStr> - : InstAlias<OpcodeStr#" $rd, $rs1, $rs2", - (Inst FPR16:$rd, FPR16:$rs1, FPR16:$rs2, 0b111)>; - -let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in -class FPCmpH_rr<bits<3> funct3, string opcodestr> - : RVInstR<0b1010010, funct3, OPC_OP_FP, (outs GPR:$rd), - (ins FPR16:$rs1, FPR16:$rs2), opcodestr, "$rd, $rs1, $rs2">, - Sched<[WriteFCmp16, ReadFCmp16, ReadFCmp16]>; - -//===----------------------------------------------------------------------===// // Instructions //===----------------------------------------------------------------------===// @@ -84,145 +49,120 @@ def FSH : RVInstS<0b001, OPC_STORE_FP, (outs), } // Predicates = [HasStdExtZfhmin] let Predicates = [HasStdExtZfh] in { -def FMADD_H : FPFMAH_rrr_frm<OPC_MADD, "fmadd.h">, - Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>; -def : FPFMAHDynFrmAlias<FMADD_H, "fmadd.h">; -def FMSUB_H : FPFMAH_rrr_frm<OPC_MSUB, "fmsub.h">, - Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>; -def : FPFMAHDynFrmAlias<FMSUB_H, "fmsub.h">; -def FNMSUB_H : FPFMAH_rrr_frm<OPC_NMSUB, "fnmsub.h">, - Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>; -def : FPFMAHDynFrmAlias<FNMSUB_H, "fnmsub.h">; -def FNMADD_H : FPFMAH_rrr_frm<OPC_NMADD, "fnmadd.h">, - Sched<[WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16]>; -def : FPFMAHDynFrmAlias<FNMADD_H, "fnmadd.h">; +let SchedRW = [WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16] in { +def FMADD_H : FPFMA_rrr_frm<OPC_MADD, 0b10, "fmadd.h", FPR16>; +def FMSUB_H : FPFMA_rrr_frm<OPC_MSUB, 0b10, "fmsub.h", FPR16>; +def FNMSUB_H : FPFMA_rrr_frm<OPC_NMSUB, 0b10, "fnmsub.h", FPR16>; +def FNMADD_H : FPFMA_rrr_frm<OPC_NMADD, 0b10, "fnmadd.h", FPR16>; +} -def FADD_H : FPALUH_rr_frm<0b0000010, "fadd.h">, +def : FPFMADynFrmAlias<FMADD_H, "fmadd.h", FPR16>; +def : FPFMADynFrmAlias<FMSUB_H, "fmsub.h", FPR16>; +def : FPFMADynFrmAlias<FNMSUB_H, "fnmsub.h", FPR16>; +def : FPFMADynFrmAlias<FNMADD_H, "fnmadd.h", FPR16>; + +def FADD_H : FPALU_rr_frm<0b0000010, "fadd.h", FPR16>, Sched<[WriteFALU16, ReadFALU16, ReadFALU16]>; -def : FPALUHDynFrmAlias<FADD_H, "fadd.h">; -def FSUB_H : FPALUH_rr_frm<0b0000110, "fsub.h">, +def FSUB_H : FPALU_rr_frm<0b0000110, "fsub.h", FPR16>, Sched<[WriteFALU16, ReadFALU16, ReadFALU16]>; -def : FPALUHDynFrmAlias<FSUB_H, "fsub.h">; -def FMUL_H : FPALUH_rr_frm<0b0001010, "fmul.h">, +def FMUL_H : FPALU_rr_frm<0b0001010, "fmul.h", FPR16>, Sched<[WriteFMul16, ReadFMul16, ReadFMul16]>; -def : FPALUHDynFrmAlias<FMUL_H, "fmul.h">; -def FDIV_H : FPALUH_rr_frm<0b0001110, "fdiv.h">, +def FDIV_H : FPALU_rr_frm<0b0001110, "fdiv.h", FPR16>, Sched<[WriteFDiv16, ReadFDiv16, ReadFDiv16]>; -def : FPALUHDynFrmAlias<FDIV_H, "fdiv.h">; -def FSQRT_H : FPUnaryOp_r_frm<0b0101110, FPR16, FPR16, "fsqrt.h">, - Sched<[WriteFSqrt16, ReadFSqrt16]> { - let rs2 = 0b00000; -} +def : FPALUDynFrmAlias<FADD_H, "fadd.h", FPR16>; +def : FPALUDynFrmAlias<FSUB_H, "fsub.h", FPR16>; +def : FPALUDynFrmAlias<FMUL_H, "fmul.h", FPR16>; +def : FPALUDynFrmAlias<FDIV_H, "fdiv.h", FPR16>; + +def FSQRT_H : FPUnaryOp_r_frm<0b0101110, 0b00000, FPR16, FPR16, "fsqrt.h">, + Sched<[WriteFSqrt16, ReadFSqrt16]>; def : FPUnaryOpDynFrmAlias<FSQRT_H, "fsqrt.h", FPR16, FPR16>; -def FSGNJ_H : FPALUH_rr<0b0010010, 0b000, "fsgnj.h">, - Sched<[WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16]>; -def FSGNJN_H : FPALUH_rr<0b0010010, 0b001, "fsgnjn.h">, - Sched<[WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16]>; -def FSGNJX_H : FPALUH_rr<0b0010010, 0b010, "fsgnjx.h">, - Sched<[WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16]>; -def FMIN_H : FPALUH_rr<0b0010110, 0b000, "fmin.h">, - Sched<[WriteFMinMax16, ReadFMinMax16, ReadFMinMax16]>; -def FMAX_H : FPALUH_rr<0b0010110, 0b001, "fmax.h">, - Sched<[WriteFMinMax16, ReadFMinMax16, ReadFMinMax16]>; +let SchedRW = [WriteFSGNJ16, ReadFSGNJ16, ReadFSGNJ16], + mayRaiseFPException = 0 in { +def FSGNJ_H : FPALU_rr<0b0010010, 0b000, "fsgnj.h", FPR16>; +def FSGNJN_H : FPALU_rr<0b0010010, 0b001, "fsgnjn.h", FPR16>; +def FSGNJX_H : FPALU_rr<0b0010010, 0b010, "fsgnjx.h", FPR16>; +} -def FCVT_W_H : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.w.h">, - Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]> { - let rs2 = 0b00000; +let SchedRW = [WriteFMinMax16, ReadFMinMax16, ReadFMinMax16] in { +def FMIN_H : FPALU_rr<0b0010110, 0b000, "fmin.h", FPR16>; +def FMAX_H : FPALU_rr<0b0010110, 0b001, "fmax.h", FPR16>; } + +def FCVT_W_H : FPUnaryOp_r_frm<0b1100010, 0b00000, GPR, FPR16, "fcvt.w.h">, + Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]>; def : FPUnaryOpDynFrmAlias<FCVT_W_H, "fcvt.w.h", GPR, FPR16>; -def FCVT_WU_H : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.wu.h">, - Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]> { - let rs2 = 0b00001; -} +def FCVT_WU_H : FPUnaryOp_r_frm<0b1100010, 0b00001, GPR, FPR16, "fcvt.wu.h">, + Sched<[WriteFCvtF16ToI32, ReadFCvtF16ToI32]>; def : FPUnaryOpDynFrmAlias<FCVT_WU_H, "fcvt.wu.h", GPR, FPR16>; -def FCVT_H_W : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.w">, - Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]> { - let rs2 = 0b00000; -} +def FCVT_H_W : FPUnaryOp_r_frm<0b1101010, 0b00000, FPR16, GPR, "fcvt.h.w">, + Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]>; def : FPUnaryOpDynFrmAlias<FCVT_H_W, "fcvt.h.w", FPR16, GPR>; -def FCVT_H_WU : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.wu">, - Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]> { - let rs2 = 0b00001; -} +def FCVT_H_WU : FPUnaryOp_r_frm<0b1101010, 0b00001, FPR16, GPR, "fcvt.h.wu">, + Sched<[WriteFCvtI32ToF16, ReadFCvtI32ToF16]>; def : FPUnaryOpDynFrmAlias<FCVT_H_WU, "fcvt.h.wu", FPR16, GPR>; } // Predicates = [HasStdExtZfh] let Predicates = [HasStdExtZfhmin] in { -def FCVT_H_S : FPUnaryOp_r_frm<0b0100010, FPR16, FPR32, "fcvt.h.s">, - Sched<[WriteFCvtF32ToF16, ReadFCvtF32ToF16]> { - let rs2 = 0b00000; -} +def FCVT_H_S : FPUnaryOp_r_frm<0b0100010, 0b00000, FPR16, FPR32, "fcvt.h.s">, + Sched<[WriteFCvtF32ToF16, ReadFCvtF32ToF16]>; def : FPUnaryOpDynFrmAlias<FCVT_H_S, "fcvt.h.s", FPR16, FPR32>; -def FCVT_S_H : FPUnaryOp_r<0b0100000, 0b000, FPR32, FPR16, "fcvt.s.h">, - Sched<[WriteFCvtF16ToF32, ReadFCvtF16ToF32]> { - let rs2 = 0b00010; -} +def FCVT_S_H : FPUnaryOp_r<0b0100000, 0b00010, 0b000, FPR32, FPR16, "fcvt.s.h">, + Sched<[WriteFCvtF16ToF32, ReadFCvtF16ToF32]>; -def FMV_X_H : FPUnaryOp_r<0b1110010, 0b000, GPR, FPR16, "fmv.x.h">, - Sched<[WriteFMovF16ToI16, ReadFMovF16ToI16]> { - let rs2 = 0b00000; -} +let mayRaiseFPException = 0 in +def FMV_X_H : FPUnaryOp_r<0b1110010, 0b00000, 0b000, GPR, FPR16, "fmv.x.h">, + Sched<[WriteFMovF16ToI16, ReadFMovF16ToI16]>; -def FMV_H_X : FPUnaryOp_r<0b1111010, 0b000, FPR16, GPR, "fmv.h.x">, - Sched<[WriteFMovI16ToF16, ReadFMovI16ToF16]> { - let rs2 = 0b00000; -} +let mayRaiseFPException = 0 in +def FMV_H_X : FPUnaryOp_r<0b1111010, 0b00000, 0b000, FPR16, GPR, "fmv.h.x">, + Sched<[WriteFMovI16ToF16, ReadFMovI16ToF16]>; } // Predicates = [HasStdExtZfhmin] let Predicates = [HasStdExtZfh] in { -def FEQ_H : FPCmpH_rr<0b010, "feq.h">; -def FLT_H : FPCmpH_rr<0b001, "flt.h">; -def FLE_H : FPCmpH_rr<0b000, "fle.h">; -def FCLASS_H : FPUnaryOp_r<0b1110010, 0b001, GPR, FPR16, "fclass.h">, - Sched<[WriteFClass16, ReadFClass16]> { - let rs2 = 0b00000; +let SchedRW = [WriteFCmp16, ReadFCmp16, ReadFCmp16] in { +def FEQ_H : FPCmp_rr<0b1010010, 0b010, "feq.h", FPR16>; +def FLT_H : FPCmp_rr<0b1010010, 0b001, "flt.h", FPR16>; +def FLE_H : FPCmp_rr<0b1010010, 0b000, "fle.h", FPR16>; } + +let mayRaiseFPException = 0 in +def FCLASS_H : FPUnaryOp_r<0b1110010, 0b00000, 0b001, GPR, FPR16, "fclass.h">, + Sched<[WriteFClass16, ReadFClass16]>; } // Predicates = [HasStdExtZfh] let Predicates = [HasStdExtZfh, IsRV64] in { -def FCVT_L_H : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.l.h">, - Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]> { - let rs2 = 0b00010; -} +def FCVT_L_H : FPUnaryOp_r_frm<0b1100010, 0b00010, GPR, FPR16, "fcvt.l.h">, + Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]>; def : FPUnaryOpDynFrmAlias<FCVT_L_H, "fcvt.l.h", GPR, FPR16>; -def FCVT_LU_H : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.lu.h">, - Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]> { - let rs2 = 0b00011; -} +def FCVT_LU_H : FPUnaryOp_r_frm<0b1100010, 0b00011, GPR, FPR16, "fcvt.lu.h">, + Sched<[WriteFCvtF16ToI64, ReadFCvtF16ToI64]>; def : FPUnaryOpDynFrmAlias<FCVT_LU_H, "fcvt.lu.h", GPR, FPR16>; -def FCVT_H_L : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.l">, - Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]> { - let rs2 = 0b00010; -} +def FCVT_H_L : FPUnaryOp_r_frm<0b1101010, 0b00010, FPR16, GPR, "fcvt.h.l">, + Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]>; def : FPUnaryOpDynFrmAlias<FCVT_H_L, "fcvt.h.l", FPR16, GPR>; -def FCVT_H_LU : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.lu">, - Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]> { - let rs2 = 0b00011; -} +def FCVT_H_LU : FPUnaryOp_r_frm<0b1101010, 0b00011, FPR16, GPR, "fcvt.h.lu">, + Sched<[WriteFCvtI64ToF16, ReadFCvtI64ToF16]>; def : FPUnaryOpDynFrmAlias<FCVT_H_LU, "fcvt.h.lu", FPR16, GPR>; } // Predicates = [HasStdExtZfh, IsRV64] let Predicates = [HasStdExtZfhmin, HasStdExtD] in { -def FCVT_H_D : FPUnaryOp_r_frm<0b0100010, FPR16, FPR64, "fcvt.h.d">, - Sched<[WriteFCvtF64ToF16, ReadFCvtF64ToF16]> { - let rs2 = 0b00001; -} +def FCVT_H_D : FPUnaryOp_r_frm<0b0100010, 0b00001, FPR16, FPR64, "fcvt.h.d">, + Sched<[WriteFCvtF64ToF16, ReadFCvtF64ToF16]>; def : FPUnaryOpDynFrmAlias<FCVT_H_D, "fcvt.h.d", FPR16, FPR64>; -def FCVT_D_H : FPUnaryOp_r<0b0100001, 0b000, FPR64, FPR16, "fcvt.d.h">, - Sched<[WriteFCvtF16ToF64, ReadFCvtF16ToF64]> { - let rs2 = 0b00010; -} +def FCVT_D_H : FPUnaryOp_r<0b0100001, 0b00010, 0b000, FPR64, FPR16, "fcvt.d.h">, + Sched<[WriteFCvtF16ToF64, ReadFCvtF16ToF64]>; } // Predicates = [HasStdExtZfhmin, HasStdExtD] //===----------------------------------------------------------------------===// @@ -275,12 +215,12 @@ def : Pat<(f16 (fpimm0)), (FMV_H_X X0)>; /// Float arithmetic operations -def : PatFpr16Fpr16DynFrm<fadd, FADD_H>; -def : PatFpr16Fpr16DynFrm<fsub, FSUB_H>; -def : PatFpr16Fpr16DynFrm<fmul, FMUL_H>; -def : PatFpr16Fpr16DynFrm<fdiv, FDIV_H>; +def : PatFpr16Fpr16DynFrm<any_fadd, FADD_H>; +def : PatFpr16Fpr16DynFrm<any_fsub, FSUB_H>; +def : PatFpr16Fpr16DynFrm<any_fmul, FMUL_H>; +def : PatFpr16Fpr16DynFrm<any_fdiv, FDIV_H>; -def : Pat<(fsqrt FPR16:$rs1), (FSQRT_H FPR16:$rs1, 0b111)>; +def : Pat<(any_fsqrt FPR16:$rs1), (FSQRT_H FPR16:$rs1, 0b111)>; def : Pat<(fneg FPR16:$rs1), (FSGNJN_H $rs1, $rs1)>; def : Pat<(fabs FPR16:$rs1), (FSGNJX_H $rs1, $rs1)>; @@ -292,19 +232,19 @@ def : Pat<(fcopysign FPR16:$rs1, FPR32:$rs2), def : Pat<(fcopysign FPR32:$rs1, FPR16:$rs2), (FSGNJ_S $rs1, (FCVT_S_H $rs2))>; // fmadd: rs1 * rs2 + rs3 -def : Pat<(fma FPR16:$rs1, FPR16:$rs2, FPR16:$rs3), +def : Pat<(any_fma FPR16:$rs1, FPR16:$rs2, FPR16:$rs3), (FMADD_H $rs1, $rs2, $rs3, 0b111)>; // fmsub: rs1 * rs2 - rs3 -def : Pat<(fma FPR16:$rs1, FPR16:$rs2, (fneg FPR16:$rs3)), +def : Pat<(any_fma FPR16:$rs1, FPR16:$rs2, (fneg FPR16:$rs3)), (FMSUB_H FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>; // fnmsub: -rs1 * rs2 + rs3 -def : Pat<(fma (fneg FPR16:$rs1), FPR16:$rs2, FPR16:$rs3), +def : Pat<(any_fma (fneg FPR16:$rs1), FPR16:$rs2, FPR16:$rs3), (FNMSUB_H FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>; // fnmadd: -rs1 * rs2 - rs3 -def : Pat<(fma (fneg FPR16:$rs1), FPR16:$rs2, (fneg FPR16:$rs3)), +def : Pat<(any_fma (fneg FPR16:$rs1), FPR16:$rs2, (fneg FPR16:$rs3)), (FNMADD_H FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>; // The ratified 20191213 ISA spec defines fmin and fmax in a way that matches @@ -337,8 +277,8 @@ defm : StPat<store, FSH, FPR16, f16>; /// Float conversion operations // f32 -> f16, f16 -> f32 -def : Pat<(fpround FPR32:$rs1), (FCVT_H_S FPR32:$rs1, 0b111)>; -def : Pat<(fpextend FPR16:$rs1), (FCVT_S_H FPR16:$rs1)>; +def : Pat<(any_fpround FPR32:$rs1), (FCVT_H_S FPR32:$rs1, 0b111)>; +def : Pat<(any_fpextend FPR16:$rs1), (FCVT_S_H FPR16:$rs1)>; // Moves (no conversion) def : Pat<(riscv_fmv_h_x GPR:$src), (FMV_H_X GPR:$src)>; @@ -347,8 +287,8 @@ def : Pat<(riscv_fmv_x_anyexth FPR16:$src), (FMV_X_H FPR16:$src)>; let Predicates = [HasStdExtZfh, IsRV32] in { // half->[u]int. Round-to-zero must be used. -def : Pat<(i32 (fp_to_sint FPR16:$rs1)), (FCVT_W_H $rs1, 0b001)>; -def : Pat<(i32 (fp_to_uint FPR16:$rs1)), (FCVT_WU_H $rs1, 0b001)>; +def : Pat<(i32 (any_fp_to_sint FPR16:$rs1)), (FCVT_W_H $rs1, 0b001)>; +def : Pat<(i32 (any_fp_to_uint FPR16:$rs1)), (FCVT_WU_H $rs1, 0b001)>; // Saturating float->[u]int32. def : Pat<(i32 (riscv_fcvt_x_rtz FPR16:$rs1)), (FCVT_W_H $rs1, 0b001)>; @@ -361,20 +301,20 @@ def : Pat<(i32 (lrint FPR16:$rs1)), (FCVT_W_H $rs1, 0b111)>; def : Pat<(i32 (lround FPR16:$rs1)), (FCVT_W_H $rs1, 0b100)>; // [u]int->half. Match GCC and default to using dynamic rounding mode. -def : Pat<(sint_to_fp (i32 GPR:$rs1)), (FCVT_H_W $rs1, 0b111)>; -def : Pat<(uint_to_fp (i32 GPR:$rs1)), (FCVT_H_WU $rs1, 0b111)>; +def : Pat<(any_sint_to_fp (i32 GPR:$rs1)), (FCVT_H_W $rs1, 0b111)>; +def : Pat<(any_uint_to_fp (i32 GPR:$rs1)), (FCVT_H_WU $rs1, 0b111)>; } // Predicates = [HasStdExtZfh, IsRV32] let Predicates = [HasStdExtZfh, IsRV64] in { // Use target specific isd nodes to help us remember the result is sign // extended. Matching sext_inreg+fptoui/fptosi may cause the conversion to be // duplicated if it has another user that didn't need the sign_extend. -def : Pat<(riscv_fcvt_w_rtz_rv64 FPR16:$rs1), (FCVT_W_H $rs1, 0b001)>; -def : Pat<(riscv_fcvt_wu_rtz_rv64 FPR16:$rs1), (FCVT_WU_H $rs1, 0b001)>; +def : Pat<(riscv_any_fcvt_w_rtz_rv64 FPR16:$rs1), (FCVT_W_H $rs1, 0b001)>; +def : Pat<(riscv_any_fcvt_wu_rtz_rv64 FPR16:$rs1), (FCVT_WU_H $rs1, 0b001)>; // half->[u]int64. Round-to-zero must be used. -def : Pat<(i64 (fp_to_sint FPR16:$rs1)), (FCVT_L_H $rs1, 0b001)>; -def : Pat<(i64 (fp_to_uint FPR16:$rs1)), (FCVT_LU_H $rs1, 0b001)>; +def : Pat<(i64 (any_fp_to_sint FPR16:$rs1)), (FCVT_L_H $rs1, 0b001)>; +def : Pat<(i64 (any_fp_to_uint FPR16:$rs1)), (FCVT_LU_H $rs1, 0b001)>; // Saturating float->[u]int64. def : Pat<(i64 (riscv_fcvt_x_rtz FPR16:$rs1)), (FCVT_L_H $rs1, 0b001)>; @@ -389,17 +329,17 @@ def : Pat<(i64 (lround FPR16:$rs1)), (FCVT_L_H $rs1, 0b100)>; def : Pat<(i64 (llround FPR16:$rs1)), (FCVT_L_H $rs1, 0b100)>; // [u]int->fp. Match GCC and default to using dynamic rounding mode. -def : Pat<(sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_H_W $rs1, 0b111)>; -def : Pat<(uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_H_WU $rs1, 0b111)>; -def : Pat<(sint_to_fp (i64 GPR:$rs1)), (FCVT_H_L $rs1, 0b111)>; -def : Pat<(uint_to_fp (i64 GPR:$rs1)), (FCVT_H_LU $rs1, 0b111)>; +def : Pat<(any_sint_to_fp (i64 (sexti32 (i64 GPR:$rs1)))), (FCVT_H_W $rs1, 0b111)>; +def : Pat<(any_uint_to_fp (i64 (zexti32 (i64 GPR:$rs1)))), (FCVT_H_WU $rs1, 0b111)>; +def : Pat<(any_sint_to_fp (i64 GPR:$rs1)), (FCVT_H_L $rs1, 0b111)>; +def : Pat<(any_uint_to_fp (i64 GPR:$rs1)), (FCVT_H_LU $rs1, 0b111)>; } // Predicates = [HasStdExtZfh, IsRV64] let Predicates = [HasStdExtZfhmin, HasStdExtD] in { /// Float conversion operations // f64 -> f16, f16 -> f64 -def : Pat<(fpround FPR64:$rs1), (FCVT_H_D FPR64:$rs1, 0b111)>; -def : Pat<(fpextend FPR16:$rs1), (FCVT_D_H FPR16:$rs1)>; +def : Pat<(any_fpround FPR64:$rs1), (FCVT_H_D FPR64:$rs1, 0b111)>; +def : Pat<(any_fpextend FPR16:$rs1), (FCVT_D_H FPR16:$rs1)>; /// Float arithmetic operations def : Pat<(fcopysign FPR16:$rs1, FPR64:$rs2), diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 798532d5bc44..9094dff1dda1 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -105,7 +105,6 @@ BitVector RISCVRegisterInfo::getReservedRegs(const MachineFunction &MF) const { // Floating point environment registers. markSuperRegs(Reserved, RISCV::FRM); markSuperRegs(Reserved, RISCV::FFLAGS); - markSuperRegs(Reserved, RISCV::FCSR); assert(checkAllSuperRegsMarked(Reserved)); return Reserved; diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td index a56f992d320e..20903b317180 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td @@ -550,16 +550,15 @@ def VRM8NoV0 : VReg<[vint8m8_t, vint16m8_t, vint32m8_t, vint64m8_t, vfloat16m8_t, vfloat32m8_t, vfloat64m8_t], (add V8M8, V16M8, V24M8), 8>; -defvar VMaskVTs = [vbool64_t, vbool32_t, vbool16_t, vbool8_t, - vbool4_t, vbool2_t, vbool1_t]; +defvar VMaskVTs = [vbool1_t, vbool2_t, vbool4_t, vbool8_t, vbool16_t, + vbool32_t, vbool64_t]; def VMV0 : RegisterClass<"RISCV", VMaskVTs, 64, (add V0)> { let Size = 64; } // The register class is added for inline assembly for vector mask types. -def VM : VReg<[vbool1_t, vbool2_t, vbool4_t, vbool8_t, vbool16_t, - vbool32_t, vbool64_t], +def VM : VReg<VMaskVTs, (add (sequence "V%u", 8, 31), (sequence "V%u", 0, 7)), 1>; @@ -578,7 +577,6 @@ foreach m = LMULList.m in { // Special registers def FFLAGS : RISCVReg<0, "fflags">; def FRM : RISCVReg<0, "frm">; -def FCSR : RISCVReg<0, "fcsr">; // Any type register. Used for .insn directives when we don't know what the // register types could be. diff --git a/llvm/lib/Target/RISCV/RISCVSchedRocket.td b/llvm/lib/Target/RISCV/RISCVSchedRocket.td index 14f59152ed42..d5a0932c8778 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedRocket.td +++ b/llvm/lib/Target/RISCV/RISCVSchedRocket.td @@ -16,7 +16,8 @@ def RocketModel : SchedMachineModel { let IssueWidth = 1; // 1 micro-op is dispatched per cycle. let LoadLatency = 3; let MispredictPenalty = 3; - let UnsupportedFeatures = [HasStdExtV, HasStdExtZvamo, HasStdExtZvlsseg]; + let CompleteModel = false; + let UnsupportedFeatures = [HasStdExtV, HasStdExtZvlsseg]; } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td index 5b435fcb16a2..7f9d0aabc4ed 100644 --- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td +++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td @@ -15,7 +15,7 @@ def SiFive7Model : SchedMachineModel { let LoadLatency = 3; let MispredictPenalty = 3; let CompleteModel = 0; - let UnsupportedFeatures = [HasStdExtV, HasStdExtZvamo, HasStdExtZvlsseg]; + let UnsupportedFeatures = [HasStdExtV, HasStdExtZvlsseg]; } // The SiFive7 microarchitecture has two pipelines: A and B. diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h index deb2a11f98f1..d0330e6984a5 100644 --- a/llvm/lib/Target/RISCV/RISCVSubtarget.h +++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h @@ -51,7 +51,6 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo { bool HasStdExtZbt = false; bool HasStdExtV = false; bool HasStdExtZvlsseg = false; - bool HasStdExtZvamo = false; bool HasStdExtZfhmin = false; bool HasStdExtZfh = false; bool HasRV64 = false; @@ -118,7 +117,6 @@ public: bool hasStdExtZbt() const { return HasStdExtZbt; } bool hasStdExtV() const { return HasStdExtV; } bool hasStdExtZvlsseg() const { return HasStdExtZvlsseg; } - bool hasStdExtZvamo() const { return HasStdExtZvamo; } bool hasStdExtZfhmin() const { return HasStdExtZfhmin; } bool hasStdExtZfh() const { return HasStdExtZfh; } bool is64Bit() const { return HasRV64; } diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 56f0952fafc9..c435430a1288 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -162,3 +162,94 @@ InstructionCost RISCVTTIImpl::getGatherScatterOpCost( getMemoryOpCost(Opcode, VTy->getElementType(), Alignment, 0, CostKind, I); return NumLoads * MemOpCost; } + +void RISCVTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, + TTI::UnrollingPreferences &UP, + OptimizationRemarkEmitter *ORE) { + // TODO: More tuning on benchmarks and metrics with changes as needed + // would apply to all settings below to enable performance. + + // Support explicit targets enabled for SiFive with the unrolling preferences + // below + bool UseDefaultPreferences = true; + if (ST->getTuneCPU().contains("sifive-e76") || + ST->getTuneCPU().contains("sifive-s76") || + ST->getTuneCPU().contains("sifive-u74") || + ST->getTuneCPU().contains("sifive-7")) + UseDefaultPreferences = false; + + if (UseDefaultPreferences) + return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE); + + // Enable Upper bound unrolling universally, not dependant upon the conditions + // below. + UP.UpperBound = true; + + // Disable loop unrolling for Oz and Os. + UP.OptSizeThreshold = 0; + UP.PartialOptSizeThreshold = 0; + if (L->getHeader()->getParent()->hasOptSize()) + return; + + SmallVector<BasicBlock *, 4> ExitingBlocks; + L->getExitingBlocks(ExitingBlocks); + LLVM_DEBUG(dbgs() << "Loop has:\n" + << "Blocks: " << L->getNumBlocks() << "\n" + << "Exit blocks: " << ExitingBlocks.size() << "\n"); + + // Only allow another exit other than the latch. This acts as an early exit + // as it mirrors the profitability calculation of the runtime unroller. + if (ExitingBlocks.size() > 2) + return; + + // Limit the CFG of the loop body for targets with a branch predictor. + // Allowing 4 blocks permits if-then-else diamonds in the body. + if (L->getNumBlocks() > 4) + return; + + // Don't unroll vectorized loops, including the remainder loop + if (getBooleanLoopAttribute(L, "llvm.loop.isvectorized")) + return; + + // Scan the loop: don't unroll loops with calls as this could prevent + // inlining. + InstructionCost Cost = 0; + for (auto *BB : L->getBlocks()) { + for (auto &I : *BB) { + // Initial setting - Don't unroll loops containing vectorized + // instructions. + if (I.getType()->isVectorTy()) + return; + + if (isa<CallInst>(I) || isa<InvokeInst>(I)) { + if (const Function *F = cast<CallBase>(I).getCalledFunction()) { + if (!isLoweredToCall(F)) + continue; + } + return; + } + + SmallVector<const Value *> Operands(I.operand_values()); + Cost += + getUserCost(&I, Operands, TargetTransformInfo::TCK_SizeAndLatency); + } + } + + LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n"); + + UP.Partial = true; + UP.Runtime = true; + UP.UnrollRemainder = true; + UP.UnrollAndJam = true; + UP.UnrollAndJamInnerLoopThreshold = 60; + + // Force unrolling small loops can be very useful because of the branch + // taken cost of the backedge. + if (Cost < 12) + UP.Force = true; +} + +void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, + TTI::PeelingPreferences &PP) { + BaseT::getPeelingPreferences(L, SE, PP); +} diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h index 675681616d6e..7353496f4684 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h @@ -73,6 +73,13 @@ public: llvm_unreachable("Unsupported register kind"); } + void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, + TTI::UnrollingPreferences &UP, + OptimizationRemarkEmitter *ORE); + + void getPeelingPreferences(Loop *L, ScalarEvolution &SE, + TTI::PeelingPreferences &PP); + unsigned getMinVectorRegisterBitWidth() const { return ST->hasVInstructions() ? ST->getMinRVVVectorSizeInBits() : 0; } @@ -178,7 +185,9 @@ public: } unsigned getMaxInterleaveFactor(unsigned VF) { - return ST->getMaxInterleaveFactor(); + // If the loop will not be vectorized, don't interleave the loop. + // Let regular unroll to unroll the loop. + return VF == 1 ? 1 : ST->getMaxInterleaveFactor(); } }; diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp index 0f5e0b9672a9..538380263c3c 100644 --- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp +++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp @@ -28,25 +28,43 @@ static uint64_t extractBitsForFixup(MCFixupKind Kind, uint64_t Value, if (Kind < FirstTargetFixupKind) return Value; + auto checkFixupInRange = [&](int64_t Min, int64_t Max) -> bool { + int64_t SVal = int64_t(Value); + if (SVal < Min || SVal > Max) { + Ctx.reportError(Fixup.getLoc(), "operand out of range (" + Twine(SVal) + + " not between " + Twine(Min) + + " and " + Twine(Max) + ")"); + return false; + } + return true; + }; + + auto handlePCRelFixupValue = [&](unsigned W) -> uint64_t { + if (Value % 2 != 0) + Ctx.reportError(Fixup.getLoc(), "Non-even PC relative offset."); + if (!checkFixupInRange(minIntN(W) * 2, maxIntN(W) * 2)) + return 0; + return (int64_t)Value / 2; + }; + switch (unsigned(Kind)) { case SystemZ::FK_390_PC12DBL: + return handlePCRelFixupValue(12); case SystemZ::FK_390_PC16DBL: + return handlePCRelFixupValue(16); case SystemZ::FK_390_PC24DBL: + return handlePCRelFixupValue(24); case SystemZ::FK_390_PC32DBL: - return (int64_t)Value / 2; + return handlePCRelFixupValue(32); case SystemZ::FK_390_12: - if (!isUInt<12>(Value)) { - Ctx.reportError(Fixup.getLoc(), "displacement exceeds uint12"); + if (!checkFixupInRange(0, maxUIntN(12))) return 0; - } return Value; case SystemZ::FK_390_20: { - if (!isInt<20>(Value)) { - Ctx.reportError(Fixup.getLoc(), "displacement exceeds int20"); + if (!checkFixupInRange(minIntN(20), maxIntN(20))) return 0; - } // The high byte of a 20 bit displacement value comes first. uint64_t DLo = Value & 0xfff; uint64_t DHi = (Value >> 12) & 0xff; diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp index e280e4aaf3d8..c83796b8579b 100644 --- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp +++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp @@ -197,7 +197,8 @@ getDispOpValue(const MCInst &MI, unsigned OpNum, // All instructions follow the pattern where the first displacement has a // 2 bytes offset, and the second one 4 bytes. unsigned ByteOffs = Fixups.size() == 0 ? 2 : 4; - Fixups.push_back(MCFixup::create(ByteOffs, MO.getExpr(), (MCFixupKind)Kind)); + Fixups.push_back(MCFixup::create(ByteOffs, MO.getExpr(), (MCFixupKind)Kind, + MI.getLoc())); assert(Fixups.size() <= 2 && "More than two memory operands in MI?"); return 0; } @@ -296,6 +297,7 @@ SystemZMCCodeEmitter::getPCRelEncoding(const MCInst &MI, unsigned OpNum, SmallVectorImpl<MCFixup> &Fixups, unsigned Kind, int64_t Offset, bool AllowTLS) const { + SMLoc Loc = MI.getLoc(); const MCOperand &MO = MI.getOperand(OpNum); const MCExpr *Expr; if (MO.isImm()) @@ -311,13 +313,13 @@ SystemZMCCodeEmitter::getPCRelEncoding(const MCInst &MI, unsigned OpNum, Expr = MCBinaryExpr::createAdd(Expr, OffsetExpr, Ctx); } } - Fixups.push_back(MCFixup::create(Offset, Expr, (MCFixupKind)Kind)); + Fixups.push_back(MCFixup::create(Offset, Expr, (MCFixupKind)Kind, Loc)); // Output the fixup for the TLS marker if present. if (AllowTLS && OpNum + 1 < MI.getNumOperands()) { const MCOperand &MOTLS = MI.getOperand(OpNum + 1); - Fixups.push_back(MCFixup::create(0, MOTLS.getExpr(), - (MCFixupKind)SystemZ::FK_390_TLS_CALL)); + Fixups.push_back(MCFixup::create( + 0, MOTLS.getExpr(), (MCFixupKind)SystemZ::FK_390_TLS_CALL, Loc)); } return 0; } diff --git a/llvm/lib/Target/SystemZ/SystemZCallingConv.td b/llvm/lib/Target/SystemZ/SystemZCallingConv.td index 373023effb4a..a7ea5e1e4bf8 100644 --- a/llvm/lib/Target/SystemZ/SystemZCallingConv.td +++ b/llvm/lib/Target/SystemZ/SystemZCallingConv.td @@ -166,6 +166,7 @@ def CSR_SystemZ_NoRegs : CalleeSavedRegs<(add)>; // any non-leaf function and restored in the epilogue for use by the // return instruction so it functions exactly like a callee-saved register. def CSR_SystemZ_XPLINK64 : CalleeSavedRegs<(add (sequence "R%dD", 7, 15), + (sequence "R%dD", 4, 4), (sequence "F%dD", 15, 8))>; def CSR_SystemZ_XPLINK64_Vector : CalleeSavedRegs<(add CSR_SystemZ_XPLINK64, diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp index 2f7cdfcf7bde..99ab4c5455d6 100644 --- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp @@ -818,7 +818,7 @@ bool SystemZELFFrameLowering::usePackedStack(MachineFunction &MF) const { } SystemZXPLINKFrameLowering::SystemZXPLINKFrameLowering() - : SystemZFrameLowering(TargetFrameLowering::StackGrowsUp, Align(32), 128, + : SystemZFrameLowering(TargetFrameLowering::StackGrowsDown, Align(32), 0, Align(32), /* StackRealignable */ false), RegSpillOffsets(-1) { @@ -990,12 +990,184 @@ bool SystemZXPLINKFrameLowering::spillCalleeSavedRegisters( return true; } +bool SystemZXPLINKFrameLowering::restoreCalleeSavedRegisters( + MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, + MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const { + + if (CSI.empty()) + return false; + + MachineFunction &MF = *MBB.getParent(); + SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>(); + const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>(); + const TargetInstrInfo *TII = Subtarget.getInstrInfo(); + auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>(); + + DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); + + // Restore FPRs in the normal TargetInstrInfo way. + for (unsigned I = 0, E = CSI.size(); I != E; ++I) { + unsigned Reg = CSI[I].getReg(); + if (SystemZ::FP64BitRegClass.contains(Reg)) + TII->loadRegFromStackSlot(MBB, MBBI, Reg, CSI[I].getFrameIdx(), + &SystemZ::FP64BitRegClass, TRI); + if (SystemZ::VR128BitRegClass.contains(Reg)) + TII->loadRegFromStackSlot(MBB, MBBI, Reg, CSI[I].getFrameIdx(), + &SystemZ::VR128BitRegClass, TRI); + } + + // Restore call-saved GPRs (but not call-clobbered varargs, which at + // this point might hold return values). + SystemZ::GPRRegs RestoreGPRs = ZFI->getRestoreGPRRegs(); + if (RestoreGPRs.LowGPR) { + assert(isInt<20>(Regs.getStackPointerBias() + RestoreGPRs.GPROffset)); + if (RestoreGPRs.LowGPR == RestoreGPRs.HighGPR) + // Build an LG/L instruction. + BuildMI(MBB, MBBI, DL, TII->get(SystemZ::LG), RestoreGPRs.LowGPR) + .addReg(Regs.getStackPointerRegister()) + .addImm(Regs.getStackPointerBias() + RestoreGPRs.GPROffset) + .addReg(0); + else { + // Build an LMG/LM instruction. + MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(SystemZ::LMG)); + + // Add the explicit register operands. + MIB.addReg(RestoreGPRs.LowGPR, RegState::Define); + MIB.addReg(RestoreGPRs.HighGPR, RegState::Define); + + // Add the address. + MIB.addReg(Regs.getStackPointerRegister()); + MIB.addImm(Regs.getStackPointerBias() + RestoreGPRs.GPROffset); + + // Do a second scan adding regs as being defined by instruction + for (unsigned I = 0, E = CSI.size(); I != E; ++I) { + unsigned Reg = CSI[I].getReg(); + if (Reg > RestoreGPRs.LowGPR && Reg < RestoreGPRs.HighGPR) + MIB.addReg(Reg, RegState::ImplicitDefine); + } + } + } + + return true; +} + void SystemZXPLINKFrameLowering::emitPrologue(MachineFunction &MF, - MachineBasicBlock &MBB) const {} + MachineBasicBlock &MBB) const { + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); + const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>(); + SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>(); + MachineBasicBlock::iterator MBBI = MBB.begin(); + auto *ZII = static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); + auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>(); + MachineFrameInfo &MFFrame = MF.getFrameInfo(); + MachineInstr *StoreInstr = nullptr; + bool HasFP = hasFP(MF); + // Debug location must be unknown since the first debug location is used + // to determine the end of the prologue. + DebugLoc DL; + uint64_t Offset = 0; + + // TODO: Support leaf functions; only add size of save+reserved area when + // function is non-leaf. + MFFrame.setStackSize(MFFrame.getStackSize() + Regs.getCallFrameSize()); + uint64_t StackSize = MFFrame.getStackSize(); + + // FIXME: Implement support for large stack sizes, when the stack extension + // routine needs to be called. + if (StackSize > 1024 * 1024) { + llvm_unreachable("Huge Stack Frame not yet supported on z/OS"); + } + + if (ZFI->getSpillGPRRegs().LowGPR) { + // Skip over the GPR saves. + if ((MBBI != MBB.end()) && ((MBBI->getOpcode() == SystemZ::STMG))) { + const int Operand = 3; + // Now we can set the offset for the operation, since now the Stack + // has been finalized. + Offset = Regs.getStackPointerBias() + MBBI->getOperand(Operand).getImm(); + // Maximum displacement for STMG instruction. + if (isInt<20>(Offset - StackSize)) + Offset -= StackSize; + else + StoreInstr = &*MBBI; + MBBI->getOperand(Operand).setImm(Offset); + ++MBBI; + } else + llvm_unreachable("Couldn't skip over GPR saves"); + } + + if (StackSize) { + MachineBasicBlock::iterator InsertPt = StoreInstr ? StoreInstr : MBBI; + // Allocate StackSize bytes. + int64_t Delta = -int64_t(StackSize); + + // In case the STM(G) instruction also stores SP (R4), but the displacement + // is too large, the SP register is manipulated first before storing, + // resulting in the wrong value stored and retrieved later. In this case, we + // need to temporarily save the value of SP, and store it later to memory. + if (StoreInstr && HasFP) { + // Insert LR r0,r4 before STMG instruction. + BuildMI(MBB, InsertPt, DL, ZII->get(SystemZ::LGR)) + .addReg(SystemZ::R0D, RegState::Define) + .addReg(SystemZ::R4D); + // Insert ST r0,xxx(,r4) after STMG instruction. + BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::STG)) + .addReg(SystemZ::R0D, RegState::Kill) + .addReg(SystemZ::R4D) + .addImm(Offset) + .addReg(0); + } + + emitIncrement(MBB, InsertPt, DL, Regs.getStackPointerRegister(), Delta, + ZII); + } + + if (HasFP) { + // Copy the base of the frame to Frame Pointer Register. + BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR), + Regs.getFramePointerRegister()) + .addReg(Regs.getStackPointerRegister()); + + // Mark the FramePtr as live at the beginning of every block except + // the entry block. (We'll have marked R8 as live on entry when + // saving the GPRs.) + for (auto I = std::next(MF.begin()), E = MF.end(); I != E; ++I) + I->addLiveIn(Regs.getFramePointerRegister()); + } +} void SystemZXPLINKFrameLowering::emitEpilogue(MachineFunction &MF, - MachineBasicBlock &MBB) const {} + MachineBasicBlock &MBB) const { + const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>(); + MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr(); + SystemZMachineFunctionInfo *ZFI = MF.getInfo<SystemZMachineFunctionInfo>(); + MachineFrameInfo &MFFrame = MF.getFrameInfo(); + auto *ZII = static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); + auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>(); + + // Skip the return instruction. + assert(MBBI->isReturn() && "Can only insert epilogue into returning blocks"); + + uint64_t StackSize = MFFrame.getStackSize(); + if (StackSize) { + unsigned SPReg = Regs.getStackPointerRegister(); + if (ZFI->getRestoreGPRRegs().LowGPR != SPReg) { + DebugLoc DL = MBBI->getDebugLoc(); + emitIncrement(MBB, MBBI, DL, SPReg, StackSize, ZII); + } + } +} bool SystemZXPLINKFrameLowering::hasFP(const MachineFunction &MF) const { - return false; + return (MF.getFrameInfo().hasVarSizedObjects()); +} + +void SystemZXPLINKFrameLowering::processFunctionBeforeFrameFinalized( + MachineFunction &MF, RegScavenger *RS) const { + MachineFrameInfo &MFFrame = MF.getFrameInfo(); + const SystemZSubtarget &Subtarget = MF.getSubtarget<SystemZSubtarget>(); + auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>(); + + // Setup stack frame offset + MFFrame.setOffsetAdjustment(Regs.getStackPointerBias()); } diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.h b/llvm/lib/Target/SystemZ/SystemZFrameLowering.h index af219da79c32..106b9e8ebe06 100644 --- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.h @@ -115,11 +115,20 @@ public: ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const override; + bool + restoreCalleeSavedRegisters(MachineBasicBlock &MBB, + MachineBasicBlock::iterator MBBII, + MutableArrayRef<CalleeSavedInfo> CSI, + const TargetRegisterInfo *TRI) const override; + void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; bool hasFP(const MachineFunction &MF) const override; + + void processFunctionBeforeFrameFinalized(MachineFunction &MF, + RegScavenger *RS) const override; }; } // end namespace llvm diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp index 71432218068e..24de52850771 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -1500,8 +1500,16 @@ SDValue SystemZTargetLowering::LowerFormalArguments( assert(VA.isMemLoc() && "Argument not register or memory"); // Create the frame index object for this incoming parameter. - int FI = MFI.CreateFixedObject(LocVT.getSizeInBits() / 8, - VA.getLocMemOffset(), true); + // FIXME: Pre-include call frame size in the offset, should not + // need to manually add it here. + int64_t ArgSPOffset = VA.getLocMemOffset(); + if (Subtarget.isTargetXPLINK64()) { + auto &XPRegs = + Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>(); + ArgSPOffset += XPRegs.getCallFrameSize(); + } + int FI = + MFI.CreateFixedObject(LocVT.getSizeInBits() / 8, ArgSPOffset, true); // Create the SelectionDAG nodes corresponding to a load // from this parameter. Unpromoted ints and floats are @@ -5714,6 +5722,7 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const { OPCODE(OC); OPCODE(XC); OPCODE(CLC); + OPCODE(MEMSET_MVC); OPCODE(STPCPY); OPCODE(STRCMP); OPCODE(SEARCH_STRING); @@ -7860,8 +7869,10 @@ MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI, return MBB; } -MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper( - MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const { +MachineBasicBlock * +SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI, + MachineBasicBlock *MBB, + unsigned Opcode, bool IsMemset) const { MachineFunction &MF = *MBB->getParent(); const SystemZInstrInfo *TII = static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo()); @@ -7870,18 +7881,64 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper( MachineOperand DestBase = earlyUseOperand(MI.getOperand(0)); uint64_t DestDisp = MI.getOperand(1).getImm(); - MachineOperand SrcBase = earlyUseOperand(MI.getOperand(2)); - uint64_t SrcDisp = MI.getOperand(3).getImm(); - MachineOperand &LengthMO = MI.getOperand(4); + MachineOperand SrcBase = MachineOperand::CreateReg(0U, false); + uint64_t SrcDisp; + + // Fold the displacement Disp if it is out of range. + auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void { + if (!isUInt<12>(Disp)) { + Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); + unsigned Opcode = TII->getOpcodeForOffset(SystemZ::LA, Disp); + BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(Opcode), Reg) + .add(Base).addImm(Disp).addReg(0); + Base = MachineOperand::CreateReg(Reg, false); + Disp = 0; + } + }; + + if (!IsMemset) { + SrcBase = earlyUseOperand(MI.getOperand(2)); + SrcDisp = MI.getOperand(3).getImm(); + } else { + SrcBase = DestBase; + SrcDisp = DestDisp++; + foldDisplIfNeeded(DestBase, DestDisp); + } + + MachineOperand &LengthMO = MI.getOperand(IsMemset ? 2 : 4); bool IsImmForm = LengthMO.isImm(); bool IsRegForm = !IsImmForm; + // Build and insert one Opcode of Length, with special treatment for memset. + auto insertMemMemOp = [&](MachineBasicBlock *InsMBB, + MachineBasicBlock::iterator InsPos, + MachineOperand DBase, uint64_t DDisp, + MachineOperand SBase, uint64_t SDisp, + unsigned Length) -> void { + assert(Length > 0 && Length <= 256 && "Building memory op with bad length."); + if (IsMemset) { + MachineOperand ByteMO = earlyUseOperand(MI.getOperand(3)); + if (ByteMO.isImm()) + BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::MVI)) + .add(SBase).addImm(SDisp).add(ByteMO); + else + BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::STC)) + .add(ByteMO).add(SBase).addImm(SDisp).addReg(0); + if (--Length == 0) + return; + } + BuildMI(*MBB, InsPos, DL, TII->get(Opcode)) + .add(DBase).addImm(DDisp).addImm(Length) + .add(SBase).addImm(SDisp) + .setMemRefs(MI.memoperands()); + }; + bool NeedsLoop = false; uint64_t ImmLength = 0; - Register LenMinus1Reg = SystemZ::NoRegister; + Register LenAdjReg = SystemZ::NoRegister; if (IsImmForm) { ImmLength = LengthMO.getImm(); - ImmLength++; // Add back the '1' subtracted originally. + ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment. if (ImmLength == 0) { MI.eraseFromParent(); return MBB; @@ -7905,7 +7962,7 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper( NeedsLoop = true; } else { NeedsLoop = true; - LenMinus1Reg = LengthMO.getReg(); + LenAdjReg = LengthMO.getReg(); } // When generating more than one CLC, all but the last will need to @@ -7923,17 +7980,17 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper( ImmLength &= 255; } else { BuildMI(*MBB, MI, DL, TII->get(SystemZ::SRLG), StartCountReg) - .addReg(LenMinus1Reg) + .addReg(LenAdjReg) .addReg(0) .addImm(8); } + bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase); auto loadZeroAddress = [&]() -> MachineOperand { Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); BuildMI(*MBB, MI, DL, TII->get(SystemZ::LGHI), Reg).addImm(0); return MachineOperand::CreateReg(Reg, false); }; - bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase); if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister) DestBase = loadZeroAddress(); if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister) @@ -7968,14 +8025,41 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper( DoneMBB = SystemZ::emitBlockAfter(NextMBB); // MBB: - // # Jump to AllDoneMBB if LenMinus1Reg is -1, or fall thru to StartMBB. + // # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB. BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) - .addReg(LenMinus1Reg).addImm(-1); + .addReg(LenAdjReg).addImm(IsMemset ? -2 : -1); BuildMI(MBB, DL, TII->get(SystemZ::BRC)) .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ) .addMBB(AllDoneMBB); MBB->addSuccessor(AllDoneMBB); - MBB->addSuccessor(StartMBB); + if (!IsMemset) + MBB->addSuccessor(StartMBB); + else { + // MemsetOneCheckMBB: + // # Jump to MemsetOneMBB for a memset of length 1, or + // # fall thru to StartMBB. + MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB); + MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(&*MF.rbegin()); + MBB->addSuccessor(MemsetOneCheckMBB); + MBB = MemsetOneCheckMBB; + BuildMI(MBB, DL, TII->get(SystemZ::CGHI)) + .addReg(LenAdjReg).addImm(-1); + BuildMI(MBB, DL, TII->get(SystemZ::BRC)) + .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ) + .addMBB(MemsetOneMBB); + MBB->addSuccessor(MemsetOneMBB, {10, 100}); + MBB->addSuccessor(StartMBB, {90, 100}); + + // MemsetOneMBB: + // # Jump back to AllDoneMBB after a single MVI or STC. + MBB = MemsetOneMBB; + insertMemMemOp(MBB, MBB->end(), + MachineOperand::CreateReg(StartDestReg, false), DestDisp, + MachineOperand::CreateReg(StartSrcReg, false), SrcDisp, + 1); + BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(AllDoneMBB); + MBB->addSuccessor(AllDoneMBB); + } // StartMBB: // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB. @@ -8032,10 +8116,10 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper( if (Opcode == SystemZ::MVC) BuildMI(MBB, DL, TII->get(SystemZ::PFD)) .addImm(SystemZ::PFD_WRITE) - .addReg(ThisDestReg).addImm(DestDisp + 768).addReg(0); - BuildMI(MBB, DL, TII->get(Opcode)) - .addReg(ThisDestReg).addImm(DestDisp).addImm(256) - .addReg(ThisSrcReg).addImm(SrcDisp); + .addReg(ThisDestReg).addImm(DestDisp - IsMemset + 768).addReg(0); + insertMemMemOp(MBB, MBB->end(), + MachineOperand::CreateReg(ThisDestReg, false), DestDisp, + MachineOperand::CreateReg(ThisSrcReg, false), SrcDisp, 256); if (EndMBB) { BuildMI(MBB, DL, TII->get(SystemZ::BRC)) .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE) @@ -8075,7 +8159,7 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper( // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run. // # Use EXecute Relative Long for the remainder of the bytes. The target // instruction of the EXRL will have a length field of 1 since 0 is an - // illegal value. The number of bytes processed becomes (%LenMinus1Reg & + // illegal value. The number of bytes processed becomes (%LenAdjReg & // 0xff) + 1. // # Fall through to AllDoneMBB. Register RemSrcReg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); @@ -8088,10 +8172,14 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper( BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemSrcReg) .addReg(StartSrcReg).addMBB(StartMBB) .addReg(NextSrcReg).addMBB(NextMBB); + if (IsMemset) + insertMemMemOp(MBB, MBB->end(), + MachineOperand::CreateReg(RemDestReg, false), DestDisp, + MachineOperand::CreateReg(RemSrcReg, false), SrcDisp, 1); MachineInstrBuilder EXRL_MIB = BuildMI(MBB, DL, TII->get(SystemZ::EXRL_Pseudo)) .addImm(Opcode) - .addReg(LenMinus1Reg) + .addReg(LenAdjReg) .addReg(RemDestReg).addImm(DestDisp) .addReg(RemSrcReg).addImm(SrcDisp); MBB->addSuccessor(AllDoneMBB); @@ -8107,32 +8195,10 @@ MachineBasicBlock *SystemZTargetLowering::emitMemMemWrapper( while (ImmLength > 0) { uint64_t ThisLength = std::min(ImmLength, uint64_t(256)); // The previous iteration might have created out-of-range displacements. - // Apply them using LAY if so. - if (!isUInt<12>(DestDisp)) { - Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); - BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LAY), Reg) - .add(DestBase) - .addImm(DestDisp) - .addReg(0); - DestBase = MachineOperand::CreateReg(Reg, false); - DestDisp = 0; - } - if (!isUInt<12>(SrcDisp)) { - Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass); - BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LAY), Reg) - .add(SrcBase) - .addImm(SrcDisp) - .addReg(0); - SrcBase = MachineOperand::CreateReg(Reg, false); - SrcDisp = 0; - } - BuildMI(*MBB, MI, DL, TII->get(Opcode)) - .add(DestBase) - .addImm(DestDisp) - .addImm(ThisLength) - .add(SrcBase) - .addImm(SrcDisp) - .setMemRefs(MI.memoperands()); + // Apply them using LA/LAY if so. + foldDisplIfNeeded(DestBase, DestDisp); + foldDisplIfNeeded(SrcBase, SrcDisp); + insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength); DestDisp += ThisLength; SrcDisp += ThisLength; ImmLength -= ThisLength; @@ -8630,6 +8696,11 @@ MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter( case SystemZ::CLCImm: case SystemZ::CLCReg: return emitMemMemWrapper(MI, MBB, SystemZ::CLC); + case SystemZ::MemsetImmImm: + case SystemZ::MemsetImmReg: + case SystemZ::MemsetRegImm: + case SystemZ::MemsetRegReg: + return emitMemMemWrapper(MI, MBB, SystemZ::MVC, true/*IsMemset*/); case SystemZ::CLSTLoop: return emitStringWrapper(MI, MBB, SystemZ::CLST); case SystemZ::MVSTLoop: diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h index 461f804ca55e..940c0a857ea4 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h @@ -126,6 +126,9 @@ enum NodeType : unsigned { // as for MVC. CLC, + // Use MVC to set a block of memory after storing the first byte. + MEMSET_MVC, + // Use an MVST-based sequence to implement stpcpy(). STPCPY, @@ -709,7 +712,8 @@ private: MachineBasicBlock *emitAtomicCmpSwapW(MachineInstr &MI, MachineBasicBlock *BB) const; MachineBasicBlock *emitMemMemWrapper(MachineInstr &MI, MachineBasicBlock *BB, - unsigned Opcode) const; + unsigned Opcode, + bool IsMemset = false) const; MachineBasicBlock *emitStringWrapper(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode) const; MachineBasicBlock *emitTransactionBegin(MachineInstr &MI, diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td index cd60fff1ab11..e513befd0d6f 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td +++ b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td @@ -5256,6 +5256,16 @@ class RotateSelectAliasRIEf<RegisterOperand cls1, RegisterOperand cls2> let Constraints = "$R1 = $R1src"; } +class MemsetPseudo<DAGOperand lenop, DAGOperand byteop> + : Pseudo<(outs), (ins bdaddr12only:$dest, lenop:$length, byteop:$B), + [(z_memset_mvc bdaddr12only:$dest, lenop:$length, byteop:$B)]> { + let Defs = [CC]; + let mayLoad = 1; + let mayStore = 1; + let usesCustomInserter = 1; + let hasNoSchedulingInfo = 1; +} + //===----------------------------------------------------------------------===// // Multiclasses that emit both real and pseudo instructions //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td index e4760229fd6b..84f1e0fb428c 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td +++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td @@ -510,6 +510,12 @@ let mayLoad = 1, mayStore = 1, Defs = [CC] in { def MVCLU : SideEffectTernaryMemMemRSY<"mvclu", 0xEB8E, GR128, GR128>; } +// Memset[Length][Byte] pseudos. +def MemsetImmImm : MemsetPseudo<imm64, imm32zx8trunc>; +def MemsetImmReg : MemsetPseudo<imm64, GR32>; +def MemsetRegImm : MemsetPseudo<ADDR64, imm32zx8trunc>; +def MemsetRegReg : MemsetPseudo<ADDR64, GR32>; + // Move right. let Predicates = [FeatureMiscellaneousExtensions3], mayLoad = 1, mayStore = 1, Uses = [R0L] in diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td index 927d97233286..9935416559bc 100644 --- a/llvm/lib/Target/SystemZ/SystemZOperators.td +++ b/llvm/lib/Target/SystemZ/SystemZOperators.td @@ -102,6 +102,10 @@ def SDT_ZMemMemLengthCC : SDTypeProfile<1, 3, SDTCisPtrTy<1>, SDTCisPtrTy<2>, SDTCisVT<3, i64>]>; +def SDT_ZMemsetMVC : SDTypeProfile<0, 3, + [SDTCisPtrTy<0>, + SDTCisVT<1, i64>, + SDTCisVT<2, i32>]>; def SDT_ZString : SDTypeProfile<1, 3, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, @@ -413,6 +417,8 @@ def z_xc : SDNode<"SystemZISD::XC", SDT_ZMemMemLength, [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>; def z_clc : SDNode<"SystemZISD::CLC", SDT_ZMemMemLengthCC, [SDNPHasChain, SDNPMayLoad]>; +def z_memset_mvc : SDNode<"SystemZISD::MEMSET_MVC", SDT_ZMemsetMVC, + [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>; def z_strcmp : SDNode<"SystemZISD::STRCMP", SDT_ZStringCC, [SDNPHasChain, SDNPMayLoad]>; def z_stpcpy : SDNode<"SystemZISD::STPCPY", SDT_ZString, diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp index f38e93109967..db4b4879b33a 100644 --- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp +++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp @@ -17,29 +17,44 @@ using namespace llvm; #define DEBUG_TYPE "systemz-selectiondag-info" -static SDVTList getMemMemVTs(unsigned Op, SelectionDAG &DAG) { - return Op == SystemZISD::CLC ? DAG.getVTList(MVT::i32, MVT::Other) - : DAG.getVTList(MVT::Other); +static unsigned getMemMemLenAdj(unsigned Op) { + return Op == SystemZISD::MEMSET_MVC ? 2 : 1; } -// Emit a mem-mem operation after subtracting one from size, which will be -// added back during pseudo expansion. As the Reg case emitted here may be -// converted by DAGCombiner into having an Imm length, they are both emitted -// the same way. +static SDValue createMemMemNode(SelectionDAG &DAG, const SDLoc &DL, unsigned Op, + SDValue Chain, SDValue Dst, SDValue Src, + SDValue LenAdj, SDValue Byte) { + SDVTList VTs = Op == SystemZISD::CLC ? DAG.getVTList(MVT::i32, MVT::Other) + : DAG.getVTList(MVT::Other); + SmallVector<SDValue, 6> Ops; + if (Op == SystemZISD::MEMSET_MVC) + Ops = { Chain, Dst, LenAdj, Byte }; + else + Ops = { Chain, Dst, Src, LenAdj }; + return DAG.getNode(Op, DL, VTs, Ops); +} + +// Emit a mem-mem operation after subtracting one (or two for memset) from +// size, which will be added back during pseudo expansion. As the Reg case +// emitted here may be converted by DAGCombiner into having an Imm length, +// they are both emitted the same way. static SDValue emitMemMemImm(SelectionDAG &DAG, const SDLoc &DL, unsigned Op, SDValue Chain, SDValue Dst, SDValue Src, - uint64_t Size) { - return DAG.getNode(Op, DL, getMemMemVTs(Op, DAG), Chain, Dst, Src, - DAG.getConstant(Size - 1, DL, Src.getValueType())); + uint64_t Size, SDValue Byte = SDValue()) { + unsigned Adj = getMemMemLenAdj(Op); + assert(Size >= Adj && "Adjusted length overflow."); + SDValue LenAdj = DAG.getConstant(Size - Adj, DL, Dst.getValueType()); + return createMemMemNode(DAG, DL, Op, Chain, Dst, Src, LenAdj, Byte); } static SDValue emitMemMemReg(SelectionDAG &DAG, const SDLoc &DL, unsigned Op, SDValue Chain, SDValue Dst, SDValue Src, - SDValue Size) { - SDValue LenMinus1 = DAG.getNode(ISD::ADD, DL, MVT::i64, - DAG.getZExtOrTrunc(Size, DL, MVT::i64), - DAG.getConstant(-1, DL, MVT::i64)); - return DAG.getNode(Op, DL, getMemMemVTs(Op, DAG), Chain, Dst, Src, LenMinus1); + SDValue Size, SDValue Byte = SDValue()) { + int64_t Adj = getMemMemLenAdj(Op); + SDValue LenAdj = DAG.getNode(ISD::ADD, DL, MVT::i64, + DAG.getZExtOrTrunc(Size, DL, MVT::i64), + DAG.getConstant(0 - Adj, DL, MVT::i64)); + return createMemMemNode(DAG, DL, Op, Chain, Dst, Src, LenAdj, Byte); } SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy( @@ -127,13 +142,8 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset( if (CByte && CByte->getZExtValue() == 0) return emitMemMemImm(DAG, DL, SystemZISD::XC, Chain, Dst, Dst, Bytes); - // Copy the byte to the first location and then use MVC to copy - // it to the rest. - Chain = DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Alignment); - SDValue DstPlus1 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst, - DAG.getConstant(1, DL, PtrVT)); - return emitMemMemImm(DAG, DL, SystemZISD::MVC, Chain, DstPlus1, Dst, - Bytes - 1); + return emitMemMemImm(DAG, DL, SystemZISD::MEMSET_MVC, Chain, Dst, SDValue(), + Bytes, DAG.getAnyExtOrTrunc(Byte, DL, MVT::i32)); } // Variable length @@ -141,7 +151,8 @@ SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset( // Handle the special case of a variable length memset of 0 with XC. return emitMemMemReg(DAG, DL, SystemZISD::XC, Chain, Dst, Dst, Size); - return SDValue(); + return emitMemMemReg(DAG, DL, SystemZISD::MEMSET_MVC, Chain, Dst, SDValue(), + Size, DAG.getAnyExtOrTrunc(Byte, DL, MVT::i32)); } // Convert the current CC value into an integer that is 0 if CC == 0, diff --git a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp index 7e92e4b33812..fd9dc32b04f5 100644 --- a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp +++ b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp @@ -84,6 +84,8 @@ class VEAsmParser : public MCTargetAsmParser { StringRef splitMnemonic(StringRef Name, SMLoc NameLoc, OperandVector *Operands); + bool parseLiteralValues(unsigned Size, SMLoc L); + public: VEAsmParser(const MCSubtargetInfo &sti, MCAsmParser &parser, const MCInstrInfo &MII, const MCTargetOptions &Options) @@ -994,10 +996,43 @@ bool VEAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, } bool VEAsmParser::ParseDirective(AsmToken DirectiveID) { + std::string IDVal = DirectiveID.getIdentifier().lower(); + + // Defines VE specific directives. Reference is "Vector Engine Assembly + // Language Reference Manual": + // https://www.hpc.nec/documents/sdk/pdfs/VectorEngine-as-manual-v1.3.pdf + + // The .word is 4 bytes long on VE. + if (IDVal == ".word") + return parseLiteralValues(4, DirectiveID.getLoc()); + + // The .long is 8 bytes long on VE. + if (IDVal == ".long") + return parseLiteralValues(8, DirectiveID.getLoc()); + + // The .llong is 8 bytes long on VE. + if (IDVal == ".llong") + return parseLiteralValues(8, DirectiveID.getLoc()); + // Let the MC layer to handle other directives. return true; } +/// parseLiteralValues +/// ::= .word expression [, expression]* +/// ::= .long expression [, expression]* +/// ::= .llong expression [, expression]* +bool VEAsmParser::parseLiteralValues(unsigned Size, SMLoc L) { + auto parseOne = [&]() -> bool { + const MCExpr *Value; + if (getParser().parseExpression(Value)) + return true; + getParser().getStreamer().emitValue(Value, Size, L); + return false; + }; + return (parseMany(parseOne)); +} + /// Extract \code @lo32/@hi32/etc \endcode modifier from expression. /// Recursively scan the expression and check for VK_VE_HI32/LO32/etc /// symbol variants. If all symbols with modifier use the same diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp index 29c209934680..38d163b37080 100644 --- a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp +++ b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp @@ -42,6 +42,7 @@ static uint64_t adjustFixupValue(unsigned Kind, uint64_t Value) { case VE::fixup_ve_tpoff_hi32: return (Value >> 32) & 0xffffffff; case VE::fixup_ve_reflong: + case VE::fixup_ve_srel32: case VE::fixup_ve_lo32: case VE::fixup_ve_pc_lo32: case VE::fixup_ve_got_lo32: @@ -68,6 +69,7 @@ static unsigned getFixupKindNumBytes(unsigned Kind) { case FK_Data_4: case FK_PCRel_4: case VE::fixup_ve_reflong: + case VE::fixup_ve_srel32: case VE::fixup_ve_hi32: case VE::fixup_ve_lo32: case VE::fixup_ve_pc_hi32: @@ -103,6 +105,7 @@ public: const static MCFixupKindInfo Infos[VE::NumTargetFixupKinds] = { // name, offset, bits, flags {"fixup_ve_reflong", 0, 32, 0}, + {"fixup_ve_srel32", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, {"fixup_ve_hi32", 0, 32, 0}, {"fixup_ve_lo32", 0, 32, 0}, {"fixup_ve_pc_hi32", 0, 32, MCFixupKindInfo::FKF_IsPCRel}, diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp index 741e8320a941..ae065407409a 100644 --- a/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp +++ b/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp @@ -9,6 +9,7 @@ #include "VEFixupKinds.h" #include "VEMCExpr.h" #include "VEMCTargetDesc.h" +#include "llvm/MC/MCContext.h" #include "llvm/MC/MCELFObjectWriter.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCObjectWriter.h" @@ -46,16 +47,29 @@ unsigned VEELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target, if (IsPCRel) { switch (Fixup.getTargetKind()) { default: - llvm_unreachable("Unimplemented fixup -> relocation"); + Ctx.reportError(Fixup.getLoc(), "Unsupported pc-relative fixup kind"); + return ELF::R_VE_NONE; + case FK_Data_1: case FK_PCRel_1: - llvm_unreachable("Unimplemented fixup fk_data_1 -> relocation"); + Ctx.reportError(Fixup.getLoc(), + "1-byte pc-relative data relocation is not supported"); + return ELF::R_VE_NONE; + case FK_Data_2: case FK_PCRel_2: - llvm_unreachable("Unimplemented fixup fk_data_2 -> relocation"); - // FIXME: relative kind? + Ctx.reportError(Fixup.getLoc(), + "2-byte pc-relative data relocation is not supported"); + return ELF::R_VE_NONE; + case FK_Data_4: case FK_PCRel_4: - return ELF::R_VE_REFLONG; + return ELF::R_VE_SREL32; + case FK_Data_8: case FK_PCRel_8: - return ELF::R_VE_REFQUAD; + Ctx.reportError(Fixup.getLoc(), + "8-byte pc-relative data relocation is not supported"); + return ELF::R_VE_NONE; + case VE::fixup_ve_reflong: + case VE::fixup_ve_srel32: + return ELF::R_VE_SREL32; case VE::fixup_ve_pc_hi32: return ELF::R_VE_PC_HI32; case VE::fixup_ve_pc_lo32: @@ -65,25 +79,36 @@ unsigned VEELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target, switch (Fixup.getTargetKind()) { default: - llvm_unreachable("Unimplemented fixup -> relocation"); + Ctx.reportError(Fixup.getLoc(), "Unknown ELF relocation type"); + return ELF::R_VE_NONE; case FK_Data_1: - llvm_unreachable("Unimplemented fixup fk_data_1 -> relocation"); + Ctx.reportError(Fixup.getLoc(), "1-byte data relocation is not supported"); + return ELF::R_VE_NONE; case FK_Data_2: - llvm_unreachable("Unimplemented fixup fk_data_2 -> relocation"); + Ctx.reportError(Fixup.getLoc(), "2-byte data relocation is not supported"); + return ELF::R_VE_NONE; case FK_Data_4: return ELF::R_VE_REFLONG; case FK_Data_8: return ELF::R_VE_REFQUAD; case VE::fixup_ve_reflong: return ELF::R_VE_REFLONG; + case VE::fixup_ve_srel32: + Ctx.reportError(Fixup.getLoc(), + "A non pc-relative srel32 relocation is not supported"); + return ELF::R_VE_NONE; case VE::fixup_ve_hi32: return ELF::R_VE_HI32; case VE::fixup_ve_lo32: return ELF::R_VE_LO32; case VE::fixup_ve_pc_hi32: - llvm_unreachable("Unimplemented fixup pc_hi32 -> relocation"); + Ctx.reportError(Fixup.getLoc(), + "A non pc-relative pc_hi32 relocation is not supported"); + return ELF::R_VE_NONE; case VE::fixup_ve_pc_lo32: - llvm_unreachable("Unimplemented fixup pc_lo32 -> relocation"); + Ctx.reportError(Fixup.getLoc(), + "A non pc-relative pc_lo32 relocation is not supported"); + return ELF::R_VE_NONE; case VE::fixup_ve_got_hi32: return ELF::R_VE_GOT_HI32; case VE::fixup_ve_got_lo32: diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEFixupKinds.h b/llvm/lib/Target/VE/MCTargetDesc/VEFixupKinds.h index 5d5dc1c5c891..46b995cee840 100644 --- a/llvm/lib/Target/VE/MCTargetDesc/VEFixupKinds.h +++ b/llvm/lib/Target/VE/MCTargetDesc/VEFixupKinds.h @@ -17,6 +17,9 @@ enum Fixups { /// fixup_ve_reflong - 32-bit fixup corresponding to foo fixup_ve_reflong = FirstTargetFixupKind, + /// fixup_ve_srel32 - 32-bit fixup corresponding to foo for relative branch + fixup_ve_srel32, + /// fixup_ve_hi32 - 32-bit fixup corresponding to foo@hi fixup_ve_hi32, diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp index d50d8fcae9da..65bb0cf8b0d7 100644 --- a/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp +++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp @@ -102,11 +102,11 @@ unsigned VEMCCodeEmitter::getMachineOpValue(const MCInst &MI, const MCSubtargetInfo &STI) const { if (MO.isReg()) return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()); - if (MO.isImm()) - return MO.getImm(); + return static_cast<unsigned>(MO.getImm()); assert(MO.isExpr()); + const MCExpr *Expr = MO.getExpr(); if (const VEMCExpr *SExpr = dyn_cast<VEMCExpr>(Expr)) { MCFixupKind Kind = (MCFixupKind)SExpr->getFixupKind(); @@ -131,7 +131,7 @@ VEMCCodeEmitter::getBranchTargetOpValue(const MCInst &MI, unsigned OpNo, return getMachineOpValue(MI, MO, Fixups, STI); Fixups.push_back( - MCFixup::create(0, MO.getExpr(), (MCFixupKind)VE::fixup_ve_pc_lo32)); + MCFixup::create(0, MO.getExpr(), (MCFixupKind)VE::fixup_ve_srel32)); return 0; } diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp index a3ce3b3309be..4d45918ad0aa 100644 --- a/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp +++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp @@ -12,11 +12,12 @@ //===----------------------------------------------------------------------===// #include "VEMCExpr.h" +#include "llvm/BinaryFormat/ELF.h" #include "llvm/MC/MCAssembler.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCObjectStreamer.h" #include "llvm/MC/MCSymbolELF.h" -#include "llvm/BinaryFormat/ELF.h" +#include "llvm/MC/MCValue.h" using namespace llvm; @@ -174,7 +175,13 @@ VE::Fixups VEMCExpr::getFixupKind(VEMCExpr::VariantKind Kind) { bool VEMCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAsmLayout *Layout, const MCFixup *Fixup) const { - return getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup); + if (!getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup)) + return false; + + Res = + MCValue::get(Res.getSymA(), Res.getSymB(), Res.getConstant(), getKind()); + + return true; } static void fixELFSymbolsInTLSFixupsImpl(const MCExpr *Expr, MCAssembler &Asm) { diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp index 32315543826a..5ef223d6030b 100644 --- a/llvm/lib/Target/VE/VEISelLowering.cpp +++ b/llvm/lib/Target/VE/VEISelLowering.cpp @@ -1720,7 +1720,7 @@ SDValue VETargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::EXTRACT_VECTOR_ELT: return lowerEXTRACT_VECTOR_ELT(Op, DAG); -#define ADD_BINARY_VVP_OP(VVP_NAME, ISD_NAME) case ISD::ISD_NAME: +#define ADD_BINARY_VVP_OP(VVP_NAME, VP_NAME, ISD_NAME) case ISD::ISD_NAME: #include "VVPNodes.def" return lowerToVVP(Op, DAG); } diff --git a/llvm/lib/Target/VE/VVPInstrInfo.td b/llvm/lib/Target/VE/VVPInstrInfo.td index 2c88d5099a7b..99566e91ec11 100644 --- a/llvm/lib/Target/VE/VVPInstrInfo.td +++ b/llvm/lib/Target/VE/VVPInstrInfo.td @@ -29,6 +29,16 @@ def SDTIntBinOpVVP : SDTypeProfile<1, 4, [ // vp_add, vp_and, etc. IsVLVT<4> ]>; +// BinaryFPOp(x,y,mask,vl) +def SDTFPBinOpVVP : SDTypeProfile<1, 4, [ // vvp_fadd, etc. + SDTCisSameAs<0, 1>, + SDTCisSameAs<0, 2>, + SDTCisFP<0>, + SDTCisInt<3>, + SDTCisSameNumEltsAs<0, 3>, + IsVLVT<4> +]>; + // Binary operator commutative pattern. class vvp_commutative<SDNode RootOp> : PatFrags< @@ -40,7 +50,32 @@ class vvp_commutative<SDNode RootOp> : def vvp_add : SDNode<"VEISD::VVP_ADD", SDTIntBinOpVVP>; def c_vvp_add : vvp_commutative<vvp_add>; +def vvp_sub : SDNode<"VEISD::VVP_SUB", SDTIntBinOpVVP>; + +def vvp_mul : SDNode<"VEISD::VVP_MUL", SDTIntBinOpVVP>; +def c_vvp_mul : vvp_commutative<vvp_mul>; + +def vvp_sdiv : SDNode<"VEISD::VVP_SDIV", SDTIntBinOpVVP>; +def vvp_udiv : SDNode<"VEISD::VVP_UDIV", SDTIntBinOpVVP>; + def vvp_and : SDNode<"VEISD::VVP_AND", SDTIntBinOpVVP>; def c_vvp_and : vvp_commutative<vvp_and>; +def vvp_or : SDNode<"VEISD::VVP_OR", SDTIntBinOpVVP>; +def c_vvp_or : vvp_commutative<vvp_or>; + +def vvp_xor : SDNode<"VEISD::VVP_XOR", SDTIntBinOpVVP>; +def c_vvp_xor : vvp_commutative<vvp_xor>; + +def vvp_srl : SDNode<"VEISD::VVP_SRL", SDTIntBinOpVVP>; +def vvp_sra : SDNode<"VEISD::VVP_SRA", SDTIntBinOpVVP>; +def vvp_shl : SDNode<"VEISD::VVP_SHL", SDTIntBinOpVVP>; + +def vvp_fadd : SDNode<"VEISD::VVP_FADD", SDTFPBinOpVVP>; +def c_vvp_fadd : vvp_commutative<vvp_fadd>; +def vvp_fsub : SDNode<"VEISD::VVP_FSUB", SDTFPBinOpVVP>; +def vvp_fmul : SDNode<"VEISD::VVP_FMUL", SDTFPBinOpVVP>; +def c_vvp_fmul : vvp_commutative<vvp_fmul>; +def vvp_fdiv : SDNode<"VEISD::VVP_FDIV", SDTFPBinOpVVP>; + // } Binary Operators diff --git a/llvm/lib/Target/VE/VVPInstrPatternsVec.td b/llvm/lib/Target/VE/VVPInstrPatternsVec.td index ac03e0bf627e..8d5d9d103547 100644 --- a/llvm/lib/Target/VE/VVPInstrPatternsVec.td +++ b/llvm/lib/Target/VE/VVPInstrPatternsVec.td @@ -17,54 +17,177 @@ //===----------------------------------------------------------------------===// include "VVPInstrInfo.td" -multiclass VectorBinaryArith< - SDPatternOperator OpNode, - ValueType ScalarVT, ValueType DataVT, ValueType MaskVT, - string OpBaseName> { - // No mask. +multiclass Binary_rv<SDPatternOperator OpNode, + ValueType ScalarVT, ValueType DataVT, + ValueType MaskVT, string OpBaseName> { + // Masked with select, broadcast. + // TODO + + // Unmasked, broadcast. def : Pat<(OpNode - (any_broadcast ScalarVT:$sx), - DataVT:$vy, (MaskVT true_mask), i32:$avl), + (any_broadcast ScalarVT:$sx), DataVT:$vy, + (MaskVT true_mask), + i32:$avl), (!cast<Instruction>(OpBaseName#"rvl") ScalarVT:$sx, $vy, $avl)>; - def : Pat<(OpNode DataVT:$vx, DataVT:$vy, (MaskVT true_mask), i32:$avl), + // Masked, broadcast. + def : Pat<(OpNode + (any_broadcast ScalarVT:$sx), DataVT:$vy, + MaskVT:$mask, + i32:$avl), + (!cast<Instruction>(OpBaseName#"rvml") + ScalarVT:$sx, $vy, $mask, $avl)>; +} + +multiclass Binary_vr<SDPatternOperator OpNode, + ValueType ScalarVT, ValueType DataVT, + ValueType MaskVT, string OpBaseName> { + // Masked with select, broadcast. + // TODO + + // Unmasked, broadcast. + def : Pat<(OpNode + DataVT:$vx, (any_broadcast ScalarVT:$sy), + (MaskVT true_mask), + i32:$avl), + (!cast<Instruction>(OpBaseName#"vrl") + $vx, ScalarVT:$sy, $avl)>; + // Masked, broadcast. + def : Pat<(OpNode + DataVT:$vx, (any_broadcast ScalarVT:$sy), + MaskVT:$mask, + i32:$avl), + (!cast<Instruction>(OpBaseName#"vrml") + $vx, ScalarVT:$sy, $mask, $avl)>; +} + +multiclass Binary_vv<SDPatternOperator OpNode, + ValueType DataVT, + ValueType MaskVT, string OpBaseName> { + // Masked with select. + // TODO + + // Unmasked. + def : Pat<(OpNode + DataVT:$vx, DataVT:$vy, + (MaskVT true_mask), + i32:$avl), (!cast<Instruction>(OpBaseName#"vvl") $vx, $vy, $avl)>; - // Mask. + // Masked. def : Pat<(OpNode - (any_broadcast ScalarVT:$sx), - DataVT:$vy, MaskVT:$mask, i32:$avl), - (!cast<Instruction>(OpBaseName#"rvml") - ScalarVT:$sx, $vy, $mask, $avl)>; - def : Pat<(OpNode DataVT:$vx, DataVT:$vy, MaskVT:$mask, i32:$avl), + DataVT:$vx, DataVT:$vy, + MaskVT:$mask, + i32:$avl), (!cast<Instruction>(OpBaseName#"vvml") $vx, $vy, $mask, $avl)>; +} - // TODO We do not specify patterns for the immediate variants here. There - // will be an immediate folding pass that takes care of switching to the - // immediate variant where applicable. +multiclass Binary_rv_vv< + SDPatternOperator OpNode, + ValueType ScalarVT, ValueType DataVT, ValueType MaskVT, + string OpBaseName> { + defm : Binary_rv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>; + defm : Binary_vv<OpNode, DataVT, MaskVT, OpBaseName>; +} + +multiclass Binary_vr_vv< + SDPatternOperator OpNode, + ValueType ScalarVT, ValueType DataVT, ValueType MaskVT, + string OpBaseName> { + defm : Binary_vr<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>; + defm : Binary_vv<OpNode, DataVT, MaskVT, OpBaseName>; +} - // TODO Fold vvp_select into passthru. +multiclass Binary_rv_vr_vv< + SDPatternOperator OpNode, + ValueType ScalarVT, ValueType DataVT, ValueType MaskVT, + string OpBaseName> { + defm : Binary_rv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>; + defm : Binary_vr_vv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>; } // Expand both 64bit and 32 bit variant (256 elements) -multiclass VectorBinaryArith_ShortLong< +multiclass Binary_rv_vv_ShortLong< + SDPatternOperator OpNode, + ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName, + ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> { + defm : Binary_rv_vv<OpNode, + LongScalarVT, LongDataVT, v256i1, + LongOpBaseName>; + defm : Binary_rv_vv<OpNode, + ShortScalarVT, ShortDataVT, v256i1, + ShortOpBaseName>; +} + +multiclass Binary_vr_vv_ShortLong< + SDPatternOperator OpNode, + ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName, + ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> { + defm : Binary_vr_vv<OpNode, + LongScalarVT, LongDataVT, v256i1, + LongOpBaseName>; + defm : Binary_vr_vv<OpNode, + ShortScalarVT, ShortDataVT, v256i1, + ShortOpBaseName>; +} + +multiclass Binary_rv_vr_vv_ShortLong< SDPatternOperator OpNode, ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName, ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> { - defm : VectorBinaryArith<OpNode, - LongScalarVT, LongDataVT, v256i1, - LongOpBaseName>; - defm : VectorBinaryArith<OpNode, - ShortScalarVT, ShortDataVT, v256i1, - ShortOpBaseName>; + defm : Binary_rv_vr_vv<OpNode, + LongScalarVT, LongDataVT, v256i1, + LongOpBaseName>; + defm : Binary_rv_vr_vv<OpNode, + ShortScalarVT, ShortDataVT, v256i1, + ShortOpBaseName>; } +defm : Binary_rv_vv_ShortLong<c_vvp_add, + i64, v256i64, "VADDSL", + i32, v256i32, "VADDSWSX">; +defm : Binary_rv_vv_ShortLong<vvp_sub, + i64, v256i64, "VSUBSL", + i32, v256i32, "VSUBSWSX">; +defm : Binary_rv_vv_ShortLong<c_vvp_mul, + i64, v256i64, "VMULSL", + i32, v256i32, "VMULSWSX">; +defm : Binary_rv_vr_vv_ShortLong<vvp_sdiv, + i64, v256i64, "VDIVSL", + i32, v256i32, "VDIVSWSX">; +defm : Binary_rv_vr_vv_ShortLong<vvp_udiv, + i64, v256i64, "VDIVUL", + i32, v256i32, "VDIVUW">; +defm : Binary_rv_vv_ShortLong<c_vvp_and, + i64, v256i64, "VAND", + i32, v256i32, "PVANDLO">; +defm : Binary_rv_vv_ShortLong<c_vvp_or, + i64, v256i64, "VOR", + i32, v256i32, "PVORLO">; +defm : Binary_rv_vv_ShortLong<c_vvp_xor, + i64, v256i64, "VXOR", + i32, v256i32, "PVXORLO">; +defm : Binary_vr_vv_ShortLong<vvp_shl, + i64, v256i64, "VSLL", + i32, v256i32, "PVSLLLO">; +defm : Binary_vr_vv_ShortLong<vvp_sra, + i64, v256i64, "VSRAL", + i32, v256i32, "PVSRALO">; +defm : Binary_vr_vv_ShortLong<vvp_srl, + i64, v256i64, "VSRL", + i32, v256i32, "PVSRLLO">; -defm : VectorBinaryArith_ShortLong<c_vvp_add, - i64, v256i64, "VADDSL", - i32, v256i32, "VADDSWSX">; -defm : VectorBinaryArith_ShortLong<c_vvp_and, - i64, v256i64, "VAND", - i32, v256i32, "PVANDLO">; +defm : Binary_rv_vv_ShortLong<c_vvp_fadd, + f64, v256f64, "VFADDD", + f32, v256f32, "PVFADDUP">; +defm : Binary_rv_vv_ShortLong<c_vvp_fmul, + f64, v256f64, "VFMULD", + f32, v256f32, "PVFMULUP">; +defm : Binary_rv_vv_ShortLong<vvp_fsub, + f64, v256f64, "VFSUBD", + f32, v256f32, "PVFSUBUP">; +defm : Binary_rv_vr_vv_ShortLong<vvp_fdiv, + f64, v256f64, "VFDIVD", + f32, v256f32, "VFDIVS">; diff --git a/llvm/lib/Target/VE/VVPNodes.def b/llvm/lib/Target/VE/VVPNodes.def index a68402e9ea10..8a9231f7d3e6 100644 --- a/llvm/lib/Target/VE/VVPNodes.def +++ b/llvm/lib/Target/VE/VVPNodes.def @@ -28,14 +28,38 @@ /// \p VVPName is a VVP Binary operator. /// \p SDNAME is the generic SD opcode corresponding to \p VVPName. #ifndef ADD_BINARY_VVP_OP -#define ADD_BINARY_VVP_OP(X,Y) ADD_VVP_OP(X,Y) HANDLE_VP_TO_VVP(VP_##Y, X) +#define ADD_BINARY_VVP_OP(VVPNAME,VPNAME,SDNAME) \ + ADD_VVP_OP(VVPNAME,SDNAME) \ + HANDLE_VP_TO_VVP(VPNAME, VVPNAME) +#endif + +#ifndef ADD_BINARY_VVP_OP_COMPACT +#define ADD_BINARY_VVP_OP_COMPACT(NAME) \ + ADD_BINARY_VVP_OP(VVP_##NAME,VP_##NAME,NAME) #endif // Integer arithmetic. -ADD_BINARY_VVP_OP(VVP_ADD,ADD) +ADD_BINARY_VVP_OP_COMPACT(ADD) +ADD_BINARY_VVP_OP_COMPACT(SUB) +ADD_BINARY_VVP_OP_COMPACT(MUL) +ADD_BINARY_VVP_OP_COMPACT(UDIV) +ADD_BINARY_VVP_OP_COMPACT(SDIV) -ADD_BINARY_VVP_OP(VVP_AND,AND) +ADD_BINARY_VVP_OP(VVP_SRA,VP_ASHR,SRA) +ADD_BINARY_VVP_OP(VVP_SRL,VP_LSHR,SRL) +ADD_BINARY_VVP_OP_COMPACT(SHL) + +ADD_BINARY_VVP_OP_COMPACT(AND) +ADD_BINARY_VVP_OP_COMPACT(OR) +ADD_BINARY_VVP_OP_COMPACT(XOR) + +// FP arithmetic. +ADD_BINARY_VVP_OP_COMPACT(FADD) +ADD_BINARY_VVP_OP_COMPACT(FSUB) +ADD_BINARY_VVP_OP_COMPACT(FMUL) +ADD_BINARY_VVP_OP_COMPACT(FDIV) -#undef HANDLE_VP_TO_VVP #undef ADD_BINARY_VVP_OP +#undef ADD_BINARY_VVP_OP_COMPACT #undef ADD_VVP_OP +#undef HANDLE_VP_TO_VVP diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp index 7d1e6c553f81..56689d3ee06b 100644 --- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp +++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp @@ -571,7 +571,6 @@ public: // proper nesting. bool ExpectBlockType = false; bool ExpectFuncType = false; - bool ExpectHeapType = false; std::unique_ptr<WebAssemblyOperand> FunctionTable; if (Name == "block") { push(Block); @@ -624,8 +623,6 @@ public: if (parseFunctionTableOperand(&FunctionTable)) return true; ExpectFuncType = true; - } else if (Name == "ref.null") { - ExpectHeapType = true; } if (ExpectFuncType || (ExpectBlockType && Lexer.is(AsmToken::LParen))) { @@ -670,15 +667,6 @@ public: return error("Unknown block type: ", Id); addBlockTypeOperand(Operands, NameLoc, BT); Parser.Lex(); - } else if (ExpectHeapType) { - auto HeapType = WebAssembly::parseHeapType(Id.getString()); - if (HeapType == WebAssembly::HeapType::Invalid) { - return error("Expected a heap type: ", Id); - } - Operands.push_back(std::make_unique<WebAssemblyOperand>( - WebAssemblyOperand::Integer, Id.getLoc(), Id.getEndLoc(), - WebAssemblyOperand::IntOp{static_cast<int64_t>(HeapType)})); - Parser.Lex(); } else { // Assume this identifier is a label. const MCExpr *Val; diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp index a6b5d4252f2f..128ce5c4fec0 100644 --- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp +++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp @@ -112,9 +112,18 @@ bool WebAssemblyAsmTypeCheck::getLocal(SMLoc ErrorLoc, const MCInst &Inst, return false; } -bool WebAssemblyAsmTypeCheck::checkEnd(SMLoc ErrorLoc) { +bool WebAssemblyAsmTypeCheck::checkEnd(SMLoc ErrorLoc, bool PopVals) { if (LastSig.Returns.size() > Stack.size()) return typeError(ErrorLoc, "end: insufficient values on the type stack"); + + if (PopVals) { + for (auto VT : llvm::reverse(LastSig.Returns)) { + if (popType(ErrorLoc, VT)) + return true; + } + return false; + } + for (size_t i = 0; i < LastSig.Returns.size(); i++) { auto EVT = LastSig.Returns[i]; auto PVT = Stack[Stack.size() - LastSig.Returns.size() + i]; @@ -221,7 +230,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst) { return true; } else if (Name == "end_block" || Name == "end_loop" || Name == "end_if" || Name == "else" || Name == "end_try") { - if (checkEnd(ErrorLoc)) + if (checkEnd(ErrorLoc, Name == "else")) return true; if (Name == "end_block") Unreachable = false; diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h index aa35213ccca3..2b07faf67a18 100644 --- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h +++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h @@ -39,7 +39,7 @@ class WebAssemblyAsmTypeCheck final { bool typeError(SMLoc ErrorLoc, const Twine &Msg); bool popType(SMLoc ErrorLoc, Optional<wasm::ValType> EVT); bool getLocal(SMLoc ErrorLoc, const MCInst &Inst, wasm::ValType &Type); - bool checkEnd(SMLoc ErrorLoc); + bool checkEnd(SMLoc ErrorLoc, bool PopVals = false); bool checkSig(SMLoc ErrorLoc, const wasm::WasmSignature &Sig); bool getSymRef(SMLoc ErrorLoc, const MCInst &Inst, const MCSymbolRefExpr *&SymRef); diff --git a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp index 2e1e4f061219..5d38145559da 100644 --- a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp +++ b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp @@ -241,28 +241,6 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction( } break; } - // heap_type operands, for e.g. ref.null: - case WebAssembly::OPERAND_HEAPTYPE: { - int64_t Val; - uint64_t PrevSize = Size; - if (!nextLEB(Val, Bytes, Size, true)) - return MCDisassembler::Fail; - if (Val < 0 && Size == PrevSize + 1) { - // The HeapType encoding is like BlockType, in that encodings that - // decode as negative values indicate ValTypes. In practice we expect - // either wasm::ValType::EXTERNREF or wasm::ValType::FUNCREF here. - // - // The positive SLEB values are reserved for future expansion and are - // expected to be type indices in the typed function references - // proposal, and should disassemble as MCSymbolRefExpr as in BlockType - // above. - MI.addOperand(MCOperand::createImm(Val & 0x7f)); - } else { - MI.addOperand( - MCOperand::createImm(int64_t(WebAssembly::HeapType::Invalid))); - } - break; - } // FP operands. case WebAssembly::OPERAND_F32IMM: { if (!parseImmediate<float>(MI, Size, Bytes)) diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp index 2967aaa00ad4..d72bfdbbfb99 100644 --- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp @@ -366,26 +366,3 @@ void WebAssemblyInstPrinter::printWebAssemblySignatureOperand(const MCInst *MI, } } } - -void WebAssemblyInstPrinter::printWebAssemblyHeapTypeOperand(const MCInst *MI, - unsigned OpNo, - raw_ostream &O) { - const MCOperand &Op = MI->getOperand(OpNo); - if (Op.isImm()) { - switch (Op.getImm()) { - case long(wasm::ValType::EXTERNREF): - O << "extern"; - break; - case long(wasm::ValType::FUNCREF): - O << "func"; - break; - default: - O << "unsupported_heap_type_value"; - break; - } - } else { - // Typed function references and other subtypes of funcref and externref - // currently unimplemented. - O << "unsupported_heap_type_operand"; - } -} diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h index 7d980c78c3c9..fe104cbca12e 100644 --- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h +++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h @@ -47,8 +47,6 @@ public: raw_ostream &O); void printWebAssemblySignatureOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O); - void printWebAssemblyHeapTypeOperand(const MCInst *MI, unsigned OpNo, - raw_ostream &O); // Autogenerated by tblgen. std::pair<const char *, uint64_t> getMnemonic(const MCInst *MI) override; diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp index c3d259e6ff20..d8122950e061 100644 --- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp +++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "WebAssemblyMCAsmInfo.h" +#include "Utils/WebAssemblyUtilities.h" #include "llvm/ADT/Triple.h" using namespace llvm; @@ -44,5 +45,13 @@ WebAssemblyMCAsmInfo::WebAssemblyMCAsmInfo(const Triple &T, SupportsDebugInformation = true; + // When compilation is done on a cpp file by clang, the exception model info + // is stored in LangOptions, which is later used to set the info in + // TargetOptions and then MCAsmInfo in LLVMTargetMachine::initAsmInfo(). But + // this process does not happen when compiling bitcode directly with clang, so + // we make sure this info is set correctly. + if (WebAssembly::WasmEnableEH || WebAssembly::WasmEnableSjLj) + ExceptionsType = ExceptionHandling::Wasm; + // TODO: UseIntegratedAssembler? } diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp index 4961c2ef9529..6e494b9430f7 100644 --- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp +++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp @@ -106,9 +106,6 @@ void WebAssemblyMCCodeEmitter::encodeInstruction( encodeSLEB128(int64_t(MO.getImm()), OS); break; case WebAssembly::OPERAND_SIGNATURE: - case WebAssembly::OPERAND_HEAPTYPE: - OS << uint8_t(MO.getImm()); - break; case WebAssembly::OPERAND_VEC_I8IMM: support::endian::write<uint8_t>(OS, MO.getImm(), support::little); break; diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h index d07bfce9abc1..b2f10ca93a4f 100644 --- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h +++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h @@ -78,8 +78,6 @@ enum OperandType { OPERAND_BRLIST, /// 32-bit unsigned table number. OPERAND_TABLE, - /// heap type immediate for ref.null. - OPERAND_HEAPTYPE, }; } // end namespace WebAssembly diff --git a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.cpp b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.cpp index 6f81431bba2d..0412e524f800 100644 --- a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.cpp +++ b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.cpp @@ -41,13 +41,6 @@ Optional<wasm::ValType> WebAssembly::parseType(StringRef Type) { return Optional<wasm::ValType>(); } -WebAssembly::HeapType WebAssembly::parseHeapType(StringRef Type) { - return StringSwitch<WebAssembly::HeapType>(Type) - .Case("extern", WebAssembly::HeapType::Externref) - .Case("func", WebAssembly::HeapType::Funcref) - .Default(WebAssembly::HeapType::Invalid); -} - WebAssembly::BlockType WebAssembly::parseBlockType(StringRef Type) { // Multivalue block types are handled separately in parseSignature return StringSwitch<WebAssembly::BlockType>(Type) diff --git a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h index 8d757df27b34..042d51c7d6cb 100644 --- a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h +++ b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyTypeUtilities.h @@ -41,17 +41,9 @@ enum class BlockType : unsigned { Multivalue = 0xffff, }; -/// Used as immediate MachineOperands for heap types, e.g. for ref.null. -enum class HeapType : unsigned { - Invalid = 0x00, - Externref = unsigned(wasm::ValType::EXTERNREF), - Funcref = unsigned(wasm::ValType::FUNCREF), -}; - // Convert StringRef to ValType / HealType / BlockType Optional<wasm::ValType> parseType(StringRef Type); -HeapType parseHeapType(StringRef Type); BlockType parseBlockType(StringRef Type); MVT parseMVT(StringRef Type); diff --git a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.cpp b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.cpp index 3da80f4fc875..b87c884c9e4a 100644 --- a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.cpp +++ b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.cpp @@ -18,6 +18,31 @@ #include "llvm/MC/MCContext.h" using namespace llvm; +// Exception handling & setjmp-longjmp handling related options. These are +// defined here to be shared between WebAssembly and its subdirectories. + +// Emscripten's asm.js-style exception handling +cl::opt<bool> WebAssembly::WasmEnableEmEH( + "enable-emscripten-cxx-exceptions", + cl::desc("WebAssembly Emscripten-style exception handling"), + cl::init(false)); +// Emscripten's asm.js-style setjmp/longjmp handling +cl::opt<bool> WebAssembly::WasmEnableEmSjLj( + "enable-emscripten-sjlj", + cl::desc("WebAssembly Emscripten-style setjmp/longjmp handling"), + cl::init(false)); +// Exception handling using wasm EH instructions +cl::opt<bool> + WebAssembly::WasmEnableEH("wasm-enable-eh", + cl::desc("WebAssembly exception handling"), + cl::init(false)); +// setjmp/longjmp handling using wasm EH instrutions +cl::opt<bool> + WebAssembly::WasmEnableSjLj("wasm-enable-sjlj", + cl::desc("WebAssembly setjmp/longjmp handling"), + cl::init(false)); + +// Function names in libc++abi and libunwind const char *const WebAssembly::CxaBeginCatchFn = "__cxa_begin_catch"; const char *const WebAssembly::CxaRethrowFn = "__cxa_rethrow"; const char *const WebAssembly::StdTerminateFn = "_ZSt9terminatev"; diff --git a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.h b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.h index f6e96d9b2877..d024185defb4 100644 --- a/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.h +++ b/llvm/lib/Target/WebAssembly/Utils/WebAssemblyUtilities.h @@ -16,6 +16,7 @@ #define LLVM_LIB_TARGET_WEBASSEMBLY_UTILS_WEBASSEMBLYUTILITIES_H #include "llvm/IR/DerivedTypes.h" +#include "llvm/Support/CommandLine.h" namespace llvm { @@ -70,6 +71,12 @@ inline bool isRefType(const Type *Ty) { bool isChild(const MachineInstr &MI, const WebAssemblyFunctionInfo &MFI); bool mayThrow(const MachineInstr &MI); +// Exception handling / setjmp-longjmp handling command-line options +extern cl::opt<bool> WasmEnableEmEH; // asm.js-style EH +extern cl::opt<bool> WasmEnableEmSjLj; // asm.js-style SjLJ +extern cl::opt<bool> WasmEnableEH; // EH using Wasm EH instructions +extern cl::opt<bool> WasmEnableSjLj; // SjLj using Wasm EH instructions + // Exception-related function names extern const char *const ClangCallTerminateFn; extern const char *const CxaBeginCatchFn; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp index 0d3f51693261..e3af6b2662ef 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp @@ -51,8 +51,6 @@ using namespace llvm; #define DEBUG_TYPE "asm-printer" extern cl::opt<bool> WasmKeepRegisters; -extern cl::opt<bool> WasmEnableEmEH; -extern cl::opt<bool> WasmEnableEmSjLj; //===----------------------------------------------------------------------===// // Helpers. @@ -196,6 +194,13 @@ void WebAssemblyAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) { Sym->setGlobalType(wasm::WasmGlobalType{uint8_t(Type), Mutable}); } + // If the GlobalVariable refers to a table, we handle it here instead of + // in emitExternalDecls + if (Sym->isTable()) { + getTargetStreamer()->emitTableType(Sym); + return; + } + emitVisibility(Sym, GV->getVisibility(), !GV->isDeclaration()); if (GV->hasInitializer()) { assert(getSymbolPreferLocal(*GV) == Sym); @@ -315,8 +320,9 @@ void WebAssemblyAsmPrinter::emitExternalDecls(const Module &M) { // will discard it later if it turns out not to be necessary. auto Signature = signatureFromMVTs(Results, Params); bool InvokeDetected = false; - auto *Sym = getMCSymbolForFunction(&F, WasmEnableEmEH || WasmEnableEmSjLj, - Signature.get(), InvokeDetected); + auto *Sym = getMCSymbolForFunction( + &F, WebAssembly::WasmEnableEmEH || WebAssembly::WasmEnableEmSjLj, + Signature.get(), InvokeDetected); // Multiple functions can be mapped to the same invoke symbol. For // example, two IR functions '__invoke_void_i8*' and '__invoke_void_i32' diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp index 7832f199a2cc..17e867e4c7d8 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp @@ -1741,7 +1741,7 @@ void WebAssemblyCFGStackify::rewriteDepthImmediates(MachineFunction &MF) { void WebAssemblyCFGStackify::cleanupFunctionData(MachineFunction &MF) { if (FakeCallerBB) - MF.DeleteMachineBasicBlock(FakeCallerBB); + MF.deleteMachineBasicBlock(FakeCallerBB); AppendixBB = FakeCallerBB = nullptr; } diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def index 1fa0ea3867c7..a3a33f4a5b3a 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def @@ -31,6 +31,7 @@ HANDLE_NODETYPE(SWIZZLE) HANDLE_NODETYPE(VEC_SHL) HANDLE_NODETYPE(VEC_SHR_S) HANDLE_NODETYPE(VEC_SHR_U) +HANDLE_NODETYPE(NARROW_U) HANDLE_NODETYPE(EXTEND_LOW_S) HANDLE_NODETYPE(EXTEND_LOW_U) HANDLE_NODETYPE(EXTEND_HIGH_S) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp index 0df8f3e0e09c..38ed4c73fb93 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp @@ -176,6 +176,8 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering( setTargetDAGCombine(ISD::FP_ROUND); setTargetDAGCombine(ISD::CONCAT_VECTORS); + setTargetDAGCombine(ISD::TRUNCATE); + // Support saturating add for i8x16 and i16x8 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT}) for (auto T : {MVT::v16i8, MVT::v8i16}) @@ -644,8 +646,7 @@ LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB, Register RegFuncref = MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass); MachineInstr *RefNull = - BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref) - .addImm(static_cast<int32_t>(WebAssembly::HeapType::Funcref)); + BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref); BB->insertAfter(Const0->getIterator(), RefNull); MachineInstr *TableSet = @@ -2610,6 +2611,114 @@ performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { return DAG.getNode(Op, SDLoc(N), ResVT, Source); } +// Helper to extract VectorWidth bits from Vec, starting from IdxVal. +static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, + const SDLoc &DL, unsigned VectorWidth) { + EVT VT = Vec.getValueType(); + EVT ElVT = VT.getVectorElementType(); + unsigned Factor = VT.getSizeInBits() / VectorWidth; + EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT, + VT.getVectorNumElements() / Factor); + + // Extract the relevant VectorWidth bits. Generate an EXTRACT_SUBVECTOR + unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits(); + assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2"); + + // This is the index of the first element of the VectorWidth-bit chunk + // we want. Since ElemsPerChunk is a power of 2 just need to clear bits. + IdxVal &= ~(ElemsPerChunk - 1); + + // If the input is a buildvector just emit a smaller one. + if (Vec.getOpcode() == ISD::BUILD_VECTOR) + return DAG.getBuildVector(ResultVT, DL, + Vec->ops().slice(IdxVal, ElemsPerChunk)); + + SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, DL); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, VecIdx); +} + +// Helper to recursively truncate vector elements in half with NARROW_U. DstVT +// is the expected destination value type after recursion. In is the initial +// input. Note that the input should have enough leading zero bits to prevent +// NARROW_U from saturating results. +static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL, + SelectionDAG &DAG) { + EVT SrcVT = In.getValueType(); + + // No truncation required, we might get here due to recursive calls. + if (SrcVT == DstVT) + return In; + + unsigned SrcSizeInBits = SrcVT.getSizeInBits(); + unsigned NumElems = SrcVT.getVectorNumElements(); + if (!isPowerOf2_32(NumElems)) + return SDValue(); + assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation"); + assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation"); + + LLVMContext &Ctx = *DAG.getContext(); + EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2); + + // Narrow to the largest type possible: + // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u. + EVT InVT = MVT::i16, OutVT = MVT::i8; + if (SrcVT.getScalarSizeInBits() > 16) { + InVT = MVT::i32; + OutVT = MVT::i16; + } + unsigned SubSizeInBits = SrcSizeInBits / 2; + InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits()); + OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits()); + + // Split lower/upper subvectors. + SDValue Lo = extractSubVector(In, 0, DAG, DL, SubSizeInBits); + SDValue Hi = extractSubVector(In, NumElems / 2, DAG, DL, SubSizeInBits); + + // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors. + if (SrcVT.is256BitVector() && DstVT.is128BitVector()) { + Lo = DAG.getBitcast(InVT, Lo); + Hi = DAG.getBitcast(InVT, Hi); + SDValue Res = DAG.getNode(WebAssemblyISD::NARROW_U, DL, OutVT, Lo, Hi); + return DAG.getBitcast(DstVT, Res); + } + + // Recursively narrow lower/upper subvectors, concat result and narrow again. + EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2); + Lo = truncateVectorWithNARROW(PackedVT, Lo, DL, DAG); + Hi = truncateVectorWithNARROW(PackedVT, Hi, DL, DAG); + + PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems); + SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi); + return truncateVectorWithNARROW(DstVT, Res, DL, DAG); +} + +static SDValue performTruncateCombine(SDNode *N, + TargetLowering::DAGCombinerInfo &DCI) { + auto &DAG = DCI.DAG; + + SDValue In = N->getOperand(0); + EVT InVT = In.getValueType(); + if (!InVT.isSimple()) + return SDValue(); + + EVT OutVT = N->getValueType(0); + if (!OutVT.isVector()) + return SDValue(); + + EVT OutSVT = OutVT.getVectorElementType(); + EVT InSVT = InVT.getVectorElementType(); + // Currently only cover truncate to v16i8 or v8i16. + if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) && + (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector())) + return SDValue(); + + SDLoc DL(N); + APInt Mask = APInt::getLowBitsSet(InVT.getScalarSizeInBits(), + OutVT.getScalarSizeInBits()); + In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT)); + return truncateVectorWithNARROW(OutVT, In, DL, DAG); +} + SDValue WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { @@ -2626,5 +2735,7 @@ WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N, case ISD::FP_ROUND: case ISD::CONCAT_VECTORS: return performVectorTruncZeroCombine(N, DCI); + case ISD::TRUNCATE: + return performTruncateCombine(N, DCI); } } diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td index ee9247a8bef9..3fb0af1d47a0 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.td @@ -202,11 +202,6 @@ def Signature : Operand<i32> { let PrintMethod = "printWebAssemblySignatureOperand"; } -let OperandType = "OPERAND_HEAPTYPE" in -def HeapType : Operand<i32> { - let PrintMethod = "printWebAssemblyHeapTypeOperand"; -} - let OperandType = "OPERAND_TYPEINDEX" in def TypeIndex : Operand<i32>; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td index ef9bd35d004a..76a88caafc47 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td @@ -11,13 +11,14 @@ /// //===----------------------------------------------------------------------===// -multiclass REF_I<WebAssemblyRegClass rc, ValueType vt> { - defm REF_NULL_#rc : I<(outs rc:$res), (ins HeapType:$heaptype), - (outs), (ins HeapType:$heaptype), - [], - "ref.null\t$res, $heaptype", - "ref.null\t$heaptype", - 0xd0>, +multiclass REF_I<WebAssemblyRegClass rc, ValueType vt, string ht> { + defm REF_NULL_#rc : I<(outs rc:$dst), (ins), + (outs), (ins), + [(set rc:$dst, (!cast<Intrinsic>("int_wasm_ref_null_" # ht)))], + "ref.null_" # ht # "$dst", + "ref.null_" # ht, + !cond(!eq(ht, "func") : 0xd070, + !eq(ht, "extern") : 0xd06f)>, Requires<[HasReferenceTypes]>; defm SELECT_#rc: I<(outs rc:$dst), (ins rc:$lhs, rc:$rhs, I32:$cond), (outs), (ins), @@ -28,8 +29,8 @@ multiclass REF_I<WebAssemblyRegClass rc, ValueType vt> { Requires<[HasReferenceTypes]>; } -defm "" : REF_I<FUNCREF, funcref>; -defm "" : REF_I<EXTERNREF, externref>; +defm "" : REF_I<FUNCREF, funcref, "func">; +defm "" : REF_I<EXTERNREF, externref, "extern">; foreach rc = [FUNCREF, EXTERNREF] in { def : Pat<(select (i32 (setne I32:$cond, 0)), rc:$lhs, rc:$rhs), diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td index 30b99c3a69a9..5bb12c7fbdc7 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td @@ -1278,6 +1278,14 @@ multiclass SIMDNarrow<Vec vec, bits<32> baseInst> { defm "" : SIMDNarrow<I16x8, 101>; defm "" : SIMDNarrow<I32x4, 133>; +// WebAssemblyISD::NARROW_U +def wasm_narrow_t : SDTypeProfile<1, 2, []>; +def wasm_narrow_u : SDNode<"WebAssemblyISD::NARROW_U", wasm_narrow_t>; +def : Pat<(v16i8 (wasm_narrow_u (v8i16 V128:$left), (v8i16 V128:$right))), + (NARROW_U_I8x16 $left, $right)>; +def : Pat<(v8i16 (wasm_narrow_u (v4i32 V128:$left), (v4i32 V128:$right))), + (NARROW_U_I16x8 $left, $right)>; + // Bitcasts are nops // Matching bitcast t1 to t1 causes strange errors, so avoid repeating types foreach t1 = AllVecs in diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrTable.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrTable.td index e44c2073eaeb..1fd00bf1cbc8 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrTable.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrTable.td @@ -20,7 +20,7 @@ def WebAssemblyTableGet : SDNode<"WebAssemblyISD::TABLE_GET", WebAssemblyTableGe [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; -multiclass TABLE<WebAssemblyRegClass rc> { +multiclass TABLE<WebAssemblyRegClass rc, string suffix> { let mayLoad = 1 in defm TABLE_GET_#rc : I<(outs rc:$res), (ins table32_op:$table, I32:$i), (outs), (ins table32_op:$table), @@ -39,14 +39,14 @@ multiclass TABLE<WebAssemblyRegClass rc> { defm TABLE_GROW_#rc : I<(outs I32:$sz), (ins table32_op:$table, rc:$val, I32:$n), (outs), (ins table32_op:$table), - [], + [(set I32:$sz, (!cast<Intrinsic>("int_wasm_table_grow_" # suffix) (WebAssemblyWrapper tglobaladdr:$table), rc:$val, I32:$n))], "table.grow\t$sz, $table, $val, $n", "table.grow\t$table", 0xfc0f>; defm TABLE_FILL_#rc : I<(outs), (ins table32_op:$table, I32:$i, rc:$val, I32:$n), (outs), (ins table32_op:$table), - [], + [(!cast<Intrinsic>("int_wasm_table_fill_" # suffix) (WebAssemblyWrapper tglobaladdr:$table), I32:$i, rc:$val, I32:$n)], "table.fill\t$table, $i, $val, $n", "table.fill\t$table", 0xfc11>; @@ -62,8 +62,8 @@ multiclass TABLE<WebAssemblyRegClass rc> { } } -defm "" : TABLE<FUNCREF>, Requires<[HasReferenceTypes]>; -defm "" : TABLE<EXTERNREF>, Requires<[HasReferenceTypes]>; +defm "" : TABLE<FUNCREF, "funcref">, Requires<[HasReferenceTypes]>; +defm "" : TABLE<EXTERNREF, "externref">, Requires<[HasReferenceTypes]>; def : Pat<(WebAssemblyTableSet mcsym:$table, i32:$idx, funcref:$r), (TABLE_SET_FUNCREF mcsym:$table, i32:$idx, funcref:$r)>, @@ -71,7 +71,7 @@ def : Pat<(WebAssemblyTableSet mcsym:$table, i32:$idx, funcref:$r), defm TABLE_SIZE : I<(outs I32:$sz), (ins table32_op:$table), (outs), (ins table32_op:$table), - [], + [(set I32:$sz, (int_wasm_table_size (WebAssemblyWrapper tglobaladdr:$table)))], "table.size\t$sz, $table", "table.size\t$table", 0xfc10>, @@ -80,7 +80,9 @@ defm TABLE_SIZE : I<(outs I32:$sz), (ins table32_op:$table), defm TABLE_COPY : I<(outs), (ins table32_op:$table1, table32_op:$table2, I32:$d, I32:$s, I32:$n), (outs), (ins table32_op:$table1, table32_op:$table2), - [], + [(int_wasm_table_copy (WebAssemblyWrapper tglobaladdr:$table1), + (WebAssemblyWrapper tglobaladdr:$table2), + I32:$d, I32:$s, I32:$n)], "table.copy\t$table1, $table2, $d, $s, $n", "table.copy\t$table1, $table2", 0xfc0e>, diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp index 4eacc921b6cd..23aaa5160abd 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp @@ -267,6 +267,7 @@ /// ///===----------------------------------------------------------------------===// +#include "Utils/WebAssemblyUtilities.h" #include "WebAssembly.h" #include "WebAssemblyTargetMachine.h" #include "llvm/ADT/StringExtras.h" @@ -285,13 +286,6 @@ using namespace llvm; #define DEBUG_TYPE "wasm-lower-em-ehsjlj" -// Emscripten's asm.js-style exception handling -extern cl::opt<bool> WasmEnableEmEH; -// Emscripten's asm.js-style setjmp/longjmp handling -extern cl::opt<bool> WasmEnableEmSjLj; -// Wasm setjmp/longjmp handling using wasm EH instructions -extern cl::opt<bool> WasmEnableSjLj; - static cl::list<std::string> EHAllowlist("emscripten-cxx-exceptions-allowed", cl::desc("The list of function names in which Emscripten-style " @@ -370,8 +364,9 @@ public: static char ID; WebAssemblyLowerEmscriptenEHSjLj() - : ModulePass(ID), EnableEmEH(WasmEnableEmEH), - EnableEmSjLj(WasmEnableEmSjLj), EnableWasmSjLj(WasmEnableSjLj) { + : ModulePass(ID), EnableEmEH(WebAssembly::WasmEnableEmEH), + EnableEmSjLj(WebAssembly::WasmEnableEmSjLj), + EnableWasmSjLj(WebAssembly::WasmEnableSjLj) { assert(!(EnableEmSjLj && EnableWasmSjLj) && "Two SjLj modes cannot be turned on at the same time"); assert(!(EnableEmEH && EnableWasmSjLj) && diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp index 0b953a90aeab..09bccef17ab0 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp @@ -40,9 +40,6 @@ cl::opt<bool> " instruction output for test purposes only."), cl::init(false)); -extern cl::opt<bool> WasmEnableEmEH; -extern cl::opt<bool> WasmEnableEmSjLj; - static void removeRegisterOperands(const MachineInstr *MI, MCInst &OutMI); MCSymbol * @@ -66,9 +63,11 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const { // they reach this point as aggregate Array types with an element type // that is a reference type. wasm::ValType Type; + bool IsTable = false; if (GlobalVT->isArrayTy() && WebAssembly::isRefType(GlobalVT->getArrayElementType())) { MVT VT; + IsTable = true; switch (GlobalVT->getArrayElementType()->getPointerAddressSpace()) { case WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF: VT = MVT::funcref; @@ -85,9 +84,14 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const { } else report_fatal_error("Aggregate globals not yet implemented"); - WasmSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL); - WasmSym->setGlobalType( - wasm::WasmGlobalType{uint8_t(Type), /*Mutable=*/true}); + if (IsTable) { + WasmSym->setType(wasm::WASM_SYMBOL_TYPE_TABLE); + WasmSym->setTableType(Type); + } else { + WasmSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL); + WasmSym->setGlobalType( + wasm::WasmGlobalType{uint8_t(Type), /*Mutable=*/true}); + } } return WasmSym; } @@ -105,7 +109,8 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const { bool InvokeDetected = false; auto *WasmSym = Printer.getMCSymbolForFunction( - F, WasmEnableEmEH || WasmEnableEmSjLj, Signature.get(), InvokeDetected); + F, WebAssembly::WasmEnableEmEH || WebAssembly::WasmEnableEmSjLj, + Signature.get(), InvokeDetected); WasmSym->setSignature(Signature.get()); Printer.addSignature(std::move(Signature)); WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION); @@ -275,11 +280,6 @@ void WebAssemblyMCInstLower::lower(const MachineInstr *MI, SmallVector<wasm::ValType, 4>()); break; } - } else if (Info.OperandType == WebAssembly::OPERAND_HEAPTYPE) { - assert(static_cast<WebAssembly::HeapType>(MO.getImm()) != - WebAssembly::HeapType::Invalid); - // With typed function references, this will need a case for type - // index operands. Otherwise, fall through. } } MCOp = MCOperand::createImm(MO.getImm()); diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp index 7b70d99b5f52..482837178f3d 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp @@ -14,6 +14,7 @@ #include "WebAssemblyTargetMachine.h" #include "MCTargetDesc/WebAssemblyMCTargetDesc.h" #include "TargetInfo/WebAssemblyTargetInfo.h" +#include "Utils/WebAssemblyUtilities.h" #include "WebAssembly.h" #include "WebAssemblyMachineFunctionInfo.h" #include "WebAssemblyTargetObjectFile.h" @@ -24,6 +25,7 @@ #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/Function.h" +#include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/Scalar.h" @@ -33,28 +35,6 @@ using namespace llvm; #define DEBUG_TYPE "wasm" -// Emscripten's asm.js-style exception handling -cl::opt<bool> - WasmEnableEmEH("enable-emscripten-cxx-exceptions", - cl::desc("WebAssembly Emscripten-style exception handling"), - cl::init(false)); - -// Emscripten's asm.js-style setjmp/longjmp handling -cl::opt<bool> WasmEnableEmSjLj( - "enable-emscripten-sjlj", - cl::desc("WebAssembly Emscripten-style setjmp/longjmp handling"), - cl::init(false)); - -// Exception handling using wasm EH instructions -cl::opt<bool> WasmEnableEH("wasm-enable-eh", - cl::desc("WebAssembly exception handling"), - cl::init(false)); - -// setjmp/longjmp handling using wasm EH instrutions -cl::opt<bool> WasmEnableSjLj("wasm-enable-sjlj", - cl::desc("WebAssembly setjmp/longjmp handling"), - cl::init(false)); - // A command-line option to keep implicit locals // for the purpose of testing with lit/llc ONLY. // This produces output which is not valid WebAssembly, and is not supported @@ -368,7 +348,23 @@ FunctionPass *WebAssemblyPassConfig::createTargetRegisterAllocator(bool) { return nullptr; // No reg alloc } -static void basicCheckForEHAndSjLj(const TargetMachine *TM) { +using WebAssembly::WasmEnableEH; +using WebAssembly::WasmEnableEmEH; +using WebAssembly::WasmEnableEmSjLj; +using WebAssembly::WasmEnableSjLj; + +static void basicCheckForEHAndSjLj(TargetMachine *TM) { + // Before checking, we make sure TargetOptions.ExceptionModel is the same as + // MCAsmInfo.ExceptionsType. Normally these have to be the same, because clang + // stores the exception model info in LangOptions, which is later transferred + // to TargetOptions and MCAsmInfo. But when clang compiles bitcode directly, + // clang's LangOptions is not used and thus the exception model info is not + // correctly transferred to TargetOptions and MCAsmInfo, so we make sure we + // have the correct exception model in in WebAssemblyMCAsmInfo constructor. + // But in this case TargetOptions is still not updated, so we make sure they + // are the same. + TM->Options.ExceptionModel = TM->getMCAsmInfo()->getExceptionHandlingType(); + // Basic Correctness checking related to -exception-model if (TM->Options.ExceptionModel != ExceptionHandling::None && TM->Options.ExceptionModel != ExceptionHandling::Wasm) diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp index 8ce6b47d10e8..2ba0b97229cc 100644 --- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp +++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp @@ -1759,7 +1759,8 @@ bool X86AsmParser::CreateMemForMSInlineAsm( // registers in a mmory expression, and though unaccessible via rip/eip. if (IsGlobalLV && (BaseReg || IndexReg)) { Operands.push_back(X86Operand::CreateMem(getPointerWidth(), Disp, Start, - End, Size, Identifier, Decl)); + End, Size, Identifier, Decl, + FrontendSize)); return false; } // Otherwise, we set the base register to a non-zero value @@ -2551,8 +2552,6 @@ bool X86AsmParser::ParseIntelOperand(OperandVector &Operands) { StringRef ErrMsg; unsigned BaseReg = SM.getBaseReg(); unsigned IndexReg = SM.getIndexReg(); - if (IndexReg && BaseReg == X86::RIP) - BaseReg = 0; unsigned Scale = SM.getScale(); if (!PtrInOperand) Size = SM.getElementSize() << 3; @@ -4430,8 +4429,7 @@ bool X86AsmParser::MatchAndEmitATTInstruction(SMLoc IDLoc, unsigned &Opcode, // If exactly one matched, then we treat that as a successful match (and the // instruction will already have been filled in correctly, since the failing // matches won't have modified it). - unsigned NumSuccessfulMatches = - std::count(std::begin(Match), std::end(Match), Match_Success); + unsigned NumSuccessfulMatches = llvm::count(Match, Match_Success); if (NumSuccessfulMatches == 1) { if (!MatchingInlineAsm && validateInstruction(Inst, Operands)) return true; @@ -4479,7 +4477,7 @@ bool X86AsmParser::MatchAndEmitATTInstruction(SMLoc IDLoc, unsigned &Opcode, // If all of the instructions reported an invalid mnemonic, then the original // mnemonic was invalid. - if (std::count(std::begin(Match), std::end(Match), Match_MnemonicFail) == 4) { + if (llvm::count(Match, Match_MnemonicFail) == 4) { if (OriginalError == Match_MnemonicFail) return Error(IDLoc, "invalid instruction mnemonic '" + Base + "'", Op.getLocRange(), MatchingInlineAsm); @@ -4508,16 +4506,14 @@ bool X86AsmParser::MatchAndEmitATTInstruction(SMLoc IDLoc, unsigned &Opcode, } // If one instruction matched as unsupported, report this as unsupported. - if (std::count(std::begin(Match), std::end(Match), - Match_Unsupported) == 1) { + if (llvm::count(Match, Match_Unsupported) == 1) { return Error(IDLoc, "unsupported instruction", EmptyRange, MatchingInlineAsm); } // If one instruction matched with a missing feature, report this as a // missing feature. - if (std::count(std::begin(Match), std::end(Match), - Match_MissingFeature) == 1) { + if (llvm::count(Match, Match_MissingFeature) == 1) { ErrorInfo = Match_MissingFeature; return ErrorMissingFeature(IDLoc, ErrorInfoMissingFeatures, MatchingInlineAsm); @@ -4525,8 +4521,7 @@ bool X86AsmParser::MatchAndEmitATTInstruction(SMLoc IDLoc, unsigned &Opcode, // If one instruction matched with an invalid operand, report this as an // operand failure. - if (std::count(std::begin(Match), std::end(Match), - Match_InvalidOperand) == 1) { + if (llvm::count(Match, Match_InvalidOperand) == 1) { return Error(IDLoc, "invalid operand for instruction", EmptyRange, MatchingInlineAsm); } @@ -4674,8 +4669,7 @@ bool X86AsmParser::MatchAndEmitIntelInstruction(SMLoc IDLoc, unsigned &Opcode, Op.getLocRange(), MatchingInlineAsm); } - unsigned NumSuccessfulMatches = - std::count(std::begin(Match), std::end(Match), Match_Success); + unsigned NumSuccessfulMatches = llvm::count(Match, Match_Success); // If matching was ambiguous and we had size information from the frontend, // try again with that. This handles cases like "movxz eax, m8/m16". @@ -4721,16 +4715,14 @@ bool X86AsmParser::MatchAndEmitIntelInstruction(SMLoc IDLoc, unsigned &Opcode, } // If one instruction matched as unsupported, report this as unsupported. - if (std::count(std::begin(Match), std::end(Match), - Match_Unsupported) == 1) { + if (llvm::count(Match, Match_Unsupported) == 1) { return Error(IDLoc, "unsupported instruction", EmptyRange, MatchingInlineAsm); } // If one instruction matched with a missing feature, report this as a // missing feature. - if (std::count(std::begin(Match), std::end(Match), - Match_MissingFeature) == 1) { + if (llvm::count(Match, Match_MissingFeature) == 1) { ErrorInfo = Match_MissingFeature; return ErrorMissingFeature(IDLoc, ErrorInfoMissingFeatures, MatchingInlineAsm); @@ -4738,14 +4730,12 @@ bool X86AsmParser::MatchAndEmitIntelInstruction(SMLoc IDLoc, unsigned &Opcode, // If one instruction matched with an invalid operand, report this as an // operand failure. - if (std::count(std::begin(Match), std::end(Match), - Match_InvalidOperand) == 1) { + if (llvm::count(Match, Match_InvalidOperand) == 1) { return Error(IDLoc, "invalid operand for instruction", EmptyRange, MatchingInlineAsm); } - if (std::count(std::begin(Match), std::end(Match), - Match_InvalidImmUnsignedi4) == 1) { + if (llvm::count(Match, Match_InvalidImmUnsignedi4) == 1) { SMLoc ErrorLoc = ((X86Operand &)*Operands[ErrorInfo]).getStartLoc(); if (ErrorLoc == SMLoc()) ErrorLoc = IDLoc; diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h index 9164c699b569..67b1244708a8 100644 --- a/llvm/lib/Target/X86/AsmParser/X86Operand.h +++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h @@ -285,6 +285,12 @@ struct X86Operand final : public MCParsedAsmOperand { bool isOffsetOfLocal() const override { return isImm() && Imm.LocalRef; } + bool isMemPlaceholder(const MCInstrDesc &Desc) const override { + // Only MS InlineAsm uses global variables with registers rather than + // rip/eip. + return isMem() && !Mem.DefaultBaseReg && Mem.FrontendSize; + } + bool needAddressOf() const override { return AddressOf; } bool isMem() const override { return Kind == Memory; } diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp index b51011e2c52f..a903c5f455a2 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp @@ -948,39 +948,39 @@ bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, break; CASE_UNPCK(PUNPCKHBW, r) - case X86::MMX_PUNPCKHBWirr: + case X86::MMX_PUNPCKHBWrr: Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg()); RegForm = true; LLVM_FALLTHROUGH; CASE_UNPCK(PUNPCKHBW, m) - case X86::MMX_PUNPCKHBWirm: + case X86::MMX_PUNPCKHBWrm: Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); DecodeUNPCKHMask(getRegOperandNumElts(MI, 8, 0), 8, ShuffleMask); break; CASE_UNPCK(PUNPCKHWD, r) - case X86::MMX_PUNPCKHWDirr: + case X86::MMX_PUNPCKHWDrr: Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg()); RegForm = true; LLVM_FALLTHROUGH; CASE_UNPCK(PUNPCKHWD, m) - case X86::MMX_PUNPCKHWDirm: + case X86::MMX_PUNPCKHWDrm: Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); DecodeUNPCKHMask(getRegOperandNumElts(MI, 16, 0), 16, ShuffleMask); break; CASE_UNPCK(PUNPCKHDQ, r) - case X86::MMX_PUNPCKHDQirr: + case X86::MMX_PUNPCKHDQrr: Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg()); RegForm = true; LLVM_FALLTHROUGH; CASE_UNPCK(PUNPCKHDQ, m) - case X86::MMX_PUNPCKHDQirm: + case X86::MMX_PUNPCKHDQrm: Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); DecodeUNPCKHMask(getRegOperandNumElts(MI, 32, 0), 32, ShuffleMask); @@ -998,39 +998,39 @@ bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, break; CASE_UNPCK(PUNPCKLBW, r) - case X86::MMX_PUNPCKLBWirr: + case X86::MMX_PUNPCKLBWrr: Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg()); RegForm = true; LLVM_FALLTHROUGH; CASE_UNPCK(PUNPCKLBW, m) - case X86::MMX_PUNPCKLBWirm: + case X86::MMX_PUNPCKLBWrm: Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); DecodeUNPCKLMask(getRegOperandNumElts(MI, 8, 0), 8, ShuffleMask); break; CASE_UNPCK(PUNPCKLWD, r) - case X86::MMX_PUNPCKLWDirr: + case X86::MMX_PUNPCKLWDrr: Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg()); RegForm = true; LLVM_FALLTHROUGH; CASE_UNPCK(PUNPCKLWD, m) - case X86::MMX_PUNPCKLWDirm: + case X86::MMX_PUNPCKLWDrm: Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); DecodeUNPCKLMask(getRegOperandNumElts(MI, 16, 0), 16, ShuffleMask); break; CASE_UNPCK(PUNPCKLDQ, r) - case X86::MMX_PUNPCKLDQirr: + case X86::MMX_PUNPCKLDQrr: Src2Name = getRegName(MI->getOperand(NumOperands - 1).getReg()); RegForm = true; LLVM_FALLTHROUGH; CASE_UNPCK(PUNPCKLDQ, m) - case X86::MMX_PUNPCKLDQirm: + case X86::MMX_PUNPCKLDQrm: Src1Name = getRegName(MI->getOperand(NumOperands-(RegForm?2:6)).getReg()); DestName = getRegName(MI->getOperand(0).getReg()); DecodeUNPCKLMask(getRegOperandNumElts(MI, 32, 0), 32, ShuffleMask); diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp index 11251fb2b2ba..bf3f4e990ecc 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp @@ -236,7 +236,7 @@ bool X86WinCOFFTargetStreamer::emitFPOStackAlloc(unsigned StackAlloc, SMLoc L) { bool X86WinCOFFTargetStreamer::emitFPOStackAlign(unsigned Align, SMLoc L) { if (checkInFPOPrologue(L)) return true; - if (!llvm::any_of(CurFPOData->Instructions, [](const FPOInstruction &Inst) { + if (llvm::none_of(CurFPOData->Instructions, [](const FPOInstruction &Inst) { return Inst.Op == FPOInstruction::SetFrame; })) { getContext().reportError( diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp index 2e08482e4ff6..d48b8e458219 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.cpp +++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp @@ -754,8 +754,6 @@ static void emitNonLazyStubs(MachineModuleInfo *MMI, MCStreamer &OutStreamer) { void X86AsmPrinter::emitEndOfAsmFile(Module &M) { const Triple &TT = TM.getTargetTriple(); - emitAsanMemaccessSymbols(M); - if (TT.isOSBinFormatMachO()) { // Mach-O uses non-lazy symbol stubs to encode per-TU information into // global table for symbol lookup. diff --git a/llvm/lib/Target/X86/X86AsmPrinter.h b/llvm/lib/Target/X86/X86AsmPrinter.h index 3b0983a7d935..b22f25af26cf 100644 --- a/llvm/lib/Target/X86/X86AsmPrinter.h +++ b/llvm/lib/Target/X86/X86AsmPrinter.h @@ -31,6 +31,7 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter { FaultMaps FM; std::unique_ptr<MCCodeEmitter> CodeEmitter; bool EmitFPOData = false; + bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false; // This utility class tracks the length of a stackmap instruction's 'shadow'. // It is used by the X86AsmPrinter to ensure that the stackmap shadow @@ -100,20 +101,6 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter { // Address sanitizer specific lowering for X86. void LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI); - void emitAsanMemaccessSymbols(Module &M); - void emitAsanMemaccessPartial(Module &M, unsigned Reg, - const ASanAccessInfo &AccessInfo, - MCSubtargetInfo &STI); - void emitAsanMemaccessFull(Module &M, unsigned Reg, - const ASanAccessInfo &AccessInfo, - MCSubtargetInfo &STI); - void emitAsanReportError(Module &M, unsigned Reg, - const ASanAccessInfo &AccessInfo, - MCSubtargetInfo &STI); - - typedef std::tuple<unsigned /*Reg*/, uint32_t /*AccessInfo*/> - AsanMemaccessTuple; - std::map<AsanMemaccessTuple, MCSymbol *> AsanMemaccessSymbols; // Choose between emitting .seh_ directives and .cv_fpo_ directives. void EmitSEHInstruction(const MachineInstr *MI); @@ -165,6 +152,10 @@ public: bool runOnMachineFunction(MachineFunction &MF) override; void emitFunctionBodyStart() override; void emitFunctionBodyEnd() override; + + bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override { + return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags; + } }; } // end namespace llvm diff --git a/llvm/lib/Target/X86/X86CmovConversion.cpp b/llvm/lib/Target/X86/X86CmovConversion.cpp index 863438793acf..96d3d1390a59 100644 --- a/llvm/lib/Target/X86/X86CmovConversion.cpp +++ b/llvm/lib/Target/X86/X86CmovConversion.cpp @@ -186,7 +186,7 @@ bool X86CmovConverterPass::runOnMachineFunction(MachineFunction &MF) { if (collectCmovCandidates(Blocks, AllCmovGroups, /*IncludeLoads*/ true)) { for (auto &Group : AllCmovGroups) { // Skip any group that doesn't do at least one memory operand cmov. - if (!llvm::any_of(Group, [&](MachineInstr *I) { return I->mayLoad(); })) + if (llvm::none_of(Group, [&](MachineInstr *I) { return I->mayLoad(); })) continue; // For CMOV groups which we can rewrite and which contain a memory load, diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp index 93bc23006dc4..6a047838f0b5 100644 --- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -191,8 +191,6 @@ void X86ExpandPseudo::expandCALL_RVMARKER(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) { // Expand CALL_RVMARKER pseudo to call instruction, followed by the special //"movq %rax, %rdi" marker. - // TODO: Mark the sequence as bundle, to avoid passes moving other code - // in between. MachineInstr &MI = *MBBI; MachineInstr *OriginalCall; @@ -236,15 +234,23 @@ void X86ExpandPseudo::expandCALL_RVMARKER(MachineBasicBlock &MBB, // Emit call to ObjC runtime. const uint32_t *RegMask = TRI->getCallPreservedMask(*MBB.getParent(), CallingConv::C); - BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::CALL64pcrel32)) - .addGlobalAddress(MI.getOperand(0).getGlobal(), 0, 0) - .addRegMask(RegMask) - .addReg(X86::RAX, - RegState::Implicit | - (RAXImplicitDead ? (RegState::Dead | RegState::Define) - : RegState::Define)) - .getInstr(); + MachineInstr *RtCall = + BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(X86::CALL64pcrel32)) + .addGlobalAddress(MI.getOperand(0).getGlobal(), 0, 0) + .addRegMask(RegMask) + .addReg(X86::RAX, + RegState::Implicit | + (RAXImplicitDead ? (RegState::Dead | RegState::Define) + : RegState::Define)) + .getInstr(); MI.eraseFromParent(); + + auto &TM = MBB.getParent()->getTarget(); + // On Darwin platforms, wrap the expanded sequence in a bundle to prevent + // later optimizations from breaking up the sequence. + if (TM.getTargetTriple().isOSDarwin()) + finalizeBundle(MBB, OriginalCall->getIterator(), + std::next(RtCall->getIterator())); } /// If \p MBBI is a pseudo instruction, this method expands diff --git a/llvm/lib/Target/X86/X86FastTileConfig.cpp b/llvm/lib/Target/X86/X86FastTileConfig.cpp index 87c04a07cd13..47874e82ff3b 100644 --- a/llvm/lib/Target/X86/X86FastTileConfig.cpp +++ b/llvm/lib/Target/X86/X86FastTileConfig.cpp @@ -134,11 +134,7 @@ bool X86FastTileConfig::isAMXInstr(MachineInstr &MI) { if (MI.getOpcode() == X86::PLDTILECFGV || MI.isDebugInstr()) return false; - for (MachineOperand &MO : MI.operands()) - if (isTilePhysReg(MO)) - return true; - - return false; + return llvm::any_of(MI.operands(), isTilePhysReg); } MachineInstr *X86FastTileConfig::getKeyAMXInstr(MachineInstr *MI) { diff --git a/llvm/lib/Target/X86/X86FixupBWInsts.cpp b/llvm/lib/Target/X86/X86FixupBWInsts.cpp index e1d4b4c34772..16bff201dd03 100644 --- a/llvm/lib/Target/X86/X86FixupBWInsts.cpp +++ b/llvm/lib/Target/X86/X86FixupBWInsts.cpp @@ -457,14 +457,12 @@ void FixupBWInstPass::processBasicBlock(MachineFunction &MF, OptForSize = MF.getFunction().hasOptSize() || llvm::shouldOptimizeForSize(&MBB, PSI, MBFI); - for (auto I = MBB.rbegin(); I != MBB.rend(); ++I) { - MachineInstr *MI = &*I; - - if (MachineInstr *NewMI = tryReplaceInstr(MI, MBB)) - MIReplacements.push_back(std::make_pair(MI, NewMI)); + for (MachineInstr &MI : llvm::reverse(MBB)) { + if (MachineInstr *NewMI = tryReplaceInstr(&MI, MBB)) + MIReplacements.push_back(std::make_pair(&MI, NewMI)); // We're done with this instruction, update liveness for the next one. - LiveRegs.stepBackward(*MI); + LiveRegs.stepBackward(MI); } while (!MIReplacements.empty()) { diff --git a/llvm/lib/Target/X86/X86FloatingPoint.cpp b/llvm/lib/Target/X86/X86FloatingPoint.cpp index 4d9160f35226..2f0ab4ca9de4 100644 --- a/llvm/lib/Target/X86/X86FloatingPoint.cpp +++ b/llvm/lib/Target/X86/X86FloatingPoint.cpp @@ -1442,7 +1442,7 @@ void FPS::handleTwoArgFP(MachineBasicBlock::iterator &I) { assert(UpdatedSlot < StackTop && Dest < 7); Stack[UpdatedSlot] = Dest; RegMap[Dest] = UpdatedSlot; - MBB->getParent()->DeleteMachineInstr(&MI); // Remove the old instruction + MBB->getParent()->deleteMachineInstr(&MI); // Remove the old instruction } /// handleCompareFP - Handle FUCOM and FUCOMI instructions, which have two FP diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp index c29ae9f6af4c..0a7aea467809 100644 --- a/llvm/lib/Target/X86/X86FrameLowering.cpp +++ b/llvm/lib/Target/X86/X86FrameLowering.cpp @@ -2496,8 +2496,8 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots( } // Assign slots for GPRs. It increases frame size. - for (unsigned i = CSI.size(); i != 0; --i) { - unsigned Reg = CSI[i - 1].getReg(); + for (CalleeSavedInfo &I : llvm::reverse(CSI)) { + unsigned Reg = I.getReg(); if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg)) continue; @@ -2506,15 +2506,15 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots( CalleeSavedFrameSize += SlotSize; int SlotIndex = MFI.CreateFixedSpillStackObject(SlotSize, SpillSlotOffset); - CSI[i - 1].setFrameIdx(SlotIndex); + I.setFrameIdx(SlotIndex); } X86FI->setCalleeSavedFrameSize(CalleeSavedFrameSize); MFI.setCVBytesOfCalleeSavedRegisters(CalleeSavedFrameSize); // Assign slots for XMMs. - for (unsigned i = CSI.size(); i != 0; --i) { - unsigned Reg = CSI[i - 1].getReg(); + for (CalleeSavedInfo &I : llvm::reverse(CSI)) { + unsigned Reg = I.getReg(); if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg)) continue; @@ -2533,7 +2533,7 @@ bool X86FrameLowering::assignCalleeSavedSpillSlots( // spill into slot SpillSlotOffset -= Size; int SlotIndex = MFI.CreateFixedSpillStackObject(Size, SpillSlotOffset); - CSI[i - 1].setFrameIdx(SlotIndex); + I.setFrameIdx(SlotIndex); MFI.ensureMaxAlignment(Alignment); // Save the start offset and size of XMM in stack frame for funclets. @@ -2559,8 +2559,8 @@ bool X86FrameLowering::spillCalleeSavedRegisters( // Push GPRs. It increases frame size. const MachineFunction &MF = *MBB.getParent(); unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r; - for (unsigned i = CSI.size(); i != 0; --i) { - unsigned Reg = CSI[i - 1].getReg(); + for (const CalleeSavedInfo &I : llvm::reverse(CSI)) { + unsigned Reg = I.getReg(); if (!X86::GR64RegClass.contains(Reg) && !X86::GR32RegClass.contains(Reg)) continue; @@ -2593,8 +2593,8 @@ bool X86FrameLowering::spillCalleeSavedRegisters( // Make XMM regs spilled. X86 does not have ability of push/pop XMM. // It can be done by spilling XMMs to stack frame. - for (unsigned i = CSI.size(); i != 0; --i) { - unsigned Reg = CSI[i-1].getReg(); + for (const CalleeSavedInfo &I : llvm::reverse(CSI)) { + unsigned Reg = I.getReg(); if (X86::GR64RegClass.contains(Reg) || X86::GR32RegClass.contains(Reg)) continue; @@ -2607,8 +2607,7 @@ bool X86FrameLowering::spillCalleeSavedRegisters( MBB.addLiveIn(Reg); const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT); - TII.storeRegToStackSlot(MBB, MI, Reg, true, CSI[i - 1].getFrameIdx(), RC, - TRI); + TII.storeRegToStackSlot(MBB, MI, Reg, true, I.getFrameIdx(), RC, TRI); --MI; MI->setFlag(MachineInstr::FrameSetup); ++MI; diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 62b2387396be..6f6361b6757b 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -1091,17 +1091,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::SRL, VT, Custom); setOperationAction(ISD::SHL, VT, Custom); setOperationAction(ISD::SRA, VT, Custom); + if (VT == MVT::v2i64) continue; + setOperationAction(ISD::ROTL, VT, Custom); + setOperationAction(ISD::ROTR, VT, Custom); } - setOperationAction(ISD::ROTL, MVT::v4i32, Custom); - setOperationAction(ISD::ROTL, MVT::v8i16, Custom); - - // With 512-bit registers or AVX512VL+BW, expanding (and promoting the - // shifts) is better. - if (!Subtarget.useAVX512Regs() && - !(Subtarget.hasBWI() && Subtarget.hasVLX())) - setOperationAction(ISD::ROTL, MVT::v16i8, Custom); - setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal); @@ -1199,8 +1193,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) { for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, - MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) + MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { setOperationAction(ISD::ROTL, VT, Custom); + setOperationAction(ISD::ROTR, VT, Custom); + } // XOP can efficiently perform BITREVERSE with VPPERM. for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) @@ -1283,6 +1279,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::SRL, VT, Custom); setOperationAction(ISD::SHL, VT, Custom); setOperationAction(ISD::SRA, VT, Custom); + if (VT == MVT::v4i64) continue; + setOperationAction(ISD::ROTL, VT, Custom); + setOperationAction(ISD::ROTR, VT, Custom); } // These types need custom splitting if their input is a 128-bit vector. @@ -1291,13 +1290,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom); setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); - setOperationAction(ISD::ROTL, MVT::v8i32, Custom); - setOperationAction(ISD::ROTL, MVT::v16i16, Custom); - - // With BWI, expanding (and promoting the shifts) is the better. - if (!Subtarget.useBWIRegs()) - setOperationAction(ISD::ROTL, MVT::v32i8, Custom); - setOperationAction(ISD::SELECT, MVT::v4f64, Custom); setOperationAction(ISD::SELECT, MVT::v4i64, Custom); setOperationAction(ISD::SELECT, MVT::v8i32, Custom); @@ -1662,6 +1654,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::SRL, VT, Custom); setOperationAction(ISD::SHL, VT, Custom); setOperationAction(ISD::SRA, VT, Custom); + setOperationAction(ISD::ROTL, VT, Custom); + setOperationAction(ISD::ROTR, VT, Custom); setOperationAction(ISD::SETCC, VT, Custom); // The condition codes aren't legal in SSE/AVX and under AVX512 we use @@ -1676,16 +1670,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::UMIN, VT, Legal); setOperationAction(ISD::ABS, VT, Legal); setOperationAction(ISD::CTPOP, VT, Custom); - setOperationAction(ISD::ROTL, VT, Custom); - setOperationAction(ISD::ROTR, VT, Custom); setOperationAction(ISD::STRICT_FSETCC, VT, Custom); setOperationAction(ISD::STRICT_FSETCCS, VT, Custom); } - // With BWI, expanding (and promoting the shifts) is the better. - if (!Subtarget.useBWIRegs()) - setOperationAction(ISD::ROTL, MVT::v32i16, Custom); - for (auto VT : { MVT::v64i8, MVT::v32i16 }) { setOperationAction(ISD::ABS, VT, HasBWI ? Legal : Custom); setOperationAction(ISD::CTPOP, VT, Subtarget.hasBITALG() ? Legal : Custom); @@ -5926,8 +5914,7 @@ static bool isSequentialOrUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos, /// from position Pos and ending in Pos+Size is undef or is zero. static bool isUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos, unsigned Size) { - return llvm::all_of(Mask.slice(Pos, Size), - [](int M) { return isUndefOrZero(M); }); + return llvm::all_of(Mask.slice(Pos, Size), isUndefOrZero); } /// Helper function to test whether a shuffle mask could be @@ -6788,12 +6775,33 @@ void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, } } +// Attempt to constant fold, else just create a VECTOR_SHUFFLE. +static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl, + SDValue V1, SDValue V2, ArrayRef<int> Mask) { + if ((ISD::isBuildVectorOfConstantSDNodes(V1.getNode()) || V1.isUndef()) && + (ISD::isBuildVectorOfConstantSDNodes(V2.getNode()) || V2.isUndef())) { + SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType())); + for (int I = 0, NumElts = Mask.size(); I != NumElts; ++I) { + int M = Mask[I]; + if (M < 0) + continue; + SDValue V = (M < NumElts) ? V1 : V2; + if (V.isUndef()) + continue; + Ops[I] = V.getOperand(M % NumElts); + } + return DAG.getBuildVector(VT, dl, Ops); + } + + return DAG.getVectorShuffle(VT, dl, V1, V2, Mask); +} + /// Returns a vector_shuffle node for an unpackl operation. static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2) { SmallVector<int, 8> Mask; createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false); - return DAG.getVectorShuffle(VT, dl, V1, V2, Mask); + return getVectorShuffle(DAG, VT, dl, V1, V2, Mask); } /// Returns a vector_shuffle node for an unpackh operation. @@ -6801,12 +6809,11 @@ static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2) { SmallVector<int, 8> Mask; createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false); - return DAG.getVectorShuffle(VT, dl, V1, V2, Mask); + return getVectorShuffle(DAG, VT, dl, V1, V2, Mask); } /// Returns a node that packs the LHS + RHS nodes together at half width. /// May return X86ISD::PACKSS/PACKUS, packing the top/bottom half. -/// TODO: Add vXi64 -> vXi32 pack support with vector_shuffle node. /// TODO: Add subvector splitting if/when we have a need for it. static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS, @@ -6818,9 +6825,24 @@ static SDValue getPack(SelectionDAG &DAG, const X86Subtarget &Subtarget, VT.getSizeInBits() == OpVT.getSizeInBits() && (EltSizeInBits * 2) == OpVT.getScalarSizeInBits() && "Unexpected PACK operand types"); - assert((EltSizeInBits == 8 || EltSizeInBits == 16) && + assert((EltSizeInBits == 8 || EltSizeInBits == 16 || EltSizeInBits == 32) && "Unexpected PACK result type"); + // Rely on vector shuffles for vXi64 -> vXi32 packing. + if (EltSizeInBits == 32) { + SmallVector<int> PackMask; + int Offset = PackHiHalf ? 1 : 0; + int NumElts = VT.getVectorNumElements(); + for (int I = 0; I != NumElts; I += 4) { + PackMask.push_back(I + Offset); + PackMask.push_back(I + Offset + 2); + PackMask.push_back(I + Offset + NumElts); + PackMask.push_back(I + Offset + NumElts + 2); + } + return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, LHS), + DAG.getBitcast(VT, RHS), PackMask); + } + // See if we already have sufficient leading bits for PACKSS/PACKUS. if (!PackHiHalf) { if (UsePackUS && @@ -15192,12 +15214,10 @@ static SDValue lowerV8I16GeneralSingleInputShuffle( // need // to balance this to ensure we don't form a 3-1 shuffle in the other // half. - int NumFlippedAToBInputs = - std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord) + - std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord + 1); - int NumFlippedBToBInputs = - std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord) + - std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord + 1); + int NumFlippedAToBInputs = llvm::count(AToBInputs, 2 * ADWord) + + llvm::count(AToBInputs, 2 * ADWord + 1); + int NumFlippedBToBInputs = llvm::count(BToBInputs, 2 * BDWord) + + llvm::count(BToBInputs, 2 * BDWord + 1); if ((NumFlippedAToBInputs == 1 && (NumFlippedBToBInputs == 0 || NumFlippedBToBInputs == 2)) || (NumFlippedBToBInputs == 1 && @@ -25599,6 +25619,7 @@ static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT, /// Handle vector element shifts where the shift amount may or may not be a /// constant. Takes immediate version of shift as input. +/// TODO: Replace with vector + (splat) idx to avoid extract_element nodes. static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, const X86Subtarget &Subtarget, @@ -25606,11 +25627,6 @@ static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT, MVT SVT = ShAmt.getSimpleValueType(); assert((SVT == MVT::i32 || SVT == MVT::i64) && "Unexpected value type!"); - // Catch shift-by-constant. - if (ConstantSDNode *CShAmt = dyn_cast<ConstantSDNode>(ShAmt)) - return getTargetVShiftByConstNode(Opc, dl, VT, SrcOp, - CShAmt->getZExtValue(), DAG); - // Change opcode to non-immediate version. Opc = getTargetVShiftUniformOpcode(Opc, true); @@ -26342,10 +26358,19 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, DAG.getBitcast(MVT::i16, Ins)); } - case VSHIFT: + case VSHIFT: { + SDValue SrcOp = Op.getOperand(1); + SDValue ShAmt = Op.getOperand(2); + + // Catch shift-by-constant. + if (auto *CShAmt = dyn_cast<ConstantSDNode>(ShAmt)) + return getTargetVShiftByConstNode(IntrData->Opc0, dl, + Op.getSimpleValueType(), SrcOp, + CShAmt->getZExtValue(), DAG); + return getTargetVShiftNode(IntrData->Opc0, dl, Op.getSimpleValueType(), - Op.getOperand(1), Op.getOperand(2), Subtarget, - DAG); + SrcOp, ShAmt, Subtarget, DAG); + } case COMPRESS_EXPAND_IN_REG: { SDValue Mask = Op.getOperand(3); SDValue DataToCompress = Op.getOperand(1); @@ -26638,7 +26663,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, X86CC = X86::COND_E; break; } - SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end()); + SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops())); SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32); SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2); SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG); @@ -26653,7 +26678,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, else Opcode = X86ISD::PCMPESTR; - SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end()); + SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops())); SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32); return DAG.getNode(Opcode, dl, VTs, NewOps); } @@ -26666,7 +26691,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, else Opcode = X86ISD::PCMPESTR; - SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end()); + SmallVector<SDValue, 5> NewOps(llvm::drop_begin(Op->ops())); SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32); return DAG.getNode(Opcode, dl, VTs, NewOps).getValue(1); } @@ -28892,10 +28917,13 @@ SDValue X86TargetLowering::LowerWin64_INT128_TO_FP(SDValue Op, // supported by the Subtarget static bool supportedVectorShiftWithImm(MVT VT, const X86Subtarget &Subtarget, unsigned Opcode) { + if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) + return false; + if (VT.getScalarSizeInBits() < 16) return false; - if (VT.is512BitVector() && Subtarget.hasAVX512() && + if (VT.is512BitVector() && Subtarget.useAVX512Regs() && (VT.getScalarSizeInBits() > 16 || Subtarget.hasBWI())) return true; @@ -28919,6 +28947,8 @@ bool supportedVectorShiftWithBaseAmnt(MVT VT, const X86Subtarget &Subtarget, // natively supported by the Subtarget static bool supportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget, unsigned Opcode) { + if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) + return false; if (!Subtarget.hasInt256() || VT.getScalarSizeInBits() < 16) return false; @@ -28927,7 +28957,8 @@ static bool supportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget, if (VT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI()) return false; - if (Subtarget.hasAVX512()) + if (Subtarget.hasAVX512() && + (Subtarget.useAVX512Regs() || !VT.is512BitVector())) return true; bool LShift = VT.is128BitVector() || VT.is256BitVector(); @@ -28935,8 +28966,8 @@ static bool supportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget, return (Opcode == ISD::SRA) ? AShift : LShift; } -static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG, - const X86Subtarget &Subtarget) { +static SDValue LowerShiftByScalarImmediate(SDValue Op, SelectionDAG &DAG, + const X86Subtarget &Subtarget) { MVT VT = Op.getSimpleValueType(); SDLoc dl(Op); SDValue R = Op.getOperand(0); @@ -29066,8 +29097,8 @@ static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG, return SDValue(); } -static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG, - const X86Subtarget &Subtarget) { +static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG, + const X86Subtarget &Subtarget) { MVT VT = Op.getSimpleValueType(); SDLoc dl(Op); SDValue R = Op.getOperand(0); @@ -29166,28 +29197,20 @@ static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl, (Subtarget.hasBWI() && VT == MVT::v64i8))) return SDValue(); - if (ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) { - SmallVector<SDValue, 8> Elts; - MVT SVT = VT.getVectorElementType(); - unsigned SVTBits = SVT.getSizeInBits(); - APInt One(SVTBits, 1); - unsigned NumElems = VT.getVectorNumElements(); - - for (unsigned i = 0; i != NumElems; ++i) { - SDValue Op = Amt->getOperand(i); - if (Op->isUndef()) { - Elts.push_back(Op); - continue; - } + MVT SVT = VT.getVectorElementType(); + unsigned SVTBits = SVT.getSizeInBits(); + unsigned NumElems = VT.getVectorNumElements(); - ConstantSDNode *ND = cast<ConstantSDNode>(Op); - APInt C(SVTBits, ND->getZExtValue()); - uint64_t ShAmt = C.getZExtValue(); - if (ShAmt >= SVTBits) { - Elts.push_back(DAG.getUNDEF(SVT)); + APInt UndefElts; + SmallVector<APInt> EltBits; + if (getTargetConstantBitsFromNode(Amt, SVTBits, UndefElts, EltBits)) { + APInt One(SVTBits, 1); + SmallVector<SDValue> Elts(NumElems, DAG.getUNDEF(SVT)); + for (unsigned I = 0; I != NumElems; ++I) { + if (UndefElts[I] || EltBits[I].uge(SVTBits)) continue; - } - Elts.push_back(DAG.getConstant(One.shl(ShAmt), dl, SVT)); + uint64_t ShAmt = EltBits[I].getZExtValue(); + Elts[I] = DAG.getConstant(One.shl(ShAmt), dl, SVT); } return DAG.getBuildVector(VT, dl, Elts); } @@ -29233,10 +29256,10 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget, assert(VT.isVector() && "Custom lowering only for vector shifts!"); assert(Subtarget.hasSSE2() && "Only custom lower when we have SSE2!"); - if (SDValue V = LowerScalarImmediateShift(Op, DAG, Subtarget)) + if (SDValue V = LowerShiftByScalarImmediate(Op, DAG, Subtarget)) return V; - if (SDValue V = LowerScalarVariableShift(Op, DAG, Subtarget)) + if (SDValue V = LowerShiftByScalarVariable(Op, DAG, Subtarget)) return V; if (supportedVectorVarShift(VT, Subtarget, Opc)) @@ -29818,14 +29841,29 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget, return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt); } - assert(IsROTL && "Only ROTL supported"); + SDValue Z = DAG.getConstant(0, DL, VT); + + if (!IsROTL) { + // If the ISD::ROTR amount is constant, we're always better converting to + // ISD::ROTL. + if (SDValue NegAmt = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {Z, Amt})) + return DAG.getNode(ISD::ROTL, DL, VT, R, NegAmt); + + // XOP targets always prefers ISD::ROTL. + if (Subtarget.hasXOP()) + return DAG.getNode(ISD::ROTL, DL, VT, R, + DAG.getNode(ISD::SUB, DL, VT, Z, Amt)); + } + + // Split 256-bit integers on XOP/pre-AVX2 targets. + if (VT.is256BitVector() && (Subtarget.hasXOP() || !Subtarget.hasAVX2())) + return splitVectorIntBinary(Op, DAG); // XOP has 128-bit vector variable + immediate rotates. // +ve/-ve Amt = rotate left/right - just need to handle ISD::ROTL. // XOP implicitly uses modulo rotation amounts. if (Subtarget.hasXOP()) { - if (VT.is256BitVector()) - return splitVectorIntBinary(Op, DAG); + assert(IsROTL && "Only ROTL expected"); assert(VT.is128BitVector() && "Only rotate 128-bit vectors!"); // Attempt to rotate by immediate. @@ -29839,55 +29877,89 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget, return Op; } - // Split 256-bit integers on pre-AVX2 targets. - if (VT.is256BitVector() && !Subtarget.hasAVX2()) - return splitVectorIntBinary(Op, DAG); - - assert((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 || - ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8 || - VT == MVT::v32i16) && - Subtarget.hasAVX2())) && - "Only vXi32/vXi16/vXi8 vector rotates supported"); - // Rotate by an uniform constant - expand back to shifts. if (IsCstSplat) return SDValue(); - bool IsSplatAmt = DAG.isSplatValue(Amt); - SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT); + // Split 512-bit integers on non 512-bit BWI targets. + if (VT.is512BitVector() && !Subtarget.useBWIRegs()) + return splitVectorIntBinary(Op, DAG); - // v16i8/v32i8: Split rotation into rot4/rot2/rot1 stages and select by - // the amount bit. - if (EltSizeInBits == 8) { - if (ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) - return SDValue(); + assert( + (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 || + ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) && + Subtarget.hasAVX2()) || + ((VT == MVT::v32i16 || VT == MVT::v64i8) && Subtarget.useBWIRegs())) && + "Only vXi32/vXi16/vXi8 vector rotates supported"); - // Check for a hidden ISD::ROTR, vXi8 lowering can handle both, but we - // currently hit infinite loops in legalization if we allow ISD::ROTR. - // FIXME: Infinite ROTL<->ROTR legalization in TargetLowering::expandROT. - SDValue HiddenROTRAmt; - if (Amt.getOpcode() == ISD::SUB && - ISD::isBuildVectorAllZeros(Amt.getOperand(0).getNode())) - HiddenROTRAmt = Amt.getOperand(1); + MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits); + MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2); - MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2); + SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT); + SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); - // If the amount is a splat, attempt to fold as unpack(x,x) << zext(y): - // rotl(x,y) -> (((aext(x) << bw) | zext(x)) << (y & (bw-1))) >> bw. - // rotr(x,y) -> (((aext(x) << bw) | zext(x)) >> (y & (bw-1))). - if (SDValue BaseRotAmt = DAG.getSplatValue(DAG.getNode( - ISD::AND, DL, VT, HiddenROTRAmt ? HiddenROTRAmt : Amt, AmtMask))) { - unsigned ShiftX86Opc = HiddenROTRAmt ? X86ISD::VSRLI : X86ISD::VSHLI; - BaseRotAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, BaseRotAmt); + // Attempt to fold as unpack(x,x) << zext(splat(y)): + // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw. + // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))). + // TODO: Handle vXi16 cases. + if (EltSizeInBits == 8 || EltSizeInBits == 32) { + if (SDValue BaseRotAmt = DAG.getSplatValue(AmtMod)) { + unsigned ShiftX86Opc = IsROTL ? X86ISD::VSHLI : X86ISD::VSRLI; SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R)); SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R)); + BaseRotAmt = DAG.getZExtOrTrunc(BaseRotAmt, DL, MVT::i32); Lo = getTargetVShiftNode(ShiftX86Opc, DL, ExtVT, Lo, BaseRotAmt, Subtarget, DAG); Hi = getTargetVShiftNode(ShiftX86Opc, DL, ExtVT, Hi, BaseRotAmt, Subtarget, DAG); - return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !HiddenROTRAmt); + return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL); + } + } + + // v16i8/v32i8/v64i8: Split rotation into rot4/rot2/rot1 stages and select by + // the amount bit. + // TODO: We're doing nothing here that we couldn't do for funnel shifts. + if (EltSizeInBits == 8) { + bool IsConstAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()); + MVT WideVT = + MVT::getVectorVT(Subtarget.hasBWI() ? MVT::i16 : MVT::i32, NumElts); + unsigned ShiftOpc = IsROTL ? ISD::SHL : ISD::SRL; + + // Attempt to fold as: + // rotl(x,y) -> (((aext(x) << bw) | zext(x)) << (y & (bw-1))) >> bw. + // rotr(x,y) -> (((aext(x) << bw) | zext(x)) >> (y & (bw-1))). + if (supportedVectorVarShift(WideVT, Subtarget, ShiftOpc) && + supportedVectorShiftWithImm(WideVT, Subtarget, ShiftOpc)) { + // If we're rotating by constant, just use default promotion. + if (IsConstAmt) + return SDValue(); + // See if we can perform this by widening to vXi16 or vXi32. + R = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, R); + R = DAG.getNode( + ISD::OR, DL, WideVT, R, + getTargetVShiftByConstNode(X86ISD::VSHLI, DL, WideVT, R, 8, DAG)); + Amt = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, AmtMod); + R = DAG.getNode(ShiftOpc, DL, WideVT, R, Amt); + if (IsROTL) + R = getTargetVShiftByConstNode(X86ISD::VSRLI, DL, WideVT, R, 8, DAG); + return DAG.getNode(ISD::TRUNCATE, DL, VT, R); } + // Attempt to fold as unpack(x,x) << zext(y): + // rotl(x,y) -> (unpack(x,x) << (y & (bw-1))) >> bw. + // rotr(x,y) -> (unpack(x,x) >> (y & (bw-1))). + if (IsConstAmt || supportedVectorVarShift(ExtVT, Subtarget, ShiftOpc)) { + // See if we can perform this by unpacking to lo/hi vXi16. + SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R)); + SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R)); + SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z)); + SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z)); + SDValue Lo = DAG.getNode(ShiftOpc, DL, ExtVT, RLo, ALo); + SDValue Hi = DAG.getNode(ShiftOpc, DL, ExtVT, RHi, AHi); + return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL); + } + assert((VT == MVT::v16i8 || VT == MVT::v32i8) && "Unsupported vXi8 type"); + // We don't need ModuloAmt here as we just peek at individual bits. auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) { if (Subtarget.hasSSE41()) { @@ -29907,15 +29979,15 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget, return DAG.getSelect(DL, SelVT, C, V0, V1); }; - // 'Hidden' ROTR is currently only profitable on AVX512 targets where we - // have VPTERNLOG. - unsigned ShiftLHS = ISD::SHL; - unsigned ShiftRHS = ISD::SRL; - if (HiddenROTRAmt && useVPTERNLOG(Subtarget, VT)) { - std::swap(ShiftLHS, ShiftRHS); - Amt = HiddenROTRAmt; + // ISD::ROTR is currently only profitable on AVX512 targets with VPTERNLOG. + if (!IsROTL && !useVPTERNLOG(Subtarget, VT)) { + Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt); + IsROTL = true; } + unsigned ShiftLHS = IsROTL ? ISD::SHL : ISD::SRL; + unsigned ShiftRHS = IsROTL ? ISD::SRL : ISD::SHL; + // Turn 'a' into a mask suitable for VSELECT: a = a << 5; // We can safely do this using i16 shifts as we're only interested in // the 3 lower bits of each byte. @@ -29952,18 +30024,7 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget, return SignBitSelect(VT, Amt, M, R); } - // ISD::ROT* uses modulo rotate amounts. - if (SDValue BaseRotAmt = DAG.getSplatValue(Amt)) { - // If the amount is a splat, perform the modulo BEFORE the splat, - // this helps LowerScalarVariableShift to remove the splat later. - Amt = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, BaseRotAmt); - Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); - Amt = DAG.getVectorShuffle(VT, DL, Amt, DAG.getUNDEF(VT), - SmallVector<int>(NumElts, 0)); - } else { - Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); - } - + bool IsSplatAmt = DAG.isSplatValue(Amt); bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()); bool LegalVarShifts = supportedVectorVarShift(VT, Subtarget, ISD::SHL) && supportedVectorVarShift(VT, Subtarget, ISD::SRL); @@ -29971,13 +30032,25 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget, // Fallback for splats + all supported variable shifts. // Fallback for non-constants AVX2 vXi16 as well. if (IsSplatAmt || LegalVarShifts || (Subtarget.hasAVX2() && !ConstantAmt)) { + Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT); AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt); - SDValue SHL = DAG.getNode(ISD::SHL, DL, VT, R, Amt); - SDValue SRL = DAG.getNode(ISD::SRL, DL, VT, R, AmtR); + SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt); + SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR); return DAG.getNode(ISD::OR, DL, VT, SHL, SRL); } + // Everything below assumes ISD::ROTL. + if (!IsROTL) { + Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt); + IsROTL = true; + } + + // ISD::ROT* uses modulo rotate amounts. + Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); + + assert(IsROTL && "Only ROTL supported"); + // As with shifts, attempt to convert the rotation amount to a multiplication // factor, fallback to general expansion. SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG); @@ -32927,11 +33000,6 @@ bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL, bool X86TargetLowering::isVectorShiftByScalarCheap(Type *Ty) const { unsigned Bits = Ty->getScalarSizeInBits(); - // 8-bit shifts are always expensive, but versions with a scalar amount aren't - // particularly cheaper than those without. - if (Bits == 8) - return false; - // XOP has v16i8/v8i16/v4i32/v2i64 variable vector shifts. // Splitting for v32i8/v16i16 on XOP+AVX2 targets is still preferred. if (Subtarget.hasXOP() && @@ -36249,9 +36317,10 @@ static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask, (V1.getOpcode() == ISD::SCALAR_TO_VECTOR && isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1))) { Shuffle = X86ISD::VZEXT_MOVL; - SrcVT = DstVT = MaskEltSize == 16 ? MVT::v8f16 - : !Subtarget.hasSSE2() ? MVT::v4f32 - : MaskVT; + if (MaskEltSize == 16) + SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16); + else + SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; return true; } } @@ -36300,9 +36369,10 @@ static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask, isUndefOrEqual(Mask[0], 0) && isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1)) { Shuffle = X86ISD::VZEXT_MOVL; - SrcVT = DstVT = MaskEltSize == 16 ? MVT::v8f16 - : !Subtarget.hasSSE2() ? MVT::v4f32 - : MaskVT; + if (MaskEltSize == 16) + SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16); + else + SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; return true; } @@ -40981,6 +41051,28 @@ SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode( Op, DemandedBits, DemandedElts, DAG, Depth); } +bool X86TargetLowering::isSplatValueForTargetNode(SDValue Op, + const APInt &DemandedElts, + APInt &UndefElts, + unsigned Depth) const { + unsigned NumElts = DemandedElts.getBitWidth(); + unsigned Opc = Op.getOpcode(); + + switch (Opc) { + case X86ISD::VBROADCAST: + case X86ISD::VBROADCAST_LOAD: + // TODO: Permit vXi64 types on 32-bit targets. + if (isTypeLegal(Op.getValueType().getVectorElementType())) { + UndefElts = APInt::getNullValue(NumElts); + return true; + } + return false; + } + + return TargetLowering::isSplatValueForTargetNode(Op, DemandedElts, UndefElts, + Depth); +} + // Helper to peek through bitops/trunc/setcc to determine size of source vector. // Allows combineBitcastvxi1 to determine what size vector generated a <X x i1>. static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size, @@ -46204,25 +46296,27 @@ static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG, static SDValue combineAnd(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); EVT VT = N->getValueType(0); + SDLoc dl(N); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); // If this is SSE1 only convert to FAND to avoid scalarization. if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { - return DAG.getBitcast( - MVT::v4i32, DAG.getNode(X86ISD::FAND, SDLoc(N), MVT::v4f32, - DAG.getBitcast(MVT::v4f32, N->getOperand(0)), - DAG.getBitcast(MVT::v4f32, N->getOperand(1)))); + return DAG.getBitcast(MVT::v4i32, + DAG.getNode(X86ISD::FAND, dl, MVT::v4f32, + DAG.getBitcast(MVT::v4f32, N0), + DAG.getBitcast(MVT::v4f32, N1))); } // Use a 32-bit and+zext if upper bits known zero. - if (VT == MVT::i64 && Subtarget.is64Bit() && - !isa<ConstantSDNode>(N->getOperand(1))) { + if (VT == MVT::i64 && Subtarget.is64Bit() && !isa<ConstantSDNode>(N1)) { APInt HiMask = APInt::getHighBitsSet(64, 32); - if (DAG.MaskedValueIsZero(N->getOperand(1), HiMask) || - DAG.MaskedValueIsZero(N->getOperand(0), HiMask)) { - SDLoc dl(N); - SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(0)); - SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(1)); + if (DAG.MaskedValueIsZero(N1, HiMask) || + DAG.MaskedValueIsZero(N0, HiMask)) { + SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N0); + SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N1); return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS)); } @@ -46235,8 +46329,6 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG, SmallVector<APInt, 2> SrcPartials; if (matchScalarReduction(SDValue(N, 0), ISD::AND, SrcOps, &SrcPartials) && SrcOps.size() == 1) { - SDLoc dl(N); - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements(); EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts); SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget); @@ -46276,33 +46368,57 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG, if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget)) return R; - // Attempt to recursively combine a bitmask AND with shuffles. if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) { + // Attempt to recursively combine a bitmask AND with shuffles. SDValue Op(N, 0); if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) return Res; + + // If either operand is a constant mask, then only the elements that aren't + // zero are actually demanded by the other operand. + auto SimplifyUndemandedElts = [&](SDValue Op, SDValue OtherOp) { + APInt UndefElts; + SmallVector<APInt> EltBits; + int NumElts = VT.getVectorNumElements(); + int EltSizeInBits = VT.getScalarSizeInBits(); + if (!getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts, EltBits)) + return false; + + APInt DemandedElts = APInt::getZero(NumElts); + for (int I = 0; I != NumElts; ++I) + if (!EltBits[I].isZero()) + DemandedElts.setBit(I); + + APInt KnownUndef, KnownZero; + return TLI.SimplifyDemandedVectorElts(OtherOp, DemandedElts, KnownUndef, + KnownZero, DCI); + }; + if (SimplifyUndemandedElts(N0, N1) || SimplifyUndemandedElts(N1, N0)) { + if (N->getOpcode() != ISD::DELETED_NODE) + DCI.AddToWorklist(N); + return SDValue(N, 0); + } } // Attempt to combine a scalar bitmask AND with an extracted shuffle. if ((VT.getScalarSizeInBits() % 8) == 0 && - N->getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT && - isa<ConstantSDNode>(N->getOperand(0).getOperand(1))) { - SDValue BitMask = N->getOperand(1); - SDValue SrcVec = N->getOperand(0).getOperand(0); + N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && + isa<ConstantSDNode>(N0.getOperand(1))) { + SDValue BitMask = N1; + SDValue SrcVec = N0.getOperand(0); EVT SrcVecVT = SrcVec.getValueType(); // Check that the constant bitmask masks whole bytes. APInt UndefElts; SmallVector<APInt, 64> EltBits; - if (VT == SrcVecVT.getScalarType() && - N->getOperand(0)->isOnlyUserOf(SrcVec.getNode()) && + if (VT == SrcVecVT.getScalarType() && N0->isOnlyUserOf(SrcVec.getNode()) && getTargetConstantBitsFromNode(BitMask, 8, UndefElts, EltBits) && llvm::all_of(EltBits, [](const APInt &M) { return M.isZero() || M.isAllOnes(); })) { unsigned NumElts = SrcVecVT.getVectorNumElements(); unsigned Scale = SrcVecVT.getScalarSizeInBits() / 8; - unsigned Idx = N->getOperand(0).getConstantOperandVal(1); + unsigned Idx = N0.getConstantOperandVal(1); // Create a root shuffle mask from the byte mask and the extracted index. SmallVector<int, 16> ShuffleMask(NumElts * Scale, SM_SentinelUndef); @@ -46318,8 +46434,8 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG, X86::MaxShuffleCombineDepth, /*HasVarMask*/ false, /*AllowVarCrossLaneMask*/ true, /*AllowVarPerLaneMask*/ true, DAG, Subtarget)) - return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), VT, Shuffle, - N->getOperand(0).getOperand(1)); + return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle, + N0.getOperand(1)); } } @@ -46644,11 +46760,13 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG, SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); EVT VT = N->getValueType(0); + SDLoc dl(N); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); // If this is SSE1 only convert to FOR to avoid scalarization. if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { return DAG.getBitcast(MVT::v4i32, - DAG.getNode(X86ISD::FOR, SDLoc(N), MVT::v4f32, + DAG.getNode(X86ISD::FOR, dl, MVT::v4f32, DAG.getBitcast(MVT::v4f32, N0), DAG.getBitcast(MVT::v4f32, N1))); } @@ -46660,8 +46778,6 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG, SmallVector<APInt, 2> SrcPartials; if (matchScalarReduction(SDValue(N, 0), ISD::OR, SrcOps, &SrcPartials) && SrcOps.size() == 1) { - SDLoc dl(N); - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements(); EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts); SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget); @@ -46707,7 +46823,6 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG, if (NumElts >= 16 && N1.getOpcode() == X86ISD::KSHIFTL && N1.getConstantOperandAPInt(1) == HalfElts && DAG.MaskedValueIsZero(N0, APInt(1, 1), UpperElts)) { - SDLoc dl(N); return DAG.getNode( ISD::CONCAT_VECTORS, dl, VT, extractSubVector(N0, 0, DAG, dl, HalfElts), @@ -46716,7 +46831,6 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG, if (NumElts >= 16 && N0.getOpcode() == X86ISD::KSHIFTL && N0.getConstantOperandAPInt(1) == HalfElts && DAG.MaskedValueIsZero(N1, APInt(1, 1), UpperElts)) { - SDLoc dl(N); return DAG.getNode( ISD::CONCAT_VECTORS, dl, VT, extractSubVector(N1, 0, DAG, dl, HalfElts), @@ -46724,11 +46838,36 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG, } } - // Attempt to recursively combine an OR of shuffles. if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) { + // Attempt to recursively combine an OR of shuffles. SDValue Op(N, 0); if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) return Res; + + // If either operand is a constant mask, then only the elements that aren't + // allones are actually demanded by the other operand. + auto SimplifyUndemandedElts = [&](SDValue Op, SDValue OtherOp) { + APInt UndefElts; + SmallVector<APInt> EltBits; + int NumElts = VT.getVectorNumElements(); + int EltSizeInBits = VT.getScalarSizeInBits(); + if (!getTargetConstantBitsFromNode(Op, EltSizeInBits, UndefElts, EltBits)) + return false; + + APInt DemandedElts = APInt::getZero(NumElts); + for (int I = 0; I != NumElts; ++I) + if (!EltBits[I].isAllOnes()) + DemandedElts.setBit(I); + + APInt KnownUndef, KnownZero; + return TLI.SimplifyDemandedVectorElts(OtherOp, DemandedElts, KnownUndef, + KnownZero, DCI); + }; + if (SimplifyUndemandedElts(N0, N1) || SimplifyUndemandedElts(N1, N0)) { + if (N->getOpcode() != ISD::DELETED_NODE) + DCI.AddToWorklist(N); + return SDValue(N, 0); + } } // We should fold "masked merge" patterns when `andn` is not available. @@ -52111,7 +52250,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, case X86ISD::VSHLI: case X86ISD::VSRLI: // Special case: SHL/SRL AVX1 V4i64 by 32-bits can lower as a shuffle. - // TODO: Move this to LowerScalarImmediateShift? + // TODO: Move this to LowerShiftByScalarImmediate? if (VT == MVT::v4i64 && !Subtarget.hasInt256() && llvm::all_of(Ops, [](SDValue Op) { return Op.getConstantOperandAPInt(1) == 32; diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 6805cb75f0f2..d1d6e319f16b 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -1100,6 +1100,12 @@ namespace llvm { bool shouldSplatInsEltVarIndex(EVT VT) const override; + bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override { + // Converting to sat variants holds little benefit on X86 as we will just + // need to saturate the value back using fp arithmatic. + return Op != ISD::FP_TO_UINT_SAT && isOperationLegalOrCustom(Op, VT); + } + bool convertSetCCLogicToBitwiseLogic(EVT VT) const override { return VT.isScalarInteger(); } @@ -1153,6 +1159,10 @@ namespace llvm { SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const override; + bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, + APInt &UndefElts, + unsigned Depth) const override; + const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const override; SDValue unwrapAddress(SDValue N) const override; diff --git a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp index 732b2b1a5ada..6642f46e64b2 100644 --- a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp +++ b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp @@ -137,8 +137,10 @@ bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) { Changed |= addENDBR(MBB, MBB.begin()); for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) { - if (I->isCall() && IsCallReturnTwice(I->getOperand(0))) + if (I->isCall() && I->getNumOperands() > 0 && + IsCallReturnTwice(I->getOperand(0))) { Changed |= addENDBR(MBB, std::next(I)); + } } // Exception handle may indirectly jump to catch pad, So we should add diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td index 1db83033ba35..ecd4777c3533 100644 --- a/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/llvm/lib/Target/X86/X86InstrAVX512.td @@ -9958,74 +9958,74 @@ multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode, } defm VPMOVQB : avx512_trunc_qb<0x32, "vpmovqb", - WriteShuffle256, truncstorevi8, + WriteVPMOV256, truncstorevi8, masked_truncstorevi8, X86vtrunc, X86vmtrunc>; defm VPMOVSQB : avx512_trunc_qb<0x22, "vpmovsqb", - WriteShuffle256, truncstore_s_vi8, + WriteVPMOV256, truncstore_s_vi8, masked_truncstore_s_vi8, X86vtruncs, X86vmtruncs>; defm VPMOVUSQB : avx512_trunc_qb<0x12, "vpmovusqb", - WriteShuffle256, truncstore_us_vi8, + WriteVPMOV256, truncstore_us_vi8, masked_truncstore_us_vi8, X86vtruncus, X86vmtruncus>; defm VPMOVQW : avx512_trunc_qw<0x34, "vpmovqw", trunc, select_trunc, - WriteShuffle256, truncstorevi16, + WriteVPMOV256, truncstorevi16, masked_truncstorevi16, X86vtrunc, X86vmtrunc>; defm VPMOVSQW : avx512_trunc_qw<0x24, "vpmovsqw", X86vtruncs, select_truncs, - WriteShuffle256, truncstore_s_vi16, + WriteVPMOV256, truncstore_s_vi16, masked_truncstore_s_vi16, X86vtruncs, X86vmtruncs>; defm VPMOVUSQW : avx512_trunc_qw<0x14, "vpmovusqw", X86vtruncus, - select_truncus, WriteShuffle256, + select_truncus, WriteVPMOV256, truncstore_us_vi16, masked_truncstore_us_vi16, X86vtruncus, X86vmtruncus>; defm VPMOVQD : avx512_trunc_qd<0x35, "vpmovqd", trunc, select_trunc, - WriteShuffle256, truncstorevi32, + WriteVPMOV256, truncstorevi32, masked_truncstorevi32, X86vtrunc, X86vmtrunc>; defm VPMOVSQD : avx512_trunc_qd<0x25, "vpmovsqd", X86vtruncs, select_truncs, - WriteShuffle256, truncstore_s_vi32, + WriteVPMOV256, truncstore_s_vi32, masked_truncstore_s_vi32, X86vtruncs, X86vmtruncs>; defm VPMOVUSQD : avx512_trunc_qd<0x15, "vpmovusqd", X86vtruncus, - select_truncus, WriteShuffle256, + select_truncus, WriteVPMOV256, truncstore_us_vi32, masked_truncstore_us_vi32, X86vtruncus, X86vmtruncus>; defm VPMOVDB : avx512_trunc_db<0x31, "vpmovdb", trunc, select_trunc, - WriteShuffle256, truncstorevi8, + WriteVPMOV256, truncstorevi8, masked_truncstorevi8, X86vtrunc, X86vmtrunc>; defm VPMOVSDB : avx512_trunc_db<0x21, "vpmovsdb", X86vtruncs, select_truncs, - WriteShuffle256, truncstore_s_vi8, + WriteVPMOV256, truncstore_s_vi8, masked_truncstore_s_vi8, X86vtruncs, X86vmtruncs>; defm VPMOVUSDB : avx512_trunc_db<0x11, "vpmovusdb", X86vtruncus, - select_truncus, WriteShuffle256, + select_truncus, WriteVPMOV256, truncstore_us_vi8, masked_truncstore_us_vi8, X86vtruncus, X86vmtruncus>; defm VPMOVDW : avx512_trunc_dw<0x33, "vpmovdw", trunc, select_trunc, - WriteShuffle256, truncstorevi16, + WriteVPMOV256, truncstorevi16, masked_truncstorevi16, X86vtrunc, X86vmtrunc>; defm VPMOVSDW : avx512_trunc_dw<0x23, "vpmovsdw", X86vtruncs, select_truncs, - WriteShuffle256, truncstore_s_vi16, + WriteVPMOV256, truncstore_s_vi16, masked_truncstore_s_vi16, X86vtruncs, X86vmtruncs>; defm VPMOVUSDW : avx512_trunc_dw<0x13, "vpmovusdw", X86vtruncus, - select_truncus, WriteShuffle256, + select_truncus, WriteVPMOV256, truncstore_us_vi16, masked_truncstore_us_vi16, X86vtruncus, X86vmtruncus>; defm VPMOVWB : avx512_trunc_wb<0x30, "vpmovwb", trunc, select_trunc, - WriteShuffle256, truncstorevi8, + WriteVPMOV256, truncstorevi8, masked_truncstorevi8, X86vtrunc, X86vmtrunc>; defm VPMOVSWB : avx512_trunc_wb<0x20, "vpmovswb", X86vtruncs, select_truncs, - WriteShuffle256, truncstore_s_vi8, + WriteVPMOV256, truncstore_s_vi8, masked_truncstore_s_vi8, X86vtruncs, X86vmtruncs>; defm VPMOVUSWB : avx512_trunc_wb<0x10, "vpmovuswb", X86vtruncus, - select_truncus, WriteShuffle256, + select_truncus, WriteVPMOV256, truncstore_us_vi8, masked_truncstore_us_vi8, X86vtruncus, X86vmtruncus>; @@ -10084,7 +10084,7 @@ defm : mtrunc_lowering<"VPMOVSQWZ", X86vmtruncs, v8i16x_info, v8i64_info>; defm : mtrunc_lowering<"VPMOVUSQWZ", X86vmtruncus, v8i16x_info, v8i64_info>; } -multiclass WriteShuffle256_common<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched, +multiclass avx512_pmovx_common<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched, X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo, X86MemOperand x86memop, PatFrag LdFrag, SDNode OpNode>{ let ExeDomain = DestInfo.ExeDomain in { @@ -10100,135 +10100,140 @@ multiclass WriteShuffle256_common<bits<8> opc, string OpcodeStr, X86FoldableSche } } -multiclass WriteShuffle256_BW<bits<8> opc, string OpcodeStr, +multiclass avx512_pmovx_bw<bits<8> opc, string OpcodeStr, SDNode OpNode, SDNode InVecNode, string ExtTy, - X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { + X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ, + PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { let Predicates = [HasVLX, HasBWI] in { - defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v8i16x_info, + defm Z128: avx512_pmovx_common<opc, OpcodeStr, schedX, v8i16x_info, v16i8x_info, i64mem, LdFrag, InVecNode>, EVEX_CD8<8, CD8VH>, T8PD, EVEX_V128, VEX_WIG; - defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v16i16x_info, + defm Z256: avx512_pmovx_common<opc, OpcodeStr, schedYZ, v16i16x_info, v16i8x_info, i128mem, LdFrag, OpNode>, EVEX_CD8<8, CD8VH>, T8PD, EVEX_V256, VEX_WIG; } let Predicates = [HasBWI] in { - defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v32i16_info, + defm Z : avx512_pmovx_common<opc, OpcodeStr, schedYZ, v32i16_info, v32i8x_info, i256mem, LdFrag, OpNode>, EVEX_CD8<8, CD8VH>, T8PD, EVEX_V512, VEX_WIG; } } -multiclass WriteShuffle256_BD<bits<8> opc, string OpcodeStr, +multiclass avx512_pmovx_bd<bits<8> opc, string OpcodeStr, SDNode OpNode, SDNode InVecNode, string ExtTy, - X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { + X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ, + PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { let Predicates = [HasVLX, HasAVX512] in { - defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v4i32x_info, + defm Z128: avx512_pmovx_common<opc, OpcodeStr, schedX, v4i32x_info, v16i8x_info, i32mem, LdFrag, InVecNode>, EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V128, VEX_WIG; - defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v8i32x_info, + defm Z256: avx512_pmovx_common<opc, OpcodeStr, schedYZ, v8i32x_info, v16i8x_info, i64mem, LdFrag, InVecNode>, EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V256, VEX_WIG; } let Predicates = [HasAVX512] in { - defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v16i32_info, + defm Z : avx512_pmovx_common<opc, OpcodeStr, schedYZ, v16i32_info, v16i8x_info, i128mem, LdFrag, OpNode>, EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V512, VEX_WIG; } } -multiclass WriteShuffle256_BQ<bits<8> opc, string OpcodeStr, +multiclass avx512_pmovx_bq<bits<8> opc, string OpcodeStr, SDNode InVecNode, string ExtTy, - X86FoldableSchedWrite sched, + X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { let Predicates = [HasVLX, HasAVX512] in { - defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v2i64x_info, + defm Z128: avx512_pmovx_common<opc, OpcodeStr, schedX, v2i64x_info, v16i8x_info, i16mem, LdFrag, InVecNode>, EVEX_CD8<8, CD8VO>, T8PD, EVEX_V128, VEX_WIG; - defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v4i64x_info, + defm Z256: avx512_pmovx_common<opc, OpcodeStr, schedYZ, v4i64x_info, v16i8x_info, i32mem, LdFrag, InVecNode>, EVEX_CD8<8, CD8VO>, T8PD, EVEX_V256, VEX_WIG; } let Predicates = [HasAVX512] in { - defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v8i64_info, + defm Z : avx512_pmovx_common<opc, OpcodeStr, schedYZ, v8i64_info, v16i8x_info, i64mem, LdFrag, InVecNode>, EVEX_CD8<8, CD8VO>, T8PD, EVEX_V512, VEX_WIG; } } -multiclass WriteShuffle256_WD<bits<8> opc, string OpcodeStr, +multiclass avx512_pmovx_wd<bits<8> opc, string OpcodeStr, SDNode OpNode, SDNode InVecNode, string ExtTy, - X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> { + X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ, + PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> { let Predicates = [HasVLX, HasAVX512] in { - defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v4i32x_info, + defm Z128: avx512_pmovx_common<opc, OpcodeStr, schedX, v4i32x_info, v8i16x_info, i64mem, LdFrag, InVecNode>, EVEX_CD8<16, CD8VH>, T8PD, EVEX_V128, VEX_WIG; - defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v8i32x_info, + defm Z256: avx512_pmovx_common<opc, OpcodeStr, schedYZ, v8i32x_info, v8i16x_info, i128mem, LdFrag, OpNode>, EVEX_CD8<16, CD8VH>, T8PD, EVEX_V256, VEX_WIG; } let Predicates = [HasAVX512] in { - defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v16i32_info, + defm Z : avx512_pmovx_common<opc, OpcodeStr, schedYZ, v16i32_info, v16i16x_info, i256mem, LdFrag, OpNode>, EVEX_CD8<16, CD8VH>, T8PD, EVEX_V512, VEX_WIG; } } -multiclass WriteShuffle256_WQ<bits<8> opc, string OpcodeStr, +multiclass avx512_pmovx_wq<bits<8> opc, string OpcodeStr, SDNode OpNode, SDNode InVecNode, string ExtTy, - X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> { + X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ, + PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> { let Predicates = [HasVLX, HasAVX512] in { - defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v2i64x_info, + defm Z128: avx512_pmovx_common<opc, OpcodeStr, schedX, v2i64x_info, v8i16x_info, i32mem, LdFrag, InVecNode>, EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V128, VEX_WIG; - defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v4i64x_info, + defm Z256: avx512_pmovx_common<opc, OpcodeStr, schedYZ, v4i64x_info, v8i16x_info, i64mem, LdFrag, InVecNode>, EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V256, VEX_WIG; } let Predicates = [HasAVX512] in { - defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v8i64_info, + defm Z : avx512_pmovx_common<opc, OpcodeStr, schedYZ, v8i64_info, v8i16x_info, i128mem, LdFrag, OpNode>, EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V512, VEX_WIG; } } -multiclass WriteShuffle256_DQ<bits<8> opc, string OpcodeStr, +multiclass avx512_pmovx_dq<bits<8> opc, string OpcodeStr, SDNode OpNode, SDNode InVecNode, string ExtTy, - X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> { + X86FoldableSchedWrite schedX, X86FoldableSchedWrite schedYZ, + PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> { let Predicates = [HasVLX, HasAVX512] in { - defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v2i64x_info, + defm Z128: avx512_pmovx_common<opc, OpcodeStr, schedX, v2i64x_info, v4i32x_info, i64mem, LdFrag, InVecNode>, EVEX_CD8<32, CD8VH>, T8PD, EVEX_V128; - defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v4i64x_info, + defm Z256: avx512_pmovx_common<opc, OpcodeStr, schedYZ, v4i64x_info, v4i32x_info, i128mem, LdFrag, OpNode>, EVEX_CD8<32, CD8VH>, T8PD, EVEX_V256; } let Predicates = [HasAVX512] in { - defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v8i64_info, + defm Z : avx512_pmovx_common<opc, OpcodeStr, schedYZ, v8i64_info, v8i32x_info, i256mem, LdFrag, OpNode>, EVEX_CD8<32, CD8VH>, T8PD, EVEX_V512; } } -defm VPMOVZXBW : WriteShuffle256_BW<0x30, "vpmovzxbw", zext, zext_invec, "z", WriteShuffle256>; -defm VPMOVZXBD : WriteShuffle256_BD<0x31, "vpmovzxbd", zext, zext_invec, "z", WriteShuffle256>; -defm VPMOVZXBQ : WriteShuffle256_BQ<0x32, "vpmovzxbq", zext_invec, "z", WriteShuffle256>; -defm VPMOVZXWD : WriteShuffle256_WD<0x33, "vpmovzxwd", zext, zext_invec, "z", WriteShuffle256>; -defm VPMOVZXWQ : WriteShuffle256_WQ<0x34, "vpmovzxwq", zext, zext_invec, "z", WriteShuffle256>; -defm VPMOVZXDQ : WriteShuffle256_DQ<0x35, "vpmovzxdq", zext, zext_invec, "z", WriteShuffle256>; +defm VPMOVZXBW : avx512_pmovx_bw<0x30, "vpmovzxbw", zext, zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>; +defm VPMOVZXBD : avx512_pmovx_bd<0x31, "vpmovzxbd", zext, zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>; +defm VPMOVZXBQ : avx512_pmovx_bq<0x32, "vpmovzxbq", zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>; +defm VPMOVZXWD : avx512_pmovx_wd<0x33, "vpmovzxwd", zext, zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>; +defm VPMOVZXWQ : avx512_pmovx_wq<0x34, "vpmovzxwq", zext, zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>; +defm VPMOVZXDQ : avx512_pmovx_dq<0x35, "vpmovzxdq", zext, zext_invec, "z", SchedWriteShuffle.XMM, WriteVPMOV256>; -defm VPMOVSXBW: WriteShuffle256_BW<0x20, "vpmovsxbw", sext, sext_invec, "s", WriteShuffle256>; -defm VPMOVSXBD: WriteShuffle256_BD<0x21, "vpmovsxbd", sext, sext_invec, "s", WriteShuffle256>; -defm VPMOVSXBQ: WriteShuffle256_BQ<0x22, "vpmovsxbq", sext_invec, "s", WriteShuffle256>; -defm VPMOVSXWD: WriteShuffle256_WD<0x23, "vpmovsxwd", sext, sext_invec, "s", WriteShuffle256>; -defm VPMOVSXWQ: WriteShuffle256_WQ<0x24, "vpmovsxwq", sext, sext_invec, "s", WriteShuffle256>; -defm VPMOVSXDQ: WriteShuffle256_DQ<0x25, "vpmovsxdq", sext, sext_invec, "s", WriteShuffle256>; +defm VPMOVSXBW: avx512_pmovx_bw<0x20, "vpmovsxbw", sext, sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>; +defm VPMOVSXBD: avx512_pmovx_bd<0x21, "vpmovsxbd", sext, sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>; +defm VPMOVSXBQ: avx512_pmovx_bq<0x22, "vpmovsxbq", sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>; +defm VPMOVSXWD: avx512_pmovx_wd<0x23, "vpmovsxwd", sext, sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>; +defm VPMOVSXWQ: avx512_pmovx_wq<0x24, "vpmovsxwq", sext, sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>; +defm VPMOVSXDQ: avx512_pmovx_dq<0x25, "vpmovsxdq", sext, sext_invec, "s", SchedWriteShuffle.XMM, WriteVPMOV256>; // Patterns that we also need any extend versions of. aext_vector_inreg @@ -10523,21 +10528,22 @@ defm VSCATTERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dpd defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd", VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; -multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr > { +multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr, SchedWrite Sched> { def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src), !strconcat(OpcodeStr#Vec.Suffix, "\t{$src, $dst|$dst, $src}"), [(set Vec.RC:$dst, (Vec.VT (sext Vec.KRC:$src)))]>, - EVEX, Sched<[WriteMove]>; // TODO - WriteVecTrunc? + EVEX, Sched<[Sched]>; } multiclass cvt_mask_by_elt_width<bits<8> opc, AVX512VLVectorVTInfo VTInfo, string OpcodeStr, Predicate prd> { +// TODO - Replace WriteMove with WriteVecTrunc? let Predicates = [prd] in - defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512; + defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr, WriteMove>, EVEX_V512; let Predicates = [prd, HasVLX] in { - defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256; - defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128; + defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr, WriteMove>, EVEX_V256; + defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr, WriteMove>, EVEX_V128; } } diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td index ba52283b570d..7288ce812138 100644 --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -260,10 +260,10 @@ let isPseudo = 1, SchedRW = [WriteSystem] in { // Pseudo instructions used by address sanitizer. //===----------------------------------------------------------------------===// let - Defs = [R8, EFLAGS] in { + Defs = [R10, R11, EFLAGS] in { def ASAN_CHECK_MEMACCESS : PseudoI< - (outs), (ins GR64NoR8:$addr, i32imm:$accessinfo), - [(int_asan_check_memaccess GR64NoR8:$addr, (i32 timm:$accessinfo))]>, + (outs), (ins GR64PLTSafe:$addr, i32imm:$accessinfo), + [(int_asan_check_memaccess GR64PLTSafe:$addr, (i32 timm:$accessinfo))]>, Sched<[]>; } diff --git a/llvm/lib/Target/X86/X86InstrFoldTables.cpp b/llvm/lib/Target/X86/X86InstrFoldTables.cpp index 6d4ad08842c7..226349485238 100644 --- a/llvm/lib/Target/X86/X86InstrFoldTables.cpp +++ b/llvm/lib/Target/X86/X86InstrFoldTables.cpp @@ -529,11 +529,11 @@ static const X86MemoryFoldTableEntry MemoryFoldTable1[] = { { X86::LZCNT16rr, X86::LZCNT16rm, 0 }, { X86::LZCNT32rr, X86::LZCNT32rm, 0 }, { X86::LZCNT64rr, X86::LZCNT64rm, 0 }, - { X86::MMX_CVTPD2PIirr, X86::MMX_CVTPD2PIirm, TB_ALIGN_16 }, - { X86::MMX_CVTPI2PDirr, X86::MMX_CVTPI2PDirm, 0 }, - { X86::MMX_CVTPS2PIirr, X86::MMX_CVTPS2PIirm, TB_NO_REVERSE }, - { X86::MMX_CVTTPD2PIirr, X86::MMX_CVTTPD2PIirm, TB_ALIGN_16 }, - { X86::MMX_CVTTPS2PIirr, X86::MMX_CVTTPS2PIirm, TB_NO_REVERSE }, + { X86::MMX_CVTPD2PIrr, X86::MMX_CVTPD2PIrm, TB_ALIGN_16 }, + { X86::MMX_CVTPI2PDrr, X86::MMX_CVTPI2PDrm, 0 }, + { X86::MMX_CVTPS2PIrr, X86::MMX_CVTPS2PIrm, TB_NO_REVERSE }, + { X86::MMX_CVTTPD2PIrr, X86::MMX_CVTTPD2PIrm, TB_ALIGN_16 }, + { X86::MMX_CVTTPS2PIrr, X86::MMX_CVTTPS2PIrm, TB_NO_REVERSE }, { X86::MMX_MOVD64to64rr, X86::MMX_MOVQ64rm, 0 }, { X86::MMX_PABSBrr, X86::MMX_PABSBrm, 0 }, { X86::MMX_PABSDrr, X86::MMX_PABSDrm, 0 }, @@ -1339,29 +1339,29 @@ static const X86MemoryFoldTableEntry MemoryFoldTable2[] = { { X86::MINSDrr_Int, X86::MINSDrm_Int, TB_NO_REVERSE }, { X86::MINSSrr, X86::MINSSrm, 0 }, { X86::MINSSrr_Int, X86::MINSSrm_Int, TB_NO_REVERSE }, - { X86::MMX_CVTPI2PSirr, X86::MMX_CVTPI2PSirm, 0 }, - { X86::MMX_PACKSSDWirr, X86::MMX_PACKSSDWirm, 0 }, - { X86::MMX_PACKSSWBirr, X86::MMX_PACKSSWBirm, 0 }, - { X86::MMX_PACKUSWBirr, X86::MMX_PACKUSWBirm, 0 }, - { X86::MMX_PADDBirr, X86::MMX_PADDBirm, 0 }, - { X86::MMX_PADDDirr, X86::MMX_PADDDirm, 0 }, - { X86::MMX_PADDQirr, X86::MMX_PADDQirm, 0 }, - { X86::MMX_PADDSBirr, X86::MMX_PADDSBirm, 0 }, - { X86::MMX_PADDSWirr, X86::MMX_PADDSWirm, 0 }, - { X86::MMX_PADDUSBirr, X86::MMX_PADDUSBirm, 0 }, - { X86::MMX_PADDUSWirr, X86::MMX_PADDUSWirm, 0 }, - { X86::MMX_PADDWirr, X86::MMX_PADDWirm, 0 }, + { X86::MMX_CVTPI2PSrr, X86::MMX_CVTPI2PSrm, 0 }, + { X86::MMX_PACKSSDWrr, X86::MMX_PACKSSDWrm, 0 }, + { X86::MMX_PACKSSWBrr, X86::MMX_PACKSSWBrm, 0 }, + { X86::MMX_PACKUSWBrr, X86::MMX_PACKUSWBrm, 0 }, + { X86::MMX_PADDBrr, X86::MMX_PADDBrm, 0 }, + { X86::MMX_PADDDrr, X86::MMX_PADDDrm, 0 }, + { X86::MMX_PADDQrr, X86::MMX_PADDQrm, 0 }, + { X86::MMX_PADDSBrr, X86::MMX_PADDSBrm, 0 }, + { X86::MMX_PADDSWrr, X86::MMX_PADDSWrm, 0 }, + { X86::MMX_PADDUSBrr, X86::MMX_PADDUSBrm, 0 }, + { X86::MMX_PADDUSWrr, X86::MMX_PADDUSWrm, 0 }, + { X86::MMX_PADDWrr, X86::MMX_PADDWrm, 0 }, { X86::MMX_PALIGNRrri, X86::MMX_PALIGNRrmi, 0 }, - { X86::MMX_PANDNirr, X86::MMX_PANDNirm, 0 }, - { X86::MMX_PANDirr, X86::MMX_PANDirm, 0 }, - { X86::MMX_PAVGBirr, X86::MMX_PAVGBirm, 0 }, - { X86::MMX_PAVGWirr, X86::MMX_PAVGWirm, 0 }, - { X86::MMX_PCMPEQBirr, X86::MMX_PCMPEQBirm, 0 }, - { X86::MMX_PCMPEQDirr, X86::MMX_PCMPEQDirm, 0 }, - { X86::MMX_PCMPEQWirr, X86::MMX_PCMPEQWirm, 0 }, - { X86::MMX_PCMPGTBirr, X86::MMX_PCMPGTBirm, 0 }, - { X86::MMX_PCMPGTDirr, X86::MMX_PCMPGTDirm, 0 }, - { X86::MMX_PCMPGTWirr, X86::MMX_PCMPGTWirm, 0 }, + { X86::MMX_PANDNrr, X86::MMX_PANDNrm, 0 }, + { X86::MMX_PANDrr, X86::MMX_PANDrm, 0 }, + { X86::MMX_PAVGBrr, X86::MMX_PAVGBrm, 0 }, + { X86::MMX_PAVGWrr, X86::MMX_PAVGWrm, 0 }, + { X86::MMX_PCMPEQBrr, X86::MMX_PCMPEQBrm, 0 }, + { X86::MMX_PCMPEQDrr, X86::MMX_PCMPEQDrm, 0 }, + { X86::MMX_PCMPEQWrr, X86::MMX_PCMPEQWrm, 0 }, + { X86::MMX_PCMPGTBrr, X86::MMX_PCMPGTBrm, 0 }, + { X86::MMX_PCMPGTDrr, X86::MMX_PCMPGTDrm, 0 }, + { X86::MMX_PCMPGTWrr, X86::MMX_PCMPGTWrm, 0 }, { X86::MMX_PHADDDrr, X86::MMX_PHADDDrm, 0 }, { X86::MMX_PHADDSWrr, X86::MMX_PHADDSWrm, 0 }, { X86::MMX_PHADDWrr, X86::MMX_PHADDWrm, 0 }, @@ -1370,18 +1370,18 @@ static const X86MemoryFoldTableEntry MemoryFoldTable2[] = { { X86::MMX_PHSUBWrr, X86::MMX_PHSUBWrm, 0 }, { X86::MMX_PINSRWrr, X86::MMX_PINSRWrm, TB_NO_REVERSE }, { X86::MMX_PMADDUBSWrr, X86::MMX_PMADDUBSWrm, 0 }, - { X86::MMX_PMADDWDirr, X86::MMX_PMADDWDirm, 0 }, - { X86::MMX_PMAXSWirr, X86::MMX_PMAXSWirm, 0 }, - { X86::MMX_PMAXUBirr, X86::MMX_PMAXUBirm, 0 }, - { X86::MMX_PMINSWirr, X86::MMX_PMINSWirm, 0 }, - { X86::MMX_PMINUBirr, X86::MMX_PMINUBirm, 0 }, + { X86::MMX_PMADDWDrr, X86::MMX_PMADDWDrm, 0 }, + { X86::MMX_PMAXSWrr, X86::MMX_PMAXSWrm, 0 }, + { X86::MMX_PMAXUBrr, X86::MMX_PMAXUBrm, 0 }, + { X86::MMX_PMINSWrr, X86::MMX_PMINSWrm, 0 }, + { X86::MMX_PMINUBrr, X86::MMX_PMINUBrm, 0 }, { X86::MMX_PMULHRSWrr, X86::MMX_PMULHRSWrm, 0 }, - { X86::MMX_PMULHUWirr, X86::MMX_PMULHUWirm, 0 }, - { X86::MMX_PMULHWirr, X86::MMX_PMULHWirm, 0 }, - { X86::MMX_PMULLWirr, X86::MMX_PMULLWirm, 0 }, - { X86::MMX_PMULUDQirr, X86::MMX_PMULUDQirm, 0 }, - { X86::MMX_PORirr, X86::MMX_PORirm, 0 }, - { X86::MMX_PSADBWirr, X86::MMX_PSADBWirm, 0 }, + { X86::MMX_PMULHUWrr, X86::MMX_PMULHUWrm, 0 }, + { X86::MMX_PMULHWrr, X86::MMX_PMULHWrm, 0 }, + { X86::MMX_PMULLWrr, X86::MMX_PMULLWrm, 0 }, + { X86::MMX_PMULUDQrr, X86::MMX_PMULUDQrm, 0 }, + { X86::MMX_PORrr, X86::MMX_PORrm, 0 }, + { X86::MMX_PSADBWrr, X86::MMX_PSADBWrm, 0 }, { X86::MMX_PSHUFBrr, X86::MMX_PSHUFBrm, 0 }, { X86::MMX_PSIGNBrr, X86::MMX_PSIGNBrm, 0 }, { X86::MMX_PSIGNDrr, X86::MMX_PSIGNDrm, 0 }, @@ -1394,21 +1394,21 @@ static const X86MemoryFoldTableEntry MemoryFoldTable2[] = { { X86::MMX_PSRLDrr, X86::MMX_PSRLDrm, 0 }, { X86::MMX_PSRLQrr, X86::MMX_PSRLQrm, 0 }, { X86::MMX_PSRLWrr, X86::MMX_PSRLWrm, 0 }, - { X86::MMX_PSUBBirr, X86::MMX_PSUBBirm, 0 }, - { X86::MMX_PSUBDirr, X86::MMX_PSUBDirm, 0 }, - { X86::MMX_PSUBQirr, X86::MMX_PSUBQirm, 0 }, - { X86::MMX_PSUBSBirr, X86::MMX_PSUBSBirm, 0 }, - { X86::MMX_PSUBSWirr, X86::MMX_PSUBSWirm, 0 }, - { X86::MMX_PSUBUSBirr, X86::MMX_PSUBUSBirm, 0 }, - { X86::MMX_PSUBUSWirr, X86::MMX_PSUBUSWirm, 0 }, - { X86::MMX_PSUBWirr, X86::MMX_PSUBWirm, 0 }, - { X86::MMX_PUNPCKHBWirr, X86::MMX_PUNPCKHBWirm, 0 }, - { X86::MMX_PUNPCKHDQirr, X86::MMX_PUNPCKHDQirm, 0 }, - { X86::MMX_PUNPCKHWDirr, X86::MMX_PUNPCKHWDirm, 0 }, - { X86::MMX_PUNPCKLBWirr, X86::MMX_PUNPCKLBWirm, TB_NO_REVERSE }, - { X86::MMX_PUNPCKLDQirr, X86::MMX_PUNPCKLDQirm, TB_NO_REVERSE }, - { X86::MMX_PUNPCKLWDirr, X86::MMX_PUNPCKLWDirm, TB_NO_REVERSE }, - { X86::MMX_PXORirr, X86::MMX_PXORirm, 0 }, + { X86::MMX_PSUBBrr, X86::MMX_PSUBBrm, 0 }, + { X86::MMX_PSUBDrr, X86::MMX_PSUBDrm, 0 }, + { X86::MMX_PSUBQrr, X86::MMX_PSUBQrm, 0 }, + { X86::MMX_PSUBSBrr, X86::MMX_PSUBSBrm, 0 }, + { X86::MMX_PSUBSWrr, X86::MMX_PSUBSWrm, 0 }, + { X86::MMX_PSUBUSBrr, X86::MMX_PSUBUSBrm, 0 }, + { X86::MMX_PSUBUSWrr, X86::MMX_PSUBUSWrm, 0 }, + { X86::MMX_PSUBWrr, X86::MMX_PSUBWrm, 0 }, + { X86::MMX_PUNPCKHBWrr, X86::MMX_PUNPCKHBWrm, 0 }, + { X86::MMX_PUNPCKHDQrr, X86::MMX_PUNPCKHDQrm, 0 }, + { X86::MMX_PUNPCKHWDrr, X86::MMX_PUNPCKHWDrm, 0 }, + { X86::MMX_PUNPCKLBWrr, X86::MMX_PUNPCKLBWrm, TB_NO_REVERSE }, + { X86::MMX_PUNPCKLDQrr, X86::MMX_PUNPCKLDQrm, TB_NO_REVERSE }, + { X86::MMX_PUNPCKLWDrr, X86::MMX_PUNPCKLWDrm, TB_NO_REVERSE }, + { X86::MMX_PXORrr, X86::MMX_PXORrm, 0 }, { X86::MOVLHPSrr, X86::MOVHPSrm, TB_NO_REVERSE }, { X86::MOVSDrr, X86::MOVLPDrm, TB_NO_REVERSE }, { X86::MPSADBWrri, X86::MPSADBWrmi, TB_ALIGN_16 }, diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index bb5637a31947..c379aa8d9258 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -4088,8 +4088,8 @@ bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg, bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI, Register SrcReg, Register SrcReg2, int64_t ImmMask, int64_t ImmValue, - const MachineInstr &OI, bool *IsSwapped, - int64_t *ImmDelta) const { + const MachineInstr &OI, + bool *IsSwapped) const { switch (OI.getOpcode()) { case X86::CMP64rr: case X86::CMP32rr: @@ -4140,21 +4140,10 @@ bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI, int64_t OIMask; int64_t OIValue; if (analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) && - SrcReg == OISrcReg && ImmMask == OIMask) { - if (OIValue == ImmValue) { - *ImmDelta = 0; - return true; - } else if (static_cast<uint64_t>(ImmValue) == - static_cast<uint64_t>(OIValue) - 1) { - *ImmDelta = -1; - return true; - } else if (static_cast<uint64_t>(ImmValue) == - static_cast<uint64_t>(OIValue) + 1) { - *ImmDelta = 1; - return true; - } else { - return false; - } + SrcReg == OISrcReg && ImmMask == OIMask && OIValue == ImmValue) { + assert(SrcReg2 == X86::NoRegister && OISrcReg2 == X86::NoRegister && + "should not have 2nd register"); + return true; } } return FlagI.isIdenticalTo(OI); @@ -4404,7 +4393,6 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, bool ShouldUpdateCC = false; bool IsSwapped = false; X86::CondCode NewCC = X86::COND_INVALID; - int64_t ImmDelta = 0; // Search backward from CmpInstr for the next instruction defining EFLAGS. const TargetRegisterInfo *TRI = &getRegisterInfo(); @@ -4451,7 +4439,7 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, // ... // EFLAGS not changed // cmp x, y // <-- can be removed if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue, - Inst, &IsSwapped, &ImmDelta)) { + Inst, &IsSwapped)) { Sub = &Inst; break; } @@ -4485,7 +4473,7 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, // It is safe to remove CmpInstr if EFLAGS is redefined or killed. // If we are done with the basic block, we need to check whether EFLAGS is // live-out. - bool FlagsMayLiveOut = true; + bool IsSafe = false; SmallVector<std::pair<MachineInstr*, X86::CondCode>, 4> OpsToUpdate; MachineBasicBlock::iterator AfterCmpInstr = std::next(MachineBasicBlock::iterator(CmpInstr)); @@ -4495,7 +4483,7 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, // We should check the usage if this instruction uses and updates EFLAGS. if (!UseEFLAGS && ModifyEFLAGS) { // It is safe to remove CmpInstr if EFLAGS is updated again. - FlagsMayLiveOut = false; + IsSafe = true; break; } if (!UseEFLAGS && !ModifyEFLAGS) @@ -4503,7 +4491,7 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, // EFLAGS is used by this instruction. X86::CondCode OldCC = X86::COND_INVALID; - if (MI || IsSwapped || ImmDelta != 0) { + if (MI || IsSwapped) { // We decode the condition code from opcode. if (Instr.isBranch()) OldCC = X86::getCondFromBranch(Instr); @@ -4555,60 +4543,11 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc. // We swap the condition code and synthesize the new opcode. ReplacementCC = getSwappedCondition(OldCC); - if (ReplacementCC == X86::COND_INVALID) return false; - ShouldUpdateCC = true; - } else if (ImmDelta != 0) { - unsigned BitWidth = TRI->getRegSizeInBits(*MRI->getRegClass(SrcReg)); - // Shift amount for min/max constants to adjust for 8/16/32 instruction - // sizes. - switch (OldCC) { - case X86::COND_L: // x <s (C + 1) --> x <=s C - if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue) - return false; - ReplacementCC = X86::COND_LE; - break; - case X86::COND_B: // x <u (C + 1) --> x <=u C - if (ImmDelta != 1 || CmpValue == 0) - return false; - ReplacementCC = X86::COND_BE; - break; - case X86::COND_GE: // x >=s (C + 1) --> x >s C - if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue) - return false; - ReplacementCC = X86::COND_G; - break; - case X86::COND_AE: // x >=u (C + 1) --> x >u C - if (ImmDelta != 1 || CmpValue == 0) - return false; - ReplacementCC = X86::COND_A; - break; - case X86::COND_G: // x >s (C - 1) --> x >=s C - if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue) - return false; - ReplacementCC = X86::COND_GE; - break; - case X86::COND_A: // x >u (C - 1) --> x >=u C - if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue) - return false; - ReplacementCC = X86::COND_AE; - break; - case X86::COND_LE: // x <=s (C - 1) --> x <s C - if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue) - return false; - ReplacementCC = X86::COND_L; - break; - case X86::COND_BE: // x <=u (C - 1) --> x <u C - if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue) - return false; - ReplacementCC = X86::COND_B; - break; - default: + if (ReplacementCC == X86::COND_INVALID) return false; - } - ShouldUpdateCC = true; } - if (ShouldUpdateCC && ReplacementCC != OldCC) { + if ((ShouldUpdateCC || IsSwapped) && ReplacementCC != OldCC) { // Push the MachineInstr to OpsToUpdate. // If it is safe to remove CmpInstr, the condition code of these // instructions will be modified. @@ -4616,14 +4555,14 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg, } if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) { // It is safe to remove CmpInstr if EFLAGS is updated again or killed. - FlagsMayLiveOut = false; + IsSafe = true; break; } } - // If we have to update users but EFLAGS is live-out abort, since we cannot - // easily find all of the users. - if (ShouldUpdateCC && FlagsMayLiveOut) { + // If EFLAGS is not killed nor re-defined, we should check whether it is + // live-out. If it is live-out, do not optimize. + if ((MI || IsSwapped) && !IsSafe) { for (MachineBasicBlock *Successor : CmpMBB.successors()) if (Successor->isLiveIn(X86::EFLAGS)) return false; @@ -4944,7 +4883,7 @@ bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const { case X86::SETB_C64r: return Expand2AddrUndef(MIB, get(X86::SBB64rr)); case X86::MMX_SET0: - return Expand2AddrUndef(MIB, get(X86::MMX_PXORirr)); + return Expand2AddrUndef(MIB, get(X86::MMX_PXORrr)); case X86::V_SET0: case X86::FsFLD0SS: case X86::FsFLD0SD: @@ -5217,12 +5156,12 @@ static bool hasUndefRegUpdate(unsigned Opcode, unsigned OpNum, bool ForLoadFold = false) { // Set the OpNum parameter to the first source operand. switch (Opcode) { - case X86::MMX_PUNPCKHBWirr: - case X86::MMX_PUNPCKHWDirr: - case X86::MMX_PUNPCKHDQirr: - case X86::MMX_PUNPCKLBWirr: - case X86::MMX_PUNPCKLWDirr: - case X86::MMX_PUNPCKLDQirr: + case X86::MMX_PUNPCKHBWrr: + case X86::MMX_PUNPCKHWDrr: + case X86::MMX_PUNPCKHDQrr: + case X86::MMX_PUNPCKLBWrr: + case X86::MMX_PUNPCKLWDrr: + case X86::MMX_PUNPCKLDQrr: case X86::MOVHLPSrr: case X86::PACKSSWBrr: case X86::PACKUSWBrr: diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h index 33ce55bbdb2b..537ada6222bf 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.h +++ b/llvm/lib/Target/X86/X86InstrInfo.h @@ -643,8 +643,7 @@ private: /// CMP %1, %2 and %3 = SUB %2, %1 ; IsSwapped=true bool isRedundantFlagInstr(const MachineInstr &FlagI, Register SrcReg, Register SrcReg2, int64_t ImmMask, int64_t ImmValue, - const MachineInstr &OI, bool *IsSwapped, - int64_t *ImmDelta) const; + const MachineInstr &OI, bool *IsSwapped) const; }; } // namespace llvm diff --git a/llvm/lib/Target/X86/X86InstrMMX.td b/llvm/lib/Target/X86/X86InstrMMX.td index bb3e6df3bf3e..aeecc25ddea2 100644 --- a/llvm/lib/Target/X86/X86InstrMMX.td +++ b/llvm/lib/Target/X86/X86InstrMMX.td @@ -34,14 +34,14 @@ let Constraints = "$src1 = $dst" in { multiclass MMXI_binop_rm_int<bits<8> opc, string OpcodeStr, Intrinsic IntId, X86FoldableSchedWrite sched, bit Commutable = 0, X86MemOperand OType = i64mem> { - def irr : MMXI<opc, MRMSrcReg, (outs VR64:$dst), + def rr : MMXI<opc, MRMSrcReg, (outs VR64:$dst), (ins VR64:$src1, VR64:$src2), !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), [(set VR64:$dst, (IntId VR64:$src1, VR64:$src2))]>, Sched<[sched]> { let isCommutable = Commutable; } - def irm : MMXI<opc, MRMSrcMem, (outs VR64:$dst), + def rm : MMXI<opc, MRMSrcMem, (outs VR64:$dst), (ins VR64:$src1, OType:$src2), !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), [(set VR64:$dst, (IntId VR64:$src1, (load_mmx addr:$src2)))]>, @@ -123,25 +123,25 @@ multiclass ssse3_palign_mm<string asm, Intrinsic IntId, multiclass sse12_cvt_pint<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, Intrinsic Int, X86MemOperand x86memop, PatFrag ld_frag, string asm, X86FoldableSchedWrite sched, Domain d> { - def irr : MMXPI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm, - [(set DstRC:$dst, (Int SrcRC:$src))], d>, - Sched<[sched]>; - def irm : MMXPI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm, - [(set DstRC:$dst, (Int (ld_frag addr:$src)))], d>, - Sched<[sched.Folded]>; + def rr : MMXPI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm, + [(set DstRC:$dst, (Int SrcRC:$src))], d>, + Sched<[sched]>; + def rm : MMXPI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm, + [(set DstRC:$dst, (Int (ld_frag addr:$src)))], d>, + Sched<[sched.Folded]>; } multiclass sse12_cvt_pint_3addr<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, Intrinsic Int, X86MemOperand x86memop, PatFrag ld_frag, string asm, Domain d> { - def irr : MMXPI<opc, MRMSrcReg, (outs DstRC:$dst), - (ins DstRC:$src1, SrcRC:$src2), asm, - [(set DstRC:$dst, (Int DstRC:$src1, SrcRC:$src2))], d>, - Sched<[WriteCvtI2PS]>; - def irm : MMXPI<opc, MRMSrcMem, (outs DstRC:$dst), - (ins DstRC:$src1, x86memop:$src2), asm, - [(set DstRC:$dst, (Int DstRC:$src1, (ld_frag addr:$src2)))], d>, - Sched<[WriteCvtI2PS.Folded]>; + def rr : MMXPI<opc, MRMSrcReg, (outs DstRC:$dst), + (ins DstRC:$src1, SrcRC:$src2), asm, + [(set DstRC:$dst, (Int DstRC:$src1, SrcRC:$src2))], d>, + Sched<[WriteCvtI2PS]>; + def rm : MMXPI<opc, MRMSrcMem, (outs DstRC:$dst), + (ins DstRC:$src1, x86memop:$src2), asm, + [(set DstRC:$dst, (Int DstRC:$src1, (ld_frag addr:$src2)))], d>, + Sched<[WriteCvtI2PS.Folded]>; } //===----------------------------------------------------------------------===// @@ -569,14 +569,14 @@ def : Pat<(x86mmx (bitconvert (f64 FR64:$src))), (MMX_MOVFR642Qrr FR64:$src)>; def : Pat<(x86mmx (MMX_X86movdq2q (bc_v2i64 (v4i32 (X86cvtp2Int (v4f32 VR128:$src)))))), - (MMX_CVTPS2PIirr VR128:$src)>; + (MMX_CVTPS2PIrr VR128:$src)>; def : Pat<(x86mmx (MMX_X86movdq2q (bc_v2i64 (v4i32 (X86cvttp2si (v4f32 VR128:$src)))))), - (MMX_CVTTPS2PIirr VR128:$src)>; + (MMX_CVTTPS2PIrr VR128:$src)>; def : Pat<(x86mmx (MMX_X86movdq2q (bc_v2i64 (v4i32 (X86cvtp2Int (v2f64 VR128:$src)))))), - (MMX_CVTPD2PIirr VR128:$src)>; + (MMX_CVTPD2PIrr VR128:$src)>; def : Pat<(x86mmx (MMX_X86movdq2q (bc_v2i64 (v4i32 (X86cvttp2si (v2f64 VR128:$src)))))), - (MMX_CVTTPD2PIirr VR128:$src)>; + (MMX_CVTTPD2PIrr VR128:$src)>; } diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp index c3cd634612a4..9044f10ec630 100644 --- a/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -48,6 +48,7 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Instrumentation/AddressSanitizer.h" #include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h" +#include <string> using namespace llvm; @@ -1336,235 +1337,29 @@ void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) { return; } - unsigned Reg = MI.getOperand(0).getReg().id(); + const auto &Reg = MI.getOperand(0).getReg(); ASanAccessInfo AccessInfo(MI.getOperand(1).getImm()); - MCSymbol *&Sym = - AsanMemaccessSymbols[AsanMemaccessTuple(Reg, AccessInfo.Packed)]; - if (!Sym) { - std::string Name = AccessInfo.IsWrite ? "store" : "load"; - std::string SymName = "__asan_check_" + Name + - utostr(1ULL << AccessInfo.AccessSizeIndex) + "_rn" + - utostr(Reg); - Sym = OutContext.getOrCreateSymbol(SymName); - } - - EmitAndCountInstruction( - MCInstBuilder(X86::CALL64pcrel32) - .addExpr(MCSymbolRefExpr::create(Sym, OutContext))); -} - -void X86AsmPrinter::emitAsanMemaccessPartial(Module &M, unsigned Reg, - const ASanAccessInfo &AccessInfo, - MCSubtargetInfo &STI) { - assert(AccessInfo.AccessSizeIndex == 0 || AccessInfo.AccessSizeIndex == 1 || - AccessInfo.AccessSizeIndex == 2); - assert(Reg != X86::R8); - uint64_t ShadowBase; int MappingScale; bool OrShadowOffset; - getAddressSanitizerParams( - Triple(M.getTargetTriple()), M.getDataLayout().getPointerSizeInBits(), - AccessInfo.CompileKernel, &ShadowBase, &MappingScale, &OrShadowOffset); - - OutStreamer->emitInstruction( - MCInstBuilder(X86::MOV64rr).addReg(X86::R8).addReg(X86::NoRegister + Reg), - STI); - OutStreamer->emitInstruction(MCInstBuilder(X86::SHR64ri) - .addReg(X86::R8) - .addReg(X86::R8) - .addImm(MappingScale), - STI); - if (OrShadowOffset) { - OutStreamer->emitInstruction(MCInstBuilder(X86::OR64ri32) - .addReg(X86::R8) - .addReg(X86::R8) - .addImm(ShadowBase), - STI); - OutStreamer->emitInstruction(MCInstBuilder(X86::MOV8rm) - .addReg(X86::R8B) - .addReg(X86::R8) - .addImm(1) - .addReg(X86::NoRegister) - .addImm(0) - .addReg(X86::NoRegister), - STI); - OutStreamer->emitInstruction( - MCInstBuilder(X86::TEST8rr).addReg(X86::R8B).addReg(X86::R8B), STI); - } else { - OutStreamer->emitInstruction(MCInstBuilder(X86::MOVSX32rm8) - .addReg(X86::R8D) - .addReg(X86::R8) - .addImm(1) - .addReg(X86::NoRegister) - .addImm(ShadowBase) - .addReg(X86::NoRegister), - STI); - OutStreamer->emitInstruction( - MCInstBuilder(X86::TEST32rr).addReg(X86::R8D).addReg(X86::R8D), STI); - } - MCSymbol *AdditionalCheck = OutContext.createTempSymbol(); - OutStreamer->emitInstruction( - MCInstBuilder(X86::JCC_1) - .addExpr(MCSymbolRefExpr::create(AdditionalCheck, OutContext)) - .addImm(X86::COND_NE), - STI); - MCSymbol *ReturnSym = OutContext.createTempSymbol(); - OutStreamer->emitLabel(ReturnSym); - OutStreamer->emitInstruction(MCInstBuilder(getRetOpcode(*Subtarget)), STI); - - // Shadow byte is non-zero so we need to perform additional checks. - OutStreamer->emitLabel(AdditionalCheck); - OutStreamer->emitInstruction(MCInstBuilder(X86::PUSH64r).addReg(X86::RCX), - STI); - OutStreamer->emitInstruction(MCInstBuilder(X86::MOV64rr) - .addReg(X86::RCX) - .addReg(X86::NoRegister + Reg), - STI); - const size_t Granularity = 1ULL << MappingScale; - OutStreamer->emitInstruction(MCInstBuilder(X86::AND32ri8) - .addReg(X86::NoRegister) - .addReg(X86::ECX) - .addImm(Granularity - 1), - STI); - if (AccessInfo.AccessSizeIndex == 1) { - OutStreamer->emitInstruction(MCInstBuilder(X86::ADD32ri8) - .addReg(X86::NoRegister) - .addReg(X86::ECX) - .addImm(1), - STI); - } else if (AccessInfo.AccessSizeIndex == 2) { - OutStreamer->emitInstruction(MCInstBuilder(X86::ADD32ri8) - .addReg(X86::NoRegister) - .addReg(X86::ECX) - .addImm(3), - STI); - } - - OutStreamer->emitInstruction( - MCInstBuilder(X86::CMP32rr).addReg(X86::ECX).addReg(X86::R8D).addImm(1), - STI); - OutStreamer->emitInstruction(MCInstBuilder(X86::POP64r).addReg(X86::RCX), - STI); - OutStreamer->emitInstruction( - MCInstBuilder(X86::JCC_1) - .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)) - .addImm(X86::COND_L), - STI); - - emitAsanReportError(M, Reg, AccessInfo, STI); -} - -void X86AsmPrinter::emitAsanMemaccessFull(Module &M, unsigned Reg, - const ASanAccessInfo &AccessInfo, - MCSubtargetInfo &STI) { - assert(AccessInfo.AccessSizeIndex == 3 || AccessInfo.AccessSizeIndex == 4); - assert(Reg != X86::R8); - - uint64_t ShadowBase; - int MappingScale; - bool OrShadowOffset; - getAddressSanitizerParams( - Triple(M.getTargetTriple()), M.getDataLayout().getPointerSizeInBits(), - AccessInfo.CompileKernel, &ShadowBase, &MappingScale, &OrShadowOffset); - - OutStreamer->emitInstruction( - MCInstBuilder(X86::MOV64rr).addReg(X86::R8).addReg(X86::NoRegister + Reg), - STI); - OutStreamer->emitInstruction(MCInstBuilder(X86::SHR64ri) - .addReg(X86::R8) - .addReg(X86::R8) - .addImm(MappingScale), - STI); - if (OrShadowOffset) { - OutStreamer->emitInstruction(MCInstBuilder(X86::OR64ri32) - .addReg(X86::R8) - .addReg(X86::R8) - .addImm(ShadowBase), - STI); - auto OpCode = AccessInfo.AccessSizeIndex == 3 ? X86::CMP8mi : X86::CMP16mi8; - OutStreamer->emitInstruction(MCInstBuilder(OpCode) - .addReg(X86::R8) - .addImm(1) - .addReg(X86::NoRegister) - .addImm(0) - .addReg(X86::NoRegister) - .addImm(0), - STI); - } else { - auto OpCode = AccessInfo.AccessSizeIndex == 3 ? X86::CMP8mi : X86::CMP16mi8; - OutStreamer->emitInstruction(MCInstBuilder(OpCode) - .addReg(X86::R8) - .addImm(1) - .addReg(X86::NoRegister) - .addImm(ShadowBase) - .addReg(X86::NoRegister) - .addImm(0), - STI); - } - MCSymbol *ReportCode = OutContext.createTempSymbol(); - OutStreamer->emitInstruction( - MCInstBuilder(X86::JCC_1) - .addExpr(MCSymbolRefExpr::create(ReportCode, OutContext)) - .addImm(X86::COND_NE), - STI); - MCSymbol *ReturnSym = OutContext.createTempSymbol(); - OutStreamer->emitLabel(ReturnSym); - OutStreamer->emitInstruction(MCInstBuilder(getRetOpcode(*Subtarget)), STI); - - OutStreamer->emitLabel(ReportCode); - emitAsanReportError(M, Reg, AccessInfo, STI); -} + getAddressSanitizerParams(Triple(TM.getTargetTriple()), 64, + AccessInfo.CompileKernel, &ShadowBase, + &MappingScale, &OrShadowOffset); -void X86AsmPrinter::emitAsanReportError(Module &M, unsigned Reg, - const ASanAccessInfo &AccessInfo, - MCSubtargetInfo &STI) { std::string Name = AccessInfo.IsWrite ? "store" : "load"; - MCSymbol *ReportError = OutContext.getOrCreateSymbol( - "__asan_report_" + Name + utostr(1ULL << AccessInfo.AccessSizeIndex)); - OutStreamer->emitInstruction(MCInstBuilder(X86::MOV64rr) - .addReg(X86::RDI) - .addReg(X86::NoRegister + Reg), - STI); - OutStreamer->emitInstruction( - MCInstBuilder(X86::JMP_4) - .addExpr(MCSymbolRefExpr::create(ReportError, MCSymbolRefExpr::VK_PLT, - OutContext)), - STI); -} - -void X86AsmPrinter::emitAsanMemaccessSymbols(Module &M) { - if (AsanMemaccessSymbols.empty()) - return; - - const Triple &TT = TM.getTargetTriple(); - assert(TT.isOSBinFormatELF()); - std::unique_ptr<MCSubtargetInfo> STI( - TM.getTarget().createMCSubtargetInfo(TT.str(), "", "")); - assert(STI && "Unable to create subtarget info"); - - for (auto &P : AsanMemaccessSymbols) { - MCSymbol *Sym = P.second; - OutStreamer->SwitchSection(OutContext.getELFSection( - ".text.hot", ELF::SHT_PROGBITS, - ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(), - /*IsComdat=*/true)); - - OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction); - OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak); - OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden); - OutStreamer->emitLabel(Sym); + std::string Op = OrShadowOffset ? "or" : "add"; + std::string SymName = "__asan_check_" + Name + "_" + Op + "_" + + utostr(1ULL << AccessInfo.AccessSizeIndex) + "_" + + TM.getMCRegisterInfo()->getName(Reg.asMCReg()); + if (OrShadowOffset) + report_fatal_error( + "OrShadowOffset is not supported with optimized callbacks"); - unsigned Reg = std::get<0>(P.first); - ASanAccessInfo AccessInfo(std::get<1>(P.first)); - - if (AccessInfo.AccessSizeIndex < 3) { - emitAsanMemaccessPartial(M, Reg, AccessInfo, *STI); - } else { - emitAsanMemaccessFull(M, Reg, AccessInfo, *STI); - } - } + EmitAndCountInstruction( + MCInstBuilder(X86::CALL64pcrel32) + .addExpr(MCSymbolRefExpr::create( + OutContext.getOrCreateSymbol(SymName), OutContext))); } void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI, @@ -2615,6 +2410,15 @@ void X86AsmPrinter::emitInstruction(const MachineInstr *MI) { const X86RegisterInfo *RI = MF->getSubtarget<X86Subtarget>().getRegisterInfo(); + if (MI->getOpcode() == X86::OR64rm) { + for (auto &Opd : MI->operands()) { + if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) == + "swift_async_extendedFramePointerFlags") { + ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true; + } + } + } + // Add a comment about EVEX-2-VEX compression for AVX-512 instrs that // are compressed from EVEX encoding to VEX encoding. if (TM.Options.MCOptions.ShowMCEncoding) { diff --git a/llvm/lib/Target/X86/X86RegisterInfo.td b/llvm/lib/Target/X86/X86RegisterInfo.td index d835f452b67e..1b704bcb8e08 100644 --- a/llvm/lib/Target/X86/X86RegisterInfo.td +++ b/llvm/lib/Target/X86/X86RegisterInfo.td @@ -430,11 +430,11 @@ def GR64 : RegisterClass<"X86", [i64], 64, (add RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11, RBX, R14, R15, R12, R13, RBP, RSP, RIP)>; -// GR64 - 64-bit GPRs without R8 and RIP. Could be used when emitting code for -// intrinsics, which use implict input registers. -def GR64NoR8 : RegisterClass<"X86", [i64], 64, - (add RAX, RCX, RDX, RSI, RDI, R9, R10, R11, - RBX, R14, R15, R12, R13, RBP, RSP)>; +// GR64PLTSafe - 64-bit GPRs without R10, R11, RSP and RIP. Could be used when +// emitting code for intrinsics, which use implict input registers. +def GR64PLTSafe : RegisterClass<"X86", [i64], 64, + (add RAX, RCX, RDX, RSI, RDI, R8, R9, + RBX, R14, R15, R12, R13, RBP)>; // Segment registers for use by MOV instructions (and others) that have a // segment register as one operand. Always contain a 16-bit segment diff --git a/llvm/lib/Target/X86/X86SchedBroadwell.td b/llvm/lib/Target/X86/X86SchedBroadwell.td index 2827981b7fb0..a6ff472aac6f 100644 --- a/llvm/lib/Target/X86/X86SchedBroadwell.td +++ b/llvm/lib/Target/X86/X86SchedBroadwell.td @@ -783,7 +783,7 @@ def BWWriteResGroup27 : SchedWriteRes<[BWPort1]> { let NumMicroOps = 1; let ResourceCycles = [1]; } -def: InstRW<[BWWriteResGroup27], (instrs MMX_CVTPI2PSirr)>; +def: InstRW<[BWWriteResGroup27], (instrs MMX_CVTPI2PSrr)>; def: InstRW<[BWWriteResGroup27], (instregex "P(DEP|EXT)(32|64)rr", "(V?)CVTDQ2PS(Y?)rr")>; @@ -800,9 +800,9 @@ def BWWriteResGroup33 : SchedWriteRes<[BWPort5,BWPort0156]> { let NumMicroOps = 3; let ResourceCycles = [2,1]; } -def: InstRW<[BWWriteResGroup33], (instrs MMX_PACKSSDWirr, - MMX_PACKSSWBirr, - MMX_PACKUSWBirr)>; +def: InstRW<[BWWriteResGroup33], (instrs MMX_PACKSSDWrr, + MMX_PACKSSWBrr, + MMX_PACKUSWBrr)>; def BWWriteResGroup34 : SchedWriteRes<[BWPort6,BWPort0156]> { let Latency = 3; @@ -862,9 +862,9 @@ def BWWriteResGroup42 : SchedWriteRes<[BWPort1,BWPort5]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[BWWriteResGroup42], (instrs MMX_CVTPI2PDirr)>; -def: InstRW<[BWWriteResGroup42], (instregex "MMX_CVT(T?)PD2PIirr", - "MMX_CVT(T?)PS2PIirr", +def: InstRW<[BWWriteResGroup42], (instrs MMX_CVTPI2PDrr)>; +def: InstRW<[BWWriteResGroup42], (instregex "MMX_CVT(T?)PD2PIrr", + "MMX_CVT(T?)PS2PIrr", "(V?)CVTDQ2PDrr", "(V?)CVTPD2PSrr", "(V?)CVTSD2SSrr", @@ -1086,9 +1086,9 @@ def BWWriteResGroup79 : SchedWriteRes<[BWPort5,BWPort23]> { let NumMicroOps = 3; let ResourceCycles = [2,1]; } -def: InstRW<[BWWriteResGroup79], (instrs MMX_PACKSSDWirm, - MMX_PACKSSWBirm, - MMX_PACKUSWBirm)>; +def: InstRW<[BWWriteResGroup79], (instrs MMX_PACKSSDWrm, + MMX_PACKSSWBrm, + MMX_PACKUSWBrm)>; def BWWriteResGroup80 : SchedWriteRes<[BWPort23,BWPort0156]> { let Latency = 7; @@ -1155,7 +1155,7 @@ def BWWriteResGroup91 : SchedWriteRes<[BWPort1,BWPort23]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[BWWriteResGroup91], (instrs MMX_CVTPI2PSirm, +def: InstRW<[BWWriteResGroup91], (instrs MMX_CVTPI2PSrm, CVTDQ2PSrm, VCVTDQ2PSrm)>; def: InstRW<[BWWriteResGroup91], (instregex "P(DEP|EXT)(32|64)rm")>; @@ -1236,8 +1236,8 @@ def BWWriteResGroup107 : SchedWriteRes<[BWPort1,BWPort5,BWPort23]> { def: InstRW<[BWWriteResGroup107], (instrs CVTPD2PSrm, CVTPD2DQrm, CVTTPD2DQrm, - MMX_CVTPI2PDirm)>; -def: InstRW<[BWWriteResGroup107], (instregex "MMX_CVT(T?)PD2PIirm", + MMX_CVTPI2PDrm)>; +def: InstRW<[BWWriteResGroup107], (instregex "MMX_CVT(T?)PD2PIrm", "(V?)CVTDQ2PDrm", "(V?)CVTSD2SSrm")>; diff --git a/llvm/lib/Target/X86/X86SchedHaswell.td b/llvm/lib/Target/X86/X86SchedHaswell.td index 68961d6245ab..371a9571ae39 100644 --- a/llvm/lib/Target/X86/X86SchedHaswell.td +++ b/llvm/lib/Target/X86/X86SchedHaswell.td @@ -995,7 +995,7 @@ def HWWriteResGroup12 : SchedWriteRes<[HWPort1,HWPort23]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[HWWriteResGroup12], (instrs MMX_CVTPI2PSirm)>; +def: InstRW<[HWWriteResGroup12], (instrs MMX_CVTPI2PSrm)>; def: InstRW<[HWWriteResGroup12], (instregex "P(DEP|EXT)(32|64)rm")>; def HWWriteResGroup13 : SchedWriteRes<[HWPort5,HWPort23]> { @@ -1164,9 +1164,9 @@ def HWWriteResGroup36_2 : SchedWriteRes<[HWPort5,HWPort23]> { let NumMicroOps = 3; let ResourceCycles = [2,1]; } -def: InstRW<[HWWriteResGroup36_2], (instrs MMX_PACKSSDWirm, - MMX_PACKSSWBirm, - MMX_PACKUSWBirm)>; +def: InstRW<[HWWriteResGroup36_2], (instrs MMX_PACKSSDWrm, + MMX_PACKSSWBrm, + MMX_PACKUSWBrm)>; def HWWriteResGroup37 : SchedWriteRes<[HWPort23,HWPort0156]> { let Latency = 7; @@ -1240,7 +1240,7 @@ def HWWriteResGroup50 : SchedWriteRes<[HWPort1]> { let NumMicroOps = 1; let ResourceCycles = [1]; } -def: InstRW<[HWWriteResGroup50], (instrs MMX_CVTPI2PSirr)>; +def: InstRW<[HWWriteResGroup50], (instrs MMX_CVTPI2PSrr)>; def: InstRW<[HWWriteResGroup50], (instregex "P(DEP|EXT)(32|64)rr", "(V?)CVTDQ2PS(Y?)rr")>; @@ -1285,9 +1285,9 @@ def HWWriteResGroup57 : SchedWriteRes<[HWPort5,HWPort0156]> { let NumMicroOps = 3; let ResourceCycles = [2,1]; } -def: InstRW<[HWWriteResGroup57], (instrs MMX_PACKSSDWirr, - MMX_PACKSSWBirr, - MMX_PACKUSWBirr)>; +def: InstRW<[HWWriteResGroup57], (instrs MMX_PACKSSDWrr, + MMX_PACKSSWBrr, + MMX_PACKUSWBrr)>; def HWWriteResGroup58 : SchedWriteRes<[HWPort6,HWPort0156]> { let Latency = 3; @@ -1373,11 +1373,11 @@ def HWWriteResGroup73 : SchedWriteRes<[HWPort1,HWPort5]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[HWWriteResGroup73], (instrs MMX_CVTPI2PDirr, - MMX_CVTPD2PIirr, - MMX_CVTPS2PIirr, - MMX_CVTTPD2PIirr, - MMX_CVTTPS2PIirr)>; +def: InstRW<[HWWriteResGroup73], (instrs MMX_CVTPI2PDrr, + MMX_CVTPD2PIrr, + MMX_CVTPS2PIrr, + MMX_CVTTPD2PIrr, + MMX_CVTTPS2PIrr)>; def: InstRW<[HWWriteResGroup73], (instregex "(V?)CVTDQ2PDrr", "(V?)CVTPD2PSrr", "(V?)CVTSD2SSrr", @@ -1418,8 +1418,8 @@ def HWWriteResGroup78 : SchedWriteRes<[HWPort1,HWPort5,HWPort23]> { def: InstRW<[HWWriteResGroup78], (instrs CVTPD2PSrm, CVTPD2DQrm, CVTTPD2DQrm, - MMX_CVTPD2PIirm, - MMX_CVTTPD2PIirm, + MMX_CVTPD2PIrm, + MMX_CVTTPD2PIrm, CVTDQ2PDrm, VCVTDQ2PDrm)>; @@ -1428,7 +1428,7 @@ def HWWriteResGroup78_1 : SchedWriteRes<[HWPort1,HWPort5,HWPort23]> { let NumMicroOps = 3; let ResourceCycles = [1,1,1]; } -def: InstRW<[HWWriteResGroup78_1], (instrs MMX_CVTPI2PDirm, +def: InstRW<[HWWriteResGroup78_1], (instrs MMX_CVTPI2PDrm, CVTSD2SSrm, CVTSD2SSrm_Int, VCVTSD2SSrm, VCVTSD2SSrm_Int)>; diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td index 889b9b7fa666..789de9eb5751 100644 --- a/llvm/lib/Target/X86/X86SchedIceLake.td +++ b/llvm/lib/Target/X86/X86SchedIceLake.td @@ -331,12 +331,12 @@ defm : ICXWriteResPair<WriteFLogicZ, [ICXPort05], 1, [1], 1, 7>; defm : ICXWriteResPair<WriteFTest, [ICXPort0], 2, [1], 1, 6>; // Floating point TEST instructions. defm : ICXWriteResPair<WriteFTestY, [ICXPort0], 2, [1], 1, 7>; defm : ICXWriteResPair<WriteFTestZ, [ICXPort0], 2, [1], 1, 7>; -defm : ICXWriteResPair<WriteFShuffle, [ICXPort5], 1, [1], 1, 6>; // Floating point vector shuffles. -defm : ICXWriteResPair<WriteFShuffleY, [ICXPort5], 1, [1], 1, 7>; -defm : ICXWriteResPair<WriteFShuffleZ, [ICXPort5], 1, [1], 1, 7>; -defm : ICXWriteResPair<WriteFVarShuffle, [ICXPort5], 1, [1], 1, 6>; // Floating point vector variable shuffles. -defm : ICXWriteResPair<WriteFVarShuffleY, [ICXPort5], 1, [1], 1, 7>; -defm : ICXWriteResPair<WriteFVarShuffleZ, [ICXPort5], 1, [1], 1, 7>; +defm : ICXWriteResPair<WriteFShuffle, [ICXPort15], 1, [1], 1, 6>; // Floating point vector shuffles. +defm : ICXWriteResPair<WriteFShuffleY, [ICXPort15], 1, [1], 1, 7>; +defm : ICXWriteResPair<WriteFShuffleZ, [ICXPort5], 1, [1], 1, 7>; +defm : ICXWriteResPair<WriteFVarShuffle, [ICXPort15], 1, [1], 1, 6>; // Floating point vector variable shuffles. +defm : ICXWriteResPair<WriteFVarShuffleY, [ICXPort15], 1, [1], 1, 7>; +defm : ICXWriteResPair<WriteFVarShuffleZ, [ICXPort5], 1, [1], 1, 7>; defm : ICXWriteResPair<WriteFBlend, [ICXPort015], 1, [1], 1, 6>; // Floating point vector blends. defm : ICXWriteResPair<WriteFBlendY,[ICXPort015], 1, [1], 1, 7>; defm : ICXWriteResPair<WriteFBlendZ,[ICXPort015], 1, [1], 1, 7>; @@ -388,14 +388,14 @@ defm : ICXWriteResPair<WriteVecIMulZ, [ICXPort05], 5, [1], 1, 7>; defm : ICXWriteResPair<WritePMULLD, [ICXPort01], 10, [2], 2, 6>; // Vector PMULLD. defm : ICXWriteResPair<WritePMULLDY, [ICXPort01], 10, [2], 2, 7>; defm : ICXWriteResPair<WritePMULLDZ, [ICXPort05], 10, [2], 2, 7>; -defm : ICXWriteResPair<WriteShuffle, [ICXPort5], 1, [1], 1, 5>; // Vector shuffles. -defm : ICXWriteResPair<WriteShuffleX, [ICXPort5], 1, [1], 1, 6>; -defm : ICXWriteResPair<WriteShuffleY, [ICXPort5], 1, [1], 1, 7>; -defm : ICXWriteResPair<WriteShuffleZ, [ICXPort5], 1, [1], 1, 7>; -defm : ICXWriteResPair<WriteVarShuffle, [ICXPort5], 1, [1], 1, 5>; // Vector variable shuffles. -defm : ICXWriteResPair<WriteVarShuffleX, [ICXPort5], 1, [1], 1, 6>; -defm : ICXWriteResPair<WriteVarShuffleY, [ICXPort5], 1, [1], 1, 7>; -defm : ICXWriteResPair<WriteVarShuffleZ, [ICXPort5], 1, [1], 1, 7>; +defm : ICXWriteResPair<WriteShuffle, [ICXPort5], 1, [1], 1, 5>; // Vector shuffles. +defm : ICXWriteResPair<WriteShuffleX, [ICXPort15], 1, [1], 1, 6>; +defm : ICXWriteResPair<WriteShuffleY, [ICXPort15], 1, [1], 1, 7>; +defm : ICXWriteResPair<WriteShuffleZ, [ICXPort5], 1, [1], 1, 7>; +defm : ICXWriteResPair<WriteVarShuffle, [ICXPort5], 1, [1], 1, 5>; // Vector variable shuffles. +defm : ICXWriteResPair<WriteVarShuffleX, [ICXPort15], 1, [1], 1, 6>; +defm : ICXWriteResPair<WriteVarShuffleY, [ICXPort15], 1, [1], 1, 7>; +defm : ICXWriteResPair<WriteVarShuffleZ, [ICXPort5], 1, [1], 1, 7>; defm : ICXWriteResPair<WriteBlend, [ICXPort5], 1, [1], 1, 6>; // Vector blends. defm : ICXWriteResPair<WriteBlendY,[ICXPort5], 1, [1], 1, 7>; defm : ICXWriteResPair<WriteBlendZ,[ICXPort5], 1, [1], 1, 7>; @@ -642,15 +642,15 @@ def: InstRW<[ICXWriteResGroup1], (instregex "KAND(B|D|Q|W)rr", "KXOR(B|D|Q|W)rr", "KSET0(B|D|Q|W)", // Same as KXOR "KSET1(B|D|Q|W)", // Same as KXNOR - "MMX_PADDS(B|W)irr", - "MMX_PADDUS(B|W)irr", - "MMX_PAVG(B|W)irr", - "MMX_PCMPEQ(B|D|W)irr", - "MMX_PCMPGT(B|D|W)irr", - "MMX_P(MAX|MIN)SWirr", - "MMX_P(MAX|MIN)UBirr", - "MMX_PSUBS(B|W)irr", - "MMX_PSUBUS(B|W)irr", + "MMX_PADDS(B|W)rr", + "MMX_PADDUS(B|W)rr", + "MMX_PAVG(B|W)rr", + "MMX_PCMPEQ(B|D|W)rr", + "MMX_PCMPGT(B|D|W)rr", + "MMX_P(MAX|MIN)SWrr", + "MMX_P(MAX|MIN)UBrr", + "MMX_PSUBS(B|W)rr", + "MMX_PSUBUS(B|W)rr", "VPMOVB2M(Z|Z128|Z256)rr", "VPMOVD2M(Z|Z128|Z256)rr", "VPMOVQ2M(Z|Z128|Z256)rr", @@ -663,7 +663,16 @@ def ICXWriteResGroup3 : SchedWriteRes<[ICXPort5]> { } def: InstRW<[ICXWriteResGroup3], (instregex "COM(P?)_FST0r", "KMOV(B|D|Q|W)kr", - "UCOM_F(P?)r")>; + "UCOM_F(P?)r", + "VPBROADCAST(D|Q)rr", + "(V?)INSERTPS(Z?)rr", + "(V?)MOV(HL|LH)PS(Z?)rr", + "(V?)MOVDDUP(Y|Z|Z128|Z256)?rr", + "(V?)PALIGNR(Y|Z|Z128|Z256)?rri", + "(V?)PERMIL(PD|PS)(Y|Z|Z128|Z256)?ri", + "(V?)PERMIL(PD|PS)(Y|Z|Z128|Z256)?rr", + "(V?)PACK(U|S)S(DW|WB)(Y|Z|Z128|Z256)?rr", + "(V?)UNPCK(L|H)(PD|PS)(Y|Z|Z128|Z256)?rr")>; def ICXWriteResGroup4 : SchedWriteRes<[ICXPort6]> { let Latency = 1; @@ -702,6 +711,7 @@ def: InstRW<[ICXWriteResGroup9], (instregex "VBLENDMPD(Z128|Z256)rr", "VBLENDMPS(Z128|Z256)rr", "VPADD(B|D|Q|W)(Y|Z|Z128|Z256)rr", "(V?)PADD(B|D|Q|W)rr", + "(V?)MOV(SD|SS)(Z?)rr", "VPBLENDD(Y?)rri", "VPBLENDMB(Z128|Z256)rr", "VPBLENDMD(Z128|Z256)rr", @@ -892,9 +902,9 @@ def ICXWriteResGroup41 : SchedWriteRes<[ICXPort5,ICXPort0156]> { let NumMicroOps = 3; let ResourceCycles = [2,1]; } -def: InstRW<[ICXWriteResGroup41], (instrs MMX_PACKSSDWirr, - MMX_PACKSSWBirr, - MMX_PACKUSWBirr)>; +def: InstRW<[ICXWriteResGroup41], (instrs MMX_PACKSSDWrr, + MMX_PACKSSWBrr, + MMX_PACKUSWBrr)>; def ICXWriteResGroup42 : SchedWriteRes<[ICXPort6,ICXPort0156]> { let Latency = 3; @@ -1055,8 +1065,8 @@ def ICXWriteResGroup61 : SchedWriteRes<[ICXPort5,ICXPort015]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[ICXWriteResGroup61], (instregex "MMX_CVT(T?)PD2PIirr", - "MMX_CVT(T?)PS2PIirr", +def: InstRW<[ICXWriteResGroup61], (instregex "MMX_CVT(T?)PD2PIrr", + "MMX_CVT(T?)PS2PIrr", "VCVTDQ2PDZ128rr", "VCVTPD2DQZ128rr", "(V?)CVT(T?)PD2DQrr", @@ -1162,7 +1172,7 @@ def ICXWriteResGroup72 : SchedWriteRes<[ICXPort5]> { let NumMicroOps = 2; let ResourceCycles = [2]; } -def: InstRW<[ICXWriteResGroup72], (instrs MMX_CVTPI2PSirr)>; +def: InstRW<[ICXWriteResGroup72], (instrs MMX_CVTPI2PSrr)>; def: InstRW<[ICXWriteResGroup72], (instregex "VCOMPRESSPD(Z|Z128|Z256)rr", "VCOMPRESSPS(Z|Z128|Z256)rr", "VPCOMPRESSD(Z|Z128|Z256)rr", @@ -1174,26 +1184,26 @@ def ICXWriteResGroup73 : SchedWriteRes<[ICXPort0,ICXPort23]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[ICXWriteResGroup73], (instrs MMX_PADDSBirm, - MMX_PADDSWirm, - MMX_PADDUSBirm, - MMX_PADDUSWirm, - MMX_PAVGBirm, - MMX_PAVGWirm, - MMX_PCMPEQBirm, - MMX_PCMPEQDirm, - MMX_PCMPEQWirm, - MMX_PCMPGTBirm, - MMX_PCMPGTDirm, - MMX_PCMPGTWirm, - MMX_PMAXSWirm, - MMX_PMAXUBirm, - MMX_PMINSWirm, - MMX_PMINUBirm, - MMX_PSUBSBirm, - MMX_PSUBSWirm, - MMX_PSUBUSBirm, - MMX_PSUBUSWirm)>; +def: InstRW<[ICXWriteResGroup73], (instrs MMX_PADDSBrm, + MMX_PADDSWrm, + MMX_PADDUSBrm, + MMX_PADDUSWrm, + MMX_PAVGBrm, + MMX_PAVGWrm, + MMX_PCMPEQBrm, + MMX_PCMPEQDrm, + MMX_PCMPEQWrm, + MMX_PCMPGTBrm, + MMX_PCMPGTDrm, + MMX_PCMPGTWrm, + MMX_PMAXSWrm, + MMX_PMAXUBrm, + MMX_PMINSWrm, + MMX_PMINUBrm, + MMX_PSUBSBrm, + MMX_PSUBSWrm, + MMX_PSUBUSBrm, + MMX_PSUBUSWrm)>; def ICXWriteResGroup76 : SchedWriteRes<[ICXPort6,ICXPort23]> { let Latency = 6; @@ -1295,20 +1305,14 @@ def ICXWriteResGroup92 : SchedWriteRes<[ICXPort5,ICXPort23]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[ICXWriteResGroup92], (instregex "VMOVSDZrm(b?)", - "VMOVSSZrm(b?)")>; - -def ICXWriteResGroup92a : SchedWriteRes<[ICXPort5,ICXPort23]> { - let Latency = 6; - let NumMicroOps = 2; - let ResourceCycles = [1,1]; -} -def: InstRW<[ICXWriteResGroup92a], (instregex "(V?)PMOV(SX|ZX)BDrm", - "(V?)PMOV(SX|ZX)BQrm", - "(V?)PMOV(SX|ZX)BWrm", - "(V?)PMOV(SX|ZX)DQrm", - "(V?)PMOV(SX|ZX)WDrm", - "(V?)PMOV(SX|ZX)WQrm")>; +def: InstRW<[ICXWriteResGroup92], (instregex "VMOV(SD|SS)Zrm(b?)", + "VPBROADCAST(B|W)(Z128)?rm", + "(V?)INSERTPS(Z?)rm", + "(V?)PALIGNR(Z128)?rmi", + "(V?)PERMIL(PD|PS)(Z128)?m(b?)i", + "(V?)PERMIL(PD|PS)(Z128)?rm", + "(V?)PACK(U|S)S(DW|WB)(Z128)?rm", + "(V?)UNPCK(L|H)(PD|PS)(Z128)?rm")>; def ICXWriteResGroup93 : SchedWriteRes<[ICXPort5,ICXPort015]> { let Latency = 7; @@ -1391,9 +1395,9 @@ def ICXWriteResGroup96 : SchedWriteRes<[ICXPort5,ICXPort23]> { let NumMicroOps = 3; let ResourceCycles = [2,1]; } -def: InstRW<[ICXWriteResGroup96], (instrs MMX_PACKSSDWirm, - MMX_PACKSSWBirm, - MMX_PACKUSWBirm)>; +def: InstRW<[ICXWriteResGroup96], (instrs MMX_PACKSSDWrm, + MMX_PACKSSWBrm, + MMX_PACKUSWBrm)>; def ICXWriteResGroup97 : SchedWriteRes<[ICXPort5,ICXPort015]> { let Latency = 7; @@ -1546,7 +1550,12 @@ def ICXWriteResGroup119 : SchedWriteRes<[ICXPort5,ICXPort23]> { } def: InstRW<[ICXWriteResGroup119], (instregex "FCOM(P?)(32|64)m", "VPBROADCASTB(Z|Z256)rm(b?)", - "VPBROADCASTW(Z|Z256)rm(b?)")>; + "VPBROADCASTW(Z|Z256)rm(b?)", + "(V?)PALIGNR(Y|Z|Z256)rmi", + "(V?)PERMIL(PD|PS)(Y|Z|Z256)m(b?)i", + "(V?)PERMIL(PD|PS)(Y|Z|Z256)rm", + "(V?)PACK(U|S)S(DW|WB)(Y|Z|Z256)rm", + "(V?)UNPCK(L|H)(PD|PS)(Y|Z|Z256)rm")>; def: InstRW<[ICXWriteResGroup119], (instrs VPBROADCASTBYrm, VPBROADCASTWYrm, VPMOVSXBDYrm, @@ -1683,7 +1692,7 @@ def ICXWriteResGroup135 : SchedWriteRes<[ICXPort0,ICXPort23]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[ICXWriteResGroup135], (instrs MMX_CVTPI2PSirm)>; +def: InstRW<[ICXWriteResGroup135], (instrs MMX_CVTPI2PSrm)>; def ICXWriteResGroup136 : SchedWriteRes<[ICXPort5,ICXPort23]> { let Latency = 9; @@ -1709,19 +1718,7 @@ def: InstRW<[ICXWriteResGroup136], (instregex "VALIGN(D|Q)Z128rm(b?)i", "VPMAXSQZ128rm(b?)", "VPMAXUQZ128rm(b?)", "VPMINSQZ128rm(b?)", - "VPMINUQZ128rm(b?)", - "VPMOVSXBDZ128rm(b?)", - "VPMOVSXBQZ128rm(b?)", - "VPMOVSXBWZ128rm(b?)", - "VPMOVSXDQZ128rm(b?)", - "VPMOVSXWDZ128rm(b?)", - "VPMOVSXWQZ128rm(b?)", - "VPMOVZXBDZ128rm(b?)", - "VPMOVZXBQZ128rm(b?)", - "VPMOVZXBWZ128rm(b?)", - "VPMOVZXDQZ128rm(b?)", - "VPMOVZXWDZ128rm(b?)", - "VPMOVZXWQZ128rm(b?)")>; + "VPMINUQZ128rm(b?)")>; def ICXWriteResGroup136_2 : SchedWriteRes<[ICXPort5,ICXPort23]> { let Latency = 10; @@ -1753,7 +1750,7 @@ def ICXWriteResGroup137 : SchedWriteRes<[ICXPort23,ICXPort015]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[ICXWriteResGroup137], (instregex "MMX_CVT(T?)PS2PIirm", +def: InstRW<[ICXWriteResGroup137], (instregex "MMX_CVT(T?)PS2PIrm", "(V?)CVTPS2PDrm")>; def ICXWriteResGroup143 : SchedWriteRes<[ICXPort5,ICXPort01,ICXPort23]> { @@ -1950,8 +1947,8 @@ def ICXWriteResGroup166 : SchedWriteRes<[ICXPort5,ICXPort23,ICXPort015]> { def: InstRW<[ICXWriteResGroup166], (instrs CVTPD2PSrm, CVTPD2DQrm, CVTTPD2DQrm, - MMX_CVTPD2PIirm, - MMX_CVTTPD2PIirm)>; + MMX_CVTPD2PIrm, + MMX_CVTTPD2PIrm)>; def ICXWriteResGroup167 : SchedWriteRes<[ICXPort5,ICXPort23,ICXPort015]> { let Latency = 11; diff --git a/llvm/lib/Target/X86/X86SchedSandyBridge.td b/llvm/lib/Target/X86/X86SchedSandyBridge.td index c8d7b0f72c1c..af5c0540deb5 100644 --- a/llvm/lib/Target/X86/X86SchedSandyBridge.td +++ b/llvm/lib/Target/X86/X86SchedSandyBridge.td @@ -623,7 +623,7 @@ def SBWriteResGroup5 : SchedWriteRes<[SBPort15]> { def: InstRW<[SBWriteResGroup5], (instrs MMX_PABSBrr, MMX_PABSDrr, MMX_PABSWrr, - MMX_PADDQirr, + MMX_PADDQrr, MMX_PALIGNRrri, MMX_PSIGNBrr, MMX_PSIGNDrr, @@ -870,7 +870,7 @@ def SBWriteResGroup59 : SchedWriteRes<[SBPort23,SBPort15]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[SBWriteResGroup59], (instrs MMX_PADDQirm)>; +def: InstRW<[SBWriteResGroup59], (instrs MMX_PADDQrm)>; def SBWriteResGroup62 : SchedWriteRes<[SBPort5,SBPort23]> { let Latency = 7; diff --git a/llvm/lib/Target/X86/X86SchedSkylakeClient.td b/llvm/lib/Target/X86/X86SchedSkylakeClient.td index 7d3229c3b023..b3c13c72dd01 100644 --- a/llvm/lib/Target/X86/X86SchedSkylakeClient.td +++ b/llvm/lib/Target/X86/X86SchedSkylakeClient.td @@ -624,15 +624,15 @@ def SKLWriteResGroup1 : SchedWriteRes<[SKLPort0]> { let NumMicroOps = 1; let ResourceCycles = [1]; } -def: InstRW<[SKLWriteResGroup1], (instregex "MMX_PADDS(B|W)irr", - "MMX_PADDUS(B|W)irr", - "MMX_PAVG(B|W)irr", - "MMX_PCMPEQ(B|D|W)irr", - "MMX_PCMPGT(B|D|W)irr", - "MMX_P(MAX|MIN)SWirr", - "MMX_P(MAX|MIN)UBirr", - "MMX_PSUBS(B|W)irr", - "MMX_PSUBUS(B|W)irr")>; +def: InstRW<[SKLWriteResGroup1], (instregex "MMX_PADDS(B|W)rr", + "MMX_PADDUS(B|W)rr", + "MMX_PAVG(B|W)rr", + "MMX_PCMPEQ(B|D|W)rr", + "MMX_PCMPGT(B|D|W)rr", + "MMX_P(MAX|MIN)SWrr", + "MMX_P(MAX|MIN)UBrr", + "MMX_PSUBS(B|W)rr", + "MMX_PSUBUS(B|W)rr")>; def SKLWriteResGroup3 : SchedWriteRes<[SKLPort5]> { let Latency = 1; @@ -815,9 +815,9 @@ def SKLWriteResGroup39 : SchedWriteRes<[SKLPort5,SKLPort0156]> { let NumMicroOps = 3; let ResourceCycles = [2,1]; } -def: InstRW<[SKLWriteResGroup39], (instrs MMX_PACKSSDWirr, - MMX_PACKSSWBirr, - MMX_PACKUSWBirr)>; +def: InstRW<[SKLWriteResGroup39], (instrs MMX_PACKSSDWrr, + MMX_PACKSSWBrr, + MMX_PACKUSWBrr)>; def SKLWriteResGroup40 : SchedWriteRes<[SKLPort6,SKLPort0156]> { let Latency = 3; @@ -927,7 +927,7 @@ def SKLWriteResGroup59 : SchedWriteRes<[SKLPort0,SKLPort5]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[SKLWriteResGroup59], (instrs MMX_CVTPI2PDirr, +def: InstRW<[SKLWriteResGroup59], (instrs MMX_CVTPI2PDrr, CVTDQ2PDrr, VCVTDQ2PDrr)>; @@ -936,8 +936,8 @@ def SKLWriteResGroup60 : SchedWriteRes<[SKLPort5,SKLPort015]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[SKLWriteResGroup60], (instregex "MMX_CVT(T?)PD2PIirr", - "MMX_CVT(T?)PS2PIirr", +def: InstRW<[SKLWriteResGroup60], (instregex "MMX_CVT(T?)PD2PIrr", + "MMX_CVT(T?)PS2PIrr", "(V?)CVT(T?)PD2DQrr", "(V?)CVTPD2PSrr", "(V?)CVTPS2PDrr", @@ -984,33 +984,33 @@ def SKLWriteResGroup68 : SchedWriteRes<[SKLPort0]> { let NumMicroOps = 2; let ResourceCycles = [2]; } -def: InstRW<[SKLWriteResGroup68], (instrs MMX_CVTPI2PSirr)>; +def: InstRW<[SKLWriteResGroup68], (instrs MMX_CVTPI2PSrr)>; def SKLWriteResGroup69 : SchedWriteRes<[SKLPort0,SKLPort23]> { let Latency = 6; let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[SKLWriteResGroup69], (instrs MMX_PADDSBirm, - MMX_PADDSWirm, - MMX_PADDUSBirm, - MMX_PADDUSWirm, - MMX_PAVGBirm, - MMX_PAVGWirm, - MMX_PCMPEQBirm, - MMX_PCMPEQDirm, - MMX_PCMPEQWirm, - MMX_PCMPGTBirm, - MMX_PCMPGTDirm, - MMX_PCMPGTWirm, - MMX_PMAXSWirm, - MMX_PMAXUBirm, - MMX_PMINSWirm, - MMX_PMINUBirm, - MMX_PSUBSBirm, - MMX_PSUBSWirm, - MMX_PSUBUSBirm, - MMX_PSUBUSWirm)>; +def: InstRW<[SKLWriteResGroup69], (instrs MMX_PADDSBrm, + MMX_PADDSWrm, + MMX_PADDUSBrm, + MMX_PADDUSWrm, + MMX_PAVGBrm, + MMX_PAVGWrm, + MMX_PCMPEQBrm, + MMX_PCMPEQDrm, + MMX_PCMPEQWrm, + MMX_PCMPGTBrm, + MMX_PCMPGTDrm, + MMX_PCMPGTWrm, + MMX_PMAXSWrm, + MMX_PMAXUBrm, + MMX_PMINSWrm, + MMX_PMINUBrm, + MMX_PSUBSBrm, + MMX_PSUBSWrm, + MMX_PSUBUSBrm, + MMX_PSUBUSWrm)>; def SKLWriteResGroup70 : SchedWriteRes<[SKLPort0,SKLPort01]> { let Latency = 6; @@ -1144,9 +1144,9 @@ def SKLWriteResGroup92 : SchedWriteRes<[SKLPort5,SKLPort23]> { let NumMicroOps = 3; let ResourceCycles = [2,1]; } -def: InstRW<[SKLWriteResGroup92], (instrs MMX_PACKSSDWirm, - MMX_PACKSSWBirm, - MMX_PACKUSWBirm)>; +def: InstRW<[SKLWriteResGroup92], (instrs MMX_PACKSSDWrm, + MMX_PACKSSWBrm, + MMX_PACKUSWBrm)>; def SKLWriteResGroup94 : SchedWriteRes<[SKLPort23,SKLPort0156]> { let Latency = 7; @@ -1283,7 +1283,7 @@ def SKLWriteResGroup120 : SchedWriteRes<[SKLPort0,SKLPort23]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[SKLWriteResGroup120], (instrs MMX_CVTPI2PSirm)>; +def: InstRW<[SKLWriteResGroup120], (instrs MMX_CVTPI2PSrm)>; def SKLWriteResGroup121 : SchedWriteRes<[SKLPort5,SKLPort23]> { let Latency = 9; @@ -1302,7 +1302,7 @@ def SKLWriteResGroup123 : SchedWriteRes<[SKLPort23,SKLPort01]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[SKLWriteResGroup123], (instregex "MMX_CVT(T?)PS2PIirm", +def: InstRW<[SKLWriteResGroup123], (instregex "MMX_CVT(T?)PS2PIrm", "(V?)CVTPS2PDrm")>; def SKLWriteResGroup128 : SchedWriteRes<[SKLPort5,SKLPort01,SKLPort23]> { @@ -1345,7 +1345,7 @@ def SKLWriteResGroup138 : SchedWriteRes<[SKLPort0,SKLPort5,SKLPort23]> { let NumMicroOps = 3; let ResourceCycles = [1,1,1]; } -def: InstRW<[SKLWriteResGroup138], (instrs MMX_CVTPI2PDirm)>; +def: InstRW<[SKLWriteResGroup138], (instrs MMX_CVTPI2PDrm)>; def SKLWriteResGroup139 : SchedWriteRes<[SKLPort5,SKLPort23,SKLPort01]> { let Latency = 10; @@ -1425,8 +1425,8 @@ def SKLWriteResGroup152 : SchedWriteRes<[SKLPort5,SKLPort23,SKLPort01]> { def: InstRW<[SKLWriteResGroup152], (instrs CVTPD2PSrm, CVTPD2DQrm, CVTTPD2DQrm, - MMX_CVTPD2PIirm, - MMX_CVTTPD2PIirm)>; + MMX_CVTPD2PIrm, + MMX_CVTTPD2PIrm)>; def SKLWriteResGroup154 : SchedWriteRes<[SKLPort1,SKLPort06,SKLPort0156]> { let Latency = 11; diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td index 1d8417aef41e..74f9da158353 100644 --- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td +++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td @@ -634,15 +634,15 @@ def: InstRW<[SKXWriteResGroup1], (instregex "KAND(B|D|Q|W)rr", "KXOR(B|D|Q|W)rr", "KSET0(B|D|Q|W)", // Same as KXOR "KSET1(B|D|Q|W)", // Same as KXNOR - "MMX_PADDS(B|W)irr", - "MMX_PADDUS(B|W)irr", - "MMX_PAVG(B|W)irr", - "MMX_PCMPEQ(B|D|W)irr", - "MMX_PCMPGT(B|D|W)irr", - "MMX_P(MAX|MIN)SWirr", - "MMX_P(MAX|MIN)UBirr", - "MMX_PSUBS(B|W)irr", - "MMX_PSUBUS(B|W)irr", + "MMX_PADDS(B|W)rr", + "MMX_PADDUS(B|W)rr", + "MMX_PAVG(B|W)rr", + "MMX_PCMPEQ(B|D|W)rr", + "MMX_PCMPGT(B|D|W)rr", + "MMX_P(MAX|MIN)SWrr", + "MMX_P(MAX|MIN)UBrr", + "MMX_PSUBS(B|W)rr", + "MMX_PSUBUS(B|W)rr", "VPMOVB2M(Z|Z128|Z256)rr", "VPMOVD2M(Z|Z128|Z256)rr", "VPMOVQ2M(Z|Z128|Z256)rr", @@ -884,9 +884,9 @@ def SKXWriteResGroup41 : SchedWriteRes<[SKXPort5,SKXPort0156]> { let NumMicroOps = 3; let ResourceCycles = [2,1]; } -def: InstRW<[SKXWriteResGroup41], (instrs MMX_PACKSSDWirr, - MMX_PACKSSWBirr, - MMX_PACKUSWBirr)>; +def: InstRW<[SKXWriteResGroup41], (instrs MMX_PACKSSDWrr, + MMX_PACKSSWBrr, + MMX_PACKUSWBrr)>; def SKXWriteResGroup42 : SchedWriteRes<[SKXPort6,SKXPort0156]> { let Latency = 3; @@ -1047,8 +1047,8 @@ def SKXWriteResGroup61 : SchedWriteRes<[SKXPort5,SKXPort015]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[SKXWriteResGroup61], (instregex "MMX_CVT(T?)PD2PIirr", - "MMX_CVT(T?)PS2PIirr", +def: InstRW<[SKXWriteResGroup61], (instregex "MMX_CVT(T?)PD2PIrr", + "MMX_CVT(T?)PS2PIrr", "VCVTDQ2PDZ128rr", "VCVTPD2DQZ128rr", "(V?)CVT(T?)PD2DQrr", @@ -1154,7 +1154,7 @@ def SKXWriteResGroup72 : SchedWriteRes<[SKXPort5]> { let NumMicroOps = 2; let ResourceCycles = [2]; } -def: InstRW<[SKXWriteResGroup72], (instrs MMX_CVTPI2PSirr)>; +def: InstRW<[SKXWriteResGroup72], (instrs MMX_CVTPI2PSrr)>; def: InstRW<[SKXWriteResGroup72], (instregex "VCOMPRESSPD(Z|Z128|Z256)rr", "VCOMPRESSPS(Z|Z128|Z256)rr", "VPCOMPRESSD(Z|Z128|Z256)rr", @@ -1166,26 +1166,26 @@ def SKXWriteResGroup73 : SchedWriteRes<[SKXPort0,SKXPort23]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[SKXWriteResGroup73], (instrs MMX_PADDSBirm, - MMX_PADDSWirm, - MMX_PADDUSBirm, - MMX_PADDUSWirm, - MMX_PAVGBirm, - MMX_PAVGWirm, - MMX_PCMPEQBirm, - MMX_PCMPEQDirm, - MMX_PCMPEQWirm, - MMX_PCMPGTBirm, - MMX_PCMPGTDirm, - MMX_PCMPGTWirm, - MMX_PMAXSWirm, - MMX_PMAXUBirm, - MMX_PMINSWirm, - MMX_PMINUBirm, - MMX_PSUBSBirm, - MMX_PSUBSWirm, - MMX_PSUBUSBirm, - MMX_PSUBUSWirm)>; +def: InstRW<[SKXWriteResGroup73], (instrs MMX_PADDSBrm, + MMX_PADDSWrm, + MMX_PADDUSBrm, + MMX_PADDUSWrm, + MMX_PAVGBrm, + MMX_PAVGWrm, + MMX_PCMPEQBrm, + MMX_PCMPEQDrm, + MMX_PCMPEQWrm, + MMX_PCMPGTBrm, + MMX_PCMPGTDrm, + MMX_PCMPGTWrm, + MMX_PMAXSWrm, + MMX_PMAXUBrm, + MMX_PMINSWrm, + MMX_PMINUBrm, + MMX_PSUBSBrm, + MMX_PSUBSWrm, + MMX_PSUBUSBrm, + MMX_PSUBUSWrm)>; def SKXWriteResGroup76 : SchedWriteRes<[SKXPort6,SKXPort23]> { let Latency = 6; @@ -1383,9 +1383,9 @@ def SKXWriteResGroup96 : SchedWriteRes<[SKXPort5,SKXPort23]> { let NumMicroOps = 3; let ResourceCycles = [2,1]; } -def: InstRW<[SKXWriteResGroup96], (instrs MMX_PACKSSDWirm, - MMX_PACKSSWBirm, - MMX_PACKUSWBirm)>; +def: InstRW<[SKXWriteResGroup96], (instrs MMX_PACKSSDWrm, + MMX_PACKSSWBrm, + MMX_PACKUSWBrm)>; def SKXWriteResGroup97 : SchedWriteRes<[SKXPort5,SKXPort015]> { let Latency = 7; @@ -1675,7 +1675,7 @@ def SKXWriteResGroup135 : SchedWriteRes<[SKXPort0,SKXPort23]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[SKXWriteResGroup135], (instrs MMX_CVTPI2PSirm)>; +def: InstRW<[SKXWriteResGroup135], (instrs MMX_CVTPI2PSrm)>; def SKXWriteResGroup136 : SchedWriteRes<[SKXPort5,SKXPort23]> { let Latency = 9; @@ -1701,19 +1701,7 @@ def: InstRW<[SKXWriteResGroup136], (instregex "VALIGN(D|Q)Z128rm(b?)i", "VPMAXSQZ128rm(b?)", "VPMAXUQZ128rm(b?)", "VPMINSQZ128rm(b?)", - "VPMINUQZ128rm(b?)", - "VPMOVSXBDZ128rm(b?)", - "VPMOVSXBQZ128rm(b?)", - "VPMOVSXBWZ128rm(b?)", - "VPMOVSXDQZ128rm(b?)", - "VPMOVSXWDZ128rm(b?)", - "VPMOVSXWQZ128rm(b?)", - "VPMOVZXBDZ128rm(b?)", - "VPMOVZXBQZ128rm(b?)", - "VPMOVZXBWZ128rm(b?)", - "VPMOVZXDQZ128rm(b?)", - "VPMOVZXWDZ128rm(b?)", - "VPMOVZXWQZ128rm(b?)")>; + "VPMINUQZ128rm(b?)")>; def SKXWriteResGroup136_2 : SchedWriteRes<[SKXPort5,SKXPort23]> { let Latency = 10; @@ -1745,7 +1733,7 @@ def SKXWriteResGroup137 : SchedWriteRes<[SKXPort23,SKXPort015]> { let NumMicroOps = 2; let ResourceCycles = [1,1]; } -def: InstRW<[SKXWriteResGroup137], (instregex "MMX_CVT(T?)PS2PIirm", +def: InstRW<[SKXWriteResGroup137], (instregex "MMX_CVT(T?)PS2PIrm", "(V?)CVTPS2PDrm")>; def SKXWriteResGroup143 : SchedWriteRes<[SKXPort5,SKXPort01,SKXPort23]> { @@ -1942,8 +1930,8 @@ def SKXWriteResGroup166 : SchedWriteRes<[SKXPort5,SKXPort23,SKXPort015]> { def: InstRW<[SKXWriteResGroup166], (instrs CVTPD2PSrm, CVTPD2DQrm, CVTTPD2DQrm, - MMX_CVTPD2PIirm, - MMX_CVTTPD2PIirm)>; + MMX_CVTPD2PIrm, + MMX_CVTTPD2PIrm)>; def SKXWriteResGroup167 : SchedWriteRes<[SKXPort5,SKXPort23,SKXPort015]> { let Latency = 11; diff --git a/llvm/lib/Target/X86/X86ScheduleAtom.td b/llvm/lib/Target/X86/X86ScheduleAtom.td index 6fd98280f560..0fedfc01092c 100644 --- a/llvm/lib/Target/X86/X86ScheduleAtom.td +++ b/llvm/lib/Target/X86/X86ScheduleAtom.td @@ -320,30 +320,30 @@ defm : X86WriteResPairUnsupported<WriteFVarShuffle256>; // Conversions. //////////////////////////////////////////////////////////////////////////////// -defm : AtomWriteResPair<WriteCvtSS2I, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 8, 9, [7,7], [6,6]>; -defm : AtomWriteResPair<WriteCvtPS2I, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [5,5], [6,6]>; +defm : AtomWriteResPair<WriteCvtSS2I, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 8, 9, [8,8], [9,9], 3, 4>; +defm : AtomWriteResPair<WriteCvtPS2I, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [6,6], [7,7], 3, 4>; defm : X86WriteResPairUnsupported<WriteCvtPS2IY>; defm : X86WriteResPairUnsupported<WriteCvtPS2IZ>; -defm : AtomWriteResPair<WriteCvtSD2I, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 8, 9, [7,7], [6,6]>; -defm : AtomWriteResPair<WriteCvtPD2I, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 7, 8, [6,6], [7,7]>; +defm : AtomWriteResPair<WriteCvtSD2I, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 8, 9, [8,8],[10,10], 3, 4>; +defm : AtomWriteResPair<WriteCvtPD2I, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 7, 8, [7,7], [8,8], 4, 5>; defm : X86WriteResPairUnsupported<WriteCvtPD2IY>; defm : X86WriteResPairUnsupported<WriteCvtPD2IZ>; -defm : AtomWriteResPair<WriteCvtI2SS, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [5,5], [6,6]>; -defm : AtomWriteResPair<WriteCvtI2PS, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [5,5], [6,6]>; +defm : AtomWriteResPair<WriteCvtI2SS, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [6,6], [6,6], 3, 1>; +defm : AtomWriteResPair<WriteCvtI2PS, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [6,6], [7,7], 3, 4>; defm : X86WriteResPairUnsupported<WriteCvtI2PSY>; defm : X86WriteResPairUnsupported<WriteCvtI2PSZ>; -defm : AtomWriteResPair<WriteCvtI2SD, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [5,5], [6,6]>; -defm : AtomWriteResPair<WriteCvtI2PD, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 7, 8, [6,6], [7,7]>; +defm : AtomWriteResPair<WriteCvtI2SD, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [6,6], [7,7], 3, 3>; +defm : AtomWriteResPair<WriteCvtI2PD, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 7, 8, [6,6], [7,7], 3, 4>; defm : X86WriteResPairUnsupported<WriteCvtI2PDY>; defm : X86WriteResPairUnsupported<WriteCvtI2PDZ>; -defm : AtomWriteResPair<WriteCvtSS2SD, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [5,5], [6,6]>; -defm : AtomWriteResPair<WriteCvtPS2PD, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 7, 8, [6,6], [7,7]>; +defm : AtomWriteResPair<WriteCvtSS2SD, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [6,6], [7,7], 3, 4>; +defm : AtomWriteResPair<WriteCvtPS2PD, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 7, 8, [6,6], [7,7], 4, 5>; defm : X86WriteResPairUnsupported<WriteCvtPS2PDY>; defm : X86WriteResPairUnsupported<WriteCvtPS2PDZ>; -defm : AtomWriteResPair<WriteCvtSD2SS, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 6, 7, [5,5], [6,6]>; -defm : AtomWriteResPair<WriteCvtPD2PS, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 7, 8, [6,6], [7,7]>; +defm : AtomWriteResPair<WriteCvtSD2SS, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 10, 11,[10,10],[12,12], 3, 4>; +defm : AtomWriteResPair<WriteCvtPD2PS, [AtomPort0,AtomPort1], [AtomPort0,AtomPort1], 11, 12,[11,11],[12,12], 4, 5>; defm : X86WriteResPairUnsupported<WriteCvtPD2PSY>; defm : X86WriteResPairUnsupported<WriteCvtPD2PSZ>; @@ -525,8 +525,8 @@ def AtomWrite1_5 : SchedWriteRes<[AtomPort1]> { let Latency = 5; let ResourceCycles = [5]; } -def : InstRW<[AtomWrite1_5], (instrs MMX_CVTPI2PSirr, MMX_CVTPI2PSirm, - MMX_CVTPS2PIirr, MMX_CVTTPS2PIirr)>; +def : InstRW<[AtomWrite1_5], (instrs MMX_CVTPI2PSrr, MMX_CVTPI2PSrm, + MMX_CVTPS2PIrr, MMX_CVTTPS2PIrr)>; // Port0 and Port1 def AtomWrite0_1_1 : SchedWriteRes<[AtomPort0, AtomPort1]> { @@ -547,9 +547,43 @@ def AtomWrite0_1_5 : SchedWriteRes<[AtomPort0, AtomPort1]> { let Latency = 5; let ResourceCycles = [5, 5]; } -def : InstRW<[AtomWrite0_1_5], (instrs MMX_CVTPS2PIirm, MMX_CVTTPS2PIirm)>; +def : InstRW<[AtomWrite0_1_5], (instrs MMX_CVTPS2PIrm, MMX_CVTTPS2PIrm)>; def : InstRW<[AtomWrite0_1_5], (instregex "ILD_F(16|32|64)")>; +def AtomWrite0_1_7 : SchedWriteRes<[AtomPort0,AtomPort1]> { + let Latency = 7; + let ResourceCycles = [6,6]; +} +def : InstRW<[AtomWrite0_1_7], (instregex "CVTSI642SDrm(_Int)?")>; + +def AtomWrite0_1_7_4 : SchedWriteRes<[AtomPort0,AtomPort1]> { + let Latency = 7; + let ResourceCycles = [8,8]; + let NumMicroOps = 4; +} +def : InstRW<[AtomWrite0_1_7_4], (instregex "CVTSI642SSrr(_Int)?")>; + +def AtomWrite0_1_8_4 : SchedWriteRes<[AtomPort0,AtomPort1]> { + let Latency = 8; + let ResourceCycles = [8,8]; + let NumMicroOps = 4; +} +def : InstRW<[AtomWrite0_1_7_4], (instregex "CVTSI642SSrm(_Int)?")>; + +def AtomWrite0_1_9 : SchedWriteRes<[AtomPort0,AtomPort1]> { + let Latency = 9; + let ResourceCycles = [9,9]; + let NumMicroOps = 4; +} +def : InstRW<[AtomWrite0_1_9], (instregex "CVT(T)?SS2SI64rr(_Int)?")>; + +def AtomWrite0_1_10 : SchedWriteRes<[AtomPort0,AtomPort1]> { + let Latency = 10; + let ResourceCycles = [11,11]; + let NumMicroOps = 5; +} +def : InstRW<[AtomWrite0_1_10], (instregex "CVT(T)?SS2SI64rm(_Int)?")>; + // Port0 or Port1 def AtomWrite01_1 : SchedWriteRes<[AtomPort01]> { let Latency = 1; @@ -570,7 +604,7 @@ def : InstRW<[AtomWrite01_2], (instrs LEAVE, LEAVE64, POP16r, SCASB, SCASL, SCASQ, SCASW)>; def : InstRW<[AtomWrite01_2], (instregex "PUSH(CS|DS|ES|FS|GS|SS)(16|32|64)", "(ST|ISTT)_F(P)?(16|32|64)?(m|rr)", - "MMX_P(ADD|SUB)Qirr", + "MMX_P(ADD|SUB)Qrr", "MOV(S|Z)X16rr8", "MOV(UPS|UPD|DQU)mr", "MASKMOVDQU(64)?", @@ -589,7 +623,7 @@ def : InstRW<[AtomWrite01_3], (instregex "XADD(8|16|32|64)rm", "XCHG(8|16|32|64)rm", "PH(ADD|SUB)Drr", "MOV(S|Z)X16rm8", - "MMX_P(ADD|SUB)Qirm", + "MMX_P(ADD|SUB)Qrm", "MOV(UPS|UPD|DQU)rm", "P(ADD|SUB)Qrm")>; @@ -647,15 +681,13 @@ def : InstRW<[AtomWrite01_9], (instrs POPA16, POPA32, SHLD64mri8, SHRD64mri8, SHLD64rri8, SHRD64rri8, CMPXCHG8rr)>; -def : InstRW<[AtomWrite01_9], (instregex "(U)?COM_FI", "TST_F", - "CVT(T)?SS2SI64rr(_Int)?")>; +def : InstRW<[AtomWrite01_9], (instregex "(U)?COM_FI", "TST_F")>; def AtomWrite01_10 : SchedWriteRes<[AtomPort01]> { let Latency = 10; let ResourceCycles = [10]; } def : SchedAlias<WriteFLDC, AtomWrite01_10>; -def : InstRW<[AtomWrite01_10], (instregex "CVT(T)?SS2SI64rm(_Int)?")>; def AtomWrite01_11 : SchedWriteRes<[AtomPort01]> { let Latency = 11; diff --git a/llvm/lib/Target/X86/X86ScheduleBdVer2.td b/llvm/lib/Target/X86/X86ScheduleBdVer2.td index 4c16b5b52b1d..0f6f24f9f1fe 100644 --- a/llvm/lib/Target/X86/X86ScheduleBdVer2.td +++ b/llvm/lib/Target/X86/X86ScheduleBdVer2.td @@ -1008,11 +1008,11 @@ defm : PdWriteResXMMPair<WriteCvtPD2I, [PdFPU0, PdFPCVT, PdFPSTO], 8, defm : PdWriteResYMMPair<WriteCvtPD2IY, [PdFPU0, PdFPCVT, PdFPSTO, PdFPFMA], 8, [1, 2, 1, 1], 4>; defm : X86WriteResPairUnsupported<WriteCvtPD2IZ>; -def PdWriteMMX_CVTTPD2PIirr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> { +def PdWriteMMX_CVTTPD2PIrr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> { let Latency = 6; let NumMicroOps = 2; } -def : InstRW<[PdWriteMMX_CVTTPD2PIirr], (instrs MMX_CVTTPD2PIirr)>; +def : InstRW<[PdWriteMMX_CVTTPD2PIrr], (instrs MMX_CVTTPD2PIrr)>; // FIXME: f+3 ST, LD+STC latency defm : PdWriteResXMMPair<WriteCvtI2SS, [PdFPU0, PdFPCVT, PdFPSTO], 4, [], 2>; @@ -1048,18 +1048,18 @@ defm : PdWriteResXMMPair<WriteCvtPD2PS, [PdFPU0, PdFPCVT, PdFPSTO], 8, defm : PdWriteResYMMPair<WriteCvtPD2PSY, [PdFPU0, PdFPCVT, PdFPSTO, PdFPFMA], 8, [1, 2, 1, 1], 4>; defm : X86WriteResPairUnsupported<WriteCvtPD2PSZ>; -def PdWriteMMX_CVTPD2PIirrMMX_CVTPI2PDirr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> { +def PdWriteMMX_CVTPD2PIrrMMX_CVTPI2PDrr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> { let Latency = 6; let NumMicroOps = 2; } -def : InstRW<[PdWriteMMX_CVTPD2PIirrMMX_CVTPI2PDirr], (instrs MMX_CVTPD2PIirr, - MMX_CVTPI2PDirr)>; +def : InstRW<[PdWriteMMX_CVTPD2PIrrMMX_CVTPI2PDrr], (instrs MMX_CVTPD2PIrr, + MMX_CVTPI2PDrr)>; -def PdWriteMMX_CVTPI2PSirr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> { +def PdWriteMMX_CVTPI2PSrr : SchedWriteRes<[PdFPU0, PdFPCVT, PdFPSTO]> { let Latency = 4; let NumMicroOps = 2; } -def : InstRW<[PdWriteMMX_CVTPI2PSirr], (instrs MMX_CVTPI2PSirr)>; +def : InstRW<[PdWriteMMX_CVTPI2PSrr], (instrs MMX_CVTPI2PSrr)>; defm : PdWriteResXMMPair<WriteCvtPH2PS, [PdFPU0, PdFPCVT, PdFPSTO], 8, [1, 2, 1], 2, 1>; defm : PdWriteResYMMPair<WriteCvtPH2PSY, [PdFPU0, PdFPCVT, PdFPSTO], 8, [1, 2, 1], 4, 3>; @@ -1365,7 +1365,7 @@ def PdWriteVZeroIdiomLogic : SchedWriteVariant<[ SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [PdWriteZeroLatency]>, SchedVar<MCSchedPredicate<TruePred>, [WriteVecLogic]> ]>; -def : InstRW<[PdWriteVZeroIdiomLogic], (instrs MMX_PXORirr, MMX_PANDNirr)>; +def : InstRW<[PdWriteVZeroIdiomLogic], (instrs MMX_PXORrr, MMX_PANDNrr)>; def PdWriteVZeroIdiomLogicX : SchedWriteVariant<[ SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [PdWriteZeroLatency]>, @@ -1378,11 +1378,11 @@ def PdWriteVZeroIdiomALU : SchedWriteVariant<[ SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [PdWriteZeroLatency]>, SchedVar<MCSchedPredicate<TruePred>, [WriteVecALU]> ]>; -def : InstRW<[PdWriteVZeroIdiomALU], (instrs MMX_PSUBBirr, MMX_PSUBDirr, - MMX_PSUBQirr, MMX_PSUBWirr, - MMX_PCMPGTBirr, - MMX_PCMPGTDirr, - MMX_PCMPGTWirr)>; +def : InstRW<[PdWriteVZeroIdiomALU], (instrs MMX_PSUBBrr, MMX_PSUBDrr, + MMX_PSUBQrr, MMX_PSUBWrr, + MMX_PCMPGTBrr, + MMX_PCMPGTDrr, + MMX_PCMPGTWrr)>; def PdWriteVZeroIdiomALUX : SchedWriteVariant<[ SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [PdWriteZeroLatency]>, @@ -1408,10 +1408,10 @@ def : IsZeroIdiomFunction<[ // MMX Zero-idioms. DepBreakingClass<[ - MMX_PXORirr, MMX_PANDNirr, MMX_PSUBBirr, - MMX_PSUBDirr, MMX_PSUBQirr, MMX_PSUBWirr, - MMX_PSUBSBirr, MMX_PSUBSWirr, MMX_PSUBUSBirr, MMX_PSUBUSWirr, - MMX_PCMPGTBirr, MMX_PCMPGTDirr, MMX_PCMPGTWirr + MMX_PXORrr, MMX_PANDNrr, MMX_PSUBBrr, + MMX_PSUBDrr, MMX_PSUBQrr, MMX_PSUBWrr, + MMX_PSUBSBrr, MMX_PSUBSWrr, MMX_PSUBUSBrr, MMX_PSUBUSWrr, + MMX_PCMPGTBrr, MMX_PCMPGTDrr, MMX_PCMPGTWrr ], ZeroIdiomPredicate>, // SSE Zero-idioms. @@ -1449,7 +1449,7 @@ def : IsDepBreakingFunction<[ // MMX DepBreakingClass<[ - MMX_PCMPEQBirr, MMX_PCMPEQDirr, MMX_PCMPEQWirr + MMX_PCMPEQBrr, MMX_PCMPEQDrr, MMX_PCMPEQWrr ], ZeroIdiomPredicate>, // SSE diff --git a/llvm/lib/Target/X86/X86ScheduleBtVer2.td b/llvm/lib/Target/X86/X86ScheduleBtVer2.td index 68ebaa244acf..a070da34cab5 100644 --- a/llvm/lib/Target/X86/X86ScheduleBtVer2.td +++ b/llvm/lib/Target/X86/X86ScheduleBtVer2.td @@ -888,7 +888,7 @@ def JWriteVZeroIdiomLogic : SchedWriteVariant<[ SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [JWriteZeroLatency]>, SchedVar<NoSchedPred, [WriteVecLogic]> ]>; -def : InstRW<[JWriteVZeroIdiomLogic], (instrs MMX_PXORirr, MMX_PANDNirr)>; +def : InstRW<[JWriteVZeroIdiomLogic], (instrs MMX_PXORrr, MMX_PANDNrr)>; def JWriteVZeroIdiomLogicX : SchedWriteVariant<[ SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [JWriteZeroLatency]>, @@ -901,12 +901,12 @@ def JWriteVZeroIdiomALU : SchedWriteVariant<[ SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [JWriteZeroLatency]>, SchedVar<NoSchedPred, [WriteVecALU]> ]>; -def : InstRW<[JWriteVZeroIdiomALU], (instrs MMX_PSUBBirr, MMX_PSUBDirr, - MMX_PSUBQirr, MMX_PSUBWirr, - MMX_PSUBSBirr, MMX_PSUBSWirr, - MMX_PSUBUSBirr, MMX_PSUBUSWirr, - MMX_PCMPGTBirr, MMX_PCMPGTDirr, - MMX_PCMPGTWirr)>; +def : InstRW<[JWriteVZeroIdiomALU], (instrs MMX_PSUBBrr, MMX_PSUBDrr, + MMX_PSUBQrr, MMX_PSUBWrr, + MMX_PSUBSBrr, MMX_PSUBSWrr, + MMX_PSUBUSBrr, MMX_PSUBUSWrr, + MMX_PCMPGTBrr, MMX_PCMPGTDrr, + MMX_PCMPGTWrr)>; def JWriteVZeroIdiomALUX : SchedWriteVariant<[ SchedVar<MCSchedPredicate<ZeroIdiomPredicate>, [JWriteZeroLatency]>, @@ -974,10 +974,10 @@ def : IsZeroIdiomFunction<[ // MMX Zero-idioms. DepBreakingClass<[ - MMX_PXORirr, MMX_PANDNirr, MMX_PSUBBirr, - MMX_PSUBDirr, MMX_PSUBQirr, MMX_PSUBWirr, - MMX_PSUBSBirr, MMX_PSUBSWirr, MMX_PSUBUSBirr, MMX_PSUBUSWirr, - MMX_PCMPGTBirr, MMX_PCMPGTDirr, MMX_PCMPGTWirr + MMX_PXORrr, MMX_PANDNrr, MMX_PSUBBrr, + MMX_PSUBDrr, MMX_PSUBQrr, MMX_PSUBWrr, + MMX_PSUBSBrr, MMX_PSUBSWrr, MMX_PSUBUSBrr, MMX_PSUBUSWrr, + MMX_PCMPGTBrr, MMX_PCMPGTDrr, MMX_PCMPGTWrr ], ZeroIdiomPredicate>, // SSE Zero-idioms. @@ -1017,7 +1017,7 @@ def : IsDepBreakingFunction<[ // MMX DepBreakingClass<[ - MMX_PCMPEQBirr, MMX_PCMPEQDirr, MMX_PCMPEQWirr + MMX_PCMPEQBrr, MMX_PCMPEQDrr, MMX_PCMPEQWrr ], ZeroIdiomPredicate>, // SSE diff --git a/llvm/lib/Target/X86/X86ScheduleSLM.td b/llvm/lib/Target/X86/X86ScheduleSLM.td index 5af9835f75a7..36e5b55a4194 100644 --- a/llvm/lib/Target/X86/X86ScheduleSLM.td +++ b/llvm/lib/Target/X86/X86ScheduleSLM.td @@ -467,8 +467,8 @@ def SLMWriteResGroup1rr : SchedWriteRes<[SLM_FPC_RSV01]> { let NumMicroOps = 2; let ResourceCycles = [8]; } -def: InstRW<[SLMWriteResGroup1rr], (instrs MMX_PADDQirr, PADDQrr, - MMX_PSUBQirr, PSUBQrr, +def: InstRW<[SLMWriteResGroup1rr], (instrs MMX_PADDQrr, PADDQrr, + MMX_PSUBQrr, PSUBQrr, PCMPEQQrr)>; def SLMWriteResGroup1rm : SchedWriteRes<[SLM_MEC_RSV,SLM_FPC_RSV01]> { @@ -476,8 +476,8 @@ def SLMWriteResGroup1rm : SchedWriteRes<[SLM_MEC_RSV,SLM_FPC_RSV01]> { let NumMicroOps = 3; let ResourceCycles = [1,8]; } -def: InstRW<[SLMWriteResGroup1rm], (instrs MMX_PADDQirm, PADDQrm, - MMX_PSUBQirm, PSUBQrm, +def: InstRW<[SLMWriteResGroup1rm], (instrs MMX_PADDQrm, PADDQrm, + MMX_PSUBQrm, PSUBQrm, PCMPEQQrm)>; } // SchedModel diff --git a/llvm/lib/Target/X86/X86ScheduleZnver1.td b/llvm/lib/Target/X86/X86ScheduleZnver1.td index 8e30e5e10ca8..4343e1ed45d1 100644 --- a/llvm/lib/Target/X86/X86ScheduleZnver1.td +++ b/llvm/lib/Target/X86/X86ScheduleZnver1.td @@ -1000,12 +1000,12 @@ def ZnWriteFPU12Ym : SchedWriteRes<[ZnAGU, ZnFPU12]> { let NumMicroOps = 2; } -def : InstRW<[ZnWriteFPU12], (instrs MMX_PACKSSDWirr, - MMX_PACKSSWBirr, - MMX_PACKUSWBirr)>; -def : InstRW<[ZnWriteFPU12m], (instrs MMX_PACKSSDWirm, - MMX_PACKSSWBirm, - MMX_PACKUSWBirm)>; +def : InstRW<[ZnWriteFPU12], (instrs MMX_PACKSSDWrr, + MMX_PACKSSWBrr, + MMX_PACKUSWBrr)>; +def : InstRW<[ZnWriteFPU12m], (instrs MMX_PACKSSDWrm, + MMX_PACKSSWBrm, + MMX_PACKUSWBrm)>; def ZnWriteFPU013 : SchedWriteRes<[ZnFPU013]> ; def ZnWriteFPU013Y : SchedWriteRes<[ZnFPU013]> { @@ -1305,15 +1305,15 @@ def ZnWriteCVTPS2PIr: SchedWriteRes<[ZnFPU3]> { } // CVT(T)PS2PI. // mm,x. -def : InstRW<[ZnWriteCVTPS2PIr], (instregex "MMX_CVT(T?)PS2PIirr")>; +def : InstRW<[ZnWriteCVTPS2PIr], (instregex "MMX_CVT(T?)PS2PIrr")>; // CVTPI2PD. // x,mm. -def : InstRW<[ZnWriteCVTPS2PDr], (instrs MMX_CVTPI2PDirr)>; +def : InstRW<[ZnWriteCVTPS2PDr], (instrs MMX_CVTPI2PDrr)>; // CVT(T)PD2PI. // mm,x. -def : InstRW<[ZnWriteCVTPS2PIr], (instregex "MMX_CVT(T?)PD2PIirr")>; +def : InstRW<[ZnWriteCVTPS2PIr], (instregex "MMX_CVT(T?)PD2PIrr")>; def ZnWriteCVSTSI2SSr: SchedWriteRes<[ZnFPU3]> { let Latency = 5; diff --git a/llvm/lib/Target/X86/X86ScheduleZnver2.td b/llvm/lib/Target/X86/X86ScheduleZnver2.td index a83c89e2f28a..96d2837880c7 100644 --- a/llvm/lib/Target/X86/X86ScheduleZnver2.td +++ b/llvm/lib/Target/X86/X86ScheduleZnver2.td @@ -1012,12 +1012,12 @@ def Zn2WriteFPU12Ym : SchedWriteRes<[Zn2AGU, Zn2FPU12]> { let NumMicroOps = 2; } -def : InstRW<[Zn2WriteFPU12], (instrs MMX_PACKSSDWirr, - MMX_PACKSSWBirr, - MMX_PACKUSWBirr)>; -def : InstRW<[Zn2WriteFPU12m], (instrs MMX_PACKSSDWirm, - MMX_PACKSSWBirm, - MMX_PACKUSWBirm)>; +def : InstRW<[Zn2WriteFPU12], (instrs MMX_PACKSSDWrr, + MMX_PACKSSWBrr, + MMX_PACKUSWBrr)>; +def : InstRW<[Zn2WriteFPU12m], (instrs MMX_PACKSSDWrm, + MMX_PACKSSWBrm, + MMX_PACKUSWBrm)>; def Zn2WriteFPU013 : SchedWriteRes<[Zn2FPU013]> ; def Zn2WriteFPU013Y : SchedWriteRes<[Zn2FPU013]> ; @@ -1304,15 +1304,15 @@ def Zn2WriteCVTPS2PIr: SchedWriteRes<[Zn2FPU3]> { } // CVT(T)PS2PI. // mm,x. -def : InstRW<[Zn2WriteCVTPS2PIr], (instregex "MMX_CVT(T?)PS2PIirr")>; +def : InstRW<[Zn2WriteCVTPS2PIr], (instregex "MMX_CVT(T?)PS2PIrr")>; // CVTPI2PD. // x,mm. -def : InstRW<[Zn2WriteCVTPS2PDr], (instrs MMX_CVTPI2PDirr)>; +def : InstRW<[Zn2WriteCVTPS2PDr], (instrs MMX_CVTPI2PDrr)>; // CVT(T)PD2PI. // mm,x. -def : InstRW<[Zn2WriteCVTPS2PIr], (instregex "MMX_CVT(T?)PD2PIirr")>; +def : InstRW<[Zn2WriteCVTPS2PIr], (instregex "MMX_CVT(T?)PD2PIrr")>; def Zn2WriteCVSTSI2SSr: SchedWriteRes<[Zn2FPU3]> { let Latency = 3; diff --git a/llvm/lib/Target/X86/X86ScheduleZnver3.td b/llvm/lib/Target/X86/X86ScheduleZnver3.td index be07c069aae1..f4e03ac11f0b 100644 --- a/llvm/lib/Target/X86/X86ScheduleZnver3.td +++ b/llvm/lib/Target/X86/X86ScheduleZnver3.td @@ -1075,9 +1075,9 @@ def Zn3WriteVecALUXMMX : SchedWriteRes<[Zn3FPVAdd01]> { } def : InstRW<[Zn3WriteVecALUXMMX], (instrs MMX_PABSBrr, MMX_PABSDrr, MMX_PABSWrr, MMX_PSIGNBrr, MMX_PSIGNDrr, MMX_PSIGNWrr, - MMX_PADDSBirr, MMX_PADDSWirr, MMX_PADDUSBirr, MMX_PADDUSWirr, - MMX_PAVGBirr, MMX_PAVGWirr, - MMX_PSUBSBirr, MMX_PSUBSWirr, MMX_PSUBUSBirr, MMX_PSUBUSWirr)>; + MMX_PADDSBrr, MMX_PADDSWrr, MMX_PADDUSBrr, MMX_PADDUSWrr, + MMX_PAVGBrr, MMX_PAVGWrr, + MMX_PSUBSBrr, MMX_PSUBSWrr, MMX_PSUBUSBrr, MMX_PSUBUSWrr)>; defm : Zn3WriteResYMMPair<WriteVecALUY, [Zn3FPVAdd0123], 1, [1], 1>; // Vector integer ALU op, no logicals (YMM). @@ -1161,7 +1161,7 @@ def Zn3WriteCvtPD2IMMX : SchedWriteRes<[Zn3FPFCvt01]> { let ResourceCycles = [2]; let NumMicroOps = 2; } -def : InstRW<[Zn3WriteCvtPD2IMMX], (instrs MMX_CVTPD2PIirm, MMX_CVTTPD2PIirm, MMX_CVTPD2PIirr, MMX_CVTTPD2PIirr)>; +def : InstRW<[Zn3WriteCvtPD2IMMX], (instrs MMX_CVTPD2PIrm, MMX_CVTTPD2PIrm, MMX_CVTPD2PIrr, MMX_CVTTPD2PIrr)>; defm : Zn3WriteResXMMPair<WriteCvtSS2I, [Zn3FPFCvt01], 2, [2], 2>; // Float -> Integer. @@ -1179,7 +1179,7 @@ def Zn3WriteCvtI2PDMMX : SchedWriteRes<[Zn3FPFCvt01]> { let ResourceCycles = [6]; let NumMicroOps = 2; } -def : InstRW<[Zn3WriteCvtI2PDMMX], (instrs MMX_CVTPI2PDirm, MMX_CVTPI2PDirr)>; +def : InstRW<[Zn3WriteCvtI2PDMMX], (instrs MMX_CVTPI2PDrm, MMX_CVTPI2PDrr)>; defm : Zn3WriteResXMMPair<WriteCvtI2SS, [Zn3FPFCvt01], 3, [2], 2, /*LoadUOps=*/-1>; // Integer -> Float. defm : Zn3WriteResXMMPair<WriteCvtI2PS, [Zn3FPFCvt01], 3, [1], 1>; // Integer -> Float (XMM). @@ -1191,7 +1191,7 @@ def Zn3WriteCvtI2PSMMX : SchedWriteRes<[Zn3FPFCvt01]> { let ResourceCycles = [1]; let NumMicroOps = 2; } -def : InstRW<[Zn3WriteCvtI2PSMMX], (instrs MMX_CVTPI2PSirr)>; +def : InstRW<[Zn3WriteCvtI2PSMMX], (instrs MMX_CVTPI2PSrr)>; defm : Zn3WriteResXMMPair<WriteCvtSS2SD, [Zn3FPFCvt01], 3, [1], 1>; // Float -> Double size conversion. defm : Zn3WriteResXMMPair<WriteCvtPS2PD, [Zn3FPFCvt01], 3, [1], 1>; // Float -> Double size conversion (XMM). @@ -1621,7 +1621,7 @@ def : IsDepBreakingFunction<[ // MMX DepBreakingClass<[ - MMX_PCMPEQBirr, MMX_PCMPEQWirr, MMX_PCMPEQDirr + MMX_PCMPEQBrr, MMX_PCMPEQWrr, MMX_PCMPEQDrr ], ZeroIdiomPredicate>, // SSE diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h index 9da54dc2e9b7..5d773f0c57df 100644 --- a/llvm/lib/Target/X86/X86Subtarget.h +++ b/llvm/lib/Target/X86/X86Subtarget.h @@ -958,8 +958,7 @@ public: // extended frames should be flagged as present. const Triple &TT = getTargetTriple(); - unsigned Major, Minor, Micro; - TT.getOSVersion(Major, Minor, Micro); + unsigned Major = TT.getOSVersion().getMajor(); switch(TT.getOS()) { default: return false; diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 336985f3bf9d..78bc5519c23f 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -588,6 +588,18 @@ void X86PassConfig::addPreEmitPass2() { // Insert pseudo probe annotation for callsite profiling addPass(createPseudoProbeInserter()); + + // On Darwin platforms, BLR_RVMARKER pseudo instructions are lowered to + // bundles. + if (TT.isOSDarwin()) + addPass(createUnpackMachineBundles([](const MachineFunction &MF) { + // Only run bundle expansion if there are relevant ObjC runtime functions + // present in the module. + const Function &F = MF.getFunction(); + const Module *M = F.getParent(); + return M->getFunction("objc_retainAutoreleasedReturnValue") || + M->getFunction("objc_unsafeClaimAutoreleasedReturnValue"); + })); } bool X86PassConfig::addPostFastRegAllocRewrite() { diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 869762b35196..d8cd7311a0d5 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -236,47 +236,50 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost( } } - if ((ISD == ISD::MUL || ISD == ISD::SDIV || ISD == ISD::SREM || - ISD == ISD::UDIV || ISD == ISD::UREM) && + // Vector multiply by pow2 will be simplified to shifts. + if (ISD == ISD::MUL && (Op2Info == TargetTransformInfo::OK_UniformConstantValue || Op2Info == TargetTransformInfo::OK_NonUniformConstantValue) && - Opd2PropInfo == TargetTransformInfo::OP_PowerOf2) { - // Vector multiply by pow2 will be simplified to shifts. - if (ISD == ISD::MUL) { - InstructionCost Cost = getArithmeticInstrCost( - Instruction::Shl, Ty, CostKind, Op1Info, Op2Info, - TargetTransformInfo::OP_None, TargetTransformInfo::OP_None); - return Cost; - } - - if (ISD == ISD::SDIV || ISD == ISD::SREM) { - // On X86, vector signed division by constants power-of-two are - // normally expanded to the sequence SRA + SRL + ADD + SRA. - // The OperandValue properties may not be the same as that of the previous - // operation; conservatively assume OP_None. - InstructionCost Cost = - 2 * getArithmeticInstrCost(Instruction::AShr, Ty, CostKind, Op1Info, - Op2Info, TargetTransformInfo::OP_None, - TargetTransformInfo::OP_None); - Cost += getArithmeticInstrCost(Instruction::LShr, Ty, CostKind, Op1Info, - Op2Info, TargetTransformInfo::OP_None, - TargetTransformInfo::OP_None); - Cost += getArithmeticInstrCost(Instruction::Add, Ty, CostKind, Op1Info, - Op2Info, TargetTransformInfo::OP_None, - TargetTransformInfo::OP_None); + Opd2PropInfo == TargetTransformInfo::OP_PowerOf2) + return getArithmeticInstrCost(Instruction::Shl, Ty, CostKind, Op1Info, + Op2Info, TargetTransformInfo::OP_None, + TargetTransformInfo::OP_None); - if (ISD == ISD::SREM) { - // For SREM: (X % C) is the equivalent of (X - (X/C)*C) - Cost += getArithmeticInstrCost(Instruction::Mul, Ty, CostKind, Op1Info, - Op2Info); - Cost += getArithmeticInstrCost(Instruction::Sub, Ty, CostKind, Op1Info, - Op2Info); - } + // On X86, vector signed division by constants power-of-two are + // normally expanded to the sequence SRA + SRL + ADD + SRA. + // The OperandValue properties may not be the same as that of the previous + // operation; conservatively assume OP_None. + if ((ISD == ISD::SDIV || ISD == ISD::SREM) && + (Op2Info == TargetTransformInfo::OK_UniformConstantValue || + Op2Info == TargetTransformInfo::OK_NonUniformConstantValue) && + Opd2PropInfo == TargetTransformInfo::OP_PowerOf2) { + InstructionCost Cost = + 2 * getArithmeticInstrCost(Instruction::AShr, Ty, CostKind, Op1Info, + Op2Info, TargetTransformInfo::OP_None, + TargetTransformInfo::OP_None); + Cost += getArithmeticInstrCost(Instruction::LShr, Ty, CostKind, Op1Info, + Op2Info, TargetTransformInfo::OP_None, + TargetTransformInfo::OP_None); + Cost += getArithmeticInstrCost(Instruction::Add, Ty, CostKind, Op1Info, + Op2Info, TargetTransformInfo::OP_None, + TargetTransformInfo::OP_None); - return Cost; + if (ISD == ISD::SREM) { + // For SREM: (X % C) is the equivalent of (X - (X/C)*C) + Cost += getArithmeticInstrCost(Instruction::Mul, Ty, CostKind, Op1Info, + Op2Info); + Cost += getArithmeticInstrCost(Instruction::Sub, Ty, CostKind, Op1Info, + Op2Info); } - // Vector unsigned division/remainder will be simplified to shifts/masks. + return Cost; + } + + // Vector unsigned division/remainder will be simplified to shifts/masks. + if ((ISD == ISD::UDIV || ISD == ISD::UREM) && + (Op2Info == TargetTransformInfo::OK_UniformConstantValue || + Op2Info == TargetTransformInfo::OK_NonUniformConstantValue) && + Opd2PropInfo == TargetTransformInfo::OP_PowerOf2) { if (ISD == ISD::UDIV) return getArithmeticInstrCost(Instruction::LShr, Ty, CostKind, Op1Info, Op2Info, TargetTransformInfo::OP_None, @@ -660,6 +663,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost( { ISD::MUL, MVT::v8i32, 1 }, // pmulld (Skylake from agner.org) { ISD::MUL, MVT::v4i32, 1 }, // pmulld (Skylake from agner.org) { ISD::MUL, MVT::v8i64, 6 }, // 3*pmuludq/3*shift/2*add + { ISD::MUL, MVT::i64, 1 }, // Skylake from http://www.agner.org/ { ISD::FNEG, MVT::v8f64, 1 }, // Skylake from http://www.agner.org/ { ISD::FADD, MVT::v8f64, 1 }, // Skylake from http://www.agner.org/ @@ -5188,10 +5192,10 @@ bool X86TTIImpl::areInlineCompatible(const Function *Caller, return (RealCallerBits & RealCalleeBits) == RealCalleeBits; } -bool X86TTIImpl::areFunctionArgsABICompatible( - const Function *Caller, const Function *Callee, - SmallPtrSetImpl<Argument *> &Args) const { - if (!BaseT::areFunctionArgsABICompatible(Caller, Callee, Args)) +bool X86TTIImpl::areTypesABICompatible(const Function *Caller, + const Function *Callee, + const ArrayRef<Type *> &Types) const { + if (!BaseT::areTypesABICompatible(Caller, Callee, Types)) return false; // If we get here, we know the target features match. If one function @@ -5206,13 +5210,8 @@ bool X86TTIImpl::areFunctionArgsABICompatible( // Consider the arguments compatible if they aren't vectors or aggregates. // FIXME: Look at the size of vectors. // FIXME: Look at the element types of aggregates to see if there are vectors. - // FIXME: The API of this function seems intended to allow arguments - // to be removed from the set, but the caller doesn't check if the set - // becomes empty so that may not work in practice. - return llvm::none_of(Args, [](Argument *A) { - auto *EltTy = cast<PointerType>(A->getType())->getElementType(); - return EltTy->isVectorTy() || EltTy->isAggregateType(); - }); + return llvm::none_of(Types, + [](Type *T) { return T->isVectorTy() || T->isAggregateType(); }); } X86TTIImpl::TTI::MemCmpExpansionOptions diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h index c53424ec0026..11e9cb09c7d5 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -234,9 +234,8 @@ public: bool isFCmpOrdCheaperThanFCmpZero(Type *Ty); bool areInlineCompatible(const Function *Caller, const Function *Callee) const; - bool areFunctionArgsABICompatible(const Function *Caller, - const Function *Callee, - SmallPtrSetImpl<Argument *> &Args) const; + bool areTypesABICompatible(const Function *Caller, const Function *Callee, + const ArrayRef<Type *> &Type) const; TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const; bool prefersVectorizedAddressing() const; |