diff options
Diffstat (limited to 'llvm/lib/Target/X86/X86ISelLowering.cpp')
| -rw-r--r-- | llvm/lib/Target/X86/X86ISelLowering.cpp | 5157 |
1 files changed, 3335 insertions, 1822 deletions
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 450927aaf5cc..0dd20235aa3c 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -35,6 +35,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" +#include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetLowering.h" @@ -76,6 +77,14 @@ static cl::opt<int> ExperimentalPrefLoopAlignment( " of the loop header PC will be 0)."), cl::Hidden); +static cl::opt<int> ExperimentalPrefInnermostLoopAlignment( + "x86-experimental-pref-innermost-loop-alignment", cl::init(4), + cl::desc( + "Sets the preferable loop alignment for experiments (as log2 bytes) " + "for innermost loops only. If specified, this option overrides " + "alignment set by x86-experimental-pref-loop-alignment."), + cl::Hidden); + static cl::opt<bool> MulConstantOptimization( "mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " @@ -135,19 +144,24 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, addBypassSlowDiv(64, 32); } - if (Subtarget.isTargetWindowsMSVC() || - Subtarget.isTargetWindowsItanium()) { - // Setup Windows compiler runtime calls. - setLibcallName(RTLIB::SDIV_I64, "_alldiv"); - setLibcallName(RTLIB::UDIV_I64, "_aulldiv"); - setLibcallName(RTLIB::SREM_I64, "_allrem"); - setLibcallName(RTLIB::UREM_I64, "_aullrem"); - setLibcallName(RTLIB::MUL_I64, "_allmul"); - setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::X86_StdCall); - setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::X86_StdCall); - setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::X86_StdCall); - setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::X86_StdCall); - setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::X86_StdCall); + // Setup Windows compiler runtime calls. + if (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()) { + static const struct { + const RTLIB::Libcall Op; + const char * const Name; + const CallingConv::ID CC; + } LibraryCalls[] = { + { RTLIB::SDIV_I64, "_alldiv", CallingConv::X86_StdCall }, + { RTLIB::UDIV_I64, "_aulldiv", CallingConv::X86_StdCall }, + { RTLIB::SREM_I64, "_allrem", CallingConv::X86_StdCall }, + { RTLIB::UREM_I64, "_aullrem", CallingConv::X86_StdCall }, + { RTLIB::MUL_I64, "_allmul", CallingConv::X86_StdCall }, + }; + + for (const auto &LC : LibraryCalls) { + setLibcallName(LC.Op, LC.Name); + setLibcallCallingConv(LC.Op, LC.CC); + } } if (Subtarget.getTargetTriple().isOSMSVCRT()) { @@ -193,8 +207,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, if (Subtarget.hasCMov()) { setOperationAction(ISD::ABS , MVT::i16 , Custom); setOperationAction(ISD::ABS , MVT::i32 , Custom); + if (Subtarget.is64Bit()) + setOperationAction(ISD::ABS , MVT::i64 , Custom); } - setOperationAction(ISD::ABS , MVT::i64 , Custom); // Funnel shifts. for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) { @@ -278,6 +293,19 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, } } + if (Subtarget.hasSSE2()) { + // Custom lowering for saturating float to int conversions. + // We handle promotion to larger result types manually. + for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) { + setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom); + setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom); + } + if (Subtarget.is64Bit()) { + setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom); + setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom); + } + } + // Handle address space casts between mixed sized pointers. setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom); setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom); @@ -384,6 +412,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setTruncStoreAction(MVT::f80, MVT::f16, Expand); setTruncStoreAction(MVT::f128, MVT::f16, Expand); + setOperationAction(ISD::PARITY, MVT::i8, Custom); if (Subtarget.hasPOPCNT()) { setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32); } else { @@ -394,6 +423,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::CTPOP , MVT::i64 , Expand); else setOperationAction(ISD::CTPOP , MVT::i64 , Custom); + + setOperationAction(ISD::PARITY, MVT::i16, Custom); + setOperationAction(ISD::PARITY, MVT::i32, Custom); + if (Subtarget.is64Bit()) + setOperationAction(ISD::PARITY, MVT::i64, Custom); } setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); @@ -487,6 +521,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::TRAP, MVT::Other, Legal); setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); + setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal); // VASTART needs to be custom lowered to use the VarArgsFrameIndex setOperationAction(ISD::VASTART , MVT::Other, Custom); @@ -915,9 +950,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal); setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal); setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal); - setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom); setOperationAction(ISD::USUBSAT, MVT::v4i32, Custom); - setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom); setOperationAction(ISD::USUBSAT, MVT::v2i64, Custom); setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); @@ -1081,6 +1114,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::STRICT_FRINT, RoundedTy, Legal); setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal); setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy, Legal); + setOperationAction(ISD::FROUNDEVEN, RoundedTy, Legal); + setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy, Legal); setOperationAction(ISD::FROUND, RoundedTy, Custom); } @@ -1094,6 +1129,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::UMIN, MVT::v8i16, Legal); setOperationAction(ISD::UMIN, MVT::v4i32, Legal); + setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom); + // FIXME: Do we need to handle scalar-to-vector here? setOperationAction(ISD::MUL, MVT::v4i32, Legal); @@ -1134,6 +1171,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, } } + if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) { + setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom); + } + if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) { for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) @@ -1175,6 +1216,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::STRICT_FRINT, VT, Legal); setOperationAction(ISD::FNEARBYINT, VT, Legal); setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal); + setOperationAction(ISD::FROUNDEVEN, VT, Legal); + setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal); setOperationAction(ISD::FROUND, VT, Custom); @@ -1302,6 +1345,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom); setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom); setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom); + setOperationAction(ISD::UADDSAT, MVT::v8i32, Custom); + setOperationAction(ISD::USUBSAT, MVT::v8i32, Custom); + setOperationAction(ISD::UADDSAT, MVT::v4i64, Custom); + setOperationAction(ISD::USUBSAT, MVT::v4i64, Custom); for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) { setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom); @@ -1560,6 +1607,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::STRICT_FRINT, VT, Legal); setOperationAction(ISD::FNEARBYINT, VT, Legal); setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal); + setOperationAction(ISD::FROUNDEVEN, VT, Legal); + setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal); setOperationAction(ISD::FROUND, VT, Custom); } @@ -1688,10 +1737,17 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, } if (Subtarget.hasVBMI2()) { - for (auto VT : { MVT::v32i16, MVT::v16i32, MVT::v8i64 }) { + for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64, + MVT::v16i16, MVT::v8i32, MVT::v4i64, + MVT::v32i16, MVT::v16i32, MVT::v8i64 }) { setOperationAction(ISD::FSHL, VT, Custom); setOperationAction(ISD::FSHR, VT, Custom); } + + setOperationAction(ISD::ROTL, MVT::v32i16, Custom); + setOperationAction(ISD::ROTR, MVT::v8i16, Custom); + setOperationAction(ISD::ROTR, MVT::v16i16, Custom); + setOperationAction(ISD::ROTR, MVT::v32i16, Custom); } }// useAVX512Regs @@ -1858,20 +1914,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal); } - if (Subtarget.hasVBMI2()) { - // TODO: Make these legal even without VLX? - for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64, - MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { - setOperationAction(ISD::FSHL, VT, Custom); - setOperationAction(ISD::FSHR, VT, Custom); - } - } - setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom); setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom); setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom); } + if (Subtarget.hasAMXTILE()) { + addRegisterClass(MVT::x86amx, &X86::TILERegClass); + } + // We want to custom lower some of our intrinsics. setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); @@ -1901,6 +1952,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::ADDCARRY, VT, Custom); setOperationAction(ISD::SUBCARRY, VT, Custom); setOperationAction(ISD::SETCCCARRY, VT, Custom); + setOperationAction(ISD::SADDO_CARRY, VT, Custom); + setOperationAction(ISD::SSUBO_CARRY, VT, Custom); } if (!Subtarget.is64Bit()) { @@ -1923,8 +1976,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM, setOperationAction(ISD::UDIV, MVT::i128, Custom); setOperationAction(ISD::SREM, MVT::i128, Custom); setOperationAction(ISD::UREM, MVT::i128, Custom); - setOperationAction(ISD::SDIVREM, MVT::i128, Custom); - setOperationAction(ISD::UDIVREM, MVT::i128, Custom); } // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)` @@ -2456,13 +2507,23 @@ Value *X86TargetLowering::getIRStackGuard(IRBuilder<> &IRB) const { // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value. return SegmentOffset(IRB, 0x10, getAddressSpace()); } else { + unsigned AddressSpace = getAddressSpace(); + // Specially, some users may customize the base reg and offset. + unsigned Offset = getTargetMachine().Options.StackProtectorGuardOffset; + // If we don't set -stack-protector-guard-offset value: // %fs:0x28, unless we're using a Kernel code model, in which case // it's %gs:0x28. gs:0x14 on i386. - unsigned Offset = (Subtarget.is64Bit()) ? 0x28 : 0x14; - return SegmentOffset(IRB, Offset, getAddressSpace()); + if (Offset == (unsigned)-1) + Offset = (Subtarget.is64Bit()) ? 0x28 : 0x14; + + const auto &GuardReg = getTargetMachine().Options.StackProtectorGuardReg; + if (GuardReg == "fs") + AddressSpace = X86AS::FS; + else if (GuardReg == "gs") + AddressSpace = X86AS::GS; + return SegmentOffset(IRB, Offset, AddressSpace); } } - return TargetLowering::getIRStackGuard(IRB); } @@ -2484,8 +2545,13 @@ void X86TargetLowering::insertSSPDeclarations(Module &M) const { } return; } + + auto GuardMode = getTargetMachine().Options.StackProtectorGuard; + // glibc, bionic, and Fuchsia have a special slot for the stack guard. - if (hasStackGuardSlotTLS(Subtarget.getTargetTriple())) + if ((GuardMode == llvm::StackProtectorGuards::TLS || + GuardMode == llvm::StackProtectorGuards::None) + && hasStackGuardSlotTLS(Subtarget.getTargetTriple())) return; TargetLowering::insertSSPDeclarations(M); } @@ -2531,17 +2597,6 @@ Value *X86TargetLowering::getSafeStackPointerLocation(IRBuilder<> &IRB) const { return TargetLowering::getSafeStackPointerLocation(IRB); } -bool X86TargetLowering::isNoopAddrSpaceCast(unsigned SrcAS, - unsigned DestAS) const { - assert(SrcAS != DestAS && "Expected different address spaces!"); - - const TargetMachine &TM = getTargetMachine(); - if (TM.getPointerSize(SrcAS) != TM.getPointerSize(DestAS)) - return false; - - return SrcAS < 256 && DestAS < 256; -} - //===----------------------------------------------------------------------===// // Return Value Calling Convention Implementation //===----------------------------------------------------------------------===// @@ -3046,8 +3101,9 @@ SDValue X86TargetLowering::LowerCallResult( // This truncation won't change the value. DAG.getIntPtrConstant(1, dl)); - if (VA.isExtInLoc() && (VA.getValVT().getScalarType() == MVT::i1)) { + if (VA.isExtInLoc()) { if (VA.getValVT().isVector() && + VA.getValVT().getScalarType() == MVT::i1 && ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) || (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) { // promoting a mask type (v*i1) into a register of type i64/i32/i16/i8 @@ -3115,7 +3171,7 @@ argsAreStructReturn(ArrayRef<ISD::InputArg> Ins, bool IsMCU) { static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl) { - SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); + SDValue SizeNode = DAG.getIntPtrConstant(Flags.getByValSize(), dl); return DAG.getMemcpy( Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(), @@ -3208,13 +3264,23 @@ X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv, return DAG.getFrameIndex(FI, PtrVT); } + EVT ArgVT = Ins[i].ArgVT; + + // If this is a vector that has been split into multiple parts, and the + // scalar size of the parts don't match the vector element size, then we can't + // elide the copy. The parts will have padding between them instead of being + // packed like a vector. + bool ScalarizedAndExtendedVector = + ArgVT.isVector() && !VA.getLocVT().isVector() && + VA.getLocVT().getSizeInBits() != ArgVT.getScalarSizeInBits(); + // This is an argument in memory. We might be able to perform copy elision. // If the argument is passed directly in memory without any extension, then we // can perform copy elision. Large vector types, for example, may be passed // indirectly by pointer. if (Flags.isCopyElisionCandidate() && - VA.getLocInfo() != CCValAssign::Indirect && !ExtendedInMem) { - EVT ArgVT = Ins[i].ArgVT; + VA.getLocInfo() != CCValAssign::Indirect && !ExtendedInMem && + !ScalarizedAndExtendedVector) { SDValue PartAddr; if (Ins[i].PartOffset == 0) { // If this is a one-part value or the first part of a multi-part value, @@ -3354,8 +3420,8 @@ private: void forwardMustTailParameters(SDValue &Chain); - bool is64Bit() { return Subtarget.is64Bit(); } - bool isWin64() { return Subtarget.isCallingConvWin64(CallConv); } + bool is64Bit() const { return Subtarget.is64Bit(); } + bool isWin64() const { return Subtarget.isCallingConvWin64(CallConv); } X86MachineFunctionInfo *FuncInfo; const SDLoc &DL; @@ -3466,11 +3532,10 @@ void VarArgsLoweringHelper::createVarArgAreaAndStoreRegisters( SaveXMMOps.push_back(Chain); SaveXMMOps.push_back(ALVal); SaveXMMOps.push_back( - DAG.getIntPtrConstant(FuncInfo->getRegSaveFrameIndex(), DL)); + DAG.getTargetConstant(FuncInfo->getRegSaveFrameIndex(), DL, MVT::i32)); SaveXMMOps.push_back( - DAG.getIntPtrConstant(FuncInfo->getVarArgsFPOffset(), DL)); - SaveXMMOps.insert(SaveXMMOps.end(), LiveXMMRegs.begin(), - LiveXMMRegs.end()); + DAG.getTargetConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32)); + llvm::append_range(SaveXMMOps, LiveXMMRegs); MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, DL, MVT::Other, SaveXMMOps)); } @@ -3744,7 +3809,7 @@ SDValue X86TargetLowering::LowerFormalArguments( // same, so the size of funclets' (mostly empty) frames is dictated by // how far this slot is from the bottom (since they allocate just enough // space to accommodate holding this slot at the correct offset). - int PSPSymFI = MFI.CreateStackObject(8, Align(8), /*isSS=*/false); + int PSPSymFI = MFI.CreateStackObject(8, Align(8), /*isSpillSlot=*/false); EHInfo->PSPSymFrameIdx = PSPSymFI; } } @@ -3851,6 +3916,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, const auto *II = dyn_cast_or_null<InvokeInst>(CLI.CB); bool HasNoCfCheck = (CI && CI->doesNoCfCheck()) || (II && II->doesNoCfCheck()); + bool IsIndirectCall = (CI && CI->isIndirectCall()); const Module *M = MF.getMMI().getModule(); Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch"); @@ -4090,9 +4156,13 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, if (Subtarget.isPICStyleGOT()) { // ELF / PIC requires GOT in the EBX register before function calls via PLT - // GOT pointer. + // GOT pointer (except regcall). if (!isTailCall) { - RegsToPass.push_back(std::make_pair( + // Indirect call with RegCall calling convertion may use up all the + // general registers, so it is not suitable to bind EBX reister for + // GOT address, just let register allocator handle it. + if (CallConv != CallingConv::X86_RegCall) + RegsToPass.push_back(std::make_pair( Register(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), getPointerTy(DAG.getDataLayout())))); } else { @@ -4259,7 +4329,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, Ops.push_back(Callee); if (isTailCall) - Ops.push_back(DAG.getConstant(FPDiff, dl, MVT::i32)); + Ops.push_back(DAG.getTargetConstant(FPDiff, dl, MVT::i32)); // Add argument registers to the end of the list so that they are known live // into the call. @@ -4333,7 +4403,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, return Ret; } - if (HasNoCfCheck && IsCFProtectionSupported) { + if (HasNoCfCheck && IsCFProtectionSupported && IsIndirectCall) { Chain = DAG.getNode(X86ISD::NT_CALL, dl, NodeTys, Ops); } else { Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); @@ -4452,7 +4522,7 @@ bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, int FI = INT_MAX; if (Arg.getOpcode() == ISD::CopyFromReg) { Register VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg(); - if (!Register::isVirtualRegister(VR)) + if (!VR.isVirtual()) return false; MachineInstr *Def = MRI->getVRegDef(VR); if (!Def) @@ -4504,7 +4574,8 @@ bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, if (!Flags.isByVal() && !MFI.isImmutableObjectIndex(FI)) return false; - if (VA.getLocVT().getSizeInBits() > Arg.getValueSizeInBits()) { + if (VA.getLocVT().getFixedSizeInBits() > + Arg.getValueSizeInBits().getFixedSize()) { // If the argument location is wider than the argument type, check that any // extension flags match. if (Flags.isZExt() != MFI.isObjectZExt(FI) || @@ -5012,13 +5083,47 @@ bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { + Info.flags = MachineMemOperand::MONone; + Info.offset = 0; const IntrinsicData* IntrData = getIntrinsicWithChain(Intrinsic); - if (!IntrData) + if (!IntrData) { + switch (Intrinsic) { + case Intrinsic::x86_aesenc128kl: + case Intrinsic::x86_aesdec128kl: + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.ptrVal = I.getArgOperand(1); + Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48); + Info.align = Align(1); + Info.flags |= MachineMemOperand::MOLoad; + return true; + case Intrinsic::x86_aesenc256kl: + case Intrinsic::x86_aesdec256kl: + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.ptrVal = I.getArgOperand(1); + Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64); + Info.align = Align(1); + Info.flags |= MachineMemOperand::MOLoad; + return true; + case Intrinsic::x86_aesencwide128kl: + case Intrinsic::x86_aesdecwide128kl: + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.ptrVal = I.getArgOperand(0); + Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48); + Info.align = Align(1); + Info.flags |= MachineMemOperand::MOLoad; + return true; + case Intrinsic::x86_aesencwide256kl: + case Intrinsic::x86_aesdecwide256kl: + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.ptrVal = I.getArgOperand(0); + Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64); + Info.align = Align(1); + Info.flags |= MachineMemOperand::MOLoad; + return true; + } return false; - - Info.flags = MachineMemOperand::MONone; - Info.offset = 0; + } switch (IntrData->Type) { case TRUNCATE_TO_MEM_VI8: @@ -5088,7 +5193,7 @@ bool X86TargetLowering::shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const { assert(cast<LoadSDNode>(Load)->isSimple() && "illegal to narrow"); - + // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF // relocation target a movq or addq instruction: don't let the load shrink. SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr(); @@ -5261,6 +5366,7 @@ bool X86TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT, // width. if (MemVT.getSizeInBits() > Subtarget.getPreferVectorWidth()) return false; + return true; } @@ -5404,6 +5510,14 @@ static bool isUndefOrEqual(int Val, int CmpVal) { return ((Val == SM_SentinelUndef) || (Val == CmpVal)); } +/// Return true if every element in Mask is the undef sentinel value or equal to +/// the specified value.. +static bool isUndefOrEqual(ArrayRef<int> Mask, int CmpVal) { + return llvm::all_of(Mask, [CmpVal](int M) { + return (M == SM_SentinelUndef) || (M == CmpVal); + }); +} + /// Val is either the undef or zero sentinel value. static bool isUndefOrZero(int Val) { return ((Val == SM_SentinelUndef) || (Val == SM_SentinelZero)); @@ -5810,7 +5924,7 @@ static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal, static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) { - assert(Vec.getValueSizeInBits() < VT.getSizeInBits() && + assert(Vec.getValueSizeInBits().getFixedSize() < VT.getFixedSizeInBits() && Vec.getValueType().getScalarType() == VT.getScalarType() && "Unsupported vector widening type"); SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl) @@ -6174,6 +6288,22 @@ static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) { return DAG.getBitcast(VT, Vec); } +// Convert *_EXTEND_VECTOR_INREG to *_EXTEND opcode. +static unsigned getOpcode_EXTEND(unsigned Opcode) { + switch (Opcode) { + case ISD::ANY_EXTEND: + case ISD::ANY_EXTEND_VECTOR_INREG: + return ISD::ANY_EXTEND; + case ISD::ZERO_EXTEND: + case ISD::ZERO_EXTEND_VECTOR_INREG: + return ISD::ZERO_EXTEND; + case ISD::SIGN_EXTEND: + case ISD::SIGN_EXTEND_VECTOR_INREG: + return ISD::SIGN_EXTEND; + } + llvm_unreachable("Unknown opcode"); +} + // Convert *_EXTEND to *_EXTEND_VECTOR_INREG opcode. static unsigned getOpcode_EXTEND_VECTOR_INREG(unsigned Opcode) { switch (Opcode) { @@ -6190,8 +6320,8 @@ static unsigned getOpcode_EXTEND_VECTOR_INREG(unsigned Opcode) { llvm_unreachable("Unknown opcode"); } -static SDValue getExtendInVec(unsigned Opcode, const SDLoc &DL, EVT VT, - SDValue In, SelectionDAG &DAG) { +static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT, + SDValue In, SelectionDAG &DAG) { EVT InVT = In.getValueType(); assert(VT.isVector() && InVT.isVector() && "Expected vector VTs."); assert((ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode || @@ -6243,8 +6373,10 @@ static SDValue IsNOT(SDValue V, SelectionDAG &DAG, bool OneUse = false) { return SDValue(); } -void llvm::createUnpackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, +void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, bool Lo, bool Unary) { + assert(VT.getScalarType().isSimple() && (VT.getSizeInBits() % 128) == 0 && + "Illegal vector type to unpack"); assert(Mask.empty() && "Expected an empty shuffle mask vector"); int NumElts = VT.getVectorNumElements(); int NumEltsInLane = 128 / VT.getScalarSizeInBits(); @@ -6273,7 +6405,7 @@ void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, } /// Returns a vector_shuffle node for an unpackl operation. -static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, MVT VT, +static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2) { SmallVector<int, 8> Mask; createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false); @@ -6281,7 +6413,7 @@ static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, MVT VT, } /// Returns a vector_shuffle node for an unpackh operation. -static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, MVT VT, +static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2) { SmallVector<int, 8> Mask; createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false); @@ -6528,15 +6660,30 @@ static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits, } // Extract constant bits from a subvector broadcast. - if (Op.getOpcode() == X86ISD::SUBV_BROADCAST) { - SmallVector<APInt, 16> SubEltBits; - if (getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits, - UndefElts, SubEltBits, AllowWholeUndefs, - AllowPartialUndefs)) { - UndefElts = APInt::getSplat(NumElts, UndefElts); - while (EltBits.size() < NumElts) - EltBits.append(SubEltBits.begin(), SubEltBits.end()); - return true; + if (Op.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) { + auto *MemIntr = cast<MemIntrinsicSDNode>(Op); + SDValue Ptr = MemIntr->getBasePtr(); + if (const Constant *Cst = getTargetConstantFromBasePtr(Ptr)) { + Type *CstTy = Cst->getType(); + unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits(); + if (!CstTy->isVectorTy() || (SizeInBits % CstSizeInBits) != 0) + return false; + unsigned SubEltSizeInBits = CstTy->getScalarSizeInBits(); + unsigned NumSubElts = CstSizeInBits / SubEltSizeInBits; + unsigned NumSubVecs = SizeInBits / CstSizeInBits; + APInt UndefSubElts(NumSubElts, 0); + SmallVector<APInt, 64> SubEltBits(NumSubElts * NumSubVecs, + APInt(SubEltSizeInBits, 0)); + for (unsigned i = 0; i != NumSubElts; ++i) { + if (!CollectConstantBits(Cst->getAggregateElement(i), SubEltBits[i], + UndefSubElts, i)) + return false; + for (unsigned j = 1; j != NumSubVecs; ++j) + SubEltBits[i + (j * NumSubElts)] = SubEltBits[i]; + } + UndefSubElts = APInt::getSplat(NumSubVecs * UndefSubElts.getBitWidth(), + UndefSubElts); + return CastBitData(UndefSubElts, SubEltBits); } } @@ -6557,23 +6704,26 @@ static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits, // Insert constant bits from a base and sub vector sources. if (Op.getOpcode() == ISD::INSERT_SUBVECTOR) { - // TODO - support insert_subvector through bitcasts. - if (EltSizeInBits != VT.getScalarSizeInBits()) - return false; + // If bitcasts to larger elements we might lose track of undefs - don't + // allow any to be safe. + unsigned SrcEltSizeInBits = VT.getScalarSizeInBits(); + bool AllowUndefs = EltSizeInBits >= SrcEltSizeInBits; - APInt UndefSubElts; - SmallVector<APInt, 32> EltSubBits; - if (getTargetConstantBitsFromNode(Op.getOperand(1), EltSizeInBits, + APInt UndefSrcElts, UndefSubElts; + SmallVector<APInt, 32> EltSrcBits, EltSubBits; + if (getTargetConstantBitsFromNode(Op.getOperand(1), SrcEltSizeInBits, UndefSubElts, EltSubBits, - AllowWholeUndefs, AllowPartialUndefs) && - getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits, - UndefElts, EltBits, AllowWholeUndefs, - AllowPartialUndefs)) { + AllowWholeUndefs && AllowUndefs, + AllowPartialUndefs && AllowUndefs) && + getTargetConstantBitsFromNode(Op.getOperand(0), SrcEltSizeInBits, + UndefSrcElts, EltSrcBits, + AllowWholeUndefs && AllowUndefs, + AllowPartialUndefs && AllowUndefs)) { unsigned BaseIdx = Op.getConstantOperandVal(2); - UndefElts.insertBits(UndefSubElts, BaseIdx); + UndefSrcElts.insertBits(UndefSubElts, BaseIdx); for (unsigned i = 0, e = EltSubBits.size(); i != e; ++i) - EltBits[BaseIdx + i] = EltSubBits[i]; - return true; + EltSrcBits[BaseIdx + i] = EltSubBits[i]; + return CastBitData(UndefSrcElts, EltSrcBits); } } @@ -6686,7 +6836,7 @@ static bool getTargetShuffleMaskIndices(SDValue MaskNode, return false; // Insert the extracted elements into the mask. - for (APInt Elt : EltBits) + for (const APInt &Elt : EltBits) RawMask.push_back(Elt.getZExtValue()); return true; @@ -6916,25 +7066,16 @@ static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero, DecodeZeroMoveLowMask(NumElems, Mask); IsUnary = true; break; - case X86ISD::VBROADCAST: { - SDValue N0 = N->getOperand(0); - // See if we're broadcasting from index 0 of an EXTRACT_SUBVECTOR. If so, - // add the pre-extracted value to the Ops vector. - if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && - N0.getOperand(0).getValueType() == VT && - N0.getConstantOperandVal(1) == 0) - Ops.push_back(N0.getOperand(0)); - - // We only decode broadcasts of same-sized vectors, unless the broadcast - // came from an extract from the original width. If we found one, we - // pushed it the Ops vector above. - if (N0.getValueType() == VT || !Ops.empty()) { + case X86ISD::VBROADCAST: + // We only decode broadcasts of same-sized vectors, peeking through to + // extracted subvectors is likely to cause hasOneUse issues with + // SimplifyDemandedBits etc. + if (N->getOperand(0).getValueType() == VT) { DecodeVectorBroadcast(NumElems, Mask); IsUnary = true; break; } return false; - } case X86ISD::VPERMILPV: { assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); IsUnary = true; @@ -7374,44 +7515,10 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts, return true; } case ISD::OR: { - // Inspect each operand at the byte level. We can merge these into a - // blend shuffle mask if for each byte at least one is masked out (zero). - KnownBits Known0 = - DAG.computeKnownBits(N.getOperand(0), DemandedElts, Depth + 1); - KnownBits Known1 = - DAG.computeKnownBits(N.getOperand(1), DemandedElts, Depth + 1); - if (Known0.One.isNullValue() && Known1.One.isNullValue()) { - bool IsByteMask = true; - APInt ZeroMask = APInt::getNullValue(NumBytesPerElt); - APInt SelectMask = APInt::getNullValue(NumBytesPerElt); - for (unsigned i = 0; i != NumBytesPerElt && IsByteMask; ++i) { - unsigned LHS = Known0.Zero.extractBits(8, i * 8).getZExtValue(); - unsigned RHS = Known1.Zero.extractBits(8, i * 8).getZExtValue(); - if (LHS == 255 && RHS == 0) - SelectMask.setBit(i); - else if (LHS == 255 && RHS == 255) - ZeroMask.setBit(i); - else if (!(LHS == 0 && RHS == 255)) - IsByteMask = false; - } - if (IsByteMask) { - for (unsigned i = 0; i != NumSizeInBytes; i += NumBytesPerElt) { - for (unsigned j = 0; j != NumBytesPerElt; ++j) { - unsigned Ofs = (SelectMask[j] ? NumSizeInBytes : 0); - int Idx = (ZeroMask[j] ? (int)SM_SentinelZero : (i + j + Ofs)); - Mask.push_back(Idx); - } - } - Ops.push_back(N.getOperand(0)); - Ops.push_back(N.getOperand(1)); - return true; - } - } - // Handle OR(SHUFFLE,SHUFFLE) case where one source is zero and the other // is a valid shuffle index. - SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0)); - SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1)); + SDValue N0 = peekThroughBitcasts(N.getOperand(0)); + SDValue N1 = peekThroughBitcasts(N.getOperand(1)); if (!N0.getValueType().isVector() || !N1.getValueType().isVector()) return false; SmallVector<int, 64> SrcMask0, SrcMask1; @@ -7422,34 +7529,24 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts, true)) return false; - // Shuffle inputs must be the same size as the result. - if (llvm::any_of(SrcInputs0, [VT](SDValue Op) { - return VT.getSizeInBits() != Op.getValueSizeInBits(); - })) - return false; - if (llvm::any_of(SrcInputs1, [VT](SDValue Op) { - return VT.getSizeInBits() != Op.getValueSizeInBits(); - })) - return false; - size_t MaskSize = std::max(SrcMask0.size(), SrcMask1.size()); SmallVector<int, 64> Mask0, Mask1; narrowShuffleMaskElts(MaskSize / SrcMask0.size(), SrcMask0, Mask0); narrowShuffleMaskElts(MaskSize / SrcMask1.size(), SrcMask1, Mask1); - for (size_t i = 0; i != MaskSize; ++i) { + for (int i = 0; i != (int)MaskSize; ++i) { if (Mask0[i] == SM_SentinelUndef && Mask1[i] == SM_SentinelUndef) Mask.push_back(SM_SentinelUndef); else if (Mask0[i] == SM_SentinelZero && Mask1[i] == SM_SentinelZero) Mask.push_back(SM_SentinelZero); else if (Mask1[i] == SM_SentinelZero) - Mask.push_back(Mask0[i]); + Mask.push_back(i); else if (Mask0[i] == SM_SentinelZero) - Mask.push_back(Mask1[i] + (int)(MaskSize * SrcInputs0.size())); + Mask.push_back(i + MaskSize); else return false; } - Ops.append(SrcInputs0.begin(), SrcInputs0.end()); - Ops.append(SrcInputs1.begin(), SrcInputs1.end()); + Ops.push_back(N0); + Ops.push_back(N1); return true; } case ISD::INSERT_SUBVECTOR: { @@ -7481,7 +7578,8 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts, // Subvector shuffle inputs must not be larger than the subvector. if (llvm::any_of(SubInputs, [SubVT](SDValue SubInput) { - return SubVT.getSizeInBits() < SubInput.getValueSizeInBits(); + return SubVT.getFixedSizeInBits() < + SubInput.getValueSizeInBits().getFixedSize(); })) return false; @@ -7502,8 +7600,11 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts, } Ops.push_back(Src); Ops.append(SubInputs.begin(), SubInputs.end()); - for (int i = 0; i != (int)NumElts; ++i) - Mask.push_back(i); + if (ISD::isBuildVectorAllZeros(Src.getNode())) + Mask.append(NumElts, SM_SentinelZero); + else + for (int i = 0; i != (int)NumElts; ++i) + Mask.push_back(i); for (int i = 0; i != (int)NumSubElts; ++i) { int M = SubMask[i]; if (0 <= M) { @@ -7604,19 +7705,33 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts, APInt EltsLHS, EltsRHS; getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS); - // If we know input saturation won't happen we can treat this - // as a truncation shuffle. + // If we know input saturation won't happen (or we don't care for particular + // lanes), we can treat this as a truncation shuffle. + bool Offset0 = false, Offset1 = false; if (Opcode == X86ISD::PACKSS) { - if ((!N0.isUndef() && + if ((!(N0.isUndef() || EltsLHS.isNullValue()) && DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) || - (!N1.isUndef() && + (!(N1.isUndef() || EltsRHS.isNullValue()) && DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt)) return false; + // We can't easily fold ASHR into a shuffle, but if it was feeding a + // PACKSS then it was likely being used for sign-extension for a + // truncation, so just peek through and adjust the mask accordingly. + if (N0.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N0.getNode()) && + N0.getConstantOperandAPInt(1) == NumBitsPerElt) { + Offset0 = true; + N0 = N0.getOperand(0); + } + if (N1.getOpcode() == X86ISD::VSRAI && N->isOnlyUserOf(N1.getNode()) && + N1.getConstantOperandAPInt(1) == NumBitsPerElt) { + Offset1 = true; + N1 = N1.getOperand(0); + } } else { APInt ZeroMask = APInt::getHighBitsSet(2 * NumBitsPerElt, NumBitsPerElt); - if ((!N0.isUndef() && + if ((!(N0.isUndef() || EltsLHS.isNullValue()) && !DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) || - (!N1.isUndef() && + (!(N1.isUndef() || EltsRHS.isNullValue()) && !DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1))) return false; } @@ -7628,6 +7743,13 @@ static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts, Ops.push_back(N1); createPackShuffleMask(VT, Mask, IsUnary); + + if (Offset0 || Offset1) { + for (int &M : Mask) + if ((Offset0 && isInRange(M, 0, NumElts)) || + (Offset1 && isInRange(M, NumElts, 2 * NumElts))) + ++M; + } return true; } case X86ISD::VTRUNC: { @@ -7915,7 +8037,7 @@ static SDValue getShuffleScalarElt(SDValue Op, unsigned Index, } // Use PINSRB/PINSRW/PINSRD to create a build vector. -static SDValue LowerBuildVectorAsInsert(SDValue Op, unsigned NonZeros, +static SDValue LowerBuildVectorAsInsert(SDValue Op, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget) { @@ -7930,7 +8052,7 @@ static SDValue LowerBuildVectorAsInsert(SDValue Op, unsigned NonZeros, bool First = true; for (unsigned i = 0; i < NumElts; ++i) { - bool IsNonZero = (NonZeros & (1 << i)) != 0; + bool IsNonZero = NonZeroMask[i]; if (!IsNonZero) continue; @@ -7957,7 +8079,7 @@ static SDValue LowerBuildVectorAsInsert(SDValue Op, unsigned NonZeros, } /// Custom lower build_vector of v16i8. -static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, +static SDValue LowerBuildVectorv16i8(SDValue Op, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget) { @@ -7966,7 +8088,7 @@ static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, // SSE4.1 - use PINSRB to insert each byte directly. if (Subtarget.hasSSE41()) - return LowerBuildVectorAsInsert(Op, NonZeros, NumNonZero, NumZero, DAG, + return LowerBuildVectorAsInsert(Op, NonZeroMask, NumNonZero, NumZero, DAG, Subtarget); SDLoc dl(Op); @@ -7974,8 +8096,8 @@ static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, // Pre-SSE4.1 - merge byte pairs and insert with PINSRW. for (unsigned i = 0; i < 16; i += 2) { - bool ThisIsNonZero = (NonZeros & (1 << i)) != 0; - bool NextIsNonZero = (NonZeros & (1 << (i + 1))) != 0; + bool ThisIsNonZero = NonZeroMask[i]; + bool NextIsNonZero = NonZeroMask[i + 1]; if (!ThisIsNonZero && !NextIsNonZero) continue; @@ -8023,7 +8145,7 @@ static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros, } /// Custom lower build_vector of v8i16. -static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, +static SDValue LowerBuildVectorv8i16(SDValue Op, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget) { @@ -8031,7 +8153,7 @@ static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros, return SDValue(); // Use PINSRW to insert each byte directly. - return LowerBuildVectorAsInsert(Op, NonZeros, NumNonZero, NumZero, DAG, + return LowerBuildVectorAsInsert(Op, NonZeroMask, NumNonZero, NumZero, DAG, Subtarget); } @@ -8351,8 +8473,6 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts, // Handle Special Cases - all undef or undef/zero. if (UndefMask.countPopulation() == NumElems) return DAG.getUNDEF(VT); - - // FIXME: Should we return this as a BUILD_VECTOR instead? if ((ZeroMask.countPopulation() + UndefMask.countPopulation()) == NumElems) return VT.isInteger() ? DAG.getConstant(0, DL, VT) : DAG.getConstantFP(0.0, DL, VT); @@ -8367,7 +8487,8 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts, assert(LDBase && "Did not find base load for merging consecutive loads"); unsigned BaseSizeInBits = EltBaseVT.getStoreSizeInBits(); unsigned BaseSizeInBytes = BaseSizeInBits / 8; - int LoadSizeInBits = (1 + LastLoadedElt - FirstLoadedElt) * BaseSizeInBits; + int NumLoadedElts = (1 + LastLoadedElt - FirstLoadedElt); + int LoadSizeInBits = NumLoadedElts * BaseSizeInBits; assert((BaseSizeInBits % 8) == 0 && "Sub-byte element loads detected"); // TODO: Support offsetting the base load. @@ -8429,7 +8550,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts, // base pointer. If the vector contains zeros, then attempt to shuffle those // elements. if (FirstLoadedElt == 0 && - (LastLoadedElt == (int)(NumElems - 1) || IsDereferenceable) && + (NumLoadedElts == (int)NumElems || IsDereferenceable) && (IsConsecutiveLoad || IsConsecutiveLoadWithZeros)) { if (isAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT)) return SDValue(); @@ -8517,6 +8638,11 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts, if (!Subtarget.hasAVX2() && ScalarSize < 32) continue; + // Don't attempt a 1:N subvector broadcast - it should be caught by + // combineConcatVectorOps, else will cause infinite loops. + if (RepeatSize > ScalarSize && SubElems == 1) + continue; + bool Match = true; SmallVector<SDValue, 8> RepeatedLoads(SubElems, DAG.getUNDEF(EltBaseVT)); for (unsigned i = 0; i != NumElems && Match; ++i) { @@ -8548,9 +8674,14 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts, if (TLI.isTypeLegal(BroadcastVT)) { if (SDValue RepeatLoad = EltsFromConsecutiveLoads( RepeatVT, RepeatedLoads, DL, DAG, Subtarget, isAfterLegalize)) { - unsigned Opcode = RepeatSize > ScalarSize ? X86ISD::SUBV_BROADCAST - : X86ISD::VBROADCAST; - SDValue Broadcast = DAG.getNode(Opcode, DL, BroadcastVT, RepeatLoad); + SDValue Broadcast = RepeatLoad; + if (RepeatSize > ScalarSize) { + while (Broadcast.getValueSizeInBits() < VT.getSizeInBits()) + Broadcast = concatSubVectors(Broadcast, Broadcast, DAG, DL); + } else { + Broadcast = + DAG.getNode(X86ISD::VBROADCAST, DL, BroadcastVT, RepeatLoad); + } return DAG.getBitcast(VT, Broadcast); } } @@ -8621,43 +8752,6 @@ static bool isFoldableUseOfShuffle(SDNode *N) { return false; } -// Check if the current node of build vector is a zero extended vector. -// // If so, return the value extended. -// // For example: (0,0,0,a,0,0,0,a,0,0,0,a,0,0,0,a) returns a. -// // NumElt - return the number of zero extended identical values. -// // EltType - return the type of the value include the zero extend. -static SDValue isSplatZeroExtended(const BuildVectorSDNode *Op, - unsigned &NumElt, MVT &EltType) { - SDValue ExtValue = Op->getOperand(0); - unsigned NumElts = Op->getNumOperands(); - unsigned Delta = NumElts; - - for (unsigned i = 1; i < NumElts; i++) { - if (Op->getOperand(i) == ExtValue) { - Delta = i; - break; - } - if (!(Op->getOperand(i).isUndef() || isNullConstant(Op->getOperand(i)))) - return SDValue(); - } - if (!isPowerOf2_32(Delta) || Delta == 1) - return SDValue(); - - for (unsigned i = Delta; i < NumElts; i++) { - if (i % Delta == 0) { - if (Op->getOperand(i) != ExtValue) - return SDValue(); - } else if (!(isNullConstant(Op->getOperand(i)) || - Op->getOperand(i).isUndef())) - return SDValue(); - } - unsigned EltSize = Op->getSimpleValueType(0).getScalarSizeInBits(); - unsigned ExtVTSize = EltSize * Delta; - EltType = MVT::getIntegerVT(ExtVTSize); - NumElt = NumElts / Delta; - return ExtValue; -} - /// Attempt to use the vbroadcast instruction to generate a splat value /// from a splat BUILD_VECTOR which uses: /// a. A single scalar load, or a constant. @@ -8675,13 +8769,21 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp, return SDValue(); MVT VT = BVOp->getSimpleValueType(0); + unsigned NumElts = VT.getVectorNumElements(); SDLoc dl(BVOp); assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && "Unsupported vector type for broadcast."); + // See if the build vector is a repeating sequence of scalars (inc. splat). + SDValue Ld; BitVector UndefElements; - SDValue Ld = BVOp->getSplatValue(&UndefElements); + SmallVector<SDValue, 16> Sequence; + if (BVOp->getRepeatedSequence(Sequence, &UndefElements)) { + assert((NumElts % Sequence.size()) == 0 && "Sequence doesn't fit."); + if (Sequence.size() == 1) + Ld = Sequence[0]; + } // Attempt to use VBROADCASTM // From this pattern: @@ -8689,30 +8791,38 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp, // b. t1 = (build_vector t0 t0) // // Create (VBROADCASTM v2i1 X) - if (Subtarget.hasCDI() && (VT.is512BitVector() || Subtarget.hasVLX())) { - MVT EltType = VT.getScalarType(); - unsigned NumElts = VT.getVectorNumElements(); - SDValue BOperand; - SDValue ZeroExtended = isSplatZeroExtended(BVOp, NumElts, EltType); - if ((ZeroExtended && ZeroExtended.getOpcode() == ISD::BITCAST) || - (Ld && Ld.getOpcode() == ISD::ZERO_EXTEND && - Ld.getOperand(0).getOpcode() == ISD::BITCAST)) { - if (ZeroExtended) - BOperand = ZeroExtended.getOperand(0); - else - BOperand = Ld.getOperand(0).getOperand(0); + if (!Sequence.empty() && Subtarget.hasCDI()) { + // If not a splat, are the upper sequence values zeroable? + unsigned SeqLen = Sequence.size(); + bool UpperZeroOrUndef = + SeqLen == 1 || + llvm::all_of(makeArrayRef(Sequence).drop_front(), [](SDValue V) { + return !V || V.isUndef() || isNullConstant(V); + }); + SDValue Op0 = Sequence[0]; + if (UpperZeroOrUndef && ((Op0.getOpcode() == ISD::BITCAST) || + (Op0.getOpcode() == ISD::ZERO_EXTEND && + Op0.getOperand(0).getOpcode() == ISD::BITCAST))) { + SDValue BOperand = Op0.getOpcode() == ISD::BITCAST + ? Op0.getOperand(0) + : Op0.getOperand(0).getOperand(0); MVT MaskVT = BOperand.getSimpleValueType(); - if ((EltType == MVT::i64 && MaskVT == MVT::v8i1) || // for broadcastmb2q + MVT EltType = MVT::getIntegerVT(VT.getScalarSizeInBits() * SeqLen); + if ((EltType == MVT::i64 && MaskVT == MVT::v8i1) || // for broadcastmb2q (EltType == MVT::i32 && MaskVT == MVT::v16i1)) { // for broadcastmw2d - SDValue Brdcst = - DAG.getNode(X86ISD::VBROADCASTM, dl, - MVT::getVectorVT(EltType, NumElts), BOperand); - return DAG.getBitcast(VT, Brdcst); + MVT BcstVT = MVT::getVectorVT(EltType, NumElts / SeqLen); + if (!VT.is512BitVector() && !Subtarget.hasVLX()) { + unsigned Scale = 512 / VT.getSizeInBits(); + BcstVT = MVT::getVectorVT(EltType, Scale * (NumElts / SeqLen)); + } + SDValue Bcst = DAG.getNode(X86ISD::VBROADCASTM, dl, BcstVT, BOperand); + if (BcstVT.getSizeInBits() != VT.getSizeInBits()) + Bcst = extractSubVector(Bcst, 0, DAG, dl, VT.getSizeInBits()); + return DAG.getBitcast(VT, Bcst); } } } - unsigned NumElts = VT.getVectorNumElements(); unsigned NumUndefElts = UndefElements.count(); if (!Ld || (NumElts - NumUndefElts) <= 1) { APInt SplatValue, Undef; @@ -8754,18 +8864,19 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp, } if (SplatBitSize > 64) { // Load the vector of constants and broadcast it. - MVT CVT = VT.getScalarType(); Constant *VecC = getConstantVector(VT, SplatValue, SplatBitSize, *Ctx); SDValue VCP = DAG.getConstantPool(VecC, PVT); unsigned NumElm = SplatBitSize / VT.getScalarSizeInBits(); + MVT VVT = MVT::getVectorVT(VT.getScalarType(), NumElm); Align Alignment = cast<ConstantPoolSDNode>(VCP)->getAlign(); - Ld = DAG.getLoad( - MVT::getVectorVT(CVT, NumElm), dl, DAG.getEntryNode(), VCP, - MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), - Alignment); - SDValue Brdcst = DAG.getNode(X86ISD::SUBV_BROADCAST, dl, VT, Ld); - return DAG.getBitcast(VT, Brdcst); + SDVTList Tys = DAG.getVTList(VT, MVT::Other); + SDValue Ops[] = {DAG.getEntryNode(), VCP}; + MachinePointerInfo MPI = + MachinePointerInfo::getConstantPool(DAG.getMachineFunction()); + return DAG.getMemIntrinsicNode( + X86ISD::SUBV_BROADCAST_LOAD, dl, Tys, Ops, VVT, MPI, Alignment, + MachineMemOperand::MOLoad); } } } @@ -8786,6 +8897,8 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp, (Ld.getOpcode() == ISD::Constant || Ld.getOpcode() == ISD::ConstantFP); bool IsLoad = ISD::isNormalLoad(Ld.getNode()); + // TODO: Handle broadcasts of non-constant sequences. + // Make sure that all of the users of a non-constant load are from the // BUILD_VECTOR node. // FIXME: Is the use count needed for non-constant, non-load case? @@ -10119,45 +10232,69 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { if (SDValue VectorConstant = materializeVectorConstant(Op, DAG, Subtarget)) return VectorConstant; - BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode()); - if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, Subtarget, DAG)) - return AddSub; - if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG)) - return HorizontalOp; - if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG)) - return Broadcast; - if (SDValue BitOp = lowerBuildVectorToBitOp(BV, Subtarget, DAG)) - return BitOp; - unsigned EVTBits = EltVT.getSizeInBits(); - - unsigned NumZero = 0; - unsigned NumNonZero = 0; - uint64_t NonZeros = 0; + APInt UndefMask = APInt::getNullValue(NumElems); + APInt ZeroMask = APInt::getNullValue(NumElems); + APInt NonZeroMask = APInt::getNullValue(NumElems); bool IsAllConstants = true; SmallSet<SDValue, 8> Values; unsigned NumConstants = NumElems; for (unsigned i = 0; i < NumElems; ++i) { SDValue Elt = Op.getOperand(i); - if (Elt.isUndef()) + if (Elt.isUndef()) { + UndefMask.setBit(i); continue; + } Values.insert(Elt); if (!isa<ConstantSDNode>(Elt) && !isa<ConstantFPSDNode>(Elt)) { IsAllConstants = false; NumConstants--; } - if (X86::isZeroNode(Elt)) - NumZero++; - else { - assert(i < sizeof(NonZeros) * 8); // Make sure the shift is within range. - NonZeros |= ((uint64_t)1 << i); - NumNonZero++; + if (X86::isZeroNode(Elt)) { + ZeroMask.setBit(i); + } else { + NonZeroMask.setBit(i); } } - // All undef vector. Return an UNDEF. All zero vectors were handled above. - if (NumNonZero == 0) + // All undef vector. Return an UNDEF. All zero vectors were handled above. + if (NonZeroMask == 0) { + assert(UndefMask.isAllOnesValue() && "Fully undef mask expected"); return DAG.getUNDEF(VT); + } + + BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode()); + + // If the upper elts of a ymm/zmm are undef/zero then we might be better off + // lowering to a smaller build vector and padding with undef/zero. + if ((VT.is256BitVector() || VT.is512BitVector()) && + !isFoldableUseOfShuffle(BV)) { + unsigned UpperElems = NumElems / 2; + APInt UndefOrZeroMask = UndefMask | ZeroMask; + unsigned NumUpperUndefsOrZeros = UndefOrZeroMask.countLeadingOnes(); + if (NumUpperUndefsOrZeros >= UpperElems) { + if (VT.is512BitVector() && + NumUpperUndefsOrZeros >= (NumElems - (NumElems / 4))) + UpperElems = NumElems - (NumElems / 4); + bool UndefUpper = UndefMask.countLeadingOnes() >= UpperElems; + MVT LowerVT = MVT::getVectorVT(EltVT, NumElems - UpperElems); + SDValue NewBV = + DAG.getBuildVector(LowerVT, dl, Op->ops().drop_back(UpperElems)); + return widenSubVector(VT, NewBV, !UndefUpper, Subtarget, DAG, dl); + } + } + + if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, Subtarget, DAG)) + return AddSub; + if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG)) + return HorizontalOp; + if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG)) + return Broadcast; + if (SDValue BitOp = lowerBuildVectorToBitOp(BV, Subtarget, DAG)) + return BitOp; + + unsigned NumZero = ZeroMask.countPopulation(); + unsigned NumNonZero = NonZeroMask.countPopulation(); // If we are inserting one variable into a vector of non-zero constants, try // to avoid loading each constant element as a scalar. Load the constants as a @@ -10221,7 +10358,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { // Special case for single non-zero, non-undef, element. if (NumNonZero == 1) { - unsigned Idx = countTrailingZeros(NonZeros); + unsigned Idx = NonZeroMask.countTrailingZeros(); SDValue Item = Op.getOperand(Idx); // If we have a constant or non-constant insertion into the low element of @@ -10285,7 +10422,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0> // Check if it's possible to issue this instead. // shuffle (vload ptr)), undef, <1, 1, 1, 1> - unsigned Idx = countTrailingZeros(NonZeros); + unsigned Idx = NonZeroMask.countTrailingZeros(); SDValue Item = Op.getOperand(Idx); if (Op.getNode()->isOnlyUserOf(Item.getNode())) return LowerAsSplatVectorLoad(Item, VT, dl, DAG); @@ -10354,7 +10491,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { if (EVTBits == 64) { if (NumNonZero == 1) { // One half is zero or undef. - unsigned Idx = countTrailingZeros(NonZeros); + unsigned Idx = NonZeroMask.countTrailingZeros(); SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(Idx)); return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG); @@ -10364,12 +10501,12 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { // If element VT is < 32 bits, convert it to inserts into a zero vector. if (EVTBits == 8 && NumElems == 16) - if (SDValue V = LowerBuildVectorv16i8(Op, NonZeros, NumNonZero, NumZero, + if (SDValue V = LowerBuildVectorv16i8(Op, NonZeroMask, NumNonZero, NumZero, DAG, Subtarget)) return V; if (EVTBits == 16 && NumElems == 8) - if (SDValue V = LowerBuildVectorv8i16(Op, NonZeros, NumNonZero, NumZero, + if (SDValue V = LowerBuildVectorv8i16(Op, NonZeroMask, NumNonZero, NumZero, DAG, Subtarget)) return V; @@ -10382,7 +10519,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { if (NumElems == 4 && NumZero > 0) { SmallVector<SDValue, 8> Ops(NumElems); for (unsigned i = 0; i < 4; ++i) { - bool isZero = !(NonZeros & (1ULL << i)); + bool isZero = !NonZeroMask[i]; if (isZero) Ops[i] = getZeroVector(VT, Subtarget, DAG, dl); else @@ -10390,7 +10527,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { } for (unsigned i = 0; i < 2; ++i) { - switch ((NonZeros >> (i*2)) & 0x3) { + switch (NonZeroMask.extractBitsAsZExtValue(2, i * 2)) { default: llvm_unreachable("Unexpected NonZero count"); case 0: Ops[i] = Ops[i*2]; // Must be a zero vector. @@ -10407,8 +10544,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { } } - bool Reverse1 = (NonZeros & 0x3) == 2; - bool Reverse2 = ((NonZeros & (0x3 << 2)) >> 2) == 2; + bool Reverse1 = NonZeroMask.extractBitsAsZExtValue(2, 0) == 2; + bool Reverse2 = NonZeroMask.extractBitsAsZExtValue(2, 2) == 2; int MaskVec[] = { Reverse1 ? 1 : 0, Reverse1 ? 0 : 1, @@ -10680,6 +10817,35 @@ static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) { return isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), Mask); } +/// Test whether elements in each LaneSizeInBits lane in this shuffle mask come +/// from multiple lanes - this is different to isLaneCrossingShuffleMask to +/// better support 'repeated mask + lane permute' style shuffles. +static bool isMultiLaneShuffleMask(unsigned LaneSizeInBits, + unsigned ScalarSizeInBits, + ArrayRef<int> Mask) { + assert(LaneSizeInBits && ScalarSizeInBits && + (LaneSizeInBits % ScalarSizeInBits) == 0 && + "Illegal shuffle lane size"); + int NumElts = Mask.size(); + int NumEltsPerLane = LaneSizeInBits / ScalarSizeInBits; + int NumLanes = NumElts / NumEltsPerLane; + if (NumLanes > 1) { + for (int i = 0; i != NumLanes; ++i) { + int SrcLane = -1; + for (int j = 0; j != NumEltsPerLane; ++j) { + int M = Mask[(i * NumEltsPerLane) + j]; + if (M < 0) + continue; + int Lane = (M % NumElts) / NumEltsPerLane; + if (SrcLane >= 0 && SrcLane != Lane) + return true; + SrcLane = Lane; + } + } + } + return false; +} + /// Test whether a shuffle mask is equivalent within each sub-lane. /// /// This checks a shuffle mask to see if it is performing the same @@ -10741,10 +10907,11 @@ is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask, /// Test whether a target shuffle mask is equivalent within each sub-lane. /// Unlike isRepeatedShuffleMask we must respect SM_SentinelZero. -static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT, +static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, + unsigned EltSizeInBits, ArrayRef<int> Mask, SmallVectorImpl<int> &RepeatedMask) { - int LaneSize = LaneSizeInBits / VT.getScalarSizeInBits(); + int LaneSize = LaneSizeInBits / EltSizeInBits; RepeatedMask.assign(LaneSize, SM_SentinelUndef); int Size = Mask.size(); for (int i = 0; i < Size; ++i) { @@ -10775,6 +10942,67 @@ static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT, return true; } +/// Test whether a target shuffle mask is equivalent within each sub-lane. +/// Unlike isRepeatedShuffleMask we must respect SM_SentinelZero. +static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT, + ArrayRef<int> Mask, + SmallVectorImpl<int> &RepeatedMask) { + return isRepeatedTargetShuffleMask(LaneSizeInBits, VT.getScalarSizeInBits(), + Mask, RepeatedMask); +} + +/// Checks whether the vector elements referenced by two shuffle masks are +/// equivalent. +static bool IsElementEquivalent(int MaskSize, SDValue Op, SDValue ExpectedOp, + int Idx, int ExpectedIdx) { + assert(0 <= Idx && Idx < MaskSize && 0 <= ExpectedIdx && + ExpectedIdx < MaskSize && "Out of range element index"); + if (!Op || !ExpectedOp || Op.getOpcode() != ExpectedOp.getOpcode()) + return false; + + switch (Op.getOpcode()) { + case ISD::BUILD_VECTOR: + // If the values are build vectors, we can look through them to find + // equivalent inputs that make the shuffles equivalent. + // TODO: Handle MaskSize != Op.getNumOperands()? + if (MaskSize == (int)Op.getNumOperands() && + MaskSize == (int)ExpectedOp.getNumOperands()) + return Op.getOperand(Idx) == ExpectedOp.getOperand(ExpectedIdx); + break; + case X86ISD::VBROADCAST: + case X86ISD::VBROADCAST_LOAD: + // TODO: Handle MaskSize != Op.getValueType().getVectorNumElements()? + return (Op == ExpectedOp && + (int)Op.getValueType().getVectorNumElements() == MaskSize); + case X86ISD::HADD: + case X86ISD::HSUB: + case X86ISD::FHADD: + case X86ISD::FHSUB: + case X86ISD::PACKSS: + case X86ISD::PACKUS: + // HOP(X,X) can refer to the elt from the lower/upper half of a lane. + // TODO: Handle MaskSize != NumElts? + // TODO: Handle HOP(X,Y) vs HOP(Y,X) equivalence cases. + if (Op == ExpectedOp && Op.getOperand(0) == Op.getOperand(1)) { + MVT VT = Op.getSimpleValueType(); + int NumElts = VT.getVectorNumElements(); + if (MaskSize == NumElts) { + int NumLanes = VT.getSizeInBits() / 128; + int NumEltsPerLane = NumElts / NumLanes; + int NumHalfEltsPerLane = NumEltsPerLane / 2; + bool SameLane = + (Idx / NumEltsPerLane) == (ExpectedIdx / NumEltsPerLane); + bool SameElt = + (Idx % NumHalfEltsPerLane) == (ExpectedIdx % NumHalfEltsPerLane); + return SameLane && SameElt; + } + } + break; + } + + return false; +} + /// Checks whether a shuffle mask is equivalent to an explicit list of /// arguments. /// @@ -10785,30 +11013,26 @@ static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT, /// It returns true if the mask is exactly as wide as the argument list, and /// each element of the mask is either -1 (signifying undef) or the value given /// in the argument. -static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef<int> Mask, - ArrayRef<int> ExpectedMask) { - if (Mask.size() != ExpectedMask.size()) - return false; - +static bool isShuffleEquivalent(ArrayRef<int> Mask, ArrayRef<int> ExpectedMask, + SDValue V1 = SDValue(), + SDValue V2 = SDValue()) { int Size = Mask.size(); - - // If the values are build vectors, we can look through them to find - // equivalent inputs that make the shuffles equivalent. - auto *BV1 = dyn_cast<BuildVectorSDNode>(V1); - auto *BV2 = dyn_cast<BuildVectorSDNode>(V2); + if (Size != (int)ExpectedMask.size()) + return false; for (int i = 0; i < Size; ++i) { assert(Mask[i] >= -1 && "Out of bound mask element!"); - if (Mask[i] >= 0 && Mask[i] != ExpectedMask[i]) { - auto *MaskBV = Mask[i] < Size ? BV1 : BV2; - auto *ExpectedBV = ExpectedMask[i] < Size ? BV1 : BV2; - if (!MaskBV || !ExpectedBV || - MaskBV->getOperand(Mask[i] % Size) != - ExpectedBV->getOperand(ExpectedMask[i] % Size)) + int MaskIdx = Mask[i]; + int ExpectedIdx = ExpectedMask[i]; + if (0 <= MaskIdx && MaskIdx != ExpectedIdx) { + SDValue MaskV = MaskIdx < Size ? V1 : V2; + SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2; + MaskIdx = MaskIdx < Size ? MaskIdx : (MaskIdx - Size); + ExpectedIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size); + if (!IsElementEquivalent(Size, MaskV, ExpectedV, MaskIdx, ExpectedIdx)) return false; } } - return true; } @@ -10821,7 +11045,7 @@ static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef<int> Mask, /// /// SM_SentinelZero is accepted as a valid negative index but must match in /// both. -static bool isTargetShuffleEquivalent(ArrayRef<int> Mask, +static bool isTargetShuffleEquivalent(MVT VT, ArrayRef<int> Mask, ArrayRef<int> ExpectedMask, SDValue V1 = SDValue(), SDValue V2 = SDValue()) { @@ -10835,22 +11059,23 @@ static bool isTargetShuffleEquivalent(ArrayRef<int> Mask, if (!isUndefOrZeroOrInRange(Mask, 0, 2 * Size)) return false; - // If the values are build vectors, we can look through them to find - // equivalent inputs that make the shuffles equivalent. - auto *BV1 = dyn_cast_or_null<BuildVectorSDNode>(V1); - auto *BV2 = dyn_cast_or_null<BuildVectorSDNode>(V2); - BV1 = ((BV1 && Size != (int)BV1->getNumOperands()) ? nullptr : BV1); - BV2 = ((BV2 && Size != (int)BV2->getNumOperands()) ? nullptr : BV2); + // Don't use V1/V2 if they're not the same size as the shuffle mask type. + if (V1 && V1.getValueSizeInBits() != VT.getSizeInBits()) + V1 = SDValue(); + if (V2 && V2.getValueSizeInBits() != VT.getSizeInBits()) + V2 = SDValue(); for (int i = 0; i < Size; ++i) { - if (Mask[i] == SM_SentinelUndef || Mask[i] == ExpectedMask[i]) + int MaskIdx = Mask[i]; + int ExpectedIdx = ExpectedMask[i]; + if (MaskIdx == SM_SentinelUndef || MaskIdx == ExpectedIdx) continue; - if (0 <= Mask[i] && 0 <= ExpectedMask[i]) { - auto *MaskBV = Mask[i] < Size ? BV1 : BV2; - auto *ExpectedBV = ExpectedMask[i] < Size ? BV1 : BV2; - if (MaskBV && ExpectedBV && - MaskBV->getOperand(Mask[i] % Size) == - ExpectedBV->getOperand(ExpectedMask[i] % Size)) + if (0 <= MaskIdx && 0 <= ExpectedIdx) { + SDValue MaskV = MaskIdx < Size ? V1 : V2; + SDValue ExpectedV = ExpectedIdx < Size ? V1 : V2; + MaskIdx = MaskIdx < Size ? MaskIdx : (MaskIdx - Size); + ExpectedIdx = ExpectedIdx < Size ? ExpectedIdx : (ExpectedIdx - Size); + if (IsElementEquivalent(Size, MaskV, ExpectedV, MaskIdx, ExpectedIdx)) continue; } // TODO - handle SM_Sentinel equivalences. @@ -10862,20 +11087,25 @@ static bool isTargetShuffleEquivalent(ArrayRef<int> Mask, // Attempt to create a shuffle mask from a VSELECT condition mask. static bool createShuffleMaskFromVSELECT(SmallVectorImpl<int> &Mask, SDValue Cond) { - if (!ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) + EVT CondVT = Cond.getValueType(); + unsigned EltSizeInBits = CondVT.getScalarSizeInBits(); + unsigned NumElts = CondVT.getVectorNumElements(); + + APInt UndefElts; + SmallVector<APInt, 32> EltBits; + if (!getTargetConstantBitsFromNode(Cond, EltSizeInBits, UndefElts, EltBits, + true, false)) return false; - unsigned Size = Cond.getValueType().getVectorNumElements(); - Mask.resize(Size, SM_SentinelUndef); + Mask.resize(NumElts, SM_SentinelUndef); - for (int i = 0; i != (int)Size; ++i) { - SDValue CondElt = Cond.getOperand(i); + for (int i = 0; i != (int)NumElts; ++i) { Mask[i] = i; // Arbitrarily choose from the 2nd operand if the select condition element // is undef. // TODO: Can we do better by matching patterns such as even/odd? - if (CondElt.isUndef() || isNullConstant(CondElt)) - Mask[i] += Size; + if (UndefElts[i] || EltBits[i].isNullValue()) + Mask[i] += NumElts; } return true; @@ -10893,8 +11123,8 @@ static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT) { SmallVector<int, 8> Unpckhwd; createUnpackShuffleMask(MVT::v8i16, Unpckhwd, /* Lo = */ false, /* Unary = */ false); - bool IsUnpackwdMask = (isTargetShuffleEquivalent(Mask, Unpcklwd) || - isTargetShuffleEquivalent(Mask, Unpckhwd)); + bool IsUnpackwdMask = (isTargetShuffleEquivalent(VT, Mask, Unpcklwd) || + isTargetShuffleEquivalent(VT, Mask, Unpckhwd)); return IsUnpackwdMask; } @@ -10911,8 +11141,8 @@ static bool is128BitUnpackShuffleMask(ArrayRef<int> Mask) { for (unsigned i = 0; i != 4; ++i) { SmallVector<int, 16> UnpackMask; createUnpackShuffleMask(VT, UnpackMask, (i >> 1) % 2, i % 2); - if (isTargetShuffleEquivalent(Mask, UnpackMask) || - isTargetShuffleEquivalent(CommutedMask, UnpackMask)) + if (isTargetShuffleEquivalent(VT, Mask, UnpackMask) || + isTargetShuffleEquivalent(VT, CommutedMask, UnpackMask)) return true; } return false; @@ -10947,6 +11177,15 @@ static unsigned getV4X86ShuffleImm(ArrayRef<int> Mask) { assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!"); assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!"); + // If the mask only uses one non-undef element, then fully 'splat' it to + // improve later broadcast matching. + int FirstIndex = find_if(Mask, [](int M) { return M >= 0; }) - Mask.begin(); + assert(0 <= FirstIndex && FirstIndex < 4 && "All undef shuffle mask"); + + int FirstElt = Mask[FirstIndex]; + if (all_of(Mask, [FirstElt](int M) { return M < 0 || M == FirstElt; })) + return (FirstElt << 6) | (FirstElt << 4) | (FirstElt << 2) | FirstElt; + unsigned Imm = 0; Imm |= (Mask[0] < 0 ? 0 : Mask[0]) << 0; Imm |= (Mask[1] < 0 ? 1 : Mask[1]) << 2; @@ -11096,7 +11335,8 @@ static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2, // Attempt to match the target mask against the unpack lo/hi mask patterns. SmallVector<int, 64> Unpckl, Unpckh; createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, IsUnary); - if (isTargetShuffleEquivalent(TargetMask, Unpckl)) { + if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, V1, + (IsUnary ? V1 : V2))) { UnpackOpcode = X86ISD::UNPCKL; V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2)); V1 = (Undef1 ? DAG.getUNDEF(VT) : V1); @@ -11104,7 +11344,8 @@ static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2, } createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, IsUnary); - if (isTargetShuffleEquivalent(TargetMask, Unpckh)) { + if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, V1, + (IsUnary ? V1 : V2))) { UnpackOpcode = X86ISD::UNPCKH; V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2)); V1 = (Undef1 ? DAG.getUNDEF(VT) : V1); @@ -11142,14 +11383,14 @@ static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2, // If a binary shuffle, commute and try again. if (!IsUnary) { ShuffleVectorSDNode::commuteMask(Unpckl); - if (isTargetShuffleEquivalent(TargetMask, Unpckl)) { + if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl)) { UnpackOpcode = X86ISD::UNPCKL; std::swap(V1, V2); return true; } ShuffleVectorSDNode::commuteMask(Unpckh); - if (isTargetShuffleEquivalent(TargetMask, Unpckh)) { + if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh)) { UnpackOpcode = X86ISD::UNPCKH; std::swap(V1, V2); return true; @@ -11166,21 +11407,21 @@ static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT, SelectionDAG &DAG) { SmallVector<int, 8> Unpckl; createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false); - if (isShuffleEquivalent(V1, V2, Mask, Unpckl)) + if (isShuffleEquivalent(Mask, Unpckl, V1, V2)) return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2); SmallVector<int, 8> Unpckh; createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, /* Unary = */ false); - if (isShuffleEquivalent(V1, V2, Mask, Unpckh)) + if (isShuffleEquivalent(Mask, Unpckh, V1, V2)) return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2); // Commute and try again. ShuffleVectorSDNode::commuteMask(Unpckl); - if (isShuffleEquivalent(V1, V2, Mask, Unpckl)) + if (isShuffleEquivalent(Mask, Unpckl, V1, V2)) return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1); ShuffleVectorSDNode::commuteMask(Unpckh); - if (isShuffleEquivalent(V1, V2, Mask, Unpckh)) + if (isShuffleEquivalent(Mask, Unpckh, V1, V2)) return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1); return SDValue(); @@ -11196,9 +11437,9 @@ static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT, createSplat2ShuffleMask(VT, Unpckh, /* Lo */ false); unsigned UnpackOpcode; - if (isShuffleEquivalent(V1, V2, Mask, Unpckl)) + if (isShuffleEquivalent(Mask, Unpckl, V1, V2)) UnpackOpcode = X86ISD::UNPCKL; - else if (isShuffleEquivalent(V1, V2, Mask, Unpckh)) + else if (isShuffleEquivalent(Mask, Unpckh, V1, V2)) UnpackOpcode = X86ISD::UNPCKH; else return SDValue(); @@ -11214,7 +11455,6 @@ static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT, // Check if the mask can be mapped to a TRUNCATE or VTRUNC, truncating the // source into the lower elements and zeroing the upper elements. -// TODO: Merge with matchShuffleAsVPMOV. static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT, ArrayRef<int> Mask, const APInt &Zeroable, const X86Subtarget &Subtarget) { @@ -11251,22 +11491,51 @@ static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT, return false; } -static bool matchShuffleAsVPMOV(ArrayRef<int> Mask, bool SwappedOps, - int Delta) { - int Size = (int)Mask.size(); - int Split = Size / Delta; - int TruncatedVectorStart = SwappedOps ? Size : 0; +// Helper to create TRUNCATE/VTRUNC nodes, optionally with zero/undef upper +// element padding to the final DstVT. +static SDValue getAVX512TruncNode(const SDLoc &DL, MVT DstVT, SDValue Src, + const X86Subtarget &Subtarget, + SelectionDAG &DAG, bool ZeroUppers) { + MVT SrcVT = Src.getSimpleValueType(); + MVT DstSVT = DstVT.getScalarType(); + unsigned NumDstElts = DstVT.getVectorNumElements(); + unsigned NumSrcElts = SrcVT.getVectorNumElements(); + unsigned DstEltSizeInBits = DstVT.getScalarSizeInBits(); - // Match for mask starting with e.g.: <8, 10, 12, 14,... or <0, 2, 4, 6,... - if (!isSequentialOrUndefInRange(Mask, 0, Split, TruncatedVectorStart, Delta)) - return false; + if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT)) + return SDValue(); - // The rest of the mask should not refer to the truncated vector's elements. - if (isAnyInRange(Mask.slice(Split, Size - Split), TruncatedVectorStart, - TruncatedVectorStart + Size)) - return false; + // Perform a direct ISD::TRUNCATE if possible. + if (NumSrcElts == NumDstElts) + return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Src); - return true; + if (NumSrcElts > NumDstElts) { + MVT TruncVT = MVT::getVectorVT(DstSVT, NumSrcElts); + SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src); + return extractSubVector(Trunc, 0, DAG, DL, DstVT.getSizeInBits()); + } + + if ((NumSrcElts * DstEltSizeInBits) >= 128) { + MVT TruncVT = MVT::getVectorVT(DstSVT, NumSrcElts); + SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Src); + return widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL, + DstVT.getSizeInBits()); + } + + // Non-VLX targets must truncate from a 512-bit type, so we need to + // widen, truncate and then possibly extract the original subvector. + if (!Subtarget.hasVLX() && !SrcVT.is512BitVector()) { + SDValue NewSrc = widenSubVector(Src, ZeroUppers, Subtarget, DAG, DL, 512); + return getAVX512TruncNode(DL, DstVT, NewSrc, Subtarget, DAG, ZeroUppers); + } + + // Fallback to a X86ISD::VTRUNC, padding if necessary. + MVT TruncVT = MVT::getVectorVT(DstSVT, 128 / DstEltSizeInBits); + SDValue Trunc = DAG.getNode(X86ISD::VTRUNC, DL, TruncVT, Src); + if (DstVT != TruncVT) + Trunc = widenSubVector(Trunc, ZeroUppers, Subtarget, DAG, DL, + DstVT.getSizeInBits()); + return Trunc; } // Try to lower trunc+vector_shuffle to a vpmovdb or a vpmovdw instruction. @@ -11282,66 +11551,99 @@ static bool matchShuffleAsVPMOV(ArrayRef<int> Mask, bool SwappedOps, // t51: v8i16 = vector_shuffle<0,2,4,6,12,13,14,15> t41, t21 // t18: v2i64 = bitcast t51 // -// Without avx512vl, this is lowered to: -// -// vpmovqd %zmm0, %ymm0 -// vpshufb {{.*#+}} xmm0 = -// xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero -// -// But when avx512vl is available, one can just use a single vpmovdw -// instruction. -static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, ArrayRef<int> Mask, - MVT VT, SDValue V1, SDValue V2, - SelectionDAG &DAG, - const X86Subtarget &Subtarget) { - if (VT != MVT::v16i8 && VT != MVT::v8i16) +// One can just use a single vpmovdw instruction, without avx512vl we need to +// use the zmm variant and extract the lower subvector, padding with zeroes. +// TODO: Merge with lowerShuffleAsVTRUNC. +static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef<int> Mask, + const APInt &Zeroable, + const X86Subtarget &Subtarget, + SelectionDAG &DAG) { + assert((VT == MVT::v16i8 || VT == MVT::v8i16) && "Unexpected VTRUNC type"); + if (!Subtarget.hasAVX512()) return SDValue(); - if (Mask.size() != VT.getVectorNumElements()) - return SDValue(); + unsigned NumElts = VT.getVectorNumElements(); + unsigned EltSizeInBits = VT.getScalarSizeInBits(); + unsigned MaxScale = 64 / EltSizeInBits; + for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) { + unsigned NumSrcElts = NumElts / Scale; + unsigned UpperElts = NumElts - NumSrcElts; + if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale) || + !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnesValue()) + continue; - bool SwappedOps = false; + SDValue Src = V1; + if (!Src.hasOneUse()) + return SDValue(); - if (!ISD::isBuildVectorAllZeros(V2.getNode())) { - if (!ISD::isBuildVectorAllZeros(V1.getNode())) + Src = peekThroughOneUseBitcasts(Src); + if (Src.getOpcode() != ISD::TRUNCATE || + Src.getScalarValueSizeInBits() != (EltSizeInBits * Scale)) return SDValue(); + Src = Src.getOperand(0); - std::swap(V1, V2); - SwappedOps = true; + // VPMOVWB is only available with avx512bw. + MVT SrcVT = Src.getSimpleValueType(); + if (SrcVT.getVectorElementType() == MVT::i16 && VT == MVT::v16i8 && + !Subtarget.hasBWI()) + return SDValue(); + + bool UndefUppers = isUndefInRange(Mask, NumSrcElts, UpperElts); + return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers); } - // Look for: - // - // bitcast (truncate <8 x i32> %vec to <8 x i16>) to <16 x i8> - // bitcast (truncate <4 x i64> %vec to <4 x i32>) to <8 x i16> - // - // and similar ones. - if (V1.getOpcode() != ISD::BITCAST) - return SDValue(); - if (V1.getOperand(0).getOpcode() != ISD::TRUNCATE) + return SDValue(); +} + +// Attempt to match binary shuffle patterns as a truncate. +static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1, + SDValue V2, ArrayRef<int> Mask, + const APInt &Zeroable, + const X86Subtarget &Subtarget, + SelectionDAG &DAG) { + assert((VT.is128BitVector() || VT.is256BitVector()) && + "Unexpected VTRUNC type"); + if (!Subtarget.hasAVX512()) return SDValue(); - SDValue Src = V1.getOperand(0).getOperand(0); - MVT SrcVT = Src.getSimpleValueType(); + unsigned NumElts = VT.getVectorNumElements(); + unsigned EltSizeInBits = VT.getScalarSizeInBits(); + unsigned MaxScale = 64 / EltSizeInBits; + for (unsigned Scale = 2; Scale <= MaxScale; Scale += Scale) { + // TODO: Support non-BWI VPMOVWB truncations? + unsigned SrcEltBits = EltSizeInBits * Scale; + if (SrcEltBits < 32 && !Subtarget.hasBWI()) + continue; - // The vptrunc** instructions truncating 128 bit and 256 bit vectors - // are only available with avx512vl. - if (!SrcVT.is512BitVector() && !Subtarget.hasVLX()) - return SDValue(); + // Match shuffle <0,Scale,2*Scale,..,undef_or_zero,undef_or_zero,...> + // Bail if the V2 elements are undef. + unsigned NumHalfSrcElts = NumElts / Scale; + unsigned NumSrcElts = 2 * NumHalfSrcElts; + if (!isSequentialOrUndefInRange(Mask, 0, NumSrcElts, 0, Scale) || + isUndefInRange(Mask, NumHalfSrcElts, NumHalfSrcElts)) + continue; - // Down Convert Word to Byte is only available with avx512bw. The case with - // 256-bit output doesn't contain a shuffle and is therefore not handled here. - if (SrcVT.getVectorElementType() == MVT::i16 && VT == MVT::v16i8 && - !Subtarget.hasBWI()) - return SDValue(); + // The elements beyond the truncation must be undef/zero. + unsigned UpperElts = NumElts - NumSrcElts; + if (UpperElts > 0 && + !Zeroable.extractBits(UpperElts, NumSrcElts).isAllOnesValue()) + continue; + bool UndefUppers = + UpperElts > 0 && isUndefInRange(Mask, NumSrcElts, UpperElts); - // The first half/quarter of the mask should refer to every second/fourth - // element of the vector truncated and bitcasted. - if (!matchShuffleAsVPMOV(Mask, SwappedOps, 2) && - !matchShuffleAsVPMOV(Mask, SwappedOps, 4)) - return SDValue(); + // As we're using both sources then we need to concat them together + // and truncate from the double-sized src. + MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2); + SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2); - return DAG.getNode(X86ISD::VTRUNC, DL, VT, Src); + MVT SrcSVT = MVT::getIntegerVT(SrcEltBits); + MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts); + Src = DAG.getBitcast(SrcVT, Src); + return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers); + } + + return SDValue(); } /// Check whether a compaction lowering can be done by dropping even @@ -11459,14 +11761,14 @@ static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, // Try binary shuffle. SmallVector<int, 32> BinaryMask; createPackShuffleMask(VT, BinaryMask, false, NumStages); - if (isTargetShuffleEquivalent(TargetMask, BinaryMask, V1, V2)) + if (isTargetShuffleEquivalent(VT, TargetMask, BinaryMask, V1, V2)) if (MatchPACK(V1, V2, PackVT)) return true; // Try unary shuffle. SmallVector<int, 32> UnaryMask; createPackShuffleMask(VT, UnaryMask, true, NumStages); - if (isTargetShuffleEquivalent(TargetMask, UnaryMask, V1)) + if (isTargetShuffleEquivalent(VT, TargetMask, UnaryMask, V1)) if (MatchPACK(V1, V1, PackVT)) return true; } @@ -12015,23 +12317,32 @@ static SDValue lowerShuffleAsByteRotateAndPermute( /// This matches the extremely common pattern for handling combined /// shuffle+blend operations on newer X86 ISAs where we have very fast blend /// operations. It will try to pick the best arrangement of shuffles and -/// blends. -static SDValue lowerShuffleAsDecomposedShuffleBlend( +/// blends. For vXi8/vXi16 shuffles we may use unpack instead of blend. +static SDValue lowerShuffleAsDecomposedShuffleMerge( const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) { + int NumElts = Mask.size(); + int NumLanes = VT.getSizeInBits() / 128; + int NumEltsPerLane = NumElts / NumLanes; + // Shuffle the input elements into the desired positions in V1 and V2 and - // blend them together. - SmallVector<int, 32> V1Mask(Mask.size(), -1); - SmallVector<int, 32> V2Mask(Mask.size(), -1); - SmallVector<int, 32> BlendMask(Mask.size(), -1); - for (int i = 0, Size = Mask.size(); i < Size; ++i) - if (Mask[i] >= 0 && Mask[i] < Size) { - V1Mask[i] = Mask[i]; - BlendMask[i] = i; - } else if (Mask[i] >= Size) { - V2Mask[i] = Mask[i] - Size; - BlendMask[i] = i + Size; + // unpack/blend them together. + bool IsAlternating = true; + SmallVector<int, 32> V1Mask(NumElts, -1); + SmallVector<int, 32> V2Mask(NumElts, -1); + SmallVector<int, 32> FinalMask(NumElts, -1); + for (int i = 0; i < NumElts; ++i) { + int M = Mask[i]; + if (M >= 0 && M < NumElts) { + V1Mask[i] = M; + FinalMask[i] = i; + IsAlternating &= (i & 1) == 0; + } else if (M >= NumElts) { + V2Mask[i] = M - NumElts; + FinalMask[i] = i + NumElts; + IsAlternating &= (i & 1) == 1; } + } // Try to lower with the simpler initial blend/unpack/rotate strategies unless // one of the input shuffles would be a no-op. We prefer to shuffle inputs as @@ -12055,9 +12366,30 @@ static SDValue lowerShuffleAsDecomposedShuffleBlend( return BlendPerm; } + // If the final mask is an alternating blend of vXi8/vXi16, convert to an + // UNPCKL(SHUFFLE, SHUFFLE) pattern. + // TODO: It doesn't have to be alternating - but each lane mustn't have more + // than half the elements coming from each source. + if (IsAlternating && VT.getScalarSizeInBits() < 32) { + V1Mask.assign(NumElts, -1); + V2Mask.assign(NumElts, -1); + FinalMask.assign(NumElts, -1); + for (int i = 0; i != NumElts; i += NumEltsPerLane) + for (int j = 0; j != NumEltsPerLane; ++j) { + int M = Mask[i + j]; + if (M >= 0 && M < NumElts) { + V1Mask[i + (j / 2)] = M; + FinalMask[i + j] = i + (j / 2); + } else if (M >= NumElts) { + V2Mask[i + (j / 2)] = M - NumElts; + FinalMask[i + j] = i + (j / 2) + NumElts; + } + } + } + V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); - return DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask); + return DAG.getVectorShuffle(VT, DL, V1, V2, FinalMask); } /// Try to lower a vector shuffle as a bit rotation. @@ -12715,8 +13047,8 @@ static SDValue lowerShuffleAsSpecificZeroOrAnyExtend( MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale), NumElements / Scale); InputV = ShuffleOffset(InputV); - InputV = getExtendInVec(AnyExt ? ISD::ANY_EXTEND : ISD::ZERO_EXTEND, DL, - ExtVT, InputV, DAG); + InputV = getEXTEND_VECTOR_INREG(AnyExt ? ISD::ANY_EXTEND : ISD::ZERO_EXTEND, + DL, ExtVT, InputV, DAG); return DAG.getBitcast(VT, InputV); } @@ -13324,7 +13656,8 @@ static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1, MVT SVT = VT.getScalarType(); unsigned Offset = BroadcastIdx * SVT.getStoreSize(); assert((int)(Offset * 8) == BitOffset && "Unexpected bit-offset"); - SDValue NewAddr = DAG.getMemBasePlusOffset(BaseAddr, Offset, DL); + SDValue NewAddr = + DAG.getMemBasePlusOffset(BaseAddr, TypeSize::Fixed(Offset), DL); // Directly form VBROADCAST_LOAD if we're using VBROADCAST opcode rather // than MOVDDUP. @@ -13497,7 +13830,7 @@ static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2, assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); // Attempt to match the insertps pattern. - unsigned InsertPSMask; + unsigned InsertPSMask = 0; if (!matchShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG)) return SDValue(); @@ -13685,8 +14018,8 @@ static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // Try to use one of the special instruction patterns to handle two common // blend patterns if a zero-blend above didn't work. - if (isShuffleEquivalent(V1, V2, Mask, {0, 3}) || - isShuffleEquivalent(V1, V2, Mask, {1, 3})) + if (isShuffleEquivalent(Mask, {0, 3}, V1, V2) || + isShuffleEquivalent(Mask, {1, 3}, V1, V2)) if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG)) // We can either use a special instruction to load over the low double or // to move just the low double. @@ -13732,9 +14065,10 @@ static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // onward this has a single fast instruction with no scary immediates. // We have to map the mask as it is actually a v4i32 shuffle instruction. V1 = DAG.getBitcast(MVT::v4i32, V1); - int WidenedMask[4] = { - std::max(Mask[0], 0) * 2, std::max(Mask[0], 0) * 2 + 1, - std::max(Mask[1], 0) * 2, std::max(Mask[1], 0) * 2 + 1}; + int WidenedMask[4] = {Mask[0] < 0 ? -1 : (Mask[0] * 2), + Mask[0] < 0 ? -1 : ((Mask[0] * 2) + 1), + Mask[1] < 0 ? -1 : (Mask[1] * 2), + Mask[1] < 0 ? -1 : ((Mask[1] * 2) + 1)}; return DAG.getBitcast( MVT::v2i64, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, @@ -13794,7 +14128,7 @@ static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // If we have direct support for blends, we should lower by decomposing into // a permute. That will be faster than the domain cross. if (IsBlendSupported) - return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2, Mask, + return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG); // We implement this with SHUFPD which is pretty lame because it will likely @@ -13888,6 +14222,12 @@ static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT, NewMask[2] = Mask[2] < 4 ? 1 : 3; NewMask[3] = Mask[2] < 4 ? 3 : 1; } + } else if (NumV2Elements == 3) { + // Ideally canonicalizeShuffleMaskWithCommute should have caught this, but + // we can get here due to other paths (e.g repeated mask matching) that we + // don't want to do another round of lowerVECTOR_SHUFFLE. + ShuffleVectorSDNode::commuteMask(NewMask); + return lowerShuffleWithSHUFPS(DL, VT, NewMask, V2, V1, DAG); } return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV, getV4X86ShuffleImm8ForMask(NewMask, DL, DAG)); @@ -13916,9 +14256,9 @@ static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // Use even/odd duplicate instructions for masks that match their pattern. if (Subtarget.hasSSE3()) { - if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 2, 2})) + if (isShuffleEquivalent(Mask, {0, 0, 2, 2}, V1, V2)) return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1); - if (isShuffleEquivalent(V1, V2, Mask, {1, 1, 3, 3})) + if (isShuffleEquivalent(Mask, {1, 1, 3, 3}, V1, V2)) return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1); } @@ -13932,9 +14272,9 @@ static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // Use MOVLHPS/MOVHLPS to simulate unary shuffles. These are only valid // in SSE1 because otherwise they are widened to v2f64 and never get here. if (!Subtarget.hasSSE2()) { - if (isShuffleEquivalent(V1, V2, Mask, {0, 1, 0, 1})) + if (isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1, V2)) return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1); - if (isShuffleEquivalent(V1, V2, Mask, {2, 3, 2, 3})) + if (isShuffleEquivalent(Mask, {2, 3, 2, 3}, V1, V2)) return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1); } @@ -13976,9 +14316,9 @@ static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // Use low/high mov instructions. These are only valid in SSE1 because // otherwise they are widened to v2f64 and never get here. if (!Subtarget.hasSSE2()) { - if (isShuffleEquivalent(V1, V2, Mask, {0, 1, 4, 5})) + if (isShuffleEquivalent(Mask, {0, 1, 4, 5}, V1, V2)) return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2); - if (isShuffleEquivalent(V1, V2, Mask, {2, 3, 6, 7})) + if (isShuffleEquivalent(Mask, {2, 3, 6, 7}, V1, V2)) return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1); } @@ -14026,9 +14366,9 @@ static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // so prevents folding a load into this instruction or making a copy. const int UnpackLoMask[] = {0, 0, 1, 1}; const int UnpackHiMask[] = {2, 2, 3, 3}; - if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 1, 1})) + if (isShuffleEquivalent(Mask, {0, 0, 1, 1}, V1, V2)) Mask = UnpackLoMask; - else if (isShuffleEquivalent(V1, V2, Mask, {2, 2, 3, 3})) + else if (isShuffleEquivalent(Mask, {2, 2, 3, 3}, V1, V2)) Mask = UnpackHiMask; return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, @@ -14086,7 +14426,7 @@ static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // If we have direct support for blends, we should lower by decomposing into // a permute. That will be faster than the domain cross. if (IsBlendSupported) - return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2, Mask, + return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG); // Try to lower by permuting the inputs into an unpack instruction. @@ -14695,6 +15035,11 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Zeroable, Subtarget, DAG)) return ZExt; + // Try to use lower using a truncation. + if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v8i16, V1, V2, Mask, Zeroable, + Subtarget, DAG)) + return V; + int NumV2Inputs = count_if(Mask, [](int M) { return M >= 8; }); if (NumV2Inputs == 0) { @@ -14775,6 +15120,11 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Subtarget)) return V; + // Try to use lower using a truncation. + if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v8i16, V1, V2, Mask, Zeroable, + Subtarget, DAG)) + return V; + // Try to use byte rotation instructions. if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG)) @@ -14826,22 +15176,49 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask, } // We can always bit-blend if we have to so the fallback strategy is to - // decompose into single-input permutes and blends. - return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2, + // decompose into single-input permutes and blends/unpacks. + return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG); } +// Lowers unary/binary shuffle as VPERMV/VPERMV3, for non-VLX targets, +// sub-512-bit shuffles are padded to 512-bits for the shuffle and then +// the active subvector is extracted. static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT, - ArrayRef<int> Mask, SDValue V1, - SDValue V2, SelectionDAG &DAG) { - MVT MaskEltVT = MVT::getIntegerVT(VT.getScalarSizeInBits()); - MVT MaskVecVT = MVT::getVectorVT(MaskEltVT, VT.getVectorNumElements()); + ArrayRef<int> Mask, SDValue V1, SDValue V2, + const X86Subtarget &Subtarget, + SelectionDAG &DAG) { + MVT MaskVT = VT.changeTypeToInteger(); + SDValue MaskNode; + MVT ShuffleVT = VT; + if (!VT.is512BitVector() && !Subtarget.hasVLX()) { + V1 = widenSubVector(V1, false, Subtarget, DAG, DL, 512); + V2 = widenSubVector(V2, false, Subtarget, DAG, DL, 512); + ShuffleVT = V1.getSimpleValueType(); + + // Adjust mask to correct indices for the second input. + int NumElts = VT.getVectorNumElements(); + unsigned Scale = 512 / VT.getSizeInBits(); + SmallVector<int, 32> AdjustedMask(Mask.begin(), Mask.end()); + for (int &M : AdjustedMask) + if (NumElts <= M) + M += (Scale - 1) * NumElts; + MaskNode = getConstVector(AdjustedMask, MaskVT, DAG, DL, true); + MaskNode = widenSubVector(MaskNode, false, Subtarget, DAG, DL, 512); + } else { + MaskNode = getConstVector(Mask, MaskVT, DAG, DL, true); + } - SDValue MaskNode = getConstVector(Mask, MaskVecVT, DAG, DL, true); + SDValue Result; if (V2.isUndef()) - return DAG.getNode(X86ISD::VPERMV, DL, VT, MaskNode, V1); + Result = DAG.getNode(X86ISD::VPERMV, DL, ShuffleVT, MaskNode, V1); + else + Result = DAG.getNode(X86ISD::VPERMV3, DL, ShuffleVT, V1, MaskNode, V2); - return DAG.getNode(X86ISD::VPERMV3, DL, VT, V1, MaskNode, V2); + if (VT != ShuffleVT) + Result = extractSubVector(Result, 0, DAG, DL, VT.getSizeInBits()); + + return Result; } /// Generic lowering of v16i8 shuffles. @@ -14879,6 +15256,15 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Zeroable, Subtarget, DAG)) return ZExt; + // Try to use lower using a truncation. + if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v16i8, V1, V2, Mask, Zeroable, + Subtarget, DAG)) + return V; + + if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i8, V1, V2, Mask, Zeroable, + Subtarget, DAG)) + return V; + // See if we can use SSE4A Extraction / Insertion. if (Subtarget.hasSSE4A()) if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask, @@ -15061,9 +15447,16 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask, DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) return Unpack; - // If we have VBMI we can use one VPERM instead of multiple PSHUFBs. - if (Subtarget.hasVBMI() && Subtarget.hasVLX()) - return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, DAG); + // AVX512VBMI can lower to VPERMB (non-VLX will pad to v64i8). + if (Subtarget.hasVBMI()) + return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, Subtarget, + DAG); + + // If we have XOP we can use one VPPERM instead of multiple PSHUFBs. + if (Subtarget.hasXOP()) { + SDValue MaskNode = getConstVector(Mask, MVT::v16i8, DAG, DL, true); + return DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, V1, V2, MaskNode); + } // Use PALIGNR+Permute if possible - permute might become PSHUFB but the // PALIGNR will be cheaper than the second PSHUFB+OR. @@ -15119,9 +15512,9 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask, return Result; } - // Handle multi-input cases by blending single-input shuffles. + // Handle multi-input cases by blending/unpacking single-input shuffles. if (NumV2Elements > 0) - return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2, Mask, + return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG); // The fallback path for single-input shuffles widens this into two v8i16 @@ -15301,7 +15694,7 @@ static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1, } /// Either split a vector in halves or decompose the shuffles and the -/// blend. +/// blend/unpack. /// /// This is provided as a good fallback for many lowerings of non-single-input /// shuffles with more than one 128-bit lane. In those cases, we want to select @@ -15336,8 +15729,8 @@ static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1, return true; }; if (DoBothBroadcast()) - return lowerShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, - Subtarget, DAG); + return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget, + DAG); // If the inputs all stem from a single 128-bit lane of each input, then we // split them rather than blending because the split will decompose to @@ -15353,9 +15746,9 @@ static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1, if (LaneInputs[0].count() <= 1 && LaneInputs[1].count() <= 1) return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG); - // Otherwise, just fall back to decomposed shuffles and a blend. This requires - // that the decomposed single-input shuffles don't end up here. - return lowerShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, Subtarget, + // Otherwise, just fall back to decomposed shuffles and a blend/unpack. This + // requires that the decomposed single-input shuffles don't end up here. + return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget, DAG); } @@ -15403,53 +15796,94 @@ static SDValue lowerShuffleAsLanePermuteAndPermute( int NumElts = VT.getVectorNumElements(); int NumLanes = VT.getSizeInBits() / 128; int NumEltsPerLane = NumElts / NumLanes; + bool CanUseSublanes = Subtarget.hasAVX2() && V2.isUndef(); - SmallVector<int, 4> SrcLaneMask(NumLanes, SM_SentinelUndef); - SmallVector<int, 16> PermMask(NumElts, SM_SentinelUndef); + /// Attempts to find a sublane permute with the given size + /// that gets all elements into their target lanes. + /// + /// If successful, fills CrossLaneMask and InLaneMask and returns true. + /// If unsuccessful, returns false and may overwrite InLaneMask. + auto getSublanePermute = [&](int NumSublanes) -> SDValue { + int NumSublanesPerLane = NumSublanes / NumLanes; + int NumEltsPerSublane = NumElts / NumSublanes; - for (int i = 0; i != NumElts; ++i) { - int M = Mask[i]; - if (M < 0) - continue; + SmallVector<int, 16> CrossLaneMask; + SmallVector<int, 16> InLaneMask(NumElts, SM_SentinelUndef); + // CrossLaneMask but one entry == one sublane. + SmallVector<int, 16> CrossLaneMaskLarge(NumSublanes, SM_SentinelUndef); - // Ensure that each lane comes from a single source lane. - int SrcLane = M / NumEltsPerLane; - int DstLane = i / NumEltsPerLane; - if (!isUndefOrEqual(SrcLaneMask[DstLane], SrcLane)) - return SDValue(); - SrcLaneMask[DstLane] = SrcLane; + for (int i = 0; i != NumElts; ++i) { + int M = Mask[i]; + if (M < 0) + continue; - PermMask[i] = (DstLane * NumEltsPerLane) + (M % NumEltsPerLane); - } + int SrcSublane = M / NumEltsPerSublane; + int DstLane = i / NumEltsPerLane; - // Make sure we set all elements of the lane mask, to avoid undef propagation. - SmallVector<int, 16> LaneMask(NumElts, SM_SentinelUndef); - for (int DstLane = 0; DstLane != NumLanes; ++DstLane) { - int SrcLane = SrcLaneMask[DstLane]; - if (0 <= SrcLane) - for (int j = 0; j != NumEltsPerLane; ++j) { - LaneMask[(DstLane * NumEltsPerLane) + j] = - (SrcLane * NumEltsPerLane) + j; + // We only need to get the elements into the right lane, not sublane. + // So search all sublanes that make up the destination lane. + bool Found = false; + int DstSubStart = DstLane * NumSublanesPerLane; + int DstSubEnd = DstSubStart + NumSublanesPerLane; + for (int DstSublane = DstSubStart; DstSublane < DstSubEnd; ++DstSublane) { + if (!isUndefOrEqual(CrossLaneMaskLarge[DstSublane], SrcSublane)) + continue; + + Found = true; + CrossLaneMaskLarge[DstSublane] = SrcSublane; + int DstSublaneOffset = DstSublane * NumEltsPerSublane; + InLaneMask[i] = DstSublaneOffset + M % NumEltsPerSublane; + break; } - } + if (!Found) + return SDValue(); + } - // If we're only shuffling a single lowest lane and the rest are identity - // then don't bother. - // TODO - isShuffleMaskInputInPlace could be extended to something like this. - int NumIdentityLanes = 0; - bool OnlyShuffleLowestLane = true; - for (int i = 0; i != NumLanes; ++i) { - if (isSequentialOrUndefInRange(PermMask, i * NumEltsPerLane, NumEltsPerLane, - i * NumEltsPerLane)) - NumIdentityLanes++; - else if (SrcLaneMask[i] != 0 && SrcLaneMask[i] != NumLanes) - OnlyShuffleLowestLane = false; - } - if (OnlyShuffleLowestLane && NumIdentityLanes == (NumLanes - 1)) + // Fill CrossLaneMask using CrossLaneMaskLarge. + narrowShuffleMaskElts(NumEltsPerSublane, CrossLaneMaskLarge, CrossLaneMask); + + if (!CanUseSublanes) { + // If we're only shuffling a single lowest lane and the rest are identity + // then don't bother. + // TODO - isShuffleMaskInputInPlace could be extended to something like + // this. + int NumIdentityLanes = 0; + bool OnlyShuffleLowestLane = true; + for (int i = 0; i != NumLanes; ++i) { + int LaneOffset = i * NumEltsPerLane; + if (isSequentialOrUndefInRange(InLaneMask, LaneOffset, NumEltsPerLane, + i * NumEltsPerLane)) + NumIdentityLanes++; + else if (CrossLaneMask[LaneOffset] != 0) + OnlyShuffleLowestLane = false; + } + if (OnlyShuffleLowestLane && NumIdentityLanes == (NumLanes - 1)) + return SDValue(); + } + + SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask); + return DAG.getVectorShuffle(VT, DL, CrossLane, DAG.getUNDEF(VT), + InLaneMask); + }; + + // First attempt a solution with full lanes. + if (SDValue V = getSublanePermute(/*NumSublanes=*/NumLanes)) + return V; + + // The rest of the solutions use sublanes. + if (!CanUseSublanes) + return SDValue(); + + // Then attempt a solution with 64-bit sublanes (vpermq). + if (SDValue V = getSublanePermute(/*NumSublanes=*/NumLanes * 2)) + return V; + + // If that doesn't work and we have fast variable shuffle, + // attempt 32-bit sublanes (vpermd). + if (!Subtarget.hasFastVariableShuffle()) return SDValue(); - SDValue LanePermute = DAG.getVectorShuffle(VT, DL, V1, V2, LaneMask); - return DAG.getVectorShuffle(VT, DL, LanePermute, DAG.getUNDEF(VT), PermMask); + return getSublanePermute(/*NumSublanes=*/NumLanes * 4); } /// Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one @@ -15562,8 +15996,8 @@ static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1, if (!IsLowZero && !IsHighZero) { // Check for patterns which can be matched with a single insert of a 128-bit // subvector. - bool OnlyUsesV1 = isShuffleEquivalent(V1, V2, Mask, {0, 1, 0, 1}); - if (OnlyUsesV1 || isShuffleEquivalent(V1, V2, Mask, {0, 1, 4, 5})) { + bool OnlyUsesV1 = isShuffleEquivalent(Mask, {0, 1, 0, 1}, V1, V2); + if (OnlyUsesV1 || isShuffleEquivalent(Mask, {0, 1, 4, 5}, V1, V2)) { // With AVX1, use vperm2f128 (below) to allow load folding. Otherwise, // this will likely become vinsertf128 which can't fold a 256-bit memop. @@ -16305,7 +16739,7 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, return Broadcast; // Use low duplicate instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 2, 2})) + if (isShuffleEquivalent(Mask, {0, 0, 2, 2}, V1, V2)) return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1); if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) { @@ -16366,7 +16800,7 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // If we have one input in place, then we can permute the other input and // blend the result. if (isShuffleMaskInputInPlace(0, Mask) || isShuffleMaskInputInPlace(1, Mask)) - return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, Mask, + return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG); // Try to create an in-lane repeating shuffle mask and then shuffle the @@ -16394,7 +16828,7 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // If we have AVX2 then we always want to lower with a blend because an v4 we // can fully permute the elements. if (Subtarget.hasAVX2()) - return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, Mask, + return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG); // Otherwise fall back on generic lowering. @@ -16476,7 +16910,7 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // If we have one input in place, then we can permute the other input and // blend the result. if (isShuffleMaskInputInPlace(0, Mask) || isShuffleMaskInputInPlace(1, Mask)) - return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, Mask, + return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG); // Try to create an in-lane repeating shuffle mask and then shuffle the @@ -16496,7 +16930,7 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, return Result; // Otherwise fall back on generic blend lowering. - return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, Mask, + return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG); } @@ -16529,9 +16963,9 @@ static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, "Repeated masks must be half the mask width!"); // Use even/odd duplicate instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, RepeatedMask, {0, 0, 2, 2})) + if (isShuffleEquivalent(RepeatedMask, {0, 0, 2, 2}, V1, V2)) return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1); - if (isShuffleEquivalent(V1, V2, RepeatedMask, {1, 1, 3, 3})) + if (isShuffleEquivalent(RepeatedMask, {1, 1, 3, 3}, V1, V2)) return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1); if (V2.isUndef()) @@ -16585,14 +17019,13 @@ static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // since after split we get a more efficient code using vpunpcklwd and // vpunpckhwd instrs than vblend. if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32)) - if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, - Subtarget, DAG)) - return V; + return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Subtarget, + DAG); // If we have AVX2 then we always want to lower with a blend because at v8 we // can fully permute the elements. if (Subtarget.hasAVX2()) - return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2, Mask, + return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG); // Otherwise fall back on generic lowering. @@ -16625,9 +17058,8 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // vpunpcklwd and vpunpckhwd instrs. if (isUnpackWdShuffleMask(Mask, MVT::v8i32) && !V2.isUndef() && !Subtarget.hasAVX512()) - if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, - Subtarget, DAG)) - return V; + return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Subtarget, + DAG); if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) @@ -16712,7 +17144,7 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, return Result; // Otherwise fall back on generic blend lowering. - return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2, Mask, + return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG); } @@ -16754,6 +17186,11 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Subtarget)) return V; + // Try to use lower using a truncation. + if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i16, V1, V2, Mask, Zeroable, + Subtarget, DAG)) + return V; + // Try to use shift instructions. if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) @@ -16806,9 +17243,9 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Zeroable, Subtarget, DAG)) return PSHUFB; - // AVX512BWVL can lower to VPERMW. - if (Subtarget.hasBWI() && Subtarget.hasVLX()) - return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, DAG); + // AVX512BW can lower to VPERMW (non-VLX will pad to v32i16). + if (Subtarget.hasBWI()) + return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, Subtarget, DAG); // Try to simplify this by merging 128-bit lanes to enable a lane-based // shuffle. @@ -16864,6 +17301,11 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Subtarget)) return V; + // Try to use lower using a truncation. + if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v32i8, V1, V2, Mask, Zeroable, + Subtarget, DAG)) + return V; + // Try to use shift instructions. if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) @@ -16906,9 +17348,9 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Zeroable, Subtarget, DAG)) return PSHUFB; - // AVX512VBMIVL can lower to VPERMB. - if (Subtarget.hasVBMI() && Subtarget.hasVLX()) - return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, DAG); + // AVX512VBMI can lower to VPERMB (non-VLX will pad to v64i8). + if (Subtarget.hasVBMI()) + return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, Subtarget, DAG); // Try to simplify this by merging 128-bit lanes to enable a lane-based // shuffle. @@ -17035,9 +17477,9 @@ static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask, // Check for patterns which can be matched with a single insert of a 256-bit // subvector. - bool OnlyUsesV1 = isShuffleEquivalent(V1, V2, Mask, {0, 1, 2, 3, 0, 1, 2, 3}); + bool OnlyUsesV1 = isShuffleEquivalent(Mask, {0, 1, 2, 3, 0, 1, 2, 3}, V1, V2); if (OnlyUsesV1 || - isShuffleEquivalent(V1, V2, Mask, {0, 1, 2, 3, 8, 9, 10, 11})) { + isShuffleEquivalent(Mask, {0, 1, 2, 3, 8, 9, 10, 11}, V1, V2)) { MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4); SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, OnlyUsesV1 ? V1 : V2, @@ -17122,7 +17564,7 @@ static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, if (V2.isUndef()) { // Use low duplicate instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 2, 2, 4, 4, 6, 6})) + if (isShuffleEquivalent(Mask, {0, 0, 2, 2, 4, 4, 6, 6}, V1, V2)) return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1); if (!is128BitLaneCrossingShuffleMask(MVT::v8f64, Mask)) { @@ -17162,7 +17604,7 @@ static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Zeroable, Subtarget, DAG)) return Blend; - return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, DAG); + return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, Subtarget, DAG); } /// Handle lowering of 16-lane 32-bit floating point shuffles. @@ -17181,9 +17623,9 @@ static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!"); // Use even/odd duplicate instructions for masks that match their pattern. - if (isShuffleEquivalent(V1, V2, RepeatedMask, {0, 0, 2, 2})) + if (isShuffleEquivalent(RepeatedMask, {0, 0, 2, 2}, V1, V2)) return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1); - if (isShuffleEquivalent(V1, V2, RepeatedMask, {1, 1, 3, 3})) + if (isShuffleEquivalent(RepeatedMask, {1, 1, 3, 3}, V1, V2)) return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1); if (V2.isUndef()) @@ -17221,7 +17663,7 @@ static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, V1, V2, DAG, Subtarget)) return V; - return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, DAG); + return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, Subtarget, DAG); } /// Handle lowering of 8-lane 64-bit integer shuffles. @@ -17269,12 +17711,14 @@ static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, return Rotate; // Try to use PALIGNR. - if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask, - Subtarget, DAG)) - return Rotate; + if (Subtarget.hasBWI()) + if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask, + Subtarget, DAG)) + return Rotate; if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG)) return Unpck; + // If we have AVX512F support, we can use VEXPAND. if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2, DAG, Subtarget)) @@ -17284,7 +17728,7 @@ static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Zeroable, Subtarget, DAG)) return Blend; - return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, DAG); + return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, Subtarget, DAG); } /// Handle lowering of 16-lane 32-bit integer shuffles. @@ -17361,7 +17805,7 @@ static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Zeroable, Subtarget, DAG)) return Blend; - return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, DAG); + return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, Subtarget, DAG); } /// Handle lowering of 32-lane 16-bit integer shuffles. @@ -17424,7 +17868,7 @@ static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask, Zeroable, Subtarget, DAG)) return PSHUFB; - return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, DAG); + return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, Subtarget, DAG); } /// Handle lowering of 64-lane 8-bit integer shuffles. @@ -17480,7 +17924,7 @@ static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask, // VBMI can use VPERMV/VPERMV3 byte shuffles. if (Subtarget.hasVBMI()) - return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, DAG); + return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, Subtarget, DAG); // Try to create an in-lane repeating shuffle mask and then shuffle the // results into the target lanes. @@ -17934,7 +18378,7 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget, // Modify the new Mask to take all zeros from the all-zero vector. // Choose indices that are blend-friendly. bool UsedZeroVector = false; - assert(find(WidenedMask, SM_SentinelZero) != WidenedMask.end() && + assert(is_contained(WidenedMask, SM_SentinelZero) && "V2's non-undef elements are used?!"); for (int i = 0; i != NewNumElts; ++i) if (WidenedMask[i] == SM_SentinelZero) { @@ -17960,9 +18404,6 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget, std::swap(V1, V2); } - if (SDValue V = lowerShuffleWithVPMOV(DL, Mask, VT, V1, V2, DAG, Subtarget)) - return V; - // For each vector width, delegate to a specialized lowering routine. if (VT.is128BitVector()) return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); @@ -17990,9 +18431,11 @@ static SDValue lowerVSELECTtoVectorShuffle(SDValue Op, // Only non-legal VSELECTs reach this lowering, convert those into generic // shuffles and re-use the shuffle lowering path for blends. - SmallVector<int, 32> Mask; - if (createShuffleMaskFromVSELECT(Mask, Cond)) - return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask); + if (ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) { + SmallVector<int, 32> Mask; + if (createShuffleMaskFromVSELECT(Mask, Cond)) + return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask); + } return SDValue(); } @@ -18106,7 +18549,9 @@ static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) { DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, DAG.getBitcast(MVT::v4i32, Vec), Idx)); - SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec, Idx); + unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); + SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec, + DAG.getTargetConstant(IdxVal, dl, MVT::i8)); return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract); } @@ -18261,7 +18706,8 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, DAG.getBitcast(MVT::v4i32, Vec), Idx)); - SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec, Idx); + SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec, + DAG.getTargetConstant(IdxVal, dl, MVT::i8)); return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract); } @@ -18455,10 +18901,9 @@ SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, Opc = X86ISD::PINSRB; } - if (N1.getValueType() != MVT::i32) - N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); - if (N2.getValueType() != MVT::i32) - N2 = DAG.getIntPtrConstant(IdxVal, dl); + assert(N1.getValueType() != MVT::i32 && "Unexpected VT"); + N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); + N2 = DAG.getTargetConstant(IdxVal, dl, MVT::i8); return DAG.getNode(Opc, dl, VT, N0, N1, N2); } @@ -18706,9 +19151,12 @@ SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG, if (GV) { // Create a target global address if this is a global. If possible, fold the // offset into the global address reference. Otherwise, ADD it on later. + // Suppress the folding if Offset is negative: movl foo-1, %eax is not + // allowed because if the address of foo is 0, the ELF R_X86_64_32 + // relocation will compute to a negative value, which is invalid. int64_t GlobalOffset = 0; - if (OpFlags == X86II::MO_NO_FLAG && - X86::isOffsetSuitableForCodeModel(Offset, M)) { + if (OpFlags == X86II::MO_NO_FLAG && Offset >= 0 && + X86::isOffsetSuitableForCodeModel(Offset, M, true)) { std::swap(GlobalOffset, Offset); } Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, GlobalOffset, OpFlags); @@ -18795,7 +19243,7 @@ LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG, return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD); } -// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit +// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit LP64 static SDValue LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT) { @@ -18803,10 +19251,17 @@ LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG, X86::RAX, X86II::MO_TLSGD); } +// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit ILP32 +static SDValue +LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA, SelectionDAG &DAG, + const EVT PtrVT) { + return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, + X86::EAX, X86II::MO_TLSGD); +} + static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA, - SelectionDAG &DAG, - const EVT PtrVT, - bool is64Bit) { + SelectionDAG &DAG, const EVT PtrVT, + bool Is64Bit, bool Is64BitLP64) { SDLoc dl(GA); // Get the start address of the TLS block for this module. @@ -18815,8 +19270,9 @@ static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA, MFI->incNumLocalDynamicTLSAccesses(); SDValue Base; - if (is64Bit) { - Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, X86::RAX, + if (Is64Bit) { + unsigned ReturnReg = Is64BitLP64 ? X86::RAX : X86::EAX; + Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, ReturnReg, X86II::MO_TLSLD, /*LocalDynamic=*/true); } else { SDValue InFlag; @@ -18913,12 +19369,15 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { TLSModel::Model model = DAG.getTarget().getTLSModel(GV); switch (model) { case TLSModel::GeneralDynamic: - if (Subtarget.is64Bit()) - return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT); + if (Subtarget.is64Bit()) { + if (Subtarget.isTarget64BitLP64()) + return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT); + return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT); + } return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT); case TLSModel::LocalDynamic: - return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, - Subtarget.is64Bit()); + return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(), + Subtarget.isTarget64BitLP64()); case TLSModel::InitialExec: case TLSModel::LocalExec: return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(), @@ -19018,7 +19477,7 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { else IDX = DAG.getLoad(PtrVT, dl, Chain, IDX, MachinePointerInfo()); - auto &DL = DAG.getDataLayout(); + const DataLayout &DL = DAG.getDataLayout(); SDValue Scale = DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, MVT::i8); IDX = DAG.getNode(ISD::SHL, dl, PtrVT, IDX, Scale); @@ -19111,15 +19570,29 @@ static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget, if (IsFSHR) std::swap(Op0, Op1); + // With AVX512, but not VLX we need to widen to get a 512-bit result type. + if (!Subtarget.hasVLX() && !VT.is512BitVector()) { + Op0 = widenSubVector(Op0, false, Subtarget, DAG, DL, 512); + Op1 = widenSubVector(Op1, false, Subtarget, DAG, DL, 512); + } + + SDValue Funnel; APInt APIntShiftAmt; + MVT ResultVT = Op0.getSimpleValueType(); if (X86::isConstantSplat(Amt, APIntShiftAmt)) { uint64_t ShiftAmt = APIntShiftAmt.urem(VT.getScalarSizeInBits()); - return DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT, Op0, - Op1, DAG.getTargetConstant(ShiftAmt, DL, MVT::i8)); + Funnel = + DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, ResultVT, Op0, + Op1, DAG.getTargetConstant(ShiftAmt, DL, MVT::i8)); + } else { + if (!Subtarget.hasVLX() && !VT.is512BitVector()) + Amt = widenSubVector(Amt, false, Subtarget, DAG, DL, 512); + Funnel = DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, + ResultVT, Op0, Op1, Amt); } - - return DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT, - Op0, Op1, Amt); + if (!Subtarget.hasVLX() && !VT.is512BitVector()) + Funnel = extractSubVector(Funnel, 0, DAG, DL, VT.getSizeInBits()); + return Funnel; } assert( (VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) && @@ -19471,7 +19944,7 @@ SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op, } if (VT == MVT::f128) - return LowerF128Call(Op, DAG, RTLIB::getSINTTOFP(SrcVT, VT)); + return SDValue(); SDValue ValueToStore = Src; if (SrcVT == MVT::i64 && Subtarget.hasSSE2() && !Subtarget.is64Bit()) @@ -19552,6 +20025,10 @@ static bool shouldUseHorizontalOp(bool IsSingleSource, SelectionDAG &DAG, /// 64-bit unsigned integer to double expansion. static SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) { + // We can't use this algorithm for strict fp. It produces -0.0 instead of +0.0 + // when converting 0 when rounding toward negative infinity. Caller will + // fall back to Expand for when i64 or is legal or use FILD in 32-bit mode. + assert(!Op->isStrictFPOpcode() && "Expected non-strict uint_to_fp!"); // This algorithm is not obvious. Here it is what we're trying to output: /* movq %rax, %xmm0 @@ -19565,8 +20042,6 @@ static SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG, #endif */ - bool IsStrict = Op->isStrictFPOpcode(); - unsigned OpNo = IsStrict ? 1 : 0; SDLoc dl(Op); LLVMContext *Context = DAG.getContext(); @@ -19588,48 +20063,30 @@ static SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG, // Load the 64-bit value into an XMM register. SDValue XR1 = - DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Op.getOperand(OpNo)); - SDValue CLod0 = - DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, - MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), - /* Alignment = */ 16); + DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Op.getOperand(0)); + SDValue CLod0 = DAG.getLoad( + MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, + MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16)); SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0); - SDValue CLod1 = - DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1, - MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), - /* Alignment = */ 16); + SDValue CLod1 = DAG.getLoad( + MVT::v2f64, dl, CLod0.getValue(1), CPIdx1, + MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Align(16)); SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1); - SDValue Sub; - SDValue Chain; // TODO: Are there any fast-math-flags to propagate here? - if (IsStrict) { - Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other}, - {Op.getOperand(0), XR2F, CLod1}); - Chain = Sub.getValue(1); - } else - Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); + SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); SDValue Result; - if (!IsStrict && Subtarget.hasSSE3() && + if (Subtarget.hasSSE3() && shouldUseHorizontalOp(true, DAG, Subtarget)) { - // FIXME: Do we need a STRICT version of FHADD? Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub); } else { SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1}); - if (IsStrict) { - Result = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::v2f64, MVT::Other}, - {Chain, Shuffle, Sub}); - Chain = Result.getValue(1); - } else - Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub); + Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub); } Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result, DAG.getIntPtrConstant(0, dl)); - if (IsStrict) - return DAG.getMergeValues({Result, Chain}, dl); - return Result; } @@ -19928,7 +20385,7 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op, SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode(); if (DstVT == MVT::f128) - return LowerF128Call(Op, DAG, RTLIB::getUINTTOFP(SrcVT, DstVT)); + return SDValue(); if (DstVT.isVector()) return lowerUINT_TO_FP_vec(Op, DAG, Subtarget); @@ -19955,26 +20412,30 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op, if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget)) return V; - if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64) + // The transform for i64->f64 isn't correct for 0 when rounding to negative + // infinity. It produces -0.0, so disable under strictfp. + if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64 && !IsStrict) return LowerUINT_TO_FP_i64(Op, DAG, Subtarget); if (SrcVT == MVT::i32 && X86ScalarSSEf64 && DstVT != MVT::f80) return LowerUINT_TO_FP_i32(Op, DAG, Subtarget); - if (Subtarget.is64Bit() && SrcVT == MVT::i64 && DstVT == MVT::f32) + if (Subtarget.is64Bit() && SrcVT == MVT::i64 && + (DstVT == MVT::f32 || DstVT == MVT::f64)) return SDValue(); // Make a 64-bit buffer, and use it to build an FILD. SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64, 8); int SSFI = cast<FrameIndexSDNode>(StackSlot)->getIndex(); + Align SlotAlign(8); MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI); if (SrcVT == MVT::i32) { - SDValue OffsetSlot = DAG.getMemBasePlusOffset(StackSlot, 4, dl); - SDValue Store1 = - DAG.getStore(Chain, dl, Src, StackSlot, MPI, 8 /*Align*/); + SDValue OffsetSlot = + DAG.getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), dl); + SDValue Store1 = DAG.getStore(Chain, dl, Src, StackSlot, MPI, SlotAlign); SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32), - OffsetSlot, MPI.getWithOffset(4), 4); + OffsetSlot, MPI.getWithOffset(4), SlotAlign); std::pair<SDValue, SDValue> Tmp = - BuildFILD(DstVT, MVT::i64, dl, Store2, StackSlot, MPI, Align(8), DAG); + BuildFILD(DstVT, MVT::i64, dl, Store2, StackSlot, MPI, SlotAlign, DAG); if (IsStrict) return DAG.getMergeValues({Tmp.first, Tmp.second}, dl); @@ -19990,17 +20451,15 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op, ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore); } SDValue Store = - DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, Align(8)); + DAG.getStore(Chain, dl, ValueToStore, StackSlot, MPI, SlotAlign); // For i64 source, we need to add the appropriate power of 2 if the input - // was negative. This is the same as the optimization in - // DAGTypeLegalizer::ExpandIntOp_UNIT_TO_FP, and for it to be safe here, - // we must be careful to do the computation in x87 extended precision, not - // in SSE. (The generic code can't know it's OK to do this, or how to.) + // was negative. We must be careful to do the computation in x87 extended + // precision, not in SSE. SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); SDValue Ops[] = { Store, StackSlot }; SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, MVT::i64, MPI, - Align(8), MachineMemOperand::MOLoad); + SlotAlign, MachineMemOperand::MOLoad); Chain = Fild.getValue(1); @@ -20103,8 +20562,8 @@ X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, // of a signed i64. Let Thresh be the FP equivalent of // 0x8000000000000000ULL. // - // Adjust = (Value < Thresh) ? 0 : 0x80000000; - // FltOfs = (Value < Thresh) ? 0 : 0x80000000; + // Adjust = (Value >= Thresh) ? 0x80000000 : 0; + // FltOfs = (Value >= Thresh) ? 0x80000000 : 0; // FistSrc = (Value - FltOfs); // Fist-to-mem64 FistSrc // Add 0 or 0x800...0ULL to the 64-bit result, which is equivalent @@ -20134,20 +20593,30 @@ X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, *DAG.getContext(), TheVT); SDValue Cmp; if (IsStrict) { - Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETLT, - Chain, /*IsSignaling*/ true); + Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE, Chain, + /*IsSignaling*/ true); Chain = Cmp.getValue(1); } else { - Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETLT); + Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETGE); } - Adjust = DAG.getSelect(DL, MVT::i64, Cmp, - DAG.getConstant(0, DL, MVT::i64), - DAG.getConstant(APInt::getSignMask(64), - DL, MVT::i64)); - SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp, - DAG.getConstantFP(0.0, DL, TheVT), - ThreshVal); + // Our preferred lowering of + // + // (Value >= Thresh) ? 0x8000000000000000ULL : 0 + // + // is + // + // (Value >= Thresh) << 63 + // + // but since we can get here after LegalOperations, DAGCombine might do the + // wrong thing if we create a select. So, directly create the preferred + // version. + SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Cmp); + SDValue Const63 = DAG.getConstant(63, DL, MVT::i8); + Adjust = DAG.getNode(ISD::SHL, DL, MVT::i64, Zext, Const63); + + SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp, ThreshVal, + DAG.getConstantFP(0.0, DL, TheVT)); if (IsStrict) { Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other}, @@ -20606,30 +21075,29 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { assert(VT.is128BitVector() && InVT.is256BitVector() && "Unexpected types!"); if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) { + In = DAG.getBitcast(MVT::v8i32, In); + // On AVX2, v4i64 -> v4i32 becomes VPERMD. if (Subtarget.hasInt256()) { static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1}; - In = DAG.getBitcast(MVT::v8i32, In); In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask); return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In, DAG.getIntPtrConstant(0, DL)); } - SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, + SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, DAG.getIntPtrConstant(0, DL)); - SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, - DAG.getIntPtrConstant(2, DL)); - OpLo = DAG.getBitcast(MVT::v4i32, OpLo); - OpHi = DAG.getBitcast(MVT::v4i32, OpHi); + SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, + DAG.getIntPtrConstant(4, DL)); static const int ShufMask[] = {0, 2, 4, 6}; return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask); } if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) { + In = DAG.getBitcast(MVT::v32i8, In); + // On AVX2, v8i32 -> v8i16 becomes PSHUFB. if (Subtarget.hasInt256()) { - In = DAG.getBitcast(MVT::v32i8, In); - // The PSHUFB mask: static const int ShufMask1[] = { 0, 1, 4, 5, 8, 9, 12, 13, -1, -1, -1, -1, -1, -1, -1, -1, @@ -20638,21 +21106,17 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1); In = DAG.getBitcast(MVT::v4i64, In); - static const int ShufMask2[] = {0, 2, -1, -1}; - In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2); - In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, - DAG.getIntPtrConstant(0, DL)); - return DAG.getBitcast(VT, In); + static const int ShufMask2[] = {0, 2, -1, -1}; + In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, + DAG.getBitcast(MVT::v16i16, In), + DAG.getIntPtrConstant(0, DL)); } - SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, + SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v16i8, In, DAG.getIntPtrConstant(0, DL)); - - SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, - DAG.getIntPtrConstant(4, DL)); - - OpLo = DAG.getBitcast(MVT::v16i8, OpLo); - OpHi = DAG.getBitcast(MVT::v16i8, OpHi); + SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v16i8, In, + DAG.getIntPtrConstant(16, DL)); // The PSHUFB mask: static const int ShufMask1[] = {0, 1, 4, 5, 8, 9, 12, 13, @@ -20988,6 +21452,155 @@ SDValue X86TargetLowering::LRINT_LLRINTHelper(SDNode *N, return DAG.getLoad(DstVT, DL, Chain, StackPtr, MPI); } +SDValue +X86TargetLowering::LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const { + // This is based on the TargetLowering::expandFP_TO_INT_SAT implementation, + // but making use of X86 specifics to produce better instruction sequences. + SDNode *Node = Op.getNode(); + bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT; + unsigned FpToIntOpcode = IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT; + SDLoc dl(SDValue(Node, 0)); + SDValue Src = Node->getOperand(0); + + // There are three types involved here: SrcVT is the source floating point + // type, DstVT is the type of the result, and TmpVT is the result of the + // intermediate FP_TO_*INT operation we'll use (which may be a promotion of + // DstVT). + EVT SrcVT = Src.getValueType(); + EVT DstVT = Node->getValueType(0); + EVT TmpVT = DstVT; + + // This code is only for floats and doubles. Fall back to generic code for + // anything else. + if (!isScalarFPTypeInSSEReg(SrcVT)) + return SDValue(); + + unsigned SatWidth = Node->getConstantOperandVal(1); + unsigned DstWidth = DstVT.getScalarSizeInBits(); + unsigned TmpWidth = TmpVT.getScalarSizeInBits(); + assert(SatWidth <= DstWidth && SatWidth <= TmpWidth && + "Expected saturation width smaller than result width"); + + // Promote result of FP_TO_*INT to at least 32 bits. + if (TmpWidth < 32) { + TmpVT = MVT::i32; + TmpWidth = 32; + } + + // Promote conversions to unsigned 32-bit to 64-bit, because it will allow + // us to use a native signed conversion instead. + if (SatWidth == 32 && !IsSigned && Subtarget.is64Bit()) { + TmpVT = MVT::i64; + TmpWidth = 64; + } + + // If the saturation width is smaller than the size of the temporary result, + // we can always use signed conversion, which is native. + if (SatWidth < TmpWidth) + FpToIntOpcode = ISD::FP_TO_SINT; + + // Determine minimum and maximum integer values and their corresponding + // floating-point values. + APInt MinInt, MaxInt; + if (IsSigned) { + MinInt = APInt::getSignedMinValue(SatWidth).sextOrSelf(DstWidth); + MaxInt = APInt::getSignedMaxValue(SatWidth).sextOrSelf(DstWidth); + } else { + MinInt = APInt::getMinValue(SatWidth).zextOrSelf(DstWidth); + MaxInt = APInt::getMaxValue(SatWidth).zextOrSelf(DstWidth); + } + + APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT)); + APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT)); + + APFloat::opStatus MinStatus = MinFloat.convertFromAPInt( + MinInt, IsSigned, APFloat::rmTowardZero); + APFloat::opStatus MaxStatus = MaxFloat.convertFromAPInt( + MaxInt, IsSigned, APFloat::rmTowardZero); + bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) + && !(MaxStatus & APFloat::opStatus::opInexact); + + SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT); + SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT); + + // If the integer bounds are exactly representable as floats, emit a + // min+max+fptoi sequence. Otherwise use comparisons and selects. + if (AreExactFloatBounds) { + if (DstVT != TmpVT) { + // Clamp by MinFloat from below. If Src is NaN, propagate NaN. + SDValue MinClamped = DAG.getNode( + X86ISD::FMAX, dl, SrcVT, MinFloatNode, Src); + // Clamp by MaxFloat from above. If Src is NaN, propagate NaN. + SDValue BothClamped = DAG.getNode( + X86ISD::FMIN, dl, SrcVT, MaxFloatNode, MinClamped); + // Convert clamped value to integer. + SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, BothClamped); + + // NaN will become INDVAL, with the top bit set and the rest zero. + // Truncation will discard the top bit, resulting in zero. + return DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt); + } + + // Clamp by MinFloat from below. If Src is NaN, the result is MinFloat. + SDValue MinClamped = DAG.getNode( + X86ISD::FMAX, dl, SrcVT, Src, MinFloatNode); + // Clamp by MaxFloat from above. NaN cannot occur. + SDValue BothClamped = DAG.getNode( + X86ISD::FMINC, dl, SrcVT, MinClamped, MaxFloatNode); + // Convert clamped value to integer. + SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, DstVT, BothClamped); + + if (!IsSigned) { + // In the unsigned case we're done, because we mapped NaN to MinFloat, + // which is zero. + return FpToInt; + } + + // Otherwise, select zero if Src is NaN. + SDValue ZeroInt = DAG.getConstant(0, dl, DstVT); + return DAG.getSelectCC( + dl, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO); + } + + SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT); + SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT); + + // Result of direct conversion, which may be selected away. + SDValue FpToInt = DAG.getNode(FpToIntOpcode, dl, TmpVT, Src); + + if (DstVT != TmpVT) { + // NaN will become INDVAL, with the top bit set and the rest zero. + // Truncation will discard the top bit, resulting in zero. + FpToInt = DAG.getNode(ISD::TRUNCATE, dl, DstVT, FpToInt); + } + + SDValue Select = FpToInt; + // For signed conversions where we saturate to the same size as the + // result type of the fptoi instructions, INDVAL coincides with integer + // minimum, so we don't need to explicitly check it. + if (!IsSigned || SatWidth != TmpVT.getScalarSizeInBits()) { + // If Src ULT MinFloat, select MinInt. In particular, this also selects + // MinInt if Src is NaN. + Select = DAG.getSelectCC( + dl, Src, MinFloatNode, MinIntNode, Select, ISD::CondCode::SETULT); + } + + // If Src OGT MaxFloat, select MaxInt. + Select = DAG.getSelectCC( + dl, Src, MaxFloatNode, MaxIntNode, Select, ISD::CondCode::SETOGT); + + // In the unsigned case we are done, because we mapped NaN to MinInt, which + // is already zero. The promoted case was already handled above. + if (!IsSigned || DstVT != TmpVT) { + return Select; + } + + // Otherwise, select 0 if Src is NaN. + SDValue ZeroInt = DAG.getConstant(0, dl, DstVT); + return DAG.getSelectCC( + dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO); +} + SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { bool IsStrict = Op->isStrictFPOpcode(); @@ -20996,10 +21609,8 @@ SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { SDValue In = Op.getOperand(IsStrict ? 1 : 0); MVT SVT = In.getSimpleValueType(); - if (VT == MVT::f128) { - RTLIB::Libcall LC = RTLIB::getFPEXT(SVT, VT); - return LowerF128Call(Op, DAG, LC); - } + if (VT == MVT::f128) + return SDValue(); assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!"); @@ -21013,31 +21624,12 @@ SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { bool IsStrict = Op->isStrictFPOpcode(); - - MVT VT = Op.getSimpleValueType(); SDValue In = Op.getOperand(IsStrict ? 1 : 0); - MVT SVT = In.getSimpleValueType(); - // It's legal except when f128 is involved - if (SVT != MVT::f128) + if (In.getSimpleValueType() != MVT::f128) return Op; - RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, VT); - - // FP_ROUND node has a second operand indicating whether it is known to be - // precise. That doesn't take part in the LibCall so we can't directly use - // LowerF128Call. - - SDLoc dl(Op); - SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue(); - MakeLibCallOptions CallOptions; - std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, LC, VT, In, CallOptions, - dl, Chain); - - if (IsStrict) - return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl); - - return Tmp.first; + return SDValue(); } static SDValue LowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) { @@ -21402,8 +21994,7 @@ static bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp, if (M == SrcOpMap.end()) { VT = Src.getValueType(); // Quit if not the same type. - if (SrcOpMap.begin() != SrcOpMap.end() && - VT != SrcOpMap.begin()->first.getValueType()) + if (!SrcOpMap.empty() && VT != SrcOpMap.begin()->first.getValueType()) return false; unsigned NumElts = VT.getVectorNumElements(); APInt EltCount = APInt::getNullValue(NumElts); @@ -21441,8 +22032,11 @@ static SDValue LowerVectorAllZero(const SDLoc &DL, SDValue V, ISD::CondCode CC, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC) { EVT VT = V.getValueType(); - assert(Mask.getBitWidth() == VT.getScalarSizeInBits() && - "Element Mask vs Vector bitwidth mismatch"); + unsigned ScalarSize = VT.getScalarSizeInBits(); + if (Mask.getBitWidth() != ScalarSize) { + assert(ScalarSize == 1 && "Element Mask vs Vector bitwidth mismatch"); + return SDValue(); + } assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode"); X86CC = (CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE); @@ -22346,7 +22940,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget, Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)); } - if (VT.getSizeInBits() > Op.getSimpleValueType().getSizeInBits()) { + if (VT.getFixedSizeInBits() > + Op.getSimpleValueType().getFixedSizeInBits()) { // We emitted a compare with an XMM/YMM result. Finish converting to a // mask register using a vptestm. EVT CastVT = EVT(VT).changeVectorElementTypeToInteger(); @@ -22521,8 +23116,10 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget, } // Try to use SUBUS and PCMPEQ. - if (SDValue V = LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG)) - return V; + if (FlipSigns) + if (SDValue V = + LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG)) + return V; // We are handling one of the integer comparisons here. Since SSE only has // GT and EQ comparisons for integer, swapping operands and multiple @@ -23317,7 +23914,7 @@ static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op, MVT SVT = VT.getVectorElementType(); MVT InSVT = InVT.getVectorElementType(); - assert(SVT.getSizeInBits() > InSVT.getSizeInBits()); + assert(SVT.getFixedSizeInBits() > InSVT.getFixedSizeInBits()); if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16) return SDValue(); @@ -23492,7 +24089,8 @@ static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG) { std::tie(Value0, Value1) = splitVector(StoredVal, DAG, DL); unsigned HalfOffset = Value0.getValueType().getStoreSize(); SDValue Ptr0 = Store->getBasePtr(); - SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, HalfOffset, DL); + SDValue Ptr1 = + DAG.getMemBasePlusOffset(Ptr0, TypeSize::Fixed(HalfOffset), DL); SDValue Ch0 = DAG.getStore(Store->getChain(), DL, Value0, Ptr0, Store->getPointerInfo(), Store->getOriginalAlign(), @@ -23527,7 +24125,8 @@ static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT, SmallVector<SDValue, 4> Stores; for (unsigned i = 0; i != NumElems; ++i) { unsigned Offset = i * ScalarSize; - SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(), Offset, DL); + SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(), + TypeSize::Fixed(Offset), DL); SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal, DAG.getIntPtrConstant(i, DL)); SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr, @@ -23548,17 +24147,22 @@ static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget, // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 stores. if (StoredVal.getValueType().isVector() && StoredVal.getValueType().getVectorElementType() == MVT::i1) { - assert(StoredVal.getValueType().getVectorNumElements() <= 8 && - "Unexpected VT"); + unsigned NumElts = StoredVal.getValueType().getVectorNumElements(); + assert(NumElts <= 8 && "Unexpected VT"); assert(!St->isTruncatingStore() && "Expected non-truncating store"); assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() && "Expected AVX512F without AVX512DQI"); + // We must pad with zeros to ensure we store zeroes to any unused bits. StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1, DAG.getUNDEF(MVT::v16i1), StoredVal, DAG.getIntPtrConstant(0, dl)); StoredVal = DAG.getBitcast(MVT::i16, StoredVal); StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal); + // Make sure we store zeros in the extra bits. + if (NumElts < 8) + StoredVal = DAG.getZeroExtendInReg( + StoredVal, dl, EVT::getIntegerVT(*DAG.getContext(), NumElts)); return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), St->getPointerInfo(), St->getOriginalAlign(), @@ -23814,7 +24418,7 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SDValue Result; if (!Lower) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); + Register SPReg = TLI.getStackPointerRegisterToSaveRestore(); assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" " not tell us which reg is the stack pointer!"); @@ -23915,7 +24519,7 @@ SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { MemOps.push_back(Store); // Store fp_offset - FIN = DAG.getMemBasePlusOffset(FIN, 4, DL); + FIN = DAG.getMemBasePlusOffset(FIN, TypeSize::Fixed(4), DL); Store = DAG.getStore( Op.getOperand(0), DL, DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN, @@ -23980,15 +24584,18 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { Subtarget.hasSSE1()); } - // Insert VAARG_64 node into the DAG - // VAARG_64 returns two values: Variable Argument Address, Chain - SDValue InstOps[] = {Chain, SrcPtr, DAG.getConstant(ArgSize, dl, MVT::i32), - DAG.getConstant(ArgMode, dl, MVT::i8), - DAG.getConstant(Align, dl, MVT::i32)}; + // Insert VAARG node into the DAG + // VAARG returns two values: Variable Argument Address, Chain + SDValue InstOps[] = {Chain, SrcPtr, + DAG.getTargetConstant(ArgSize, dl, MVT::i32), + DAG.getTargetConstant(ArgMode, dl, MVT::i8), + DAG.getTargetConstant(Align, dl, MVT::i32)}; SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other); SDValue VAARG = DAG.getMemIntrinsicNode( - X86ISD::VAARG_64, dl, VTs, InstOps, MVT::i64, MachinePointerInfo(SV), - /*Align=*/None, MachineMemOperand::MOLoad | MachineMemOperand::MOStore); + Subtarget.isTarget64BitLP64() ? X86ISD::VAARG_64 : X86ISD::VAARG_X32, dl, + VTs, InstOps, MVT::i64, MachinePointerInfo(SV), + /*Alignment=*/None, + MachineMemOperand::MOLoad | MachineMemOperand::MOStore); Chain = VAARG.getValue(1); // Load the next argument and return it @@ -24012,9 +24619,11 @@ static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget, const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); SDLoc DL(Op); - return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(24, DL), - Align(8), /*isVolatile*/ false, false, false, - MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); + return DAG.getMemcpy( + Chain, DL, DstPtr, SrcPtr, + DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 24 : 16, DL), + Align(Subtarget.isTarget64BitLP64() ? 8 : 4), /*isVolatile*/ false, false, + false, MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); } // Helper to get immediate/variable SSE shift opcode from other shift opcodes. @@ -24461,6 +25070,12 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SDValue Src2 = Op.getOperand(2); SDValue Src3 = Op.getOperand(3); + if (IntrData->Type == INTR_TYPE_3OP_IMM8 && + Src3.getValueType() != MVT::i8) { + Src3 = DAG.getTargetConstant( + cast<ConstantSDNode>(Src3)->getZExtValue() & 0xff, dl, MVT::i8); + } + // We specify 2 possible opcodes for intrinsics with rounding modes. // First, we check if the intrinsic may have non-default rounding mode, // (IntrData->Opc1 != 0), then we check the rounding mode operand. @@ -24479,9 +25094,18 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), {Src1, Src2, Src3}); } - case INTR_TYPE_4OP: - return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1), - Op.getOperand(2), Op.getOperand(3), Op.getOperand(4)); + case INTR_TYPE_4OP_IMM8: { + assert(Op.getOperand(4)->getOpcode() == ISD::TargetConstant); + SDValue Src4 = Op.getOperand(4); + if (Src4.getValueType() != MVT::i8) { + Src4 = DAG.getTargetConstant( + cast<ConstantSDNode>(Src4)->getZExtValue() & 0xff, dl, MVT::i8); + } + + return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), + Op.getOperand(1), Op.getOperand(2), Op.getOperand(3), + Src4); + } case INTR_TYPE_1OP_MASK: { SDValue Src = Op.getOperand(1); SDValue PassThru = Op.getOperand(2); @@ -24714,20 +25338,21 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, case CMP_MASK_CC: { MVT MaskVT = Op.getSimpleValueType(); SDValue CC = Op.getOperand(3); + SDValue Mask = Op.getOperand(4); // We specify 2 possible opcodes for intrinsics with rounding modes. // First, we check if the intrinsic may have non-default rounding mode, // (IntrData->Opc1 != 0), then we check the rounding mode operand. if (IntrData->Opc1 != 0) { - SDValue Sae = Op.getOperand(4); + SDValue Sae = Op.getOperand(5); if (isRoundModeSAE(Sae)) return DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1), - Op.getOperand(2), CC, Sae); + Op.getOperand(2), CC, Mask, Sae); if (!isRoundModeCurDirection(Sae)) return SDValue(); } //default rounding mode return DAG.getNode(IntrData->Opc0, dl, MaskVT, - {Op.getOperand(1), Op.getOperand(2), CC}); + {Op.getOperand(1), Op.getOperand(2), CC, Mask}); } case CMP_MASK_SCALAR_CC: { SDValue Src1 = Op.getOperand(1); @@ -24882,12 +25507,11 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, Op.getOperand(1), Op.getOperand(2), RoundingMode); } case BEXTRI: { - assert(IntrData->Opc0 == X86ISD::BEXTR && "Unexpected opcode"); + assert(IntrData->Opc0 == X86ISD::BEXTRI && "Unexpected opcode"); - // The control is a TargetConstant, but we need to convert it to a - // ConstantSDNode. uint64_t Imm = Op.getConstantOperandVal(2); - SDValue Control = DAG.getConstant(Imm, dl, Op.getValueType()); + SDValue Control = DAG.getTargetConstant(Imm & 0xffff, dl, + Op.getValueType()); return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1), Control); } @@ -25278,9 +25902,9 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, // MMX register. ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt); return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), - DAG.getConstant(NewIntrinsic, DL, MVT::i32), + DAG.getTargetConstant(NewIntrinsic, DL, + getPointerTy(DAG.getDataLayout())), Op.getOperand(1), ShAmt); - } } } @@ -25649,6 +26273,96 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget, return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, Operation.getValue(1)); } + case Intrinsic::x86_aesenc128kl: + case Intrinsic::x86_aesdec128kl: + case Intrinsic::x86_aesenc256kl: + case Intrinsic::x86_aesdec256kl: { + SDLoc DL(Op); + SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::i32, MVT::Other); + SDValue Chain = Op.getOperand(0); + unsigned Opcode; + + switch (IntNo) { + default: llvm_unreachable("Impossible intrinsic"); + case Intrinsic::x86_aesenc128kl: + Opcode = X86ISD::AESENC128KL; + break; + case Intrinsic::x86_aesdec128kl: + Opcode = X86ISD::AESDEC128KL; + break; + case Intrinsic::x86_aesenc256kl: + Opcode = X86ISD::AESENC256KL; + break; + case Intrinsic::x86_aesdec256kl: + Opcode = X86ISD::AESDEC256KL; + break; + } + + MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op); + MachineMemOperand *MMO = MemIntr->getMemOperand(); + EVT MemVT = MemIntr->getMemoryVT(); + SDValue Operation = DAG.getMemIntrinsicNode( + Opcode, DL, VTs, {Chain, Op.getOperand(2), Op.getOperand(3)}, MemVT, + MMO); + SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(1), DL, DAG); + + return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), + {ZF, Operation.getValue(0), Operation.getValue(2)}); + } + case Intrinsic::x86_aesencwide128kl: + case Intrinsic::x86_aesdecwide128kl: + case Intrinsic::x86_aesencwide256kl: + case Intrinsic::x86_aesdecwide256kl: { + SDLoc DL(Op); + SDVTList VTs = DAG.getVTList( + {MVT::i32, MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::v2i64, + MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::Other}); + SDValue Chain = Op.getOperand(0); + unsigned Opcode; + + switch (IntNo) { + default: llvm_unreachable("Impossible intrinsic"); + case Intrinsic::x86_aesencwide128kl: + Opcode = X86ISD::AESENCWIDE128KL; + break; + case Intrinsic::x86_aesdecwide128kl: + Opcode = X86ISD::AESDECWIDE128KL; + break; + case Intrinsic::x86_aesencwide256kl: + Opcode = X86ISD::AESENCWIDE256KL; + break; + case Intrinsic::x86_aesdecwide256kl: + Opcode = X86ISD::AESDECWIDE256KL; + break; + } + + MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op); + MachineMemOperand *MMO = MemIntr->getMemOperand(); + EVT MemVT = MemIntr->getMemoryVT(); + SDValue Operation = DAG.getMemIntrinsicNode( + Opcode, DL, VTs, + {Chain, Op.getOperand(2), Op.getOperand(3), Op.getOperand(4), + Op.getOperand(5), Op.getOperand(6), Op.getOperand(7), + Op.getOperand(8), Op.getOperand(9), Op.getOperand(10)}, + MemVT, MMO); + SDValue ZF = getSETCC(X86::COND_E, Operation.getValue(0), DL, DAG); + + return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), + {ZF, Operation.getValue(1), Operation.getValue(2), + Operation.getValue(3), Operation.getValue(4), + Operation.getValue(5), Operation.getValue(6), + Operation.getValue(7), Operation.getValue(8), + Operation.getValue(9)}); + } + case Intrinsic::x86_testui: { + SDLoc dl(Op); + SDValue Chain = Op.getOperand(0); + SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other); + SDValue Operation = DAG.getNode(X86ISD::TESTUI, dl, VTs, Chain); + SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG); + return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC, + Operation.getValue(1)); + } } return SDValue(); } @@ -26019,9 +26733,8 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, DAG.getConstant(2, dl, MVT::i64)); - OutChains[1] = - DAG.getStore(Root, dl, FPtr, Addr, MachinePointerInfo(TrmpAddr, 2), - /* Alignment = */ 2); + OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr, + MachinePointerInfo(TrmpAddr, 2), Align(2)); // Load the 'nest' parameter value into R10. // R10 is specified in X86CallingConv.td @@ -26033,9 +26746,8 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, DAG.getConstant(12, dl, MVT::i64)); - OutChains[3] = - DAG.getStore(Root, dl, Nest, Addr, MachinePointerInfo(TrmpAddr, 12), - /* Alignment = */ 2); + OutChains[3] = DAG.getStore(Root, dl, Nest, Addr, + MachinePointerInfo(TrmpAddr, 12), Align(2)); // Jump to the nested function. OpCode = (JMP64r << 8) | REX_WB; // jmpq *... @@ -26077,7 +26789,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, for (FunctionType::param_iterator I = FTy->param_begin(), E = FTy->param_end(); I != E; ++I, ++Idx) if (Attrs.hasAttribute(Idx, Attribute::InReg)) { - auto &DL = DAG.getDataLayout(); + const DataLayout &DL = DAG.getDataLayout(); // FIXME: should only count parameters that are lowered to integers. InRegCount += (DL.getTypeSizeInBits(*I) + 31) / 32; } @@ -26115,22 +26827,20 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, DAG.getConstant(1, dl, MVT::i32)); - OutChains[1] = - DAG.getStore(Root, dl, Nest, Addr, MachinePointerInfo(TrmpAddr, 1), - /* Alignment = */ 1); + OutChains[1] = DAG.getStore(Root, dl, Nest, Addr, + MachinePointerInfo(TrmpAddr, 1), Align(1)); const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode. Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, DAG.getConstant(5, dl, MVT::i32)); - OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8), - Addr, MachinePointerInfo(TrmpAddr, 5), - /* Alignment = */ 1); + OutChains[2] = + DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8), Addr, + MachinePointerInfo(TrmpAddr, 5), Align(1)); Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, DAG.getConstant(6, dl, MVT::i32)); - OutChains[3] = - DAG.getStore(Root, dl, Disp, Addr, MachinePointerInfo(TrmpAddr, 6), - /* Alignment = */ 1); + OutChains[3] = DAG.getStore(Root, dl, Disp, Addr, + MachinePointerInfo(TrmpAddr, 6), Align(1)); return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); } @@ -26424,50 +27134,47 @@ static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG, MVT VT = Op.getSimpleValueType(); SDValue X = Op.getOperand(0), Y = Op.getOperand(1); unsigned Opcode = Op.getOpcode(); + SDLoc DL(Op); + if (VT.getScalarType() == MVT::i1) { - SDLoc dl(Op); switch (Opcode) { default: llvm_unreachable("Expected saturated arithmetic opcode"); case ISD::UADDSAT: case ISD::SADDSAT: // *addsat i1 X, Y --> X | Y - return DAG.getNode(ISD::OR, dl, VT, X, Y); + return DAG.getNode(ISD::OR, DL, VT, X, Y); case ISD::USUBSAT: case ISD::SSUBSAT: // *subsat i1 X, Y --> X & ~Y - return DAG.getNode(ISD::AND, dl, VT, X, DAG.getNOT(dl, Y, VT)); + return DAG.getNode(ISD::AND, DL, VT, X, DAG.getNOT(DL, Y, VT)); } } - if (VT.is128BitVector()) { - // Avoid the generic expansion with min/max if we don't have pminu*/pmaxu*. - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - EVT SetCCResultType = TLI.getSetCCResultType(DAG.getDataLayout(), - *DAG.getContext(), VT); - SDLoc DL(Op); - if (Opcode == ISD::UADDSAT && !TLI.isOperationLegal(ISD::UMIN, VT)) { - // uaddsat X, Y --> (X >u (X + Y)) ? -1 : X + Y - SDValue Add = DAG.getNode(ISD::ADD, DL, VT, X, Y); - SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Add, ISD::SETUGT); - return DAG.getSelect(DL, VT, Cmp, DAG.getAllOnesConstant(DL, VT), Add); - } - if (Opcode == ISD::USUBSAT && !TLI.isOperationLegal(ISD::UMAX, VT)) { - // usubsat X, Y --> (X >u Y) ? X - Y : 0 - SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y); - SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT); - return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT)); - } - // Use default expansion. - return SDValue(); + if (VT == MVT::v32i16 || VT == MVT::v64i8 || + (VT.is256BitVector() && !Subtarget.hasInt256())) { + assert(Op.getSimpleValueType().isInteger() && + "Only handle AVX vector integer operation"); + return splitVectorIntBinary(Op, DAG); } - if (VT == MVT::v32i16 || VT == MVT::v64i8) - return splitVectorIntBinary(Op, DAG); + // Avoid the generic expansion with min/max if we don't have pminu*/pmaxu*. + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + EVT SetCCResultType = + TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); - assert(Op.getSimpleValueType().is256BitVector() && - Op.getSimpleValueType().isInteger() && - "Only handle AVX 256-bit vector integer operation"); - return splitVectorIntBinary(Op, DAG); + if (Opcode == ISD::USUBSAT && !TLI.isOperationLegal(ISD::UMAX, VT)) { + // usubsat X, Y --> (X >u Y) ? X - Y : 0 + SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y); + SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT); + // TODO: Move this to DAGCombiner? + if (SetCCResultType == VT && + DAG.ComputeNumSignBits(Cmp) == VT.getScalarSizeInBits()) + return DAG.getNode(ISD::AND, DL, VT, Cmp, Sub); + return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT)); + } + + // Use default expansion. + return SDValue(); } static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget, @@ -26517,36 +27224,8 @@ static SDValue LowerMINMAX(SDValue Op, SelectionDAG &DAG) { if (VT == MVT::v32i16 || VT == MVT::v64i8) return splitVectorIntBinary(Op, DAG); - SDLoc DL(Op); - unsigned Opcode = Op.getOpcode(); - SDValue N0 = Op.getOperand(0); - SDValue N1 = Op.getOperand(1); - - // For pre-SSE41, we can perform UMIN/UMAX v8i16 by flipping the signbit, - // using the SMIN/SMAX instructions and flipping the signbit back. - if (VT == MVT::v8i16) { - assert((Opcode == ISD::UMIN || Opcode == ISD::UMAX) && - "Unexpected MIN/MAX opcode"); - SDValue Sign = DAG.getConstant(APInt::getSignedMinValue(16), DL, VT); - N0 = DAG.getNode(ISD::XOR, DL, VT, N0, Sign); - N1 = DAG.getNode(ISD::XOR, DL, VT, N1, Sign); - Opcode = (Opcode == ISD::UMIN ? ISD::SMIN : ISD::SMAX); - SDValue Result = DAG.getNode(Opcode, DL, VT, N0, N1); - return DAG.getNode(ISD::XOR, DL, VT, Result, Sign); - } - - // Else, expand to a compare/select. - ISD::CondCode CC; - switch (Opcode) { - case ISD::SMIN: CC = ISD::CondCode::SETLT; break; - case ISD::SMAX: CC = ISD::CondCode::SETGT; break; - case ISD::UMIN: CC = ISD::CondCode::SETULT; break; - case ISD::UMAX: CC = ISD::CondCode::SETUGT; break; - default: llvm_unreachable("Unknown MINMAX opcode"); - } - - SDValue Cond = DAG.getSetCC(DL, VT, N0, N1, CC); - return DAG.getSelect(DL, VT, Cond, N0, N1); + // Default to expand. + return SDValue(); } static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget, @@ -26902,8 +27581,6 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons case ISD::UDIV: isSigned = false; LC = RTLIB::UDIV_I128; break; case ISD::SREM: isSigned = true; LC = RTLIB::SREM_I128; break; case ISD::UREM: isSigned = false; LC = RTLIB::UREM_I128; break; - case ISD::SDIVREM: isSigned = true; LC = RTLIB::SDIVREM_I128; break; - case ISD::UDIVREM: isSigned = false; LC = RTLIB::UDIVREM_I128; break; } SDLoc dl(Op); @@ -26920,8 +27597,8 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); Entry.Node = StackPtr; - InChain = DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, - MPI, /* Alignment = */ 16); + InChain = + DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MPI, Align(16)); Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); Entry.Ty = PointerType::get(ArgTy,0); Entry.IsSExt = false; @@ -27212,6 +27889,7 @@ static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl, MVT VT = Amt.getSimpleValueType(); if (!(VT == MVT::v8i16 || VT == MVT::v4i32 || (Subtarget.hasInt256() && VT == MVT::v16i16) || + (Subtarget.hasVBMI2() && VT == MVT::v32i16) || (!Subtarget.hasAVX512() && VT == MVT::v16i8))) return SDValue(); @@ -27789,6 +28467,12 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget, return Op; } + // AVX512 VBMI2 vXi16 - lower to funnel shifts. + if (Subtarget.hasVBMI2() && 16 == EltSizeInBits) { + unsigned FunnelOpc = (Opcode == ISD::ROTL ? ISD::FSHL : ISD::FSHR); + return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt); + } + assert((Opcode == ISD::ROTL) && "Only ROTL supported"); // XOP has 128-bit vector variable + immediate rotates. @@ -27815,7 +28499,8 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget, return splitVectorIntBinary(Op, DAG); assert((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 || - ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) && + ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8 || + VT == MVT::v32i16) && Subtarget.hasAVX2())) && "Only vXi32/vXi16/vXi8 vector rotates supported"); @@ -28112,8 +28797,8 @@ bool X86TargetLowering::lowerAtomicLoadAsLoadSDNode(const LoadInst &LI) const { /// a) very likely accessed only by a single thread to minimize cache traffic, /// and b) definitely dereferenceable. Returns the new Chain result. static SDValue emitLockedStackOp(SelectionDAG &DAG, - const X86Subtarget &Subtarget, - SDValue Chain, SDLoc DL) { + const X86Subtarget &Subtarget, SDValue Chain, + const SDLoc &DL) { // Implementation notes: // 1) LOCK prefix creates a full read/write reordering barrier for memory // operations issued by the current processor. As such, the location @@ -28551,18 +29236,28 @@ static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget, SDValue In = Op.getOperand(0); SDLoc DL(Op); + assert(VT.getScalarType() == MVT::i8 && + "Only byte vector BITREVERSE supported"); + // Split v64i8 without BWI so that we can still use the PSHUFB lowering. if (VT == MVT::v64i8 && !Subtarget.hasBWI()) return splitVectorIntUnary(Op, DAG); - unsigned NumElts = VT.getVectorNumElements(); - assert(VT.getScalarType() == MVT::i8 && - "Only byte vector BITREVERSE supported"); - // Decompose 256-bit ops into smaller 128-bit ops on pre-AVX2. - if (VT.is256BitVector() && !Subtarget.hasInt256()) + if (VT == MVT::v32i8 && !Subtarget.hasInt256()) return splitVectorIntUnary(Op, DAG); + unsigned NumElts = VT.getVectorNumElements(); + + // If we have GFNI, we can use GF2P8AFFINEQB to reverse the bits. + if (Subtarget.hasGFNI()) { + MVT MatrixVT = MVT::getVectorVT(MVT::i64, NumElts / 8); + SDValue Matrix = DAG.getConstant(0x8040201008040201ULL, DL, MatrixVT); + Matrix = DAG.getBitcast(VT, Matrix); + return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix, + DAG.getTargetConstant(0, DL, MVT::i8)); + } + // Perform BITREVERSE using PSHUFB lookups. Each byte is split into // two nibbles and a PSHUFB lookup to find the bitreverse of each // 0-15 value (moved to the other nibble). @@ -28594,6 +29289,58 @@ static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget, return DAG.getNode(ISD::OR, DL, VT, Lo, Hi); } +static SDValue LowerPARITY(SDValue Op, const X86Subtarget &Subtarget, + SelectionDAG &DAG) { + SDLoc DL(Op); + SDValue X = Op.getOperand(0); + MVT VT = Op.getSimpleValueType(); + + // Special case. If the input fits in 8-bits we can use a single 8-bit TEST. + if (VT == MVT::i8 || + DAG.MaskedValueIsZero(X, APInt::getBitsSetFrom(VT.getSizeInBits(), 8))) { + X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X); + SDValue Flags = DAG.getNode(X86ISD::CMP, DL, MVT::i32, X, + DAG.getConstant(0, DL, MVT::i8)); + // Copy the inverse of the parity flag into a register with setcc. + SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG); + // Extend to the original type. + return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp); + } + + if (VT == MVT::i64) { + // Xor the high and low 16-bits together using a 32-bit operation. + SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, + DAG.getNode(ISD::SRL, DL, MVT::i64, X, + DAG.getConstant(32, DL, MVT::i8))); + SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X); + X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi); + } + + if (VT != MVT::i16) { + // Xor the high and low 16-bits together using a 32-bit operation. + SDValue Hi16 = DAG.getNode(ISD::SRL, DL, MVT::i32, X, + DAG.getConstant(16, DL, MVT::i8)); + X = DAG.getNode(ISD::XOR, DL, MVT::i32, X, Hi16); + } else { + // If the input is 16-bits, we need to extend to use an i32 shift below. + X = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, X); + } + + // Finally xor the low 2 bytes together and use a 8-bit flag setting xor. + // This should allow an h-reg to be used to save a shift. + SDValue Hi = DAG.getNode( + ISD::TRUNCATE, DL, MVT::i8, + DAG.getNode(ISD::SRL, DL, MVT::i32, X, DAG.getConstant(8, DL, MVT::i8))); + SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X); + SDVTList VTs = DAG.getVTList(MVT::i8, MVT::i32); + SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1); + + // Copy the inverse of the parity flag into a register with setcc. + SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG); + // Extend to the original type. + return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp); +} + static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget) { unsigned NewOpc = 0; @@ -28730,7 +29477,7 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); Chain = DAG.getStore(Node->getChain(), dl, Node->getOperand(2), StackPtr, - MPI, /*Align*/ 0, MachineMemOperand::MOStore); + MPI, MaybeAlign(), MachineMemOperand::MOStore); SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); SDValue LdOps[] = {Chain, StackPtr}; SDValue Value = @@ -28770,6 +29517,7 @@ static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG, static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) { SDNode *N = Op.getNode(); MVT VT = N->getSimpleValueType(0); + unsigned Opc = Op.getOpcode(); // Let legalize expand this if it isn't a legal type yet. if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) @@ -28784,11 +29532,14 @@ static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) { Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), Carry, DAG.getAllOnesConstant(DL, CarryVT)); - unsigned Opc = Op.getOpcode() == ISD::ADDCARRY ? X86ISD::ADC : X86ISD::SBB; - SDValue Sum = DAG.getNode(Opc, DL, VTs, Op.getOperand(0), - Op.getOperand(1), Carry.getValue(1)); + bool IsAdd = Opc == ISD::ADDCARRY || Opc == ISD::SADDO_CARRY; + SDValue Sum = DAG.getNode(IsAdd ? X86ISD::ADC : X86ISD::SBB, DL, VTs, + Op.getOperand(0), Op.getOperand(1), + Carry.getValue(1)); - SDValue SetCC = getSETCC(X86::COND_B, Sum.getValue(1), DL, DAG); + bool IsSigned = Opc == ISD::SADDO_CARRY || Opc == ISD::SSUBO_CARRY; + SDValue SetCC = getSETCC(IsSigned ? X86::COND_O : X86::COND_B, + Sum.getValue(1), DL, DAG); if (N->getValueType(1) == MVT::i1) SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC); @@ -29164,25 +29915,6 @@ SDValue X86TargetLowering::LowerGC_TRANSITION(SDValue Op, return NOOP; } -SDValue X86TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG, - RTLIB::Libcall Call) const { - - bool IsStrict = Op->isStrictFPOpcode(); - unsigned Offset = IsStrict ? 1 : 0; - SmallVector<SDValue, 2> Ops(Op->op_begin() + Offset, Op->op_end()); - - SDLoc dl(Op); - SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue(); - MakeLibCallOptions CallOptions; - std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, Call, MVT::f128, Ops, - CallOptions, dl, Chain); - - if (IsStrict) - return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl); - - return Tmp.first; -} - // Custom split CVTPS2PH with wide types. static SDValue LowerCVTPS2PH(SDValue Op, SelectionDAG &DAG) { SDLoc dl(Op); @@ -29212,6 +29944,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::ATOMIC_LOAD_AND: return lowerAtomicArith(Op, DAG, Subtarget); case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op, DAG, Subtarget); case ISD::BITREVERSE: return LowerBITREVERSE(Op, Subtarget, DAG); + case ISD::PARITY: return LowerPARITY(Op, Subtarget, DAG); case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, Subtarget, DAG); case ISD::VECTOR_SHUFFLE: return lowerVECTOR_SHUFFLE(Op, Subtarget, DAG); @@ -29246,6 +29979,8 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::STRICT_FP_TO_SINT: case ISD::FP_TO_UINT: case ISD::STRICT_FP_TO_UINT: return LowerFP_TO_INT(Op, DAG); + case ISD::FP_TO_SINT_SAT: + case ISD::FP_TO_UINT_SAT: return LowerFP_TO_INT_SAT(Op, DAG); case ISD::FP_EXTEND: case ISD::STRICT_FP_EXTEND: return LowerFP_EXTEND(Op, DAG); case ISD::FP_ROUND: @@ -29312,6 +30047,8 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::UMULO: return LowerXALUO(Op, DAG); case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG); case ISD::BITCAST: return LowerBITCAST(Op, Subtarget, DAG); + case ISD::SADDO_CARRY: + case ISD::SSUBO_CARRY: case ISD::ADDCARRY: case ISD::SUBCARRY: return LowerADDSUBCARRY(Op, DAG); case ISD::ADD: @@ -29337,35 +30074,6 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { } } -/// Places new result values for the node in Results (their number -/// and types must exactly match those of the original return values of -/// the node), or leaves Results empty, which indicates that the node is not -/// to be custom lowered after all. -void X86TargetLowering::LowerOperationWrapper(SDNode *N, - SmallVectorImpl<SDValue> &Results, - SelectionDAG &DAG) const { - SDValue Res = LowerOperation(SDValue(N, 0), DAG); - - if (!Res.getNode()) - return; - - // If the original node has one result, take the return value from - // LowerOperation as is. It might not be result number 0. - if (N->getNumValues() == 1) { - Results.push_back(Res); - return; - } - - // If the original node has multiple results, then the return node should - // have the same number of results. - assert((N->getNumValues() == Res->getNumValues()) && - "Lowering returned the wrong number of results!"); - - // Places new result values base on N result number. - for (unsigned I = 0, E = N->getNumValues(); I != E; ++I) - Results.push_back(Res.getValue(I)); -} - /// Replace a node with an illegal result type with a new node built out of /// custom code. void X86TargetLowering::ReplaceNodeResults(SDNode *N, @@ -29408,6 +30116,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, Results.push_back(Chain); return; } + case X86ISD::CVTPS2PH: + Results.push_back(LowerCVTPS2PH(SDValue(N, 0), DAG)); + return; case ISD::CTPOP: { assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!"); // Use a v2i64 if possible. @@ -29476,28 +30187,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, Results.push_back(Res); return; } - case ISD::ABS: { - assert(N->getValueType(0) == MVT::i64 && - "Unexpected type (!= i64) on ABS."); - MVT HalfT = MVT::i32; - SDValue Lo, Hi, Tmp; - SDVTList VTList = DAG.getVTList(HalfT, MVT::i1); - - Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(0), - DAG.getConstant(0, dl, HalfT)); - Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(0), - DAG.getConstant(1, dl, HalfT)); - Tmp = DAG.getNode( - ISD::SRA, dl, HalfT, Hi, - DAG.getShiftAmountConstant(HalfT.getSizeInBits() - 1, HalfT, dl)); - Lo = DAG.getNode(ISD::UADDO, dl, VTList, Tmp, Lo); - Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Tmp, Hi, - SDValue(Lo.getNode(), 1)); - Hi = DAG.getNode(ISD::XOR, dl, HalfT, Tmp, Hi); - Lo = DAG.getNode(ISD::XOR, dl, HalfT, Tmp, Lo); - Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi)); - return; - } // We might have generated v2f32 FMIN/FMAX operations. Widen them to v4f32. case X86ISD::FMINC: case X86ISD::FMIN: @@ -29538,10 +30227,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, return; } - LLVM_FALLTHROUGH; - } - case ISD::SDIVREM: - case ISD::UDIVREM: { SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG); Results.push_back(V); return; @@ -29675,7 +30360,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); assert(isTypeLegal(LoVT) && "Split VT not legal?"); - SDValue Lo = getExtendInVec(N->getOpcode(), dl, LoVT, In, DAG); + SDValue Lo = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, LoVT, In, DAG); // We need to shift the input over by half the number of elements. unsigned NumElts = InVT.getVectorNumElements(); @@ -29685,7 +30370,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, ShufMask[i] = i + HalfNumElts; SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask); - Hi = getExtendInVec(N->getOpcode(), dl, HiVT, Hi, DAG); + Hi = getEXTEND_VECTOR_INREG(N->getOpcode(), dl, HiVT, Hi, DAG); SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); Results.push_back(Res); @@ -30036,46 +30721,30 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, swapInH = DAG.getCopyToReg(cpInH.getValue(0), dl, Regs64bit ? X86::RCX : X86::ECX, swapInH, cpInH.getValue(1)); - // If the current function needs the base pointer, RBX, - // we shouldn't use cmpxchg directly. - // Indeed the lowering of that instruction will clobber - // that register and since RBX will be a reserved register - // the register allocator will not make sure its value will - // be properly saved and restored around this live-range. - const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); + + // In 64-bit mode we might need the base pointer in RBX, but we can't know + // until later. So we keep the RBX input in a vreg and use a custom + // inserter. + // Since RBX will be a reserved register the register allocator will not + // make sure its value will be properly saved and restored around this + // live-range. SDValue Result; SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); - Register BasePtr = TRI->getBaseRegister(); MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand(); - if (TRI->hasBasePointer(DAG.getMachineFunction()) && - (BasePtr == X86::RBX || BasePtr == X86::EBX)) { - // ISel prefers the LCMPXCHG64 variant. - // If that assert breaks, that means it is not the case anymore, - // and we need to teach LCMPXCHG8_SAVE_EBX_DAG how to save RBX, - // not just EBX. This is a matter of accepting i64 input for that - // pseudo, and restoring into the register of the right wide - // in expand pseudo. Everything else should just work. - assert(((Regs64bit == (BasePtr == X86::RBX)) || BasePtr == X86::EBX) && - "Saving only half of the RBX"); - unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_SAVE_RBX_DAG - : X86ISD::LCMPXCHG8_SAVE_EBX_DAG; - SDValue RBXSave = DAG.getCopyFromReg(swapInH.getValue(0), dl, - Regs64bit ? X86::RBX : X86::EBX, - HalfT, swapInH.getValue(1)); - SDValue Ops[] = {/*Chain*/ RBXSave.getValue(1), N->getOperand(1), swapInL, - RBXSave, - /*Glue*/ RBXSave.getValue(2)}; - Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, T, MMO); + if (Regs64bit) { + SDValue Ops[] = {swapInH.getValue(0), N->getOperand(1), swapInL, + swapInH.getValue(1)}; + Result = + DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG16_DAG, dl, Tys, Ops, T, MMO); } else { - unsigned Opcode = - Regs64bit ? X86ISD::LCMPXCHG16_DAG : X86ISD::LCMPXCHG8_DAG; - swapInL = DAG.getCopyToReg(swapInH.getValue(0), dl, - Regs64bit ? X86::RBX : X86::EBX, swapInL, + swapInL = DAG.getCopyToReg(swapInH.getValue(0), dl, X86::EBX, swapInL, swapInH.getValue(1)); SDValue Ops[] = {swapInL.getValue(0), N->getOperand(1), swapInL.getValue(1)}; - Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, T, MMO); + Result = + DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, Ops, T, MMO); } + SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, Regs64bit ? X86::RAX : X86::EAX, HalfT, Result.getValue(1)); @@ -30284,6 +30953,13 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N, Results.push_back(V); return; } + case ISD::BITREVERSE: + assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!"); + assert(Subtarget.hasXOP() && "Expected XOP"); + // We can use VPPERM by copying to a vector register and back. We'll need + // to move the scalar in two i32 pieces. + Results.push_back(LowerBITREVERSE(SDValue(N, 0), Subtarget, DAG)); + return; } } @@ -30313,8 +30989,9 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(COMI) NODE_NAME_CASE(UCOMI) NODE_NAME_CASE(CMPM) + NODE_NAME_CASE(CMPMM) NODE_NAME_CASE(STRICT_CMPM) - NODE_NAME_CASE(CMPM_SAE) + NODE_NAME_CASE(CMPMM_SAE) NODE_NAME_CASE(SETCC) NODE_NAME_CASE(SETCC_CARRY) NODE_NAME_CASE(FSETCC) @@ -30373,7 +31050,6 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(LCMPXCHG_DAG) NODE_NAME_CASE(LCMPXCHG8_DAG) NODE_NAME_CASE(LCMPXCHG16_DAG) - NODE_NAME_CASE(LCMPXCHG8_SAVE_EBX_DAG) NODE_NAME_CASE(LCMPXCHG16_SAVE_RBX_DAG) NODE_NAME_CASE(LADD) NODE_NAME_CASE(LSUB) @@ -30433,6 +31109,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(XOR) NODE_NAME_CASE(AND) NODE_NAME_CASE(BEXTR) + NODE_NAME_CASE(BEXTRI) NODE_NAME_CASE(BZHI) NODE_NAME_CASE(PDEP) NODE_NAME_CASE(PEXT) @@ -30470,7 +31147,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(VBROADCAST) NODE_NAME_CASE(VBROADCAST_LOAD) NODE_NAME_CASE(VBROADCASTM) - NODE_NAME_CASE(SUBV_BROADCAST) + NODE_NAME_CASE(SUBV_BROADCAST_LOAD) NODE_NAME_CASE(VPERMILPV) NODE_NAME_CASE(VPERMILPI) NODE_NAME_CASE(VPERM2X128) @@ -30492,6 +31169,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(DBPSADBW) NODE_NAME_CASE(VASTART_SAVE_XMM_REGS) NODE_NAME_CASE(VAARG_64) + NODE_NAME_CASE(VAARG_X32) NODE_NAME_CASE(WIN_ALLOCA) NODE_NAME_CASE(MEMBARRIER) NODE_NAME_CASE(MFENCE) @@ -30648,6 +31326,15 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(ENQCMD) NODE_NAME_CASE(ENQCMDS) NODE_NAME_CASE(VP2INTERSECT) + NODE_NAME_CASE(AESENC128KL) + NODE_NAME_CASE(AESDEC128KL) + NODE_NAME_CASE(AESENC256KL) + NODE_NAME_CASE(AESDEC256KL) + NODE_NAME_CASE(AESENCWIDE128KL) + NODE_NAME_CASE(AESDECWIDE128KL) + NODE_NAME_CASE(AESENCWIDE256KL) + NODE_NAME_CASE(AESDECWIDE256KL) + NODE_NAME_CASE(TESTUI) } return nullptr; #undef NODE_NAME_CASE @@ -30962,10 +31649,38 @@ bool X86TargetLowering::areJTsAllowed(const Function *Fn) const { // X86 Scheduler Hooks //===----------------------------------------------------------------------===// +// Returns true if EFLAG is consumed after this iterator in the rest of the +// basic block or any successors of the basic block. +static bool isEFLAGSLiveAfter(MachineBasicBlock::iterator Itr, + MachineBasicBlock *BB) { + // Scan forward through BB for a use/def of EFLAGS. + for (MachineBasicBlock::iterator miI = std::next(Itr), miE = BB->end(); + miI != miE; ++miI) { + const MachineInstr& mi = *miI; + if (mi.readsRegister(X86::EFLAGS)) + return true; + // If we found a def, we can stop searching. + if (mi.definesRegister(X86::EFLAGS)) + return false; + } + + // If we hit the end of the block, check whether EFLAGS is live into a + // successor. + for (MachineBasicBlock::succ_iterator sItr = BB->succ_begin(), + sEnd = BB->succ_end(); + sItr != sEnd; ++sItr) { + MachineBasicBlock* succ = *sItr; + if (succ->isLiveIn(X86::EFLAGS)) + return true; + } + + return false; +} + /// Utility function to emit xbegin specifying the start of an RTM region. static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII) { - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); const BasicBlock *BB = MBB->getBasicBlock(); MachineFunction::iterator I = ++MBB->getIterator(); @@ -30994,6 +31709,12 @@ static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB, MF->insert(I, fallMBB); MF->insert(I, sinkMBB); + if (isEFLAGSLiveAfter(MI, MBB)) { + mainMBB->addLiveIn(X86::EFLAGS); + fallMBB->addLiveIn(X86::EFLAGS); + sinkMBB->addLiveIn(X86::EFLAGS); + } + // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)), MBB->end()); @@ -31038,11 +31759,9 @@ static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB, return sinkMBB; } - - MachineBasicBlock * -X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI, - MachineBasicBlock *MBB) const { +X86TargetLowering::EmitVAARGWithCustomInserter(MachineInstr &MI, + MachineBasicBlock *MBB) const { // Emit va_arg instruction on X86-64. // Operands to this pseudo-instruction: @@ -31053,9 +31772,8 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI, // 8 ) Align : Alignment of type // 9 ) EFLAGS (implicit-def) - assert(MI.getNumOperands() == 10 && "VAARG_64 should have 10 operands!"); - static_assert(X86::AddrNumOperands == 5, - "VAARG_64 assumes 5 address operands"); + assert(MI.getNumOperands() == 10 && "VAARG should have 10 operands!"); + static_assert(X86::AddrNumOperands == 5, "VAARG assumes 5 address operands"); Register DestReg = MI.getOperand(0).getReg(); MachineOperand &Base = MI.getOperand(1); @@ -31070,7 +31788,7 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI, MachineFunction *MF = MBB->getParent(); // Memory Reference - assert(MI.hasOneMemOperand() && "Expected VAARG_64 to have one memoperand"); + assert(MI.hasOneMemOperand() && "Expected VAARG to have one memoperand"); MachineMemOperand *OldMMO = MI.memoperands().front(); @@ -31083,9 +31801,10 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI, // Machine Information const TargetInstrInfo *TII = Subtarget.getInstrInfo(); MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); - const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64); + const TargetRegisterClass *AddrRegClass = + getRegClassFor(getPointerTy(MBB->getParent()->getDataLayout())); const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32); - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); // struct va_list { // i32 gp_offset @@ -31194,25 +31913,35 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI, // Read the reg_save_area address. Register RegSaveReg = MRI.createVirtualRegister(AddrRegClass); - BuildMI(offsetMBB, DL, TII->get(X86::MOV64rm), RegSaveReg) + BuildMI( + offsetMBB, DL, + TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm), + RegSaveReg) .add(Base) .add(Scale) .add(Index) - .addDisp(Disp, 16) + .addDisp(Disp, Subtarget.isTarget64BitLP64() ? 16 : 12) .add(Segment) .setMemRefs(LoadOnlyMMO); - // Zero-extend the offset - Register OffsetReg64 = MRI.createVirtualRegister(AddrRegClass); - BuildMI(offsetMBB, DL, TII->get(X86::SUBREG_TO_REG), OffsetReg64) - .addImm(0) - .addReg(OffsetReg) - .addImm(X86::sub_32bit); + if (Subtarget.isTarget64BitLP64()) { + // Zero-extend the offset + Register OffsetReg64 = MRI.createVirtualRegister(AddrRegClass); + BuildMI(offsetMBB, DL, TII->get(X86::SUBREG_TO_REG), OffsetReg64) + .addImm(0) + .addReg(OffsetReg) + .addImm(X86::sub_32bit); - // Add the offset to the reg_save_area to get the final address. - BuildMI(offsetMBB, DL, TII->get(X86::ADD64rr), OffsetDestReg) - .addReg(OffsetReg64) - .addReg(RegSaveReg); + // Add the offset to the reg_save_area to get the final address. + BuildMI(offsetMBB, DL, TII->get(X86::ADD64rr), OffsetDestReg) + .addReg(OffsetReg64) + .addReg(RegSaveReg); + } else { + // Add the offset to the reg_save_area to get the final address. + BuildMI(offsetMBB, DL, TII->get(X86::ADD32rr), OffsetDestReg) + .addReg(OffsetReg) + .addReg(RegSaveReg); + } // Compute the offset for the next argument Register NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass); @@ -31241,7 +31970,9 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI, // Load the overflow_area address into a register. Register OverflowAddrReg = MRI.createVirtualRegister(AddrRegClass); - BuildMI(overflowMBB, DL, TII->get(X86::MOV64rm), OverflowAddrReg) + BuildMI(overflowMBB, DL, + TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64rm : X86::MOV32rm), + OverflowAddrReg) .add(Base) .add(Scale) .add(Index) @@ -31256,11 +31987,17 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI, Register TmpReg = MRI.createVirtualRegister(AddrRegClass); // aligned_addr = (addr + (align-1)) & ~(align-1) - BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), TmpReg) + BuildMI( + overflowMBB, DL, + TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri), + TmpReg) .addReg(OverflowAddrReg) .addImm(Alignment.value() - 1); - BuildMI(overflowMBB, DL, TII->get(X86::AND64ri32), OverflowDestReg) + BuildMI( + overflowMBB, DL, + TII->get(Subtarget.isTarget64BitLP64() ? X86::AND64ri32 : X86::AND32ri), + OverflowDestReg) .addReg(TmpReg) .addImm(~(uint64_t)(Alignment.value() - 1)); } else { @@ -31271,12 +32008,16 @@ X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI, // Compute the next overflow address after this argument. // (the overflow address should be kept 8-byte aligned) Register NextAddrReg = MRI.createVirtualRegister(AddrRegClass); - BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), NextAddrReg) - .addReg(OverflowDestReg) - .addImm(ArgSizeA8); + BuildMI( + overflowMBB, DL, + TII->get(Subtarget.isTarget64BitLP64() ? X86::ADD64ri32 : X86::ADD32ri), + NextAddrReg) + .addReg(OverflowDestReg) + .addImm(ArgSizeA8); // Store the new overflow address. - BuildMI(overflowMBB, DL, TII->get(X86::MOV64mr)) + BuildMI(overflowMBB, DL, + TII->get(Subtarget.isTarget64BitLP64() ? X86::MOV64mr : X86::MOV32mr)) .add(Base) .add(Scale) .add(Index) @@ -31332,10 +32073,10 @@ MachineBasicBlock *X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter( // Now add the instructions. const TargetInstrInfo *TII = Subtarget.getInstrInfo(); - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); Register CountReg = MI.getOperand(0).getReg(); - int64_t RegSaveFrameIndex = MI.getOperand(1).getImm(); + int RegSaveFrameIndex = MI.getOperand(1).getImm(); int64_t VarArgsFPOffset = MI.getOperand(2).getImm(); if (!Subtarget.isCallingConvWin64(F->getFunction().getCallingConv())) { @@ -31382,27 +32123,8 @@ MachineBasicBlock *X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter( static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr, MachineBasicBlock* BB, const TargetRegisterInfo* TRI) { - // Scan forward through BB for a use/def of EFLAGS. - MachineBasicBlock::iterator miI(std::next(SelectItr)); - for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) { - const MachineInstr& mi = *miI; - if (mi.readsRegister(X86::EFLAGS)) - return false; - if (mi.definesRegister(X86::EFLAGS)) - break; // Should have kill-flag - update below. - } - - // If we hit the end of the block, check whether EFLAGS is live into a - // successor. - if (miI == BB->end()) { - for (MachineBasicBlock::succ_iterator sItr = BB->succ_begin(), - sEnd = BB->succ_end(); - sItr != sEnd; ++sItr) { - MachineBasicBlock* succ = *sItr; - if (succ->isLiveIn(X86::EFLAGS)) - return false; - } - } + if (isEFLAGSLiveAfter(SelectItr, BB)) + return false; // We found a def, or hit the end of the basic block and EFLAGS wasn't live // out. SelectMI should have a kill flag on EFLAGS. @@ -31663,7 +32385,7 @@ MachineBasicBlock * X86TargetLowering::EmitLoweredSelect(MachineInstr &MI, MachineBasicBlock *ThisMBB) const { const TargetInstrInfo *TII = Subtarget.getInstrInfo(); - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); // To "insert" a SELECT_CC instruction, we actually have to insert the // diamond control-flow pattern. The incoming instruction knows the @@ -31818,7 +32540,7 @@ X86TargetLowering::EmitLoweredProbedAlloca(MachineInstr &MI, MachineFunction *MF = MBB->getParent(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); const X86FrameLowering &TFI = *Subtarget.getFrameLowering(); - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); const BasicBlock *LLVM_BB = MBB->getBasicBlock(); const unsigned ProbeSize = getStackProbeSize(*MF); @@ -31860,7 +32582,7 @@ X86TargetLowering::EmitLoweredProbedAlloca(MachineInstr &MI, BuildMI(testMBB, DL, TII->get(X86::JCC_1)) .addMBB(tailMBB) - .addImm(X86::COND_L); + .addImm(X86::COND_GE); testMBB->addSuccessor(blockMBB); testMBB->addSuccessor(tailMBB); @@ -31876,9 +32598,9 @@ X86TargetLowering::EmitLoweredProbedAlloca(MachineInstr &MI, // // The property we want to enforce is to never have more than [page alloc] between two probes. - const unsigned MovMIOpc = - TFI.Uses64BitFramePtr ? X86::MOV64mi32 : X86::MOV32mi; - addRegOffset(BuildMI(blockMBB, DL, TII->get(MovMIOpc)), physSPReg, false, 0) + const unsigned XORMIOpc = + TFI.Uses64BitFramePtr ? X86::XOR64mi8 : X86::XOR32mi8; + addRegOffset(BuildMI(blockMBB, DL, TII->get(XORMIOpc)), physSPReg, false, 0) .addImm(0); BuildMI(blockMBB, DL, @@ -31911,7 +32633,7 @@ X86TargetLowering::EmitLoweredSegAlloca(MachineInstr &MI, MachineBasicBlock *BB) const { MachineFunction *MF = BB->getParent(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); const BasicBlock *LLVM_BB = BB->getBasicBlock(); assert(MF->shouldSplitStack()); @@ -31946,7 +32668,7 @@ X86TargetLowering::EmitLoweredSegAlloca(MachineInstr &MI, const TargetRegisterClass *AddrRegClass = getRegClassFor(getPointerTy(MF->getDataLayout())); - unsigned mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass), + Register mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass), bumpSPPtrVReg = MRI.createVirtualRegister(AddrRegClass), tmpSPVReg = MRI.createVirtualRegister(AddrRegClass), SPLimitVReg = MRI.createVirtualRegister(AddrRegClass), @@ -32046,7 +32768,7 @@ X86TargetLowering::EmitLoweredCatchRet(MachineInstr &MI, MachineFunction *MF = BB->getParent(); const TargetInstrInfo &TII = *Subtarget.getInstrInfo(); MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB(); - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); assert(!isAsynchronousEHPersonality( classifyEHPersonality(MF->getFunction().getPersonalityFn())) && @@ -32084,7 +32806,7 @@ X86TargetLowering::EmitLoweredTLSAddr(MachineInstr &MI, // inside MC, therefore without the two markers shrink-wrapping // may push the prologue/epilogue pass them. const TargetInstrInfo &TII = *Subtarget.getInstrInfo(); - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); MachineFunction &MF = *BB->getParent(); // Emit CALLSEQ_START right before the instruction. @@ -32113,7 +32835,7 @@ X86TargetLowering::EmitLoweredTLSCall(MachineInstr &MI, // be in the normal return register. MachineFunction *F = BB->getParent(); const X86InstrInfo *TII = Subtarget.getInstrInfo(); - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); assert(Subtarget.isTargetDarwin() && "Darwin only instr emitted?"); assert(MI.getOperand(3).isGlobal() && "This should be a global"); @@ -32252,7 +32974,7 @@ X86TargetLowering::EmitLoweredIndirectThunk(MachineInstr &MI, MachineBasicBlock *BB) const { // Copy the virtual register into the R11 physical register and // call the retpoline thunk. - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); const X86InstrInfo *TII = Subtarget.getInstrInfo(); Register CalleeVReg = MI.getOperand(0).getReg(); unsigned Opc = getOpcodeForIndirectThunk(MI.getOpcode()); @@ -32314,7 +33036,7 @@ X86TargetLowering::EmitLoweredIndirectThunk(MachineInstr &MI, /// \param [in] MBB The Machine Basic Block that will be modified. void X86TargetLowering::emitSetJmpShadowStackFix(MachineInstr &MI, MachineBasicBlock *MBB) const { - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); MachineFunction *MF = MBB->getParent(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); MachineRegisterInfo &MRI = MF->getRegInfo(); @@ -32357,7 +33079,7 @@ void X86TargetLowering::emitSetJmpShadowStackFix(MachineInstr &MI, MachineBasicBlock * X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI, MachineBasicBlock *MBB) const { - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); MachineFunction *MF = MBB->getParent(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); @@ -32517,7 +33239,7 @@ X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI, MachineBasicBlock * X86TargetLowering::emitLongJmpShadowStackFix(MachineInstr &MI, MachineBasicBlock *MBB) const { - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); MachineFunction *MF = MBB->getParent(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); MachineRegisterInfo &MRI = MF->getRegInfo(); @@ -32698,7 +33420,7 @@ X86TargetLowering::emitLongJmpShadowStackFix(MachineInstr &MI, MachineBasicBlock * X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI, MachineBasicBlock *MBB) const { - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); MachineFunction *MF = MBB->getParent(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); MachineRegisterInfo &MRI = MF->getRegInfo(); @@ -32782,7 +33504,7 @@ void X86TargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock *MBB, MachineBasicBlock *DispatchBB, int FI) const { - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); MachineFunction *MF = MBB->getParent(); MachineRegisterInfo *MRI = &MF->getRegInfo(); const X86InstrInfo *TII = Subtarget.getInstrInfo(); @@ -32831,7 +33553,7 @@ void X86TargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock * X86TargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI, MachineBasicBlock *BB) const { - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); MachineFunction *MF = BB->getParent(); MachineRegisterInfo *MRI = &MF->getRegInfo(); const X86InstrInfo *TII = Subtarget.getInstrInfo(); @@ -33061,7 +33783,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const { MachineFunction *MF = BB->getParent(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); - DebugLoc DL = MI.getDebugLoc(); + const DebugLoc &DL = MI.getDebugLoc(); auto TMMImmToTMMReg = [](unsigned Imm) { assert (Imm < 8 && "Illegal tmm index"); @@ -33071,8 +33793,10 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, default: llvm_unreachable("Unexpected instr type to insert"); case X86::TLS_addr32: case X86::TLS_addr64: + case X86::TLS_addrX32: case X86::TLS_base_addr32: case X86::TLS_base_addr64: + case X86::TLS_base_addrX32: return EmitLoweredTLSAddr(MI, BB); case X86::INDIRECT_THUNK_CALL32: case X86::INDIRECT_THUNK_CALL64: @@ -33228,7 +33952,8 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, return EmitVAStartSaveXMMRegsWithCustomInserter(MI, BB); case X86::VAARG_64: - return EmitVAARG64WithCustomInserter(MI, BB); + case X86::VAARG_X32: + return EmitVAARGWithCustomInserter(MI, BB); case X86::EH_SjLj_SetJmp32: case X86::EH_SjLj_SetJmp64: @@ -33251,10 +33976,8 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, return emitPatchPoint(MI, BB); case TargetOpcode::PATCHABLE_EVENT_CALL: - return emitXRayCustomEvent(MI, BB); - case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL: - return emitXRayTypedEvent(MI, BB); + return BB; case X86::LCMPXCHG8B: { const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); @@ -33309,14 +34032,75 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, return BB; } - case X86::LCMPXCHG16B: + case X86::LCMPXCHG16B_NO_RBX: { + const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); + Register BasePtr = TRI->getBaseRegister(); + if (TRI->hasBasePointer(*MF) && + (BasePtr == X86::RBX || BasePtr == X86::EBX)) { + if (!BB->isLiveIn(BasePtr)) + BB->addLiveIn(BasePtr); + // Save RBX into a virtual register. + Register SaveRBX = + MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), SaveRBX) + .addReg(X86::RBX); + Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass); + MachineInstrBuilder MIB = + BuildMI(*BB, MI, DL, TII->get(X86::LCMPXCHG16B_SAVE_RBX), Dst); + for (unsigned Idx = 0; Idx < X86::AddrNumOperands; ++Idx) + MIB.add(MI.getOperand(Idx)); + MIB.add(MI.getOperand(X86::AddrNumOperands)); + MIB.addReg(SaveRBX); + } else { + // Simple case, just copy the virtual register to RBX. + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), X86::RBX) + .add(MI.getOperand(X86::AddrNumOperands)); + MachineInstrBuilder MIB = + BuildMI(*BB, MI, DL, TII->get(X86::LCMPXCHG16B)); + for (unsigned Idx = 0; Idx < X86::AddrNumOperands; ++Idx) + MIB.add(MI.getOperand(Idx)); + } + MI.eraseFromParent(); return BB; - case X86::LCMPXCHG8B_SAVE_EBX: - case X86::LCMPXCHG16B_SAVE_RBX: { - unsigned BasePtr = - MI.getOpcode() == X86::LCMPXCHG8B_SAVE_EBX ? X86::EBX : X86::RBX; - if (!BB->isLiveIn(BasePtr)) - BB->addLiveIn(BasePtr); + } + case X86::MWAITX: { + const X86RegisterInfo *TRI = Subtarget.getRegisterInfo(); + Register BasePtr = TRI->getBaseRegister(); + bool IsRBX = (BasePtr == X86::RBX || BasePtr == X86::EBX); + // If no need to save the base pointer, we generate MWAITXrrr, + // else we generate pseudo MWAITX_SAVE_RBX. + if (!IsRBX || !TRI->hasBasePointer(*MF)) { + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), X86::ECX) + .addReg(MI.getOperand(0).getReg()); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), X86::EAX) + .addReg(MI.getOperand(1).getReg()); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), X86::EBX) + .addReg(MI.getOperand(2).getReg()); + BuildMI(*BB, MI, DL, TII->get(X86::MWAITXrrr)); + MI.eraseFromParent(); + } else { + if (!BB->isLiveIn(BasePtr)) { + BB->addLiveIn(BasePtr); + } + // Parameters can be copied into ECX and EAX but not EBX yet. + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), X86::ECX) + .addReg(MI.getOperand(0).getReg()); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), X86::EAX) + .addReg(MI.getOperand(1).getReg()); + assert(Subtarget.is64Bit() && "Expected 64-bit mode!"); + // Save RBX into a virtual register. + Register SaveRBX = + MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), SaveRBX) + .addReg(X86::RBX); + // Generate mwaitx pseudo. + Register Dst = MF->getRegInfo().createVirtualRegister(&X86::GR64RegClass); + BuildMI(*BB, MI, DL, TII->get(X86::MWAITX_SAVE_RBX)) + .addDef(Dst) // Destination tied in with SaveRBX. + .addReg(MI.getOperand(2).getReg()) // input value of EBX. + .addUse(SaveRBX); // Save of base pointer. + MI.eraseFromParent(); + } return BB; } case TargetOpcode::PREALLOCATED_SETUP: { @@ -33354,7 +34138,6 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, case X86::PTDPBUSD: case X86::PTDPBUUD: case X86::PTDPBF16PS: { - const DebugLoc &DL = MI.getDebugLoc(); unsigned Opc; switch (MI.getOpcode()) { case X86::PTDPBSSD: Opc = X86::TDPBSSD; break; @@ -33374,7 +34157,6 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, return BB; } case X86::PTILEZERO: { - const DebugLoc &DL = MI.getDebugLoc(); unsigned Imm = MI.getOperand(0).getImm(); BuildMI(*BB, MI, DL, TII->get(X86::TILEZERO), TMMImmToTMMReg(Imm)); MI.eraseFromParent(); // The pseudo is gone now. @@ -33383,7 +34165,6 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, case X86::PTILELOADD: case X86::PTILELOADDT1: case X86::PTILESTORED: { - const DebugLoc &DL = MI.getDebugLoc(); unsigned Opc; switch (MI.getOpcode()) { case X86::PTILELOADD: Opc = X86::TILELOADD; break; @@ -33584,13 +34365,11 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op, KnownBits Known2; if (!!DemandedLHS) { Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedLHS, Depth + 1); - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); } if (!!DemandedRHS) { Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedRHS, Depth + 1); - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); } if (Known.countMinLeadingZeros() < BitWidth) @@ -33633,11 +34412,11 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op, KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1); // Only known if known in both the LHS and RHS. - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); break; } - case X86ISD::BEXTR: { + case X86ISD::BEXTR: + case X86ISD::BEXTRI: { SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); @@ -33659,6 +34438,28 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op, } break; } + case X86ISD::PDEP: { + KnownBits Known2; + Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); + Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); + // Zeros are retained from the mask operand. But not ones. + Known.One.clearAllBits(); + // The result will have at least as many trailing zeros as the non-mask + // operand since bits can only map to the same or higher bit position. + Known.Zero.setLowBits(Known2.countMinTrailingZeros()); + break; + } + case X86ISD::PEXT: { + Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); + // The result has as many leading zeros as the number of zeroes in the mask. + unsigned Count = Known.Zero.countPopulation(); + Known.Zero = APInt::getHighBitsSet(BitWidth, Count); + Known.One.clearAllBits(); + break; + } + case X86ISD::VTRUNC: + case X86ISD::VTRUNCS: + case X86ISD::VTRUNCUS: case X86ISD::CVTSI2P: case X86ISD::CVTUI2P: case X86ISD::CVTP2SI: @@ -33675,7 +34476,7 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op, case X86ISD::VMFPROUND: case X86ISD::CVTPS2PH: case X86ISD::MCVTPS2PH: { - // Conversions - upper elements are known zero. + // Truncations/Conversions - upper elements are known zero. EVT SrcVT = Op.getOperand(0).getValueType(); if (SrcVT.isVector()) { unsigned NumSrcElts = SrcVT.getVectorNumElements(); @@ -33753,8 +34554,7 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op, continue; KnownBits Known2 = DAG.computeKnownBits(Ops[i], DemandedOps[i], Depth + 1); - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + Known = KnownBits::commonBits(Known, Known2); } } } @@ -33933,11 +34733,18 @@ static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask, unsigned MaskEltSize = MaskVT.getScalarSizeInBits(); // Match against a VZEXT_MOVL vXi32 zero-extending instruction. - if (MaskEltSize == 32 && isUndefOrEqual(Mask[0], 0) && - isUndefOrZero(Mask[1]) && isUndefInRange(Mask, 2, NumMaskElts - 2)) { - Shuffle = X86ISD::VZEXT_MOVL; - SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; - return true; + if (MaskEltSize == 32 && Mask[0] == 0) { + if (isUndefOrZero(Mask[1]) && isUndefInRange(Mask, 2, NumMaskElts - 2)) { + Shuffle = X86ISD::VZEXT_MOVL; + SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; + return true; + } + if (V1.getOpcode() == ISD::SCALAR_TO_VECTOR && + isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1)) { + Shuffle = X86ISD::VZEXT_MOVL; + SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; + return true; + } } // Match against a ANY/ZERO_EXTEND_VECTOR_INREG instruction. @@ -33991,17 +34798,17 @@ static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask, // instructions are no slower than UNPCKLPD but has the option to // fold the input operand into even an unaligned memory load. if (MaskVT.is128BitVector() && Subtarget.hasSSE3() && AllowFloatDomain) { - if (isTargetShuffleEquivalent(Mask, {0, 0})) { + if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}, V1)) { Shuffle = X86ISD::MOVDDUP; SrcVT = DstVT = MVT::v2f64; return true; } - if (isTargetShuffleEquivalent(Mask, {0, 0, 2, 2})) { + if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, V1)) { Shuffle = X86ISD::MOVSLDUP; SrcVT = DstVT = MVT::v4f32; return true; } - if (isTargetShuffleEquivalent(Mask, {1, 1, 3, 3})) { + if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3}, V1)) { Shuffle = X86ISD::MOVSHDUP; SrcVT = DstVT = MVT::v4f32; return true; @@ -34010,17 +34817,17 @@ static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask, if (MaskVT.is256BitVector() && AllowFloatDomain) { assert(Subtarget.hasAVX() && "AVX required for 256-bit vector shuffles"); - if (isTargetShuffleEquivalent(Mask, {0, 0, 2, 2})) { + if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2}, V1)) { Shuffle = X86ISD::MOVDDUP; SrcVT = DstVT = MVT::v4f64; return true; } - if (isTargetShuffleEquivalent(Mask, {0, 0, 2, 2, 4, 4, 6, 6})) { + if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, V1)) { Shuffle = X86ISD::MOVSLDUP; SrcVT = DstVT = MVT::v8f32; return true; } - if (isTargetShuffleEquivalent(Mask, {1, 1, 3, 3, 5, 5, 7, 7})) { + if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1, 3, 3, 5, 5, 7, 7}, V1)) { Shuffle = X86ISD::MOVSHDUP; SrcVT = DstVT = MVT::v8f32; return true; @@ -34030,19 +34837,21 @@ static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask, if (MaskVT.is512BitVector() && AllowFloatDomain) { assert(Subtarget.hasAVX512() && "AVX512 required for 512-bit vector shuffles"); - if (isTargetShuffleEquivalent(Mask, {0, 0, 2, 2, 4, 4, 6, 6})) { + if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0, 2, 2, 4, 4, 6, 6}, V1)) { Shuffle = X86ISD::MOVDDUP; SrcVT = DstVT = MVT::v8f64; return true; } if (isTargetShuffleEquivalent( - Mask, {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14})) { + MaskVT, Mask, + {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14}, V1)) { Shuffle = X86ISD::MOVSLDUP; SrcVT = DstVT = MVT::v16f32; return true; } if (isTargetShuffleEquivalent( - Mask, {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15})) { + MaskVT, Mask, + {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15}, V1)) { Shuffle = X86ISD::MOVSHDUP; SrcVT = DstVT = MVT::v16f32; return true; @@ -34124,7 +34933,10 @@ static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask, } // Handle PSHUFLW/PSHUFHW vXi16 repeated patterns. - if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits == 16) { + if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits == 16 && + ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) || + (MaskVT.is256BitVector() && Subtarget.hasAVX2()) || + (MaskVT.is512BitVector() && Subtarget.hasBWI()))) { SmallVector<int, 4> RepeatedMask; if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) { ArrayRef<int> LoMask(RepeatedMask.data() + 0, 4); @@ -34194,30 +35006,31 @@ static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, bool IsUnary) { + unsigned NumMaskElts = Mask.size(); unsigned EltSizeInBits = MaskVT.getScalarSizeInBits(); if (MaskVT.is128BitVector()) { - if (isTargetShuffleEquivalent(Mask, {0, 0}) && AllowFloatDomain) { + if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 0}) && AllowFloatDomain) { V2 = V1; V1 = (SM_SentinelUndef == Mask[0] ? DAG.getUNDEF(MVT::v4f32) : V1); Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKL : X86ISD::MOVLHPS; SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32; return true; } - if (isTargetShuffleEquivalent(Mask, {1, 1}) && AllowFloatDomain) { + if (isTargetShuffleEquivalent(MaskVT, Mask, {1, 1}) && AllowFloatDomain) { V2 = V1; Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKH : X86ISD::MOVHLPS; SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32; return true; } - if (isTargetShuffleEquivalent(Mask, {0, 3}) && Subtarget.hasSSE2() && - (AllowFloatDomain || !Subtarget.hasSSE41())) { + if (isTargetShuffleEquivalent(MaskVT, Mask, {0, 3}) && + Subtarget.hasSSE2() && (AllowFloatDomain || !Subtarget.hasSSE41())) { std::swap(V1, V2); Shuffle = X86ISD::MOVSD; SrcVT = DstVT = MVT::v2f64; return true; } - if (isTargetShuffleEquivalent(Mask, {4, 1, 2, 3}) && + if (isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3}) && (AllowFloatDomain || !Subtarget.hasSSE41())) { Shuffle = X86ISD::MOVSS; SrcVT = DstVT = MVT::v4f32; @@ -34251,6 +35064,46 @@ static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask, } } + // Attempt to match against a OR if we're performing a blend shuffle and the + // non-blended source element is zero in each case. + if ((EltSizeInBits % V1.getScalarValueSizeInBits()) == 0 && + (EltSizeInBits % V2.getScalarValueSizeInBits()) == 0) { + bool IsBlend = true; + unsigned NumV1Elts = V1.getValueType().getVectorNumElements(); + unsigned NumV2Elts = V2.getValueType().getVectorNumElements(); + unsigned Scale1 = NumV1Elts / NumMaskElts; + unsigned Scale2 = NumV2Elts / NumMaskElts; + APInt DemandedZeroV1 = APInt::getNullValue(NumV1Elts); + APInt DemandedZeroV2 = APInt::getNullValue(NumV2Elts); + for (unsigned i = 0; i != NumMaskElts; ++i) { + int M = Mask[i]; + if (M == SM_SentinelUndef) + continue; + if (M == SM_SentinelZero) { + DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1); + DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2); + continue; + } + if (M == (int)i) { + DemandedZeroV2.setBits(i * Scale2, (i + 1) * Scale2); + continue; + } + if (M == (int)(i + NumMaskElts)) { + DemandedZeroV1.setBits(i * Scale1, (i + 1) * Scale1); + continue; + } + IsBlend = false; + break; + } + if (IsBlend && + DAG.computeKnownBits(V1, DemandedZeroV1).isZero() && + DAG.computeKnownBits(V2, DemandedZeroV2).isZero()) { + Shuffle = ISD::OR; + SrcVT = DstVT = MaskVT.changeTypeToInteger(); + return true; + } + } + return false; } @@ -34439,6 +35292,16 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, assert((Inputs.size() == 1 || Inputs.size() == 2) && "Unexpected number of shuffle inputs!"); + MVT RootVT = Root.getSimpleValueType(); + unsigned RootSizeInBits = RootVT.getSizeInBits(); + unsigned NumRootElts = RootVT.getVectorNumElements(); + + // Canonicalize shuffle input op to the requested type. + // TODO: Support cases where Op is smaller than VT. + auto CanonicalizeShuffleInput = [&](MVT VT, SDValue Op) { + return DAG.getBitcast(VT, Op); + }; + // Find the inputs that enter the chain. Note that multiple uses are OK // here, we're not going to remove the operands we find. bool UnaryShuffle = (Inputs.size() == 1); @@ -34448,10 +35311,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, MVT VT1 = V1.getSimpleValueType(); MVT VT2 = V2.getSimpleValueType(); - MVT RootVT = Root.getSimpleValueType(); - assert(VT1.getSizeInBits() == RootVT.getSizeInBits() && - VT2.getSizeInBits() == RootVT.getSizeInBits() && - "Vector size mismatch"); + assert(VT1.getSizeInBits() == RootSizeInBits && + VT2.getSizeInBits() == RootSizeInBits && "Vector size mismatch"); SDLoc DL(Root); SDValue Res; @@ -34459,12 +35320,10 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, unsigned NumBaseMaskElts = BaseMask.size(); if (NumBaseMaskElts == 1) { assert(BaseMask[0] == 0 && "Invalid shuffle index found!"); - return DAG.getBitcast(RootVT, V1); + return CanonicalizeShuffleInput(RootVT, V1); } bool OptForSize = DAG.shouldOptForSize(); - unsigned RootSizeInBits = RootVT.getSizeInBits(); - unsigned NumRootElts = RootVT.getVectorNumElements(); unsigned BaseMaskEltSizeInBits = RootSizeInBits / NumBaseMaskElts; bool FloatDomain = VT1.isFloatingPoint() || VT2.isFloatingPoint() || (RootVT.isFloatingPoint() && Depth >= 1) || @@ -34485,33 +35344,14 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, // we can just use the broadcast directly. This works for smaller broadcast // elements as well as they already repeat across each mask element if (UnaryShuffle && isTargetShuffleSplat(V1) && !isAnyZero(BaseMask) && - (BaseMaskEltSizeInBits % V1.getScalarValueSizeInBits()) == 0) { - return DAG.getBitcast(RootVT, V1); - } - - // Attempt to match a subvector broadcast. - // shuffle(insert_subvector(undef, sub, 0), undef, 0, 0, 0, 0) - if (UnaryShuffle && - (BaseMaskEltSizeInBits == 128 || BaseMaskEltSizeInBits == 256)) { - SmallVector<int, 64> BroadcastMask(NumBaseMaskElts, 0); - if (isTargetShuffleEquivalent(BaseMask, BroadcastMask)) { - SDValue Src = Inputs[0]; - if (Src.getOpcode() == ISD::INSERT_SUBVECTOR && - Src.getOperand(0).isUndef() && - Src.getOperand(1).getValueSizeInBits() == BaseMaskEltSizeInBits && - MayFoldLoad(Src.getOperand(1)) && isNullConstant(Src.getOperand(2))) { - return DAG.getBitcast(RootVT, DAG.getNode(X86ISD::SUBV_BROADCAST, DL, - Src.getValueType(), - Src.getOperand(1))); - } - } + (BaseMaskEltSizeInBits % V1.getScalarValueSizeInBits()) == 0 && + V1.getValueSizeInBits() >= RootSizeInBits) { + return CanonicalizeShuffleInput(RootVT, V1); } // Handle 128/256-bit lane shuffles of 512-bit vectors. if (RootVT.is512BitVector() && (NumBaseMaskElts == 2 || NumBaseMaskElts == 4)) { - MVT ShuffleVT = (FloatDomain ? MVT::v8f64 : MVT::v8i64); - // If the upper subvectors are zeroable, then an extract+insert is more // optimal than using X86ISD::SHUF128. The insertion is free, even if it has // to zero the upper subvectors. @@ -34520,12 +35360,11 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, return SDValue(); // Nothing to do! assert(isInRange(BaseMask[0], 0, NumBaseMaskElts) && "Unexpected lane shuffle"); - Res = DAG.getBitcast(ShuffleVT, V1); - unsigned SubIdx = BaseMask[0] * (8 / NumBaseMaskElts); + Res = CanonicalizeShuffleInput(RootVT, V1); + unsigned SubIdx = BaseMask[0] * (NumRootElts / NumBaseMaskElts); bool UseZero = isAnyZero(BaseMask); Res = extractSubVector(Res, SubIdx, DAG, DL, BaseMaskEltSizeInBits); - Res = widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits); - return DAG.getBitcast(RootVT, Res); + return widenSubVector(Res, UseZero, Subtarget, DAG, DL, RootSizeInBits); } // Narrow shuffle mask to v4x128. @@ -34534,8 +35373,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, narrowShuffleMaskElts(BaseMaskEltSizeInBits / 128, BaseMask, Mask); // Try to lower to vshuf64x2/vshuf32x4. - auto MatchSHUF128 = [](MVT ShuffleVT, const SDLoc &DL, ArrayRef<int> Mask, - SDValue V1, SDValue V2, SelectionDAG &DAG) { + auto MatchSHUF128 = [&](MVT ShuffleVT, const SDLoc &DL, ArrayRef<int> Mask, + SDValue V1, SDValue V2, SelectionDAG &DAG) { unsigned PermMask = 0; // Insure elements came from the same Op. SDValue Ops[2] = {DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT)}; @@ -34558,8 +35397,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, } return DAG.getNode(X86ISD::SHUF128, DL, ShuffleVT, - DAG.getBitcast(ShuffleVT, Ops[0]), - DAG.getBitcast(ShuffleVT, Ops[1]), + CanonicalizeShuffleInput(ShuffleVT, Ops[0]), + CanonicalizeShuffleInput(ShuffleVT, Ops[1]), DAG.getTargetConstant(PermMask, DL, MVT::i8)); }; @@ -34574,6 +35413,9 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, (Mask[1] < 0 || Mask[3] < 0 || Mask[1] == (Mask[3] % 2)); if (!isAnyZero(Mask) && !PreferPERMQ) { + if (Depth == 0 && Root.getOpcode() == X86ISD::SHUF128) + return SDValue(); // Nothing to do! + MVT ShuffleVT = (FloatDomain ? MVT::v8f64 : MVT::v8i64); if (SDValue V = MatchSHUF128(ShuffleVT, DL, Mask, V1, V2, DAG)) return DAG.getBitcast(RootVT, V); } @@ -34581,8 +35423,6 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, // Handle 128-bit lane shuffles of 256-bit vectors. if (RootVT.is256BitVector() && NumBaseMaskElts == 2) { - MVT ShuffleVT = (FloatDomain ? MVT::v4f64 : MVT::v4i64); - // If the upper half is zeroable, then an extract+insert is more optimal // than using X86ISD::VPERM2X128. The insertion is free, even if it has to // zero the upper half. @@ -34590,11 +35430,10 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, if (Depth == 0 && Root.getOpcode() == ISD::INSERT_SUBVECTOR) return SDValue(); // Nothing to do! assert(isInRange(BaseMask[0], 0, 2) && "Unexpected lane shuffle"); - Res = DAG.getBitcast(ShuffleVT, V1); - Res = extract128BitVector(Res, BaseMask[0] * 2, DAG, DL); - Res = widenSubVector(Res, BaseMask[1] == SM_SentinelZero, Subtarget, DAG, - DL, 256); - return DAG.getBitcast(RootVT, Res); + Res = CanonicalizeShuffleInput(RootVT, V1); + Res = extract128BitVector(Res, BaseMask[0] * (NumRootElts / 2), DAG, DL); + return widenSubVector(Res, BaseMask[1] == SM_SentinelZero, Subtarget, DAG, + DL, 256); } if (Depth == 0 && Root.getOpcode() == X86ISD::VPERM2X128) @@ -34609,12 +35448,9 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, unsigned PermMask = 0; PermMask |= ((BaseMask[0] < 0 ? 0x8 : (BaseMask[0] & 1)) << 0); PermMask |= ((BaseMask[1] < 0 ? 0x8 : (BaseMask[1] & 1)) << 4); - - Res = DAG.getBitcast(ShuffleVT, V1); - Res = DAG.getNode(X86ISD::VPERM2X128, DL, ShuffleVT, Res, - DAG.getUNDEF(ShuffleVT), - DAG.getTargetConstant(PermMask, DL, MVT::i8)); - return DAG.getBitcast(RootVT, Res); + return DAG.getNode( + X86ISD::VPERM2X128, DL, RootVT, CanonicalizeShuffleInput(RootVT, V1), + DAG.getUNDEF(RootVT), DAG.getTargetConstant(PermMask, DL, MVT::i8)); } if (Depth == 0 && Root.getOpcode() == X86ISD::SHUF128) @@ -34630,13 +35466,12 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, unsigned PermMask = 0; PermMask |= ((BaseMask[0] & 3) << 0); PermMask |= ((BaseMask[1] & 3) << 4); - - Res = DAG.getNode( - X86ISD::VPERM2X128, DL, ShuffleVT, - DAG.getBitcast(ShuffleVT, isInRange(BaseMask[0], 0, 2) ? V1 : V2), - DAG.getBitcast(ShuffleVT, isInRange(BaseMask[1], 0, 2) ? V1 : V2), - DAG.getTargetConstant(PermMask, DL, MVT::i8)); - return DAG.getBitcast(RootVT, Res); + SDValue LHS = isInRange(BaseMask[0], 0, 2) ? V1 : V2; + SDValue RHS = isInRange(BaseMask[1], 0, 2) ? V1 : V2; + return DAG.getNode(X86ISD::VPERM2X128, DL, RootVT, + CanonicalizeShuffleInput(RootVT, LHS), + CanonicalizeShuffleInput(RootVT, RHS), + DAG.getTargetConstant(PermMask, DL, MVT::i8)); } } } @@ -34698,8 +35533,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, if ((Subtarget.hasAVX2() || (Subtarget.hasAVX() && 32 <= MaskEltSizeInBits)) && (!IsMaskedShuffle || NumRootElts == NumMaskElts)) { - SmallVector<int, 64> BroadcastMask(NumMaskElts, 0); - if (isTargetShuffleEquivalent(Mask, BroadcastMask)) { + if (isUndefOrEqual(Mask, 0)) { if (V1.getValueType() == MaskVT && V1.getOpcode() == ISD::SCALAR_TO_VECTOR && MayFoldLoad(V1.getOperand(0))) { @@ -34712,7 +35546,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, if (Subtarget.hasAVX2()) { if (Depth == 0 && Root.getOpcode() == X86ISD::VBROADCAST) return SDValue(); // Nothing to do! - Res = DAG.getBitcast(MaskVT, V1); + Res = CanonicalizeShuffleInput(MaskVT, V1); Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res); return DAG.getBitcast(RootVT, Res); } @@ -34727,7 +35561,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, (NumRootElts == ShuffleVT.getVectorNumElements()))) { if (Depth == 0 && Root.getOpcode() == Shuffle) return SDValue(); // Nothing to do! - Res = DAG.getBitcast(ShuffleSrcVT, NewV1); + Res = CanonicalizeShuffleInput(ShuffleSrcVT, NewV1); Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res); return DAG.getBitcast(RootVT, Res); } @@ -34739,7 +35573,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, (NumRootElts == ShuffleVT.getVectorNumElements()))) { if (Depth == 0 && Root.getOpcode() == Shuffle) return SDValue(); // Nothing to do! - Res = DAG.getBitcast(ShuffleVT, V1); + Res = CanonicalizeShuffleInput(ShuffleVT, V1); Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res, DAG.getTargetConstant(PermuteImm, DL, MVT::i8)); return DAG.getBitcast(RootVT, Res); @@ -34750,16 +35584,32 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, // from a scalar. // TODO: Handle other insertions here as well? if (!UnaryShuffle && AllowFloatDomain && RootSizeInBits == 128 && - MaskEltSizeInBits == 32 && Subtarget.hasSSE41() && - !isTargetShuffleEquivalent(Mask, {4, 1, 2, 3})) { - SDValue SrcV1 = V1, SrcV2 = V2; - if (matchShuffleAsInsertPS(SrcV1, SrcV2, PermuteImm, Zeroable, Mask, DAG) && - SrcV2.getOpcode() == ISD::SCALAR_TO_VECTOR) { + Subtarget.hasSSE41() && + !isTargetShuffleEquivalent(MaskVT, Mask, {4, 1, 2, 3})) { + if (MaskEltSizeInBits == 32) { + SDValue SrcV1 = V1, SrcV2 = V2; + if (matchShuffleAsInsertPS(SrcV1, SrcV2, PermuteImm, Zeroable, Mask, + DAG) && + SrcV2.getOpcode() == ISD::SCALAR_TO_VECTOR) { + if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTPS) + return SDValue(); // Nothing to do! + Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, + CanonicalizeShuffleInput(MVT::v4f32, SrcV1), + CanonicalizeShuffleInput(MVT::v4f32, SrcV2), + DAG.getTargetConstant(PermuteImm, DL, MVT::i8)); + return DAG.getBitcast(RootVT, Res); + } + } + if (MaskEltSizeInBits == 64 && + isTargetShuffleEquivalent(MaskVT, Mask, {0, 2}) && + V2.getOpcode() == ISD::SCALAR_TO_VECTOR && + V2.getScalarValueSizeInBits() <= 32) { if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTPS) return SDValue(); // Nothing to do! + PermuteImm = (/*DstIdx*/2 << 4) | (/*SrcIdx*/0 << 0); Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, - DAG.getBitcast(MVT::v4f32, SrcV1), - DAG.getBitcast(MVT::v4f32, SrcV2), + CanonicalizeShuffleInput(MVT::v4f32, V1), + CanonicalizeShuffleInput(MVT::v4f32, V2), DAG.getTargetConstant(PermuteImm, DL, MVT::i8)); return DAG.getBitcast(RootVT, Res); } @@ -34773,8 +35623,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, (!IsMaskedShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) { if (Depth == 0 && Root.getOpcode() == Shuffle) return SDValue(); // Nothing to do! - NewV1 = DAG.getBitcast(ShuffleSrcVT, NewV1); - NewV2 = DAG.getBitcast(ShuffleSrcVT, NewV2); + NewV1 = CanonicalizeShuffleInput(ShuffleSrcVT, NewV1); + NewV2 = CanonicalizeShuffleInput(ShuffleSrcVT, NewV2); Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2); return DAG.getBitcast(RootVT, Res); } @@ -34787,8 +35637,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, (!IsMaskedShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) { if (Depth == 0 && Root.getOpcode() == Shuffle) return SDValue(); // Nothing to do! - NewV1 = DAG.getBitcast(ShuffleVT, NewV1); - NewV2 = DAG.getBitcast(ShuffleVT, NewV2); + NewV1 = CanonicalizeShuffleInput(ShuffleVT, NewV1); + NewV2 = CanonicalizeShuffleInput(ShuffleVT, NewV2); Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2, DAG.getTargetConstant(PermuteImm, DL, MVT::i8)); return DAG.getBitcast(RootVT, Res); @@ -34805,7 +35655,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, Zeroable)) { if (Depth == 0 && Root.getOpcode() == X86ISD::EXTRQI) return SDValue(); // Nothing to do! - V1 = DAG.getBitcast(IntMaskVT, V1); + V1 = CanonicalizeShuffleInput(IntMaskVT, V1); Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1, DAG.getTargetConstant(BitLen, DL, MVT::i8), DAG.getTargetConstant(BitIdx, DL, MVT::i8)); @@ -34815,8 +35665,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, if (matchShuffleAsINSERTQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx)) { if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTQI) return SDValue(); // Nothing to do! - V1 = DAG.getBitcast(IntMaskVT, V1); - V2 = DAG.getBitcast(IntMaskVT, V2); + V1 = CanonicalizeShuffleInput(IntMaskVT, V1); + V2 = CanonicalizeShuffleInput(IntMaskVT, V2); Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2, DAG.getTargetConstant(BitLen, DL, MVT::i8), DAG.getTargetConstant(BitIdx, DL, MVT::i8)); @@ -34835,7 +35685,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, IsTRUNCATE ? (unsigned)ISD::TRUNCATE : (unsigned)X86ISD::VTRUNC; if (Depth == 0 && Root.getOpcode() == Opc) return SDValue(); // Nothing to do! - V1 = DAG.getBitcast(ShuffleSrcVT, V1); + V1 = CanonicalizeShuffleInput(ShuffleSrcVT, V1); Res = DAG.getNode(Opc, DL, ShuffleVT, V1); if (ShuffleVT.getSizeInBits() < RootSizeInBits) Res = widenSubVector(Res, true, Subtarget, DAG, DL, RootSizeInBits); @@ -34852,8 +35702,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, return SDValue(); // Nothing to do! ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2); ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts / 2); - V1 = DAG.getBitcast(ShuffleSrcVT, V1); - V2 = DAG.getBitcast(ShuffleSrcVT, V2); + V1 = CanonicalizeShuffleInput(ShuffleSrcVT, V1); + V2 = CanonicalizeShuffleInput(ShuffleSrcVT, V2); ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2); ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts); Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShuffleSrcVT, V1, V2); @@ -34870,49 +35720,56 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, // Depth threshold above which we can efficiently use variable mask shuffles. int VariableShuffleDepth = Subtarget.hasFastVariableShuffle() ? 1 : 2; AllowVariableMask &= (Depth >= VariableShuffleDepth) || HasVariableMask; + // VPERMI2W/VPERMI2B are 3 uops on Skylake and Icelake so we require a + // higher depth before combining them. + bool AllowBWIVPERMV3 = (Depth >= 2 || HasVariableMask); bool MaskContainsZeros = isAnyZero(Mask); if (is128BitLaneCrossingShuffleMask(MaskVT, Mask)) { // If we have a single input lane-crossing shuffle then lower to VPERMV. - if (UnaryShuffle && AllowVariableMask && !MaskContainsZeros && - ((Subtarget.hasAVX2() && - (MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) || - (Subtarget.hasAVX512() && - (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || - MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || - (Subtarget.hasBWI() && MaskVT == MVT::v32i16) || - (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || - (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || - (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { - SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true); - Res = DAG.getBitcast(MaskVT, V1); - Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res); - return DAG.getBitcast(RootVT, Res); + if (UnaryShuffle && AllowVariableMask && !MaskContainsZeros) { + if (Subtarget.hasAVX2() && + (MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) { + SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true); + Res = CanonicalizeShuffleInput(MaskVT, V1); + Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res); + return DAG.getBitcast(RootVT, Res); + } + // AVX512 variants (non-VLX will pad to 512-bit shuffles). + if ((Subtarget.hasAVX512() && + (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || + MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || + (Subtarget.hasBWI() && + (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) || + (Subtarget.hasVBMI() && + (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8))) { + V1 = CanonicalizeShuffleInput(MaskVT, V1); + V2 = DAG.getUNDEF(MaskVT); + Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); + return DAG.getBitcast(RootVT, Res); + } } // Lower a unary+zero lane-crossing shuffle as VPERMV3 with a zero - // vector as the second source. + // vector as the second source (non-VLX will pad to 512-bit shuffles). if (UnaryShuffle && AllowVariableMask && ((Subtarget.hasAVX512() && (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || + MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 || + MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32 || MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || - (Subtarget.hasVLX() && - (MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 || - MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) || - (Subtarget.hasBWI() && MaskVT == MVT::v32i16) || - (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || - (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || - (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { + (Subtarget.hasBWI() && AllowBWIVPERMV3 && + (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) || + (Subtarget.hasVBMI() && AllowBWIVPERMV3 && + (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) { // Adjust shuffle mask - replace SM_SentinelZero with second source index. for (unsigned i = 0; i != NumMaskElts; ++i) if (Mask[i] == SM_SentinelZero) Mask[i] = NumMaskElts + i; - - SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true); - Res = DAG.getBitcast(MaskVT, V1); - SDValue Zero = getZeroVector(MaskVT, Subtarget, DAG, DL); - Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, Res, VPermMask, Zero); + V1 = CanonicalizeShuffleInput(MaskVT, V1); + V2 = getZeroVector(MaskVT, Subtarget, DAG, DL); + Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); return DAG.getBitcast(RootVT, Res); } @@ -34923,22 +35780,21 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, DAG, Subtarget)) return WideShuffle; - // If we have a dual input lane-crossing shuffle then lower to VPERMV3. + // If we have a dual input lane-crossing shuffle then lower to VPERMV3, + // (non-VLX will pad to 512-bit shuffles). if (AllowVariableMask && !MaskContainsZeros && ((Subtarget.hasAVX512() && (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || - MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || - (Subtarget.hasVLX() && - (MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 || + MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 || + MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32 || MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) || - (Subtarget.hasBWI() && MaskVT == MVT::v32i16) || - (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || - (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || - (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { - SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true); - V1 = DAG.getBitcast(MaskVT, V1); - V2 = DAG.getBitcast(MaskVT, V2); - Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, V1, VPermMask, V2); + (Subtarget.hasBWI() && AllowBWIVPERMV3 && + (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) || + (Subtarget.hasVBMI() && AllowBWIVPERMV3 && + (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) { + V1 = CanonicalizeShuffleInput(MaskVT, V1); + V2 = CanonicalizeShuffleInput(MaskVT, V2); + Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); return DAG.getBitcast(RootVT, Res); } return SDValue(); @@ -34964,7 +35820,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, EltBits[i] = AllOnes; } SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL); - Res = DAG.getBitcast(MaskVT, V1); + Res = CanonicalizeShuffleInput(MaskVT, V1); unsigned AndOpcode = MaskVT.isFloatingPoint() ? unsigned(X86ISD::FAND) : unsigned(ISD::AND); Res = DAG.getNode(AndOpcode, DL, MaskVT, Res, BitMask); @@ -34984,7 +35840,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, VPermIdx.push_back(Idx); } SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx); - Res = DAG.getBitcast(MaskVT, V1); + Res = CanonicalizeShuffleInput(MaskVT, V1); Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask); return DAG.getBitcast(RootVT, Res); } @@ -35016,8 +35872,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, Index = (MaskVT.getScalarSizeInBits() == 64 ? Index << 1 : Index); VPerm2Idx.push_back(Index); } - V1 = DAG.getBitcast(MaskVT, V1); - V2 = DAG.getBitcast(MaskVT, V2); + V1 = CanonicalizeShuffleInput(MaskVT, V1); + V2 = CanonicalizeShuffleInput(MaskVT, V2); SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true); Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp, DAG.getTargetConstant(M2ZImm, DL, MVT::i8)); @@ -35051,7 +35907,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8)); } MVT ByteVT = MVT::getVectorVT(MVT::i8, NumBytes); - Res = DAG.getBitcast(ByteVT, V1); + Res = CanonicalizeShuffleInput(ByteVT, V1); SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask); Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp); return DAG.getBitcast(RootVT, Res); @@ -35081,8 +35937,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, VPPERMMask.push_back(DAG.getConstant(M, DL, MVT::i8)); } MVT ByteVT = MVT::v16i8; - V1 = DAG.getBitcast(ByteVT, V1); - V2 = DAG.getBitcast(ByteVT, V2); + V1 = CanonicalizeShuffleInput(ByteVT, V1); + V2 = CanonicalizeShuffleInput(ByteVT, V2); SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask); Res = DAG.getNode(X86ISD::VPPERM, DL, ByteVT, V1, V2, VPPERMMaskOp); return DAG.getBitcast(RootVT, Res); @@ -35095,25 +35951,22 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root, DAG, Subtarget)) return WideShuffle; - // If we have a dual input shuffle then lower to VPERMV3. + // If we have a dual input shuffle then lower to VPERMV3, + // (non-VLX will pad to 512-bit shuffles) if (!UnaryShuffle && AllowVariableMask && !MaskContainsZeros && ((Subtarget.hasAVX512() && - (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || - MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || - (Subtarget.hasVLX() && - (MaskVT == MVT::v2f64 || MaskVT == MVT::v2i64 || MaskVT == MVT::v4f64 || - MaskVT == MVT::v4i64 || MaskVT == MVT::v4f32 || MaskVT == MVT::v4i32 || - MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) || - (Subtarget.hasBWI() && MaskVT == MVT::v32i16) || - (Subtarget.hasBWI() && Subtarget.hasVLX() && - (MaskVT == MVT::v8i16 || MaskVT == MVT::v16i16)) || - (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || - (Subtarget.hasVBMI() && Subtarget.hasVLX() && - (MaskVT == MVT::v16i8 || MaskVT == MVT::v32i8)))) { - SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true); - V1 = DAG.getBitcast(MaskVT, V1); - V2 = DAG.getBitcast(MaskVT, V2); - Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, V1, VPermMask, V2); + (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v8f64 || + MaskVT == MVT::v2i64 || MaskVT == MVT::v4i64 || MaskVT == MVT::v8i64 || + MaskVT == MVT::v4f32 || MaskVT == MVT::v4i32 || MaskVT == MVT::v8f32 || + MaskVT == MVT::v8i32 || MaskVT == MVT::v16f32 || + MaskVT == MVT::v16i32)) || + (Subtarget.hasBWI() && AllowBWIVPERMV3 && + (MaskVT == MVT::v8i16 || MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) || + (Subtarget.hasVBMI() && AllowBWIVPERMV3 && + (MaskVT == MVT::v16i8 || MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) { + V1 = CanonicalizeShuffleInput(MaskVT, V1); + V2 = CanonicalizeShuffleInput(MaskVT, V2); + Res = lowerShuffleWithPERMV(DL, MaskVT, Mask, V1, V2, Subtarget, DAG); return DAG.getBitcast(RootVT, Res); } @@ -35138,12 +35991,16 @@ static SDValue combineX86ShuffleChainWithExtract( if (NumInputs == 0) return SDValue(); + EVT RootVT = Root.getValueType(); + unsigned RootSizeInBits = RootVT.getSizeInBits(); + assert((RootSizeInBits % NumMaskElts) == 0 && "Unexpected root shuffle mask"); + SmallVector<SDValue, 4> WideInputs(Inputs.begin(), Inputs.end()); SmallVector<unsigned, 4> Offsets(NumInputs, 0); // Peek through subvectors. // TODO: Support inter-mixed EXTRACT_SUBVECTORs + BITCASTs? - unsigned WideSizeInBits = WideInputs[0].getValueSizeInBits(); + unsigned WideSizeInBits = RootSizeInBits; for (unsigned i = 0; i != NumInputs; ++i) { SDValue &Src = WideInputs[i]; unsigned &Offset = Offsets[i]; @@ -35166,8 +36023,6 @@ static SDValue combineX86ShuffleChainWithExtract( if (llvm::all_of(Offsets, [](unsigned Offset) { return Offset == 0; })) return SDValue(); - EVT RootVT = Root.getValueType(); - unsigned RootSizeInBits = RootVT.getSizeInBits(); unsigned Scale = WideSizeInBits / RootSizeInBits; assert((WideSizeInBits % RootSizeInBits) == 0 && "Unexpected subvector extraction"); @@ -35227,6 +36082,149 @@ static SDValue combineX86ShuffleChainWithExtract( return SDValue(); } +// Canonicalize the combined shuffle mask chain with horizontal ops. +// NOTE: This may update the Ops and Mask. +static SDValue canonicalizeShuffleMaskWithHorizOp( + MutableArrayRef<SDValue> Ops, MutableArrayRef<int> Mask, + unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG, + const X86Subtarget &Subtarget) { + if (Mask.empty() || Ops.empty()) + return SDValue(); + + SmallVector<SDValue> BC; + for (SDValue Op : Ops) + BC.push_back(peekThroughBitcasts(Op)); + + // All ops must be the same horizop + type. + SDValue BC0 = BC[0]; + EVT VT0 = BC0.getValueType(); + unsigned Opcode0 = BC0.getOpcode(); + if (VT0.getSizeInBits() != RootSizeInBits || llvm::any_of(BC, [&](SDValue V) { + return V.getOpcode() != Opcode0 || V.getValueType() != VT0; + })) + return SDValue(); + + bool isHoriz = (Opcode0 == X86ISD::FHADD || Opcode0 == X86ISD::HADD || + Opcode0 == X86ISD::FHSUB || Opcode0 == X86ISD::HSUB); + bool isPack = (Opcode0 == X86ISD::PACKSS || Opcode0 == X86ISD::PACKUS); + if (!isHoriz && !isPack) + return SDValue(); + + int NumElts = VT0.getVectorNumElements(); + int NumLanes = VT0.getSizeInBits() / 128; + int NumEltsPerLane = NumElts / NumLanes; + int NumHalfEltsPerLane = NumEltsPerLane / 2; + + // See if we can remove the shuffle by resorting the HOP chain so that + // the HOP args are pre-shuffled. + // TODO: Generalize to any sized/depth chain. + // TODO: Add support for PACKSS/PACKUS. + if (isHoriz && NumEltsPerLane == 4 && VT0.is128BitVector() && + shouldUseHorizontalOp(Ops.size() == 1, DAG, Subtarget)) { + SmallVector<int> ScaledMask; + if (scaleShuffleElements(Mask, 4, ScaledMask)) { + // Attempt to find a HOP(HOP(X,Y),HOP(Z,W)) source operand. + auto GetHOpSrc = [&](int M) { + if (M == SM_SentinelUndef) + return DAG.getUNDEF(VT0); + if (M == SM_SentinelZero) + return getZeroVector(VT0.getSimpleVT(), Subtarget, DAG, DL); + SDValue Src0 = BC[M / NumElts]; + SDValue Src1 = Src0.getOperand((M % 4) >= 2); + if (Src1.getOpcode() == Opcode0 && Src0->isOnlyUserOf(Src1.getNode())) + return Src1.getOperand(M % 2); + return SDValue(); + }; + SDValue M0 = GetHOpSrc(ScaledMask[0]); + SDValue M1 = GetHOpSrc(ScaledMask[1]); + SDValue M2 = GetHOpSrc(ScaledMask[2]); + SDValue M3 = GetHOpSrc(ScaledMask[3]); + if (M0 && M1 && M2 && M3) { + SDValue LHS = DAG.getNode(Opcode0, DL, VT0, M0, M1); + SDValue RHS = DAG.getNode(Opcode0, DL, VT0, M2, M3); + return DAG.getNode(Opcode0, DL, VT0, LHS, RHS); + } + } + } + + if (2 < Ops.size()) + return SDValue(); + + SDValue BC1 = BC[BC.size() - 1]; + if (Mask.size() == VT0.getVectorNumElements()) { + // Canonicalize binary shuffles of horizontal ops that use the + // same sources to an unary shuffle. + // TODO: Try to perform this fold even if the shuffle remains. + if (Ops.size() == 2) { + auto ContainsOps = [](SDValue HOp, SDValue Op) { + return Op == HOp.getOperand(0) || Op == HOp.getOperand(1); + }; + // Commute if all BC0's ops are contained in BC1. + if (ContainsOps(BC1, BC0.getOperand(0)) && + ContainsOps(BC1, BC0.getOperand(1))) { + ShuffleVectorSDNode::commuteMask(Mask); + std::swap(Ops[0], Ops[1]); + std::swap(BC0, BC1); + } + + // If BC1 can be represented by BC0, then convert to unary shuffle. + if (ContainsOps(BC0, BC1.getOperand(0)) && + ContainsOps(BC0, BC1.getOperand(1))) { + for (int &M : Mask) { + if (M < NumElts) // BC0 element or UNDEF/Zero sentinel. + continue; + int SubLane = ((M % NumEltsPerLane) >= NumHalfEltsPerLane) ? 1 : 0; + M -= NumElts + (SubLane * NumHalfEltsPerLane); + if (BC1.getOperand(SubLane) != BC0.getOperand(0)) + M += NumHalfEltsPerLane; + } + } + } + + // Canonicalize unary horizontal ops to only refer to lower halves. + for (int i = 0; i != NumElts; ++i) { + int &M = Mask[i]; + if (isUndefOrZero(M)) + continue; + if (M < NumElts && BC0.getOperand(0) == BC0.getOperand(1) && + (M % NumEltsPerLane) >= NumHalfEltsPerLane) + M -= NumHalfEltsPerLane; + if (NumElts <= M && BC1.getOperand(0) == BC1.getOperand(1) && + (M % NumEltsPerLane) >= NumHalfEltsPerLane) + M -= NumHalfEltsPerLane; + } + } + + // Combine binary shuffle of 2 similar 'Horizontal' instructions into a + // single instruction. Attempt to match a v2X64 repeating shuffle pattern that + // represents the LHS/RHS inputs for the lower/upper halves. + unsigned EltSizeInBits = RootSizeInBits / Mask.size(); + SmallVector<int, 16> TargetMask128, WideMask128; + if (isRepeatedTargetShuffleMask(128, EltSizeInBits, Mask, TargetMask128) && + scaleShuffleElements(TargetMask128, 2, WideMask128)) { + assert(isUndefOrZeroOrInRange(WideMask128, 0, 4) && "Illegal shuffle"); + bool SingleOp = (Ops.size() == 1); + if (!isHoriz || shouldUseHorizontalOp(SingleOp, DAG, Subtarget)) { + SDValue Lo = isInRange(WideMask128[0], 0, 2) ? BC0 : BC1; + SDValue Hi = isInRange(WideMask128[1], 0, 2) ? BC0 : BC1; + Lo = Lo.getOperand(WideMask128[0] & 1); + Hi = Hi.getOperand(WideMask128[1] & 1); + if (SingleOp) { + MVT SrcVT = BC0.getOperand(0).getSimpleValueType(); + SDValue Undef = DAG.getUNDEF(SrcVT); + SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL); + Lo = (WideMask128[0] == SM_SentinelZero ? Zero : Lo); + Hi = (WideMask128[1] == SM_SentinelZero ? Zero : Hi); + Lo = (WideMask128[0] == SM_SentinelUndef ? Undef : Lo); + Hi = (WideMask128[1] == SM_SentinelUndef ? Undef : Hi); + } + return DAG.getNode(Opcode0, DL, VT0, Lo, Hi); + } + } + + return SDValue(); +} + // Attempt to constant fold all of the constant source ops. // Returns true if the entire shuffle is folded to a constant. // TODO: Extend this to merge multiple constant Ops and update the mask. @@ -35318,6 +36316,14 @@ static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops, return DAG.getBitcast(VT, CstOp); } +namespace llvm { + namespace X86 { + enum { + MaxShuffleCombineDepth = 8 + }; + } +} // namespace llvm + /// Fully generic combining of x86 shuffle instructions. /// /// This should be the last combine run over the x86 shuffle instructions. Once @@ -35350,31 +36356,30 @@ static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops, static SDValue combineX86ShufflesRecursively( ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root, ArrayRef<int> RootMask, ArrayRef<const SDNode *> SrcNodes, unsigned Depth, - bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG, - const X86Subtarget &Subtarget) { + unsigned MaxDepth, bool HasVariableMask, bool AllowVariableMask, + SelectionDAG &DAG, const X86Subtarget &Subtarget) { assert(RootMask.size() > 0 && (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) && "Illegal shuffle root mask"); + assert(Root.getSimpleValueType().isVector() && + "Shuffles operate on vector types!"); + unsigned RootSizeInBits = Root.getSimpleValueType().getSizeInBits(); // Bound the depth of our recursive combine because this is ultimately // quadratic in nature. - const unsigned MaxRecursionDepth = 8; - if (Depth >= MaxRecursionDepth) + if (Depth >= MaxDepth) return SDValue(); // Directly rip through bitcasts to find the underlying operand. SDValue Op = SrcOps[SrcOpIndex]; Op = peekThroughOneUseBitcasts(Op); - MVT VT = Op.getSimpleValueType(); - if (!VT.isVector()) - return SDValue(); // Bail if we hit a non-vector. + EVT VT = Op.getValueType(); + if (!VT.isVector() || !VT.isSimple()) + return SDValue(); // Bail if we hit a non-simple non-vector. - assert(Root.getSimpleValueType().isVector() && - "Shuffles operate on vector types!"); - unsigned RootSizeInBits = Root.getSimpleValueType().getSizeInBits(); - assert(VT.getSizeInBits() == RootSizeInBits && - "Can only combine shuffles of the same vector register size."); + assert((RootSizeInBits % VT.getSizeInBits()) == 0 && + "Can only combine shuffles upto size of the root op."); // Extract target shuffle mask and resolve sentinels and inputs. // TODO - determine Op's demanded elts from RootMask. @@ -35387,17 +36392,32 @@ static SDValue combineX86ShufflesRecursively( OpZero, DAG, Depth, false)) return SDValue(); - // Shuffle inputs must be the same size as the result, bail on any larger - // inputs and widen any smaller inputs. - if (llvm::any_of(OpInputs, [RootSizeInBits](SDValue Op) { - return Op.getValueSizeInBits() > RootSizeInBits; + // Shuffle inputs must not be larger than the shuffle result. + // TODO: Relax this for single input faux shuffles (trunc/extract_subvector). + if (llvm::any_of(OpInputs, [VT](SDValue OpInput) { + return OpInput.getValueSizeInBits() > VT.getSizeInBits(); })) return SDValue(); - for (SDValue &Op : OpInputs) - if (Op.getValueSizeInBits() < RootSizeInBits) - Op = widenSubVector(peekThroughOneUseBitcasts(Op), false, Subtarget, DAG, - SDLoc(Op), RootSizeInBits); + // If the shuffle result was smaller than the root, we need to adjust the + // mask indices and pad the mask with undefs. + if (RootSizeInBits > VT.getSizeInBits()) { + unsigned NumSubVecs = RootSizeInBits / VT.getSizeInBits(); + unsigned OpMaskSize = OpMask.size(); + if (OpInputs.size() > 1) { + unsigned PaddedMaskSize = NumSubVecs * OpMaskSize; + for (int &M : OpMask) { + if (M < 0) + continue; + int EltIdx = M % OpMaskSize; + int OpIdx = M / OpMaskSize; + M = (PaddedMaskSize * OpIdx) + EltIdx; + } + } + OpZero = OpZero.zext(NumSubVecs * OpMaskSize); + OpUndef = OpUndef.zext(NumSubVecs * OpMaskSize); + OpMask.append((NumSubVecs - 1) * OpMaskSize, SM_SentinelUndef); + } SmallVector<int, 64> Mask; SmallVector<SDValue, 16> Ops; @@ -35538,10 +36558,6 @@ static SDValue combineX86ShufflesRecursively( // Handle the all undef/zero cases early. if (all_of(Mask, [](int Idx) { return Idx == SM_SentinelUndef; })) return DAG.getUNDEF(Root.getValueType()); - - // TODO - should we handle the mixed zero/undef case as well? Just returning - // a zero mask will lose information on undef elements possibly reducing - // future combine possibilities. if (all_of(Mask, [](int Idx) { return Idx < 0; })) return getZeroVector(Root.getSimpleValueType(), Subtarget, DAG, SDLoc(Root)); @@ -35561,7 +36577,7 @@ static SDValue combineX86ShufflesRecursively( // shuffles to avoid constant pool bloat. // Don't recurse if we already have more source ops than we can combine in // the remaining recursion depth. - if (Ops.size() < (MaxRecursionDepth - Depth)) { + if (Ops.size() < (MaxDepth - Depth)) { for (int i = 0, e = Ops.size(); i < e; ++i) { // For empty roots, we need to resolve zeroable elements before combining // them with other shuffles. @@ -35573,7 +36589,7 @@ static SDValue combineX86ShufflesRecursively( SDNode::areOnlyUsersOf(CombinedNodes, Ops[i].getNode())) AllowVar = AllowVariableMask; if (SDValue Res = combineX86ShufflesRecursively( - Ops, i, Root, ResolvedMask, CombinedNodes, Depth + 1, + Ops, i, Root, ResolvedMask, CombinedNodes, Depth + 1, MaxDepth, HasVariableMask, AllowVar, DAG, Subtarget)) return Res; } @@ -35584,6 +36600,24 @@ static SDValue combineX86ShufflesRecursively( Ops, Mask, Root, HasVariableMask, DAG, Subtarget)) return Cst; + // Canonicalize the combined shuffle mask chain with horizontal ops. + // NOTE: This will update the Ops and Mask. + if (SDValue HOp = canonicalizeShuffleMaskWithHorizOp( + Ops, Mask, RootSizeInBits, SDLoc(Root), DAG, Subtarget)) + return DAG.getBitcast(Root.getValueType(), HOp); + + // Widen any subvector shuffle inputs we've collected. + if (any_of(Ops, [RootSizeInBits](SDValue Op) { + return Op.getValueSizeInBits() < RootSizeInBits; + })) { + for (SDValue &Op : Ops) + if (Op.getValueSizeInBits() < RootSizeInBits) + Op = widenSubVector(Op, false, Subtarget, DAG, SDLoc(Op), + RootSizeInBits); + // Reresolve - we might have repeated subvector sources. + resolveTargetShuffleInputsAndMask(Ops, Mask); + } + // We can only combine unary and binary shuffle mask cases. if (Ops.size() <= 2) { // Minor canonicalization of the accumulated shuffle mask to make it easier @@ -35591,8 +36625,10 @@ static SDValue combineX86ShufflesRecursively( // elements, and shrink them to the half-width mask. It does this in a loop // so it will reduce the size of the mask to the minimal width mask which // performs an equivalent shuffle. - SmallVector<int, 64> WidenedMask; - while (Mask.size() > 1 && canWidenShuffleElements(Mask, WidenedMask)) { + while (Mask.size() > 1) { + SmallVector<int, 64> WidenedMask; + if (!canWidenShuffleElements(Mask, WidenedMask)) + break; Mask = std::move(WidenedMask); } @@ -35619,6 +36655,7 @@ static SDValue combineX86ShufflesRecursively( static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) { return combineX86ShufflesRecursively({Op}, 0, Op, {0}, {}, /*Depth*/ 0, + X86::MaxShuffleCombineDepth, /*HasVarMask*/ false, /*AllowVarMask*/ true, DAG, Subtarget); } @@ -35852,6 +36889,54 @@ static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL, return SDValue(); } +/// Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()). +static SDValue canonicalizeLaneShuffleWithRepeatedOps(SDValue V, + SelectionDAG &DAG, + const SDLoc &DL) { + assert(V.getOpcode() == X86ISD::VPERM2X128 && "Unknown lane shuffle"); + + MVT VT = V.getSimpleValueType(); + SDValue Src0 = peekThroughBitcasts(V.getOperand(0)); + SDValue Src1 = peekThroughBitcasts(V.getOperand(1)); + unsigned SrcOpc0 = Src0.getOpcode(); + unsigned SrcOpc1 = Src1.getOpcode(); + EVT SrcVT0 = Src0.getValueType(); + EVT SrcVT1 = Src1.getValueType(); + + if (!Src1.isUndef() && (SrcVT0 != SrcVT1 || SrcOpc0 != SrcOpc1)) + return SDValue(); + + switch (SrcOpc0) { + case X86ISD::MOVDDUP: { + SDValue LHS = DAG.getBitcast(VT, Src0.getOperand(0)); + SDValue RHS = + DAG.getBitcast(VT, Src1.isUndef() ? Src1 : Src1.getOperand(0)); + SDValue Res = + DAG.getNode(X86ISD::VPERM2X128, DL, VT, LHS, RHS, V.getOperand(2)); + Res = DAG.getNode(SrcOpc0, DL, SrcVT0, DAG.getBitcast(SrcVT0, Res)); + return DAG.getBitcast(VT, Res); + } + case X86ISD::VSHLI: + case X86ISD::VSRLI: + case X86ISD::VSRAI: + case X86ISD::PSHUFD: + case X86ISD::VPERMILPI: + if (Src1.isUndef() || Src0.getOperand(1) == Src1.getOperand(1)) { + SDValue LHS = DAG.getBitcast(VT, Src0.getOperand(0)); + SDValue RHS = + DAG.getBitcast(VT, Src1.isUndef() ? Src1 : Src1.getOperand(0)); + SDValue Res = + DAG.getNode(X86ISD::VPERM2X128, DL, VT, LHS, RHS, V.getOperand(2)); + Res = DAG.getNode(SrcOpc0, DL, SrcVT0, DAG.getBitcast(SrcVT0, Res), + Src0.getOperand(1)); + return DAG.getBitcast(VT, Res); + } + break; + } + + return SDValue(); +} + /// Try to combine x86 target specific shuffles. static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, @@ -35861,59 +36946,6 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG, SmallVector<int, 4> Mask; unsigned Opcode = N.getOpcode(); - bool IsUnary; - SmallVector<int, 64> TargetMask; - SmallVector<SDValue, 2> TargetOps; - if (isTargetShuffle(Opcode)) - getTargetShuffleMask(N.getNode(), VT, true, TargetOps, TargetMask, IsUnary); - - // Combine binary shuffle of 2 similar 'Horizontal' instructions into a - // single instruction. Attempt to match a v2X64 repeating shuffle pattern that - // represents the LHS/RHS inputs for the lower/upper halves. - SmallVector<int, 16> TargetMask128; - if (!TargetMask.empty() && 0 < TargetOps.size() && TargetOps.size() <= 2 && - isRepeatedTargetShuffleMask(128, VT, TargetMask, TargetMask128)) { - SmallVector<int, 16> WidenedMask128 = TargetMask128; - while (WidenedMask128.size() > 2) { - SmallVector<int, 16> WidenedMask; - if (!canWidenShuffleElements(WidenedMask128, WidenedMask)) - break; - WidenedMask128 = std::move(WidenedMask); - } - if (WidenedMask128.size() == 2) { - assert(isUndefOrZeroOrInRange(WidenedMask128, 0, 4) && "Illegal shuffle"); - SDValue BC0 = peekThroughBitcasts(TargetOps.front()); - SDValue BC1 = peekThroughBitcasts(TargetOps.back()); - EVT VT0 = BC0.getValueType(); - EVT VT1 = BC1.getValueType(); - unsigned Opcode0 = BC0.getOpcode(); - unsigned Opcode1 = BC1.getOpcode(); - bool isHoriz = (Opcode0 == X86ISD::FHADD || Opcode0 == X86ISD::HADD || - Opcode0 == X86ISD::FHSUB || Opcode0 == X86ISD::HSUB); - if (Opcode0 == Opcode1 && VT0 == VT1 && - (isHoriz || Opcode0 == X86ISD::PACKSS || Opcode0 == X86ISD::PACKUS)) { - bool SingleOp = (TargetOps.size() == 1); - if (!isHoriz || shouldUseHorizontalOp(SingleOp, DAG, Subtarget)) { - SDValue Lo = isInRange(WidenedMask128[0], 0, 2) ? BC0 : BC1; - SDValue Hi = isInRange(WidenedMask128[1], 0, 2) ? BC0 : BC1; - Lo = Lo.getOperand(WidenedMask128[0] & 1); - Hi = Hi.getOperand(WidenedMask128[1] & 1); - if (SingleOp) { - MVT SrcVT = BC0.getOperand(0).getSimpleValueType(); - SDValue Undef = DAG.getUNDEF(SrcVT); - SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL); - Lo = (WidenedMask128[0] == SM_SentinelZero ? Zero : Lo); - Hi = (WidenedMask128[1] == SM_SentinelZero ? Zero : Hi); - Lo = (WidenedMask128[0] == SM_SentinelUndef ? Undef : Lo); - Hi = (WidenedMask128[1] == SM_SentinelUndef ? Undef : Hi); - } - SDValue Horiz = DAG.getNode(Opcode0, DL, VT0, Lo, Hi); - return DAG.getBitcast(VT, Horiz); - } - } - } - } - if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG)) return R; @@ -35977,6 +37009,7 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG, DemandedMask[i] = i; if (SDValue Res = combineX86ShufflesRecursively( {BC}, 0, BC, DemandedMask, {}, /*Depth*/ 0, + X86::MaxShuffleCombineDepth, /*HasVarMask*/ false, /*AllowVarMask*/ true, DAG, Subtarget)) return DAG.getNode(X86ISD::VBROADCAST, DL, VT, DAG.getBitcast(SrcVT, Res)); @@ -36002,9 +37035,12 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG, return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0)); // Share broadcast with the longest vector and extract low subvector (free). + // Ensure the same SDValue from the SDNode use is being used. for (SDNode *User : Src->uses()) if (User != N.getNode() && User->getOpcode() == X86ISD::VBROADCAST && - User->getValueSizeInBits(0) > VT.getSizeInBits()) { + Src == User->getOperand(0) && + User->getValueSizeInBits(0).getFixedSize() > + VT.getFixedSizeInBits()) { return extractSubVector(SDValue(User, 0), 0, DAG, DL, VT.getSizeInBits()); } @@ -36090,7 +37126,8 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG, LN->isSimple()) { unsigned Offset = ShiftAmt / 8; SDVTList Tys = DAG.getVTList(VT, MVT::Other); - SDValue Ptr = DAG.getMemBasePlusOffset(LN->getBasePtr(), Offset, DL); + SDValue Ptr = DAG.getMemBasePlusOffset(LN->getBasePtr(), + TypeSize::Fixed(Offset), DL); SDValue Ops[] = { LN->getChain(), Ptr }; SDValue BcastLd = DAG.getMemIntrinsicNode( X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16, @@ -36122,15 +37159,16 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG, } // vbroadcast(vector load X) -> vbroadcast_load - if (SrcVT == MVT::v2f64 && Src.hasOneUse() && - ISD::isNormalLoad(Src.getNode())) { + if ((SrcVT == MVT::v2f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v2i64 || + SrcVT == MVT::v4i32) && + Src.hasOneUse() && ISD::isNormalLoad(Src.getNode())) { LoadSDNode *LN = cast<LoadSDNode>(Src); // Unless the load is volatile or atomic. if (LN->isSimple()) { SDVTList Tys = DAG.getVTList(VT, MVT::Other); - SDValue Ops[] = { LN->getChain(), LN->getBasePtr() }; + SDValue Ops[] = {LN->getChain(), LN->getBasePtr()}; SDValue BcastLd = DAG.getMemIntrinsicNode( - X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::f64, + X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, SrcVT.getScalarType(), LN->getPointerInfo(), LN->getOriginalAlign(), LN->getMemOperand()->getFlags()); DCI.CombineTo(N.getNode(), BcastLd); @@ -36217,6 +37255,27 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG, } } + // Pull subvector inserts into undef through VZEXT_MOVL by making it an + // insert into a zero vector. This helps get VZEXT_MOVL closer to + // scalar_to_vectors where 256/512 are canonicalized to an insert and a + // 128-bit scalar_to_vector. This reduces the number of isel patterns. + if (!DCI.isBeforeLegalizeOps() && N0.hasOneUse()) { + SDValue V = peekThroughOneUseBitcasts(N0); + + if (V.getOpcode() == ISD::INSERT_SUBVECTOR && V.getOperand(0).isUndef() && + isNullConstant(V.getOperand(2))) { + SDValue In = V.getOperand(1); + MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), + In.getValueSizeInBits() / + VT.getScalarSizeInBits()); + In = DAG.getBitcast(SubVT, In); + SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, DL, SubVT, In); + return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, + getZeroVector(VT, Subtarget, DAG, DL), Movl, + V.getOperand(2)); + } + } + return SDValue(); } case X86ISD::BLENDI: { @@ -36258,32 +37317,51 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG, return SDValue(); } case X86ISD::VPERM2X128: { - // If both 128-bit values were inserted into high halves of 256-bit values, - // the shuffle can be reduced to a concatenation of subvectors: - // vperm2x128 (ins ?, X, C1), (ins ?, Y, C2), 0x31 --> concat X, Y - // Note: We are only looking for the exact high/high shuffle mask because we - // expect to fold other similar patterns before creating this opcode. - SDValue Ins0 = peekThroughBitcasts(N.getOperand(0)); - SDValue Ins1 = peekThroughBitcasts(N.getOperand(1)); - unsigned Imm = N.getConstantOperandVal(2); - if (!(Imm == 0x31 && - Ins0.getOpcode() == ISD::INSERT_SUBVECTOR && - Ins1.getOpcode() == ISD::INSERT_SUBVECTOR && - Ins0.getValueType() == Ins1.getValueType())) - return SDValue(); + // Fold vperm2x128(bitcast(x),bitcast(y),c) -> bitcast(vperm2x128(x,y,c)). + SDValue LHS = N->getOperand(0); + SDValue RHS = N->getOperand(1); + if (LHS.getOpcode() == ISD::BITCAST && + (RHS.getOpcode() == ISD::BITCAST || RHS.isUndef())) { + EVT SrcVT = LHS.getOperand(0).getValueType(); + if (RHS.isUndef() || SrcVT == RHS.getOperand(0).getValueType()) { + return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT, + DAG.getBitcast(SrcVT, LHS), + DAG.getBitcast(SrcVT, RHS), + N->getOperand(2))); + } + } - SDValue X = Ins0.getOperand(1); - SDValue Y = Ins1.getOperand(1); - unsigned C1 = Ins0.getConstantOperandVal(2); - unsigned C2 = Ins1.getConstantOperandVal(2); - MVT SrcVT = X.getSimpleValueType(); - unsigned SrcElts = SrcVT.getVectorNumElements(); - if (SrcVT != Y.getSimpleValueType() || SrcVT.getSizeInBits() != 128 || - C1 != SrcElts || C2 != SrcElts) - return SDValue(); + // Fold vperm2x128(op(),op()) -> op(vperm2x128(),vperm2x128()). + if (SDValue Res = canonicalizeLaneShuffleWithRepeatedOps(N, DAG, DL)) + return Res; - return DAG.getBitcast(VT, DAG.getNode(ISD::CONCAT_VECTORS, DL, - Ins1.getValueType(), X, Y)); + // Fold vperm2x128 subvector shuffle with an inner concat pattern. + // vperm2x128(concat(X,Y),concat(Z,W)) --> concat X,Y etc. + auto FindSubVector128 = [&](unsigned Idx) { + if (Idx > 3) + return SDValue(); + SDValue Src = peekThroughBitcasts(N.getOperand(Idx < 2 ? 0 : 1)); + SmallVector<SDValue> SubOps; + if (collectConcatOps(Src.getNode(), SubOps) && SubOps.size() == 2) + return SubOps[Idx & 1]; + unsigned NumElts = Src.getValueType().getVectorNumElements(); + if ((Idx & 1) == 1 && Src.getOpcode() == ISD::INSERT_SUBVECTOR && + Src.getOperand(1).getValueSizeInBits() == 128 && + Src.getConstantOperandAPInt(2) == (NumElts / 2)) { + return Src.getOperand(1); + } + return SDValue(); + }; + unsigned Imm = N.getConstantOperandVal(2); + if (SDValue SubLo = FindSubVector128(Imm & 0x0F)) { + if (SDValue SubHi = FindSubVector128((Imm & 0xF0) >> 4)) { + MVT SubVT = VT.getHalfNumVectorElementsVT(); + SubLo = DAG.getBitcast(SubVT, SubLo); + SubHi = DAG.getBitcast(SubVT, SubHi); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubLo, SubHi); + } + } + return SDValue(); } case X86ISD::PSHUFD: case X86ISD::PSHUFLW: @@ -36726,10 +37804,12 @@ static SDValue combineShuffleOfConcatUndef(SDNode *N, SelectionDAG &DAG, /// Eliminate a redundant shuffle of a horizontal math op. static SDValue foldShuffleOfHorizOp(SDNode *N, SelectionDAG &DAG) { + // TODO: Can we use getTargetShuffleInputs instead? unsigned Opcode = N->getOpcode(); if (Opcode != X86ISD::MOVDDUP && Opcode != X86ISD::VBROADCAST) - if (Opcode != ISD::VECTOR_SHUFFLE || !N->getOperand(1).isUndef()) - return SDValue(); + if (Opcode != X86ISD::UNPCKL && Opcode != X86ISD::UNPCKH) + if (Opcode != ISD::VECTOR_SHUFFLE || !N->getOperand(1).isUndef()) + return SDValue(); // For a broadcast, peek through an extract element of index 0 to find the // horizontal op: broadcast (ext_vec_elt HOp, 0) @@ -36748,6 +37828,28 @@ static SDValue foldShuffleOfHorizOp(SDNode *N, SelectionDAG &DAG) { HOp.getOpcode() != X86ISD::HSUB && HOp.getOpcode() != X86ISD::FHSUB) return SDValue(); + // unpcklo(hop(x,y),hop(z,w)) -> permute(hop(x,z)). + // unpckhi(hop(x,y),hop(z,w)) -> permute(hop(y,w)). + // Don't fold if hop(x,y) == hop(z,w). + if (Opcode == X86ISD::UNPCKL || Opcode == X86ISD::UNPCKH) { + SDValue HOp2 = N->getOperand(1); + if (HOp.getOpcode() != HOp2.getOpcode() || VT.getScalarSizeInBits() != 32) + return SDValue(); + if (HOp == HOp2) + return SDValue(); + SDLoc DL(HOp); + unsigned LoHi = Opcode == X86ISD::UNPCKL ? 0 : 1; + SDValue Res = DAG.getNode(HOp.getOpcode(), DL, VT, HOp.getOperand(LoHi), + HOp2.getOperand(LoHi)); + // Use SHUFPS for the permute so this will work on SSE3 targets, shuffle + // combining and domain handling will simplify this later on. + EVT ShuffleVT = VT.changeVectorElementType(MVT::f32); + Res = DAG.getBitcast(ShuffleVT, Res); + Res = DAG.getNode(X86ISD::SHUFP, DL, ShuffleVT, Res, Res, + getV4X86ShuffleImm8ForMask({0, 2, 1, 3}, DL, DAG)); + return DAG.getBitcast(VT, Res); + } + // 128-bit horizontal math instructions are defined to operate on adjacent // lanes of each operand as: // v4X32: A[0] + A[1] , A[2] + A[3] , B[0] + B[1] , B[2] + B[3] @@ -36792,19 +37894,20 @@ static SDValue foldShuffleOfHorizOp(SDNode *N, SelectionDAG &DAG) { // shuffle (hadd X, X), undef, [low half...high half] --> hadd X, X ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask(); + // TODO: Other mask possibilities like {1,1} and {1,0} could be added here, // but this should be tied to whatever horizontal op matching and shuffle // canonicalization are producing. if (HOp.getValueSizeInBits() == 128 && - (isTargetShuffleEquivalent(Mask, {0, 0}) || - isTargetShuffleEquivalent(Mask, {0, 1, 0, 1}) || - isTargetShuffleEquivalent(Mask, {0, 1, 2, 3, 0, 1, 2, 3}))) + (isShuffleEquivalent(Mask, {0, 0}) || + isShuffleEquivalent(Mask, {0, 1, 0, 1}) || + isShuffleEquivalent(Mask, {0, 1, 2, 3, 0, 1, 2, 3}))) return updateHOp(HOp, DAG); if (HOp.getValueSizeInBits() == 256 && - (isTargetShuffleEquivalent(Mask, {0, 0, 2, 2}) || - isTargetShuffleEquivalent(Mask, {0, 1, 0, 1, 4, 5, 4, 5}) || - isTargetShuffleEquivalent( + (isShuffleEquivalent(Mask, {0, 0, 2, 2}) || + isShuffleEquivalent(Mask, {0, 1, 0, 1, 4, 5, 4, 5}) || + isShuffleEquivalent( Mask, {0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11}))) return updateHOp(HOp, DAG); @@ -36862,6 +37965,34 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG, if (SDValue HAddSub = foldShuffleOfHorizOp(N, DAG)) return HAddSub; + + // Merge shuffles through binops if its likely we'll be able to merge it + // with other shuffles (as long as they aren't splats). + // shuffle(bop(shuffle(x,y),shuffle(z,w)),bop(shuffle(a,b),shuffle(c,d))) + // TODO: We might be able to move this to DAGCombiner::visitVECTOR_SHUFFLE. + if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N)) { + unsigned SrcOpcode = N->getOperand(0).getOpcode(); + if (SrcOpcode == N->getOperand(1).getOpcode() && TLI.isBinOp(SrcOpcode) && + N->isOnlyUserOf(N->getOperand(0).getNode()) && + N->isOnlyUserOf(N->getOperand(1).getNode())) { + SDValue Op00 = N->getOperand(0).getOperand(0); + SDValue Op10 = N->getOperand(1).getOperand(0); + SDValue Op01 = N->getOperand(0).getOperand(1); + SDValue Op11 = N->getOperand(1).getOperand(1); + auto *SVN00 = dyn_cast<ShuffleVectorSDNode>(Op00); + auto *SVN10 = dyn_cast<ShuffleVectorSDNode>(Op10); + auto *SVN01 = dyn_cast<ShuffleVectorSDNode>(Op01); + auto *SVN11 = dyn_cast<ShuffleVectorSDNode>(Op11); + if (((SVN00 && !SVN00->isSplat()) || (SVN10 && !SVN10->isSplat())) && + ((SVN01 && !SVN01->isSplat()) || (SVN11 && !SVN11->isSplat()))) { + SDLoc DL(N); + ArrayRef<int> Mask = SVN->getMask(); + SDValue LHS = DAG.getVectorShuffle(VT, DL, Op00, Op10, Mask); + SDValue RHS = DAG.getVectorShuffle(VT, DL, Op01, Op11, Mask); + return DAG.getNode(SrcOpcode, DL, VT, LHS, RHS); + } + } + } } // Attempt to combine into a vector load/broadcast. @@ -36895,32 +38026,11 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG, // TODO - merge this into combineX86ShufflesRecursively. APInt KnownUndef, KnownZero; APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements()); - if (TLI.SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, DCI)) + if (TLI.SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, + DCI)) return SDValue(N, 0); } - // Pull subvector inserts into undef through VZEXT_MOVL by making it an - // insert into a zero vector. This helps get VZEXT_MOVL closer to - // scalar_to_vectors where 256/512 are canonicalized to an insert and a - // 128-bit scalar_to_vector. This reduces the number of isel patterns. - if (N->getOpcode() == X86ISD::VZEXT_MOVL && !DCI.isBeforeLegalizeOps() && - N->getOperand(0).hasOneUse()) { - SDValue V = peekThroughOneUseBitcasts(N->getOperand(0)); - - if (V.getOpcode() == ISD::INSERT_SUBVECTOR && - V.getOperand(0).isUndef() && isNullConstant(V.getOperand(2))) { - SDValue In = V.getOperand(1); - MVT SubVT = - MVT::getVectorVT(VT.getSimpleVT().getVectorElementType(), - In.getValueSizeInBits() / VT.getScalarSizeInBits()); - In = DAG.getBitcast(SubVT, In); - SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, dl, SubVT, In); - return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, - getZeroVector(VT.getSimpleVT(), Subtarget, DAG, dl), - Movl, V.getOperand(2)); - } - } - return SDValue(); } @@ -37051,7 +38161,13 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( if (SimplifyDemandedVectorElts(Src, DemandedElts, SrcUndef, KnownZero, TLO, Depth + 1)) return true; - // TODO convert SrcUndef to KnownUndef. + + // Aggressively peek through ops to get at the demanded elts. + if (!DemandedElts.isAllOnesValue()) + if (SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts( + Src, DemandedElts, TLO.DAG, Depth + 1)) + return TLO.CombineTo( + Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1))); break; } case X86ISD::KSHIFTL: { @@ -37240,7 +38356,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( SDValue Src = Op.getOperand(0); MVT SrcVT = Src.getSimpleValueType(); if (!SrcVT.isVector()) - return false; + break; // Don't bother broadcasting if we just need the 0'th element. if (DemandedElts == 1) { if (Src.getValueType() != VT) @@ -37293,21 +38409,62 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( ExtSizeInBits = SizeInBits / 4; switch (Opc) { - // Subvector broadcast. - case X86ISD::SUBV_BROADCAST: { + // Scalar broadcast. + case X86ISD::VBROADCAST: { SDLoc DL(Op); SDValue Src = Op.getOperand(0); if (Src.getValueSizeInBits() > ExtSizeInBits) Src = extractSubVector(Src, 0, TLO.DAG, DL, ExtSizeInBits); - else if (Src.getValueSizeInBits() < ExtSizeInBits) { - MVT SrcSVT = Src.getSimpleValueType().getScalarType(); - MVT SrcVT = - MVT::getVectorVT(SrcSVT, ExtSizeInBits / SrcSVT.getSizeInBits()); - Src = TLO.DAG.getNode(X86ISD::SUBV_BROADCAST, DL, SrcVT, Src); - } - return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Src, 0, + EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), + ExtSizeInBits / VT.getScalarSizeInBits()); + SDValue Bcst = TLO.DAG.getNode(X86ISD::VBROADCAST, DL, BcstVT, Src); + return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0, + TLO.DAG, DL, ExtSizeInBits)); + } + case X86ISD::VBROADCAST_LOAD: { + SDLoc DL(Op); + auto *MemIntr = cast<MemIntrinsicSDNode>(Op); + EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), + ExtSizeInBits / VT.getScalarSizeInBits()); + SDVTList Tys = TLO.DAG.getVTList(BcstVT, MVT::Other); + SDValue Ops[] = {MemIntr->getOperand(0), MemIntr->getOperand(1)}; + SDValue Bcst = TLO.DAG.getMemIntrinsicNode( + X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MemIntr->getMemoryVT(), + MemIntr->getMemOperand()); + TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1), + Bcst.getValue(1)); + return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0, TLO.DAG, DL, ExtSizeInBits)); } + // Subvector broadcast. + case X86ISD::SUBV_BROADCAST_LOAD: { + auto *MemIntr = cast<MemIntrinsicSDNode>(Op); + EVT MemVT = MemIntr->getMemoryVT(); + if (ExtSizeInBits == MemVT.getStoreSizeInBits()) { + SDLoc DL(Op); + SDValue Ld = + TLO.DAG.getLoad(MemVT, DL, MemIntr->getChain(), + MemIntr->getBasePtr(), MemIntr->getMemOperand()); + TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1), + Ld.getValue(1)); + return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Ld, 0, + TLO.DAG, DL, ExtSizeInBits)); + } else if ((ExtSizeInBits % MemVT.getStoreSizeInBits()) == 0) { + SDLoc DL(Op); + EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), + ExtSizeInBits / VT.getScalarSizeInBits()); + SDVTList Tys = TLO.DAG.getVTList(BcstVT, MVT::Other); + SDValue Ops[] = {MemIntr->getOperand(0), MemIntr->getOperand(1)}; + SDValue Bcst = + TLO.DAG.getMemIntrinsicNode(X86ISD::SUBV_BROADCAST_LOAD, DL, Tys, + Ops, MemVT, MemIntr->getMemOperand()); + TLO.DAG.makeEquivalentMemoryOrdering(SDValue(MemIntr, 1), + Bcst.getValue(1)); + return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0, + TLO.DAG, DL, ExtSizeInBits)); + } + break; + } // Byte shifts by immediate. case X86ISD::VSHLDQ: case X86ISD::VSRLDQ: @@ -37359,7 +38516,8 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( case X86ISD::UNPCKL: case X86ISD::UNPCKH: case X86ISD::BLENDI: - // Saturated Packs. + // Integer ops. + case X86ISD::AVG: case X86ISD::PACKSS: case X86ISD::PACKUS: // Horizontal Ops. @@ -37452,16 +38610,22 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode( // If we don't demand all elements, then attempt to combine to a simpler // shuffle. - // TODO: Handle other depths, but first we need to handle the fact that - // it might combine to the same shuffle. - if (!DemandedElts.isAllOnesValue() && Depth == 0) { + // We need to convert the depth to something combineX86ShufflesRecursively + // can handle - so pretend its Depth == 0 again, and reduce the max depth + // to match. This prevents combineX86ShuffleChain from returning a + // combined shuffle that's the same as the original root, causing an + // infinite loop. + if (!DemandedElts.isAllOnesValue()) { + assert(Depth < X86::MaxShuffleCombineDepth && "Depth out of range"); + SmallVector<int, 64> DemandedMask(NumElts, SM_SentinelUndef); for (int i = 0; i != NumElts; ++i) if (DemandedElts[i]) DemandedMask[i] = i; SDValue NewShuffle = combineX86ShufflesRecursively( - {Op}, 0, Op, DemandedMask, {}, Depth, /*HasVarMask*/ false, + {Op}, 0, Op, DemandedMask, {}, 0, X86::MaxShuffleCombineDepth - Depth, + /*HasVarMask*/ false, /*AllowVarMask*/ true, TLO.DAG, Subtarget); if (NewShuffle) return TLO.CombineTo(Op, NewShuffle); @@ -37564,7 +38728,7 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode( // Low bits known zero. Known.Zero.setLowBits(ShAmt); - break; + return false; } case X86ISD::VSRLI: { unsigned ShAmt = Op.getConstantOperandVal(1); @@ -37583,7 +38747,7 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode( // High bits known zero. Known.Zero.setHighBits(ShAmt); - break; + return false; } case X86ISD::VSRAI: { SDValue Op0 = Op.getOperand(0); @@ -37632,7 +38796,7 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode( // High bits are known one. if (Known.One[BitWidth - ShAmt - 1]) Known.One.setHighBits(ShAmt); - break; + return false; } case X86ISD::PEXTRB: case X86ISD::PEXTRW: { @@ -37698,8 +38862,7 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode( return true; KnownScl = KnownScl.trunc(VecVT.getScalarSizeInBits()); - Known.One = KnownVec.One & KnownScl.One; - Known.Zero = KnownVec.Zero & KnownScl.Zero; + Known = KnownBits::commonBits(KnownVec, KnownScl); return false; } break; @@ -37779,34 +38942,83 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode( return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); return false; } - case X86ISD::BEXTR: { + case X86ISD::BEXTR: + case X86ISD::BEXTRI: { SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); // Only bottom 16-bits of the control bits are required. if (auto *Cst1 = dyn_cast<ConstantSDNode>(Op1)) { // NOTE: SimplifyDemandedBits won't do this for constants. - const APInt &Val1 = Cst1->getAPIntValue(); - APInt MaskedVal1 = Val1 & 0xFFFF; - if (MaskedVal1 != Val1) { + uint64_t Val1 = Cst1->getZExtValue(); + uint64_t MaskedVal1 = Val1 & 0xFFFF; + if (Opc == X86ISD::BEXTR && MaskedVal1 != Val1) { SDLoc DL(Op); return TLO.CombineTo( Op, TLO.DAG.getNode(X86ISD::BEXTR, DL, VT, Op0, TLO.DAG.getConstant(MaskedVal1, DL, VT))); } + + unsigned Shift = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 0); + unsigned Length = Cst1->getAPIntValue().extractBitsAsZExtValue(8, 8); + + // If the length is 0, the result is 0. + if (Length == 0) { + Known.setAllZero(); + return false; + } + + if ((Shift + Length) <= BitWidth) { + APInt DemandedMask = APInt::getBitsSet(BitWidth, Shift, Shift + Length); + if (SimplifyDemandedBits(Op0, DemandedMask, Known, TLO, Depth + 1)) + return true; + + Known = Known.extractBits(Length, Shift); + Known = Known.zextOrTrunc(BitWidth); + return false; + } + } else { + assert(Opc == X86ISD::BEXTR && "Unexpected opcode!"); + KnownBits Known1; + APInt DemandedMask(APInt::getLowBitsSet(BitWidth, 16)); + if (SimplifyDemandedBits(Op1, DemandedMask, Known1, TLO, Depth + 1)) + return true; + + // If the length is 0, replace with 0. + KnownBits LengthBits = Known1.extractBits(8, 8); + if (LengthBits.isZero()) + return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); } - KnownBits Known1; - APInt DemandedMask(APInt::getLowBitsSet(BitWidth, 16)); - if (SimplifyDemandedBits(Op1, DemandedMask, Known1, TLO, Depth + 1)) + break; + } + case X86ISD::PDEP: { + SDValue Op0 = Op.getOperand(0); + SDValue Op1 = Op.getOperand(1); + + unsigned DemandedBitsLZ = OriginalDemandedBits.countLeadingZeros(); + APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ); + + // If the demanded bits has leading zeroes, we don't demand those from the + // mask. + if (SimplifyDemandedBits(Op1, LoMask, Known, TLO, Depth + 1)) return true; - // If the length is 0, replace with 0. - KnownBits LengthBits = Known1.extractBits(8, 8); - if (LengthBits.isZero()) - return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); + // The number of possible 1s in the mask determines the number of LSBs of + // operand 0 used. Undemanded bits from the mask don't matter so filter + // them before counting. + KnownBits Known2; + uint64_t Count = (~Known.Zero & LoMask).countPopulation(); + APInt DemandedMask(APInt::getLowBitsSet(BitWidth, Count)); + if (SimplifyDemandedBits(Op0, DemandedMask, Known2, TLO, Depth + 1)) + return true; - break; + // Zeroes are retained from the mask, but not ones. + Known.One.clearAllBits(); + // The result will have at least as many trailing zeros as the non-mask + // operand since bits can only map to the same or higher bit position. + Known.Zero.setLowBits(Known2.countMinTrailingZeros()); + return false; } } @@ -38217,6 +39429,8 @@ static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG, // Convert build vector ops to MMX data in the bottom elements. SmallVector<SDValue, 8> Ops; + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + // Broadcast - use (PUNPCKL+)PSHUFW to broadcast single element. if (Splat) { if (Splat.isUndef()) @@ -38229,14 +39443,16 @@ static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG, if (NumElts == 8) Splat = DAG.getNode( ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, - DAG.getConstant(Intrinsic::x86_mmx_punpcklbw, DL, MVT::i32), Splat, - Splat); + DAG.getTargetConstant(Intrinsic::x86_mmx_punpcklbw, DL, + TLI.getPointerTy(DAG.getDataLayout())), + Splat, Splat); // Use PSHUFW to repeat 16-bit elements. unsigned ShufMask = (NumElts > 2 ? 0 : 0x44); return DAG.getNode( ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, - DAG.getTargetConstant(Intrinsic::x86_sse_pshuf_w, DL, MVT::i32), + DAG.getTargetConstant(Intrinsic::x86_sse_pshuf_w, DL, + TLI.getPointerTy(DAG.getDataLayout())), Splat, DAG.getTargetConstant(ShufMask, DL, MVT::i8)); } Ops.append(NumElts, Splat); @@ -38252,7 +39468,8 @@ static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG, (NumOps == 2 ? Intrinsic::x86_mmx_punpckldq : (NumOps == 4 ? Intrinsic::x86_mmx_punpcklwd : Intrinsic::x86_mmx_punpcklbw)); - SDValue Intrin = DAG.getConstant(IntrinOp, DL, MVT::i32); + SDValue Intrin = DAG.getTargetConstant( + IntrinOp, DL, TLI.getPointerTy(DAG.getDataLayout())); for (unsigned i = 0; i != NumOps; i += 2) Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin, Ops[i], Ops[i + 1]); @@ -38266,7 +39483,7 @@ static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG, // a vector/float/double that got truncated/extended/bitcast to/from a scalar // integer. If so, replace the scalar ops with bool vector equivalents back down // the chain. -static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, SDLoc DL, +static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); @@ -38319,6 +39536,10 @@ static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, SDLoc DL, case ISD::SHL: { // If we find a suitable source, a SHL becomes a KSHIFTL. SDValue Src0 = V.getOperand(0); + if ((VT == MVT::v8i1 && !Subtarget.hasDQI()) || + ((VT == MVT::v32i1 || VT == MVT::v64i1) && !Subtarget.hasBWI())) + break; + if (auto *Amt = dyn_cast<ConstantSDNode>(V.getOperand(1))) if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget)) return DAG.getNode( @@ -38661,8 +39882,8 @@ static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0, // Attempt to replace an min/max v8i16/v16i8 horizontal reduction with // PHMINPOSUW. -static SDValue combineHorizontalMinMaxResult(SDNode *Extract, SelectionDAG &DAG, - const X86Subtarget &Subtarget) { +static SDValue combineMinMaxReduction(SDNode *Extract, SelectionDAG &DAG, + const X86Subtarget &Subtarget) { // Bail without SSE41. if (!Subtarget.hasSSE41()) return SDValue(); @@ -38735,9 +39956,8 @@ static SDValue combineHorizontalMinMaxResult(SDNode *Extract, SelectionDAG &DAG, } // Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK. -static SDValue combineHorizontalPredicateResult(SDNode *Extract, - SelectionDAG &DAG, - const X86Subtarget &Subtarget) { +static SDValue combinePredicateReduction(SDNode *Extract, SelectionDAG &DAG, + const X86Subtarget &Subtarget) { // Bail without SSE2. if (!Subtarget.hasSSE2()) return SDValue(); @@ -38851,10 +40071,8 @@ static SDValue combineHorizontalPredicateResult(SDNode *Extract, MVT CmpVT = NumElts == 64 ? MVT::i64 : MVT::i32; if (BinOp == ISD::XOR) { - // parity -> (AND (CTPOP(MOVMSK X)), 1) - SDValue Mask = DAG.getConstant(1, DL, CmpVT); - SDValue Result = DAG.getNode(ISD::CTPOP, DL, CmpVT, Movmsk); - Result = DAG.getNode(ISD::AND, DL, CmpVT, Result, Mask); + // parity -> (PARITY(MOVMSK X)) + SDValue Result = DAG.getNode(ISD::PARITY, DL, CmpVT, Movmsk); return DAG.getZExtOrTrunc(Result, DL, ExtractVT); } @@ -39042,10 +40260,12 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG, // Handle extract(truncate(x)) for 0'th index. // TODO: Treat this as a faux shuffle? // TODO: When can we use this for general indices? - if (ISD::TRUNCATE == Src.getOpcode() && SrcVT.is128BitVector() && IdxC == 0) { + if (ISD::TRUNCATE == Src.getOpcode() && IdxC == 0 && + (SrcVT.getSizeInBits() % 128) == 0) { Src = extract128BitVector(Src.getOperand(0), 0, DAG, dl); - Src = DAG.getBitcast(SrcVT, Src); - return DAG.getNode(N->getOpcode(), dl, VT, Src, Idx); + MVT ExtractVT = MVT::getVectorVT(SrcSVT.getSimpleVT(), 128 / SrcEltBits); + return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src), + Idx); } // Resolve the target shuffle inputs and mask. @@ -39121,7 +40341,7 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG, unsigned OpCode = (SrcVT == MVT::v8i16 ? X86ISD::PEXTRW : X86ISD::PEXTRB); SrcOp = DAG.getBitcast(SrcVT, SrcOp); SDValue ExtOp = DAG.getNode(OpCode, dl, MVT::i32, SrcOp, - DAG.getIntPtrConstant(SrcIdx, dl)); + DAG.getTargetConstant(SrcIdx, dl, MVT::i8)); return DAG.getZExtOrTrunc(ExtOp, dl, VT); } @@ -39228,8 +40448,8 @@ static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG) { /// Try to convert a vector reduction sequence composed of binops and shuffles /// into horizontal ops. -static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG, - const X86Subtarget &Subtarget) { +static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG, + const X86Subtarget &Subtarget) { assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unexpected caller"); // We need at least SSE2 to anything here. @@ -39237,8 +40457,8 @@ static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG, return SDValue(); ISD::NodeType Opc; - SDValue Rdx = - DAG.matchBinOpReduction(ExtElt, Opc, {ISD::ADD, ISD::FADD}, true); + SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc, + {ISD::ADD, ISD::MUL, ISD::FADD}, true); if (!Rdx) return SDValue(); @@ -39253,7 +40473,46 @@ static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG, SDLoc DL(ExtElt); - // vXi8 reduction - sub 128-bit vector. + // vXi8 mul reduction - promote to vXi16 mul reduction. + if (Opc == ISD::MUL) { + unsigned NumElts = VecVT.getVectorNumElements(); + if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts)) + return SDValue(); + if (VecVT.getSizeInBits() >= 128) { + EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2); + SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT)); + SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT)); + Lo = DAG.getBitcast(WideVT, Lo); + Hi = DAG.getBitcast(WideVT, Hi); + Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi); + while (Rdx.getValueSizeInBits() > 128) { + std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL); + Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi); + } + } else { + if (VecVT == MVT::v4i8) + Rdx = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Rdx, + DAG.getUNDEF(MVT::v4i8)); + Rdx = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, Rdx, + DAG.getUNDEF(MVT::v8i8)); + Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8)); + Rdx = DAG.getBitcast(MVT::v8i16, Rdx); + } + if (NumElts >= 8) + Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx, + DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx, + {4, 5, 6, 7, -1, -1, -1, -1})); + Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx, + DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx, + {2, 3, -1, -1, -1, -1, -1, -1})); + Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx, + DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx, + {1, -1, -1, -1, -1, -1, -1, -1})); + Rdx = DAG.getBitcast(MVT::v16i8, Rdx); + return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); + } + + // vXi8 add reduction - sub 128-bit vector. if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) { if (VecVT == MVT::v4i8) { // Pad with zero. @@ -39284,7 +40543,7 @@ static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG, !isPowerOf2_32(VecVT.getVectorNumElements())) return SDValue(); - // vXi8 reduction - sum lo/hi halves then use PSADBW. + // vXi8 add reduction - sum lo/hi halves then use PSADBW. if (VT == MVT::i8) { while (Rdx.getValueSizeInBits() > 128) { SDValue Lo, Hi; @@ -39390,7 +40649,7 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG, } // TODO - Remove this once we can handle the implicit zero-extension of - // X86ISD::PEXTRW/X86ISD::PEXTRB in combineHorizontalPredicateResult and + // X86ISD::PEXTRW/X86ISD::PEXTRB in combinePredicateReduction and // combineBasicSADPattern. return SDValue(); } @@ -39422,14 +40681,15 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG, return SAD; // Attempt to replace an all_of/any_of horizontal reduction with a MOVMSK. - if (SDValue Cmp = combineHorizontalPredicateResult(N, DAG, Subtarget)) + if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget)) return Cmp; // Attempt to replace min/max v8i16/v16i8 reductions with PHMINPOSUW. - if (SDValue MinMax = combineHorizontalMinMaxResult(N, DAG, Subtarget)) + if (SDValue MinMax = combineMinMaxReduction(N, DAG, Subtarget)) return MinMax; - if (SDValue V = combineReductionToHorizontal(N, DAG, Subtarget)) + // Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc.. + if (SDValue V = combineArithReduction(N, DAG, Subtarget)) return V; if (SDValue V = scalarizeExtEltFP(N, DAG)) @@ -39553,7 +40813,7 @@ combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG, if (TValIsAllOnes && FValIsAllZeros) return DAG.getBitcast(VT, Cond); - if (!DCI.isBeforeLegalize() && !TLI.isTypeLegal(CondVT)) + if (!TLI.isTypeLegal(CondVT)) return SDValue(); // vselect Cond, 111..., X -> or Cond, X @@ -39572,10 +40832,14 @@ combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG, // vselect Cond, 000..., X -> andn Cond, X if (TValIsAllZeros) { - MVT AndNVT = MVT::getVectorVT(MVT::i64, CondVT.getSizeInBits() / 64); - SDValue CastCond = DAG.getBitcast(AndNVT, Cond); - SDValue CastRHS = DAG.getBitcast(AndNVT, RHS); - SDValue AndN = DAG.getNode(X86ISD::ANDNP, DL, AndNVT, CastCond, CastRHS); + SDValue CastRHS = DAG.getBitcast(CondVT, RHS); + SDValue AndN; + // The canonical form differs for i1 vectors - x86andnp is not used + if (CondVT.getScalarType() == MVT::i1) + AndN = DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT), + CastRHS); + else + AndN = DAG.getNode(X86ISD::ANDNP, DL, CondVT, Cond, CastRHS); return DAG.getBitcast(VT, AndN); } @@ -39872,6 +41136,36 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG, return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask); } + // fold vselect(cond, pshufb(x), pshufb(y)) -> or (pshufb(x), pshufb(y)) + // by forcing the unselected elements to zero. + // TODO: Can we handle more shuffles with this? + if (N->getOpcode() == ISD::VSELECT && CondVT.isVector() && + LHS.getOpcode() == X86ISD::PSHUFB && RHS.getOpcode() == X86ISD::PSHUFB && + LHS.hasOneUse() && RHS.hasOneUse()) { + MVT SimpleVT = VT.getSimpleVT(); + bool LHSUnary, RHSUnary; + SmallVector<SDValue, 1> LHSOps, RHSOps; + SmallVector<int, 64> LHSMask, RHSMask, CondMask; + if (createShuffleMaskFromVSELECT(CondMask, Cond) && + getTargetShuffleMask(LHS.getNode(), SimpleVT, true, LHSOps, LHSMask, + LHSUnary) && + getTargetShuffleMask(RHS.getNode(), SimpleVT, true, RHSOps, RHSMask, + RHSUnary)) { + int NumElts = VT.getVectorNumElements(); + for (int i = 0; i != NumElts; ++i) { + if (CondMask[i] < NumElts) + RHSMask[i] = 0x80; + else + LHSMask[i] = 0x80; + } + LHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, LHS.getOperand(0), + getConstVector(LHSMask, SimpleVT, DAG, DL, true)); + RHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, RHS.getOperand(0), + getConstVector(RHSMask, SimpleVT, DAG, DL, true)); + return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); + } + } + // If we have SSE[12] support, try to form min/max nodes. SSE min/max // instructions match the semantics of the common C idiom x<y?x:y but not // x<=y?x:y, because of how they handle negative zero (which can be @@ -40098,13 +41392,12 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG, if (SDValue V = combineSelectOfTwoConstants(N, DAG)) return V; - // Canonicalize max and min: - // (x > y) ? x : y -> (x >= y) ? x : y - // (x < y) ? x : y -> (x <= y) ? x : y + // Canonicalize min/max: + // (x > 0) ? x : 0 -> (x >= 0) ? x : 0 + // (x < -1) ? x : -1 -> (x <= -1) ? x : -1 // This allows use of COND_S / COND_NS (see TranslateX86CC) which eliminates - // the need for an extra compare - // against zero. e.g. - // (x - y) > 0 : (x - y) ? 0 -> (x - y) >= 0 : (x - y) ? 0 + // the need for an extra compare against zero. e.g. + // (a - b) > 0 : (a - b) ? 0 -> (a - b) >= 0 : (a - b) ? 0 // subl %esi, %edi // testl %edi, %edi // movl $0, %eax @@ -40113,142 +41406,27 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG, // xorl %eax, %eax // subl %esi, $edi // cmovsl %eax, %edi + // + // We can also canonicalize + // (x s> 1) ? x : 1 -> (x s>= 1) ? x : 1 -> (x s> 0) ? x : 1 + // (x u> 1) ? x : 1 -> (x u>= 1) ? x : 1 -> (x != 0) ? x : 1 + // This allows the use of a test instruction for the compare. if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC && Cond.hasOneUse() && - DAG.isEqualTo(LHS, Cond.getOperand(0)) && - DAG.isEqualTo(RHS, Cond.getOperand(1))) { + LHS == Cond.getOperand(0) && RHS == Cond.getOperand(1)) { ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); - switch (CC) { - default: break; - case ISD::SETLT: - case ISD::SETGT: { - ISD::CondCode NewCC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGE; + if ((CC == ISD::SETGT && (isNullConstant(RHS) || isOneConstant(RHS))) || + (CC == ISD::SETLT && isAllOnesConstant(RHS))) { + ISD::CondCode NewCC = CC == ISD::SETGT ? ISD::SETGE : ISD::SETLE; Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), Cond.getOperand(0), Cond.getOperand(1), NewCC); return DAG.getSelect(DL, VT, Cond, LHS, RHS); } - } - } - - // Match VSELECTs into subs with unsigned saturation. - if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC && - // psubus is available in SSE2 for i8 and i16 vectors. - Subtarget.hasSSE2() && VT.getVectorNumElements() >= 2 && - isPowerOf2_32(VT.getVectorNumElements()) && - (VT.getVectorElementType() == MVT::i8 || - VT.getVectorElementType() == MVT::i16)) { - ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); - - // Check if one of the arms of the VSELECT is a zero vector. If it's on the - // left side invert the predicate to simplify logic below. - SDValue Other; - if (ISD::isBuildVectorAllZeros(LHS.getNode())) { - Other = RHS; - CC = ISD::getSetCCInverse(CC, VT.getVectorElementType()); - } else if (ISD::isBuildVectorAllZeros(RHS.getNode())) { - Other = LHS; - } - - if (Other.getNode() && Other->getNumOperands() == 2 && - Other->getOperand(0) == Cond.getOperand(0)) { - SDValue OpLHS = Other->getOperand(0), OpRHS = Other->getOperand(1); - SDValue CondRHS = Cond->getOperand(1); - - // Look for a general sub with unsigned saturation first. - // x >= y ? x-y : 0 --> subus x, y - // x > y ? x-y : 0 --> subus x, y - if ((CC == ISD::SETUGE || CC == ISD::SETUGT) && - Other->getOpcode() == ISD::SUB && OpRHS == CondRHS) - return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); - - if (auto *OpRHSBV = dyn_cast<BuildVectorSDNode>(OpRHS)) { - if (isa<BuildVectorSDNode>(CondRHS)) { - // If the RHS is a constant we have to reverse the const - // canonicalization. - // x > C-1 ? x+-C : 0 --> subus x, C - auto MatchUSUBSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) { - return (!Op && !Cond) || - (Op && Cond && - Cond->getAPIntValue() == (-Op->getAPIntValue() - 1)); - }; - if (CC == ISD::SETUGT && Other->getOpcode() == ISD::ADD && - ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT, - /*AllowUndefs*/ true)) { - OpRHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), - OpRHS); - return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); - } - - // Another special case: If C was a sign bit, the sub has been - // canonicalized into a xor. - // FIXME: Would it be better to use computeKnownBits to determine - // whether it's safe to decanonicalize the xor? - // x s< 0 ? x^C : 0 --> subus x, C - if (auto *OpRHSConst = OpRHSBV->getConstantSplatNode()) { - if (CC == ISD::SETLT && Other.getOpcode() == ISD::XOR && - ISD::isBuildVectorAllZeros(CondRHS.getNode()) && - OpRHSConst->getAPIntValue().isSignMask()) { - // Note that we have to rebuild the RHS constant here to ensure we - // don't rely on particular values of undef lanes. - OpRHS = DAG.getConstant(OpRHSConst->getAPIntValue(), DL, VT); - return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); - } - } - } - } - } - } - - // Match VSELECTs into add with unsigned saturation. - if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC && - // paddus is available in SSE2 for i8 and i16 vectors. - Subtarget.hasSSE2() && VT.getVectorNumElements() >= 2 && - isPowerOf2_32(VT.getVectorNumElements()) && - (VT.getVectorElementType() == MVT::i8 || - VT.getVectorElementType() == MVT::i16)) { - ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); - - SDValue CondLHS = Cond->getOperand(0); - SDValue CondRHS = Cond->getOperand(1); - - // Check if one of the arms of the VSELECT is vector with all bits set. - // If it's on the left side invert the predicate to simplify logic below. - SDValue Other; - if (ISD::isBuildVectorAllOnes(LHS.getNode())) { - Other = RHS; - CC = ISD::getSetCCInverse(CC, VT.getVectorElementType()); - } else if (ISD::isBuildVectorAllOnes(RHS.getNode())) { - Other = LHS; - } - - if (Other.getNode() && Other.getOpcode() == ISD::ADD) { - SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1); - - // Canonicalize condition operands. - if (CC == ISD::SETUGE) { - std::swap(CondLHS, CondRHS); - CC = ISD::SETULE; - } - - // We can test against either of the addition operands. - // x <= x+y ? x+y : ~0 --> addus x, y - // x+y >= x ? x+y : ~0 --> addus x, y - if (CC == ISD::SETULE && Other == CondRHS && - (OpLHS == CondLHS || OpRHS == CondLHS)) - return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); - - if (isa<BuildVectorSDNode>(OpRHS) && isa<BuildVectorSDNode>(CondRHS) && - CondLHS == OpLHS) { - // If the RHS is a constant we have to reverse the const - // canonicalization. - // x > ~C ? x+C : ~0 --> addus x, C - auto MatchUADDSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) { - return Cond->getAPIntValue() == ~Op->getAPIntValue(); - }; - if (CC == ISD::SETULE && - ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUADDSAT)) - return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); - } + if (CC == ISD::SETUGT && isOneConstant(RHS)) { + ISD::CondCode NewCC = ISD::SETUGE; + Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), + Cond.getOperand(0), Cond.getOperand(1), NewCC); + return DAG.getSelect(DL, VT, Cond, LHS, RHS); } } @@ -40279,10 +41457,18 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG, return V; // select(~Cond, X, Y) -> select(Cond, Y, X) - if (CondVT.getScalarType() != MVT::i1) + if (CondVT.getScalarType() != MVT::i1) { if (SDValue CondNot = IsNOT(Cond, DAG)) return DAG.getNode(N->getOpcode(), DL, VT, DAG.getBitcast(CondVT, CondNot), RHS, LHS); + // pcmpgt(X, -1) -> pcmpgt(0, X) to help select/blendv just use the signbit. + if (Cond.getOpcode() == X86ISD::PCMPGT && Cond.hasOneUse() && + ISD::isBuildVectorAllOnes(Cond.getOperand(1).getNode())) { + Cond = DAG.getNode(X86ISD::PCMPGT, DL, CondVT, + DAG.getConstant(0, DL, CondVT), Cond.getOperand(0)); + return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS); + } + } // Try to optimize vXi1 selects if both operands are either all constants or // bitcasts from scalar integer type. In that case we can convert the operands @@ -41899,73 +43085,115 @@ static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG, return SDValue(); } -static SDValue combineVectorPackWithShuffle(SDNode *N, SelectionDAG &DAG) { +static SDValue combineHorizOpWithShuffle(SDNode *N, SelectionDAG &DAG, + const X86Subtarget &Subtarget) { unsigned Opcode = N->getOpcode(); - assert((X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) && - "Unexpected pack opcode"); + assert((X86ISD::HADD == Opcode || X86ISD::FHADD == Opcode || + X86ISD::HSUB == Opcode || X86ISD::FHSUB == Opcode || + X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) && + "Unexpected hadd/hsub/pack opcode"); EVT VT = N->getValueType(0); SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); - unsigned NumDstElts = VT.getVectorNumElements(); + EVT SrcVT = N0.getValueType(); - // Attempt to fold PACK(LOSUBVECTOR(SHUFFLE(X)),HISUBVECTOR(SHUFFLE(X))) - // to SHUFFLE(PACK(LOSUBVECTOR(X),HISUBVECTOR(X))), this is mainly for + // Attempt to fold HOP(LOSUBVECTOR(SHUFFLE(X)),HISUBVECTOR(SHUFFLE(X))) + // to SHUFFLE(HOP(LOSUBVECTOR(X),HISUBVECTOR(X))), this is mainly for // truncation trees that help us avoid lane crossing shuffles. // TODO: There's a lot more we can do for PACK/HADD style shuffle combines. + // TODO: We don't handle vXf64 shuffles yet. if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.getConstantOperandAPInt(1) == 0 && - N1.getConstantOperandAPInt(1) == (NumDstElts / 2) && + N1.getConstantOperandAPInt(1) == SrcVT.getVectorNumElements() && N0.getOperand(0) == N1.getOperand(0) && VT.is128BitVector() && - N0.getOperand(0).getValueType().is256BitVector()) { + N0.getOperand(0).getValueType().is256BitVector() && + SrcVT.getScalarSizeInBits() <= 32) { // TODO - support target/faux shuffles. SDValue Vec = peekThroughBitcasts(N0.getOperand(0)); if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Vec)) { - // To keep the PACK LHS/RHS coherency, we must be able to scale the unary + // To keep the HOP LHS/RHS coherency, we must be able to scale the unary // shuffle to a vXi64 width - we can probably relax this in the future. SmallVector<int, 4> ShuffleMask; if (SVN->getOperand(1).isUndef() && scaleShuffleElements(SVN->getMask(), 4, ShuffleMask)) { SDLoc DL(N); SDValue Lo, Hi; + MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32; std::tie(Lo, Hi) = DAG.SplitVector(SVN->getOperand(0), DL); Lo = DAG.getBitcast(N0.getValueType(), Lo); Hi = DAG.getBitcast(N1.getValueType(), Hi); SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi); - Res = DAG.getBitcast(MVT::v4i32, Res); - Res = DAG.getVectorShuffle(MVT::v4i32, DL, Res, Res, ShuffleMask); + Res = DAG.getBitcast(ShufVT, Res); + Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ShuffleMask); return DAG.getBitcast(VT, Res); } } } - // Attempt to fold PACK(SHUFFLE(X,Y),SHUFFLE(X,Y)) -> SHUFFLE(PACK(X,Y)). + // Attempt to fold HOP(SHUFFLE(X),SHUFFLE(Y)) -> SHUFFLE(HOP(X,Y)). + // TODO: Merge with binary shuffle folds below. + if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) { + int PostShuffle[4] = {0, 1, 2, 3}; + + // If the op is an unary shuffle that can scale to v2x64, + // then we can perform this as a v4x32 post shuffle. + auto AdjustOp = [&](SDValue V, int Offset) { + auto *SVN = dyn_cast<ShuffleVectorSDNode>(V); + SmallVector<int, 2> ScaledMask; + if (!SVN || !SVN->getOperand(1).isUndef() || + !scaleShuffleElements(SVN->getMask(), 2, ScaledMask) || + !N->isOnlyUserOf(V.getNode())) + return SDValue(); + PostShuffle[Offset + 0] = ScaledMask[0] < 0 ? -1 : Offset + ScaledMask[0]; + PostShuffle[Offset + 1] = ScaledMask[1] < 0 ? -1 : Offset + ScaledMask[1]; + return SVN->getOperand(0); + }; + + SDValue Src0 = AdjustOp(N0, 0); + SDValue Src1 = AdjustOp(N1, 2); + if (Src0 || Src1) { + Src0 = Src0 ? Src0 : N0; + Src1 = Src1 ? Src1 : N1; + SDLoc DL(N); + MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32; + SDValue Res = DAG.getNode(Opcode, DL, VT, Src0, Src1); + Res = DAG.getBitcast(ShufVT, Res); + Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, PostShuffle); + return DAG.getBitcast(VT, Res); + } + } + + // Attempt to fold HOP(SHUFFLE(X,Y),SHUFFLE(X,Y)) -> SHUFFLE(HOP(X,Y)). // TODO: Relax shuffle scaling to support sub-128-bit subvector shuffles. - if (VT.is256BitVector()) { - if (auto *SVN0 = dyn_cast<ShuffleVectorSDNode>(N0)) { - if (auto *SVN1 = dyn_cast<ShuffleVectorSDNode>(N1)) { - SmallVector<int, 2> ShuffleMask0, ShuffleMask1; - if (scaleShuffleElements(SVN0->getMask(), 2, ShuffleMask0) && - scaleShuffleElements(SVN1->getMask(), 2, ShuffleMask1)) { - SDValue Op00 = SVN0->getOperand(0); - SDValue Op01 = SVN0->getOperand(1); - SDValue Op10 = SVN1->getOperand(0); - SDValue Op11 = SVN1->getOperand(1); - if ((Op00 == Op11) && (Op01 == Op10)) { - std::swap(Op10, Op11); - ShuffleVectorSDNode::commuteMask(ShuffleMask1); - } - if ((Op00 == Op10) && (Op01 == Op11)) { - SmallVector<int, 4> ShuffleMask; - ShuffleMask.append(ShuffleMask0.begin(), ShuffleMask0.end()); - ShuffleMask.append(ShuffleMask1.begin(), ShuffleMask1.end()); - SDLoc DL(N); - SDValue Res = DAG.getNode(Opcode, DL, VT, Op00, Op01); - Res = DAG.getBitcast(MVT::v4i64, Res); - Res = DAG.getVectorShuffle(MVT::v4i64, DL, Res, Res, ShuffleMask); - return DAG.getBitcast(VT, Res); - } + if (VT.is256BitVector() && Subtarget.hasInt256()) { + SmallVector<int> Mask0, Mask1; + SmallVector<SDValue> Ops0, Ops1; + if (getTargetShuffleInputs(N0, Ops0, Mask0, DAG) && !isAnyZero(Mask0) && + getTargetShuffleInputs(N1, Ops1, Mask1, DAG) && !isAnyZero(Mask1) && + !Ops0.empty() && !Ops1.empty()) { + SDValue Op00 = Ops0.front(), Op01 = Ops0.back(); + SDValue Op10 = Ops1.front(), Op11 = Ops1.back(); + SmallVector<int, 2> ShuffleMask0, ShuffleMask1; + if (Op00.getValueType() == SrcVT && Op01.getValueType() == SrcVT && + Op11.getValueType() == SrcVT && Op11.getValueType() == SrcVT && + scaleShuffleElements(Mask0, 2, ShuffleMask0) && + scaleShuffleElements(Mask1, 2, ShuffleMask1)) { + if ((Op00 == Op11) && (Op01 == Op10)) { + std::swap(Op10, Op11); + ShuffleVectorSDNode::commuteMask(ShuffleMask1); + } + if ((Op00 == Op10) && (Op01 == Op11)) { + SmallVector<int, 4> ShuffleMask; + ShuffleMask.append(ShuffleMask0.begin(), ShuffleMask0.end()); + ShuffleMask.append(ShuffleMask1.begin(), ShuffleMask1.end()); + SDLoc DL(N); + MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64; + SDValue Res = DAG.getNode(Opcode, DL, VT, Op00, Op01); + Res = DAG.getBitcast(ShufVT, Res); + Res = DAG.getVectorShuffle(ShufVT, DL, Res, Res, ShuffleMask); + return DAG.getBitcast(VT, Res); } } } @@ -42048,7 +43276,7 @@ static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG, } // Try to fold PACK(SHUFFLE(),SHUFFLE()) -> SHUFFLE(PACK()). - if (SDValue V = combineVectorPackWithShuffle(N, DAG)) + if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget)) return V; // Try to combine a PACKUSWB/PACKSSWB implemented truncate with a regular @@ -42070,6 +43298,28 @@ static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG, } } + // Try to fold PACK(EXTEND(X),EXTEND(Y)) -> CONCAT(X,Y) subvectors. + if (VT.is128BitVector()) { + unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; + SDValue Src0, Src1; + if (N0.getOpcode() == ExtOpc && + N0.getOperand(0).getValueType().is64BitVector() && + N0.getOperand(0).getScalarValueSizeInBits() == DstBitsPerElt) { + Src0 = N0.getOperand(0); + } + if (N1.getOpcode() == ExtOpc && + N1.getOperand(0).getValueType().is64BitVector() && + N1.getOperand(0).getScalarValueSizeInBits() == DstBitsPerElt) { + Src1 = N1.getOperand(0); + } + if ((Src0 || N0.isUndef()) && (Src1 || N1.isUndef())) { + assert((Src0 || Src1) && "Found PACK(UNDEF,UNDEF)"); + Src0 = Src0 ? Src0 : DAG.getUNDEF(Src1.getValueType()); + Src1 = Src1 ? Src1 : DAG.getUNDEF(Src0.getValueType()); + return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Src0, Src1); + } + } + // Attempt to combine as shuffle. SDValue Op(N, 0); if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) @@ -42078,6 +43328,20 @@ static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG, return SDValue(); } +static SDValue combineVectorHADDSUB(SDNode *N, SelectionDAG &DAG, + TargetLowering::DAGCombinerInfo &DCI, + const X86Subtarget &Subtarget) { + assert((X86ISD::HADD == N->getOpcode() || X86ISD::FHADD == N->getOpcode() || + X86ISD::HSUB == N->getOpcode() || X86ISD::FHSUB == N->getOpcode()) && + "Unexpected horizontal add/sub opcode"); + + // Try to fold HOP(SHUFFLE(),SHUFFLE()) -> SHUFFLE(HOP()). + if (SDValue V = combineHorizOpWithShuffle(N, DAG, Subtarget)) + return V; + + return SDValue(); +} + static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) { @@ -42706,74 +43970,6 @@ static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG, return SDValue(); } -// Look for (and (ctpop X), 1) which is the IR form of __builtin_parity. -// Turn it into series of XORs and a setnp. -static SDValue combineParity(SDNode *N, SelectionDAG &DAG, - const X86Subtarget &Subtarget) { - EVT VT = N->getValueType(0); - - // We only support 64-bit and 32-bit. 64-bit requires special handling - // unless the 64-bit popcnt instruction is legal. - if (VT != MVT::i32 && VT != MVT::i64) - return SDValue(); - - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - if (TLI.isTypeLegal(VT) && TLI.isOperationLegal(ISD::CTPOP, VT)) - return SDValue(); - - SDValue N0 = N->getOperand(0); - SDValue N1 = N->getOperand(1); - - // LHS needs to be a single use CTPOP. - if (N0.getOpcode() != ISD::CTPOP || !N0.hasOneUse()) - return SDValue(); - - // RHS needs to be 1. - if (!isOneConstant(N1)) - return SDValue(); - - SDLoc DL(N); - SDValue X = N0.getOperand(0); - - // If this is 64-bit, its always best to xor the two 32-bit pieces together - // even if we have popcnt. - if (VT == MVT::i64) { - SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, - DAG.getNode(ISD::SRL, DL, VT, X, - DAG.getConstant(32, DL, MVT::i8))); - SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X); - X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi); - // Generate a 32-bit parity idiom. This will bring us back here if we need - // to expand it too. - SDValue Parity = DAG.getNode(ISD::AND, DL, MVT::i32, - DAG.getNode(ISD::CTPOP, DL, MVT::i32, X), - DAG.getConstant(1, DL, MVT::i32)); - return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Parity); - } - assert(VT == MVT::i32 && "Unexpected VT!"); - - // Xor the high and low 16-bits together using a 32-bit operation. - SDValue Hi16 = DAG.getNode(ISD::SRL, DL, VT, X, - DAG.getConstant(16, DL, MVT::i8)); - X = DAG.getNode(ISD::XOR, DL, VT, X, Hi16); - - // Finally xor the low 2 bytes together and use a 8-bit flag setting xor. - // This should allow an h-reg to be used to save a shift. - // FIXME: We only get an h-reg in 32-bit mode. - SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, - DAG.getNode(ISD::SRL, DL, VT, X, - DAG.getConstant(8, DL, MVT::i8))); - SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X); - SDVTList VTs = DAG.getVTList(MVT::i8, MVT::i32); - SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1); - - // Copy the inverse of the parity flag into a register with setcc. - SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG); - // Zero extend to original type. - return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0), Setnp); -} - - // Look for (and (bitcast (vXi1 (concat_vectors (vYi1 setcc), undef,))), C) // Where C is a mask containing the same number of bits as the setcc and // where the setcc will freely 0 upper bits of k-register. We can replace the @@ -42865,10 +44061,6 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG, } } - // This must be done before legalization has expanded the ctpop. - if (SDValue V = combineParity(N, DAG, Subtarget)) - return V; - // Match all-of bool scalar reductions into a bitcast/movmsk + cmp. // TODO: Support multiple SrcOps. if (VT == MVT::i1) { @@ -42938,7 +44130,7 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG, if (VT == SrcVecVT.getScalarType() && N->getOperand(0)->isOnlyUserOf(SrcVec.getNode()) && getTargetConstantBitsFromNode(BitMask, 8, UndefElts, EltBits) && - llvm::all_of(EltBits, [](APInt M) { + llvm::all_of(EltBits, [](const APInt &M) { return M.isNullValue() || M.isAllOnesValue(); })) { unsigned NumElts = SrcVecVT.getVectorNumElements(); @@ -42957,6 +44149,7 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG, if (SDValue Shuffle = combineX86ShufflesRecursively( {SrcVec}, 0, SrcVec, ShuffleMask, {}, /*Depth*/ 1, + X86::MaxShuffleCombineDepth, /*HasVarMask*/ false, /*AllowVarMask*/ true, DAG, Subtarget)) return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), VT, Shuffle, N->getOperand(0).getOperand(1)); @@ -43624,14 +44817,13 @@ static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG, unsigned NumElems = VT.getVectorNumElements(); EVT ScalarVT = VT.getVectorElementType(); - if (!((ScalarVT == MVT::i8 || ScalarVT == MVT::i16) && - NumElems >= 2 && isPowerOf2_32(NumElems))) + if (!((ScalarVT == MVT::i8 || ScalarVT == MVT::i16) && NumElems >= 2)) return SDValue(); // InScalarVT is the intermediate type in AVG pattern and it should be greater // than the original input type (i8/i16). EVT InScalarVT = InVT.getVectorElementType(); - if (InScalarVT.getSizeInBits() <= ScalarVT.getSizeInBits()) + if (InScalarVT.getFixedSizeInBits() <= ScalarVT.getFixedSizeInBits()) return SDValue(); if (!Subtarget.hasSSE2()) @@ -43659,8 +44851,8 @@ static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG, }; // Check if each element of the vector is right-shifted by one. - auto LHS = In.getOperand(0); - auto RHS = In.getOperand(1); + SDValue LHS = In.getOperand(0); + SDValue RHS = In.getOperand(1); if (!IsConstVectorInRange(RHS, 1, 1)) return SDValue(); if (LHS.getOpcode() != ISD::ADD) @@ -43676,6 +44868,29 @@ static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG, return DAG.getNode(X86ISD::AVG, DL, Ops[0].getValueType(), Ops); }; + auto AVGSplitter = [&](SDValue Op0, SDValue Op1) { + // Pad to a power-of-2 vector, split+apply and extract the original vector. + unsigned NumElemsPow2 = PowerOf2Ceil(NumElems); + EVT Pow2VT = EVT::getVectorVT(*DAG.getContext(), ScalarVT, NumElemsPow2); + if (NumElemsPow2 != NumElems) { + SmallVector<SDValue, 32> Ops0(NumElemsPow2, DAG.getUNDEF(ScalarVT)); + SmallVector<SDValue, 32> Ops1(NumElemsPow2, DAG.getUNDEF(ScalarVT)); + for (unsigned i = 0; i != NumElems; ++i) { + SDValue Idx = DAG.getIntPtrConstant(i, DL); + Ops0[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Op0, Idx); + Ops1[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Op1, Idx); + } + Op0 = DAG.getBuildVector(Pow2VT, DL, Ops0); + Op1 = DAG.getBuildVector(Pow2VT, DL, Ops1); + } + SDValue Res = + SplitOpsAndApply(DAG, Subtarget, DL, Pow2VT, {Op0, Op1}, AVGBuilder); + if (NumElemsPow2 == NumElems) + return Res; + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, + DAG.getIntPtrConstant(0, DL)); + }; + // Take care of the case when one of the operands is a constant vector whose // element is in the range [1, 256]. if (IsConstVectorInRange(Operands[1], 1, ScalarVT == MVT::i8 ? 256 : 65536) && @@ -43686,9 +44901,7 @@ static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG, SDValue VecOnes = DAG.getConstant(1, DL, InVT); Operands[1] = DAG.getNode(ISD::SUB, DL, InVT, Operands[1], VecOnes); Operands[1] = DAG.getNode(ISD::TRUNCATE, DL, VT, Operands[1]); - return SplitOpsAndApply(DAG, Subtarget, DL, VT, - { Operands[0].getOperand(0), Operands[1] }, - AVGBuilder); + return AVGSplitter(Operands[0].getOperand(0), Operands[1]); } // Matches 'add like' patterns: add(Op0,Op1) + zext(or(Op0,Op1)). @@ -43735,8 +44948,7 @@ static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG, } // The pattern is detected, emit X86ISD::AVG instruction(s). - return SplitOpsAndApply(DAG, Subtarget, DL, VT, {Operands[0], Operands[1]}, - AVGBuilder); + return AVGSplitter(Operands[0], Operands[1]); } return SDValue(); @@ -43769,7 +44981,8 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, unsigned HalfOffset = 16; SDValue Ptr1 = Ld->getBasePtr(); - SDValue Ptr2 = DAG.getMemBasePlusOffset(Ptr1, HalfOffset, dl); + SDValue Ptr2 = + DAG.getMemBasePlusOffset(Ptr1, TypeSize::Fixed(HalfOffset), dl); EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), NumElems / 2); SDValue Load1 = @@ -43803,6 +45016,29 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, } } + // If we also broadcast this as a subvector to a wider type, then just extract + // the lowest subvector. + if (Ext == ISD::NON_EXTLOAD && Subtarget.hasAVX() && Ld->isSimple() && + (RegVT.is128BitVector() || RegVT.is256BitVector())) { + SDValue Ptr = Ld->getBasePtr(); + SDValue Chain = Ld->getChain(); + for (SDNode *User : Ptr->uses()) { + if (User != N && User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD && + cast<MemIntrinsicSDNode>(User)->getBasePtr() == Ptr && + cast<MemIntrinsicSDNode>(User)->getChain() == Chain && + cast<MemIntrinsicSDNode>(User)->getMemoryVT().getSizeInBits() == + MemVT.getSizeInBits() && + !User->hasAnyUseOfValue(1) && + User->getValueSizeInBits(0).getFixedSize() > + RegVT.getFixedSizeInBits()) { + SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N), + RegVT.getSizeInBits()); + Extract = DAG.getBitcast(RegVT, Extract); + return DCI.CombineTo(N, Extract, SDValue(User, 1)); + } + } + } + // Cast ptr32 and ptr64 pointers to the default address space before a load. unsigned AddrSpace = Ld->getAddressSpace(); if (AddrSpace == X86AS::PTR64 || AddrSpace == X86AS::PTR32_SPTR || @@ -43844,7 +45080,7 @@ static int getOneTrueElt(SDValue V) { auto *ConstNode = dyn_cast<ConstantSDNode>(Op); if (!ConstNode) return -1; - if (ConstNode->getAPIntValue().isAllOnesValue()) { + if (ConstNode->getAPIntValue().countTrailingOnes() >= 1) { // If we already found a one, this is too many. if (TrueIndex >= 0) return -1; @@ -43860,7 +45096,8 @@ static int getOneTrueElt(SDValue V) { /// scalar element, and the alignment for the scalar memory access. static bool getParamsForOneTrueMaskedElt(MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, - SDValue &Index, unsigned &Alignment) { + SDValue &Index, Align &Alignment, + unsigned &Offset) { int TrueMaskElt = getOneTrueElt(MaskedOp->getMask()); if (TrueMaskElt < 0) return false; @@ -43868,14 +45105,17 @@ static bool getParamsForOneTrueMaskedElt(MaskedLoadStoreSDNode *MaskedOp, // Get the address of the one scalar element that is specified by the mask // using the appropriate offset from the base pointer. EVT EltVT = MaskedOp->getMemoryVT().getVectorElementType(); + Offset = 0; Addr = MaskedOp->getBasePtr(); if (TrueMaskElt != 0) { - unsigned Offset = TrueMaskElt * EltVT.getStoreSize(); - Addr = DAG.getMemBasePlusOffset(Addr, Offset, SDLoc(MaskedOp)); + Offset = TrueMaskElt * EltVT.getStoreSize(); + Addr = DAG.getMemBasePlusOffset(Addr, TypeSize::Fixed(Offset), + SDLoc(MaskedOp)); } Index = DAG.getIntPtrConstant(TrueMaskElt, SDLoc(MaskedOp)); - Alignment = MinAlign(MaskedOp->getAlignment(), EltVT.getStoreSize()); + Alignment = commonAlignment(MaskedOp->getOriginalAlign(), + EltVT.getStoreSize()); return true; } @@ -43885,15 +45125,17 @@ static bool getParamsForOneTrueMaskedElt(MaskedLoadStoreSDNode *MaskedOp, /// mask have already been optimized in IR, so we don't bother with those here. static SDValue reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG, - TargetLowering::DAGCombinerInfo &DCI) { + TargetLowering::DAGCombinerInfo &DCI, + const X86Subtarget &Subtarget) { assert(ML->isUnindexed() && "Unexpected indexed masked load!"); // TODO: This is not x86-specific, so it could be lifted to DAGCombiner. // However, some target hooks may need to be added to know when the transform // is profitable. Endianness would also have to be considered. SDValue Addr, VecIndex; - unsigned Alignment; - if (!getParamsForOneTrueMaskedElt(ML, DAG, Addr, VecIndex, Alignment)) + Align Alignment; + unsigned Offset; + if (!getParamsForOneTrueMaskedElt(ML, DAG, Addr, VecIndex, Alignment, Offset)) return SDValue(); // Load the one scalar element that is specified by the mask using the @@ -43901,13 +45143,25 @@ reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG, SDLoc DL(ML); EVT VT = ML->getValueType(0); EVT EltVT = VT.getVectorElementType(); + + EVT CastVT = VT; + if (EltVT == MVT::i64 && !Subtarget.is64Bit()) { + EltVT = MVT::f64; + CastVT = + EVT::getVectorVT(*DAG.getContext(), EltVT, VT.getVectorNumElements()); + } + SDValue Load = - DAG.getLoad(EltVT, DL, ML->getChain(), Addr, ML->getPointerInfo(), + DAG.getLoad(EltVT, DL, ML->getChain(), Addr, + ML->getPointerInfo().getWithOffset(Offset), Alignment, ML->getMemOperand()->getFlags()); + SDValue PassThru = DAG.getBitcast(CastVT, ML->getPassThru()); + // Insert the loaded element into the appropriate place in the vector. - SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, - ML->getPassThru(), Load, VecIndex); + SDValue Insert = + DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, CastVT, PassThru, Load, VecIndex); + Insert = DAG.getBitcast(VT, Insert); return DCI.CombineTo(ML, Insert, Load.getValue(1), true); } @@ -43970,7 +45224,8 @@ static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG, return SDValue(); if (Mld->getExtensionType() == ISD::NON_EXTLOAD) { - if (SDValue ScalarLoad = reduceMaskedLoadToScalarLoad(Mld, DAG, DCI)) + if (SDValue ScalarLoad = + reduceMaskedLoadToScalarLoad(Mld, DAG, DCI, Subtarget)) return ScalarLoad; // TODO: Do some AVX512 subsets benefit from this transform? @@ -44007,25 +45262,35 @@ static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG, /// Note: It is expected that the degenerate cases of an all-zeros or all-ones /// mask have already been optimized in IR, so we don't bother with those here. static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS, - SelectionDAG &DAG) { + SelectionDAG &DAG, + const X86Subtarget &Subtarget) { // TODO: This is not x86-specific, so it could be lifted to DAGCombiner. // However, some target hooks may need to be added to know when the transform // is profitable. Endianness would also have to be considered. SDValue Addr, VecIndex; - unsigned Alignment; - if (!getParamsForOneTrueMaskedElt(MS, DAG, Addr, VecIndex, Alignment)) + Align Alignment; + unsigned Offset; + if (!getParamsForOneTrueMaskedElt(MS, DAG, Addr, VecIndex, Alignment, Offset)) return SDValue(); // Extract the one scalar element that is actually being stored. SDLoc DL(MS); - EVT VT = MS->getValue().getValueType(); + SDValue Value = MS->getValue(); + EVT VT = Value.getValueType(); EVT EltVT = VT.getVectorElementType(); - SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, - MS->getValue(), VecIndex); + if (EltVT == MVT::i64 && !Subtarget.is64Bit()) { + EltVT = MVT::f64; + EVT CastVT = + EVT::getVectorVT(*DAG.getContext(), EltVT, VT.getVectorNumElements()); + Value = DAG.getBitcast(CastVT, Value); + } + SDValue Extract = + DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Value, VecIndex); // Store that element at the appropriate offset from the base pointer. - return DAG.getStore(MS->getChain(), DL, Extract, Addr, MS->getPointerInfo(), + return DAG.getStore(MS->getChain(), DL, Extract, Addr, + MS->getPointerInfo().getWithOffset(Offset), Alignment, MS->getMemOperand()->getFlags()); } @@ -44043,7 +45308,7 @@ static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG, if (Mst->isTruncatingStore()) return SDValue(); - if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG)) + if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG, Subtarget)) return ScalarStore; // If the mask value has been legalized to a non-boolean vector, try to @@ -44104,17 +45369,21 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG, if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() && StoredVal.getOpcode() == ISD::SCALAR_TO_VECTOR && StoredVal.getOperand(0).getValueType() == MVT::i8) { - return DAG.getStore(St->getChain(), dl, StoredVal.getOperand(0), + SDValue Val = StoredVal.getOperand(0); + // We must store zeros to the unused bits. + Val = DAG.getZeroExtendInReg(Val, dl, MVT::i1); + return DAG.getStore(St->getChain(), dl, Val, St->getBasePtr(), St->getPointerInfo(), St->getOriginalAlign(), St->getMemOperand()->getFlags()); } // Widen v2i1/v4i1 stores to v8i1. - if ((VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT && + if ((VT == MVT::v1i1 || VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT && Subtarget.hasAVX512()) { unsigned NumConcats = 8 / VT.getVectorNumElements(); - SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(VT)); + // We must store zeros to the unused bits. + SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, VT)); Ops[0] = StoredVal; StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops); return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(), @@ -44136,7 +45405,7 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG, Hi = combinevXi1ConstantToInteger(Hi, DAG); SDValue Ptr0 = St->getBasePtr(); - SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, 4, dl); + SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, TypeSize::Fixed(4), dl); SDValue Ch0 = DAG.getStore(St->getChain(), dl, Lo, Ptr0, St->getPointerInfo(), @@ -44215,6 +45484,36 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG, VT, St->getMemOperand(), DAG); } + // Try to fold a extract_element(VTRUNC) pattern into a truncating store. + if (!St->isTruncatingStore() && StoredVal.hasOneUse()) { + auto IsExtractedElement = [](SDValue V) { + if (V.getOpcode() == ISD::TRUNCATE && V.getOperand(0).hasOneUse()) + V = V.getOperand(0); + unsigned Opc = V.getOpcode(); + if (Opc == ISD::EXTRACT_VECTOR_ELT || Opc == X86ISD::PEXTRW) { + if (V.getOperand(0).hasOneUse() && isNullConstant(V.getOperand(1))) + return V.getOperand(0); + } + return SDValue(); + }; + if (SDValue Extract = IsExtractedElement(StoredVal)) { + SDValue Trunc = peekThroughOneUseBitcasts(Extract); + if (Trunc.getOpcode() == X86ISD::VTRUNC) { + SDValue Src = Trunc.getOperand(0); + MVT DstVT = Trunc.getSimpleValueType(); + MVT SrcVT = Src.getSimpleValueType(); + unsigned NumSrcElts = SrcVT.getVectorNumElements(); + unsigned NumTruncBits = DstVT.getScalarSizeInBits() * NumSrcElts; + MVT TruncVT = MVT::getVectorVT(DstVT.getScalarType(), NumSrcElts); + if (NumTruncBits == VT.getSizeInBits() && + TLI.isTruncStoreLegal(SrcVT, TruncVT)) { + return DAG.getTruncStore(St->getChain(), dl, Src, St->getBasePtr(), + TruncVT, St->getMemOperand()); + } + } + } + } + // Optimize trunc store (of multiple scalars) to shuffle and store. // First, pack all of the elements in one place. Next, store to memory // in fewer chunks. @@ -44357,9 +45656,10 @@ static SDValue combineVEXTRACT_STORE(SDNode *N, SelectionDAG &DAG, /// In short, LHS and RHS are inspected to see if LHS op RHS is of the form /// A horizontal-op B, for some already available A and B, and if so then LHS is /// set to A, RHS to B, and the routine returns 'true'. -static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, - const X86Subtarget &Subtarget, - bool IsCommutative) { +static bool isHorizontalBinOp(unsigned HOpcode, SDValue &LHS, SDValue &RHS, + SelectionDAG &DAG, const X86Subtarget &Subtarget, + bool IsCommutative, + SmallVectorImpl<int> &PostShuffleMask) { // If either operand is undef, bail out. The binop should be simplified. if (LHS.isUndef() || RHS.isUndef()) return false; @@ -44462,6 +45762,9 @@ static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, if (!(A == C && B == D)) return false; + PostShuffleMask.clear(); + PostShuffleMask.append(NumElts, SM_SentinelUndef); + // LHS and RHS are now: // LHS = shuffle A, B, LMask // RHS = shuffle A, B, RMask @@ -44470,6 +45773,7 @@ static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, // so we just repeat the inner loop if this is a 256-bit op. unsigned Num128BitChunks = VT.getSizeInBits() / 128; unsigned NumEltsPer128BitChunk = NumElts / Num128BitChunks; + unsigned NumEltsPer64BitChunk = NumEltsPer128BitChunk / 2; assert((NumEltsPer128BitChunk % 2 == 0) && "Vector type should have an even number of elements in each lane"); for (unsigned j = 0; j != NumElts; j += NumEltsPer128BitChunk) { @@ -44481,29 +45785,60 @@ static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts))) continue; + // Check that successive odd/even elements are being operated on. If not, + // this is not a horizontal operation. + if (!((RIdx & 1) == 1 && (LIdx + 1) == RIdx) && + !((LIdx & 1) == 1 && (RIdx + 1) == LIdx && IsCommutative)) + return false; + + // Compute the post-shuffle mask index based on where the element + // is stored in the HOP result, and where it needs to be moved to. + int Base = LIdx & ~1u; + int Index = ((Base % NumEltsPer128BitChunk) / 2) + + ((Base % NumElts) & ~(NumEltsPer128BitChunk - 1)); + // The low half of the 128-bit result must choose from A. // The high half of the 128-bit result must choose from B, // unless B is undef. In that case, we are always choosing from A. - unsigned NumEltsPer64BitChunk = NumEltsPer128BitChunk / 2; - unsigned Src = B.getNode() ? i >= NumEltsPer64BitChunk : 0; - - // Check that successive elements are being operated on. If not, this is - // not a horizontal operation. - int Index = 2 * (i % NumEltsPer64BitChunk) + NumElts * Src + j; - if (!(LIdx == Index && RIdx == Index + 1) && - !(IsCommutative && LIdx == Index + 1 && RIdx == Index)) - return false; + if ((B && Base >= (int)NumElts) || (!B && i >= NumEltsPer64BitChunk)) + Index += NumEltsPer64BitChunk; + PostShuffleMask[i + j] = Index; } } - LHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it. - RHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it. + SDValue NewLHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it. + SDValue NewRHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it. - if (!shouldUseHorizontalOp(LHS == RHS && NumShuffles < 2, DAG, Subtarget)) + bool IsIdentityPostShuffle = + isSequentialOrUndefInRange(PostShuffleMask, 0, NumElts, 0); + if (IsIdentityPostShuffle) + PostShuffleMask.clear(); + + // Avoid 128-bit multi lane shuffles if pre-AVX2 and FP (integer will split). + if (!IsIdentityPostShuffle && !Subtarget.hasAVX2() && VT.isFloatingPoint() && + isMultiLaneShuffleMask(128, VT.getScalarSizeInBits(), PostShuffleMask)) + return false; + + // If the source nodes are already used in HorizOps then always accept this. + // Shuffle folding should merge these back together. + bool FoundHorizLHS = llvm::any_of(NewLHS->uses(), [&](SDNode *User) { + return User->getOpcode() == HOpcode && User->getValueType(0) == VT; + }); + bool FoundHorizRHS = llvm::any_of(NewRHS->uses(), [&](SDNode *User) { + return User->getOpcode() == HOpcode && User->getValueType(0) == VT; + }); + bool ForceHorizOp = FoundHorizLHS && FoundHorizRHS; + + // Assume a SingleSource HOP if we only shuffle one input and don't need to + // shuffle the result. + if (!ForceHorizOp && + !shouldUseHorizontalOp(NewLHS == NewRHS && + (NumShuffles < 2 || !IsIdentityPostShuffle), + DAG, Subtarget)) return false; - LHS = DAG.getBitcast(VT, LHS); - RHS = DAG.getBitcast(VT, RHS); + LHS = DAG.getBitcast(VT, NewLHS); + RHS = DAG.getBitcast(VT, NewRHS); return true; } @@ -44518,10 +45853,17 @@ static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG, assert((IsFadd || N->getOpcode() == ISD::FSUB) && "Wrong opcode"); // Try to synthesize horizontal add/sub from adds/subs of shuffles. + SmallVector<int, 8> PostShuffleMask; if (((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && - isHorizontalBinOp(LHS, RHS, DAG, Subtarget, IsFadd)) - return DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS); + isHorizontalBinOp(HorizOpcode, LHS, RHS, DAG, Subtarget, IsFadd, + PostShuffleMask)) { + SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS); + if (!PostShuffleMask.empty()) + HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp, + DAG.getUNDEF(VT), PostShuffleMask); + return HorizBinOp; + } return SDValue(); } @@ -44660,7 +46002,7 @@ static SDValue combineVectorTruncation(SDNode *N, SelectionDAG &DAG, EVT OutSVT = OutVT.getVectorElementType(); EVT InSVT = InVT.getVectorElementType(); - if (!((InSVT == MVT::i32 || InSVT == MVT::i64) && + if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) && (OutSVT == MVT::i8 || OutSVT == MVT::i16) && isPowerOf2_32(NumElems) && NumElems >= 8)) return SDValue(); @@ -44722,8 +46064,13 @@ static SDValue combineVectorSignBitsTruncation(SDNode *N, const SDLoc &DL, // there's no harm in trying pack. if (Subtarget.hasAVX512() && !(!Subtarget.useAVX512Regs() && VT.is256BitVector() && - InVT.is512BitVector())) + InVT.is512BitVector())) { + // PACK should still be worth it for 128-bit vectors if the sources were + // originally concatenated from subvectors. + SmallVector<SDValue> ConcatOps; + if (VT.getSizeInBits() > 128 || !collectConcatOps(In.getNode(), ConcatOps)) return SDValue(); + } unsigned NumPackedSignBits = std::min<unsigned>(SVT.getSizeInBits(), 16); unsigned NumPackedZeroBits = Subtarget.hasSSE41() ? NumPackedSignBits : 8; @@ -44745,9 +46092,23 @@ static SDValue combineVectorSignBitsTruncation(SDNode *N, const SDLoc &DL, if (SVT == MVT::i32 && NumSignBits != InSVT.getSizeInBits()) return SDValue(); - if (NumSignBits > (InSVT.getSizeInBits() - NumPackedSignBits)) + unsigned MinSignBits = InSVT.getSizeInBits() - NumPackedSignBits; + if (NumSignBits > MinSignBits) return truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget); + // If we have a srl that only generates signbits that we will discard in + // the truncation then we can use PACKSS by converting the srl to a sra. + // SimplifyDemandedBits often relaxes sra to srl so we need to reverse it. + if (In.getOpcode() == ISD::SRL && N->isOnlyUserOf(In.getNode())) + if (const APInt *ShAmt = DAG.getValidShiftAmountConstant( + In, APInt::getAllOnesValue(VT.getVectorNumElements()))) { + if (*ShAmt == MinSignBits) { + SDValue NewIn = DAG.getNode(ISD::SRA, DL, InVT, In->ops()); + return truncateVectorWithPACK(X86ISD::PACKSS, VT, NewIn, DL, DAG, + Subtarget); + } + } + return SDValue(); } @@ -46065,7 +47426,6 @@ static SDValue combineSext(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) { SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); - EVT InVT = N0.getValueType(); SDLoc DL(N); // (i32 (sext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry)) @@ -46094,23 +47454,17 @@ static SDValue combineSext(SDNode *N, SelectionDAG &DAG, if (SDValue V = combineExtSetcc(N, DAG, Subtarget)) return V; - if (InVT == MVT::i1 && N0.getOpcode() == ISD::XOR && - isAllOnesConstant(N0.getOperand(1)) && N0.hasOneUse()) { - // Invert and sign-extend a boolean is the same as zero-extend and subtract - // 1 because 0 becomes -1 and 1 becomes 0. The subtract is efficiently - // lowered with an LEA or a DEC. This is the same as: select Bool, 0, -1. - // sext (xor Bool, -1) --> sub (zext Bool), 1 - SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); - return DAG.getNode(ISD::SUB, DL, VT, Zext, DAG.getConstant(1, DL, VT)); - } - if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget)) return V; - if (VT.isVector()) + if (VT.isVector()) { if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget)) return R; + if (N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) + return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0)); + } + if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget)) return NewAdd; @@ -46129,14 +47483,23 @@ static SDValue combineFMA(SDNode *N, SelectionDAG &DAG, if (!TLI.isTypeLegal(VT)) return SDValue(); - EVT ScalarVT = VT.getScalarType(); - if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) || !Subtarget.hasAnyFMA()) - return SDValue(); - SDValue A = N->getOperand(IsStrict ? 1 : 0); SDValue B = N->getOperand(IsStrict ? 2 : 1); SDValue C = N->getOperand(IsStrict ? 3 : 2); + // If the operation allows fast-math and the target does not support FMA, + // split this into mul+add to avoid libcall(s). + SDNodeFlags Flags = N->getFlags(); + if (!IsStrict && Flags.hasAllowReassociation() && + TLI.isOperationExpand(ISD::FMA, VT)) { + SDValue Fmul = DAG.getNode(ISD::FMUL, dl, VT, A, B, Flags); + return DAG.getNode(ISD::FADD, dl, VT, Fmul, C, Flags); + } + + EVT ScalarVT = VT.getScalarType(); + if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) || !Subtarget.hasAnyFMA()) + return SDValue(); + auto invertIfNegative = [&DAG, &TLI, &DCI](SDValue &V) { bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize(); bool LegalOperations = !DCI.isBeforeLegalizeOps(); @@ -46559,7 +47922,7 @@ static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG, Src.getOperand(0).getScalarValueSizeInBits() == EltWidth) return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0)); - // Fold movmsk(not(x)) -> not(movmsk) to improve folding of movmsk results + // Fold movmsk(not(x)) -> not(movmsk(x)) to improve folding of movmsk results // with scalar comparisons. if (SDValue NotSrc = IsNOT(Src, DAG)) { SDLoc DL(N); @@ -46570,6 +47933,17 @@ static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG, DAG.getConstant(NotMask, DL, VT)); } + // Fold movmsk(icmp_sgt(x,-1)) -> not(movmsk(x)) to improve folding of movmsk + // results with scalar comparisons. + if (Src.getOpcode() == X86ISD::PCMPGT && + ISD::isBuildVectorAllOnes(Src.getOperand(1).getNode())) { + SDLoc DL(N); + APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts); + return DAG.getNode(ISD::XOR, DL, VT, + DAG.getNode(X86ISD::MOVMSK, DL, VT, Src.getOperand(0)), + DAG.getConstant(NotMask, DL, VT)); + } + // Simplify the inputs. const TargetLowering &TLI = DAG.getTargetLoweringInfo(); APInt DemandedMask(APInt::getAllOnesValue(NumBits)); @@ -46607,7 +47981,8 @@ static SDValue rebuildGatherScatter(MaskedGatherScatterSDNode *GorS, return DAG.getMaskedGather(Gather->getVTList(), Gather->getMemoryVT(), DL, Ops, Gather->getMemOperand(), - Gather->getIndexType()); + Gather->getIndexType(), + Gather->getExtensionType()); } auto *Scatter = cast<MaskedScatterSDNode>(GorS); SDValue Ops[] = { Scatter->getChain(), Scatter->getValue(), @@ -46615,7 +47990,8 @@ static SDValue rebuildGatherScatter(MaskedGatherScatterSDNode *GorS, return DAG.getMaskedScatter(Scatter->getVTList(), Scatter->getMemoryVT(), DL, Ops, Scatter->getMemOperand(), - Scatter->getIndexType()); + Scatter->getIndexType(), + Scatter->isTruncatingStore()); } static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG, @@ -47604,6 +48980,36 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1, PMADDBuilder); } +static SDValue combineAddOrSubToHADDorHSUB(SDNode *N, SelectionDAG &DAG, + const X86Subtarget &Subtarget) { + EVT VT = N->getValueType(0); + SDValue Op0 = N->getOperand(0); + SDValue Op1 = N->getOperand(1); + bool IsAdd = N->getOpcode() == ISD::ADD; + auto HorizOpcode = IsAdd ? X86ISD::HADD : X86ISD::HSUB; + assert((IsAdd || N->getOpcode() == ISD::SUB) && "Wrong opcode"); + + SmallVector<int, 8> PostShuffleMask; + if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 || + VT == MVT::v8i32) && + Subtarget.hasSSSE3() && + isHorizontalBinOp(HorizOpcode, Op0, Op1, DAG, Subtarget, IsAdd, + PostShuffleMask)) { + auto HOpBuilder = [HorizOpcode](SelectionDAG &DAG, const SDLoc &DL, + ArrayRef<SDValue> Ops) { + return DAG.getNode(HorizOpcode, DL, Ops[0].getValueType(), Ops); + }; + SDValue HorizBinOp = + SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {Op0, Op1}, HOpBuilder); + if (!PostShuffleMask.empty()) + HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp, + DAG.getUNDEF(VT), PostShuffleMask); + return HorizBinOp; + } + + return SDValue(); +} + static SDValue combineAdd(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) { @@ -47617,17 +49023,8 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG, return MAdd; // Try to synthesize horizontal adds from adds of shuffles. - if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 || - VT == MVT::v8i32) && - Subtarget.hasSSSE3() && - isHorizontalBinOp(Op0, Op1, DAG, Subtarget, true)) { - auto HADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL, - ArrayRef<SDValue> Ops) { - return DAG.getNode(X86ISD::HADD, DL, Ops[0].getValueType(), Ops); - }; - return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {Op0, Op1}, - HADDBuilder); - } + if (SDValue V = combineAddOrSubToHADDorHSUB(N, DAG, Subtarget)) + return V; // If vectors of i1 are legal, turn (add (zext (vXi1 X)), Y) into // (sub Y, (sext (vXi1 X))). @@ -47665,12 +49062,11 @@ static SDValue combineSubToSubus(SDNode *N, SelectionDAG &DAG, if (!VT.isVector()) return SDValue(); - // PSUBUS is supported, starting from SSE2, but truncation for v8i32 - // is only worth it with SSSE3 (PSHUFB). + // PSUBUS is supported, starting from SSE2. EVT EltVT = VT.getVectorElementType(); - if (!(Subtarget.hasSSE2() && (EltVT == MVT::i8 || EltVT == MVT::i16)) && - !(Subtarget.hasSSSE3() && (VT == MVT::v8i32 || VT == MVT::v8i64)) && - !(Subtarget.useBWIRegs() && (VT == MVT::v16i32))) + if (!(Subtarget.hasSSE2() && + (EltVT == MVT::i8 || EltVT == MVT::i16 || VT == MVT::v8i32 || + VT == MVT::v8i64 || VT == MVT::v16i32))) return SDValue(); SDValue SubusLHS, SubusRHS; @@ -47706,9 +49102,9 @@ static SDValue combineSubToSubus(SDNode *N, SelectionDAG &DAG, SDValue MinLHS = Op1.getOperand(0).getOperand(0); SDValue MinRHS = Op1.getOperand(0).getOperand(1); EVT TruncVT = Op1.getOperand(0).getValueType(); - if (!(Subtarget.hasSSSE3() && (TruncVT == MVT::v8i32 || - TruncVT == MVT::v8i64)) && - !(Subtarget.useBWIRegs() && (TruncVT == MVT::v16i32))) + if (!(Subtarget.hasSSE2() && + (TruncVT == MVT::v8i32 || TruncVT == MVT::v8i64 || + TruncVT == MVT::v16i32))) return SDValue(); SDValue OpToSaturate; if (MinLHS.getOpcode() == ISD::ZERO_EXTEND && @@ -47746,7 +49142,7 @@ static SDValue combineSubToSubus(SDNode *N, SelectionDAG &DAG, // values, or first 48 bits for 64 bit values. KnownBits Known = DAG.computeKnownBits(SubusLHS); unsigned NumZeros = Known.countMinLeadingZeros(); - if ((VT == MVT::v8i64 && NumZeros < 48) || NumZeros < 16) + if (NumZeros < (VT.getScalarSizeInBits() - 16)) return SDValue(); EVT ExtType = SubusLHS.getValueType(); @@ -47800,18 +49196,8 @@ static SDValue combineSub(SDNode *N, SelectionDAG &DAG, } // Try to synthesize horizontal subs from subs of shuffles. - EVT VT = N->getValueType(0); - if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 || - VT == MVT::v8i32) && - Subtarget.hasSSSE3() && - isHorizontalBinOp(Op0, Op1, DAG, Subtarget, false)) { - auto HSUBBuilder = [](SelectionDAG &DAG, const SDLoc &DL, - ArrayRef<SDValue> Ops) { - return DAG.getNode(X86ISD::HSUB, DL, Ops[0].getValueType(), Ops); - }; - return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {Op0, Op1}, - HSUBBuilder); - } + if (SDValue V = combineAddOrSubToHADDorHSUB(N, DAG, Subtarget)) + return V; // Try to create PSUBUS if SUB's argument is max/min if (SDValue V = combineSubToSubus(N, DAG, Subtarget)) @@ -47856,43 +49242,47 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, SDValue Op0 = Ops[0]; bool IsSplat = llvm::all_of(Ops, [&Op0](SDValue Op) { return Op == Op0; }); - // Fold subvector loads into one. - // If needed, look through bitcasts to get to the load. - if (auto *FirstLd = dyn_cast<LoadSDNode>(peekThroughBitcasts(Op0))) { - bool Fast; - const X86TargetLowering *TLI = Subtarget.getTargetLowering(); - if (TLI->allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, - *FirstLd->getMemOperand(), &Fast) && - Fast) { - if (SDValue Ld = - EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false)) - return Ld; - } - } - // Repeated subvectors. - if (IsSplat) { - // If this broadcast/subv_broadcast is inserted into both halves, use a - // larger broadcast/subv_broadcast. - if (Op0.getOpcode() == X86ISD::VBROADCAST || - Op0.getOpcode() == X86ISD::SUBV_BROADCAST) + if (IsSplat && + (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) { + // If this broadcast is inserted into both halves, use a larger broadcast. + if (Op0.getOpcode() == X86ISD::VBROADCAST) return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0)); - // If this broadcast_load is inserted into both halves, use a larger - // broadcast_load. Update other uses to use an extracted subvector. - if (Op0.getOpcode() == X86ISD::VBROADCAST_LOAD) { + // If this scalar/subvector broadcast_load is inserted into both halves, use + // a larger broadcast_load. Update other uses to use an extracted subvector. + if (Op0.getOpcode() == X86ISD::VBROADCAST_LOAD || + Op0.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) { auto *MemIntr = cast<MemIntrinsicSDNode>(Op0); SDVTList Tys = DAG.getVTList(VT, MVT::Other); SDValue Ops[] = {MemIntr->getChain(), MemIntr->getBasePtr()}; - SDValue BcastLd = DAG.getMemIntrinsicNode( - X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MemIntr->getMemoryVT(), - MemIntr->getMemOperand()); + SDValue BcastLd = DAG.getMemIntrinsicNode(Op0.getOpcode(), DL, Tys, Ops, + MemIntr->getMemoryVT(), + MemIntr->getMemOperand()); DAG.ReplaceAllUsesOfValueWith( Op0, extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits())); DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1)); return BcastLd; } + // If this is a simple subvector load repeated across multiple lanes, then + // broadcast the load. Update other uses to use an extracted subvector. + if (auto *Ld = dyn_cast<LoadSDNode>(Op0)) { + if (Ld->isSimple() && !Ld->isNonTemporal() && + Ld->getExtensionType() == ISD::NON_EXTLOAD) { + SDVTList Tys = DAG.getVTList(VT, MVT::Other); + SDValue Ops[] = {Ld->getChain(), Ld->getBasePtr()}; + SDValue BcastLd = + DAG.getMemIntrinsicNode(X86ISD::SUBV_BROADCAST_LOAD, DL, Tys, Ops, + Ld->getMemoryVT(), Ld->getMemOperand()); + DAG.ReplaceAllUsesOfValueWith( + Op0, + extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits())); + DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), BcastLd.getValue(1)); + return BcastLd; + } + } + // concat_vectors(movddup(x),movddup(x)) -> broadcast(x) if (Op0.getOpcode() == X86ISD::MOVDDUP && VT == MVT::v4f64 && (Subtarget.hasAVX2() || MayFoldLoad(Op0.getOperand(0)))) @@ -47970,6 +49360,38 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, return DAG.getBitcast(VT, Res); } break; + case X86ISD::VPERMV3: + if (!IsSplat && NumOps == 2 && VT.is512BitVector()) { + MVT OpVT = Op0.getSimpleValueType(); + int NumSrcElts = OpVT.getVectorNumElements(); + SmallVector<int, 64> ConcatMask; + for (unsigned i = 0; i != NumOps; ++i) { + bool IsUnary; + SmallVector<int, 64> SubMask; + SmallVector<SDValue, 2> SubOps; + if (!getTargetShuffleMask(Ops[i].getNode(), OpVT, false, SubOps, + SubMask, IsUnary)) + break; + for (int M : SubMask) { + if (0 <= M) { + M += M < NumSrcElts ? 0 : NumSrcElts; + M += i * NumSrcElts; + } + ConcatMask.push_back(M); + } + } + if (ConcatMask.size() == (NumOps * NumSrcElts)) { + SDValue Src0 = concatSubVectors(Ops[0].getOperand(0), + Ops[1].getOperand(0), DAG, DL); + SDValue Src1 = concatSubVectors(Ops[0].getOperand(2), + Ops[1].getOperand(2), DAG, DL); + MVT IntMaskSVT = MVT::getIntegerVT(VT.getScalarSizeInBits()); + MVT IntMaskVT = MVT::getVectorVT(IntMaskSVT, NumOps * NumSrcElts); + SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true); + return DAG.getNode(X86ISD::VPERMV3, DL, VT, Src0, Mask, Src1); + } + } + break; case X86ISD::VSHLI: case X86ISD::VSRAI: case X86ISD::VSRLI: @@ -48002,10 +49424,33 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, Op0.getOperand(1)); } break; + case ISD::AND: + case ISD::OR: + case ISD::XOR: + case X86ISD::ANDNP: + // TODO: Add 256-bit support. + if (!IsSplat && VT.is512BitVector()) { + SmallVector<SDValue, 2> LHS, RHS; + for (unsigned i = 0; i != NumOps; ++i) { + LHS.push_back(Ops[i].getOperand(0)); + RHS.push_back(Ops[i].getOperand(1)); + } + MVT SrcVT = Op0.getOperand(0).getSimpleValueType(); + SrcVT = MVT::getVectorVT(SrcVT.getScalarType(), + NumOps * SrcVT.getVectorNumElements()); + return DAG.getNode(Op0.getOpcode(), DL, VT, + DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, LHS), + DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, RHS)); + } + break; + case X86ISD::HADD: + case X86ISD::HSUB: + case X86ISD::FHADD: + case X86ISD::FHSUB: case X86ISD::PACKSS: case X86ISD::PACKUS: - if (!IsSplat && NumOps == 2 && VT.is256BitVector() && - Subtarget.hasInt256()) { + if (!IsSplat && VT.is256BitVector() && + (VT.isFloatingPoint() || Subtarget.hasInt256())) { SmallVector<SDValue, 2> LHS, RHS; for (unsigned i = 0; i != NumOps; ++i) { LHS.push_back(Ops[i].getOperand(0)); @@ -48040,6 +49485,20 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, } } + // Fold subvector loads into one. + // If needed, look through bitcasts to get to the load. + if (auto *FirstLd = dyn_cast<LoadSDNode>(peekThroughBitcasts(Op0))) { + bool Fast; + const X86TargetLowering *TLI = Subtarget.getTargetLowering(); + if (TLI->allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, + *FirstLd->getMemOperand(), &Fast) && + Fast) { + if (SDValue Ld = + EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false)) + return Ld; + } + } + return SDValue(); } @@ -48111,7 +49570,8 @@ static SDValue combineInsertSubvector(SDNode *N, SelectionDAG &DAG, SDValue Ins = SubVec.getOperand(0); if (isNullConstant(Ins.getOperand(2)) && ISD::isBuildVectorAllZeros(Ins.getOperand(0).getNode()) && - Ins.getOperand(1).getValueSizeInBits() <= SubVecVT.getSizeInBits()) + Ins.getOperand(1).getValueSizeInBits().getFixedSize() <= + SubVecVT.getFixedSizeInBits()) return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, getZeroVector(OpVT, Subtarget, DAG, dl), Ins.getOperand(1), N->getOperand(2)); @@ -48264,12 +49724,14 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG, unsigned IdxVal = N->getConstantOperandVal(1); SDValue InVecBC = peekThroughBitcasts(InVec); EVT InVecVT = InVec.getValueType(); + unsigned SizeInBits = VT.getSizeInBits(); + unsigned InSizeInBits = InVecVT.getSizeInBits(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (Subtarget.hasAVX() && !Subtarget.hasAVX2() && TLI.isTypeLegal(InVecVT) && - InVecVT.getSizeInBits() == 256 && InVecBC.getOpcode() == ISD::AND) { - auto isConcatenatedNot = [] (SDValue V) { + InSizeInBits == 256 && InVecBC.getOpcode() == ISD::AND) { + auto isConcatenatedNot = [](SDValue V) { V = peekThroughBitcasts(V); if (!isBitwiseNot(V)) return false; @@ -48312,53 +49774,32 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG, InVec.getOpcode() == ISD::INSERT_SUBVECTOR && IdxVal == 0 && InVec.hasOneUse() && isNullConstant(InVec.getOperand(2)) && ISD::isBuildVectorAllZeros(InVec.getOperand(0).getNode()) && - InVec.getOperand(1).getValueSizeInBits() <= VT.getSizeInBits()) { + InVec.getOperand(1).getValueSizeInBits() <= SizeInBits) { SDLoc DL(N); return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, getZeroVector(VT, Subtarget, DAG, DL), InVec.getOperand(1), InVec.getOperand(2)); } - // If we're extracting from a broadcast then we're better off just - // broadcasting to the smaller type directly, assuming this is the only use. - // As its a broadcast we don't care about the extraction index. - if (InVec.getOpcode() == X86ISD::VBROADCAST && InVec.hasOneUse() && - InVec.getOperand(0).getValueSizeInBits() <= VT.getSizeInBits()) - return DAG.getNode(X86ISD::VBROADCAST, SDLoc(N), VT, InVec.getOperand(0)); - - if (InVec.getOpcode() == X86ISD::VBROADCAST_LOAD && InVec.hasOneUse()) { - auto *MemIntr = cast<MemIntrinsicSDNode>(InVec); - if (MemIntr->getMemoryVT().getSizeInBits() <= VT.getSizeInBits()) { - SDVTList Tys = DAG.getVTList(VT, MVT::Other); - SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() }; - SDValue BcastLd = - DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops, - MemIntr->getMemoryVT(), - MemIntr->getMemOperand()); - DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1)); - return BcastLd; - } - } - // If we're extracting an upper subvector from a broadcast we should just // extract the lowest subvector instead which should allow // SimplifyDemandedVectorElts do more simplifications. if (IdxVal != 0 && (InVec.getOpcode() == X86ISD::VBROADCAST || InVec.getOpcode() == X86ISD::VBROADCAST_LOAD)) - return extractSubVector(InVec, 0, DAG, SDLoc(N), VT.getSizeInBits()); + return extractSubVector(InVec, 0, DAG, SDLoc(N), SizeInBits); - // If we're extracting a broadcasted subvector, just use the source. - if (InVec.getOpcode() == X86ISD::SUBV_BROADCAST && - InVec.getOperand(0).getValueType() == VT) - return InVec.getOperand(0); + // If we're extracting a broadcasted subvector, just use the lowest subvector. + if (IdxVal != 0 && InVec.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD && + cast<MemIntrinsicSDNode>(InVec)->getMemoryVT() == VT) + return extractSubVector(InVec, 0, DAG, SDLoc(N), SizeInBits); // Attempt to extract from the source of a shuffle vector. - if ((InVecVT.getSizeInBits() % VT.getSizeInBits()) == 0 && + if ((InSizeInBits % SizeInBits) == 0 && (IdxVal % VT.getVectorNumElements()) == 0) { SmallVector<int, 32> ShuffleMask; SmallVector<int, 32> ScaledMask; SmallVector<SDValue, 2> ShuffleInputs; - unsigned NumSubVecs = InVecVT.getSizeInBits() / VT.getSizeInBits(); + unsigned NumSubVecs = InSizeInBits / SizeInBits; // Decode the shuffle mask and scale it so its shuffling subvectors. if (getTargetShuffleInputs(InVecBC, ShuffleInputs, ShuffleMask, DAG) && scaleShuffleElements(ShuffleMask, NumSubVecs, ScaledMask)) { @@ -48368,19 +49809,19 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG, if (ScaledMask[SubVecIdx] == SM_SentinelZero) return getZeroVector(VT, Subtarget, DAG, SDLoc(N)); SDValue Src = ShuffleInputs[ScaledMask[SubVecIdx] / NumSubVecs]; - if (Src.getValueSizeInBits() == InVecVT.getSizeInBits()) { + if (Src.getValueSizeInBits() == InSizeInBits) { unsigned SrcSubVecIdx = ScaledMask[SubVecIdx] % NumSubVecs; unsigned SrcEltIdx = SrcSubVecIdx * VT.getVectorNumElements(); return extractSubVector(DAG.getBitcast(InVecVT, Src), SrcEltIdx, DAG, - SDLoc(N), VT.getSizeInBits()); + SDLoc(N), SizeInBits); } } } // If we're extracting the lowest subvector and we're the only user, // we may be able to perform this with a smaller vector width. + unsigned InOpcode = InVec.getOpcode(); if (IdxVal == 0 && InVec.hasOneUse()) { - unsigned InOpcode = InVec.getOpcode(); if (VT == MVT::v2f64 && InVecVT == MVT::v4f64) { // v2f64 CVTDQ2PD(v4i32). if (InOpcode == ISD::SINT_TO_FP && @@ -48404,10 +49845,14 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG, InOpcode == ISD::ZERO_EXTEND_VECTOR_INREG || InOpcode == ISD::SIGN_EXTEND || InOpcode == ISD::SIGN_EXTEND_VECTOR_INREG) && - VT.is128BitVector() && - InVec.getOperand(0).getSimpleValueType().is128BitVector()) { + (SizeInBits == 128 || SizeInBits == 256) && + InVec.getOperand(0).getValueSizeInBits() >= SizeInBits) { + SDLoc DL(N); + SDValue Ext = InVec.getOperand(0); + if (Ext.getValueSizeInBits() > SizeInBits) + Ext = extractSubVector(Ext, 0, DAG, DL, SizeInBits); unsigned ExtOp = getOpcode_EXTEND_VECTOR_INREG(InOpcode); - return DAG.getNode(ExtOp, SDLoc(N), VT, InVec.getOperand(0)); + return DAG.getNode(ExtOp, DL, VT, Ext); } if (InOpcode == ISD::VSELECT && InVec.getOperand(0).getValueType().is256BitVector() && @@ -48419,6 +49864,25 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG, SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128); return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2); } + if (InOpcode == ISD::TRUNCATE && Subtarget.hasVLX() && + (VT.is128BitVector() || VT.is256BitVector())) { + SDLoc DL(N); + SDValue InVecSrc = InVec.getOperand(0); + unsigned Scale = InVecSrc.getValueSizeInBits() / InSizeInBits; + SDValue Ext = extractSubVector(InVecSrc, 0, DAG, DL, Scale * SizeInBits); + return DAG.getNode(InOpcode, DL, VT, Ext); + } + } + + // Always split vXi64 logical shifts where we're extracting the upper 32-bits + // as this is very likely to fold into a shuffle/truncation. + if ((InOpcode == X86ISD::VSHLI || InOpcode == X86ISD::VSRLI) && + InVecVT.getScalarSizeInBits() == 64 && + InVec.getConstantOperandAPInt(1) == 32) { + SDLoc DL(N); + SDValue Ext = + extractSubVector(InVec.getOperand(0), IdxVal, DAG, DL, SizeInBits); + return DAG.getNode(InOpcode, DL, VT, Ext, InVec.getOperand(1)); } return SDValue(); @@ -48502,7 +49966,7 @@ static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG, // If the input is an extend_invec and the SimplifyDemandedBits call didn't // convert it to any_extend_invec, due to the LegalOperations check, do the // conversion directly to a vector shuffle manually. This exposes combine - // opportunities missed by combineExtInVec not calling + // opportunities missed by combineEXTEND_VECTOR_INREG not calling // combineX86ShufflesRecursively on SSE4.1 targets. // FIXME: This is basically a hack around several other issues related to // ANY_EXTEND_VECTOR_INREG. @@ -48530,11 +49994,13 @@ static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG, return SDValue(); } -static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG, - TargetLowering::DAGCombinerInfo &DCI, - const X86Subtarget &Subtarget) { +static SDValue combineEXTEND_VECTOR_INREG(SDNode *N, SelectionDAG &DAG, + TargetLowering::DAGCombinerInfo &DCI, + const X86Subtarget &Subtarget) { EVT VT = N->getValueType(0); SDValue In = N->getOperand(0); + unsigned Opcode = N->getOpcode(); + unsigned InOpcode = In.getOpcode(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); // Try to merge vector loads and extend_inreg to an extload. @@ -48543,7 +50009,7 @@ static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG, auto *Ld = cast<LoadSDNode>(In); if (Ld->isSimple()) { MVT SVT = In.getSimpleValueType().getVectorElementType(); - ISD::LoadExtType Ext = N->getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG + ISD::LoadExtType Ext = Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ? ISD::SEXTLOAD : ISD::ZEXTLOAD; EVT MemVT = @@ -48551,8 +50017,7 @@ static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG, if (TLI.isLoadExtLegal(Ext, VT, MemVT)) { SDValue Load = DAG.getExtLoad(Ext, SDLoc(N), VT, Ld->getChain(), Ld->getBasePtr(), - Ld->getPointerInfo(), MemVT, - Ld->getOriginalAlign(), + Ld->getPointerInfo(), MemVT, Ld->getOriginalAlign(), Ld->getMemOperand()->getFlags()); DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1)); return Load; @@ -48560,9 +50025,23 @@ static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG, } } + // Fold EXTEND_VECTOR_INREG(EXTEND_VECTOR_INREG(X)) -> EXTEND_VECTOR_INREG(X). + if (Opcode == InOpcode) + return DAG.getNode(Opcode, SDLoc(N), VT, In.getOperand(0)); + + // Fold EXTEND_VECTOR_INREG(EXTRACT_SUBVECTOR(EXTEND(X),0)) + // -> EXTEND_VECTOR_INREG(X). + // TODO: Handle non-zero subvector indices. + if (InOpcode == ISD::EXTRACT_SUBVECTOR && In.getConstantOperandVal(1) == 0 && + In.getOperand(0).getOpcode() == getOpcode_EXTEND(Opcode) && + In.getOperand(0).getOperand(0).getValueSizeInBits() == + In.getValueSizeInBits()) + return DAG.getNode(Opcode, SDLoc(N), VT, In.getOperand(0).getOperand(0)); + // Attempt to combine as a shuffle. - // TODO: SSE41 support - if (Subtarget.hasAVX() && N->getOpcode() != ISD::SIGN_EXTEND_VECTOR_INREG) { + // TODO: General ZERO_EXTEND_VECTOR_INREG support. + if (Opcode == ISD::ANY_EXTEND_VECTOR_INREG || + (Opcode == ISD::ZERO_EXTEND_VECTOR_INREG && Subtarget.hasSSE41())) { SDValue Op(N, 0); if (TLI.isTypeLegal(VT) && TLI.isTypeLegal(In.getValueType())) if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget)) @@ -48683,11 +50162,15 @@ static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG, return DAG.getNode(ISD::FP_EXTEND, dl, VT, Cvt); } -// Try to find a larger VBROADCAST_LOAD that we can extract from. Limit this to -// cases where the loads have the same input chain and the output chains are -// unused. This avoids any memory ordering issues. -static SDValue combineVBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG, - TargetLowering::DAGCombinerInfo &DCI) { +// Try to find a larger VBROADCAST_LOAD/SUBV_BROADCAST_LOAD that we can extract +// from. Limit this to cases where the loads have the same input chain and the +// output chains are unused. This avoids any memory ordering issues. +static SDValue combineBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG, + TargetLowering::DAGCombinerInfo &DCI) { + assert((N->getOpcode() == X86ISD::VBROADCAST_LOAD || + N->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) && + "Unknown broadcast load type"); + // Only do this if the chain result is unused. if (N->hasAnyUseOfValue(1)) return SDValue(); @@ -48702,13 +50185,13 @@ static SDValue combineVBROADCAST_LOAD(SDNode *N, SelectionDAG &DAG, // Look at other users of our base pointer and try to find a wider broadcast. // The input chain and the size of the memory VT must match. for (SDNode *User : Ptr->uses()) - if (User != N && User->getOpcode() == X86ISD::VBROADCAST_LOAD && + if (User != N && User->getOpcode() == N->getOpcode() && cast<MemIntrinsicSDNode>(User)->getBasePtr() == Ptr && cast<MemIntrinsicSDNode>(User)->getChain() == Chain && cast<MemIntrinsicSDNode>(User)->getMemoryVT().getSizeInBits() == MemVT.getSizeInBits() && !User->hasAnyUseOfValue(1) && - User->getValueSizeInBits(0) > VT.getSizeInBits()) { + User->getValueSizeInBits(0).getFixedSize() > VT.getFixedSizeInBits()) { SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N), VT.getSizeInBits()); Extract = DAG.getBitcast(VT, Extract); @@ -48779,6 +50262,17 @@ static SDValue combineMOVDQ2Q(SDNode *N, SelectionDAG &DAG) { return SDValue(); } +static SDValue combinePDEP(SDNode *N, SelectionDAG &DAG, + TargetLowering::DAGCombinerInfo &DCI) { + unsigned NumBits = N->getSimpleValueType(0).getSizeInBits(); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + if (TLI.SimplifyDemandedBits(SDValue(N, 0), + APInt::getAllOnesValue(NumBits), DCI)) + return SDValue(N, 0); + + return SDValue(); +} + SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; @@ -48815,7 +50309,8 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, case ISD::AND: return combineAnd(N, DAG, DCI, Subtarget); case ISD::OR: return combineOr(N, DAG, DCI, Subtarget); case ISD::XOR: return combineXor(N, DAG, DCI, Subtarget); - case X86ISD::BEXTR: return combineBEXTR(N, DAG, DCI, Subtarget); + case X86ISD::BEXTR: + case X86ISD::BEXTRI: return combineBEXTR(N, DAG, DCI, Subtarget); case ISD::LOAD: return combineLoad(N, DAG, DCI, Subtarget); case ISD::MLOAD: return combineMaskedLoad(N, DAG, DCI, Subtarget); case ISD::STORE: return combineStore(N, DAG, DCI, Subtarget); @@ -48860,13 +50355,17 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, case ISD::SIGN_EXTEND_INREG: return combineSignExtendInReg(N, DAG, Subtarget); case ISD::ANY_EXTEND_VECTOR_INREG: case ISD::SIGN_EXTEND_VECTOR_INREG: - case ISD::ZERO_EXTEND_VECTOR_INREG: return combineExtInVec(N, DAG, DCI, - Subtarget); + case ISD::ZERO_EXTEND_VECTOR_INREG: + return combineEXTEND_VECTOR_INREG(N, DAG, DCI, Subtarget); case ISD::SETCC: return combineSetCC(N, DAG, Subtarget); case X86ISD::SETCC: return combineX86SetCC(N, DAG, Subtarget); case X86ISD::BRCOND: return combineBrCond(N, DAG, Subtarget); case X86ISD::PACKSS: case X86ISD::PACKUS: return combineVectorPack(N, DAG, DCI, Subtarget); + case X86ISD::HADD: + case X86ISD::HSUB: + case X86ISD::FHADD: + case X86ISD::FHSUB: return combineVectorHADDSUB(N, DAG, DCI, Subtarget); case X86ISD::VSHL: case X86ISD::VSRA: case X86ISD::VSRL: @@ -48943,8 +50442,10 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N, case ISD::STRICT_FP_EXTEND: case ISD::FP_EXTEND: return combineFP_EXTEND(N, DAG, Subtarget); case ISD::FP_ROUND: return combineFP_ROUND(N, DAG, Subtarget); - case X86ISD::VBROADCAST_LOAD: return combineVBROADCAST_LOAD(N, DAG, DCI); + case X86ISD::VBROADCAST_LOAD: + case X86ISD::SUBV_BROADCAST_LOAD: return combineBROADCAST_LOAD(N, DAG, DCI); case X86ISD::MOVDQ2Q: return combineMOVDQ2Q(N, DAG); + case X86ISD::PDEP: return combinePDEP(N, DAG, DCI); } return SDValue(); @@ -49233,7 +50734,7 @@ static X86::CondCode parseConstraintCode(llvm::StringRef Constraint) { .Case("{@ccnl}", X86::COND_GE) .Case("{@ccnle}", X86::COND_G) .Case("{@ccno}", X86::COND_NO) - .Case("{@ccnp}", X86::COND_P) + .Case("{@ccnp}", X86::COND_NP) .Case("{@ccns}", X86::COND_NS) .Case("{@cco}", X86::COND_O) .Case("{@ccp}", X86::COND_P) @@ -49469,8 +50970,8 @@ LowerXConstraint(EVT ConstraintVT) const { // Lower @cc targets via setcc. SDValue X86TargetLowering::LowerAsmOutputForConstraint( - SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo, - SelectionDAG &DAG) const { + SDValue &Chain, SDValue &Flag, const SDLoc &DL, + const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const { X86::CondCode Cond = parseConstraintCode(OpInfo.ConstraintCode); if (Cond == X86::COND_INVALID) return SDValue(); @@ -49906,30 +51407,35 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, // Not found as a standard register? if (!Res.second) { - // Map st(0) -> st(7) -> ST0 - if (Constraint.size() == 7 && Constraint[0] == '{' && - tolower(Constraint[1]) == 's' && tolower(Constraint[2]) == 't' && - Constraint[3] == '(' && - (Constraint[4] >= '0' && Constraint[4] <= '7') && - Constraint[5] == ')' && Constraint[6] == '}') { - // st(7) is not allocatable and thus not a member of RFP80. Return - // singleton class in cases where we have a reference to it. - if (Constraint[4] == '7') - return std::make_pair(X86::FP7, &X86::RFP80_7RegClass); - return std::make_pair(X86::FP0 + Constraint[4] - '0', - &X86::RFP80RegClass); - } + // Only match x87 registers if the VT is one SelectionDAGBuilder can convert + // to/from f80. + if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) { + // Map st(0) -> st(7) -> ST0 + if (Constraint.size() == 7 && Constraint[0] == '{' && + tolower(Constraint[1]) == 's' && tolower(Constraint[2]) == 't' && + Constraint[3] == '(' && + (Constraint[4] >= '0' && Constraint[4] <= '7') && + Constraint[5] == ')' && Constraint[6] == '}') { + // st(7) is not allocatable and thus not a member of RFP80. Return + // singleton class in cases where we have a reference to it. + if (Constraint[4] == '7') + return std::make_pair(X86::FP7, &X86::RFP80_7RegClass); + return std::make_pair(X86::FP0 + Constraint[4] - '0', + &X86::RFP80RegClass); + } - // GCC allows "st(0)" to be called just plain "st". - if (StringRef("{st}").equals_lower(Constraint)) - return std::make_pair(X86::FP0, &X86::RFP80RegClass); + // GCC allows "st(0)" to be called just plain "st". + if (StringRef("{st}").equals_lower(Constraint)) + return std::make_pair(X86::FP0, &X86::RFP80RegClass); + } // flags -> EFLAGS if (StringRef("{flags}").equals_lower(Constraint)) return std::make_pair(X86::EFLAGS, &X86::CCRRegClass); // dirflag -> DF - if (StringRef("{dirflag}").equals_lower(Constraint)) + // Only allow for clobber. + if (StringRef("{dirflag}").equals_lower(Constraint) && VT == MVT::Other) return std::make_pair(X86::DF, &X86::DFCCRRegClass); // fpsr -> FPSW @@ -50203,3 +51709,10 @@ X86TargetLowering::getStackProbeSize(MachineFunction &MF) const { .getAsInteger(0, StackProbeSize); return StackProbeSize; } + +Align X86TargetLowering::getPrefLoopAlignment(MachineLoop *ML) const { + if (ML->isInnermost() && + ExperimentalPrefInnermostLoopAlignment.getNumOccurrences()) + return Align(1ULL << ExperimentalPrefInnermostLoopAlignment); + return TargetLowering::getPrefLoopAlignment(); +} |