diff options
Diffstat (limited to 'lib/Target/RISCV/RISCVISelLowering.cpp')
-rw-r--r-- | lib/Target/RISCV/RISCVISelLowering.cpp | 328 |
1 files changed, 255 insertions, 73 deletions
diff --git a/lib/Target/RISCV/RISCVISelLowering.cpp b/lib/Target/RISCV/RISCVISelLowering.cpp index 87796e5b1097..508dcbd009ed 100644 --- a/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/lib/Target/RISCV/RISCVISelLowering.cpp @@ -80,6 +80,13 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, for (auto VT : {MVT::i1, MVT::i8, MVT::i16}) setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); + if (Subtarget.is64Bit()) { + setTargetDAGCombine(ISD::SHL); + setTargetDAGCombine(ISD::SRL); + setTargetDAGCombine(ISD::SRA); + setTargetDAGCombine(ISD::ANY_EXTEND); + } + if (!Subtarget.hasStdExtM()) { setOperationAction(ISD::MUL, XLenVT, Expand); setOperationAction(ISD::MULHS, XLenVT, Expand); @@ -111,6 +118,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT, ISD::SETGE, ISD::SETNE}; + ISD::NodeType FPOpToExtend[] = { + ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM}; + if (Subtarget.hasStdExtF()) { setOperationAction(ISD::FMINNUM, MVT::f32, Legal); setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); @@ -119,6 +129,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); setOperationAction(ISD::SELECT, MVT::f32, Custom); setOperationAction(ISD::BR_CC, MVT::f32, Expand); + for (auto Op : FPOpToExtend) + setOperationAction(Op, MVT::f32, Expand); } if (Subtarget.hasStdExtD()) { @@ -131,16 +143,20 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::BR_CC, MVT::f64, Expand); setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); setTruncStoreAction(MVT::f64, MVT::f32, Expand); + for (auto Op : FPOpToExtend) + setOperationAction(Op, MVT::f64, Expand); } setOperationAction(ISD::GlobalAddress, XLenVT, Custom); setOperationAction(ISD::BlockAddress, XLenVT, Custom); setOperationAction(ISD::ConstantPool, XLenVT, Custom); - if (Subtarget.hasStdExtA()) + if (Subtarget.hasStdExtA()) { setMaxAtomicSizeInBitsSupported(Subtarget.getXLen()); - else + setMinCmpXchgSizeInBits(32); + } else { setMaxAtomicSizeInBitsSupported(0); + } setBooleanContents(ZeroOrOneBooleanContent); @@ -160,6 +176,34 @@ EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &, return VT.changeVectorElementTypeToInteger(); } +bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, + const CallInst &I, + MachineFunction &MF, + unsigned Intrinsic) const { + switch (Intrinsic) { + default: + return false; + case Intrinsic::riscv_masked_atomicrmw_xchg_i32: + case Intrinsic::riscv_masked_atomicrmw_add_i32: + case Intrinsic::riscv_masked_atomicrmw_sub_i32: + case Intrinsic::riscv_masked_atomicrmw_nand_i32: + case Intrinsic::riscv_masked_atomicrmw_max_i32: + case Intrinsic::riscv_masked_atomicrmw_min_i32: + case Intrinsic::riscv_masked_atomicrmw_umax_i32: + case Intrinsic::riscv_masked_atomicrmw_umin_i32: + case Intrinsic::riscv_masked_cmpxchg_i32: + PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType()); + Info.opc = ISD::INTRINSIC_W_CHAIN; + Info.memVT = MVT::getVT(PtrTy->getElementType()); + Info.ptrVal = I.getArgOperand(0); + Info.offset = 0; + Info.align = 4; + Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore | + MachineMemOperand::MOVolatile; + return true; + } +} + bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, @@ -228,6 +272,10 @@ bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { return TargetLowering::isZExtFree(Val, VT2); } +bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const { + return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64; +} + // Changes the condition code and swaps operands if necessary, so the SetCC // operation matches one of the comparisons supported directly in the RISC-V // ISA. @@ -283,9 +331,9 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op, case ISD::VASTART: return lowerVASTART(Op, DAG); case ISD::FRAMEADDR: - return LowerFRAMEADDR(Op, DAG); + return lowerFRAMEADDR(Op, DAG); case ISD::RETURNADDR: - return LowerRETURNADDR(Op, DAG); + return lowerRETURNADDR(Op, DAG); } } @@ -298,7 +346,7 @@ SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op, int64_t Offset = N->getOffset(); MVT XLenVT = Subtarget.getXLenVT(); - if (isPositionIndependent() || Subtarget.is64Bit()) + if (isPositionIndependent()) report_fatal_error("Unable to lowerGlobalAddress"); // In order to maximise the opportunity for common subexpression elimination, // emit a separate ADD node for the global address offset instead of folding @@ -323,7 +371,7 @@ SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op, const BlockAddress *BA = N->getBlockAddress(); int64_t Offset = N->getOffset(); - if (isPositionIndependent() || Subtarget.is64Bit()) + if (isPositionIndependent()) report_fatal_error("Unable to lowerBlockAddress"); SDValue BAHi = DAG.getTargetBlockAddress(BA, Ty, Offset, RISCVII::MO_HI); @@ -357,26 +405,6 @@ SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op, } } -SDValue RISCVTargetLowering::lowerExternalSymbol(SDValue Op, - SelectionDAG &DAG) const { - SDLoc DL(Op); - EVT Ty = Op.getValueType(); - ExternalSymbolSDNode *N = cast<ExternalSymbolSDNode>(Op); - const char *Sym = N->getSymbol(); - - // TODO: should also handle gp-relative loads. - - if (isPositionIndependent() || Subtarget.is64Bit()) - report_fatal_error("Unable to lowerExternalSymbol"); - - SDValue GAHi = DAG.getTargetExternalSymbol(Sym, Ty, RISCVII::MO_HI); - SDValue GALo = DAG.getTargetExternalSymbol(Sym, Ty, RISCVII::MO_LO); - SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, GAHi), 0); - SDValue MNLo = - SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, GALo), 0); - return MNLo; -} - SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const { SDValue CondV = Op.getOperand(0); SDValue TrueV = Op.getOperand(1); @@ -432,7 +460,7 @@ SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const { MachinePointerInfo(SV)); } -SDValue RISCVTargetLowering::LowerFRAMEADDR(SDValue Op, +SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo(); MachineFunction &MF = DAG.getMachineFunction(); @@ -455,7 +483,7 @@ SDValue RISCVTargetLowering::LowerFRAMEADDR(SDValue Op, return FrameAddr; } -SDValue RISCVTargetLowering::LowerRETURNADDR(SDValue Op, +SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const { const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo(); MachineFunction &MF = DAG.getMachineFunction(); @@ -472,7 +500,7 @@ SDValue RISCVTargetLowering::LowerRETURNADDR(SDValue Op, unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); if (Depth) { int Off = -XLenInBytes; - SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); + SDValue FrameAddr = lowerFRAMEADDR(Op, DAG); SDValue Offset = DAG.getConstant(Off, DL, VT); return DAG.getLoad(VT, DL, DAG.getEntryNode(), DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), @@ -485,6 +513,84 @@ SDValue RISCVTargetLowering::LowerRETURNADDR(SDValue Op, return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT); } +// Return true if the given node is a shift with a non-constant shift amount. +static bool isVariableShift(SDValue Val) { + switch (Val.getOpcode()) { + default: + return false; + case ISD::SHL: + case ISD::SRA: + case ISD::SRL: + return Val.getOperand(1).getOpcode() != ISD::Constant; + } +} + +// Returns true if the given node is an sdiv, udiv, or urem with non-constant +// operands. +static bool isVariableSDivUDivURem(SDValue Val) { + switch (Val.getOpcode()) { + default: + return false; + case ISD::SDIV: + case ISD::UDIV: + case ISD::UREM: + return Val.getOperand(0).getOpcode() != ISD::Constant && + Val.getOperand(1).getOpcode() != ISD::Constant; + } +} + +SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N, + DAGCombinerInfo &DCI) const { + SelectionDAG &DAG = DCI.DAG; + + switch (N->getOpcode()) { + default: + break; + case ISD::SHL: + case ISD::SRL: + case ISD::SRA: { + assert(Subtarget.getXLen() == 64 && "Combine should be 64-bit only"); + if (!DCI.isBeforeLegalize()) + break; + SDValue RHS = N->getOperand(1); + if (N->getValueType(0) != MVT::i32 || RHS->getOpcode() == ISD::Constant || + (RHS->getOpcode() == ISD::AssertZext && + cast<VTSDNode>(RHS->getOperand(1))->getVT().getSizeInBits() <= 5)) + break; + SDValue LHS = N->getOperand(0); + SDLoc DL(N); + SDValue NewRHS = + DAG.getNode(ISD::AssertZext, DL, RHS.getValueType(), RHS, + DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), 5))); + return DCI.CombineTo( + N, DAG.getNode(N->getOpcode(), DL, LHS.getValueType(), LHS, NewRHS)); + } + case ISD::ANY_EXTEND: { + // If any-extending an i32 variable-length shift or sdiv/udiv/urem to i64, + // then instead sign-extend in order to increase the chance of being able + // to select the sllw/srlw/sraw/divw/divuw/remuw instructions. + SDValue Src = N->getOperand(0); + if (N->getValueType(0) != MVT::i64 || Src.getValueType() != MVT::i32) + break; + if (!isVariableShift(Src) && + !(Subtarget.hasStdExtM() && isVariableSDivUDivURem(Src))) + break; + SDLoc DL(N); + return DCI.CombineTo(N, DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Src)); + } + case RISCVISD::SplitF64: { + // If the input to SplitF64 is just BuildPairF64 then the operation is + // redundant. Instead, use BuildPairF64's operands directly. + SDValue Op0 = N->getOperand(0); + if (Op0->getOpcode() != RISCVISD::BuildPairF64) + break; + return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1)); + } + } + + return SDValue(); +} + static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB) { assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction"); @@ -807,10 +913,14 @@ static bool CC_RISCV(const DataLayout &DL, unsigned ValNo, MVT ValVT, MVT LocVT, if (Reg) { State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); - } else { - State.addLoc( - CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo)); + return false; } + + if (ValVT == MVT::f32) { + LocVT = MVT::f32; + LocInfo = CCValAssign::Full; + } + State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo)); return false; } @@ -859,6 +969,22 @@ void RISCVTargetLowering::analyzeOutputArgs( } } +// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect +// values. +static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, + const CCValAssign &VA, const SDLoc &DL) { + switch (VA.getLocInfo()) { + default: + llvm_unreachable("Unexpected CCValAssign::LocInfo"); + case CCValAssign::Full: + break; + case CCValAssign::BCvt: + Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); + break; + } + return Val; +} + // The caller is responsible for loading the full value if the argument is // passed with CCValAssign::Indirect. static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain, @@ -866,21 +992,29 @@ static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain, MachineFunction &MF = DAG.getMachineFunction(); MachineRegisterInfo &RegInfo = MF.getRegInfo(); EVT LocVT = VA.getLocVT(); - EVT ValVT = VA.getValVT(); SDValue Val; unsigned VReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass); RegInfo.addLiveIn(VA.getLocReg(), VReg); Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); + if (VA.getLocInfo() == CCValAssign::Indirect) + return Val; + + return convertLocVTToValVT(DAG, Val, VA, DL); +} + +static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, + const CCValAssign &VA, const SDLoc &DL) { + EVT LocVT = VA.getLocVT(); + switch (VA.getLocInfo()) { default: llvm_unreachable("Unexpected CCValAssign::LocInfo"); case CCValAssign::Full: - case CCValAssign::Indirect: break; case CCValAssign::BCvt: - Val = DAG.getNode(ISD::BITCAST, DL, ValVT, Val); + Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val); break; } return Val; @@ -995,7 +1129,6 @@ SDValue RISCVTargetLowering::LowerFormalArguments( for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { CCValAssign &VA = ArgLocs[i]; - assert(VA.getLocVT() == XLenVT && "Unhandled argument type"); SDValue ArgValue; // Passing f64 on RV32D with a soft float ABI must be handled as a special // case. @@ -1282,13 +1415,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, // Promote the value if needed. // For now, only handle fully promoted and indirect arguments. - switch (VA.getLocInfo()) { - case CCValAssign::Full: - break; - case CCValAssign::BCvt: - ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), ArgValue); - break; - case CCValAssign::Indirect: { + if (VA.getLocInfo() == CCValAssign::Indirect) { // Store the argument in a stack slot and pass its address. SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT); int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); @@ -1310,10 +1437,8 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, ++i; } ArgValue = SpillSlot; - break; - } - default: - llvm_unreachable("Unknown loc info!"); + } else { + ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL); } // Use local copy if it is a byval arg. @@ -1415,6 +1540,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, // Glue the RetValue to the end of the call sequence Chain = RetValue.getValue(1); Glue = RetValue.getValue(2); + if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) { assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment"); SDValue RetValue2 = @@ -1425,15 +1551,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, RetValue2); } - switch (VA.getLocInfo()) { - default: - llvm_unreachable("Unknown loc info!"); - case CCValAssign::Full: - break; - case CCValAssign::BCvt: - RetValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), RetValue); - break; - } + RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL); InVals.push_back(RetValue); } @@ -1456,22 +1574,6 @@ bool RISCVTargetLowering::CanLowerReturn( return true; } -static SDValue packIntoRegLoc(SelectionDAG &DAG, SDValue Val, - const CCValAssign &VA, const SDLoc &DL) { - EVT LocVT = VA.getLocVT(); - - switch (VA.getLocInfo()) { - default: - llvm_unreachable("Unexpected CCValAssign::LocInfo"); - case CCValAssign::Full: - break; - case CCValAssign::BCvt: - Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val); - break; - } - return Val; -} - SDValue RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, @@ -1514,7 +1616,7 @@ RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, RetOps.push_back(DAG.getRegister(RegHi, MVT::i32)); } else { // Handle a 'normal' return. - Val = packIntoRegLoc(DAG, Val, VA, DL); + Val = convertValVTToLocVT(DAG, Val, VA, DL); Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue); // Guarantee that all emitted copies are stuck together. @@ -1616,3 +1718,83 @@ Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilder<> &Builder, return Builder.CreateFence(AtomicOrdering::Acquire); return nullptr; } + +TargetLowering::AtomicExpansionKind +RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { + unsigned Size = AI->getType()->getPrimitiveSizeInBits(); + if (Size == 8 || Size == 16) + return AtomicExpansionKind::MaskedIntrinsic; + return AtomicExpansionKind::None; +} + +static Intrinsic::ID +getIntrinsicForMaskedAtomicRMWBinOp32(AtomicRMWInst::BinOp BinOp) { + switch (BinOp) { + default: + llvm_unreachable("Unexpected AtomicRMW BinOp"); + case AtomicRMWInst::Xchg: + return Intrinsic::riscv_masked_atomicrmw_xchg_i32; + case AtomicRMWInst::Add: + return Intrinsic::riscv_masked_atomicrmw_add_i32; + case AtomicRMWInst::Sub: + return Intrinsic::riscv_masked_atomicrmw_sub_i32; + case AtomicRMWInst::Nand: + return Intrinsic::riscv_masked_atomicrmw_nand_i32; + case AtomicRMWInst::Max: + return Intrinsic::riscv_masked_atomicrmw_max_i32; + case AtomicRMWInst::Min: + return Intrinsic::riscv_masked_atomicrmw_min_i32; + case AtomicRMWInst::UMax: + return Intrinsic::riscv_masked_atomicrmw_umax_i32; + case AtomicRMWInst::UMin: + return Intrinsic::riscv_masked_atomicrmw_umin_i32; + } +} + +Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic( + IRBuilder<> &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, + Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const { + Value *Ordering = Builder.getInt32(static_cast<uint32_t>(AI->getOrdering())); + Type *Tys[] = {AlignedAddr->getType()}; + Function *LrwOpScwLoop = Intrinsic::getDeclaration( + AI->getModule(), + getIntrinsicForMaskedAtomicRMWBinOp32(AI->getOperation()), Tys); + + // Must pass the shift amount needed to sign extend the loaded value prior + // to performing a signed comparison for min/max. ShiftAmt is the number of + // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which + // is the number of bits to left+right shift the value in order to + // sign-extend. + if (AI->getOperation() == AtomicRMWInst::Min || + AI->getOperation() == AtomicRMWInst::Max) { + const DataLayout &DL = AI->getModule()->getDataLayout(); + unsigned ValWidth = + DL.getTypeStoreSizeInBits(AI->getValOperand()->getType()); + Value *SextShamt = Builder.CreateSub( + Builder.getInt32(Subtarget.getXLen() - ValWidth), ShiftAmt); + return Builder.CreateCall(LrwOpScwLoop, + {AlignedAddr, Incr, Mask, SextShamt, Ordering}); + } + + return Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering}); +} + +TargetLowering::AtomicExpansionKind +RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR( + AtomicCmpXchgInst *CI) const { + unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits(); + if (Size == 8 || Size == 16) + return AtomicExpansionKind::MaskedIntrinsic; + return AtomicExpansionKind::None; +} + +Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic( + IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, + Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const { + Value *Ordering = Builder.getInt32(static_cast<uint32_t>(Ord)); + Type *Tys[] = {AlignedAddr->getType()}; + Function *MaskedCmpXchg = Intrinsic::getDeclaration( + CI->getModule(), Intrinsic::riscv_masked_cmpxchg_i32, Tys); + return Builder.CreateCall(MaskedCmpXchg, + {AlignedAddr, CmpVal, NewVal, Mask, Ordering}); +} |