diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG')
15 files changed, 385 insertions, 313 deletions
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index eafa95ce7fcf..2327664516cc 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -7987,7 +7987,7 @@ SDValue DAGCombiner::visitOR(SDNode *N) { // If OR can be rewritten into ADD, try combines based on ADD. if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && - DAG.haveNoCommonBitsSet(N0, N1)) + DAG.isADDLike(SDValue(N, 0))) if (SDValue Combined = visitADDLike(N)) return Combined; @@ -10055,7 +10055,11 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) { SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1); AddToWorklist(Shl0.getNode()); - return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1); + SDNodeFlags Flags; + // Preserve the disjoint flag for Or. + if (N0.getOpcode() == ISD::OR && N0->getFlags().hasDisjoint()) + Flags.setDisjoint(true); + return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1, Flags); } } @@ -14709,7 +14713,7 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { SDValue EltNo = N0->getOperand(1); if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) { - int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue(); + int Elt = EltNo->getAsZExtVal(); int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1)); SDLoc DL(N); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index f3d8edb8926b..6d80b282a1ed 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -560,15 +560,13 @@ bool FastISel::selectGetElementPtr(const User *I) { } } } else { - Type *Ty = GTI.getIndexedType(); - // If this is a constant subscript, handle it quickly. if (const auto *CI = dyn_cast<ConstantInt>(Idx)) { if (CI->isZero()) continue; // N = N + Offset uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue(); - TotalOffs += DL.getTypeAllocSize(Ty) * IdxN; + TotalOffs += GTI.getSequentialElementStride(DL) * IdxN; if (TotalOffs >= MaxOffs) { N = fastEmit_ri_(VT, ISD::ADD, N, TotalOffs, VT); if (!N) // Unhandled operand. Halt "fast" selection and bail. @@ -585,7 +583,7 @@ bool FastISel::selectGetElementPtr(const User *I) { } // N = N + Idx * ElementSize; - uint64_t ElementSize = DL.getTypeAllocSize(Ty); + uint64_t ElementSize = GTI.getSequentialElementStride(DL); Register IdxN = getRegForGEPIndex(Idx); if (!IdxN) // Unhandled operand. Halt "fast" selection and bail. return false; @@ -1182,6 +1180,184 @@ bool FastISel::selectCall(const User *I) { return lowerCall(Call); } +void FastISel::handleDbgInfo(const Instruction *II) { + if (!II->hasDbgValues()) + return; + + // Clear any metadata. + MIMD = MIMetadata(); + + // Reverse order of debug records, because fast-isel walks through backwards. + for (DPValue &DPV : llvm::reverse(II->getDbgValueRange())) { + flushLocalValueMap(); + recomputeInsertPt(); + + Value *V = nullptr; + if (!DPV.hasArgList()) + V = DPV.getVariableLocationOp(0); + + bool Res = false; + if (DPV.getType() == DPValue::LocationType::Value) { + Res = lowerDbgValue(V, DPV.getExpression(), DPV.getVariable(), + DPV.getDebugLoc()); + } else { + assert(DPV.getType() == DPValue::LocationType::Declare); + if (FuncInfo.PreprocessedDPVDeclares.contains(&DPV)) + continue; + Res = lowerDbgDeclare(V, DPV.getExpression(), DPV.getVariable(), + DPV.getDebugLoc()); + } + + if (!Res) + LLVM_DEBUG(dbgs() << "Dropping debug-info for " << DPV << "\n";); + } +} + +bool FastISel::lowerDbgValue(const Value *V, DIExpression *Expr, + DILocalVariable *Var, const DebugLoc &DL) { + // This form of DBG_VALUE is target-independent. + const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE); + if (!V || isa<UndefValue>(V)) { + // DI is either undef or cannot produce a valid DBG_VALUE, so produce an + // undef DBG_VALUE to terminate any prior location. + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, false, 0U, Var, Expr); + return true; + } + if (const auto *CI = dyn_cast<ConstantInt>(V)) { + // See if there's an expression to constant-fold. + if (Expr) + std::tie(Expr, CI) = Expr->constantFold(CI); + if (CI->getBitWidth() > 64) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II) + .addCImm(CI) + .addImm(0U) + .addMetadata(Var) + .addMetadata(Expr); + else + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II) + .addImm(CI->getZExtValue()) + .addImm(0U) + .addMetadata(Var) + .addMetadata(Expr); + return true; + } + if (const auto *CF = dyn_cast<ConstantFP>(V)) { + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II) + .addFPImm(CF) + .addImm(0U) + .addMetadata(Var) + .addMetadata(Expr); + return true; + } + if (const auto *Arg = dyn_cast<Argument>(V); + Arg && Expr && Expr->isEntryValue()) { + // As per the Verifier, this case is only valid for swift async Args. + assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync)); + + Register Reg = getRegForValue(Arg); + for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins()) + if (Reg == VirtReg || Reg == PhysReg) { + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, false /*IsIndirect*/, + PhysReg, Var, Expr); + return true; + } + + LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but " + "couldn't find a physical register\n"); + return false; + } + if (auto SI = FuncInfo.StaticAllocaMap.find(dyn_cast<AllocaInst>(V)); + SI != FuncInfo.StaticAllocaMap.end()) { + MachineOperand FrameIndexOp = MachineOperand::CreateFI(SI->second); + bool IsIndirect = false; + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, IsIndirect, FrameIndexOp, + Var, Expr); + return true; + } + if (Register Reg = lookUpRegForValue(V)) { + // FIXME: This does not handle register-indirect values at offset 0. + if (!FuncInfo.MF->useDebugInstrRef()) { + bool IsIndirect = false; + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, IsIndirect, Reg, Var, + Expr); + return true; + } + // If using instruction referencing, produce this as a DBG_INSTR_REF, + // to be later patched up by finalizeDebugInstrRefs. + SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg( + /* Reg */ Reg, /* isDef */ false, /* isImp */ false, + /* isKill */ false, /* isDead */ false, + /* isUndef */ false, /* isEarlyClobber */ false, + /* SubReg */ 0, /* isDebug */ true)}); + SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0}); + auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs, + Var, NewExpr); + return true; + } + return false; +} + +bool FastISel::lowerDbgDeclare(const Value *Address, DIExpression *Expr, + DILocalVariable *Var, const DebugLoc &DL) { + if (!Address || isa<UndefValue>(Address)) { + LLVM_DEBUG(dbgs() << "Dropping debug info (bad/undef address)\n"); + return false; + } + + std::optional<MachineOperand> Op; + if (Register Reg = lookUpRegForValue(Address)) + Op = MachineOperand::CreateReg(Reg, false); + + // If we have a VLA that has a "use" in a metadata node that's then used + // here but it has no other uses, then we have a problem. E.g., + // + // int foo (const int *x) { + // char a[*x]; + // return 0; + // } + // + // If we assign 'a' a vreg and fast isel later on has to use the selection + // DAG isel, it will want to copy the value to the vreg. However, there are + // no uses, which goes counter to what selection DAG isel expects. + if (!Op && !Address->use_empty() && isa<Instruction>(Address) && + (!isa<AllocaInst>(Address) || + !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address)))) + Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address), + false); + + if (Op) { + assert(Var->isValidLocationForIntrinsic(DL) && + "Expected inlined-at fields to agree"); + if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) { + // If using instruction referencing, produce this as a DBG_INSTR_REF, + // to be later patched up by finalizeDebugInstrRefs. Tack a deref onto + // the expression, we don't have an "indirect" flag in DBG_INSTR_REF. + SmallVector<uint64_t, 3> Ops( + {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref}); + auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, *Op, + Var, NewExpr); + return true; + } + + // A dbg.declare describes the address of a source variable, so lower it + // into an indirect DBG_VALUE. + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, + TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, *Op, Var, + Expr); + return true; + } + + // We can't yet handle anything else here because it would require + // generating code, thus altering codegen because of debug info. + LLVM_DEBUG( + dbgs() << "Dropping debug info (no materialized reg for address)\n"); + return false; +} + bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) { switch (II->getIntrinsicID()) { default: @@ -1211,153 +1387,28 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) { return true; const Value *Address = DI->getAddress(); - if (!Address || isa<UndefValue>(Address)) { - LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI - << " (bad/undef address)\n"); - return true; - } + if (!lowerDbgDeclare(Address, DI->getExpression(), DI->getVariable(), + MIMD.getDL())) + LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI); - std::optional<MachineOperand> Op; - if (Register Reg = lookUpRegForValue(Address)) - Op = MachineOperand::CreateReg(Reg, false); - - // If we have a VLA that has a "use" in a metadata node that's then used - // here but it has no other uses, then we have a problem. E.g., - // - // int foo (const int *x) { - // char a[*x]; - // return 0; - // } - // - // If we assign 'a' a vreg and fast isel later on has to use the selection - // DAG isel, it will want to copy the value to the vreg. However, there are - // no uses, which goes counter to what selection DAG isel expects. - if (!Op && !Address->use_empty() && isa<Instruction>(Address) && - (!isa<AllocaInst>(Address) || - !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address)))) - Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address), - false); - - if (Op) { - assert(DI->getVariable()->isValidLocationForIntrinsic(MIMD.getDL()) && - "Expected inlined-at fields to agree"); - if (FuncInfo.MF->useDebugInstrRef() && Op->isReg()) { - // If using instruction referencing, produce this as a DBG_INSTR_REF, - // to be later patched up by finalizeDebugInstrRefs. Tack a deref onto - // the expression, we don't have an "indirect" flag in DBG_INSTR_REF. - SmallVector<uint64_t, 3> Ops( - {dwarf::DW_OP_LLVM_arg, 0, dwarf::DW_OP_deref}); - auto *NewExpr = DIExpression::prependOpcodes(DI->getExpression(), Ops); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), - TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, *Op, - DI->getVariable(), NewExpr); - } else { - // A dbg.declare describes the address of a source variable, so lower it - // into an indirect DBG_VALUE. - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), - TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, *Op, - DI->getVariable(), DI->getExpression()); - } - } else { - // We can't yet handle anything else here because it would require - // generating code, thus altering codegen because of debug info. - LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI - << " (no materialized reg for address)\n"); - } return true; } case Intrinsic::dbg_value: { // This form of DBG_VALUE is target-independent. const DbgValueInst *DI = cast<DbgValueInst>(II); - const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE); const Value *V = DI->getValue(); DIExpression *Expr = DI->getExpression(); DILocalVariable *Var = DI->getVariable(); + if (DI->hasArgList()) + // Signal that we don't have a location for this. + V = nullptr; + assert(Var->isValidLocationForIntrinsic(MIMD.getDL()) && "Expected inlined-at fields to agree"); - if (!V || isa<UndefValue>(V) || DI->hasArgList()) { - // DI is either undef or cannot produce a valid DBG_VALUE, so produce an - // undef DBG_VALUE to terminate any prior location. - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), II, false, 0U, - Var, Expr); - return true; - } - if (const auto *CI = dyn_cast<ConstantInt>(V)) { - // See if there's an expression to constant-fold. - if (Expr) - std::tie(Expr, CI) = Expr->constantFold(CI); - if (CI->getBitWidth() > 64) - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II) - .addCImm(CI) - .addImm(0U) - .addMetadata(Var) - .addMetadata(Expr); - else - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II) - .addImm(CI->getZExtValue()) - .addImm(0U) - .addMetadata(Var) - .addMetadata(Expr); - return true; - } - if (const auto *CF = dyn_cast<ConstantFP>(V)) { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II) - .addFPImm(CF) - .addImm(0U) - .addMetadata(Var) - .addMetadata(Expr); - return true; - } - if (const auto *Arg = dyn_cast<Argument>(V); - Arg && Expr && Expr->isEntryValue()) { - // As per the Verifier, this case is only valid for swift async Args. - assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync)); - - Register Reg = getRegForValue(Arg); - for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins()) - if (Reg == VirtReg || Reg == PhysReg) { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), II, - false /*IsIndirect*/, PhysReg, Var, Expr); - return true; - } - LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but " - "couldn't find a physical register\n" - << *DI << "\n"); - return true; - } - if (auto SI = FuncInfo.StaticAllocaMap.find(dyn_cast<AllocaInst>(V)); - SI != FuncInfo.StaticAllocaMap.end()) { - MachineOperand FrameIndexOp = MachineOperand::CreateFI(SI->second); - bool IsIndirect = false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), II, IsIndirect, - FrameIndexOp, Var, Expr); - return true; - } - if (Register Reg = lookUpRegForValue(V)) { - // FIXME: This does not handle register-indirect values at offset 0. - if (!FuncInfo.MF->useDebugInstrRef()) { - bool IsIndirect = false; - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), II, IsIndirect, - Reg, Var, Expr); - return true; - } - // If using instruction referencing, produce this as a DBG_INSTR_REF, - // to be later patched up by finalizeDebugInstrRefs. - SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg( - /* Reg */ Reg, /* isDef */ false, /* isImp */ false, - /* isKill */ false, /* isDead */ false, - /* isUndef */ false, /* isEarlyClobber */ false, - /* SubReg */ 0, /* isDebug */ true)}); - SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0}); - auto *NewExpr = DIExpression::prependOpcodes(Expr, Ops); - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD.getDL(), - TII.get(TargetOpcode::DBG_INSTR_REF), /*IsIndirect*/ false, MOs, - Var, NewExpr); - return true; - } - // We don't know how to handle other cases, so we drop. - LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); + if (!lowerDbgValue(V, Expr, Var, MIMD.getDL())) + LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); + return true; } case Intrinsic::dbg_label: { diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 34fa1f5a7ed1..032cff416cda 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -551,7 +551,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, SDValue N0 = Node->getOperand(0); SDValue N1 = Node->getOperand(1); SDValue N2 = Node->getOperand(2); - unsigned SubIdx = cast<ConstantSDNode>(N2)->getZExtValue(); + unsigned SubIdx = N2->getAsZExtVal(); // Figure out the register class to create for the destreg. It should be // the largest legal register class supporting SubIdx sub-registers. @@ -650,7 +650,7 @@ void InstrEmitter::EmitRegSequence(SDNode *Node, // Skip physical registers as they don't have a vreg to get and we'll // insert copies for them in TwoAddressInstructionPass anyway. if (!R || !R->getReg().isPhysical()) { - unsigned SubIdx = cast<ConstantSDNode>(Op)->getZExtValue(); + unsigned SubIdx = Op->getAsZExtVal(); unsigned SubReg = getVR(Node->getOperand(i-1), VRBaseMap); const TargetRegisterClass *TRC = MRI->getRegClass(SubReg); const TargetRegisterClass *SRC = diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 4e317062cec4..296ed3a3c3dc 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -3199,7 +3199,16 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { return true; } break; - case ISD::FP_ROUND: + case ISD::FP_ROUND: { + EVT VT = Node->getValueType(0); + if (VT.getScalarType() == MVT::bf16) { + Results.push_back( + DAG.getNode(ISD::FP_TO_BF16, SDLoc(Node), VT, Node->getOperand(0))); + break; + } + + LLVM_FALLTHROUGH; + } case ISD::BITCAST: if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), Node->getValueType(0), dl))) @@ -3226,12 +3235,19 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { return true; } break; - case ISD::FP_EXTEND: - if ((Tmp1 = EmitStackConvert(Node->getOperand(0), - Node->getOperand(0).getValueType(), - Node->getValueType(0), dl))) + case ISD::FP_EXTEND: { + SDValue Op = Node->getOperand(0); + EVT SrcVT = Op.getValueType(); + EVT DstVT = Node->getValueType(0); + if (SrcVT.getScalarType() == MVT::bf16) { + Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op)); + break; + } + + if ((Tmp1 = EmitStackConvert(Op, SrcVT, DstVT, dl))) Results.push_back(Tmp1); break; + } case ISD::BF16_TO_FP: { // Always expand bf16 to f32 casts, they lower to ext + shift. // diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 65919a64b806..589fec0e56f7 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -2181,6 +2181,24 @@ static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { report_fatal_error("Attempt at an invalid promotion-related conversion"); } +static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) { + if (OpVT == MVT::f16) + return ISD::STRICT_FP16_TO_FP; + + if (RetVT == MVT::f16) + return ISD::STRICT_FP_TO_FP16; + + if (OpVT == MVT::bf16) { + // TODO: return ISD::STRICT_BF16_TO_FP; + } + + if (RetVT == MVT::bf16) { + // TODO: return ISD::STRICT_FP_TO_BF16; + } + + report_fatal_error("Attempt at an invalid promotion-related conversion"); +} + bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG)); SDValue R = SDValue(); @@ -2281,7 +2299,7 @@ SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N, unsigned OpNo) { - assert(OpNo == 1); + assert(OpNo == 1 && "Promoting unpromotable operand"); SDValue Op = GetPromotedFloat(N->getOperand(1)); EVT VT = N->getValueType(0); @@ -2416,6 +2434,9 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break; case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; + case ISD::STRICT_FP_ROUND: + R = PromoteFloatRes_STRICT_FP_ROUND(N); + break; case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; @@ -2490,7 +2511,7 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { EVT VecVT = Vec->getValueType(0); EVT EltVT = VecVT.getVectorElementType(); - uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); + uint64_t IdxVal = Idx->getAsZExtVal(); switch (getTypeAction(VecVT)) { default: break; @@ -2621,6 +2642,29 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); } +// Explicit operation to reduce precision. Reduce the value to half precision +// and promote it back to the legal type. +SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) { + SDLoc DL(N); + + SDValue Chain = N->getOperand(0); + SDValue Op = N->getOperand(1); + EVT VT = N->getValueType(0); + EVT OpVT = Op->getValueType(0); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); + + // Round promoted float to desired precision + SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL, + DAG.getVTList(IVT, MVT::Other), Chain, Op); + // Promote it back to the legal output type + SDValue Res = + DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL, + DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round); + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + return Res; +} + SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { LoadSDNode *L = cast<LoadSDNode>(N); EVT VT = N->getValueType(0); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 3d21bd22e6ef..814f746f5a4d 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -165,7 +165,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16_BF16(N); break; - + case ISD::STRICT_FP_TO_FP16: + Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N); + break; case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break; case ISD::AND: @@ -787,6 +789,16 @@ SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) { return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); } +SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) { + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + SDLoc dl(N); + + SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), + N->getOperand(0), N->getOperand(1)); + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + return Res; +} + SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDLoc dl(N); @@ -1804,6 +1816,7 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { case ISD::FP16_TO_FP: case ISD::VP_UINT_TO_FP: case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; + case ISD::STRICT_FP16_TO_FP: case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break; @@ -5557,7 +5570,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { getTypeAction(InVT) == TargetLowering::TypeLegal) { EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext()); unsigned NElts = NInVT.getVectorMinNumElements(); - uint64_t IdxVal = cast<ConstantSDNode>(BaseIdx)->getZExtValue(); + uint64_t IdxVal = BaseIdx->getAsZExtVal(); SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0, DAG.getConstant(alignDown(IdxVal, NElts), dl, diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 84b1b2c71fd0..09f0bca8b861 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -326,6 +326,7 @@ private: SDValue PromoteIntRes_FP_TO_XINT(SDNode *N); SDValue PromoteIntRes_FP_TO_XINT_SAT(SDNode *N); SDValue PromoteIntRes_FP_TO_FP16_BF16(SDNode *N); + SDValue PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N); SDValue PromoteIntRes_XRINT(SDNode *N); SDValue PromoteIntRes_FREEZE(SDNode *N); SDValue PromoteIntRes_INT_EXTEND(SDNode *N); @@ -699,6 +700,7 @@ private: SDValue PromoteFloatRes_ExpOp(SDNode *N); SDValue PromoteFloatRes_FFREXP(SDNode *N); SDValue PromoteFloatRes_FP_ROUND(SDNode *N); + SDValue PromoteFloatRes_STRICT_FP_ROUND(SDNode *N); SDValue PromoteFloatRes_LOAD(SDNode *N); SDValue PromoteFloatRes_SELECT(SDNode *N); SDValue PromoteFloatRes_SELECT_CC(SDNode *N); diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 66461b26468f..ec74d2940099 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -1442,7 +1442,7 @@ void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx); - uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); + uint64_t IdxVal = Idx->getAsZExtVal(); Hi = DAG.getNode( ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl)); @@ -1466,7 +1466,7 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, // If we know the index is in the first half, and we know the subvector // doesn't cross the boundary between the halves, we can avoid spilling the // vector, and insert into the lower half of the split vector directly. - unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); + unsigned IdxVal = Idx->getAsZExtVal(); if (IdxVal + SubElems <= LoElems) { Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx); return; @@ -3279,7 +3279,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N, SDValue Lo, Hi; GetSplitVector(SubVec, Lo, Hi); - uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); + uint64_t IdxVal = Idx->getAsZExtVal(); uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); SDValue FirstInsertion = @@ -3301,7 +3301,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) { GetSplitVector(N->getOperand(0), Lo, Hi); uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements(); - uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); + uint64_t IdxVal = Idx->getAsZExtVal(); if (IdxVal < LoEltsMin) { assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin && @@ -5257,7 +5257,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) { EVT InVT = InOp.getValueType(); // Check if we can just return the input vector after widening. - uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue(); + uint64_t IdxVal = Idx->getAsZExtVal(); if (IdxVal == 0 && InVT == WidenVT) return InOp; diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 0e17bba2398e..b39be64c06f9 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5022,7 +5022,6 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, case ISD::CONCAT_VECTORS: case ISD::INSERT_SUBVECTOR: case ISD::AND: - case ISD::OR: case ISD::XOR: case ISD::ROTL: case ISD::ROTR: @@ -5062,6 +5061,10 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap()); + // Matches hasPoisonGeneratingFlags(). + case ISD::OR: + return ConsiderFlags && Op->getFlags().hasDisjoint(); + case ISD::INSERT_VECTOR_ELT:{ // Ensure that the element index is in bounds. EVT VecVT = Op.getOperand(0).getValueType(); @@ -5085,7 +5088,8 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, bool SelectionDAG::isADDLike(SDValue Op) const { unsigned Opcode = Op.getOpcode(); if (Opcode == ISD::OR) - return haveNoCommonBitsSet(Op.getOperand(0), Op.getOperand(1)); + return Op->getFlags().hasDisjoint() || + haveNoCommonBitsSet(Op.getOperand(0), Op.getOperand(1)); if (Opcode == ISD::XOR) return isMinSignedConstant(Op.getOperand(1)); return false; @@ -7193,8 +7197,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, assert(isa<ConstantSDNode>(N3) && "Insert subvector index must be constant"); assert((VT.isScalableVector() != N2VT.isScalableVector() || - (N2VT.getVectorMinNumElements() + - cast<ConstantSDNode>(N3)->getZExtValue()) <= + (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <= VT.getVectorMinNumElements()) && "Insert subvector overflow!"); assert(cast<ConstantSDNode>(N3)->getAPIntValue().getBitWidth() == @@ -9982,8 +9985,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, Ops[1].getValueType().isFloatingPoint() && VTList.VTs[0].bitsLT(Ops[1].getValueType()) && isa<ConstantSDNode>(Ops[2]) && - (cast<ConstantSDNode>(Ops[2])->getZExtValue() == 0 || - cast<ConstantSDNode>(Ops[2])->getZExtValue() == 1) && + (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) && "Invalid STRICT_FP_ROUND!"); break; #if 0 diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp index 39a1e09e83c5..66825d845c19 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp @@ -38,15 +38,18 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other, return true; // Match GlobalAddresses - if (auto *A = dyn_cast<GlobalAddressSDNode>(Base)) + if (auto *A = dyn_cast<GlobalAddressSDNode>(Base)) { if (auto *B = dyn_cast<GlobalAddressSDNode>(Other.Base)) if (A->getGlobal() == B->getGlobal()) { Off += B->getOffset() - A->getOffset(); return true; } + return false; + } + // Match Constants - if (auto *A = dyn_cast<ConstantPoolSDNode>(Base)) + if (auto *A = dyn_cast<ConstantPoolSDNode>(Base)) { if (auto *B = dyn_cast<ConstantPoolSDNode>(Other.Base)) { bool IsMatch = A->isMachineConstantPoolEntry() == B->isMachineConstantPoolEntry(); @@ -62,7 +65,8 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other, } } - const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); + return false; + } // Match FrameIndexes. if (auto *A = dyn_cast<FrameIndexSDNode>(Base)) @@ -73,6 +77,7 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other, // Non-equal FrameIndexes - If both frame indices are fixed // we know their relative offsets and can compare them. Otherwise // we must be conservative. + const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); if (MFI.isFixedObjectIndex(A->getIndex()) && MFI.isFixedObjectIndex(B->getIndex())) { Off += MFI.getObjectOffset(B->getIndex()) - @@ -81,6 +86,7 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other, } } } + return false; } @@ -91,10 +97,13 @@ bool BaseIndexOffset::computeAliasing(const SDNode *Op0, const SelectionDAG &DAG, bool &IsAlias) { BaseIndexOffset BasePtr0 = match(Op0, DAG); - BaseIndexOffset BasePtr1 = match(Op1, DAG); + if (!BasePtr0.getBase().getNode()) + return false; - if (!(BasePtr0.getBase().getNode() && BasePtr1.getBase().getNode())) + BaseIndexOffset BasePtr1 = match(Op1, DAG); + if (!BasePtr1.getBase().getNode()) return false; + int64_t PtrDiff; if (NumBytes0 && NumBytes1 && BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) { diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 3c4b285cb067..2c477b947430 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3354,6 +3354,8 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) { } if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) Flags.setExact(ExactOp->isExact()); + if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I)) + Flags.setDisjoint(DisjointOp->isDisjoint()); if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) Flags.copyFMF(*FPOp); @@ -4112,7 +4114,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS); MVT IdxTy = MVT::getIntegerVT(IdxSize); TypeSize ElementSize = - DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType()); + GTI.getSequentialElementStride(DAG.getDataLayout()); // We intentionally mask away the high bits here; ElementSize may not // fit in IdxTy. APInt ElementMul(IdxSize, ElementSize.getKnownMinValue()); @@ -5642,7 +5644,7 @@ static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, // expansion/promotion) if it was possible to expand a libcall of an // illegal type during operation legalization. But it's not, so things // get a bit hacky. - unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue(); + unsigned ScaleInt = Scale->getAsZExtVal(); if ((ScaleInt > 0 || (Saturating && Signed)) && (TLI.isTypeLegal(VT) || (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) { @@ -7655,8 +7657,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, // suitable for the target. Convert the index as required. MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); if (Index.getValueType() != VectorIdxTy) - Index = DAG.getVectorIdxConstant( - cast<ConstantSDNode>(Index)->getZExtValue(), sdl); + Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl); EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec, @@ -7672,8 +7673,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, // suitable for the target. Convert the index as required. MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); if (Index.getValueType() != VectorIdxTy) - Index = DAG.getVectorIdxConstant( - cast<ConstantSDNode>(Index)->getZExtValue(), sdl); + Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl); setValue(&I, DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index)); @@ -8136,7 +8136,7 @@ void SelectionDAGBuilder::visitVectorPredicationIntrinsic( case ISD::VP_IS_FPCLASS: { const DataLayout DLayout = DAG.getDataLayout(); EVT DestVT = TLI.getValueType(DLayout, VPIntrin.getType()); - auto Constant = cast<ConstantSDNode>(OpValues[1])->getZExtValue(); + auto Constant = OpValues[1]->getAsZExtVal(); SDValue Check = DAG.getTargetConstant(Constant, DL, MVT::i32); SDValue V = DAG.getNode(ISD::VP_IS_FPCLASS, DL, DestVT, {OpValues[0], Check, OpValues[2], OpValues[3]}); @@ -9173,8 +9173,7 @@ findMatchingInlineAsmOperand(unsigned OperandNo, unsigned CurOp = InlineAsm::Op_FirstOperand; for (; OperandNo; --OperandNo) { // Advance to the next operand. - unsigned OpFlag = - cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); + unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal(); const InlineAsm::Flag F(OpFlag); assert( (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) && @@ -9480,8 +9479,7 @@ void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call, // just use its register. auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(), AsmNodeOperands); - InlineAsm::Flag Flag( - cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue()); + InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal()); if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) { if (OpInfo.isIndirect) { // This happens on gcc/testsuite/gcc.dg/pr8788-1.c @@ -9985,14 +9983,14 @@ void SelectionDAGBuilder::visitStackmap(const CallInst &CI) { // constant nodes. SDValue ID = getValue(CI.getArgOperand(0)); assert(ID.getValueType() == MVT::i64); - SDValue IDConst = DAG.getTargetConstant( - cast<ConstantSDNode>(ID)->getZExtValue(), DL, ID.getValueType()); + SDValue IDConst = + DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType()); Ops.push_back(IDConst); SDValue Shad = getValue(CI.getArgOperand(1)); assert(Shad.getValueType() == MVT::i32); - SDValue ShadConst = DAG.getTargetConstant( - cast<ConstantSDNode>(Shad)->getZExtValue(), DL, Shad.getValueType()); + SDValue ShadConst = + DAG.getTargetConstant(Shad->getAsZExtVal(), DL, Shad.getValueType()); Ops.push_back(ShadConst); // Add the live variables. @@ -10041,7 +10039,7 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB, // Get the real number of arguments participating in the call <numArgs> SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos)); - unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue(); + unsigned NumArgs = NArgVal->getAsZExtVal(); // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs> // Intrinsics include all meta-operands up to but not including CC. @@ -10088,12 +10086,9 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB, // Add the <id> and <numBytes> constants. SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos)); - Ops.push_back(DAG.getTargetConstant( - cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64)); + Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64)); SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos)); - Ops.push_back(DAG.getTargetConstant( - cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl, - MVT::i32)); + Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32)); // Add the callee. Ops.push_back(Callee); @@ -11637,92 +11632,16 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, } } -unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC, - CaseClusterIt First, - CaseClusterIt Last) { - return std::count_if(First, Last + 1, [&](const CaseCluster &X) { - if (X.Prob != CC.Prob) - return X.Prob > CC.Prob; - - // Ties are broken by comparing the case value. - return X.Low->getValue().slt(CC.Low->getValue()); - }); -} - void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList, const SwitchWorkListItem &W, Value *Cond, MachineBasicBlock *SwitchMBB) { assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) && "Clusters not sorted?"); - assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!"); - // Balance the tree based on branch probabilities to create a near-optimal (in - // terms of search time given key frequency) binary search tree. See e.g. Kurt - // Mehlhorn "Nearly Optimal Binary Search Trees" (1975). - CaseClusterIt LastLeft = W.FirstCluster; - CaseClusterIt FirstRight = W.LastCluster; - auto LeftProb = LastLeft->Prob + W.DefaultProb / 2; - auto RightProb = FirstRight->Prob + W.DefaultProb / 2; - - // Move LastLeft and FirstRight towards each other from opposite directions to - // find a partitioning of the clusters which balances the probability on both - // sides. If LeftProb and RightProb are equal, alternate which side is - // taken to ensure 0-probability nodes are distributed evenly. - unsigned I = 0; - while (LastLeft + 1 < FirstRight) { - if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1))) - LeftProb += (++LastLeft)->Prob; - else - RightProb += (--FirstRight)->Prob; - I++; - } - - while (true) { - // Our binary search tree differs from a typical BST in that ours can have up - // to three values in each leaf. The pivot selection above doesn't take that - // into account, which means the tree might require more nodes and be less - // efficient. We compensate for this here. - - unsigned NumLeft = LastLeft - W.FirstCluster + 1; - unsigned NumRight = W.LastCluster - FirstRight + 1; - - if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) { - // If one side has less than 3 clusters, and the other has more than 3, - // consider taking a cluster from the other side. - - if (NumLeft < NumRight) { - // Consider moving the first cluster on the right to the left side. - CaseCluster &CC = *FirstRight; - unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster); - unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft); - if (LeftSideRank <= RightSideRank) { - // Moving the cluster to the left does not demote it. - ++LastLeft; - ++FirstRight; - continue; - } - } else { - assert(NumRight < NumLeft); - // Consider moving the last element on the left to the right side. - CaseCluster &CC = *LastLeft; - unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft); - unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster); - if (RightSideRank <= LeftSideRank) { - // Moving the cluster to the right does not demot it. - --LastLeft; - --FirstRight; - continue; - } - } - } - break; - } - - assert(LastLeft + 1 == FirstRight); - assert(LastLeft >= W.FirstCluster); - assert(FirstRight <= W.LastCluster); + auto [LastLeft, FirstRight, LeftProb, RightProb] = + SL->computeSplitWorkItemInfo(W); // Use the first element on the right as pivot since we will make less-than // comparisons against it. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 2e102c002c09..6dcb8c816ad0 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -200,12 +200,6 @@ private: /// create. unsigned SDNodeOrder; - /// Determine the rank by weight of CC in [First,Last]. If CC has more weight - /// than each cluster in the range, its rank is 0. - unsigned caseClusterRank(const SwitchCG::CaseCluster &CC, - SwitchCG::CaseClusterIt First, - SwitchCG::CaseClusterIt Last); - /// Emit comparison and split W into two subtrees. void splitWorkItem(SwitchCG::SwitchWorkList &WorkList, const SwitchCG::SwitchWorkListItem &W, Value *Cond, diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 78cc60084068..9ebef642e423 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -149,7 +149,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::INTRINSIC_VOID: case ISD::INTRINSIC_W_CHAIN: { unsigned OpNo = getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1; - unsigned IID = cast<ConstantSDNode>(getOperand(OpNo))->getZExtValue(); + unsigned IID = getOperand(OpNo)->getAsZExtVal(); if (IID < Intrinsic::num_intrinsics) return Intrinsic::getBaseName((Intrinsic::ID)IID).str(); if (!G) @@ -597,6 +597,9 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { if (getFlags().hasExact()) OS << " exact"; + if (getFlags().hasDisjoint()) + OS << " disjoint"; + if (getFlags().hasNonNeg()) OS << " nneg"; diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index f28211ac113c..9acfc76d7d5e 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -1614,6 +1614,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { if (isFoldedOrDeadInstruction(Inst, *FuncInfo) || ElidedArgCopyInstrs.count(Inst)) { --NumFastIselRemaining; + FastIS->handleDbgInfo(Inst); continue; } @@ -1625,6 +1626,8 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { if (FastIS->selectInstruction(Inst)) { --NumFastIselRemaining; ++NumFastIselSuccess; + + FastIS->handleDbgInfo(Inst); // If fast isel succeeded, skip over all the folded instructions, and // then see if there is a load right before the selected instructions. // Try to fold the load if so. @@ -1640,6 +1643,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // If we succeeded, don't re-select the load. LLVM_DEBUG(dbgs() << "FastISel folded load: " << *BeforeInst << "\n"); + FastIS->handleDbgInfo(BeforeInst); BI = std::next(BasicBlock::const_iterator(BeforeInst)); --NumFastIselRemaining; ++NumFastIselSuccess; @@ -2121,7 +2125,7 @@ void SelectionDAGISel::SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, --e; // Don't process a glue operand if it is here. while (i != e) { - InlineAsm::Flag Flags(cast<ConstantSDNode>(InOps[i])->getZExtValue()); + InlineAsm::Flag Flags(InOps[i]->getAsZExtVal()); if (!Flags.isMemKind() && !Flags.isFuncKind()) { // Just skip over this operand, copying the operands verbatim. Ops.insert(Ops.end(), InOps.begin() + i, @@ -2135,12 +2139,10 @@ void SelectionDAGISel::SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, if (Flags.isUseOperandTiedToDef(TiedToOperand)) { // We need the constraint ID from the operand this is tied to. unsigned CurOp = InlineAsm::Op_FirstOperand; - Flags = - InlineAsm::Flag(cast<ConstantSDNode>(InOps[CurOp])->getZExtValue()); + Flags = InlineAsm::Flag(InOps[CurOp]->getAsZExtVal()); for (; TiedToOperand; --TiedToOperand) { CurOp += Flags.getNumOperandRegisters() + 1; - Flags = InlineAsm::Flag( - cast<ConstantSDNode>(InOps[CurOp])->getZExtValue()); + Flags = InlineAsm::Flag(InOps[CurOp]->getAsZExtVal()); } } @@ -2380,9 +2382,8 @@ void SelectionDAGISel::pushStackMapLiveVariable(SmallVectorImpl<SDValue> &Ops, if (OpNode->getOpcode() == ISD::Constant) { Ops.push_back( CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64)); - Ops.push_back( - CurDAG->getTargetConstant(cast<ConstantSDNode>(OpNode)->getZExtValue(), - DL, OpVal.getValueType())); + Ops.push_back(CurDAG->getTargetConstant(OpNode->getAsZExtVal(), DL, + OpVal.getValueType())); } else { Ops.push_back(OpVal); } @@ -2452,7 +2453,7 @@ void SelectionDAGISel::Select_PATCHPOINT(SDNode *N) { Ops.push_back(*It++); // Push the args for the call. - for (uint64_t I = cast<ConstantSDNode>(NumArgs)->getZExtValue(); I != 0; I--) + for (uint64_t I = NumArgs->getAsZExtVal(); I != 0; I--) Ops.push_back(*It++); // Now push the live variables. diff --git a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index c5977546828f..e3e3e375d6a6 100644 --- a/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/contrib/llvm-project/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -1064,10 +1064,9 @@ static SDValue combineShiftToAVG(SDValue Op, SelectionDAG &DAG, SDLoc DL(Op); SDValue ResultAVG = - DAG.getNode(AVGOpc, DL, NVT, DAG.getNode(ISD::TRUNCATE, DL, NVT, ExtOpA), - DAG.getNode(ISD::TRUNCATE, DL, NVT, ExtOpB)); - return DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, VT, - ResultAVG); + DAG.getNode(AVGOpc, DL, NVT, DAG.getExtOrTrunc(IsSigned, ExtOpA, DL, NVT), + DAG.getExtOrTrunc(IsSigned, ExtOpB, DL, NVT)); + return DAG.getExtOrTrunc(IsSigned, ResultAVG, DL, VT); } /// Look at Op. At this point, we know that only the OriginalDemandedBits of the @@ -1468,14 +1467,24 @@ bool TargetLowering::SimplifyDemandedBits( case ISD::OR: { SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); - + SDNodeFlags Flags = Op.getNode()->getFlags(); if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, - Depth + 1)) + Depth + 1)) { + if (Flags.hasDisjoint()) { + Flags.setDisjoint(false); + Op->setFlags(Flags); + } return true; + } assert(!Known.hasConflict() && "Bits known to be one AND zero?"); if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts, - Known2, TLO, Depth + 1)) + Known2, TLO, Depth + 1)) { + if (Flags.hasDisjoint()) { + Flags.setDisjoint(false); + Op->setFlags(Flags); + } return true; + } assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); // If all of the demanded bits are known zero on one side, return the other. @@ -1636,11 +1645,11 @@ bool TargetLowering::SimplifyDemandedBits( break; } case ISD::SELECT: - if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO, - Depth + 1)) + if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts, + Known, TLO, Depth + 1)) return true; - if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO, - Depth + 1)) + if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, DemandedElts, + Known2, TLO, Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); @@ -1666,11 +1675,11 @@ bool TargetLowering::SimplifyDemandedBits( Known = Known.intersectWith(Known2); break; case ISD::SELECT_CC: - if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO, - Depth + 1)) + if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, DemandedElts, + Known, TLO, Depth + 1)) return true; - if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO, - Depth + 1)) + if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, DemandedElts, + Known2, TLO, Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); @@ -2435,6 +2444,13 @@ bool TargetLowering::SimplifyDemandedBits( unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1; bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG; + APInt InDemandedElts = DemandedElts.zext(InElts); + APInt InDemandedBits = DemandedBits.trunc(InBits); + + // Since some of the sign extended bits are demanded, we know that the sign + // bit is demanded. + InDemandedBits.setBit(InBits - 1); + // If none of the top bits are demanded, convert this into an any_extend. if (DemandedBits.getActiveBits() <= InBits) { // If we only need the non-extended bits of the bottom element @@ -2443,19 +2459,17 @@ bool TargetLowering::SimplifyDemandedBits( VT.getSizeInBits() == SrcVT.getSizeInBits()) return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); - unsigned Opc = - IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND; - if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) - return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); + // Don't lose an all signbits 0/-1 splat on targets with 0/-1 booleans. + if (getBooleanContents(VT) != ZeroOrNegativeOneBooleanContent || + TLO.DAG.ComputeNumSignBits(Src, InDemandedElts, Depth + 1) != + InBits) { + unsigned Opc = + IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND; + if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) + return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); + } } - APInt InDemandedBits = DemandedBits.trunc(InBits); - APInt InDemandedElts = DemandedElts.zext(InElts); - - // Since some of the sign extended bits are demanded, we know that the sign - // bit is demanded. - InDemandedBits.setBit(InBits - 1); - if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO, Depth + 1)) return true; @@ -3344,8 +3358,8 @@ bool TargetLowering::SimplifyDemandedVectorElts( // Try to transform the select condition based on the current demanded // elements. - APInt UndefSel, UndefZero; - if (SimplifyDemandedVectorElts(Sel, DemandedElts, UndefSel, UndefZero, TLO, + APInt UndefSel, ZeroSel; + if (SimplifyDemandedVectorElts(Sel, DemandedElts, UndefSel, ZeroSel, TLO, Depth + 1)) return true; @@ -3368,7 +3382,7 @@ bool TargetLowering::SimplifyDemandedVectorElts( // select value element. APInt DemandedSel = DemandedElts & ~KnownZero; if (DemandedSel != DemandedElts) - if (SimplifyDemandedVectorElts(Sel, DemandedSel, UndefSel, UndefZero, TLO, + if (SimplifyDemandedVectorElts(Sel, DemandedSel, UndefSel, ZeroSel, TLO, Depth + 1)) return true; |