diff options
author | Bartosz Sobczak <bartosz.sobczak@intel.com> | 2024-02-16 21:55:08 +0000 |
---|---|---|
committer | Eric Joyner <erj@FreeBSD.org> | 2024-02-23 22:25:31 +0000 |
commit | b33556f035b4cf78b848f256d6218f49ab99c509 (patch) | |
tree | 4ea14082d302faa5171007ab267f5146da1b8c54 | |
parent | 6831288c8c4091c533c590d2d3cbb07459f2d28b (diff) | |
download | src-b33556f035b4cf78b848f256d6218f49ab99c509.tar.gz src-b33556f035b4cf78b848f256d6218f49ab99c509.zip |
irdma(4): Upgrade to 1.2.36-k
Update Intel irdma driver to version 1.2.36-k.
Notable changes:
- Start using ib_sge directly instead of irdma_sge
- Turn off flush completion generator for libirdma
- Minor formatting changes
Signed-off-by: Bartosz Sobczak <bartosz.sobczak@intel.com>
Signed-off-by: Eric Joyner <erj@FreeBSD.org>
Reviewed by: erj@
Approved by: re (cperciva@)
Sponsored by: Intel Corporation
Differential Revision: https://reviews.freebsd.org/D43567
(cherry picked from commit 5b5f7d0e77a9eee73eb5d596f43aef4e1a3674d8)
(cherry picked from commit a4510dafa8045bfa521303db7f23123a47d8749f)
37 files changed, 424 insertions, 791 deletions
diff --git a/contrib/ofed/libirdma/abi.h b/contrib/ofed/libirdma/abi.h index 1d84fbc747f6..a3e159828bec 100644 --- a/contrib/ofed/libirdma/abi.h +++ b/contrib/ofed/libirdma/abi.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (C) 2019 - 2022 Intel Corporation + * Copyright (C) 2019 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -85,6 +85,7 @@ struct irdma_ucreate_qp { struct ibv_create_qp ibv_cmd; __aligned_u64 user_wqe_bufs; __aligned_u64 user_compl_ctx; + __aligned_u64 comp_mask; }; struct irdma_ucreate_qp_resp { @@ -97,6 +98,9 @@ struct irdma_ucreate_qp_resp { __u8 lsmm; __u8 rsvd; __u32 qp_caps; + __aligned_u64 comp_mask; + __u8 start_wqe_idx; + __u8 rsvd2[7]; }; struct irdma_umodify_qp_resp { @@ -137,6 +141,8 @@ struct irdma_get_context_resp { __u8 hw_rev; __u8 rsvd2; __aligned_u64 comp_mask; + __u16 min_hw_wq_size; + __u8 rsvd3[6]; }; struct irdma_ureg_mr { diff --git a/contrib/ofed/libirdma/irdma-abi.h b/contrib/ofed/libirdma/irdma-abi.h index 8c04dcce2e8b..ae805919ea55 100644 --- a/contrib/ofed/libirdma/irdma-abi.h +++ b/contrib/ofed/libirdma/irdma-abi.h @@ -54,6 +54,11 @@ enum irdma_memreg_type { enum { IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0, + IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1, +}; + +enum { + IRDMA_CREATE_QP_USE_START_WQE_IDX = 1 << 0, }; struct irdma_alloc_ucontext_req { @@ -82,6 +87,8 @@ struct irdma_alloc_ucontext_resp { __u8 hw_rev; __u8 rsvd2; __aligned_u64 comp_mask; + __u16 min_hw_wq_size; + __u8 rsvd3[6]; }; struct irdma_alloc_pd_resp { @@ -101,6 +108,7 @@ struct irdma_create_cq_req { struct irdma_create_qp_req { __aligned_u64 user_wqe_bufs; __aligned_u64 user_compl_ctx; + __aligned_u64 comp_mask; }; struct irdma_mem_reg_req { @@ -130,6 +138,9 @@ struct irdma_create_qp_resp { __u8 lsmm; __u8 rsvd; __u32 qp_caps; + __aligned_u64 comp_mask; + __u8 start_wqe_idx; + __u8 rsvd2[7]; }; struct irdma_modify_qp_resp { diff --git a/contrib/ofed/libirdma/irdma.h b/contrib/ofed/libirdma/irdma.h index 0e2a5189acd1..f4a5a4796f82 100644 --- a/contrib/ofed/libirdma/irdma.h +++ b/contrib/ofed/libirdma/irdma.h @@ -46,7 +46,6 @@ enum irdma_vers { IRDMA_GEN_RSVD = 0, IRDMA_GEN_1 = 1, IRDMA_GEN_2 = 2, - IRDMA_GEN_MAX = 2, }; struct irdma_uk_attrs { diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h index da9eea472568..39d4e7772c31 100644 --- a/contrib/ofed/libirdma/irdma_defs.h +++ b/contrib/ofed/libirdma/irdma_defs.h @@ -456,19 +456,6 @@ enum irdma_qp_wqe_size { IRDMA_WQE_SIZE_256 = 256, }; -enum irdma_ws_op_type { - IRDMA_WS_OP_TYPE_NODE = 0, - IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP, -}; - -enum irdma_ws_rate_limit_flags { - IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1, - IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2, - IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4, - IRDMA_WS_TREE_RATE_LIMITING = 0x8, - IRDMA_WS_PACING_CONTROL = 0x10, -}; - /** * set_64bit_val - set 64 bit value to hw wqe * @wqe_words: wqe addr to write diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c index 5fa9d792745f..115c5f0a27f0 100644 --- a/contrib/ofed/libirdma/irdma_uk.c +++ b/contrib/ofed/libirdma/irdma_uk.c @@ -45,16 +45,16 @@ * @valid: The wqe valid */ static void -irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge, +irdma_set_fragment(__le64 * wqe, u32 offset, struct ibv_sge *sge, u8 valid) { if (sge) { set_64bit_val(wqe, offset, - FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr)); set_64bit_val(wqe, offset + IRDMA_BYTE_8, FIELD_PREP(IRDMAQPSQ_VALID, valid) | - FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) | - FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag)); + FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) | + FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey)); } else { set_64bit_val(wqe, offset, 0); set_64bit_val(wqe, offset + IRDMA_BYTE_8, @@ -71,14 +71,14 @@ irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge, */ static void irdma_set_fragment_gen_1(__le64 * wqe, u32 offset, - struct irdma_sge *sge, u8 valid) + struct ibv_sge *sge, u8 valid) { if (sge) { set_64bit_val(wqe, offset, - FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr)); set_64bit_val(wqe, offset + IRDMA_BYTE_8, - FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) | - FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag)); + FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) | + FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey)); } else { set_64bit_val(wqe, offset, 0); set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0); @@ -209,8 +209,7 @@ irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 * wqe, u16 quanta, if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) != IRDMA_RING_CURRENT_TAIL(qp->sq_ring) && !qp->push_mode) { - if (post_sq) - irdma_uk_qp_post_wr(qp); + irdma_uk_qp_post_wr(qp); } else { push = (__le64 *) ((uintptr_t)qp->push_wqe + (wqe_idx & 0x7) * 0x20); @@ -338,7 +337,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, return EINVAL; for (i = 0; i < op_info->num_lo_sges; i++) - total_size += op_info->lo_sg_list[i].len; + total_size += op_info->lo_sg_list[i].length; read_fence |= info->read_fence; @@ -357,7 +356,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; set_64bit_val(wqe, IRDMA_BYTE_16, - FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); if (info->imm_data_valid) { set_64bit_val(wqe, IRDMA_BYTE_0, @@ -386,7 +385,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, ++addl_frag_cnt; } - hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | + hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) | FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) | @@ -437,7 +436,7 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, return EINVAL; for (i = 0; i < op_info->num_lo_sges; i++) - total_size += op_info->lo_sg_list[i].len; + total_size += op_info->lo_sg_list[i].length; ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta); if (ret_code) @@ -475,8 +474,8 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, ++addl_frag_cnt; } set_64bit_val(wqe, IRDMA_BYTE_16, - FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); - hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | + FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); + hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | FIELD_PREP(IRDMAQPSQ_OPCODE, @@ -525,7 +524,7 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, return EINVAL; for (i = 0; i < op_info->num_sges; i++) - total_size += op_info->sg_list[i].len; + total_size += op_info->sg_list[i].length; if (info->imm_data_valid) frag_cnt = op_info->num_sges + 1; @@ -620,15 +619,15 @@ irdma_set_mw_bind_wqe_gen_1(__le64 * wqe, * @polarity: compatibility parameter */ static void -irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list, +irdma_copy_inline_data_gen_1(u8 *wqe, struct ibv_sge *sge_list, u32 num_sges, u8 polarity) { u32 quanta_bytes_remaining = 16; u32 i; for (i = 0; i < num_sges; i++) { - u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off; - u32 sge_len = sge_list[i].len; + u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr; + u32 sge_len = sge_list[i].length; while (sge_len) { u32 bytes_copied; @@ -683,7 +682,7 @@ irdma_set_mw_bind_wqe(__le64 * wqe, * @polarity: polarity of wqe valid bit */ static void -irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, +irdma_copy_inline_data(u8 *wqe, struct ibv_sge *sge_list, u32 num_sges, u8 polarity) { u8 inline_valid = polarity << IRDMA_INLINE_VALID_S; @@ -694,8 +693,8 @@ irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, wqe += 8; for (i = 0; i < num_sges; i++) { - u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off; - u32 sge_len = sge_list[i].len; + u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr; + u32 sge_len = sge_list[i].length; while (sge_len) { u32 bytes_copied; @@ -775,7 +774,7 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, return EINVAL; for (i = 0; i < op_info->num_lo_sges; i++) - total_size += op_info->lo_sg_list[i].len; + total_size += op_info->lo_sg_list[i].length; if (unlikely(total_size > qp->max_inline_data)) return EINVAL; @@ -788,9 +787,9 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; read_fence |= info->read_fence; set_64bit_val(wqe, IRDMA_BYTE_16, - FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); - hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | + hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) | FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) | @@ -846,7 +845,7 @@ irdma_uk_inline_send(struct irdma_qp_uk *qp, return EINVAL; for (i = 0; i < op_info->num_sges; i++) - total_size += op_info->sg_list[i].len; + total_size += op_info->sg_list[i].length; if (unlikely(total_size > qp->max_inline_data)) return EINVAL; @@ -911,7 +910,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, u64 hdr; u32 wqe_idx; bool local_fence = false; - struct irdma_sge sge = {0}; + struct ibv_sge sge = {0}; u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; info->push_wqe = qp->push_db ? true : false; @@ -922,7 +921,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, if (!wqe) return ENOSPC; - sge.stag = op_info->target_stag; + sge.lkey = op_info->target_stag; qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0); set_64bit_val(wqe, IRDMA_BYTE_16, 0); @@ -1436,8 +1435,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, IRDMA_RING_MOVE_TAIL(cq->cq_ring); set_64bit_val(cq->shadow_area, IRDMA_BYTE_0, IRDMA_RING_CURRENT_HEAD(cq->cq_ring)); - memset(info, 0, - sizeof(struct irdma_cq_poll_info)); + memset(info, 0, sizeof(*info)); return irdma_uk_cq_poll_cmpl(cq, info); } } @@ -1510,7 +1508,6 @@ exit: if (pring && IRDMA_RING_MORE_WORK(*pring)) move_cq_head = false; } - if (move_cq_head) { IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring)) @@ -1591,10 +1588,12 @@ irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge, int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth) { + u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; + *sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD); - if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift)) - *sqdepth = uk_attrs->min_hw_wq_size << shift; + if (*sqdepth < min_size) + *sqdepth = min_size; else if (*sqdepth > uk_attrs->max_hw_wq_quanta) return EINVAL; @@ -1608,10 +1607,12 @@ irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *s int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth) { + u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; + *rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD); - if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift)) - *rqdepth = uk_attrs->min_hw_wq_size << shift; + if (*rqdepth < min_size) + *rqdepth = min_size; else if (*rqdepth > uk_attrs->max_hw_rq_quanta) return EINVAL; @@ -1644,9 +1645,10 @@ irdma_setup_connection_wqes(struct irdma_qp_uk *qp, { u16 move_cnt = 1; - if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE) + if (info->start_wqe_idx) + move_cnt = info->start_wqe_idx; + else if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE) move_cnt = 3; - qp->conn_wqes = move_cnt; IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt); IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt); @@ -1654,32 +1656,6 @@ irdma_setup_connection_wqes(struct irdma_qp_uk *qp, } /** - * irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ - * @ukinfo: qp initialization info - * @sq_shift: Returns shift of SQ - * @rq_shift: Returns shift of RQ - */ -void -irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift, - u8 *rq_shift) -{ - bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false; - - irdma_get_wqe_shift(ukinfo->uk_attrs, - imm_support ? ukinfo->max_sq_frag_cnt + 1 : - ukinfo->max_sq_frag_cnt, - ukinfo->max_inline_data, sq_shift); - - irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0, - rq_shift); - - if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) { - if (ukinfo->abi_ver > 4) - *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; - } -} - -/** * irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size. * @ukinfo: qp initialization info * @sq_depth: Returns depth of SQ @@ -1786,6 +1762,8 @@ irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info) qp->wqe_ops = iw_wqe_uk_ops_gen_1; else qp->wqe_ops = iw_wqe_uk_ops; + qp->start_wqe_idx = info->start_wqe_idx; + return ret_code; } diff --git a/contrib/ofed/libirdma/irdma_umain.c b/contrib/ofed/libirdma/irdma_umain.c index 2902dd963e18..e8d27c31a0dc 100644 --- a/contrib/ofed/libirdma/irdma_umain.c +++ b/contrib/ofed/libirdma/irdma_umain.c @@ -48,7 +48,7 @@ /** * Driver version */ -char libirdma_version[] = "1.2.17-k"; +char libirdma_version[] = "1.2.36-k"; unsigned int irdma_dbg; diff --git a/contrib/ofed/libirdma/irdma_umain.h b/contrib/ofed/libirdma/irdma_umain.h index 1fa18dd7a847..269609f8c77f 100644 --- a/contrib/ofed/libirdma/irdma_umain.h +++ b/contrib/ofed/libirdma/irdma_umain.h @@ -96,7 +96,6 @@ struct irdma_cq_buf { LIST_ENTRY(irdma_cq_buf) list; struct irdma_cq_uk cq; struct verbs_mr vmr; - size_t buf_size; }; extern pthread_mutex_t sigusr1_wait_mutex; @@ -142,7 +141,6 @@ struct irdma_uqp { struct ibv_recv_wr *pend_rx_wr; struct irdma_qp_uk qp; enum ibv_qp_type qp_type; - struct irdma_sge *recv_sges; }; /* irdma_uverbs.c */ diff --git a/contrib/ofed/libirdma/irdma_user.h b/contrib/ofed/libirdma/irdma_user.h index 0f0aa30bec91..aeb6aa9feebd 100644 --- a/contrib/ofed/libirdma/irdma_user.h +++ b/contrib/ofed/libirdma/irdma_user.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -50,7 +50,7 @@ #define irdma_access_privileges u32 #define irdma_physical_fragment u64 #define irdma_address_list u64 * -#define irdma_sgl struct irdma_sge * +#define irdma_sgl struct ibv_sge * #define IRDMA_MAX_MR_SIZE 0x200000000000ULL @@ -80,96 +80,6 @@ #define IRDMA_OP_TYPE_REC_IMM 0x3f #define IRDMA_FLUSH_MAJOR_ERR 1 -#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe - -/* Async Events codes */ -#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102 -#define IRDMA_AE_AMP_INVALID_STAG 0x0103 -#define IRDMA_AE_AMP_BAD_QP 0x0104 -#define IRDMA_AE_AMP_BAD_PD 0x0105 -#define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106 -#define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107 -#define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108 -#define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109 -#define IRDMA_AE_AMP_TO_WRAP 0x010a -#define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c -#define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d -#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e -#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110 -#define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111 -#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112 -#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113 -#define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114 -#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115 -#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116 -#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117 -#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118 -#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119 -#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a -#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b -#define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c -#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d -#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e -#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f -#define IRDMA_AE_AMP_MWBIND_WRONG_TYPE 0x0120 -#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121 -#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132 -#define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133 -#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134 -#define IRDMA_AE_UDA_L4LEN_INVALID 0x0135 -#define IRDMA_AE_BAD_CLOSE 0x0201 -#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202 -#define IRDMA_AE_CQ_OPERATION_ERROR 0x0203 -#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205 -#define IRDMA_AE_STAG_ZERO_INVALID 0x0206 -#define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207 -#define IRDMA_AE_IB_INVALID_REQUEST 0x0208 -#define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a -#define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b -#define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c -#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d -#define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e -#define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220 -#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301 -#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303 -#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304 -#define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305 -#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306 -#define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307 -#define IRDMA_AE_DDP_NO_L_BIT 0x0308 -#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311 -#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312 -#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313 -#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314 -#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316 -#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380 -#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381 -#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382 -#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383 -#define IRDMA_AE_INVALID_ARP_ENTRY 0x0401 -#define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402 -#define IRDMA_AE_STALE_ARP_ENTRY 0x0403 -#define IRDMA_AE_INVALID_AH_ENTRY 0x0406 -#define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501 -#define IRDMA_AE_LLP_CONNECTION_RESET 0x0502 -#define IRDMA_AE_LLP_FIN_RECEIVED 0x0503 -#define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504 -#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505 -#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507 -#define IRDMA_AE_LLP_SYN_RECEIVED 0x0508 -#define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509 -#define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a -#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b -#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c -#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e -#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520 -#define IRDMA_AE_RESET_SENT 0x0601 -#define IRDMA_AE_TERMINATE_SENT 0x0602 -#define IRDMA_AE_RESET_NOT_SENT 0x0603 -#define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700 -#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701 -#define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702 -#define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900 enum irdma_device_caps_const { IRDMA_WQE_SIZE = 4, @@ -201,8 +111,7 @@ enum irdma_device_caps_const { IRDMA_MAX_OUTBOUND_MSG_SIZE = 65537, /* 64K +1 */ IRDMA_MAX_INBOUND_MSG_SIZE = 65537, - IRDMA_MAX_PUSH_PAGE_COUNT = 1024, - IRDMA_MAX_PE_ENA_VF_COUNT = 32, + IRDMA_MAX_PE_ENA_VF_COUNT = 32, IRDMA_MAX_VF_FPM_ID = 47, IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496, IRDMA_MAX_INLINE_DATA_SIZE = 101, @@ -229,12 +138,7 @@ enum irdma_flush_opcode { FLUSH_RETRY_EXC_ERR, FLUSH_MW_BIND_ERR, FLUSH_REM_INV_REQ_ERR, -}; - -enum irdma_qp_event_type { - IRDMA_QP_EVENT_CATASTROPHIC, - IRDMA_QP_EVENT_ACCESS_ERR, - IRDMA_QP_EVENT_REQ_ERR, + FLUSH_RNR_RETRY_EXC_ERR, }; enum irdma_cmpl_status { @@ -282,12 +186,6 @@ struct irdma_cq_uk; struct irdma_qp_uk_init_info; struct irdma_cq_uk_init_info; -struct irdma_sge { - irdma_tagged_offset tag_off; - u32 len; - irdma_stag stag; -}; - struct irdma_ring { volatile u32 head; volatile u32 tail; /* effective tail */ @@ -319,13 +217,13 @@ struct irdma_post_rq_info { struct irdma_rdma_write { irdma_sgl lo_sg_list; u32 num_lo_sges; - struct irdma_sge rem_addr; + struct ibv_sge rem_addr; }; struct irdma_rdma_read { irdma_sgl lo_sg_list; u32 num_lo_sges; - struct irdma_sge rem_addr; + struct ibv_sge rem_addr; }; struct irdma_bind_window { @@ -399,11 +297,6 @@ struct irdma_cq_poll_info { } stat; }; -struct qp_err_code { - enum irdma_flush_opcode flush_code; - enum irdma_qp_event_type event_type; -}; - int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, bool post_sq); int irdma_uk_inline_send(struct irdma_qp_uk *qp, @@ -426,9 +319,9 @@ int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, bool post_sq); struct irdma_wqe_uk_ops { - void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity); + void (*iw_copy_inline_data)(u8 *dest, struct ibv_sge *sge_list, u32 num_sges, u8 polarity); u16 (*iw_inline_data_size_to_quanta)(u32 data_size); - void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge, + void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ibv_sge *sge, u8 valid); void (*iw_set_mw_bind_wqe)(__le64 *wqe, struct irdma_bind_window *op_info); @@ -444,8 +337,6 @@ int irdma_uk_cq_init(struct irdma_cq_uk *cq, struct irdma_cq_uk_init_info *info); int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info); -void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift, - u8 *rq_shift); int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo, u32 *sq_depth, u8 *sq_shift); int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo, @@ -494,6 +385,7 @@ struct irdma_qp_uk { u8 rwqe_polarity; u8 rq_wqe_size; u8 rq_wqe_size_multiplier; + u8 start_wqe_idx; bool deferred_flag:1; bool push_mode:1; /* whether the last post wqe was pushed */ bool push_dropped:1; @@ -541,6 +433,7 @@ struct irdma_qp_uk_init_info { u32 sq_depth; u32 rq_depth; u8 first_sq_wq; + u8 start_wqe_idx; u8 type; u8 sq_shift; u8 rq_shift; @@ -574,75 +467,4 @@ int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u3 void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta, u32 wqe_idx, bool post_sq); void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx); - -static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id) -{ - struct qp_err_code qp_err = { 0 }; - - switch (ae_id) { - case IRDMA_AE_AMP_BOUNDS_VIOLATION: - case IRDMA_AE_AMP_INVALID_STAG: - case IRDMA_AE_AMP_RIGHTS_VIOLATION: - case IRDMA_AE_AMP_UNALLOCATED_STAG: - case IRDMA_AE_AMP_BAD_PD: - case IRDMA_AE_AMP_BAD_QP: - case IRDMA_AE_AMP_BAD_STAG_KEY: - case IRDMA_AE_AMP_BAD_STAG_INDEX: - case IRDMA_AE_AMP_TO_WRAP: - case IRDMA_AE_PRIV_OPERATION_DENIED: - qp_err.flush_code = FLUSH_PROT_ERR; - qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR; - break; - case IRDMA_AE_UDA_XMIT_BAD_PD: - case IRDMA_AE_WQE_UNEXPECTED_OPCODE: - qp_err.flush_code = FLUSH_LOC_QP_OP_ERR; - qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; - break; - case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT: - case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: - case IRDMA_AE_UDA_L4LEN_INVALID: - case IRDMA_AE_DDP_UBE_INVALID_MO: - case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: - qp_err.flush_code = FLUSH_LOC_LEN_ERR; - qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; - break; - case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: - case IRDMA_AE_IB_REMOTE_ACCESS_ERROR: - qp_err.flush_code = FLUSH_REM_ACCESS_ERR; - qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR; - break; - case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS: - case IRDMA_AE_AMP_MWBIND_BIND_DISABLED: - case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS: - case IRDMA_AE_AMP_MWBIND_VALID_STAG: - qp_err.flush_code = FLUSH_MW_BIND_ERR; - qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR; - break; - case IRDMA_AE_LLP_TOO_MANY_RETRIES: - qp_err.flush_code = FLUSH_RETRY_EXC_ERR; - qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; - break; - case IRDMA_AE_IB_INVALID_REQUEST: - qp_err.flush_code = FLUSH_REM_INV_REQ_ERR; - qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR; - break; - case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: - case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: - case IRDMA_AE_ROCE_RSP_LENGTH_ERROR: - case IRDMA_AE_IB_REMOTE_OP_ERROR: - qp_err.flush_code = FLUSH_REM_OP_ERR; - qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; - break; - case IRDMA_AE_LCE_QP_CATASTROPHIC: - qp_err.flush_code = FLUSH_FATAL_ERR; - qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; - break; - default: - qp_err.flush_code = FLUSH_GENERAL_ERR; - qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; - break; - } - - return qp_err; -} #endif /* IRDMA_USER_H */ diff --git a/contrib/ofed/libirdma/irdma_uverbs.c b/contrib/ofed/libirdma/irdma_uverbs.c index 962af399ed31..e52ce1cfa229 100644 --- a/contrib/ofed/libirdma/irdma_uverbs.c +++ b/contrib/ofed/libirdma/irdma_uverbs.c @@ -152,6 +152,7 @@ irdma_ualloc_pd(struct ibv_context *context) err_free: free(iwupd); + errno = err; return NULL; } @@ -163,7 +164,6 @@ err_free: int irdma_ufree_pd(struct ibv_pd *pd) { - struct irdma_uvcontext *iwvctx = container_of(pd->context, struct irdma_uvcontext, ibv_ctx); struct irdma_upd *iwupd; int ret; @@ -374,12 +374,12 @@ irdma_free_hw_buf(void *buf, size_t size) * @cqe_64byte_ena: enable 64byte cqe */ static inline int -get_cq_size(int ncqe, u8 hw_rev, bool cqe_64byte_ena) +get_cq_size(int ncqe, u8 hw_rev) { ncqe++; /* Completions with immediate require 1 extra entry */ - if (!cqe_64byte_ena && hw_rev > IRDMA_GEN_1) + if (hw_rev > IRDMA_GEN_1) ncqe *= 2; if (ncqe < IRDMA_U_MINCQ_SIZE) @@ -388,11 +388,8 @@ get_cq_size(int ncqe, u8 hw_rev, bool cqe_64byte_ena) return ncqe; } -static inline size_t get_cq_total_bytes(u32 cq_size, bool cqe_64byte_ena){ - if (cqe_64byte_ena) - return roundup(cq_size * sizeof(struct irdma_extended_cqe), IRDMA_HW_PAGE_SIZE); - else - return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE); +static inline size_t get_cq_total_bytes(u32 cq_size) { + return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE); } /** @@ -420,7 +417,6 @@ ucreate_cq(struct ibv_context *context, u32 cq_pages; int ret, ncqe; u8 hw_rev; - bool cqe_64byte_ena; iwvctx = container_of(context, struct irdma_uvcontext, ibv_ctx); uk_attrs = &iwvctx->uk_attrs; @@ -454,11 +450,10 @@ ucreate_cq(struct ibv_context *context, return NULL; } - cqe_64byte_ena = uk_attrs->feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? true : false; - info.cq_size = get_cq_size(attr_ex->cqe, hw_rev, cqe_64byte_ena); + info.cq_size = get_cq_size(attr_ex->cqe, hw_rev); + total_size = get_cq_total_bytes(info.cq_size); iwucq->comp_vector = attr_ex->comp_vector; LIST_INIT(&iwucq->resize_list); - total_size = get_cq_total_bytes(info.cq_size, cqe_64byte_ena); cq_pages = total_size >> IRDMA_HW_PAGE_SHIFT; if (!(uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE)) @@ -528,8 +523,6 @@ ucreate_cq(struct ibv_context *context, info.cq_id = resp.cq_id; /* Do not report the CQE's reserved for immediate and burned by HW */ iwucq->verbs_cq.cq.cqe = ncqe; - if (cqe_64byte_ena) - info.avoid_mem_cflct = true; info.cqe_alloc_db = (u32 *)((u8 *)iwvctx->db + IRDMA_DB_CQ_OFFSET); irdma_uk_cq_init(&iwucq->cq, &info); return &iwucq->verbs_cq.cq_ex; @@ -585,7 +578,7 @@ static void irdma_free_cq_buf(struct irdma_cq_buf *cq_buf) { ibv_cmd_dereg_mr(&cq_buf->vmr.ibv_mr); - irdma_free_hw_buf(cq_buf->cq.cq_base, cq_buf->buf_size); + irdma_free_hw_buf(cq_buf->cq.cq_base, get_cq_total_bytes(cq_buf->cq.cq_size)); free(cq_buf); } @@ -1322,6 +1315,8 @@ irdma_vmapped_qp(struct irdma_uqp *iwuqp, struct ibv_pd *pd, cmd.user_wqe_bufs = (__u64) ((uintptr_t)info->sq); cmd.user_compl_ctx = (__u64) (uintptr_t)&iwuqp->qp; + cmd.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX; + ret = ibv_cmd_create_qp(pd, &iwuqp->ibv_qp, attr, &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp, sizeof(struct irdma_ucreate_qp_resp)); @@ -1331,6 +1326,8 @@ irdma_vmapped_qp(struct irdma_uqp *iwuqp, struct ibv_pd *pd, info->sq_size = resp.actual_sq_size; info->rq_size = resp.actual_rq_size; info->first_sq_wq = legacy_mode ? 1 : resp.lsmm; + if (resp.comp_mask & IRDMA_CREATE_QP_USE_START_WQE_IDX) + info->start_wqe_idx = resp.start_wqe_idx; info->qp_caps = resp.qp_caps; info->qp_id = resp.qp_id; iwuqp->irdma_drv_opt = resp.irdma_drv_opt; @@ -1379,6 +1376,8 @@ irdma_ucreate_qp(struct ibv_pd *pd, if (attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags || + attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta || + attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta || attr->cap.max_inline_data > uk_attrs->max_hw_inline) { errno = EINVAL; return NULL; @@ -1430,18 +1429,12 @@ irdma_ucreate_qp(struct ibv_pd *pd, attr->cap.max_recv_wr = info.rq_size; } - iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges)); - if (!iwuqp->recv_sges) { - status = errno; /* preserve errno */ - goto err_destroy_lock; - } - info.wqe_alloc_db = (u32 *)iwvctx->db; info.legacy_mode = iwvctx->legacy_mode; info.sq_wrtrk_array = calloc(info.sq_depth, sizeof(*info.sq_wrtrk_array)); if (!info.sq_wrtrk_array) { status = errno; /* preserve errno */ - goto err_free_rsges; + goto err_destroy_lock; } info.rq_wrid_array = calloc(info.rq_depth, sizeof(*info.rq_wrid_array)); @@ -1475,8 +1468,6 @@ err_free_rq_wrid: free(info.rq_wrid_array); err_free_sq_wrtrk: free(info.sq_wrtrk_array); -err_free_rsges: - free(iwuqp->recv_sges); err_destroy_lock: pthread_spin_destroy(&iwuqp->lock); err_free_qp: @@ -1635,7 +1626,6 @@ irdma_udestroy_qp(struct ibv_qp *qp) free(iwuqp->qp.rq_wrid_array); irdma_free_hw_buf(iwuqp->qp.sq_base, iwuqp->buf_size); - free(iwuqp->recv_sges); free(iwuqp); return 0; @@ -1646,26 +1636,6 @@ err: } /** - * irdma_copy_sg_list - copy sg list for qp - * @sg_list: copied into sg_list - * @sgl: copy from sgl - * @num_sges: count of sg entries - * @max_sges: count of max supported sg entries - */ -static void -irdma_copy_sg_list(struct irdma_sge *sg_list, struct ibv_sge *sgl, - int num_sges) -{ - int i; - - for (i = 0; i < num_sges; i++) { - sg_list[i].tag_off = sgl[i].addr; - sg_list[i].len = sgl[i].length; - sg_list[i].stag = sgl[i].lkey; - } -} - -/** * calc_type2_mw_stag - calculate type 2 MW stag * @rkey: desired rkey of the MW * @mw_rkey: type2 memory window rkey @@ -1743,7 +1713,7 @@ irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, info.stag_to_inv = ib_wr->imm_data; } info.op.send.num_sges = ib_wr->num_sge; - info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list; + info.op.send.sg_list = (struct ibv_sge *)ib_wr->sg_list; if (ib_qp->qp_type == IBV_QPT_UD) { struct irdma_uah *ah = container_of(ib_wr->wr.ud.ah, struct irdma_uah, ibv_ah); @@ -1774,9 +1744,9 @@ irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, info.op_type = IRDMA_OP_TYPE_RDMA_WRITE; info.op.rdma_write.num_lo_sges = ib_wr->num_sge; - info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list; - info.op.rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr; - info.op.rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey; + info.op.rdma_write.lo_sg_list = ib_wr->sg_list; + info.op.rdma_write.rem_addr.addr = ib_wr->wr.rdma.remote_addr; + info.op.rdma_write.rem_addr.lkey = ib_wr->wr.rdma.rkey; if (ib_wr->send_flags & IBV_SEND_INLINE) err = irdma_uk_inline_rdma_write(&iwuqp->qp, &info, false); else @@ -1788,10 +1758,10 @@ irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, break; } info.op_type = IRDMA_OP_TYPE_RDMA_READ; - info.op.rdma_read.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr; - info.op.rdma_read.rem_addr.stag = ib_wr->wr.rdma.rkey; + info.op.rdma_read.rem_addr.addr = ib_wr->wr.rdma.remote_addr; + info.op.rdma_read.rem_addr.lkey = ib_wr->wr.rdma.rkey; - info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; + info.op.rdma_read.lo_sg_list = ib_wr->sg_list; info.op.rdma_read.num_lo_sges = ib_wr->num_sge; err = irdma_uk_rdma_read(&iwuqp->qp, &info, false, false); break; @@ -1873,14 +1843,11 @@ irdma_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, struct ibv_recv_wr **bad_wr) { struct irdma_post_rq_info post_recv = {}; - struct irdma_sge *sg_list; struct irdma_uqp *iwuqp; bool reflush = false; int err = 0; iwuqp = container_of(ib_qp, struct irdma_uqp, ibv_qp); - sg_list = iwuqp->recv_sges; - err = pthread_spin_lock(&iwuqp->lock); if (err) return err; @@ -1897,8 +1864,7 @@ irdma_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, } post_recv.num_sges = ib_wr->num_sge; post_recv.wr_id = ib_wr->wr_id; - irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge); - post_recv.sg_list = sg_list; + post_recv.sg_list = ib_wr->sg_list; err = irdma_uk_post_receive(&iwuqp->qp, &post_recv); if (err) { *bad_wr = ib_wr; @@ -2022,7 +1988,6 @@ irdma_uresize_cq(struct ibv_cq *cq, int cqe) u32 cq_pages; int cqe_needed; int ret = 0; - bool cqe_64byte_ena; iwucq = container_of(cq, struct irdma_ucq, verbs_cq.cq); iwvctx = container_of(cq->context, struct irdma_uvcontext, ibv_ctx); @@ -2034,14 +1999,11 @@ irdma_uresize_cq(struct ibv_cq *cq, int cqe) if (cqe < uk_attrs->min_hw_cq_size || cqe > uk_attrs->max_hw_cq_size - 1) return EINVAL; - cqe_64byte_ena = uk_attrs->feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? true : false; - - cqe_needed = get_cq_size(cqe, uk_attrs->hw_rev, cqe_64byte_ena); - + cqe_needed = get_cq_size(cqe, uk_attrs->hw_rev); if (cqe_needed == iwucq->cq.cq_size) return 0; - cq_size = get_cq_total_bytes(cqe_needed, cqe_64byte_ena); + cq_size = get_cq_total_bytes(cqe_needed); cq_pages = cq_size >> IRDMA_HW_PAGE_SHIFT; cq_base = irdma_alloc_hw_buf(cq_size); if (!cq_base) @@ -2077,7 +2039,6 @@ irdma_uresize_cq(struct ibv_cq *cq, int cqe) goto err_resize; memcpy(&cq_buf->cq, &iwucq->cq, sizeof(cq_buf->cq)); - cq_buf->buf_size = cq_size; cq_buf->vmr = iwucq->vmr; iwucq->vmr = new_mr; irdma_uk_cq_resize(&iwucq->cq, cq_base, cqe_needed); diff --git a/contrib/ofed/libirdma/osdep.h b/contrib/ofed/libirdma/osdep.h index 961e9cc99cf6..f59cbfdc68b5 100644 --- a/contrib/ofed/libirdma/osdep.h +++ b/contrib/ofed/libirdma/osdep.h @@ -77,7 +77,6 @@ /* constants */ #define STATS_TIMER_DELAY 60000 -/* a couple of linux size defines */ #define BIT_ULL(a) (1ULL << (a)) #define min(a, b) ((a) > (b) ? (b) : (a)) #ifndef likely diff --git a/sys/dev/irdma/fbsd_kcompat.c b/sys/dev/irdma/fbsd_kcompat.c index 19c3943b5151..bf119ba88d07 100644 --- a/sys/dev/irdma/fbsd_kcompat.c +++ b/sys/dev/irdma/fbsd_kcompat.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -660,8 +660,7 @@ static const struct irdma_sw_stats_tunable_info irdma_sws_list[] = { "manage_hmc_pm_func_table", 0}, {IRDMA_OP_SUSPEND, "suspend", "suspend", 0}, {IRDMA_OP_RESUME, "resume", "resume", 0}, - {IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP, "manage_vchnl_req_pble_bp", - "manage_vchnl_req_pble_bp", 0}, + {25, "manage_vchnl_req_pble_bp", "manage_vchnl_req_pble_bp", 0}, {IRDMA_OP_QUERY_FPM_VAL, "query_fpm_val", "query_fpm_val", 0}, {IRDMA_OP_COMMIT_FPM_VAL, "commit_fpm_val", "commit_fpm_val", 0}, {IRDMA_OP_AH_CREATE, "ah_create", "ah_create", 0}, diff --git a/sys/dev/irdma/fbsd_kcompat.h b/sys/dev/irdma/fbsd_kcompat.h index 5e656cd9be9f..dd422a63ccb4 100644 --- a/sys/dev/irdma/fbsd_kcompat.h +++ b/sys/dev/irdma/fbsd_kcompat.h @@ -53,15 +53,12 @@ ibdev.dma_device = (dev) #define set_max_sge(props, rf) \ ((props)->max_sge = (rf)->sc_dev.hw_attrs.uk_attrs.max_hw_wq_frags) -#define rdma_query_gid(ibdev, port, index, gid) \ - ib_get_cached_gid(ibdev, port, index, gid, NULL) #define kmap(pg) page_address(pg) #define kmap_local_page(pg) page_address(pg) #define kunmap(pg) #define kunmap_local(pg) #define IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION IB_CQ_FLAGS_TIMESTAMP_COMPLETION -#define kc_irdma_destroy_qp(ibqp, udata) irdma_destroy_qp(ibqp) #ifndef IB_QP_ATTR_STANDARD_BITS #define IB_QP_ATTR_STANDARD_BITS GENMASK(20, 0) #endif @@ -71,34 +68,26 @@ #define IRDMA_VER_LEN 24 -#ifndef EVNT_HNDLR_CRITERR -#if ICE_RDMA_MAJOR_VERSION == 1 && ICE_RDMA_MINOR_VERSION == 1 -#define EVNT_HNDLR_CRITERR -#else -#undef EVNT_HNDLR_CRITERR -#endif - -#endif void kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev); void kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev); struct irdma_tunable_info { struct sysctl_ctx_list irdma_sysctl_ctx; struct sysctl_oid *irdma_sysctl_tree; + struct sysctl_oid *qos_sysctl_tree; struct sysctl_oid *sws_sysctl_tree; char drv_ver[IRDMA_VER_LEN]; u8 roce_ena; }; typedef u_int if_addr_cb_t(void *, struct ifaddr *, u_int); -u_int if_foreach_addr_type(if_t ifp, int type, if_addr_cb_t cb, void *cb_arg); typedef int (*if_foreach_cb_t)(if_t, void *); -int if_foreach(if_foreach_cb_t cb, void *cb_arg); -#ifndef if_iter struct if_iter { void *context[4]; }; -#endif + +u_int if_foreach_addr_type(if_t ifp, int type, if_addr_cb_t cb, void *cb_arg); +int if_foreach(if_foreach_cb_t cb, void *cb_arg); if_t if_iter_start(struct if_iter *iter); if_t if_iter_next(struct if_iter *iter); void if_iter_finish(struct if_iter *iter); @@ -241,8 +230,8 @@ void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq); int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, struct irdma_device *iwdev); void irdma_setup_virt_qp(struct irdma_device *iwdev, - struct irdma_qp *iwqp, - struct irdma_qp_init_info *init_info); + struct irdma_qp *iwqp, + struct irdma_qp_init_info *init_info); int irdma_setup_kmode_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, struct irdma_qp_init_info *info, @@ -258,7 +247,7 @@ void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, struct irdma_qp_host_ctx_info *ctx_info); int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp); void irdma_dealloc_push_page(struct irdma_pci_f *rf, - struct irdma_sc_qp *qp); + struct irdma_qp *iwqp); int irdma_process_resize_list(struct irdma_cq *iwcq, struct irdma_device *iwdev, struct irdma_cq_buf *lcqe_buf); int irdma_destroy_cq(struct ib_cq *ib_cq); diff --git a/sys/dev/irdma/icrdma.c b/sys/dev/irdma/icrdma.c index 8efd9d5c6034..576a185b013f 100644 --- a/sys/dev/irdma/icrdma.c +++ b/sys/dev/irdma/icrdma.c @@ -52,7 +52,7 @@ /** * Driver version */ -char irdma_driver_version[] = "1.2.17-k"; +char irdma_driver_version[] = "1.2.36-k"; /** * irdma_init_tunable - prepare tunables diff --git a/sys/dev/irdma/icrdma_hw.c b/sys/dev/irdma/icrdma_hw.c index d1da9b81e5b7..a046bf18a616 100644 --- a/sys/dev/irdma/icrdma_hw.c +++ b/sys/dev/irdma/icrdma_hw.c @@ -76,7 +76,7 @@ static u64 icrdma_masks[IRDMA_MAX_MASKS] = { ICRDMA_CQPSQ_CQ_CEQID, ICRDMA_CQPSQ_CQ_CQID, ICRDMA_COMMIT_FPM_CQCNT, - ICRDMA_CQPSQ_UPESD_HMCFNID + ICRDMA_CQPSQ_UPESD_HMCFNID, }; static u8 icrdma_shifts[IRDMA_MAX_SHIFTS] = { @@ -86,7 +86,7 @@ static u8 icrdma_shifts[IRDMA_MAX_SHIFTS] = { ICRDMA_CQPSQ_CQ_CEQID_S, ICRDMA_CQPSQ_CQ_CQID_S, ICRDMA_COMMIT_FPM_CQCNT_S, - ICRDMA_CQPSQ_UPESD_HMCFNID_S + ICRDMA_CQPSQ_UPESD_HMCFNID_S, }; /** @@ -210,8 +210,6 @@ icrdma_init_hw(struct irdma_sc_dev *dev) dev->hw_regs[i] = (u32 IOMEM *) (hw_addr + icrdma_regs[i]); } - dev->hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID; - dev->hw_attrs.first_hw_vf_fpm_id = IRDMA_FIRST_VF_FPM_ID; for (i = 0; i < IRDMA_MAX_SHIFTS; ++i) dev->hw_shifts[i] = icrdma_shifts[i]; @@ -231,6 +229,7 @@ icrdma_init_hw(struct irdma_sc_dev *dev) dev->hw_attrs.max_hw_ord = ICRDMA_MAX_ORD_SIZE; dev->hw_attrs.max_stat_inst = ICRDMA_MAX_STATS_COUNT; dev->hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_2; + dev->hw_attrs.max_hw_device_pages = ICRDMA_MAX_PUSH_PAGE_COUNT; dev->hw_attrs.uk_attrs.max_hw_wq_frags = ICRDMA_MAX_WQ_FRAGMENT_COUNT; dev->hw_attrs.uk_attrs.max_hw_read_sges = ICRDMA_MAX_SGE_RD; diff --git a/sys/dev/irdma/icrdma_hw.h b/sys/dev/irdma/icrdma_hw.h index 8ed50fe6aa48..b413b478538a 100644 --- a/sys/dev/irdma/icrdma_hw.h +++ b/sys/dev/irdma/icrdma_hw.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2017 - 2022 Intel Corporation + * Copyright (c) 2017 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -123,6 +123,7 @@ enum icrdma_device_caps_const { ICRDMA_MAX_IRD_SIZE = 32, ICRDMA_MAX_ORD_SIZE = 32, ICRDMA_MIN_WQ_SIZE = 8 /* WQEs */, + ICRDMA_MAX_PUSH_PAGE_COUNT = 256, }; diff --git a/sys/dev/irdma/irdma-abi.h b/sys/dev/irdma/irdma-abi.h index c9da13ad7831..d19e92d57754 100644 --- a/sys/dev/irdma/irdma-abi.h +++ b/sys/dev/irdma/irdma-abi.h @@ -54,6 +54,11 @@ enum irdma_memreg_type { enum { IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0, + IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1, +}; + +enum { + IRDMA_CREATE_QP_USE_START_WQE_IDX = 1 << 0, }; struct irdma_alloc_ucontext_req { @@ -82,6 +87,8 @@ struct irdma_alloc_ucontext_resp { __u8 hw_rev; __u8 rsvd2; __aligned_u64 comp_mask; + __u16 min_hw_wq_size; + __u8 rsvd3[6]; }; struct irdma_alloc_pd_resp { @@ -101,6 +108,7 @@ struct irdma_create_cq_req { struct irdma_create_qp_req { __aligned_u64 user_wqe_bufs; __aligned_u64 user_compl_ctx; + __aligned_u64 comp_mask; }; struct irdma_mem_reg_req { @@ -130,6 +138,9 @@ struct irdma_create_qp_resp { __u8 lsmm; __u8 rsvd; __u32 qp_caps; + __aligned_u64 comp_mask; + __u8 start_wqe_idx; + __u8 rsvd2[7]; }; struct irdma_modify_qp_resp { diff --git a/sys/dev/irdma/irdma.h b/sys/dev/irdma/irdma.h index 0b8fdf1440a9..e6e493f1854a 100644 --- a/sys/dev/irdma/irdma.h +++ b/sys/dev/irdma/irdma.h @@ -183,7 +183,7 @@ enum irdma_vers { IRDMA_GEN_RSVD = 0, IRDMA_GEN_1 = 1, IRDMA_GEN_2 = 2, - IRDMA_GEN_MAX = 2, + IRDMA_GEN_MAX = IRDMA_GEN_2, }; struct irdma_uk_attrs { diff --git a/sys/dev/irdma/irdma_cm.c b/sys/dev/irdma/irdma_cm.c index d34da57cc84d..cda333d8c549 100644 --- a/sys/dev/irdma/irdma_cm.c +++ b/sys/dev/irdma/irdma_cm.c @@ -1079,14 +1079,14 @@ irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type, *type = IRDMA_MPA_REQUEST_ACCEPT; - if (len < sizeof(struct ietf_mpa_v1)) { + if (len < sizeof(*mpa_frame)) { irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "ietf buffer small (%x)\n", len); return -EINVAL; } mpa_frame = (struct ietf_mpa_v1 *)buf; - mpa_hdr_len = sizeof(struct ietf_mpa_v1); + mpa_hdr_len = sizeof(*mpa_frame); priv_data_len = ntohs(mpa_frame->priv_data_len); if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) { @@ -1616,7 +1616,8 @@ static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4) } /** - * irdma_netdev_vlan_ipv6 - Gets the netdev and mac + * irdma_get_vlan_mac_ipv6 - Get the vlan and mac for an IPv6 + * address * @addr: local IPv6 address * @vlan_id: vlan id for the given IPv6 address * @mac: mac address for the given IPv6 address @@ -1624,14 +1625,12 @@ static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4) * Returns the net_device of the IPv6 address and also sets the * vlan id and mac for that address. */ -if_t -irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac) +void +irdma_get_vlan_mac_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac) { if_t ip_dev = NULL; struct in6_addr laddr6; -#ifdef VIMAGE - struct vnet *vnet = irdma_cmid_to_vnet(cm_id); -#endif + struct vnet *vnet = &init_net; struct ifaddr *ifa; u16 scope_id = 0; @@ -1646,10 +1645,9 @@ irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac) scope_id = ntohs(laddr6.__u6_addr.__u6_addr16[1]); #ifdef VIMAGE - ip_dev = ip6_ifp_find(vnet, laddr6, scope_id); -#else - ip_dev = ip6_ifp_find(&init_net, laddr6, scope_id); + vnet = irdma_cmid_to_vnet(cm_id); #endif + ip_dev = ip6_ifp_find(vnet, laddr6, scope_id); if (ip_dev) { if (vlan_id) *vlan_id = rdma_vlan_dev_vlan_id(ip_dev); @@ -1657,8 +1655,6 @@ irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac) if (ifa && ifa->ifa_addr && mac) ether_addr_copy(mac, if_getlladdr(ip_dev)); } - - return ip_dev; } /** @@ -1669,16 +1665,13 @@ u16 irdma_get_vlan_ipv4(struct iw_cm_id *cm_id, u32 *addr) { if_t netdev; -#ifdef VIMAGE - struct vnet *vnet = irdma_cmid_to_vnet(cm_id); -#endif + struct vnet *vnet = &init_net; u16 vlan_id = 0xFFFF; #ifdef VIMAGE - netdev = ip_ifp_find(vnet, htonl(addr[0])); -#else - netdev = ip_ifp_find(&init_net, htonl(addr[0])); + vnet = irdma_cmid_to_vnet(cm_id); #endif + netdev = ip_ifp_find(vnet, htonl(addr[0])); if (netdev) { vlan_id = rdma_vlan_dev_vlan_id(netdev); dev_put(netdev); @@ -2546,7 +2539,7 @@ irdma_handle_syn_pkt(struct irdma_cm_node *cm_node, u32 inc_sequence; int optionsize; - optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr); + optionsize = (tcph->th_off << 2) - sizeof(*tcph); inc_sequence = ntohl(tcph->th_seq); switch (cm_node->state) { @@ -2613,7 +2606,7 @@ irdma_handle_synack_pkt(struct irdma_cm_node *cm_node, u32 inc_sequence; int optionsize; - optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr); + optionsize = (tcph->th_off << 2) - sizeof(*tcph); inc_sequence = ntohl(tcph->th_seq); switch (cm_node->state) { case IRDMA_CM_STATE_SYN_SENT: @@ -2687,7 +2680,7 @@ irdma_handle_ack_pkt(struct irdma_cm_node *cm_node, int optionsize; u32 datasize = rbuf->datalen; - optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr); + optionsize = (tcph->th_off << 2) - sizeof(*tcph); if (irdma_check_seq(cm_node, tcph)) return -EINVAL; @@ -3537,8 +3530,8 @@ irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) cm_node->vlan_id = irdma_get_vlan_ipv4(cm_id, cm_node->loc_addr); } else { cm_node->ipv4 = false; - irdma_netdev_vlan_ipv6(cm_id, cm_node->loc_addr, - &cm_node->vlan_id, NULL); + irdma_get_vlan_mac_ipv6(cm_id, cm_node->loc_addr, &cm_node->vlan_id, + NULL); } irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Accept vlan_id=%d\n", cm_node->vlan_id); @@ -3745,7 +3738,7 @@ irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) raddr6->sin6_addr.__u6_addr.__u6_addr32); cm_info.loc_port = ntohs(laddr6->sin6_port); cm_info.rem_port = ntohs(raddr6->sin6_port); - irdma_netdev_vlan_ipv6(cm_id, cm_info.loc_addr, &cm_info.vlan_id, NULL); + irdma_get_vlan_mac_ipv6(cm_id, cm_info.loc_addr, &cm_info.vlan_id, NULL); } cm_info.cm_id = cm_id; cm_info.qh_qpid = iwdev->vsi.ilq->qp_id; @@ -3874,8 +3867,8 @@ irdma_create_listen(struct iw_cm_id *cm_id, int backlog) laddr6->sin6_addr.__u6_addr.__u6_addr32); cm_info.loc_port = ntohs(laddr6->sin6_port); if (!IN6_IS_ADDR_UNSPECIFIED(&laddr6->sin6_addr)) { - irdma_netdev_vlan_ipv6(cm_id, cm_info.loc_addr, - &cm_info.vlan_id, NULL); + irdma_get_vlan_mac_ipv6(cm_id, cm_info.loc_addr, + &cm_info.vlan_id, NULL); } else { cm_info.vlan_id = 0xFFFF; wildcard = true; diff --git a/sys/dev/irdma/irdma_cm.h b/sys/dev/irdma/irdma_cm.h index 9f71f2450147..36cebdb5bf19 100644 --- a/sys/dev/irdma/irdma_cm.h +++ b/sys/dev/irdma/irdma_cm.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU diff --git a/sys/dev/irdma/irdma_ctrl.c b/sys/dev/irdma/irdma_ctrl.c index b0f5357a21e5..3a080f1d0c75 100644 --- a/sys/dev/irdma/irdma_ctrl.c +++ b/sys/dev/irdma/irdma_ctrl.c @@ -785,14 +785,14 @@ irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 * qp_ctx, set_64bit_val(qp_ctx, IRDMA_BYTE_144, FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); set_64bit_val(qp_ctx, IRDMA_BYTE_152, - FIELD_PREP(IRDMAQPC_MACADDRESS, irdma_mac_to_u64(roce_info->mac_addr))); + FIELD_PREP(IRDMAQPC_MACADDRESS, + irdma_mac_to_u64(roce_info->mac_addr))); set_64bit_val(qp_ctx, IRDMA_BYTE_160, FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) | FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) | FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) | FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) | FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) | - FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) | FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) | FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) | FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) | @@ -1016,7 +1016,6 @@ irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 * qp_ctx, FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) | FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) | FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) | - FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) | FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) | FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) | FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) | @@ -1466,6 +1465,15 @@ irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp) set_64bit_val(wqe, IRDMA_BYTE_24, hdr); irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "CONN EST WQE", wqe, IRDMA_QP_WQE_MIN_SIZE); + if (qp->qp_uk.start_wqe_idx) { + wqe = qp_uk->sq_base[3].elem; + hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | + FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) | + FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); + irdma_wmb(); /* make sure WQE is written before valid bit is set */ + + set_64bit_val(wqe, IRDMA_BYTE_24, hdr); + } } /** @@ -1831,7 +1839,7 @@ irdma_bld_terminate_hdr(struct irdma_sc_qp *qp, if (copy_len) irdma_memcpy(termhdr + 1, pkt, copy_len); - return sizeof(struct irdma_terminate_hdr) + copy_len; + return sizeof(*termhdr) + copy_len; } /** @@ -2124,9 +2132,6 @@ irdma_sc_gather_stats(struct irdma_sc_cqp *cqp, IRDMA_CQP_WQE_SIZE * 8); irdma_sc_cqp_post_sq(cqp); - irdma_debug(cqp->dev, IRDMA_DEBUG_STATS, - "CQP SQ head 0x%x tail 0x%x size 0x%x\n", cqp->sq_ring.head, - cqp->sq_ring.tail, cqp->sq_ring.size); return 0; } @@ -2607,17 +2612,18 @@ irdma_sc_cq_create(struct irdma_sc_cq *cq, u64 scratch, set_64bit_val(wqe, IRDMA_BYTE_0, cq->cq_uk.cq_size); set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1)); set_64bit_val(wqe, IRDMA_BYTE_16, - FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold)); - set_64bit_val(wqe, IRDMA_BYTE_32, (cq->virtual_map ? 0 : cq->cq_pa)); + FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, + cq->shadow_read_threshold)); + set_64bit_val(wqe, IRDMA_BYTE_32, cq->virtual_map ? 0 : cq->cq_pa); set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa); set_64bit_val(wqe, IRDMA_BYTE_48, - FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0))); + FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, + cq->virtual_map ? cq->first_pm_pbl_idx : 0)); set_64bit_val(wqe, IRDMA_BYTE_56, FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) | FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx)); - hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) | - FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0), + FLD_LS_64(cq->dev, cq->ceq_id_valid ? cq->ceq_id : 0, IRDMA_CQPSQ_CQ_CEQID) | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) | FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) | @@ -2935,10 +2941,12 @@ irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 * buf, IRDMA_HMC_IW_HDR); irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_152, info, IRDMA_HMC_IW_MD); - irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_160, info, - IRDMA_HMC_IW_OOISC); - irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_168, info, - IRDMA_HMC_IW_OOISCFFL); + if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) { + irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_160, info, + IRDMA_HMC_IW_OOISC); + irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_168, info, + IRDMA_HMC_IW_OOISCFFL); + } } /* searching for the last object in HMC to find the size of the HMC area. */ @@ -3071,15 +3079,18 @@ irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 * buf, irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR); irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD); - irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC); - - get_64bit_val(buf, IRDMA_BYTE_168, &temp); - obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp; - obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4; - hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp); - if (!hmc_fpm_misc->ooiscf_block_size && - obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt) - return -EINVAL; + + if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) { + irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC); + + get_64bit_val(buf, IRDMA_BYTE_168, &temp); + obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp; + obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4; + hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp); + if (!hmc_fpm_misc->ooiscf_block_size && + obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt) + return -EINVAL; + } return 0; } @@ -3251,6 +3262,7 @@ irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err) temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) | FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile); + if (hw_rev >= IRDMA_GEN_2) temp |= FIELD_PREP(IRDMA_CQPHC_EN_REM_ENDPOINT_TRK, cqp->en_rem_endpoint_trk); @@ -3399,11 +3411,13 @@ irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp, bool free_hwcqp) void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq) { + unsigned long flags; u64 temp_val; u16 sw_cq_sel; u8 arm_next_se; u8 arm_seq_num; + spin_lock_irqsave(&ccq->dev->cqp_lock, flags); get_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, &temp_val); sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); @@ -3414,6 +3428,7 @@ irdma_sc_ccq_arm(struct irdma_sc_cq *ccq) FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1); set_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, temp_val); + spin_unlock_irqrestore(&ccq->dev->cqp_lock, flags); irdma_wmb(); /* make sure shadow area is updated before arming */ @@ -3436,6 +3451,7 @@ irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq, u32 error; u8 polarity; int ret_code = 0; + unsigned long flags; if (ccq->cq_uk.avoid_mem_cflct) cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk); @@ -3484,7 +3500,9 @@ irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq, irdma_wmb(); /* make sure shadow area is updated before moving tail */ + spin_lock_irqsave(&cqp->dev->cqp_lock, flags); IRDMA_RING_MOVE_TAIL(cqp->sq_ring); + spin_unlock_irqrestore(&cqp->dev->cqp_lock, flags); atomic64_inc(&cqp->completed_ops); return ret_code; @@ -4152,6 +4170,7 @@ irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq, case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST: case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP: case IRDMA_AE_ROCE_RSP_LENGTH_ERROR: + case IRDMA_AE_ROCE_REQ_LENGTH_ERROR: case IRDMA_AE_INVALID_ARP_ENTRY: case IRDMA_AE_INVALID_TCP_OPTION_RCVD: case IRDMA_AE_STALE_ARP_ENTRY: @@ -4704,10 +4723,11 @@ static u32 irdma_est_sd(struct irdma_sc_dev *dev, u64 sd; int i; - for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) + for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) { if (i != IRDMA_HMC_IW_PBLE) size += round_up(hmc_info->hmc_obj[i].cnt * hmc_info->hmc_obj[i].size, 512); + } pble_info = &hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE]; size += round_up(pble_info->cnt * pble_info->size, 512); @@ -4717,7 +4737,7 @@ static u32 irdma_est_sd(struct irdma_sc_dev *dev, sd = size >> 21; if (sd > 0xFFFFFFFF) { irdma_debug(dev, IRDMA_DEBUG_HMC, "sd overflow[%ld]\n", sd); - sd = 0xFFFFFFFF - 1; + sd = 0xFFFFFFFE; } return (u32)sd; @@ -4773,10 +4793,9 @@ irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp, int irdma_get_rdma_features(struct irdma_sc_dev *dev) { - int ret_code; + int ret_code, byte_idx, feat_type, feat_cnt, feat_idx; struct irdma_dma_mem feat_buf; u64 temp; - u16 byte_idx, feat_type, feat_cnt, feat_idx; feat_buf.size = IRDMA_FEATURE_BUF_SIZE; feat_buf.va = irdma_allocate_dma_mem(dev->hw, &feat_buf, feat_buf.size, @@ -4819,7 +4838,7 @@ irdma_get_rdma_features(struct irdma_sc_dev *dev) irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QUERY RDMA FEATURES", feat_buf.va, feat_cnt * 8); - for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, (u16)IRDMA_MAX_FEATURES); + for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, IRDMA_MAX_FEATURES); feat_idx++, byte_idx += 8) { get_64bit_val(feat_buf.va, byte_idx, &temp); feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp); @@ -5421,14 +5440,16 @@ irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable) void sc_vsi_update_stats(struct irdma_sc_vsi *vsi) { - struct irdma_gather_stats *gather_stats; - struct irdma_gather_stats *last_gather_stats; - - gather_stats = vsi->pestat->gather_info.gather_stats_va; - last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va; - irdma_update_stats(&vsi->pestat->hw_stats, gather_stats, - last_gather_stats, vsi->dev->hw_stats_map, - vsi->dev->hw_attrs.max_stat_idx); + struct irdma_dev_hw_stats *hw_stats = &vsi->pestat->hw_stats; + struct irdma_gather_stats *gather_stats = + vsi->pestat->gather_info.gather_stats_va; + struct irdma_gather_stats *last_gather_stats = + vsi->pestat->gather_info.last_gather_stats_va; + const struct irdma_hw_stat_map *map = vsi->dev->hw_stats_map; + u16 max_stat_idx = vsi->dev->hw_attrs.max_stat_idx; + + irdma_update_stats(hw_stats, gather_stats, last_gather_stats, + map, max_stat_idx); } /** @@ -5498,7 +5519,6 @@ irdma_sc_dev_init(struct irdma_sc_dev *dev, struct irdma_device_init_info *info) dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE; dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE; dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE; - dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT; dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE; dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES; dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR); @@ -5526,7 +5546,6 @@ irdma_sc_dev_init(struct irdma_sc_dev *dev, struct irdma_device_init_info *info) val, db_size); return -ENODEV; } - dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET]; return ret_code; } diff --git a/sys/dev/irdma/irdma_defs.h b/sys/dev/irdma/irdma_defs.h index dcd6a0b5956b..fd3bf82c9ad6 100644 --- a/sys/dev/irdma/irdma_defs.h +++ b/sys/dev/irdma/irdma_defs.h @@ -249,7 +249,6 @@ enum irdma_cqp_op_type { IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE = 22, IRDMA_OP_SUSPEND = 23, IRDMA_OP_RESUME = 24, - IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP = 25, IRDMA_OP_QUERY_FPM_VAL = 26, IRDMA_OP_COMMIT_FPM_VAL = 27, IRDMA_OP_AH_CREATE = 28, @@ -292,7 +291,6 @@ enum irdma_cqp_op_type { #define IRDMA_CQP_OP_DEALLOC_STAG 0x0d #define IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE 0x0e #define IRDMA_CQP_OP_MANAGE_ARP 0x0f -#define IRDMA_CQP_OP_MANAGE_VCHNL_REQ_PBLE_BP 0x10 #define IRDMA_CQP_OP_MANAGE_PUSH_PAGES 0x11 #define IRDMA_CQP_OP_QUERY_RDMA_FEATURES 0x12 #define IRDMA_CQP_OP_UPLOAD_CONTEXT 0x13 @@ -849,7 +847,6 @@ enum irdma_cqp_op_type { #define IRDMA_CQPSQ_UCTX_RAWFORMAT BIT_ULL(61) #define IRDMA_CQPSQ_UCTX_FREEZEQP_S 62 #define IRDMA_CQPSQ_UCTX_FREEZEQP BIT_ULL(62) - #define IRDMA_CQPSQ_MHMC_VFIDX_S 0 #define IRDMA_CQPSQ_MHMC_VFIDX GENMASK_ULL(15, 0) #define IRDMA_CQPSQ_MHMC_FREEPMFN_S 62 diff --git a/sys/dev/irdma/irdma_hw.c b/sys/dev/irdma/irdma_hw.c index e80ffc1d2fd7..05004b1ccc83 100644 --- a/sys/dev/irdma/irdma_hw.c +++ b/sys/dev/irdma/irdma_hw.c @@ -394,6 +394,7 @@ irdma_process_aeq(struct irdma_pci_f *rf) case IRDMA_AE_LLP_TOO_MANY_RETRIES: case IRDMA_AE_LCE_QP_CATASTROPHIC: case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC: + case IRDMA_AE_LLP_TOO_MANY_RNRS: case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: default: irdma_dev_err(&iwdev->ibdev, @@ -490,7 +491,7 @@ irdma_save_msix_info(struct irdma_pci_f *rf) { struct irdma_qvlist_info *iw_qvlist; struct irdma_qv_info *iw_qvinfo; - u32 ceq_idx; + u16 ceq_idx; u32 i; u32 size; @@ -500,8 +501,8 @@ irdma_save_msix_info(struct irdma_pci_f *rf) } size = sizeof(struct irdma_msix_vector) * rf->msix_count; - size += sizeof(struct irdma_qvlist_info); - size += sizeof(struct irdma_qv_info) * rf->msix_count - 1; + size += sizeof(*iw_qvlist); + size += sizeof(*iw_qvinfo) * rf->msix_count - 1; rf->iw_msixtbl = kzalloc(size, GFP_KERNEL); if (!rf->iw_msixtbl) return -ENOMEM; @@ -600,6 +601,13 @@ irdma_destroy_irq(struct irdma_pci_f *rf, dev->irq_ops->irdma_dis_irq(dev, msix_vec->idx); irdma_free_irq(rf, msix_vec); + if (rf == dev_id) { + tasklet_kill(&rf->dpc_tasklet); + } else { + struct irdma_ceq *iwceq = (struct irdma_ceq *)dev_id; + + tasklet_kill(&iwceq->dpc_tasklet); + } } /** @@ -964,13 +972,13 @@ irdma_create_cqp(struct irdma_pci_f *rf) u16 maj_err, min_err; int i, status; - cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL); - memset(cqp->cqp_requests, 0, sqsize * sizeof(*cqp->cqp_requests)); + cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), + GFP_KERNEL); if (!cqp->cqp_requests) return -ENOMEM; - cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL); - memset(cqp->scratch_array, 0, sqsize * sizeof(*cqp->scratch_array)); + cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), + GFP_KERNEL); if (!cqp->scratch_array) { status = -ENOMEM; goto err_scratch; @@ -1190,7 +1198,7 @@ fail_intr: */ static int irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq, - u32 ceq_id, struct irdma_msix_vector *msix_vec) + u16 ceq_id, struct irdma_msix_vector *msix_vec) { int status; @@ -1264,7 +1272,7 @@ irdma_cfg_aeq_vector(struct irdma_pci_f *rf) */ static int irdma_create_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq, - u32 ceq_id, struct irdma_sc_vsi *vsi) + u16 ceq_id, struct irdma_sc_vsi *vsi) { int status; struct irdma_ceq_init_info info = {0}; @@ -1381,7 +1389,7 @@ static int irdma_setup_ceqs(struct irdma_pci_f *rf, struct irdma_sc_vsi *vsi) { u32 i; - u32 ceq_id; + u16 ceq_id; struct irdma_ceq *iwceq; struct irdma_msix_vector *msix_vec; int status; @@ -2212,14 +2220,19 @@ irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) cqp_request = (struct irdma_cqp_request *) (uintptr_t)info.scratch; - if (info.error && irdma_cqp_crit_err(dev, cqp_request->info.cqp_cmd, + if (info.error && irdma_cqp_crit_err(dev, + cqp_request->info.cqp_cmd, info.maj_err_code, info.min_err_code)) - irdma_dev_err(&rf->iwdev->ibdev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n", - info.op_code, info.maj_err_code, info.min_err_code); + irdma_dev_err(&rf->iwdev->ibdev, + "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n", + info.op_code, info.maj_err_code, + info.min_err_code); if (cqp_request) { - cqp_request->compl_info.maj_err_code = info.maj_err_code; - cqp_request->compl_info.min_err_code = info.min_err_code; + cqp_request->compl_info.maj_err_code = + info.maj_err_code; + cqp_request->compl_info.min_err_code = + info.min_err_code; cqp_request->compl_info.op_ret_val = info.op_ret_val; cqp_request->compl_info.error = info.error; irdma_complete_cqp_request(&rf->cqp, cqp_request); diff --git a/sys/dev/irdma/irdma_kcompat.c b/sys/dev/irdma/irdma_kcompat.c index 1ddfd6791fc0..7f40cffcde27 100644 --- a/sys/dev/irdma/irdma_kcompat.c +++ b/sys/dev/irdma/irdma_kcompat.c @@ -160,6 +160,7 @@ err: #define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8) #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd) + /** * irdma_alloc_ucontext - Allocate the user context data structure * @ibdev: ib device pointer @@ -228,6 +229,8 @@ irdma_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata) uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; uresp.hw_rev = uk_attrs->hw_rev; uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR; + uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size; + uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE; bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; @@ -285,6 +288,7 @@ irdma_dealloc_ucontext(struct ib_ucontext *context) } #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd) + /** * irdma_alloc_pd - allocate protection domain * @ibdev: IB device @@ -481,23 +485,41 @@ static int irdma_create_ah_wait(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, bool sleep) { + int ret; + if (!sleep) { int cnt = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms * CQP_TIMEOUT_THRESHOLD; + struct irdma_cqp_request *cqp_request = + sc_ah->ah_info.cqp_request; do { irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); mdelay(1); - } while (!sc_ah->ah_info.ah_valid && --cnt); + } while (!READ_ONCE(cqp_request->request_done) && --cnt); - if (!cnt) - return -ETIMEDOUT; + if (cnt && !cqp_request->compl_info.op_ret_val) { + irdma_put_cqp_request(&rf->cqp, cqp_request); + sc_ah->ah_info.ah_valid = true; + } else { + ret = !cnt ? -ETIMEDOUT : -EINVAL; + irdma_dev_err(&rf->iwdev->ibdev, "CQP create AH error ret = %d opt_ret_val = %d", + ret, cqp_request->compl_info.op_ret_val); + irdma_put_cqp_request(&rf->cqp, cqp_request); + if (!cnt && !rf->reset) { + rf->reset = true; + rf->gen_ops.request_reset(rf); + } + return ret; + } } + return 0; } #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) + void irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr) { @@ -610,17 +632,15 @@ irdma_create_ah(struct ib_pd *ibpd, goto err_gid_l2; err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, - sleep, irdma_gsi_ud_qp_ah_cb, sc_ah); + sleep, NULL, sc_ah); if (err) { irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "CQP-OP Create AH fail"); goto err_gid_l2; } err = irdma_create_ah_wait(rf, sc_ah, sleep); - if (err) { - irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP create AH timed out"); + if (err) goto err_gid_l2; - } if (udata) { uresp.ah_id = ah->sc_ah.ah_info.ah_idx; @@ -652,7 +672,7 @@ irdma_free_qp_rsrc(struct irdma_qp *iwqp) u32 qp_num = iwqp->ibqp.qp_num; irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); - irdma_dealloc_push_page(rf, &iwqp->sc_qp); + irdma_dealloc_push_page(rf, iwqp); if (iwqp->sc_qp.vsi) { irdma_qp_rem_qos(&iwqp->sc_qp); iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, @@ -846,12 +866,17 @@ irdma_create_qp(struct ib_pd *ibpd, if (udata) { /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ - if (udata->outlen < sizeof(uresp)) { + if (udata->outlen == IRDMA_CREATE_QP_MIN_RESP_LEN) { uresp.lsmm = 1; uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; } else { - if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) + if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) { uresp.lsmm = 1; + if (qp->qp_uk.start_wqe_idx) { + uresp.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX; + uresp.start_wqe_idx = qp->qp_uk.start_wqe_idx; + } + } } uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size; uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size; @@ -862,7 +887,7 @@ irdma_create_qp(struct ib_pd *ibpd, min(sizeof(uresp), udata->outlen)); if (err_code) { irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); - kc_irdma_destroy_qp(&iwqp->ibqp, udata); + irdma_destroy_qp(&iwqp->ibqp); return ERR_PTR(err_code); } } @@ -1358,50 +1383,6 @@ irdma_destroy_cq(struct ib_cq *ib_cq) } /** - * irdma_alloc_mw - Allocate memory window - * @pd: Protection domain - * @type: Window type - * @udata: user data pointer - */ -struct ib_mw * -irdma_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, - struct ib_udata *udata) -{ - struct irdma_device *iwdev = to_iwdev(pd->device); - struct irdma_mr *iwmr; - int err_code; - u32 stag; - - if (type != IB_MW_TYPE_1 && type != IB_MW_TYPE_2) - return ERR_PTR(-EINVAL); - - iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); - if (!iwmr) - return ERR_PTR(-ENOMEM); - - stag = irdma_create_stag(iwdev); - if (!stag) { - kfree(iwmr); - return ERR_PTR(-ENOMEM); - } - - iwmr->stag = stag; - iwmr->ibmw.rkey = stag; - iwmr->ibmw.pd = pd; - iwmr->ibmw.type = type; - iwmr->ibmw.device = pd->device; - - err_code = irdma_hw_alloc_mw(iwdev, iwmr); - if (err_code) { - irdma_free_stag(iwdev, stag); - kfree(iwmr); - return ERR_PTR(err_code); - } - - return &iwmr->ibmw; -} - -/** * kc_set_loc_seq_num_mss - Set local seq number and mss * @cm_node: cm node info */ @@ -1569,7 +1550,7 @@ irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index, { int ret; - ret = rdma_query_gid(ibdev, port, index, gid); + ret = ib_get_cached_gid(ibdev, port, index, gid, NULL); if (ret == -EAGAIN) { memcpy(gid, &zgid, sizeof(*gid)); return 0; @@ -1861,9 +1842,6 @@ kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev) BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) | BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) | - BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) | - BIT_ULL(IB_USER_VERBS_CMD_BIND_MW) | - BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) | BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) | BIT_ULL(IB_USER_VERBS_CMD_POST_SEND); iwdev->ibdev.uverbs_ex_cmd_mask = diff --git a/sys/dev/irdma/irdma_main.h b/sys/dev/irdma/irdma_main.h index 3d6dd4e3f9db..d8a015f5a265 100644 --- a/sys/dev/irdma/irdma_main.h +++ b/sys/dev/irdma/irdma_main.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -181,7 +181,7 @@ struct irdma_cqp_request { void (*callback_fcn)(struct irdma_cqp_request *cqp_request); void *param; struct irdma_cqp_compl_info compl_info; - bool request_done; /* READ/WRITE_ONCE macros operate on it */ + u8 request_done; /* READ/WRITE_ONCE macros operate on it */ bool waiting:1; bool dynamic:1; }; @@ -232,7 +232,7 @@ struct irdma_msix_vector { u32 idx; u32 irq; u32 cpu_affinity; - u32 ceq_id; + u16 ceq_id; char name[IRDMA_IRQ_NAME_STR_LEN]; struct resource *res; void *tag; @@ -374,7 +374,6 @@ struct irdma_device { u32 roce_ackcreds; u32 vendor_id; u32 vendor_part_id; - u32 push_mode; u32 rcv_wnd; u16 mac_ip_table_idx; u16 vsi_num; @@ -388,6 +387,7 @@ struct irdma_device { bool override_ooo:1; bool override_rd_fence_rate:1; bool override_rtomin:1; + bool push_mode:1; bool roce_mode:1; bool roce_dcqcn_en:1; bool dcb_vlan_mode:1; @@ -573,8 +573,8 @@ void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp, void irdma_copy_ip_ntohl(u32 *dst, __be32 *src); void irdma_copy_ip_htonl(__be32 *dst, u32 *src); u16 irdma_get_vlan_ipv4(struct iw_cm_id *cm_id, u32 *addr); -if_t irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, - u8 *mac); +void irdma_get_vlan_mac_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, + u8 *mac); struct ib_mr *irdma_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size, int acc, u64 *iova_start); int irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw); @@ -586,7 +586,6 @@ int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, bool wait, void (*callback_fcn)(struct irdma_cqp_request *cqp_request), void *cb_param); -void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request); void irdma_udqp_qs_worker(struct work_struct *work); bool irdma_cq_empty(struct irdma_cq *iwcq); int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, diff --git a/sys/dev/irdma/irdma_pble.h b/sys/dev/irdma/irdma_pble.h index 213be2018457..44e43a8437a1 100644 --- a/sys/dev/irdma/irdma_pble.h +++ b/sys/dev/irdma/irdma_pble.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU diff --git a/sys/dev/irdma/irdma_protos.h b/sys/dev/irdma/irdma_protos.h index 4f9a97d54088..0663f9591d52 100644 --- a/sys/dev/irdma/irdma_protos.h +++ b/sys/dev/irdma/irdma_protos.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2016 - 2022 Intel Corporation + * Copyright (c) 2016 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU diff --git a/sys/dev/irdma/irdma_puda.c b/sys/dev/irdma/irdma_puda.c index 86bdb334e658..0c5b9c164d76 100644 --- a/sys/dev/irdma/irdma_puda.c +++ b/sys/dev/irdma/irdma_puda.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -182,7 +182,7 @@ irdma_puda_alloc_buf(struct irdma_sc_dev *dev, struct irdma_puda_buf *buf; struct irdma_virt_mem buf_mem; - buf_mem.size = sizeof(struct irdma_puda_buf); + buf_mem.size = sizeof(*buf); buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL); if (!buf_mem.va) return NULL; @@ -1010,7 +1010,7 @@ irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc, u32 count) bool virtdma = false; unsigned long flags; - buf_mem.size = count * sizeof(struct irdma_puda_buf); + buf_mem.size = count * sizeof(*buf); buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL); if (!buf_mem.va) { irdma_debug(rsrc->dev, IRDMA_DEBUG_PUDA, @@ -1105,7 +1105,7 @@ irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi, struct irdma_virt_mem *vmem; info->count = 1; - pudasize = sizeof(struct irdma_puda_rsrc); + pudasize = sizeof(*rsrc); sqwridsize = info->sq_size * sizeof(struct irdma_sq_uk_wr_trk_info); rqwridsize = info->rq_size * 8; switch (info->type) { diff --git a/sys/dev/irdma/irdma_type.h b/sys/dev/irdma/irdma_type.h index 2f3ab8c471d5..ee77c6064295 100644 --- a/sys/dev/irdma/irdma_type.h +++ b/sys/dev/irdma/irdma_type.h @@ -42,6 +42,8 @@ #include "irdma_hmc.h" #include "irdma_uda.h" #include "irdma_ws.h" +#include "irdma_pble.h" + enum irdma_debug_flag { IRDMA_DEBUG_NONE = 0x00000000, IRDMA_DEBUG_ERR = 0x00000001, @@ -70,6 +72,8 @@ enum irdma_debug_flag { IRDMA_DEBUG_ALL = 0xFFFFFFFF, }; +#define RSVD_OFFSET 0xFFFFFFFF + enum irdma_page_size { IRDMA_PAGE_SIZE_4K = 0, IRDMA_PAGE_SIZE_2M, @@ -472,7 +476,7 @@ struct irdma_sc_cq { bool virtual_map:1; bool check_overflow:1; bool ceq_id_valid:1; - bool tph_en; + bool tph_en:1; }; struct irdma_sc_qp { @@ -520,9 +524,9 @@ struct irdma_sc_qp { }; struct irdma_stats_inst_info { - bool use_hmc_fcn_index; u16 hmc_fn_id; u16 stats_idx; + bool use_hmc_fcn_index:1; }; struct irdma_up_info { @@ -570,7 +574,7 @@ struct irdma_qos { u8 traffic_class; u8 rel_bw; u8 prio_type; - bool valid; + bool valid:1; }; struct irdma_config_check { @@ -623,7 +627,6 @@ struct irdma_sc_dev { __le64 *fpm_query_buf; __le64 *fpm_commit_buf; struct irdma_hw *hw; - u8 IOMEM *db_addr; u32 IOMEM *wqe_alloc_db; u32 IOMEM *cq_arm_db; u32 IOMEM *aeq_alloc_db; @@ -649,8 +652,6 @@ struct irdma_sc_dev { u32 debug_mask; u16 num_vfs; u16 hmc_fn_id; - u8 vf_id; - bool vchnl_up:1; bool ceq_valid:1; u8 pci_rev; int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri); @@ -666,7 +667,7 @@ struct irdma_modify_cq_info { u8 pbl_chunk_size; u32 first_pm_pbl_idx; bool virtual_map:1; - bool check_overflow; + bool check_overflow:1; bool cq_resize:1; }; @@ -676,7 +677,7 @@ struct irdma_create_qp_info { bool cq_num_valid:1; bool arp_cache_idx_valid:1; bool mac_valid:1; - bool force_lpb; + bool force_lpb:1; u8 next_iwarp_state; }; @@ -709,7 +710,7 @@ struct irdma_ccq_cqe_info { u16 maj_err_code; u16 min_err_code; u8 op_code; - bool error; + bool error:1; }; struct irdma_qos_tc_info { @@ -751,7 +752,7 @@ struct irdma_vsi_init_info { struct irdma_vsi_stats_info { struct irdma_vsi_pestat *pestat; u8 fcn_id; - bool alloc_stats_inst; + bool alloc_stats_inst:1; }; struct irdma_device_init_info { @@ -789,7 +790,7 @@ struct irdma_aeq_init_info { u32 *aeqe_base; void *pbl_list; u32 elem_cnt; - bool virtual_map; + bool virtual_map:1; u8 pbl_chunk_size; u32 first_pm_pbl_idx; u32 msix_idx; @@ -856,7 +857,6 @@ struct irdma_roce_offload_info { bool dcqcn_en:1; bool rcv_no_icrc:1; bool wr_rdresp_en:1; - bool bind_en:1; bool fast_reg_en:1; bool priv_mode_en:1; bool rd_en:1; @@ -888,7 +888,6 @@ struct irdma_iwarp_offload_info { bool snd_mark_en:1; bool rcv_mark_en:1; bool wr_rdresp_en:1; - bool bind_en:1; bool fast_reg_en:1; bool priv_mode_en:1; bool rd_en:1; @@ -1130,12 +1129,12 @@ struct irdma_add_arp_cache_entry_info { u8 mac_addr[ETHER_ADDR_LEN]; u32 reach_max; u16 arp_index; - bool permanent; + bool permanent:1; }; struct irdma_apbvt_info { u16 port; - bool add; + bool add:1; }; struct irdma_qhash_table_info { diff --git a/sys/dev/irdma/irdma_uda.c b/sys/dev/irdma/irdma_uda.c index 850bfa4ae6d6..6aaceac68cdb 100644 --- a/sys/dev/irdma/irdma_uda.c +++ b/sys/dev/irdma/irdma_uda.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2016 - 2022 Intel Corporation + * Copyright (c) 2016 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU diff --git a/sys/dev/irdma/irdma_uda.h b/sys/dev/irdma/irdma_uda.h index 4be9a144d18b..9850f986ee67 100644 --- a/sys/dev/irdma/irdma_uda.h +++ b/sys/dev/irdma/irdma_uda.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2016 - 2021 Intel Corporation + * Copyright (c) 2016 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -43,6 +43,7 @@ struct irdma_sc_cqp; struct irdma_ah_info { struct irdma_sc_vsi *vsi; + struct irdma_cqp_request *cqp_request; u32 pd_idx; u32 dst_arpindex; u32 dest_ip_addr[4]; diff --git a/sys/dev/irdma/irdma_uk.c b/sys/dev/irdma/irdma_uk.c index 1333b1ca4845..e0c9dc20a7fd 100644 --- a/sys/dev/irdma/irdma_uk.c +++ b/sys/dev/irdma/irdma_uk.c @@ -45,16 +45,16 @@ * @valid: The wqe valid */ static void -irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge, +irdma_set_fragment(__le64 * wqe, u32 offset, struct ib_sge *sge, u8 valid) { if (sge) { set_64bit_val(wqe, offset, - FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr)); set_64bit_val(wqe, offset + IRDMA_BYTE_8, FIELD_PREP(IRDMAQPSQ_VALID, valid) | - FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) | - FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag)); + FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) | + FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey)); } else { set_64bit_val(wqe, offset, 0); set_64bit_val(wqe, offset + IRDMA_BYTE_8, @@ -71,14 +71,14 @@ irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge, */ static void irdma_set_fragment_gen_1(__le64 * wqe, u32 offset, - struct irdma_sge *sge, u8 valid) + struct ib_sge *sge, u8 valid) { if (sge) { set_64bit_val(wqe, offset, - FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr)); set_64bit_val(wqe, offset + IRDMA_BYTE_8, - FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) | - FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag)); + FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) | + FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey)); } else { set_64bit_val(wqe, offset, 0); set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0); @@ -209,8 +209,7 @@ irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 * wqe, u16 quanta, if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) != IRDMA_RING_CURRENT_TAIL(qp->sq_ring) && !qp->push_mode) { - if (post_sq) - irdma_uk_qp_post_wr(qp); + irdma_uk_qp_post_wr(qp); } else { push = (__le64 *) ((uintptr_t)qp->push_wqe + (wqe_idx & 0x7) * 0x20); @@ -338,7 +337,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, return -EINVAL; for (i = 0; i < op_info->num_lo_sges; i++) - total_size += op_info->lo_sg_list[i].len; + total_size += op_info->lo_sg_list[i].length; read_fence |= info->read_fence; @@ -357,7 +356,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; set_64bit_val(wqe, IRDMA_BYTE_16, - FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); if (info->imm_data_valid) { set_64bit_val(wqe, IRDMA_BYTE_0, @@ -386,7 +385,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, ++addl_frag_cnt; } - hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | + hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) | FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) | @@ -437,7 +436,7 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, return -EINVAL; for (i = 0; i < op_info->num_lo_sges; i++) - total_size += op_info->lo_sg_list[i].len; + total_size += op_info->lo_sg_list[i].length; ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta); if (ret_code) @@ -475,8 +474,8 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, ++addl_frag_cnt; } set_64bit_val(wqe, IRDMA_BYTE_16, - FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); - hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | + FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); + hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | FIELD_PREP(IRDMAQPSQ_OPCODE, @@ -525,7 +524,7 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, return -EINVAL; for (i = 0; i < op_info->num_sges; i++) - total_size += op_info->sg_list[i].len; + total_size += op_info->sg_list[i].length; if (info->imm_data_valid) frag_cnt = op_info->num_sges + 1; @@ -604,15 +603,15 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, * @polarity: compatibility parameter */ static void -irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list, +irdma_copy_inline_data_gen_1(u8 *wqe, struct ib_sge *sge_list, u32 num_sges, u8 polarity) { u32 quanta_bytes_remaining = 16; u32 i; for (i = 0; i < num_sges; i++) { - u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off; - u32 sge_len = sge_list[i].len; + u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr; + u32 sge_len = sge_list[i].length; while (sge_len) { u32 bytes_copied; @@ -651,7 +650,7 @@ static inline u16 irdma_inline_data_size_to_quanta_gen_1(u32 data_size) { * @polarity: polarity of wqe valid bit */ static void -irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, +irdma_copy_inline_data(u8 *wqe, struct ib_sge *sge_list, u32 num_sges, u8 polarity) { u8 inline_valid = polarity << IRDMA_INLINE_VALID_S; @@ -662,8 +661,8 @@ irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, wqe += 8; for (i = 0; i < num_sges; i++) { - u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off; - u32 sge_len = sge_list[i].len; + u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr; + u32 sge_len = sge_list[i].length; while (sge_len) { u32 bytes_copied; @@ -743,7 +742,7 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, return -EINVAL; for (i = 0; i < op_info->num_lo_sges; i++) - total_size += op_info->lo_sg_list[i].len; + total_size += op_info->lo_sg_list[i].length; if (unlikely(total_size > qp->max_inline_data)) return -EINVAL; @@ -756,9 +755,9 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp, qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; read_fence |= info->read_fence; set_64bit_val(wqe, IRDMA_BYTE_16, - FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); + FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr)); - hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | + hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) | FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) | @@ -814,7 +813,7 @@ irdma_uk_inline_send(struct irdma_qp_uk *qp, return -EINVAL; for (i = 0; i < op_info->num_sges; i++) - total_size += op_info->sg_list[i].len; + total_size += op_info->sg_list[i].length; if (unlikely(total_size > qp->max_inline_data)) return -EINVAL; @@ -879,7 +878,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, u64 hdr; u32 wqe_idx; bool local_fence = false; - struct irdma_sge sge = {0}; + struct ib_sge sge = {0}; u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; info->push_wqe = qp->push_db ? true : false; @@ -890,7 +889,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, if (!wqe) return -ENOSPC; - sge.stag = op_info->target_stag; + sge.lkey = op_info->target_stag; qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0); set_64bit_val(wqe, IRDMA_BYTE_16, 0); @@ -1327,8 +1326,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, IRDMA_RING_MOVE_TAIL(cq->cq_ring); set_64bit_val(cq->shadow_area, IRDMA_BYTE_0, IRDMA_RING_CURRENT_HEAD(cq->cq_ring)); - memset(info, 0, - sizeof(struct irdma_cq_poll_info)); + memset(info, 0, sizeof(*info)); return irdma_uk_cq_poll_cmpl(cq, info); } } @@ -1403,7 +1401,6 @@ exit: if (pring && IRDMA_RING_MORE_WORK(*pring)) move_cq_head = false; } - if (move_cq_head) { IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring)) @@ -1484,10 +1481,12 @@ irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge, int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth) { + u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; + *sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD); - if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift)) - *sqdepth = uk_attrs->min_hw_wq_size << shift; + if (*sqdepth < min_size) + *sqdepth = min_size; else if (*sqdepth > uk_attrs->max_hw_wq_quanta) return -EINVAL; @@ -1501,10 +1500,12 @@ irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *s int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth) { + u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift; + *rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD); - if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift)) - *rqdepth = uk_attrs->min_hw_wq_size << shift; + if (*rqdepth < min_size) + *rqdepth = min_size; else if (*rqdepth > uk_attrs->max_hw_rq_quanta) return -EINVAL; @@ -1535,9 +1536,10 @@ irdma_setup_connection_wqes(struct irdma_qp_uk *qp, { u16 move_cnt = 1; - if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE) + if (info->start_wqe_idx) + move_cnt = info->start_wqe_idx; + else if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE) move_cnt = 3; - qp->conn_wqes = move_cnt; IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt); IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt); @@ -1677,6 +1679,8 @@ irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info) qp->wqe_ops = iw_wqe_uk_ops_gen_1; else qp->wqe_ops = iw_wqe_uk_ops; + qp->start_wqe_idx = info->start_wqe_idx; + return ret_code; } diff --git a/sys/dev/irdma/irdma_user.h b/sys/dev/irdma/irdma_user.h index 6793aa8eb706..05828ebbd7d6 100644 --- a/sys/dev/irdma/irdma_user.h +++ b/sys/dev/irdma/irdma_user.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -35,6 +35,8 @@ #ifndef IRDMA_USER_H #define IRDMA_USER_H +#include <rdma/ib_verbs.h> + #define irdma_handle void * #define irdma_adapter_handle irdma_handle #define irdma_qp_handle irdma_handle @@ -48,7 +50,7 @@ #define irdma_access_privileges u32 #define irdma_physical_fragment u64 #define irdma_address_list u64 * -#define irdma_sgl struct irdma_sge * +#define irdma_sgl struct ib_sge * #define IRDMA_MAX_MR_SIZE 0x200000000000ULL @@ -78,8 +80,6 @@ #define IRDMA_OP_TYPE_REC_IMM 0x3f #define IRDMA_FLUSH_MAJOR_ERR 1 -#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe - /* Async Events codes */ #define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102 #define IRDMA_AE_AMP_INVALID_STAG 0x0103 @@ -140,6 +140,7 @@ #define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313 #define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314 #define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316 +#define IRDMA_AE_ROCE_REQ_LENGTH_ERROR 0x0318 #define IRDMA_AE_ROCE_EMPTY_MCG 0x0380 #define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381 #define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382 @@ -160,6 +161,7 @@ #define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b #define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c #define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e +#define IRDMA_AE_LLP_TOO_MANY_RNRS 0x050f #define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520 #define IRDMA_AE_RESET_SENT 0x0601 #define IRDMA_AE_TERMINATE_SENT 0x0602 @@ -199,8 +201,7 @@ enum irdma_device_caps_const { IRDMA_MAX_OUTBOUND_MSG_SIZE = 65537, /* 64K +1 */ IRDMA_MAX_INBOUND_MSG_SIZE = 65537, - IRDMA_MAX_PUSH_PAGE_COUNT = 1024, - IRDMA_MAX_PE_ENA_VF_COUNT = 32, + IRDMA_MAX_PE_ENA_VF_COUNT = 32, IRDMA_MAX_VF_FPM_ID = 47, IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496, IRDMA_MAX_INLINE_DATA_SIZE = 101, @@ -227,6 +228,7 @@ enum irdma_flush_opcode { FLUSH_RETRY_EXC_ERR, FLUSH_MW_BIND_ERR, FLUSH_REM_INV_REQ_ERR, + FLUSH_RNR_RETRY_EXC_ERR, }; enum irdma_qp_event_type { @@ -280,12 +282,6 @@ struct irdma_cq_uk; struct irdma_qp_uk_init_info; struct irdma_cq_uk_init_info; -struct irdma_sge { - irdma_tagged_offset tag_off; - u32 len; - irdma_stag stag; -}; - struct irdma_ring { volatile u32 head; volatile u32 tail; /* effective tail */ @@ -317,13 +313,13 @@ struct irdma_post_rq_info { struct irdma_rdma_write { irdma_sgl lo_sg_list; u32 num_lo_sges; - struct irdma_sge rem_addr; + struct ib_sge rem_addr; }; struct irdma_rdma_read { irdma_sgl lo_sg_list; u32 num_lo_sges; - struct irdma_sge rem_addr; + struct ib_sge rem_addr; }; struct irdma_bind_window { @@ -422,9 +418,9 @@ int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp, bool post_sq); struct irdma_wqe_uk_ops { - void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity); + void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list, u32 num_sges, u8 polarity); u16 (*iw_inline_data_size_to_quanta)(u32 data_size); - void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge, + void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge, u8 valid); void (*iw_set_mw_bind_wqe)(__le64 *wqe, struct irdma_bind_window *op_info); @@ -490,6 +486,7 @@ struct irdma_qp_uk { u8 rwqe_polarity; u8 rq_wqe_size; u8 rq_wqe_size_multiplier; + u8 start_wqe_idx; bool deferred_flag:1; bool push_mode:1; /* whether the last post wqe was pushed */ bool push_dropped:1; @@ -537,6 +534,7 @@ struct irdma_qp_uk_init_info { u32 sq_depth; u32 rq_depth; u8 first_sq_wq; + u8 start_wqe_idx; u8 type; u8 sq_shift; u8 rq_shift; @@ -625,10 +623,15 @@ static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id) case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: case IRDMA_AE_ROCE_RSP_LENGTH_ERROR: + case IRDMA_AE_ROCE_REQ_LENGTH_ERROR: case IRDMA_AE_IB_REMOTE_OP_ERROR: qp_err.flush_code = FLUSH_REM_OP_ERR; qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; break; + case IRDMA_AE_LLP_TOO_MANY_RNRS: + qp_err.flush_code = FLUSH_RNR_RETRY_EXC_ERR; + qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; + break; case IRDMA_AE_LCE_QP_CATASTROPHIC: qp_err.flush_code = FLUSH_FATAL_ERR; qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC; diff --git a/sys/dev/irdma/irdma_utils.c b/sys/dev/irdma/irdma_utils.c index acffd36fd445..5fc37022981f 100644 --- a/sys/dev/irdma/irdma_utils.c +++ b/sys/dev/irdma/irdma_utils.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -164,6 +164,7 @@ static const struct ae_desc ae_desc_list[] = { "Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"}, {IRDMA_AE_LLP_CONNECTION_ESTABLISHED, "iWARP event: Connection established"}, + {IRDMA_AE_LLP_TOO_MANY_RNRS, "RoCEv2: Too many RNR NACKs"}, {IRDMA_AE_RESOURCE_EXHAUSTION, "QP error: Resource exhaustion"}, {IRDMA_AE_RESET_SENT, @@ -437,11 +438,11 @@ static void irdma_free_pending_cqp_request(struct irdma_cqp *cqp, struct irdma_cqp_request *cqp_request) { - if (cqp_request->waiting) { - cqp_request->compl_info.error = true; - WRITE_ONCE(cqp_request->request_done, true); + cqp_request->compl_info.error = true; + WRITE_ONCE(cqp_request->request_done, true); + + if (cqp_request->waiting) wake_up(&cqp_request->waitq); - } wait_event_timeout(cqp->remove_wq, atomic_read(&cqp_request->refcnt) == 1, 1000); irdma_put_cqp_request(cqp, cqp_request); @@ -558,8 +559,6 @@ static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = { [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd", [IRDMA_OP_SUSPEND] = "Suspend QP Cmd", [IRDMA_OP_RESUME] = "Resume QP Cmd", - [IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP] = - "Manage Virtual Channel Requester Function PBLE Backing Pages Cmd", [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd", [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd", [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd", @@ -591,7 +590,7 @@ static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = { {0xffff, 0x8007, "Modify QP Bad Close"}, {0xffff, 0x8009, "LLP Closed"}, {0xffff, 0x800a, "Reset Not Sent"}, - {0xffff, 0x200, "Failover Pending"} + {0xffff, 0x0200, "Failover Pending"}, }; /** @@ -1055,15 +1054,16 @@ irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) /** * irdma_dealloc_push_page - free a push page for qp * @rf: RDMA PCI function - * @qp: hardware control qp + * @iwqp: QP pointer */ void irdma_dealloc_push_page(struct irdma_pci_f *rf, - struct irdma_sc_qp *qp) + struct irdma_qp *iwqp) { struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; int status; + struct irdma_sc_qp *qp = &iwqp->sc_qp; if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) return; @@ -1565,16 +1565,6 @@ irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi) } /** - * irdma_process_stats - Checking for wrap and update stats - * @pestat: stats structure pointer - */ -static inline void -irdma_process_stats(struct irdma_vsi_pestat *pestat) -{ - sc_vsi_update_stats(pestat->vsi); -} - -/** * irdma_process_cqp_stats - Checking for wrap and update stats * @cqp_request: cqp_request structure pointer */ @@ -1583,7 +1573,7 @@ irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request) { struct irdma_vsi_pestat *pestat = cqp_request->param; - irdma_process_stats(pestat); + sc_vsi_update_stats(pestat->vsi); } /** @@ -1619,7 +1609,7 @@ irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev, cqp_request->callback_fcn = irdma_process_cqp_stats; status = irdma_handle_cqp_op(rf, cqp_request); if (wait) - irdma_process_stats(pestat); + sc_vsi_update_stats(pestat->vsi); irdma_put_cqp_request(&rf->cqp, cqp_request); return status; @@ -1814,6 +1804,10 @@ irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, cqp_info->cqp_cmd = cmd; cqp_info->post_sq = 1; if (cmd == IRDMA_OP_AH_CREATE) { + if (!wait) + irdma_get_cqp_request(cqp_request); + sc_ah->ah_info.cqp_request = cqp_request; + cqp_info->in.u.ah_create.info = sc_ah->ah_info; cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; @@ -1949,21 +1943,6 @@ irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah) } /** - * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP - * @cqp_request: pointer to cqp_request of create AH - */ -void -irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request) -{ - struct irdma_sc_ah *sc_ah = cqp_request->param; - - if (!cqp_request->compl_info.op_ret_val) - sc_ah->ah_info.ah_valid = true; - else - sc_ah->ah_info.ah_valid = false; -} - -/** * irdma_prm_add_pble_mem - add moemory to pble resources * @pprm: pble resource manager * @pchunk: chunk of memory to add diff --git a/sys/dev/irdma/irdma_verbs.c b/sys/dev/irdma/irdma_verbs.c index c0726d25b8ec..50aeaea0f36a 100644 --- a/sys/dev/irdma/irdma_verbs.c +++ b/sys/dev/irdma/irdma_verbs.c @@ -72,7 +72,6 @@ irdma_query_device(struct ib_device *ibdev, props->max_cq = rf->max_cq - rf->used_cqs; props->max_cqe = rf->max_cqe - 1; props->max_mr = rf->max_mr - rf->used_mrs; - props->max_mw = props->max_mr; props->max_pd = rf->max_pd - rf->used_pds; props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; props->max_qp_rd_atom = hw_attrs->max_hw_ird; @@ -500,6 +499,9 @@ irdma_setup_umode_qp(struct ib_udata *udata, ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift; ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift; } + if (req.comp_mask & IRDMA_CREATE_QP_USE_START_WQE_IDX && + iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) + ukinfo->start_wqe_idx = 4; irdma_setup_virt_qp(iwdev, iwqp, info); return 0; @@ -642,7 +644,6 @@ irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, roce_info->rd_en = true; roce_info->wr_rdresp_en = true; - roce_info->bind_en = true; roce_info->dcqcn_en = false; roce_info->rtomin = iwdev->roce_rtomin; @@ -674,7 +675,6 @@ irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, ether_addr_copy(iwarp_info->mac_addr, if_getlladdr(iwdev->netdev)); iwarp_info->rd_en = true; iwarp_info->wr_rdresp_en = true; - iwarp_info->bind_en = true; iwarp_info->ecn_en = true; iwarp_info->rtomin = 5; @@ -705,6 +705,8 @@ irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline || init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || + init_attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta || + init_attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta || init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags) return -EINVAL; @@ -759,8 +761,6 @@ irdma_get_ib_acc_flags(struct irdma_qp *iwqp) } if (iwqp->roce_info.rd_en) acc_flags |= IB_ACCESS_REMOTE_READ; - if (iwqp->roce_info.bind_en) - acc_flags |= IB_ACCESS_MW_BIND; } else { if (iwqp->iwarp_info.wr_rdresp_en) { acc_flags |= IB_ACCESS_LOCAL_WRITE; @@ -768,8 +768,6 @@ irdma_get_ib_acc_flags(struct irdma_qp *iwqp) } if (iwqp->iwarp_info.rd_en) acc_flags |= IB_ACCESS_REMOTE_READ; - if (iwqp->iwarp_info.bind_en) - acc_flags |= IB_ACCESS_MW_BIND; } return acc_flags; } @@ -1181,7 +1179,8 @@ irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, udata->outlen)); if (ret) { irdma_remove_push_mmap_entries(iwqp); - irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); return ret; } @@ -1647,8 +1646,10 @@ error: /** * irdma_get_mr_access - get hw MR access permissions from IB access flags * @access: IB access flags + * @hw_rev: Hardware version */ -static inline u16 irdma_get_mr_access(int access){ +static inline u16 irdma_get_mr_access(int access, u8 hw_rev) +{ u16 hw_access = 0; hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ? @@ -1657,8 +1658,6 @@ static inline u16 irdma_get_mr_access(int access){ IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0; hw_access |= (access & IB_ACCESS_REMOTE_READ) ? IRDMA_ACCESS_FLAGS_REMOTEREAD : 0; - hw_access |= (access & IB_ACCESS_MW_BIND) ? - IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0; hw_access |= (access & IB_ZERO_BASED) ? IRDMA_ACCESS_FLAGS_ZERO_BASED : 0; hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD; @@ -1894,81 +1893,6 @@ irdma_handle_q_mem(struct irdma_device *iwdev, } /** - * irdma_hw_alloc_mw - create the hw memory window - * @iwdev: irdma device - * @iwmr: pointer to memory window info - */ -int -irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr) -{ - struct irdma_mw_alloc_info *info; - struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); - struct irdma_cqp_request *cqp_request; - struct cqp_cmds_info *cqp_info; - int status; - - cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); - if (!cqp_request) - return -ENOMEM; - - cqp_info = &cqp_request->info; - info = &cqp_info->in.u.mw_alloc.info; - memset(info, 0, sizeof(*info)); - if (iwmr->ibmw.type == IB_MW_TYPE_1) - info->mw_wide = true; - - info->page_size = PAGE_SIZE; - info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; - info->pd_id = iwpd->sc_pd.pd_id; - info->remote_access = true; - cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC; - cqp_info->post_sq = 1; - cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev; - cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request; - status = irdma_handle_cqp_op(iwdev->rf, cqp_request); - irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); - - return status; -} - -/** - * irdma_dealloc_mw - Dealloc memory window - * @ibmw: memory window structure. - */ -static int -irdma_dealloc_mw(struct ib_mw *ibmw) -{ - struct ib_pd *ibpd = ibmw->pd; - struct irdma_pd *iwpd = to_iwpd(ibpd); - struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw); - struct irdma_device *iwdev = to_iwdev(ibmw->device); - struct irdma_cqp_request *cqp_request; - struct cqp_cmds_info *cqp_info; - struct irdma_dealloc_stag_info *info; - - cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); - if (!cqp_request) - return -ENOMEM; - - cqp_info = &cqp_request->info; - info = &cqp_info->in.u.dealloc_stag.info; - memset(info, 0, sizeof(*info)); - info->pd_id = iwpd->sc_pd.pd_id; - info->stag_idx = RS_64_1(ibmw->rkey, IRDMA_CQPSQ_STAG_IDX_S); - info->mr = false; - cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; - cqp_info->post_sq = 1; - cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; - cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; - irdma_handle_cqp_op(iwdev->rf, cqp_request); - irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); - irdma_free_stag(iwdev, iwmr->stag); - kfree(iwmr); - - return 0; -} - -/** * irdma_hw_alloc_stag - cqp command to allocate stag * @iwdev: irdma device * @iwmr: irdma mr pointer @@ -2088,7 +2012,8 @@ irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, stag_info->stag_key = (u8)iwmr->stag; stag_info->total_len = iwmr->len; stag_info->all_memory = (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) ? true : false; - stag_info->access_rights = irdma_get_mr_access(access); + stag_info->access_rights = irdma_get_mr_access(access, + iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev); stag_info->pd_id = iwpd->sc_pd.pd_id; if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; @@ -2162,14 +2087,18 @@ irdma_free_iwmr(struct irdma_mr *iwmr) } /* - * irdma_reg_user_mr_type_mem - Handle memory registration @iwmr - irdma mr @access - access rights + * irdma_reg_user_mr_type_mem - Handle memory registration + * @iwmr - irdma mr + * @access - access rights + * @create_stag - flag to create stag or not */ static int -irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access) +irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, + bool create_stag) { struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); struct irdma_pbl *iwpbl = &iwmr->iwpbl; - u32 stag; + u32 stag = 0; int err; u8 lvl; @@ -2188,15 +2117,17 @@ irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access) } } - stag = irdma_create_stag(iwdev); - if (!stag) { - err = -ENOMEM; - goto free_pble; - } + if (create_stag) { + stag = irdma_create_stag(iwdev); + if (!stag) { + err = -ENOMEM; + goto free_pble; + } - iwmr->stag = stag; - iwmr->ibmr.rkey = stag; - iwmr->ibmr.lkey = stag; + iwmr->stag = stag; + iwmr->ibmr.rkey = stag; + iwmr->ibmr.lkey = stag; + } iwmr->access = access; err = irdma_hwreg_mr(iwdev, iwmr, access); if (err) @@ -2205,7 +2136,8 @@ irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access) return 0; err_hwreg: - irdma_free_stag(iwdev, stag); + if (stag) + irdma_free_stag(iwdev, stag); free_pble: if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) @@ -2344,7 +2276,7 @@ irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, break; case IRDMA_MEMREG_TYPE_MEM: - err = irdma_reg_user_mr_type_mem(iwmr, access); + err = irdma_reg_user_mr_type_mem(iwmr, access, true); if (err) goto error; @@ -2421,10 +2353,8 @@ irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, { struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); struct irdma_pbl *iwpbl = &iwmr->iwpbl; - struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct ib_pd *pd = iwmr->ibmr.pd; struct ib_umem *region; - u8 lvl; int err; region = ib_umem_get(pd->uobject->context, start, len, iwmr->access, 0); @@ -2445,35 +2375,14 @@ irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size, virt); - lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; - - err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); + err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false); if (err) - goto error; - - if (lvl) { - err = irdma_check_mr_contiguous(palloc, - iwmr->page_size); - if (err) { - irdma_free_pble(iwdev->rf->pble_rsrc, palloc); - iwpbl->pbl_allocated = false; - } - } - - err = irdma_hwreg_mr(iwdev, iwmr, iwmr->access); - if (err) - goto error; + goto err; return &iwmr->ibmr; -error: - if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) { - irdma_free_pble(iwdev->rf->pble_rsrc, palloc); - iwpbl->pbl_allocated = false; - } +err: ib_umem_release(region); - iwmr->region = NULL; - return ERR_PTR(err); } @@ -2581,25 +2490,6 @@ irdma_del_memlist(struct irdma_mr *iwmr, } /** - * irdma_copy_sg_list - copy sg list for qp - * @sg_list: copied into sg_list - * @sgl: copy from sgl - * @num_sges: count of sg entries - */ -static void -irdma_copy_sg_list(struct irdma_sge *sg_list, struct ib_sge *sgl, - int num_sges) -{ - unsigned int i; - - for (i = 0; i < num_sges; i++) { - sg_list[i].tag_off = sgl[i].addr; - sg_list[i].len = sgl[i].length; - sg_list[i].stag = sgl[i].lkey; - } -} - -/** * irdma_post_send - kernel application wr * @ibqp: qp ptr for wr * @ib_wr: work request ptr @@ -2659,7 +2549,7 @@ irdma_post_send(struct ib_qp *ibqp, } info.op.send.num_sges = ib_wr->num_sge; - info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list; + info.op.send.sg_list = ib_wr->sg_list; if (iwqp->ibqp.qp_type == IB_QPT_UD || iwqp->ibqp.qp_type == IB_QPT_GSI) { ah = to_iwah(ud_wr(ib_wr)->ah); @@ -2690,8 +2580,8 @@ irdma_post_send(struct ib_qp *ibqp, info.op.rdma_write.num_lo_sges = ib_wr->num_sge; info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list; - info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; - info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey; + info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; + info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey; if (ib_wr->send_flags & IB_SEND_INLINE) err = irdma_uk_inline_rdma_write(ukqp, &info, false); else @@ -2707,8 +2597,8 @@ irdma_post_send(struct ib_qp *ibqp, break; } info.op_type = IRDMA_OP_TYPE_RDMA_READ; - info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; - info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey; + info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; + info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey; info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; info.op.rdma_read.num_lo_sges = ib_wr->num_sge; err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false); @@ -2726,7 +2616,9 @@ irdma_post_send(struct ib_qp *ibqp, stag_info.signaled = info.signaled; stag_info.read_fence = info.read_fence; - stag_info.access_rights = irdma_get_mr_access(reg_wr(ib_wr)->access); + stag_info.access_rights = + irdma_get_mr_access(reg_wr(ib_wr)->access, + dev->hw_attrs.uk_attrs.hw_rev); stag_info.stag_key = reg_wr(ib_wr)->key & 0xff; stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; stag_info.page_size = reg_wr(ib_wr)->mr->page_size; @@ -2788,7 +2680,6 @@ irdma_post_recv(struct ib_qp *ibqp, struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_qp_uk *ukqp = &iwqp->sc_qp.qp_uk; struct irdma_post_rq_info post_recv = {0}; - struct irdma_sge *sg_list = iwqp->sg_list; unsigned long flags; int err = 0; @@ -2801,8 +2692,7 @@ irdma_post_recv(struct ib_qp *ibqp, } post_recv.num_sges = ib_wr->num_sge; post_recv.wr_id = ib_wr->wr_id; - irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge); - post_recv.sg_list = sg_list; + post_recv.sg_list = ib_wr->sg_list; err = irdma_uk_post_receive(ukqp, &post_recv); if (err) { irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, @@ -3217,7 +3107,7 @@ irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) { irdma_copy_ip_ntohl(ip_addr, sgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32); - irdma_netdev_vlan_ipv6(iwqp->cm_id, ip_addr, &vlan_id, NULL); + irdma_get_vlan_mac_ipv6(iwqp->cm_id, ip_addr, &vlan_id, NULL); ipv4 = false; irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "qp_id=%d, IP6address=%x:%x:%x:%x\n", ibqp->qp_num, @@ -3440,12 +3330,10 @@ irdma_set_device_ops(struct ib_device *ibdev) dev_ops->alloc_hw_stats = irdma_alloc_hw_stats; dev_ops->alloc_mr = irdma_alloc_mr; - dev_ops->alloc_mw = irdma_alloc_mw; dev_ops->alloc_pd = irdma_alloc_pd; dev_ops->alloc_ucontext = irdma_alloc_ucontext; dev_ops->create_cq = irdma_create_cq; dev_ops->create_qp = irdma_create_qp; - dev_ops->dealloc_mw = irdma_dealloc_mw; dev_ops->dealloc_pd = irdma_dealloc_pd; dev_ops->dealloc_ucontext = irdma_dealloc_ucontext; dev_ops->dereg_mr = irdma_dereg_mr; diff --git a/sys/dev/irdma/irdma_verbs.h b/sys/dev/irdma/irdma_verbs.h index 890840c5d1d9..0e8cf06791d5 100644 --- a/sys/dev/irdma/irdma_verbs.h +++ b/sys/dev/irdma/irdma_verbs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -254,7 +254,7 @@ struct irdma_qp { struct irdma_dma_mem host_ctx; struct timer_list terminate_timer; struct irdma_pbl *iwpbl; - struct irdma_sge *sg_list; + struct ib_sge *sg_list; struct irdma_dma_mem q2_ctx_mem; struct irdma_dma_mem ietf_mem; struct completion free_qp; diff --git a/sys/dev/irdma/irdma_ws.c b/sys/dev/irdma/irdma_ws.c index a2afba5c738a..d311343111b9 100644 --- a/sys/dev/irdma/irdma_ws.c +++ b/sys/dev/irdma/irdma_ws.c @@ -57,7 +57,7 @@ irdma_alloc_node(struct irdma_sc_vsi *vsi, struct irdma_ws_node *node; u16 node_index = 0; - ws_mem.size = sizeof(struct irdma_ws_node); + ws_mem.size = sizeof(*node); ws_mem.va = kzalloc(ws_mem.size, GFP_KERNEL); if (!ws_mem.va) return NULL; @@ -109,7 +109,7 @@ irdma_free_node(struct irdma_sc_vsi *vsi, irdma_free_ws_node_id(vsi->dev, node->index); ws_mem.va = node; - ws_mem.size = sizeof(struct irdma_ws_node); + ws_mem.size = sizeof(*node); kfree(ws_mem.va); } diff --git a/sys/dev/irdma/irdma_ws.h b/sys/dev/irdma/irdma_ws.h index cbb1bddf96c1..91deecffac62 100644 --- a/sys/dev/irdma/irdma_ws.h +++ b/sys/dev/irdma/irdma_ws.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU |