diff options
Diffstat (limited to 'sys/dev/nvmf/nvmf_tcp.c')
-rw-r--r-- | sys/dev/nvmf/nvmf_tcp.c | 93 |
1 files changed, 56 insertions, 37 deletions
diff --git a/sys/dev/nvmf/nvmf_tcp.c b/sys/dev/nvmf/nvmf_tcp.c index 57c81eceee02..6ad5229f6043 100644 --- a/sys/dev/nvmf/nvmf_tcp.c +++ b/sys/dev/nvmf/nvmf_tcp.c @@ -18,6 +18,7 @@ #include <sys/mbuf.h> #include <sys/module.h> #include <sys/mutex.h> +#include <sys/nv.h> #include <sys/protosw.h> #include <sys/refcount.h> #include <sys/socket.h> @@ -138,7 +139,7 @@ static void tcp_free_qpair(struct nvmf_qpair *nq); SYSCTL_NODE(_kern_nvmf, OID_AUTO, tcp, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "TCP transport"); static u_int tcp_max_transmit_data = 256 * 1024; -SYSCTL_UINT(_kern_nvmf_tcp, OID_AUTO, max_c2hdata, CTLFLAG_RWTUN, +SYSCTL_UINT(_kern_nvmf_tcp, OID_AUTO, max_transmit_data, CTLFLAG_RWTUN, &tcp_max_transmit_data, 0, "Maximum size of data payload in a transmitted PDU"); @@ -442,7 +443,7 @@ nvmf_tcp_construct_pdu(struct nvmf_tcp_qpair *qp, void *hdr, size_t hlen, plen += sizeof(digest); if (data_len != 0) { KASSERT(m_length(data, NULL) == data_len, ("length mismatch")); - pdo = roundup2(plen, qp->txpda); + pdo = roundup(plen, qp->txpda); pad = pdo - plen; plen = pdo + data_len; if (qp->data_digests) @@ -623,10 +624,7 @@ mbuf_copyto_io(struct mbuf *m, u_int skip, u_int len, while (len != 0) { MPASS((m->m_flags & M_EXTPG) == 0); - todo = m->m_len - skip; - if (todo > len) - todo = len; - + todo = min(m->m_len - skip, len); memdesc_copyback(&io->io_mem, io_offset, todo, mtodo(m, skip)); skip = 0; io_offset += todo; @@ -887,7 +885,7 @@ nvmf_tcp_mext_pg(void *arg, int how) struct nvmf_tcp_command_buffer *cb = arg; struct mbuf *m; - m = mb_alloc_ext_pgs(how, nvmf_tcp_free_mext_pg); + m = mb_alloc_ext_pgs(how, nvmf_tcp_free_mext_pg, M_RDONLY); m->m_ext.ext_arg1 = cb; tcp_hold_command_buffer(cb); return (m); @@ -1000,9 +998,7 @@ nvmf_tcp_handle_r2t(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu) struct mbuf *m; uint32_t sent, todo; - todo = data_len; - if (todo > qp->max_tx_data) - todo = qp->max_tx_data; + todo = min(data_len, qp->max_tx_data); m = nvmf_tcp_command_buffer_mbuf(cb, data_offset, todo, &sent, todo < data_len); tcp_send_h2c_pdu(qp, r2t->cccid, r2t->ttag, data_offset, m, @@ -1418,8 +1414,7 @@ nvmf_soupcall_send(struct socket *so, void *arg, int waitflag) } static struct nvmf_qpair * -tcp_allocate_qpair(bool controller, - const struct nvmf_handoff_qpair_params *params) +tcp_allocate_qpair(bool controller, const nvlist_t *nvl) { struct nvmf_tcp_qpair *qp; struct socket *so; @@ -1427,8 +1422,18 @@ tcp_allocate_qpair(bool controller, cap_rights_t rights; int error; - error = fget(curthread, params->tcp.fd, cap_rights_init_one(&rights, - CAP_SOCK_CLIENT), &fp); + if (!nvlist_exists_number(nvl, "fd") || + !nvlist_exists_number(nvl, "rxpda") || + !nvlist_exists_number(nvl, "txpda") || + !nvlist_exists_bool(nvl, "header_digests") || + !nvlist_exists_bool(nvl, "data_digests") || + !nvlist_exists_number(nvl, "maxr2t") || + !nvlist_exists_number(nvl, "maxh2cdata") || + !nvlist_exists_number(nvl, "max_icd")) + return (NULL); + + error = fget(curthread, nvlist_get_number(nvl, "fd"), + cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp); if (error != 0) return (NULL); if (fp->f_type != DTYPE_SOCKET) { @@ -1450,26 +1455,28 @@ tcp_allocate_qpair(bool controller, qp = malloc(sizeof(*qp), M_NVMF_TCP, M_WAITOK | M_ZERO); qp->so = so; refcount_init(&qp->refs, 1); - qp->txpda = params->tcp.txpda; - qp->rxpda = params->tcp.rxpda; - qp->header_digests = params->tcp.header_digests; - qp->data_digests = params->tcp.data_digests; - qp->maxr2t = params->tcp.maxr2t; - qp->maxh2cdata = params->tcp.maxh2cdata; + qp->txpda = nvlist_get_number(nvl, "txpda"); + qp->rxpda = nvlist_get_number(nvl, "rxpda"); + qp->header_digests = nvlist_get_bool(nvl, "header_digests"); + qp->data_digests = nvlist_get_bool(nvl, "data_digests"); + qp->maxr2t = nvlist_get_number(nvl, "maxr2t"); + if (controller) + qp->maxh2cdata = nvlist_get_number(nvl, "maxh2cdata"); qp->max_tx_data = tcp_max_transmit_data; if (!controller) { - if (qp->max_tx_data > params->tcp.maxh2cdata) - qp->max_tx_data = params->tcp.maxh2cdata; + qp->max_tx_data = min(qp->max_tx_data, + nvlist_get_number(nvl, "maxh2cdata")); + qp->max_icd = nvlist_get_number(nvl, "max_icd"); } - qp->max_icd = params->tcp.max_icd; if (controller) { /* Use the SUCCESS flag if SQ flow control is disabled. */ - qp->send_success = !params->sq_flow_control; + qp->send_success = !nvlist_get_bool(nvl, "sq_flow_control"); /* NB: maxr2t is 0's based. */ qp->num_ttags = MIN((u_int)UINT16_MAX + 1, - (uint64_t)params->qsize * (uint64_t)qp->maxr2t + 1); + nvlist_get_number(nvl, "qsize") * + ((uint64_t)qp->maxr2t + 1)); qp->open_ttags = mallocarray(qp->num_ttags, sizeof(*qp->open_ttags), M_NVMF_TCP, M_WAITOK | M_ZERO); } @@ -1558,6 +1565,7 @@ tcp_free_qpair(struct nvmf_qpair *nq) for (u_int i = 0; i < qp->num_ttags; i++) { cb = qp->open_ttags[i]; if (cb != NULL) { + cb->tc->active_r2ts--; cb->error = ECONNABORTED; tcp_release_command_buffer(cb); } @@ -1569,6 +1577,10 @@ tcp_free_qpair(struct nvmf_qpair *nq) TAILQ_FOREACH_SAFE(cb, &qp->rx_buffers.head, link, ncb) { tcp_remove_command_buffer(&qp->rx_buffers, cb); mtx_unlock(&qp->rx_buffers.lock); +#ifdef INVARIANTS + if (cb->tc != NULL) + cb->tc->pending_r2ts--; +#endif cb->error = ECONNABORTED; tcp_release_command_buffer(cb); mtx_lock(&qp->rx_buffers.lock); @@ -1784,7 +1796,6 @@ tcp_send_controller_data(struct nvmf_capsule *nc, uint32_t data_offset, { struct nvmf_tcp_qpair *qp = TQP(nc->nc_qpair); struct nvme_sgl_descriptor *sgl; - struct mbuf *n, *p; uint32_t data_len; bool last_pdu, last_xfer; @@ -1813,21 +1824,29 @@ tcp_send_controller_data(struct nvmf_capsule *nc, uint32_t data_offset, /* Queue one more C2H_DATA PDUs containing the data from 'm'. */ while (m != NULL) { + struct mbuf *n; uint32_t todo; - todo = m->m_len; - p = m; - n = p->m_next; - while (n != NULL) { - if (todo + n->m_len > qp->max_tx_data) { - p->m_next = NULL; - break; - } - todo += n->m_len; - p = n; + if (m->m_len > qp->max_tx_data) { + n = m_split(m, qp->max_tx_data, M_WAITOK); + todo = m->m_len; + } else { + struct mbuf *p; + + todo = m->m_len; + p = m; n = p->m_next; + while (n != NULL) { + if (todo + n->m_len > qp->max_tx_data) { + p->m_next = NULL; + break; + } + todo += n->m_len; + p = n; + n = p->m_next; + } + MPASS(m_length(m, NULL) == todo); } - MPASS(m_length(m, NULL) == todo); last_pdu = (n == NULL && last_xfer); tcp_send_c2h_pdu(qp, nc->nc_sqe.cid, data_offset, m, todo, |