aboutsummaryrefslogtreecommitdiff
path: root/sys/dev/nvmf/nvmf_tcp.c
diff options
context:
space:
mode:
Diffstat (limited to 'sys/dev/nvmf/nvmf_tcp.c')
-rw-r--r--sys/dev/nvmf/nvmf_tcp.c93
1 files changed, 56 insertions, 37 deletions
diff --git a/sys/dev/nvmf/nvmf_tcp.c b/sys/dev/nvmf/nvmf_tcp.c
index 57c81eceee02..6ad5229f6043 100644
--- a/sys/dev/nvmf/nvmf_tcp.c
+++ b/sys/dev/nvmf/nvmf_tcp.c
@@ -18,6 +18,7 @@
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/mutex.h>
+#include <sys/nv.h>
#include <sys/protosw.h>
#include <sys/refcount.h>
#include <sys/socket.h>
@@ -138,7 +139,7 @@ static void tcp_free_qpair(struct nvmf_qpair *nq);
SYSCTL_NODE(_kern_nvmf, OID_AUTO, tcp, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"TCP transport");
static u_int tcp_max_transmit_data = 256 * 1024;
-SYSCTL_UINT(_kern_nvmf_tcp, OID_AUTO, max_c2hdata, CTLFLAG_RWTUN,
+SYSCTL_UINT(_kern_nvmf_tcp, OID_AUTO, max_transmit_data, CTLFLAG_RWTUN,
&tcp_max_transmit_data, 0,
"Maximum size of data payload in a transmitted PDU");
@@ -442,7 +443,7 @@ nvmf_tcp_construct_pdu(struct nvmf_tcp_qpair *qp, void *hdr, size_t hlen,
plen += sizeof(digest);
if (data_len != 0) {
KASSERT(m_length(data, NULL) == data_len, ("length mismatch"));
- pdo = roundup2(plen, qp->txpda);
+ pdo = roundup(plen, qp->txpda);
pad = pdo - plen;
plen = pdo + data_len;
if (qp->data_digests)
@@ -623,10 +624,7 @@ mbuf_copyto_io(struct mbuf *m, u_int skip, u_int len,
while (len != 0) {
MPASS((m->m_flags & M_EXTPG) == 0);
- todo = m->m_len - skip;
- if (todo > len)
- todo = len;
-
+ todo = min(m->m_len - skip, len);
memdesc_copyback(&io->io_mem, io_offset, todo, mtodo(m, skip));
skip = 0;
io_offset += todo;
@@ -887,7 +885,7 @@ nvmf_tcp_mext_pg(void *arg, int how)
struct nvmf_tcp_command_buffer *cb = arg;
struct mbuf *m;
- m = mb_alloc_ext_pgs(how, nvmf_tcp_free_mext_pg);
+ m = mb_alloc_ext_pgs(how, nvmf_tcp_free_mext_pg, M_RDONLY);
m->m_ext.ext_arg1 = cb;
tcp_hold_command_buffer(cb);
return (m);
@@ -1000,9 +998,7 @@ nvmf_tcp_handle_r2t(struct nvmf_tcp_qpair *qp, struct nvmf_tcp_rxpdu *pdu)
struct mbuf *m;
uint32_t sent, todo;
- todo = data_len;
- if (todo > qp->max_tx_data)
- todo = qp->max_tx_data;
+ todo = min(data_len, qp->max_tx_data);
m = nvmf_tcp_command_buffer_mbuf(cb, data_offset, todo, &sent,
todo < data_len);
tcp_send_h2c_pdu(qp, r2t->cccid, r2t->ttag, data_offset, m,
@@ -1418,8 +1414,7 @@ nvmf_soupcall_send(struct socket *so, void *arg, int waitflag)
}
static struct nvmf_qpair *
-tcp_allocate_qpair(bool controller,
- const struct nvmf_handoff_qpair_params *params)
+tcp_allocate_qpair(bool controller, const nvlist_t *nvl)
{
struct nvmf_tcp_qpair *qp;
struct socket *so;
@@ -1427,8 +1422,18 @@ tcp_allocate_qpair(bool controller,
cap_rights_t rights;
int error;
- error = fget(curthread, params->tcp.fd, cap_rights_init_one(&rights,
- CAP_SOCK_CLIENT), &fp);
+ if (!nvlist_exists_number(nvl, "fd") ||
+ !nvlist_exists_number(nvl, "rxpda") ||
+ !nvlist_exists_number(nvl, "txpda") ||
+ !nvlist_exists_bool(nvl, "header_digests") ||
+ !nvlist_exists_bool(nvl, "data_digests") ||
+ !nvlist_exists_number(nvl, "maxr2t") ||
+ !nvlist_exists_number(nvl, "maxh2cdata") ||
+ !nvlist_exists_number(nvl, "max_icd"))
+ return (NULL);
+
+ error = fget(curthread, nvlist_get_number(nvl, "fd"),
+ cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp);
if (error != 0)
return (NULL);
if (fp->f_type != DTYPE_SOCKET) {
@@ -1450,26 +1455,28 @@ tcp_allocate_qpair(bool controller,
qp = malloc(sizeof(*qp), M_NVMF_TCP, M_WAITOK | M_ZERO);
qp->so = so;
refcount_init(&qp->refs, 1);
- qp->txpda = params->tcp.txpda;
- qp->rxpda = params->tcp.rxpda;
- qp->header_digests = params->tcp.header_digests;
- qp->data_digests = params->tcp.data_digests;
- qp->maxr2t = params->tcp.maxr2t;
- qp->maxh2cdata = params->tcp.maxh2cdata;
+ qp->txpda = nvlist_get_number(nvl, "txpda");
+ qp->rxpda = nvlist_get_number(nvl, "rxpda");
+ qp->header_digests = nvlist_get_bool(nvl, "header_digests");
+ qp->data_digests = nvlist_get_bool(nvl, "data_digests");
+ qp->maxr2t = nvlist_get_number(nvl, "maxr2t");
+ if (controller)
+ qp->maxh2cdata = nvlist_get_number(nvl, "maxh2cdata");
qp->max_tx_data = tcp_max_transmit_data;
if (!controller) {
- if (qp->max_tx_data > params->tcp.maxh2cdata)
- qp->max_tx_data = params->tcp.maxh2cdata;
+ qp->max_tx_data = min(qp->max_tx_data,
+ nvlist_get_number(nvl, "maxh2cdata"));
+ qp->max_icd = nvlist_get_number(nvl, "max_icd");
}
- qp->max_icd = params->tcp.max_icd;
if (controller) {
/* Use the SUCCESS flag if SQ flow control is disabled. */
- qp->send_success = !params->sq_flow_control;
+ qp->send_success = !nvlist_get_bool(nvl, "sq_flow_control");
/* NB: maxr2t is 0's based. */
qp->num_ttags = MIN((u_int)UINT16_MAX + 1,
- (uint64_t)params->qsize * (uint64_t)qp->maxr2t + 1);
+ nvlist_get_number(nvl, "qsize") *
+ ((uint64_t)qp->maxr2t + 1));
qp->open_ttags = mallocarray(qp->num_ttags,
sizeof(*qp->open_ttags), M_NVMF_TCP, M_WAITOK | M_ZERO);
}
@@ -1558,6 +1565,7 @@ tcp_free_qpair(struct nvmf_qpair *nq)
for (u_int i = 0; i < qp->num_ttags; i++) {
cb = qp->open_ttags[i];
if (cb != NULL) {
+ cb->tc->active_r2ts--;
cb->error = ECONNABORTED;
tcp_release_command_buffer(cb);
}
@@ -1569,6 +1577,10 @@ tcp_free_qpair(struct nvmf_qpair *nq)
TAILQ_FOREACH_SAFE(cb, &qp->rx_buffers.head, link, ncb) {
tcp_remove_command_buffer(&qp->rx_buffers, cb);
mtx_unlock(&qp->rx_buffers.lock);
+#ifdef INVARIANTS
+ if (cb->tc != NULL)
+ cb->tc->pending_r2ts--;
+#endif
cb->error = ECONNABORTED;
tcp_release_command_buffer(cb);
mtx_lock(&qp->rx_buffers.lock);
@@ -1784,7 +1796,6 @@ tcp_send_controller_data(struct nvmf_capsule *nc, uint32_t data_offset,
{
struct nvmf_tcp_qpair *qp = TQP(nc->nc_qpair);
struct nvme_sgl_descriptor *sgl;
- struct mbuf *n, *p;
uint32_t data_len;
bool last_pdu, last_xfer;
@@ -1813,21 +1824,29 @@ tcp_send_controller_data(struct nvmf_capsule *nc, uint32_t data_offset,
/* Queue one more C2H_DATA PDUs containing the data from 'm'. */
while (m != NULL) {
+ struct mbuf *n;
uint32_t todo;
- todo = m->m_len;
- p = m;
- n = p->m_next;
- while (n != NULL) {
- if (todo + n->m_len > qp->max_tx_data) {
- p->m_next = NULL;
- break;
- }
- todo += n->m_len;
- p = n;
+ if (m->m_len > qp->max_tx_data) {
+ n = m_split(m, qp->max_tx_data, M_WAITOK);
+ todo = m->m_len;
+ } else {
+ struct mbuf *p;
+
+ todo = m->m_len;
+ p = m;
n = p->m_next;
+ while (n != NULL) {
+ if (todo + n->m_len > qp->max_tx_data) {
+ p->m_next = NULL;
+ break;
+ }
+ todo += n->m_len;
+ p = n;
+ n = p->m_next;
+ }
+ MPASS(m_length(m, NULL) == todo);
}
- MPASS(m_length(m, NULL) == todo);
last_pdu = (n == NULL && last_xfer);
tcp_send_c2h_pdu(qp, nc->nc_sqe.cid, data_offset, m, todo,
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-30 19:15:10 +0000