diff options
Diffstat (limited to 'sys/dev/nvmf/controller')
| -rw-r--r-- | sys/dev/nvmf/controller/ctl_frontend_nvmf.c | 196 | ||||
| -rw-r--r-- | sys/dev/nvmf/controller/nvmft_controller.c | 78 | ||||
| -rw-r--r-- | sys/dev/nvmf/controller/nvmft_qpair.c | 72 | ||||
| -rw-r--r-- | sys/dev/nvmf/controller/nvmft_var.h | 19 |
4 files changed, 278 insertions, 87 deletions
diff --git a/sys/dev/nvmf/controller/ctl_frontend_nvmf.c b/sys/dev/nvmf/controller/ctl_frontend_nvmf.c index a203bb1c90a6..658b47699c1d 100644 --- a/sys/dev/nvmf/controller/ctl_frontend_nvmf.c +++ b/sys/dev/nvmf/controller/ctl_frontend_nvmf.c @@ -19,7 +19,9 @@ #include <sys/queue.h> #include <sys/refcount.h> #include <sys/sbuf.h> +#include <sys/smp.h> #include <sys/sx.h> +#include <sys/taskqueue.h> #include <machine/bus.h> #include <machine/bus_dma.h> @@ -31,8 +33,10 @@ #include <cam/ctl/ctl.h> #include <cam/ctl/ctl_error.h> +#include <cam/ctl/ctl_ha.h> #include <cam/ctl/ctl_io.h> #include <cam/ctl/ctl_frontend.h> +#include <cam/ctl/ctl_private.h> /* * Store pointers to the capsule and qpair in the two pointer members @@ -47,6 +51,7 @@ static int nvmft_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int flag, struct thread *td); static int nvmft_shutdown(void); +static struct taskqueue *nvmft_taskq; static TAILQ_HEAD(, nvmft_port) nvmft_ports; static struct sx nvmft_ports_lock; @@ -65,9 +70,9 @@ nvmft_online(void *arg) { struct nvmft_port *np = arg; - sx_xlock(&np->lock); + mtx_lock(&np->lock); np->online = true; - sx_xunlock(&np->lock); + mtx_unlock(&np->lock); } static void @@ -76,7 +81,7 @@ nvmft_offline(void *arg) struct nvmft_port *np = arg; struct nvmft_controller *ctrlr; - sx_xlock(&np->lock); + mtx_lock(&np->lock); np->online = false; TAILQ_FOREACH(ctrlr, &np->controllers, link) { @@ -86,8 +91,32 @@ nvmft_offline(void *arg) } while (!TAILQ_EMPTY(&np->controllers)) - sx_sleep(np, &np->lock, 0, "nvmfoff", 0); - sx_xunlock(&np->lock); + mtx_sleep(np, &np->lock, 0, "nvmfoff", 0); + mtx_unlock(&np->lock); +} + +static int +nvmft_info(void *arg, struct sbuf *sb) +{ + struct nvmft_port *np = arg; + struct nvmft_controller *ctrlr; + int retval; + + mtx_lock(&np->lock); + retval = sbuf_printf(sb, "\t<port>%s,p,%u</port>\n", np->cdata.subnqn, + np->portid); + if (retval != 0) + goto out; + + TAILQ_FOREACH(ctrlr, &np->controllers, link) { + retval = sbuf_printf(sb, "\t<host id=\"%u\">%s</host>\n", + ctrlr->cntlid, ctrlr->hostnqn); + if (retval != 0) + break; + } +out: + mtx_unlock(&np->lock); + return (retval); } static int @@ -97,7 +126,7 @@ nvmft_lun_enable(void *arg, int lun_id) struct nvmft_controller *ctrlr; uint32_t *old_ns, *new_ns; uint32_t nsid; - u_int i; + u_int i, new_count; if (lun_id >= le32toh(np->cdata.nn)) { printf("NVMFT: %s lun %d larger than maximum nsid %u\n", @@ -106,14 +135,22 @@ nvmft_lun_enable(void *arg, int lun_id) } nsid = lun_id + 1; - sx_xlock(&np->lock); - new_ns = mallocarray(np->num_ns + 1, sizeof(*new_ns), M_NVMFT, - M_WAITOK); + mtx_lock(&np->lock); + for (;;) { + new_count = np->num_ns + 1; + mtx_unlock(&np->lock); + new_ns = mallocarray(new_count, sizeof(*new_ns), M_NVMFT, + M_WAITOK); + mtx_lock(&np->lock); + if (np->num_ns + 1 <= new_count) + break; + free(new_ns, M_NVMFT); + } for (i = 0; i < np->num_ns; i++) { if (np->active_ns[i] < nsid) continue; if (np->active_ns[i] == nsid) { - sx_xunlock(&np->lock); + mtx_unlock(&np->lock); free(new_ns, M_NVMFT); printf("NVMFT: %s duplicate lun %d\n", np->cdata.subnqn, lun_id); @@ -140,7 +177,7 @@ nvmft_lun_enable(void *arg, int lun_id) nvmft_controller_lun_changed(ctrlr, lun_id); } - sx_xunlock(&np->lock); + mtx_unlock(&np->lock); free(old_ns, M_NVMFT); return (0); @@ -158,12 +195,12 @@ nvmft_lun_disable(void *arg, int lun_id) return (0); nsid = lun_id + 1; - sx_xlock(&np->lock); + mtx_lock(&np->lock); for (i = 0; i < np->num_ns; i++) { if (np->active_ns[i] == nsid) goto found; } - sx_xunlock(&np->lock); + mtx_unlock(&np->lock); printf("NVMFT: %s request to disable nonexistent lun %d\n", np->cdata.subnqn, lun_id); return (EINVAL); @@ -180,7 +217,7 @@ found: nvmft_controller_lun_changed(ctrlr, lun_id); } - sx_xunlock(&np->lock); + mtx_unlock(&np->lock); return (0); } @@ -191,7 +228,7 @@ nvmft_populate_active_nslist(struct nvmft_port *np, uint32_t nsid, { u_int i, count; - sx_slock(&np->lock); + mtx_lock(&np->lock); count = 0; for (i = 0; i < np->num_ns; i++) { if (np->active_ns[i] <= nsid) @@ -201,7 +238,7 @@ nvmft_populate_active_nslist(struct nvmft_port *np, uint32_t nsid, if (count == nitems(nslist->ns)) break; } - sx_sunlock(&np->lock); + mtx_unlock(&np->lock); } void @@ -458,8 +495,8 @@ nvmft_datamove_in(struct ctl_nvmeio *ctnio, struct nvmft_qpair *qp, ctl_datamove_done((union ctl_io *)ctnio, true); } -static void -nvmft_datamove(union ctl_io *io) +void +nvmft_handle_datamove(union ctl_io *io) { struct nvmf_capsule *nc; struct nvmft_qpair *qp; @@ -478,6 +515,35 @@ nvmft_datamove(union ctl_io *io) nvmft_datamove_out(&io->nvmeio, qp, nc); } +void +nvmft_abort_datamove(union ctl_io *io) +{ + io->io_hdr.port_status = 1; + io->io_hdr.flags |= CTL_FLAG_ABORT; + ctl_datamove_done(io, true); +} + +static void +nvmft_datamove(union ctl_io *io) +{ + struct nvmft_qpair *qp; + + qp = NVMFT_QP(io); + nvmft_qpair_datamove(qp, io); +} + +void +nvmft_enqueue_task(struct task *task) +{ + taskqueue_enqueue(nvmft_taskq, task); +} + +void +nvmft_drain_task(struct task *task) +{ + taskqueue_drain(nvmft_taskq, task); +} + static void hip_add(uint64_t pair[2], uint64_t addend) { @@ -561,6 +627,17 @@ end: static int nvmft_init(void) { + int error; + + nvmft_taskq = taskqueue_create("nvmft", M_WAITOK, + taskqueue_thread_enqueue, &nvmft_taskq); + error = taskqueue_start_threads_in_proc(&nvmft_taskq, mp_ncpus, PWAIT, + control_softc->ctl_proc, "nvmft"); + if (error != 0) { + taskqueue_free(nvmft_taskq); + return (error); + } + TAILQ_INIT(&nvmft_ports); sx_init(&nvmft_ports_lock, "nvmft ports"); return (0); @@ -580,7 +657,7 @@ nvmft_port_free(struct nvmft_port *np) free(np->active_ns, M_NVMFT); clean_unrhdr(np->ids); delete_unrhdr(np->ids); - sx_destroy(&np->lock); + mtx_destroy(&np->lock); free(np, M_NVMFT); } @@ -750,9 +827,10 @@ nvmft_port_create(struct ctl_req *req) np = malloc(sizeof(*np), M_NVMFT, M_WAITOK | M_ZERO); refcount_init(&np->refs, 1); + np->portid = portid; np->max_io_qsize = max_io_qsize; np->cap = _nvmf_controller_cap(max_io_qsize, enable_timeout / 500); - sx_init(&np->lock, "nvmft port"); + mtx_init(&np->lock, "nvmft port", NULL, MTX_DEF); np->ids = new_unrhdr(0, MIN(CTL_MAX_INIT_PER_PORT - 1, NVMF_CNTLID_STATIC_MAX), UNR_NO_MTX); TAILQ_INIT(&np->controllers); @@ -781,6 +859,7 @@ nvmft_port_create(struct ctl_req *req) port->virtual_port = 0; port->port_online = nvmft_online; port->port_offline = nvmft_offline; + port->port_info = nvmft_info; port->onoff_arg = np; port->lun_enable = nvmft_lun_enable; port->lun_disable = nvmft_lun_disable; @@ -870,7 +949,13 @@ nvmft_port_remove(struct ctl_req *req) TAILQ_REMOVE(&nvmft_ports, np, link); sx_xunlock(&nvmft_ports_lock); - ctl_port_offline(&np->port); + mtx_lock(&np->lock); + if (np->online) { + mtx_unlock(&np->lock); + ctl_port_offline(&np->port); + } else + mtx_unlock(&np->lock); + nvmft_port_rele(np); req->status = CTL_LUN_OK; } @@ -878,29 +963,55 @@ nvmft_port_remove(struct ctl_req *req) static void nvmft_handoff(struct ctl_nvmf *cn) { - struct nvmf_fabric_connect_cmd cmd; - struct nvmf_handoff_controller_qpair *handoff; - struct nvmf_fabric_connect_data *data; + const struct nvmf_fabric_connect_cmd *cmd; + const struct nvmf_fabric_connect_data *data; + const nvlist_t *params; struct nvmft_port *np; + nvlist_t *nvl; + size_t len; + enum nvmf_trtype trtype; int error; np = NULL; - data = NULL; - handoff = &cn->data.handoff; - error = copyin(handoff->cmd, &cmd, sizeof(cmd)); + error = nvmf_unpack_ioc_nvlist(&cn->data.handoff, &nvl); if (error != 0) { cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), - "Failed to copyin CONNECT SQE"); + "Failed to copyin and unpack handoff arguments"); return; } - data = malloc(sizeof(*data), M_NVMFT, M_WAITOK); - error = copyin(handoff->data, data, sizeof(*data)); - if (error != 0) { + if (!nvlist_exists_number(nvl, "trtype") || + !nvlist_exists_nvlist(nvl, "params") || + !nvlist_exists_binary(nvl, "cmd") || + !nvlist_exists_binary(nvl, "data")) { cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), - "Failed to copyin CONNECT data"); + "Handoff arguments missing required value"); + goto out; + } + + params = nvlist_get_nvlist(nvl, "params"); + if (!nvmf_validate_qpair_nvlist(params, true)) { + cn->status = CTL_NVMF_ERROR; + snprintf(cn->error_str, sizeof(cn->error_str), + "Invalid queue pair parameters"); + goto out; + } + + cmd = nvlist_get_binary(nvl, "cmd", &len); + if (len != sizeof(*cmd)) { + cn->status = CTL_NVMF_ERROR; + snprintf(cn->error_str, sizeof(cn->error_str), + "Wrong size for CONNECT SQE"); + goto out; + } + + data = nvlist_get_binary(nvl, "data", &len); + if (len != sizeof(*data)) { + cn->status = CTL_NVMF_ERROR; + snprintf(cn->error_str, sizeof(cn->error_str), + "Wrong size for CONNECT data"); goto out; } @@ -931,8 +1042,10 @@ nvmft_handoff(struct ctl_nvmf *cn) nvmft_port_ref(np); sx_sunlock(&nvmft_ports_lock); - if (handoff->params.admin) { - error = nvmft_handoff_admin_queue(np, handoff, &cmd, data); + trtype = nvlist_get_number(nvl, "trtype"); + if (nvlist_get_bool(params, "admin")) { + error = nvmft_handoff_admin_queue(np, trtype, params, cmd, + data); if (error != 0) { cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), @@ -940,11 +1053,11 @@ nvmft_handoff(struct ctl_nvmf *cn) goto out; } } else { - error = nvmft_handoff_io_queue(np, handoff, &cmd, data); + error = nvmft_handoff_io_queue(np, trtype, params, cmd, data); if (error != 0) { cn->status = CTL_NVMF_ERROR; snprintf(cn->error_str, sizeof(cn->error_str), - "Failed to handoff admin queue: %d", error); + "Failed to handoff I/O queue: %d", error); goto out; } } @@ -953,7 +1066,7 @@ nvmft_handoff(struct ctl_nvmf *cn) out: if (np != NULL) nvmft_port_rele(np); - free(data, M_NVMFT); + nvlist_destroy(nvl); } static void @@ -979,7 +1092,7 @@ nvmft_list(struct ctl_nvmf *cn) sbuf_printf(sb, "<ctlnvmflist>\n"); sx_slock(&nvmft_ports_lock); TAILQ_FOREACH(np, &nvmft_ports, link) { - sx_slock(&np->lock); + mtx_lock(&np->lock); TAILQ_FOREACH(ctrlr, &np->controllers, link) { sbuf_printf(sb, "<connection id=\"%d\">" "<hostnqn>%s</hostnqn>" @@ -991,7 +1104,7 @@ nvmft_list(struct ctl_nvmf *cn) np->cdata.subnqn, ctrlr->trtype); } - sx_sunlock(&np->lock); + mtx_unlock(&np->lock); } sx_sunlock(&nvmft_ports_lock); sbuf_printf(sb, "</ctlnvmflist>\n"); @@ -1029,7 +1142,7 @@ nvmft_terminate(struct ctl_nvmf *cn) found = false; sx_slock(&nvmft_ports_lock); TAILQ_FOREACH(np, &nvmft_ports, link) { - sx_slock(&np->lock); + mtx_lock(&np->lock); TAILQ_FOREACH(ctrlr, &np->controllers, link) { if (tp->all != 0) match = true; @@ -1047,7 +1160,7 @@ nvmft_terminate(struct ctl_nvmf *cn) nvmft_controller_error(ctrlr, NULL, ECONNABORTED); found = true; } - sx_sunlock(&np->lock); + mtx_unlock(&np->lock); } sx_sunlock(&nvmft_ports_lock); @@ -1115,6 +1228,7 @@ nvmft_shutdown(void) if (!TAILQ_EMPTY(&nvmft_ports)) return (EBUSY); + taskqueue_free(nvmft_taskq); sx_destroy(&nvmft_ports_lock); return (0); } diff --git a/sys/dev/nvmf/controller/nvmft_controller.c b/sys/dev/nvmf/controller/nvmft_controller.c index f3783eac1275..390467534ca2 100644 --- a/sys/dev/nvmf/controller/nvmft_controller.c +++ b/sys/dev/nvmf/controller/nvmft_controller.c @@ -14,7 +14,6 @@ #include <sys/memdesc.h> #include <sys/mutex.h> #include <sys/sbuf.h> -#include <sys/sx.h> #include <sys/taskqueue.h> #include <dev/nvmf/nvmf_transport.h> @@ -55,8 +54,6 @@ nvmft_controller_alloc(struct nvmft_port *np, uint16_t cntlid, ctrlr = malloc(sizeof(*ctrlr), M_NVMFT, M_WAITOK | M_ZERO); ctrlr->cntlid = cntlid; - nvmft_port_ref(np); - TAILQ_INSERT_TAIL(&np->controllers, ctrlr, link); ctrlr->np = np; mtx_init(&ctrlr->lock, "nvmft controller", NULL, MTX_DEF); callout_init(&ctrlr->ka_timer, 1); @@ -107,9 +104,8 @@ nvmft_keep_alive_timer(void *arg) } int -nvmft_handoff_admin_queue(struct nvmft_port *np, - const struct nvmf_handoff_controller_qpair *handoff, - const struct nvmf_fabric_connect_cmd *cmd, +nvmft_handoff_admin_queue(struct nvmft_port *np, enum nvmf_trtype trtype, + const nvlist_t *params, const struct nvmf_fabric_connect_cmd *cmd, const struct nvmf_fabric_connect_data *data) { struct nvmft_controller *ctrlr; @@ -120,13 +116,17 @@ nvmft_handoff_admin_queue(struct nvmft_port *np, if (cmd->qid != htole16(0)) return (EINVAL); - qp = nvmft_qpair_init(handoff->trtype, &handoff->params, 0, - "admin queue"); + qp = nvmft_qpair_init(trtype, params, 0, "admin queue"); + if (qp == NULL) { + printf("NVMFT: Failed to setup admin queue from %.*s\n", + (int)sizeof(data->hostnqn), data->hostnqn); + return (ENXIO); + } - sx_xlock(&np->lock); + mtx_lock(&np->lock); cntlid = alloc_unr(np->ids); if (cntlid == -1) { - sx_xunlock(&np->lock); + mtx_unlock(&np->lock); printf("NVMFT: Unable to allocate controller for %.*s\n", (int)sizeof(data->hostnqn), data->hostnqn); nvmft_connect_error(qp, cmd, NVME_SCT_COMMAND_SPECIFIC, @@ -141,12 +141,25 @@ nvmft_handoff_admin_queue(struct nvmft_port *np, ("%s: duplicate controllers with id %d", __func__, cntlid)); } #endif + mtx_unlock(&np->lock); ctrlr = nvmft_controller_alloc(np, cntlid, data); + + mtx_lock(&np->lock); + if (!np->online) { + mtx_unlock(&np->lock); + nvmft_controller_free(ctrlr); + free_unr(np->ids, cntlid); + nvmft_qpair_destroy(qp); + return (ENXIO); + } + nvmft_port_ref(np); + TAILQ_INSERT_TAIL(&np->controllers, ctrlr, link); + nvmft_printf(ctrlr, "associated with %.*s\n", (int)sizeof(data->hostnqn), data->hostnqn); ctrlr->admin = qp; - ctrlr->trtype = handoff->trtype; + ctrlr->trtype = trtype; /* * The spec requires a non-zero KeepAlive timer, but allow a @@ -162,17 +175,16 @@ nvmft_handoff_admin_queue(struct nvmft_port *np, callout_reset_sbt(&ctrlr->ka_timer, ctrlr->ka_sbt, 0, nvmft_keep_alive_timer, ctrlr, C_HARDCLOCK); } + mtx_unlock(&np->lock); nvmft_finish_accept(qp, cmd, ctrlr); - sx_xunlock(&np->lock); return (0); } int -nvmft_handoff_io_queue(struct nvmft_port *np, - const struct nvmf_handoff_controller_qpair *handoff, - const struct nvmf_fabric_connect_cmd *cmd, +nvmft_handoff_io_queue(struct nvmft_port *np, enum nvmf_trtype trtype, + const nvlist_t *params, const struct nvmf_fabric_connect_cmd *cmd, const struct nvmf_fabric_connect_data *data) { struct nvmft_controller *ctrlr; @@ -186,15 +198,20 @@ nvmft_handoff_io_queue(struct nvmft_port *np, cntlid = le16toh(data->cntlid); snprintf(name, sizeof(name), "I/O queue %u", qid); - qp = nvmft_qpair_init(handoff->trtype, &handoff->params, qid, name); + qp = nvmft_qpair_init(trtype, params, qid, name); + if (qp == NULL) { + printf("NVMFT: Failed to setup I/O queue %u from %.*s\n", qid, + (int)sizeof(data->hostnqn), data->hostnqn); + return (ENXIO); + } - sx_slock(&np->lock); + mtx_lock(&np->lock); TAILQ_FOREACH(ctrlr, &np->controllers, link) { if (ctrlr->cntlid == cntlid) break; } if (ctrlr == NULL) { - sx_sunlock(&np->lock); + mtx_unlock(&np->lock); printf("NVMFT: Nonexistent controller %u for I/O queue %u from %.*s\n", ctrlr->cntlid, qid, (int)sizeof(data->hostnqn), data->hostnqn); @@ -205,7 +222,7 @@ nvmft_handoff_io_queue(struct nvmft_port *np, } if (memcmp(ctrlr->hostid, data->hostid, sizeof(ctrlr->hostid)) != 0) { - sx_sunlock(&np->lock); + mtx_unlock(&np->lock); nvmft_printf(ctrlr, "hostid mismatch for I/O queue %u from %.*s\n", qid, (int)sizeof(data->hostnqn), data->hostnqn); @@ -215,7 +232,7 @@ nvmft_handoff_io_queue(struct nvmft_port *np, return (EINVAL); } if (memcmp(ctrlr->hostnqn, data->hostnqn, sizeof(ctrlr->hostnqn)) != 0) { - sx_sunlock(&np->lock); + mtx_unlock(&np->lock); nvmft_printf(ctrlr, "hostnqn mismatch for I/O queue %u from %.*s\n", qid, (int)sizeof(data->hostnqn), data->hostnqn); @@ -225,12 +242,12 @@ nvmft_handoff_io_queue(struct nvmft_port *np, return (EINVAL); } - /* XXX: Require handoff->trtype == ctrlr->trtype? */ + /* XXX: Require trtype == ctrlr->trtype? */ mtx_lock(&ctrlr->lock); if (ctrlr->shutdown) { mtx_unlock(&ctrlr->lock); - sx_sunlock(&np->lock); + mtx_unlock(&np->lock); nvmft_printf(ctrlr, "attempt to create I/O queue %u on disabled controller from %.*s\n", qid, (int)sizeof(data->hostnqn), data->hostnqn); @@ -241,7 +258,7 @@ nvmft_handoff_io_queue(struct nvmft_port *np, } if (ctrlr->num_io_queues == 0) { mtx_unlock(&ctrlr->lock); - sx_sunlock(&np->lock); + mtx_unlock(&np->lock); nvmft_printf(ctrlr, "attempt to create I/O queue %u without enabled queues from %.*s\n", qid, (int)sizeof(data->hostnqn), data->hostnqn); @@ -252,7 +269,7 @@ nvmft_handoff_io_queue(struct nvmft_port *np, } if (cmd->qid > ctrlr->num_io_queues) { mtx_unlock(&ctrlr->lock); - sx_sunlock(&np->lock); + mtx_unlock(&np->lock); nvmft_printf(ctrlr, "attempt to create invalid I/O queue %u from %.*s\n", qid, (int)sizeof(data->hostnqn), data->hostnqn); @@ -263,7 +280,7 @@ nvmft_handoff_io_queue(struct nvmft_port *np, } if (ctrlr->io_qpairs[qid - 1].qp != NULL) { mtx_unlock(&ctrlr->lock); - sx_sunlock(&np->lock); + mtx_unlock(&np->lock); nvmft_printf(ctrlr, "attempt to re-create I/O queue %u from %.*s\n", qid, (int)sizeof(data->hostnqn), data->hostnqn); @@ -275,8 +292,8 @@ nvmft_handoff_io_queue(struct nvmft_port *np, ctrlr->io_qpairs[qid - 1].qp = qp; mtx_unlock(&ctrlr->lock); + mtx_unlock(&np->lock); nvmft_finish_accept(qp, cmd, ctrlr); - sx_sunlock(&np->lock); return (0); } @@ -375,11 +392,11 @@ nvmft_controller_terminate(void *arg, int pending) /* Remove association (CNTLID). */ np = ctrlr->np; - sx_xlock(&np->lock); + mtx_lock(&np->lock); TAILQ_REMOVE(&np->controllers, ctrlr, link); - free_unr(np->ids, ctrlr->cntlid); wakeup_np = (!np->online && TAILQ_EMPTY(&np->controllers)); - sx_xunlock(&np->lock); + mtx_unlock(&np->lock); + free_unr(np->ids, ctrlr->cntlid); if (wakeup_np) wakeup(np); @@ -770,6 +787,7 @@ handle_set_features(struct nvmft_controller *ctrlr, ctrlr->aer_mask = aer_mask; mtx_unlock(&ctrlr->lock); nvmft_send_success(ctrlr->admin, nc); + nvmf_free_capsule(nc); return; } default: @@ -944,7 +962,7 @@ nvmft_handle_admin_command(struct nvmft_controller *ctrlr, if (NVMEV(NVME_CC_REG_EN, ctrlr->cc) == 0 && cmd->opc != NVME_OPC_FABRICS_COMMANDS) { nvmft_printf(ctrlr, - "Unsupported admin opcode %#x whiled disabled\n", cmd->opc); + "Unsupported admin opcode %#x while disabled\n", cmd->opc); nvmft_send_generic_error(ctrlr->admin, nc, NVME_SC_COMMAND_SEQUENCE_ERROR); nvmf_free_capsule(nc); diff --git a/sys/dev/nvmf/controller/nvmft_qpair.c b/sys/dev/nvmf/controller/nvmft_qpair.c index 6cb3ebd76884..73c7bb280780 100644 --- a/sys/dev/nvmf/controller/nvmft_qpair.c +++ b/sys/dev/nvmf/controller/nvmft_qpair.c @@ -31,9 +31,11 @@ struct nvmft_qpair { uint16_t qid; u_int qsize; uint16_t sqhd; - uint16_t sqtail; volatile u_int qp_refs; /* Internal references on 'qp'. */ + struct task datamove_task; + STAILQ_HEAD(, ctl_io_hdr) datamove_queue; + struct mtx lock; char name[16]; @@ -41,6 +43,7 @@ struct nvmft_qpair { static int _nvmft_send_generic_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, uint8_t sc_status); +static void nvmft_datamove_task(void *context, int pending); static void nvmft_qpair_error(void *arg, int error) @@ -98,24 +101,24 @@ nvmft_receive_capsule(void *arg, struct nvmf_capsule *nc) } struct nvmft_qpair * -nvmft_qpair_init(enum nvmf_trtype trtype, - const struct nvmf_handoff_qpair_params *handoff, uint16_t qid, +nvmft_qpair_init(enum nvmf_trtype trtype, const nvlist_t *params, uint16_t qid, const char *name) { struct nvmft_qpair *qp; qp = malloc(sizeof(*qp), M_NVMFT, M_WAITOK | M_ZERO); - qp->admin = handoff->admin; - qp->sq_flow_control = handoff->sq_flow_control; - qp->qsize = handoff->qsize; + qp->admin = nvlist_get_bool(params, "admin"); + qp->sq_flow_control = nvlist_get_bool(params, "sq_flow_control"); + qp->qsize = nvlist_get_number(params, "qsize"); qp->qid = qid; - qp->sqhd = handoff->sqhd; - qp->sqtail = handoff->sqtail; + qp->sqhd = nvlist_get_number(params, "sqhd"); strlcpy(qp->name, name, sizeof(qp->name)); mtx_init(&qp->lock, "nvmft qp", NULL, MTX_DEF); qp->cids = BITSET_ALLOC(NUM_CIDS, M_NVMFT, M_WAITOK | M_ZERO); + STAILQ_INIT(&qp->datamove_queue); + TASK_INIT(&qp->datamove_task, 0, nvmft_datamove_task, qp); - qp->qp = nvmf_allocate_qpair(trtype, true, handoff, nvmft_qpair_error, + qp->qp = nvmf_allocate_qpair(trtype, true, params, nvmft_qpair_error, qp, nvmft_receive_capsule, qp); if (qp->qp == NULL) { mtx_destroy(&qp->lock); @@ -131,14 +134,25 @@ nvmft_qpair_init(enum nvmf_trtype trtype, void nvmft_qpair_shutdown(struct nvmft_qpair *qp) { + STAILQ_HEAD(, ctl_io_hdr) datamove_queue; struct nvmf_qpair *nq; + union ctl_io *io; + STAILQ_INIT(&datamove_queue); mtx_lock(&qp->lock); nq = qp->qp; qp->qp = NULL; + STAILQ_CONCAT(&datamove_queue, &qp->datamove_queue); mtx_unlock(&qp->lock); if (nq != NULL && refcount_release(&qp->qp_refs)) nvmf_free_qpair(nq); + + while (!STAILQ_EMPTY(&datamove_queue)) { + io = (union ctl_io *)STAILQ_FIRST(&datamove_queue); + STAILQ_REMOVE_HEAD(&datamove_queue, links); + nvmft_abort_datamove(io); + } + nvmft_drain_task(&qp->datamove_task); } void @@ -359,3 +373,43 @@ nvmft_finish_accept(struct nvmft_qpair *qp, rsp.status_code_specific.success.cntlid = htole16(ctrlr->cntlid); return (nvmft_send_connect_response(qp, &rsp)); } + +void +nvmft_qpair_datamove(struct nvmft_qpair *qp, union ctl_io *io) +{ + bool enqueue_task; + + mtx_lock(&qp->lock); + if (qp->qp == NULL) { + mtx_unlock(&qp->lock); + nvmft_abort_datamove(io); + return; + } + enqueue_task = STAILQ_EMPTY(&qp->datamove_queue); + STAILQ_INSERT_TAIL(&qp->datamove_queue, &io->io_hdr, links); + mtx_unlock(&qp->lock); + if (enqueue_task) + nvmft_enqueue_task(&qp->datamove_task); +} + +static void +nvmft_datamove_task(void *context, int pending __unused) +{ + struct nvmft_qpair *qp = context; + union ctl_io *io; + bool abort; + + mtx_lock(&qp->lock); + while (!STAILQ_EMPTY(&qp->datamove_queue)) { + io = (union ctl_io *)STAILQ_FIRST(&qp->datamove_queue); + STAILQ_REMOVE_HEAD(&qp->datamove_queue, links); + abort = (qp->qp == NULL); + mtx_unlock(&qp->lock); + if (abort) + nvmft_abort_datamove(io); + else + nvmft_handle_datamove(io); + mtx_lock(&qp->lock); + } + mtx_unlock(&qp->lock); +} diff --git a/sys/dev/nvmf/controller/nvmft_var.h b/sys/dev/nvmf/controller/nvmft_var.h index fc1f86754382..85032b2dc55f 100644 --- a/sys/dev/nvmf/controller/nvmft_var.h +++ b/sys/dev/nvmf/controller/nvmft_var.h @@ -9,6 +9,7 @@ #define __NVMFT_VAR_H__ #include <sys/_callout.h> +#include <sys/_nv.h> #include <sys/refcount.h> #include <sys/taskqueue.h> @@ -32,9 +33,10 @@ struct nvmft_port { struct nvme_firmware_page fp; uint64_t cap; uint32_t max_io_qsize; + uint16_t portid; bool online; - struct sx lock; + struct mtx lock; struct unrhdr *ids; TAILQ_HEAD(, nvmft_controller) controllers; @@ -110,6 +112,10 @@ void nvmft_populate_active_nslist(struct nvmft_port *np, uint32_t nsid, void nvmft_dispatch_command(struct nvmft_qpair *qp, struct nvmf_capsule *nc, bool admin); void nvmft_terminate_commands(struct nvmft_controller *ctrlr); +void nvmft_abort_datamove(union ctl_io *io); +void nvmft_handle_datamove(union ctl_io *io); +void nvmft_drain_task(struct task *task); +void nvmft_enqueue_task(struct task *task); /* nvmft_controller.c */ void nvmft_controller_error(struct nvmft_controller *ctrlr, @@ -121,23 +127,22 @@ void nvmft_handle_admin_command(struct nvmft_controller *ctrlr, void nvmft_handle_io_command(struct nvmft_qpair *qp, uint16_t qid, struct nvmf_capsule *nc); int nvmft_handoff_admin_queue(struct nvmft_port *np, - const struct nvmf_handoff_controller_qpair *handoff, + enum nvmf_trtype trtype, const nvlist_t *params, const struct nvmf_fabric_connect_cmd *cmd, const struct nvmf_fabric_connect_data *data); -int nvmft_handoff_io_queue(struct nvmft_port *np, - const struct nvmf_handoff_controller_qpair *handoff, - const struct nvmf_fabric_connect_cmd *cmd, +int nvmft_handoff_io_queue(struct nvmft_port *np, enum nvmf_trtype trtype, + const nvlist_t *params, const struct nvmf_fabric_connect_cmd *cmd, const struct nvmf_fabric_connect_data *data); int nvmft_printf(struct nvmft_controller *ctrlr, const char *fmt, ...) __printflike(2, 3); /* nvmft_qpair.c */ struct nvmft_qpair *nvmft_qpair_init(enum nvmf_trtype trtype, - const struct nvmf_handoff_qpair_params *handoff, uint16_t qid, - const char *name); + const nvlist_t *params, uint16_t qid, const char *name); void nvmft_qpair_shutdown(struct nvmft_qpair *qp); void nvmft_qpair_destroy(struct nvmft_qpair *qp); struct nvmft_controller *nvmft_qpair_ctrlr(struct nvmft_qpair *qp); +void nvmft_qpair_datamove(struct nvmft_qpair *qp, union ctl_io *io); uint16_t nvmft_qpair_id(struct nvmft_qpair *qp); const char *nvmft_qpair_name(struct nvmft_qpair *qp); void nvmft_command_completed(struct nvmft_qpair *qp, |