From 1058c12197aba80d0777e3484f350436fca55fd7 Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Thu, 2 May 2024 16:30:10 -0700 Subject: nvmecontrol: New commands to support Fabrics hosts - discover: Connects to a remote Discovery controller, fetches its Discovery Log Page, and enumerates the remote controllers described in the log page. The -v option can be used to display the Identify Controller data structure for the Discovery controller. This is only really useful for debugging. - connect: Connects to a remote I/O controller and establishes an association of an admin queue and a single I/O queue. The association is handed off to the in-kernel host to create a new nvmeX device. - connect-all: Connects to a Discovery controller and attempts to create an association with each I/O controller enumerated in the Discovery controller's Discovery Log Page. - reconnect: Establishes a new association with a remote I/O controller for an existing nvmeX device. This can be used to restore access to a remote I/O controller after the loss of a prior association due to a transport error, controller reboot, etc. - disconnect: Deletes one or more nvmeX devices after detaching its namespaces and terminating any active associations. The devices to delete can be identified by either a nvmeX device name or the NQN of the remote controller. - disconnect-all: Deletes all active associations with remote controllers. Reviewed by: imp Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D44715 --- sbin/nvmecontrol/connect.c | 283 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 sbin/nvmecontrol/connect.c (limited to 'sbin/nvmecontrol/connect.c') diff --git a/sbin/nvmecontrol/connect.c b/sbin/nvmecontrol/connect.c new file mode 100644 index 000000000000..afb78725a3c7 --- /dev/null +++ b/sbin/nvmecontrol/connect.c @@ -0,0 +1,283 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2023-2024 Chelsio Communications, Inc. + * Written by: John Baldwin + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "comnd.h" +#include "fabrics.h" + +/* + * Settings that are currently hardcoded but could be exposed to the + * user via additional command line options: + * + * - ADMIN queue entries + * - MaxR2T + */ + +static struct options { + const char *transport; + const char *address; + const char *cntlid; + const char *subnqn; + const char *hostnqn; + uint32_t kato; + uint16_t num_io_queues; + uint16_t queue_size; + bool data_digests; + bool flow_control; + bool header_digests; +} opt = { + .transport = "tcp", + .address = NULL, + .cntlid = "dynamic", + .subnqn = NULL, + .hostnqn = NULL, + .kato = NVMF_KATO_DEFAULT / 1000, + .num_io_queues = 1, + .queue_size = 0, + .data_digests = false, + .flow_control = false, + .header_digests = false, +}; + +static void +tcp_association_params(struct nvmf_association_params *params) +{ + params->tcp.pda = 0; + params->tcp.header_digests = opt.header_digests; + params->tcp.data_digests = opt.data_digests; + /* XXX */ + params->tcp.maxr2t = 1; +} + +static int +connect_nvm_controller(enum nvmf_trtype trtype, int adrfam, const char *address, + const char *port, uint16_t cntlid, const char *subnqn) +{ + struct nvme_controller_data cdata; + struct nvmf_association_params aparams; + struct nvmf_qpair *admin, **io; + int error; + + memset(&aparams, 0, sizeof(aparams)); + aparams.sq_flow_control = opt.flow_control; + switch (trtype) { + case NVMF_TRTYPE_TCP: + tcp_association_params(&aparams); + break; + default: + warnx("Unsupported transport %s", nvmf_transport_type(trtype)); + return (EX_UNAVAILABLE); + } + + io = calloc(opt.num_io_queues, sizeof(*io)); + error = connect_nvm_queues(&aparams, trtype, adrfam, address, port, + cntlid, subnqn, opt.hostnqn, opt.kato, &admin, io, + opt.num_io_queues, opt.queue_size, &cdata); + if (error != 0) + return (error); + + error = nvmf_handoff_host(admin, opt.num_io_queues, io, &cdata); + if (error != 0) { + warnc(error, "Failed to handoff queues to kernel"); + return (EX_IOERR); + } + free(io); + return (0); +} + +static void +connect_discovery_entry(struct nvme_discovery_log_entry *entry) +{ + int adrfam; + + switch (entry->trtype) { + case NVMF_TRTYPE_TCP: + switch (entry->adrfam) { + case NVMF_ADRFAM_IPV4: + adrfam = AF_INET; + break; + case NVMF_ADRFAM_IPV6: + adrfam = AF_INET6; + break; + default: + warnx("Skipping unsupported address family for %s", + entry->subnqn); + return; + } + switch (entry->tsas.tcp.sectype) { + case NVME_TCP_SECURITY_NONE: + break; + default: + warnx("Skipping unsupported TCP security type for %s", + entry->subnqn); + return; + } + break; + default: + warnx("Skipping unsupported transport %s for %s", + nvmf_transport_type(entry->trtype), entry->subnqn); + return; + } + + /* + * XXX: Track portids and avoid duplicate connections for a + * given (subnqn,portid)? + */ + + /* XXX: Should this make use of entry->aqsz in some way? */ + connect_nvm_controller(entry->trtype, adrfam, entry->traddr, + entry->trsvcid, entry->cntlid, entry->subnqn); +} + +static void +connect_discovery_log_page(struct nvmf_qpair *qp) +{ + struct nvme_discovery_log *log; + int error; + + error = nvmf_host_fetch_discovery_log_page(qp, &log); + if (error != 0) + errc(EX_IOERR, error, "Failed to fetch discovery log page"); + + for (u_int i = 0; i < log->numrec; i++) + connect_discovery_entry(&log->entries[i]); + free(log); +} + +static void +discover_controllers(enum nvmf_trtype trtype, const char *address, + const char *port) +{ + struct nvmf_qpair *qp; + + qp = connect_discovery_adminq(trtype, address, port, opt.hostnqn); + + connect_discovery_log_page(qp); + + nvmf_free_qpair(qp); +} + +static void +connect_fn(const struct cmd *f, int argc, char *argv[]) +{ + enum nvmf_trtype trtype; + const char *address, *port; + char *tofree; + u_long cntlid; + int error; + + if (arg_parse(argc, argv, f)) + return; + + if (opt.num_io_queues <= 0) + errx(EX_USAGE, "Invalid number of I/O queues"); + + if (strcasecmp(opt.transport, "tcp") == 0) { + trtype = NVMF_TRTYPE_TCP; + } else + errx(EX_USAGE, "Unsupported or invalid transport"); + + nvmf_parse_address(opt.address, &address, &port, &tofree); + if (port == NULL) + errx(EX_USAGE, "Explicit port required"); + + cntlid = nvmf_parse_cntlid(opt.cntlid); + + error = connect_nvm_controller(trtype, AF_UNSPEC, address, port, cntlid, + opt.subnqn); + if (error != 0) + exit(error); + + free(tofree); +} + +static void +connect_all_fn(const struct cmd *f, int argc, char *argv[]) +{ + enum nvmf_trtype trtype; + const char *address, *port; + char *tofree; + + if (arg_parse(argc, argv, f)) + return; + + if (opt.num_io_queues <= 0) + errx(EX_USAGE, "Invalid number of I/O queues"); + + if (strcasecmp(opt.transport, "tcp") == 0) { + trtype = NVMF_TRTYPE_TCP; + } else + errx(EX_USAGE, "Unsupported or invalid transport"); + + nvmf_parse_address(opt.address, &address, &port, &tofree); + discover_controllers(trtype, address, port); + + free(tofree); +} + +static const struct opts connect_opts[] = { +#define OPT(l, s, t, opt, addr, desc) { l, s, t, &opt.addr, desc } + OPT("transport", 't', arg_string, opt, transport, + "Transport type"), + OPT("cntlid", 'c', arg_string, opt, cntlid, + "Controller ID"), + OPT("nr-io-queues", 'i', arg_uint16, opt, num_io_queues, + "Number of I/O queues"), + OPT("queue-size", 'Q', arg_uint16, opt, queue_size, + "Number of entries in each I/O queue"), + OPT("keep-alive-tmo", 'k', arg_uint32, opt, kato, + "Keep Alive timeout (in seconds)"), + OPT("hostnqn", 'q', arg_string, opt, hostnqn, + "Host NQN"), + OPT("flow_control", 'F', arg_none, opt, flow_control, + "Request SQ flow control"), + OPT("hdr_digests", 'g', arg_none, opt, header_digests, + "Enable TCP PDU header digests"), + OPT("data_digests", 'G', arg_none, opt, data_digests, + "Enable TCP PDU data digests"), + { NULL, 0, arg_none, NULL, NULL } +}; +#undef OPT + +static const struct args connect_args[] = { + { arg_string, &opt.address, "address" }, + { arg_string, &opt.subnqn, "SubNQN" }, + { arg_none, NULL, NULL }, +}; + +static const struct args connect_all_args[] = { + { arg_string, &opt.address, "address" }, + { arg_none, NULL, NULL }, +}; + +static struct cmd connect_cmd = { + .name = "connect", + .fn = connect_fn, + .descr = "Connect to a fabrics controller", + .ctx_size = sizeof(opt), + .opts = connect_opts, + .args = connect_args, +}; + +static struct cmd connect_all_cmd = { + .name = "connect-all", + .fn = connect_all_fn, + .descr = "Discover and connect to fabrics controllers", + .ctx_size = sizeof(opt), + .opts = connect_opts, + .args = connect_all_args, +}; + +CMD_COMMAND(connect_cmd); +CMD_COMMAND(connect_all_cmd); -- cgit v1.2.3