diff options
Diffstat (limited to 'lib/geom/multipath')
| -rw-r--r-- | lib/geom/multipath/Makefile | 7 | ||||
| -rw-r--r-- | lib/geom/multipath/Makefile.depend | 16 | ||||
| -rw-r--r-- | lib/geom/multipath/geom_multipath.c | 323 | ||||
| -rw-r--r-- | lib/geom/multipath/gmultipath.8 | 375 |
4 files changed, 721 insertions, 0 deletions
diff --git a/lib/geom/multipath/Makefile b/lib/geom/multipath/Makefile new file mode 100644 index 000000000000..1c7341145017 --- /dev/null +++ b/lib/geom/multipath/Makefile @@ -0,0 +1,7 @@ +PACKAGE=geom + +GEOM_CLASS= multipath + +CFLAGS+= -I${SRCTOP}/sys + +.include <bsd.lib.mk> diff --git a/lib/geom/multipath/Makefile.depend b/lib/geom/multipath/Makefile.depend new file mode 100644 index 000000000000..0dd05cace3c0 --- /dev/null +++ b/lib/geom/multipath/Makefile.depend @@ -0,0 +1,16 @@ +# Autogenerated - do NOT edit! + +DIRDEPS = \ + include \ + include/xlocale \ + lib/${CSU_DIR} \ + lib/libc \ + lib/libcompiler_rt \ + lib/libgeom \ + + +.include <dirdeps.mk> + +.if ${DEP_RELDIR} == ${_DEP_RELDIR} +# local dependencies - needed for -jN in clean tree +.endif diff --git a/lib/geom/multipath/geom_multipath.c b/lib/geom/multipath/geom_multipath.c new file mode 100644 index 000000000000..e4cb123594c3 --- /dev/null +++ b/lib/geom/multipath/geom_multipath.c @@ -0,0 +1,323 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2006 Mathew Jacob <mjacob@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <errno.h> +#include <paths.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <strings.h> +#include <assert.h> +#include <libgeom.h> +#include <unistd.h> +#include <uuid.h> +#include <geom/multipath/g_multipath.h> + +#include "core/geom.h" +#include "misc/subr.h" + +uint32_t lib_version = G_LIB_VERSION; +uint32_t version = G_MULTIPATH_VERSION; + +static void mp_main(struct gctl_req *, unsigned int); +static void mp_label(struct gctl_req *); +static void mp_clear(struct gctl_req *); +static void mp_prefer(struct gctl_req *); + +struct g_command class_commands[] = { + { + "create", G_FLAG_VERBOSE | G_FLAG_LOADKLD, NULL, + { + { 'A', "active_active", NULL, G_TYPE_BOOL }, + { 'R', "active_read", NULL, G_TYPE_BOOL }, + G_OPT_SENTINEL + }, + "[-vAR] name prov ..." + }, + { + "label", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main, + { + { 'A', "active_active", NULL, G_TYPE_BOOL }, + { 'R', "active_read", NULL, G_TYPE_BOOL }, + G_OPT_SENTINEL + }, + "[-vAR] name prov ..." + }, + { "configure", G_FLAG_VERBOSE, NULL, + { + { 'A', "active_active", NULL, G_TYPE_BOOL }, + { 'P', "active_passive", NULL, G_TYPE_BOOL }, + { 'R', "active_read", NULL, G_TYPE_BOOL }, + G_OPT_SENTINEL + }, + "[-vAPR] name" + }, + { + "add", G_FLAG_VERBOSE, NULL, G_NULL_OPTS, + "[-v] name prov" + }, + { + "remove", G_FLAG_VERBOSE, NULL, G_NULL_OPTS, + "[-v] name prov" + }, + { + "prefer", G_FLAG_VERBOSE, mp_main, G_NULL_OPTS, + "[-v] prov ..." + }, + { + "fail", G_FLAG_VERBOSE, NULL, G_NULL_OPTS, + "[-v] name prov" + }, + { + "restore", G_FLAG_VERBOSE, NULL, G_NULL_OPTS, + "[-v] name prov" + }, + { + "rotate", G_FLAG_VERBOSE, NULL, G_NULL_OPTS, + "[-v] name" + }, + { + "getactive", G_FLAG_VERBOSE, NULL, G_NULL_OPTS, + "[-v] name" + }, + { + "destroy", G_FLAG_VERBOSE, NULL, G_NULL_OPTS, + "[-v] name" + }, + { + "stop", G_FLAG_VERBOSE, NULL, G_NULL_OPTS, + "[-v] name" + }, + { + "clear", G_FLAG_VERBOSE, mp_main, G_NULL_OPTS, + "[-v] prov ..." + }, + G_CMD_SENTINEL +}; + +static void +mp_main(struct gctl_req *req, unsigned int flags __unused) +{ + const char *name; + + name = gctl_get_ascii(req, "verb"); + if (name == NULL) { + gctl_error(req, "No '%s' argument.", "verb"); + return; + } + if (strcmp(name, "label") == 0) { + mp_label(req); + } else if (strcmp(name, "clear") == 0) { + mp_clear(req); + } else if (strcmp(name, "prefer") == 0) { + mp_prefer(req); + } else { + gctl_error(req, "Unknown command: %s.", name); + } +} + +static void +mp_label(struct gctl_req *req) +{ + struct g_multipath_metadata md; + off_t disksize = 0, msize; + uint8_t *sector, *rsector; + char *ptr; + uuid_t uuid; + ssize_t secsize = 0, ssize; + uint32_t status; + const char *name, *name2, *mpname; + int error, i, nargs, fd; + + nargs = gctl_get_int(req, "nargs"); + if (nargs < 2) { + gctl_error(req, "wrong number of arguments."); + return; + } + + /* + * First, check each provider to make sure it's the same size. + * This also gets us our size and sectorsize for the metadata. + */ + for (i = 1; i < nargs; i++) { + name = gctl_get_ascii(req, "arg%d", i); + msize = g_get_mediasize(name); + ssize = g_get_sectorsize(name); + if (msize == 0 || ssize == 0) { + gctl_error(req, "cannot get information about %s: %s.", + name, strerror(errno)); + return; + } + if (i == 1) { + secsize = ssize; + disksize = msize; + } else { + if (secsize != ssize) { + gctl_error(req, "%s sector size %ju different.", + name, (intmax_t)ssize); + return; + } + if (disksize != msize) { + gctl_error(req, "%s media size %ju different.", + name, (intmax_t)msize); + return; + } + } + + } + + /* + * Generate metadata. + */ + strlcpy(md.md_magic, G_MULTIPATH_MAGIC, sizeof(md.md_magic)); + md.md_version = G_MULTIPATH_VERSION; + mpname = gctl_get_ascii(req, "arg0"); + strlcpy(md.md_name, mpname, sizeof(md.md_name)); + md.md_size = disksize; + md.md_sectorsize = secsize; + uuid_create(&uuid, &status); + if (status != uuid_s_ok) { + gctl_error(req, "cannot create a UUID."); + return; + } + uuid_to_string(&uuid, &ptr, &status); + if (status != uuid_s_ok) { + gctl_error(req, "cannot stringify a UUID."); + return; + } + strlcpy(md.md_uuid, ptr, sizeof (md.md_uuid)); + md.md_active_active = gctl_get_int(req, "active_active"); + if (gctl_get_int(req, "active_read")) + md.md_active_active = 2; + free(ptr); + + /* + * Allocate a sector to write as metadata. + */ + sector = calloc(1, secsize); + if (sector == NULL) { + gctl_error(req, "unable to allocate metadata buffer"); + return; + } + rsector = malloc(secsize); + if (rsector == NULL) { + gctl_error(req, "unable to allocate metadata buffer"); + goto done; + } + + /* + * encode the metadata + */ + multipath_metadata_encode(&md, sector); + + /* + * Store metadata on the initial provider. + */ + name = gctl_get_ascii(req, "arg1"); + error = g_metadata_store(name, sector, secsize); + if (error != 0) { + gctl_error(req, "cannot store metadata on %s: %s.", name, strerror(error)); + goto done; + } + + /* + * Now touch the rest of the providers to hint retaste. + */ + for (i = 2; i < nargs; i++) { + name2 = gctl_get_ascii(req, "arg%d", i); + fd = g_open(name2, 1); + if (fd < 0) { + fprintf(stderr, "Unable to open %s: %s.\n", + name2, strerror(errno)); + continue; + } + if (pread(fd, rsector, secsize, disksize - secsize) != + (ssize_t)secsize) { + fprintf(stderr, "Unable to read metadata from %s: %s.\n", + name2, strerror(errno)); + g_close(fd); + continue; + } + g_close(fd); + if (memcmp(sector, rsector, secsize)) { + fprintf(stderr, "No metadata found on %s." + " It is not a path of %s.\n", + name2, name); + } + } +done: + free(rsector); + free(sector); +} + + +static void +mp_clear(struct gctl_req *req) +{ + const char *name; + int error, i, nargs; + + nargs = gctl_get_int(req, "nargs"); + if (nargs < 1) { + gctl_error(req, "Too few arguments."); + return; + } + + for (i = 0; i < nargs; i++) { + name = gctl_get_ascii(req, "arg%d", i); + error = g_metadata_clear(name, G_MULTIPATH_MAGIC); + if (error != 0) { + fprintf(stderr, "Can't clear metadata on %s: %s.\n", + name, strerror(error)); + gctl_error(req, "Not fully done."); + continue; + } + } +} + +static void +mp_prefer(struct gctl_req *req) +{ + const char *name, *comp, *errstr; + int nargs; + + nargs = gctl_get_int(req, "nargs"); + if (nargs != 2) { + gctl_error(req, "Usage: prefer GEOM PROVIDER"); + return; + } + name = gctl_get_ascii(req, "arg0"); + comp = gctl_get_ascii(req, "arg1"); + errstr = gctl_issue (req); + if (errstr != NULL) { + fprintf(stderr, "Can't set %s preferred provider to %s: %s.\n", + name, comp, errstr); + } +} diff --git a/lib/geom/multipath/gmultipath.8 b/lib/geom/multipath/gmultipath.8 new file mode 100644 index 000000000000..0f007196f7d4 --- /dev/null +++ b/lib/geom/multipath/gmultipath.8 @@ -0,0 +1,375 @@ +.\" Copyright (c) 2007 Matthew Jacob +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.Dd March 17, 2022 +.Dt GMULTIPATH 8 +.Os +.Sh NAME +.Nm gmultipath +.Nd "disk multipath control utility" +.Sh SYNOPSIS +.Nm +.Cm create +.Op Fl ARv +.Ar name +.Ar prov ... +.Nm +.Cm label +.Op Fl ARv +.Ar name +.Ar prov ... +.Nm +.Cm configure +.Op Fl APRv +.Ar name +.Nm +.Cm add +.Op Fl v +.Ar name prov +.Nm +.Cm remove +.Op Fl v +.Ar name prov +.Nm +.Cm fail +.Op Fl v +.Ar name prov +.Nm +.Cm restore +.Op Fl v +.Ar name prov +.Nm +.Cm rotate +.Op Fl v +.Ar name +.Nm +.Cm prefer +.Op Fl v +.Ar name +.Ar prov +.Nm +.Cm getactive +.Op Fl v +.Ar name +.Nm +.Cm destroy +.Op Fl v +.Ar name +.Nm +.Cm stop +.Op Fl v +.Ar name +.Nm +.Cm clear +.Op Fl v +.Ar prov ... +.Nm +.Cm list +.Nm +.Cm status +.Nm +.Cm load +.Nm +.Cm unload +.Sh DESCRIPTION +The +.Nm +utility is used for device multipath configuration. +.Pp +The multipath device can be configured using two different methods: +.Dq manual +or +.Dq automatic . +When using the +.Dq manual +method, no metadata are stored on the devices, so the multipath +device has to be configured by hand every time it is needed. +Additional device paths also will not be detected automatically. +The +.Dq automatic +method uses on-disk metadata to detect device and all its paths. +Metadata use the last sector of the underlying disk device and +include device name and UUID. +The UUID guarantees uniqueness in a shared storage environment +but is in general too cumbersome to use. +The name is what is exported via the device interface. +.Pp +The first argument to +.Nm +indicates an action to be performed: +.Bl -tag -width ".Cm destroy" +.It Cm create +Create multipath device with +.Dq manual +method without writing any on-disk metadata. +It is up to administrator, how to properly identify device paths. +Kernel will only check that all given providers have same media and +sector sizes. +.Pp +.Fl A +option enables Active/Active mode, +.Fl R +option enables Active/Read mode, otherwise Active/Passive mode is used +by default. +.It Cm label +Create multipath device with +.Dq automatic +method. +Label the first given provider with on-disk metadata using the specified +.Ar name . +The rest of given providers will be retasted to detect these metadata. +It reliably protects against specifying unrelated providers. +Providers with no matching metadata detected will not be added to the device. +.Pp +.Fl A +option enables Active/Active mode, +.Fl R +option enables Active/Read mode, otherwise Active/Passive mode is used +by default. +.It Cm configure +Configure the given multipath device. +.Pp +.Fl A +option enables Active/Active mode, +.Fl P +option enables Active/Passive mode, +.Fl R +option enables Active/Read mode. +.It Cm add +Add the given provider as a path to the given multipath device. +Should normally be used only for devices created with +.Dq manual +method, unless you know what you are doing (you are sure that it is another +device path, but tasting its metadata in regular +.Dq automatic +way is not possible). +.It Cm remove +Remove the given provider as a path from the given multipath device. +If the last path removed, the multipath device will be destroyed. +.It Cm fail +Mark specified provider as a path of the specified multipath device as failed. +If there are other paths present, new requests will be forwarded there. +.It Cm restore +Mark specified provider as a path of the specified multipath device as +operational, allowing it to handle requests. +.It Cm rotate +Change the active provider/path to the next available provider in Active/Passive mode. +.It Cm prefer +Change the active provider/path to the specified provider in Active/Passive mode. +.It Cm getactive +Get the currently active provider(s)/path(s). +.It Cm destroy +Destroy the given multipath device clearing metadata. +.It Cm stop +Stop the given multipath device without clearing metadata. +.It Cm clear +Clear metadata on the given provider. +.It Cm list +See +.Xr geom 8 . +.It Cm status +See +.Xr geom 8 . +.It Cm load +See +.Xr geom 8 . +.It Cm unload +See +.Xr geom 8 . +.El +.Sh SYSCTL VARIABLES +The following +.Xr sysctl 8 +variable can be used to control the behavior of the +.Nm MULTIPATH +GEOM class. +.Bl -tag -width indent +.It Va kern.geom.multipath.debug : No 0 +Debug level of the +.Nm MULTIPATH +GEOM class. +This can be set to 0 (default) or 1 to disable or enable various +forms of chattiness. +.It Va kern.geom.multipath.exclusive : No 1 +Open underlying providers exclusively, preventing individual paths access. +.El +.Sh EXIT STATUS +Exit status is 0 on success, and 1 if the command fails. +.Sh MULTIPATH ARCHITECTURE +This is a multiple path architecture with no device knowledge or +presumptions other than size matching built in. +Therefore the user must exercise some care +in selecting providers that do indeed represent multiple paths to the +same underlying disk device. +The reason for this is that there are several +criteria across multiple underlying transport types that can +.Ar indicate +identity, but in all respects such identity can rarely be considered +.Ar definitive . +.Pp +For example, if you use the World Wide Port Name of a Fibre Channel +disk object you might believe that two disks that have the same WWPN +on different paths (or even disjoint fabrics) might be considered +the same disk. +Nearly always this would be a safe assumption, until +you realize that a WWPN, like an Ethernet MAC address, is a soft +programmable entity, and that a misconfigured Director Class switch +could lead you to believe incorrectly that you have found multiple +paths to the same device. +This is an extreme and theoretical case, but +it is possible enough to indicate that the policy for deciding which +of multiple pathnames refer to the same device should be left to the +system operator who will use tools and knowledge of their own storage +subsystem to make the correct configuration selection. +.Pp +There are Active/Passive, Active/Read and Active/Active operation modes +supported. +In Active/Passive mode only one path has I/O moving on it +at any point in time. +This I/O continues until an I/O is returned with +a generic I/O error or a "Nonexistent Device" error. +When this occurs, that path is marked FAIL, the next path +in a list is selected as active and the failed I/O reissued. +In Active/Active mode all paths not marked FAIL may handle I/O at the same time. +Requests are distributed between paths to equalize load. +For capable devices it allows the utilisation of the bandwidth available on all paths. +In Active/Read mode all paths not marked FAIL may handle reads at the same time, +but unlike in Active/Active mode only one path handles write requests at any +point in time; closely following the original write request order if the layer +above needs it for data consistency (not waiting for requisite write completion +before sending dependent write). +.Pp +When new devices are added to the system the +.Nm MULTIPATH +GEOM class is given an opportunity to taste these new devices. +If a new +device has a +.Nm MULTIPATH +on-disk metadata label, the device is either used to create a new +.Nm MULTIPATH +GEOM, or added to the list of paths for an existing +.Nm MULTIPATH +GEOM. +.Pp +It is this mechanism that works reasonably with +.Xr isp 4 +and +.Xr mpt 4 +based Fibre Channel disk devices. +For these devices, when a device disappears +(due to e.g., a cable pull or power failure to a switch), the device is +proactively marked as gone and I/O to it failed. +This causes the +.Nm MULTIPATH +failure event just described. +.Pp +When Fibre Channel events inform either +.Xr isp 4 +or +.Xr mpt 4 +host bus adapters that new devices may have arrived (e.g., the arrival +of an RSCN event from the Fabric Domain Controller), they can cause +a rescan to occur and cause the attachment and configuration of any +(now) new devices to occur, causing the taste event described above. +.Pp +This means that this multipath architecture is not a one-shot path +failover, but can be considered to be steady state as long as failed +paths are repaired (automatically or otherwise). +.Pp +Automatic rescanning is not a requirement. +Nor is Fibre Channel. +The +same failover mechanisms work equally well for traditional "Parallel" +SCSI but may require manual intervention with +.Xr camcontrol 8 +to cause the reattachment of repaired device links. +.Sh EXAMPLES +The following example shows how to use +.Xr camcontrol 8 +to find possible multiple path devices and to create a +.Nm MULTIPATH +GEOM class for them. +.Bd -literal -offset indent +mysys# camcontrol devlist +<ECNCTX @WESTVILLE > at scbus0 target 0 lun 0 (da0,pass0) +<ECNCTX @WESTVILLE > at scbus0 target 0 lun 1 (da1,pass1) +<ECNCTX @WESTVILLE > at scbus1 target 0 lun 0 (da2,pass2) +<ECNCTX @WESTVILLE > at scbus1 target 0 lun 1 (da3,pass3) +mysys# camcontrol inquiry da0 -S +ECNTX0LUN000000SER10ac0d01 +mysys# camcontrol inquiry da2 -S +ECNTX0LUN000000SER10ac0d01 +.Ed +.Pp +Now that you have used the Serial Number to compare two disk paths +it is not entirely unreasonable to conclude that these are multiple +paths to the same device. +However, only the user who is familiar +with their storage is qualified to make this judgement. +.Pp +You can then use the +.Nm +command to label and create a +.Nm MULTIPATH +GEOM provider named +.Ar FRED . +.Bd -literal -offset indent +gmultipath label -v FRED /dev/da0 /dev/da2 +disklabel -Bw /dev/multipath/FRED auto +newfs /dev/multipath/FREDa +mount /dev/multipath/FREDa /mnt.... +.Ed +.Pp +The resultant console output looks something like: +.Bd -literal -offset indent +GEOM_MULTIPATH: da0 added to FRED +GEOM_MULTIPATH: da0 is now active path in FRED +GEOM_MULTIPATH: da2 added to FRED +.Ed +.Pp +To load the +.Nm +module at boot time, add this entry to +.Pa /boot/loader.conf : +.Bd -literal -offset ident +geom_multipath_load="YES" +.Ed +.Sh SEE ALSO +.Xr geom 4 , +.Xr isp 4 , +.Xr mpt 4 , +.Xr loader.conf 5 , +.Xr camcontrol 8 , +.Xr geom 8 , +.Xr mount 8 , +.Xr newfs 8 , +.Xr sysctl 8 +.Sh HISTORY +The +.Nm +utility first appeared in +.Fx 7.0 +.Sh AUTHORS +.An Matthew Jacob Aq Mt mjacob@FreeBSD.org +.An Alexander Motin Aq Mt mav@FreeBSD.org |