Major GEOM MULTIPATH class rewrite:

 - Improved locking and destruction process to fix crashes.
 - Improved "automatic" configuration method to make it consistent and safe
by reading metadata back from all specified paths after writing to one.
 - Added provider size check to reduce chance of ordering conflict with
other GEOM classes.
 - Added "manual" configuration method without using on-disk metadata.
 - Added "add" and "remove" commands to allow manage paths manually.
 - Failed paths are no longer dropped from geom, but only marked as FAIL
and excluded from I/O operations.
 - Automatically restore failed paths when all others paths are marked
as failed, for example, because of device-caused (not transport) errors.
 - Added "fail" and "restore" commands to manually control FAIL flag.
 - geom is now destroyed on last path disconnection.
 - Added optional Active/Active mode support. Unlike Active/Passive
mode, load evenly distributed between all working paths. If supported by
the device, it allows to significantly improve performance, utilizing
bandwidth of all paths. It is controlled by -A option during creation.
Disabled by default now.
 - Improved `status` and `list` commands output.

Sponsored by:	iXsystems, inc.
MFC after:	1 month

2 files changed, 194 insertions, 98 deletions

diff --git a/sbin/geom/class/multipath/geom_multipath.c b/sbin/geom/class/multipath/geom_multipath.c
index 6364bc3..ce0028d 100644
--- a/sbin/geom/class/multipath/geom_multipath.c
+++ b/sbin/geom/class/multipath/geom_multipath.c
@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
 #include <strings.h>
 #include <assert.h>
 #include <libgeom.h>
+#include <unistd.h>
 #include <uuid.h>
 #include <geom/multipath/g_multipath.h>
 
@@ -48,31 +49,58 @@ uint32_t version = G_MULTIPATH_VERSION;
 static void mp_main(struct gctl_req *, unsigned int);
 static void mp_label(struct gctl_req *);
 static void mp_clear(struct gctl_req *);
-static void mp_add(struct gctl_req *);
 
 struct g_command class_commands[] = {
 	{
-		"label", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main, G_NULL_OPTS,
-		"[-v] name prov ..."
+		"create", G_FLAG_VERBOSE | G_FLAG_LOADKLD, NULL,
+		{
+			{ 'A', "active_active", NULL, G_TYPE_BOOL },
+			G_OPT_SENTINEL
+		},
+		"[-vA] name prov ..."
 	},
 	{
-		"add", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main, G_NULL_OPTS,
-		"[-v] name prov ..."
+		"label", G_FLAG_VERBOSE | G_FLAG_LOADKLD, mp_main,
+		{
+			{ 'A', "active_active", NULL, G_TYPE_BOOL },
+			G_OPT_SENTINEL
+		},
+		"[-vA] name prov ..."
 	},
 	{
-		"destroy", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
-		"[-v] prov ..."
+		"add", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
 	},
 	{
-		"clear", G_FLAG_VERBOSE, mp_main, G_NULL_OPTS,
-		"[-v] prov ..."
+		"remove", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
+	},
+	{
+		"fail", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
+	},
+	{
+		"restore", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name prov"
 	},
 	{
 		"rotate", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
-		"[-v] prov ..."
+		"[-v] name"
 	},
 	{
 		"getactive", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name"
+	},
+	{
+		"destroy", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name"
+	},
+	{
+		"stop", G_FLAG_VERBOSE, NULL, G_NULL_OPTS,
+		"[-v] name"
+	},
+	{
+		"clear", G_FLAG_VERBOSE, mp_main, G_NULL_OPTS,
 		"[-v] prov ..."
 	},
 	G_CMD_SENTINEL
@@ -90,8 +118,6 @@ mp_main(struct gctl_req *req, unsigned int flags __unused)
 	}
 	if (strcmp(name, "label") == 0) {
 		mp_label(req);
-	} else if (strcmp(name, "add") == 0) {
-		mp_add(req);
 	} else if (strcmp(name, "clear") == 0) {
 		mp_clear(req);
 	} else {
@@ -103,13 +129,13 @@ static void
 mp_label(struct gctl_req *req)
 {
 	struct g_multipath_metadata md;
-	off_t disksiz = 0, msize;
-	uint8_t *sector;
+	off_t disksize = 0, msize;
+	uint8_t *sector, *rsector;
 	char *ptr;
 	uuid_t uuid;
 	uint32_t secsize = 0, ssize, status;
-	const char *name, *mpname;
-	int error, i, nargs;
+	const char *name, *name2, *mpname;
+	int error, i, nargs, fd;
 
 	nargs = gctl_get_int(req, "nargs");
 	if (nargs < 2) {
@@ -132,14 +158,14 @@ mp_label(struct gctl_req *req)
 		}
 		if (i == 1) {
 			secsize = ssize;
-			disksiz	= msize;
+			disksize = msize;
 		} else {
 			if (secsize != ssize) {
 				gctl_error(req, "%s sector size %u different.",
 				    name, ssize);
 				return;
 			}
-			if (disksiz != msize) {
+			if (disksize != msize) {
 				gctl_error(req, "%s media size %ju different.",
 				    name, (intmax_t)msize);
 				return;
@@ -155,7 +181,7 @@ mp_label(struct gctl_req *req)
 	md.md_version = G_MULTIPATH_VERSION;
 	mpname = gctl_get_ascii(req, "arg0");
 	strlcpy(md.md_name, mpname, sizeof(md.md_name));
-	md.md_size = disksiz;
+	md.md_size = disksize;
 	md.md_sectorsize = secsize;
 	uuid_create(&uuid, &status);
 	if (status != uuid_s_ok) {
@@ -168,19 +194,10 @@ mp_label(struct gctl_req *req)
 		return;
 	}
 	strlcpy(md.md_uuid, ptr, sizeof (md.md_uuid));
+	md.md_active_active = gctl_get_int(req, "active_active");
 	free(ptr);
 
 	/*
-	 * Clear metadata on initial provider first.
-	 */
-	name = gctl_get_ascii(req, "arg1");
-	error = g_metadata_clear(name, NULL);
-	if (error != 0) {
-		gctl_error(req, "cannot clear metadata on %s: %s.", name, strerror(error));
-		return;
-	}
-
-	/*
 	 * Allocate a sector to write as metadata.
 	 */
 	sector = malloc(secsize);
@@ -189,6 +206,12 @@ mp_label(struct gctl_req *req)
 		return;
 	}
 	memset(sector, 0, secsize);
+	rsector = malloc(secsize);
+	if (rsector == NULL) {
+		free(sector);
+		gctl_error(req, "unable to allocate metadata buffer");
+		return;
+	}
 
 	/*
 	 * encode the metadata
@@ -198,6 +221,7 @@ mp_label(struct gctl_req *req)
 	/*
 	 * Store metadata on the initial provider.
 	 */
+	name = gctl_get_ascii(req, "arg1");
 	error = g_metadata_store(name, sector, secsize);
 	if (error != 0) {
 		gctl_error(req, "cannot store metadata on %s: %s.", name, strerror(error));
@@ -205,20 +229,29 @@ mp_label(struct gctl_req *req)
 	}
 
 	/*
-	 * Now add the rest of the providers.
+	 * Now touch the rest of the providers to hint retaste.
 	 */
-	error = gctl_change_param(req, "verb", -1, "add");
-	if (error) {
-		gctl_error(req, "unable to change verb to \"add\": %s.", strerror(error));
-		return;
-	}
 	for (i = 2; i < nargs; i++) {
-		error = gctl_change_param(req, "arg1", -1, gctl_get_ascii(req, "arg%d", i));
-		if (error) {
-			gctl_error(req, "unable to add %s to %s: %s.", gctl_get_ascii(req, "arg%d", i), mpname, strerror(error));
+		name2 = gctl_get_ascii(req, "arg%d", i);
+		fd = g_open(name2, 1);
+		if (fd < 0) {
+			fprintf(stderr, "Unable to open %s: %s.\n",
+			    name2, strerror(errno));
+			continue;
+		}
+		if (pread(fd, rsector, secsize, disksize - secsize) !=
+		    secsize) {
+			fprintf(stderr, "Unable to read metadata from %s: %s.\n",
+			    name2, strerror(errno));
+			g_close(fd);
 			continue;
 		}
-		mp_add(req);
+		g_close(fd);
+		if (memcmp(sector, rsector, secsize)) {
+			fprintf(stderr, "No metadata found on %s."
+			    " It is not a path of %s.\n",
+			    name2, name);
+		}
 	}
 }
 
@@ -247,13 +280,3 @@ mp_clear(struct gctl_req *req)
 	}
 }
 
-static void
-mp_add(struct gctl_req *req)
-{
-	const char *errstr;
-
-	errstr = gctl_issue(req);
-	if (errstr != NULL && errstr[0] != '\0') {
-		gctl_error(req, "%s", errstr);
-	}
-}
diff --git a/sbin/geom/class/multipath/gmultipath.8 b/sbin/geom/class/multipath/gmultipath.8
index 8c354e4..a009b2b 100644
--- a/sbin/geom/class/multipath/gmultipath.8
+++ b/sbin/geom/class/multipath/gmultipath.8
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd February 26, 2007
+.Dd October 31, 2011
 .Dt GMULTIPATH 8
 .Os
 .Sh NAME
@@ -32,11 +32,48 @@
 .Nd "disk multipath control utility"
 .Sh SYNOPSIS
 .Nm
+.Cm create
+.Op Fl Av
+.Ar name
+.Ar prov ...
+.Nm
 .Cm label
-.Op Fl hv
+.Op Fl Av
 .Ar name
 .Ar prov ...
 .Nm
+.Cm add
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm remove
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm fail
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm restore
+.Op Fl v
+.Ar name prov
+.Nm
+.Cm rotate
+.Op Fl v
+.Ar name
+.Nm
+.Cm getactive
+.Op Fl v
+.Ar name
+.Nm
+.Cm destroy
+.Op Fl v
+.Ar name
+.Nm
+.Cm stop
+.Op Fl v
+.Ar name
+.Nm
 .Cm clear
 .Op Fl v
 .Ar prov ...
@@ -53,27 +90,79 @@ The
 .Nm
 utility is used for device multipath configuration.
 .Pp
-Only automatic configuration is supported at the present time via the
-.Cm label
-command.
-This operation writes a label on the last sector of the underlying
-disk device with a contained name and UUID.
-The UUID guarantees uniqueness
-in a shared storage environment but is in general too cumbersome to use.
+The multipath device can be configured using two different methods:
+.Dq manual
+or
+.Dq automatic .
+When using the
+.Dq manual
+method, no metadata are stored on the devices, so the multipath
+device has to be configured by hand every time it is needed.
+Additional device paths also won't be detected automatically.
+The
+.Dq automatic
+method uses on-disk metadata to detect device and all it's paths.
+Metadata use the last sector of the underlying disk device and
+include device name and UUID.
+The UUID guarantees uniqueness in a shared storage environment
+but is in general too cumbersome to use.
 The name is what is exported via the device interface.
 .Pp
 The first argument to
 .Nm
 indicates an action to be performed:
 .Bl -tag -width ".Cm destroy"
+.It Cm create
+Create multipath device with
+.Dq manual
+method without writing any on-disk metadata.
+It is up to administrator, how to properly identify device paths.
+Kernel will only check that all given providers have same media and
+sector sizes.
+.Pp
+.Fl A
+option enables Active/Active mode, otherwise Active/Passive mode is used
+by default.
 .It Cm label
-Label the given underlying device with the specified
+Create multipath device with
+.Dq automatic
+method.
+Label the first given provider with on-disk metadata using the specified
 .Ar name .
-The kernel module
-.Pa geom_multipath.ko
-will be loaded if it is not loaded already.
+The rest of given providers will be retasted to detect these metadata.
+It reliably protects against specifying unrelated providers.
+Providers with no matching metadata detected will not be added to the device.
+.Pp
+.Fl A
+option enables Active/Active mode, otherwise Active/Passive mode is used
+by default.
+.It Cm add
+Add the given provider as a path to the given multipath device.
+Should normally be used only for devices created with
+.Dq manual
+method, unless you know what you are doing (you are sure that it is another
+device path, but tasting its metadata in regular
+.Dq automatic
+way is not possible).
+.It Cm remove
+Remove the given provider as a path from the given multipath device.
+If the last path removed, the multipath device will be destroyed.
+.It Cm fail
+Mark specified provider as a path of the specified multipath device as failed.
+If there are other paths present, new requests will be forwarded there.
+.It Cm restore
+Mark specified provider as a path of the specified multipath device as
+operational, allowing it to handle requests.
+.It Cm rotate
+Change the active provider/path in Active/Passive mode.
+.It Cm getactive
+Get the currently active provider(s)/path(s).
+.It Cm destroy
+Destroy the given multipath device clearing metadata.
+.It Cm stop
+Stop the given multipath device without clearing metadata.
 .It Cm clear
-Clear metadata on the given device.
+Clear metadata on the given provider.
 .It Cm list
 See
 .Xr geom 8 .
@@ -101,14 +190,15 @@ Debug level of the
 GEOM class.
 This can be set to 0 (default) or 1 to disable or enable various
 forms of chattiness.
+.It Va kern.geom.multipath.exclusive : No 1
+Open underlying providers exclusively, preventing individual paths access.
 .El
 .Sh EXIT STATUS
 Exit status is 0 on success, and 1 if the command fails.
 .Sh MULTIPATH ARCHITECTURE
 .Pp
-This is an active/passive
-multiple path architecture with no device knowledge or presumptions other
-than size matching built in.
+This is a multiple path architecture with no device knowledge or
+presumptions other than size matching built in.
 Therefore the user must exercise some care
 in selecting providers that do indeed represent multiple paths to the
 same underlying disk device.
@@ -133,15 +223,16 @@ of multiple pathnames refer to the same device should be left to the
 system operator who will use tools and knowledge of their own storage
 subsystem to make the correct configuration selection.
 .Pp
-As an active/passive architecture, only one path has I/O moving on it
+There are Active/Passive and Active/Active operation modes supported.
+In Active/Passive mode only one path has I/O moving on it
 at any point in time.
 This I/O continues until an I/O is returned with
 a generic I/O error or a "Nonexistent Device" error.
-When this occurs,
-the active device is kicked out of the
-.Nm MULTIPATH
-GEOM class and the next in a list is selected, the failed I/O reissued
-and the system proceeds.
+When this occurs, that path is marked FAIL, the next path
+in a list is selected as active and the failed I/O reissued.
+In Active/Active mode all paths not marked FAIL may handle I/O same time.
+Requests are distributed between paths to equalize load.
+For capable devices it allows to utilize bandwidth of all paths.
 .Pp
 When new devices are added to the system the
 .Nm MULTIPATH
@@ -149,9 +240,9 @@ GEOM class is given an opportunity to taste these new devices.
 If a new
 device has a
 .Nm MULTIPATH
-label, the device is used to either create a new
+on-disk metadata label, the device is used to either create a new
 .Nm MULTIPATH
-GEOM, or to attach to the end of the list of devices for an existing
+GEOM, or been added the list of paths for an existing
 .Nm MULTIPATH
 GEOM.
 .Pp
@@ -176,7 +267,7 @@ of an RSCN event from the Fabric Domain Controller), they can cause
 a rescan to occur and cause the attachment and configuration of any
 (now) new devices to occur, causing the taste event described above.
 .Pp
-This means that this active/passive architecture is not a one-shot path
+This means that this multipath architecture is not a one-shot path
 failover, but can be considered to be steady state as long as failed
 paths are repaired (automatically or otherwise).
 .Pp
@@ -184,7 +275,7 @@ Automatic rescanning is not a requirement.
 Nor is Fibre Channel.
 The
 same failover mechanisms work equally well for traditional "Parallel"
-SCSI but require manual intervention with
+SCSI but may require manual intervention with
 .Xr camcontrol 8
 to cause the reattachment of repaired device links.
 .Sh EXAMPLES
@@ -226,9 +317,9 @@ mount /dev/multipath/FREDa /mnt....
 .Pp
 The resultant console output looks something like:
 .Bd -literal -offset indent
-GEOM_MULTIPATH: adding da0 to Fred/b631385f-c61c-11db-b884-0011116ae789
-GEOM_MULTIPATH: da0 now active path in Fred
-GEOM_MULTIPATH: adding da2 to Fred/b631385f-c61c-11db-b884-0011116ae789
+GEOM_MULTIPATH: da0 added to FRED
+GEOM_MULTIPATH: da0 is now active path in FRED
+GEOM_MULTIPATH: da2 added to FRED
 .Ed
 .Sh SEE ALSO
 .Xr geom 4 ,
@@ -240,24 +331,6 @@ GEOM_MULTIPATH: adding da2 to Fred/b631385f-c61c-11db-b884-0011116ae789
 .Xr mount 8 ,
 .Xr newfs 8 ,
 .Xr sysctl 8
-.Sh BUGS
-The
-.Nm
-should allow for a manual method of pairing disks.
-.Pp
-There is currently no way for
-.Pa geom_multipath.ko
-to distinguish between various label instances of the same provider.
-That
-is devices such as
-.Ar da0
-and
-.Ar da0c
-can be tasted and instantiated as multiple paths for the same device.
-Technically, this is correct, but pretty useless.
-This will be fixed soon
-(I hope), but to avoid this it is a good idea to destroy any label on
-the disk object prior to labelling it with
-.Nm .
 .Sh AUTHOR
 .An Matthew Jacob Aq mjacob@FreeBSD.org
+.An Alexander Motin Aq mav@FreeBSD.org