Import the gvinum work that have been done during and after Summer of Code 2007.

The work have been under testing and fixing since then, and it is mature enough
to be put into HEAD for further testing.

A lot have changed in this time, and here are the most important:
- Gvinum now uses one single workerthread instead of one thread for each
  volume and each plex. The reason for this is that the previous scheme was
  very complex, and was the cause of many of the bugs discovered in gvinum.
  Instead, gvinum now uses one worker thread with an event queue, quite
  similar to what used in gmirror.
- The rebuild/grow/initialize/parity check routines no longer runs in
  separate threads, but are run as regular I/O requests with special flags.
  This made it easier to support mounted growing and parity rebuild.
- Support for growing striped and raid5-plexes, meaning that one can extend the
  volumes for these plex types in addition to the concat type. Also works while
  the volume is mounted.
- Implementation of many of the missing commands from the old vinum:
  attach/detach, start (was partially implemented), stop (was partially
  implemented), concat, mirror, stripe, raid5 (shortcuts for creating volumes
  with one plex of these organizations).
- The parity check and rebuild no longer goes between userland/kernel, meaning
  that the gvinum command will not stay and wait forever for the rebuild to
  finish. You can instead watch the status with the list command.
- Many problems with gvinum have been reported since 5.x, and some has been hard
  to fix due to the complicated architecture. Hopefully, it should be more
  stable and better handle edge cases that previously made gvinum crash.
- Failed drives no longer disappears entirely, but now leave behind a dummy
  drive that makes sure the original state is not forgotten in case the system
  is rebooted between drive failures/swaps.
- Update manpage to reflect new commands and extend it with some examples.

Sponsored by:   Google Summer of Code 2007
Mentored by:    le
Tested by:      Rick C. Petty <rick-freebsd2008 -at- kiwi-computer.com>

1 files changed, 460 insertions, 67 deletions

diff --git a/sbin/gvinum/gvinum.c b/sbin/gvinum/gvinum.c
index ea33b75..bea4f7c 100644
--- a/sbin/gvinum/gvinum.c
+++ b/sbin/gvinum/gvinum.c
@@ -1,5 +1,7 @@
 /*
- *  Copyright (c) 2004 Lukas Ertl, 2005 Chris Jones
+ *  Copyright (c) 2004 Lukas Ertl
+ *  Copyright (c) 2005 Chris Jones
+ *  Copyright (c) 2007 Ulf Lilleengen
  *  All rights reserved.
  *
  * Portions of this software were developed for the FreeBSD Project
@@ -43,6 +45,7 @@
 
 #include <ctype.h>
 #include <err.h>
+#include <errno.h>
 #include <libgeom.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -54,12 +57,17 @@
 
 #include "gvinum.h"
 
+void	gvinum_attach(int, char **);
+void	gvinum_concat(int, char **);
 void	gvinum_create(int, char **);
+void	gvinum_detach(int, char **);
 void	gvinum_help(void);
 void	gvinum_list(int, char **);
 void	gvinum_move(int, char **);
+void	gvinum_mirror(int, char **);
 void	gvinum_parityop(int, char **, int);
 void	gvinum_printconfig(int, char **);
+void	gvinum_raid5(int, char **);
 void	gvinum_rename(int, char **);
 void	gvinum_resetconfig(void);
 void	gvinum_rm(int, char **);
@@ -67,9 +75,15 @@ void	gvinum_saveconfig(void);
 void	gvinum_setstate(int, char **);
 void	gvinum_start(int, char **);
 void	gvinum_stop(int, char **);
+void	gvinum_stripe(int, char **);
 void	parseline(int, char **);
 void	printconfig(FILE *, char *);
 
+char	*create_drive(char *);
+void	 create_volume(int, char **, char *);
+char	*find_name(const char *, int, int);
+char	*find_pattern(char *, char *);
+
 int
 main(int argc, char **argv)
 {
@@ -111,6 +125,44 @@ main(int argc, char **argv)
 	exit(0);
 }
 
+/* Attach a plex to a volume or a subdisk to a plex. */
+void
+gvinum_attach(int argc, char **argv)
+{
+	struct gctl_req *req;
+	const char *errstr;
+	int rename;
+	off_t offset;
+
+	rename = 0;
+	offset = -1;
+	if (argc < 3) {
+		warnx("usage:\tattach <subdisk> <plex> [rename] "
+		    "[<plexoffset>]\n"
+		    "\tattach <plex> <volume> [rename]");
+		return;
+	}
+	if (argc > 3) {
+		if (!strcmp(argv[3], "rename")) {
+			rename = 1;
+			if (argc == 5)
+				offset = strtol(argv[4], NULL, 0);
+		} else
+			offset = strtol(argv[3], NULL, 0);
+	}
+	req = gctl_get_handle();
+	gctl_ro_param(req, "class", -1, "VINUM");
+	gctl_ro_param(req, "verb", -1, "attach");
+	gctl_ro_param(req, "child", -1, argv[1]);
+	gctl_ro_param(req, "parent", -1, argv[2]);
+	gctl_ro_param(req, "offset", sizeof(off_t), &offset);
+	gctl_ro_param(req, "rename", sizeof(int), &rename);
+	errstr = gctl_issue(req);
+	if (errstr != NULL)
+		warnx("attach failed: %s", errstr);
+	gctl_free(req);
+}
+
 void
 gvinum_create(int argc, char **argv)
 {
@@ -120,19 +172,30 @@ gvinum_create(int argc, char **argv)
 	struct gv_sd *s;
 	struct gv_volume *v;
 	FILE *tmp;
-	int drives, errors, fd, line, plexes, plex_in_volume;
-	int sd_in_plex, status, subdisks, tokens, volumes;
+	int drives, errors, fd, flags, i, line, plexes, plex_in_volume;
+	int sd_in_plex, status, subdisks, tokens, undeffd, volumes;
 	const char *errstr;
-	char buf[BUFSIZ], buf1[BUFSIZ], commandline[BUFSIZ], *ed;
+	char buf[BUFSIZ], buf1[BUFSIZ], commandline[BUFSIZ], *ed, *sdname;
 	char original[BUFSIZ], tmpfile[20], *token[GV_MAXARGS];
 	char plex[GV_MAXPLEXNAME], volume[GV_MAXVOLNAME];
 
-	if (argc == 2) {
-		if ((tmp = fopen(argv[1], "r")) == NULL) {
-			warn("can't open '%s' for reading", argv[1]);
-			return;
-		}
-	} else {
+	tmp = NULL;
+	flags = 0;
+	for (i = 1; i < argc; i++) {
+		/* Force flag used to ignore already created drives. */
+		if (!strcmp(argv[i], "-f")) {
+			flags |= GV_FLAG_F;
+		/* Else it must be a file. */
+		} else {
+			if ((tmp = fopen(argv[1], "r")) == NULL) {
+				warn("can't open '%s' for reading", argv[1]);
+				return;
+			}
+		}	
+	}
+
+	/* We didn't get a file. */
+	if (tmp == NULL) {
 		snprintf(tmpfile, sizeof(tmpfile), "/tmp/gvinum.XXXXXX");
 		
 		if ((fd = mkstemp(tmpfile)) == -1) {
@@ -167,9 +230,11 @@ gvinum_create(int argc, char **argv)
 	req = gctl_get_handle();
 	gctl_ro_param(req, "class", -1, "VINUM");
 	gctl_ro_param(req, "verb", -1, "create");
+	gctl_ro_param(req, "flags", sizeof(int), &flags);
 
 	drives = volumes = plexes = subdisks = 0;
-	plex_in_volume = sd_in_plex = 0;
+	plex_in_volume = sd_in_plex = undeffd = 0;
+	plex[0] = '\0';
 	errors = 0;
 	line = 1;
 	while ((fgets(buf, BUFSIZ, tmp)) != NULL) {
@@ -187,7 +252,7 @@ gvinum_create(int argc, char **argv)
 		 * Copy the original input line in case we need it for error
 		 * output.
 		 */
-		strncpy(original, buf, sizeof(buf));
+		strlcpy(original, buf, sizeof(original));
 
 		tokens = gv_tokenize(buf, token, GV_MAXARGS);
 		if (tokens <= 0) {
@@ -214,7 +279,7 @@ gvinum_create(int argc, char **argv)
 			 * Set default volume name for following plex
 			 * definitions.
 			 */
-			strncpy(volume, v->name, sizeof(volume));
+			strlcpy(volume, v->name, sizeof(volume));
 
 			snprintf(buf1, sizeof(buf1), "volume%d", volumes);
 			gctl_ro_param(req, buf1, sizeof(*v), v);
@@ -236,13 +301,13 @@ gvinum_create(int argc, char **argv)
 
 			/* Default name. */
 			if (strlen(p->name) == 0) {
-				snprintf(p->name, GV_MAXPLEXNAME, "%s.p%d",
+				snprintf(p->name, sizeof(p->name), "%s.p%d",
 				    volume, plex_in_volume++);
 			}
 
 			/* Default volume. */
 			if (strlen(p->volume) == 0) {
-				snprintf(p->volume, GV_MAXVOLNAME, "%s",
+				snprintf(p->volume, sizeof(p->volume), "%s",
 				    volume);
 			}
 
@@ -250,7 +315,7 @@ gvinum_create(int argc, char **argv)
 			 * Set default plex name for following subdisk
 			 * definitions.
 			 */
-			strncpy(plex, p->name, GV_MAXPLEXNAME);
+			strlcpy(plex, p->name, sizeof(plex));
 
 			snprintf(buf1, sizeof(buf1), "plex%d", plexes);
 			gctl_ro_param(req, buf1, sizeof(*p), p);
@@ -270,13 +335,21 @@ gvinum_create(int argc, char **argv)
 
 			/* Default name. */
 			if (strlen(s->name) == 0) {
-				snprintf(s->name, GV_MAXSDNAME, "%s.s%d",
-				    plex, sd_in_plex++);
+				if (strlen(plex) == 0) {
+					sdname = find_name("gvinumsubdisk.p",
+					    GV_TYPE_SD, GV_MAXSDNAME);
+					snprintf(s->name, sizeof(s->name),
+					    "%s.s%d", sdname, undeffd++);
+					free(sdname);
+				} else {
+					snprintf(s->name, sizeof(s->name),
+					    "%s.s%d",plex, sd_in_plex++);
+				}
 			}
 
 			/* Default plex. */
 			if (strlen(s->plex) == 0)
-				snprintf(s->plex, GV_MAXPLEXNAME, "%s", plex);
+				snprintf(s->plex, sizeof(s->plex), "%s", plex);
 
 			snprintf(buf1, sizeof(buf1), "sd%d", subdisks);
 			gctl_ro_param(req, buf1, sizeof(*s), s);
@@ -320,7 +393,279 @@ gvinum_create(int argc, char **argv)
 			warnx("create failed: %s", errstr);
 	}
 	gctl_free(req);
-	gvinum_list(0, NULL);
+}
+
+/* Create a concatenated volume. */
+void
+gvinum_concat(int argc, char **argv)
+{
+
+	if (argc < 2) {
+		warnx("usage:\tconcat [-fv] [-n name] drives\n");
+		return;
+	}
+	create_volume(argc, argv, "concat");
+}
+
+
+/* Create a drive quick and dirty. */
+char *
+create_drive(char *device)
+{
+	struct gv_drive *d;
+	struct gctl_req *req;
+	const char *errstr;
+	char *drivename, *dname;
+	int drives, i, flags, volumes, subdisks, plexes;
+
+	flags = plexes = subdisks = volumes = 0;
+	drives = 1;
+	dname = NULL;
+
+	/* Strip away eventual /dev/ in front. */
+	if (strncmp(device, "/dev/", 5) == 0)
+		device += 5;
+
+	drivename = find_name("gvinumdrive", GV_TYPE_DRIVE, GV_MAXDRIVENAME);
+	if (drivename == NULL)
+		return (NULL);
+
+	req = gctl_get_handle();
+	gctl_ro_param(req, "class", -1, "VINUM");
+	gctl_ro_param(req, "verb", -1, "create");
+	d = malloc(sizeof(struct gv_drive));
+	if (d == NULL)
+		err(1, "unable to allocate for gv_drive object");
+	memset(d, 0, sizeof(struct gv_drive));
+
+	strlcpy(d->name, drivename, sizeof(d->name));
+	strlcpy(d->device, device, sizeof(d->device));
+	gctl_ro_param(req, "drive0", sizeof(*d), d);
+	gctl_ro_param(req, "flags", sizeof(int), &flags);
+	gctl_ro_param(req, "drives", sizeof(int), &drives);
+	gctl_ro_param(req, "volumes", sizeof(int), &volumes);
+	gctl_ro_param(req, "plexes", sizeof(int), &plexes);
+	gctl_ro_param(req, "subdisks", sizeof(int), &subdisks);
+	errstr = gctl_issue(req);
+	if (errstr != NULL) {
+		warnx("error creating drive: %s", errstr);
+		gctl_free(req);
+		return (NULL);
+	} else {
+		gctl_free(req);
+		/* XXX: This is needed because we have to make sure the drives
+		 * are created before we return. */
+		/* Loop until it's in the config. */
+		for (i = 0; i < 100000; i++) {
+			dname = find_name("gvinumdrive", GV_TYPE_DRIVE,
+			    GV_MAXDRIVENAME);
+			/* If we got a different name, quit. */
+			if (dname == NULL)
+				continue;
+			if (strcmp(dname, drivename)) {
+				free(dname);
+				return (drivename);
+			}
+			free(dname);
+			dname = NULL;
+			usleep(100000); /* Sleep for 0.1s */
+		}
+	}
+	gctl_free(req);
+	return (drivename);
+}
+
+/* 
+ * General routine for creating a volume. Mainly for use by concat, mirror,
+ * raid5 and stripe commands.
+ */
+void
+create_volume(int argc, char **argv, char *verb)
+{
+	struct gctl_req *req;
+	const char *errstr;
+	char buf[BUFSIZ], *drivename, *volname;
+	int drives, flags, i;
+	off_t stripesize;
+
+	flags = 0;
+	drives = 0;
+	volname = NULL;
+	stripesize = 262144;
+
+	/* XXX: Should we check for argument length? */
+
+	req = gctl_get_handle();
+	gctl_ro_param(req, "class", -1, "VINUM");
+
+	for (i = 1; i < argc; i++) {
+		if (!strcmp(argv[i], "-f")) {
+			flags |= GV_FLAG_F;
+		} else if (!strcmp(argv[i], "-n")) {
+			volname = argv[++i];
+		} else if (!strcmp(argv[i], "-v")) {
+			flags |= GV_FLAG_V;
+		} else if (!strcmp(argv[i], "-s")) {
+			flags |= GV_FLAG_S;
+			if (!strcmp(verb, "raid5"))
+				stripesize = gv_sizespec(argv[++i]);
+		} else {
+			/* Assume it's a drive. */
+			snprintf(buf, sizeof(buf), "drive%d", drives++);
+
+			/* First we create the drive. */
+			drivename = create_drive(argv[i]); 
+			if (drivename == NULL)
+				goto bad;
+			/* Then we add it to the request. */
+			gctl_ro_param(req, buf, -1, drivename);
+		}
+	}
+
+	gctl_ro_param(req, "stripesize", sizeof(off_t), &stripesize);
+
+	/* Find a free volume name. */
+	if (volname == NULL)
+		volname = find_name("gvinumvolume", GV_TYPE_VOL, GV_MAXVOLNAME);
+
+	/* Then we send a request to actually create the volumes. */
+	gctl_ro_param(req, "verb", -1, verb);
+	gctl_ro_param(req, "flags", sizeof(int), &flags); 
+	gctl_ro_param(req, "drives", sizeof(int), &drives);
+	gctl_ro_param(req, "name", -1, volname);
+	errstr = gctl_issue(req);
+	if (errstr != NULL)
+		warnx("creating %s volume failed: %s", verb, errstr);
+bad:
+	gctl_free(req);
+}
+
+/* Parse a line of the config, return the word after <pattern>. */
+char *
+find_pattern(char *line, char *pattern)
+{
+	char *ptr;
+
+	ptr = strsep(&line, " ");
+	while (ptr != NULL) {
+		if (!strcmp(ptr, pattern)) {
+			/* Return the next. */
+			ptr = strsep(&line, " ");
+			return (ptr);
+		}
+		ptr = strsep(&line, " ");
+	}
+	return (NULL);
+}
+
+/* Find a free name for an object given a a prefix. */
+char *
+find_name(const char *prefix, int type, int namelen)
+{
+	struct gctl_req *req;
+	char comment[1], buf[GV_CFG_LEN - 1], *name, *sname, *ptr;
+	const char *errstr;
+	int i, n, begin, len, conflict;
+	char line[1024];
+
+	comment[0] = '\0';
+
+	/* Find a name. Fetch out configuration first. */
+	req = gctl_get_handle();
+	gctl_ro_param(req, "class", -1, "VINUM");
+	gctl_ro_param(req, "verb", -1, "getconfig");
+	gctl_ro_param(req, "comment", -1, comment);
+	gctl_rw_param(req, "config", sizeof(buf), buf);
+	errstr = gctl_issue(req);
+	if (errstr != NULL) {
+		warnx("can't get configuration: %s", errstr);
+		return (NULL);
+	}
+	gctl_free(req);
+
+	begin = 0;
+	len = strlen(buf);
+	i = 0;
+	sname = malloc(namelen + 1);
+
+	/* XXX: Max object setting? */
+	for (n = 0; n < 10000; n++) {
+		snprintf(sname, namelen, "%s%d", prefix, n);
+		conflict = 0;
+		begin = 0;
+		/* Loop through the configuration line by line. */
+		for (i = 0; i < len; i++) {
+			if (buf[i] == '\n' || buf[i] == '\0') {
+				ptr = buf + begin;
+				strlcpy(line, ptr, (i - begin) + 1);
+				begin = i + 1;
+				switch (type) {
+				case GV_TYPE_DRIVE:
+					name = find_pattern(line, "drive");
+					break;
+				case GV_TYPE_VOL:
+					name = find_pattern(line, "volume");
+					break;
+				case GV_TYPE_PLEX:
+				case GV_TYPE_SD:
+					name = find_pattern(line, "name");
+					break;
+				default:
+					printf("Invalid type given\n");
+					continue;
+				}
+				if (name == NULL)
+					continue;
+				if (!strcmp(sname, name)) {
+					conflict = 1;
+					/* XXX: Could quit the loop earlier. */
+				}
+			}
+		}
+		if (!conflict)
+			return (sname);
+	}
+	free(sname);
+	return (NULL);
+}
+
+/* Detach a plex or subdisk from its parent. */
+void
+gvinum_detach(int argc, char **argv)
+{
+	const char *errstr;
+	struct gctl_req *req;
+	int flags, i;
+
+	optreset = 1;
+	optind = 1;
+	while ((i = getopt(argc, argv, "f")) != -1) {
+		switch(i) {
+		case 'f':
+			flags |= GV_FLAG_F;
+			break;
+		default:
+			warn("invalid flag: %c", i);
+			return;
+		}
+	}
+	argc -= optind;
+	argv += optind;
+	if (argc != 1) {
+		warnx("usage: detach [-f] <subdisk> | <plex>");
+		return;
+	}
+
+	req = gctl_get_handle();
+	gctl_ro_param(req, "class", -1, "VINUM");
+	gctl_ro_param(req, "verb", -1, "detach");
+	gctl_ro_param(req, "object", -1, argv[0]);
+	gctl_ro_param(req, "flags", sizeof(int), &flags);
+
+	errstr = gctl_issue(req);
+	if (errstr != NULL)
+		warnx("detach failed: %s", errstr);
+	gctl_free(req);
 }
 
 void
@@ -329,8 +674,16 @@ gvinum_help(void)
 	printf("COMMANDS\n"
 	    "checkparity [-f] plex\n"
 	    "        Check the parity blocks of a RAID-5 plex.\n"
-	    "create description-file\n"
+	    "create [-f] description-file\n"
 	    "        Create as per description-file or open editor.\n"
+	    "attach plex volume [rename]\n"
+	    "attach subdisk plex [offset] [rename]\n"
+	    "        Attach a plex to a volume, or a subdisk to a plex\n"
+	    "concat [-fv] [-n name] drives\n"
+	    "        Create a concatenated volume from the specified drives.\n"
+	    "detach [-f] [plex | subdisk]\n"
+	    "        Detach a plex or a subdisk from the volume or plex to\n"
+	    "        which it is attached.\n"
 	    "l | list [-r] [-v] [-V] [volume | plex | subdisk]\n"
 	    "        List information about specified objects.\n"
 	    "ld [-r] [-v] [-V] [volume]\n"
@@ -341,18 +694,22 @@ gvinum_help(void)
 	    "        List information about plexes.\n"
 	    "lv [-r] [-v] [-V] [volume]\n"
 	    "        List information about volumes.\n"
+	    "mirror [-fsv] [-n name] drives\n"
+	    "        Create a mirrored volume from the specified drives.\n"
 	    "move | mv -f drive object ...\n"
 	    "        Move the object(s) to the specified drive.\n"
 	    "quit    Exit the vinum program when running in interactive mode."
 	    "  Nor-\n"
 	    "        mally this would be done by entering the EOF character.\n"
+	    "raid5 [-fv] [-s stripesize] [-n name] drives\n"
+	    "        Create a RAID-5 volume from the specified drives.\n"
 	    "rename [-r] [drive | subdisk | plex | volume] newname\n"
 	    "        Change the name of the specified object.\n"
 	    "rebuildparity plex [-f]\n"
 	    "        Rebuild the parity blocks of a RAID-5 plex.\n"
 	    "resetconfig\n"
 	    "        Reset the complete gvinum configuration\n"
-	    "rm [-r] volume | plex | subdisk | drive\n"
+	    "rm [-r] [-f] volume | plex | subdisk | drive\n"
 	    "        Remove an object.\n"
 	    "saveconfig\n"
 	    "        Save vinum configuration to disk after configuration"
@@ -363,6 +720,8 @@ gvinum_help(void)
 	    "        poses only.\n"
 	    "start [-S size] volume | plex | subdisk\n"
 	    "        Allow the system to access the objects.\n"
+	    "stripe [-fv] [-n name] drives\n"
+	    "        Create a striped volume from the specified drives.\n"
 	);
 
 	return;
@@ -488,6 +847,18 @@ gvinum_list(int argc, char **argv)
 	return;
 }
 
+/* Create a mirrored volume. */
+void
+gvinum_mirror(int argc, char **argv)
+{
+
+	if (argc < 2) {
+		warnx("usage\tmirror [-fsv] [-n name] drives\n");
+		return;
+	}
+	create_volume(argc, argv, "mirror");
+}
+
 /* Note that move is currently of form '[-r] target object [...]' */
 void
 gvinum_move(int argc, char **argv)
@@ -553,8 +924,7 @@ void
 gvinum_parityop(int argc, char **argv, int rebuild)
 {
 	struct gctl_req *req;
-	int flags, i, rv;
-	off_t offset;
+	int flags, i;
 	const char *errstr;
 	char *op, *msg;
 
@@ -591,47 +961,32 @@ gvinum_parityop(int argc, char **argv, int rebuild)
 		return;
 	}
 
-	do {
-		rv = 0;
-		req = gctl_get_handle();
-		gctl_ro_param(req, "class", -1, "VINUM");
-		gctl_ro_param(req, "verb", -1, "parityop");
-		gctl_ro_param(req, "flags", sizeof(int), &flags);
-		gctl_ro_param(req, "rebuild", sizeof(int), &rebuild);
-		gctl_rw_param(req, "rv", sizeof(int), &rv);
-		gctl_rw_param(req, "offset", sizeof(off_t), &offset);
-		gctl_ro_param(req, "plex", -1, argv[0]);
-		errstr = gctl_issue(req);
-		if (errstr) {
-			warnx("%s\n", errstr);
-			gctl_free(req);
-			break;
-		}
-		gctl_free(req);
-		if (flags & GV_FLAG_V) {
-			printf("\r%s at %s ... ", msg,
-			    gv_roughlength(offset, 1));
-		}
-		if (rv == 1) {
-			printf("Parity incorrect at offset 0x%jx\n",
-			    (intmax_t)offset);
-			if (!rebuild)
-				break;
-		}
-		fflush(stdout);
+	req = gctl_get_handle();
+	gctl_ro_param(req, "class", -1, "VINUM");
+	gctl_ro_param(req, "verb", -1, op);
+	gctl_ro_param(req, "rebuild", sizeof(int), &rebuild);
+	gctl_ro_param(req, "flags", sizeof(int), &flags);
+	gctl_ro_param(req, "plex", -1, argv[0]);
+
+	errstr = gctl_issue(req);
+	if (errstr)
+		warnx("%s\n", errstr);
+	gctl_free(req);
+}
 
-		/* Clear the -f flag. */
-		flags &= ~GV_FLAG_F;
-	} while (rv >= 0);
+/* Create a RAID-5 volume. */
+void
+gvinum_raid5(int argc, char **argv)
+{
 
-	if ((rv == 2) && (flags & GV_FLAG_V)) {
-		if (rebuild)
-			printf("Rebuilt parity on %s\n", argv[0]);
-		else
-			printf("%s has correct parity\n", argv[0]);
+	if (argc < 2) {
+		warnx("usage:\traid5 [-fv] [-s stripesize] [-n name] drives\n");
+		return;
 	}
+	create_volume(argc, argv, "raid5");
 }
 
+
 void
 gvinum_rename(int argc, char **argv)
 {
@@ -697,8 +1052,11 @@ gvinum_rm(int argc, char **argv)
 	flags = 0;
 	optreset = 1;
 	optind = 1;
-	while ((j = getopt(argc, argv, "r")) != -1) {
+	while ((j = getopt(argc, argv, "rf")) != -1) {
 		switch (j) {
+		case 'f':
+			flags |= GV_FLAG_F;
+			break;
 		case 'r':
 			flags |= GV_FLAG_R;
 			break;
@@ -728,7 +1086,6 @@ gvinum_rm(int argc, char **argv)
 		return;
 	}
 	gctl_free(req);
-	gvinum_list(0, NULL);
 }
 
 void
@@ -763,7 +1120,6 @@ gvinum_resetconfig(void)
 		return;
 	}
 	gctl_free(req);
-	gvinum_list(0, NULL);
 	printf("gvinum configuration obliterated\n");
 }
 
@@ -833,28 +1189,53 @@ gvinum_start(int argc, char **argv)
 	}
 
 	gctl_free(req);
-	gvinum_list(0, NULL);
 }
 
 void
 gvinum_stop(int argc, char **argv)
 {
-	int fileid;
+	int err, fileid;
 
 	fileid = kldfind(GVINUMMOD);
 	if (fileid == -1) {
 		warn("cannot find " GVINUMMOD);
 		return;
 	}
-	if (kldunload(fileid) != 0) {
+
+	/*
+	 * This little hack prevents that we end up in an infinite loop in
+	 * g_unload_class().  gv_unload() will return EAGAIN so that the GEOM
+	 * event thread will be free for the g_wither_geom() call from
+	 * gv_unload().  It's silly, but it works.
+	 */
+	printf("unloading " GVINUMMOD " kernel module... ");
+	fflush(stdout);
+	if ((err = kldunload(fileid)) != 0 && (errno == EAGAIN)) {
+		sleep(1);
+		err = kldunload(fileid);
+	}
+	if (err != 0) {
+		printf(" failed!\n");
 		warn("cannot unload " GVINUMMOD);
 		return;
 	}
 
-	warnx(GVINUMMOD " unloaded");
+	printf("done\n");
 	exit(0);
 }
 
+/* Create a striped volume. */
+void
+gvinum_stripe(int argc, char **argv)
+{
+
+	if (argc < 2) {
+		warnx("usage:\tstripe [-fv] [-n name] drives\n");
+		return;
+	}
+	create_volume(argc, argv, "stripe");
+}
+
 void
 parseline(int argc, char **argv)
 {
@@ -865,6 +1246,12 @@ parseline(int argc, char **argv)
 		gvinum_create(argc, argv);
 	else if (!strcmp(argv[0], "exit") || !strcmp(argv[0], "quit"))
 		exit(0);
+	else if (!strcmp(argv[0], "attach"))
+		gvinum_attach(argc, argv);
+	else if (!strcmp(argv[0], "detach"))
+		gvinum_detach(argc, argv);
+	else if (!strcmp(argv[0], "concat"))
+		gvinum_concat(argc, argv);
 	else if (!strcmp(argv[0], "help"))
 		gvinum_help();
 	else if (!strcmp(argv[0], "list") || !strcmp(argv[0], "l"))
@@ -877,12 +1264,16 @@ parseline(int argc, char **argv)
 		gvinum_list(argc, argv);
 	else if (!strcmp(argv[0], "lv"))
 		gvinum_list(argc, argv);
+	else if (!strcmp(argv[0], "mirror"))
+		gvinum_mirror(argc, argv);
 	else if (!strcmp(argv[0], "move"))
 		gvinum_move(argc, argv);
 	else if (!strcmp(argv[0], "mv"))
 		gvinum_move(argc, argv);
 	else if (!strcmp(argv[0], "printconfig"))
 		gvinum_printconfig(argc, argv);
+	else if (!strcmp(argv[0], "raid5"))
+		gvinum_raid5(argc, argv);
 	else if (!strcmp(argv[0], "rename"))
 		gvinum_rename(argc, argv);
 	else if (!strcmp(argv[0], "resetconfig"))
@@ -897,6 +1288,8 @@ parseline(int argc, char **argv)
 		gvinum_start(argc, argv);
 	else if (!strcmp(argv[0], "stop"))
 		gvinum_stop(argc, argv);
+	else if (!strcmp(argv[0], "stripe"))
+		gvinum_stripe(argc, argv);
 	else if (!strcmp(argv[0], "checkparity"))
 		gvinum_parityop(argc, argv, 0);
 	else if (!strcmp(argv[0], "rebuildparity"))