Implement checkparity/rebuildparity.

7 files changed, 316 insertions, 13 deletions

diff --git a/sys/geom/vinum/geom_vinum.c b/sys/geom/vinum/geom_vinum.c
index a25e5ca..5a54bee 100644
--- a/sys/geom/vinum/geom_vinum.c
+++ b/sys/geom/vinum/geom_vinum.c
@@ -503,6 +503,9 @@ gv_config(struct gctl_req *req, struct g_class *mp, char const *verb)
 	} else if (!strcmp(verb, "create")) {
 		gv_create(gp, req);
 
+	} else if (!strcmp(verb, "parityop")) {
+		gv_parityop(gp, req);
+
 	} else if (!strcmp(verb, "remove")) {
 		gv_remove(gp, req);
 
diff --git a/sys/geom/vinum/geom_vinum.h b/sys/geom/vinum/geom_vinum.h
index de48c7d..c215e2e 100644
--- a/sys/geom/vinum/geom_vinum.h
+++ b/sys/geom/vinum/geom_vinum.h
@@ -38,6 +38,7 @@ void	gv_save_config(struct g_consumer *, struct gv_drive *,
 	    struct gv_softc *);
 
 /* geom_vinum_init.c */
+void	gv_parityop(struct g_geom *, struct gctl_req *);
 void	gv_start_obj(struct g_geom *, struct gctl_req *);
 
 /* geom_vinum_list.c */
diff --git a/sys/geom/vinum/geom_vinum_init.c b/sys/geom/vinum/geom_vinum_init.c
index 46d9d51..95b3d6a 100644
--- a/sys/geom/vinum/geom_vinum_init.c
+++ b/sys/geom/vinum/geom_vinum_init.c
@@ -58,6 +58,125 @@ struct gv_sync_args {
 };
 
 void
+gv_parityop(struct g_geom *gp, struct gctl_req *req)
+{
+	struct gv_softc *sc;
+	struct gv_plex *p;
+	struct bio *bp;
+	struct g_consumer *cp;
+	int error, *flags, type, *rebuild, rv;
+	char *plex;
+
+	rv = -1;
+
+	plex = gctl_get_param(req, "plex", NULL);
+	if (plex == NULL) {
+		gctl_error(req, "no plex given");
+		goto out;
+	}
+
+	flags = gctl_get_paraml(req, "flags", sizeof(*flags));
+	if (flags == NULL) {
+		gctl_error(req, "no flags given");
+		goto out;
+	}
+
+	rebuild = gctl_get_paraml(req, "rebuild", sizeof(*rebuild));
+	if (rebuild == NULL) {
+		gctl_error(req, "no rebuild op given");
+		goto out;
+	}
+
+	sc = gp->softc;
+	type = gv_object_type(sc, plex);
+	switch (type) {
+	case GV_TYPE_PLEX:
+		break;
+	case GV_TYPE_VOL:
+	case GV_TYPE_SD:
+	case GV_TYPE_DRIVE:
+	default:
+		gctl_error(req, "'%s' is not a plex", plex);
+		goto out;
+	}
+
+	p = gv_find_plex(sc, plex);
+	if (p->state != GV_PLEX_UP) {
+		gctl_error(req, "plex %s is not completely accessible",
+		    p->name);
+		goto out;
+	}
+
+	cp = p->consumer;
+	error = g_access(cp, 1, 1, 0);
+	if (error) {
+		gctl_error(req, "cannot access consumer");
+		goto out;
+	}
+	g_topology_unlock();
+
+	/* Reset the check pointer when using -f. */
+	if (*flags & GV_FLAG_F)
+		p->synced = 0;
+
+	bp = g_new_bio();
+	if (bp == NULL) {
+		gctl_error(req, "cannot create BIO - out of memory");
+		g_topology_lock();
+		error = g_access(cp, -1, -1, 0);
+		goto out;
+	}
+	bp->bio_cmd = BIO_WRITE;
+	bp->bio_done = NULL;
+	bp->bio_data = g_malloc(p->stripesize, M_WAITOK | M_ZERO);
+	bp->bio_cflags |= GV_BIO_CHECK;
+	if (*rebuild)
+		bp->bio_cflags |= GV_BIO_PARITY;
+	bp->bio_offset = p->synced;
+	bp->bio_length = p->stripesize;
+
+	/* Schedule it down ... */
+	g_io_request(bp, cp);
+
+	/* ... and wait for the result. */
+	error = biowait(bp, "gwrite");
+	g_free(bp->bio_data);
+	g_destroy_bio(bp);
+
+	if (error) {
+		/* Incorrect parity. */
+		if (error == EAGAIN)
+			rv = 1;
+
+		/* Some other error happened. */
+		else
+			gctl_error(req, "Parity check failed at offset 0x%jx, "
+			    "errno %d", (intmax_t)p->synced, error);
+
+	/* Correct parity. */
+	} else
+		rv = 0;
+
+	gctl_set_param(req, "offset", &p->synced, sizeof(p->synced));
+
+	/* Advance the checkpointer if there was no error. */
+	if (rv == 0)
+		p->synced += p->stripesize;
+
+	/* End of plex; reset the check pointer and signal it to the caller. */
+	if (p->synced >= p->size) {
+		p->synced = 0;
+		rv = -2;
+	}
+
+	g_topology_lock();
+	error = g_access(cp, -1, -1, 0);
+
+out:
+	gctl_set_param(req, "rv", &rv, sizeof(rv));
+}
+
+void
 gv_start_obj(struct g_geom *gp, struct gctl_req *req)
 {
 	struct gv_softc *sc;
diff --git a/sys/geom/vinum/geom_vinum_plex.c b/sys/geom/vinum/geom_vinum_plex.c
index 31266f9..8fa2923 100644
--- a/sys/geom/vinum/geom_vinum_plex.c
+++ b/sys/geom/vinum/geom_vinum_plex.c
@@ -46,6 +46,10 @@ __FBSDID("$FreeBSD$");
 static void gv_plex_completed_request(struct gv_plex *, struct bio *);
 static void gv_plex_normal_request(struct gv_plex *, struct bio *);
 static void gv_plex_worker(void *);
+static int gv_check_parity(struct gv_plex *, struct bio *,
+    struct gv_raid5_packet *);
+static int gv_normal_parity(struct gv_plex *, struct bio *,
+    struct gv_raid5_packet *);
 
 /* XXX: is this the place to catch dying subdisks? */
 static void
@@ -346,6 +350,85 @@ gv_plex_worker(void *arg)
 	kthread_exit(ENXIO);
 }
 
+static int
+gv_normal_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
+{
+	struct bio *cbp, *pbp;
+	int finished, i;
+
+	finished = 1;
+
+	if (wp->waiting != NULL) {
+		pbp = wp->waiting;
+		wp->waiting = NULL;
+		cbp = wp->parity;
+		for (i = 0; i < wp->length; i++)
+			cbp->bio_data[i] ^= pbp->bio_data[i];
+		g_io_request(pbp, pbp->bio_caller2);
+		finished = 0;
+
+	} else if (wp->parity != NULL) {
+		cbp = wp->parity;
+		wp->parity = NULL;
+		g_io_request(cbp, cbp->bio_caller2);
+		finished = 0;
+	}
+
+	return (finished);
+}
+
+static int
+gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp)
+{
+	struct bio *cbp, *pbp;
+	int err, finished, i;
+
+	err = 0;
+	finished = 1;
+
+	if (wp->waiting != NULL) {
+		pbp = wp->waiting;
+		wp->waiting = NULL;
+		g_io_request(pbp, pbp->bio_caller2);
+		finished = 0;
+
+	} else if (wp->parity != NULL) {
+		cbp = wp->parity;
+		wp->parity = NULL;
+
+		/* Check if the parity is correct. */
+		for (i = 0; i < wp->length; i++) {
+			if (bp->bio_data[i] != cbp->bio_data[i]) {
+				err = 1;
+				break;
+			}
+		}
+
+		/* The parity is not correct... */
+		if (err) {
+			bp->bio_parent->bio_error = EAGAIN;
+
+			/* ... but we rebuild it. */
+			if (bp->bio_parent->bio_cflags & GV_BIO_PARITY) {
+				g_io_request(cbp, cbp->bio_caller2);
+				finished = 0;
+			}
+		}
+
+		/*
+		 * Clean up the BIO we would have used for rebuilding the
+		 * parity.
+		 */
+		if (finished) {
+			bp->bio_parent->bio_inbed++;
+			g_destroy_bio(cbp);
+		}
+
+	}
+
+	return (finished);
+}
+
 void
 gv_plex_completed_request(struct gv_plex *p, struct bio *bp)
 {
@@ -405,18 +488,13 @@ gv_plex_completed_request(struct gv_plex *p, struct bio *bp)
 
 		/* Handle parity data. */
 		if (TAILQ_EMPTY(&wp->bits)) {
-			if (wp->waiting != NULL) {
-				pbp = wp->waiting;
-				wp->waiting = NULL;
-				cbp = wp->parity;
-				for (i = 0; i < wp->length; i++)
-					cbp->bio_data[i] ^= pbp->bio_data[i];
-				g_io_request(pbp, pbp->bio_caller2);
-			} else if (wp->parity != NULL) {
-				cbp = wp->parity;
-				wp->parity = NULL;
-				g_io_request(cbp, cbp->bio_caller2);
-			} else {
+			if (bp->bio_parent->bio_cflags & GV_BIO_CHECK)
+				i = gv_check_parity(p, bp, wp);
+			else
+				i = gv_normal_parity(p, bp, wp);
+
+			/* All of our sub-requests have finished. */
+			if (i) {
 				bp->bio_parent->bio_completed += wp->length;
 				TAILQ_REMOVE(&p->packets, wp, list);
 				/* Bring the waiting bios back into the game. */
@@ -475,6 +553,9 @@ gv_plex_normal_request(struct gv_plex *p, struct bio *bp)
 			if (bp->bio_cflags & GV_BIO_REBUILD)
 				err = gv_rebuild_raid5(p, wp, bp, addr,
 				    boff, bcount);
+			else if (bp->bio_cflags & GV_BIO_CHECK)
+				err = gv_check_raid5(p, wp, bp, addr,
+				    boff, bcount);
 			else
 				err = gv_build_raid5_req(p, wp, bp, addr,
 				    boff, bcount);
diff --git a/sys/geom/vinum/geom_vinum_raid5.c b/sys/geom/vinum/geom_vinum_raid5.c
index 11bb931..789db19 100644
--- a/sys/geom/vinum/geom_vinum_raid5.c
+++ b/sys/geom/vinum/geom_vinum_raid5.c
@@ -81,6 +81,101 @@ gv_stripe_active(struct gv_plex *p, struct bio *bp)
 }
 
 int
+gv_check_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
+    caddr_t addr, off_t boff, off_t bcount)
+{
+	struct gv_sd *parity, *s;
+	struct gv_bioq *bq;
+	struct bio *cbp, *pbp;
+	int i, psdno;
+	off_t real_len, real_off;
+
+	if (p == NULL || LIST_EMPTY(&p->subdisks))
+		return (ENXIO);
+
+	gv_raid5_offset(p, boff, bcount, &real_off, &real_len, NULL, &psdno);
+
+	/* Find the right subdisk. */
+	parity = NULL;
+	i = 0;
+	LIST_FOREACH(s, &p->subdisks, in_plex) {
+		if (i == psdno) {
+			parity = s;
+			break;
+		}
+		i++;
+	}
+
+	/* Parity stripe not found. */
+	if (parity == NULL)
+		return (ENXIO);
+
+	if (parity->state != GV_SD_UP)
+		return (ENXIO);
+
+	wp->length = real_len;
+	wp->data = addr;
+	wp->lockbase = real_off;
+
+	/* Read all subdisks. */
+	LIST_FOREACH(s, &p->subdisks, in_plex) {
+		/* Skip the parity subdisk. */
+		if (s == parity)
+			continue;
+
+		cbp = g_clone_bio(bp);
+		if (cbp == NULL)
+			return (ENOMEM);
+		cbp->bio_cmd = BIO_READ;
+		cbp->bio_data = g_malloc(real_len, M_WAITOK);
+		cbp->bio_cflags |= GV_BIO_MALLOC;
+		cbp->bio_offset = real_off;
+		cbp->bio_length = real_len;
+		cbp->bio_done = gv_plex_done;
+		cbp->bio_caller2 = s->consumer;
+		cbp->bio_driver1 = wp;
+
+		GV_ENQUEUE(bp, cbp, pbp);
+
+		bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO);
+		bq->bp = cbp;
+		TAILQ_INSERT_TAIL(&wp->bits, bq, queue);
+	}
+
+	/* Read the parity data. */
+	cbp = g_clone_bio(bp);
+	if (cbp == NULL)
+		return (ENOMEM);
+	cbp->bio_cmd = BIO_READ;
+	cbp->bio_data = g_malloc(real_len, M_WAITOK | M_ZERO);
+	cbp->bio_cflags |= GV_BIO_MALLOC;
+	cbp->bio_offset = real_off;
+	cbp->bio_length = real_len;
+	cbp->bio_done = gv_plex_done;
+	cbp->bio_caller2 = parity->consumer;
+	cbp->bio_driver1 = wp;
+	wp->waiting = cbp;
+
+	/*
+	 * In case we want to rebuild the parity, create an extra BIO to write
+	 * it out.  It also acts as buffer for the XOR operations.
+	 */
+	cbp = g_clone_bio(bp);
+	if (cbp == NULL)
+		return (ENOMEM);
+	cbp->bio_data = addr;
+	cbp->bio_offset = real_off;
+	cbp->bio_length = real_len;
+	cbp->bio_done = gv_plex_done;
+	cbp->bio_caller2 = parity->consumer;
+	cbp->bio_driver1 = wp;
+	wp->parity = cbp;
+
+	return (0);
+}
+
+/* Rebuild a degraded RAID5 plex. */
+int
 gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
     caddr_t addr, off_t boff, off_t bcount)
 {
@@ -101,7 +196,7 @@ gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp,
 			broken = s;
 	}
 
-	/* Parity stripe not found. */
+	/* Broken stripe not found. */
 	if (broken == NULL)
 		return (ENXIO);
 
diff --git a/sys/geom/vinum/geom_vinum_raid5.h b/sys/geom/vinum/geom_vinum_raid5.h
index 212f6c6..9637db1 100644
--- a/sys/geom/vinum/geom_vinum_raid5.h
+++ b/sys/geom/vinum/geom_vinum_raid5.h
@@ -67,6 +67,8 @@ struct gv_raid5_packet {
 int	gv_stripe_active(struct gv_plex *, struct bio *);
 int	gv_build_raid5_req(struct gv_plex *, struct gv_raid5_packet *,
 	    struct bio *, caddr_t, off_t, off_t);
+int	gv_check_raid5(struct gv_plex *, struct gv_raid5_packet *,
+	    struct bio *, caddr_t, off_t, off_t);
 int	gv_rebuild_raid5(struct gv_plex *, struct gv_raid5_packet *,
 	    struct bio *, caddr_t, off_t, off_t);
 void	gv_raid5_worker(void *);
diff --git a/sys/geom/vinum/geom_vinum_var.h b/sys/geom/vinum/geom_vinum_var.h
index 95d8956..05a5e8c 100644
--- a/sys/geom/vinum/geom_vinum_var.h
+++ b/sys/geom/vinum/geom_vinum_var.h
@@ -114,6 +114,8 @@
 #define	GV_BIO_SYNCREQ	0x08
 #define	GV_BIO_SUCCEED	0x10
 #define	GV_BIO_REBUILD	0x20
+#define	GV_BIO_CHECK	0x40
+#define	GV_BIO_PARITY	0x80
 
 /*
  * hostname is 256 bytes long, but we don't need to shlep multiple copies in