- Add genid field to the metadata which will allow to improve reliability a bit.

  After this change, when component is disconnected because of an I/O error,
  it will not be connected and synchronized automatically, it will be logged
  as broken and skipped. Autosynchronization can occur, when component is
  disconnected (on orphan event) and connected again - there were no I/O
  error, so there is no need to not connected the component, but when there were
  writes while it wasn't connected, it will be synchronized.
  This fix cases, when component is disconnected because of I/O error and can be
  connected again and again.
- Bump version number.
- Implement backward compatibility mechanism. After this change when metadata in
  old version is detected, it is automatically upgraded to the new (current)
  version.

3 files changed, 206 insertions, 61 deletions

diff --git a/sys/geom/raid3/g_raid3.c b/sys/geom/raid3/g_raid3.c
index b40a8f4..6130de8 100644
--- a/sys/geom/raid3/g_raid3.c
+++ b/sys/geom/raid3/g_raid3.c
@@ -507,6 +507,7 @@ g_raid3_init_disk(struct g_raid3_softc *sc, struct g_provider *pp,
 	disk->d_sync.ds_offset = md->md_sync_offset;
 	disk->d_sync.ds_offset_done = md->md_sync_offset;
 	disk->d_sync.ds_resync = -1;
+	disk->d_genid = md->md_genid;
 	disk->d_sync.ds_syncid = md->md_syncid;
 	if (errorp != NULL)
 		*errorp = 0;
@@ -610,7 +611,7 @@ g_raid3_orphan(struct g_consumer *cp)
 	disk = cp->private;
 	if (disk == NULL)
 		return;
-	disk->d_softc->sc_bump_syncid = G_RAID3_BUMP_ON_FIRST_WRITE;
+	disk->d_softc->sc_bump_id = G_RAID3_BUMP_SYNCID_OFW;
 	g_raid3_event_send(disk, G_RAID3_DISK_STATE_DISCONNECTED,
 	    G_RAID3_EVENT_DONTWAIT);
 }
@@ -625,7 +626,7 @@ g_raid3_spoiled(struct g_consumer *cp)
 	disk = cp->private;
 	if (disk == NULL)
 		return;
-	disk->d_softc->sc_bump_syncid = G_RAID3_BUMP_IMMEDIATELY;
+	disk->d_softc->sc_bump_id = G_RAID3_BUMP_SYNCID_IMM;
 	g_raid3_event_send(disk, G_RAID3_DISK_STATE_DISCONNECTED,
 	    G_RAID3_EVENT_DONTWAIT);
 }
@@ -658,7 +659,7 @@ g_raid3_write_metadata(struct g_raid3_disk *disk, struct g_raid3_metadata *md)
 	g_topology_lock();
 	free(sector, M_RAID3);
 	if (error != 0) {
-		disk->d_softc->sc_bump_syncid = G_RAID3_BUMP_IMMEDIATELY;
+		disk->d_softc->sc_bump_id = G_RAID3_BUMP_GENID_IMM;
 		g_raid3_event_send(disk, G_RAID3_DISK_STATE_DISCONNECTED,
 		    G_RAID3_EVENT_DONTWAIT);
 	}
@@ -694,6 +695,7 @@ g_raid3_fill_metadata(struct g_raid3_disk *disk, struct g_raid3_metadata *md)
 	strlcpy(md->md_name, sc->sc_name, sizeof(md->md_name));
 	md->md_id = sc->sc_id;
 	md->md_all = sc->sc_ndisks;
+	md->md_genid = sc->sc_genid;
 	md->md_mediasize = sc->sc_mediasize;
 	md->md_sectorsize = sc->sc_sectorsize;
 	md->md_mflags = (sc->sc_flags & G_RAID3_DEVICE_FLAG_MASK);
@@ -744,6 +746,8 @@ g_raid3_bump_syncid(struct g_raid3_softc *sc)
 	    sc->sc_name));
 
 	sc->sc_syncid++;
+	G_RAID3_DEBUG(1, "Device %s: syncid bumped to %u.", sc->sc_name,
+	    sc->sc_syncid);
 	for (n = 0; n < sc->sc_ndisks; n++) {
 		disk = &sc->sc_disks[n];
 		if (disk->d_state == G_RAID3_DISK_STATE_ACTIVE ||
@@ -755,6 +759,30 @@ g_raid3_bump_syncid(struct g_raid3_softc *sc)
 }
 
 static void
+g_raid3_bump_genid(struct g_raid3_softc *sc)
+{
+	struct g_raid3_disk *disk;
+	u_int n;
+
+	g_topology_assert();
+	KASSERT(g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) > 0,
+	    ("%s called with no active disks (device=%s).", __func__,
+	    sc->sc_name));
+
+	sc->sc_genid++;
+	G_RAID3_DEBUG(1, "Device %s: genid bumped to %u.", sc->sc_name,
+	    sc->sc_genid);
+	for (n = 0; n < sc->sc_ndisks; n++) {
+		disk = &sc->sc_disks[n];
+		if (disk->d_state == G_RAID3_DISK_STATE_ACTIVE ||
+		    disk->d_state == G_RAID3_DISK_STATE_SYNCHRONIZING) {
+			disk->d_genid = sc->sc_genid;
+			g_raid3_update_metadata(disk);
+		}
+	}
+}
+
+static void
 g_raid3_idle(struct g_raid3_softc *sc)
 {
 	struct g_raid3_disk *disk;
@@ -1062,10 +1090,10 @@ g_raid3_gather(struct bio *pbp)
 		disk = cbp->bio_caller2;
 		if (disk != NULL) {
 			/*
-			 * Actually this is pointless to bump syncid,
+			 * Actually this is pointless to bump genid,
 			 * because whole device is fucked up.
 			 */
-			sc->sc_bump_syncid = G_RAID3_BUMP_IMMEDIATELY;
+			sc->sc_bump_id |= G_RAID3_BUMP_GENID_IMM;
 			g_raid3_event_send(disk,
 			    G_RAID3_DISK_STATE_DISCONNECTED,
 			    G_RAID3_EVENT_DONTWAIT);
@@ -1228,8 +1256,8 @@ g_raid3_regular_request(struct bio *cbp)
 			if (cbp->bio_error != 0) {
 				disk = cbp->bio_caller2;
 				if (disk != NULL) {
-					sc->sc_bump_syncid =
-					    G_RAID3_BUMP_IMMEDIATELY;
+					sc->sc_bump_id |=
+					    G_RAID3_BUMP_GENID_IMM;
 					g_raid3_event_send(disk,
 					    G_RAID3_DISK_STATE_DISCONNECTED,
 					    G_RAID3_EVENT_DONTWAIT);
@@ -1429,7 +1457,7 @@ g_raid3_sync_request(struct bio *bp)
 			    "Synchronization request failed (error=%d).",
 			    bp->bio_error);
 			g_destroy_bio(bp);
-			sc->sc_bump_syncid = G_RAID3_BUMP_IMMEDIATELY;
+			sc->sc_bump_id |= G_RAID3_BUMP_GENID_IMM;
 			g_raid3_event_send(disk,
 			    G_RAID3_DISK_STATE_DISCONNECTED,
 			    G_RAID3_EVENT_DONTWAIT);
@@ -1629,8 +1657,8 @@ g_raid3_register_request(struct bio *pbp)
 		/*
 		 * Bump syncid on first write.
 		 */
-		if (sc->sc_bump_syncid == G_RAID3_BUMP_ON_FIRST_WRITE) {
-			sc->sc_bump_syncid = 0;
+		if ((sc->sc_bump_id & G_RAID3_BUMP_SYNCID_OFW) != 0) {
+			sc->sc_bump_id &= ~G_RAID3_BUMP_SYNCID;
 			g_topology_lock();
 			g_raid3_bump_syncid(sc);
 			g_topology_unlock();
@@ -1667,23 +1695,18 @@ static int
 g_raid3_try_destroy(struct g_raid3_softc *sc)
 {
  
+	g_topology_lock();
+	if (!g_raid3_can_destroy(sc)) {
+		g_topology_unlock();
+		return (0);
+	}
 	if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_WAIT) != 0) {
-		g_topology_lock();
-		if (!g_raid3_can_destroy(sc)) {
-			g_topology_unlock();
-			return (0);
-		}
 		g_topology_unlock();
 		G_RAID3_DEBUG(4, "%s: Waking up %p.", __func__,
 		    &sc->sc_worker);
 		wakeup(&sc->sc_worker);
 		sc->sc_worker = NULL;
 	} else {
-		g_topology_lock();
-		if (!g_raid3_can_destroy(sc)) {
-			g_topology_unlock();
-			return (0);
-		}
 		g_raid3_destroy_device(sc);
 		g_topology_unlock();
 		free(sc->sc_disks, M_RAID3);
@@ -1755,8 +1778,14 @@ g_raid3_worker(void *arg)
 			G_RAID3_DEBUG(5, "%s: I'm here 1.", __func__);
 			continue;
 		}
-		if (ep != NULL)
+#if 1
+		if (ep != NULL) {
 			printf("The topology lock is already held.\n");
+#if 0
+			tsleep(&ep, PRIBIO, "r3:hmm", hz * 3);
+#endif
+		}
+#endif
 		/*
 		 * Now I/O requests.
 		 */
@@ -1770,6 +1799,7 @@ g_raid3_worker(void *arg)
 				 * already held? Try again.
 				 */
 				mtx_unlock(&sc->sc_queue_mtx);
+				tsleep(ep, PRIBIO, "r3:hmm1", hz / 5);
 				continue;
 			}
 			if ((sc->sc_flags &
@@ -1849,6 +1879,7 @@ sleep:
 				 * We have some pending events, don't sleep now.
 				 */
 				G_RAID3_DEBUG(5, "%s: I'm here 7.", __func__);
+				tsleep(ep, PRIBIO, "r3:hmm2", hz / 5);
 				continue;
 			}
 			mtx_lock(&sc->sc_queue_mtx);
@@ -2114,7 +2145,7 @@ g_raid3_update_device(struct g_raid3_softc *sc, boolean_t force)
 	switch (sc->sc_state) {
 	case G_RAID3_DEVICE_STATE_STARTING:
 	    {
-		u_int n, ndirty, ndisks, syncid;
+		u_int n, ndirty, ndisks, genid, syncid;
 
 		KASSERT(sc->sc_provider == NULL,
 		    ("Non-NULL provider in STARTING state (%s).", sc->sc_name));
@@ -2136,6 +2167,33 @@ g_raid3_update_device(struct g_raid3_softc *sc, boolean_t force)
 		}
 
 		/*
+		 * Find the biggest genid.
+		 */
+		genid = 0;
+		for (n = 0; n < sc->sc_ndisks; n++) {
+			disk = &sc->sc_disks[n];
+			if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
+				continue;
+			if (disk->d_genid > genid)
+				genid = disk->d_genid;
+		}
+		sc->sc_genid = genid;
+		/*
+		 * Remove all disks without the biggest genid.
+		 */
+		for (n = 0; n < sc->sc_ndisks; n++) {
+			disk = &sc->sc_disks[n];
+			if (disk->d_state == G_RAID3_DISK_STATE_NODISK)
+				continue;
+			if (disk->d_genid < genid) {
+				G_RAID3_DEBUG(0,
+				    "Component %s (device %s) broken, skipping.",
+				    g_raid3_get_diskname(disk), sc->sc_name);
+				g_raid3_destroy_disk(disk);
+			}
+		}
+
+		/*
 		 * There must be at least 'sc->sc_ndisks - 1' components
 		 * with the same syncid and without SYNCHRONIZING flag.
 		 */
@@ -2196,7 +2254,7 @@ g_raid3_update_device(struct g_raid3_softc *sc, boolean_t force)
 		sc->sc_syncid = syncid;
 		if (force) {
 			/* Remember to bump syncid on first write. */
-			sc->sc_bump_syncid = G_RAID3_BUMP_ON_FIRST_WRITE;
+			sc->sc_bump_id |= G_RAID3_BUMP_SYNCID_OFW;
 		}
 		if (ndisks == sc->sc_ndisks)
 			state = G_RAID3_DEVICE_STATE_COMPLETE;
@@ -2212,21 +2270,25 @@ g_raid3_update_device(struct g_raid3_softc *sc, boolean_t force)
 				continue;
 			state = g_raid3_determine_state(disk);
 			g_raid3_event_send(disk, state, G_RAID3_EVENT_DONTWAIT);
-			if (state == G_RAID3_DISK_STATE_STALE) {
-				sc->sc_bump_syncid =
-				    G_RAID3_BUMP_ON_FIRST_WRITE;
-			}
+			if (state == G_RAID3_DISK_STATE_STALE)
+				sc->sc_bump_id |= G_RAID3_BUMP_SYNCID_OFW;
 		}
 		break;
 	    }
 	case G_RAID3_DEVICE_STATE_DEGRADED:
 		/*
-		 * Bump syncid here, if we need to do it immediately.
+		 * Bump syncid and/or genid here, if we need to do it
+		 * immediately.
 		 */
-		if (sc->sc_bump_syncid == G_RAID3_BUMP_IMMEDIATELY) {
-			sc->sc_bump_syncid = 0;
+		if ((sc->sc_bump_id & G_RAID3_BUMP_SYNCID_IMM) != 0) {
+			sc->sc_bump_id &= ~G_RAID3_BUMP_SYNCID;
 			g_raid3_bump_syncid(sc);
 		}
+		if ((sc->sc_bump_id & G_RAID3_BUMP_GENID_IMM) != 0) {
+			sc->sc_bump_id &= ~G_RAID3_BUMP_GENID;
+			g_raid3_bump_genid(sc);
+		}
+
 		if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_NEW) > 0)
 			return;
 		if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) <
@@ -2250,12 +2312,18 @@ g_raid3_update_device(struct g_raid3_softc *sc, boolean_t force)
 		break;
 	case G_RAID3_DEVICE_STATE_COMPLETE:
 		/*
-		 * Bump syncid here, if we need to do it immediately.
+		 * Bump syncid and/or genid here, if we need to do it
+		 * immediately.
 		 */
-		if (sc->sc_bump_syncid == G_RAID3_BUMP_IMMEDIATELY) {
-			sc->sc_bump_syncid = 0;
+		if ((sc->sc_bump_id & G_RAID3_BUMP_SYNCID_IMM) != 0) {
+			sc->sc_bump_id &= ~G_RAID3_BUMP_SYNCID;
 			g_raid3_bump_syncid(sc);
 		}
+		if ((sc->sc_bump_id & G_RAID3_BUMP_GENID_IMM) != 0) {
+			sc->sc_bump_id &= ~G_RAID3_BUMP_GENID;
+			g_raid3_bump_genid(sc);
+		}
+
 		if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_NEW) > 0)
 			return;
 		KASSERT(g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) >=
@@ -2448,8 +2516,8 @@ again:
 			 * Reset bumping syncid if disk disappeared in STARTING
 			 * state.
 			 */
-			if (sc->sc_bump_syncid == G_RAID3_BUMP_ON_FIRST_WRITE)
-				sc->sc_bump_syncid = 0;
+			if ((sc->sc_bump_id & G_RAID3_BUMP_SYNCID_OFW) != 0)
+				sc->sc_bump_id &= ~G_RAID3_BUMP_SYNCID;
 #ifdef	INVARIANTS
 		} else {
 			KASSERT(1 == 0, ("Wrong device state (%s, %s, %s, %s).",
@@ -2493,6 +2561,8 @@ g_raid3_read_metadata(struct g_consumer *cp, struct g_raid3_metadata *md)
 	g_topology_lock();
 	g_access(cp, -1, 0, 0);
 	if (error != 0) {
+		G_RAID3_DEBUG(1, "Cannot read metadata from %s (error=%d).",
+		    cp->provider->name, error);
 		if (buf != NULL)
 			g_free(buf);
 		return (error);
@@ -2503,6 +2573,12 @@ g_raid3_read_metadata(struct g_consumer *cp, struct g_raid3_metadata *md)
 	g_free(buf);
 	if (strcmp(md->md_magic, G_RAID3_MAGIC) != 0)
 		return (EINVAL);
+	if (md->md_version > G_RAID3_VERSION) {
+		G_RAID3_DEBUG(0,
+		    "Kernel module is too old to handle metadata from %s.",
+		    cp->provider->name);
+		return (EINVAL);
+	}
 	if (error != 0) {
 		G_RAID3_DEBUG(1, "MD5 metadata hash mismatch for provider %s.",
 		    cp->provider->name);
@@ -2606,12 +2682,25 @@ g_raid3_add_disk(struct g_raid3_softc *sc, struct g_provider *pp,
 	error = g_raid3_check_metadata(sc, pp, md);
 	if (error != 0)
 		return (error);
+	if (sc->sc_state != G_RAID3_DEVICE_STATE_STARTING &&
+	    md->md_genid < sc->sc_genid) {
+		G_RAID3_DEBUG(0, "Component %s (device %s) broken, skipping.",
+		    pp->name, sc->sc_name);
+		return (EINVAL);
+	}
 	disk = g_raid3_init_disk(sc, pp, md, &error);
 	if (disk == NULL)
 		return (error);
 	error = g_raid3_event_send(disk, G_RAID3_DISK_STATE_NEW,
 	    G_RAID3_EVENT_WAIT);
-	return (error);
+	if (error != 0)
+		return (error);
+	if (md->md_version < G_RAID3_VERSION) {
+		G_RAID3_DEBUG(0, "Upgrading metadata on %s (v%d->v%d).",
+		    pp->name, md->md_version, G_RAID3_VERSION);
+		g_raid3_update_metadata(disk);
+	}
+	return (0);
 }
 
 static int
@@ -2694,7 +2783,7 @@ g_raid3_create(struct g_class *mp, const struct g_raid3_metadata *md)
 	sc->sc_ndisks = md->md_all;
 	sc->sc_round_robin = 0;
 	sc->sc_flags = md->md_mflags;
-	sc->sc_bump_syncid = 0;
+	sc->sc_bump_id = 0;
 	sc->sc_idle = 0;
 	for (n = 0; n < sc->sc_ndisks; n++) {
 		sc->sc_disks[n].d_softc = sc;
@@ -2828,11 +2917,6 @@ g_raid3_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 		return (NULL);
 	gp = NULL;
 
-	if (md.md_version > G_RAID3_VERSION) {
-		printf("geom_raid3.ko module is too old to handle %s.\n",
-		    pp->name);
-		return (NULL);
-	}
 	if (md.md_provider[0] != '\0' && strcmp(md.md_provider, pp->name) != 0)
 		return (NULL);
 	if (g_raid3_debug >= 2)
@@ -2931,6 +3015,7 @@ g_raid3_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 		}
 		sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent,
 		    disk->d_sync.ds_syncid);
+		sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, disk->d_genid);
 		sbuf_printf(sb, "%s<Flags>", indent);
 		if (disk->d_flags == 0)
 			sbuf_printf(sb, "NONE");
@@ -2959,6 +3044,7 @@ g_raid3_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 	} else {
 		sbuf_printf(sb, "%s<ID>%u</ID>\n", indent, (u_int)sc->sc_id);
 		sbuf_printf(sb, "%s<SyncID>%u</SyncID>\n", indent, sc->sc_syncid);
+		sbuf_printf(sb, "%s<GenID>%u</GenID>\n", indent, sc->sc_genid);
 		sbuf_printf(sb, "%s<Flags>", indent);
 		if (sc->sc_flags == 0)
 			sbuf_printf(sb, "NONE");
diff --git a/sys/geom/raid3/g_raid3.h b/sys/geom/raid3/g_raid3.h
index 1eb6e87..ffb463c 100644
--- a/sys/geom/raid3/g_raid3.h
+++ b/sys/geom/raid3/g_raid3.h
@@ -40,8 +40,9 @@
  * 0 - Initial version number.
  * 1 - Added 'round-robin reading' algorithm.
  * 2 - Added 'verify reading' algorithm.
+ * 3 - Added md_genid field to metadata.
  */
-#define	G_RAID3_VERSION		2
+#define	G_RAID3_VERSION		3
 
 #define	G_RAID3_DISK_FLAG_DIRTY		0x0000000000000001ULL
 #define	G_RAID3_DISK_FLAG_SYNCHRONIZING	0x0000000000000002ULL
@@ -136,6 +137,7 @@ struct g_raid3_disk {
 	struct g_raid3_softc *d_softc;	/* Back-pointer to softc. */
 	int		 d_state;	/* Disk state. */
 	uint64_t	 d_flags;	/* Additional flags. */
+	u_int		 d_genid;	/* Disk's generation ID. */
 	struct g_raid3_disk_sync d_sync; /* Sync information. */
 	LIST_ENTRY(g_raid3_disk) d_next;
 };
@@ -160,8 +162,15 @@ struct g_raid3_event {
 #define	G_RAID3_DEVICE_STATE_DEGRADED		1
 #define	G_RAID3_DEVICE_STATE_COMPLETE		2
 
-#define	G_RAID3_BUMP_ON_FIRST_WRITE		1
-#define	G_RAID3_BUMP_IMMEDIATELY		2
+/* Bump syncid on first write. */
+#define	G_RAID3_BUMP_SYNCID_OFW	0x1
+/* Bump syncid immediately. */
+#define	G_RAID3_BUMP_SYNCID_IMM	0x2
+#define	G_RAID3_BUMP_SYNCID	(G_RAID3_BUMP_SYNCID_OFW | \
+				 G_RAID3_BUMP_SYNCID_IMM)
+/* Bump genid immediately. */
+#define	G_RAID3_BUMP_GENID_IMM	0x4
+#define	G_RAID3_BUMP_GENID	(G_RAID3_BUMP_GENID_IMM)
 
 struct g_raid3_softc {
 	u_int		sc_state;	/* Device state. */
@@ -187,8 +196,9 @@ struct g_raid3_softc {
 	uma_zone_t	sc_zone_16k;
 	uma_zone_t	sc_zone_4k;
 
+	u_int		sc_genid;	/* Generation ID. */
 	u_int		sc_syncid;	/* Synchronization ID. */
-	int		sc_bump_syncid;
+	int		sc_bump_id;
 	struct g_raid3_device_sync sc_sync;
 	int		sc_idle;	/* DIRTY flags removed. */
 
@@ -219,6 +229,7 @@ struct g_raid3_metadata {
 	uint32_t	md_id;		/* Device unique ID. */
 	uint16_t	md_no;		/* Component number. */
 	uint16_t	md_all;		/* Number of disks in device. */
+	uint32_t	md_genid;	/* Generation ID. */
 	uint32_t	md_syncid;	/* Synchronization ID. */
 	uint64_t	md_mediasize;	/* Size of whole device. */
 	uint32_t	md_sectorsize;	/* Sector size. */
@@ -239,25 +250,24 @@ raid3_metadata_encode(struct g_raid3_metadata *md, u_char *data)
 	le32enc(data + 36, md->md_id);
 	le16enc(data + 40, md->md_no);
 	le16enc(data + 42, md->md_all);
-	le32enc(data + 44, md->md_syncid);
-	le64enc(data + 48, md->md_mediasize);
-	le32enc(data + 56, md->md_sectorsize);
-	le64enc(data + 60, md->md_sync_offset);
-	le64enc(data + 68, md->md_mflags);
-	le64enc(data + 76, md->md_dflags);
-	bcopy(md->md_provider, data + 84, 16);
+	le32enc(data + 44, md->md_genid);
+	le32enc(data + 48, md->md_syncid);
+	le64enc(data + 52, md->md_mediasize);
+	le32enc(data + 60, md->md_sectorsize);
+	le64enc(data + 64, md->md_sync_offset);
+	le64enc(data + 72, md->md_mflags);
+	le64enc(data + 80, md->md_dflags);
+	bcopy(md->md_provider, data + 88, 16);
 	MD5Init(&ctx);
-	MD5Update(&ctx, data, 100);
+	MD5Update(&ctx, data, 104);
 	MD5Final(md->md_hash, &ctx);
-	bcopy(md->md_hash, data + 100, 16);
+	bcopy(md->md_hash, data + 104, 16);
 }
 static __inline int
-raid3_metadata_decode(const u_char *data, struct g_raid3_metadata *md)
+raid3_metadata_decode_v0v1v2(const u_char *data, struct g_raid3_metadata *md)
 {
 	MD5_CTX ctx;
 
-	bcopy(data, md->md_magic, 16);
-	md->md_version = le32dec(data + 16);
 	bcopy(data + 20, md->md_name, 16);
 	md->md_id = le32dec(data + 36);
 	md->md_no = le16dec(data + 40);
@@ -277,6 +287,53 @@ raid3_metadata_decode(const u_char *data, struct g_raid3_metadata *md)
 		return (EINVAL);
 	return (0);
 }
+static __inline int
+raid3_metadata_decode_v3(const u_char *data, struct g_raid3_metadata *md)
+{
+	MD5_CTX ctx;
+
+	bcopy(data + 20, md->md_name, 16);
+	md->md_id = le32dec(data + 36);
+	md->md_no = le16dec(data + 40);
+	md->md_all = le16dec(data + 42);
+	md->md_genid = le32dec(data + 44);
+	md->md_syncid = le32dec(data + 48);
+	md->md_mediasize = le64dec(data + 52);
+	md->md_sectorsize = le32dec(data + 60);
+	md->md_sync_offset = le64dec(data + 64);
+	md->md_mflags = le64dec(data + 72);
+	md->md_dflags = le64dec(data + 80);
+	bcopy(data + 88, md->md_provider, 16);
+	bcopy(data + 104, md->md_hash, 16);
+	MD5Init(&ctx);
+	MD5Update(&ctx, data, 104);
+	MD5Final(md->md_hash, &ctx);
+	if (bcmp(md->md_hash, data + 104, 16) != 0)
+		return (EINVAL);
+	return (0);
+}
+static __inline int
+raid3_metadata_decode(const u_char *data, struct g_raid3_metadata *md)
+{
+	int error;
+ 
+	bcopy(data, md->md_magic, 16);
+	md->md_version = le32dec(data + 16);
+	switch (md->md_version) {
+	case 0:
+	case 1:
+	case 2:
+		error = raid3_metadata_decode_v0v1v2(data, md);
+		break;
+	case 3:
+		error = raid3_metadata_decode_v3(data, md);
+		break;
+	default:
+		error = EINVAL;
+		break;
+	}
+	return (error);
+}
 
 static __inline void
 raid3_metadata_dump(const struct g_raid3_metadata *md)
@@ -291,6 +348,7 @@ raid3_metadata_dump(const struct g_raid3_metadata *md)
 	printf("        id: %u\n", (u_int)md->md_id);
 	printf("        no: %u\n", (u_int)md->md_no);
 	printf("       all: %u\n", (u_int)md->md_all);
+	printf("     genid: %u\n", (u_int)md->md_genid);
 	printf("    syncid: %u\n", (u_int)md->md_syncid);
 	printf(" mediasize: %jd\n", (intmax_t)md->md_mediasize);
 	printf("sectorsize: %u\n", (u_int)md->md_sectorsize);
diff --git a/sys/geom/raid3/g_raid3_ctl.c b/sys/geom/raid3/g_raid3_ctl.c
index 94ec67e..41d2fcf 100644
--- a/sys/geom/raid3/g_raid3_ctl.c
+++ b/sys/geom/raid3/g_raid3_ctl.c
@@ -507,10 +507,11 @@ g_raid3_ctl_remove(struct gctl_req *req, struct g_class *mp)
 		if (g_raid3_clear_metadata(disk) != 0) {
 			gctl_error(req, "Cannot clear metadata on %s.",
 			    g_raid3_get_diskname(disk));
-			sc->sc_bump_syncid = G_RAID3_BUMP_IMMEDIATELY;
+		} else {
+			g_raid3_event_send(disk,
+			    G_RAID3_DISK_STATE_DISCONNECTED,
+			    G_RAID3_EVENT_WAIT);
 		}
-		g_raid3_event_send(disk, G_RAID3_DISK_STATE_DISCONNECTED,
-		    G_RAID3_EVENT_WAIT);
 		break;
 	case G_RAID3_DISK_STATE_NODISK:
 		break;