Introduce and use delayed-destruction functionality from a pre-sync hook,

which means that devices will be destroyed on last close.

This fixes destruction order problems when, eg. RAID3 array is build on
top of RAID1 arrays.

Requested, reviewed and tested by:	ru
MFC after:	2 weeks

3 files changed, 82 insertions, 55 deletions

diff --git a/sys/geom/raid3/g_raid3.c b/sys/geom/raid3/g_raid3.c
index 1cc2325..55687c6 100644
--- a/sys/geom/raid3/g_raid3.c
+++ b/sys/geom/raid3/g_raid3.c
@@ -97,7 +97,7 @@ SYSCTL_UINT(_kern_geom_raid3_stat, OID_AUTO, parity_mismatch, CTLFLAG_RD,
 	G_RAID3_DEBUG(4, "%s: Woken up %p.", __func__, (ident));	\
 } while (0)
 
-static eventhandler_tag g_raid3_pre_sync = NULL, g_raid3_post_sync = NULL;
+static eventhandler_tag g_raid3_pre_sync = NULL;
 
 static int g_raid3_destroy_geom(struct gctl_req *req, struct g_class *mp,
     struct g_geom *gp);
@@ -2896,11 +2896,37 @@ g_raid3_add_disk(struct g_raid3_softc *sc, struct g_provider *pp,
 	return (0);
 }
 
+static void
+g_raid3_destroy_delayed(void *arg, int flag)
+{
+	struct g_raid3_softc *sc;
+	int error;
+
+	if (flag == EV_CANCEL) {
+		G_RAID3_DEBUG(1, "Destroying canceled.");
+		return;
+	}
+	sc = arg;
+	g_topology_unlock();
+	sx_xlock(&sc->sc_lock);
+	KASSERT((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) == 0,
+	    ("DESTROY flag set on %s.", sc->sc_name));
+	KASSERT((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROYING) != 0,
+	    ("DESTROYING flag not set on %s.", sc->sc_name));
+	G_RAID3_DEBUG(0, "Destroying %s (delayed).", sc->sc_name);
+	error = g_raid3_destroy(sc, G_RAID3_DESTROY_SOFT);
+	if (error != 0) {
+		G_RAID3_DEBUG(0, "Cannot destroy %s.", sc->sc_name);
+		sx_xunlock(&sc->sc_lock);
+	}
+	g_topology_lock();
+}
+
 static int
 g_raid3_access(struct g_provider *pp, int acr, int acw, int ace)
 {
 	struct g_raid3_softc *sc;
-	int dcr, dcw, dce, error;
+	int dcr, dcw, dce, error = 0;
 
 	g_topology_assert();
 	G_RAID3_DEBUG(2, "Access request for %s: r%dw%de%d.", pp->name, acr,
@@ -2910,17 +2936,12 @@ g_raid3_access(struct g_provider *pp, int acr, int acw, int ace)
 	dcw = pp->acw + acw;
 	dce = pp->ace + ace;
 
-	error = 0;
 	sc = pp->geom->softc;
-	if (sc != NULL) {
-		if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) != 0)
-			sc = NULL;
-		else {
-			g_topology_unlock();
-			sx_xlock(&sc->sc_lock);
-		}
-	}
-	if (sc == NULL ||
+	KASSERT(sc != NULL, ("NULL softc (provider=%s).", pp->name));
+
+	g_topology_unlock();
+	sx_xlock(&sc->sc_lock);
+	if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROY) != 0 ||
 	    g_raid3_ndisks(sc, G_RAID3_DISK_STATE_ACTIVE) < sc->sc_ndisks - 1) {
 		if (acr > 0 || acw > 0 || ace > 0)
 			error = ENXIO;
@@ -2928,11 +2949,19 @@ g_raid3_access(struct g_provider *pp, int acr, int acw, int ace)
 	}
 	if (dcw == 0 && !sc->sc_idle)
 		g_raid3_idle(sc, dcw);
-end:
-	if (sc != NULL) {
-		sx_xunlock(&sc->sc_lock);
-		g_topology_lock();
+	if ((sc->sc_flags & G_RAID3_DEVICE_FLAG_DESTROYING) != 0) {
+		if (acr > 0 || acw > 0 || ace > 0) {
+			error = ENXIO;
+			goto end;
+		}
+		if (dcr == 0 && dcw == 0 && dce == 0) {
+			g_post_event(g_raid3_destroy_delayed, sc, M_WAITOK,
+			    sc, NULL);
+		}
 	}
+end:
+	sx_xunlock(&sc->sc_lock);
+	g_topology_lock();
 	return (error);
 }
 
@@ -3049,7 +3078,7 @@ g_raid3_create(struct g_class *mp, const struct g_raid3_metadata *md)
 }
 
 int
-g_raid3_destroy(struct g_raid3_softc *sc, boolean_t force)
+g_raid3_destroy(struct g_raid3_softc *sc, int how)
 {
 	struct g_provider *pp;
 
@@ -3060,14 +3089,24 @@ g_raid3_destroy(struct g_raid3_softc *sc, boolean_t force)
 
 	pp = sc->sc_provider;
 	if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
-		if (force) {
-			G_RAID3_DEBUG(1, "Device %s is still open, so it "
-			    "can't be definitely removed.", pp->name);
-		} else {
+		switch (how) {
+		case G_RAID3_DESTROY_SOFT:
 			G_RAID3_DEBUG(1,
 			    "Device %s is still open (r%dw%de%d).", pp->name,
 			    pp->acr, pp->acw, pp->ace);
 			return (EBUSY);
+		case G_RAID3_DESTROY_DELAYED:
+			G_RAID3_DEBUG(1,
+			    "Device %s will be destroyed on last close.",
+			    pp->name);
+			if (sc->sc_syncdisk != NULL)
+				g_raid3_sync_stop(sc, 1);
+			sc->sc_flags |= G_RAID3_DEVICE_FLAG_DESTROYING;
+			return (EBUSY);
+		case G_RAID3_DESTROY_HARD:
+			G_RAID3_DEBUG(1, "Device %s is still open, so it "
+			    "can't be definitely removed.", pp->name);
+			break;
 		}
 	}
 
@@ -3168,6 +3207,7 @@ g_raid3_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 		    pp->name, gp->name, error);
 		if (g_raid3_ndisks(sc, G_RAID3_DISK_STATE_NODISK) ==
 		    sc->sc_ndisks) {
+			g_cancel_event(sc);
 			g_raid3_destroy(sc, 1);
 			g_topology_lock();
 			return (NULL);
@@ -3189,6 +3229,7 @@ g_raid3_destroy_geom(struct gctl_req *req __unused, struct g_class *mp __unused,
 	g_topology_unlock();
 	sc = gp->softc;
 	sx_xlock(&sc->sc_lock);
+	g_cancel_event(sc);
 	error = g_raid3_destroy(gp->softc, 0);
 	if (error != 0)
 		sx_xunlock(&sc->sc_lock);
@@ -3325,6 +3366,7 @@ g_raid3_shutdown_pre_sync(void *arg, int howto)
 	struct g_class *mp;
 	struct g_geom *gp, *gp2;
 	struct g_raid3_softc *sc;
+	int error;
 
 	mp = arg;
 	DROP_GIANT();
@@ -3332,40 +3374,19 @@ g_raid3_shutdown_pre_sync(void *arg, int howto)
 	LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
 		if ((sc = gp->softc) == NULL)
 			continue;
-		g_topology_unlock();
-		sx_xlock(&sc->sc_lock);
-		if (sc->sc_syncdisk != NULL)
-			g_raid3_sync_stop(sc, 1);
-		sx_xunlock(&sc->sc_lock);
-		g_topology_lock();
-	}
-	g_topology_unlock();
-	PICKUP_GIANT();
-}
-
-static void
-g_raid3_shutdown_post_sync(void *arg, int howto)
-{
-	struct g_class *mp;
-	struct g_geom *gp, *gp2;
-	struct g_raid3_softc *sc;
-
-	mp = arg;
-	DROP_GIANT();
-	g_topology_lock();
-	LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
-		if ((sc = gp->softc) == NULL)
+		/* Skip synchronization geom. */
+		if (gp == sc->sc_sync.ds_geom)
 			continue;
 		g_topology_unlock();
 		sx_xlock(&sc->sc_lock);
-		g_raid3_destroy(sc, 1);
+		g_cancel_event(sc);
+		error = g_raid3_destroy(sc, G_RAID3_DESTROY_DELAYED);
+		if (error != 0)
+			sx_xunlock(&sc->sc_lock);
 		g_topology_lock();
 	}
 	g_topology_unlock();
 	PICKUP_GIANT();
-#if 0
-	tsleep(&gp, PRIBIO, "r3:shutdown", hz * 20);
-#endif
 }
 
 static void
@@ -3374,9 +3395,7 @@ g_raid3_init(struct g_class *mp)
 
 	g_raid3_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync,
 	    g_raid3_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST);
-	g_raid3_post_sync = EVENTHANDLER_REGISTER(shutdown_post_sync,
-	    g_raid3_shutdown_post_sync, mp, SHUTDOWN_PRI_FIRST);
-	if (g_raid3_pre_sync == NULL || g_raid3_post_sync == NULL)
+	if (g_raid3_pre_sync == NULL)
 		G_RAID3_DEBUG(0, "Warning! Cannot register shutdown event.");
 }
 
@@ -3386,8 +3405,6 @@ g_raid3_fini(struct g_class *mp)
 
 	if (g_raid3_pre_sync != NULL)
 		EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_raid3_pre_sync);
-	if (g_raid3_post_sync != NULL)
-		EVENTHANDLER_DEREGISTER(shutdown_post_sync, g_raid3_post_sync);
 }
 
 DECLARE_GEOM_CLASS(g_raid3_class, g_raid3);
diff --git a/sys/geom/raid3/g_raid3.h b/sys/geom/raid3/g_raid3.h
index c6e6b90..1032142 100644
--- a/sys/geom/raid3/g_raid3.h
+++ b/sys/geom/raid3/g_raid3.h
@@ -160,6 +160,7 @@ struct g_raid3_event {
 
 #define	G_RAID3_DEVICE_FLAG_DESTROY	0x0100000000000000ULL
 #define	G_RAID3_DEVICE_FLAG_WAIT	0x0200000000000000ULL
+#define	G_RAID3_DEVICE_FLAG_DESTROYING	0x0400000000000000ULL
 
 #define	G_RAID3_DEVICE_STATE_STARTING		0
 #define	G_RAID3_DEVICE_STATE_DEGRADED		1
@@ -243,7 +244,10 @@ struct g_raid3_softc {
 
 const char *g_raid3_get_diskname(struct g_raid3_disk *disk);
 u_int g_raid3_ndisks(struct g_raid3_softc *sc, int state);
-int g_raid3_destroy(struct g_raid3_softc *sc, boolean_t force);
+#define	G_RAID3_DESTROY_SOFT	0
+#define	G_RAID3_DESTROY_DELAYED	1
+#define	G_RAID3_DESTROY_HARD	2
+int g_raid3_destroy(struct g_raid3_softc *sc, int how);
 int g_raid3_event_send(void *arg, int state, int flags);
 struct g_raid3_metadata;
 int g_raid3_add_disk(struct g_raid3_softc *sc, struct g_provider *pp,
diff --git a/sys/geom/raid3/g_raid3_ctl.c b/sys/geom/raid3/g_raid3_ctl.c
index fe23fb5..ef55741 100644
--- a/sys/geom/raid3/g_raid3_ctl.c
+++ b/sys/geom/raid3/g_raid3_ctl.c
@@ -310,6 +310,7 @@ g_raid3_ctl_stop(struct gctl_req *req, struct g_class *mp)
 	const char *name;
 	char param[16];
 	u_int i;
+	int how;
 
 	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
 	if (nargs == NULL) {
@@ -325,6 +326,10 @@ g_raid3_ctl_stop(struct gctl_req *req, struct g_class *mp)
 		gctl_error(req, "No '%s' argument.", "force");
 		return;
 	}
+	if (*force)
+		how = G_RAID3_DESTROY_HARD;
+	else
+		how = G_RAID3_DESTROY_SOFT;
 
 	for (i = 0; i < (u_int)*nargs; i++) {
 		snprintf(param, sizeof(param), "arg%u", i);
@@ -338,7 +343,8 @@ g_raid3_ctl_stop(struct gctl_req *req, struct g_class *mp)
 			gctl_error(req, "No such device: %s.", name);
 			return;
 		}
-		error = g_raid3_destroy(sc, *force);
+		g_cancel_event(sc);
+		error = g_raid3_destroy(sc, how);
 		if (error != 0) {
 			gctl_error(req, "Cannot destroy device %s (error=%d).",
 			    sc->sc_geom->name, error);