Add 'initialized' state for subdisks. After initializing, the subdisk

goes into initialized state, not 'up'.  This makes it easier to ensure
consistency in multi-plex volumes.

update_plex_state: redo transitions from empty and initialized
subdisks to up or reviving, depending on the number of plexes.

Reported-by:	Bernd Walter <ticso@cicely.de>
		Remy Nonnenmacher <remy@synx.com>

5 files changed, 163 insertions, 94 deletions

diff --git a/sys/dev/vinum/statetexts.h b/sys/dev/vinum/statetexts.h
index c932c93..7c12804 100644
--- a/sys/dev/vinum/statetexts.h
+++ b/sys/dev/vinum/statetexts.h
@@ -20,7 +20,7 @@
  * 4. Neither the name of the Company nor the names of its contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
- *  
+ *
  * This software is provided ``as is'', and any express or implied
  * warranties, including, but not limited to, the implied warranties of
  * merchantability and fitness for a particular purpose are disclaimed.
@@ -35,7 +35,7 @@
  *
  */
 
-/* Created by ./makestatetext on Thu Mar 25 12:50:46 CST 1999. Do not edit */
+/* Created by ./makestatetext on Sun Aug 15 10:34:15 CST 1999. Do not edit */
 
 /* Drive state texts */
 char *drivestatetext[] =
@@ -55,6 +55,7 @@ char *sdstatetext[] =
     "init",
     "empty",
     "initializing",
+    "initialized",
     "obsolete",
     "stale",
     "crashed",
diff --git a/sys/dev/vinum/vinumio.h b/sys/dev/vinum/vinumio.h
index 2a02f15..c3bd528 100644
--- a/sys/dev/vinum/vinumio.h
+++ b/sys/dev/vinum/vinumio.h
@@ -20,7 +20,7 @@
  * 4. Neither the name of the Company nor the names of its contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
- *  
+ *
  * This software is provided ``as is'', and any express or implied
  * warranties, including, but not limited to, the implied warranties of
  * merchantability and fitness for a particular purpose are disclaimed.
@@ -92,11 +92,12 @@ enum objecttype {
 /*
  * The state to set with VINUM_SETSTATE.  Since
  * each object has a different set of states, we
- * need to translate later 
+ * need to translate later
  */
 enum objectstate {
     object_down,
     object_initializing,
+    object_initialized,
     object_up
 };
 
diff --git a/sys/dev/vinum/vinumstate.c b/sys/dev/vinum/vinumstate.c
index 86587a1..18fe853 100644
--- a/sys/dev/vinum/vinumstate.c
+++ b/sys/dev/vinum/vinumstate.c
@@ -45,7 +45,7 @@
 
 /* Update drive state */
 /* Return 1 if the state changes, otherwise 0 */
-int 
+int
 set_drive_state(int driveno, enum drivestate newstate, enum setstateflags flags)
 {
     struct drive *drive = &DRIVE[driveno];
@@ -97,7 +97,7 @@ set_drive_state(int driveno, enum drivestate newstate, enum setstateflags flags)
  * if it could be dangerous to data consistency.  It shouldn't allow
  * nonsense.
  */
-int 
+int
 set_sd_state(int sdno, enum sdstate newstate, enum setstateflags flags)
 {
     struct sd *sd = &SD[sdno];
@@ -132,6 +132,12 @@ set_sd_state(int sdno, enum sdstate newstate, enum setstateflags flags)
 		return 0;				    /* don't do it */
 	    break;
 
+	case sd_initialized:
+	    if ((sd->state == sd_initializing)		    /* we were initializing */
+	    ||(flags & setstate_force))			    /* or we forced it */
+		break;
+	    return 0;					    /* can't do it otherwise */
+
 	case sd_up:
 	    if (DRIVE[sd->driveno].state != drive_up)	    /* can't bring the sd up if the drive isn't, */
 		return 0;				    /* not even by force */
@@ -144,19 +150,12 @@ set_sd_state(int sdno, enum sdstate newstate, enum setstateflags flags)
 		/*
 		 * If we're associated with a plex, and
 		 * the plex isn't up, or we're the only
-		 * subdisk in the plex, we can do it
+		 * subdisk in the plex, we can do it.
 		 */
 		if ((sd->plexno >= 0)
 		    && (((PLEX[sd->plexno].state < plex_firstup)
 			    || (PLEX[sd->plexno].subdisks > 1))))
 		    break;				    /* do it */
-		/*
-		 * XXX Get this right: make sure that other plexes in
-		 * the volume cover this address space, otherwise
-		 * we make this one sd_up.
-		 *
-		 * Do we even want this any more?
-		 */
 		if (oldstate != sd_reborn) {
 		    sd->state = sd_reborn;		    /* here it is again */
 		    log(LOG_INFO,
@@ -174,7 +173,6 @@ set_sd_state(int sdno, enum sdstate newstate, enum setstateflags flags)
 		/* otherwise it's like being empty */
 		/* FALLTHROUGH */
 
-	    case sd_initializing:
 	    case sd_empty:
 		/*
 		 * If we're associated with a plex which is down, or which is
@@ -227,11 +225,16 @@ set_sd_state(int sdno, enum sdstate newstate, enum setstateflags flags)
 		if (((vol == NULL)
 			|| (vol->plexes == 1))
 		    && ((plex->organization != plex_raid5)
-			|| (plex->sddowncount > 1)))
-		    return 0;				    /* can't do it */
-		sd->state = sd_reviving;		    /* put in reviving state */
-		sd->revived = 0;			    /* nothing done yet */
-		status = EAGAIN;			    /* need to repeat */
+			|| (plex->sddowncount > 1))) {
+		    if (sd->state == sd_initializing)	    /* it's finished initializing  */
+			sd->state = sd_initialized;
+		    else
+			return 0;			    /* can't do it */
+		} else {
+		    sd->state = sd_reviving;		    /* put in reviving state */
+		    sd->revived = 0;			    /* nothing done yet */
+		    status = EAGAIN;			    /* need to repeat */
+		}
 		break;
 
 	    case sd_reviving:
@@ -243,7 +246,7 @@ set_sd_state(int sdno, enum sdstate newstate, enum setstateflags flags)
 		/*
 		 * There's no way to bring subdisks up directly from
 		 * other states.  First they need to be initialized
-		 * or revived
+		 * or revived.
 		 */
 		return 0;
 	    }
@@ -280,7 +283,7 @@ set_sd_state(int sdno, enum sdstate newstate, enum setstateflags flags)
  * magnitude less complicated.  In particular, ignore
  * the requested state.
  */
-int 
+int
 set_plex_state(int plexno, enum plexstate state, enum setstateflags flags)
 {
     struct plex *plex;					    /* point to our plex */
@@ -363,7 +366,7 @@ set_plex_state(int plexno, enum plexstate state, enum setstateflags flags)
 }
 
 /* Update the state of a plex dependent on its plexes. */
-int 
+int
 set_volume_state(int volno, enum volumestate state, enum setstateflags flags)
 {
     struct volume *vol = &VOL[volno];			    /* point to our volume */
@@ -392,7 +395,7 @@ set_volume_state(int volno, enum volumestate state, enum setstateflags flags)
 }
 
 /* Set the state of a subdisk based on its environment */
-void 
+void
 update_sd_state(int sdno)
 {
     struct sd *sd;
@@ -440,7 +443,7 @@ update_sd_state(int sdno)
  * into an 'up' state.  This is a helper
  * for update_plex_state.
  */
-void 
+void
 forceup(int plexno)
 {
     struct plex *plex;
@@ -459,7 +462,7 @@ forceup(int plexno)
 }
 
 /* Set the state of a plex based on its environment */
-void 
+void
 update_plex_state(int plexno)
 {
     struct plex *plex;					    /* point to our plex */
@@ -475,32 +478,54 @@ update_plex_state(int plexno)
     if (statemap & sd_initstate)			    /* something initializing? */
 	plex->state = plex_initializing;		    /* yup, that makes the plex the same */
     if ((statemap == sd_emptystate)			    /* all subdisks empty */
-&&((vps & volplex_otherup) == 0)			    /* and no other plex is up */ &&((plex->organization == plex_concat) /* and we're not RAID-5 */
-    ||(plex->organization == plex_striped))) {
-	struct volume *vol = &VOL[plex->volno];		    /* possible volume to which it points */
-
-	/*
-	 * If we're a striped or concat plex associated with a
-	 * volume, none of whose plexes are up, and we're new and
-	 * untested, and the volume has the setupstate bit set, we
-	 * can pretend to be in a consistent state.
-	 *
-	 * We need to do this in one swell foop: on the next call
-	 * we will no longer be just empty.
-	 *
-	 * This code assumes that all the other plexes are also
-	 * capable of coming up (i.e. all the sds are up), but
-	 * that's OK: we'll come back to this function for the
-	 * remaining plexes in the volume.
-	 */
+    ||(statemap == sd_initializedstate)) {		    /* or all initialized */
+	if ((vps & volplex_otherup) == 0) {		    /*  no other plex is up */
+	    struct volume *vol = &VOL[plex->volno];	    /* possible volume to which it points */
 
-	if ((plex->state == plex_init)
-	    && (plex->volno >= 0)
-	    && (vol->flags & VF_CONFIG_SETUPSTATE)) {
-	    for (plexno = 0; plexno < vol->plexes; plexno++)
-		forceup(VOL[plex->volno].plex[plexno]);
-	} else
-	    forceup(plexno);				    /* we'll do it */
+	    /*
+	     * If we're a striped or concat plex
+	     * associated with a volume, none of whose
+	     * plexes are up, and we're new and untested,
+	     * and the volume has the setupstate bit set,
+	     * we can pretend to be in a consistent state.
+	     *
+	     * We need to do this in one swell foop: on
+	     * the next call we will no longer be just
+	     * empty.
+	     *
+	     * This code assumes that all the other plexes
+	     * are also capable of coming up (i.e. all the
+	     * sds are up), but that's OK: we'll come back
+	     * to this function for the remaining plexes
+	     * in the volume.
+	     */
+	    if ((plex->state == plex_init)
+		&& (plex->volno >= 0)
+		&& (vol->flags & VF_CONFIG_SETUPSTATE)) {
+		for (plexno = 0; plexno < vol->plexes; plexno++)
+		    forceup(VOL[plex->volno].plex[plexno]);
+	    } else if ((statemap == sd_initializedstate)    /* if it's initialized (not empty) */
+||(plex->organization == plex_concat)			    /* and we're not RAID-5 */
+	    ||(plex->organization == plex_striped))
+		forceup(plexno);			    /* we'll do it */
+	    /*
+	     * This leaves a case where things don't get
+	     * done: the plex is RAID-5, and the subdisks
+	     * are all empty.  They need to be initialized
+	     * first.
+	     */
+	} else {					    /* another plex is up */
+	    int sdno;
+
+	    plex->state = plex_faulty;			    /* and bring it up */
+	    /* change the subdisks to up state */
+	    for (sdno = 0; sdno < plex->subdisks; sdno++) {
+		SD[plex->sdnos[sdno]].state = sd_reviving;
+		log(LOG_INFO,				    /* tell them about it */
+		    "vinum: %s is reviving\n",
+		    SD[plex->sdnos[sdno]].name);
+	    }
+	}
     } else if (statemap == sd_upstate)
 	/*
 	 * All the subdisks are up.  This also means that
@@ -530,7 +555,7 @@ update_plex_state(int plexno)
 }
 
 /* Set volume state based on its components */
-void 
+void
 update_volume_state(int volno)
 {
     struct volume *vol;					    /* our volume */
@@ -565,7 +590,7 @@ update_volume_state(int volno)
  * don't want to stop when we hit a subdisk which
  * is down.  Return a separate indication instead.
  */
-enum requeststatus 
+enum requeststatus
 sddownstate(struct request *rq)
 {
     if (rq->bp->b_flags & B_READ)			    /* read operation? */
@@ -588,7 +613,7 @@ sddownstate(struct request *rq)
  * and use the subdisk state only.  The last version of
  * this file with the old logic was 2.7. XXX
  */
-enum requeststatus 
+enum requeststatus
 checksdstate(struct sd *sd, struct request *rq, daddr_t diskaddr, daddr_t diskend)
 {
     struct plex *plex = &PLEX[sd->plexno];
@@ -669,7 +694,7 @@ checksdstate(struct sd *sd, struct request *rq, daddr_t diskaddr, daddr_t disken
 }
 
 /* return a state map for the subdisks of a plex */
-enum sdstates 
+enum sdstates
 sdstatemap(struct plex *plex)
 {
     int sdno;
@@ -723,6 +748,11 @@ sdstatemap(struct plex *plex)
 	    (plex->sddowncount)++;			    /* another unusable subdisk */
 	    break;
 
+	case sd_initialized:
+	    statemap |= sd_initializedstate;
+	    (plex->sddowncount)++;			    /* another unusable subdisk */
+	    break;
+
 	case sd_unallocated:
 	case sd_uninit:
 	case sd_reviving:
@@ -735,7 +765,7 @@ sdstatemap(struct plex *plex)
 }
 
 /* determine the state of the volume relative to this plex */
-enum volplexstate 
+enum volplexstate
 vpstate(struct plex *plex)
 {
     struct volume *vol;
@@ -763,14 +793,14 @@ vpstate(struct plex *plex)
 /* Check if all bits b are set in a */
 int allset(int a, int b);
 
-int 
+int
 allset(int a, int b)
 {
     return (a & b) == b;
 }
 
 /* Invalidate the subdisks belonging to a plex */
-void 
+void
 invalidate_subdisks(struct plex *plex, enum sdstate state)
 {
     int sdno;
@@ -783,6 +813,7 @@ invalidate_subdisks(struct plex *plex, enum sdstate state)
 	case sd_uninit:
 	case sd_init:
 	case sd_initializing:
+	case sd_initialized:
 	case sd_empty:
 	case sd_obsolete:
 	case sd_stale:
@@ -804,7 +835,7 @@ invalidate_subdisks(struct plex *plex, enum sdstate state)
  * This is called from vinumioctl (VINUMSTART).
  * Return error indications via ioctl_reply
  */
-void 
+void
 start_object(struct vinum_ioctl_msg *data)
 {
     int status;
@@ -877,7 +908,7 @@ start_object(struct vinum_ioctl_msg *data)
  * This is called from vinumioctl (VINUMSTOP).
  * Return error indications via ioctl_reply.
  */
-void 
+void
 stop_object(struct vinum_ioctl_msg *data)
 {
     int status = 1;
@@ -917,7 +948,7 @@ stop_object(struct vinum_ioctl_msg *data)
  * VINUM_SETSTATE ioctl: set an object state
  * msg is the message passed by the user
  */
-void 
+void
 setstate(struct vinum_ioctl_msg *msg)
 {
     int sdno;
@@ -980,7 +1011,31 @@ setstate(struct vinum_ioctl_msg *msg)
 	}
 	break;
 
+    case object_initialized:
+	if (msg->type == sd_object) {
+	    sd = &SD[msg->index];
+	    if ((msg->index >= vinum_conf.subdisks_allocated)
+		|| (sd->state <= sd_referenced)) {
+		sprintf(ioctl_reply->msg, "Invalid subdisk %d", msg->index);
+		ioctl_reply->error = EFAULT;
+		return;
+	    }
+	    set_sd_state(msg->index, sd_initialized, msg->force);
+	    if (sd->state != sd_initializing) {
+		strcpy(ioctl_reply->msg, "Can't set state");
+		ioctl_reply->error = EBUSY;
+	    } else
+		ioctl_reply->error = 0;
+	} else {
+	    strcpy(ioctl_reply->msg, "Invalid object");
+	    ioctl_reply->error = EINVAL;
+	}
+	break;
+
     case object_up:
 	start_object(msg);
     }
 }
+/* Local Variables: */
+/* fill-column: 50 */
+/* End: */
diff --git a/sys/dev/vinum/vinumstate.h b/sys/dev/vinum/vinumstate.h
index 6f49d20..38efa31 100644
--- a/sys/dev/vinum/vinumstate.h
+++ b/sys/dev/vinum/vinumstate.h
@@ -33,13 +33,13 @@
  * otherwise) arising in any way out of the use of this software, even if
  * advised of the possibility of such damage.
  *
- * $Id: vinumstate.h,v 1.13 1999/02/28 04:58:47 grog Exp grog $
+ * $Id: vinumstate.h,v 1.14 1999/03/25 02:17:38 grog Exp grog $
  */
 
 /*
  * This file gets read by makestatetext to create text files
  * with the names of the states, so don't change the file
- * format 
+ * format
  */
 
 enum volumestate {
@@ -64,19 +64,19 @@ enum plexstate {
     plex_referenced,
     /*
      * The plex has been allocated, but there configuration
-     * is not complete 
+     * is not complete
      */
     plex_init,
 
     /*
      * A plex which has gone completely down because of
-     * I/O errors. 
+     * I/O errors.
      */
     plex_faulty,
 
     /*
      * A plex which has been taken down by the
-     * administrator. 
+     * administrator.
      */
     plex_down,
 
@@ -86,14 +86,14 @@ enum plexstate {
     /*
      * *** The remaining states represent plexes which are
      * at least partially up.  Keep these separate so that
-     * they can be checked more easily. 
+     * they can be checked more easily.
      */
 
     /*
      * A plex entry which is at least partially up.  Not
      * all subdisks are available, and an inconsistency
      * has occurred.  If no other plex is uncorrupted,
-     * the volume is no longer consistent. 
+     * the volume is no longer consistent.
      */
     plex_corrupt,
 
@@ -102,20 +102,20 @@ enum plexstate {
     /*
      * A RAID-5 plex entry which is accessible, but one
      * subdisk is down, requiring recovery for many
-     * I/O requests. 
+     * I/O requests.
      */
     plex_degraded,
 
     /*
      * A plex which is really up, but which has a reborn
      * subdisk which we don't completely trust, and
-     * which we don't want to read if we can avoid it 
+     * which we don't want to read if we can avoid it
      */
     plex_flaky,
 
     /*
      * A plex entry which is completely up.  All subdisks
-     * are up. 
+     * are up.
      */
     plex_up,
 
@@ -152,10 +152,17 @@ enum sdstate {
 
     /*
      * A subdisk entry which has been created completely and
-     * which is currently being initialized 
+     * which is currently being initialized
      */
     sd_initializing,
 
+    /*
+     * A subdisk entry which has been initialized,
+     * but which can't come up because it would
+     * cause inconsistencies.
+     */
+    sd_initialized,
+
     /* *** The following states represent invalid data */
     /*
      * A subdisk entry which has been created completely.
@@ -188,20 +195,20 @@ enum sdstate {
     /*
      * A subdisk entry which was up, which contained
      * valid data, and which was taken down by the
-     * administrator.  The data is valid. 
+     * administrator.  The data is valid.
      */
     sd_down,
 
     /*
      * *** This is invalid data (the subdisk previously had
      * a numerically lower state), but it is currently in the
-     * process of being revived.  We can write but not read. 
+     * process of being revived.  We can write but not read.
      */
     sd_reviving,
 
     /*
      * *** The following states represent accessible subdisks
-     * with valid data 
+     * with valid data
      */
 
     /*
@@ -215,7 +222,7 @@ enum sdstate {
      * covers this address space in the plex, we set its
      * state to sd_up under these circumstances, so this
      * status implies that there is another subdisk to
-     * fulfil the request. 
+     * fulfil the request.
      */
     sd_reborn,
 
@@ -244,3 +251,7 @@ enum drivestate {
 
     drive_laststate = drive_up				    /* last value, for table dimensions */
 };
+
+/* Local Variables: */
+/* fill-column: 50 */
+/* End: */
diff --git a/sys/dev/vinum/vinumvar.h b/sys/dev/vinum/vinumvar.h
index 288c34e..f844010 100644
--- a/sys/dev/vinum/vinumvar.h
+++ b/sys/dev/vinum/vinumvar.h
@@ -79,7 +79,7 @@ enum constants {
 
 /*
    * Shifts for the second half of raw plex and
-   * subdisk numbers 
+   * subdisk numbers
  */
     VINUM_RAWPLEX_SHIFT = 8,				    /* shift the second half this much */
     VINUM_RAWPLEX_WIDTH = 12,				    /* width of second half */
@@ -202,7 +202,7 @@ enum constants {
 struct devcode {
 /*
  * CARE.  These fields assume a big-endian word.  On a
- * little-endian system, they're the wrong way around 
+ * little-endian system, they're the wrong way around
  */
     unsigned volume:8;					    /* up to 256 volumes */
     unsigned major:8;					    /* this is where the major number fits */
@@ -217,7 +217,7 @@ struct devcode {
      VINUM_SUBDISK = 2,
      VINUM_DRIVE = 3,
      VINUM_SUPERDEV = 4,
-     VINUM_RAWPLEX = 5,                                       
+     VINUM_RAWPLEX = 5,
      VINUM_RAWSD = 6 */
     unsigned signbit:1;					    /* to make 32 bits */
 };
@@ -241,7 +241,7 @@ struct devcode {
 /*
  * Flags for all objects.  Most of them only apply to
  * specific objects, but we have space for all in any
- * 32 bit flags word. 
+ * 32 bit flags word.
  */
 enum objflags {
     VF_LOCKED = 1,					    /* somebody has locked access to this object */
@@ -338,7 +338,7 @@ enum {
  * hostname is 256 bytes long, but we don't need to shlep
  * multiple copies in vinum.  We use the host name just
  * to identify this system, and 32 bytes should be ample
- * for that purpose 
+ * for that purpose
  */
 
 struct vinum_label {
@@ -379,7 +379,7 @@ enum drive_label_info {
 /*
  * A drive corresponds to a disk slice.  We use a different term to show
  * the difference in usage: it doesn't have to be a slice, and could
- * theoretically be a complete, unpartitioned disk 
+ * theoretically be a complete, unpartitioned disk
  */
 
 struct drive {
@@ -427,7 +427,7 @@ struct sd {
      * plexoffset is the offset from the beginning of the
      * plex to the very first part of the subdisk, in
      * sectors.  For striped and RAID-5 plexes, only
-     * the first stripe is located at this offset 
+     * the first stripe is located at this offset
      */
     int64_t plexoffset;					    /* offset in plex */
     u_int64_t sectors;					    /* and length in sectors */
@@ -532,7 +532,7 @@ struct volume {
 /*
  * Table expansion.  Expand table, which contains oldcount
  * entries of type element, by increment entries, and change
- * oldcount accordingly 
+ * oldcount accordingly
  */
 #define EXPAND(table, element, oldcount, increment)         \
 {							    \
@@ -567,7 +567,7 @@ struct mc {
  * Bit 1: Other plexes in the volume are up
  * Bit 2: The current plex is up
  * Maybe they should be local to
- * state.c 
+ * state.c
  */
 enum volplexstate {
     volplex_onlyusdown = 0,				    /* 0: we're the only plex, and we're down */
@@ -583,21 +583,22 @@ enum volplexstate {
 /* state map for plex */
 enum sdstates {
     sd_emptystate = 1,
-    sd_downstate = 2,					    /* found an SD which is down */
-    sd_crashedstate = 4,				    /* found an SD which is crashed */
-    sd_obsoletestate = 8,				    /* found an SD which is obsolete */
-    sd_stalestate = 16,					    /* found an SD which is stale */
-    sd_rebornstate = 32,				    /* found an SD which is reborn */
-    sd_upstate = 64,					    /* found an SD which is up */
-    sd_initstate = 128,					    /* found an SD which is init */
-    sd_otherstate = 256					    /* found an SD in some other state */
+    sd_downstate = 2,					    /* SD is down */
+    sd_crashedstate = 4,				    /* SD is crashed */
+    sd_obsoletestate = 8,				    /* SD is obsolete */
+    sd_stalestate = 16,					    /* SD is stale */
+    sd_rebornstate = 32,				    /* SD is reborn */
+    sd_upstate = 64,					    /* SD is up */
+    sd_initstate = 128,					    /* SD is initializing */
+    sd_initializedstate = 256,				    /* SD is initialized */
+    sd_otherstate = 512,				    /* SD is in some other state */
 };
 
 /*
  * This is really just a parameter to pass to
  * set_<foo>_state, but since it needs to be known
  * in the external definitions, we need to define
- * it here 
+ * it here
  */
 enum setstateflags {
     setstate_none = 0,					    /* no flags */