From 84a309987ce847321ccbfac2c62a793218ddf771 Mon Sep 17 00:00:00 2001 From: phk Date: Wed, 18 Jun 2003 09:29:28 +0000 Subject: Add "GEOM_FOX", a class which detects and selects between multiple redundant paths to the same device. This class reacts to a label in the first sector of the device, which is created the following way: # "0123456789abcdef012345..." # "<----magic-----><-id-...> echo "GEOM::FOX someid" | dd of=/dev/da0 conv=sync NB: Since the fact that multiple disk devices are in fact the same device is not known to GEOM, the geom taste/spoil process cannot fully catch all corner cases and this module can therefore be confused if you do the right wrong things. NB: The disk level drivers need to do the right thing for this to be useful, and that is not by definition currently the case. --- sys/geom/geom_fox.c | 468 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 468 insertions(+) create mode 100644 sys/geom/geom_fox.c (limited to 'sys/geom/geom_fox.c') diff --git a/sys/geom/geom_fox.c b/sys/geom/geom_fox.c new file mode 100644 index 0000000..295840f --- /dev/null +++ b/sys/geom/geom_fox.c @@ -0,0 +1,468 @@ +/*- + * Copyright (c) 2003 Poul-Henning Kamp + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The names of the authors may not be used to endorse or promote + * products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + * + * This is a GEOM module for handling path selection for multi-path + * storage devices. It is named "fox" because it, like they, prefer + * to have multiple exits to choose from. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FOX_CLASS_NAME "FOX" +#define FOX_MAGIC "GEOM::FOX" + +struct g_fox_softc { + off_t mediasize; + u_int sectorsize; + TAILQ_HEAD(, bio) queue; + struct mtx lock; + u_char magic[16]; + struct g_consumer *path; + struct g_consumer *opath; + int waiting; + int cr, cw, ce; +}; + +/* + * This function is called whenever we need to select a new path. + */ +static void +g_fox_select_path(void *arg, int flag) +{ + struct g_geom *gp; + struct g_fox_softc *sc; + struct g_consumer *cp1; + struct bio *bp; + int error; + + g_topology_assert(); + if (flag == EV_CANCEL) + return; + gp = arg; + sc = gp->softc; + + if (sc->opath != NULL) { + /* + * First, close the old path entirely. + */ + printf("Closing old path (%s) on fox (%s)\n", + sc->opath->provider->name, gp->name); + + cp1 = LIST_NEXT(sc->opath, consumer); + + error = g_access_rel(sc->opath, -sc->cr, -sc->cw, -(sc->ce + 1)); + KASSERT(error == 0, ("Failed close of old path %d", error)); + + /* + * The attempt to reopen it with a exclusive count + */ + error = g_access_rel(sc->opath, 0, 0, 1); + if (error) { + /* + * Ok, ditch this consumer, we can't use it. + */ + printf("Drop old path (%s) on fox (%s)\n", + sc->opath->provider->name, gp->name); + g_detach(sc->opath); + g_destroy_consumer(sc->opath); + if (LIST_EMPTY(&gp->consumer)) { + /* No consumers left */ + g_wither_geom(gp, ENXIO); + for (;;) { + bp = TAILQ_FIRST(&sc->queue); + if (bp == NULL) + break; + TAILQ_REMOVE(&sc->queue, bp, bio_queue); + bp->bio_error = ENXIO; + g_std_done(bp); + } + return; + } + } else { + printf("Got e-bit on old path (%s) on fox (%s)\n", + sc->opath->provider->name, gp->name); + } + sc->opath = NULL; + } else { + cp1 = LIST_FIRST(&gp->consumer); + } + if (cp1 == NULL) + cp1 = LIST_FIRST(&gp->consumer); + printf("Open new path (%s) on fox (%s)\n", + cp1->provider->name, gp->name); + error = g_access_rel(cp1, sc->cr, sc->cw, sc->ce); + if (error) { + /* + * If we failed, we take another trip through here + */ + printf("Open new path (%s) on fox (%s) failed, reselect.\n", + cp1->provider->name, gp->name); + sc->opath = cp1; + g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL); + } else { + printf("Open new path (%s) on fox (%s) succeeded\n", + cp1->provider->name, gp->name); + mtx_lock(&sc->lock); + sc->path = cp1; + sc->waiting = 0; + for (;;) { + bp = TAILQ_FIRST(&sc->queue); + if (bp == NULL) + break; + TAILQ_REMOVE(&sc->queue, bp, bio_queue); + g_io_request(bp, sc->path); + } + mtx_unlock(&sc->lock); + } +} + +static void +g_fox_orphan(struct g_consumer *cp) +{ + struct g_geom *gp; + struct g_fox_softc *sc; + int error, mark; + + g_topology_assert(); + gp = cp->geom; + sc = gp->softc; + printf("Removing path (%s) from fox (%s)\n", + cp->provider->name, gp->name); + mtx_lock(&sc->lock); + if (cp == sc->path) { + sc->opath = NULL; + sc->path = NULL; + sc->waiting = 1; + mark = 1; + } else { + mark = 0; + } + mtx_unlock(&sc->lock); + + g_access_rel(cp, -cp->acr, -cp->acw, -cp->ace); + error = cp->provider->error; + g_detach(cp); + g_destroy_consumer(cp); + if (!LIST_EMPTY(&gp->consumer)) { + if (mark) + g_post_event(g_fox_select_path, gp, M_WAITOK, gp, NULL); + return; + } + + mtx_destroy(&sc->lock); + gp->softc = NULL; + g_free(gp->softc); + g_wither_geom(gp, ENXIO); +} + +static void +g_fox_done(struct bio *bp) +{ + struct g_geom *gp; + struct g_fox_softc *sc; + int error; + + if (bp->bio_error == 0) { + g_std_done(bp); + return; + } + gp = bp->bio_from->geom; + sc = gp->softc; + if (bp->bio_from != sc->path) { + g_io_request(bp, sc->path); + return; + } + mtx_lock(&sc->lock); + sc->opath = sc->path; + sc->path = NULL; + error = g_post_event(g_fox_select_path, gp, M_NOWAIT, gp, NULL); + if (error) { + bp->bio_error = ENOMEM; + g_std_done(bp); + } else { + sc->waiting = 1; + TAILQ_INSERT_TAIL(&sc->queue, bp, bio_queue); + } + mtx_unlock(&sc->lock); +} + +static void +g_fox_start(struct bio *bp) +{ + struct g_geom *gp; + struct bio *bp2; + struct g_fox_softc *sc; + int error; + + gp = bp->bio_to->geom; + sc = gp->softc; + if (sc == NULL) { + g_io_deliver(bp, ENXIO); + return; + } + switch(bp->bio_cmd) { + case BIO_READ: + case BIO_WRITE: + case BIO_DELETE: + bp2 = g_clone_bio(bp); + if (bp2 == NULL) { + g_io_deliver(bp, ENOMEM); + break; + } + bp2->bio_offset += sc->sectorsize; + bp2->bio_done = g_fox_done; + mtx_lock(&sc->lock); + if (sc->path == NULL || !TAILQ_EMPTY(&sc->queue)) { + if (sc->waiting == 0) { + error = g_post_event(g_fox_select_path, gp, + M_NOWAIT, gp, NULL); + if (error) { + g_destroy_bio(bp2); + bp2 = NULL; + g_io_deliver(bp, error); + } else { + sc->waiting = 1; + } + } + if (bp2 != NULL) + TAILQ_INSERT_TAIL(&sc->queue, bp2, + bio_queue); + } else { + g_io_request(bp2, sc->path); + } + mtx_unlock(&sc->lock); + break; + default: + g_io_deliver(bp, EOPNOTSUPP); + break; + } + return; +} + +static int +g_fox_access(struct g_provider *pp, int dr, int dw, int de) +{ + struct g_geom *gp; + struct g_fox_softc *sc; + struct g_consumer *cp1; + int error; + + g_topology_assert(); + gp = pp->geom; + sc = gp->softc; + if (sc == NULL) + return (ENXIO); + + if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) { + /* + * First open, open all consumers with an exclusive bit + */ + error = 0; + LIST_FOREACH(cp1, &gp->consumer, consumer) { + error = g_access_rel(cp1, 0, 0, 1); + if (error) { + printf("FOX: access(%s,0,0,1) = %d\n", + cp1->provider->name, error); + break; + } + } + if (error) { + LIST_FOREACH(cp1, &gp->consumer, consumer) { + if (cp1->ace) + g_access_rel(cp1, 0, 0, -1); + } + return (error); + } + } + if (sc->path == NULL) + g_fox_select_path(gp, 0); + if (sc->path == NULL) + error = ENXIO; + else + error = g_access_rel(sc->path, dr, dw, de); + if (error == 0) { + sc->cr += dr; + sc->cw += dw; + sc->ce += de; + if (sc->cr == 0 && sc->cw == 0 && sc->ce == 0) { + /* + * Last close, remove e-bit on all consumers + */ + LIST_FOREACH(cp1, &gp->consumer, consumer) + g_access_rel(cp1, 0, 0, -1); + } + } + return (error); +} + +static struct g_geom * +g_fox_taste(struct g_class *mp, struct g_provider *pp, int flags __unused) +{ + struct g_geom *gp, *gp2; + struct g_provider *pp2; + struct g_consumer *cp, *cp2; + struct g_fox_softc *sc, *sc2; + int error; + u_int sectorsize; + u_char *buf; + + g_trace(G_T_TOPOLOGY, "fox_taste(%s, %s)", mp->name, pp->name); + g_topology_assert(); + if (!strcmp(pp->geom->class->name, mp->name)) + return (NULL); + gp = g_new_geomf(mp, "%s.fox", pp->name); + gp->softc = g_malloc(sizeof(struct g_fox_softc), M_WAITOK | M_ZERO); + sc = gp->softc; + + gp->start = g_fox_start; + gp->spoiled = g_fox_orphan; + gp->orphan = g_fox_orphan; + gp->access= g_fox_access; + cp = g_new_consumer(gp); + g_attach(cp, pp); + error = g_access_rel(cp, 1, 0, 0); + if (error) { + g_free(sc); + g_detach(cp); + g_destroy_consumer(cp); + g_destroy_geom(gp); + return(NULL); + } + do { + sectorsize = cp->provider->sectorsize; + g_topology_unlock(); + buf = g_read_data(cp, 0, sectorsize, &error); + g_topology_lock(); + if (buf == NULL || error != 0) + break; + if (memcmp(buf, FOX_MAGIC, strlen(FOX_MAGIC))) + break; + + /* + * First we need to see if this a new path for an existing fox. + */ + LIST_FOREACH(gp2, &mp->geom, geom) { + sc2 = gp2->softc; + if (sc == NULL) + continue; + if (memcmp(buf + 16, sc2->magic, sizeof sc2->magic)) + continue; + break; + } + if (gp2 != NULL) { + /* + * It was. Create a new consumer for that fox, + * attach it, and if the fox is open, open this + * path with an exclusive count of one. + */ + printf("Adding path (%s) to fox (%s)\n", + pp->name, gp2->name); + cp2 = g_new_consumer(gp2); + g_attach(cp2, pp); + pp2 = LIST_FIRST(&gp2->provider); + if (pp2->acr > 0 || pp2->acw > 0 || pp2->ace > 0) { + error = g_access_rel(cp2, 0, 0, 1); + if (error) { + /* + * This is bad, or more likely, + * the user is doing something stupid + */ + printf( + "WARNING: New path (%s) to fox(%s) not added: %s\n%s", + cp->provider->name, gp2->name, + "Could not get exclusive bit.", + "WARNING: This indicates a risk of data inconsistency." + ); + g_detach(cp2); + g_destroy_consumer(cp2); + } + } + break; + } + printf("Creating new fox (%s)\n", pp->name); + sc->path = cp; + memcpy(sc->magic, buf + 16, sizeof sc->magic); + pp2 = g_new_providerf(gp, "%s", gp->name); + pp2->mediasize = sc->mediasize = pp->mediasize - pp->sectorsize; + pp2->sectorsize = sc->sectorsize = pp->sectorsize; +printf("fox %s lock %p\n", gp->name, &sc->lock); + + mtx_init(&sc->lock, "fox queue", NULL, MTX_DEF); + TAILQ_INIT(&sc->queue); + g_error_provider(pp2, 0); + } while (0); + if (buf != NULL) + g_free(buf); + g_access_rel(cp, -1, 0, 0); + + if (!LIST_EMPTY(&gp->provider)) + return (gp); + + g_free(gp->softc); + g_detach(cp); + g_destroy_consumer(cp); + g_destroy_geom(gp); + return (NULL); +} + +static int +g_fox_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp) +{ + struct g_fox_softc *sc; + + g_topology_assert(); + sc = gp->softc; + gp->softc = NULL; + mtx_destroy(&sc->lock); + g_free(gp->softc); + g_wither_geom(gp, ENXIO); + return (0); +} + +static struct g_class g_fox_class = { + .name = FOX_CLASS_NAME, + .taste = g_fox_taste, + .destroy_geom = g_fox_destroy_geom, +}; + +DECLARE_GEOM_CLASS(g_fox_class, g_fox); -- cgit v1.1