From e257308b07ececec004a50897bcb88d5ae23203e Mon Sep 17 00:00:00 2001 From: le Date: Fri, 26 Nov 2004 12:01:00 +0000 Subject: Implement checkparity/rebuildparity. --- sys/geom/vinum/geom_vinum.c | 3 + sys/geom/vinum/geom_vinum.h | 1 + sys/geom/vinum/geom_vinum_init.c | 119 ++++++++++++++++++++++++++++++++++++++ sys/geom/vinum/geom_vinum_plex.c | 105 +++++++++++++++++++++++++++++---- sys/geom/vinum/geom_vinum_raid5.c | 97 ++++++++++++++++++++++++++++++- sys/geom/vinum/geom_vinum_raid5.h | 2 + sys/geom/vinum/geom_vinum_var.h | 2 + 7 files changed, 316 insertions(+), 13 deletions(-) (limited to 'sys/geom/vinum') diff --git a/sys/geom/vinum/geom_vinum.c b/sys/geom/vinum/geom_vinum.c index a25e5ca..5a54bee 100644 --- a/sys/geom/vinum/geom_vinum.c +++ b/sys/geom/vinum/geom_vinum.c @@ -503,6 +503,9 @@ gv_config(struct gctl_req *req, struct g_class *mp, char const *verb) } else if (!strcmp(verb, "create")) { gv_create(gp, req); + } else if (!strcmp(verb, "parityop")) { + gv_parityop(gp, req); + } else if (!strcmp(verb, "remove")) { gv_remove(gp, req); diff --git a/sys/geom/vinum/geom_vinum.h b/sys/geom/vinum/geom_vinum.h index de48c7d..c215e2e 100644 --- a/sys/geom/vinum/geom_vinum.h +++ b/sys/geom/vinum/geom_vinum.h @@ -38,6 +38,7 @@ void gv_save_config(struct g_consumer *, struct gv_drive *, struct gv_softc *); /* geom_vinum_init.c */ +void gv_parityop(struct g_geom *, struct gctl_req *); void gv_start_obj(struct g_geom *, struct gctl_req *); /* geom_vinum_list.c */ diff --git a/sys/geom/vinum/geom_vinum_init.c b/sys/geom/vinum/geom_vinum_init.c index 46d9d51..95b3d6a 100644 --- a/sys/geom/vinum/geom_vinum_init.c +++ b/sys/geom/vinum/geom_vinum_init.c @@ -58,6 +58,125 @@ struct gv_sync_args { }; void +gv_parityop(struct g_geom *gp, struct gctl_req *req) +{ + struct gv_softc *sc; + struct gv_plex *p; + struct bio *bp; + struct g_consumer *cp; + int error, *flags, type, *rebuild, rv; + char *plex; + + rv = -1; + + plex = gctl_get_param(req, "plex", NULL); + if (plex == NULL) { + gctl_error(req, "no plex given"); + goto out; + } + + flags = gctl_get_paraml(req, "flags", sizeof(*flags)); + if (flags == NULL) { + gctl_error(req, "no flags given"); + goto out; + } + + rebuild = gctl_get_paraml(req, "rebuild", sizeof(*rebuild)); + if (rebuild == NULL) { + gctl_error(req, "no rebuild op given"); + goto out; + } + + sc = gp->softc; + type = gv_object_type(sc, plex); + switch (type) { + case GV_TYPE_PLEX: + break; + case GV_TYPE_VOL: + case GV_TYPE_SD: + case GV_TYPE_DRIVE: + default: + gctl_error(req, "'%s' is not a plex", plex); + goto out; + } + + p = gv_find_plex(sc, plex); + if (p->state != GV_PLEX_UP) { + gctl_error(req, "plex %s is not completely accessible", + p->name); + goto out; + } + + cp = p->consumer; + error = g_access(cp, 1, 1, 0); + if (error) { + gctl_error(req, "cannot access consumer"); + goto out; + } + g_topology_unlock(); + + /* Reset the check pointer when using -f. */ + if (*flags & GV_FLAG_F) + p->synced = 0; + + bp = g_new_bio(); + if (bp == NULL) { + gctl_error(req, "cannot create BIO - out of memory"); + g_topology_lock(); + error = g_access(cp, -1, -1, 0); + goto out; + } + bp->bio_cmd = BIO_WRITE; + bp->bio_done = NULL; + bp->bio_data = g_malloc(p->stripesize, M_WAITOK | M_ZERO); + bp->bio_cflags |= GV_BIO_CHECK; + if (*rebuild) + bp->bio_cflags |= GV_BIO_PARITY; + bp->bio_offset = p->synced; + bp->bio_length = p->stripesize; + + /* Schedule it down ... */ + g_io_request(bp, cp); + + /* ... and wait for the result. */ + error = biowait(bp, "gwrite"); + g_free(bp->bio_data); + g_destroy_bio(bp); + + if (error) { + /* Incorrect parity. */ + if (error == EAGAIN) + rv = 1; + + /* Some other error happened. */ + else + gctl_error(req, "Parity check failed at offset 0x%jx, " + "errno %d", (intmax_t)p->synced, error); + + /* Correct parity. */ + } else + rv = 0; + + gctl_set_param(req, "offset", &p->synced, sizeof(p->synced)); + + /* Advance the checkpointer if there was no error. */ + if (rv == 0) + p->synced += p->stripesize; + + /* End of plex; reset the check pointer and signal it to the caller. */ + if (p->synced >= p->size) { + p->synced = 0; + rv = -2; + } + + g_topology_lock(); + error = g_access(cp, -1, -1, 0); + +out: + gctl_set_param(req, "rv", &rv, sizeof(rv)); +} + +void gv_start_obj(struct g_geom *gp, struct gctl_req *req) { struct gv_softc *sc; diff --git a/sys/geom/vinum/geom_vinum_plex.c b/sys/geom/vinum/geom_vinum_plex.c index 31266f9..8fa2923 100644 --- a/sys/geom/vinum/geom_vinum_plex.c +++ b/sys/geom/vinum/geom_vinum_plex.c @@ -46,6 +46,10 @@ __FBSDID("$FreeBSD$"); static void gv_plex_completed_request(struct gv_plex *, struct bio *); static void gv_plex_normal_request(struct gv_plex *, struct bio *); static void gv_plex_worker(void *); +static int gv_check_parity(struct gv_plex *, struct bio *, + struct gv_raid5_packet *); +static int gv_normal_parity(struct gv_plex *, struct bio *, + struct gv_raid5_packet *); /* XXX: is this the place to catch dying subdisks? */ static void @@ -346,6 +350,85 @@ gv_plex_worker(void *arg) kthread_exit(ENXIO); } +static int +gv_normal_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp) +{ + struct bio *cbp, *pbp; + int finished, i; + + finished = 1; + + if (wp->waiting != NULL) { + pbp = wp->waiting; + wp->waiting = NULL; + cbp = wp->parity; + for (i = 0; i < wp->length; i++) + cbp->bio_data[i] ^= pbp->bio_data[i]; + g_io_request(pbp, pbp->bio_caller2); + finished = 0; + + } else if (wp->parity != NULL) { + cbp = wp->parity; + wp->parity = NULL; + g_io_request(cbp, cbp->bio_caller2); + finished = 0; + } + + return (finished); +} + +static int +gv_check_parity(struct gv_plex *p, struct bio *bp, struct gv_raid5_packet *wp) +{ + struct bio *cbp, *pbp; + int err, finished, i; + + err = 0; + finished = 1; + + if (wp->waiting != NULL) { + pbp = wp->waiting; + wp->waiting = NULL; + g_io_request(pbp, pbp->bio_caller2); + finished = 0; + + } else if (wp->parity != NULL) { + cbp = wp->parity; + wp->parity = NULL; + + /* Check if the parity is correct. */ + for (i = 0; i < wp->length; i++) { + if (bp->bio_data[i] != cbp->bio_data[i]) { + err = 1; + break; + } + } + + /* The parity is not correct... */ + if (err) { + bp->bio_parent->bio_error = EAGAIN; + + /* ... but we rebuild it. */ + if (bp->bio_parent->bio_cflags & GV_BIO_PARITY) { + g_io_request(cbp, cbp->bio_caller2); + finished = 0; + } + } + + /* + * Clean up the BIO we would have used for rebuilding the + * parity. + */ + if (finished) { + bp->bio_parent->bio_inbed++; + g_destroy_bio(cbp); + } + + } + + return (finished); +} + void gv_plex_completed_request(struct gv_plex *p, struct bio *bp) { @@ -405,18 +488,13 @@ gv_plex_completed_request(struct gv_plex *p, struct bio *bp) /* Handle parity data. */ if (TAILQ_EMPTY(&wp->bits)) { - if (wp->waiting != NULL) { - pbp = wp->waiting; - wp->waiting = NULL; - cbp = wp->parity; - for (i = 0; i < wp->length; i++) - cbp->bio_data[i] ^= pbp->bio_data[i]; - g_io_request(pbp, pbp->bio_caller2); - } else if (wp->parity != NULL) { - cbp = wp->parity; - wp->parity = NULL; - g_io_request(cbp, cbp->bio_caller2); - } else { + if (bp->bio_parent->bio_cflags & GV_BIO_CHECK) + i = gv_check_parity(p, bp, wp); + else + i = gv_normal_parity(p, bp, wp); + + /* All of our sub-requests have finished. */ + if (i) { bp->bio_parent->bio_completed += wp->length; TAILQ_REMOVE(&p->packets, wp, list); /* Bring the waiting bios back into the game. */ @@ -475,6 +553,9 @@ gv_plex_normal_request(struct gv_plex *p, struct bio *bp) if (bp->bio_cflags & GV_BIO_REBUILD) err = gv_rebuild_raid5(p, wp, bp, addr, boff, bcount); + else if (bp->bio_cflags & GV_BIO_CHECK) + err = gv_check_raid5(p, wp, bp, addr, + boff, bcount); else err = gv_build_raid5_req(p, wp, bp, addr, boff, bcount); diff --git a/sys/geom/vinum/geom_vinum_raid5.c b/sys/geom/vinum/geom_vinum_raid5.c index 11bb931..789db19 100644 --- a/sys/geom/vinum/geom_vinum_raid5.c +++ b/sys/geom/vinum/geom_vinum_raid5.c @@ -81,6 +81,101 @@ gv_stripe_active(struct gv_plex *p, struct bio *bp) } int +gv_check_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp, + caddr_t addr, off_t boff, off_t bcount) +{ + struct gv_sd *parity, *s; + struct gv_bioq *bq; + struct bio *cbp, *pbp; + int i, psdno; + off_t real_len, real_off; + + if (p == NULL || LIST_EMPTY(&p->subdisks)) + return (ENXIO); + + gv_raid5_offset(p, boff, bcount, &real_off, &real_len, NULL, &psdno); + + /* Find the right subdisk. */ + parity = NULL; + i = 0; + LIST_FOREACH(s, &p->subdisks, in_plex) { + if (i == psdno) { + parity = s; + break; + } + i++; + } + + /* Parity stripe not found. */ + if (parity == NULL) + return (ENXIO); + + if (parity->state != GV_SD_UP) + return (ENXIO); + + wp->length = real_len; + wp->data = addr; + wp->lockbase = real_off; + + /* Read all subdisks. */ + LIST_FOREACH(s, &p->subdisks, in_plex) { + /* Skip the parity subdisk. */ + if (s == parity) + continue; + + cbp = g_clone_bio(bp); + if (cbp == NULL) + return (ENOMEM); + cbp->bio_cmd = BIO_READ; + cbp->bio_data = g_malloc(real_len, M_WAITOK); + cbp->bio_cflags |= GV_BIO_MALLOC; + cbp->bio_offset = real_off; + cbp->bio_length = real_len; + cbp->bio_done = gv_plex_done; + cbp->bio_caller2 = s->consumer; + cbp->bio_driver1 = wp; + + GV_ENQUEUE(bp, cbp, pbp); + + bq = g_malloc(sizeof(*bq), M_WAITOK | M_ZERO); + bq->bp = cbp; + TAILQ_INSERT_TAIL(&wp->bits, bq, queue); + } + + /* Read the parity data. */ + cbp = g_clone_bio(bp); + if (cbp == NULL) + return (ENOMEM); + cbp->bio_cmd = BIO_READ; + cbp->bio_data = g_malloc(real_len, M_WAITOK | M_ZERO); + cbp->bio_cflags |= GV_BIO_MALLOC; + cbp->bio_offset = real_off; + cbp->bio_length = real_len; + cbp->bio_done = gv_plex_done; + cbp->bio_caller2 = parity->consumer; + cbp->bio_driver1 = wp; + wp->waiting = cbp; + + /* + * In case we want to rebuild the parity, create an extra BIO to write + * it out. It also acts as buffer for the XOR operations. + */ + cbp = g_clone_bio(bp); + if (cbp == NULL) + return (ENOMEM); + cbp->bio_data = addr; + cbp->bio_offset = real_off; + cbp->bio_length = real_len; + cbp->bio_done = gv_plex_done; + cbp->bio_caller2 = parity->consumer; + cbp->bio_driver1 = wp; + wp->parity = cbp; + + return (0); +} + +/* Rebuild a degraded RAID5 plex. */ +int gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp, caddr_t addr, off_t boff, off_t bcount) { @@ -101,7 +196,7 @@ gv_rebuild_raid5(struct gv_plex *p, struct gv_raid5_packet *wp, struct bio *bp, broken = s; } - /* Parity stripe not found. */ + /* Broken stripe not found. */ if (broken == NULL) return (ENXIO); diff --git a/sys/geom/vinum/geom_vinum_raid5.h b/sys/geom/vinum/geom_vinum_raid5.h index 212f6c6..9637db1 100644 --- a/sys/geom/vinum/geom_vinum_raid5.h +++ b/sys/geom/vinum/geom_vinum_raid5.h @@ -67,6 +67,8 @@ struct gv_raid5_packet { int gv_stripe_active(struct gv_plex *, struct bio *); int gv_build_raid5_req(struct gv_plex *, struct gv_raid5_packet *, struct bio *, caddr_t, off_t, off_t); +int gv_check_raid5(struct gv_plex *, struct gv_raid5_packet *, + struct bio *, caddr_t, off_t, off_t); int gv_rebuild_raid5(struct gv_plex *, struct gv_raid5_packet *, struct bio *, caddr_t, off_t, off_t); void gv_raid5_worker(void *); diff --git a/sys/geom/vinum/geom_vinum_var.h b/sys/geom/vinum/geom_vinum_var.h index 95d8956..05a5e8c 100644 --- a/sys/geom/vinum/geom_vinum_var.h +++ b/sys/geom/vinum/geom_vinum_var.h @@ -114,6 +114,8 @@ #define GV_BIO_SYNCREQ 0x08 #define GV_BIO_SUCCEED 0x10 #define GV_BIO_REBUILD 0x20 +#define GV_BIO_CHECK 0x40 +#define GV_BIO_PARITY 0x80 /* * hostname is 256 bytes long, but we don't need to shlep multiple copies in -- cgit v1.1