Bring in geli suspend/resume functionality (finally).

Before this change if you wanted to suspend your laptop and be sure that your
encryption keys are safe, you had to stop all processes that use file system
stored on encrypted device, unmount the file system and detach geli provider.

This isn't very handy. If you are a lucky user of a laptop where suspend/resume
actually works with FreeBSD (I'm not!) you most likely want to suspend your
laptop, because you don't want to start everything over again when you turn
your laptop back on.

And this is where geli suspend/resume steps in. When you execute:

	# geli suspend -a

geli will wait for all in-flight I/O requests, suspend new I/O requests, remove
all geli sensitive data from the kernel memory (like encryption keys) and will
wait for either 'geli resume' or 'geli detach'.

Now with no keys in memory you can suspend your laptop without stopping any
processes or unmounting any file systems.

When you resume your laptop you have to resume geli devices using 'geli resume'
command. You need to provide your passphrase, etc. again so the keys can be
restored and suspended I/O requests released.

Of course you need to remember that 'geli suspend' won't clear file system
cache and other places where data from your geli-encrypted file system might be
present. But to get rid of those stopping processes and unmounting file system
won't help either - you have to turn your laptop off. Be warned.

Also note, that suspending geli device which contains file system with geli
utility (or anything used by 'geli resume') is not very good idea, as you won't
be able to resume it - when you execute geli(8), the kernel will try to read it
and this read I/O request will be suspended.

5 files changed, 379 insertions, 35 deletions

diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c
index 4e15877..66f641b 100644
--- a/sys/geom/eli/g_eli.c
+++ b/sys/geom/eli/g_eli.c
@@ -106,7 +106,7 @@ struct g_class g_eli_class = {
 /*
  * Code paths:
  * BIO_READ:
- *	g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
+ *	g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
  * BIO_WRITE:
  *	g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
@@ -148,7 +148,7 @@ g_eli_crypto_rerun(struct cryptop *crp)
 /*
  * The function is called afer reading encrypted data from the provider.
  *
- * g_eli_start -> g_io_request -> G_ELI_READ_DONE -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
+ * g_eli_start -> g_eli_crypto_read -> g_io_request -> G_ELI_READ_DONE -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
  */
 void
 g_eli_read_done(struct bio *bp)
@@ -167,6 +167,7 @@ g_eli_read_done(struct bio *bp)
 	if (pbp->bio_inbed < pbp->bio_children)
 		return;
 	g_destroy_bio(bp);
+	sc = pbp->bio_to->geom->softc;
 	if (pbp->bio_error != 0) {
 		G_ELI_LOGREQ(0, pbp, "%s() failed", __func__);
 		pbp->bio_completed = 0;
@@ -175,9 +176,9 @@ g_eli_read_done(struct bio *bp)
 			pbp->bio_driver2 = NULL;
 		}
 		g_io_deliver(pbp, pbp->bio_error);
+		atomic_subtract_int(&sc->sc_inflight, 1);
 		return;
 	}
-	sc = pbp->bio_to->geom->softc;
 	mtx_lock(&sc->sc_queue_mtx);
 	bioq_insert_tail(&sc->sc_queue, pbp);
 	mtx_unlock(&sc->sc_queue_mtx);
@@ -192,6 +193,7 @@ g_eli_read_done(struct bio *bp)
 void
 g_eli_write_done(struct bio *bp)
 {
+	struct g_eli_softc *sc;
 	struct bio *pbp;
 
 	G_ELI_LOGREQ(2, bp, "Request done.");
@@ -218,7 +220,9 @@ g_eli_write_done(struct bio *bp)
 	 * Write is finished, send it up.
 	 */
 	pbp->bio_completed = pbp->bio_length;
+	sc = pbp->bio_to->geom->softc;
 	g_io_deliver(pbp, pbp->bio_error);
+	atomic_subtract_int(&sc->sc_inflight, 1);
 }
 
 /*
@@ -241,12 +245,12 @@ g_eli_orphan(struct g_consumer *cp)
 	sc = cp->geom->softc;
 	if (sc == NULL)
 		return;
-	g_eli_destroy(sc, 1);
+	g_eli_destroy(sc, TRUE);
 }
 
 /*
  * BIO_READ:
- *	G_ELI_START -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
+ *	G_ELI_START -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
  * BIO_WRITE:
  *	G_ELI_START -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
@@ -284,24 +288,16 @@ g_eli_start(struct bio *bp)
 		g_io_deliver(bp, ENOMEM);
 		return;
 	}
+	bp->bio_driver1 = cbp;
+	bp->bio_pflags = G_ELI_NEW_BIO;
 	switch (bp->bio_cmd) {
 	case BIO_READ:
 		if (!(sc->sc_flags & G_ELI_FLAG_AUTH)) {
-			bp->bio_driver2 = NULL;
-			cbp->bio_done = g_eli_read_done;
-			cp = LIST_FIRST(&sc->sc_geom->consumer);
-			cbp->bio_to = cp->provider;
-			G_ELI_LOGREQ(2, cbp, "Sending request.");
-			/*
-			 * Read encrypted data from provider.
-			 */
-			g_io_request(cbp, cp);
+			g_eli_crypto_read(sc, bp, 0);
 			break;
 		}
-		bp->bio_pflags = 255;
 		/* FALLTHROUGH */
 	case BIO_WRITE:
-		bp->bio_driver1 = cbp;
 		mtx_lock(&sc->sc_queue_mtx);
 		bioq_insert_tail(&sc->sc_queue, bp);
 		mtx_unlock(&sc->sc_queue_mtx);
@@ -318,6 +314,41 @@ g_eli_start(struct bio *bp)
 	}
 }
 
+static void
+g_eli_cancel(struct g_eli_softc *sc)
+{
+	struct bio *bp;
+
+	mtx_assert(&sc->sc_queue_mtx, MA_OWNED);
+
+	while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL) {
+		KASSERT(bp->bio_pflags == G_ELI_NEW_BIO,
+		    ("Not new bio when canceling (bp=%p).", bp));
+		g_io_deliver(bp, ENXIO);
+	}
+}
+
+static struct bio *
+g_eli_takefirst(struct g_eli_softc *sc)
+{
+	struct bio *bp;
+
+	mtx_assert(&sc->sc_queue_mtx, MA_OWNED);
+
+	if (!(sc->sc_flags & G_ELI_FLAG_SUSPEND))
+		return (bioq_takefirst(&sc->sc_queue));
+	/*
+	 * Device suspended, so we skip new I/O requests.
+	 */
+	TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
+		if (bp->bio_pflags != G_ELI_NEW_BIO)
+			break;
+	}
+	if (bp != NULL)
+		bioq_remove(&sc->sc_queue, bp);
+	return (bp);
+}
+
 /*
  * This is the main function for kernel worker thread when we don't have
  * hardware acceleration and we have to do cryptography in software.
@@ -351,9 +382,11 @@ g_eli_worker(void *arg)
 
 	for (;;) {
 		mtx_lock(&sc->sc_queue_mtx);
-		bp = bioq_takefirst(&sc->sc_queue);
+again:
+		bp = g_eli_takefirst(sc);
 		if (bp == NULL) {
 			if (sc->sc_flags & G_ELI_FLAG_DESTROY) {
+				g_eli_cancel(sc);
 				LIST_REMOVE(wr, w_next);
 				crypto_freesession(wr->w_sid);
 				free(wr, M_ELI);
@@ -363,16 +396,54 @@ g_eli_worker(void *arg)
 				mtx_unlock(&sc->sc_queue_mtx);
 				kproc_exit(0);
 			}
+			while (sc->sc_flags & G_ELI_FLAG_SUSPEND) {
+				if (sc->sc_inflight > 0) {
+					G_ELI_DEBUG(0, "inflight=%d", sc->sc_inflight);
+					/*
+					 * We still have inflight BIOs, so
+					 * sleep and retry.
+					 */
+					msleep(sc, &sc->sc_queue_mtx, PRIBIO,
+					    "geli:inf", hz / 5);
+					goto again;
+				}
+				/*
+				 * Suspend requested, mark the worker as
+				 * suspended and go to sleep.
+				 */
+				wr->w_active = 0;
+				wakeup(&sc->sc_workers);
+				msleep(sc, &sc->sc_queue_mtx, PRIBIO,
+				    "geli:suspend", 0);
+				if (!(sc->sc_flags & G_ELI_FLAG_SUSPEND))
+					wr->w_active = 1;
+				goto again;
+			}
 			msleep(sc, &sc->sc_queue_mtx, PDROP, "geli:w", 0);
 			continue;
 		}
+		if (bp->bio_pflags == G_ELI_NEW_BIO)
+			atomic_add_int(&sc->sc_inflight, 1);
 		mtx_unlock(&sc->sc_queue_mtx);
-		if (bp->bio_cmd == BIO_READ && bp->bio_pflags == 255)
-			g_eli_auth_read(sc, bp);
-		else if (sc->sc_flags & G_ELI_FLAG_AUTH)
-			g_eli_auth_run(wr, bp);
-		else
-			g_eli_crypto_run(wr, bp);
+		if (bp->bio_pflags == G_ELI_NEW_BIO) {
+			bp->bio_pflags = 0;
+			if (sc->sc_flags & G_ELI_FLAG_AUTH) {
+				if (bp->bio_cmd == BIO_READ)
+					g_eli_auth_read(sc, bp);
+				else
+					g_eli_auth_run(wr, bp);
+			} else {
+				if (bp->bio_cmd == BIO_READ)
+					g_eli_crypto_read(sc, bp, 1);
+				else
+					g_eli_crypto_run(wr, bp);
+			}
+		} else {
+			if (sc->sc_flags & G_ELI_FLAG_AUTH)
+				g_eli_auth_run(wr, bp);
+			else
+				g_eli_crypto_run(wr, bp);
+		}
 	}
 }
 
@@ -502,7 +573,7 @@ g_eli_last_close(struct g_eli_softc *sc)
 	gp = sc->sc_geom;
 	pp = LIST_FIRST(&gp->provider);
 	strlcpy(ppname, pp->name, sizeof(ppname));
-	error = g_eli_destroy(sc, 1);
+	error = g_eli_destroy(sc, TRUE);
 	KASSERT(error == 0, ("Cannot detach %s on last close (error=%d).",
 	    ppname, error));
 	G_ELI_DEBUG(0, "Detached %s on last close.", ppname);
@@ -586,6 +657,7 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 	else
 		gp->access = g_std_access;
 
+	sc->sc_inflight = 0;
 	sc->sc_crypto = G_ELI_CRYPTO_SW;
 	sc->sc_flags = md->md_flags;
 	/* Backward compatibility. */
@@ -730,6 +802,7 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 		wr = malloc(sizeof(*wr), M_ELI, M_WAITOK | M_ZERO);
 		wr->w_softc = sc;
 		wr->w_number = i;
+		wr->w_active = TRUE;
 
 		/*
 		 * If this is the first pass, try to get hardware support.
@@ -877,7 +950,7 @@ g_eli_destroy_geom(struct gctl_req *req __unused,
 	struct g_eli_softc *sc;
 
 	sc = gp->softc;
-	return (g_eli_destroy(sc, 0));
+	return (g_eli_destroy(sc, FALSE));
 }
 
 static int
@@ -1108,6 +1181,7 @@ g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 		sbuf_printf(sb, name);					\
 	}								\
 } while (0)
+		ADD_FLAG(G_ELI_FLAG_SUSPEND, "SUSPEND");
 		ADD_FLAG(G_ELI_FLAG_SINGLE_KEY, "SINGLE-KEY");
 		ADD_FLAG(G_ELI_FLAG_NATIVE_BYTE_ORDER, "NATIVE-BYTE-ORDER");
 		ADD_FLAG(G_ELI_FLAG_ONETIME, "ONETIME");
@@ -1169,7 +1243,7 @@ g_eli_shutdown_pre_sync(void *arg, int howto)
 		pp = LIST_FIRST(&gp->provider);
 		KASSERT(pp != NULL, ("No provider? gp=%p (%s)", gp, gp->name));
 		if (pp->acr + pp->acw + pp->ace == 0)
-			error = g_eli_destroy(sc, 1);
+			error = g_eli_destroy(sc, TRUE);
 		else {
 			sc->sc_flags |= G_ELI_FLAG_RW_DETACH;
 			gp->access = g_eli_access;
diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h
index e6b311d..fd53d5f 100644
--- a/sys/geom/eli/g_eli.h
+++ b/sys/geom/eli/g_eli.h
@@ -86,6 +86,10 @@
 #define	G_ELI_FLAG_NATIVE_BYTE_ORDER	0x00040000
 /* Provider uses single encryption key. */
 #define	G_ELI_FLAG_SINGLE_KEY		0x00080000
+/* Device suspended. */
+#define	G_ELI_FLAG_SUSPEND		0x00100000
+
+#define	G_ELI_NEW_BIO	255
 
 #define	SHA512_MDLEN		64
 #define	G_ELI_AUTH_SECKEYLEN	SHA256_DIGEST_LENGTH
@@ -140,6 +144,7 @@ struct g_eli_worker {
 	struct proc		*w_proc;
 	u_int			 w_number;
 	uint64_t		 w_sid;
+	boolean_t		 w_active;
 	LIST_ENTRY(g_eli_worker) w_next;
 };
 
@@ -160,6 +165,7 @@ struct g_eli_softc {
 	SHA256_CTX	  sc_ivctx;
 	int		  sc_nkey;
 	uint32_t	  sc_flags;
+	int		  sc_inflight;
 	off_t		  sc_mediasize;
 	size_t		  sc_sectorsize;
 	u_int		  sc_bytes_per_sector;
@@ -499,6 +505,7 @@ uint8_t *g_eli_crypto_key(struct g_eli_softc *sc, off_t offset,
 void g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv,
     size_t size);
 
+void g_eli_crypto_read(struct g_eli_softc *sc, struct bio *bp, boolean_t fromworker);
 void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp);
 
 void g_eli_auth_read(struct g_eli_softc *sc, struct bio *bp);
diff --git a/sys/geom/eli/g_eli_ctl.c b/sys/geom/eli/g_eli_ctl.c
index 02ede13..7147b27 100644
--- a/sys/geom/eli/g_eli_ctl.c
+++ b/sys/geom/eli/g_eli_ctl.c
@@ -217,7 +217,7 @@ g_eli_ctl_detach(struct gctl_req *req, struct g_class *mp)
 			sc->sc_flags |= G_ELI_FLAG_RW_DETACH;
 			sc->sc_geom->access = g_eli_access;
 		} else {
-			error = g_eli_destroy(sc, *force);
+			error = g_eli_destroy(sc, *force ? TRUE : FALSE);
 			if (error != 0) {
 				gctl_error(req,
 				    "Cannot destroy device %s (error=%d).",
@@ -700,6 +700,213 @@ g_eli_ctl_delkey(struct gctl_req *req, struct g_class *mp)
 }
 
 static int
+g_eli_suspend_one(struct g_eli_softc *sc)
+{
+	struct g_eli_worker *wr;
+
+	g_topology_assert();
+
+	if (sc == NULL)
+		return (ENOENT);
+	if (sc->sc_flags & G_ELI_FLAG_ONETIME)
+		return (EOPNOTSUPP);
+
+	mtx_lock(&sc->sc_queue_mtx);
+	if (sc->sc_flags & G_ELI_FLAG_SUSPEND) {
+		mtx_unlock(&sc->sc_queue_mtx);
+		return (EALREADY);
+	}
+	sc->sc_flags |= G_ELI_FLAG_SUSPEND;
+	wakeup(sc);
+	for (;;) {
+		LIST_FOREACH(wr, &sc->sc_workers, w_next) {
+			if (wr->w_active)
+				break;
+		}
+		if (wr == NULL)
+			break;
+		/* Not all threads suspended. */
+		msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO,
+		    "geli:suspend", 0);
+	}
+	/*
+	 * Clear sensitive data on suspend, they will be recovered on resume.
+	 */
+	bzero(sc->sc_mkey, sizeof(sc->sc_mkey));
+	bzero(sc->sc_ekeys,
+	    sc->sc_nekeys * (sizeof(uint8_t *) + G_ELI_DATAKEYLEN));
+	free(sc->sc_ekeys, M_ELI);
+	sc->sc_ekeys = NULL;
+	bzero(sc->sc_akey, sizeof(sc->sc_akey));
+	bzero(&sc->sc_akeyctx, sizeof(sc->sc_akeyctx));
+	bzero(sc->sc_ivkey, sizeof(sc->sc_ivkey));
+	bzero(&sc->sc_ivctx, sizeof(sc->sc_ivctx));
+	mtx_unlock(&sc->sc_queue_mtx);
+	G_ELI_DEBUG(0, "%s has been suspended.", sc->sc_name);
+	return (0);
+}
+
+static void
+g_eli_ctl_suspend(struct gctl_req *req, struct g_class *mp)
+{
+	struct g_eli_softc *sc;
+	int *all, *nargs;
+	int error;
+
+	g_topology_assert();
+
+	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
+	if (nargs == NULL) {
+		gctl_error(req, "No '%s' argument.", "nargs");
+		return;
+	}
+	all = gctl_get_paraml(req, "all", sizeof(*all));
+	if (all == NULL) {
+		gctl_error(req, "No '%s' argument.", "all");
+		return;
+	}
+	if (!*all && *nargs == 0) {
+		gctl_error(req, "Too few arguments.");
+		return;
+	}
+
+	if (*all) {
+		struct g_geom *gp, *gp2;
+
+		LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
+			sc = gp->softc;
+			if (sc->sc_flags & G_ELI_FLAG_ONETIME)
+				continue;
+			error = g_eli_suspend_one(sc);
+			if (error != 0)
+				gctl_error(req, "Not fully done.");
+		}
+	} else {
+		const char *prov;
+		char param[16];
+		int i;
+
+		for (i = 0; i < *nargs; i++) {
+			snprintf(param, sizeof(param), "arg%d", i);
+			prov = gctl_get_asciiparam(req, param);
+			if (prov == NULL) {
+				G_ELI_DEBUG(0, "No 'arg%d' argument.", i);
+				continue;
+			}
+
+			sc = g_eli_find_device(mp, prov);
+			if (sc == NULL) {
+				G_ELI_DEBUG(0, "No such provider: %s.", prov);
+				continue;
+			}
+			error = g_eli_suspend_one(sc);
+			if (error != 0)
+				gctl_error(req, "Not fully done.");
+		}
+	}
+}
+
+static void
+g_eli_ctl_resume(struct gctl_req *req, struct g_class *mp)
+{
+	struct g_eli_metadata md;
+	struct g_eli_softc *sc;
+	struct g_provider *pp;
+	struct g_consumer *cp;
+	const char *name;
+	u_char *key, mkey[G_ELI_DATAIVKEYLEN];
+	int *nargs, keysize, error;
+	u_int nkey;
+
+	g_topology_assert();
+
+	nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs));
+	if (nargs == NULL) {
+		gctl_error(req, "No '%s' argument.", "nargs");
+		return;
+	}
+	if (*nargs != 1) {
+		gctl_error(req, "Invalid number of arguments.");
+		return;
+	}
+
+	name = gctl_get_asciiparam(req, "arg0");
+	if (name == NULL) {
+		gctl_error(req, "No 'arg%u' argument.", 0);
+		return;
+	}
+	sc = g_eli_find_device(mp, name);
+	if (sc == NULL) {
+		gctl_error(req, "Provider %s is invalid.", name);
+		return;
+	}
+	if (!(sc->sc_flags & G_ELI_FLAG_SUSPEND)) {
+		gctl_error(req, "Provider %s not suspended.", name);
+		return;
+	}
+	cp = LIST_FIRST(&sc->sc_geom->consumer);
+	pp = cp->provider;
+	error = g_eli_read_metadata(mp, pp, &md);
+	if (error != 0) {
+		gctl_error(req, "Cannot read metadata from %s (error=%d).",
+		    name, error);
+		return;
+	}
+	if (md.md_keys == 0x00) {
+		bzero(&md, sizeof(md));
+		gctl_error(req, "No valid keys on %s.", pp->name);
+		return;
+	}
+
+	key = gctl_get_param(req, "key", &keysize);
+	if (key == NULL || keysize != G_ELI_USERKEYLEN) {
+		bzero(&md, sizeof(md));
+		gctl_error(req, "No '%s' argument.", "key");
+		return;
+	}
+
+	error = g_eli_mkey_decrypt(&md, key, mkey, &nkey);
+	bzero(key, keysize);
+	if (error == -1) {
+		bzero(&md, sizeof(md));
+		gctl_error(req, "Wrong key for %s.", pp->name);
+		return;
+	} else if (error > 0) {
+		bzero(&md, sizeof(md));
+		gctl_error(req, "Cannot decrypt Master Key for %s (error=%d).",
+		    pp->name, error);
+		return;
+	}
+	G_ELI_DEBUG(1, "Using Master Key %u for %s.", nkey, pp->name);
+
+	mtx_lock(&sc->sc_queue_mtx);
+	/* Restore sc_mkey, sc_ekeys, sc_akey and sc_ivkey. */
+	g_eli_mkey_propagate(sc, mkey);
+	bzero(mkey, sizeof(mkey));
+	bzero(&md, sizeof(md));
+	/* Restore sc_akeyctx. */
+	if (sc->sc_flags & G_ELI_FLAG_AUTH) {
+		SHA256_Init(&sc->sc_akeyctx);
+		SHA256_Update(&sc->sc_akeyctx, sc->sc_akey,
+		    sizeof(sc->sc_akey));
+	}
+	/* Restore sc_ivctx. */
+	switch (sc->sc_ealgo) {
+	case CRYPTO_AES_XTS:
+		break;
+	default:
+		SHA256_Init(&sc->sc_ivctx);
+		SHA256_Update(&sc->sc_ivctx, sc->sc_ivkey,
+		    sizeof(sc->sc_ivkey));
+		break;
+	}
+	sc->sc_flags &= ~G_ELI_FLAG_SUSPEND;
+	mtx_unlock(&sc->sc_queue_mtx);
+	G_ELI_DEBUG(1, "Resumed %s.", pp->name);
+	wakeup(sc);
+}
+
+static int
 g_eli_kill_one(struct g_eli_softc *sc)
 {
 	struct g_provider *pp;
@@ -749,7 +956,7 @@ g_eli_kill_one(struct g_eli_softc *sc)
 	}
 	if (error == 0)
 		G_ELI_DEBUG(0, "%s has been killed.", pp->name);
-	g_eli_destroy(sc, 1);
+	g_eli_destroy(sc, TRUE);
 	return (error);
 }
 
@@ -839,6 +1046,10 @@ g_eli_config(struct gctl_req *req, struct g_class *mp, const char *verb)
 		g_eli_ctl_setkey(req, mp);
 	else if (strcmp(verb, "delkey") == 0)
 		g_eli_ctl_delkey(req, mp);
+	else if (strcmp(verb, "suspend") == 0)
+		g_eli_ctl_suspend(req, mp);
+	else if (strcmp(verb, "resume") == 0)
+		g_eli_ctl_resume(req, mp);
 	else if (strcmp(verb, "kill") == 0)
 		g_eli_ctl_kill(req, mp);
 	else
diff --git a/sys/geom/eli/g_eli_integrity.c b/sys/geom/eli/g_eli_integrity.c
index 0103a32..24586bd 100644
--- a/sys/geom/eli/g_eli_integrity.c
+++ b/sys/geom/eli/g_eli_integrity.c
@@ -129,6 +129,7 @@ g_eli_auth_keygen(struct g_eli_softc *sc, off_t offset, u_char *key)
 static int
 g_eli_auth_read_done(struct cryptop *crp)
 {
+	struct g_eli_softc *sc;
 	struct bio *bp;
 
 	if (crp->crp_etype == EAGAIN) {
@@ -152,8 +153,8 @@ g_eli_auth_read_done(struct cryptop *crp)
 	 */
 	if (bp->bio_inbed < bp->bio_children)
 		return (0);
+	sc = bp->bio_to->geom->softc;
 	if (bp->bio_error == 0) {
-		struct g_eli_softc *sc;
 		u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
 		u_char *srcdata, *dstdata, *auth;
 		off_t coroff, corsize;
@@ -161,7 +162,6 @@ g_eli_auth_read_done(struct cryptop *crp)
 		/*
 		 * Verify data integrity based on calculated and read HMACs.
 		 */
-		sc = bp->bio_to->geom->softc;
 		/* Sectorsize of decrypted provider eg. 4096. */
 		decr_secsize = bp->bio_to->sectorsize;
 		/* The real sectorsize of encrypted provider, eg. 512. */
@@ -240,6 +240,7 @@ g_eli_auth_read_done(struct cryptop *crp)
 	 * Read is finished, send it up.
 	 */
 	g_io_deliver(bp, bp->bio_error);
+	atomic_subtract_int(&sc->sc_inflight, 1);
 	return (0);
 }
 
@@ -276,6 +277,7 @@ g_eli_auth_write_done(struct cryptop *crp)
 	 */
 	if (bp->bio_inbed < bp->bio_children)
 		return (0);
+	sc = bp->bio_to->geom->softc;
 	if (bp->bio_error != 0) {
 		G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
 		    bp->bio_error);
@@ -285,9 +287,9 @@ g_eli_auth_write_done(struct cryptop *crp)
 		bp->bio_driver1 = NULL;
 		g_destroy_bio(cbp);
 		g_io_deliver(bp, bp->bio_error);
+		atomic_subtract_int(&sc->sc_inflight, 1);
 		return (0);
 	}
-	sc = bp->bio_to->geom->softc;
 	cp = LIST_FIRST(&sc->sc_geom->consumer);
 	cbp = bp->bio_driver1;
 	bp->bio_driver1 = NULL;
diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c
index c2814bc..ee133c6 100644
--- a/sys/geom/eli/g_eli_privacy.c
+++ b/sys/geom/eli/g_eli_privacy.c
@@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
 /*
  * Code paths:
  * BIO_READ:
- *	g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
+ *	g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
  * BIO_WRITE:
  *	g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
@@ -63,11 +63,12 @@ MALLOC_DECLARE(M_ELI);
 /*
  * The function is called after we read and decrypt data.
  *
- * g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
+ * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
  */
 static int
 g_eli_crypto_read_done(struct cryptop *crp)
 {
+	struct g_eli_softc *sc;
 	struct bio *bp;
 
 	if (crp->crp_etype == EAGAIN) {
@@ -101,7 +102,9 @@ g_eli_crypto_read_done(struct cryptop *crp)
 	/*
 	 * Read is finished, send it up.
 	 */
+	sc = bp->bio_to->geom->softc;
 	g_io_deliver(bp, bp->bio_error);
+	atomic_subtract_int(&sc->sc_inflight, 1);
 	return (0);
 }
 
@@ -113,6 +116,7 @@ g_eli_crypto_read_done(struct cryptop *crp)
 static int
 g_eli_crypto_write_done(struct cryptop *crp)
 {
+	struct g_eli_softc *sc;
 	struct g_geom *gp;
 	struct g_consumer *cp;
 	struct bio *bp, *cbp;
@@ -141,18 +145,20 @@ g_eli_crypto_write_done(struct cryptop *crp)
 	bp->bio_children = 1;
 	cbp = bp->bio_driver1;
 	bp->bio_driver1 = NULL;
+	gp = bp->bio_to->geom;
 	if (bp->bio_error != 0) {
 		G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
 		    bp->bio_error);
 		free(bp->bio_driver2, M_ELI);
 		bp->bio_driver2 = NULL;
 		g_destroy_bio(cbp);
+		sc = gp->softc;
 		g_io_deliver(bp, bp->bio_error);
+		atomic_subtract_int(&sc->sc_inflight, 1);
 		return (0);
 	}
 	cbp->bio_data = bp->bio_driver2;
 	cbp->bio_done = g_eli_write_done;
-	gp = bp->bio_to->geom;
 	cp = LIST_FIRST(&gp->consumer);
 	cbp->bio_to = cp->provider;
 	G_ELI_LOGREQ(2, cbp, "Sending request.");
@@ -164,11 +170,55 @@ g_eli_crypto_write_done(struct cryptop *crp)
 }
 
 /*
+ * The function is called to read encrypted data.
+ *
+ * g_eli_start -> G_ELI_CRYPTO_READ -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
+ */
+void
+g_eli_crypto_read(struct g_eli_softc *sc, struct bio *bp, boolean_t fromworker)
+{
+	struct g_consumer *cp;
+	struct bio *cbp;
+
+	if (!fromworker) {
+		/*
+		 * We are not called from the worker thread, so check if
+		 * device is suspended.
+		 */
+		mtx_lock(&sc->sc_queue_mtx);
+		if (sc->sc_flags & G_ELI_FLAG_SUSPEND) {
+			/*
+			 * If device is suspended, we place the request onto
+			 * the queue, so it can be handled after resume.
+			 */
+			G_ELI_DEBUG(0, "device suspended, move onto queue");
+			bioq_insert_tail(&sc->sc_queue, bp);
+			mtx_unlock(&sc->sc_queue_mtx);
+			wakeup(sc);
+			return;
+		}
+		atomic_add_int(&sc->sc_inflight, 1);
+		mtx_unlock(&sc->sc_queue_mtx);
+	}
+	bp->bio_pflags = 0;
+	bp->bio_driver2 = NULL;
+	cbp = bp->bio_driver1;
+	cbp->bio_done = g_eli_read_done;
+	cp = LIST_FIRST(&sc->sc_geom->consumer);
+	cbp->bio_to = cp->provider;
+	G_ELI_LOGREQ(2, cbp, "Sending request.");
+	/*
+	 * Read encrypted data from provider.
+	 */
+	g_io_request(cbp, cp);
+}
+
+/*
  * This is the main function responsible for cryptography (ie. communication
  * with crypto(9) subsystem).
  *
  * BIO_READ:
- *	g_eli_start -> g_io_request -> g_eli_read_done -> G_ELI_CRYPTO_RUN -> g_eli_crypto_read_done -> g_io_deliver
+ *	g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> G_ELI_CRYPTO_RUN -> g_eli_crypto_read_done -> g_io_deliver
  * BIO_WRITE:
  *	g_eli_start -> G_ELI_CRYPTO_RUN -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
  */