Implement data integrity verification (data authentication) for geli(8).

Supported by:	Wheel Sp. z o.o. (http://www.wheel.pl)

6 files changed, 1101 insertions, 275 deletions

diff --git a/sys/geom/eli/g_eli.c b/sys/geom/eli/g_eli.c
index e6e872a..47d6a0a 100644
--- a/sys/geom/eli/g_eli.c
+++ b/sys/geom/eli/g_eli.c
@@ -76,10 +76,13 @@ static u_int g_eli_threads = 0;
 TUNABLE_INT("kern.geom.eli.threads", &g_eli_threads);
 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, threads, CTLFLAG_RW, &g_eli_threads, 0,
     "Number of threads doing crypto work");
+u_int g_eli_batch = 0;
+TUNABLE_INT("kern.geom.eli.batch", &g_eli_batch);
+SYSCTL_UINT(_kern_geom_eli, OID_AUTO, batch, CTLFLAG_RW, &g_eli_batch, 0,
+    "Use crypto operations batching");
 
 static int g_eli_destroy_geom(struct gctl_req *req, struct g_class *mp,
     struct g_geom *gp);
-static void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp);
 
 static g_taste_t g_eli_taste;
 static g_dumpconf_t g_eli_dumpconf;
@@ -107,7 +110,7 @@ struct g_class g_eli_class = {
  * accelerator or something like this.
  * The function updates the SID and rerun the operation.
  */
-static int
+int
 g_eli_crypto_rerun(struct cryptop *crp)
 {
 	struct g_eli_softc *sc;
@@ -140,7 +143,7 @@ g_eli_crypto_rerun(struct cryptop *crp)
  *
  * g_eli_start -> g_io_request -> G_ELI_READ_DONE -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
  */
-static void
+void
 g_eli_read_done(struct bio *bp)
 {
 	struct g_eli_softc *sc;
@@ -150,10 +153,20 @@ g_eli_read_done(struct bio *bp)
 	pbp = bp->bio_parent;
 	if (pbp->bio_error == 0)
 		pbp->bio_error = bp->bio_error;
+	/*
+	 * Do we have all sectors already?
+	 */
+	pbp->bio_inbed++;
+	if (pbp->bio_inbed < pbp->bio_children)
+		return;
 	g_destroy_bio(bp);
 	if (pbp->bio_error != 0) {
 		G_ELI_LOGREQ(0, pbp, "%s() failed", __func__);
 		pbp->bio_completed = 0;
+		if (pbp->bio_driver2 != NULL) {
+			free(pbp->bio_driver2, M_ELI);
+			pbp->bio_driver2 = NULL;
+		}
 		g_io_deliver(pbp, pbp->bio_error);
 		return;
 	}
@@ -165,69 +178,30 @@ g_eli_read_done(struct bio *bp)
 }
 
 /*
- * The function is called after we read and decrypt data.
- *
- * g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
- */
-static int
-g_eli_crypto_read_done(struct cryptop *crp)
-{
-	struct bio *bp;
-
-	if (crp->crp_etype == EAGAIN) {
-		if (g_eli_crypto_rerun(crp) == 0)
-			return (0);
-	}
-	bp = (struct bio *)crp->crp_opaque;
-	bp->bio_inbed++;
-	if (crp->crp_etype == 0) {
-		G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).",
-		    bp->bio_inbed, bp->bio_children);
-		bp->bio_completed += crp->crp_olen;
-	} else {
-		G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
-		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
-		if (bp->bio_error == 0)
-			bp->bio_error = crp->crp_etype;
-	}
-	/*
-	 * Do we have all sectors already?
-	 */
-	if (bp->bio_inbed < bp->bio_children)
-		return (0);
-	free(bp->bio_driver2, M_ELI);
-	bp->bio_driver2 = NULL;
-	if (bp->bio_error != 0) {
-		G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
-		    bp->bio_error);
-		bp->bio_completed = 0;
-	}
-	/*
-	 * Read is finished, send it up.
-	 */
-	g_io_deliver(bp, bp->bio_error);
-	return (0);
-}
-
-/*
  * The function is called after we encrypt and write data.
  *
  * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> G_ELI_WRITE_DONE -> g_io_deliver
  */
-static void
+void
 g_eli_write_done(struct bio *bp)
 {
 	struct bio *pbp;
 
 	G_ELI_LOGREQ(2, bp, "Request done.");
 	pbp = bp->bio_parent;
-	if (pbp->bio_error == 0)
-		pbp->bio_error = bp->bio_error;
+	if (pbp->bio_error == 0) {
+		if (bp->bio_error != 0)
+			pbp->bio_error = bp->bio_error;
+	}
+	/*
+	 * Do we have all sectors already?
+	 */
+	pbp->bio_inbed++;
+	if (pbp->bio_inbed < pbp->bio_children)
+		return;
 	free(pbp->bio_driver2, M_ELI);
 	pbp->bio_driver2 = NULL;
-	if (pbp->bio_error == 0)
-		pbp->bio_completed = pbp->bio_length;
-	else {
+	if (pbp->bio_error != 0) {
 		G_ELI_LOGREQ(0, pbp, "Crypto WRITE request failed (error=%d).",
 		    pbp->bio_error);
 		pbp->bio_completed = 0;
@@ -236,68 +210,11 @@ g_eli_write_done(struct bio *bp)
 	/*
 	 * Write is finished, send it up.
 	 */
+	pbp->bio_completed = pbp->bio_length;
 	g_io_deliver(pbp, pbp->bio_error);
 }
 
 /*
- * The function is called after data encryption.
- *
- * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
- */
-static int
-g_eli_crypto_write_done(struct cryptop *crp)
-{
-	struct g_geom *gp;
-	struct g_consumer *cp;
-	struct bio *bp, *cbp;
-
-	if (crp->crp_etype == EAGAIN) {
-		if (g_eli_crypto_rerun(crp) == 0)
-			return (0);
-	}
-	bp = (struct bio *)crp->crp_opaque;
-	bp->bio_inbed++;
-	if (crp->crp_etype == 0) {
-		G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
-		    bp->bio_inbed, bp->bio_children);
-	} else {
-		G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
-		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
-		if (bp->bio_error == 0)
-			bp->bio_error = crp->crp_etype;
-	}
-	/*
-	 * All sectors are already encrypted?
-	 */
-	if (bp->bio_inbed < bp->bio_children)
-		return (0);
-	bp->bio_inbed = 0;
-	bp->bio_children = 1;
-	cbp = bp->bio_driver1;
-	bp->bio_driver1 = NULL;
-	if (bp->bio_error != 0) {
-		G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
-		    bp->bio_error);
-		free(bp->bio_driver2, M_ELI);
-		bp->bio_driver2 = NULL;
-		g_destroy_bio(cbp);
-		g_io_deliver(bp, bp->bio_error);
-		return (0);
-	}
-	cbp->bio_data = bp->bio_driver2;
-	cbp->bio_done = g_eli_write_done;
-	gp = bp->bio_to->geom;
-	cp = LIST_FIRST(&gp->consumer);
-	cbp->bio_to = cp->provider;
-	G_ELI_LOGREQ(2, cbp, "Sending request.");
-	/*
-	 * Send encrypted data to the provider.
-	 */
-	g_io_request(cbp, cp);
-	return (0);
-}
-
-/*
  * This function should never be called, but GEOM made as it set ->orphan()
  * method for every geom.
  */
@@ -359,15 +276,20 @@ g_eli_start(struct bio *bp)
 	}
 	switch (bp->bio_cmd) {
 	case BIO_READ:
-		cbp->bio_done = g_eli_read_done;
-		cp = LIST_FIRST(&sc->sc_geom->consumer);
-		cbp->bio_to = cp->provider;
-		G_ELI_LOGREQ(2, cbp, "Sending request.");
-		/*
-		 * Read encrypted data from provider.
-		 */
-		g_io_request(cbp, cp);
-		break;
+		if (!(sc->sc_flags & G_ELI_FLAG_AUTH)) {
+			bp->bio_driver2 = NULL;
+			cbp->bio_done = g_eli_read_done;
+			cp = LIST_FIRST(&sc->sc_geom->consumer);
+			cbp->bio_to = cp->provider;
+			G_ELI_LOGREQ(2, cbp, "Sending request.");
+			/*
+			 * Read encrypted data from provider.
+			 */
+			g_io_request(cbp, cp);
+			break;
+		}
+		bp->bio_pflags = 255;
+		/* FALLTHROUGH */
 	case BIO_WRITE:
 		bp->bio_driver1 = cbp;
 		mtx_lock(&sc->sc_queue_mtx);
@@ -412,7 +334,7 @@ g_eli_worker(void *arg)
 		mtx_lock(&sc->sc_queue_mtx);
 		bp = bioq_takefirst(&sc->sc_queue);
 		if (bp == NULL) {
-			if ((sc->sc_flags & G_ELI_FLAG_DESTROY) != 0) {
+			if (sc->sc_flags & G_ELI_FLAG_DESTROY) {
 				LIST_REMOVE(wr, w_next);
 				crypto_freesession(wr->w_sid);
 				free(wr, M_ELI);
@@ -427,14 +349,19 @@ g_eli_worker(void *arg)
 			continue;
 		}
 		mtx_unlock(&sc->sc_queue_mtx);
-		g_eli_crypto_run(wr, bp);
+		if (bp->bio_cmd == BIO_READ && bp->bio_pflags == 255)
+			g_eli_auth_read(sc, bp);
+		else if (sc->sc_flags & G_ELI_FLAG_AUTH)
+			g_eli_auth_run(wr, bp);
+		else
+			g_eli_crypto_run(wr, bp);
 	}
 }
 
 /*
  * Here we generate IV. It is unique for every sector.
  */
-static void
+void
 g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv,
     size_t size)
 {
@@ -448,110 +375,6 @@ g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv,
 	bcopy(hash, iv, size);
 }
 
-/*
- * This is the main function responsible for cryptography (ie. communication
- * with crypto(9) subsystem).
- */
-static void
-g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
-{
-	struct g_eli_softc *sc;
-	struct cryptop *crp;
-	struct cryptodesc *crd;
-	struct uio *uio;
-	struct iovec *iov;
-	u_int i, nsec, add, secsize;
-	int err, error;
-	size_t size;
-	u_char *p, *data;
-
-	G_ELI_LOGREQ(3, bp, "%s", __func__);
-
-	bp->bio_pflags = wr->w_number;
-	sc = wr->w_softc;
-	secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize;
-	nsec = bp->bio_length / secsize;
-
-	/*
-	 * Calculate how much memory do we need.
-	 * We need separate crypto operation for every single sector.
-	 * It is much faster to calculate total amount of needed memory here and
-	 * do the allocation once instead of allocating memory in pieces (many,
-	 * many pieces).
-	 */
-	size = sizeof(*crp) * nsec;
-	size += sizeof(*crd) * nsec;
-	size += sizeof(*uio) * nsec;
-	size += sizeof(*iov) * nsec;
-	/*
-	 * If we write the data we cannot destroy current bio_data content,
-	 * so we need to allocate more memory for encrypted data.
-	 */
-	if (bp->bio_cmd == BIO_WRITE)
-		size += bp->bio_length;
-	p = malloc(size, M_ELI, M_WAITOK);
-
-	bp->bio_inbed = 0;
-	bp->bio_children = nsec;
-	bp->bio_driver2 = p;
-
-	if (bp->bio_cmd == BIO_READ)
-		data = bp->bio_data;
-	else {
-		data = p;
-		p += bp->bio_length;
-		bcopy(bp->bio_data, data, bp->bio_length);
-	}
-
-	error = 0;
-	for (i = 0, add = 0; i < nsec; i++, add += secsize) {
-		crp = (struct cryptop *)p;	p += sizeof(*crp);
-		crd = (struct cryptodesc *)p;	p += sizeof(*crd);
-		uio = (struct uio *)p;		p += sizeof(*uio);
-		iov = (struct iovec *)p;	p += sizeof(*iov);
-
-		iov->iov_len = secsize;
-		iov->iov_base = data;
-		data += secsize;
-
-		uio->uio_iov = iov;
-		uio->uio_iovcnt = 1;
-		uio->uio_segflg = UIO_SYSSPACE;
-		uio->uio_resid = secsize;
-
-		crp->crp_sid = wr->w_sid;
-		crp->crp_ilen = secsize;
-		crp->crp_olen = secsize;
-		crp->crp_opaque = (void *)bp;
-		crp->crp_buf = (void *)uio;
-		if (bp->bio_cmd == BIO_WRITE)
-			crp->crp_callback = g_eli_crypto_write_done;
-		else /* if (bp->bio_cmd == BIO_READ) */
-			crp->crp_callback = g_eli_crypto_read_done;
-		crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC | CRYPTO_F_REL;
-		crp->crp_desc = crd;
-
-		crd->crd_skip = 0;
-		crd->crd_len = secsize;
-		crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
-		if (bp->bio_cmd == BIO_WRITE)
-			crd->crd_flags |= CRD_F_ENCRYPT;
-		crd->crd_alg = sc->sc_algo;
-		crd->crd_key = sc->sc_datakey;
-		crd->crd_klen = sc->sc_keylen;
-		g_eli_crypto_ivgen(sc, bp->bio_offset + add, crd->crd_iv,
-		    sizeof(crd->crd_iv));
-		crd->crd_next = NULL;
-
-		crp->crp_etype = 0;
-		err = crypto_dispatch(crp);
-		if (error == 0)
-			error = err;
-	}
-	if (bp->bio_error == 0)
-		bp->bio_error = error;
-}
-
 int
 g_eli_read_metadata(struct g_class *mp, struct g_provider *pp,
     struct g_eli_metadata *md)
@@ -659,7 +482,7 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 	struct g_geom *gp;
 	struct g_provider *pp;
 	struct g_consumer *cp;
-	struct cryptoini cri;
+	struct cryptoini crie, cria;
 	u_int i, threads;
 	int error;
 
@@ -688,15 +511,41 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 
 	sc->sc_crypto = G_ELI_CRYPTO_SW;
 	sc->sc_flags = md->md_flags;
-	sc->sc_algo = md->md_algo;
+	sc->sc_ealgo = md->md_ealgo;
 	sc->sc_nkey = nkey;
 	/*
 	 * Remember the keys in our softc structure.
 	 */
-	bcopy(mkey, sc->sc_ivkey, sizeof(sc->sc_ivkey));
-	mkey += sizeof(sc->sc_ivkey);
-	bcopy(mkey, sc->sc_datakey, sizeof(sc->sc_datakey));
-	sc->sc_keylen = md->md_keylen;
+	g_eli_mkey_propagate(sc, mkey);
+	sc->sc_ekeylen = md->md_keylen;
+
+	if (sc->sc_flags & G_ELI_FLAG_AUTH) {
+		sc->sc_akeylen = sizeof(sc->sc_akey) * 8;
+		sc->sc_aalgo = md->md_aalgo;
+		sc->sc_alen = g_eli_hashlen(sc->sc_aalgo);
+
+		sc->sc_data_per_sector = bpp->sectorsize - sc->sc_alen;
+		/*
+		 * Some hash functions (like SHA1 and RIPEMD160) generates hash
+		 * which length is not multiple of 128 bits, but we want data
+		 * length to be multiple of 128, so we can encrypt without
+		 * padding. The line below rounds down data length to multiple
+		 * of 128 bits.
+		 */
+		sc->sc_data_per_sector -= sc->sc_data_per_sector % 16;
+
+		sc->sc_bytes_per_sector =
+		    (md->md_sectorsize - 1) / sc->sc_data_per_sector + 1;
+		sc->sc_bytes_per_sector *= bpp->sectorsize;
+		/*
+		 * Precalculate SHA256 for HMAC key generation.
+		 * This is expensive operation and we can do it only once now or
+		 * for every access to sector, so now will be much better.
+		 */
+		SHA256_Init(&sc->sc_akeyctx);
+		SHA256_Update(&sc->sc_akeyctx, sc->sc_akey,
+		    sizeof(sc->sc_akey));
+	}
 
 	/*
 	 * Precalculate SHA256 for IV generation.
@@ -744,10 +593,17 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 
 	LIST_INIT(&sc->sc_workers);
 
-	bzero(&cri, sizeof(cri));
-	cri.cri_alg = sc->sc_algo;
-	cri.cri_klen = sc->sc_keylen;
-	cri.cri_key = sc->sc_datakey;
+	bzero(&crie, sizeof(crie));
+	crie.cri_alg = sc->sc_ealgo;
+	crie.cri_klen = sc->sc_ekeylen;
+	crie.cri_key = sc->sc_ekey;
+	if (sc->sc_flags & G_ELI_FLAG_AUTH) {
+		bzero(&cria, sizeof(cria));
+		cria.cri_alg = sc->sc_aalgo;
+		cria.cri_klen = sc->sc_akeylen;
+		cria.cri_key = sc->sc_akey;
+		crie.cri_next = &cria;
+	}
 
 	threads = g_eli_threads;
 	if (threads == 0)
@@ -767,12 +623,12 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 		 * Use software cryptography, if we cannot get it.
 		 */
 		if (i == 0) {
-			error = crypto_newsession(&wr->w_sid, &cri, 1);
+			error = crypto_newsession(&wr->w_sid, &crie, 1);
 			if (error == 0)
 				sc->sc_crypto = G_ELI_CRYPTO_HW;
 		}
 		if (sc->sc_crypto == G_ELI_CRYPTO_SW)
-			error = crypto_newsession(&wr->w_sid, &cri, 0);
+			error = crypto_newsession(&wr->w_sid, &crie, 0);
 		if (error != 0) {
 			free(wr, M_ELI);
 			if (req != NULL) {
@@ -811,14 +667,22 @@ g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
 	pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX);
 	pp->sectorsize = md->md_sectorsize;
 	pp->mediasize = bpp->mediasize;
-	if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0)
+	if (!(sc->sc_flags & G_ELI_FLAG_ONETIME))
 		pp->mediasize -= bpp->sectorsize;
-	pp->mediasize -= (pp->mediasize % pp->sectorsize);
+	if (!(sc->sc_flags & G_ELI_FLAG_AUTH))
+		pp->mediasize -= (pp->mediasize % pp->sectorsize);
+	else {
+		pp->mediasize /= sc->sc_bytes_per_sector;
+		pp->mediasize *= pp->sectorsize;
+	}
+
 	g_error_provider(pp, 0);
 
 	G_ELI_DEBUG(0, "Device %s created.", pp->name);
-	G_ELI_DEBUG(0, "    Cipher: %s", g_eli_algo2str(sc->sc_algo));
-	G_ELI_DEBUG(0, "Key length: %u", sc->sc_keylen);
+	G_ELI_DEBUG(0, "Encryption: %s %u", g_eli_algo2str(sc->sc_ealgo),
+	    sc->sc_ekeylen);
+	if (sc->sc_flags & G_ELI_FLAG_AUTH)
+		G_ELI_DEBUG(0, " Integrity: %s", g_eli_algo2str(sc->sc_aalgo));
 	G_ELI_DEBUG(0, "    Crypto: %s",
 	    sc->sc_crypto == G_ELI_CRYPTO_SW ? "software" : "hardware");
 	return (gp);
@@ -996,7 +860,7 @@ g_eli_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 	if (md.md_provsize != pp->mediasize)
 		return (NULL);
 	/* Should we attach it on boot? */
-	if ((md.md_flags & G_ELI_FLAG_BOOT) == 0)
+	if (!(md.md_flags & G_ELI_FLAG_BOOT))
 		return (NULL);
 	if (md.md_keys == 0x00) {
 		G_ELI_DEBUG(0, "No valid keys on %s.", pp->name);
@@ -1118,7 +982,7 @@ g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 		int first = 1;
 
 #define ADD_FLAG(flag, name)	do {					\
-	if ((sc->sc_flags & (flag)) != 0) {				\
+	if (sc->sc_flags & (flag)) {					\
 		if (!first)						\
 			sbuf_printf(sb, ", ");				\
 		else							\
@@ -1130,13 +994,14 @@ g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 		ADD_FLAG(G_ELI_FLAG_BOOT, "BOOT");
 		ADD_FLAG(G_ELI_FLAG_WO_DETACH, "W-DETACH");
 		ADD_FLAG(G_ELI_FLAG_RW_DETACH, "RW-DETACH");
+		ADD_FLAG(G_ELI_FLAG_AUTH, "AUTH");
 		ADD_FLAG(G_ELI_FLAG_WOPEN, "W-OPEN");
 		ADD_FLAG(G_ELI_FLAG_DESTROY, "DESTROY");
 #undef  ADD_FLAG
 	}
 	sbuf_printf(sb, "</Flags>\n");
 
-	if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0) {
+	if (!(sc->sc_flags & G_ELI_FLAG_ONETIME)) {
 		sbuf_printf(sb, "%s<UsedKey>%u</UsedKey>\n", indent,
 		    sc->sc_nkey);
 	}
@@ -1153,9 +1018,15 @@ g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
 		break;
 	}
 	sbuf_printf(sb, "</Crypto>\n");
-	sbuf_printf(sb, "%s<KeyLength>%u</KeyLength>\n", indent, sc->sc_keylen);
-	sbuf_printf(sb, "%s<Cipher>%s</Cipher>\n", indent,
-	    g_eli_algo2str(sc->sc_algo));
+	if (sc->sc_flags & G_ELI_FLAG_AUTH) {
+		sbuf_printf(sb,
+		    "%s<AuthenticationAlgorithm>%s</AuthenticationAlgorithm>\n",
+		    indent, g_eli_algo2str(sc->sc_aalgo));
+	}
+	sbuf_printf(sb, "%s<KeyLength>%u</KeyLength>\n", indent,
+	    sc->sc_ekeylen);
+	sbuf_printf(sb, "%s<EncryptionAlgorithm>%s</EncryptionAlgorithm>\n", indent,
+	    g_eli_algo2str(sc->sc_ealgo));
 }
 
 DECLARE_GEOM_CLASS(g_eli_class, g_eli);
diff --git a/sys/geom/eli/g_eli.h b/sys/geom/eli/g_eli.h
index 363b38b..82ce64f 100644
--- a/sys/geom/eli/g_eli.h
+++ b/sys/geom/eli/g_eli.h
@@ -54,8 +54,10 @@
 /*
  * Version history:
  * 0 - Initial version number.
+ * 1 - Added data authentication support (md_aalgo field and
+ *     G_ELI_FLAG_AUTH flag).
  */
-#define	G_ELI_VERSION		0
+#define	G_ELI_VERSION		1
 
 /* Use random, onetime keys. */
 #define	G_ELI_FLAG_ONETIME	0x00000001
@@ -65,17 +67,21 @@
 #define	G_ELI_FLAG_WO_DETACH	0x00000004
 /* Detach on last close. */
 #define	G_ELI_FLAG_RW_DETACH	0x00000008
+/* Provide data authentication. */
+#define	G_ELI_FLAG_AUTH		0x00000010
 /* Provider was open for writing. */
 #define	G_ELI_FLAG_WOPEN	0x00010000
 /* Destroy device. */
 #define	G_ELI_FLAG_DESTROY	0x00020000
 
 #define	SHA512_MDLEN		64
+#define	G_ELI_AUTH_SECKEYLEN	SHA256_DIGEST_LENGTH
 
 #define	G_ELI_MAXMKEYS		2
 #define	G_ELI_MAXKEYLEN		64
 #define	G_ELI_USERKEYLEN	G_ELI_MAXKEYLEN
 #define	G_ELI_DATAKEYLEN	G_ELI_MAXKEYLEN
+#define	G_ELI_AUTHKEYLEN	G_ELI_MAXKEYLEN
 #define	G_ELI_IVKEYLEN		G_ELI_MAXKEYLEN
 #define	G_ELI_SALTLEN		64
 #define	G_ELI_DATAIVKEYLEN	(G_ELI_DATAKEYLEN + G_ELI_IVKEYLEN)
@@ -85,6 +91,7 @@
 #ifdef _KERNEL
 extern u_int g_eli_debug;
 extern u_int g_eli_overwrites;
+extern u_int g_eli_batch;
 
 #define	G_ELI_CRYPTO_HW		1
 #define	G_ELI_CRYPTO_SW		2
@@ -123,13 +130,21 @@ struct g_eli_worker {
 struct g_eli_softc {
 	struct g_geom	*sc_geom;
 	u_int		 sc_crypto;
-	uint8_t		 sc_datakey[G_ELI_DATAKEYLEN];
+	uint8_t		 sc_mkey[G_ELI_DATAIVKEYLEN];
+	uint8_t		 sc_ekey[G_ELI_DATAKEYLEN];
+	u_int		 sc_ealgo;
+	u_int		 sc_ekeylen;
+	uint8_t		 sc_akey[G_ELI_AUTHKEYLEN];
+	u_int		 sc_aalgo;
+	u_int		 sc_akeylen;
+	u_int		 sc_alen;
+	SHA256_CTX	 sc_akeyctx;
 	uint8_t		 sc_ivkey[G_ELI_IVKEYLEN];
 	SHA256_CTX	 sc_ivctx;
-	u_int		 sc_algo;
-	u_int		 sc_keylen;
 	int		 sc_nkey;
 	uint32_t	 sc_flags;
+	u_int		 sc_bytes_per_sector;
+	u_int		 sc_data_per_sector;
 
 	/* Only for software cryptography. */
 	struct bio_queue_head sc_queue;
@@ -143,8 +158,9 @@ struct g_eli_metadata {
 	char		md_magic[16];	/* Magic value. */
 	uint32_t	md_version;	/* Version number. */
 	uint32_t	md_flags;	/* Additional flags. */
-	uint16_t	md_algo;	/* Encryption algorithm. */
+	uint16_t	md_ealgo;	/* Encryption algorithm. */
 	uint16_t	md_keylen;	/* Key length. */
+	uint16_t	md_aalgo;	/* Authentication algorithm. */
 	uint64_t	md_provsize;	/* Provider's size. */
 	uint32_t	md_sectorsize;	/* Sector size. */
 	uint8_t		md_keys;	/* Available keys. */
@@ -165,8 +181,9 @@ eli_metadata_encode(struct g_eli_metadata *md, u_char *data)
 	bcopy(md->md_magic, p, sizeof(md->md_magic)); p += sizeof(md->md_magic);
 	le32enc(p, md->md_version);	p += sizeof(md->md_version);
 	le32enc(p, md->md_flags);	p += sizeof(md->md_flags);
-	le16enc(p, md->md_algo);	p += sizeof(md->md_algo);
+	le16enc(p, md->md_ealgo);	p += sizeof(md->md_ealgo);
 	le16enc(p, md->md_keylen);	p += sizeof(md->md_keylen);
+	le16enc(p, md->md_aalgo);	p += sizeof(md->md_aalgo);
 	le64enc(p, md->md_provsize);	p += sizeof(md->md_provsize);
 	le32enc(p, md->md_sectorsize);	p += sizeof(md->md_sectorsize);
 	*p = md->md_keys;		p += sizeof(md->md_keys);
@@ -186,7 +203,7 @@ eli_metadata_decode_v0(const u_char *data, struct g_eli_metadata *md)
 
 	p = data + sizeof(md->md_magic) + sizeof(md->md_version);
 	md->md_flags = le32dec(p);	p += sizeof(md->md_flags);
-	md->md_algo = le16dec(p);	p += sizeof(md->md_algo);
+	md->md_ealgo = le16dec(p);	p += sizeof(md->md_ealgo);
 	md->md_keylen = le16dec(p);	p += sizeof(md->md_keylen);
 	md->md_provsize = le64dec(p);	p += sizeof(md->md_provsize);
 	md->md_sectorsize = le32dec(p);	p += sizeof(md->md_sectorsize);
@@ -202,6 +219,30 @@ eli_metadata_decode_v0(const u_char *data, struct g_eli_metadata *md)
 	return (0);
 }
 static __inline int
+eli_metadata_decode_v1(const u_char *data, struct g_eli_metadata *md)
+{
+	MD5_CTX ctx;
+	const u_char *p;
+
+	p = data + sizeof(md->md_magic) + sizeof(md->md_version);
+	md->md_flags = le32dec(p);	p += sizeof(md->md_flags);
+	md->md_ealgo = le16dec(p);	p += sizeof(md->md_ealgo);
+	md->md_keylen = le16dec(p);	p += sizeof(md->md_keylen);
+	md->md_aalgo = le16dec(p);	p += sizeof(md->md_aalgo);
+	md->md_provsize = le64dec(p);	p += sizeof(md->md_provsize);
+	md->md_sectorsize = le32dec(p);	p += sizeof(md->md_sectorsize);
+	md->md_keys = *p;		p += sizeof(md->md_keys);
+	md->md_iterations = le32dec(p);	p += sizeof(md->md_iterations);
+	bcopy(p, md->md_salt, sizeof(md->md_salt)); p += sizeof(md->md_salt);
+	bcopy(p, md->md_mkeys, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys);
+	MD5Init(&ctx);
+	MD5Update(&ctx, data, p - data);
+	MD5Final(md->md_hash, &ctx);
+	if (bcmp(md->md_hash, p, 16) != 0)
+		return (EINVAL);
+	return (0);
+}
+static __inline int
 eli_metadata_decode(const u_char *data, struct g_eli_metadata *md)
 {
 	int error;
@@ -212,6 +253,9 @@ eli_metadata_decode(const u_char *data, struct g_eli_metadata *md)
 	case 0:
 		error = eli_metadata_decode_v0(data, md);
 		break;
+	case 1:
+		error = eli_metadata_decode_v1(data, md);
+		break;
 	default:
 		error = EINVAL;
 		break;
@@ -221,7 +265,7 @@ eli_metadata_decode(const u_char *data, struct g_eli_metadata *md)
 #endif	/* !_OpenSSL */
 
 static __inline u_int
-g_eli_str2algo(const char *name)
+g_eli_str2ealgo(const char *name)
 {
 
 	if (strcasecmp("null", name) == 0)
@@ -235,6 +279,25 @@ g_eli_str2algo(const char *name)
 	return (CRYPTO_ALGORITHM_MIN - 1);
 }
 
+static __inline u_int
+g_eli_str2aalgo(const char *name)
+{
+
+	if (strcasecmp("hmac/md5", name) == 0)
+		return (CRYPTO_MD5_HMAC);
+	else if (strcasecmp("hmac/sha1", name) == 0)
+		return (CRYPTO_SHA1_HMAC);
+	else if (strcasecmp("hmac/ripemd160", name) == 0)
+		return (CRYPTO_RIPEMD160_HMAC);
+	else if (strcasecmp("hmac/sha256", name) == 0)
+		return (CRYPTO_SHA2_256_HMAC);
+	else if (strcasecmp("hmac/sha384", name) == 0)
+		return (CRYPTO_SHA2_384_HMAC);
+	else if (strcasecmp("hmac/sha512", name) == 0)
+		return (CRYPTO_SHA2_512_HMAC);
+	return (CRYPTO_ALGORITHM_MIN - 1);
+}
+
 static __inline const char *
 g_eli_algo2str(u_int algo)
 {
@@ -243,11 +306,23 @@ g_eli_algo2str(u_int algo)
 	case CRYPTO_NULL_CBC:
 		return ("NULL");
 	case CRYPTO_AES_CBC:
-		return ("AES");
+		return ("AES-CBC");
 	case CRYPTO_BLF_CBC:
-		return ("Blowfish");
+		return ("Blowfish-CBC");
 	case CRYPTO_3DES_CBC:
-		return ("3DES");
+		return ("3DES-CBC");
+	case CRYPTO_MD5_HMAC:
+		return ("HMAC/MD5");
+	case CRYPTO_SHA1_HMAC:
+		return ("HMAC/SHA1");
+	case CRYPTO_RIPEMD160_HMAC:
+		return ("HMAC/RIPEMD160");
+	case CRYPTO_SHA2_256_HMAC:
+		return ("HMAC/SHA256");
+	case CRYPTO_SHA2_384_HMAC:
+		return ("HMAC/SHA384");
+	case CRYPTO_SHA2_512_HMAC:
+		return ("HMAC/SHA512");
 	}
 	return ("unknown");
 }
@@ -262,8 +337,10 @@ eli_metadata_dump(const struct g_eli_metadata *md)
 	printf("     magic: %s\n", md->md_magic);
 	printf("   version: %u\n", (u_int)md->md_version);
 	printf("     flags: 0x%x\n", (u_int)md->md_flags);
-	printf("      algo: %s\n", g_eli_algo2str(md->md_algo));
+	printf("     ealgo: %s\n", g_eli_algo2str(md->md_ealgo));
 	printf("    keylen: %u\n", (u_int)md->md_keylen);
+	if (md->md_flags & G_ELI_FLAG_AUTH)
+		printf("     aalgo: %s\n", g_eli_algo2str(md->md_aalgo));
 	printf("  provsize: %ju\n", (uintmax_t)md->md_provsize);
 	printf("sectorsize: %u\n", (u_int)md->md_sectorsize);
 	printf("      keys: 0x%02x\n", (u_int)md->md_keys);
@@ -329,6 +406,27 @@ g_eli_keylen(u_int algo, u_int keylen)
 	}
 }
 
+static __inline u_int
+g_eli_hashlen(u_int algo)
+{
+
+	switch (algo) {
+	case CRYPTO_MD5_HMAC:
+		return (16);
+	case CRYPTO_SHA1_HMAC:
+		return (20);
+	case CRYPTO_RIPEMD160_HMAC:
+		return (20);
+	case CRYPTO_SHA2_256_HMAC:
+		return (32);
+	case CRYPTO_SHA2_384_HMAC:
+		return (48);
+	case CRYPTO_SHA2_512_HMAC:
+		return (64);
+	}
+	return (0);
+}
+
 #ifdef _KERNEL
 int g_eli_read_metadata(struct g_class *mp, struct g_provider *pp,
     struct g_eli_metadata *md);
@@ -339,6 +437,17 @@ int g_eli_destroy(struct g_eli_softc *sc, boolean_t force);
 
 int g_eli_access(struct g_provider *pp, int dr, int dw, int de);
 void g_eli_config(struct gctl_req *req, struct g_class *mp, const char *verb);
+
+void g_eli_read_done(struct bio *bp);
+void g_eli_write_done(struct bio *bp);
+int g_eli_crypto_rerun(struct cryptop *crp);
+void g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv,
+    size_t size);
+
+void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp);
+
+void g_eli_auth_read(struct g_eli_softc *sc, struct bio *bp);
+void g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp);
 #endif
 
 void g_eli_mkey_hmac(unsigned char *mkey, const unsigned char *key);
@@ -346,6 +455,9 @@ int g_eli_mkey_decrypt(const struct g_eli_metadata *md,
     const unsigned char *key, unsigned char *mkey, unsigned *nkeyp);
 int g_eli_mkey_encrypt(unsigned algo, const unsigned char *key, unsigned keylen,
     unsigned char *mkey);
+#ifdef _KERNEL
+void g_eli_mkey_propagate(struct g_eli_softc *sc, const unsigned char *mkey);
+#endif
 
 int g_eli_crypto_encrypt(u_int algo, u_char *data, size_t datasize,
     const u_char *key, size_t keysize);
diff --git a/sys/geom/eli/g_eli_ctl.c b/sys/geom/eli/g_eli_ctl.c
index e8297d3..a7b6b45 100644
--- a/sys/geom/eli/g_eli_ctl.c
+++ b/sys/geom/eli/g_eli_ctl.c
@@ -250,15 +250,30 @@ g_eli_ctl_onetime(struct gctl_req *req, struct g_class *mp)
 	if (*detach)
 		md.md_flags |= G_ELI_FLAG_WO_DETACH;
 
-	name = gctl_get_asciiparam(req, "algo");
+	name = gctl_get_asciiparam(req, "aalgo");
 	if (name == NULL) {
-		gctl_error(req, "No '%s' argument.", "algo");
+		gctl_error(req, "No '%s' argument.", "aalgo");
 		return;
 	}
-	md.md_algo = g_eli_str2algo(name);
-	if (md.md_algo < CRYPTO_ALGORITHM_MIN ||
-	    md.md_algo > CRYPTO_ALGORITHM_MAX) {
-		gctl_error(req, "Invalid '%s' argument.", "algo");
+	if (strcmp(name, "none") != 0) {
+		md.md_aalgo = g_eli_str2aalgo(name);
+		if (md.md_aalgo < CRYPTO_ALGORITHM_MIN ||
+		    md.md_aalgo > CRYPTO_ALGORITHM_MAX) {
+			gctl_error(req, "Invalid authentication algorithm.");
+			return;
+		}
+		md.md_flags |= G_ELI_FLAG_AUTH;
+	}
+
+	name = gctl_get_asciiparam(req, "ealgo");
+	if (name == NULL) {
+		gctl_error(req, "No '%s' argument.", "ealgo");
+		return;
+	}
+	md.md_ealgo = g_eli_str2ealgo(name);
+	if (md.md_ealgo < CRYPTO_ALGORITHM_MIN ||
+	    md.md_ealgo > CRYPTO_ALGORITHM_MAX) {
+		gctl_error(req, "Invalid encryption algorithm.");
 		return;
 	}
 
@@ -267,7 +282,7 @@ g_eli_ctl_onetime(struct gctl_req *req, struct g_class *mp)
 		gctl_error(req, "No '%s' argument.", "keylen");
 		return;
 	}
-	md.md_keylen = g_eli_keylen(md.md_algo, *keylen);
+	md.md_keylen = g_eli_keylen(md.md_ealgo, *keylen);
 	if (md.md_keylen == 0) {
 		gctl_error(req, "Invalid '%s' argument.", "keylen");
 		return;
@@ -395,12 +410,10 @@ g_eli_ctl_setkey(struct gctl_req *req, struct g_class *mp)
 	mkeydst = md.md_mkeys + nkey * G_ELI_MKEYLEN;
 	md.md_keys |= (1 << nkey);
 
-	bcopy(sc->sc_ivkey, mkeydst, sizeof(sc->sc_ivkey));
-	bcopy(sc->sc_datakey, mkeydst + sizeof(sc->sc_ivkey),
-	    sizeof(sc->sc_datakey));
+	bcopy(sc->sc_mkey, mkeydst, sizeof(sc->sc_mkey));
 
 	/* Encrypt Master Key with the new key. */
-	error = g_eli_mkey_encrypt(md.md_algo, key, md.md_keylen, mkeydst);
+	error = g_eli_mkey_encrypt(md.md_ealgo, key, md.md_keylen, mkeydst);
 	bzero(key, sizeof(key));
 	if (error != 0) {
 		bzero(&md, sizeof(md));
diff --git a/sys/geom/eli/g_eli_integrity.c b/sys/geom/eli/g_eli_integrity.c
new file mode 100644
index 0000000..e30b0a8
--- /dev/null
+++ b/sys/geom/eli/g_eli_integrity.c
@@ -0,0 +1,530 @@
+/*-
+ * Copyright (c) 2005-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/linker.h>
+#include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/bio.h>
+#include <sys/sysctl.h>
+#include <sys/malloc.h>
+#include <sys/kthread.h>
+#include <sys/proc.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+#include <sys/uio.h>
+#include <sys/vnode.h>
+
+#include <vm/uma.h>
+
+#include <geom/geom.h>
+#include <geom/eli/g_eli.h>
+#include <geom/eli/pkcs5v2.h>
+
+/*
+ * The data layout description when integrity verification is configured.
+ *
+ * One of the most important assumption here is that authenticated data and its
+ * HMAC has to be stored in the same place (namely in the same sector) to make
+ * it work reliable.
+ * The problem is that file systems work only with sectors that are multiple of
+ * 512 bytes and a power of two number.
+ * My idea to implement it is as follows.
+ * Let's store HMAC in sector. This is a must. This leaves us 480 bytes for
+ * data. We can't use that directly (ie. we can't create provider with 480 bytes
+ * sector size). We need another sector from where we take only 32 bytes of data
+ * and we store HMAC of this data as well. This takes two sectors from the
+ * original provider at the input and leaves us one sector of authenticated data
+ * at the output. Not very efficient, but you got the idea.
+ * Now, let's assume, we want to create provider with 4096 bytes sector.
+ * To output 4096 bytes of authenticated data we need 8x480 plus 1x256, so we
+ * need nine 512-bytes sectors at the input to get one 4096-bytes sector at the
+ * output. That's better. With 4096 bytes sector we can use 89% of size of the
+ * original provider. I find it as an acceptable cost.
+ * The reliability comes from the fact, that every HMAC stored inside the sector
+ * is calculated only for the data in the same sector, so its impossible to
+ * write new data and leave old HMAC or vice versa.
+ *
+ * And here is the picture:
+ *
+ * da0: +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+
+ *      |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |256b |
+ *      |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data |
+ *      +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+
+ *      |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |288 bytes |
+ *      +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ |224 unused|
+ *                                                                                                      +----------+
+ * da0.eli: +----+----+----+----+----+----+----+----+----+
+ *          |480b|480b|480b|480b|480b|480b|480b|480b|256b|
+ *          +----+----+----+----+----+----+----+----+----+
+ *          |                 4096 bytes                 |
+ *          +--------------------------------------------+
+ *
+ * PS. You can use any sector size with geli(8). My example is using 4kB,
+ *     because it's most efficient. For 8kB sectors you need 2 extra sectors,
+ *     so the cost is the same as for 4kB sectors.
+ */
+
+/*
+ * Code paths:
+ * BIO_READ:
+ *	g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> g_eli_auth_read_done -> g_io_deliver
+ * BIO_WRITE:
+ *	g_eli_start -> g_eli_auth_run -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
+ */
+
+MALLOC_DECLARE(M_ELI);
+
+/*
+ * Here we generate key for HMAC. Every sector has its own HMAC key, so it is
+ * not possible to copy sectors.
+ * We cannot depend on fact, that every sector has its own IV, because different
+ * IV doesn't change HMAC, when we use encrypt-then-authenticate method.
+ */
+static void
+g_eli_auth_keygen(struct g_eli_softc *sc, off_t offset, u_char *key)
+{
+	SHA256_CTX ctx;
+
+	/* Copy precalculated SHA256 context. */
+	bcopy(&sc->sc_akeyctx, &ctx, sizeof(ctx));
+	SHA256_Update(&ctx, (uint8_t *)&offset, sizeof(offset));
+	SHA256_Final(key, &ctx);
+}
+
+/*
+ * The function is called after we read and decrypt data.
+ *
+ * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> G_ELI_AUTH_READ_DONE -> g_io_deliver
+ */
+static int
+g_eli_auth_read_done(struct cryptop *crp)
+{
+	struct bio *bp;
+
+	if (crp->crp_etype == EAGAIN) {
+		if (g_eli_crypto_rerun(crp) == 0)
+			return (0);
+	}
+	bp = (struct bio *)crp->crp_opaque;
+	bp->bio_inbed++;
+	if (crp->crp_etype == 0) {
+		bp->bio_completed += crp->crp_olen;
+		G_ELI_DEBUG(3, "Crypto READ request done (%d/%d) (add=%jd completed=%jd).",
+		    bp->bio_inbed, bp->bio_children, (intmax_t)crp->crp_olen, (intmax_t)bp->bio_completed);
+	} else {
+		G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
+		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
+		if (bp->bio_error == 0)
+			bp->bio_error = crp->crp_etype;
+	}
+	/*
+	 * Do we have all sectors already?
+	 */
+	if (bp->bio_inbed < bp->bio_children)
+		return (0);
+	if (bp->bio_error == 0) {
+		struct g_eli_softc *sc;
+		u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
+		u_char *srcdata, *dstdata, *auth;
+		off_t coroff, corsize;
+
+		/*
+		 * Verify data integrity based on calculated and read HMACs.
+		 */
+		sc = bp->bio_to->geom->softc;
+		/* Sectorsize of decrypted provider eg. 4096. */
+		decr_secsize = bp->bio_to->sectorsize;
+		/* The real sectorsize of encrypted provider, eg. 512. */
+		encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
+		/* Number of data bytes in one encrypted sector, eg. 480. */
+		data_secsize = sc->sc_data_per_sector;
+		/* Number of sectors from decrypted provider, eg. 2. */
+		nsec = bp->bio_length / decr_secsize;
+		/* Number of sectors from encrypted provider, eg. 18. */
+		nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
+		/* Last sector number in every big sector, eg. 9. */
+		lsec = sc->sc_bytes_per_sector / encr_secsize;
+
+		srcdata = bp->bio_driver2;
+		dstdata = bp->bio_data;
+		auth = srcdata + encr_secsize * nsec;
+		coroff = -1;
+		corsize = 0;
+
+		for (i = 1; i <= nsec; i++) {
+			data_secsize = sc->sc_data_per_sector;
+			if ((i % lsec) == 0)
+				data_secsize = decr_secsize % data_secsize;
+			if (bcmp(srcdata, auth, sc->sc_alen) != 0) {
+				/*
+				 * Curruption detected, remember the offset if
+				 * this is the first corrupted sector and
+				 * increase size.
+				 */
+				if (bp->bio_error == 0)
+					bp->bio_error = -1;
+				if (coroff == -1) {
+					coroff = bp->bio_offset +
+					    (dstdata - (u_char *)bp->bio_data);
+				}
+				corsize += data_secsize;
+			} else {
+				/*
+				 * No curruption, good.
+				 * Report previous corruption if there was one.
+				 */
+				if (coroff != -1) {
+					G_ELI_DEBUG(0, "%s: %jd bytes "
+					    "corrupted at offset %jd.",
+					    sc->sc_name, (intmax_t)corsize,
+					    (intmax_t)coroff);
+					coroff = -1;
+					corsize = 0;
+				}
+				bcopy(srcdata + sc->sc_alen, dstdata,
+				    data_secsize);
+			}
+			srcdata += encr_secsize;
+			dstdata += data_secsize;
+			auth += sc->sc_alen;
+		}
+		/* Report previous corruption if there was one. */
+		if (coroff != -1) {
+			G_ELI_DEBUG(0, "%s: %jd bytes corrupted at offset %jd.",
+			    sc->sc_name, (intmax_t)corsize, (intmax_t)coroff);
+		}
+	}
+	free(bp->bio_driver2, M_ELI);
+	bp->bio_driver2 = NULL;
+	if (bp->bio_error != 0) {
+		if (bp->bio_error == -1)
+			bp->bio_error = EINVAL;
+		else {
+			G_ELI_LOGREQ(0, bp,
+			    "Crypto READ request failed (error=%d).",
+			    bp->bio_error);
+		}
+		bp->bio_completed = 0;
+	}
+	/*
+	 * Read is finished, send it up.
+	 */
+	g_io_deliver(bp, bp->bio_error);
+	return (0);
+}
+
+/*
+ * The function is called after data encryption.
+ *
+ * g_eli_start -> g_eli_auth_run -> G_ELI_AUTH_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
+ */
+static int
+g_eli_auth_write_done(struct cryptop *crp)
+{
+	struct g_eli_softc *sc;
+	struct g_consumer *cp;
+	struct bio *bp, *cbp, *cbp2;
+	u_int nsec;
+
+	if (crp->crp_etype == EAGAIN) {
+		if (g_eli_crypto_rerun(crp) == 0)
+			return (0);
+	}
+	bp = (struct bio *)crp->crp_opaque;
+	bp->bio_inbed++;
+	if (crp->crp_etype == 0) {
+		G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
+		    bp->bio_inbed, bp->bio_children);
+	} else {
+		G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
+		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
+		if (bp->bio_error == 0)
+			bp->bio_error = crp->crp_etype;
+	}
+	/*
+	 * All sectors are already encrypted?
+	 */
+	if (bp->bio_inbed < bp->bio_children)
+		return (0);
+	if (bp->bio_error != 0) {
+		G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
+		    bp->bio_error);
+		free(bp->bio_driver2, M_ELI);
+		bp->bio_driver2 = NULL;
+		cbp = bp->bio_driver1;
+		bp->bio_driver1 = NULL;
+		g_destroy_bio(cbp);
+		g_io_deliver(bp, bp->bio_error);
+		return (0);
+	}
+	sc = bp->bio_to->geom->softc;
+	cp = LIST_FIRST(&sc->sc_geom->consumer);
+	cbp = bp->bio_driver1;
+	bp->bio_driver1 = NULL;
+	cbp->bio_to = cp->provider;
+	cbp->bio_done = g_eli_write_done;
+
+	/* Number of sectors from decrypted provider, eg. 1. */
+	nsec = bp->bio_length / bp->bio_to->sectorsize;
+	/* Number of sectors from encrypted provider, eg. 9. */
+	nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize;
+
+	cbp->bio_length = cp->provider->sectorsize * nsec;
+	cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
+	cbp->bio_data = bp->bio_driver2;
+
+	/*
+	 * We write more than what is requested, so we have to be ready to write
+	 * more than MAXPHYS.
+	 */
+	cbp2 = NULL;
+	if (cbp->bio_length > MAXPHYS) {
+		cbp2 = g_duplicate_bio(bp);
+		cbp2->bio_length = cbp->bio_length - MAXPHYS;
+		cbp2->bio_data = cbp->bio_data + MAXPHYS;
+		cbp2->bio_offset = cbp->bio_offset + MAXPHYS;
+		cbp2->bio_to = cp->provider;
+		cbp2->bio_done = g_eli_write_done;
+		cbp->bio_length = MAXPHYS;
+	}
+	/*
+	 * Send encrypted data to the provider.
+	 */
+	G_ELI_LOGREQ(2, cbp, "Sending request.");
+	bp->bio_inbed = 0;
+	bp->bio_children = (cbp2 != NULL ? 2 : 1);
+	g_io_request(cbp, cp);
+	if (cbp2 != NULL) {
+		G_ELI_LOGREQ(2, cbp2, "Sending request.");
+		g_io_request(cbp2, cp);
+	}
+	return (0);
+}
+
+void
+g_eli_auth_read(struct g_eli_softc *sc, struct bio *bp)
+{
+	struct g_consumer *cp;
+	struct bio *cbp, *cbp2;
+	size_t size;
+	off_t nsec;
+
+	bp->bio_pflags = 0;
+
+	cp = LIST_FIRST(&sc->sc_geom->consumer);
+	cbp = bp->bio_driver1;
+	bp->bio_driver1 = NULL;
+	cbp->bio_to = cp->provider;
+	cbp->bio_done = g_eli_read_done;
+
+	/* Number of sectors from decrypted provider, eg. 1. */
+	nsec = bp->bio_length / bp->bio_to->sectorsize;
+	/* Number of sectors from encrypted provider, eg. 9. */
+	nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize;
+
+	cbp->bio_length = cp->provider->sectorsize * nsec;
+	size = cbp->bio_length;
+	size += sc->sc_alen * nsec;
+	size += sizeof(struct cryptop) * nsec;
+	size += sizeof(struct cryptodesc) * nsec * 2;
+	size += G_ELI_AUTH_SECKEYLEN * nsec;
+	size += sizeof(struct uio) * nsec;
+	size += sizeof(struct iovec) * nsec;
+	cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
+	bp->bio_driver2 = malloc(size, M_ELI, M_WAITOK);
+	cbp->bio_data = bp->bio_driver2;
+
+	/*
+	 * We read more than what is requested, so we have to be ready to read
+	 * more than MAXPHYS.
+	 */
+	cbp2 = NULL;
+	if (cbp->bio_length > MAXPHYS) {
+		cbp2 = g_duplicate_bio(bp);
+		cbp2->bio_length = cbp->bio_length - MAXPHYS;
+		cbp2->bio_data = cbp->bio_data + MAXPHYS;
+		cbp2->bio_offset = cbp->bio_offset + MAXPHYS;
+		cbp2->bio_to = cp->provider;
+		cbp2->bio_done = g_eli_read_done;
+		cbp->bio_length = MAXPHYS;
+	}
+	/*
+	 * Read encrypted data from provider.
+	 */
+	G_ELI_LOGREQ(2, cbp, "Sending request.");
+	g_io_request(cbp, cp);
+	if (cbp2 != NULL) {
+		G_ELI_LOGREQ(2, cbp2, "Sending request.");
+		g_io_request(cbp2, cp);
+	}
+}
+
+/*
+ * This is the main function responsible for cryptography (ie. communication
+ * with crypto(9) subsystem).
+ */
+void
+g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp)
+{
+	struct g_eli_softc *sc;
+	struct cryptop *crp;
+	struct cryptodesc *crde, *crda;
+	struct uio *uio;
+	struct iovec *iov;
+	u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
+	off_t dstoff;
+	int err, error;
+	u_char *p, *data, *auth, *authkey, *plaindata;
+
+	G_ELI_LOGREQ(3, bp, "%s", __func__);
+
+	bp->bio_pflags = wr->w_number;
+	sc = wr->w_softc;
+	/* Sectorsize of decrypted provider eg. 4096. */
+	decr_secsize = bp->bio_to->sectorsize;
+	/* The real sectorsize of encrypted provider, eg. 512. */
+	encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
+	/* Number of data bytes in one encrypted sector, eg. 480. */
+	data_secsize = sc->sc_data_per_sector;
+	/* Number of sectors from decrypted provider, eg. 2. */
+	nsec = bp->bio_length / decr_secsize;
+	/* Number of sectors from encrypted provider, eg. 18. */
+	nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
+	/* Last sector number in every big sector, eg. 9. */
+	lsec = sc->sc_bytes_per_sector / encr_secsize;
+	/* Destination offset, used for IV generation. */
+	dstoff = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
+
+	plaindata = bp->bio_data;
+	if (bp->bio_cmd == BIO_READ) {
+		data = bp->bio_driver2;
+		auth = data + encr_secsize * nsec;
+		p = auth + sc->sc_alen * nsec;
+	} else {
+		size_t size;
+
+		size = encr_secsize * nsec;
+		size += sizeof(*crp) * nsec;
+		size += sizeof(*crde) * nsec;
+		size += sizeof(*crda) * nsec;
+		size += G_ELI_AUTH_SECKEYLEN * nsec;
+		size += sizeof(*uio) * nsec;
+		size += sizeof(*iov) * nsec;
+		data = malloc(size, M_ELI, M_WAITOK);
+		bp->bio_driver2 = data;
+		p = data + encr_secsize * nsec;
+	}
+	bp->bio_inbed = 0;
+	bp->bio_children = nsec;
+
+	error = 0;
+	for (i = 1; i <= nsec; i++, dstoff += encr_secsize) {
+		crp = (struct cryptop *)p;	p += sizeof(*crp);
+		crde = (struct cryptodesc *)p;	p += sizeof(*crde);
+		crda = (struct cryptodesc *)p;	p += sizeof(*crda);
+		authkey = (u_char *)p;		p += G_ELI_AUTH_SECKEYLEN;
+		uio = (struct uio *)p;		p += sizeof(*uio);
+		iov = (struct iovec *)p;	p += sizeof(*iov);
+
+		data_secsize = sc->sc_data_per_sector;
+		if ((i % lsec) == 0)
+			data_secsize = decr_secsize % data_secsize;
+
+		if (bp->bio_cmd == BIO_READ) {
+			/* Remember read HMAC. */
+			bcopy(data, auth, sc->sc_alen);
+			auth += sc->sc_alen;
+			/* TODO: bzero(9) can be commented out later. */
+			bzero(data, sc->sc_alen);
+		} else {
+			bcopy(plaindata, data + sc->sc_alen, data_secsize);
+			plaindata += data_secsize;
+		}
+
+		iov->iov_len = sc->sc_alen + data_secsize;
+		iov->iov_base = data;
+		data += encr_secsize;
+
+		uio->uio_iov = iov;
+		uio->uio_iovcnt = 1;
+		uio->uio_segflg = UIO_SYSSPACE;
+		uio->uio_resid = iov->iov_len;
+
+		crp->crp_sid = wr->w_sid;
+		crp->crp_ilen = data_secsize;
+		crp->crp_olen = data_secsize;
+		crp->crp_opaque = (void *)bp;
+		crp->crp_buf = (void *)uio;
+		crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC | CRYPTO_F_REL;
+		if (g_eli_batch)
+			crp->crp_flags |= CRYPTO_F_BATCH;
+		if (bp->bio_cmd == BIO_WRITE) {
+			crp->crp_callback = g_eli_auth_write_done;
+			crp->crp_desc = crde;
+			crde->crd_next = crda;
+			crda->crd_next = NULL;
+		} else {
+			crp->crp_callback = g_eli_auth_read_done;
+			crp->crp_desc = crda;
+			crda->crd_next = crde;
+			crde->crd_next = NULL;
+		}
+
+		crde->crd_skip = sc->sc_alen;
+		crde->crd_len = data_secsize;
+		crde->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
+		if (bp->bio_cmd == BIO_WRITE)
+			crde->crd_flags |= CRD_F_ENCRYPT;
+		crde->crd_alg = sc->sc_ealgo;
+		crde->crd_key = sc->sc_ekey;
+		crde->crd_klen = sc->sc_ekeylen;
+		g_eli_crypto_ivgen(sc, dstoff, crde->crd_iv,
+		    sizeof(crde->crd_iv));
+
+		crda->crd_skip = sc->sc_alen;
+		crda->crd_len = data_secsize;
+		crda->crd_inject = 0;
+		crda->crd_flags = CRD_F_KEY_EXPLICIT;
+		crda->crd_alg = sc->sc_aalgo;
+		g_eli_auth_keygen(sc, dstoff, authkey);
+		crda->crd_key = authkey;
+		crda->crd_klen = G_ELI_AUTH_SECKEYLEN * 8;
+
+		crp->crp_etype = 0;
+		err = crypto_dispatch(crp);
+		if (err != 0 && error == 0)
+			error = err;
+	}
+	if (bp->bio_error == 0)
+		bp->bio_error = error;
+}
diff --git a/sys/geom/eli/g_eli_key.c b/sys/geom/eli/g_eli_key.c
index aef6697..3f7be95 100644
--- a/sys/geom/eli/g_eli_key.c
+++ b/sys/geom/eli/g_eli_key.c
@@ -123,10 +123,10 @@ g_eli_mkey_decrypt(const struct g_eli_metadata *md, const unsigned char *key,
 	nkey = 0;
 	for (nkey = 0; nkey < G_ELI_MAXMKEYS; nkey++, mmkey += G_ELI_MKEYLEN) {
 		bit = (1 << nkey);
-		if ((md->md_keys & bit) == 0)
+		if (!(md->md_keys & bit))
 			continue;
 		bcopy(mmkey, tmpmkey, G_ELI_MKEYLEN);
-		error = g_eli_crypto_decrypt(md->md_algo, tmpmkey,
+		error = g_eli_crypto_decrypt(md->md_ealgo, tmpmkey,
 		    G_ELI_MKEYLEN, enckey, md->md_keylen);
 		if (error != 0) {
 			bzero(tmpmkey, sizeof(tmpmkey));
@@ -177,3 +177,33 @@ g_eli_mkey_encrypt(unsigned algo, const unsigned char *key, unsigned keylen,
 
 	return (error);
 }
+
+#ifdef _KERNEL
+/*
+ * When doing encryption only, copy IV key and encryption key.
+ * When doing encryption and authentication, copy IV key, generate encryption
+ * key and generate authentication key.
+ */
+void
+g_eli_mkey_propagate(struct g_eli_softc *sc, const unsigned char *mkey)
+{
+
+	/* Remember the Master Key. */
+	bcopy(mkey, sc->sc_mkey, sizeof(sc->sc_mkey));
+
+	bcopy(mkey, sc->sc_ivkey, sizeof(sc->sc_ivkey));
+	mkey += sizeof(sc->sc_ivkey);
+
+	if (!(sc->sc_flags & G_ELI_FLAG_AUTH)) {
+		bcopy(mkey, sc->sc_ekey, sizeof(sc->sc_ekey));
+	} else {
+		/*
+		 * The encryption key is: ekey = HMAC_SHA512(Master-Key, 0x10)
+		 * The authentication key is: akey = HMAC_SHA512(Master-Key, 0x11)
+		 */
+		g_eli_crypto_hmac(mkey, G_ELI_MAXKEYLEN, "\x10", 1, sc->sc_ekey, 0);
+		g_eli_crypto_hmac(mkey, G_ELI_MAXKEYLEN, "\x11", 1, sc->sc_akey, 0);
+	}
+
+}
+#endif
diff --git a/sys/geom/eli/g_eli_privacy.c b/sys/geom/eli/g_eli_privacy.c
new file mode 100644
index 0000000..94cc4fc
--- /dev/null
+++ b/sys/geom/eli/g_eli_privacy.c
@@ -0,0 +1,270 @@
+/*-
+ * Copyright (c) 2005-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/linker.h>
+#include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/bio.h>
+#include <sys/sysctl.h>
+#include <sys/malloc.h>
+#include <sys/kthread.h>
+#include <sys/proc.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+#include <sys/uio.h>
+#include <sys/vnode.h>
+
+#include <vm/uma.h>
+
+#include <geom/geom.h>
+#include <geom/eli/g_eli.h>
+#include <geom/eli/pkcs5v2.h>
+
+/*
+ * Code paths:
+ * BIO_READ:
+ *	g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
+ * BIO_WRITE:
+ *	g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
+ */
+
+MALLOC_DECLARE(M_ELI);
+
+/*
+ * The function is called after we read and decrypt data.
+ *
+ * g_eli_start -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
+ */
+static int
+g_eli_crypto_read_done(struct cryptop *crp)
+{
+	struct bio *bp;
+
+	if (crp->crp_etype == EAGAIN) {
+		if (g_eli_crypto_rerun(crp) == 0)
+			return (0);
+	}
+	bp = (struct bio *)crp->crp_opaque;
+	bp->bio_inbed++;
+	if (crp->crp_etype == 0) {
+		G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).",
+		    bp->bio_inbed, bp->bio_children);
+		bp->bio_completed += crp->crp_olen;
+	} else {
+		G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
+		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
+		if (bp->bio_error == 0)
+			bp->bio_error = crp->crp_etype;
+	}
+	/*
+	 * Do we have all sectors already?
+	 */
+	if (bp->bio_inbed < bp->bio_children)
+		return (0);
+	free(bp->bio_driver2, M_ELI);
+	bp->bio_driver2 = NULL;
+	if (bp->bio_error != 0) {
+		G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
+		    bp->bio_error);
+		bp->bio_completed = 0;
+	}
+	/*
+	 * Read is finished, send it up.
+	 */
+	g_io_deliver(bp, bp->bio_error);
+	return (0);
+}
+
+/*
+ * The function is called after data encryption.
+ *
+ * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
+ */
+static int
+g_eli_crypto_write_done(struct cryptop *crp)
+{
+	struct g_geom *gp;
+	struct g_consumer *cp;
+	struct bio *bp, *cbp;
+
+	if (crp->crp_etype == EAGAIN) {
+		if (g_eli_crypto_rerun(crp) == 0)
+			return (0);
+	}
+	bp = (struct bio *)crp->crp_opaque;
+	bp->bio_inbed++;
+	if (crp->crp_etype == 0) {
+		G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
+		    bp->bio_inbed, bp->bio_children);
+	} else {
+		G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
+		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
+		if (bp->bio_error == 0)
+			bp->bio_error = crp->crp_etype;
+	}
+	/*
+	 * All sectors are already encrypted?
+	 */
+	if (bp->bio_inbed < bp->bio_children)
+		return (0);
+	bp->bio_inbed = 0;
+	bp->bio_children = 1;
+	cbp = bp->bio_driver1;
+	bp->bio_driver1 = NULL;
+	if (bp->bio_error != 0) {
+		G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
+		    bp->bio_error);
+		free(bp->bio_driver2, M_ELI);
+		bp->bio_driver2 = NULL;
+		g_destroy_bio(cbp);
+		g_io_deliver(bp, bp->bio_error);
+		return (0);
+	}
+	cbp->bio_data = bp->bio_driver2;
+	cbp->bio_done = g_eli_write_done;
+	gp = bp->bio_to->geom;
+	cp = LIST_FIRST(&gp->consumer);
+	cbp->bio_to = cp->provider;
+	G_ELI_LOGREQ(2, cbp, "Sending request.");
+	/*
+	 * Send encrypted data to the provider.
+	 */
+	g_io_request(cbp, cp);
+	return (0);
+}
+
+/*
+ * This is the main function responsible for cryptography (ie. communication
+ * with crypto(9) subsystem).
+ */
+void
+g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
+{
+	struct g_eli_softc *sc;
+	struct cryptop *crp;
+	struct cryptodesc *crd;
+	struct uio *uio;
+	struct iovec *iov;
+	u_int i, nsec, add, secsize;
+	int err, error;
+	size_t size;
+	u_char *p, *data;
+
+	G_ELI_LOGREQ(3, bp, "%s", __func__);
+
+	bp->bio_pflags = wr->w_number;
+	sc = wr->w_softc;
+	secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize;
+	nsec = bp->bio_length / secsize;
+
+	/*
+	 * Calculate how much memory do we need.
+	 * We need separate crypto operation for every single sector.
+	 * It is much faster to calculate total amount of needed memory here and
+	 * do the allocation once instead of allocating memory in pieces (many,
+	 * many pieces).
+	 */
+	size = sizeof(*crp) * nsec;
+	size += sizeof(*crd) * nsec;
+	size += sizeof(*uio) * nsec;
+	size += sizeof(*iov) * nsec;
+	/*
+	 * If we write the data we cannot destroy current bio_data content,
+	 * so we need to allocate more memory for encrypted data.
+	 */
+	if (bp->bio_cmd == BIO_WRITE)
+		size += bp->bio_length;
+	p = malloc(size, M_ELI, M_WAITOK);
+
+	bp->bio_inbed = 0;
+	bp->bio_children = nsec;
+	bp->bio_driver2 = p;
+
+	if (bp->bio_cmd == BIO_READ)
+		data = bp->bio_data;
+	else {
+		data = p;
+		p += bp->bio_length;
+		bcopy(bp->bio_data, data, bp->bio_length);
+	}
+
+	error = 0;
+	for (i = 0, add = 0; i < nsec; i++, add += secsize) {
+		crp = (struct cryptop *)p;	p += sizeof(*crp);
+		crd = (struct cryptodesc *)p;	p += sizeof(*crd);
+		uio = (struct uio *)p;		p += sizeof(*uio);
+		iov = (struct iovec *)p;	p += sizeof(*iov);
+
+		iov->iov_len = secsize;
+		iov->iov_base = data;
+		data += secsize;
+
+		uio->uio_iov = iov;
+		uio->uio_iovcnt = 1;
+		uio->uio_segflg = UIO_SYSSPACE;
+		uio->uio_resid = secsize;
+
+		crp->crp_sid = wr->w_sid;
+		crp->crp_ilen = secsize;
+		crp->crp_olen = secsize;
+		crp->crp_opaque = (void *)bp;
+		crp->crp_buf = (void *)uio;
+		if (bp->bio_cmd == BIO_WRITE)
+			crp->crp_callback = g_eli_crypto_write_done;
+		else /* if (bp->bio_cmd == BIO_READ) */
+			crp->crp_callback = g_eli_crypto_read_done;
+		crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIFSYNC | CRYPTO_F_REL;
+		if (g_eli_batch)
+			crp->crp_flags |= CRYPTO_F_BATCH;
+		crp->crp_desc = crd;
+
+		crd->crd_skip = 0;
+		crd->crd_len = secsize;
+		crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
+		if (bp->bio_cmd == BIO_WRITE)
+			crd->crd_flags |= CRD_F_ENCRYPT;
+		crd->crd_alg = sc->sc_ealgo;
+		crd->crd_key = sc->sc_ekey;
+		crd->crd_klen = sc->sc_ekeylen;
+		g_eli_crypto_ivgen(sc, bp->bio_offset + add, crd->crd_iv,
+		    sizeof(crd->crd_iv));
+		crd->crd_next = NULL;
+
+		crp->crp_etype = 0;
+		err = crypto_dispatch(crp);
+		if (error == 0)
+			error = err;
+	}
+	if (bp->bio_error == 0)
+		bp->bio_error = error;
+}