Add some new modes to OpenCrypto. These modes are AES-ICM (can be used

for counter mode), and AES-GCM.  Both of these modes have been added to
the aesni module.

Included is a set of tests to validate that the software and aesni
module calculate the correct values.  These use the NIST KAT test
vectors.  To run the test, you will need to install a soon to be
committed port, nist-kat that will install the vectors.  Using a port
is necessary as the test vectors are around 25MB.

All the man pages were updated.  I have added a new man page, crypto.7,
which includes a description of how to use each mode.  All the new modes
and some other AES modes are present.  It would be good for someone
else to go through and document the other modes.

A new ioctl was added to support AEAD modes which AES-GCM is one of them.
Without this ioctl, it is not possible to test AEAD modes from userland.

Add a timing safe bcmp for use to compare MACs.  Previously we were using
bcmp which could leak timing info and result in the ability to forge
messages.

Add a minor optimization to the aesni module so that single segment
mbufs don't get copied and instead are updated in place.  The aesni
module needs to be updated to support blocked IO so segmented mbufs
don't have to be copied.

We require that the IV be specified for all calls for both GCM and ICM.
This is to ensure proper use of these functions.

Obtained from:	p4: //depot/projects/opencrypto
Relnotes:	yes
Sponsored by:	FreeBSD Foundation
Sponsored by:	NetGate

1 files changed, 252 insertions, 19 deletions

diff --git a/sys/opencrypto/cryptodev.c b/sys/opencrypto/cryptodev.c
index bf46138..1401504 100644
--- a/sys/opencrypto/cryptodev.c
+++ b/sys/opencrypto/cryptodev.c
@@ -3,6 +3,12 @@
 /*-
  * Copyright (c) 2001 Theo de Raadt
  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
+ * Copyright (c) 2014 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by John-Mark Gurney
+ * under sponsorship of the FreeBSD Foundation and
+ * Rubicon Communications, LLC (Netgate).
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -36,6 +42,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_compat.h"
+#include "opt_kdtrace.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -55,10 +62,15 @@ __FBSDID("$FreeBSD$");
 #include <sys/fcntl.h>
 #include <sys/bus.h>
 #include <sys/user.h>
+#include <sys/sdt.h>
 
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 
+SDT_PROVIDER_DECLARE(opencrypto);
+
+SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/);
+
 #ifdef COMPAT_FREEBSD32
 #include <sys/mount.h>
 #include <compat/freebsd32/freebsd32.h>
@@ -315,6 +327,8 @@ static int csefree(struct csession *);
 
 static	int cryptodev_op(struct csession *, struct crypt_op *,
 			struct ucred *, struct thread *td);
+static	int cryptodev_aead(struct csession *, struct crypt_aead *,
+			struct ucred *, struct thread *);
 static	int cryptodev_key(struct crypt_kop *);
 static	int cryptodev_find(struct crypt_find_op *);
 
@@ -324,15 +338,23 @@ static	int cryptodev_find(struct crypt_find_op *);
  * by device name/class or through search constraints.
  */
 static int
-checkforsoftware(int crid)
+checkforsoftware(int *cridp)
 {
+	int crid;
+
+	crid = *cridp;
 
 	if (!crypto_devallowsoft) {
-		if (crid & CRYPTOCAP_F_SOFTWARE)
-			return EINVAL;		/* XXX */
+		if (crid & CRYPTOCAP_F_SOFTWARE) {
+			if (crid & CRYPTOCAP_F_HARDWARE) {
+				*cridp = CRYPTOCAP_F_HARDWARE;
+				return 0;
+			}
+			return EINVAL;
+		}
 		if ((crid & CRYPTOCAP_F_HARDWARE) == 0 &&
 		    (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0)
-			return EINVAL;		/* XXX */
+			return EINVAL;
 	}
 	return 0;
 }
@@ -352,6 +374,7 @@ cryptof_ioctl(
 	struct csession *cse;
 	struct session_op *sop;
 	struct crypt_op *cop;
+	struct crypt_aead *caead;
 	struct enc_xform *txform = NULL;
 	struct auth_hash *thash = NULL;
 	struct crypt_kop *kop;
@@ -412,7 +435,15 @@ cryptof_ioctl(
  		case CRYPTO_CAMELLIA_CBC:
  			txform = &enc_xform_camellia;
  			break;
+		case CRYPTO_AES_ICM:
+			txform = &enc_xform_aes_icm;
+ 			break;
+		case CRYPTO_AES_NIST_GCM_16:
+			txform = &enc_xform_aes_nist_gcm;
+ 			break;
+
 		default:
+			CRYPTDEB("invalid cipher");
 			return (EINVAL);
 		}
 
@@ -437,6 +468,16 @@ cryptof_ioctl(
 		case CRYPTO_RIPEMD160_HMAC:
 			thash = &auth_hash_hmac_ripemd_160;
 			break;
+		case CRYPTO_AES_128_NIST_GMAC:
+			thash = &auth_hash_nist_gmac_aes_128;
+			break;
+		case CRYPTO_AES_192_NIST_GMAC:
+			thash = &auth_hash_nist_gmac_aes_192;
+			break;
+		case CRYPTO_AES_256_NIST_GMAC:
+			thash = &auth_hash_nist_gmac_aes_256;
+			break;
+
 #ifdef notdef
 		case CRYPTO_MD5:
 			thash = &auth_hash_md5;
@@ -449,6 +490,7 @@ cryptof_ioctl(
 			thash = &auth_hash_null;
 			break;
 		default:
+			CRYPTDEB("invalid mac");
 			return (EINVAL);
 		}
 
@@ -460,6 +502,7 @@ cryptof_ioctl(
 			crie.cri_klen = sop->keylen * 8;
 			if (sop->keylen > txform->maxkey ||
 			    sop->keylen < txform->minkey) {
+				CRYPTDEB("invalid cipher parameters");
 				error = EINVAL;
 				goto bail;
 			}
@@ -467,8 +510,10 @@ cryptof_ioctl(
 			crie.cri_key = malloc(crie.cri_klen / 8,
 			    M_XDATA, M_WAITOK);
 			if ((error = copyin(sop->key, crie.cri_key,
-			    crie.cri_klen / 8)))
+			    crie.cri_klen / 8))) {
+				CRYPTDEB("invalid key");
 				goto bail;
+			}
 			if (thash)
 				crie.cri_next = &cria;
 		}
@@ -477,6 +522,7 @@ cryptof_ioctl(
 			cria.cri_alg = thash->type;
 			cria.cri_klen = sop->mackeylen * 8;
 			if (sop->mackeylen != thash->keysize) {
+				CRYPTDEB("invalid mac key length");
 				error = EINVAL;
 				goto bail;
 			}
@@ -485,8 +531,10 @@ cryptof_ioctl(
 				cria.cri_key = malloc(cria.cri_klen / 8,
 				    M_XDATA, M_WAITOK);
 				if ((error = copyin(sop->mackey, cria.cri_key,
-				    cria.cri_klen / 8)))
+				    cria.cri_klen / 8))) {
+					CRYPTDEB("invalid mac key");
 					goto bail;
+				}
 			}
 		}
 
@@ -497,14 +545,18 @@ cryptof_ioctl(
 #endif
 			) {
 			crid = SES2(sop)->crid;
-			error = checkforsoftware(crid);
-			if (error)
+			error = checkforsoftware(&crid);
+			if (error) {
+				CRYPTDEB("checkforsoftware");
 				goto bail;
+			}
 		} else
 			crid = CRYPTOCAP_F_HARDWARE;
 		error = crypto_newsession(&sid, (txform ? &crie : &cria), crid);
-		if (error)
+		if (error) {
+			CRYPTDEB("crypto_newsession");
 			goto bail;
+		}
 
 		cse = csecreate(fcr, sid, crie.cri_key, crie.cri_klen,
 		    cria.cri_key, cria.cri_klen, sop->cipher, sop->mac, txform,
@@ -513,6 +565,7 @@ cryptof_ioctl(
 		if (cse == NULL) {
 			crypto_freesession(sid);
 			error = EINVAL;
+			CRYPTDEB("csecreate");
 			goto bail;
 		}
 		sop->ses = cse->ses;
@@ -559,8 +612,10 @@ bail:
 #endif
 			cop = (struct crypt_op *)data;
 		cse = csefind(fcr, cop->ses);
-		if (cse == NULL)
+		if (cse == NULL) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EINVAL);
+		}
 		error = cryptodev_op(cse, cop, active_cred, td);
 #ifdef COMPAT_FREEBSD32
 		if (error == 0 && cmd == CIOCCRYPT32)
@@ -614,6 +669,13 @@ bail:
 	case CIOCFINDDEV:
 		error = cryptodev_find((struct crypt_find_op *)data);
 		break;
+	case CIOCCRYPTAEAD:
+		caead = (struct crypt_aead *)data;
+		cse = csefind(fcr, caead->ses);
+		if (cse == NULL)
+			return (EINVAL);
+		error = cryptodev_aead(cse, caead, active_cred, td);
+		break;
 	default:
 		error = EINVAL;
 		break;
@@ -636,12 +698,16 @@ cryptodev_op(
 	struct cryptodesc *crde = NULL, *crda = NULL;
 	int error;
 
-	if (cop->len > 256*1024-4)
+	if (cop->len > 256*1024-4) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (E2BIG);
+	}
 
 	if (cse->txform) {
-		if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0)
+		if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EINVAL);
+		}
 	}
 
 	cse->uio.uio_iov = &cse->iovec;
@@ -661,6 +727,7 @@ cryptodev_op(
 
 	crp = crypto_getreq((cse->txform != NULL) + (cse->thash != NULL));
 	if (crp == NULL) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		error = ENOMEM;
 		goto bail;
 	}
@@ -673,13 +740,17 @@ cryptodev_op(
 		if (cse->txform)
 			crde = crp->crp_desc;
 		else {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			error = EINVAL;
 			goto bail;
 		}
 	}
 
-	if ((error = copyin(cop->src, cse->uio.uio_iov[0].iov_base, cop->len)))
+	if ((error = copyin(cop->src, cse->uio.uio_iov[0].iov_base,
+	    cop->len))) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
+	}
 
 	if (crda) {
 		crda->crd_skip = 0;
@@ -714,15 +785,20 @@ cryptodev_op(
 
 	if (cop->iv) {
 		if (crde == NULL) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			error = EINVAL;
 			goto bail;
 		}
 		if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			error = EINVAL;
 			goto bail;
 		}
-		if ((error = copyin(cop->iv, cse->tmp_iv, cse->txform->blocksize)))
+		if ((error = copyin(cop->iv, cse->tmp_iv,
+		    cse->txform->blocksize))) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			goto bail;
+		}
 		bcopy(cse->tmp_iv, crde->crd_iv, cse->txform->blocksize);
 		crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 		crde->crd_skip = 0;
@@ -735,6 +811,7 @@ cryptodev_op(
 	}
 
 	if (cop->mac && crda == NULL) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		error = EINVAL;
 		goto bail;
 	}
@@ -753,8 +830,10 @@ again:
 		error = msleep(crp, &cse->lock, PWAIT, "crydev", 0);
 	mtx_unlock(&cse->lock);
 
-	if (error != 0)
+	if (error != 0) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
+	}
 
 	if (crp->crp_etype == EAGAIN) {
 		crp->crp_etype = 0;
@@ -763,23 +842,30 @@ again:
 	}
 
 	if (crp->crp_etype != 0) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		error = crp->crp_etype;
 		goto bail;
 	}
 
 	if (cse->error) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		error = cse->error;
 		goto bail;
 	}
 
 	if (cop->dst &&
-	    (error = copyout(cse->uio.uio_iov[0].iov_base, cop->dst, cop->len)))
+	    (error = copyout(cse->uio.uio_iov[0].iov_base, cop->dst,
+	    cop->len))) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
+	}
 
 	if (cop->mac &&
 	    (error = copyout((caddr_t)cse->uio.uio_iov[0].iov_base + cop->len,
-	    cop->mac, cse->thash->hashsize)))
+	    cop->mac, cse->thash->hashsize))) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
+	}
 
 bail:
 	if (crp)
@@ -791,6 +877,151 @@ bail:
 }
 
 static int
+cryptodev_aead(
+	struct csession *cse,
+	struct crypt_aead *caead,
+	struct ucred *active_cred,
+	struct thread *td)
+{
+	struct uio *uio;
+	struct cryptop *crp = NULL;
+	struct cryptodesc *crde = NULL, *crda = NULL;
+	int error;
+
+	if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4)
+		return (E2BIG);
+
+	if (cse->txform == NULL || cse->thash == NULL || caead->tag == NULL ||
+	    (caead->len % cse->txform->blocksize) != 0)
+		return (EINVAL);
+
+	uio = &cse->uio;
+	uio->uio_iov = &cse->iovec;
+	uio->uio_iovcnt = 1;
+	uio->uio_offset = 0;
+	uio->uio_resid = caead->len + caead->aadlen + cse->thash->hashsize;
+	uio->uio_segflg = UIO_SYSSPACE;
+	uio->uio_rw = UIO_WRITE;
+	uio->uio_td = td;
+	uio->uio_iov[0].iov_len = uio->uio_resid;
+
+	uio->uio_iov[0].iov_base = malloc(uio->uio_iov[0].iov_len,
+	    M_XDATA, M_WAITOK);
+
+	crp = crypto_getreq(2);
+	if (crp == NULL) {
+		error = ENOMEM;
+		goto bail;
+	}
+
+	crda = crp->crp_desc;
+	crde = crda->crd_next;
+
+	if ((error = copyin(caead->src, cse->uio.uio_iov[0].iov_base,
+	    caead->len)))
+		goto bail;
+
+	if ((error = copyin(caead->aad, (char *)cse->uio.uio_iov[0].iov_base +
+	    caead->len, caead->aadlen)))
+		goto bail;
+
+	crda->crd_skip = caead->len;
+	crda->crd_len = caead->aadlen;
+	crda->crd_inject = caead->len + caead->aadlen;
+
+	crda->crd_alg = cse->mac;
+	crda->crd_key = cse->mackey;
+	crda->crd_klen = cse->mackeylen * 8;
+
+	if (caead->op == COP_ENCRYPT)
+		crde->crd_flags |= CRD_F_ENCRYPT;
+	else
+		crde->crd_flags &= ~CRD_F_ENCRYPT;
+	/* crde->crd_skip set below */
+	crde->crd_len = caead->len;
+	crde->crd_inject = 0;
+
+	crde->crd_alg = cse->cipher;
+	crde->crd_key = cse->key;
+	crde->crd_klen = cse->keylen * 8;
+
+	crp->crp_ilen = caead->len + caead->aadlen;
+	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM
+		       | (caead->flags & COP_F_BATCH);
+	crp->crp_buf = (caddr_t)&cse->uio.uio_iov;
+	crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_cb;
+	crp->crp_sid = cse->sid;
+	crp->crp_opaque = (void *)cse;
+
+	if (caead->iv) {
+		if (caead->ivlen > sizeof cse->tmp_iv) {
+			error = EINVAL;
+			goto bail;
+		}
+
+		if ((error = copyin(caead->iv, cse->tmp_iv, caead->ivlen)))
+			goto bail;
+		bcopy(cse->tmp_iv, crde->crd_iv, caead->ivlen);
+		crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
+		crde->crd_skip = 0;
+	} else {
+		crde->crd_flags |= CRD_F_IV_PRESENT;
+		crde->crd_skip = cse->txform->blocksize;
+		crde->crd_len -= cse->txform->blocksize;
+	}
+
+	if ((error = copyin(caead->tag, (caddr_t)cse->uio.uio_iov[0].iov_base +
+	    caead->len + caead->aadlen, cse->thash->hashsize)))
+		goto bail;
+again:
+	/*
+	 * Let the dispatch run unlocked, then, interlock against the
+	 * callback before checking if the operation completed and going
+	 * to sleep.  This insures drivers don't inherit our lock which
+	 * results in a lock order reversal between crypto_dispatch forced
+	 * entry and the crypto_done callback into us.
+	 */
+	error = crypto_dispatch(crp);
+	mtx_lock(&cse->lock);
+	if (error == 0 && (crp->crp_flags & CRYPTO_F_DONE) == 0)
+		error = msleep(crp, &cse->lock, PWAIT, "crydev", 0);
+	mtx_unlock(&cse->lock);
+
+	if (error != 0)
+		goto bail;
+
+	if (crp->crp_etype == EAGAIN) {
+		crp->crp_etype = 0;
+		crp->crp_flags &= ~CRYPTO_F_DONE;
+		goto again;
+	}
+
+	if (crp->crp_etype != 0) {
+		error = crp->crp_etype;
+		goto bail;
+	}
+
+	if (cse->error) {
+		error = cse->error;
+		goto bail;
+	}
+
+	if (caead->dst && (error = copyout(cse->uio.uio_iov[0].iov_base,
+	    caead->dst, caead->len)))
+		goto bail;
+
+	if ((error = copyout((caddr_t)cse->uio.uio_iov[0].iov_base +
+	    caead->len + caead->aadlen, caead->tag, cse->thash->hashsize)))
+		goto bail;
+
+bail:
+	crypto_freereq(crp);
+	free(cse->uio.uio_iov[0].iov_base, M_XDATA);
+
+	return (error);
+}
+
+static int
 cryptodev_cb(void *op)
 {
 	struct cryptop *crp = (struct cryptop *) op;
@@ -919,14 +1150,16 @@ static int
 cryptodev_find(struct crypt_find_op *find)
 {
 	device_t dev;
+	size_t fnlen = sizeof find->name;
 
 	if (find->crid != -1) {
 		dev = crypto_find_device_byhid(find->crid);
 		if (dev == NULL)
 			return (ENOENT);
-		strlcpy(find->name, device_get_nameunit(dev),
-		    sizeof(find->name));
+		strncpy(find->name, device_get_nameunit(dev), fnlen);
+		find->name[fnlen - 1] = '\x0';
 	} else {
+		find->name[fnlen - 1] = '\x0';
 		find->crid = crypto_find_driver(find->name);
 		if (find->crid == -1)
 			return (ENOENT);