1 files changed, 252 insertions, 19 deletions

diff --git a/sys/opencrypto/cryptodev.c b/sys/opencrypto/cryptodev.c
index 4096a73..b11f8e8 100644
--- a/sys/opencrypto/cryptodev.c
+++ b/sys/opencrypto/cryptodev.c
@@ -3,6 +3,12 @@
 /*-
  * Copyright (c) 2001 Theo de Raadt
  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
+ * Copyright (c) 2014 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * Portions of this software were developed by John-Mark Gurney
+ * under sponsorship of the FreeBSD Foundation and
+ * Rubicon Communications, LLC (Netgate).
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -36,6 +42,7 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_compat.h"
+#include "opt_kdtrace.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -55,10 +62,15 @@ __FBSDID("$FreeBSD$");
 #include <sys/fcntl.h>
 #include <sys/bus.h>
 #include <sys/user.h>
+#include <sys/sdt.h>
 
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 
+SDT_PROVIDER_DECLARE(opencrypto);
+
+SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/);
+
 #ifdef COMPAT_FREEBSD32
 #include <sys/mount.h>
 #include <compat/freebsd32/freebsd32.h>
@@ -318,6 +330,8 @@ static int csefree(struct csession *);
 
 static	int cryptodev_op(struct csession *, struct crypt_op *,
 			struct ucred *, struct thread *td);
+static	int cryptodev_aead(struct csession *, struct crypt_aead *,
+			struct ucred *, struct thread *);
 static	int cryptodev_key(struct crypt_kop *);
 static	int cryptodev_find(struct crypt_find_op *);
 
@@ -350,15 +364,23 @@ cryptof_truncate(
  * by device name/class or through search constraints.
  */
 static int
-checkforsoftware(int crid)
+checkforsoftware(int *cridp)
 {
+	int crid;
+
+	crid = *cridp;
 
 	if (!crypto_devallowsoft) {
-		if (crid & CRYPTOCAP_F_SOFTWARE)
-			return EINVAL;		/* XXX */
+		if (crid & CRYPTOCAP_F_SOFTWARE) {
+			if (crid & CRYPTOCAP_F_HARDWARE) {
+				*cridp = CRYPTOCAP_F_HARDWARE;
+				return 0;
+			}
+			return EINVAL;
+		}
 		if ((crid & CRYPTOCAP_F_HARDWARE) == 0 &&
 		    (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0)
-			return EINVAL;		/* XXX */
+			return EINVAL;
 	}
 	return 0;
 }
@@ -378,6 +400,7 @@ cryptof_ioctl(
 	struct csession *cse;
 	struct session_op *sop;
 	struct crypt_op *cop;
+	struct crypt_aead *caead;
 	struct enc_xform *txform = NULL;
 	struct auth_hash *thash = NULL;
 	struct crypt_kop *kop;
@@ -438,7 +461,15 @@ cryptof_ioctl(
  		case CRYPTO_CAMELLIA_CBC:
  			txform = &enc_xform_camellia;
  			break;
+		case CRYPTO_AES_ICM:
+			txform = &enc_xform_aes_icm;
+ 			break;
+		case CRYPTO_AES_NIST_GCM_16:
+			txform = &enc_xform_aes_nist_gcm;
+ 			break;
+
 		default:
+			CRYPTDEB("invalid cipher");
 			return (EINVAL);
 		}
 
@@ -463,6 +494,16 @@ cryptof_ioctl(
 		case CRYPTO_RIPEMD160_HMAC:
 			thash = &auth_hash_hmac_ripemd_160;
 			break;
+		case CRYPTO_AES_128_NIST_GMAC:
+			thash = &auth_hash_nist_gmac_aes_128;
+			break;
+		case CRYPTO_AES_192_NIST_GMAC:
+			thash = &auth_hash_nist_gmac_aes_192;
+			break;
+		case CRYPTO_AES_256_NIST_GMAC:
+			thash = &auth_hash_nist_gmac_aes_256;
+			break;
+
 #ifdef notdef
 		case CRYPTO_MD5:
 			thash = &auth_hash_md5;
@@ -475,6 +516,7 @@ cryptof_ioctl(
 			thash = &auth_hash_null;
 			break;
 		default:
+			CRYPTDEB("invalid mac");
 			return (EINVAL);
 		}
 
@@ -486,6 +528,7 @@ cryptof_ioctl(
 			crie.cri_klen = sop->keylen * 8;
 			if (sop->keylen > txform->maxkey ||
 			    sop->keylen < txform->minkey) {
+				CRYPTDEB("invalid cipher parameters");
 				error = EINVAL;
 				goto bail;
 			}
@@ -493,8 +536,10 @@ cryptof_ioctl(
 			crie.cri_key = malloc(crie.cri_klen / 8,
 			    M_XDATA, M_WAITOK);
 			if ((error = copyin(sop->key, crie.cri_key,
-			    crie.cri_klen / 8)))
+			    crie.cri_klen / 8))) {
+				CRYPTDEB("invalid key");
 				goto bail;
+			}
 			if (thash)
 				crie.cri_next = &cria;
 		}
@@ -503,6 +548,7 @@ cryptof_ioctl(
 			cria.cri_alg = thash->type;
 			cria.cri_klen = sop->mackeylen * 8;
 			if (sop->mackeylen != thash->keysize) {
+				CRYPTDEB("invalid mac key length");
 				error = EINVAL;
 				goto bail;
 			}
@@ -511,8 +557,10 @@ cryptof_ioctl(
 				cria.cri_key = malloc(cria.cri_klen / 8,
 				    M_XDATA, M_WAITOK);
 				if ((error = copyin(sop->mackey, cria.cri_key,
-				    cria.cri_klen / 8)))
+				    cria.cri_klen / 8))) {
+					CRYPTDEB("invalid mac key");
 					goto bail;
+				}
 			}
 		}
 
@@ -523,14 +571,18 @@ cryptof_ioctl(
 #endif
 			) {
 			crid = SES2(sop)->crid;
-			error = checkforsoftware(crid);
-			if (error)
+			error = checkforsoftware(&crid);
+			if (error) {
+				CRYPTDEB("checkforsoftware");
 				goto bail;
+			}
 		} else
 			crid = CRYPTOCAP_F_HARDWARE;
 		error = crypto_newsession(&sid, (txform ? &crie : &cria), crid);
-		if (error)
+		if (error) {
+			CRYPTDEB("crypto_newsession");
 			goto bail;
+		}
 
 		cse = csecreate(fcr, sid, crie.cri_key, crie.cri_klen,
 		    cria.cri_key, cria.cri_klen, sop->cipher, sop->mac, txform,
@@ -539,6 +591,7 @@ cryptof_ioctl(
 		if (cse == NULL) {
 			crypto_freesession(sid);
 			error = EINVAL;
+			CRYPTDEB("csecreate");
 			goto bail;
 		}
 		sop->ses = cse->ses;
@@ -585,8 +638,10 @@ bail:
 #endif
 			cop = (struct crypt_op *)data;
 		cse = csefind(fcr, cop->ses);
-		if (cse == NULL)
+		if (cse == NULL) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EINVAL);
+		}
 		error = cryptodev_op(cse, cop, active_cred, td);
 #ifdef COMPAT_FREEBSD32
 		if (error == 0 && cmd == CIOCCRYPT32)
@@ -640,6 +695,13 @@ bail:
 	case CIOCFINDDEV:
 		error = cryptodev_find((struct crypt_find_op *)data);
 		break;
+	case CIOCCRYPTAEAD:
+		caead = (struct crypt_aead *)data;
+		cse = csefind(fcr, caead->ses);
+		if (cse == NULL)
+			return (EINVAL);
+		error = cryptodev_aead(cse, caead, active_cred, td);
+		break;
 	default:
 		error = EINVAL;
 		break;
@@ -662,12 +724,16 @@ cryptodev_op(
 	struct cryptodesc *crde = NULL, *crda = NULL;
 	int error;
 
-	if (cop->len > 256*1024-4)
+	if (cop->len > 256*1024-4) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (E2BIG);
+	}
 
 	if (cse->txform) {
-		if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0)
+		if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EINVAL);
+		}
 	}
 
 	cse->uio.uio_iov = &cse->iovec;
@@ -687,6 +753,7 @@ cryptodev_op(
 
 	crp = crypto_getreq((cse->txform != NULL) + (cse->thash != NULL));
 	if (crp == NULL) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		error = ENOMEM;
 		goto bail;
 	}
@@ -699,13 +766,17 @@ cryptodev_op(
 		if (cse->txform)
 			crde = crp->crp_desc;
 		else {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			error = EINVAL;
 			goto bail;
 		}
 	}
 
-	if ((error = copyin(cop->src, cse->uio.uio_iov[0].iov_base, cop->len)))
+	if ((error = copyin(cop->src, cse->uio.uio_iov[0].iov_base,
+	    cop->len))) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
+	}
 
 	if (crda) {
 		crda->crd_skip = 0;
@@ -740,15 +811,20 @@ cryptodev_op(
 
 	if (cop->iv) {
 		if (crde == NULL) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			error = EINVAL;
 			goto bail;
 		}
 		if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			error = EINVAL;
 			goto bail;
 		}
-		if ((error = copyin(cop->iv, cse->tmp_iv, cse->txform->blocksize)))
+		if ((error = copyin(cop->iv, cse->tmp_iv,
+		    cse->txform->blocksize))) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			goto bail;
+		}
 		bcopy(cse->tmp_iv, crde->crd_iv, cse->txform->blocksize);
 		crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 		crde->crd_skip = 0;
@@ -761,6 +837,7 @@ cryptodev_op(
 	}
 
 	if (cop->mac && crda == NULL) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		error = EINVAL;
 		goto bail;
 	}
@@ -779,8 +856,10 @@ again:
 		error = msleep(crp, &cse->lock, PWAIT, "crydev", 0);
 	mtx_unlock(&cse->lock);
 
-	if (error != 0)
+	if (error != 0) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
+	}
 
 	if (crp->crp_etype == EAGAIN) {
 		crp->crp_etype = 0;
@@ -789,23 +868,30 @@ again:
 	}
 
 	if (crp->crp_etype != 0) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		error = crp->crp_etype;
 		goto bail;
 	}
 
 	if (cse->error) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		error = cse->error;
 		goto bail;
 	}
 
 	if (cop->dst &&
-	    (error = copyout(cse->uio.uio_iov[0].iov_base, cop->dst, cop->len)))
+	    (error = copyout(cse->uio.uio_iov[0].iov_base, cop->dst,
+	    cop->len))) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
+	}
 
 	if (cop->mac &&
 	    (error = copyout((caddr_t)cse->uio.uio_iov[0].iov_base + cop->len,
-	    cop->mac, cse->thash->hashsize)))
+	    cop->mac, cse->thash->hashsize))) {
+		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
+	}
 
 bail:
 	if (crp)
@@ -817,6 +903,151 @@ bail:
 }
 
 static int
+cryptodev_aead(
+	struct csession *cse,
+	struct crypt_aead *caead,
+	struct ucred *active_cred,
+	struct thread *td)
+{
+	struct uio *uio;
+	struct cryptop *crp = NULL;
+	struct cryptodesc *crde = NULL, *crda = NULL;
+	int error;
+
+	if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4)
+		return (E2BIG);
+
+	if (cse->txform == NULL || cse->thash == NULL || caead->tag == NULL ||
+	    (caead->len % cse->txform->blocksize) != 0)
+		return (EINVAL);
+
+	uio = &cse->uio;
+	uio->uio_iov = &cse->iovec;
+	uio->uio_iovcnt = 1;
+	uio->uio_offset = 0;
+	uio->uio_resid = caead->len + caead->aadlen + cse->thash->hashsize;
+	uio->uio_segflg = UIO_SYSSPACE;
+	uio->uio_rw = UIO_WRITE;
+	uio->uio_td = td;
+	uio->uio_iov[0].iov_len = uio->uio_resid;
+
+	uio->uio_iov[0].iov_base = malloc(uio->uio_iov[0].iov_len,
+	    M_XDATA, M_WAITOK);
+
+	crp = crypto_getreq(2);
+	if (crp == NULL) {
+		error = ENOMEM;
+		goto bail;
+	}
+
+	crda = crp->crp_desc;
+	crde = crda->crd_next;
+
+	if ((error = copyin(caead->src, cse->uio.uio_iov[0].iov_base,
+	    caead->len)))
+		goto bail;
+
+	if ((error = copyin(caead->aad, (char *)cse->uio.uio_iov[0].iov_base +
+	    caead->len, caead->aadlen)))
+		goto bail;
+
+	crda->crd_skip = caead->len;
+	crda->crd_len = caead->aadlen;
+	crda->crd_inject = caead->len + caead->aadlen;
+
+	crda->crd_alg = cse->mac;
+	crda->crd_key = cse->mackey;
+	crda->crd_klen = cse->mackeylen * 8;
+
+	if (caead->op == COP_ENCRYPT)
+		crde->crd_flags |= CRD_F_ENCRYPT;
+	else
+		crde->crd_flags &= ~CRD_F_ENCRYPT;
+	/* crde->crd_skip set below */
+	crde->crd_len = caead->len;
+	crde->crd_inject = 0;
+
+	crde->crd_alg = cse->cipher;
+	crde->crd_key = cse->key;
+	crde->crd_klen = cse->keylen * 8;
+
+	crp->crp_ilen = caead->len + caead->aadlen;
+	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM
+		       | (caead->flags & COP_F_BATCH);
+	crp->crp_buf = (caddr_t)&cse->uio.uio_iov;
+	crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_cb;
+	crp->crp_sid = cse->sid;
+	crp->crp_opaque = (void *)cse;
+
+	if (caead->iv) {
+		if (caead->ivlen > sizeof cse->tmp_iv) {
+			error = EINVAL;
+			goto bail;
+		}
+
+		if ((error = copyin(caead->iv, cse->tmp_iv, caead->ivlen)))
+			goto bail;
+		bcopy(cse->tmp_iv, crde->crd_iv, caead->ivlen);
+		crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
+		crde->crd_skip = 0;
+	} else {
+		crde->crd_flags |= CRD_F_IV_PRESENT;
+		crde->crd_skip = cse->txform->blocksize;
+		crde->crd_len -= cse->txform->blocksize;
+	}
+
+	if ((error = copyin(caead->tag, (caddr_t)cse->uio.uio_iov[0].iov_base +
+	    caead->len + caead->aadlen, cse->thash->hashsize)))
+		goto bail;
+again:
+	/*
+	 * Let the dispatch run unlocked, then, interlock against the
+	 * callback before checking if the operation completed and going
+	 * to sleep.  This insures drivers don't inherit our lock which
+	 * results in a lock order reversal between crypto_dispatch forced
+	 * entry and the crypto_done callback into us.
+	 */
+	error = crypto_dispatch(crp);
+	mtx_lock(&cse->lock);
+	if (error == 0 && (crp->crp_flags & CRYPTO_F_DONE) == 0)
+		error = msleep(crp, &cse->lock, PWAIT, "crydev", 0);
+	mtx_unlock(&cse->lock);
+
+	if (error != 0)
+		goto bail;
+
+	if (crp->crp_etype == EAGAIN) {
+		crp->crp_etype = 0;
+		crp->crp_flags &= ~CRYPTO_F_DONE;
+		goto again;
+	}
+
+	if (crp->crp_etype != 0) {
+		error = crp->crp_etype;
+		goto bail;
+	}
+
+	if (cse->error) {
+		error = cse->error;
+		goto bail;
+	}
+
+	if (caead->dst && (error = copyout(cse->uio.uio_iov[0].iov_base,
+	    caead->dst, caead->len)))
+		goto bail;
+
+	if ((error = copyout((caddr_t)cse->uio.uio_iov[0].iov_base +
+	    caead->len + caead->aadlen, caead->tag, cse->thash->hashsize)))
+		goto bail;
+
+bail:
+	crypto_freereq(crp);
+	free(cse->uio.uio_iov[0].iov_base, M_XDATA);
+
+	return (error);
+}
+
+static int
 cryptodev_cb(void *op)
 {
 	struct cryptop *crp = (struct cryptop *) op;
@@ -945,14 +1176,16 @@ static int
 cryptodev_find(struct crypt_find_op *find)
 {
 	device_t dev;
+	size_t fnlen = sizeof find->name;
 
 	if (find->crid != -1) {
 		dev = crypto_find_device_byhid(find->crid);
 		if (dev == NULL)
 			return (ENOENT);
-		strlcpy(find->name, device_get_nameunit(dev),
-		    sizeof(find->name));
+		strncpy(find->name, device_get_nameunit(dev), fnlen);
+		find->name[fnlen - 1] = '\x0';
 	} else {
+		find->name[fnlen - 1] = '\x0';
 		find->crid = crypto_find_driver(find->name);
 		if (find->crid == -1)
 			return (ENOENT);