Importing pfSense patch ipsec_aescbc_aesni.diff

4 files changed, 341 insertions, 94 deletions

diff --git a/sys/crypto/aesni/aesni.c b/sys/crypto/aesni/aesni.c
index e1bd5e8..fc6bbf4 100644
--- a/sys/crypto/aesni/aesni.c
+++ b/sys/crypto/aesni/aesni.c
@@ -112,6 +112,14 @@ aesni_attach(device_t dev)
 	crypto_register(sc->cid, CRYPTO_AES_128_GMAC, 0, 0);
 	crypto_register(sc->cid, CRYPTO_AES_192_GMAC, 0, 0);
 	crypto_register(sc->cid, CRYPTO_AES_256_GMAC, 0, 0);
+
+	crypto_register(sc->cid, CRYPTO_NULL_HMAC, 0, 0);
+	crypto_register(sc->cid, CRYPTO_MD5_HMAC, 0, 0);
+	crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0);
+	crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0);
+	crypto_register(sc->cid, CRYPTO_SHA2_384_HMAC, 0, 0);
+	crypto_register(sc->cid, CRYPTO_SHA2_512_HMAC, 0, 0);
+
 	return (0);
 }
 
@@ -146,7 +154,7 @@ aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
 {
 	struct aesni_softc *sc;
 	struct aesni_session *ses;
-	struct cryptoini *encini;
+	struct cryptoini *encini, *authini;
 	int error, sessn;
 
 	if (sidp == NULL || cri == NULL) {
@@ -157,6 +165,7 @@ aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
 	sc = device_get_softc(dev);
 	ses = NULL;
 	encini = NULL;
+	authini = NULL;
 	for (; cri != NULL; cri = cri->cri_next) {
 		switch (cri->cri_alg) {
 		case CRYPTO_AES_CBC:
@@ -182,15 +191,23 @@ aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
 		 * values for GHASH
 		 */
 			break;
+		case CRYPTO_NULL_HMAC:
+		case CRYPTO_MD5_HMAC:
+		case CRYPTO_SHA1_HMAC:
+		case CRYPTO_SHA2_256_HMAC:
+		case CRYPTO_SHA2_384_HMAC:
+		case CRYPTO_SHA2_512_HMAC:
+			if (authini != NULL) {
+				printf("authini already set");
+				return (EINVAL);
+			}
+                        authini = cri;
+			break;
 		default:
 			printf("unhandled algorithm");
 			return (EINVAL);
 		}
 	}
-	if (encini == NULL) {
-		printf("no cipher");
-		return (EINVAL);
-	}
 
 	for (sessn = 1; sessn < sc->nsessions; sessn++) {
 		if (!sc->sessions[sessn].used) {
@@ -213,20 +230,32 @@ aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
 	} else if (ses->id == 0)
 		ses->id = sessn;
 
-	if (ses->fpu_ctx == NULL) {
-		ses->fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL |
-		    FPU_KERN_NOWAIT);
-		if (ses->fpu_ctx == NULL)
-			return (ENOMEM);
+	if (encini != NULL) {
+		if (ses->fpu_ctx == NULL) {
+			ses->fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL |
+			    FPU_KERN_NOWAIT);
+			if (ses->fpu_ctx == NULL)
+				return (ENOMEM);
+		}
+		ses->algo = encini->cri_alg;
+
+		error = aesni_cipher_setup(ses, encini);
+		if (error != 0) {
+			printf("setup failed");
+			aesni_freesession_locked(sc, ses);
+			return (error);
+		}
 	}
-	ses->algo = encini->cri_alg;
 
-	error = aesni_cipher_setup(ses, encini);
-	if (error != 0) {
-		printf("setup failed");
-		aesni_freesession_locked(sc, ses);
-		return (error);
+	if (authini != NULL) {
+		error = aesni_hash_setup(ses, authini);
+		if (error != 0) {
+			printf("setup failed");
+			aesni_freesession_locked(sc, ses);
+			return (error);
+		}
 	}
+
 	ses->used = 1;
 	*sidp = ses->id;
 
@@ -244,6 +273,7 @@ aesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses)
 	bzero(ses, sizeof(*ses));
 	ses->id = sid;
 	ses->fpu_ctx = ctx;
+	aesni_hash_free(ses);
 }
 
 static int
@@ -322,19 +352,33 @@ aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
 			needauth = 1;
 			break;
 
+		case CRYPTO_NULL_HMAC:
+		case CRYPTO_MD5_HMAC:
+		case CRYPTO_SHA1_HMAC:
+		case CRYPTO_SHA2_256_HMAC:
+		case CRYPTO_SHA2_384_HMAC:
+		case CRYPTO_SHA2_512_HMAC:
+			if (authcrd != NULL) {
+				error = EINVAL;
+				goto out;
+			}
+			authcrd = crd;
+			needauth = 1;
+			break;
+
 		default:
 			return (EINVAL);
 		}
 	}
 
-	if (enccrd == NULL || (needauth && authcrd == NULL)) {
+	if (needauth && authcrd == NULL) {
 		error = EINVAL;
 		goto out;
 	}
 
 	/* CBC & XTS can only handle full blocks for now */
-	if ((enccrd->crd_len == CRYPTO_AES_CBC || enccrd->crd_len ==
-	    CRYPTO_AES_XTS) && (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
+	if (enccrd != NULL && ((enccrd->crd_len == CRYPTO_AES_CBC || enccrd->crd_len ==
+	    CRYPTO_AES_XTS) && (enccrd->crd_len % AES_BLOCK_LEN) != 0)) {
 		error = EINVAL;
 		goto out;
 	}
@@ -459,89 +503,95 @@ aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
 	int error, allocated, authallocated;
 	int ivlen, encflag, i;
 
-	encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
+	allocated = authallocated = 0;
+	encflag = error = 0;
+	authbuf = NULL;
 
-	buf = aesni_cipher_alloc(enccrd, crp, &allocated);
-	if (buf == NULL)
-		return (ENOMEM);
+	if (enccrd != NULL) {
+		encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
+
+		buf = aesni_cipher_alloc(enccrd, crp, &allocated);
+		if (buf == NULL)
+			return (ENOMEM);
+
+		iv = enccrd->crd_iv;
+		/* XXX - validate that enccrd and authcrd have/use same key? */
+		switch (enccrd->crd_alg) {
+		case CRYPTO_AES_CBC:
+			ivlen = 16;
+			break;
+		case CRYPTO_AES_XTS:
+			ivlen = 8;
+			break;
+		case CRYPTO_AES_RFC4106_GCM_16:
+			/* Be smart at determining the ivlen until better ways are present */
+			ivlen = enccrd->crd_skip - enccrd->crd_inject;
+			ivlen += 4;
+			break;
+		}
+
+		/* Setup ses->iv */
+		if (encflag) {
+			if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) {
+				if (enccrd->crd_alg == CRYPTO_AES_RFC4106_GCM_16) {
+					for (i = 0; i < AESCTR_NONCESIZE; i++)
+						iv[i] = ses->nonce[i];
+					/* XXX: Is this enough? */
+					u_long counter = atomic_fetchadd_long(&ses->aesgcmcounter, 1);
+					bcopy((void *)&counter, iv + AESCTR_NONCESIZE, sizeof(uint64_t));
+					crypto_copyback(crp->crp_flags, crp->crp_buf,
+					    enccrd->crd_inject, AESCTR_IVSIZE, iv + AESCTR_NONCESIZE);
+				} else {
+					arc4rand(iv, AES_BLOCK_LEN, 0);
+					crypto_copyback(crp->crp_flags, crp->crp_buf,
+					    enccrd->crd_inject, ivlen, iv);
+				}
+			}
+		} else {
+			if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) {
+				if (enccrd->crd_alg == CRYPTO_AES_RFC4106_GCM_16) {
+					for (i = 0; i < AESCTR_NONCESIZE; i++)
+						iv[i] = ses->nonce[i];
+					crypto_copydata(crp->crp_flags, crp->crp_buf,
+					    enccrd->crd_inject, AESCTR_IVSIZE, iv + AESCTR_NONCESIZE);
+				} else
+					crypto_copydata(crp->crp_flags, crp->crp_buf,
+					    enccrd->crd_inject, ivlen, iv);
+			}
+		}
+#ifdef AESNI_DEBUG
+		aesni_printhexstr(iv, ivlen);
+		printf("\n");
+#endif
+	}
 
-	authbuf = NULL;
-	authallocated = 0;
 	if (authcrd != NULL) {
 		authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated);
 		if (authbuf == NULL) {
 			error = ENOMEM;
 			goto out1;
 		}
-		/* NOTE: GMAC_DIGEST_LEN == AES_BLOCK_LEN */
-		tag = authcrd->crd_iv;
-	}
 
-	iv = enccrd->crd_iv;
-	/* XXX - validate that enccrd and authcrd have/use same key? */
-	switch (enccrd->crd_alg) {
-	case CRYPTO_AES_CBC:
-		ivlen = 16;
-		break;
-	case CRYPTO_AES_XTS:
-		ivlen = 8;
-		break;
-	case CRYPTO_AES_RFC4106_GCM_16:
-		/* Be smart at determining the ivlen until better ways are present */
-		ivlen = enccrd->crd_skip - enccrd->crd_inject;
-		ivlen += 4;
-		break;
-	}
+		if (ses->algo == CRYPTO_AES_RFC4106_GCM_16) {
+			/* NOTE: GMAC_DIGEST_LEN == AES_BLOCK_LEN */
+			tag = authcrd->crd_iv;
 
-	/* Setup ses->iv */
-	if (encflag) {
-		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
-			bcopy(enccrd->crd_iv, iv, ivlen);
-		else if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
-			if (enccrd->crd_alg == CRYPTO_AES_RFC4106_GCM_16) {
-				for (i = 0; i < AESCTR_NONCESIZE; i++)
-					iv[i] = ses->nonce[i];
-				/* XXX: Is this enough? */
-				u_long counter = atomic_fetchadd_long(&ses->aesgcmcounter, 1);
-				bcopy((void *)&counter, iv + AESCTR_NONCESIZE, sizeof(uint64_t));
-				crypto_copyback(crp->crp_flags, crp->crp_buf,
-				    enccrd->crd_inject, AESCTR_IVSIZE, iv + AESCTR_NONCESIZE);
-			} else {
-				arc4rand(iv, AES_BLOCK_LEN, 0);
-				crypto_copyback(crp->crp_flags, crp->crp_buf,
-				    enccrd->crd_inject, ivlen, iv);
-			}
-		}
-	} else {
-		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
-			bcopy(enccrd->crd_iv, iv, ivlen);
-		else {
-			if (enccrd->crd_alg == CRYPTO_AES_RFC4106_GCM_16) {
-				for (i = 0; i < AESCTR_NONCESIZE; i++)
-					iv[i] = ses->nonce[i];
+			if (!encflag) {
 				crypto_copydata(crp->crp_flags, crp->crp_buf,
-				    enccrd->crd_inject, AESCTR_IVSIZE, iv + AESCTR_NONCESIZE);
-			} else
-				crypto_copydata(crp->crp_flags, crp->crp_buf,
-				    enccrd->crd_inject, ivlen, iv);
-		}
-	}
-#ifdef AESNI_DEBUG
-	aesni_printhexstr(iv, ivlen);
-	printf("\n");
-#endif
-
-	if (authcrd != NULL && !encflag) {
-		crypto_copydata(crp->crp_flags, crp->crp_buf,
-		    authcrd->crd_inject, GMAC_DIGEST_LEN, tag);
-	} else {
+				    authcrd->crd_inject, GMAC_DIGEST_LEN, tag);
+			} else {
 #ifdef AESNI_DEBUG
-		printf("ptag: ");
-		aesni_printhexstr(tag, sizeof tag);
-		printf("\n");
+				printf("ptag: ");
+				aesni_printhexstr(tag, sizeof tag);
+				printf("\n");
 #endif
-		bzero(tag, sizeof tag);
+				bzero(tag, sizeof tag);
+			}
+		}
 	}
+	/* Called by stack only for HASH operation? */
+	if (enccrd == NULL)
+		goto out1;
 
 	td = curthread;
 
@@ -607,8 +657,12 @@ aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
 		    enccrd->crd_len, buf);
 
 	if (!error && authcrd != NULL) {
-		crypto_copyback(crp->crp_flags, crp->crp_buf,
-		    authcrd->crd_inject, crp->crp_ilen - authcrd->crd_inject, tag);
+		if (ses->algo == CRYPTO_AES_RFC4106_GCM_16) {
+			crypto_copyback(crp->crp_flags, crp->crp_buf,
+			    authcrd->crd_inject, crp->crp_ilen - authcrd->crd_inject, tag);
+		} else if (enccrd->crd_next == authcrd) {
+			error = aesni_hash_process(ses, authcrd, crp);
+		}
 	}
 
 out1:
diff --git a/sys/crypto/aesni/aesni.h b/sys/crypto/aesni/aesni.h
index fbbbefb..b1a5ea2 100644
--- a/sys/crypto/aesni/aesni.h
+++ b/sys/crypto/aesni/aesni.h
@@ -61,9 +61,10 @@ struct aesni_session {
         volatile uint64_t aesgcmcounter;
 	int algo;
 	int rounds;
-	/* uint8_t *ses_ictx; */
-	/* uint8_t *ses_octx; */
-	/* int ses_mlen; */
+	struct auth_hash *ses_axf;
+	uint8_t *ses_ictx;
+	uint8_t *ses_octx;
+	int ses_mlen;
 	int used;
 	uint32_t id;
 	struct fpu_kern_ctx *fpu_ctx;
@@ -112,4 +113,10 @@ int aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key,
 uint8_t *aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
     int *allocated);
 
+int aesni_hash_setup(struct aesni_session *ses,
+	struct cryptoini *authini);
+int aesni_hash_process(struct aesni_session *ses,
+	struct cryptodesc *authcrd, struct cryptop *crp);
+void aesni_hash_free(struct aesni_session *ses);
+
 #endif /* _AESNI_H_ */
diff --git a/sys/crypto/aesni/aesni_hash.c b/sys/crypto/aesni/aesni_hash.c
new file mode 100644
index 0000000..84125aa
--- /dev/null
+++ b/sys/crypto/aesni/aesni_hash.c
@@ -0,0 +1,186 @@
+/*-
+ * Copyright (c) 2015 Ermal LUÇI <eri@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+
+#include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */
+#include <opencrypto/xform.h>
+
+#include "aesni.h"
+
+MALLOC_DECLARE(M_AESNI);
+
+/*
+ * Implementation notes.
+ *
+ * We implement all HMAC algorithms provided by crypto(9) framework so aesni can work
+ * with ipsec(4) for AES-CBC/AES-XTS
+ *
+ * This code was stolen from dev/glxsb/aesni_hash.c
+ */
+
+static void
+aesni_hash_key_setup(struct aesni_session *ses, caddr_t key, int klen)
+{
+	struct auth_hash *axf;
+	int i;
+
+	klen /= 8;
+	axf = ses->ses_axf;
+
+	for (i = 0; i < klen; i++)
+		key[i] ^= HMAC_IPAD_VAL;
+
+	axf->Init(ses->ses_ictx);
+	axf->Update(ses->ses_ictx, key, klen);
+	axf->Update(ses->ses_ictx, hmac_ipad_buffer, axf->blocksize - klen);
+
+	for (i = 0; i < klen; i++)
+		key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
+
+	axf->Init(ses->ses_octx);
+	axf->Update(ses->ses_octx, key, klen);
+	axf->Update(ses->ses_octx, hmac_opad_buffer, axf->blocksize - klen);
+
+	for (i = 0; i < klen; i++)
+		key[i] ^= HMAC_OPAD_VAL;
+}
+
+/*
+ * Compute keyed-hash authenticator.
+ */
+static int
+aesni_authcompute(struct aesni_session *ses, struct cryptodesc *crd,
+    caddr_t buf, int flags)
+{
+	u_char hash[HASH_MAX_LEN];
+	struct auth_hash *axf;
+	union authctx ctx;
+	int error;
+
+	axf = ses->ses_axf;
+	bcopy(ses->ses_ictx, &ctx, axf->ctxsize);
+	error = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
+	    (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
+	if (error != 0)
+		return (error);
+	axf->Final(hash, &ctx);
+
+	bcopy(ses->ses_octx, &ctx, axf->ctxsize);
+	axf->Update(&ctx, hash, axf->hashsize);
+	axf->Final(hash, &ctx);
+
+	/* Inject the authentication data */
+	crypto_copyback(flags, buf, crd->crd_inject,
+		ses->ses_mlen == 0 ? axf->hashsize : ses->ses_mlen, hash);
+
+	return (0);
+}
+
+int
+aesni_hash_setup(struct aesni_session *ses, struct cryptoini *macini)
+{
+
+	ses->ses_mlen = macini->cri_mlen;
+
+	/* Find software structure which describes HMAC algorithm. */
+	switch (macini->cri_alg) {
+	case CRYPTO_NULL_HMAC:
+		ses->ses_axf = &auth_hash_null;
+		break;
+	case CRYPTO_MD5_HMAC:
+		ses->ses_axf = &auth_hash_hmac_md5;
+		break;
+	case CRYPTO_SHA1_HMAC:
+		ses->ses_axf = &auth_hash_hmac_sha1;
+		break;
+	case CRYPTO_SHA2_256_HMAC:
+		ses->ses_axf = &auth_hash_hmac_sha2_256;
+		break;
+	case CRYPTO_SHA2_384_HMAC:
+		ses->ses_axf = &auth_hash_hmac_sha2_384;
+		break;
+	case CRYPTO_SHA2_512_HMAC:
+		ses->ses_axf = &auth_hash_hmac_sha2_512;
+		break;
+	default:
+		return EINVAL;
+		break;
+	}
+
+	/* Allocate memory for HMAC inner and outer contexts. */
+	ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_AESNI,
+	    M_ZERO | M_NOWAIT);
+	if (ses->ses_ictx == NULL)
+		return (ENOMEM);
+	ses->ses_octx = malloc(ses->ses_axf->ctxsize, M_AESNI,
+	    M_ZERO | M_NOWAIT);
+	if (ses->ses_octx == NULL) {
+		free(ses->ses_ictx, M_AESNI);
+		return (ENOMEM);
+	}
+
+	/* Setup key if given. */
+	if (macini->cri_key != NULL) {
+		aesni_hash_key_setup(ses, macini->cri_key,
+		    macini->cri_klen);
+	}
+	return (0);
+}
+
+int
+aesni_hash_process(struct aesni_session *ses, struct cryptodesc *maccrd,
+    struct cryptop *crp)
+{
+	int error;
+
+	if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0)
+		aesni_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen);
+
+	error = aesni_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags);
+	return (error);
+}
+
+void
+aesni_hash_free(struct aesni_session *ses)
+{
+
+	if (ses->ses_ictx != NULL) {
+		bzero(ses->ses_ictx, ses->ses_axf->ctxsize);
+		free(ses->ses_ictx, M_AESNI);
+		ses->ses_ictx = NULL;
+	}
+	if (ses->ses_octx != NULL) {
+		bzero(ses->ses_octx, ses->ses_axf->ctxsize);
+		free(ses->ses_octx, M_AESNI);
+		ses->ses_octx = NULL;
+	}
+}
diff --git a/sys/modules/aesni/Makefile b/sys/modules/aesni/Makefile
index e66f941..6aac042 100644
--- a/sys/modules/aesni/Makefile
+++ b/sys/modules/aesni/Makefile
@@ -3,7 +3,7 @@
 .PATH: ${.CURDIR}/../../crypto/aesni
 
 KMOD=	aesni
-SRCS=	aesni.c
+SRCS=	aesni.c aesni_hash.c
 SRCS+=	aeskeys_${MACHINE_CPUARCH}.S
 SRCS+=	device_if.h bus_if.h opt_bus.h cryptodev_if.h