diff options
Diffstat (limited to 'sys/opencrypto')
-rw-r--r-- | sys/opencrypto/criov.c | 85 | ||||
-rw-r--r-- | sys/opencrypto/crypto.c | 19 | ||||
-rw-r--r-- | sys/opencrypto/cryptodeflate.c | 18 | ||||
-rw-r--r-- | sys/opencrypto/cryptodev.c | 283 | ||||
-rw-r--r-- | sys/opencrypto/cryptodev.h | 183 | ||||
-rw-r--r-- | sys/opencrypto/cryptosoft.c | 793 | ||||
-rw-r--r-- | sys/opencrypto/gfmult.c | 275 | ||||
-rw-r--r-- | sys/opencrypto/gfmult.h | 128 | ||||
-rw-r--r-- | sys/opencrypto/gmac.c | 119 | ||||
-rw-r--r-- | sys/opencrypto/gmac.h | 56 | ||||
-rw-r--r-- | sys/opencrypto/xform.c | 275 | ||||
-rw-r--r-- | sys/opencrypto/xform.h | 21 |
12 files changed, 1761 insertions, 494 deletions
diff --git a/sys/opencrypto/criov.c b/sys/opencrypto/criov.c index 96f0dfc..499bfe3 100644 --- a/sys/opencrypto/criov.c +++ b/sys/opencrypto/criov.c @@ -99,35 +99,31 @@ cuio_copyback(struct uio* uio, int off, int len, caddr_t cp) } /* - * Return a pointer to iov/offset of location in iovec list. + * Return the index and offset of location in iovec list. */ -struct iovec * +int cuio_getptr(struct uio *uio, int loc, int *off) { - struct iovec *iov = uio->uio_iov; - int iol = uio->uio_iovcnt; + int ind, len; - while (loc >= 0) { - /* Normal end of search */ - if (loc < iov->iov_len) { + ind = 0; + while (loc >= 0 && ind < uio->uio_iovcnt) { + len = uio->uio_iov[ind].iov_len; + if (len > loc) { *off = loc; - return (iov); + return (ind); } + loc -= len; + ind++; + } - loc -= iov->iov_len; - if (iol == 0) { - if (loc == 0) { - /* Point at the end of valid data */ - *off = iov->iov_len; - return (iov); - } else - return (NULL); - } else { - iov++, iol--; - } - } + if (ind > 0 && loc == 0) { + ind--; + *off = uio->uio_iov[ind].iov_len; + return (ind); + } - return (NULL); + return (-1); } /* @@ -196,3 +192,50 @@ crypto_apply(int flags, caddr_t buf, int off, int len, error = (*f)(arg, buf + off, len); return (error); } + +int +crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iovptr, int *cnt, + int *allocated) +{ + struct iovec *iov; + struct mbuf *m, *mtmp; + int i, j; + + *allocated = 0; + iov = *iovptr; + if (iov == NULL) + *cnt = 0; + + m = mbuf; + i = 0; + while (m != NULL) { + if (i == *cnt) { + /* we need to allocate a larger array */ + j = 1; + mtmp = m; + while ((mtmp = mtmp->m_next) != NULL) + j++; + iov = malloc(sizeof *iov * (i + j), M_CRYPTO_DATA, + M_NOWAIT); + if (iov == NULL) + return ENOMEM; + *allocated = 1; + *cnt = i + j; + memcpy(iov, *iovptr, sizeof *iov * i); + } + + iov[i].iov_base = m->m_data; + iov[i].iov_len = m->m_len; + + i++; + m = m->m_next; + } + + if (*allocated) + KASSERT(*cnt == i, ("did not allocate correct amount: %d != %d", + *cnt, i)); + + *iovptr = iov; + *cnt = i; + return 0; +} diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c index 53c11e6..d084b77 100644 --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -57,7 +57,6 @@ __FBSDID("$FreeBSD$"); #define CRYPTO_TIMING /* enable timing support */ #include "opt_ddb.h" -#include "opt_kdtrace.h" #include <sys/param.h> #include <sys/systm.h> @@ -162,10 +161,10 @@ int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, &crypto_userasymcrypto, 0, "Enable/disable user-mode access to asymmetric crypto support"); -int crypto_devallowsoft = 0; /* only use hardware crypto for asym */ +int crypto_devallowsoft = 0; /* only use hardware crypto */ SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, &crypto_devallowsoft, 0, - "Enable/disable use of software asym crypto support"); + "Enable/disable use of software crypto by /dev/crypto"); MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); @@ -369,9 +368,8 @@ again: best = cap; } } - if (best != NULL) - return best; - if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { + if (best == NULL && match == CRYPTOCAP_F_HARDWARE && + (flags & CRYPTOCAP_F_SOFTWARE)) { /* sort of an Algol 68-style for loop */ match = CRYPTOCAP_F_SOFTWARE; goto again; @@ -422,9 +420,12 @@ crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int crid) (*sid) <<= 32; (*sid) |= (lid & 0xffffffff); cap->cc_sessions++; - } - } else + } else + CRYPTDEB("dev newsession failed"); + } else { + CRYPTDEB("no driver"); err = EINVAL; + } CRYPTO_DRIVER_UNLOCK(); return err; } @@ -1180,8 +1181,8 @@ crypto_kdone(struct cryptkop *krp) /* XXX: What if driver is loaded in the meantime? */ if (krp->krp_hid < crypto_drivers_num) { cap = &crypto_drivers[krp->krp_hid]; + KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0")); cap->cc_koperations--; - KASSERT(cap->cc_koperations >= 0, ("cc_koperations < 0")); if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) crypto_remove(cap); } diff --git a/sys/opencrypto/cryptodeflate.c b/sys/opencrypto/cryptodeflate.c index 2113611..f342a91 100644 --- a/sys/opencrypto/cryptodeflate.c +++ b/sys/opencrypto/cryptodeflate.c @@ -35,8 +35,6 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include "opt_kdtrace.h" - #include <sys/types.h> #include <sys/param.h> #include <sys/malloc.h> @@ -107,8 +105,8 @@ deflate_global(data, size, decomp, out) bufh = bufp = malloc(sizeof(*bufp) + (size_t)(size * i), M_CRYPTO_DATA, M_NOWAIT); if (bufp == NULL) { - SDT_PROBE3(opencrypto, deflate, deflate_global, bad, - decomp, 0, __LINE__); + SDT_PROBE5(opencrypto, deflate, deflate_global, bad, + decomp, 0, __LINE__, 0, 0); goto bad2; } bufp->next = NULL; @@ -127,8 +125,8 @@ deflate_global(data, size, decomp, out) deflateInit2(&zbuf, Z_DEFAULT_COMPRESSION, Z_METHOD, window_deflate, Z_MEMLEVEL, Z_DEFAULT_STRATEGY); if (error != Z_OK) { - SDT_PROBE3(opencrypto, deflate, deflate_global, bad, - decomp, error, __LINE__); + SDT_PROBE5(opencrypto, deflate, deflate_global, bad, + decomp, error, __LINE__, 0, 0); goto bad; } @@ -167,8 +165,8 @@ deflate_global(data, size, decomp, out) p = malloc(sizeof(*p) + (size_t)(size * i), M_CRYPTO_DATA, M_NOWAIT); if (p == NULL) { - SDT_PROBE3(opencrypto, deflate, deflate_global, - bad, decomp, 0, __LINE__); + SDT_PROBE5(opencrypto, deflate, deflate_global, + bad, decomp, 0, __LINE__, 0, 0); goto bad; } p->next = NULL; @@ -197,8 +195,8 @@ deflate_global(data, size, decomp, out) *out = malloc(result, M_CRYPTO_DATA, M_NOWAIT); if (*out == NULL) { - SDT_PROBE3(opencrypto, deflate, deflate_global, bad, - decomp, 0, __LINE__); + SDT_PROBE5(opencrypto, deflate, deflate_global, bad, + decomp, 0, __LINE__, 0, 0); goto bad; } if (decomp) diff --git a/sys/opencrypto/cryptodev.c b/sys/opencrypto/cryptodev.c index 44bfa5c..f5e80a0 100644 --- a/sys/opencrypto/cryptodev.c +++ b/sys/opencrypto/cryptodev.c @@ -3,6 +3,12 @@ /*- * Copyright (c) 2001 Theo de Raadt * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by John-Mark Gurney + * under sponsorship of the FreeBSD Foundation and + * Rubicon Communications, LLC (Netgate). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -54,10 +60,16 @@ __FBSDID("$FreeBSD$"); #include <sys/module.h> #include <sys/fcntl.h> #include <sys/bus.h> +#include <sys/user.h> +#include <sys/sdt.h> #include <opencrypto/cryptodev.h> #include <opencrypto/xform.h> +SDT_PROVIDER_DECLARE(opencrypto); + +SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); + #ifdef COMPAT_FREEBSD32 #include <sys/mount.h> #include <compat/freebsd32/freebsd32.h> @@ -317,6 +329,8 @@ static int csefree(struct csession *); static int cryptodev_op(struct csession *, struct crypt_op *, struct ucred *, struct thread *td); +static int cryptodev_aead(struct csession *, struct crypt_aead *, + struct ucred *, struct thread *); static int cryptodev_key(struct crypt_kop *); static int cryptodev_find(struct crypt_find_op *); @@ -349,13 +363,24 @@ cryptof_truncate( * by device name/class or through search constraints. */ static int -checkforsoftware(int crid) +checkforsoftware(int *cridp) { - if (crid & CRYPTOCAP_F_SOFTWARE) - return EINVAL; /* XXX */ - if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && - (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) - return EINVAL; /* XXX */ + int crid; + + crid = *cridp; + + if (!crypto_devallowsoft) { + if (crid & CRYPTOCAP_F_SOFTWARE) { + if (crid & CRYPTOCAP_F_HARDWARE) { + *cridp = CRYPTOCAP_F_HARDWARE; + return 0; + } + return EINVAL; + } + if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && + (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) + return EINVAL; + } return 0; } @@ -374,6 +399,7 @@ cryptof_ioctl( struct csession *cse; struct session_op *sop; struct crypt_op *cop; + struct crypt_aead *caead; struct enc_xform *txform = NULL; struct auth_hash *thash = NULL; struct crypt_kop *kop; @@ -434,7 +460,15 @@ cryptof_ioctl( case CRYPTO_CAMELLIA_CBC: txform = &enc_xform_camellia; break; + case CRYPTO_AES_ICM: + txform = &enc_xform_aes_icm; + break; + case CRYPTO_AES_NIST_GCM_16: + txform = &enc_xform_aes_nist_gcm; + break; + default: + CRYPTDEB("invalid cipher"); return (EINVAL); } @@ -459,6 +493,16 @@ cryptof_ioctl( case CRYPTO_RIPEMD160_HMAC: thash = &auth_hash_hmac_ripemd_160; break; + case CRYPTO_AES_128_NIST_GMAC: + thash = &auth_hash_nist_gmac_aes_128; + break; + case CRYPTO_AES_192_NIST_GMAC: + thash = &auth_hash_nist_gmac_aes_192; + break; + case CRYPTO_AES_256_NIST_GMAC: + thash = &auth_hash_nist_gmac_aes_256; + break; + #ifdef notdef case CRYPTO_MD5: thash = &auth_hash_md5; @@ -471,6 +515,7 @@ cryptof_ioctl( thash = &auth_hash_null; break; default: + CRYPTDEB("invalid mac"); return (EINVAL); } @@ -482,6 +527,7 @@ cryptof_ioctl( crie.cri_klen = sop->keylen * 8; if (sop->keylen > txform->maxkey || sop->keylen < txform->minkey) { + CRYPTDEB("invalid cipher parameters"); error = EINVAL; goto bail; } @@ -489,8 +535,10 @@ cryptof_ioctl( crie.cri_key = malloc(crie.cri_klen / 8, M_XDATA, M_WAITOK); if ((error = copyin(sop->key, crie.cri_key, - crie.cri_klen / 8))) + crie.cri_klen / 8))) { + CRYPTDEB("invalid key"); goto bail; + } if (thash) crie.cri_next = &cria; } @@ -499,6 +547,7 @@ cryptof_ioctl( cria.cri_alg = thash->type; cria.cri_klen = sop->mackeylen * 8; if (sop->mackeylen != thash->keysize) { + CRYPTDEB("invalid mac key length"); error = EINVAL; goto bail; } @@ -507,8 +556,10 @@ cryptof_ioctl( cria.cri_key = malloc(cria.cri_klen / 8, M_XDATA, M_WAITOK); if ((error = copyin(sop->mackey, cria.cri_key, - cria.cri_klen / 8))) + cria.cri_klen / 8))) { + CRYPTDEB("invalid mac key"); goto bail; + } } } @@ -519,14 +570,18 @@ cryptof_ioctl( #endif ) { crid = SES2(sop)->crid; - error = checkforsoftware(crid); - if (error) + error = checkforsoftware(&crid); + if (error) { + CRYPTDEB("checkforsoftware"); goto bail; + } } else crid = CRYPTOCAP_F_HARDWARE; error = crypto_newsession(&sid, (txform ? &crie : &cria), crid); - if (error) + if (error) { + CRYPTDEB("crypto_newsession"); goto bail; + } cse = csecreate(fcr, sid, crie.cri_key, crie.cri_klen, cria.cri_key, cria.cri_klen, sop->cipher, sop->mac, txform, @@ -535,6 +590,7 @@ cryptof_ioctl( if (cse == NULL) { crypto_freesession(sid); error = EINVAL; + CRYPTDEB("csecreate"); goto bail; } sop->ses = cse->ses; @@ -581,8 +637,10 @@ bail: #endif cop = (struct crypt_op *)data; cse = csefind(fcr, cop->ses); - if (cse == NULL) + if (cse == NULL) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); + } error = cryptodev_op(cse, cop, active_cred, td); #ifdef COMPAT_FREEBSD32 if (error == 0 && cmd == CIOCCRYPT32) @@ -636,6 +694,13 @@ bail: case CIOCFINDDEV: error = cryptodev_find((struct crypt_find_op *)data); break; + case CIOCCRYPTAEAD: + caead = (struct crypt_aead *)data; + cse = csefind(fcr, caead->ses); + if (cse == NULL) + return (EINVAL); + error = cryptodev_aead(cse, caead, active_cred, td); + break; default: error = EINVAL; break; @@ -658,12 +723,16 @@ cryptodev_op( struct cryptodesc *crde = NULL, *crda = NULL; int error; - if (cop->len > 256*1024-4) + if (cop->len > 256*1024-4) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (E2BIG); + } if (cse->txform) { - if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) + if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); return (EINVAL); + } } cse->uio.uio_iov = &cse->iovec; @@ -683,6 +752,7 @@ cryptodev_op( crp = crypto_getreq((cse->txform != NULL) + (cse->thash != NULL)); if (crp == NULL) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = ENOMEM; goto bail; } @@ -695,13 +765,17 @@ cryptodev_op( if (cse->txform) crde = crp->crp_desc; else { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } } - if ((error = copyin(cop->src, cse->uio.uio_iov[0].iov_base, cop->len))) + if ((error = copyin(cop->src, cse->uio.uio_iov[0].iov_base, + cop->len))) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; + } if (crda) { crda->crd_skip = 0; @@ -736,15 +810,20 @@ cryptodev_op( if (cop->iv) { if (crde == NULL) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */ + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } - if ((error = copyin(cop->iv, cse->tmp_iv, cse->txform->blocksize))) + if ((error = copyin(cop->iv, cse->tmp_iv, + cse->txform->blocksize))) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; + } bcopy(cse->tmp_iv, crde->crd_iv, cse->txform->blocksize); crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; crde->crd_skip = 0; @@ -757,6 +836,7 @@ cryptodev_op( } if (cop->mac && crda == NULL) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = EINVAL; goto bail; } @@ -775,8 +855,10 @@ again: error = msleep(crp, &cse->lock, PWAIT, "crydev", 0); mtx_unlock(&cse->lock); - if (error != 0) + if (error != 0) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; + } if (crp->crp_etype == EAGAIN) { crp->crp_etype = 0; @@ -785,23 +867,30 @@ again: } if (crp->crp_etype != 0) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = crp->crp_etype; goto bail; } if (cse->error) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); error = cse->error; goto bail; } if (cop->dst && - (error = copyout(cse->uio.uio_iov[0].iov_base, cop->dst, cop->len))) + (error = copyout(cse->uio.uio_iov[0].iov_base, cop->dst, + cop->len))) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; + } if (cop->mac && (error = copyout((caddr_t)cse->uio.uio_iov[0].iov_base + cop->len, - cop->mac, cse->thash->hashsize))) + cop->mac, cse->thash->hashsize))) { + SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); goto bail; + } bail: if (crp) @@ -813,6 +902,151 @@ bail: } static int +cryptodev_aead( + struct csession *cse, + struct crypt_aead *caead, + struct ucred *active_cred, + struct thread *td) +{ + struct uio *uio; + struct cryptop *crp = NULL; + struct cryptodesc *crde = NULL, *crda = NULL; + int error; + + if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) + return (E2BIG); + + if (cse->txform == NULL || cse->thash == NULL || caead->tag == NULL || + (caead->len % cse->txform->blocksize) != 0) + return (EINVAL); + + uio = &cse->uio; + uio->uio_iov = &cse->iovec; + uio->uio_iovcnt = 1; + uio->uio_offset = 0; + uio->uio_resid = caead->len + caead->aadlen + cse->thash->hashsize; + uio->uio_segflg = UIO_SYSSPACE; + uio->uio_rw = UIO_WRITE; + uio->uio_td = td; + uio->uio_iov[0].iov_len = uio->uio_resid; + + uio->uio_iov[0].iov_base = malloc(uio->uio_iov[0].iov_len, + M_XDATA, M_WAITOK); + + crp = crypto_getreq(2); + if (crp == NULL) { + error = ENOMEM; + goto bail; + } + + crda = crp->crp_desc; + crde = crda->crd_next; + + if ((error = copyin(caead->src, cse->uio.uio_iov[0].iov_base, + caead->len))) + goto bail; + + if ((error = copyin(caead->aad, (char *)cse->uio.uio_iov[0].iov_base + + caead->len, caead->aadlen))) + goto bail; + + crda->crd_skip = caead->len; + crda->crd_len = caead->aadlen; + crda->crd_inject = caead->len + caead->aadlen; + + crda->crd_alg = cse->mac; + crda->crd_key = cse->mackey; + crda->crd_klen = cse->mackeylen * 8; + + if (caead->op == COP_ENCRYPT) + crde->crd_flags |= CRD_F_ENCRYPT; + else + crde->crd_flags &= ~CRD_F_ENCRYPT; + /* crde->crd_skip set below */ + crde->crd_len = caead->len; + crde->crd_inject = 0; + + crde->crd_alg = cse->cipher; + crde->crd_key = cse->key; + crde->crd_klen = cse->keylen * 8; + + crp->crp_ilen = caead->len + caead->aadlen; + crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM + | (caead->flags & COP_F_BATCH); + crp->crp_buf = (caddr_t)&cse->uio.uio_iov; + crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_cb; + crp->crp_sid = cse->sid; + crp->crp_opaque = (void *)cse; + + if (caead->iv) { + if (caead->ivlen > sizeof cse->tmp_iv) { + error = EINVAL; + goto bail; + } + + if ((error = copyin(caead->iv, cse->tmp_iv, caead->ivlen))) + goto bail; + bcopy(cse->tmp_iv, crde->crd_iv, caead->ivlen); + crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT; + crde->crd_skip = 0; + } else { + crde->crd_flags |= CRD_F_IV_PRESENT; + crde->crd_skip = cse->txform->blocksize; + crde->crd_len -= cse->txform->blocksize; + } + + if ((error = copyin(caead->tag, (caddr_t)cse->uio.uio_iov[0].iov_base + + caead->len + caead->aadlen, cse->thash->hashsize))) + goto bail; +again: + /* + * Let the dispatch run unlocked, then, interlock against the + * callback before checking if the operation completed and going + * to sleep. This insures drivers don't inherit our lock which + * results in a lock order reversal between crypto_dispatch forced + * entry and the crypto_done callback into us. + */ + error = crypto_dispatch(crp); + mtx_lock(&cse->lock); + if (error == 0 && (crp->crp_flags & CRYPTO_F_DONE) == 0) + error = msleep(crp, &cse->lock, PWAIT, "crydev", 0); + mtx_unlock(&cse->lock); + + if (error != 0) + goto bail; + + if (crp->crp_etype == EAGAIN) { + crp->crp_etype = 0; + crp->crp_flags &= ~CRYPTO_F_DONE; + goto again; + } + + if (crp->crp_etype != 0) { + error = crp->crp_etype; + goto bail; + } + + if (cse->error) { + error = cse->error; + goto bail; + } + + if (caead->dst && (error = copyout(cse->uio.uio_iov[0].iov_base, + caead->dst, caead->len))) + goto bail; + + if ((error = copyout((caddr_t)cse->uio.uio_iov[0].iov_base + + caead->len + caead->aadlen, caead->tag, cse->thash->hashsize))) + goto bail; + +bail: + crypto_freereq(crp); + free(cse->uio.uio_iov[0].iov_base, M_XDATA); + + return (error); +} + +static int cryptodev_cb(void *op) { struct cryptop *crp = (struct cryptop *) op; @@ -941,14 +1175,16 @@ static int cryptodev_find(struct crypt_find_op *find) { device_t dev; + size_t fnlen = sizeof find->name; if (find->crid != -1) { dev = crypto_find_device_byhid(find->crid); if (dev == NULL) return (ENOENT); - strlcpy(find->name, device_get_nameunit(dev), - sizeof(find->name)); + strncpy(find->name, device_get_nameunit(dev), fnlen); + find->name[fnlen - 1] = '\x0'; } else { + find->name[fnlen - 1] = '\x0'; find->crid = crypto_find_driver(find->name); if (find->crid == -1) return (ENOENT); @@ -1044,12 +1280,7 @@ csecreate(struct fcrypt *fcr, u_int64_t sid, caddr_t key, u_int64_t keylen, { struct csession *cse; -#ifdef INVARIANTS - /* NB: required when mtx_init is built with INVARIANTS */ cse = malloc(sizeof(struct csession), M_XDATA, M_NOWAIT | M_ZERO); -#else - cse = malloc(sizeof(struct csession), M_XDATA, M_NOWAIT); -#endif if (cse == NULL) return NULL; mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); diff --git a/sys/opencrypto/cryptodev.h b/sys/opencrypto/cryptodev.h index e299522..d14fb3a 100644 --- a/sys/opencrypto/cryptodev.h +++ b/sys/opencrypto/cryptodev.h @@ -23,6 +23,12 @@ * PURPOSE. * * Copyright (c) 2001 Theo de Raadt + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by John-Mark Gurney + * under sponsorship of the FreeBSD Foundation and + * Rubicon Communications, LLC (Netgate). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -72,6 +78,7 @@ #define SHA2_512_HASH_LEN 64 #define MD5_KPDK_HASH_LEN 16 #define SHA1_KPDK_HASH_LEN 20 +#define AES_GMAC_HASH_LEN 16 /* Maximum hash algorithm result length */ #define HASH_MAX_LEN SHA2_512_HASH_LEN /* Keep this updated */ @@ -80,40 +87,87 @@ #define MD5_HMAC_BLOCK_LEN 64 #define SHA1_HMAC_BLOCK_LEN 64 #define RIPEMD160_HMAC_BLOCK_LEN 64 -#define SHA2_256_HMAC_BLOCK_LEN 64 -#define SHA2_384_HMAC_BLOCK_LEN 128 -#define SHA2_512_HMAC_BLOCK_LEN 128 +#define SHA2_256_HMAC_BLOCK_LEN 64 +#define SHA2_384_HMAC_BLOCK_LEN 128 +#define SHA2_512_HMAC_BLOCK_LEN 128 /* Maximum HMAC block length */ -#define HMAC_MAX_BLOCK_LEN SHA2_512_HMAC_BLOCK_LEN /* Keep this updated */ +#define HMAC_MAX_BLOCK_LEN SHA2_512_HMAC_BLOCK_LEN /* Keep this updated */ #define HMAC_IPAD_VAL 0x36 #define HMAC_OPAD_VAL 0x5C +/* HMAC Key Length */ +#define NULL_HMAC_KEY_LEN 0 +#define MD5_HMAC_KEY_LEN 16 +#define SHA1_HMAC_KEY_LEN 20 +#define RIPEMD160_HMAC_KEY_LEN 20 +#define SHA2_256_HMAC_KEY_LEN 32 +#define SHA2_384_HMAC_KEY_LEN 48 +#define SHA2_512_HMAC_KEY_LEN 64 +#define AES_128_GMAC_KEY_LEN 16 +#define AES_192_GMAC_KEY_LEN 24 +#define AES_256_GMAC_KEY_LEN 32 /* Encryption algorithm block sizes */ -#define NULL_BLOCK_LEN 4 -#define DES_BLOCK_LEN 8 -#define DES3_BLOCK_LEN 8 -#define BLOWFISH_BLOCK_LEN 8 -#define SKIPJACK_BLOCK_LEN 8 -#define CAST128_BLOCK_LEN 8 -#define RIJNDAEL128_BLOCK_LEN 16 -#define AES_BLOCK_LEN RIJNDAEL128_BLOCK_LEN -#define CAMELLIA_BLOCK_LEN 16 -#define EALG_MAX_BLOCK_LEN AES_BLOCK_LEN /* Keep this updated */ +#define NULL_BLOCK_LEN 4 /* IPsec to maintain alignment */ +#define DES_BLOCK_LEN 8 +#define DES3_BLOCK_LEN 8 +#define BLOWFISH_BLOCK_LEN 8 +#define SKIPJACK_BLOCK_LEN 8 +#define CAST128_BLOCK_LEN 8 +#define RIJNDAEL128_BLOCK_LEN 16 +#define AES_BLOCK_LEN 16 +#define AES_ICM_BLOCK_LEN 1 +#define ARC4_BLOCK_LEN 1 +#define CAMELLIA_BLOCK_LEN 16 +#define EALG_MAX_BLOCK_LEN AES_BLOCK_LEN /* Keep this updated */ + +/* IV Lengths */ + +#define ARC4_IV_LEN 1 +#define AES_GCM_IV_LEN 12 +#define AES_XTS_IV_LEN 8 +#define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ + +/* Min and Max Encryption Key Sizes */ +#define NULL_MIN_KEY 0 +#define NULL_MAX_KEY 256 /* 2048 bits, max key */ +#define DES_MIN_KEY 8 +#define DES_MAX_KEY DES_MIN_KEY +#define TRIPLE_DES_MIN_KEY 24 +#define TRIPLE_DES_MAX_KEY TRIPLE_DES_MIN_KEY +#define BLOWFISH_MIN_KEY 5 +#define BLOWFISH_MAX_KEY 56 /* 448 bits, max key */ +#define CAST_MIN_KEY 5 +#define CAST_MAX_KEY 16 +#define SKIPJACK_MIN_KEY 10 +#define SKIPJACK_MAX_KEY SKIPJACK_MIN_KEY +#define RIJNDAEL_MIN_KEY 16 +#define RIJNDAEL_MAX_KEY 32 +#define AES_MIN_KEY RIJNDAEL_MIN_KEY +#define AES_MAX_KEY RIJNDAEL_MAX_KEY +#define AES_XTS_MIN_KEY (2 * AES_MIN_KEY) +#define AES_XTS_MAX_KEY (2 * AES_MAX_KEY) +#define ARC4_MIN_KEY 1 +#define ARC4_MAX_KEY 32 +#define CAMELLIA_MIN_KEY 8 +#define CAMELLIA_MAX_KEY 32 + +/* Maximum hash algorithm result length */ +#define AALG_MAX_RESULT_LEN 64 /* Keep this updated */ #define CRYPTO_ALGORITHM_MIN 1 -#define CRYPTO_DES_CBC 1 -#define CRYPTO_3DES_CBC 2 -#define CRYPTO_BLF_CBC 3 -#define CRYPTO_CAST_CBC 4 -#define CRYPTO_SKIPJACK_CBC 5 -#define CRYPTO_MD5_HMAC 6 -#define CRYPTO_SHA1_HMAC 7 -#define CRYPTO_RIPEMD160_HMAC 8 -#define CRYPTO_MD5_KPDK 9 -#define CRYPTO_SHA1_KPDK 10 -#define CRYPTO_RIJNDAEL128_CBC 11 /* 128 bit blocksize */ -#define CRYPTO_AES_CBC 11 /* 128 bit blocksize -- the same as above */ -#define CRYPTO_ARC4 12 +#define CRYPTO_DES_CBC 1 +#define CRYPTO_3DES_CBC 2 +#define CRYPTO_BLF_CBC 3 +#define CRYPTO_CAST_CBC 4 +#define CRYPTO_SKIPJACK_CBC 5 +#define CRYPTO_MD5_HMAC 6 +#define CRYPTO_SHA1_HMAC 7 +#define CRYPTO_RIPEMD160_HMAC 8 +#define CRYPTO_MD5_KPDK 9 +#define CRYPTO_SHA1_KPDK 10 +#define CRYPTO_RIJNDAEL128_CBC 11 /* 128 bit blocksize */ +#define CRYPTO_AES_CBC 11 /* 128 bit blocksize -- the same as above */ +#define CRYPTO_ARC4 12 #define CRYPTO_MD5 13 #define CRYPTO_SHA1 14 #define CRYPTO_NULL_HMAC 15 @@ -122,9 +176,18 @@ #define CRYPTO_SHA2_256_HMAC 18 #define CRYPTO_SHA2_384_HMAC 19 #define CRYPTO_SHA2_512_HMAC 20 -#define CRYPTO_CAMELLIA_CBC 21 +#define CRYPTO_CAMELLIA_CBC 21 #define CRYPTO_AES_XTS 22 -#define CRYPTO_ALGORITHM_MAX 22 /* Keep updated - see below */ +#define CRYPTO_AES_ICM 23 /* commonly known as CTR mode */ +#define CRYPTO_AES_NIST_GMAC 24 /* cipher side */ +#define CRYPTO_AES_NIST_GCM_16 25 /* 16 byte ICV */ +#define CRYPTO_AES_128_NIST_GMAC 26 /* auth side */ +#define CRYPTO_AES_192_NIST_GMAC 27 /* auth side */ +#define CRYPTO_AES_256_NIST_GMAC 28 /* auth side */ +#define CRYPTO_ALGORITHM_MAX 28 /* Keep updated - see below */ + +#define CRYPTO_ALGO_VALID(x) ((x) >= CRYPTO_ALGORITHM_MIN && \ + (x) <= CRYPTO_ALGORITHM_MAX) /* Algorithm flags */ #define CRYPTO_ALG_FLAG_SUPPORTED 0x01 /* Algorithm is supported */ @@ -182,6 +245,20 @@ struct crypt_op { caddr_t iv; }; +/* op and flags the same as crypt_op */ +struct crypt_aead { + u_int32_t ses; + u_int16_t op; /* i.e. COP_ENCRYPT */ + u_int16_t flags; + u_int len; + u_int aadlen; + u_int ivlen; + caddr_t src, dst; /* become iov[] inside kernel */ + caddr_t aad; /* additional authenticated data */ + caddr_t tag; /* must fit for chosen TAG length */ + caddr_t iv; +}; + /* * Parameters for looking up a crypto driver/device by * device name or by id. The latter are returned for @@ -239,6 +316,7 @@ struct crypt_kop { #define CIOCGSESSION2 _IOWR('c', 106, struct session2_op) #define CIOCKEY2 _IOWR('c', 107, struct crypt_kop) #define CIOCFINDDEV _IOWR('c', 108, struct crypt_find_op) +#define CIOCCRYPTAEAD _IOWR('c', 109, struct crypt_aead) struct cryptotstat { struct timespec acc; /* total accumulated time */ @@ -269,6 +347,14 @@ struct cryptostats { }; #ifdef _KERNEL + +#if 0 +#define CRYPTDEB(s) do { printf("%s:%d: %s\n", __FILE__, __LINE__, s); \ + } while (0) +#else +#define CRYPTDEB(s) do { } while (0) +#endif + /* Standard initialization structure beginning */ struct cryptoini { int cri_alg; /* Algorithm to use */ @@ -292,14 +378,15 @@ struct cryptodesc { place, so don't copy. */ #define CRD_F_IV_EXPLICIT 0x04 /* IV explicitly provided */ #define CRD_F_DSA_SHA_NEEDED 0x08 /* Compute SHA-1 of buffer for DSA */ +#define CRD_F_COMP 0x0f /* Set when doing compression */ #define CRD_F_KEY_EXPLICIT 0x10 /* Key explicitly provided */ -#define CRD_F_COMP 0x0f /* Set when doing compression */ struct cryptoini CRD_INI; /* Initialization/context data */ -#define crd_iv CRD_INI.cri_iv -#define crd_key CRD_INI.cri_key -#define crd_alg CRD_INI.cri_alg -#define crd_klen CRD_INI.cri_klen +#define crd_esn CRD_INI.cri_esn +#define crd_iv CRD_INI.cri_iv +#define crd_key CRD_INI.cri_key +#define crd_alg CRD_INI.cri_alg +#define crd_klen CRD_INI.cri_klen struct cryptodesc *crd_next; }; @@ -324,9 +411,8 @@ struct cryptop { */ int crp_flags; -#define CRYPTO_F_IMBUF 0x0001 /* Input/output are mbuf chains */ -#define CRYPTO_F_IOV 0x0002 /* Input/output are uio */ -#define CRYPTO_F_REL 0x0004 /* Must return data in same place */ +#define CRYPTO_F_IMBUF 0x0001 /* Input/output are mbuf chains */ +#define CRYPTO_F_IOV 0x0002 /* Input/output are uio */ #define CRYPTO_F_BATCH 0x0008 /* Batch op if possible */ #define CRYPTO_F_CBIMM 0x0010 /* Do callback immediately */ #define CRYPTO_F_DONE 0x0020 /* Operation completed */ @@ -341,12 +427,12 @@ struct cryptop { struct bintime crp_tstamp; /* performance time stamp */ }; -#define CRYPTO_BUF_CONTIG 0x0 -#define CRYPTO_BUF_IOV 0x1 -#define CRYPTO_BUF_MBUF 0x2 +#define CRYPTO_BUF_CONTIG 0x0 +#define CRYPTO_BUF_IOV 0x1 +#define CRYPTO_BUF_MBUF 0x2 -#define CRYPTO_OP_DECRYPT 0x0 -#define CRYPTO_OP_ENCRYPT 0x1 +#define CRYPTO_OP_DECRYPT 0x0 +#define CRYPTO_OP_ENCRYPT 0x1 /* * Hints passed to process methods. @@ -381,9 +467,9 @@ MALLOC_DECLARE(M_CRYPTO_DATA); extern int crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard); extern int crypto_freesession(u_int64_t sid); -#define CRYPTOCAP_F_HARDWARE CRYPTO_FLAG_HARDWARE -#define CRYPTOCAP_F_SOFTWARE CRYPTO_FLAG_SOFTWARE -#define CRYPTOCAP_F_SYNC 0x04000000 /* operates synchronously */ +#define CRYPTOCAP_F_HARDWARE CRYPTO_FLAG_HARDWARE +#define CRYPTOCAP_F_SOFTWARE CRYPTO_FLAG_SOFTWARE +#define CRYPTOCAP_F_SYNC 0x04000000 /* operates synchronously */ extern int32_t crypto_get_driverid(device_t dev, int flags); extern int crypto_find_driver(const char *); extern device_t crypto_find_device_byhid(int hid); @@ -418,10 +504,15 @@ extern int crypto_devallowsoft; /* only use hardware crypto */ struct uio; extern void cuio_copydata(struct uio* uio, int off, int len, caddr_t cp); extern void cuio_copyback(struct uio* uio, int off, int len, caddr_t cp); -extern struct iovec *cuio_getptr(struct uio *uio, int loc, int *off); +extern int cuio_getptr(struct uio *uio, int loc, int *off); extern int cuio_apply(struct uio *uio, int off, int len, int (*f)(void *, void *, u_int), void *arg); +struct mbuf; +struct iovec; +extern int crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iovptr, + int *cnt, int *allocated); + extern void crypto_copyback(int flags, caddr_t buf, int off, int size, caddr_t in); extern void crypto_copydata(int flags, caddr_t buf, int off, int size, diff --git a/sys/opencrypto/cryptosoft.c b/sys/opencrypto/cryptosoft.c index d73f462..052916b 100644 --- a/sys/opencrypto/cryptosoft.c +++ b/sys/opencrypto/cryptosoft.c @@ -9,6 +9,12 @@ * supported the development of this code. * * Copyright (c) 2000, 2001 Angelos D. Keromytis + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by John-Mark Gurney + * under sponsorship of the FreeBSD Foundation and + * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in @@ -35,6 +41,10 @@ __FBSDID("$FreeBSD$"); #include <sys/random.h> #include <sys/kernel.h> #include <sys/uio.h> +#include <sys/lock.h> +#include <sys/rwlock.h> +#include <sys/endian.h> +#include <sys/limits.h> #include <crypto/blowfish/blowfish.h> #include <crypto/sha1.h> @@ -54,14 +64,18 @@ __FBSDID("$FreeBSD$"); static int32_t swcr_id; static struct swcr_data **swcr_sessions = NULL; static u_int32_t swcr_sesnum; +/* Protects swcr_sessions pointer, not data. */ +static struct rwlock swcr_sessions_lock; u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN]; u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN]; static int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int); static int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int); +static int swcr_authenc(struct cryptop *crp); static int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int); static int swcr_freesession(device_t dev, u_int64_t tid); +static int swcr_freesession_locked(device_t dev, u_int64_t tid); /* * Apply a symmetric encryption/decryption algorithm. @@ -71,36 +85,48 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, int flags) { unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN], *idat; - unsigned char *ivp, piv[EALG_MAX_BLOCK_LEN]; + unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; struct enc_xform *exf; - int i, k, j, blks; + int i, j, k, blks, ind, count, ivlen; + struct uio *uio, uiolcl; + struct iovec iovlcl[4]; + struct iovec *iov; + int iovcnt, iovalloc; + int error; + + error = 0; exf = sw->sw_exf; blks = exf->blocksize; + ivlen = exf->ivsize; /* Check for non-padded data */ if (crd->crd_len % blks) return EINVAL; + if (crd->crd_alg == CRYPTO_AES_ICM && + (crd->crd_flags & CRD_F_IV_EXPLICIT) == 0) + return (EINVAL); + /* Initialize the IV */ if (crd->crd_flags & CRD_F_ENCRYPT) { /* IV explicitly provided ? */ if (crd->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(crd->crd_iv, iv, blks); + bcopy(crd->crd_iv, iv, ivlen); else - arc4rand(iv, blks, 0); + arc4rand(iv, ivlen, 0); /* Do we need to write the IV */ if (!(crd->crd_flags & CRD_F_IV_PRESENT)) - crypto_copyback(flags, buf, crd->crd_inject, blks, iv); + crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv); } else { /* Decryption */ - /* IV explicitly provided ? */ + /* IV explicitly provided ? */ if (crd->crd_flags & CRD_F_IV_EXPLICIT) - bcopy(crd->crd_iv, iv, blks); + bcopy(crd->crd_iv, iv, ivlen); else { /* Get IV off buf */ - crypto_copydata(flags, buf, crd->crd_inject, blks, iv); + crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv); } } @@ -109,341 +135,186 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, if (sw->sw_kschedule) exf->zerokey(&(sw->sw_kschedule)); + error = exf->setkey(&sw->sw_kschedule, crd->crd_key, crd->crd_klen / 8); if (error) return (error); } + iov = iovlcl; + iovcnt = nitems(iovlcl); + iovalloc = 0; + uio = &uiolcl; + if ((flags & CRYPTO_F_IMBUF) != 0) { + error = crypto_mbuftoiov((struct mbuf *)buf, &iov, &iovcnt, + &iovalloc); + if (error) + return (error); + uio->uio_iov = iov; + uio->uio_iovcnt = iovcnt; + } else if ((flags & CRYPTO_F_IOV) != 0) + uio = (struct uio *)buf; + else { + iov[0].iov_base = buf; + iov[0].iov_len = crd->crd_skip + crd->crd_len; + uio->uio_iov = iov; + uio->uio_iovcnt = 1; + } + ivp = iv; - /* - * xforms that provide a reinit method perform all IV - * handling themselves. - */ - if (exf->reinit) + if (exf->reinit) { + /* + * xforms that provide a reinit method perform all IV + * handling themselves. + */ exf->reinit(sw->sw_kschedule, iv); + } - if (flags & CRYPTO_F_IMBUF) { - struct mbuf *m = (struct mbuf *) buf; + count = crd->crd_skip; + ind = cuio_getptr(uio, count, &k); + if (ind == -1) { + error = EINVAL; + goto out; + } - /* Find beginning of data */ - m = m_getptr(m, crd->crd_skip, &k); - if (m == NULL) - return EINVAL; + i = crd->crd_len; - i = crd->crd_len; - - while (i > 0) { - /* - * If there's insufficient data at the end of - * an mbuf, we have to do some copying. - */ - if (m->m_len < k + blks && m->m_len != k) { - m_copydata(m, k, blks, blk); - - /* Actual encryption/decryption */ - if (exf->reinit) { - if (crd->crd_flags & CRD_F_ENCRYPT) { - exf->encrypt(sw->sw_kschedule, - blk); - } else { - exf->decrypt(sw->sw_kschedule, - blk); - } - } else if (crd->crd_flags & CRD_F_ENCRYPT) { - /* XOR with previous block */ - for (j = 0; j < blks; j++) - blk[j] ^= ivp[j]; - - exf->encrypt(sw->sw_kschedule, blk); - - /* - * Keep encrypted block for XOR'ing - * with next block - */ - bcopy(blk, iv, blks); - ivp = iv; - } else { /* decrypt */ - /* - * Keep encrypted block for XOR'ing - * with next block - */ - if (ivp == iv) - bcopy(blk, piv, blks); - else - bcopy(blk, iv, blks); - - exf->decrypt(sw->sw_kschedule, blk); - - /* XOR with previous block */ - for (j = 0; j < blks; j++) - blk[j] ^= ivp[j]; - - if (ivp == iv) - bcopy(piv, iv, blks); - else - ivp = iv; + while (i > 0) { + /* + * If there's insufficient data at the end of + * an iovec, we have to do some copying. + */ + if (uio->uio_iov[ind].iov_len < k + blks && + uio->uio_iov[ind].iov_len != k) { + cuio_copydata(uio, count, blks, blk); + + /* Actual encryption/decryption */ + if (exf->reinit) { + if (crd->crd_flags & CRD_F_ENCRYPT) { + exf->encrypt(sw->sw_kschedule, + blk); + } else { + exf->decrypt(sw->sw_kschedule, + blk); } - - /* Copy back decrypted block */ - m_copyback(m, k, blks, blk); - - /* Advance pointer */ - m = m_getptr(m, k + blks, &k); - if (m == NULL) - return EINVAL; - - i -= blks; - - /* Could be done... */ - if (i == 0) - break; + } else if (crd->crd_flags & CRD_F_ENCRYPT) { + /* XOR with previous block */ + for (j = 0; j < blks; j++) + blk[j] ^= ivp[j]; + + exf->encrypt(sw->sw_kschedule, blk); + + /* + * Keep encrypted block for XOR'ing + * with next block + */ + bcopy(blk, iv, blks); + ivp = iv; + } else { /* decrypt */ + /* + * Keep encrypted block for XOR'ing + * with next block + */ + nivp = (ivp == iv) ? iv2 : iv; + bcopy(blk, nivp, blks); + + exf->decrypt(sw->sw_kschedule, blk); + + /* XOR with previous block */ + for (j = 0; j < blks; j++) + blk[j] ^= ivp[j]; + + ivp = nivp; } - /* Skip possibly empty mbufs */ - if (k == m->m_len) { - for (m = m->m_next; m && m->m_len == 0; - m = m->m_next) - ; - k = 0; - } - - /* Sanity check */ - if (m == NULL) - return EINVAL; + /* Copy back decrypted block */ + cuio_copyback(uio, count, blks, blk); - /* - * Warning: idat may point to garbage here, but - * we only use it in the while() loop, only if - * there are indeed enough data. - */ - idat = mtod(m, unsigned char *) + k; - - while (m->m_len >= k + blks && i > 0) { - if (exf->reinit) { - if (crd->crd_flags & CRD_F_ENCRYPT) { - exf->encrypt(sw->sw_kschedule, - idat); - } else { - exf->decrypt(sw->sw_kschedule, - idat); - } - } else if (crd->crd_flags & CRD_F_ENCRYPT) { - /* XOR with previous block/IV */ - for (j = 0; j < blks; j++) - idat[j] ^= ivp[j]; - - exf->encrypt(sw->sw_kschedule, idat); - ivp = idat; - } else { /* decrypt */ - /* - * Keep encrypted block to be used - * in next block's processing. - */ - if (ivp == iv) - bcopy(idat, piv, blks); - else - bcopy(idat, iv, blks); - - exf->decrypt(sw->sw_kschedule, idat); - - /* XOR with previous block/IV */ - for (j = 0; j < blks; j++) - idat[j] ^= ivp[j]; - - if (ivp == iv) - bcopy(piv, iv, blks); - else - ivp = iv; - } + count += blks; - idat += blks; - k += blks; - i -= blks; + /* Advance pointer */ + ind = cuio_getptr(uio, count, &k); + if (ind == -1) { + error = EINVAL; + goto out; } - } - return 0; /* Done with mbuf encryption/decryption */ - } else if (flags & CRYPTO_F_IOV) { - struct uio *uio = (struct uio *) buf; - struct iovec *iov; + i -= blks; - /* Find beginning of data */ - iov = cuio_getptr(uio, crd->crd_skip, &k); - if (iov == NULL) - return EINVAL; + /* Could be done... */ + if (i == 0) + break; + } - i = crd->crd_len; - - while (i > 0) { - /* - * If there's insufficient data at the end of - * an iovec, we have to do some copying. - */ - if (iov->iov_len < k + blks && iov->iov_len != k) { - cuio_copydata(uio, k, blks, blk); - - /* Actual encryption/decryption */ - if (exf->reinit) { - if (crd->crd_flags & CRD_F_ENCRYPT) { - exf->encrypt(sw->sw_kschedule, - blk); - } else { - exf->decrypt(sw->sw_kschedule, - blk); - } - } else if (crd->crd_flags & CRD_F_ENCRYPT) { - /* XOR with previous block */ - for (j = 0; j < blks; j++) - blk[j] ^= ivp[j]; - - exf->encrypt(sw->sw_kschedule, blk); - - /* - * Keep encrypted block for XOR'ing - * with next block - */ - bcopy(blk, iv, blks); - ivp = iv; - } else { /* decrypt */ - /* - * Keep encrypted block for XOR'ing - * with next block - */ - if (ivp == iv) - bcopy(blk, piv, blks); - else - bcopy(blk, iv, blks); - - exf->decrypt(sw->sw_kschedule, blk); - - /* XOR with previous block */ - for (j = 0; j < blks; j++) - blk[j] ^= ivp[j]; - - if (ivp == iv) - bcopy(piv, iv, blks); - else - ivp = iv; + /* + * Warning: idat may point to garbage here, but + * we only use it in the while() loop, only if + * there are indeed enough data. + */ + idat = (char *)uio->uio_iov[ind].iov_base + k; + + while (uio->uio_iov[ind].iov_len >= k + blks && i > 0) { + if (exf->reinit) { + if (crd->crd_flags & CRD_F_ENCRYPT) { + exf->encrypt(sw->sw_kschedule, + idat); + } else { + exf->decrypt(sw->sw_kschedule, + idat); } - - /* Copy back decrypted block */ - cuio_copyback(uio, k, blks, blk); - - /* Advance pointer */ - iov = cuio_getptr(uio, k + blks, &k); - if (iov == NULL) - return EINVAL; - - i -= blks; - - /* Could be done... */ - if (i == 0) - break; + } else if (crd->crd_flags & CRD_F_ENCRYPT) { + /* XOR with previous block/IV */ + for (j = 0; j < blks; j++) + idat[j] ^= ivp[j]; + + exf->encrypt(sw->sw_kschedule, idat); + ivp = idat; + } else { /* decrypt */ + /* + * Keep encrypted block to be used + * in next block's processing. + */ + nivp = (ivp == iv) ? iv2 : iv; + bcopy(idat, nivp, blks); + + exf->decrypt(sw->sw_kschedule, idat); + + /* XOR with previous block/IV */ + for (j = 0; j < blks; j++) + idat[j] ^= ivp[j]; + + ivp = nivp; } - /* - * Warning: idat may point to garbage here, but - * we only use it in the while() loop, only if - * there are indeed enough data. - */ - idat = (char *)iov->iov_base + k; - - while (iov->iov_len >= k + blks && i > 0) { - if (exf->reinit) { - if (crd->crd_flags & CRD_F_ENCRYPT) { - exf->encrypt(sw->sw_kschedule, - idat); - } else { - exf->decrypt(sw->sw_kschedule, - idat); - } - } else if (crd->crd_flags & CRD_F_ENCRYPT) { - /* XOR with previous block/IV */ - for (j = 0; j < blks; j++) - idat[j] ^= ivp[j]; - - exf->encrypt(sw->sw_kschedule, idat); - ivp = idat; - } else { /* decrypt */ - /* - * Keep encrypted block to be used - * in next block's processing. - */ - if (ivp == iv) - bcopy(idat, piv, blks); - else - bcopy(idat, iv, blks); - - exf->decrypt(sw->sw_kschedule, idat); - - /* XOR with previous block/IV */ - for (j = 0; j < blks; j++) - idat[j] ^= ivp[j]; - - if (ivp == iv) - bcopy(piv, iv, blks); - else - ivp = iv; - } - - idat += blks; - k += blks; - i -= blks; - } - if (k == iov->iov_len) { - iov++; - k = 0; - } + idat += blks; + count += blks; + k += blks; + i -= blks; } - return 0; /* Done with iovec encryption/decryption */ - } else { /* contiguous buffer */ - if (exf->reinit) { - for (i = crd->crd_skip; - i < crd->crd_skip + crd->crd_len; i += blks) { - if (crd->crd_flags & CRD_F_ENCRYPT) - exf->encrypt(sw->sw_kschedule, buf + i); - else - exf->decrypt(sw->sw_kschedule, buf + i); - } - } else if (crd->crd_flags & CRD_F_ENCRYPT) { - for (i = crd->crd_skip; - i < crd->crd_skip + crd->crd_len; i += blks) { - /* XOR with the IV/previous block, as appropriate. */ - if (i == crd->crd_skip) - for (k = 0; k < blks; k++) - buf[i + k] ^= ivp[k]; - else - for (k = 0; k < blks; k++) - buf[i + k] ^= buf[i + k - blks]; - exf->encrypt(sw->sw_kschedule, buf + i); - } - } else { /* Decrypt */ - /* - * Start at the end, so we don't need to keep the encrypted - * block as the IV for the next block. - */ - for (i = crd->crd_skip + crd->crd_len - blks; - i >= crd->crd_skip; i -= blks) { - exf->decrypt(sw->sw_kschedule, buf + i); - - /* XOR with the IV/previous block, as appropriate */ - if (i == crd->crd_skip) - for (k = 0; k < blks; k++) - buf[i + k] ^= ivp[k]; - else - for (k = 0; k < blks; k++) - buf[i + k] ^= buf[i + k - blks]; + /* + * Advance to the next iov if the end of the current iov + * is aligned with the end of a cipher block. + * Note that the code is equivalent to calling: + * ind = cuio_getptr(uio, count, &k); + */ + if (i > 0 && k == uio->uio_iov[ind].iov_len) { + k = 0; + ind++; + if (ind >= uio->uio_iovcnt) { + error = EINVAL; + goto out; } } - - return 0; /* Done with contiguous buffer encryption/decryption */ } - /* Unreachable */ - return EINVAL; +out: + if (iovalloc) + free(iov, M_CRYPTO_DATA); + + return (error); } static void @@ -578,6 +449,181 @@ swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, return 0; } +CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ +CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ + +/* + * Apply a combined encryption-authentication transformation + */ +static int +swcr_authenc(struct cryptop *crp) +{ + uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))]; + u_char *blk = (u_char *)blkbuf; + u_char aalg[AALG_MAX_RESULT_LEN]; + u_char uaalg[AALG_MAX_RESULT_LEN]; + u_char iv[EALG_MAX_BLOCK_LEN]; + union authctx ctx; + struct cryptodesc *crd, *crda = NULL, *crde = NULL; + struct swcr_data *sw, *swa, *swe = NULL; + struct auth_hash *axf = NULL; + struct enc_xform *exf = NULL; + caddr_t buf = (caddr_t)crp->crp_buf; + uint32_t *blkp; + int aadlen, blksz, i, ivlen, len, iskip, oskip, r; + + ivlen = blksz = iskip = oskip = 0; + + for (crd = crp->crp_desc; crd; crd = crd->crd_next) { + for (sw = swcr_sessions[crp->crp_sid & 0xffffffff]; + sw && sw->sw_alg != crd->crd_alg; + sw = sw->sw_next) + ; + if (sw == NULL) + return (EINVAL); + + switch (sw->sw_alg) { + case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_NIST_GMAC: + swe = sw; + crde = crd; + exf = swe->sw_exf; + ivlen = 12; + break; + case CRYPTO_AES_128_NIST_GMAC: + case CRYPTO_AES_192_NIST_GMAC: + case CRYPTO_AES_256_NIST_GMAC: + swa = sw; + crda = crd; + axf = swa->sw_axf; + if (swa->sw_ictx == 0) + return (EINVAL); + bcopy(swa->sw_ictx, &ctx, axf->ctxsize); + blksz = axf->blocksize; + break; + default: + return (EINVAL); + } + } + if (crde == NULL || crda == NULL) + return (EINVAL); + + if (crde->crd_alg == CRYPTO_AES_NIST_GCM_16 && + (crde->crd_flags & CRD_F_IV_EXPLICIT) == 0) + return (EINVAL); + + if (crde->crd_klen != crda->crd_klen) + return (EINVAL); + + /* Initialize the IV */ + if (crde->crd_flags & CRD_F_ENCRYPT) { + /* IV explicitly provided ? */ + if (crde->crd_flags & CRD_F_IV_EXPLICIT) + bcopy(crde->crd_iv, iv, ivlen); + else + arc4rand(iv, ivlen, 0); + + /* Do we need to write the IV */ + if (!(crde->crd_flags & CRD_F_IV_PRESENT)) + crypto_copyback(crp->crp_flags, buf, crde->crd_inject, + ivlen, iv); + + } else { /* Decryption */ + /* IV explicitly provided ? */ + if (crde->crd_flags & CRD_F_IV_EXPLICIT) + bcopy(crde->crd_iv, iv, ivlen); + else { + /* Get IV off buf */ + crypto_copydata(crp->crp_flags, buf, crde->crd_inject, + ivlen, iv); + } + } + + /* Supply MAC with IV */ + if (axf->Reinit) + axf->Reinit(&ctx, iv, ivlen); + + /* Supply MAC with AAD */ + aadlen = crda->crd_len; + + for (i = iskip; i < crda->crd_len; i += blksz) { + len = MIN(crda->crd_len - i, blksz - oskip); + crypto_copydata(crp->crp_flags, buf, crda->crd_skip + i, len, + blk + oskip); + bzero(blk + len + oskip, blksz - len - oskip); + axf->Update(&ctx, blk, blksz); + oskip = 0; /* reset initial output offset */ + } + + if (exf->reinit) + exf->reinit(swe->sw_kschedule, iv); + + /* Do encryption/decryption with MAC */ + for (i = 0; i < crde->crd_len; i += blksz) { + len = MIN(crde->crd_len - i, blksz); + if (len < blksz) + bzero(blk, blksz); + crypto_copydata(crp->crp_flags, buf, crde->crd_skip + i, len, + blk); + if (crde->crd_flags & CRD_F_ENCRYPT) { + exf->encrypt(swe->sw_kschedule, blk); + axf->Update(&ctx, blk, len); + crypto_copyback(crp->crp_flags, buf, + crde->crd_skip + i, len, blk); + } else { + axf->Update(&ctx, blk, len); + } + } + + /* Do any required special finalization */ + switch (crda->crd_alg) { + case CRYPTO_AES_128_NIST_GMAC: + case CRYPTO_AES_192_NIST_GMAC: + case CRYPTO_AES_256_NIST_GMAC: + /* length block */ + bzero(blk, blksz); + blkp = (uint32_t *)blk + 1; + *blkp = htobe32(aadlen * 8); + blkp = (uint32_t *)blk + 3; + *blkp = htobe32(crde->crd_len * 8); + axf->Update(&ctx, blk, blksz); + break; + } + + /* Finalize MAC */ + axf->Final(aalg, &ctx); + + /* Validate tag */ + if (!(crde->crd_flags & CRD_F_ENCRYPT)) { + crypto_copydata(crp->crp_flags, buf, crda->crd_inject, + axf->hashsize, uaalg); + + r = timingsafe_bcmp(aalg, uaalg, axf->hashsize); + if (r == 0) { + /* tag matches, decrypt data */ + for (i = 0; i < crde->crd_len; i += blksz) { + len = MIN(crde->crd_len - i, blksz); + if (len < blksz) + bzero(blk, blksz); + crypto_copydata(crp->crp_flags, buf, + crde->crd_skip + i, len, blk); + if (!(crde->crd_flags & CRD_F_ENCRYPT)) { + exf->decrypt(swe->sw_kschedule, blk); + } + crypto_copyback(crp->crp_flags, buf, + crde->crd_skip + i, len, blk); + } + } else + return (EBADMSG); + } else { + /* Inject the authentication data */ + crypto_copyback(crp->crp_flags, buf, crda->crd_inject, + axf->hashsize, aalg); + } + + return (0); +} + /* * Apply a compression/decompression algorithm */ @@ -665,11 +711,13 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) struct enc_xform *txf; struct comp_algo *cxf; u_int32_t i; + int len; int error; if (sid == NULL || cri == NULL) return EINVAL; + rw_wlock(&swcr_sessions_lock); if (swcr_sessions) { for (i = 1; i < swcr_sesnum; i++) if (swcr_sessions[i] == NULL) @@ -692,6 +740,7 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) swcr_sesnum = 0; else swcr_sesnum /= 2; + rw_wunlock(&swcr_sessions_lock); return ENOBUFS; } @@ -705,6 +754,7 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) swcr_sessions = swd; } + rw_downgrade(&swcr_sessions_lock); swd = &swcr_sessions[i]; *sid = i; @@ -712,7 +762,8 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) *swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); if (*swd == NULL) { - swcr_freesession(dev, i); + swcr_freesession_locked(dev, i); + rw_runlock(&swcr_sessions_lock); return ENOBUFS; } @@ -738,6 +789,16 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) case CRYPTO_AES_XTS: txf = &enc_xform_aes_xts; goto enccommon; + case CRYPTO_AES_ICM: + txf = &enc_xform_aes_icm; + goto enccommon; + case CRYPTO_AES_NIST_GCM_16: + txf = &enc_xform_aes_nist_gcm; + goto enccommon; + case CRYPTO_AES_NIST_GMAC: + txf = &enc_xform_aes_nist_gmac; + (*swd)->sw_exf = txf; + break; case CRYPTO_CAMELLIA_CBC: txf = &enc_xform_camellia; goto enccommon; @@ -749,7 +810,8 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) error = txf->setkey(&((*swd)->sw_kschedule), cri->cri_key, cri->cri_klen / 8); if (error) { - swcr_freesession(dev, i); + swcr_freesession_locked(dev, i); + rw_runlock(&swcr_sessions_lock); return error; } } @@ -780,14 +842,16 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) (*swd)->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if ((*swd)->sw_ictx == NULL) { - swcr_freesession(dev, i); + swcr_freesession_locked(dev, i); + rw_runlock(&swcr_sessions_lock); return ENOBUFS; } (*swd)->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if ((*swd)->sw_octx == NULL) { - swcr_freesession(dev, i); + swcr_freesession_locked(dev, i); + rw_runlock(&swcr_sessions_lock); return ENOBUFS; } @@ -810,14 +874,16 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) (*swd)->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if ((*swd)->sw_ictx == NULL) { - swcr_freesession(dev, i); + swcr_freesession_locked(dev, i); + rw_runlock(&swcr_sessions_lock); return ENOBUFS; } (*swd)->sw_octx = malloc(cri->cri_klen / 8, M_CRYPTO_DATA, M_NOWAIT); if ((*swd)->sw_octx == NULL) { - swcr_freesession(dev, i); + swcr_freesession_locked(dev, i); + rw_runlock(&swcr_sessions_lock); return ENOBUFS; } @@ -841,7 +907,8 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) (*swd)->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); if ((*swd)->sw_ictx == NULL) { - swcr_freesession(dev, i); + swcr_freesession_locked(dev, i); + rw_runlock(&swcr_sessions_lock); return ENOBUFS; } @@ -850,12 +917,41 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) (*swd)->sw_axf = axf; break; #endif + + case CRYPTO_AES_128_NIST_GMAC: + axf = &auth_hash_nist_gmac_aes_128; + goto auth4common; + + case CRYPTO_AES_192_NIST_GMAC: + axf = &auth_hash_nist_gmac_aes_192; + goto auth4common; + + case CRYPTO_AES_256_NIST_GMAC: + axf = &auth_hash_nist_gmac_aes_256; + auth4common: + len = cri->cri_klen / 8; + if (len != 16 && len != 24 && len != 32) + return EINVAL; + + (*swd)->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, + M_NOWAIT); + if ((*swd)->sw_ictx == NULL) { + swcr_freesession_locked(dev, i); + rw_runlock(&swcr_sessions_lock); + return ENOBUFS; + } + axf->Init((*swd)->sw_ictx); + axf->Setkey((*swd)->sw_ictx, cri->cri_key, len); + (*swd)->sw_axf = axf; + break; + case CRYPTO_DEFLATE_COMP: cxf = &comp_algo_deflate; (*swd)->sw_cxf = cxf; break; default: - swcr_freesession(dev, i); + swcr_freesession_locked(dev, i); + rw_runlock(&swcr_sessions_lock); return EINVAL; } @@ -863,14 +959,26 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) cri = cri->cri_next; swd = &((*swd)->sw_next); } + rw_runlock(&swcr_sessions_lock); return 0; } +static int +swcr_freesession(device_t dev, u_int64_t tid) +{ + int error; + + rw_rlock(&swcr_sessions_lock); + error = swcr_freesession_locked(dev, tid); + rw_runlock(&swcr_sessions_lock); + return error; +} + /* * Free a session. */ static int -swcr_freesession(device_t dev, u_int64_t tid) +swcr_freesession_locked(device_t dev, u_int64_t tid) { struct swcr_data *swd; struct enc_xform *txf; @@ -897,6 +1005,9 @@ swcr_freesession(device_t dev, u_int64_t tid) case CRYPTO_SKIPJACK_CBC: case CRYPTO_RIJNDAEL128_CBC: case CRYPTO_AES_XTS: + case CRYPTO_AES_ICM: + case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_NIST_GMAC: case CRYPTO_CAMELLIA_CBC: case CRYPTO_NULL_CBC: txf = swd->sw_exf; @@ -975,11 +1086,15 @@ swcr_process(device_t dev, struct cryptop *crp, int hint) goto done; } - lid = crp->crp_sid & 0xffffffff; - if (lid >= swcr_sesnum || lid == 0 || swcr_sessions[lid] == NULL) { + lid = CRYPTO_SESID2LID(crp->crp_sid); + rw_rlock(&swcr_sessions_lock); + if (swcr_sessions == NULL || lid >= swcr_sesnum || lid == 0 || + swcr_sessions[lid] == NULL) { + rw_runlock(&swcr_sessions_lock); crp->crp_etype = ENOENT; goto done; } + rw_runlock(&swcr_sessions_lock); /* Go through crypto descriptors, processing as we go */ for (crd = crp->crp_desc; crd; crd = crd->crd_next) { @@ -993,10 +1108,17 @@ swcr_process(device_t dev, struct cryptop *crp, int hint) * XXX between the various instances of an algorithm (so we can * XXX locate the correct crypto context). */ + rw_rlock(&swcr_sessions_lock); + if (swcr_sessions == NULL) { + rw_runlock(&swcr_sessions_lock); + crp->crp_etype = ENOENT; + goto done; + } for (sw = swcr_sessions[lid]; sw && sw->sw_alg != crd->crd_alg; sw = sw->sw_next) ; + rw_runlock(&swcr_sessions_lock); /* No such context ? */ if (sw == NULL) { @@ -1011,6 +1133,7 @@ swcr_process(device_t dev, struct cryptop *crp, int hint) case CRYPTO_SKIPJACK_CBC: case CRYPTO_RIJNDAEL128_CBC: case CRYPTO_AES_XTS: + case CRYPTO_AES_ICM: case CRYPTO_CAMELLIA_CBC: if ((crp->crp_etype = swcr_encdec(crd, sw, crp->crp_buf, crp->crp_flags)) != 0) @@ -1035,6 +1158,14 @@ swcr_process(device_t dev, struct cryptop *crp, int hint) goto done; break; + case CRYPTO_AES_NIST_GCM_16: + case CRYPTO_AES_NIST_GMAC: + case CRYPTO_AES_128_NIST_GMAC: + case CRYPTO_AES_192_NIST_GMAC: + case CRYPTO_AES_256_NIST_GMAC: + crp->crp_etype = swcr_authenc(crp); + goto done; + case CRYPTO_DEFLATE_COMP: if ((crp->crp_etype = swcr_compdec(crd, sw, crp->crp_buf, crp->crp_flags)) != 0) @@ -1074,6 +1205,7 @@ swcr_probe(device_t dev) static int swcr_attach(device_t dev) { + rw_init(&swcr_sessions_lock, "swcr_sessions_lock"); memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN); memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN); @@ -1104,6 +1236,12 @@ swcr_attach(device_t dev) REGISTER(CRYPTO_SHA1); REGISTER(CRYPTO_RIJNDAEL128_CBC); REGISTER(CRYPTO_AES_XTS); + REGISTER(CRYPTO_AES_ICM); + REGISTER(CRYPTO_AES_NIST_GCM_16); + REGISTER(CRYPTO_AES_NIST_GMAC); + REGISTER(CRYPTO_AES_128_NIST_GMAC); + REGISTER(CRYPTO_AES_192_NIST_GMAC); + REGISTER(CRYPTO_AES_256_NIST_GMAC); REGISTER(CRYPTO_CAMELLIA_CBC); REGISTER(CRYPTO_DEFLATE_COMP); #undef REGISTER @@ -1115,8 +1253,11 @@ static int swcr_detach(device_t dev) { crypto_unregister_all(swcr_id); - if (swcr_sessions != NULL) - free(swcr_sessions, M_CRYPTO_DATA); + rw_wlock(&swcr_sessions_lock); + free(swcr_sessions, M_CRYPTO_DATA); + swcr_sessions = NULL; + rw_wunlock(&swcr_sessions_lock); + rw_destroy(&swcr_sessions_lock); return 0; } diff --git a/sys/opencrypto/gfmult.c b/sys/opencrypto/gfmult.c new file mode 100644 index 0000000..990a7ff --- /dev/null +++ b/sys/opencrypto/gfmult.c @@ -0,0 +1,275 @@ +/*- + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by John-Mark Gurney under + * the sponsorship of the FreeBSD Foundation and + * Rubicon Communications, LLC (Netgate). + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#include "gfmult.h" + +#define REV_POLY_REDUCT 0xe1 /* 0x87 bit reversed */ + +/* reverse the bits of a nibble */ +static const uint8_t nib_rev[] = { + 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, + 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf, +}; + +/* calulate v * 2 */ +static inline struct gf128 +gf128_mulalpha(struct gf128 v) +{ + uint64_t mask; + + mask = !!(v.v[1] & 1); + mask = ~(mask - 1); + v.v[1] = (v.v[1] >> 1) | ((v.v[0] & 1) << 63); + v.v[0] = (v.v[0] >> 1) ^ ((mask & REV_POLY_REDUCT) << 56); + + return v; +} + +/* + * Generate a table for 0-16 * h. Store the results in the table w/ indexes + * bit reversed, and the words striped across the values. + */ +void +gf128_genmultable(struct gf128 h, struct gf128table *t) +{ + struct gf128 tbl[16]; + int i; + + tbl[0] = MAKE_GF128(0, 0); + tbl[1] = h; + + for (i = 2; i < 16; i += 2) { + tbl[i] = gf128_mulalpha(tbl[i / 2]); + tbl[i + 1] = gf128_add(tbl[i], h); + } + + for (i = 0; i < 16; i++) { + t->a[nib_rev[i]] = tbl[i].v[0] >> 32; + t->b[nib_rev[i]] = tbl[i].v[0]; + t->c[nib_rev[i]] = tbl[i].v[1] >> 32; + t->d[nib_rev[i]] = tbl[i].v[1]; + } +} + +/* + * Generate tables containing h, h^2, h^3 and h^4, starting at 0. + */ +void +gf128_genmultable4(struct gf128 h, struct gf128table4 *t) +{ + struct gf128 h2, h3, h4; + + gf128_genmultable(h, &t->tbls[0]); + + h2 = gf128_mul(h, &t->tbls[0]); + + gf128_genmultable(h2, &t->tbls[1]); + + h3 = gf128_mul(h, &t->tbls[1]); + gf128_genmultable(h3, &t->tbls[2]); + + h4 = gf128_mul(h2, &t->tbls[1]); + gf128_genmultable(h4, &t->tbls[3]); +} + +/* + * Read a row from the table. + */ +static inline struct gf128 +readrow(struct gf128table *tbl, unsigned bits) +{ + struct gf128 r; + + bits = bits % 16; + + r.v[0] = ((uint64_t)tbl->a[bits] << 32) | tbl->b[bits]; + r.v[1] = ((uint64_t)tbl->c[bits] << 32) | tbl->d[bits]; + + return r; +} + +/* + * These are the reduction values. Since we are dealing with bit reversed + * version, the values need to be bit reversed, AND the indexes are also + * bit reversed to make lookups quicker. + */ +static uint16_t reduction[] = { + 0x0000, 0x1c20, 0x3840, 0x2460, 0x7080, 0x6ca0, 0x48c0, 0x54e0, + 0xe100, 0xfd20, 0xd940, 0xc560, 0x9180, 0x8da0, 0xa9c0, 0xb5e0, +}; + +/* + * Calculate: + * (x*2^4 + word[3,0]*h) * + * 2^4 + word[7,4]*h) * + * ... + * 2^4 + word[63,60]*h + */ +static struct gf128 +gfmultword(uint64_t word, struct gf128 x, struct gf128table *tbl) +{ + struct gf128 row; + unsigned bits; + unsigned redbits; + int i; + + for (i = 0; i < 64; i += 4) { + bits = word % 16; + + /* fetch row */ + row = readrow(tbl, bits); + + /* x * 2^4 */ + redbits = x.v[1] % 16; + x.v[1] = (x.v[1] >> 4) | (x.v[0] % 16) << 60; + x.v[0] >>= 4; + x.v[0] ^= (uint64_t)reduction[redbits] << (64 - 16); + + word >>= 4; + + x = gf128_add(x, row); + } + + return x; +} + +/* + * Calculate + * (x*2^4 + worda[3,0]*h^4+wordb[3,0]*h^3+...+wordd[3,0]*h) * + * ... + * 2^4 + worda[63,60]*h^4+ ... + wordd[63,60]*h + * + * Passing/returning struct is .5% faster than passing in via pointer on + * amd64. + */ +static struct gf128 +gfmultword4(uint64_t worda, uint64_t wordb, uint64_t wordc, uint64_t wordd, + struct gf128 x, struct gf128table4 *tbl) +{ + struct gf128 rowa, rowb, rowc, rowd; + unsigned bitsa, bitsb, bitsc, bitsd; + unsigned redbits; + int i; + + /* + * XXX - nibble reverse words to save a shift? probably not as + * nibble reverse would take 20 ops (5 * 4) verse 16 + */ + + for (i = 0; i < 64; i += 4) { + bitsa = worda % 16; + bitsb = wordb % 16; + bitsc = wordc % 16; + bitsd = wordd % 16; + + /* fetch row */ + rowa = readrow(&tbl->tbls[3], bitsa); + rowb = readrow(&tbl->tbls[2], bitsb); + rowc = readrow(&tbl->tbls[1], bitsc); + rowd = readrow(&tbl->tbls[0], bitsd); + + /* x * 2^4 */ + redbits = x.v[1] % 16; + x.v[1] = (x.v[1] >> 4) | (x.v[0] % 16) << 60; + x.v[0] >>= 4; + x.v[0] ^= (uint64_t)reduction[redbits] << (64 - 16); + + worda >>= 4; + wordb >>= 4; + wordc >>= 4; + wordd >>= 4; + + x = gf128_add(x, gf128_add(rowa, gf128_add(rowb, + gf128_add(rowc, rowd)))); + } + + return x; +} + +struct gf128 +gf128_mul(struct gf128 v, struct gf128table *tbl) +{ + struct gf128 ret; + + ret = MAKE_GF128(0, 0); + + ret = gfmultword(v.v[1], ret, tbl); + ret = gfmultword(v.v[0], ret, tbl); + + return ret; +} + +/* + * Calculate a*h^4 + b*h^3 + c*h^2 + d*h, or: + * (((a*h+b)*h+c)*h+d)*h + */ +struct gf128 +gf128_mul4(struct gf128 a, struct gf128 b, struct gf128 c, struct gf128 d, + struct gf128table4 *tbl) +{ + struct gf128 tmp; + + tmp = MAKE_GF128(0, 0); + + tmp = gfmultword4(a.v[1], b.v[1], c.v[1], d.v[1], tmp, tbl); + tmp = gfmultword4(a.v[0], b.v[0], c.v[0], d.v[0], tmp, tbl); + + return tmp; +} + +/* + * a = data[0..15] + r + * b = data[16..31] + * c = data[32..47] + * d = data[48..63] + * + * Calculate a*h^4 + b*h^3 + c*h^2 + d*h, or: + * (((a*h+b)*h+c)*h+d)*h + */ +struct gf128 +gf128_mul4b(struct gf128 r, const uint8_t *v, struct gf128table4 *tbl) +{ + struct gf128 a, b, c, d; + struct gf128 tmp; + + tmp = MAKE_GF128(0, 0); + + a = gf128_add(r, gf128_read(&v[0*16])); + b = gf128_read(&v[1*16]); + c = gf128_read(&v[2*16]); + d = gf128_read(&v[3*16]); + + tmp = gfmultword4(a.v[1], b.v[1], c.v[1], d.v[1], tmp, tbl); + tmp = gfmultword4(a.v[0], b.v[0], c.v[0], d.v[0], tmp, tbl); + + return tmp; +} diff --git a/sys/opencrypto/gfmult.h b/sys/opencrypto/gfmult.h new file mode 100644 index 0000000..c385618 --- /dev/null +++ b/sys/opencrypto/gfmult.h @@ -0,0 +1,128 @@ +/*- + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by John-Mark Gurney under + * the sponsorship of the FreeBSD Foundation and + * Rubicon Communications, LLC (Netgate). + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef _GFMULT_H_ +#define _GFMULT_H_ + +#ifdef __APPLE__ +#define __aligned(x) __attribute__((__aligned__(x))) +#define be64dec(buf) __builtin_bswap64(*(uint64_t *)buf) +#define be64enc(buf, x) (*(uint64_t *)buf = __builtin_bswap64(x)) +#else +#include <sys/endian.h> +#endif + +#ifdef _KERNEL +#include <sys/types.h> +#else +#include <stdint.h> +#include <strings.h> +#endif + +#define REQ_ALIGN (16 * 4) +/* + * The rows are striped across cache lines. Note that the indexes + * are bit reversed to make accesses quicker. + */ +struct gf128table { + uint32_t a[16] __aligned(REQ_ALIGN); /* bits 0 - 31 */ + uint32_t b[16] __aligned(REQ_ALIGN); /* bits 63 - 32 */ + uint32_t c[16] __aligned(REQ_ALIGN); /* bits 95 - 64 */ + uint32_t d[16] __aligned(REQ_ALIGN); /* bits 127 - 96 */ +} __aligned(REQ_ALIGN); + +/* + * A set of tables that contain h, h^2, h^3, h^4. To be used w/ gf128_mul4. + */ +struct gf128table4 { + struct gf128table tbls[4]; +}; + +/* + * GCM per spec is bit reversed in memory. So byte 0 is really bit reversed + * and contains bits 0-7. We can deal w/ this by using right shifts and + * related math instead of having to bit reverse everything. This means that + * the low bits are in v[0] (bits 0-63) and reverse order, while the high + * bits are in v[1] (bits 64-127) and reverse order. The high bit of v[0] is + * bit 0, and the low bit of v[1] is bit 127. + */ +struct gf128 { + uint64_t v[2]; +}; + +/* Note that we don't bit reverse in MAKE_GF128. */ +#define MAKE_GF128(a, b) ((struct gf128){.v = { (a), (b) } }) +#define GF128_EQ(a, b) ((((a).v[0] ^ (b).v[0]) | \ + ((a).v[1] ^ (b).v[1])) == 0) + +static inline struct gf128 +gf128_read(const uint8_t *buf) +{ + struct gf128 r; + + r.v[0] = be64dec(buf); + buf += sizeof(uint64_t); + + r.v[1] = be64dec(buf); + + return r; +} + +static inline void +gf128_write(struct gf128 v, uint8_t *buf) +{ + uint64_t tmp; + + be64enc(buf, v.v[0]); + buf += sizeof tmp; + + be64enc(buf, v.v[1]); +} + +static inline struct gf128 __pure /* XXX - __pure2 instead */ +gf128_add(struct gf128 a, struct gf128 b) +{ + a.v[0] ^= b.v[0]; + a.v[1] ^= b.v[1]; + + return a; +} + +void gf128_genmultable(struct gf128 h, struct gf128table *t); +void gf128_genmultable4(struct gf128 h, struct gf128table4 *t); +struct gf128 gf128_mul(struct gf128 v, struct gf128table *tbl); +struct gf128 gf128_mul4(struct gf128 a, struct gf128 b, struct gf128 c, + struct gf128 d, struct gf128table4 *tbl); +struct gf128 gf128_mul4b(struct gf128 r, const uint8_t *v, + struct gf128table4 *tbl); + +#endif /* _GFMULT_H_ */ diff --git a/sys/opencrypto/gmac.c b/sys/opencrypto/gmac.c new file mode 100644 index 0000000..402092c --- /dev/null +++ b/sys/opencrypto/gmac.c @@ -0,0 +1,119 @@ +/*- + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by John-Mark Gurney under + * the sponsorship of the FreeBSD Foundation and + * Rubicon Communications, LLC (Netgate). + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#include <sys/types.h> +#include <sys/systm.h> +#include <opencrypto/gfmult.h> +#include <opencrypto/gmac.h> + +void +AES_GMAC_Init(struct aes_gmac_ctx *agc) +{ + + bzero(agc, sizeof *agc); +} + +void +AES_GMAC_Setkey(struct aes_gmac_ctx *agc, const uint8_t *key, uint16_t klen) +{ + const uint8_t zeros[GMAC_BLOCK_LEN] = {}; + struct gf128 h; + uint8_t hbuf[GMAC_BLOCK_LEN]; + + agc->rounds = rijndaelKeySetupEnc(agc->keysched, key, klen * 8); + + rijndaelEncrypt(agc->keysched, agc->rounds, zeros, hbuf); + + h = gf128_read(hbuf); + gf128_genmultable4(h, &agc->ghashtbl); + + explicit_bzero(&h, sizeof h); + explicit_bzero(hbuf, sizeof hbuf); +} + +void +AES_GMAC_Reinit(struct aes_gmac_ctx *agc, const uint8_t *iv, uint16_t ivlen) +{ + + KASSERT(ivlen <= sizeof agc->counter, ("passed ivlen too large!")); + bcopy(iv, agc->counter, ivlen); +} + +int +AES_GMAC_Update(struct aes_gmac_ctx *agc, const uint8_t *data, uint16_t len) +{ + struct gf128 v; + uint8_t buf[GMAC_BLOCK_LEN] = {}; + int i; + + v = agc->hash; + + while (len > 0) { + if (len >= 4*GMAC_BLOCK_LEN) { + i = 4*GMAC_BLOCK_LEN; + v = gf128_mul4b(v, data, &agc->ghashtbl); + } else if (len >= GMAC_BLOCK_LEN) { + i = GMAC_BLOCK_LEN; + v = gf128_add(v, gf128_read(data)); + v = gf128_mul(v, &agc->ghashtbl.tbls[0]); + } else { + i = len; + bcopy(data, buf, i); + v = gf128_add(v, gf128_read(&buf[0])); + v = gf128_mul(v, &agc->ghashtbl.tbls[0]); + explicit_bzero(buf, sizeof buf); + } + len -= i; + data += i; + } + + agc->hash = v; + explicit_bzero(&v, sizeof v); + + return (0); +} + +void +AES_GMAC_Final(uint8_t digest[GMAC_DIGEST_LEN], struct aes_gmac_ctx *agc) +{ + uint8_t enccntr[GMAC_BLOCK_LEN]; + struct gf128 a; + + /* XXX - zero additional bytes? */ + agc->counter[GMAC_BLOCK_LEN - 1] = 1; + + rijndaelEncrypt(agc->keysched, agc->rounds, agc->counter, enccntr); + a = gf128_add(agc->hash, gf128_read(enccntr)); + gf128_write(a, digest); + + explicit_bzero(enccntr, sizeof enccntr); +} diff --git a/sys/opencrypto/gmac.h b/sys/opencrypto/gmac.h new file mode 100644 index 0000000..909b78c --- /dev/null +++ b/sys/opencrypto/gmac.h @@ -0,0 +1,56 @@ +/*- + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by John-Mark Gurney under + * the sponsorship of the FreeBSD Foundation and + * Rubicon Communications, LLC (Netgate). + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef _GMAC_H_ +#define _GMAC_H_ + +#include "gfmult.h" +#include <crypto/rijndael/rijndael.h> + +#define GMAC_BLOCK_LEN 16 +#define GMAC_DIGEST_LEN 16 + +struct aes_gmac_ctx { + struct gf128table4 ghashtbl; + struct gf128 hash; + uint32_t keysched[4*(RIJNDAEL_MAXNR + 1)]; + uint8_t counter[GMAC_BLOCK_LEN]; + int rounds; +}; + +void AES_GMAC_Init(struct aes_gmac_ctx *); +void AES_GMAC_Setkey(struct aes_gmac_ctx *, const uint8_t *, uint16_t); +void AES_GMAC_Reinit(struct aes_gmac_ctx *, const uint8_t *, uint16_t); +int AES_GMAC_Update(struct aes_gmac_ctx *, const uint8_t *, uint16_t); +void AES_GMAC_Final(uint8_t [GMAC_DIGEST_LEN], struct aes_gmac_ctx *); + +#endif /* _GMAC_H_ */ diff --git a/sys/opencrypto/xform.c b/sys/opencrypto/xform.c index bfb061b..cb44bfa 100644 --- a/sys/opencrypto/xform.c +++ b/sys/opencrypto/xform.c @@ -24,6 +24,12 @@ * Copyright (C) 2001, Angelos D. Keromytis. * * Copyright (C) 2008, Damien Miller + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by John-Mark Gurney + * under sponsorship of the FreeBSD Foundation and + * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software with or without fee * is hereby granted, provided that this entire notice is included in @@ -76,6 +82,7 @@ static int blf_setkey(u_int8_t **, u_int8_t *, int); static int cast5_setkey(u_int8_t **, u_int8_t *, int); static int skipjack_setkey(u_int8_t **, u_int8_t *, int); static int rijndael128_setkey(u_int8_t **, u_int8_t *, int); +static int aes_icm_setkey(u_int8_t **, u_int8_t *, int); static int aes_xts_setkey(u_int8_t **, u_int8_t *, int); static int cml_setkey(u_int8_t **, u_int8_t *, int); @@ -99,6 +106,8 @@ static void rijndael128_decrypt(caddr_t, u_int8_t *); static void aes_xts_decrypt(caddr_t, u_int8_t *); static void cml_decrypt(caddr_t, u_int8_t *); +static void aes_icm_crypt(caddr_t, u_int8_t *); + static void null_zerokey(u_int8_t **); static void des1_zerokey(u_int8_t **); static void des3_zerokey(u_int8_t **); @@ -106,103 +115,149 @@ static void blf_zerokey(u_int8_t **); static void cast5_zerokey(u_int8_t **); static void skipjack_zerokey(u_int8_t **); static void rijndael128_zerokey(u_int8_t **); +static void aes_icm_zerokey(u_int8_t **); static void aes_xts_zerokey(u_int8_t **); static void cml_zerokey(u_int8_t **); +static void aes_icm_reinit(caddr_t, u_int8_t *); static void aes_xts_reinit(caddr_t, u_int8_t *); +static void aes_gcm_reinit(caddr_t, u_int8_t *); static void null_init(void *); -static int null_update(void *, u_int8_t *, u_int16_t); +static void null_reinit(void *ctx, const u_int8_t *buf, u_int16_t len); +static int null_update(void *, const u_int8_t *, u_int16_t); static void null_final(u_int8_t *, void *); -static int MD5Update_int(void *, u_int8_t *, u_int16_t); +static int MD5Update_int(void *, const u_int8_t *, u_int16_t); static void SHA1Init_int(void *); -static int SHA1Update_int(void *, u_int8_t *, u_int16_t); +static int SHA1Update_int(void *, const u_int8_t *, u_int16_t); static void SHA1Final_int(u_int8_t *, void *); -static int RMD160Update_int(void *, u_int8_t *, u_int16_t); -static int SHA256Update_int(void *, u_int8_t *, u_int16_t); -static int SHA384Update_int(void *, u_int8_t *, u_int16_t); -static int SHA512Update_int(void *, u_int8_t *, u_int16_t); +static int RMD160Update_int(void *, const u_int8_t *, u_int16_t); +static int SHA256Update_int(void *, const u_int8_t *, u_int16_t); +static int SHA384Update_int(void *, const u_int8_t *, u_int16_t); +static int SHA512Update_int(void *, const u_int8_t *, u_int16_t); static u_int32_t deflate_compress(u_int8_t *, u_int32_t, u_int8_t **); static u_int32_t deflate_decompress(u_int8_t *, u_int32_t, u_int8_t **); +#define AESICM_BLOCKSIZE AES_BLOCK_LEN + +struct aes_icm_ctx { + u_int32_t ac_ek[4*(RIJNDAEL_MAXNR + 1)]; + /* ac_block is initalized to IV */ + u_int8_t ac_block[AESICM_BLOCKSIZE]; + int ac_nr; +}; + MALLOC_DEFINE(M_XDATA, "xform", "xform data buffers"); /* Encryption instances */ struct enc_xform enc_xform_null = { CRYPTO_NULL_CBC, "NULL", /* NB: blocksize of 4 is to generate a properly aligned ESP header */ - NULL_BLOCK_LEN, 0, 256, /* 2048 bits, max key */ + NULL_BLOCK_LEN, NULL_BLOCK_LEN, NULL_MIN_KEY, NULL_MAX_KEY, null_encrypt, null_decrypt, null_setkey, null_zerokey, - NULL + NULL, }; struct enc_xform enc_xform_des = { CRYPTO_DES_CBC, "DES", - DES_BLOCK_LEN, 8, 8, + DES_BLOCK_LEN, DES_BLOCK_LEN, DES_MIN_KEY, DES_MAX_KEY, des1_encrypt, des1_decrypt, des1_setkey, des1_zerokey, - NULL + NULL, }; struct enc_xform enc_xform_3des = { CRYPTO_3DES_CBC, "3DES", - DES3_BLOCK_LEN, 24, 24, + DES3_BLOCK_LEN, DES3_BLOCK_LEN, TRIPLE_DES_MIN_KEY, + TRIPLE_DES_MAX_KEY, des3_encrypt, des3_decrypt, des3_setkey, des3_zerokey, - NULL + NULL, }; struct enc_xform enc_xform_blf = { CRYPTO_BLF_CBC, "Blowfish", - BLOWFISH_BLOCK_LEN, 5, 56 /* 448 bits, max key */, + BLOWFISH_BLOCK_LEN, BLOWFISH_BLOCK_LEN, BLOWFISH_MIN_KEY, + BLOWFISH_MAX_KEY, blf_encrypt, blf_decrypt, blf_setkey, blf_zerokey, - NULL + NULL, }; struct enc_xform enc_xform_cast5 = { CRYPTO_CAST_CBC, "CAST-128", - CAST128_BLOCK_LEN, 5, 16, + CAST128_BLOCK_LEN, CAST128_BLOCK_LEN, CAST_MIN_KEY, CAST_MAX_KEY, cast5_encrypt, cast5_decrypt, cast5_setkey, cast5_zerokey, - NULL + NULL, }; struct enc_xform enc_xform_skipjack = { CRYPTO_SKIPJACK_CBC, "Skipjack", - SKIPJACK_BLOCK_LEN, 10, 10, + SKIPJACK_BLOCK_LEN, SKIPJACK_BLOCK_LEN, SKIPJACK_MIN_KEY, + SKIPJACK_MAX_KEY, skipjack_encrypt, - skipjack_decrypt, - skipjack_setkey, + skipjack_decrypt, skipjack_setkey, skipjack_zerokey, - NULL + NULL, }; struct enc_xform enc_xform_rijndael128 = { CRYPTO_RIJNDAEL128_CBC, "Rijndael-128/AES", - RIJNDAEL128_BLOCK_LEN, 8, 32, + RIJNDAEL128_BLOCK_LEN, RIJNDAEL128_BLOCK_LEN, RIJNDAEL_MIN_KEY, + RIJNDAEL_MAX_KEY, rijndael128_encrypt, rijndael128_decrypt, rijndael128_setkey, rijndael128_zerokey, - NULL + NULL, +}; + +struct enc_xform enc_xform_aes_icm = { + CRYPTO_AES_ICM, "AES-ICM", + AES_BLOCK_LEN, AES_BLOCK_LEN, AES_MIN_KEY, AES_MAX_KEY, + aes_icm_crypt, + aes_icm_crypt, + aes_icm_setkey, + rijndael128_zerokey, + aes_icm_reinit, +}; + +struct enc_xform enc_xform_aes_nist_gcm = { + CRYPTO_AES_NIST_GCM_16, "AES-GCM", + AES_ICM_BLOCK_LEN, AES_GCM_IV_LEN, AES_MIN_KEY, AES_MAX_KEY, + aes_icm_crypt, + aes_icm_crypt, + aes_icm_setkey, + aes_icm_zerokey, + aes_gcm_reinit, +}; + +struct enc_xform enc_xform_aes_nist_gmac = { + CRYPTO_AES_NIST_GMAC, "AES-GMAC", + AES_ICM_BLOCK_LEN, AES_GCM_IV_LEN, AES_MIN_KEY, AES_MAX_KEY, + NULL, + NULL, + NULL, + NULL, + NULL, }; struct enc_xform enc_xform_aes_xts = { CRYPTO_AES_XTS, "AES-XTS", - RIJNDAEL128_BLOCK_LEN, 32, 64, + AES_BLOCK_LEN, AES_XTS_IV_LEN, AES_XTS_MIN_KEY, AES_XTS_MAX_KEY, aes_xts_encrypt, aes_xts_decrypt, aes_xts_setkey, @@ -212,85 +267,123 @@ struct enc_xform enc_xform_aes_xts = { struct enc_xform enc_xform_arc4 = { CRYPTO_ARC4, "ARC4", - 1, 1, 32, + ARC4_BLOCK_LEN, ARC4_IV_LEN, ARC4_MIN_KEY, ARC4_MAX_KEY, + NULL, NULL, NULL, NULL, NULL, - NULL }; struct enc_xform enc_xform_camellia = { CRYPTO_CAMELLIA_CBC, "Camellia", - CAMELLIA_BLOCK_LEN, 8, 32, + CAMELLIA_BLOCK_LEN, CAMELLIA_BLOCK_LEN, CAMELLIA_MIN_KEY, + CAMELLIA_MAX_KEY, cml_encrypt, cml_decrypt, cml_setkey, cml_zerokey, - NULL + NULL, }; /* Authentication instances */ -struct auth_hash auth_hash_null = { +struct auth_hash auth_hash_null = { /* NB: context isn't used */ CRYPTO_NULL_HMAC, "NULL-HMAC", - 0, NULL_HASH_LEN, NULL_HMAC_BLOCK_LEN, sizeof(int), /* NB: context isn't used */ - null_init, null_update, null_final + NULL_HMAC_KEY_LEN, NULL_HASH_LEN, sizeof(int), NULL_HMAC_BLOCK_LEN, + null_init, null_reinit, null_reinit, null_update, null_final }; struct auth_hash auth_hash_hmac_md5 = { CRYPTO_MD5_HMAC, "HMAC-MD5", - 16, MD5_HASH_LEN, MD5_HMAC_BLOCK_LEN, sizeof(MD5_CTX), - (void (*) (void *)) MD5Init, MD5Update_int, + MD5_HMAC_KEY_LEN, MD5_HASH_LEN, sizeof(MD5_CTX), MD5_HMAC_BLOCK_LEN, + (void (*) (void *)) MD5Init, NULL, NULL, MD5Update_int, (void (*) (u_int8_t *, void *)) MD5Final }; struct auth_hash auth_hash_hmac_sha1 = { CRYPTO_SHA1_HMAC, "HMAC-SHA1", - 20, SHA1_HASH_LEN, SHA1_HMAC_BLOCK_LEN, sizeof(SHA1_CTX), - SHA1Init_int, SHA1Update_int, SHA1Final_int + SHA1_HMAC_KEY_LEN, SHA1_HASH_LEN, sizeof(SHA1_CTX), SHA1_HMAC_BLOCK_LEN, + SHA1Init_int, NULL, NULL, SHA1Update_int, SHA1Final_int }; struct auth_hash auth_hash_hmac_ripemd_160 = { CRYPTO_RIPEMD160_HMAC, "HMAC-RIPEMD-160", - 20, RIPEMD160_HASH_LEN, RIPEMD160_HMAC_BLOCK_LEN, sizeof(RMD160_CTX), - (void (*)(void *)) RMD160Init, RMD160Update_int, + RIPEMD160_HMAC_KEY_LEN, RIPEMD160_HASH_LEN, sizeof(RMD160_CTX), + RIPEMD160_HMAC_BLOCK_LEN, + (void (*)(void *)) RMD160Init, NULL, NULL, RMD160Update_int, (void (*)(u_int8_t *, void *)) RMD160Final }; struct auth_hash auth_hash_key_md5 = { CRYPTO_MD5_KPDK, "Keyed MD5", - 0, MD5_KPDK_HASH_LEN, 0, sizeof(MD5_CTX), - (void (*)(void *)) MD5Init, MD5Update_int, + NULL_HMAC_KEY_LEN, MD5_KPDK_HASH_LEN, sizeof(MD5_CTX), 0, + (void (*)(void *)) MD5Init, NULL, NULL, MD5Update_int, (void (*)(u_int8_t *, void *)) MD5Final }; struct auth_hash auth_hash_key_sha1 = { CRYPTO_SHA1_KPDK, "Keyed SHA1", - 0, SHA1_KPDK_HASH_LEN, 0, sizeof(SHA1_CTX), - SHA1Init_int, SHA1Update_int, SHA1Final_int + NULL_HMAC_KEY_LEN, SHA1_KPDK_HASH_LEN, sizeof(SHA1_CTX), 0, + SHA1Init_int, NULL, NULL, SHA1Update_int, SHA1Final_int }; struct auth_hash auth_hash_hmac_sha2_256 = { CRYPTO_SHA2_256_HMAC, "HMAC-SHA2-256", - 32, SHA2_256_HASH_LEN, SHA2_256_HMAC_BLOCK_LEN, sizeof(SHA256_CTX), - (void (*)(void *)) SHA256_Init, SHA256Update_int, + SHA2_256_HMAC_KEY_LEN, SHA2_256_HASH_LEN, sizeof(SHA256_CTX), + SHA2_256_HMAC_BLOCK_LEN, + (void (*)(void *)) SHA256_Init, NULL, NULL, SHA256Update_int, (void (*)(u_int8_t *, void *)) SHA256_Final }; struct auth_hash auth_hash_hmac_sha2_384 = { CRYPTO_SHA2_384_HMAC, "HMAC-SHA2-384", - 48, SHA2_384_HASH_LEN, SHA2_384_HMAC_BLOCK_LEN, sizeof(SHA384_CTX), - (void (*)(void *)) SHA384_Init, SHA384Update_int, + SHA2_384_HMAC_KEY_LEN, SHA2_384_HASH_LEN, sizeof(SHA384_CTX), + SHA2_384_HMAC_BLOCK_LEN, + (void (*)(void *)) SHA384_Init, NULL, NULL, SHA384Update_int, (void (*)(u_int8_t *, void *)) SHA384_Final }; struct auth_hash auth_hash_hmac_sha2_512 = { CRYPTO_SHA2_512_HMAC, "HMAC-SHA2-512", - 64, SHA2_512_HASH_LEN, SHA2_512_HMAC_BLOCK_LEN, sizeof(SHA512_CTX), - (void (*)(void *)) SHA512_Init, SHA512Update_int, + SHA2_512_HMAC_KEY_LEN, SHA2_512_HASH_LEN, sizeof(SHA512_CTX), + SHA2_512_HMAC_BLOCK_LEN, + (void (*)(void *)) SHA512_Init, NULL, NULL, SHA512Update_int, (void (*)(u_int8_t *, void *)) SHA512_Final }; +struct auth_hash auth_hash_nist_gmac_aes_128 = { + CRYPTO_AES_128_NIST_GMAC, "GMAC-AES-128", + AES_128_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx), + GMAC_BLOCK_LEN, + (void (*)(void *)) AES_GMAC_Init, + (void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Setkey, + (void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Reinit, + (int (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Update, + (void (*)(u_int8_t *, void *)) AES_GMAC_Final +}; + +struct auth_hash auth_hash_nist_gmac_aes_192 = { + CRYPTO_AES_192_NIST_GMAC, "GMAC-AES-192", + AES_192_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx), + GMAC_BLOCK_LEN, + (void (*)(void *)) AES_GMAC_Init, + (void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Setkey, + (void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Reinit, + (int (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Update, + (void (*)(u_int8_t *, void *)) AES_GMAC_Final +}; + +struct auth_hash auth_hash_nist_gmac_aes_256 = { + CRYPTO_AES_256_NIST_GMAC, "GMAC-AES-256", + AES_256_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx), + GMAC_BLOCK_LEN, + (void (*)(void *)) AES_GMAC_Init, + (void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Setkey, + (void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Reinit, + (int (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Update, + (void (*)(u_int8_t *, void *)) AES_GMAC_Final +}; + /* Compression instance */ struct comp_algo comp_algo_deflate = { CRYPTO_DEFLATE_COMP, "Deflate", @@ -579,6 +672,75 @@ rijndael128_zerokey(u_int8_t **sched) *sched = NULL; } +void +aes_icm_reinit(caddr_t key, u_int8_t *iv) +{ + struct aes_icm_ctx *ctx; + + ctx = (struct aes_icm_ctx *)key; + bcopy(iv, ctx->ac_block, AESICM_BLOCKSIZE); +} + +void +aes_gcm_reinit(caddr_t key, u_int8_t *iv) +{ + struct aes_icm_ctx *ctx; + + aes_icm_reinit(key, iv); + + ctx = (struct aes_icm_ctx *)key; + /* GCM starts with 2 as counter 1 is used for final xor of tag. */ + bzero(&ctx->ac_block[AESICM_BLOCKSIZE - 4], 4); + ctx->ac_block[AESICM_BLOCKSIZE - 1] = 2; +} + +void +aes_icm_crypt(caddr_t key, u_int8_t *data) +{ + struct aes_icm_ctx *ctx; + u_int8_t keystream[AESICM_BLOCKSIZE]; + int i; + + ctx = (struct aes_icm_ctx *)key; + rijndaelEncrypt(ctx->ac_ek, ctx->ac_nr, ctx->ac_block, keystream); + for (i = 0; i < AESICM_BLOCKSIZE; i++) + data[i] ^= keystream[i]; + explicit_bzero(keystream, sizeof(keystream)); + + /* increment counter */ + for (i = AESICM_BLOCKSIZE - 1; + i >= 0; i--) + if (++ctx->ac_block[i]) /* continue on overflow */ + break; +} + +int +aes_icm_setkey(u_int8_t **sched, u_int8_t *key, int len) +{ + struct aes_icm_ctx *ctx; + + if (len != 16 && len != 24 && len != 32) + return EINVAL; + + *sched = malloc(sizeof(struct aes_icm_ctx), M_CRYPTO_DATA, + M_NOWAIT | M_ZERO); + if (*sched == NULL) + return ENOMEM; + + ctx = (struct aes_icm_ctx *)*sched; + ctx->ac_nr = rijndaelKeySetupEnc(ctx->ac_ek, (u_char *)key, len * 8); + return 0; +} + +void +aes_icm_zerokey(u_int8_t **sched) +{ + + bzero(*sched, sizeof(struct aes_icm_ctx)); + free(*sched, M_CRYPTO_DATA); + *sched = NULL; +} + #define AES_XTS_BLOCKSIZE 16 #define AES_XTS_IVSIZE 8 #define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ @@ -728,8 +890,13 @@ null_init(void *ctx) { } +static void +null_reinit(void *ctx, const u_int8_t *buf, u_int16_t len) +{ +} + static int -null_update(void *ctx, u_int8_t *buf, u_int16_t len) +null_update(void *ctx, const u_int8_t *buf, u_int16_t len) { return 0; } @@ -742,14 +909,14 @@ null_final(u_int8_t *buf, void *ctx) } static int -RMD160Update_int(void *ctx, u_int8_t *buf, u_int16_t len) +RMD160Update_int(void *ctx, const u_int8_t *buf, u_int16_t len) { RMD160Update(ctx, buf, len); return 0; } static int -MD5Update_int(void *ctx, u_int8_t *buf, u_int16_t len) +MD5Update_int(void *ctx, const u_int8_t *buf, u_int16_t len) { MD5Update(ctx, buf, len); return 0; @@ -762,7 +929,7 @@ SHA1Init_int(void *ctx) } static int -SHA1Update_int(void *ctx, u_int8_t *buf, u_int16_t len) +SHA1Update_int(void *ctx, const u_int8_t *buf, u_int16_t len) { SHA1Update(ctx, buf, len); return 0; @@ -775,21 +942,21 @@ SHA1Final_int(u_int8_t *blk, void *ctx) } static int -SHA256Update_int(void *ctx, u_int8_t *buf, u_int16_t len) +SHA256Update_int(void *ctx, const u_int8_t *buf, u_int16_t len) { SHA256_Update(ctx, buf, len); return 0; } static int -SHA384Update_int(void *ctx, u_int8_t *buf, u_int16_t len) +SHA384Update_int(void *ctx, const u_int8_t *buf, u_int16_t len) { SHA384_Update(ctx, buf, len); return 0; } static int -SHA512Update_int(void *ctx, u_int8_t *buf, u_int16_t len) +SHA512Update_int(void *ctx, const u_int8_t *buf, u_int16_t len) { SHA512_Update(ctx, buf, len); return 0; diff --git a/sys/opencrypto/xform.h b/sys/opencrypto/xform.h index 8df7b07..e0b6397 100644 --- a/sys/opencrypto/xform.h +++ b/sys/opencrypto/xform.h @@ -9,6 +9,12 @@ * supported the development of this code. * * Copyright (c) 2000 Angelos D. Keromytis + * Copyright (c) 2014 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by John-Mark Gurney + * under sponsorship of the FreeBSD Foundation and + * Rubicon Communications, LLC (Netgate). * * Permission to use, copy, and modify this software without fee * is hereby granted, provided that this entire notice is included in @@ -29,6 +35,7 @@ #include <crypto/sha1.h> #include <crypto/sha2/sha2.h> #include <opencrypto/rmd160.h> +#include <opencrypto/gmac.h> /* Declarations */ struct auth_hash { @@ -36,10 +43,12 @@ struct auth_hash { char *name; u_int16_t keysize; u_int16_t hashsize; - u_int16_t blocksize; u_int16_t ctxsize; + u_int16_t blocksize; void (*Init) (void *); - int (*Update) (void *, u_int8_t *, u_int16_t); + void (*Setkey) (void *, const u_int8_t *, u_int16_t); + void (*Reinit) (void *, const u_int8_t *, u_int16_t); + int (*Update) (void *, const u_int8_t *, u_int16_t); void (*Final) (u_int8_t *, void *); }; @@ -50,6 +59,7 @@ struct enc_xform { int type; char *name; u_int16_t blocksize; + u_int16_t ivsize; u_int16_t minkey, maxkey; void (*encrypt) (caddr_t, u_int8_t *); void (*decrypt) (caddr_t, u_int8_t *); @@ -73,6 +83,7 @@ union authctx { SHA256_CTX sha256ctx; SHA384_CTX sha384ctx; SHA512_CTX sha512ctx; + struct aes_gmac_ctx aes_gmac_ctx; }; extern struct enc_xform enc_xform_null; @@ -82,6 +93,9 @@ extern struct enc_xform enc_xform_blf; extern struct enc_xform enc_xform_cast5; extern struct enc_xform enc_xform_skipjack; extern struct enc_xform enc_xform_rijndael128; +extern struct enc_xform enc_xform_aes_icm; +extern struct enc_xform enc_xform_aes_nist_gcm; +extern struct enc_xform enc_xform_aes_nist_gmac; extern struct enc_xform enc_xform_aes_xts; extern struct enc_xform enc_xform_arc4; extern struct enc_xform enc_xform_camellia; @@ -95,6 +109,9 @@ extern struct auth_hash auth_hash_hmac_ripemd_160; extern struct auth_hash auth_hash_hmac_sha2_256; extern struct auth_hash auth_hash_hmac_sha2_384; extern struct auth_hash auth_hash_hmac_sha2_512; +extern struct auth_hash auth_hash_nist_gmac_aes_128; +extern struct auth_hash auth_hash_nist_gmac_aes_192; +extern struct auth_hash auth_hash_nist_gmac_aes_256; extern struct comp_algo comp_algo_deflate; |