diff options
Diffstat (limited to 'sys/crypto/aesni/aesni.c')
-rw-r--r-- | sys/crypto/aesni/aesni.c | 475 |
1 files changed, 352 insertions, 123 deletions
diff --git a/sys/crypto/aesni/aesni.c b/sys/crypto/aesni/aesni.c index 7d7a740..fc6bbf4 100644 --- a/sys/crypto/aesni/aesni.c +++ b/sys/crypto/aesni/aesni.c @@ -39,14 +39,15 @@ __FBSDID("$FreeBSD$"); #include <sys/rwlock.h> #include <sys/bus.h> #include <sys/uio.h> +#include <sys/mbuf.h> #include <crypto/aesni/aesni.h> #include <cryptodev_if.h> +#include <opencrypto/gmac.h> struct aesni_softc { int32_t cid; - uint32_t sid; - TAILQ_HEAD(aesni_sessions_head, aesni_session) sessions; - struct rwlock lock; + volatile uint32_t nsessions; + struct aesni_session *sessions; }; static int aesni_newsession(device_t, uint32_t *sidp, struct cryptoini *cri); @@ -56,7 +57,7 @@ static void aesni_freesession_locked(struct aesni_softc *sc, static int aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini); static int aesni_cipher_process(struct aesni_session *ses, - struct cryptodesc *enccrd, struct cryptop *crp); + struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp); MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data"); @@ -79,12 +80,12 @@ aesni_probe(device_t dev) return (EINVAL); } - if ((cpu_feature & CPUID_SSE2) == 0) { - device_printf(dev, "No SSE2 support but AESNI!?!\n"); + if ((cpu_feature & CPUID2_SSE41) == 0 && (cpu_feature2 & CPUID2_SSE41) == 0) { + device_printf(dev, "No SSE4.1 support.\n"); return (EINVAL); } - device_set_desc_copy(dev, "AES-CBC,AES-XTS"); + device_set_desc_copy(dev, "AES-CBC,AES-XTS,AES-GCM"); return (0); } @@ -94,8 +95,6 @@ aesni_attach(device_t dev) struct aesni_softc *sc; sc = device_get_softc(dev); - TAILQ_INIT(&sc->sessions); - sc->sid = 1; sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC); if (sc->cid < 0) { @@ -103,9 +102,24 @@ aesni_attach(device_t dev) return (ENOMEM); } - rw_init(&sc->lock, "aesni_lock"); + sc->nsessions = 32; + sc->sessions = malloc(sc->nsessions * sizeof(struct aesni_session), + M_AESNI, M_WAITOK | M_ZERO); + crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_RFC4106_GCM_16, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_128_GMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_192_GMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_AES_256_GMAC, 0, 0); + + crypto_register(sc->cid, CRYPTO_NULL_HMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_MD5_HMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_SHA2_384_HMAC, 0, 0); + crypto_register(sc->cid, CRYPTO_SHA2_512_HMAC, 0, 0); + return (0); } @@ -114,25 +128,24 @@ aesni_detach(device_t dev) { struct aesni_softc *sc; struct aesni_session *ses; + int i; sc = device_get_softc(dev); - rw_wlock(&sc->lock); - TAILQ_FOREACH(ses, &sc->sessions, next) { + for (i = 0; i < sc->nsessions; i++) { + ses = &sc->sessions[i]; if (ses->used) { - rw_wunlock(&sc->lock); device_printf(dev, "Cannot detach, sessions still active.\n"); return (EBUSY); } } - while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) { - TAILQ_REMOVE(&sc->sessions, ses, next); - fpu_kern_free_ctx(ses->fpu_ctx); - free(ses, M_AESNI); - } - rw_wunlock(&sc->lock); - rw_destroy(&sc->lock); crypto_unregister_all(sc->cid); + for (i = 0; i < sc->nsessions; i++) { + ses = &sc->sessions[i]; + if (ses->fpu_ctx != NULL) + fpu_kern_free_ctx(ses->fpu_ctx); + } + free(sc->sessions, M_AESNI); return (0); } @@ -141,67 +154,111 @@ aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) { struct aesni_softc *sc; struct aesni_session *ses; - struct cryptoini *encini; - int error; + struct cryptoini *encini, *authini; + int error, sessn; - if (sidp == NULL || cri == NULL) + if (sidp == NULL || cri == NULL) { + printf("no sidp or cri"); return (EINVAL); + } sc = device_get_softc(dev); ses = NULL; encini = NULL; + authini = NULL; for (; cri != NULL; cri = cri->cri_next) { switch (cri->cri_alg) { case CRYPTO_AES_CBC: + if (encini != NULL) { + printf("encini already set"); + return (EINVAL); + } + encini = cri; + break; case CRYPTO_AES_XTS: - if (encini != NULL) + case CRYPTO_AES_RFC4106_GCM_16: + if (encini != NULL) { + printf("encini already set"); return (EINVAL); - encini = cri; + } + encini = cri; + break; + case CRYPTO_AES_128_GMAC: + case CRYPTO_AES_192_GMAC: + case CRYPTO_AES_256_GMAC: + /* + * nothing to do here, maybe in the future cache some + * values for GHASH + */ + break; + case CRYPTO_NULL_HMAC: + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + if (authini != NULL) { + printf("authini already set"); + return (EINVAL); + } + authini = cri; break; default: + printf("unhandled algorithm"); return (EINVAL); } } - if (encini == NULL) - return (EINVAL); - rw_wlock(&sc->lock); - /* - * Free sessions goes first, so if first session is used, we need to - * allocate one. - */ - ses = TAILQ_FIRST(&sc->sessions); - if (ses == NULL || ses->used) { - ses = malloc(sizeof(*ses), M_AESNI, M_NOWAIT | M_ZERO); + for (sessn = 1; sessn < sc->nsessions; sessn++) { + if (!sc->sessions[sessn].used) { + ses = &sc->sessions[sessn]; + break; + } + } + if (ses == NULL) { + ses = malloc(sizeof(*ses) * sc->nsessions * 2, M_AESNI, M_NOWAIT | M_ZERO); if (ses == NULL) { - rw_wunlock(&sc->lock); + sc->sessions = ses; return (ENOMEM); } - ses->fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL | - FPU_KERN_NOWAIT); + bcopy((void *)sc->sessions, (void *)ses, sc->nsessions * sizeof(*ses)); + atomic_set_ptr((u_long *)sc->sessions, (u_long)ses); + bzero((void *)ses, sc->nsessions * sizeof(*ses)); + ses = &sc->sessions[sc->nsessions]; + ses->id = sc->nsessions; + atomic_add_int(&sc->nsessions, 1); + } else if (ses->id == 0) + ses->id = sessn; + + if (encini != NULL) { if (ses->fpu_ctx == NULL) { - free(ses, M_AESNI); - rw_wunlock(&sc->lock); - return (ENOMEM); + ses->fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL | + FPU_KERN_NOWAIT); + if (ses->fpu_ctx == NULL) + return (ENOMEM); + } + ses->algo = encini->cri_alg; + + error = aesni_cipher_setup(ses, encini); + if (error != 0) { + printf("setup failed"); + aesni_freesession_locked(sc, ses); + return (error); } - ses->id = sc->sid++; - } else { - TAILQ_REMOVE(&sc->sessions, ses, next); } - ses->used = 1; - TAILQ_INSERT_TAIL(&sc->sessions, ses, next); - rw_wunlock(&sc->lock); - ses->algo = encini->cri_alg; - error = aesni_cipher_setup(ses, encini); - if (error != 0) { - rw_wlock(&sc->lock); - aesni_freesession_locked(sc, ses); - rw_wunlock(&sc->lock); - return (error); + if (authini != NULL) { + error = aesni_hash_setup(ses, authini); + if (error != 0) { + printf("setup failed"); + aesni_freesession_locked(sc, ses); + return (error); + } } + ses->used = 1; *sidp = ses->id; + return (0); } @@ -212,12 +269,11 @@ aesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses) uint32_t sid; sid = ses->id; - TAILQ_REMOVE(&sc->sessions, ses, next); ctx = ses->fpu_ctx; bzero(ses, sizeof(*ses)); ses->id = sid; ses->fpu_ctx = ctx; - TAILQ_INSERT_HEAD(&sc->sessions, ses, next); + aesni_hash_free(ses); } static int @@ -229,17 +285,14 @@ aesni_freesession(device_t dev, uint64_t tid) sc = device_get_softc(dev); sid = ((uint32_t)tid) & 0xffffffff; - rw_wlock(&sc->lock); - TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) { - if (ses->id == sid) - break; - } - if (ses == NULL) { - rw_wunlock(&sc->lock); + if (sid >= sc->nsessions) return (EINVAL); - } + + ses = &sc->sessions[sid]; + if (ses == NULL) + return (EINVAL); + aesni_freesession_locked(sc, ses); - rw_wunlock(&sc->lock); return (0); } @@ -248,20 +301,25 @@ aesni_process(device_t dev, struct cryptop *crp, int hint __unused) { struct aesni_softc *sc = device_get_softc(dev); struct aesni_session *ses = NULL; - struct cryptodesc *crd, *enccrd; - int error; + struct cryptodesc *crd, *enccrd, *authcrd; + uint32_t sid; + int error, needauth; error = 0; enccrd = NULL; + authcrd = NULL; + needauth = 0; /* Sanity check. */ if (crp == NULL) return (EINVAL); - if (crp->crp_callback == NULL || crp->crp_desc == NULL) { - error = EINVAL; - goto out; - } + if (crp->crp_callback == NULL || crp->crp_desc == NULL) + return (EINVAL); + + sid = ((uint32_t)crp->crp_sid) & 0xffffffff; + if (sid >= sc->nsessions) + return (EINVAL); for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) { switch (crd->crd_alg) { @@ -273,27 +331,65 @@ aesni_process(device_t dev, struct cryptop *crp, int hint __unused) } enccrd = crd; break; + + case CRYPTO_AES_RFC4106_GCM_16: + if (enccrd != NULL) { + error = EINVAL; + goto out; + } + enccrd = crd; + needauth = 1; + break; + + case CRYPTO_AES_128_GMAC: + case CRYPTO_AES_192_GMAC: + case CRYPTO_AES_256_GMAC: + if (authcrd != NULL) { + error = EINVAL; + goto out; + } + authcrd = crd; + needauth = 1; + break; + + case CRYPTO_NULL_HMAC: + case CRYPTO_MD5_HMAC: + case CRYPTO_SHA1_HMAC: + case CRYPTO_SHA2_256_HMAC: + case CRYPTO_SHA2_384_HMAC: + case CRYPTO_SHA2_512_HMAC: + if (authcrd != NULL) { + error = EINVAL; + goto out; + } + authcrd = crd; + needauth = 1; + break; + default: return (EINVAL); } } - if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) { + + if (needauth && authcrd == NULL) { error = EINVAL; goto out; } - rw_rlock(&sc->lock); - TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) { - if (ses->id == (crp->crp_sid & 0xffffffff)) - break; + /* CBC & XTS can only handle full blocks for now */ + if (enccrd != NULL && ((enccrd->crd_len == CRYPTO_AES_CBC || enccrd->crd_len == + CRYPTO_AES_XTS) && (enccrd->crd_len % AES_BLOCK_LEN) != 0)) { + error = EINVAL; + goto out; } - rw_runlock(&sc->lock); + + ses = &sc->sessions[sid]; if (ses == NULL) { error = EINVAL; goto out; } - error = aesni_cipher_process(ses, enccrd, crp); + error = aesni_cipher_process(ses, enccrd, authcrd, crp); if (error != 0) goto out; @@ -307,13 +403,17 @@ uint8_t * aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp, int *allocated) { + struct mbuf *m; struct uio *uio; struct iovec *iov; uint8_t *addr; - if (crp->crp_flags & CRYPTO_F_IMBUF) - goto alloc; - else if (crp->crp_flags & CRYPTO_F_IOV) { + if (crp->crp_flags & CRYPTO_F_IMBUF) { + m = (struct mbuf *)crp->crp_buf; + if (m->m_next != NULL) + goto alloc; + addr = mtod(m, uint8_t *); + } else if (crp->crp_flags & CRYPTO_F_IOV) { uio = (struct uio *)crp->crp_buf; if (uio->uio_iovcnt != 1) goto alloc; @@ -322,6 +422,7 @@ aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp, } else addr = (u_char *)crp->crp_buf; *allocated = 0; + addr += enccrd->crd_skip; return (addr); alloc: @@ -366,83 +467,211 @@ aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini) int error; td = curthread; + critical_enter(); error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR); - if (error != 0) + if (error != 0) { + critical_exit(); return (error); + } error = aesni_cipher_setup_common(ses, encini->cri_key, encini->cri_klen); fpu_kern_leave(td, ses->fpu_ctx); + critical_exit(); return (error); } +#ifdef AESNI_DEBUG +static void +aesni_printhexstr(uint8_t *ptr, int len) +{ + int i; + + for (i = 0; i < len; i++) + printf("%02hhx", ptr[i]); +} +#endif + static int aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd, - struct cryptop *crp) + struct cryptodesc *authcrd, struct cryptop *crp) { + uint8_t *tag; + uint8_t *iv; struct thread *td; - uint8_t *buf; - int error, allocated; + uint8_t *buf, *authbuf; + int error, allocated, authallocated; + int ivlen, encflag, i; - buf = aesni_cipher_alloc(enccrd, crp, &allocated); - if (buf == NULL) - return (ENOMEM); + allocated = authallocated = 0; + encflag = error = 0; + authbuf = NULL; + + if (enccrd != NULL) { + encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT; + + buf = aesni_cipher_alloc(enccrd, crp, &allocated); + if (buf == NULL) + return (ENOMEM); + + iv = enccrd->crd_iv; + /* XXX - validate that enccrd and authcrd have/use same key? */ + switch (enccrd->crd_alg) { + case CRYPTO_AES_CBC: + ivlen = 16; + break; + case CRYPTO_AES_XTS: + ivlen = 8; + break; + case CRYPTO_AES_RFC4106_GCM_16: + /* Be smart at determining the ivlen until better ways are present */ + ivlen = enccrd->crd_skip - enccrd->crd_inject; + ivlen += 4; + break; + } + + /* Setup ses->iv */ + if (encflag) { + if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) { + if (enccrd->crd_alg == CRYPTO_AES_RFC4106_GCM_16) { + for (i = 0; i < AESCTR_NONCESIZE; i++) + iv[i] = ses->nonce[i]; + /* XXX: Is this enough? */ + u_long counter = atomic_fetchadd_long(&ses->aesgcmcounter, 1); + bcopy((void *)&counter, iv + AESCTR_NONCESIZE, sizeof(uint64_t)); + crypto_copyback(crp->crp_flags, crp->crp_buf, + enccrd->crd_inject, AESCTR_IVSIZE, iv + AESCTR_NONCESIZE); + } else { + arc4rand(iv, AES_BLOCK_LEN, 0); + crypto_copyback(crp->crp_flags, crp->crp_buf, + enccrd->crd_inject, ivlen, iv); + } + } + } else { + if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0) { + if (enccrd->crd_alg == CRYPTO_AES_RFC4106_GCM_16) { + for (i = 0; i < AESCTR_NONCESIZE; i++) + iv[i] = ses->nonce[i]; + crypto_copydata(crp->crp_flags, crp->crp_buf, + enccrd->crd_inject, AESCTR_IVSIZE, iv + AESCTR_NONCESIZE); + } else + crypto_copydata(crp->crp_flags, crp->crp_buf, + enccrd->crd_inject, ivlen, iv); + } + } +#ifdef AESNI_DEBUG + aesni_printhexstr(iv, ivlen); + printf("\n"); +#endif + } + + if (authcrd != NULL) { + authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated); + if (authbuf == NULL) { + error = ENOMEM; + goto out1; + } + + if (ses->algo == CRYPTO_AES_RFC4106_GCM_16) { + /* NOTE: GMAC_DIGEST_LEN == AES_BLOCK_LEN */ + tag = authcrd->crd_iv; + + if (!encflag) { + crypto_copydata(crp->crp_flags, crp->crp_buf, + authcrd->crd_inject, GMAC_DIGEST_LEN, tag); + } else { +#ifdef AESNI_DEBUG + printf("ptag: "); + aesni_printhexstr(tag, sizeof tag); + printf("\n"); +#endif + bzero(tag, sizeof tag); + } + } + } + /* Called by stack only for HASH operation? */ + if (enccrd == NULL) + goto out1; td = curthread; + + critical_enter(); error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR); - if (error != 0) + if (error != 0) { + critical_exit(); goto out1; - - if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) { - error = aesni_cipher_setup_common(ses, enccrd->crd_key, - enccrd->crd_klen); - if (error != 0) - goto out; } - - if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) { - if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) - bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); - if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) - crypto_copyback(crp->crp_flags, crp->crp_buf, - enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); - if (ses->algo == CRYPTO_AES_CBC) { + /* Do work */ + switch (ses->algo) { + case CRYPTO_AES_CBC: + if (encflag) aesni_encrypt_cbc(ses->rounds, ses->enc_schedule, - enccrd->crd_len, buf, buf, ses->iv); - } else /* if (ses->algo == CRYPTO_AES_XTS) */ { + enccrd->crd_len, buf, buf, iv); + else + aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, + enccrd->crd_len, buf, iv); + break; + case CRYPTO_AES_XTS: + if (encflag) aesni_encrypt_xts(ses->rounds, ses->enc_schedule, ses->xts_schedule, enccrd->crd_len, buf, buf, - ses->iv); - } - } else { - if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0) - bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN); + iv); else - crypto_copydata(crp->crp_flags, crp->crp_buf, - enccrd->crd_inject, AES_BLOCK_LEN, ses->iv); - if (ses->algo == CRYPTO_AES_CBC) { - aesni_decrypt_cbc(ses->rounds, ses->dec_schedule, - enccrd->crd_len, buf, ses->iv); - } else /* if (ses->algo == CRYPTO_AES_XTS) */ { aesni_decrypt_xts(ses->rounds, ses->dec_schedule, ses->xts_schedule, enccrd->crd_len, buf, buf, - ses->iv); + iv); + break; + case CRYPTO_AES_RFC4106_GCM_16: +#ifdef AESNI_DEBUG + printf("GCM: %d\n", encflag); + printf("buf(%d): ", enccrd->crd_len); + aesni_printhexstr(buf, enccrd->crd_len); + printf("\nauthbuf(%d): ", authcrd->crd_len); + aesni_printhexstr(authbuf, authcrd->crd_len); + printf("\niv: "); + aesni_printhexstr(iv, ivlen); + printf("\ntag: "); + aesni_printhexstr(tag, 16); + printf("\nsched: "); + aesni_printhexstr(ses->enc_schedule, 16 * (ses->rounds + 1)); + printf("\n"); +#endif + if (encflag) + AES_GCM_encrypt(buf, buf, authbuf, iv, tag, + enccrd->crd_len, authcrd->crd_len, ivlen, + ses->enc_schedule, ses->rounds); + else { + if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag, + enccrd->crd_len, authcrd->crd_len, ivlen, + ses->enc_schedule, ses->rounds)) + error = EBADMSG; } + break; } + fpu_kern_leave(td, ses->fpu_ctx); + critical_exit(); + if (allocated) crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, enccrd->crd_len, buf); - if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) - crypto_copydata(crp->crp_flags, crp->crp_buf, - enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN, - AES_BLOCK_LEN, ses->iv); -out: - fpu_kern_leave(td, ses->fpu_ctx); + + if (!error && authcrd != NULL) { + if (ses->algo == CRYPTO_AES_RFC4106_GCM_16) { + crypto_copyback(crp->crp_flags, crp->crp_buf, + authcrd->crd_inject, crp->crp_ilen - authcrd->crd_inject, tag); + } else if (enccrd->crd_next == authcrd) { + error = aesni_hash_process(ses, authcrd, crp); + } + } + out1: if (allocated) { bzero(buf, enccrd->crd_len); free(buf, M_AESNI); } + if (authallocated) + free(authbuf, M_AESNI); + return (error); } |