diff options
Diffstat (limited to 'sys/mips/cavium/cryptocteon')
| -rw-r--r-- | sys/mips/cavium/cryptocteon/cavium_crypto.c | 2222 | ||||
| -rw-r--r-- | sys/mips/cavium/cryptocteon/cryptocteon.c | 526 | ||||
| -rw-r--r-- | sys/mips/cavium/cryptocteon/cryptocteonvar.h | 94 |
3 files changed, 2842 insertions, 0 deletions
diff --git a/sys/mips/cavium/cryptocteon/cavium_crypto.c b/sys/mips/cavium/cryptocteon/cavium_crypto.c new file mode 100644 index 0000000..ff77a61 --- /dev/null +++ b/sys/mips/cavium/cryptocteon/cavium_crypto.c @@ -0,0 +1,2222 @@ +/* + * vim:sw=4 ts=8 + */ +/* + * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com> + * + * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by Cavium Networks + * 4. Cavium Networks' name may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import regulations + * in other countries. You warrant that You will comply strictly in all + * respects with all such regulations and acknowledge that you have the + * responsibility to obtain licenses to export, re-export or import the + * Software. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND + * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, + * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE + * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR + * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM + * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, + * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF + * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR + * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR + * PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +*/ +/****************************************************************************/ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/module.h> +#include <sys/malloc.h> +#include <sys/uio.h> + +#include <opencrypto/cryptodev.h> + +#include <contrib/octeon-sdk/cvmx.h> + +#include <mips/cavium/cryptocteon/cryptocteonvar.h> + +/****************************************************************************/ + +#define IOV_INIT(iov, ptr, idx, len) \ + do { \ + (idx) = 0; \ + (ptr) = (iov)[(idx)].iov_base; \ + (len) = (iov)[(idx)].iov_len; \ + } while (0) + +/* + * XXX + * It would be better if this were an IOV_READ/IOV_WRITE macro instead so + * that we could detect overflow before it happens rather than right after, + * which is especially bad since there is usually no IOV_CONSUME after the + * final read or write. + */ +#define IOV_CONSUME(iov, ptr, idx, len) \ + do { \ + if ((len) > sizeof *(ptr)) { \ + (len) -= sizeof *(ptr); \ + (ptr)++; \ + } else { \ + if ((len) != sizeof *(ptr)) \ + panic("%s: went past end of iovec.", __func__); \ + (idx)++; \ + (ptr) = (iov)[(idx)].iov_base; \ + (len) = (iov)[(idx)].iov_len; \ + } \ + } while (0) + +static inline unsigned long octeon_crypto_enable(void) +{ + register_t s; + + s = intr_disable(); + mips_wr_status(mips_rd_status() | MIPS_SR_COP_2_BIT); + + return (s); +} + +static inline void octeon_crypto_disable(register_t s) +{ + mips_wr_status(mips_rd_status() & ~MIPS_SR_COP_2_BIT); + intr_restore(s); +} + +#define ESP_HEADER_LENGTH 8 +#define DES_CBC_IV_LENGTH 8 +#define AES_CBC_IV_LENGTH 16 +#define ESP_HMAC_LEN 12 + +#define ESP_HEADER_LENGTH 8 +#define DES_CBC_IV_LENGTH 8 + +/****************************************************************************/ + +#define CVM_LOAD_SHA_UNIT(dat, next) { \ + if (next == 0) { \ + next = 1; \ + CVMX_MT_HSH_DAT (dat, 0); \ + } else if (next == 1) { \ + next = 2; \ + CVMX_MT_HSH_DAT (dat, 1); \ + } else if (next == 2) { \ + next = 3; \ + CVMX_MT_HSH_DAT (dat, 2); \ + } else if (next == 3) { \ + next = 4; \ + CVMX_MT_HSH_DAT (dat, 3); \ + } else if (next == 4) { \ + next = 5; \ + CVMX_MT_HSH_DAT (dat, 4); \ + } else if (next == 5) { \ + next = 6; \ + CVMX_MT_HSH_DAT (dat, 5); \ + } else if (next == 6) { \ + next = 7; \ + CVMX_MT_HSH_DAT (dat, 6); \ + } else { \ + CVMX_MT_HSH_STARTSHA (dat); \ + next = 0; \ + } \ +} + +#define CVM_LOAD2_SHA_UNIT(dat1, dat2, next) { \ + if (next == 0) { \ + CVMX_MT_HSH_DAT (dat1, 0); \ + CVMX_MT_HSH_DAT (dat2, 1); \ + next = 2; \ + } else if (next == 1) { \ + CVMX_MT_HSH_DAT (dat1, 1); \ + CVMX_MT_HSH_DAT (dat2, 2); \ + next = 3; \ + } else if (next == 2) { \ + CVMX_MT_HSH_DAT (dat1, 2); \ + CVMX_MT_HSH_DAT (dat2, 3); \ + next = 4; \ + } else if (next == 3) { \ + CVMX_MT_HSH_DAT (dat1, 3); \ + CVMX_MT_HSH_DAT (dat2, 4); \ + next = 5; \ + } else if (next == 4) { \ + CVMX_MT_HSH_DAT (dat1, 4); \ + CVMX_MT_HSH_DAT (dat2, 5); \ + next = 6; \ + } else if (next == 5) { \ + CVMX_MT_HSH_DAT (dat1, 5); \ + CVMX_MT_HSH_DAT (dat2, 6); \ + next = 7; \ + } else if (next == 6) { \ + CVMX_MT_HSH_DAT (dat1, 6); \ + CVMX_MT_HSH_STARTSHA (dat2); \ + next = 0; \ + } else { \ + CVMX_MT_HSH_STARTSHA (dat1); \ + CVMX_MT_HSH_DAT (dat2, 0); \ + next = 1; \ + } \ +} + +/****************************************************************************/ + +#define CVM_LOAD_MD5_UNIT(dat, next) { \ + if (next == 0) { \ + next = 1; \ + CVMX_MT_HSH_DAT (dat, 0); \ + } else if (next == 1) { \ + next = 2; \ + CVMX_MT_HSH_DAT (dat, 1); \ + } else if (next == 2) { \ + next = 3; \ + CVMX_MT_HSH_DAT (dat, 2); \ + } else if (next == 3) { \ + next = 4; \ + CVMX_MT_HSH_DAT (dat, 3); \ + } else if (next == 4) { \ + next = 5; \ + CVMX_MT_HSH_DAT (dat, 4); \ + } else if (next == 5) { \ + next = 6; \ + CVMX_MT_HSH_DAT (dat, 5); \ + } else if (next == 6) { \ + next = 7; \ + CVMX_MT_HSH_DAT (dat, 6); \ + } else { \ + CVMX_MT_HSH_STARTMD5 (dat); \ + next = 0; \ + } \ +} + +#define CVM_LOAD2_MD5_UNIT(dat1, dat2, next) { \ + if (next == 0) { \ + CVMX_MT_HSH_DAT (dat1, 0); \ + CVMX_MT_HSH_DAT (dat2, 1); \ + next = 2; \ + } else if (next == 1) { \ + CVMX_MT_HSH_DAT (dat1, 1); \ + CVMX_MT_HSH_DAT (dat2, 2); \ + next = 3; \ + } else if (next == 2) { \ + CVMX_MT_HSH_DAT (dat1, 2); \ + CVMX_MT_HSH_DAT (dat2, 3); \ + next = 4; \ + } else if (next == 3) { \ + CVMX_MT_HSH_DAT (dat1, 3); \ + CVMX_MT_HSH_DAT (dat2, 4); \ + next = 5; \ + } else if (next == 4) { \ + CVMX_MT_HSH_DAT (dat1, 4); \ + CVMX_MT_HSH_DAT (dat2, 5); \ + next = 6; \ + } else if (next == 5) { \ + CVMX_MT_HSH_DAT (dat1, 5); \ + CVMX_MT_HSH_DAT (dat2, 6); \ + next = 7; \ + } else if (next == 6) { \ + CVMX_MT_HSH_DAT (dat1, 6); \ + CVMX_MT_HSH_STARTMD5 (dat2); \ + next = 0; \ + } else { \ + CVMX_MT_HSH_STARTMD5 (dat1); \ + CVMX_MT_HSH_DAT (dat2, 0); \ + next = 1; \ + } \ +} + +/****************************************************************************/ + +void +octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer) +{ + uint8_t hash_key[64]; + uint64_t *key1; + register uint64_t xor1 = 0x3636363636363636ULL; + register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL; + register_t s; + + dprintf("%s()\n", __func__); + + memset(hash_key, 0, sizeof(hash_key)); + memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); + key1 = (uint64_t *) hash_key; + s = octeon_crypto_enable(); + if (auth) { + CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); + CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); + CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); + } else { + CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); + CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); + } + + CVMX_MT_HSH_DAT((*key1 ^ xor1), 0); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 1); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 2); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 3); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 4); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 5); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 6); + key1++; + if (auth) + CVMX_MT_HSH_STARTSHA((*key1 ^ xor1)); + else + CVMX_MT_HSH_STARTMD5((*key1 ^ xor1)); + + CVMX_MF_HSH_IV(inner[0], 0); + CVMX_MF_HSH_IV(inner[1], 1); + if (auth) { + inner[2] = 0; + CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2); + } + + memset(hash_key, 0, sizeof(hash_key)); + memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); + key1 = (uint64_t *) hash_key; + if (auth) { + CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); + CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); + CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); + } else { + CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); + CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); + } + + CVMX_MT_HSH_DAT((*key1 ^ xor2), 0); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 1); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 2); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 3); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 4); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 5); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 6); + key1++; + if (auth) + CVMX_MT_HSH_STARTSHA((*key1 ^ xor2)); + else + CVMX_MT_HSH_STARTMD5((*key1 ^ xor2)); + + CVMX_MF_HSH_IV(outer[0], 0); + CVMX_MF_HSH_IV(outer[1], 1); + if (auth) { + outer[2] = 0; + CVMX_MF_HSH_IV(outer[2], 2); + } + octeon_crypto_disable(s); + return; +} + +/****************************************************************************/ +/* DES functions */ + +int +octo_des_cbc_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + uint64_t *data; + int data_i, data_l; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + while (crypt_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + crypt_off -= 8; + } + + while (crypt_len > 0) { + CVMX_MT_3DES_ENC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + IOV_CONSUME(iov, data, data_i, data_l); + crypt_len -= 8; + } + + octeon_crypto_disable(s); + return 0; +} + + +int +octo_des_cbc_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + uint64_t *data; + int data_i, data_l; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + while (crypt_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + crypt_off -= 8; + } + + while (crypt_len > 0) { + CVMX_MT_3DES_DEC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + IOV_CONSUME(iov, data, data_i, data_l); + crypt_len -= 8; + } + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* AES functions */ + +int +octo_aes_cbc_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + uint64_t *data, *pdata; + int data_i, data_l; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + while (crypt_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + crypt_off -= 8; + } + + while (crypt_len > 0) { + pdata = data; + CVMX_MT_AES_ENC_CBC0(*data); + IOV_CONSUME(iov, data, data_i, data_l); + CVMX_MT_AES_ENC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + IOV_CONSUME(iov, data, data_i, data_l); + crypt_len -= 16; + } + + octeon_crypto_disable(s); + return 0; +} + + +int +octo_aes_cbc_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + uint64_t *data, *pdata; + int data_i, data_l; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + while (crypt_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + crypt_off -= 8; + } + + while (crypt_len > 0) { + pdata = data; + CVMX_MT_AES_DEC_CBC0(*data); + IOV_CONSUME(iov, data, data_i, data_l); + CVMX_MT_AES_DEC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + IOV_CONSUME(iov, data, data_i, data_l); + crypt_len -= 16; + } + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* MD5 */ + +int +octo_null_md5_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + uint64_t *data; + uint64_t tmp1, tmp2; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || + (auth_off & 0x7) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + s = octeon_crypto_enable(); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + + while (auth_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + auth_off -= 8; + } + + while (auth_len > 0) { + CVM_LOAD_MD5_UNIT(*data, next); + auth_len -= 8; + IOV_CONSUME(iov, data, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); + } + CVMX_ES64(tmp1, ((alen + 64) << 3)); + CVM_LOAD_MD5_UNIT(tmp1, next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_ES64(tmp1, ((64 + 16) << 3)); + CVMX_MT_HSH_STARTMD5(tmp1); + + /* save the HMAC */ + IOV_INIT(iov, data, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + icv_off -= 8; + } + CVMX_MF_HSH_IV(*data, 0); + IOV_CONSUME(iov, data, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *(uint32_t *)data = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* SHA1 */ + +int +octo_null_sha1_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + uint64_t *data; + uint64_t tmp1, tmp2, tmp3; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || + (auth_off & 0x7) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + s = octeon_crypto_enable(); + + /* Load SHA1 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + CVMX_MT_HSH_IV(od->octo_hminner[2], 2); + + while (auth_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + auth_off -= 8; + } + + while (auth_len > 0) { + CVM_LOAD_SHA_UNIT(*data, next); + auth_len -= 8; + IOV_CONSUME(iov, data, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); + } + CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + tmp3 = 0; + CVMX_MF_HSH_IV(tmp3, 2); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + tmp3 |= 0x0000000080000000; + CVMX_MT_HSH_DAT(tmp3, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); + + /* save the HMAC */ + IOV_INIT(iov, data, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + icv_off -= 8; + } + CVMX_MF_HSH_IV(*data, 0); + IOV_CONSUME(iov, data, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *(uint32_t *)data = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* DES MD5 */ + +int +octo_des_cbc_md5_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata; + uint64_t *data = &mydata.data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *first = data32; + mydata.data32[0] = *first; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata.data32[1] = *data32; + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_3DES_ENC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + crypt_len -= 8; + } + } else + crypt_off -= 8; + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_MD5_UNIT(*data, next); + auth_len -= 8; + } + } else + auth_off -= 8; + *first = mydata.data32[0]; + *data32 = mydata.data32[1]; + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); + } + CVMX_ES64(tmp1, ((alen + 64) << 3)); + CVM_LOAD_MD5_UNIT(tmp1, next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_ES64(tmp1, ((64 + 16) << 3)); + CVMX_MT_HSH_STARTMD5(tmp1); + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +int +octo_des_cbc_md5_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata; + uint64_t *data = &mydata.data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *first = data32; + mydata.data32[0] = *first; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata.data32[1] = *data32; + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_MD5_UNIT(*data, next); + auth_len -= 8; + } + } else + auth_off -= 8; + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_3DES_DEC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + crypt_len -= 8; + } + } else + crypt_off -= 8; + *first = mydata.data32[0]; + *data32 = mydata.data32[1]; + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); + } + CVMX_ES64(tmp1, ((alen + 64) << 3)); + CVM_LOAD_MD5_UNIT(tmp1, next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_ES64(tmp1, ((64 + 16) << 3)); + CVMX_MT_HSH_STARTMD5(tmp1); + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* DES SHA */ + +int +octo_des_cbc_sha1_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata; + uint64_t *data = &mydata.data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2, tmp3; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + /* Load SHA1 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + CVMX_MT_HSH_IV(od->octo_hminner[2], 2); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *first = data32; + mydata.data32[0] = *first; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata.data32[1] = *data32; + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_3DES_ENC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + crypt_len -= 8; + } + } else + crypt_off -= 8; + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_SHA_UNIT(*data, next); + auth_len -= 8; + } + } else + auth_off -= 8; + *first = mydata.data32[0]; + *data32 = mydata.data32[1]; + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_SHA_UNIT(tmp, next); + } else { + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); + } + CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + tmp3 = 0; + CVMX_MF_HSH_IV(tmp3, 2); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + tmp3 |= 0x0000000080000000; + CVMX_MT_HSH_DAT(tmp3, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +int +octo_des_cbc_sha1_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata; + uint64_t *data = &mydata.data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2, tmp3; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + /* Load SHA1 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + CVMX_MT_HSH_IV(od->octo_hminner[2], 2); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *first = data32; + mydata.data32[0] = *first; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata.data32[1] = *data32; + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_SHA_UNIT(*data, next); + auth_len -= 8; + } + } else + auth_off -= 8; + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_3DES_DEC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + crypt_len -= 8; + } + } else + crypt_off -= 8; + *first = mydata.data32[0]; + *data32 = mydata.data32[1]; + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_SHA_UNIT(tmp, next); + } else { + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); + } + CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + tmp3 = 0; + CVMX_MF_HSH_IV(tmp3, 2); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + tmp3 |= 0x0000000080000000; + CVMX_MT_HSH_DAT(tmp3, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* AES MD5 */ + +int +octo_aes_cbc_md5_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata[2]; + uint64_t *pdata = &mydata[0].data64[0]; + uint64_t *data = &mydata[1].data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *pdata32[3]; + + pdata32[0] = data32; + mydata[0].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + + pdata32[1] = data32; + mydata[0].data32[1] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + + pdata32[2] = data32; + mydata[1].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + + mydata[1].data32[1] = *data32; + + + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_AES_ENC_CBC0(*pdata); + CVMX_MT_AES_ENC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + crypt_len -= 16; + } + } else + crypt_off -= 16; + + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_MD5_UNIT(*pdata, next); + CVM_LOAD_MD5_UNIT(*data, next); + auth_len -= 16; + } + } else + auth_off -= 16; + + *pdata32[0] = mydata[0].data32[0]; + *pdata32[1] = mydata[0].data32[1]; + *pdata32[2] = mydata[1].data32[0]; + *data32 = mydata[1].data32[1]; + + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); + } + CVMX_ES64(tmp1, ((alen + 64) << 3)); + CVM_LOAD_MD5_UNIT(tmp1, next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_ES64(tmp1, ((64 + 16) << 3)); + CVMX_MT_HSH_STARTMD5(tmp1); + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +int +octo_aes_cbc_md5_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata[2]; + uint64_t *pdata = &mydata[0].data64[0]; + uint64_t *data = &mydata[1].data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *pdata32[3]; + + pdata32[0] = data32; + mydata[0].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[1] = data32; + mydata[0].data32[1] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[2] = data32; + mydata[1].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata[1].data32[1] = *data32; + + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_MD5_UNIT(*pdata, next); + CVM_LOAD_MD5_UNIT(*data, next); + auth_len -= 16; + } + } else + auth_off -= 16; + + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_AES_DEC_CBC0(*pdata); + CVMX_MT_AES_DEC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + crypt_len -= 16; + } + } else + crypt_off -= 16; + + *pdata32[0] = mydata[0].data32[0]; + *pdata32[1] = mydata[0].data32[1]; + *pdata32[2] = mydata[1].data32[0]; + *data32 = mydata[1].data32[1]; + + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); + } + CVMX_ES64(tmp1, ((alen + 64) << 3)); + CVM_LOAD_MD5_UNIT(tmp1, next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_ES64(tmp1, ((64 + 16) << 3)); + CVMX_MT_HSH_STARTMD5(tmp1); + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* AES SHA1 */ + +int +octo_aes_cbc_sha1_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata[2]; + uint64_t *pdata = &mydata[0].data64[0]; + uint64_t *data = &mydata[1].data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2, tmp3; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + /* Load SHA IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + CVMX_MT_HSH_IV(od->octo_hminner[2], 2); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *pdata32[3]; + + pdata32[0] = data32; + mydata[0].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[1] = data32; + mydata[0].data32[1] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[2] = data32; + mydata[1].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata[1].data32[1] = *data32; + + + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_AES_ENC_CBC0(*pdata); + CVMX_MT_AES_ENC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + crypt_len -= 16; + } + } else + crypt_off -= 16; + + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_SHA_UNIT(*pdata, next); + CVM_LOAD_SHA_UNIT(*data, next); + auth_len -= 16; + } + } else + auth_off -= 16; + + *pdata32[0] = mydata[0].data32[0]; + *pdata32[1] = mydata[0].data32[1]; + *pdata32[2] = mydata[1].data32[0]; + *data32 = mydata[1].data32[1]; + + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_SHA_UNIT(tmp, next); + } else { + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); + } + CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + tmp3 = 0; + CVMX_MF_HSH_IV(tmp3, 2); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + tmp3 |= 0x0000000080000000; + CVMX_MT_HSH_DAT(tmp3, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +int +octo_aes_cbc_sha1_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata[2]; + uint64_t *pdata = &mydata[0].data64[0]; + uint64_t *data = &mydata[1].data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2, tmp3; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + CVMX_MT_HSH_IV(od->octo_hminner[2], 2); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *pdata32[3]; + + pdata32[0] = data32; + mydata[0].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[1] = data32; + mydata[0].data32[1] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[2] = data32; + mydata[1].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata[1].data32[1] = *data32; + + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_SHA_UNIT(*pdata, next); + CVM_LOAD_SHA_UNIT(*data, next); + auth_len -= 16; + } + } else + auth_off -= 16; + + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_AES_DEC_CBC0(*pdata); + CVMX_MT_AES_DEC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + crypt_len -= 16; + } + } else + crypt_off -= 16; + + *pdata32[0] = mydata[0].data32[0]; + *pdata32[1] = mydata[0].data32[1]; + *pdata32[2] = mydata[1].data32[0]; + *data32 = mydata[1].data32[1]; + + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_SHA_UNIT(tmp, next); + } else { + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); + } + CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + tmp3 = 0; + CVMX_MF_HSH_IV(tmp3, 2); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + tmp3 |= 0x0000000080000000; + CVMX_MT_HSH_DAT(tmp3, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ diff --git a/sys/mips/cavium/cryptocteon/cryptocteon.c b/sys/mips/cavium/cryptocteon/cryptocteon.c new file mode 100644 index 0000000..16a0bb1 --- /dev/null +++ b/sys/mips/cavium/cryptocteon/cryptocteon.c @@ -0,0 +1,526 @@ +/* + * Octeon Crypto for OCF + * + * Written by David McCullough <david_mccullough@securecomputing.com> + * Copyright (C) 2009 David McCullough + * + * LICENSE TERMS + * + * The free distribution and use of this software in both source and binary + * form is allowed (with or without changes) provided that: + * + * 1. distributions of this source code include the above copyright + * notice, this list of conditions and the following disclaimer; + * + * 2. distributions in binary form include the above copyright + * notice, this list of conditions and the following disclaimer + * in the documentation and/or other associated materials; + * + * 3. the copyright holder's name is not used to endorse products + * built using this software without specific written permission. + * + * DISCLAIMER + * + * This software is provided 'as is' with no explicit or implied warranties + * in respect of its properties, including, but not limited to, correctness + * and/or fitness for purpose. + * --------------------------------------------------------------------------- + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/module.h> +#include <sys/malloc.h> +#include <sys/mbuf.h> +#include <sys/uio.h> + +#include <opencrypto/cryptodev.h> + +#include <contrib/octeon-sdk/cvmx.h> + +#include <mips/cavium/cryptocteon/cryptocteonvar.h> + +#include "cryptodev_if.h" + +struct cryptocteon_softc { + int32_t sc_cid; /* opencrypto id */ + struct octo_sess **sc_sessions; + uint32_t sc_sesnum; +}; + +int cryptocteon_debug = 0; +TUNABLE_INT("hw.cryptocteon.debug", &cryptocteon_debug); + +static void cryptocteon_identify(driver_t *, device_t); +static int cryptocteon_probe(device_t); +static int cryptocteon_attach(device_t); + +static int cryptocteon_process(device_t, struct cryptop *, int); +static int cryptocteon_newsession(device_t, u_int32_t *, struct cryptoini *); +static int cryptocteon_freesession(device_t, u_int64_t); + +static void +cryptocteon_identify(driver_t *drv, device_t parent) +{ + if (octeon_has_feature(OCTEON_FEATURE_CRYPTO)) + BUS_ADD_CHILD(parent, 0, "cryptocteon", 0); +} + +static int +cryptocteon_probe(device_t dev) +{ + device_set_desc(dev, "Octeon Secure Coprocessor"); + return (0); +} + +static int +cryptocteon_attach(device_t dev) +{ + struct cryptocteon_softc *sc; + + sc = device_get_softc(dev); + + sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC); + if (sc->sc_cid < 0) { + device_printf(dev, "crypto_get_driverid ret %d\n", sc->sc_cid); + return (ENXIO); + } + + crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); + crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); + crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); + crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); + crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); + + return (0); +} + +/* + * Generate a new octo session. We artifically limit it to a single + * hash/cipher or hash-cipher combo just to make it easier, most callers + * do not expect more than this anyway. + */ +static int +cryptocteon_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) +{ + struct cryptoini *c, *encini = NULL, *macini = NULL; + struct cryptocteon_softc *sc; + struct octo_sess **ocd; + int i; + + sc = device_get_softc(dev); + + if (sid == NULL || cri == NULL || sc == NULL) + return (EINVAL); + + /* + * To keep it simple, we only handle hash, cipher or hash/cipher in a + * session, you cannot currently do multiple ciphers/hashes in one + * session even though it would be possibel to code this driver to + * handle it. + */ + for (i = 0, c = cri; c && i < 2; i++) { + if (c->cri_alg == CRYPTO_MD5_HMAC || + c->cri_alg == CRYPTO_SHA1_HMAC || + c->cri_alg == CRYPTO_NULL_HMAC) { + if (macini) { + break; + } + macini = c; + } + if (c->cri_alg == CRYPTO_DES_CBC || + c->cri_alg == CRYPTO_3DES_CBC || + c->cri_alg == CRYPTO_AES_CBC || + c->cri_alg == CRYPTO_NULL_CBC) { + if (encini) { + break; + } + encini = c; + } + c = c->cri_next; + } + if (!macini && !encini) { + dprintf("%s,%d - EINVAL bad cipher/hash or combination\n", + __FILE__, __LINE__); + return EINVAL; + } + if (c) { + dprintf("%s,%d - EINVAL cannot handle chained cipher/hash combos\n", + __FILE__, __LINE__); + return EINVAL; + } + + /* + * So we have something we can do, lets setup the session + */ + + if (sc->sc_sessions) { + for (i = 1; i < sc->sc_sesnum; i++) + if (sc->sc_sessions[i] == NULL) + break; + } else + i = 1; /* NB: to silence compiler warning */ + + if (sc->sc_sessions == NULL || i == sc->sc_sesnum) { + if (sc->sc_sessions == NULL) { + i = 1; /* We leave sc->sc_sessions[0] empty */ + sc->sc_sesnum = CRYPTO_SW_SESSIONS; + } else + sc->sc_sesnum *= 2; + + ocd = malloc(sc->sc_sesnum * sizeof(struct octo_sess *), + M_DEVBUF, M_NOWAIT | M_ZERO); + if (ocd == NULL) { + /* Reset session number */ + if (sc->sc_sesnum == CRYPTO_SW_SESSIONS) + sc->sc_sesnum = 0; + else + sc->sc_sesnum /= 2; + dprintf("%s,%d: ENOBUFS\n", __FILE__, __LINE__); + return ENOBUFS; + } + + /* Copy existing sessions */ + if (sc->sc_sessions) { + memcpy(ocd, sc->sc_sessions, + (sc->sc_sesnum / 2) * sizeof(struct octo_sess *)); + free(sc->sc_sessions, M_DEVBUF); + } + + sc->sc_sessions = ocd; + } + + ocd = &sc->sc_sessions[i]; + *sid = i; + + *ocd = malloc(sizeof(struct octo_sess), M_DEVBUF, M_NOWAIT | M_ZERO); + if (*ocd == NULL) { + cryptocteon_freesession(NULL, i); + dprintf("%s,%d: ENOBUFS\n", __FILE__, __LINE__); + return ENOBUFS; + } + + if (encini && encini->cri_key) { + (*ocd)->octo_encklen = (encini->cri_klen + 7) / 8; + memcpy((*ocd)->octo_enckey, encini->cri_key, (*ocd)->octo_encklen); + } + + if (macini && macini->cri_key) { + (*ocd)->octo_macklen = (macini->cri_klen + 7) / 8; + memcpy((*ocd)->octo_mackey, macini->cri_key, (*ocd)->octo_macklen); + } + + (*ocd)->octo_mlen = 0; + if (encini && encini->cri_mlen) + (*ocd)->octo_mlen = encini->cri_mlen; + else if (macini && macini->cri_mlen) + (*ocd)->octo_mlen = macini->cri_mlen; + else + (*ocd)->octo_mlen = 12; + + /* + * point c at the enc if it exists, otherwise the mac + */ + c = encini ? encini : macini; + + switch (c->cri_alg) { + case CRYPTO_DES_CBC: + case CRYPTO_3DES_CBC: + (*ocd)->octo_ivsize = 8; + switch (macini ? macini->cri_alg : -1) { + case CRYPTO_MD5_HMAC: + (*ocd)->octo_encrypt = octo_des_cbc_md5_encrypt; + (*ocd)->octo_decrypt = octo_des_cbc_md5_decrypt; + octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + case CRYPTO_SHA1_HMAC: + (*ocd)->octo_encrypt = octo_des_cbc_sha1_encrypt; + (*ocd)->octo_decrypt = octo_des_cbc_sha1_encrypt; + octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + case -1: + (*ocd)->octo_encrypt = octo_des_cbc_encrypt; + (*ocd)->octo_decrypt = octo_des_cbc_decrypt; + break; + default: + cryptocteon_freesession(NULL, i); + dprintf("%s,%d: EINVALn", __FILE__, __LINE__); + return EINVAL; + } + break; + case CRYPTO_AES_CBC: + (*ocd)->octo_ivsize = 16; + switch (macini ? macini->cri_alg : -1) { + case CRYPTO_MD5_HMAC: + (*ocd)->octo_encrypt = octo_aes_cbc_md5_encrypt; + (*ocd)->octo_decrypt = octo_aes_cbc_md5_decrypt; + octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + case CRYPTO_SHA1_HMAC: + (*ocd)->octo_encrypt = octo_aes_cbc_sha1_encrypt; + (*ocd)->octo_decrypt = octo_aes_cbc_sha1_decrypt; + octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + case -1: + (*ocd)->octo_encrypt = octo_aes_cbc_encrypt; + (*ocd)->octo_decrypt = octo_aes_cbc_decrypt; + break; + default: + cryptocteon_freesession(NULL, i); + dprintf("%s,%d: EINVALn", __FILE__, __LINE__); + return EINVAL; + } + break; + case CRYPTO_MD5_HMAC: + (*ocd)->octo_encrypt = octo_null_md5_encrypt; + (*ocd)->octo_decrypt = octo_null_md5_encrypt; + octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + case CRYPTO_SHA1_HMAC: + (*ocd)->octo_encrypt = octo_null_sha1_encrypt; + (*ocd)->octo_decrypt = octo_null_sha1_encrypt; + octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + default: + cryptocteon_freesession(NULL, i); + dprintf("%s,%d: EINVALn", __FILE__, __LINE__); + return EINVAL; + } + + (*ocd)->octo_encalg = encini ? encini->cri_alg : -1; + (*ocd)->octo_macalg = macini ? macini->cri_alg : -1; + + return 0; +} + +/* + * Free a session. + */ +static int +cryptocteon_freesession(device_t dev, u_int64_t tid) +{ + struct cryptocteon_softc *sc; + u_int32_t sid = CRYPTO_SESID2LID(tid); + + sc = device_get_softc(dev); + + if (sc == NULL) + return (EINVAL); + + if (sid > sc->sc_sesnum || sc->sc_sessions == NULL || + sc->sc_sessions[sid] == NULL) + return (EINVAL); + + /* Silently accept and return */ + if (sid == 0) + return(0); + + if (sc->sc_sessions[sid]) + free(sc->sc_sessions[sid], M_DEVBUF); + sc->sc_sessions[sid] = NULL; + return 0; +} + +/* + * Process a request. + */ +static int +cryptocteon_process(device_t dev, struct cryptop *crp, int hint) +{ + struct cryptodesc *crd; + struct octo_sess *od; + u_int32_t lid; + size_t iovcnt, iovlen; + struct mbuf *m = NULL; + struct uio *uiop = NULL; + struct cryptodesc *enccrd = NULL, *maccrd = NULL; + unsigned char *ivp = NULL; + unsigned char iv_data[HASH_MAX_LEN]; + int auth_off = 0, auth_len = 0, crypt_off = 0, crypt_len = 0, icv_off = 0; + struct cryptocteon_softc *sc; + + sc = device_get_softc(dev); + + if (sc == NULL || crp == NULL) + return EINVAL; + + crp->crp_etype = 0; + + if (crp->crp_desc == NULL || crp->crp_buf == NULL) { + dprintf("%s,%d: EINVAL\n", __FILE__, __LINE__); + crp->crp_etype = EINVAL; + goto done; + } + + lid = crp->crp_sid & 0xffffffff; + if (lid >= sc->sc_sesnum || lid == 0 || sc->sc_sessions == NULL || + sc->sc_sessions[lid] == NULL) { + crp->crp_etype = ENOENT; + dprintf("%s,%d: ENOENT\n", __FILE__, __LINE__); + goto done; + } + od = sc->sc_sessions[lid]; + + /* + * do some error checking outside of the loop for m and IOV processing + * this leaves us with valid m or uiop pointers for later + */ + if (crp->crp_flags & CRYPTO_F_IMBUF) { + unsigned frags; + + m = (struct mbuf *) crp->crp_buf; + for (frags = 0; m != NULL; frags++) + m = m->m_next; + + if (frags >= UIO_MAXIOV) { + printf("%s,%d: %d frags > UIO_MAXIOV", __FILE__, __LINE__, frags); + goto done; + } + + m = (struct mbuf *) crp->crp_buf; + } else if (crp->crp_flags & CRYPTO_F_IOV) { + uiop = (struct uio *) crp->crp_buf; + if (uiop->uio_iovcnt > UIO_MAXIOV) { + printf("%s,%d: %d uio_iovcnt > UIO_MAXIOV", __FILE__, __LINE__, + uiop->uio_iovcnt); + goto done; + } + } + + /* point our enccrd and maccrd appropriately */ + crd = crp->crp_desc; + if (crd->crd_alg == od->octo_encalg) enccrd = crd; + if (crd->crd_alg == od->octo_macalg) maccrd = crd; + crd = crd->crd_next; + if (crd) { + if (crd->crd_alg == od->octo_encalg) enccrd = crd; + if (crd->crd_alg == od->octo_macalg) maccrd = crd; + crd = crd->crd_next; + } + if (crd) { + crp->crp_etype = EINVAL; + dprintf("%s,%d: ENOENT - descriptors do not match session\n", + __FILE__, __LINE__); + goto done; + } + + if (enccrd) { + if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) { + ivp = enccrd->crd_iv; + } else { + ivp = iv_data; + crypto_copydata(crp->crp_flags, crp->crp_buf, + enccrd->crd_inject, od->octo_ivsize, (caddr_t) ivp); + } + + if (maccrd) { + auth_off = maccrd->crd_skip; + auth_len = maccrd->crd_len; + icv_off = maccrd->crd_inject; + } + + crypt_off = enccrd->crd_skip; + crypt_len = enccrd->crd_len; + } else { /* if (maccrd) */ + auth_off = maccrd->crd_skip; + auth_len = maccrd->crd_len; + icv_off = maccrd->crd_inject; + } + + /* + * setup the I/O vector to cover the buffer + */ + if (crp->crp_flags & CRYPTO_F_IMBUF) { + iovcnt = 0; + iovlen = 0; + + while (m != NULL) { + od->octo_iov[iovcnt].iov_base = mtod(m, void *); + od->octo_iov[iovcnt].iov_len = m->m_len; + + m = m->m_next; + iovlen += od->octo_iov[iovcnt++].iov_len; + } + } else if (crp->crp_flags & CRYPTO_F_IOV) { + iovlen = 0; + for (iovcnt = 0; iovcnt < uiop->uio_iovcnt; iovcnt++) { + od->octo_iov[iovcnt].iov_base = uiop->uio_iov[iovcnt].iov_base; + od->octo_iov[iovcnt].iov_len = uiop->uio_iov[iovcnt].iov_len; + + iovlen += od->octo_iov[iovcnt].iov_len; + } + } else { + iovlen = crp->crp_ilen; + od->octo_iov[0].iov_base = crp->crp_buf; + od->octo_iov[0].iov_len = crp->crp_ilen; + iovcnt = 1; + } + + + /* + * setup a new explicit key + */ + if (enccrd) { + if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) { + od->octo_encklen = (enccrd->crd_klen + 7) / 8; + memcpy(od->octo_enckey, enccrd->crd_key, od->octo_encklen); + } + } + if (maccrd) { + if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) { + od->octo_macklen = (maccrd->crd_klen + 7) / 8; + memcpy(od->octo_mackey, maccrd->crd_key, od->octo_macklen); + od->octo_mackey_set = 0; + } + if (!od->octo_mackey_set) { + octo_calc_hash(maccrd->crd_alg == CRYPTO_MD5_HMAC ? 0 : 1, + maccrd->crd_key, od->octo_hminner, od->octo_hmouter); + od->octo_mackey_set = 1; + } + } + + + if (!enccrd || (enccrd->crd_flags & CRD_F_ENCRYPT)) + (*od->octo_encrypt)(od, od->octo_iov, iovcnt, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + else + (*od->octo_decrypt)(od, od->octo_iov, iovcnt, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + +done: + crypto_done(crp); + return 0; +} + +static device_method_t cryptocteon_methods[] = { + /* device methods */ + DEVMETHOD(device_identify, cryptocteon_identify), + DEVMETHOD(device_probe, cryptocteon_probe), + DEVMETHOD(device_attach, cryptocteon_attach), + + /* crypto device methods */ + DEVMETHOD(cryptodev_newsession, cryptocteon_newsession), + DEVMETHOD(cryptodev_freesession,cryptocteon_freesession), + DEVMETHOD(cryptodev_process, cryptocteon_process), + + { 0, 0 } +}; + +static driver_t cryptocteon_driver = { + "cryptocteon", + cryptocteon_methods, + sizeof (struct cryptocteon_softc), +}; +static devclass_t cryptocteon_devclass; +DRIVER_MODULE(cryptocteon, nexus, cryptocteon_driver, cryptocteon_devclass, 0, 0); diff --git a/sys/mips/cavium/cryptocteon/cryptocteonvar.h b/sys/mips/cavium/cryptocteon/cryptocteonvar.h new file mode 100644 index 0000000..e722298 --- /dev/null +++ b/sys/mips/cavium/cryptocteon/cryptocteonvar.h @@ -0,0 +1,94 @@ +/* + * Octeon Crypto for OCF + * + * Written by David McCullough <david_mccullough@securecomputing.com> + * Copyright (C) 2009 David McCullough + * + * LICENSE TERMS + * + * The free distribution and use of this software in both source and binary + * form is allowed (with or without changes) provided that: + * + * 1. distributions of this source code include the above copyright + * notice, this list of conditions and the following disclaimer; + * + * 2. distributions in binary form include the above copyright + * notice, this list of conditions and the following disclaimer + * in the documentation and/or other associated materials; + * + * 3. the copyright holder's name is not used to endorse products + * built using this software without specific written permission. + * + * DISCLAIMER + * + * This software is provided 'as is' with no explicit or implied warranties + * in respect of its properties, including, but not limited to, correctness + * and/or fitness for purpose. + * --------------------------------------------------------------------------- + * + * $FreeBSD$ + */ + +#ifndef _MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_ +#define _MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_ + +struct octo_sess; + +typedef int octo_encrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp); +typedef int octo_decrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp); + +struct octo_sess { + int octo_encalg; + #define MAX_CIPHER_KEYLEN 64 + char octo_enckey[MAX_CIPHER_KEYLEN]; + int octo_encklen; + + int octo_macalg; + #define MAX_HASH_KEYLEN 64 + char octo_mackey[MAX_HASH_KEYLEN]; + int octo_macklen; + int octo_mackey_set; + + int octo_mlen; + int octo_ivsize; + + octo_encrypt_t *octo_encrypt; + octo_decrypt_t *octo_decrypt; + + uint64_t octo_hminner[3]; + uint64_t octo_hmouter[3]; + + struct iovec octo_iov[UIO_MAXIOV]; +}; + +#define dprintf(fmt, ...) \ + do { \ + if (cryptocteon_debug) \ + printf("%s: " fmt, __func__, ## __VA_ARGS__); \ + } while (0) + +extern int cryptocteon_debug; + +void octo_calc_hash(uint8_t, unsigned char *, uint64_t *, uint64_t *); + +/* XXX Actually just hashing functions, not encryption. */ +octo_encrypt_t octo_null_md5_encrypt; +octo_encrypt_t octo_null_sha1_encrypt; + +octo_encrypt_t octo_des_cbc_encrypt; +octo_encrypt_t octo_des_cbc_md5_encrypt; +octo_encrypt_t octo_des_cbc_sha1_encrypt; + +octo_decrypt_t octo_des_cbc_decrypt; +octo_decrypt_t octo_des_cbc_md5_decrypt; +octo_decrypt_t octo_des_cbc_sha1_decrypt; + +octo_encrypt_t octo_aes_cbc_encrypt; +octo_encrypt_t octo_aes_cbc_md5_encrypt; +octo_encrypt_t octo_aes_cbc_sha1_encrypt; + +octo_decrypt_t octo_aes_cbc_decrypt; +octo_decrypt_t octo_aes_cbc_md5_decrypt; +octo_decrypt_t octo_aes_cbc_sha1_decrypt; + +#endif /* !_MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_ */ |
