diff options
| -rw-r--r-- | sys/mips/cavium/cryptocteon/cavium_crypto.c | 2222 | ||||
| -rw-r--r-- | sys/mips/cavium/cryptocteon/cryptocteon.c | 526 | ||||
| -rw-r--r-- | sys/mips/cavium/cryptocteon/cryptocteonvar.h | 94 | ||||
| -rw-r--r-- | sys/mips/cavium/files.octeon1 | 8 | ||||
| -rw-r--r-- | sys/mips/cavium/usb/octusb.c | 1922 | ||||
| -rw-r--r-- | sys/mips/cavium/usb/octusb.h | 147 | ||||
| -rw-r--r-- | sys/mips/cavium/usb/octusb_octeon.c | 223 |
7 files changed, 5142 insertions, 0 deletions
diff --git a/sys/mips/cavium/cryptocteon/cavium_crypto.c b/sys/mips/cavium/cryptocteon/cavium_crypto.c new file mode 100644 index 0000000..ff77a61 --- /dev/null +++ b/sys/mips/cavium/cryptocteon/cavium_crypto.c @@ -0,0 +1,2222 @@ +/* + * vim:sw=4 ts=8 + */ +/* + * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com> + * + * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by Cavium Networks + * 4. Cavium Networks' name may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * This Software, including technical data, may be subject to U.S. export + * control laws, including the U.S. Export Administration Act and its + * associated regulations, and may be subject to export or import regulations + * in other countries. You warrant that You will comply strictly in all + * respects with all such regulations and acknowledge that you have the + * responsibility to obtain licenses to export, re-export or import the + * Software. + * + * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND + * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, + * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE + * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR + * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM + * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, + * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF + * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR + * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR + * PERFORMANCE OF THE SOFTWARE LIES WITH YOU. +*/ +/****************************************************************************/ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/module.h> +#include <sys/malloc.h> +#include <sys/uio.h> + +#include <opencrypto/cryptodev.h> + +#include <contrib/octeon-sdk/cvmx.h> + +#include <mips/cavium/cryptocteon/cryptocteonvar.h> + +/****************************************************************************/ + +#define IOV_INIT(iov, ptr, idx, len) \ + do { \ + (idx) = 0; \ + (ptr) = (iov)[(idx)].iov_base; \ + (len) = (iov)[(idx)].iov_len; \ + } while (0) + +/* + * XXX + * It would be better if this were an IOV_READ/IOV_WRITE macro instead so + * that we could detect overflow before it happens rather than right after, + * which is especially bad since there is usually no IOV_CONSUME after the + * final read or write. + */ +#define IOV_CONSUME(iov, ptr, idx, len) \ + do { \ + if ((len) > sizeof *(ptr)) { \ + (len) -= sizeof *(ptr); \ + (ptr)++; \ + } else { \ + if ((len) != sizeof *(ptr)) \ + panic("%s: went past end of iovec.", __func__); \ + (idx)++; \ + (ptr) = (iov)[(idx)].iov_base; \ + (len) = (iov)[(idx)].iov_len; \ + } \ + } while (0) + +static inline unsigned long octeon_crypto_enable(void) +{ + register_t s; + + s = intr_disable(); + mips_wr_status(mips_rd_status() | MIPS_SR_COP_2_BIT); + + return (s); +} + +static inline void octeon_crypto_disable(register_t s) +{ + mips_wr_status(mips_rd_status() & ~MIPS_SR_COP_2_BIT); + intr_restore(s); +} + +#define ESP_HEADER_LENGTH 8 +#define DES_CBC_IV_LENGTH 8 +#define AES_CBC_IV_LENGTH 16 +#define ESP_HMAC_LEN 12 + +#define ESP_HEADER_LENGTH 8 +#define DES_CBC_IV_LENGTH 8 + +/****************************************************************************/ + +#define CVM_LOAD_SHA_UNIT(dat, next) { \ + if (next == 0) { \ + next = 1; \ + CVMX_MT_HSH_DAT (dat, 0); \ + } else if (next == 1) { \ + next = 2; \ + CVMX_MT_HSH_DAT (dat, 1); \ + } else if (next == 2) { \ + next = 3; \ + CVMX_MT_HSH_DAT (dat, 2); \ + } else if (next == 3) { \ + next = 4; \ + CVMX_MT_HSH_DAT (dat, 3); \ + } else if (next == 4) { \ + next = 5; \ + CVMX_MT_HSH_DAT (dat, 4); \ + } else if (next == 5) { \ + next = 6; \ + CVMX_MT_HSH_DAT (dat, 5); \ + } else if (next == 6) { \ + next = 7; \ + CVMX_MT_HSH_DAT (dat, 6); \ + } else { \ + CVMX_MT_HSH_STARTSHA (dat); \ + next = 0; \ + } \ +} + +#define CVM_LOAD2_SHA_UNIT(dat1, dat2, next) { \ + if (next == 0) { \ + CVMX_MT_HSH_DAT (dat1, 0); \ + CVMX_MT_HSH_DAT (dat2, 1); \ + next = 2; \ + } else if (next == 1) { \ + CVMX_MT_HSH_DAT (dat1, 1); \ + CVMX_MT_HSH_DAT (dat2, 2); \ + next = 3; \ + } else if (next == 2) { \ + CVMX_MT_HSH_DAT (dat1, 2); \ + CVMX_MT_HSH_DAT (dat2, 3); \ + next = 4; \ + } else if (next == 3) { \ + CVMX_MT_HSH_DAT (dat1, 3); \ + CVMX_MT_HSH_DAT (dat2, 4); \ + next = 5; \ + } else if (next == 4) { \ + CVMX_MT_HSH_DAT (dat1, 4); \ + CVMX_MT_HSH_DAT (dat2, 5); \ + next = 6; \ + } else if (next == 5) { \ + CVMX_MT_HSH_DAT (dat1, 5); \ + CVMX_MT_HSH_DAT (dat2, 6); \ + next = 7; \ + } else if (next == 6) { \ + CVMX_MT_HSH_DAT (dat1, 6); \ + CVMX_MT_HSH_STARTSHA (dat2); \ + next = 0; \ + } else { \ + CVMX_MT_HSH_STARTSHA (dat1); \ + CVMX_MT_HSH_DAT (dat2, 0); \ + next = 1; \ + } \ +} + +/****************************************************************************/ + +#define CVM_LOAD_MD5_UNIT(dat, next) { \ + if (next == 0) { \ + next = 1; \ + CVMX_MT_HSH_DAT (dat, 0); \ + } else if (next == 1) { \ + next = 2; \ + CVMX_MT_HSH_DAT (dat, 1); \ + } else if (next == 2) { \ + next = 3; \ + CVMX_MT_HSH_DAT (dat, 2); \ + } else if (next == 3) { \ + next = 4; \ + CVMX_MT_HSH_DAT (dat, 3); \ + } else if (next == 4) { \ + next = 5; \ + CVMX_MT_HSH_DAT (dat, 4); \ + } else if (next == 5) { \ + next = 6; \ + CVMX_MT_HSH_DAT (dat, 5); \ + } else if (next == 6) { \ + next = 7; \ + CVMX_MT_HSH_DAT (dat, 6); \ + } else { \ + CVMX_MT_HSH_STARTMD5 (dat); \ + next = 0; \ + } \ +} + +#define CVM_LOAD2_MD5_UNIT(dat1, dat2, next) { \ + if (next == 0) { \ + CVMX_MT_HSH_DAT (dat1, 0); \ + CVMX_MT_HSH_DAT (dat2, 1); \ + next = 2; \ + } else if (next == 1) { \ + CVMX_MT_HSH_DAT (dat1, 1); \ + CVMX_MT_HSH_DAT (dat2, 2); \ + next = 3; \ + } else if (next == 2) { \ + CVMX_MT_HSH_DAT (dat1, 2); \ + CVMX_MT_HSH_DAT (dat2, 3); \ + next = 4; \ + } else if (next == 3) { \ + CVMX_MT_HSH_DAT (dat1, 3); \ + CVMX_MT_HSH_DAT (dat2, 4); \ + next = 5; \ + } else if (next == 4) { \ + CVMX_MT_HSH_DAT (dat1, 4); \ + CVMX_MT_HSH_DAT (dat2, 5); \ + next = 6; \ + } else if (next == 5) { \ + CVMX_MT_HSH_DAT (dat1, 5); \ + CVMX_MT_HSH_DAT (dat2, 6); \ + next = 7; \ + } else if (next == 6) { \ + CVMX_MT_HSH_DAT (dat1, 6); \ + CVMX_MT_HSH_STARTMD5 (dat2); \ + next = 0; \ + } else { \ + CVMX_MT_HSH_STARTMD5 (dat1); \ + CVMX_MT_HSH_DAT (dat2, 0); \ + next = 1; \ + } \ +} + +/****************************************************************************/ + +void +octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer) +{ + uint8_t hash_key[64]; + uint64_t *key1; + register uint64_t xor1 = 0x3636363636363636ULL; + register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL; + register_t s; + + dprintf("%s()\n", __func__); + + memset(hash_key, 0, sizeof(hash_key)); + memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); + key1 = (uint64_t *) hash_key; + s = octeon_crypto_enable(); + if (auth) { + CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); + CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); + CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); + } else { + CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); + CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); + } + + CVMX_MT_HSH_DAT((*key1 ^ xor1), 0); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 1); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 2); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 3); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 4); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 5); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor1), 6); + key1++; + if (auth) + CVMX_MT_HSH_STARTSHA((*key1 ^ xor1)); + else + CVMX_MT_HSH_STARTMD5((*key1 ^ xor1)); + + CVMX_MF_HSH_IV(inner[0], 0); + CVMX_MF_HSH_IV(inner[1], 1); + if (auth) { + inner[2] = 0; + CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2); + } + + memset(hash_key, 0, sizeof(hash_key)); + memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); + key1 = (uint64_t *) hash_key; + if (auth) { + CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); + CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); + CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); + } else { + CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); + CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); + } + + CVMX_MT_HSH_DAT((*key1 ^ xor2), 0); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 1); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 2); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 3); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 4); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 5); + key1++; + CVMX_MT_HSH_DAT((*key1 ^ xor2), 6); + key1++; + if (auth) + CVMX_MT_HSH_STARTSHA((*key1 ^ xor2)); + else + CVMX_MT_HSH_STARTMD5((*key1 ^ xor2)); + + CVMX_MF_HSH_IV(outer[0], 0); + CVMX_MF_HSH_IV(outer[1], 1); + if (auth) { + outer[2] = 0; + CVMX_MF_HSH_IV(outer[2], 2); + } + octeon_crypto_disable(s); + return; +} + +/****************************************************************************/ +/* DES functions */ + +int +octo_des_cbc_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + uint64_t *data; + int data_i, data_l; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + while (crypt_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + crypt_off -= 8; + } + + while (crypt_len > 0) { + CVMX_MT_3DES_ENC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + IOV_CONSUME(iov, data, data_i, data_l); + crypt_len -= 8; + } + + octeon_crypto_disable(s); + return 0; +} + + +int +octo_des_cbc_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + uint64_t *data; + int data_i, data_l; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + while (crypt_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + crypt_off -= 8; + } + + while (crypt_len > 0) { + CVMX_MT_3DES_DEC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + IOV_CONSUME(iov, data, data_i, data_l); + crypt_len -= 8; + } + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* AES functions */ + +int +octo_aes_cbc_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + uint64_t *data, *pdata; + int data_i, data_l; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + while (crypt_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + crypt_off -= 8; + } + + while (crypt_len > 0) { + pdata = data; + CVMX_MT_AES_ENC_CBC0(*data); + IOV_CONSUME(iov, data, data_i, data_l); + CVMX_MT_AES_ENC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + IOV_CONSUME(iov, data, data_i, data_l); + crypt_len -= 16; + } + + octeon_crypto_disable(s); + return 0; +} + + +int +octo_aes_cbc_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + uint64_t *data, *pdata; + int data_i, data_l; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + while (crypt_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + crypt_off -= 8; + } + + while (crypt_len > 0) { + pdata = data; + CVMX_MT_AES_DEC_CBC0(*data); + IOV_CONSUME(iov, data, data_i, data_l); + CVMX_MT_AES_DEC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + IOV_CONSUME(iov, data, data_i, data_l); + crypt_len -= 16; + } + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* MD5 */ + +int +octo_null_md5_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + uint64_t *data; + uint64_t tmp1, tmp2; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || + (auth_off & 0x7) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + s = octeon_crypto_enable(); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + + while (auth_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + auth_off -= 8; + } + + while (auth_len > 0) { + CVM_LOAD_MD5_UNIT(*data, next); + auth_len -= 8; + IOV_CONSUME(iov, data, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); + } + CVMX_ES64(tmp1, ((alen + 64) << 3)); + CVM_LOAD_MD5_UNIT(tmp1, next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_ES64(tmp1, ((64 + 16) << 3)); + CVMX_MT_HSH_STARTMD5(tmp1); + + /* save the HMAC */ + IOV_INIT(iov, data, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + icv_off -= 8; + } + CVMX_MF_HSH_IV(*data, 0); + IOV_CONSUME(iov, data, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *(uint32_t *)data = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* SHA1 */ + +int +octo_null_sha1_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + uint64_t *data; + uint64_t tmp1, tmp2, tmp3; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || + (auth_off & 0x7) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data, data_i, data_l); + + s = octeon_crypto_enable(); + + /* Load SHA1 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + CVMX_MT_HSH_IV(od->octo_hminner[2], 2); + + while (auth_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + auth_off -= 8; + } + + while (auth_len > 0) { + CVM_LOAD_SHA_UNIT(*data, next); + auth_len -= 8; + IOV_CONSUME(iov, data, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); + } + CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + tmp3 = 0; + CVMX_MF_HSH_IV(tmp3, 2); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + tmp3 |= 0x0000000080000000; + CVMX_MT_HSH_DAT(tmp3, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); + + /* save the HMAC */ + IOV_INIT(iov, data, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data, data_i, data_l); + icv_off -= 8; + } + CVMX_MF_HSH_IV(*data, 0); + IOV_CONSUME(iov, data, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *(uint32_t *)data = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* DES MD5 */ + +int +octo_des_cbc_md5_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata; + uint64_t *data = &mydata.data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *first = data32; + mydata.data32[0] = *first; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata.data32[1] = *data32; + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_3DES_ENC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + crypt_len -= 8; + } + } else + crypt_off -= 8; + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_MD5_UNIT(*data, next); + auth_len -= 8; + } + } else + auth_off -= 8; + *first = mydata.data32[0]; + *data32 = mydata.data32[1]; + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); + } + CVMX_ES64(tmp1, ((alen + 64) << 3)); + CVM_LOAD_MD5_UNIT(tmp1, next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_ES64(tmp1, ((64 + 16) << 3)); + CVMX_MT_HSH_STARTMD5(tmp1); + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +int +octo_des_cbc_md5_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata; + uint64_t *data = &mydata.data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *first = data32; + mydata.data32[0] = *first; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata.data32[1] = *data32; + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_MD5_UNIT(*data, next); + auth_len -= 8; + } + } else + auth_off -= 8; + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_3DES_DEC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + crypt_len -= 8; + } + } else + crypt_off -= 8; + *first = mydata.data32[0]; + *data32 = mydata.data32[1]; + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); + } + CVMX_ES64(tmp1, ((alen + 64) << 3)); + CVM_LOAD_MD5_UNIT(tmp1, next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_ES64(tmp1, ((64 + 16) << 3)); + CVMX_MT_HSH_STARTMD5(tmp1); + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* DES SHA */ + +int +octo_des_cbc_sha1_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata; + uint64_t *data = &mydata.data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2, tmp3; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + /* Load SHA1 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + CVMX_MT_HSH_IV(od->octo_hminner[2], 2); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *first = data32; + mydata.data32[0] = *first; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata.data32[1] = *data32; + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_3DES_ENC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + crypt_len -= 8; + } + } else + crypt_off -= 8; + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_SHA_UNIT(*data, next); + auth_len -= 8; + } + } else + auth_off -= 8; + *first = mydata.data32[0]; + *data32 = mydata.data32[1]; + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_SHA_UNIT(tmp, next); + } else { + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); + } + CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + tmp3 = 0; + CVMX_MF_HSH_IV(tmp3, 2); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + tmp3 |= 0x0000000080000000; + CVMX_MT_HSH_DAT(tmp3, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +int +octo_des_cbc_sha1_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata; + uint64_t *data = &mydata.data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2, tmp3; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load 3DES Key */ + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + if (od->octo_encklen == 24) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + } else if (od->octo_encklen == 8) { + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); + CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + + CVMX_MT_3DES_IV(* (uint64_t *) ivp); + + /* Load SHA1 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + CVMX_MT_HSH_IV(od->octo_hminner[2], 2); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *first = data32; + mydata.data32[0] = *first; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata.data32[1] = *data32; + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_SHA_UNIT(*data, next); + auth_len -= 8; + } + } else + auth_off -= 8; + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_3DES_DEC_CBC(*data); + CVMX_MF_3DES_RESULT(*data); + crypt_len -= 8; + } + } else + crypt_off -= 8; + *first = mydata.data32[0]; + *data32 = mydata.data32[1]; + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_SHA_UNIT(tmp, next); + } else { + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); + } + CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + tmp3 = 0; + CVMX_MF_HSH_IV(tmp3, 2); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + tmp3 |= 0x0000000080000000; + CVMX_MT_HSH_DAT(tmp3, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* AES MD5 */ + +int +octo_aes_cbc_md5_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata[2]; + uint64_t *pdata = &mydata[0].data64[0]; + uint64_t *data = &mydata[1].data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *pdata32[3]; + + pdata32[0] = data32; + mydata[0].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + + pdata32[1] = data32; + mydata[0].data32[1] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + + pdata32[2] = data32; + mydata[1].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + + mydata[1].data32[1] = *data32; + + + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_AES_ENC_CBC0(*pdata); + CVMX_MT_AES_ENC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + crypt_len -= 16; + } + } else + crypt_off -= 16; + + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_MD5_UNIT(*pdata, next); + CVM_LOAD_MD5_UNIT(*data, next); + auth_len -= 16; + } + } else + auth_off -= 16; + + *pdata32[0] = mydata[0].data32[0]; + *pdata32[1] = mydata[0].data32[1]; + *pdata32[2] = mydata[1].data32[0]; + *data32 = mydata[1].data32[1]; + + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); + } + CVMX_ES64(tmp1, ((alen + 64) << 3)); + CVM_LOAD_MD5_UNIT(tmp1, next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_ES64(tmp1, ((64 + 16) << 3)); + CVMX_MT_HSH_STARTMD5(tmp1); + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +int +octo_aes_cbc_md5_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata[2]; + uint64_t *pdata = &mydata[0].data64[0]; + uint64_t *data = &mydata[1].data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *pdata32[3]; + + pdata32[0] = data32; + mydata[0].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[1] = data32; + mydata[0].data32[1] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[2] = data32; + mydata[1].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata[1].data32[1] = *data32; + + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_MD5_UNIT(*pdata, next); + CVM_LOAD_MD5_UNIT(*data, next); + auth_len -= 16; + } + } else + auth_off -= 16; + + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_AES_DEC_CBC0(*pdata); + CVMX_MT_AES_DEC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + crypt_len -= 16; + } + } else + crypt_off -= 16; + + *pdata32[0] = mydata[0].data32[0]; + *pdata32[1] = mydata[0].data32[1]; + *pdata32[2] = mydata[1].data32[0]; + *data32 = mydata[1].data32[1]; + + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); + } + CVMX_ES64(tmp1, ((alen + 64) << 3)); + CVM_LOAD_MD5_UNIT(tmp1, next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_ES64(tmp1, ((64 + 16) << 3)); + CVMX_MT_HSH_STARTMD5(tmp1); + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ +/* AES SHA1 */ + +int +octo_aes_cbc_sha1_encrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata[2]; + uint64_t *pdata = &mydata[0].data64[0]; + uint64_t *data = &mydata[1].data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2, tmp3; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + /* Load SHA IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + CVMX_MT_HSH_IV(od->octo_hminner[2], 2); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *pdata32[3]; + + pdata32[0] = data32; + mydata[0].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[1] = data32; + mydata[0].data32[1] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[2] = data32; + mydata[1].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata[1].data32[1] = *data32; + + + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_AES_ENC_CBC0(*pdata); + CVMX_MT_AES_ENC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + crypt_len -= 16; + } + } else + crypt_off -= 16; + + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_SHA_UNIT(*pdata, next); + CVM_LOAD_SHA_UNIT(*data, next); + auth_len -= 16; + } + } else + auth_off -= 16; + + *pdata32[0] = mydata[0].data32[0]; + *pdata32[1] = mydata[0].data32[1]; + *pdata32[2] = mydata[1].data32[0]; + *data32 = mydata[1].data32[1]; + + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_SHA_UNIT(tmp, next); + } else { + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); + } + CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + tmp3 = 0; + CVMX_MF_HSH_IV(tmp3, 2); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + tmp3 |= 0x0000000080000000; + CVMX_MT_HSH_DAT(tmp3, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +int +octo_aes_cbc_sha1_decrypt( + struct octo_sess *od, + struct iovec *iov, size_t iovcnt, size_t iovlen, + int auth_off, int auth_len, + int crypt_off, int crypt_len, + int icv_off, uint8_t *ivp) +{ + register int next = 0; + union { + uint32_t data32[2]; + uint64_t data64[1]; + } mydata[2]; + uint64_t *pdata = &mydata[0].data64[0]; + uint64_t *data = &mydata[1].data64[0]; + uint32_t *data32; + uint64_t tmp1, tmp2, tmp3; + int data_i, data_l, alen = auth_len; + register_t s; + + dprintf("%s()\n", __func__); + + if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || + (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || + (crypt_len & 0x7) || + (auth_len & 0x7) || + (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { + dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d " + "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " + "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + return -EINVAL; + } + + IOV_INIT(iov, data32, data_i, data_l); + + CVMX_PREFETCH0(ivp); + CVMX_PREFETCH0(od->octo_enckey); + + s = octeon_crypto_enable(); + + /* load AES Key */ + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); + + if (od->octo_encklen == 16) { + CVMX_MT_AES_KEY(0x0, 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 24) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(0x0, 3); + } else if (od->octo_encklen == 32) { + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); + CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); + } else { + octeon_crypto_disable(s); + dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); + return -EINVAL; + } + CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); + + CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); + CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); + + /* Load MD5 IV */ + CVMX_MT_HSH_IV(od->octo_hminner[0], 0); + CVMX_MT_HSH_IV(od->octo_hminner[1], 1); + CVMX_MT_HSH_IV(od->octo_hminner[2], 2); + + while (crypt_off > 0 && auth_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + crypt_off -= 4; + auth_off -= 4; + } + + while (crypt_len > 0 || auth_len > 0) { + uint32_t *pdata32[3]; + + pdata32[0] = data32; + mydata[0].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[1] = data32; + mydata[0].data32[1] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + pdata32[2] = data32; + mydata[1].data32[0] = *data32; + IOV_CONSUME(iov, data32, data_i, data_l); + mydata[1].data32[1] = *data32; + + if (auth_off <= 0) { + if (auth_len > 0) { + CVM_LOAD_SHA_UNIT(*pdata, next); + CVM_LOAD_SHA_UNIT(*data, next); + auth_len -= 16; + } + } else + auth_off -= 16; + + if (crypt_off <= 0) { + if (crypt_len > 0) { + CVMX_MT_AES_DEC_CBC0(*pdata); + CVMX_MT_AES_DEC_CBC1(*data); + CVMX_MF_AES_RESULT(*pdata, 0); + CVMX_MF_AES_RESULT(*data, 1); + crypt_len -= 16; + } + } else + crypt_off -= 16; + + *pdata32[0] = mydata[0].data32[0]; + *pdata32[1] = mydata[0].data32[1]; + *pdata32[2] = mydata[1].data32[0]; + *data32 = mydata[1].data32[1]; + + IOV_CONSUME(iov, data32, data_i, data_l); + } + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_SHA_UNIT(tmp, next); + } else { + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); +#endif + + /* Finish Inner hash */ + while (next != 7) { + CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); + } + CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); + + /* Get the inner hash of HMAC */ + CVMX_MF_HSH_IV(tmp1, 0); + CVMX_MF_HSH_IV(tmp2, 1); + tmp3 = 0; + CVMX_MF_HSH_IV(tmp3, 2); + + /* Initialize hash unit */ + CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); + CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); + CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); + + CVMX_MT_HSH_DAT(tmp1, 0); + CVMX_MT_HSH_DAT(tmp2, 1); + tmp3 |= 0x0000000080000000; + CVMX_MT_HSH_DAT(tmp3, 2); + CVMX_MT_HSH_DATZ(3); + CVMX_MT_HSH_DATZ(4); + CVMX_MT_HSH_DATZ(5); + CVMX_MT_HSH_DATZ(6); + CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); + + /* finish the hash */ + CVMX_PREFETCH0(od->octo_hmouter); +#if 0 + if (__predict_false(inplen)) { + uint64_t tmp = 0; + uint8_t *p = (uint8_t *) & tmp; + p[inplen] = 0x80; + do { + inplen--; + p[inplen] = ((uint8_t *) data)[inplen]; + } while (inplen); + CVM_LOAD_MD5_UNIT(tmp, next); + } else { + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); + } +#else + CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); +#endif + + /* save the HMAC */ + IOV_INIT(iov, data32, data_i, data_l); + while (icv_off > 0) { + IOV_CONSUME(iov, data32, data_i, data_l); + icv_off -= 4; + } + CVMX_MF_HSH_IV(tmp1, 0); + *data32 = (uint32_t) (tmp1 >> 32); + IOV_CONSUME(iov, data32, data_i, data_l); + *data32 = (uint32_t) tmp1; + IOV_CONSUME(iov, data32, data_i, data_l); + CVMX_MF_HSH_IV(tmp1, 1); + *data32 = (uint32_t) (tmp1 >> 32); + + octeon_crypto_disable(s); + return 0; +} + +/****************************************************************************/ diff --git a/sys/mips/cavium/cryptocteon/cryptocteon.c b/sys/mips/cavium/cryptocteon/cryptocteon.c new file mode 100644 index 0000000..16a0bb1 --- /dev/null +++ b/sys/mips/cavium/cryptocteon/cryptocteon.c @@ -0,0 +1,526 @@ +/* + * Octeon Crypto for OCF + * + * Written by David McCullough <david_mccullough@securecomputing.com> + * Copyright (C) 2009 David McCullough + * + * LICENSE TERMS + * + * The free distribution and use of this software in both source and binary + * form is allowed (with or without changes) provided that: + * + * 1. distributions of this source code include the above copyright + * notice, this list of conditions and the following disclaimer; + * + * 2. distributions in binary form include the above copyright + * notice, this list of conditions and the following disclaimer + * in the documentation and/or other associated materials; + * + * 3. the copyright holder's name is not used to endorse products + * built using this software without specific written permission. + * + * DISCLAIMER + * + * This software is provided 'as is' with no explicit or implied warranties + * in respect of its properties, including, but not limited to, correctness + * and/or fitness for purpose. + * --------------------------------------------------------------------------- + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/kernel.h> +#include <sys/module.h> +#include <sys/malloc.h> +#include <sys/mbuf.h> +#include <sys/uio.h> + +#include <opencrypto/cryptodev.h> + +#include <contrib/octeon-sdk/cvmx.h> + +#include <mips/cavium/cryptocteon/cryptocteonvar.h> + +#include "cryptodev_if.h" + +struct cryptocteon_softc { + int32_t sc_cid; /* opencrypto id */ + struct octo_sess **sc_sessions; + uint32_t sc_sesnum; +}; + +int cryptocteon_debug = 0; +TUNABLE_INT("hw.cryptocteon.debug", &cryptocteon_debug); + +static void cryptocteon_identify(driver_t *, device_t); +static int cryptocteon_probe(device_t); +static int cryptocteon_attach(device_t); + +static int cryptocteon_process(device_t, struct cryptop *, int); +static int cryptocteon_newsession(device_t, u_int32_t *, struct cryptoini *); +static int cryptocteon_freesession(device_t, u_int64_t); + +static void +cryptocteon_identify(driver_t *drv, device_t parent) +{ + if (octeon_has_feature(OCTEON_FEATURE_CRYPTO)) + BUS_ADD_CHILD(parent, 0, "cryptocteon", 0); +} + +static int +cryptocteon_probe(device_t dev) +{ + device_set_desc(dev, "Octeon Secure Coprocessor"); + return (0); +} + +static int +cryptocteon_attach(device_t dev) +{ + struct cryptocteon_softc *sc; + + sc = device_get_softc(dev); + + sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC); + if (sc->sc_cid < 0) { + device_printf(dev, "crypto_get_driverid ret %d\n", sc->sc_cid); + return (ENXIO); + } + + crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); + crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); + crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); + crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); + crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); + + return (0); +} + +/* + * Generate a new octo session. We artifically limit it to a single + * hash/cipher or hash-cipher combo just to make it easier, most callers + * do not expect more than this anyway. + */ +static int +cryptocteon_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri) +{ + struct cryptoini *c, *encini = NULL, *macini = NULL; + struct cryptocteon_softc *sc; + struct octo_sess **ocd; + int i; + + sc = device_get_softc(dev); + + if (sid == NULL || cri == NULL || sc == NULL) + return (EINVAL); + + /* + * To keep it simple, we only handle hash, cipher or hash/cipher in a + * session, you cannot currently do multiple ciphers/hashes in one + * session even though it would be possibel to code this driver to + * handle it. + */ + for (i = 0, c = cri; c && i < 2; i++) { + if (c->cri_alg == CRYPTO_MD5_HMAC || + c->cri_alg == CRYPTO_SHA1_HMAC || + c->cri_alg == CRYPTO_NULL_HMAC) { + if (macini) { + break; + } + macini = c; + } + if (c->cri_alg == CRYPTO_DES_CBC || + c->cri_alg == CRYPTO_3DES_CBC || + c->cri_alg == CRYPTO_AES_CBC || + c->cri_alg == CRYPTO_NULL_CBC) { + if (encini) { + break; + } + encini = c; + } + c = c->cri_next; + } + if (!macini && !encini) { + dprintf("%s,%d - EINVAL bad cipher/hash or combination\n", + __FILE__, __LINE__); + return EINVAL; + } + if (c) { + dprintf("%s,%d - EINVAL cannot handle chained cipher/hash combos\n", + __FILE__, __LINE__); + return EINVAL; + } + + /* + * So we have something we can do, lets setup the session + */ + + if (sc->sc_sessions) { + for (i = 1; i < sc->sc_sesnum; i++) + if (sc->sc_sessions[i] == NULL) + break; + } else + i = 1; /* NB: to silence compiler warning */ + + if (sc->sc_sessions == NULL || i == sc->sc_sesnum) { + if (sc->sc_sessions == NULL) { + i = 1; /* We leave sc->sc_sessions[0] empty */ + sc->sc_sesnum = CRYPTO_SW_SESSIONS; + } else + sc->sc_sesnum *= 2; + + ocd = malloc(sc->sc_sesnum * sizeof(struct octo_sess *), + M_DEVBUF, M_NOWAIT | M_ZERO); + if (ocd == NULL) { + /* Reset session number */ + if (sc->sc_sesnum == CRYPTO_SW_SESSIONS) + sc->sc_sesnum = 0; + else + sc->sc_sesnum /= 2; + dprintf("%s,%d: ENOBUFS\n", __FILE__, __LINE__); + return ENOBUFS; + } + + /* Copy existing sessions */ + if (sc->sc_sessions) { + memcpy(ocd, sc->sc_sessions, + (sc->sc_sesnum / 2) * sizeof(struct octo_sess *)); + free(sc->sc_sessions, M_DEVBUF); + } + + sc->sc_sessions = ocd; + } + + ocd = &sc->sc_sessions[i]; + *sid = i; + + *ocd = malloc(sizeof(struct octo_sess), M_DEVBUF, M_NOWAIT | M_ZERO); + if (*ocd == NULL) { + cryptocteon_freesession(NULL, i); + dprintf("%s,%d: ENOBUFS\n", __FILE__, __LINE__); + return ENOBUFS; + } + + if (encini && encini->cri_key) { + (*ocd)->octo_encklen = (encini->cri_klen + 7) / 8; + memcpy((*ocd)->octo_enckey, encini->cri_key, (*ocd)->octo_encklen); + } + + if (macini && macini->cri_key) { + (*ocd)->octo_macklen = (macini->cri_klen + 7) / 8; + memcpy((*ocd)->octo_mackey, macini->cri_key, (*ocd)->octo_macklen); + } + + (*ocd)->octo_mlen = 0; + if (encini && encini->cri_mlen) + (*ocd)->octo_mlen = encini->cri_mlen; + else if (macini && macini->cri_mlen) + (*ocd)->octo_mlen = macini->cri_mlen; + else + (*ocd)->octo_mlen = 12; + + /* + * point c at the enc if it exists, otherwise the mac + */ + c = encini ? encini : macini; + + switch (c->cri_alg) { + case CRYPTO_DES_CBC: + case CRYPTO_3DES_CBC: + (*ocd)->octo_ivsize = 8; + switch (macini ? macini->cri_alg : -1) { + case CRYPTO_MD5_HMAC: + (*ocd)->octo_encrypt = octo_des_cbc_md5_encrypt; + (*ocd)->octo_decrypt = octo_des_cbc_md5_decrypt; + octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + case CRYPTO_SHA1_HMAC: + (*ocd)->octo_encrypt = octo_des_cbc_sha1_encrypt; + (*ocd)->octo_decrypt = octo_des_cbc_sha1_encrypt; + octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + case -1: + (*ocd)->octo_encrypt = octo_des_cbc_encrypt; + (*ocd)->octo_decrypt = octo_des_cbc_decrypt; + break; + default: + cryptocteon_freesession(NULL, i); + dprintf("%s,%d: EINVALn", __FILE__, __LINE__); + return EINVAL; + } + break; + case CRYPTO_AES_CBC: + (*ocd)->octo_ivsize = 16; + switch (macini ? macini->cri_alg : -1) { + case CRYPTO_MD5_HMAC: + (*ocd)->octo_encrypt = octo_aes_cbc_md5_encrypt; + (*ocd)->octo_decrypt = octo_aes_cbc_md5_decrypt; + octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + case CRYPTO_SHA1_HMAC: + (*ocd)->octo_encrypt = octo_aes_cbc_sha1_encrypt; + (*ocd)->octo_decrypt = octo_aes_cbc_sha1_decrypt; + octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + case -1: + (*ocd)->octo_encrypt = octo_aes_cbc_encrypt; + (*ocd)->octo_decrypt = octo_aes_cbc_decrypt; + break; + default: + cryptocteon_freesession(NULL, i); + dprintf("%s,%d: EINVALn", __FILE__, __LINE__); + return EINVAL; + } + break; + case CRYPTO_MD5_HMAC: + (*ocd)->octo_encrypt = octo_null_md5_encrypt; + (*ocd)->octo_decrypt = octo_null_md5_encrypt; + octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + case CRYPTO_SHA1_HMAC: + (*ocd)->octo_encrypt = octo_null_sha1_encrypt; + (*ocd)->octo_decrypt = octo_null_sha1_encrypt; + octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner, + (*ocd)->octo_hmouter); + break; + default: + cryptocteon_freesession(NULL, i); + dprintf("%s,%d: EINVALn", __FILE__, __LINE__); + return EINVAL; + } + + (*ocd)->octo_encalg = encini ? encini->cri_alg : -1; + (*ocd)->octo_macalg = macini ? macini->cri_alg : -1; + + return 0; +} + +/* + * Free a session. + */ +static int +cryptocteon_freesession(device_t dev, u_int64_t tid) +{ + struct cryptocteon_softc *sc; + u_int32_t sid = CRYPTO_SESID2LID(tid); + + sc = device_get_softc(dev); + + if (sc == NULL) + return (EINVAL); + + if (sid > sc->sc_sesnum || sc->sc_sessions == NULL || + sc->sc_sessions[sid] == NULL) + return (EINVAL); + + /* Silently accept and return */ + if (sid == 0) + return(0); + + if (sc->sc_sessions[sid]) + free(sc->sc_sessions[sid], M_DEVBUF); + sc->sc_sessions[sid] = NULL; + return 0; +} + +/* + * Process a request. + */ +static int +cryptocteon_process(device_t dev, struct cryptop *crp, int hint) +{ + struct cryptodesc *crd; + struct octo_sess *od; + u_int32_t lid; + size_t iovcnt, iovlen; + struct mbuf *m = NULL; + struct uio *uiop = NULL; + struct cryptodesc *enccrd = NULL, *maccrd = NULL; + unsigned char *ivp = NULL; + unsigned char iv_data[HASH_MAX_LEN]; + int auth_off = 0, auth_len = 0, crypt_off = 0, crypt_len = 0, icv_off = 0; + struct cryptocteon_softc *sc; + + sc = device_get_softc(dev); + + if (sc == NULL || crp == NULL) + return EINVAL; + + crp->crp_etype = 0; + + if (crp->crp_desc == NULL || crp->crp_buf == NULL) { + dprintf("%s,%d: EINVAL\n", __FILE__, __LINE__); + crp->crp_etype = EINVAL; + goto done; + } + + lid = crp->crp_sid & 0xffffffff; + if (lid >= sc->sc_sesnum || lid == 0 || sc->sc_sessions == NULL || + sc->sc_sessions[lid] == NULL) { + crp->crp_etype = ENOENT; + dprintf("%s,%d: ENOENT\n", __FILE__, __LINE__); + goto done; + } + od = sc->sc_sessions[lid]; + + /* + * do some error checking outside of the loop for m and IOV processing + * this leaves us with valid m or uiop pointers for later + */ + if (crp->crp_flags & CRYPTO_F_IMBUF) { + unsigned frags; + + m = (struct mbuf *) crp->crp_buf; + for (frags = 0; m != NULL; frags++) + m = m->m_next; + + if (frags >= UIO_MAXIOV) { + printf("%s,%d: %d frags > UIO_MAXIOV", __FILE__, __LINE__, frags); + goto done; + } + + m = (struct mbuf *) crp->crp_buf; + } else if (crp->crp_flags & CRYPTO_F_IOV) { + uiop = (struct uio *) crp->crp_buf; + if (uiop->uio_iovcnt > UIO_MAXIOV) { + printf("%s,%d: %d uio_iovcnt > UIO_MAXIOV", __FILE__, __LINE__, + uiop->uio_iovcnt); + goto done; + } + } + + /* point our enccrd and maccrd appropriately */ + crd = crp->crp_desc; + if (crd->crd_alg == od->octo_encalg) enccrd = crd; + if (crd->crd_alg == od->octo_macalg) maccrd = crd; + crd = crd->crd_next; + if (crd) { + if (crd->crd_alg == od->octo_encalg) enccrd = crd; + if (crd->crd_alg == od->octo_macalg) maccrd = crd; + crd = crd->crd_next; + } + if (crd) { + crp->crp_etype = EINVAL; + dprintf("%s,%d: ENOENT - descriptors do not match session\n", + __FILE__, __LINE__); + goto done; + } + + if (enccrd) { + if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) { + ivp = enccrd->crd_iv; + } else { + ivp = iv_data; + crypto_copydata(crp->crp_flags, crp->crp_buf, + enccrd->crd_inject, od->octo_ivsize, (caddr_t) ivp); + } + + if (maccrd) { + auth_off = maccrd->crd_skip; + auth_len = maccrd->crd_len; + icv_off = maccrd->crd_inject; + } + + crypt_off = enccrd->crd_skip; + crypt_len = enccrd->crd_len; + } else { /* if (maccrd) */ + auth_off = maccrd->crd_skip; + auth_len = maccrd->crd_len; + icv_off = maccrd->crd_inject; + } + + /* + * setup the I/O vector to cover the buffer + */ + if (crp->crp_flags & CRYPTO_F_IMBUF) { + iovcnt = 0; + iovlen = 0; + + while (m != NULL) { + od->octo_iov[iovcnt].iov_base = mtod(m, void *); + od->octo_iov[iovcnt].iov_len = m->m_len; + + m = m->m_next; + iovlen += od->octo_iov[iovcnt++].iov_len; + } + } else if (crp->crp_flags & CRYPTO_F_IOV) { + iovlen = 0; + for (iovcnt = 0; iovcnt < uiop->uio_iovcnt; iovcnt++) { + od->octo_iov[iovcnt].iov_base = uiop->uio_iov[iovcnt].iov_base; + od->octo_iov[iovcnt].iov_len = uiop->uio_iov[iovcnt].iov_len; + + iovlen += od->octo_iov[iovcnt].iov_len; + } + } else { + iovlen = crp->crp_ilen; + od->octo_iov[0].iov_base = crp->crp_buf; + od->octo_iov[0].iov_len = crp->crp_ilen; + iovcnt = 1; + } + + + /* + * setup a new explicit key + */ + if (enccrd) { + if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) { + od->octo_encklen = (enccrd->crd_klen + 7) / 8; + memcpy(od->octo_enckey, enccrd->crd_key, od->octo_encklen); + } + } + if (maccrd) { + if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) { + od->octo_macklen = (maccrd->crd_klen + 7) / 8; + memcpy(od->octo_mackey, maccrd->crd_key, od->octo_macklen); + od->octo_mackey_set = 0; + } + if (!od->octo_mackey_set) { + octo_calc_hash(maccrd->crd_alg == CRYPTO_MD5_HMAC ? 0 : 1, + maccrd->crd_key, od->octo_hminner, od->octo_hmouter); + od->octo_mackey_set = 1; + } + } + + + if (!enccrd || (enccrd->crd_flags & CRD_F_ENCRYPT)) + (*od->octo_encrypt)(od, od->octo_iov, iovcnt, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + else + (*od->octo_decrypt)(od, od->octo_iov, iovcnt, iovlen, + auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); + +done: + crypto_done(crp); + return 0; +} + +static device_method_t cryptocteon_methods[] = { + /* device methods */ + DEVMETHOD(device_identify, cryptocteon_identify), + DEVMETHOD(device_probe, cryptocteon_probe), + DEVMETHOD(device_attach, cryptocteon_attach), + + /* crypto device methods */ + DEVMETHOD(cryptodev_newsession, cryptocteon_newsession), + DEVMETHOD(cryptodev_freesession,cryptocteon_freesession), + DEVMETHOD(cryptodev_process, cryptocteon_process), + + { 0, 0 } +}; + +static driver_t cryptocteon_driver = { + "cryptocteon", + cryptocteon_methods, + sizeof (struct cryptocteon_softc), +}; +static devclass_t cryptocteon_devclass; +DRIVER_MODULE(cryptocteon, nexus, cryptocteon_driver, cryptocteon_devclass, 0, 0); diff --git a/sys/mips/cavium/cryptocteon/cryptocteonvar.h b/sys/mips/cavium/cryptocteon/cryptocteonvar.h new file mode 100644 index 0000000..e722298 --- /dev/null +++ b/sys/mips/cavium/cryptocteon/cryptocteonvar.h @@ -0,0 +1,94 @@ +/* + * Octeon Crypto for OCF + * + * Written by David McCullough <david_mccullough@securecomputing.com> + * Copyright (C) 2009 David McCullough + * + * LICENSE TERMS + * + * The free distribution and use of this software in both source and binary + * form is allowed (with or without changes) provided that: + * + * 1. distributions of this source code include the above copyright + * notice, this list of conditions and the following disclaimer; + * + * 2. distributions in binary form include the above copyright + * notice, this list of conditions and the following disclaimer + * in the documentation and/or other associated materials; + * + * 3. the copyright holder's name is not used to endorse products + * built using this software without specific written permission. + * + * DISCLAIMER + * + * This software is provided 'as is' with no explicit or implied warranties + * in respect of its properties, including, but not limited to, correctness + * and/or fitness for purpose. + * --------------------------------------------------------------------------- + * + * $FreeBSD$ + */ + +#ifndef _MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_ +#define _MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_ + +struct octo_sess; + +typedef int octo_encrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp); +typedef int octo_decrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp); + +struct octo_sess { + int octo_encalg; + #define MAX_CIPHER_KEYLEN 64 + char octo_enckey[MAX_CIPHER_KEYLEN]; + int octo_encklen; + + int octo_macalg; + #define MAX_HASH_KEYLEN 64 + char octo_mackey[MAX_HASH_KEYLEN]; + int octo_macklen; + int octo_mackey_set; + + int octo_mlen; + int octo_ivsize; + + octo_encrypt_t *octo_encrypt; + octo_decrypt_t *octo_decrypt; + + uint64_t octo_hminner[3]; + uint64_t octo_hmouter[3]; + + struct iovec octo_iov[UIO_MAXIOV]; +}; + +#define dprintf(fmt, ...) \ + do { \ + if (cryptocteon_debug) \ + printf("%s: " fmt, __func__, ## __VA_ARGS__); \ + } while (0) + +extern int cryptocteon_debug; + +void octo_calc_hash(uint8_t, unsigned char *, uint64_t *, uint64_t *); + +/* XXX Actually just hashing functions, not encryption. */ +octo_encrypt_t octo_null_md5_encrypt; +octo_encrypt_t octo_null_sha1_encrypt; + +octo_encrypt_t octo_des_cbc_encrypt; +octo_encrypt_t octo_des_cbc_md5_encrypt; +octo_encrypt_t octo_des_cbc_sha1_encrypt; + +octo_decrypt_t octo_des_cbc_decrypt; +octo_decrypt_t octo_des_cbc_md5_decrypt; +octo_decrypt_t octo_des_cbc_sha1_decrypt; + +octo_encrypt_t octo_aes_cbc_encrypt; +octo_encrypt_t octo_aes_cbc_md5_encrypt; +octo_encrypt_t octo_aes_cbc_sha1_encrypt; + +octo_decrypt_t octo_aes_cbc_decrypt; +octo_decrypt_t octo_aes_cbc_md5_decrypt; +octo_decrypt_t octo_aes_cbc_sha1_decrypt; + +#endif /* !_MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_ */ diff --git a/sys/mips/cavium/files.octeon1 b/sys/mips/cavium/files.octeon1 index 48c36cf..c8dc43c 100644 --- a/sys/mips/cavium/files.octeon1 +++ b/sys/mips/cavium/files.octeon1 @@ -17,6 +17,9 @@ mips/mips/tick.c standard mips/cavium/octeon_rnd.c optional random +mips/cavium/cryptocteon/cavium_crypto.c optional cryptocteon +mips/cavium/cryptocteon/cryptocteon.c optional cryptocteon + mips/cavium/octe/ethernet.c optional octe mips/cavium/octe/ethernet-common.c optional octe mips/cavium/octe/ethernet-mdio.c optional octe @@ -33,6 +36,9 @@ mips/cavium/octe/octebus.c optional octe mips/cavium/octopci.c optional pci mips/cavium/octopci_bus_space.c optional pci +mips/cavium/usb/octusb.c optional usb octusb +mips/cavium/usb/octusb_octeon.c optional usb octusb + contrib/octeon-sdk/cvmx-cmd-queue.c optional octe contrib/octeon-sdk/cvmx-fpa.c optional octe contrib/octeon-sdk/cvmx-helper.c optional octe @@ -51,6 +57,8 @@ contrib/octeon-sdk/cvmx-spi.c optional octe contrib/octeon-sdk/cvmx-spi4000.c optional octe contrib/octeon-sdk/cvmx-twsi.c optional octe +contrib/octeon-sdk/cvmx-usb.c optional octusb + # XXX Some files could be excluded in some configurations. Making them # optional but on in the default config would seem reasonable. contrib/octeon-sdk/cvmx-bootmem.c standard diff --git a/sys/mips/cavium/usb/octusb.c b/sys/mips/cavium/usb/octusb.c new file mode 100644 index 0000000..d090f12 --- /dev/null +++ b/sys/mips/cavium/usb/octusb.c @@ -0,0 +1,1922 @@ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +/*- + * Copyright (c) 2010 Hans Petter Selasky. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * This file contains the driver for Octeon Executive Library USB + * Controller driver API. + */ + +/* TODO: The root HUB port callback is not yet implemented. */ + +#include <sys/stdint.h> +#include <sys/stddef.h> +#include <sys/param.h> +#include <sys/queue.h> +#include <sys/types.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/bus.h> +#include <sys/linker_set.h> +#include <sys/module.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/condvar.h> +#include <sys/sysctl.h> +#include <sys/sx.h> +#include <sys/unistd.h> +#include <sys/callout.h> +#include <sys/malloc.h> +#include <sys/priv.h> + +#include <dev/usb/usb.h> +#include <dev/usb/usbdi.h> + +#define USB_DEBUG_VAR octusbdebug + +#include <dev/usb/usb_core.h> +#include <dev/usb/usb_debug.h> +#include <dev/usb/usb_busdma.h> +#include <dev/usb/usb_process.h> +#include <dev/usb/usb_transfer.h> +#include <dev/usb/usb_device.h> +#include <dev/usb/usb_hub.h> +#include <dev/usb/usb_util.h> + +#include <dev/usb/usb_controller.h> +#include <dev/usb/usb_bus.h> + +#include <contrib/octeon-sdk/cvmx.h> +#include <contrib/octeon-sdk/cvmx-usb.h> + +#include <mips/cavium/usb/octusb.h> + +#define OCTUSB_BUS2SC(bus) \ + ((struct octusb_softc *)(((uint8_t *)(bus)) - \ + ((uint8_t *)&(((struct octusb_softc *)0)->sc_bus)))) + +#ifdef USB_DEBUG +static int octusbdebug = 0; + +SYSCTL_NODE(_hw_usb, OID_AUTO, octusb, CTLFLAG_RW, 0, "OCTUSB"); +SYSCTL_INT(_hw_usb_octusb, OID_AUTO, debug, CTLFLAG_RW, + &octusbdebug, 0, "OCTUSB debug level"); + +TUNABLE_INT("hw.usb.octusb.debug", &octusbdebug); + +#endif + +struct octusb_std_temp { + octusb_cmd_t *func; + struct octusb_td *td; + struct octusb_td *td_next; + struct usb_page_cache *pc; + uint32_t offset; + uint32_t len; + uint8_t short_pkt; + uint8_t setup_alt_next; +}; + +extern struct usb_bus_methods octusb_bus_methods; +extern struct usb_pipe_methods octusb_device_bulk_methods; +extern struct usb_pipe_methods octusb_device_ctrl_methods; +extern struct usb_pipe_methods octusb_device_intr_methods; +extern struct usb_pipe_methods octusb_device_isoc_methods; + +static void octusb_standard_done(struct usb_xfer *); +static void octusb_device_done(struct usb_xfer *, usb_error_t); +static void octusb_timeout(void *); +static void octusb_do_poll(struct usb_bus *); + +static cvmx_usb_speed_t +octusb_convert_speed(enum usb_dev_speed speed) +{ + ; /* indent fix */ + switch (speed) { + case USB_SPEED_HIGH: + return (CVMX_USB_SPEED_HIGH); + case USB_SPEED_FULL: + return (CVMX_USB_SPEED_FULL); + default: + return (CVMX_USB_SPEED_LOW); + } +} + +static cvmx_usb_transfer_t +octusb_convert_ep_type(uint8_t ep_type) +{ + ; /* indent fix */ + switch (ep_type & UE_XFERTYPE) { + case UE_CONTROL: + return (CVMX_USB_TRANSFER_CONTROL); + case UE_INTERRUPT: + return (CVMX_USB_TRANSFER_INTERRUPT); + case UE_ISOCHRONOUS: + return (CVMX_USB_TRANSFER_ISOCHRONOUS); + case UE_BULK: + return (CVMX_USB_TRANSFER_BULK); + default: + return (0); /* should not happen */ + } +} + +static uint8_t +octusb_host_alloc_endpoint(struct octusb_td *td) +{ + struct octusb_softc *sc; + int ep_handle; + + if (td->qh->fixup_pending) + return (1); /* busy */ + + if (td->qh->ep_allocated) + return (0); /* success */ + + /* get softc */ + sc = td->qh->sc; + + ep_handle = cvmx_usb_open_pipe( + &sc->sc_port[td->qh->port_index].state, + 0, + td->qh->dev_addr, + td->qh->ep_num, + octusb_convert_speed(td->qh->dev_speed), + td->qh->max_packet_size, + octusb_convert_ep_type(td->qh->ep_type), + (td->qh->ep_num & UE_DIR_IN) ? CVMX_USB_DIRECTION_IN : + CVMX_USB_DIRECTION_OUT, + td->qh->ep_interval, + td->qh->ep_mult, + td->qh->hs_hub_addr, + td->qh->hs_hub_port); + + if (ep_handle < 0) + return (1); /* busy */ + + cvmx_usb_set_toggle( + &sc->sc_port[td->qh->port_index].state, + ep_handle, td->qh->ep_toggle_next); + + td->qh->fixup_handle = -1; + td->qh->fixup_complete = 0; + td->qh->fixup_len = 0; + td->qh->fixup_off = 0; + td->qh->fixup_pending = 0; + td->qh->fixup_actlen = 0; + + td->qh->ep_handle = ep_handle; + td->qh->ep_allocated = 1; + + return (0); /* success */ +} + +static void +octusb_host_free_endpoint(struct octusb_td *td) +{ + struct octusb_softc *sc; + + if (td->qh->ep_allocated == 0) + return; + + /* get softc */ + sc = td->qh->sc; + + if (td->qh->fixup_handle >= 0) { + /* cancel, if any */ + cvmx_usb_cancel(&sc->sc_port[td->qh->port_index].state, + td->qh->ep_handle, td->qh->fixup_handle); + } + cvmx_usb_close_pipe(&sc->sc_port[td->qh->port_index].state, td->qh->ep_handle); + + td->qh->ep_allocated = 0; +} + +static void +octusb_complete_cb(cvmx_usb_state_t *state, + cvmx_usb_callback_t reason, + cvmx_usb_complete_t status, + int pipe_handle, int submit_handle, + int bytes_transferred, void *user_data) +{ + struct octusb_td *td; + + if (reason != CVMX_USB_CALLBACK_TRANSFER_COMPLETE) + return; + + td = user_data; + + td->qh->fixup_complete = 1; + td->qh->fixup_pending = 0; + td->qh->fixup_actlen = bytes_transferred; + td->qh->fixup_handle = -1; + + switch (status) { + case CVMX_USB_COMPLETE_SUCCESS: + case CVMX_USB_COMPLETE_SHORT: + td->error_any = 0; + td->error_stall = 0; + break; + case CVMX_USB_COMPLETE_STALL: + td->error_stall = 1; + td->error_any = 1; + break; + default: + td->error_any = 1; + break; + } +} + +static uint8_t +octusb_host_control_header_tx(struct octusb_td *td) +{ + int status; + + /* allocate endpoint and check pending */ + if (octusb_host_alloc_endpoint(td)) + return (1); /* busy */ + + /* check error */ + if (td->error_any) + return (0); /* done */ + + if (td->qh->fixup_complete != 0) { + /* clear complete flag */ + td->qh->fixup_complete = 0; + + /* flush data */ + usb_pc_cpu_invalidate(td->qh->fixup_pc); + return (0); /* done */ + } + /* verify length */ + if (td->remainder != 8) { + td->error_any = 1; + return (0); /* done */ + } + usbd_copy_out(td->pc, td->offset, td->qh->fixup_buf, 8); + + /* update offset and remainder */ + td->offset += 8; + td->remainder -= 8; + + /* setup data length and offset */ + td->qh->fixup_len = UGETW(td->qh->fixup_buf + 6); + td->qh->fixup_off = 0; + + if (td->qh->fixup_len > (OCTUSB_MAX_FIXUP - 8)) { + td->error_any = 1; + return (0); /* done */ + } + /* do control IN request */ + if (td->qh->fixup_buf[0] & UE_DIR_IN) { + + struct octusb_softc *sc; + + /* get softc */ + sc = td->qh->sc; + + /* flush data */ + usb_pc_cpu_flush(td->qh->fixup_pc); + + status = cvmx_usb_submit_control( + &sc->sc_port[td->qh->port_index].state, + td->qh->ep_handle, td->qh->fixup_phys, + td->qh->fixup_phys + 8, td->qh->fixup_len, + &octusb_complete_cb, td); + /* check status */ + if (status < 0) { + td->error_any = 1; + return (0); /* done */ + } + td->qh->fixup_handle = status; + td->qh->fixup_pending = 1; + td->qh->fixup_complete = 0; + + return (1); /* busy */ + } + return (0); /* done */ +} + +static uint8_t +octusb_host_control_data_tx(struct octusb_td *td) +{ + uint32_t rem; + + /* allocate endpoint and check pending */ + if (octusb_host_alloc_endpoint(td)) + return (1); /* busy */ + + /* check error */ + if (td->error_any) + return (0); /* done */ + + rem = td->qh->fixup_len - td->qh->fixup_off; + + if (td->remainder > rem) { + td->error_any = 1; + DPRINTFN(1, "Excess setup transmit data\n"); + return (0); /* done */ + } + usbd_copy_out(td->pc, td->offset, td->qh->fixup_buf + + td->qh->fixup_off + 8, td->remainder); + + td->offset += td->remainder; + td->qh->fixup_off += td->remainder; + td->remainder = 0; + + return (0); /* done */ +} + +static uint8_t +octusb_host_control_data_rx(struct octusb_td *td) +{ + uint32_t rem; + + /* allocate endpoint and check pending */ + if (octusb_host_alloc_endpoint(td)) + return (1); /* busy */ + + /* check error */ + if (td->error_any) + return (0); /* done */ + + /* copy data from buffer */ + rem = td->qh->fixup_actlen - td->qh->fixup_off; + + if (rem > td->remainder) + rem = td->remainder; + + usbd_copy_in(td->pc, td->offset, td->qh->fixup_buf + + td->qh->fixup_off + 8, rem); + + td->offset += rem; + td->remainder -= rem; + td->qh->fixup_off += rem; + + return (0); /* done */ +} + +static uint8_t +octusb_host_control_status_tx(struct octusb_td *td) +{ + int status; + + /* allocate endpoint and check pending */ + if (octusb_host_alloc_endpoint(td)) + return (1); /* busy */ + + /* check error */ + if (td->error_any) + return (0); /* done */ + + if (td->qh->fixup_complete != 0) { + /* clear complete flag */ + td->qh->fixup_complete = 0; + /* done */ + return (0); + } + /* do control IN request */ + if (!(td->qh->fixup_buf[0] & UE_DIR_IN)) { + + struct octusb_softc *sc; + + /* get softc */ + sc = td->qh->sc; + + /* flush data */ + usb_pc_cpu_flush(td->qh->fixup_pc); + + /* start USB transfer */ + status = cvmx_usb_submit_control( + &sc->sc_port[td->qh->port_index].state, + td->qh->ep_handle, td->qh->fixup_phys, + td->qh->fixup_phys + 8, td->qh->fixup_len, + &octusb_complete_cb, td); + + /* check status */ + if (status < 0) { + td->error_any = 1; + return (0); /* done */ + } + td->qh->fixup_handle = status; + td->qh->fixup_pending = 1; + td->qh->fixup_complete = 0; + + return (1); /* busy */ + } + return (0); /* done */ +} + +static uint8_t +octusb_non_control_data_tx(struct octusb_td *td) +{ + struct octusb_softc *sc; + uint32_t rem; + int status; + + /* allocate endpoint and check pending */ + if (octusb_host_alloc_endpoint(td)) + return (1); /* busy */ + + /* check error */ + if (td->error_any) + return (0); /* done */ + + if ((td->qh->fixup_complete != 0) && + ((td->qh->ep_type & UE_XFERTYPE) == UE_ISOCHRONOUS)) { + td->qh->fixup_complete = 0; + return (0); /* done */ + } + /* check complete */ + if (td->remainder == 0) { + if (td->short_pkt) + return (0); /* complete */ + /* else need to send a zero length packet */ + rem = 0; + td->short_pkt = 1; + } else { + /* get maximum length */ + rem = OCTUSB_MAX_FIXUP % td->qh->max_frame_size; + rem = OCTUSB_MAX_FIXUP - rem; + + if (rem == 0) { + /* should not happen */ + DPRINTFN(1, "Fixup buffer is too small\n"); + td->error_any = 1; + return (0); /* done */ + } + /* get minimum length */ + if (rem > td->remainder) { + rem = td->remainder; + if ((rem == 0) || (rem % td->qh->max_frame_size)) + td->short_pkt = 1; + } + /* copy data into fixup buffer */ + usbd_copy_out(td->pc, td->offset, td->qh->fixup_buf, rem); + + /* flush data */ + usb_pc_cpu_flush(td->qh->fixup_pc); + + /* pre-increment TX buffer offset */ + td->offset += rem; + td->remainder -= rem; + } + + /* get softc */ + sc = td->qh->sc; + + switch (td->qh->ep_type & UE_XFERTYPE) { + case UE_ISOCHRONOUS: + td->qh->iso_pkt.offset = 0; + td->qh->iso_pkt.length = rem; + td->qh->iso_pkt.status = 0; + /* start USB transfer */ + status = cvmx_usb_submit_isochronous(&sc->sc_port[td->qh->port_index].state, + td->qh->ep_handle, 1, CVMX_USB_ISOCHRONOUS_FLAGS_ALLOW_SHORT | + CVMX_USB_ISOCHRONOUS_FLAGS_ASAP, 1, &td->qh->iso_pkt, + td->qh->fixup_phys, rem, &octusb_complete_cb, td); + break; + case UE_BULK: + /* start USB transfer */ + status = cvmx_usb_submit_bulk(&sc->sc_port[td->qh->port_index].state, + td->qh->ep_handle, td->qh->fixup_phys, rem, &octusb_complete_cb, td); + break; + case UE_INTERRUPT: + /* start USB transfer (interrupt or interrupt) */ + status = cvmx_usb_submit_interrupt(&sc->sc_port[td->qh->port_index].state, + td->qh->ep_handle, td->qh->fixup_phys, rem, &octusb_complete_cb, td); + break; + default: + status = -1; + break; + } + + /* check status */ + if (status < 0) { + td->error_any = 1; + return (0); /* done */ + } + td->qh->fixup_handle = status; + td->qh->fixup_len = rem; + td->qh->fixup_pending = 1; + td->qh->fixup_complete = 0; + + return (1); /* busy */ +} + +static uint8_t +octusb_non_control_data_rx(struct octusb_td *td) +{ + struct octusb_softc *sc; + uint32_t rem; + int status; + uint8_t got_short; + + /* allocate endpoint and check pending */ + if (octusb_host_alloc_endpoint(td)) + return (1); /* busy */ + + /* check error */ + if (td->error_any) + return (0); /* done */ + + got_short = 0; + + if (td->qh->fixup_complete != 0) { + + /* invalidate data */ + usb_pc_cpu_invalidate(td->qh->fixup_pc); + + rem = td->qh->fixup_actlen; + + /* verify transfer length */ + if (rem != td->qh->fixup_len) { + if (rem < td->qh->fixup_len) { + /* we have a short packet */ + td->short_pkt = 1; + got_short = 1; + } else { + /* invalid USB packet */ + td->error_any = 1; + return (0); /* we are complete */ + } + } + /* copy data into fixup buffer */ + usbd_copy_in(td->pc, td->offset, td->qh->fixup_buf, rem); + + /* post-increment RX buffer offset */ + td->offset += rem; + td->remainder -= rem; + + td->qh->fixup_complete = 0; + + if ((td->qh->ep_type & UE_XFERTYPE) == UE_ISOCHRONOUS) + return (0); /* done */ + } + /* check if we are complete */ + if ((td->remainder == 0) || got_short) { + if (td->short_pkt) { + /* we are complete */ + return (0); + } + /* else need to receive a zero length packet */ + rem = 0; + td->short_pkt = 1; + } else { + /* get maximum length */ + rem = OCTUSB_MAX_FIXUP % td->qh->max_frame_size; + rem = OCTUSB_MAX_FIXUP - rem; + + if (rem == 0) { + /* should not happen */ + DPRINTFN(1, "Fixup buffer is too small\n"); + td->error_any = 1; + return (0); /* done */ + } + /* get minimum length */ + if (rem > td->remainder) + rem = td->remainder; + } + + /* invalidate data */ + usb_pc_cpu_invalidate(td->qh->fixup_pc); + + /* get softc */ + sc = td->qh->sc; + + switch (td->qh->ep_type & UE_XFERTYPE) { + case UE_ISOCHRONOUS: + td->qh->iso_pkt.offset = 0; + td->qh->iso_pkt.length = rem; + td->qh->iso_pkt.status = 0; + /* start USB transfer */ + status = cvmx_usb_submit_isochronous(&sc->sc_port[td->qh->port_index].state, + td->qh->ep_handle, 1, CVMX_USB_ISOCHRONOUS_FLAGS_ALLOW_SHORT | + CVMX_USB_ISOCHRONOUS_FLAGS_ASAP, 1, &td->qh->iso_pkt, + td->qh->fixup_phys, rem, &octusb_complete_cb, td); + break; + case UE_BULK: + /* start USB transfer */ + status = cvmx_usb_submit_bulk(&sc->sc_port[td->qh->port_index].state, + td->qh->ep_handle, td->qh->fixup_phys, rem, &octusb_complete_cb, td); + break; + case UE_INTERRUPT: + /* start USB transfer */ + status = cvmx_usb_submit_interrupt(&sc->sc_port[td->qh->port_index].state, + td->qh->ep_handle, td->qh->fixup_phys, rem, &octusb_complete_cb, td); + break; + default: + status = -1; + break; + } + + /* check status */ + if (status < 0) { + td->error_any = 1; + return (0); /* done */ + } + td->qh->fixup_handle = status; + td->qh->fixup_len = rem; + td->qh->fixup_pending = 1; + td->qh->fixup_complete = 0; + + return (1); /* busy */ +} + +static uint8_t +octusb_xfer_do_fifo(struct usb_xfer *xfer) +{ + struct octusb_td *td; + + DPRINTFN(8, "\n"); + + td = xfer->td_transfer_cache; + + while (1) { + if ((td->func) (td)) { + /* operation in progress */ + break; + } + if (((void *)td) == xfer->td_transfer_last) { + goto done; + } + if (td->error_any) { + goto done; + } else if (td->remainder > 0) { + /* + * We had a short transfer. If there is no + * alternate next, stop processing ! + */ + if (td->alt_next == 0) + goto done; + } + /* + * Fetch the next transfer descriptor and transfer + * some flags to the next transfer descriptor + */ + td = td->obj_next; + xfer->td_transfer_cache = td; + } + return (1); /* not complete */ + +done: + /* compute all actual lengths */ + + octusb_standard_done(xfer); + + return (0); /* complete */ +} + +static usb_error_t +octusb_standard_done_sub(struct usb_xfer *xfer) +{ + struct octusb_td *td; + uint32_t len; + usb_error_t error; + + DPRINTFN(8, "\n"); + + td = xfer->td_transfer_cache; + + do { + len = td->remainder; + + if (xfer->aframes != xfer->nframes) { + /* + * Verify the length and subtract + * the remainder from "frlengths[]": + */ + if (len > xfer->frlengths[xfer->aframes]) { + td->error_any = 1; + } else { + xfer->frlengths[xfer->aframes] -= len; + } + } + /* Check for transfer error */ + if (td->error_any) { + /* the transfer is finished */ + error = td->error_stall ? USB_ERR_STALLED : USB_ERR_IOERROR; + td = NULL; + break; + } + /* Check for short transfer */ + if (len > 0) { + if (xfer->flags_int.short_frames_ok) { + /* follow alt next */ + if (td->alt_next) { + td = td->obj_next; + } else { + td = NULL; + } + } else { + /* the transfer is finished */ + td = NULL; + } + error = 0; + break; + } + td = td->obj_next; + + /* this USB frame is complete */ + error = 0; + break; + + } while (0); + + /* update transfer cache */ + + xfer->td_transfer_cache = td; + + return (error); +} + +static void +octusb_standard_done(struct usb_xfer *xfer) +{ + struct octusb_softc *sc; + struct octusb_qh *qh; + usb_error_t error = 0; + + DPRINTFN(12, "xfer=%p endpoint=%p transfer done\n", + xfer, xfer->endpoint); + + /* reset scanner */ + + xfer->td_transfer_cache = xfer->td_transfer_first; + + if (xfer->flags_int.control_xfr) { + + if (xfer->flags_int.control_hdr) + error = octusb_standard_done_sub(xfer); + + xfer->aframes = 1; + + if (xfer->td_transfer_cache == NULL) + goto done; + } + while (xfer->aframes != xfer->nframes) { + + error = octusb_standard_done_sub(xfer); + + xfer->aframes++; + + if (xfer->td_transfer_cache == NULL) + goto done; + } + + if (xfer->flags_int.control_xfr && + !xfer->flags_int.control_act) + error = octusb_standard_done_sub(xfer); + +done: + /* update data toggle */ + + qh = xfer->qh_start[0]; + sc = qh->sc; + + xfer->endpoint->toggle_next = + cvmx_usb_get_toggle( + &sc->sc_port[qh->port_index].state, + qh->ep_handle) ? 1 : 0; + + octusb_device_done(xfer, error); +} + +static void +octusb_interrupt_poll(struct octusb_softc *sc) +{ + struct usb_xfer *xfer; + uint8_t x; + + /* poll all ports */ + for (x = 0; x != sc->sc_noport; x++) + cvmx_usb_poll(&sc->sc_port[x].state); + +repeat: + TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) { + if (!octusb_xfer_do_fifo(xfer)) { + /* queue has been modified */ + goto repeat; + } + } +} + +static void +octusb_start_standard_chain(struct usb_xfer *xfer) +{ + DPRINTFN(8, "\n"); + + /* poll one time */ + if (octusb_xfer_do_fifo(xfer)) { + + /* put transfer on interrupt queue */ + usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer); + + /* start timeout, if any */ + if (xfer->timeout != 0) { + usbd_transfer_timeout_ms(xfer, + &octusb_timeout, xfer->timeout); + } + } +} + +void +octusb_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb) +{ + +} + +usb_error_t +octusb_init(struct octusb_softc *sc) +{ + cvmx_usb_initialize_flags_t flags; + int status; + uint8_t x; + + /* flush all cache into memory */ + + usb_bus_mem_flush_all(&sc->sc_bus, &octusb_iterate_hw_softc); + + /* set up the bus struct */ + sc->sc_bus.methods = &octusb_bus_methods; + + /* get number of ports */ + sc->sc_noport = cvmx_usb_get_num_ports(); + + /* check number of ports */ + if (sc->sc_noport > OCTUSB_MAX_PORTS) + sc->sc_noport = OCTUSB_MAX_PORTS; + + /* set USB revision */ + sc->sc_bus.usbrev = USB_REV_2_0; + + /* flags for port initialization */ + flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_AUTO; +#ifdef USB_DEBUG + if (octusbdebug > 100) + flags |= CVMX_USB_INITIALIZE_FLAGS_DEBUG_ALL; +#endif + + USB_BUS_LOCK(&sc->sc_bus); + + /* setup all ports */ + for (x = 0; x != sc->sc_noport; x++) { + status = cvmx_usb_initialize(&sc->sc_port[x].state, x, flags); + if (status < 0) + sc->sc_port[x].disabled = 1; + } + + USB_BUS_UNLOCK(&sc->sc_bus); + + /* catch lost interrupts */ + octusb_do_poll(&sc->sc_bus); + + return (0); +} + +usb_error_t +octusb_uninit(struct octusb_softc *sc) +{ + uint8_t x; + + USB_BUS_LOCK(&sc->sc_bus); + + for (x = 0; x != sc->sc_noport; x++) { + if (sc->sc_port[x].disabled == 0) + cvmx_usb_shutdown(&sc->sc_port[x].state); + } + USB_BUS_UNLOCK(&sc->sc_bus); + + return (0); + +} + +void +octusb_suspend(struct octusb_softc *sc) +{ + +} + +void +octusb_resume(struct octusb_softc *sc) +{ + +} + +/*------------------------------------------------------------------------* + * octusb_interrupt - OCTUSB interrupt handler + *------------------------------------------------------------------------*/ +void +octusb_interrupt(struct octusb_softc *sc) +{ + USB_BUS_LOCK(&sc->sc_bus); + + DPRINTFN(16, "real interrupt\n"); + + /* poll all the USB transfers */ + octusb_interrupt_poll(sc); + + USB_BUS_UNLOCK(&sc->sc_bus); +} + +/*------------------------------------------------------------------------* + * octusb_timeout - OCTUSB transfer timeout handler + *------------------------------------------------------------------------*/ +static void +octusb_timeout(void *arg) +{ + struct usb_xfer *xfer = arg; + + DPRINTF("xfer=%p\n", xfer); + + USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED); + + /* transfer is transferred */ + octusb_device_done(xfer, USB_ERR_TIMEOUT); +} + +/*------------------------------------------------------------------------* + * octusb_do_poll - OCTUSB poll transfers + *------------------------------------------------------------------------*/ +static void +octusb_do_poll(struct usb_bus *bus) +{ + struct octusb_softc *sc = OCTUSB_BUS2SC(bus); + + USB_BUS_LOCK(&sc->sc_bus); + octusb_interrupt_poll(sc); + USB_BUS_UNLOCK(&sc->sc_bus); +} + +static void +octusb_setup_standard_chain_sub(struct octusb_std_temp *temp) +{ + struct octusb_td *td; + + /* get current Transfer Descriptor */ + td = temp->td_next; + temp->td = td; + + /* prepare for next TD */ + temp->td_next = td->obj_next; + + /* fill out the Transfer Descriptor */ + td->func = temp->func; + td->pc = temp->pc; + td->offset = temp->offset; + td->remainder = temp->len; + td->error_any = 0; + td->error_stall = 0; + td->short_pkt = temp->short_pkt; + td->alt_next = temp->setup_alt_next; +} + +static void +octusb_setup_standard_chain(struct usb_xfer *xfer) +{ + struct octusb_std_temp temp; + struct octusb_td *td; + uint32_t x; + + DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n", + xfer->address, UE_GET_ADDR(xfer->endpointno), + xfer->sumlen, usbd_get_speed(xfer->xroot->udev)); + + /* setup starting point */ + td = xfer->td_start[0]; + xfer->td_transfer_first = td; + xfer->td_transfer_cache = td; + + temp.td = NULL; + temp.td_next = td; + temp.setup_alt_next = xfer->flags_int.short_frames_ok; + temp.offset = 0; + + /* check if we should prepend a setup message */ + + if (xfer->flags_int.control_xfr) { + + if (xfer->flags_int.control_hdr) { + + temp.func = &octusb_host_control_header_tx; + temp.len = xfer->frlengths[0]; + temp.pc = xfer->frbuffers + 0; + temp.short_pkt = temp.len ? 1 : 0; + + /* check for last frame */ + if (xfer->nframes == 1) { + /* + * no STATUS stage yet, SETUP is + * last + */ + if (xfer->flags_int.control_act) + temp.setup_alt_next = 0; + } + octusb_setup_standard_chain_sub(&temp); + } + x = 1; + } else { + x = 0; + } + + if (x != xfer->nframes) { + if (xfer->endpointno & UE_DIR_IN) { + if (xfer->flags_int.control_xfr) + temp.func = &octusb_host_control_data_rx; + else + temp.func = &octusb_non_control_data_rx; + } else { + if (xfer->flags_int.control_xfr) + temp.func = &octusb_host_control_data_tx; + else + temp.func = &octusb_non_control_data_tx; + } + + /* setup "pc" pointer */ + temp.pc = xfer->frbuffers + x; + } + while (x != xfer->nframes) { + + /* DATA0 or DATA1 message */ + + temp.len = xfer->frlengths[x]; + + x++; + + if (x == xfer->nframes) { + if (xfer->flags_int.control_xfr) { + /* no STATUS stage yet, DATA is last */ + if (xfer->flags_int.control_act) + temp.setup_alt_next = 0; + } else { + temp.setup_alt_next = 0; + } + } + if (temp.len == 0) { + + /* make sure that we send an USB packet */ + + temp.short_pkt = 0; + + } else { + + /* regular data transfer */ + + temp.short_pkt = (xfer->flags.force_short_xfer) ? 0 : 1; + } + + octusb_setup_standard_chain_sub(&temp); + + if (xfer->flags_int.isochronous_xfr) { + /* get next data offset */ + temp.offset += temp.len; + } else { + /* get next Page Cache pointer */ + temp.pc = xfer->frbuffers + x; + } + } + + /* check if we should append a status stage */ + + if (xfer->flags_int.control_xfr && + !xfer->flags_int.control_act) { + + temp.func = &octusb_host_control_status_tx; + temp.len = 0; + temp.pc = NULL; + temp.short_pkt = 0; + temp.setup_alt_next = 0; + + octusb_setup_standard_chain_sub(&temp); + } + /* must have at least one frame! */ + td = temp.td; + xfer->td_transfer_last = td; + + /* properly setup QH */ + + td->qh->ep_allocated = 0; + td->qh->ep_toggle_next = xfer->endpoint->toggle_next ? 1 : 0; +} + +/*------------------------------------------------------------------------* + * octusb_device_done - OCTUSB transfers done code + * + * NOTE: This function can be called more than one time in a row. + *------------------------------------------------------------------------*/ +static void +octusb_device_done(struct usb_xfer *xfer, usb_error_t error) +{ + USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED); + + DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n", + xfer, xfer->endpoint, error); + + /* + * 1) Free any endpoints. + * 2) Control transfers can be split and we should not re-open + * the data pipe between transactions unless there is an error. + */ + if ((xfer->flags_int.control_act == 0) || (error != 0)) { + struct octusb_td *td; + + td = xfer->td_start[0]; + + octusb_host_free_endpoint(td); + } + /* dequeue transfer and start next transfer */ + usbd_transfer_done(xfer, error); +} + +/*------------------------------------------------------------------------* + * octusb bulk support + *------------------------------------------------------------------------*/ +static void +octusb_device_bulk_open(struct usb_xfer *xfer) +{ + return; +} + +static void +octusb_device_bulk_close(struct usb_xfer *xfer) +{ + octusb_device_done(xfer, USB_ERR_CANCELLED); +} + +static void +octusb_device_bulk_enter(struct usb_xfer *xfer) +{ + return; +} + +static void +octusb_device_bulk_start(struct usb_xfer *xfer) +{ + /* setup TDs */ + octusb_setup_standard_chain(xfer); + octusb_start_standard_chain(xfer); +} + +struct usb_pipe_methods octusb_device_bulk_methods = +{ + .open = octusb_device_bulk_open, + .close = octusb_device_bulk_close, + .enter = octusb_device_bulk_enter, + .start = octusb_device_bulk_start, +}; + +/*------------------------------------------------------------------------* + * octusb control support + *------------------------------------------------------------------------*/ +static void +octusb_device_ctrl_open(struct usb_xfer *xfer) +{ + return; +} + +static void +octusb_device_ctrl_close(struct usb_xfer *xfer) +{ + octusb_device_done(xfer, USB_ERR_CANCELLED); +} + +static void +octusb_device_ctrl_enter(struct usb_xfer *xfer) +{ + return; +} + +static void +octusb_device_ctrl_start(struct usb_xfer *xfer) +{ + /* setup TDs */ + octusb_setup_standard_chain(xfer); + octusb_start_standard_chain(xfer); +} + +struct usb_pipe_methods octusb_device_ctrl_methods = +{ + .open = octusb_device_ctrl_open, + .close = octusb_device_ctrl_close, + .enter = octusb_device_ctrl_enter, + .start = octusb_device_ctrl_start, +}; + +/*------------------------------------------------------------------------* + * octusb interrupt support + *------------------------------------------------------------------------*/ +static void +octusb_device_intr_open(struct usb_xfer *xfer) +{ + return; +} + +static void +octusb_device_intr_close(struct usb_xfer *xfer) +{ + octusb_device_done(xfer, USB_ERR_CANCELLED); +} + +static void +octusb_device_intr_enter(struct usb_xfer *xfer) +{ + return; +} + +static void +octusb_device_intr_start(struct usb_xfer *xfer) +{ + /* setup TDs */ + octusb_setup_standard_chain(xfer); + octusb_start_standard_chain(xfer); +} + +struct usb_pipe_methods octusb_device_intr_methods = +{ + .open = octusb_device_intr_open, + .close = octusb_device_intr_close, + .enter = octusb_device_intr_enter, + .start = octusb_device_intr_start, +}; + +/*------------------------------------------------------------------------* + * octusb isochronous support + *------------------------------------------------------------------------*/ +static void +octusb_device_isoc_open(struct usb_xfer *xfer) +{ + return; +} + +static void +octusb_device_isoc_close(struct usb_xfer *xfer) +{ + octusb_device_done(xfer, USB_ERR_CANCELLED); +} + +static void +octusb_device_isoc_enter(struct usb_xfer *xfer) +{ + struct octusb_softc *sc = OCTUSB_BUS2SC(xfer->xroot->bus); + uint32_t temp; + uint32_t frame_count; + uint32_t fs_frames; + + DPRINTFN(5, "xfer=%p next=%d nframes=%d\n", + xfer, xfer->endpoint->isoc_next, xfer->nframes); + + /* get the current frame index */ + + frame_count = cvmx_usb_get_frame_number( + &sc->sc_port[xfer->xroot->udev->port_index].state); + + /* + * check if the frame index is within the window where the frames + * will be inserted + */ + temp = (frame_count - xfer->endpoint->isoc_next) & 0x7FF; + + if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) { + fs_frames = (xfer->nframes + 7) / 8; + } else { + fs_frames = xfer->nframes; + } + + if ((xfer->endpoint->is_synced == 0) || (temp < fs_frames)) { + /* + * If there is data underflow or the pipe queue is + * empty we schedule the transfer a few frames ahead + * of the current frame position. Else two isochronous + * transfers might overlap. + */ + xfer->endpoint->isoc_next = (frame_count + 3) & 0x7FF; + xfer->endpoint->is_synced = 1; + DPRINTFN(2, "start next=%d\n", xfer->endpoint->isoc_next); + } + /* + * compute how many milliseconds the insertion is ahead of the + * current frame position: + */ + temp = (xfer->endpoint->isoc_next - frame_count) & 0x7FF; + + /* + * pre-compute when the isochronous transfer will be finished: + */ + xfer->isoc_time_complete = + usb_isoc_time_expand(&sc->sc_bus, frame_count) + temp + + fs_frames; + + /* compute frame number for next insertion */ + xfer->endpoint->isoc_next += fs_frames; +} + +static void +octusb_device_isoc_start(struct usb_xfer *xfer) +{ + /* setup TDs */ + octusb_setup_standard_chain(xfer); + octusb_start_standard_chain(xfer); +} + +struct usb_pipe_methods octusb_device_isoc_methods = +{ + .open = octusb_device_isoc_open, + .close = octusb_device_isoc_close, + .enter = octusb_device_isoc_enter, + .start = octusb_device_isoc_start, +}; + +/*------------------------------------------------------------------------* + * OCTUSB root HUB support + *------------------------------------------------------------------------* + * Simulate a hardware HUB by handling all the necessary requests. + *------------------------------------------------------------------------*/ +static const +struct usb_device_descriptor octusb_devd = { + .bLength = sizeof(octusb_devd), + .bDescriptorType = UDESC_DEVICE, + .bcdUSB = {0x00, 0x02}, + .bDeviceClass = UDCLASS_HUB, + .bDeviceSubClass = UDSUBCLASS_HUB, + .bDeviceProtocol = UDPROTO_FSHUB, + .bMaxPacketSize = 64, + .idVendor = {0}, + .idProduct = {0}, + .bcdDevice = {0x00, 0x01}, + .iManufacturer = 1, + .iProduct = 2, + .iSerialNumber = 0, + .bNumConfigurations = 1, +}; + +static const +struct usb_device_qualifier octusb_odevd = { + .bLength = sizeof(octusb_odevd), + .bDescriptorType = UDESC_DEVICE_QUALIFIER, + .bcdUSB = {0x00, 0x02}, + .bDeviceClass = UDCLASS_HUB, + .bDeviceSubClass = UDSUBCLASS_HUB, + .bDeviceProtocol = UDPROTO_FSHUB, + .bMaxPacketSize0 = 0, + .bNumConfigurations = 0, + .bReserved = 0, +}; + +static const +struct octusb_config_desc octusb_confd = { + .confd = { + .bLength = sizeof(struct usb_config_descriptor), + .bDescriptorType = UDESC_CONFIG, + .wTotalLength[0] = sizeof(octusb_confd), + .bNumInterface = 1, + .bConfigurationValue = 1, + .iConfiguration = 0, + .bmAttributes = UC_SELF_POWERED, + .bMaxPower = 0 /* max power */ + }, + .ifcd = { + .bLength = sizeof(struct usb_interface_descriptor), + .bDescriptorType = UDESC_INTERFACE, + .bNumEndpoints = 1, + .bInterfaceClass = UICLASS_HUB, + .bInterfaceSubClass = UISUBCLASS_HUB, + .bInterfaceProtocol = UIPROTO_FSHUB, + }, + .endpd = { + .bLength = sizeof(struct usb_endpoint_descriptor), + .bDescriptorType = UDESC_ENDPOINT, + .bEndpointAddress = UE_DIR_IN | OCTUSB_INTR_ENDPT, + .bmAttributes = UE_INTERRUPT, + .wMaxPacketSize[0] = 8, /* max packet (63 ports) */ + .bInterval = 255, + }, +}; + +static const +struct usb_hub_descriptor_min octusb_hubd = +{ + .bDescLength = sizeof(octusb_hubd), + .bDescriptorType = UDESC_HUB, + .bNbrPorts = 2, + .wHubCharacteristics = {UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL, 0}, + .bPwrOn2PwrGood = 50, + .bHubContrCurrent = 0, + .DeviceRemovable = {0x00}, /* all ports are removable */ +}; + +static usb_error_t +octusb_roothub_exec(struct usb_device *udev, + struct usb_device_request *req, const void **pptr, uint16_t *plength) +{ + struct octusb_softc *sc = OCTUSB_BUS2SC(udev->bus); + const void *ptr; + const char *str_ptr; + uint16_t value; + uint16_t index; + uint16_t status; + uint16_t change; + uint16_t len; + usb_error_t err; + cvmx_usb_port_status_t usb_port_status; + + USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED); + + /* XXX disable power save mode, hence it is not supported */ + udev->power_mode = USB_POWER_MODE_ON; + + /* buffer reset */ + ptr = (const void *)&sc->sc_hub_desc.temp; + len = 0; + err = 0; + + value = UGETW(req->wValue); + index = UGETW(req->wIndex); + + DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x " + "wValue=0x%04x wIndex=0x%04x\n", + req->bmRequestType, req->bRequest, + UGETW(req->wLength), value, index); + +#define C(x,y) ((x) | ((y) << 8)) + switch (C(req->bRequest, req->bmRequestType)) { + case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE): + case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE): + case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT): + break; + case C(UR_GET_CONFIG, UT_READ_DEVICE): + len = 1; + sc->sc_hub_desc.temp[0] = sc->sc_conf; + break; + case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): + switch (value >> 8) { + case UDESC_DEVICE: + if ((value & 0xff) != 0) { + err = USB_ERR_IOERROR; + goto done; + } + len = sizeof(octusb_devd); + + ptr = (const void *)&octusb_devd; + break; + + case UDESC_DEVICE_QUALIFIER: + if ((value & 0xff) != 0) { + err = USB_ERR_IOERROR; + goto done; + } + len = sizeof(octusb_odevd); + ptr = (const void *)&octusb_odevd; + break; + + case UDESC_CONFIG: + if ((value & 0xff) != 0) { + err = USB_ERR_IOERROR; + goto done; + } + len = sizeof(octusb_confd); + ptr = (const void *)&octusb_confd; + break; + + case UDESC_STRING: + switch (value & 0xff) { + case 0: /* Language table */ + str_ptr = "\001"; + break; + + case 1: /* Vendor */ + str_ptr = "Cavium Networks"; + break; + + case 2: /* Product */ + str_ptr = "OCTUSB Root HUB"; + break; + + default: + str_ptr = ""; + break; + } + + len = usb_make_str_desc(sc->sc_hub_desc.temp, + sizeof(sc->sc_hub_desc.temp), str_ptr); + break; + + default: + err = USB_ERR_IOERROR; + goto done; + } + break; + case C(UR_GET_INTERFACE, UT_READ_INTERFACE): + len = 1; + sc->sc_hub_desc.temp[0] = 0; + break; + case C(UR_GET_STATUS, UT_READ_DEVICE): + len = 2; + USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED); + break; + case C(UR_GET_STATUS, UT_READ_INTERFACE): + case C(UR_GET_STATUS, UT_READ_ENDPOINT): + len = 2; + USETW(sc->sc_hub_desc.stat.wStatus, 0); + break; + case C(UR_SET_ADDRESS, UT_WRITE_DEVICE): + if (value >= OCTUSB_MAX_DEVICES) { + err = USB_ERR_IOERROR; + goto done; + } + sc->sc_addr = value; + break; + case C(UR_SET_CONFIG, UT_WRITE_DEVICE): + if ((value != 0) && (value != 1)) { + err = USB_ERR_IOERROR; + goto done; + } + sc->sc_conf = value; + break; + case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE): + break; + case C(UR_SET_FEATURE, UT_WRITE_DEVICE): + case C(UR_SET_FEATURE, UT_WRITE_INTERFACE): + case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT): + err = USB_ERR_IOERROR; + goto done; + case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE): + break; + case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT): + break; + /* Hub requests */ + case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): + break; + case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): + DPRINTFN(4, "UR_CLEAR_PORT_FEATURE " + "port=%d feature=%d\n", + index, value); + if ((index < 1) || + (index > sc->sc_noport) || + sc->sc_port[index - 1].disabled) { + err = USB_ERR_IOERROR; + goto done; + } + index--; + + switch (value) { + case UHF_PORT_ENABLE: + cvmx_usb_disable(&sc->sc_port[index].state); + break; + case UHF_PORT_SUSPEND: + case UHF_PORT_RESET: + break; + case UHF_C_PORT_CONNECTION: + cvmx_usb_set_status(&sc->sc_port[index].state, + cvmx_usb_get_status(&sc->sc_port[index].state)); + break; + case UHF_C_PORT_ENABLE: + cvmx_usb_set_status(&sc->sc_port[index].state, + cvmx_usb_get_status(&sc->sc_port[index].state)); + break; + case UHF_C_PORT_OVER_CURRENT: + cvmx_usb_set_status(&sc->sc_port[index].state, + cvmx_usb_get_status(&sc->sc_port[index].state)); + break; + case UHF_C_PORT_RESET: + sc->sc_isreset = 0; + goto done; + case UHF_C_PORT_SUSPEND: + break; + case UHF_PORT_CONNECTION: + case UHF_PORT_OVER_CURRENT: + case UHF_PORT_POWER: + case UHF_PORT_LOW_SPEED: + default: + err = USB_ERR_IOERROR; + goto done; + } + break; + case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): + if ((value & 0xff) != 0) { + err = USB_ERR_IOERROR; + goto done; + } + sc->sc_hubd = octusb_hubd; + sc->sc_hubd.bNbrPorts = sc->sc_noport; + len = sizeof(sc->sc_hubd); + ptr = (const void *)&sc->sc_hubd; + break; + case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): + len = 16; + memset(sc->sc_hub_desc.temp, 0, 16); + break; + case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): + if ((index < 1) || + (index > sc->sc_noport) || + sc->sc_port[index - 1].disabled) { + err = USB_ERR_IOERROR; + goto done; + } + index--; + + usb_port_status = cvmx_usb_get_status(&sc->sc_port[index].state); + + status = change = 0; + if (usb_port_status.connected) + status |= UPS_CURRENT_CONNECT_STATUS; + if (usb_port_status.port_enabled) + status |= UPS_PORT_ENABLED; + if (usb_port_status.port_over_current) + status |= UPS_OVERCURRENT_INDICATOR; + if (usb_port_status.port_powered) + status |= UPS_PORT_POWER; + + switch (usb_port_status.port_speed) { + case CVMX_USB_SPEED_HIGH: + status |= UPS_HIGH_SPEED; + break; + case CVMX_USB_SPEED_FULL: + break; + default: + status |= UPS_LOW_SPEED; + break; + } + + if (usb_port_status.connect_change) + change |= UPS_C_CONNECT_STATUS; + if (sc->sc_isreset) + change |= UPS_C_PORT_RESET; + + USETW(sc->sc_hub_desc.ps.wPortStatus, status); + USETW(sc->sc_hub_desc.ps.wPortChange, change); + + len = sizeof(sc->sc_hub_desc.ps); + break; + case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): + err = USB_ERR_IOERROR; + goto done; + case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): + break; + case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): + if ((index < 1) || + (index > sc->sc_noport) || + sc->sc_port[index - 1].disabled) { + err = USB_ERR_IOERROR; + goto done; + } + index--; + + switch (value) { + case UHF_PORT_ENABLE: + break; + case UHF_PORT_RESET: + cvmx_usb_disable(&sc->sc_port[index].state); + if (cvmx_usb_enable(&sc->sc_port[index].state)) { + err = USB_ERR_IOERROR; + goto done; + } + sc->sc_isreset = 1; + goto done; + case UHF_PORT_POWER: + /* pretend we turned on power */ + goto done; + case UHF_PORT_SUSPEND: + case UHF_C_PORT_CONNECTION: + case UHF_C_PORT_ENABLE: + case UHF_C_PORT_OVER_CURRENT: + case UHF_PORT_CONNECTION: + case UHF_PORT_OVER_CURRENT: + case UHF_PORT_LOW_SPEED: + case UHF_C_PORT_SUSPEND: + case UHF_C_PORT_RESET: + default: + err = USB_ERR_IOERROR; + goto done; + } + break; + default: + err = USB_ERR_IOERROR; + goto done; + } +done: + *plength = len; + *pptr = ptr; + return (err); +} + +static void +octusb_xfer_setup(struct usb_setup_params *parm) +{ + struct usb_page_search page_info; + struct usb_page_cache *pc; + struct octusb_softc *sc; + struct octusb_qh *qh; + struct usb_xfer *xfer; + void *last_obj; + uint32_t n; + uint32_t ntd; + + sc = OCTUSB_BUS2SC(parm->udev->bus); + xfer = parm->curr_xfer; + qh = NULL; + + /* + * NOTE: This driver does not use any of the parameters that + * are computed from the following values. Just set some + * reasonable dummies: + */ + + parm->hc_max_packet_size = 0x400; + parm->hc_max_packet_count = 3; + parm->hc_max_frame_size = 0xC00; + + usbd_transfer_setup_sub(parm); + + if (parm->err) + return; + + /* Allocate a queue head */ + + if (usbd_transfer_setup_sub_malloc( + parm, &pc, sizeof(struct octusb_qh), + USB_HOST_ALIGN, 1)) { + parm->err = USB_ERR_NOMEM; + return; + } + if (parm->buf) { + usbd_get_page(pc, 0, &page_info); + + qh = page_info.buffer; + + /* fill out QH */ + + qh->sc = OCTUSB_BUS2SC(xfer->xroot->bus); + qh->max_frame_size = xfer->max_frame_size; + qh->max_packet_size = xfer->max_packet_size; + qh->ep_num = xfer->endpointno; + qh->ep_type = xfer->endpoint->edesc->bmAttributes; + qh->dev_addr = xfer->address; + qh->dev_speed = usbd_get_speed(xfer->xroot->udev); + qh->port_index = xfer->xroot->udev->port_index; + + switch (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) { + case UE_INTERRUPT: + if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) + qh->ep_interval = xfer->interval * 8; + else + qh->ep_interval = xfer->interval * 1; + break; + case UE_ISOCHRONOUS: + qh->ep_interval = 1 << xfer->fps_shift; + break; + default: + qh->ep_interval = 0; + break; + } + + qh->ep_mult = xfer->max_packet_count & 3; + qh->hs_hub_addr = xfer->xroot->udev->hs_hub_addr; + qh->hs_hub_port = xfer->xroot->udev->hs_port_no; + } + xfer->qh_start[0] = qh; + + /* Allocate a fixup buffer */ + + if (usbd_transfer_setup_sub_malloc( + parm, &pc, OCTUSB_MAX_FIXUP, + OCTUSB_MAX_FIXUP, 1)) { + parm->err = USB_ERR_NOMEM; + return; + } + if (parm->buf) { + usbd_get_page(pc, 0, &page_info); + + qh->fixup_phys = page_info.physaddr; + qh->fixup_pc = pc; + qh->fixup_buf = page_info.buffer; + } + /* Allocate transfer descriptors */ + + last_obj = NULL; + + ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC */ ; + + if (usbd_transfer_setup_sub_malloc( + parm, &pc, sizeof(struct octusb_td), + USB_HOST_ALIGN, ntd)) { + parm->err = USB_ERR_NOMEM; + return; + } + if (parm->buf) { + for (n = 0; n != ntd; n++) { + struct octusb_td *td; + + usbd_get_page(pc + n, 0, &page_info); + + td = page_info.buffer; + + td->qh = qh; + td->obj_next = last_obj; + + last_obj = td; + } + } + xfer->td_start[0] = last_obj; +} + +static void +octusb_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc, + struct usb_endpoint *ep) +{ + struct octusb_softc *sc = OCTUSB_BUS2SC(udev->bus); + + DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n", + ep, udev->address, edesc->bEndpointAddress, + udev->flags.usb_mode, sc->sc_addr); + + if (udev->flags.usb_mode != USB_MODE_HOST) { + /* not supported */ + return; + } + if (udev->device_index != sc->sc_addr) { + switch (edesc->bmAttributes & UE_XFERTYPE) { + case UE_CONTROL: + ep->methods = &octusb_device_ctrl_methods; + break; + case UE_INTERRUPT: + ep->methods = &octusb_device_intr_methods; + break; + case UE_ISOCHRONOUS: + if (udev->speed != USB_SPEED_LOW) + ep->methods = &octusb_device_isoc_methods; + break; + case UE_BULK: + ep->methods = &octusb_device_bulk_methods; + break; + default: + /* do nothing */ + break; + } + } +} + +static void +octusb_xfer_unsetup(struct usb_xfer *xfer) +{ + DPRINTF("Nothing to do.\n"); +} + +static void +octusb_get_dma_delay(struct usb_bus *bus, uint32_t *pus) +{ + /* DMA delay - wait until any use of memory is finished */ + *pus = (2125); /* microseconds */ +} + +static void +octusb_device_resume(struct usb_device *udev) +{ + DPRINTF("Nothing to do.\n"); +} + +static void +octusb_device_suspend(struct usb_device *udev) +{ + DPRINTF("Nothing to do.\n"); +} + +static void +octusb_set_hw_power(struct usb_bus *bus) +{ + DPRINTF("Nothing to do.\n"); +} + +struct usb_bus_methods octusb_bus_methods = { + .endpoint_init = octusb_ep_init, + .xfer_setup = octusb_xfer_setup, + .xfer_unsetup = octusb_xfer_unsetup, + .get_dma_delay = octusb_get_dma_delay, + .device_resume = octusb_device_resume, + .device_suspend = octusb_device_suspend, + .set_hw_power = octusb_set_hw_power, + .roothub_exec = octusb_roothub_exec, + .xfer_poll = octusb_do_poll, +}; diff --git a/sys/mips/cavium/usb/octusb.h b/sys/mips/cavium/usb/octusb.h new file mode 100644 index 0000000..31f4fc0 --- /dev/null +++ b/sys/mips/cavium/usb/octusb.h @@ -0,0 +1,147 @@ +/* $FreeBSD$ */ + +/*- + * Copyright (c) 2010 Hans Petter Selasky. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _OCTUSB_H_ +#define _OCTUSB_H_ + +#define OCTUSB_MAX_DEVICES MIN(USB_MAX_DEVICES, 64) +/* + * The second port is on a different IRQ and so we disable it for now. + */ +#if 1 +#define OCTUSB_MAX_PORTS 1 /* hardcoded */ +#else +#define OCTUSB_MAX_PORTS 2 /* hardcoded */ +#endif +#define OCTUSB_MAX_FIXUP 4096 /* bytes */ +#define OCTUSB_INTR_ENDPT 0x01 + +struct octusb_qh; +struct octusb_td; +struct octusb_softc; + +typedef uint8_t (octusb_cmd_t)(struct octusb_td *td); + +struct octusb_td { + struct octusb_qh *qh; + struct octusb_td *obj_next; + struct usb_page_cache *pc; + octusb_cmd_t *func; + + uint32_t remainder; + uint32_t offset; + + uint8_t error_any:1; + uint8_t error_stall:1; + uint8_t short_pkt:1; + uint8_t alt_next:1; + uint8_t reserved:4; +}; + +struct octusb_qh { + + uint64_t fixup_phys; + + struct octusb_softc *sc; + struct usb_page_cache *fixup_pc; + uint8_t *fixup_buf; + + cvmx_usb_iso_packet_t iso_pkt; + + uint32_t fixup_off; + + uint16_t max_frame_size; + uint16_t max_packet_size; + uint16_t fixup_actlen; + uint16_t fixup_len; + uint16_t ep_interval; + + uint8_t dev_addr; + uint8_t dev_speed; + uint8_t ep_allocated; + uint8_t ep_mult; + uint8_t ep_num; + uint8_t ep_type; + uint8_t ep_toggle_next; + uint8_t port_index; + uint8_t fixup_complete; + uint8_t fixup_pending; + uint8_t hs_hub_addr; + uint8_t hs_hub_port; + + int fixup_handle; + int ep_handle; +}; + +struct octusb_config_desc { + struct usb_config_descriptor confd; + struct usb_interface_descriptor ifcd; + struct usb_endpoint_descriptor endpd; +} __packed; + +union octusb_hub_desc { + struct usb_status stat; + struct usb_port_status ps; + uint8_t temp[128]; +}; + +struct octusb_port { + cvmx_usb_state_t state; + uint8_t disabled; +}; + +struct octusb_softc { + + struct usb_bus sc_bus; /* base device */ + union octusb_hub_desc sc_hub_desc; + + struct usb_device *sc_devices[OCTUSB_MAX_DEVICES]; + + struct resource *sc_irq_res; + void *sc_intr_hdl; + + struct octusb_port sc_port[OCTUSB_MAX_PORTS]; + device_t sc_dev; + + struct usb_hub_descriptor_min sc_hubd; + + uint8_t sc_noport; /* number of ports */ + uint8_t sc_addr; /* device address */ + uint8_t sc_conf; /* device configuration */ + uint8_t sc_isreset; /* set if current port is reset */ + + uint8_t sc_hub_idata[1]; +}; + +usb_bus_mem_cb_t octusb_iterate_hw_softc; +usb_error_t octusb_init(struct octusb_softc *); +usb_error_t octusb_uninit(struct octusb_softc *); +void octusb_suspend(struct octusb_softc *); +void octusb_resume(struct octusb_softc *); +void octusb_interrupt(struct octusb_softc *); + +#endif /* _OCTUSB_H_ */ diff --git a/sys/mips/cavium/usb/octusb_octeon.c b/sys/mips/cavium/usb/octusb_octeon.c new file mode 100644 index 0000000..edd8290 --- /dev/null +++ b/sys/mips/cavium/usb/octusb_octeon.c @@ -0,0 +1,223 @@ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +/*- + * Copyright (c) 2007-2008 Hans Petter Selasky. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/stdint.h> +#include <sys/stddef.h> +#include <sys/param.h> +#include <sys/queue.h> +#include <sys/types.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/bus.h> +#include <sys/linker_set.h> +#include <sys/module.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/condvar.h> +#include <sys/sysctl.h> +#include <sys/sx.h> +#include <sys/unistd.h> +#include <sys/callout.h> +#include <sys/malloc.h> +#include <sys/priv.h> +#include <sys/rman.h> + +#include <dev/usb/usb.h> +#include <dev/usb/usbdi.h> + +#include <dev/usb/usb_core.h> +#include <dev/usb/usb_busdma.h> +#include <dev/usb/usb_process.h> +#include <dev/usb/usb_util.h> + +#include <dev/usb/usb_controller.h> +#include <dev/usb/usb_bus.h> + +#include <contrib/octeon-sdk/cvmx.h> +#include <contrib/octeon-sdk/cvmx-interrupt.h> +#include <contrib/octeon-sdk/cvmx-usb.h> + +#include <mips/cavium/usb/octusb.h> + +#define MEM_RID 0 + +static device_identify_t octusb_octeon_identify; +static device_probe_t octusb_octeon_probe; +static device_attach_t octusb_octeon_attach; +static device_detach_t octusb_octeon_detach; +static device_shutdown_t octusb_octeon_shutdown; + +struct octusb_octeon_softc { + struct octusb_softc sc_dci; /* must be first */ +}; + +static void +octusb_octeon_identify(driver_t *drv, device_t parent) +{ + if (octeon_has_feature(OCTEON_FEATURE_USB)) + BUS_ADD_CHILD(parent, 0, "octusb", 0); +} + +static int +octusb_octeon_probe(device_t dev) +{ + device_set_desc(dev, "Cavium Octeon USB controller"); + return (0); +} + +static int +octusb_octeon_attach(device_t dev) +{ + struct octusb_octeon_softc *sc = device_get_softc(dev); + int err; + int rid; + + /* setup controller interface softc */ + + /* initialise some bus fields */ + sc->sc_dci.sc_bus.parent = dev; + sc->sc_dci.sc_bus.devices = sc->sc_dci.sc_devices; + sc->sc_dci.sc_bus.devices_max = OCTUSB_MAX_DEVICES; + + /* get all DMA memory */ + if (usb_bus_mem_alloc_all(&sc->sc_dci.sc_bus, + USB_GET_DMA_TAG(dev), NULL)) { + return (ENOMEM); + } + rid = 0; + sc->sc_dci.sc_irq_res = + bus_alloc_resource(dev, SYS_RES_IRQ, &rid, + CVMX_IRQ_USB, CVMX_IRQ_USB, 1, RF_ACTIVE); + if (!(sc->sc_dci.sc_irq_res)) { + goto error; + } + + sc->sc_dci.sc_bus.bdev = device_add_child(dev, "usbus", -1); + if (!(sc->sc_dci.sc_bus.bdev)) { + goto error; + } + device_set_ivars(sc->sc_dci.sc_bus.bdev, &sc->sc_dci.sc_bus); + +#if (__FreeBSD_version >= 700031) + err = bus_setup_intr(dev, sc->sc_dci.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE, + NULL, (driver_intr_t *)octusb_interrupt, sc, &sc->sc_dci.sc_intr_hdl); +#else + err = bus_setup_intr(dev, sc->sc_dci.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE, + (driver_intr_t *)octusb_interrupt, sc, &sc->sc_dci.sc_intr_hdl); +#endif + if (err) { + sc->sc_dci.sc_intr_hdl = NULL; + goto error; + } + err = octusb_init(&sc->sc_dci); + if (!err) { + err = device_probe_and_attach(sc->sc_dci.sc_bus.bdev); + } + if (err) { + goto error; + } + return (0); + +error: + octusb_octeon_detach(dev); + return (ENXIO); +} + +static int +octusb_octeon_detach(device_t dev) +{ + struct octusb_octeon_softc *sc = device_get_softc(dev); + device_t bdev; + int err; + + if (sc->sc_dci.sc_bus.bdev) { + bdev = sc->sc_dci.sc_bus.bdev; + device_detach(bdev); + device_delete_child(dev, bdev); + } + /* during module unload there are lots of children leftover */ + device_delete_all_children(dev); + + if (sc->sc_dci.sc_irq_res && sc->sc_dci.sc_intr_hdl) { + /* + * only call octusb_octeon_uninit() after octusb_octeon_init() + */ + octusb_uninit(&sc->sc_dci); + + err = bus_teardown_intr(dev, sc->sc_dci.sc_irq_res, + sc->sc_dci.sc_intr_hdl); + sc->sc_dci.sc_intr_hdl = NULL; + } + if (sc->sc_dci.sc_irq_res) { + bus_release_resource(dev, SYS_RES_IRQ, 0, + sc->sc_dci.sc_irq_res); + sc->sc_dci.sc_irq_res = NULL; + } + usb_bus_mem_free_all(&sc->sc_dci.sc_bus, NULL); + + return (0); +} + +static int +octusb_octeon_shutdown(device_t dev) +{ + struct octusb_octeon_softc *sc = device_get_softc(dev); + int err; + + err = bus_generic_shutdown(dev); + if (err) + return (err); + + octusb_uninit(&sc->sc_dci); + + return (0); +} + +static device_method_t octusb_octeon_methods[] = { + /* Device interface */ + DEVMETHOD(device_identify, octusb_octeon_identify), + DEVMETHOD(device_probe, octusb_octeon_probe), + DEVMETHOD(device_attach, octusb_octeon_attach), + DEVMETHOD(device_detach, octusb_octeon_detach), + DEVMETHOD(device_shutdown, octusb_octeon_shutdown), + + /* Bus interface */ + DEVMETHOD(bus_print_child, bus_generic_print_child), + + {0, 0} +}; + +static driver_t octusb_octeon_driver = { + "octusb", + octusb_octeon_methods, + sizeof(struct octusb_octeon_softc), +}; + +static devclass_t octusb_octeon_devclass; + +DRIVER_MODULE(octusb, ciu, octusb_octeon_driver, octusb_octeon_devclass, 0, 0); |
