o) Add the "octusb" controller which supports the first port of the Octeon

   on-board USB controller.  It is not currently enabled because there are
   known problems with device communication and until those are fixed I am not
   certain that it won't destabilize the system. [1]
o) Add the "cryptocteon" opencrypto device based on the OCF device written by
   David McCullough.  It is not currently enabled because until support for
   saving/restoring coprocessor 2 state on context switch is available, it runs
   with interrupts disabled, which tends to pessimize performance over using a
   software crypto facility.  Tests using this driver which are not negatively
   affected by it running with interrupts disabled show it to be substantially
   faster than software for large blocks.

Submitted by:	hps [1]

7 files changed, 5142 insertions, 0 deletions

diff --git a/sys/mips/cavium/cryptocteon/cavium_crypto.c b/sys/mips/cavium/cryptocteon/cavium_crypto.c
new file mode 100644
index 0000000..ff77a61
--- /dev/null
+++ b/sys/mips/cavium/cryptocteon/cavium_crypto.c
@@ -0,0 +1,2222 @@
+/*
+ * vim:sw=4 ts=8
+ */
+/*
+ * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com>
+ *
+ * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights
+ * reserved.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by Cavium Networks
+ * 4. Cavium Networks' name may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * 
+ * This Software, including technical data, may be subject to U.S. export
+ * control laws, including the U.S. Export Administration Act and its
+ * associated regulations, and may be subject to export or import regulations
+ * in other countries. You warrant that You will comply strictly in all
+ * respects with all such regulations and acknowledge that you have the
+ * responsibility to obtain licenses to export, re-export or import the
+ * Software.
+ * 
+ * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND
+ * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES,
+ * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE
+ * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
+ * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
+ * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
+ * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
+ * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
+ * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
+ * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
+*/
+/****************************************************************************/
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/uio.h>
+
+#include <opencrypto/cryptodev.h>
+
+#include <contrib/octeon-sdk/cvmx.h>
+
+#include <mips/cavium/cryptocteon/cryptocteonvar.h>
+
+/****************************************************************************/
+
+#define	IOV_INIT(iov, ptr, idx, len)					\
+	do {								\
+	    (idx) = 0;							\
+	    (ptr) = (iov)[(idx)].iov_base;				\
+	    (len) = (iov)[(idx)].iov_len;				\
+	} while (0)
+
+/*
+ * XXX
+ * It would be better if this were an IOV_READ/IOV_WRITE macro instead so
+ * that we could detect overflow before it happens rather than right after,
+ * which is especially bad since there is usually no IOV_CONSUME after the
+ * final read or write.
+ */
+#define	IOV_CONSUME(iov, ptr, idx, len)					\
+	do {								\
+	    if ((len) > sizeof *(ptr)) {				\
+		(len) -= sizeof *(ptr);					\
+		(ptr)++;						\
+	    } else {							\
+		if ((len) != sizeof *(ptr))				\
+			panic("%s: went past end of iovec.", __func__);	\
+		(idx)++;						\
+		(ptr) = (iov)[(idx)].iov_base;				\
+		(len) = (iov)[(idx)].iov_len;				\
+	    }								\
+	} while (0)
+
+static inline unsigned long octeon_crypto_enable(void)
+{
+    register_t s;
+    
+    s = intr_disable();
+    mips_wr_status(mips_rd_status() | MIPS_SR_COP_2_BIT);
+
+    return (s);
+}
+
+static inline void octeon_crypto_disable(register_t s)
+{
+    mips_wr_status(mips_rd_status() & ~MIPS_SR_COP_2_BIT);
+    intr_restore(s);
+}
+
+#define ESP_HEADER_LENGTH     8
+#define DES_CBC_IV_LENGTH     8
+#define AES_CBC_IV_LENGTH     16
+#define ESP_HMAC_LEN          12
+
+#define ESP_HEADER_LENGTH 8
+#define DES_CBC_IV_LENGTH 8
+
+/****************************************************************************/
+
+#define CVM_LOAD_SHA_UNIT(dat, next)  { \
+   if (next == 0) {                     \
+      next = 1;                         \
+      CVMX_MT_HSH_DAT (dat, 0);         \
+   } else if (next == 1) {              \
+      next = 2;                         \
+      CVMX_MT_HSH_DAT (dat, 1);         \
+   } else if (next == 2) {              \
+      next = 3;                    \
+      CVMX_MT_HSH_DAT (dat, 2);         \
+   } else if (next == 3) {              \
+      next = 4;                         \
+      CVMX_MT_HSH_DAT (dat, 3);         \
+   } else if (next == 4) {              \
+      next = 5;                           \
+      CVMX_MT_HSH_DAT (dat, 4);         \
+   } else if (next == 5) {              \
+      next = 6;                         \
+      CVMX_MT_HSH_DAT (dat, 5);         \
+   } else if (next == 6) {              \
+      next = 7;                         \
+      CVMX_MT_HSH_DAT (dat, 6);         \
+   } else {                             \
+     CVMX_MT_HSH_STARTSHA (dat);        \
+     next = 0;                          \
+   }                                    \
+}
+
+#define CVM_LOAD2_SHA_UNIT(dat1, dat2, next)  { \
+   if (next == 0) {                      \
+      CVMX_MT_HSH_DAT (dat1, 0);         \
+      CVMX_MT_HSH_DAT (dat2, 1);         \
+      next = 2;                          \
+   } else if (next == 1) {               \
+      CVMX_MT_HSH_DAT (dat1, 1);         \
+      CVMX_MT_HSH_DAT (dat2, 2);         \
+      next = 3;                          \
+   } else if (next == 2) {               \
+      CVMX_MT_HSH_DAT (dat1, 2);         \
+      CVMX_MT_HSH_DAT (dat2, 3);         \
+      next = 4;                          \
+   } else if (next == 3) {               \
+      CVMX_MT_HSH_DAT (dat1, 3);         \
+      CVMX_MT_HSH_DAT (dat2, 4);         \
+      next = 5;                          \
+   } else if (next == 4) {               \
+      CVMX_MT_HSH_DAT (dat1, 4);         \
+      CVMX_MT_HSH_DAT (dat2, 5);         \
+      next = 6;                          \
+   } else if (next == 5) {               \
+      CVMX_MT_HSH_DAT (dat1, 5);         \
+      CVMX_MT_HSH_DAT (dat2, 6);         \
+      next = 7;                          \
+   } else if (next == 6) {               \
+      CVMX_MT_HSH_DAT (dat1, 6);         \
+      CVMX_MT_HSH_STARTSHA (dat2);       \
+      next = 0;                          \
+   } else {                              \
+     CVMX_MT_HSH_STARTSHA (dat1);        \
+     CVMX_MT_HSH_DAT (dat2, 0);          \
+     next = 1;                           \
+   }                                     \
+}
+
+/****************************************************************************/
+
+#define CVM_LOAD_MD5_UNIT(dat, next)  { \
+   if (next == 0) {                     \
+      next = 1;                         \
+      CVMX_MT_HSH_DAT (dat, 0);         \
+   } else if (next == 1) {              \
+      next = 2;                         \
+      CVMX_MT_HSH_DAT (dat, 1);         \
+   } else if (next == 2) {              \
+      next = 3;                    \
+      CVMX_MT_HSH_DAT (dat, 2);         \
+   } else if (next == 3) {              \
+      next = 4;                         \
+      CVMX_MT_HSH_DAT (dat, 3);         \
+   } else if (next == 4) {              \
+      next = 5;                           \
+      CVMX_MT_HSH_DAT (dat, 4);         \
+   } else if (next == 5) {              \
+      next = 6;                         \
+      CVMX_MT_HSH_DAT (dat, 5);         \
+   } else if (next == 6) {              \
+      next = 7;                         \
+      CVMX_MT_HSH_DAT (dat, 6);         \
+   } else {                             \
+     CVMX_MT_HSH_STARTMD5 (dat);        \
+     next = 0;                          \
+   }                                    \
+}
+
+#define CVM_LOAD2_MD5_UNIT(dat1, dat2, next)  { \
+   if (next == 0) {                      \
+      CVMX_MT_HSH_DAT (dat1, 0);         \
+      CVMX_MT_HSH_DAT (dat2, 1);         \
+      next = 2;                          \
+   } else if (next == 1) {               \
+      CVMX_MT_HSH_DAT (dat1, 1);         \
+      CVMX_MT_HSH_DAT (dat2, 2);         \
+      next = 3;                          \
+   } else if (next == 2) {               \
+      CVMX_MT_HSH_DAT (dat1, 2);         \
+      CVMX_MT_HSH_DAT (dat2, 3);         \
+      next = 4;                          \
+   } else if (next == 3) {               \
+      CVMX_MT_HSH_DAT (dat1, 3);         \
+      CVMX_MT_HSH_DAT (dat2, 4);         \
+      next = 5;                          \
+   } else if (next == 4) {               \
+      CVMX_MT_HSH_DAT (dat1, 4);         \
+      CVMX_MT_HSH_DAT (dat2, 5);         \
+      next = 6;                          \
+   } else if (next == 5) {               \
+      CVMX_MT_HSH_DAT (dat1, 5);         \
+      CVMX_MT_HSH_DAT (dat2, 6);         \
+      next = 7;                          \
+   } else if (next == 6) {               \
+      CVMX_MT_HSH_DAT (dat1, 6);         \
+      CVMX_MT_HSH_STARTMD5 (dat2);       \
+      next = 0;                          \
+   } else {                              \
+     CVMX_MT_HSH_STARTMD5 (dat1);        \
+     CVMX_MT_HSH_DAT (dat2, 0);          \
+     next = 1;                           \
+   }                                     \
+}
+
+/****************************************************************************/
+
+void
+octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer)
+{
+    uint8_t hash_key[64];
+    uint64_t *key1;
+    register uint64_t xor1 = 0x3636363636363636ULL;
+    register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    memset(hash_key, 0, sizeof(hash_key));
+    memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
+    key1 = (uint64_t *) hash_key;
+    s = octeon_crypto_enable();
+    if (auth) {
+       CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
+       CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
+       CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
+    } else {
+       CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
+       CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
+    }
+
+    CVMX_MT_HSH_DAT((*key1 ^ xor1), 0);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor1), 1);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor1), 2);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor1), 3);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor1), 4);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor1), 5);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor1), 6);
+    key1++;
+    if (auth)
+		CVMX_MT_HSH_STARTSHA((*key1 ^ xor1));
+    else
+		CVMX_MT_HSH_STARTMD5((*key1 ^ xor1));
+
+    CVMX_MF_HSH_IV(inner[0], 0);
+    CVMX_MF_HSH_IV(inner[1], 1);
+    if (auth) {
+		inner[2] = 0;
+		CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2);
+    }
+
+    memset(hash_key, 0, sizeof(hash_key));
+    memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
+    key1 = (uint64_t *) hash_key;
+    if (auth) {
+      CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
+      CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
+      CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
+    } else {
+      CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
+      CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
+    }
+
+    CVMX_MT_HSH_DAT((*key1 ^ xor2), 0);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor2), 1);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor2), 2);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor2), 3);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor2), 4);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor2), 5);
+    key1++;
+    CVMX_MT_HSH_DAT((*key1 ^ xor2), 6);
+    key1++;
+    if (auth)
+       CVMX_MT_HSH_STARTSHA((*key1 ^ xor2));
+    else 
+       CVMX_MT_HSH_STARTMD5((*key1 ^ xor2));
+
+    CVMX_MF_HSH_IV(outer[0], 0);
+    CVMX_MF_HSH_IV(outer[1], 1);
+    if (auth) {
+      outer[2] = 0;
+      CVMX_MF_HSH_IV(outer[2], 2);
+    }
+    octeon_crypto_disable(s);
+    return;
+}
+
+/****************************************************************************/
+/* DES functions */
+
+int
+octo_des_cbc_encrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    uint64_t *data;
+    int data_i, data_l;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load 3DES Key */
+    CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    if (od->octo_encklen == 24) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+    } else if (od->octo_encklen == 8) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+
+    CVMX_MT_3DES_IV(* (uint64_t *) ivp);
+
+    while (crypt_off > 0) {
+	IOV_CONSUME(iov, data, data_i, data_l);
+	crypt_off -= 8;
+    }
+
+    while (crypt_len > 0) {
+	CVMX_MT_3DES_ENC_CBC(*data);
+	CVMX_MF_3DES_RESULT(*data);
+	IOV_CONSUME(iov, data, data_i, data_l);
+	crypt_len -= 8;
+    }
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+
+int
+octo_des_cbc_decrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    uint64_t *data;
+    int data_i, data_l;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load 3DES Key */
+    CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    if (od->octo_encklen == 24) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+    } else if (od->octo_encklen == 8) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+
+    CVMX_MT_3DES_IV(* (uint64_t *) ivp);
+
+    while (crypt_off > 0) {
+	IOV_CONSUME(iov, data, data_i, data_l);
+	crypt_off -= 8;
+    }
+
+    while (crypt_len > 0) {
+	CVMX_MT_3DES_DEC_CBC(*data);
+	CVMX_MF_3DES_RESULT(*data);
+	IOV_CONSUME(iov, data, data_i, data_l);
+	crypt_len -= 8;
+    }
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+/****************************************************************************/
+/* AES functions */
+
+int
+octo_aes_cbc_encrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    uint64_t *data, *pdata;
+    int data_i, data_l;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load AES Key */
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+
+    if (od->octo_encklen == 16) {
+	CVMX_MT_AES_KEY(0x0, 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 24) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 32) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+    CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
+
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
+
+    while (crypt_off > 0) {
+	IOV_CONSUME(iov, data, data_i, data_l);
+	crypt_off -= 8;
+    }
+
+    while (crypt_len > 0) {
+	pdata = data;
+	CVMX_MT_AES_ENC_CBC0(*data);
+	IOV_CONSUME(iov, data, data_i, data_l);
+	CVMX_MT_AES_ENC_CBC1(*data);
+	CVMX_MF_AES_RESULT(*pdata, 0);
+	CVMX_MF_AES_RESULT(*data, 1);
+	IOV_CONSUME(iov, data, data_i, data_l);
+	crypt_len -= 16;
+    }
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+
+int
+octo_aes_cbc_decrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    uint64_t *data, *pdata;
+    int data_i, data_l;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load AES Key */
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+
+    if (od->octo_encklen == 16) {
+	CVMX_MT_AES_KEY(0x0, 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 24) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 32) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+    CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
+
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
+
+    while (crypt_off > 0) {
+	IOV_CONSUME(iov, data, data_i, data_l);
+	crypt_off -= 8;
+    }
+
+    while (crypt_len > 0) {
+	pdata = data;
+	CVMX_MT_AES_DEC_CBC0(*data);
+	IOV_CONSUME(iov, data, data_i, data_l);
+	CVMX_MT_AES_DEC_CBC1(*data);
+	CVMX_MF_AES_RESULT(*pdata, 0);
+	CVMX_MF_AES_RESULT(*data, 1);
+	IOV_CONSUME(iov, data, data_i, data_l);
+	crypt_len -= 16;
+    }
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+/****************************************************************************/
+/* MD5 */
+
+int
+octo_null_md5_encrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    register int next = 0;
+    uint64_t *data;
+    uint64_t tmp1, tmp2;
+    int data_i, data_l, alen = auth_len;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
+	    (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data, data_i, data_l);
+
+    s = octeon_crypto_enable();
+
+    /* Load MD5 IV */
+    CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
+
+    while (auth_off > 0) {
+	IOV_CONSUME(iov, data, data_i, data_l);
+	auth_off -= 8;
+    }
+
+    while (auth_len > 0) {
+	CVM_LOAD_MD5_UNIT(*data, next);
+	auth_len -= 8;
+	IOV_CONSUME(iov, data, data_i, data_l);
+    }
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_MD5_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* Finish Inner hash */
+    while (next != 7) {
+	CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
+    }
+    CVMX_ES64(tmp1, ((alen + 64) << 3));
+    CVM_LOAD_MD5_UNIT(tmp1, next);
+
+    /* Get the inner hash of HMAC */
+    CVMX_MF_HSH_IV(tmp1, 0);
+    CVMX_MF_HSH_IV(tmp2, 1);
+
+    /* Initialize hash unit */
+    CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
+
+    CVMX_MT_HSH_DAT(tmp1, 0);
+    CVMX_MT_HSH_DAT(tmp2, 1);
+    CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
+    CVMX_MT_HSH_DATZ(3);
+    CVMX_MT_HSH_DATZ(4);
+    CVMX_MT_HSH_DATZ(5);
+    CVMX_MT_HSH_DATZ(6);
+    CVMX_ES64(tmp1, ((64 + 16) << 3));
+    CVMX_MT_HSH_STARTMD5(tmp1);
+
+    /* save the HMAC */
+    IOV_INIT(iov, data, data_i, data_l);
+    while (icv_off > 0) {
+	IOV_CONSUME(iov, data, data_i, data_l);
+	icv_off -= 8;
+    }
+    CVMX_MF_HSH_IV(*data, 0);
+    IOV_CONSUME(iov, data, data_i, data_l);
+    CVMX_MF_HSH_IV(tmp1, 1);
+    *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+/****************************************************************************/
+/* SHA1 */
+
+int
+octo_null_sha1_encrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    register int next = 0;
+    uint64_t *data;
+    uint64_t tmp1, tmp2, tmp3;
+    int data_i, data_l, alen = auth_len;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
+	    (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data, data_i, data_l);
+
+    s = octeon_crypto_enable();
+
+    /* Load SHA1 IV */
+    CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
+    CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
+
+    while (auth_off > 0) {
+	IOV_CONSUME(iov, data, data_i, data_l);
+	auth_off -= 8;
+    }
+
+    while (auth_len > 0) {
+	CVM_LOAD_SHA_UNIT(*data, next);
+	auth_len -= 8;
+	IOV_CONSUME(iov, data, data_i, data_l);
+    }
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_MD5_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* Finish Inner hash */
+    while (next != 7) {
+	CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
+    }
+	CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
+
+    /* Get the inner hash of HMAC */
+    CVMX_MF_HSH_IV(tmp1, 0);
+    CVMX_MF_HSH_IV(tmp2, 1);
+    tmp3 = 0;
+    CVMX_MF_HSH_IV(tmp3, 2);
+
+    /* Initialize hash unit */
+    CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
+    CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
+
+    CVMX_MT_HSH_DAT(tmp1, 0);
+    CVMX_MT_HSH_DAT(tmp2, 1);
+    tmp3 |= 0x0000000080000000;
+    CVMX_MT_HSH_DAT(tmp3, 2);
+    CVMX_MT_HSH_DATZ(3);
+    CVMX_MT_HSH_DATZ(4);
+    CVMX_MT_HSH_DATZ(5);
+    CVMX_MT_HSH_DATZ(6);
+    CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
+
+    /* save the HMAC */
+    IOV_INIT(iov, data, data_i, data_l);
+    while (icv_off > 0) {
+	IOV_CONSUME(iov, data, data_i, data_l);
+	icv_off -= 8;
+    }
+    CVMX_MF_HSH_IV(*data, 0);
+    IOV_CONSUME(iov, data, data_i, data_l);
+    CVMX_MF_HSH_IV(tmp1, 1);
+    *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+/****************************************************************************/
+/* DES MD5 */
+
+int
+octo_des_cbc_md5_encrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    register int next = 0;
+    union {
+	uint32_t data32[2];
+	uint64_t data64[1];
+    } mydata;
+    uint64_t *data = &mydata.data64[0];
+    uint32_t *data32;
+    uint64_t tmp1, tmp2;
+    int data_i, data_l, alen = auth_len;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
+	    (crypt_len  & 0x7) ||
+	    (auth_len  & 0x7) ||
+	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data32, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load 3DES Key */
+    CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    if (od->octo_encklen == 24) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+    } else if (od->octo_encklen == 8) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+
+    CVMX_MT_3DES_IV(* (uint64_t *) ivp);
+
+    /* Load MD5 IV */
+    CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
+
+    while (crypt_off > 0 && auth_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	crypt_off -= 4;
+	auth_off -= 4;
+    }
+
+    while (crypt_len > 0 || auth_len > 0) {
+    	uint32_t *first = data32;
+	mydata.data32[0] = *first;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	mydata.data32[1] = *data32;
+    	if (crypt_off <= 0) {
+	    if (crypt_len > 0) {
+		CVMX_MT_3DES_ENC_CBC(*data);
+		CVMX_MF_3DES_RESULT(*data);
+		crypt_len -= 8;
+	    }
+	} else
+	    crypt_off -= 8;
+    	if (auth_off <= 0) {
+	    if (auth_len > 0) {
+		CVM_LOAD_MD5_UNIT(*data, next);
+		auth_len -= 8;
+	    }
+	} else
+	    auth_off -= 8;
+	*first = mydata.data32[0];
+	*data32 = mydata.data32[1];
+	IOV_CONSUME(iov, data32, data_i, data_l);
+    }
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_MD5_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* Finish Inner hash */
+    while (next != 7) {
+	CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
+    }
+    CVMX_ES64(tmp1, ((alen + 64) << 3));
+    CVM_LOAD_MD5_UNIT(tmp1, next);
+
+    /* Get the inner hash of HMAC */
+    CVMX_MF_HSH_IV(tmp1, 0);
+    CVMX_MF_HSH_IV(tmp2, 1);
+
+    /* Initialize hash unit */
+    CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
+
+    CVMX_MT_HSH_DAT(tmp1, 0);
+    CVMX_MT_HSH_DAT(tmp2, 1);
+    CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
+    CVMX_MT_HSH_DATZ(3);
+    CVMX_MT_HSH_DATZ(4);
+    CVMX_MT_HSH_DATZ(5);
+    CVMX_MT_HSH_DATZ(6);
+    CVMX_ES64(tmp1, ((64 + 16) << 3));
+    CVMX_MT_HSH_STARTMD5(tmp1);
+
+    /* save the HMAC */
+    IOV_INIT(iov, data32, data_i, data_l);
+    while (icv_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	icv_off -= 4;
+    }
+    CVMX_MF_HSH_IV(tmp1, 0);
+    *data32 = (uint32_t) (tmp1 >> 32);
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    *data32 = (uint32_t) tmp1;
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    CVMX_MF_HSH_IV(tmp1, 1);
+    *data32 = (uint32_t) (tmp1 >> 32);
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+int
+octo_des_cbc_md5_decrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    register int next = 0;
+    union {
+	uint32_t data32[2];
+	uint64_t data64[1];
+    } mydata;
+    uint64_t *data = &mydata.data64[0];
+    uint32_t *data32;
+    uint64_t tmp1, tmp2;
+    int data_i, data_l, alen = auth_len;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
+	    (crypt_len  & 0x7) ||
+	    (auth_len  & 0x7) ||
+	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data32, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load 3DES Key */
+    CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    if (od->octo_encklen == 24) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+    } else if (od->octo_encklen == 8) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+
+    CVMX_MT_3DES_IV(* (uint64_t *) ivp);
+
+    /* Load MD5 IV */
+    CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
+
+    while (crypt_off > 0 && auth_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	crypt_off -= 4;
+	auth_off -= 4;
+    }
+
+    while (crypt_len > 0 || auth_len > 0) {
+    	uint32_t *first = data32;
+	mydata.data32[0] = *first;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	mydata.data32[1] = *data32;
+    	if (auth_off <= 0) {
+	    if (auth_len > 0) {
+		CVM_LOAD_MD5_UNIT(*data, next);
+		auth_len -= 8;
+	    }
+	} else
+	    auth_off -= 8;
+    	if (crypt_off <= 0) {
+	    if (crypt_len > 0) {
+		CVMX_MT_3DES_DEC_CBC(*data);
+		CVMX_MF_3DES_RESULT(*data);
+		crypt_len -= 8;
+	    }
+	} else
+	    crypt_off -= 8;
+	*first = mydata.data32[0];
+	*data32 = mydata.data32[1];
+	IOV_CONSUME(iov, data32, data_i, data_l);
+    }
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_MD5_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* Finish Inner hash */
+    while (next != 7) {
+	CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
+    }
+    CVMX_ES64(tmp1, ((alen + 64) << 3));
+    CVM_LOAD_MD5_UNIT(tmp1, next);
+
+    /* Get the inner hash of HMAC */
+    CVMX_MF_HSH_IV(tmp1, 0);
+    CVMX_MF_HSH_IV(tmp2, 1);
+
+    /* Initialize hash unit */
+    CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
+
+    CVMX_MT_HSH_DAT(tmp1, 0);
+    CVMX_MT_HSH_DAT(tmp2, 1);
+    CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
+    CVMX_MT_HSH_DATZ(3);
+    CVMX_MT_HSH_DATZ(4);
+    CVMX_MT_HSH_DATZ(5);
+    CVMX_MT_HSH_DATZ(6);
+    CVMX_ES64(tmp1, ((64 + 16) << 3));
+    CVMX_MT_HSH_STARTMD5(tmp1);
+
+    /* save the HMAC */
+    IOV_INIT(iov, data32, data_i, data_l);
+    while (icv_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	icv_off -= 4;
+    }
+    CVMX_MF_HSH_IV(tmp1, 0);
+    *data32 = (uint32_t) (tmp1 >> 32);
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    *data32 = (uint32_t) tmp1;
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    CVMX_MF_HSH_IV(tmp1, 1);
+    *data32 = (uint32_t) (tmp1 >> 32);
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+/****************************************************************************/
+/* DES SHA */
+
+int
+octo_des_cbc_sha1_encrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    register int next = 0;
+    union {
+	uint32_t data32[2];
+	uint64_t data64[1];
+    } mydata;
+    uint64_t *data = &mydata.data64[0];
+    uint32_t *data32;
+    uint64_t tmp1, tmp2, tmp3;
+    int data_i, data_l, alen = auth_len;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
+	    (crypt_len  & 0x7) ||
+	    (auth_len  & 0x7) ||
+	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data32, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load 3DES Key */
+    CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    if (od->octo_encklen == 24) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+    } else if (od->octo_encklen == 8) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+
+    CVMX_MT_3DES_IV(* (uint64_t *) ivp);
+
+    /* Load SHA1 IV */
+    CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
+    CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
+
+    while (crypt_off > 0 && auth_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	crypt_off -= 4;
+	auth_off -= 4;
+    }
+
+    while (crypt_len > 0 || auth_len > 0) {
+    	uint32_t *first = data32;
+	mydata.data32[0] = *first;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	mydata.data32[1] = *data32;
+    	if (crypt_off <= 0) {
+	    if (crypt_len > 0) {
+		CVMX_MT_3DES_ENC_CBC(*data);
+		CVMX_MF_3DES_RESULT(*data);
+		crypt_len -= 8;
+	    }
+	} else
+	    crypt_off -= 8;
+    	if (auth_off <= 0) {
+	    if (auth_len > 0) {
+		CVM_LOAD_SHA_UNIT(*data, next);
+		auth_len -= 8;
+	    }
+	} else
+	    auth_off -= 8;
+	*first = mydata.data32[0];
+	*data32 = mydata.data32[1];
+	IOV_CONSUME(iov, data32, data_i, data_l);
+    }
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_SHA_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* Finish Inner hash */
+    while (next != 7) {
+	CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
+    }
+	CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
+
+    /* Get the inner hash of HMAC */
+    CVMX_MF_HSH_IV(tmp1, 0);
+    CVMX_MF_HSH_IV(tmp2, 1);
+    tmp3 = 0;
+    CVMX_MF_HSH_IV(tmp3, 2);
+
+    /* Initialize hash unit */
+    CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
+    CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
+
+    CVMX_MT_HSH_DAT(tmp1, 0);
+    CVMX_MT_HSH_DAT(tmp2, 1);
+    tmp3 |= 0x0000000080000000;
+    CVMX_MT_HSH_DAT(tmp3, 2);
+    CVMX_MT_HSH_DATZ(3);
+    CVMX_MT_HSH_DATZ(4);
+    CVMX_MT_HSH_DATZ(5);
+    CVMX_MT_HSH_DATZ(6);
+    CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
+
+    /* save the HMAC */
+    IOV_INIT(iov, data32, data_i, data_l);
+    while (icv_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	icv_off -= 4;
+    }
+    CVMX_MF_HSH_IV(tmp1, 0);
+    *data32 = (uint32_t) (tmp1 >> 32);
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    *data32 = (uint32_t) tmp1;
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    CVMX_MF_HSH_IV(tmp1, 1);
+    *data32 = (uint32_t) (tmp1 >> 32);
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+int
+octo_des_cbc_sha1_decrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    register int next = 0;
+    union {
+	uint32_t data32[2];
+	uint64_t data64[1];
+    } mydata;
+    uint64_t *data = &mydata.data64[0];
+    uint32_t *data32;
+    uint64_t tmp1, tmp2, tmp3;
+    int data_i, data_l, alen = auth_len;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
+	    (crypt_len  & 0x7) ||
+	    (auth_len  & 0x7) ||
+	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data32, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load 3DES Key */
+    CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    if (od->octo_encklen == 24) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+    } else if (od->octo_encklen == 8) {
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
+	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+
+    CVMX_MT_3DES_IV(* (uint64_t *) ivp);
+
+    /* Load SHA1 IV */
+    CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
+    CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
+
+    while (crypt_off > 0 && auth_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	crypt_off -= 4;
+	auth_off -= 4;
+    }
+
+    while (crypt_len > 0 || auth_len > 0) {
+    	uint32_t *first = data32;
+	mydata.data32[0] = *first;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	mydata.data32[1] = *data32;
+    	if (auth_off <= 0) {
+	    if (auth_len > 0) {
+		CVM_LOAD_SHA_UNIT(*data, next);
+		auth_len -= 8;
+	    }
+	} else
+	    auth_off -= 8;
+    	if (crypt_off <= 0) {
+	    if (crypt_len > 0) {
+		CVMX_MT_3DES_DEC_CBC(*data);
+		CVMX_MF_3DES_RESULT(*data);
+		crypt_len -= 8;
+	    }
+	} else
+	    crypt_off -= 8;
+	*first = mydata.data32[0];
+	*data32 = mydata.data32[1];
+	IOV_CONSUME(iov, data32, data_i, data_l);
+    }
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_SHA_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* Finish Inner hash */
+    while (next != 7) {
+	CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
+    }
+	CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
+
+    /* Get the inner hash of HMAC */
+    CVMX_MF_HSH_IV(tmp1, 0);
+    CVMX_MF_HSH_IV(tmp2, 1);
+    tmp3 = 0;
+    CVMX_MF_HSH_IV(tmp3, 2);
+
+    /* Initialize hash unit */
+    CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
+    CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
+
+    CVMX_MT_HSH_DAT(tmp1, 0);
+    CVMX_MT_HSH_DAT(tmp2, 1);
+    tmp3 |= 0x0000000080000000;
+    CVMX_MT_HSH_DAT(tmp3, 2);
+    CVMX_MT_HSH_DATZ(3);
+    CVMX_MT_HSH_DATZ(4);
+    CVMX_MT_HSH_DATZ(5);
+    CVMX_MT_HSH_DATZ(6);
+    CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
+    /* save the HMAC */
+    IOV_INIT(iov, data32, data_i, data_l);
+    while (icv_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	icv_off -= 4;
+    }
+    CVMX_MF_HSH_IV(tmp1, 0);
+    *data32 = (uint32_t) (tmp1 >> 32);
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    *data32 = (uint32_t) tmp1;
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    CVMX_MF_HSH_IV(tmp1, 1);
+    *data32 = (uint32_t) (tmp1 >> 32);
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+/****************************************************************************/
+/* AES MD5 */
+
+int
+octo_aes_cbc_md5_encrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    register int next = 0;
+    union {
+	uint32_t data32[2];
+	uint64_t data64[1];
+    } mydata[2];
+    uint64_t *pdata = &mydata[0].data64[0];
+    uint64_t *data =  &mydata[1].data64[0];
+    uint32_t *data32;
+    uint64_t tmp1, tmp2;
+    int data_i, data_l, alen = auth_len;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
+	    (crypt_len  & 0x7) ||
+	    (auth_len  & 0x7) ||
+	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data32, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load AES Key */
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+
+    if (od->octo_encklen == 16) {
+	CVMX_MT_AES_KEY(0x0, 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 24) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 32) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+    CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
+
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
+
+    /* Load MD5 IV */
+    CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
+
+    while (crypt_off > 0 && auth_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	crypt_off -= 4;
+	auth_off -= 4;
+    }
+
+    while (crypt_len > 0 || auth_len > 0) {
+    	uint32_t *pdata32[3];
+
+	pdata32[0] = data32;
+	mydata[0].data32[0] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+
+	pdata32[1] = data32;
+	mydata[0].data32[1] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+
+	pdata32[2] = data32;
+	mydata[1].data32[0] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+
+	mydata[1].data32[1] = *data32;
+
+
+    	if (crypt_off <= 0) {
+	    if (crypt_len > 0) {
+		CVMX_MT_AES_ENC_CBC0(*pdata);
+		CVMX_MT_AES_ENC_CBC1(*data);
+		CVMX_MF_AES_RESULT(*pdata, 0);
+		CVMX_MF_AES_RESULT(*data, 1);
+		crypt_len -= 16;
+	    }
+	} else
+	    crypt_off -= 16;
+
+    	if (auth_off <= 0) {
+	    if (auth_len > 0) {
+		CVM_LOAD_MD5_UNIT(*pdata, next);
+		CVM_LOAD_MD5_UNIT(*data, next);
+		auth_len -= 16;
+	    }
+	} else
+	    auth_off -= 16;
+
+	*pdata32[0] = mydata[0].data32[0];
+	*pdata32[1] = mydata[0].data32[1];
+	*pdata32[2] = mydata[1].data32[0];
+	*data32     = mydata[1].data32[1];
+
+	IOV_CONSUME(iov, data32, data_i, data_l);
+    }
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_MD5_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* Finish Inner hash */
+    while (next != 7) {
+	CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
+    }
+    CVMX_ES64(tmp1, ((alen + 64) << 3));
+    CVM_LOAD_MD5_UNIT(tmp1, next);
+
+    /* Get the inner hash of HMAC */
+    CVMX_MF_HSH_IV(tmp1, 0);
+    CVMX_MF_HSH_IV(tmp2, 1);
+
+    /* Initialize hash unit */
+    CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
+
+    CVMX_MT_HSH_DAT(tmp1, 0);
+    CVMX_MT_HSH_DAT(tmp2, 1);
+    CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
+    CVMX_MT_HSH_DATZ(3);
+    CVMX_MT_HSH_DATZ(4);
+    CVMX_MT_HSH_DATZ(5);
+    CVMX_MT_HSH_DATZ(6);
+    CVMX_ES64(tmp1, ((64 + 16) << 3));
+    CVMX_MT_HSH_STARTMD5(tmp1);
+
+    /* save the HMAC */
+    IOV_INIT(iov, data32, data_i, data_l);
+    while (icv_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	icv_off -= 4;
+    }
+    CVMX_MF_HSH_IV(tmp1, 0);
+    *data32 = (uint32_t) (tmp1 >> 32);
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    *data32 = (uint32_t) tmp1;
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    CVMX_MF_HSH_IV(tmp1, 1);
+    *data32 = (uint32_t) (tmp1 >> 32);
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+int
+octo_aes_cbc_md5_decrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    register int next = 0;
+    union {
+	uint32_t data32[2];
+	uint64_t data64[1];
+    } mydata[2];
+    uint64_t *pdata = &mydata[0].data64[0];
+    uint64_t *data =  &mydata[1].data64[0];
+    uint32_t *data32;
+    uint64_t tmp1, tmp2;
+    int data_i, data_l, alen = auth_len;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
+	    (crypt_len  & 0x7) ||
+	    (auth_len  & 0x7) ||
+	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data32, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load AES Key */
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+
+    if (od->octo_encklen == 16) {
+	CVMX_MT_AES_KEY(0x0, 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 24) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 32) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+    CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
+
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
+
+    /* Load MD5 IV */
+    CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
+
+    while (crypt_off > 0 && auth_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	crypt_off -= 4;
+	auth_off -= 4;
+    }
+
+    while (crypt_len > 0 || auth_len > 0) {
+    	uint32_t *pdata32[3];
+
+	pdata32[0] = data32;
+	mydata[0].data32[0] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	pdata32[1] = data32;
+	mydata[0].data32[1] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	pdata32[2] = data32;
+	mydata[1].data32[0] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	mydata[1].data32[1] = *data32;
+
+    	if (auth_off <= 0) {
+	    if (auth_len > 0) {
+		CVM_LOAD_MD5_UNIT(*pdata, next);
+		CVM_LOAD_MD5_UNIT(*data, next);
+		auth_len -= 16;
+	    }
+	} else
+	    auth_off -= 16;
+
+    	if (crypt_off <= 0) {
+	    if (crypt_len > 0) {
+		CVMX_MT_AES_DEC_CBC0(*pdata);
+		CVMX_MT_AES_DEC_CBC1(*data);
+		CVMX_MF_AES_RESULT(*pdata, 0);
+		CVMX_MF_AES_RESULT(*data, 1);
+		crypt_len -= 16;
+	    }
+	} else
+	    crypt_off -= 16;
+
+	*pdata32[0] = mydata[0].data32[0];
+	*pdata32[1] = mydata[0].data32[1];
+	*pdata32[2] = mydata[1].data32[0];
+	*data32     = mydata[1].data32[1];
+
+	IOV_CONSUME(iov, data32, data_i, data_l);
+    }
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_MD5_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* Finish Inner hash */
+    while (next != 7) {
+	CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
+    }
+    CVMX_ES64(tmp1, ((alen + 64) << 3));
+    CVM_LOAD_MD5_UNIT(tmp1, next);
+
+    /* Get the inner hash of HMAC */
+    CVMX_MF_HSH_IV(tmp1, 0);
+    CVMX_MF_HSH_IV(tmp2, 1);
+
+    /* Initialize hash unit */
+    CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
+
+    CVMX_MT_HSH_DAT(tmp1, 0);
+    CVMX_MT_HSH_DAT(tmp2, 1);
+    CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
+    CVMX_MT_HSH_DATZ(3);
+    CVMX_MT_HSH_DATZ(4);
+    CVMX_MT_HSH_DATZ(5);
+    CVMX_MT_HSH_DATZ(6);
+    CVMX_ES64(tmp1, ((64 + 16) << 3));
+    CVMX_MT_HSH_STARTMD5(tmp1);
+
+    /* save the HMAC */
+    IOV_INIT(iov, data32, data_i, data_l);
+    while (icv_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	icv_off -= 4;
+    }
+    CVMX_MF_HSH_IV(tmp1, 0);
+    *data32 = (uint32_t) (tmp1 >> 32);
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    *data32 = (uint32_t) tmp1;
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    CVMX_MF_HSH_IV(tmp1, 1);
+    *data32 = (uint32_t) (tmp1 >> 32);
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+/****************************************************************************/
+/* AES SHA1 */
+
+int
+octo_aes_cbc_sha1_encrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    register int next = 0;
+    union {
+	uint32_t data32[2];
+	uint64_t data64[1];
+    } mydata[2];
+    uint64_t *pdata = &mydata[0].data64[0];
+    uint64_t *data =  &mydata[1].data64[0];
+    uint32_t *data32;
+    uint64_t tmp1, tmp2, tmp3;
+    int data_i, data_l, alen = auth_len;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
+	    (crypt_len  & 0x7) ||
+	    (auth_len  & 0x7) ||
+	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data32, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load AES Key */
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+
+    if (od->octo_encklen == 16) {
+	CVMX_MT_AES_KEY(0x0, 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 24) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 32) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+    CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
+
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
+
+    /* Load SHA IV */
+    CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
+    CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
+
+    while (crypt_off > 0 && auth_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	crypt_off -= 4;
+	auth_off -= 4;
+    }
+
+    while (crypt_len > 0 || auth_len > 0) {
+    	uint32_t *pdata32[3];
+
+	pdata32[0] = data32;
+	mydata[0].data32[0] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	pdata32[1] = data32;
+	mydata[0].data32[1] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	pdata32[2] = data32;
+	mydata[1].data32[0] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	mydata[1].data32[1] = *data32;
+
+
+    	if (crypt_off <= 0) {
+	    if (crypt_len > 0) {
+		CVMX_MT_AES_ENC_CBC0(*pdata);
+		CVMX_MT_AES_ENC_CBC1(*data);
+		CVMX_MF_AES_RESULT(*pdata, 0);
+		CVMX_MF_AES_RESULT(*data, 1);
+		crypt_len -= 16;
+	    }
+	} else
+	    crypt_off -= 16;
+
+    	if (auth_off <= 0) {
+	    if (auth_len > 0) {
+		CVM_LOAD_SHA_UNIT(*pdata, next);
+		CVM_LOAD_SHA_UNIT(*data, next);
+		auth_len -= 16;
+	    }
+	} else
+	    auth_off -= 16;
+
+	*pdata32[0] = mydata[0].data32[0];
+	*pdata32[1] = mydata[0].data32[1];
+	*pdata32[2] = mydata[1].data32[0];
+	*data32     = mydata[1].data32[1];
+
+	IOV_CONSUME(iov, data32, data_i, data_l);
+    }
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_SHA_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* Finish Inner hash */
+    while (next != 7) {
+	CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
+    }
+	CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
+
+    /* Get the inner hash of HMAC */
+    CVMX_MF_HSH_IV(tmp1, 0);
+    CVMX_MF_HSH_IV(tmp2, 1);
+    tmp3 = 0;
+    CVMX_MF_HSH_IV(tmp3, 2);
+
+    /* Initialize hash unit */
+    CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
+    CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
+
+    CVMX_MT_HSH_DAT(tmp1, 0);
+    CVMX_MT_HSH_DAT(tmp2, 1);
+    tmp3 |= 0x0000000080000000;
+    CVMX_MT_HSH_DAT(tmp3, 2);
+    CVMX_MT_HSH_DATZ(3);
+    CVMX_MT_HSH_DATZ(4);
+    CVMX_MT_HSH_DATZ(5);
+    CVMX_MT_HSH_DATZ(6);
+    CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_MD5_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* save the HMAC */
+    IOV_INIT(iov, data32, data_i, data_l);
+    while (icv_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	icv_off -= 4;
+    }
+    CVMX_MF_HSH_IV(tmp1, 0);
+    *data32 = (uint32_t) (tmp1 >> 32);
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    *data32 = (uint32_t) tmp1;
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    CVMX_MF_HSH_IV(tmp1, 1);
+    *data32 = (uint32_t) (tmp1 >> 32);
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+int
+octo_aes_cbc_sha1_decrypt(
+    struct octo_sess *od,
+    struct iovec *iov, size_t iovcnt, size_t iovlen,
+    int auth_off, int auth_len,
+    int crypt_off, int crypt_len,
+    int icv_off, uint8_t *ivp)
+{
+    register int next = 0;
+    union {
+	uint32_t data32[2];
+	uint64_t data64[1];
+    } mydata[2];
+    uint64_t *pdata = &mydata[0].data64[0];
+    uint64_t *data =  &mydata[1].data64[0];
+    uint32_t *data32;
+    uint64_t tmp1, tmp2, tmp3;
+    int data_i, data_l, alen = auth_len;
+    register_t s;
+
+    dprintf("%s()\n", __func__);
+
+    if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
+	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
+	    (crypt_len  & 0x7) ||
+	    (auth_len  & 0x7) ||
+	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
+	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%d "
+		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
+		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	return -EINVAL;
+    }
+
+    IOV_INIT(iov, data32, data_i, data_l);
+
+    CVMX_PREFETCH0(ivp);
+    CVMX_PREFETCH0(od->octo_enckey);
+
+    s = octeon_crypto_enable();
+
+    /* load AES Key */
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
+    CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
+
+    if (od->octo_encklen == 16) {
+	CVMX_MT_AES_KEY(0x0, 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 24) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(0x0, 3);
+    } else if (od->octo_encklen == 32) {
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
+	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
+    } else {
+	octeon_crypto_disable(s);
+	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
+	return -EINVAL;
+    }
+    CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
+
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
+    CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
+
+    /* Load MD5 IV */
+    CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
+    CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
+
+    while (crypt_off > 0 && auth_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	crypt_off -= 4;
+	auth_off -= 4;
+    }
+
+    while (crypt_len > 0 || auth_len > 0) {
+    	uint32_t *pdata32[3];
+
+	pdata32[0] = data32;
+	mydata[0].data32[0] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	pdata32[1] = data32;
+	mydata[0].data32[1] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	pdata32[2] = data32;
+	mydata[1].data32[0] = *data32;
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	mydata[1].data32[1] = *data32;
+
+    	if (auth_off <= 0) {
+	    if (auth_len > 0) {
+		CVM_LOAD_SHA_UNIT(*pdata, next);
+		CVM_LOAD_SHA_UNIT(*data, next);
+		auth_len -= 16;
+	    }
+	} else
+	    auth_off -= 16;
+
+    	if (crypt_off <= 0) {
+	    if (crypt_len > 0) {
+		CVMX_MT_AES_DEC_CBC0(*pdata);
+		CVMX_MT_AES_DEC_CBC1(*data);
+		CVMX_MF_AES_RESULT(*pdata, 0);
+		CVMX_MF_AES_RESULT(*data, 1);
+		crypt_len -= 16;
+	    }
+	} else
+	    crypt_off -= 16;
+
+	*pdata32[0] = mydata[0].data32[0];
+	*pdata32[1] = mydata[0].data32[1];
+	*pdata32[2] = mydata[1].data32[0];
+	*data32     = mydata[1].data32[1];
+
+	IOV_CONSUME(iov, data32, data_i, data_l);
+    }
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_SHA_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* Finish Inner hash */
+    while (next != 7) {
+	CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
+    }
+	CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
+
+    /* Get the inner hash of HMAC */
+    CVMX_MF_HSH_IV(tmp1, 0);
+    CVMX_MF_HSH_IV(tmp2, 1);
+    tmp3 = 0;
+    CVMX_MF_HSH_IV(tmp3, 2);
+
+    /* Initialize hash unit */
+    CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
+    CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
+    CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
+
+    CVMX_MT_HSH_DAT(tmp1, 0);
+    CVMX_MT_HSH_DAT(tmp2, 1);
+    tmp3 |= 0x0000000080000000;
+    CVMX_MT_HSH_DAT(tmp3, 2);
+    CVMX_MT_HSH_DATZ(3);
+    CVMX_MT_HSH_DATZ(4);
+    CVMX_MT_HSH_DATZ(5);
+    CVMX_MT_HSH_DATZ(6);
+    CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
+
+    /* finish the hash */
+    CVMX_PREFETCH0(od->octo_hmouter);
+#if 0
+    if (__predict_false(inplen)) {
+	uint64_t tmp = 0;
+	uint8_t *p = (uint8_t *) & tmp;
+	p[inplen] = 0x80;
+	do {
+	    inplen--;
+	    p[inplen] = ((uint8_t *) data)[inplen];
+	} while (inplen);
+	CVM_LOAD_MD5_UNIT(tmp, next);
+    } else {
+	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+    }
+#else
+    CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
+#endif
+
+    /* save the HMAC */
+    IOV_INIT(iov, data32, data_i, data_l);
+    while (icv_off > 0) {
+	IOV_CONSUME(iov, data32, data_i, data_l);
+	icv_off -= 4;
+    }
+    CVMX_MF_HSH_IV(tmp1, 0);
+    *data32 = (uint32_t) (tmp1 >> 32);
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    *data32 = (uint32_t) tmp1;
+    IOV_CONSUME(iov, data32, data_i, data_l);
+    CVMX_MF_HSH_IV(tmp1, 1);
+    *data32 = (uint32_t) (tmp1 >> 32);
+
+    octeon_crypto_disable(s);
+    return 0;
+}
+
+/****************************************************************************/
diff --git a/sys/mips/cavium/cryptocteon/cryptocteon.c b/sys/mips/cavium/cryptocteon/cryptocteon.c
new file mode 100644
index 0000000..16a0bb1
--- /dev/null
+++ b/sys/mips/cavium/cryptocteon/cryptocteon.c
@@ -0,0 +1,526 @@
+/*
+ * Octeon Crypto for OCF
+ *
+ * Written by David McCullough <david_mccullough@securecomputing.com>
+ * Copyright (C) 2009 David McCullough
+ *
+ * LICENSE TERMS
+ *
+ * The free distribution and use of this software in both source and binary
+ * form is allowed (with or without changes) provided that:
+ *
+ *   1. distributions of this source code include the above copyright
+ *      notice, this list of conditions and the following disclaimer;
+ *
+ *   2. distributions in binary form include the above copyright
+ *      notice, this list of conditions and the following disclaimer
+ *      in the documentation and/or other associated materials;
+ *
+ *   3. the copyright holder's name is not used to endorse products
+ *      built using this software without specific written permission.
+ *
+ * DISCLAIMER
+ *
+ * This software is provided 'as is' with no explicit or implied warranties
+ * in respect of its properties, including, but not limited to, correctness
+ * and/or fitness for purpose.
+ * ---------------------------------------------------------------------------
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/uio.h>
+
+#include <opencrypto/cryptodev.h>
+
+#include <contrib/octeon-sdk/cvmx.h>
+
+#include <mips/cavium/cryptocteon/cryptocteonvar.h>
+
+#include "cryptodev_if.h"
+
+struct cryptocteon_softc {
+	int32_t			sc_cid;		/* opencrypto id */
+	struct octo_sess	**sc_sessions;
+	uint32_t		sc_sesnum;
+};
+
+int cryptocteon_debug = 0;
+TUNABLE_INT("hw.cryptocteon.debug", &cryptocteon_debug);
+
+static void cryptocteon_identify(driver_t *, device_t);
+static int cryptocteon_probe(device_t);
+static int cryptocteon_attach(device_t);
+
+static int cryptocteon_process(device_t, struct cryptop *, int);
+static int cryptocteon_newsession(device_t, u_int32_t *, struct cryptoini *);
+static int cryptocteon_freesession(device_t, u_int64_t);
+
+static void
+cryptocteon_identify(driver_t *drv, device_t parent)
+{
+	if (octeon_has_feature(OCTEON_FEATURE_CRYPTO))
+		BUS_ADD_CHILD(parent, 0, "cryptocteon", 0);
+}
+
+static int
+cryptocteon_probe(device_t dev)
+{
+	device_set_desc(dev, "Octeon Secure Coprocessor");
+	return (0);
+}
+
+static int
+cryptocteon_attach(device_t dev)
+{
+	struct cryptocteon_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
+	if (sc->sc_cid < 0) {
+		device_printf(dev, "crypto_get_driverid ret %d\n", sc->sc_cid);
+		return (ENXIO);
+	}
+
+	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
+	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
+	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
+	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
+	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
+
+	return (0);
+}
+
+/*
+ * Generate a new octo session.  We artifically limit it to a single
+ * hash/cipher or hash-cipher combo just to make it easier, most callers
+ * do not expect more than this anyway.
+ */
+static int
+cryptocteon_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
+{
+	struct cryptoini *c, *encini = NULL, *macini = NULL;
+	struct cryptocteon_softc *sc;
+	struct octo_sess **ocd;
+	int i;
+
+	sc = device_get_softc(dev);
+
+	if (sid == NULL || cri == NULL || sc == NULL)
+		return (EINVAL);
+
+	/*
+	 * To keep it simple, we only handle hash, cipher or hash/cipher in a
+	 * session,  you cannot currently do multiple ciphers/hashes in one
+	 * session even though it would be possibel to code this driver to
+	 * handle it.
+	 */
+	for (i = 0, c = cri; c && i < 2; i++) {
+		if (c->cri_alg == CRYPTO_MD5_HMAC ||
+		    c->cri_alg == CRYPTO_SHA1_HMAC ||
+		    c->cri_alg == CRYPTO_NULL_HMAC) {
+			if (macini) {
+				break;
+			}
+			macini = c;
+		}
+		if (c->cri_alg == CRYPTO_DES_CBC ||
+		    c->cri_alg == CRYPTO_3DES_CBC ||
+		    c->cri_alg == CRYPTO_AES_CBC ||
+		    c->cri_alg == CRYPTO_NULL_CBC) {
+			if (encini) {
+				break;
+			}
+			encini = c;
+		}
+		c = c->cri_next;
+	}
+	if (!macini && !encini) {
+		dprintf("%s,%d - EINVAL bad cipher/hash or combination\n",
+				__FILE__, __LINE__);
+		return EINVAL;
+	}
+	if (c) {
+		dprintf("%s,%d - EINVAL cannot handle chained cipher/hash combos\n",
+				__FILE__, __LINE__);
+		return EINVAL;
+	}
+
+	/*
+	 * So we have something we can do, lets setup the session
+	 */
+
+	if (sc->sc_sessions) {
+		for (i = 1; i < sc->sc_sesnum; i++)
+			if (sc->sc_sessions[i] == NULL)
+				break;
+	} else
+		i = 1;		/* NB: to silence compiler warning */
+
+	if (sc->sc_sessions == NULL || i == sc->sc_sesnum) {
+		if (sc->sc_sessions == NULL) {
+			i = 1; /* We leave sc->sc_sessions[0] empty */
+			sc->sc_sesnum = CRYPTO_SW_SESSIONS;
+		} else
+			sc->sc_sesnum *= 2;
+
+		ocd = malloc(sc->sc_sesnum * sizeof(struct octo_sess *),
+			     M_DEVBUF, M_NOWAIT | M_ZERO);
+		if (ocd == NULL) {
+			/* Reset session number */
+			if (sc->sc_sesnum == CRYPTO_SW_SESSIONS)
+				sc->sc_sesnum = 0;
+			else
+				sc->sc_sesnum /= 2;
+			dprintf("%s,%d: ENOBUFS\n", __FILE__, __LINE__);
+			return ENOBUFS;
+		}
+
+		/* Copy existing sessions */
+		if (sc->sc_sessions) {
+			memcpy(ocd, sc->sc_sessions,
+			    (sc->sc_sesnum / 2) * sizeof(struct octo_sess *));
+			free(sc->sc_sessions, M_DEVBUF);
+		}
+
+		sc->sc_sessions = ocd;
+	}
+
+	ocd = &sc->sc_sessions[i];
+	*sid = i;
+
+	*ocd = malloc(sizeof(struct octo_sess), M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (*ocd == NULL) {
+		cryptocteon_freesession(NULL, i);
+		dprintf("%s,%d: ENOBUFS\n", __FILE__, __LINE__);
+		return ENOBUFS;
+	}
+
+	if (encini && encini->cri_key) {
+		(*ocd)->octo_encklen = (encini->cri_klen + 7) / 8;
+		memcpy((*ocd)->octo_enckey, encini->cri_key, (*ocd)->octo_encklen);
+	}
+
+	if (macini && macini->cri_key) {
+		(*ocd)->octo_macklen = (macini->cri_klen + 7) / 8;
+		memcpy((*ocd)->octo_mackey, macini->cri_key, (*ocd)->octo_macklen);
+	}
+
+	(*ocd)->octo_mlen = 0;
+	if (encini && encini->cri_mlen)
+		(*ocd)->octo_mlen = encini->cri_mlen;
+	else if (macini && macini->cri_mlen)
+		(*ocd)->octo_mlen = macini->cri_mlen;
+	else
+		(*ocd)->octo_mlen = 12;
+
+	/*
+	 * point c at the enc if it exists, otherwise the mac
+	 */
+	c = encini ? encini : macini;
+
+	switch (c->cri_alg) {
+	case CRYPTO_DES_CBC:
+	case CRYPTO_3DES_CBC:
+		(*ocd)->octo_ivsize  = 8;
+		switch (macini ? macini->cri_alg : -1) {
+		case CRYPTO_MD5_HMAC:
+			(*ocd)->octo_encrypt = octo_des_cbc_md5_encrypt;
+			(*ocd)->octo_decrypt = octo_des_cbc_md5_decrypt;
+			octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner,
+					(*ocd)->octo_hmouter);
+			break;
+		case CRYPTO_SHA1_HMAC:
+			(*ocd)->octo_encrypt = octo_des_cbc_sha1_encrypt;
+			(*ocd)->octo_decrypt = octo_des_cbc_sha1_encrypt;
+			octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner,
+					(*ocd)->octo_hmouter);
+			break;
+		case -1:
+			(*ocd)->octo_encrypt = octo_des_cbc_encrypt;
+			(*ocd)->octo_decrypt = octo_des_cbc_decrypt;
+			break;
+		default:
+			cryptocteon_freesession(NULL, i);
+			dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
+			return EINVAL;
+		}
+		break;
+	case CRYPTO_AES_CBC:
+		(*ocd)->octo_ivsize  = 16;
+		switch (macini ? macini->cri_alg : -1) {
+		case CRYPTO_MD5_HMAC:
+			(*ocd)->octo_encrypt = octo_aes_cbc_md5_encrypt;
+			(*ocd)->octo_decrypt = octo_aes_cbc_md5_decrypt;
+			octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner,
+					(*ocd)->octo_hmouter);
+			break;
+		case CRYPTO_SHA1_HMAC:
+			(*ocd)->octo_encrypt = octo_aes_cbc_sha1_encrypt;
+			(*ocd)->octo_decrypt = octo_aes_cbc_sha1_decrypt;
+			octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner,
+					(*ocd)->octo_hmouter);
+			break;
+		case -1:
+			(*ocd)->octo_encrypt = octo_aes_cbc_encrypt;
+			(*ocd)->octo_decrypt = octo_aes_cbc_decrypt;
+			break;
+		default:
+			cryptocteon_freesession(NULL, i);
+			dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
+			return EINVAL;
+		}
+		break;
+	case CRYPTO_MD5_HMAC:
+		(*ocd)->octo_encrypt = octo_null_md5_encrypt;
+		(*ocd)->octo_decrypt = octo_null_md5_encrypt;
+		octo_calc_hash(0, macini->cri_key, (*ocd)->octo_hminner,
+				(*ocd)->octo_hmouter);
+		break;
+	case CRYPTO_SHA1_HMAC:
+		(*ocd)->octo_encrypt = octo_null_sha1_encrypt;
+		(*ocd)->octo_decrypt = octo_null_sha1_encrypt;
+		octo_calc_hash(1, macini->cri_key, (*ocd)->octo_hminner,
+				(*ocd)->octo_hmouter);
+		break;
+	default:
+		cryptocteon_freesession(NULL, i);
+		dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
+		return EINVAL;
+	}
+
+	(*ocd)->octo_encalg = encini ? encini->cri_alg : -1;
+	(*ocd)->octo_macalg = macini ? macini->cri_alg : -1;
+
+	return 0;
+}
+
+/*
+ * Free a session.
+ */
+static int
+cryptocteon_freesession(device_t dev, u_int64_t tid)
+{
+	struct cryptocteon_softc *sc;
+	u_int32_t sid = CRYPTO_SESID2LID(tid);
+
+	sc = device_get_softc(dev);
+
+	if (sc == NULL)
+		return (EINVAL);
+
+	if (sid > sc->sc_sesnum || sc->sc_sessions == NULL ||
+	    sc->sc_sessions[sid] == NULL)
+		return (EINVAL);
+
+	/* Silently accept and return */
+	if (sid == 0)
+		return(0);
+
+	if (sc->sc_sessions[sid])
+		free(sc->sc_sessions[sid], M_DEVBUF);
+	sc->sc_sessions[sid] = NULL;
+	return 0;
+}
+
+/*
+ * Process a request.
+ */
+static int
+cryptocteon_process(device_t dev, struct cryptop *crp, int hint)
+{
+	struct cryptodesc *crd;
+	struct octo_sess *od;
+	u_int32_t lid;
+	size_t iovcnt, iovlen;
+	struct mbuf *m = NULL;
+	struct uio *uiop = NULL;
+	struct cryptodesc *enccrd = NULL, *maccrd = NULL;
+	unsigned char *ivp = NULL;
+	unsigned char iv_data[HASH_MAX_LEN];
+	int auth_off = 0, auth_len = 0, crypt_off = 0, crypt_len = 0, icv_off = 0;
+	struct cryptocteon_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (sc == NULL || crp == NULL)
+		return EINVAL;
+
+	crp->crp_etype = 0;
+
+	if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
+		dprintf("%s,%d: EINVAL\n", __FILE__, __LINE__);
+		crp->crp_etype = EINVAL;
+		goto done;
+	}
+
+	lid = crp->crp_sid & 0xffffffff;
+	if (lid >= sc->sc_sesnum || lid == 0 || sc->sc_sessions == NULL ||
+	    sc->sc_sessions[lid] == NULL) {
+		crp->crp_etype = ENOENT;
+		dprintf("%s,%d: ENOENT\n", __FILE__, __LINE__);
+		goto done;
+	}
+	od = sc->sc_sessions[lid];
+
+	/*
+	 * do some error checking outside of the loop for m and IOV processing
+	 * this leaves us with valid m or uiop pointers for later
+	 */
+	if (crp->crp_flags & CRYPTO_F_IMBUF) {
+		unsigned frags;
+
+		m = (struct mbuf *) crp->crp_buf;
+		for (frags = 0; m != NULL; frags++)
+			m = m->m_next;
+
+		if (frags >= UIO_MAXIOV) {
+			printf("%s,%d: %d frags > UIO_MAXIOV", __FILE__, __LINE__, frags);
+			goto done;
+		}
+
+		m = (struct mbuf *) crp->crp_buf;
+	} else if (crp->crp_flags & CRYPTO_F_IOV) {
+		uiop = (struct uio *) crp->crp_buf;
+		if (uiop->uio_iovcnt > UIO_MAXIOV) {
+			printf("%s,%d: %d uio_iovcnt > UIO_MAXIOV", __FILE__, __LINE__,
+			       uiop->uio_iovcnt);
+			goto done;
+		}
+	}
+
+	/* point our enccrd and maccrd appropriately */
+	crd = crp->crp_desc;
+	if (crd->crd_alg == od->octo_encalg) enccrd = crd;
+	if (crd->crd_alg == od->octo_macalg) maccrd = crd;
+	crd = crd->crd_next;
+	if (crd) {
+		if (crd->crd_alg == od->octo_encalg) enccrd = crd;
+		if (crd->crd_alg == od->octo_macalg) maccrd = crd;
+		crd = crd->crd_next;
+	}
+	if (crd) {
+		crp->crp_etype = EINVAL;
+		dprintf("%s,%d: ENOENT - descriptors do not match session\n",
+				__FILE__, __LINE__);
+		goto done;
+	}
+
+	if (enccrd) {
+		if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
+			ivp = enccrd->crd_iv;
+		} else {
+			ivp = iv_data;
+			crypto_copydata(crp->crp_flags, crp->crp_buf,
+					enccrd->crd_inject, od->octo_ivsize, (caddr_t) ivp);
+		}
+
+		if (maccrd) {
+			auth_off = maccrd->crd_skip;
+			auth_len = maccrd->crd_len;
+			icv_off  = maccrd->crd_inject;
+		}
+
+		crypt_off = enccrd->crd_skip;
+		crypt_len = enccrd->crd_len;
+	} else { /* if (maccrd) */
+		auth_off = maccrd->crd_skip;
+		auth_len = maccrd->crd_len;
+		icv_off  = maccrd->crd_inject;
+	}
+
+	/*
+	 * setup the I/O vector to cover the buffer
+	 */
+	if (crp->crp_flags & CRYPTO_F_IMBUF) {
+		iovcnt = 0;
+		iovlen = 0;
+
+		while (m != NULL) {
+			od->octo_iov[iovcnt].iov_base = mtod(m, void *);
+			od->octo_iov[iovcnt].iov_len = m->m_len;
+
+			m = m->m_next;
+			iovlen += od->octo_iov[iovcnt++].iov_len;
+		}
+	} else if (crp->crp_flags & CRYPTO_F_IOV) {
+		iovlen = 0;
+		for (iovcnt = 0; iovcnt < uiop->uio_iovcnt; iovcnt++) {
+			od->octo_iov[iovcnt].iov_base = uiop->uio_iov[iovcnt].iov_base;
+			od->octo_iov[iovcnt].iov_len = uiop->uio_iov[iovcnt].iov_len;
+
+			iovlen += od->octo_iov[iovcnt].iov_len;
+		}
+	} else {
+		iovlen = crp->crp_ilen;
+		od->octo_iov[0].iov_base = crp->crp_buf;
+		od->octo_iov[0].iov_len = crp->crp_ilen;
+		iovcnt = 1;
+	}
+
+
+	/*
+	 * setup a new explicit key
+	 */
+	if (enccrd) {
+		if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
+			od->octo_encklen = (enccrd->crd_klen + 7) / 8;
+			memcpy(od->octo_enckey, enccrd->crd_key, od->octo_encklen);
+		}
+	}
+	if (maccrd) {
+		if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
+			od->octo_macklen = (maccrd->crd_klen + 7) / 8;
+			memcpy(od->octo_mackey, maccrd->crd_key, od->octo_macklen);
+			od->octo_mackey_set = 0;
+		}
+		if (!od->octo_mackey_set) {
+			octo_calc_hash(maccrd->crd_alg == CRYPTO_MD5_HMAC ? 0 : 1,
+				maccrd->crd_key, od->octo_hminner, od->octo_hmouter);
+			od->octo_mackey_set = 1;
+		}
+	}
+
+
+	if (!enccrd || (enccrd->crd_flags & CRD_F_ENCRYPT))
+		(*od->octo_encrypt)(od, od->octo_iov, iovcnt, iovlen,
+				auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+	else
+		(*od->octo_decrypt)(od, od->octo_iov, iovcnt, iovlen,
+				auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+
+done:
+	crypto_done(crp);
+	return 0;
+}
+
+static device_method_t cryptocteon_methods[] = {
+	/* device methods */
+	DEVMETHOD(device_identify,	cryptocteon_identify),
+	DEVMETHOD(device_probe,		cryptocteon_probe),
+	DEVMETHOD(device_attach,	cryptocteon_attach),
+
+	/* crypto device methods */
+	DEVMETHOD(cryptodev_newsession,	cryptocteon_newsession),
+	DEVMETHOD(cryptodev_freesession,cryptocteon_freesession),
+	DEVMETHOD(cryptodev_process,	cryptocteon_process),
+
+	{ 0, 0 }
+};
+
+static driver_t cryptocteon_driver = {
+	"cryptocteon",
+	cryptocteon_methods,
+	sizeof (struct cryptocteon_softc),
+};
+static devclass_t cryptocteon_devclass;
+DRIVER_MODULE(cryptocteon, nexus, cryptocteon_driver, cryptocteon_devclass, 0, 0);
diff --git a/sys/mips/cavium/cryptocteon/cryptocteonvar.h b/sys/mips/cavium/cryptocteon/cryptocteonvar.h
new file mode 100644
index 0000000..e722298
--- /dev/null
+++ b/sys/mips/cavium/cryptocteon/cryptocteonvar.h
@@ -0,0 +1,94 @@
+/*
+ * Octeon Crypto for OCF
+ *
+ * Written by David McCullough <david_mccullough@securecomputing.com>
+ * Copyright (C) 2009 David McCullough
+ *
+ * LICENSE TERMS
+ *
+ * The free distribution and use of this software in both source and binary
+ * form is allowed (with or without changes) provided that:
+ *
+ *   1. distributions of this source code include the above copyright
+ *      notice, this list of conditions and the following disclaimer;
+ *
+ *   2. distributions in binary form include the above copyright
+ *      notice, this list of conditions and the following disclaimer
+ *      in the documentation and/or other associated materials;
+ *
+ *   3. the copyright holder's name is not used to endorse products
+ *      built using this software without specific written permission.
+ *
+ * DISCLAIMER
+ *
+ * This software is provided 'as is' with no explicit or implied warranties
+ * in respect of its properties, including, but not limited to, correctness
+ * and/or fitness for purpose.
+ * ---------------------------------------------------------------------------
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_
+#define	_MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_
+
+struct octo_sess;
+
+typedef	int octo_encrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp);
+typedef	int octo_decrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp);
+
+struct octo_sess {
+	int					 octo_encalg;
+	#define MAX_CIPHER_KEYLEN	64
+	char				 octo_enckey[MAX_CIPHER_KEYLEN];
+	int					 octo_encklen;
+
+	int					 octo_macalg;
+	#define MAX_HASH_KEYLEN	64
+	char				 octo_mackey[MAX_HASH_KEYLEN];
+	int					 octo_macklen;
+	int					 octo_mackey_set;
+
+	int					 octo_mlen;
+	int					 octo_ivsize;
+
+	octo_encrypt_t				*octo_encrypt;
+	octo_decrypt_t				*octo_decrypt;
+
+	uint64_t			 octo_hminner[3];
+	uint64_t			 octo_hmouter[3];
+
+	struct iovec				octo_iov[UIO_MAXIOV];
+};
+
+#define	dprintf(fmt, ...)						\
+	do {								\
+		if (cryptocteon_debug)					\
+			printf("%s: " fmt, __func__, ## __VA_ARGS__);	\
+	} while (0)
+
+extern int cryptocteon_debug;
+
+void octo_calc_hash(uint8_t, unsigned char *, uint64_t *, uint64_t *);
+
+/* XXX Actually just hashing functions, not encryption.  */
+octo_encrypt_t octo_null_md5_encrypt;
+octo_encrypt_t octo_null_sha1_encrypt;
+
+octo_encrypt_t octo_des_cbc_encrypt;
+octo_encrypt_t octo_des_cbc_md5_encrypt;
+octo_encrypt_t octo_des_cbc_sha1_encrypt;
+
+octo_decrypt_t octo_des_cbc_decrypt;
+octo_decrypt_t octo_des_cbc_md5_decrypt;
+octo_decrypt_t octo_des_cbc_sha1_decrypt;
+
+octo_encrypt_t octo_aes_cbc_encrypt;
+octo_encrypt_t octo_aes_cbc_md5_encrypt;
+octo_encrypt_t octo_aes_cbc_sha1_encrypt;
+
+octo_decrypt_t octo_aes_cbc_decrypt;
+octo_decrypt_t octo_aes_cbc_md5_decrypt;
+octo_decrypt_t octo_aes_cbc_sha1_decrypt;
+
+#endif /* !_MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_ */
diff --git a/sys/mips/cavium/files.octeon1 b/sys/mips/cavium/files.octeon1
index 48c36cf..c8dc43c 100644
--- a/sys/mips/cavium/files.octeon1
+++ b/sys/mips/cavium/files.octeon1
@@ -17,6 +17,9 @@ mips/mips/tick.c				standard
 
 mips/cavium/octeon_rnd.c			optional random
 
+mips/cavium/cryptocteon/cavium_crypto.c		optional cryptocteon
+mips/cavium/cryptocteon/cryptocteon.c		optional cryptocteon
+
 mips/cavium/octe/ethernet.c			optional octe
 mips/cavium/octe/ethernet-common.c		optional octe
 mips/cavium/octe/ethernet-mdio.c		optional octe
@@ -33,6 +36,9 @@ mips/cavium/octe/octebus.c			optional octe
 mips/cavium/octopci.c				optional pci
 mips/cavium/octopci_bus_space.c			optional pci
 
+mips/cavium/usb/octusb.c			optional usb octusb
+mips/cavium/usb/octusb_octeon.c			optional usb octusb
+
 contrib/octeon-sdk/cvmx-cmd-queue.c		optional octe
 contrib/octeon-sdk/cvmx-fpa.c			optional octe
 contrib/octeon-sdk/cvmx-helper.c		optional octe
@@ -51,6 +57,8 @@ contrib/octeon-sdk/cvmx-spi.c			optional octe
 contrib/octeon-sdk/cvmx-spi4000.c		optional octe
 contrib/octeon-sdk/cvmx-twsi.c			optional octe
 
+contrib/octeon-sdk/cvmx-usb.c			optional octusb
+
 # XXX Some files could be excluded in some configurations.  Making them
 #     optional but on in the default config would seem reasonable.
 contrib/octeon-sdk/cvmx-bootmem.c		standard
diff --git a/sys/mips/cavium/usb/octusb.c b/sys/mips/cavium/usb/octusb.c
new file mode 100644
index 0000000..d090f12
--- /dev/null
+++ b/sys/mips/cavium/usb/octusb.c
@@ -0,0 +1,1922 @@
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*-
+ * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * This file contains the driver for Octeon Executive Library USB
+ * Controller driver API.
+ */
+
+/* TODO: The root HUB port callback is not yet implemented. */
+
+#include <sys/stdint.h>
+#include <sys/stddef.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/linker_set.h>
+#include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/condvar.h>
+#include <sys/sysctl.h>
+#include <sys/sx.h>
+#include <sys/unistd.h>
+#include <sys/callout.h>
+#include <sys/malloc.h>
+#include <sys/priv.h>
+
+#include <dev/usb/usb.h>
+#include <dev/usb/usbdi.h>
+
+#define	USB_DEBUG_VAR octusbdebug
+
+#include <dev/usb/usb_core.h>
+#include <dev/usb/usb_debug.h>
+#include <dev/usb/usb_busdma.h>
+#include <dev/usb/usb_process.h>
+#include <dev/usb/usb_transfer.h>
+#include <dev/usb/usb_device.h>
+#include <dev/usb/usb_hub.h>
+#include <dev/usb/usb_util.h>
+
+#include <dev/usb/usb_controller.h>
+#include <dev/usb/usb_bus.h>
+
+#include <contrib/octeon-sdk/cvmx.h>
+#include <contrib/octeon-sdk/cvmx-usb.h>
+
+#include <mips/cavium/usb/octusb.h>
+
+#define	OCTUSB_BUS2SC(bus) \
+   ((struct octusb_softc *)(((uint8_t *)(bus)) - \
+    ((uint8_t *)&(((struct octusb_softc *)0)->sc_bus))))
+
+#ifdef USB_DEBUG
+static int octusbdebug = 0;
+
+SYSCTL_NODE(_hw_usb, OID_AUTO, octusb, CTLFLAG_RW, 0, "OCTUSB");
+SYSCTL_INT(_hw_usb_octusb, OID_AUTO, debug, CTLFLAG_RW,
+    &octusbdebug, 0, "OCTUSB debug level");
+
+TUNABLE_INT("hw.usb.octusb.debug", &octusbdebug);
+
+#endif
+
+struct octusb_std_temp {
+	octusb_cmd_t *func;
+	struct octusb_td *td;
+	struct octusb_td *td_next;
+	struct usb_page_cache *pc;
+	uint32_t offset;
+	uint32_t len;
+	uint8_t	short_pkt;
+	uint8_t	setup_alt_next;
+};
+
+extern struct usb_bus_methods octusb_bus_methods;
+extern struct usb_pipe_methods octusb_device_bulk_methods;
+extern struct usb_pipe_methods octusb_device_ctrl_methods;
+extern struct usb_pipe_methods octusb_device_intr_methods;
+extern struct usb_pipe_methods octusb_device_isoc_methods;
+
+static void octusb_standard_done(struct usb_xfer *);
+static void octusb_device_done(struct usb_xfer *, usb_error_t);
+static void octusb_timeout(void *);
+static void octusb_do_poll(struct usb_bus *);
+
+static cvmx_usb_speed_t
+octusb_convert_speed(enum usb_dev_speed speed)
+{
+	;				/* indent fix */
+	switch (speed) {
+	case USB_SPEED_HIGH:
+		return (CVMX_USB_SPEED_HIGH);
+	case USB_SPEED_FULL:
+		return (CVMX_USB_SPEED_FULL);
+	default:
+		return (CVMX_USB_SPEED_LOW);
+	}
+}
+
+static cvmx_usb_transfer_t
+octusb_convert_ep_type(uint8_t ep_type)
+{
+	;				/* indent fix */
+	switch (ep_type & UE_XFERTYPE) {
+	case UE_CONTROL:
+		return (CVMX_USB_TRANSFER_CONTROL);
+	case UE_INTERRUPT:
+		return (CVMX_USB_TRANSFER_INTERRUPT);
+	case UE_ISOCHRONOUS:
+		return (CVMX_USB_TRANSFER_ISOCHRONOUS);
+	case UE_BULK:
+		return (CVMX_USB_TRANSFER_BULK);
+	default:
+		return (0);		/* should not happen */
+	}
+}
+
+static uint8_t
+octusb_host_alloc_endpoint(struct octusb_td *td)
+{
+	struct octusb_softc *sc;
+	int ep_handle;
+
+	if (td->qh->fixup_pending)
+		return (1);		/* busy */
+
+	if (td->qh->ep_allocated)
+		return (0);		/* success */
+
+	/* get softc */
+	sc = td->qh->sc;
+
+	ep_handle = cvmx_usb_open_pipe(
+	    &sc->sc_port[td->qh->port_index].state,
+	    0,
+	    td->qh->dev_addr,
+	    td->qh->ep_num,
+	    octusb_convert_speed(td->qh->dev_speed),
+	    td->qh->max_packet_size,
+	    octusb_convert_ep_type(td->qh->ep_type),
+	    (td->qh->ep_num & UE_DIR_IN) ? CVMX_USB_DIRECTION_IN :
+	    CVMX_USB_DIRECTION_OUT,
+	    td->qh->ep_interval,
+	    td->qh->ep_mult,
+	    td->qh->hs_hub_addr,
+	    td->qh->hs_hub_port);
+
+	if (ep_handle < 0)
+		return (1);		/* busy */
+
+	cvmx_usb_set_toggle(
+	    &sc->sc_port[td->qh->port_index].state,
+	    ep_handle, td->qh->ep_toggle_next);
+
+	td->qh->fixup_handle = -1;
+	td->qh->fixup_complete = 0;
+	td->qh->fixup_len = 0;
+	td->qh->fixup_off = 0;
+	td->qh->fixup_pending = 0;
+	td->qh->fixup_actlen = 0;
+
+	td->qh->ep_handle = ep_handle;
+	td->qh->ep_allocated = 1;
+
+	return (0);			/* success */
+}
+
+static void
+octusb_host_free_endpoint(struct octusb_td *td)
+{
+	struct octusb_softc *sc;
+
+	if (td->qh->ep_allocated == 0)
+		return;
+
+	/* get softc */
+	sc = td->qh->sc;
+
+	if (td->qh->fixup_handle >= 0) {
+		/* cancel, if any */
+		cvmx_usb_cancel(&sc->sc_port[td->qh->port_index].state,
+		    td->qh->ep_handle, td->qh->fixup_handle);
+	}
+	cvmx_usb_close_pipe(&sc->sc_port[td->qh->port_index].state, td->qh->ep_handle);
+
+	td->qh->ep_allocated = 0;
+}
+
+static void
+octusb_complete_cb(cvmx_usb_state_t *state,
+    cvmx_usb_callback_t reason,
+    cvmx_usb_complete_t status,
+    int pipe_handle, int submit_handle,
+    int bytes_transferred, void *user_data)
+{
+	struct octusb_td *td;
+
+	if (reason != CVMX_USB_CALLBACK_TRANSFER_COMPLETE)
+		return;
+
+	td = user_data;
+
+	td->qh->fixup_complete = 1;
+	td->qh->fixup_pending = 0;
+	td->qh->fixup_actlen = bytes_transferred;
+	td->qh->fixup_handle = -1;
+
+	switch (status) {
+	case CVMX_USB_COMPLETE_SUCCESS:
+	case CVMX_USB_COMPLETE_SHORT:
+		td->error_any = 0;
+		td->error_stall = 0;
+		break;
+	case CVMX_USB_COMPLETE_STALL:
+		td->error_stall = 1;
+		td->error_any = 1;
+		break;
+	default:
+		td->error_any = 1;
+		break;
+	}
+}
+
+static uint8_t
+octusb_host_control_header_tx(struct octusb_td *td)
+{
+	int status;
+
+	/* allocate endpoint and check pending */
+	if (octusb_host_alloc_endpoint(td))
+		return (1);		/* busy */
+
+	/* check error */
+	if (td->error_any)
+		return (0);		/* done */
+
+	if (td->qh->fixup_complete != 0) {
+		/* clear complete flag */
+		td->qh->fixup_complete = 0;
+
+		/* flush data */
+		usb_pc_cpu_invalidate(td->qh->fixup_pc);
+		return (0);		/* done */
+	}
+	/* verify length */
+	if (td->remainder != 8) {
+		td->error_any = 1;
+		return (0);		/* done */
+	}
+	usbd_copy_out(td->pc, td->offset, td->qh->fixup_buf, 8);
+
+	/* update offset and remainder */
+	td->offset += 8;
+	td->remainder -= 8;
+
+	/* setup data length and offset */
+	td->qh->fixup_len = UGETW(td->qh->fixup_buf + 6);
+	td->qh->fixup_off = 0;
+
+	if (td->qh->fixup_len > (OCTUSB_MAX_FIXUP - 8)) {
+		td->error_any = 1;
+		return (0);		/* done */
+	}
+	/* do control IN request */
+	if (td->qh->fixup_buf[0] & UE_DIR_IN) {
+
+		struct octusb_softc *sc;
+
+		/* get softc */
+		sc = td->qh->sc;
+
+		/* flush data */
+		usb_pc_cpu_flush(td->qh->fixup_pc);
+
+		status = cvmx_usb_submit_control(
+		    &sc->sc_port[td->qh->port_index].state,
+		    td->qh->ep_handle, td->qh->fixup_phys,
+		    td->qh->fixup_phys + 8, td->qh->fixup_len,
+		    &octusb_complete_cb, td);
+		/* check status */
+		if (status < 0) {
+			td->error_any = 1;
+			return (0);	/* done */
+		}
+		td->qh->fixup_handle = status;
+		td->qh->fixup_pending = 1;
+		td->qh->fixup_complete = 0;
+
+		return (1);		/* busy */
+	}
+	return (0);			/* done */
+}
+
+static uint8_t
+octusb_host_control_data_tx(struct octusb_td *td)
+{
+	uint32_t rem;
+
+	/* allocate endpoint and check pending */
+	if (octusb_host_alloc_endpoint(td))
+		return (1);		/* busy */
+
+	/* check error */
+	if (td->error_any)
+		return (0);		/* done */
+
+	rem = td->qh->fixup_len - td->qh->fixup_off;
+
+	if (td->remainder > rem) {
+		td->error_any = 1;
+		DPRINTFN(1, "Excess setup transmit data\n");
+		return (0);		/* done */
+	}
+	usbd_copy_out(td->pc, td->offset, td->qh->fixup_buf +
+	    td->qh->fixup_off + 8, td->remainder);
+
+	td->offset += td->remainder;
+	td->qh->fixup_off += td->remainder;
+	td->remainder = 0;
+
+	return (0);			/* done */
+}
+
+static uint8_t
+octusb_host_control_data_rx(struct octusb_td *td)
+{
+	uint32_t rem;
+
+	/* allocate endpoint and check pending */
+	if (octusb_host_alloc_endpoint(td))
+		return (1);		/* busy */
+
+	/* check error */
+	if (td->error_any)
+		return (0);		/* done */
+
+	/* copy data from buffer */
+	rem = td->qh->fixup_actlen - td->qh->fixup_off;
+
+	if (rem > td->remainder)
+		rem = td->remainder;
+
+	usbd_copy_in(td->pc, td->offset, td->qh->fixup_buf +
+	    td->qh->fixup_off + 8, rem);
+
+	td->offset += rem;
+	td->remainder -= rem;
+	td->qh->fixup_off += rem;
+
+	return (0);			/* done */
+}
+
+static uint8_t
+octusb_host_control_status_tx(struct octusb_td *td)
+{
+	int status;
+
+	/* allocate endpoint and check pending */
+	if (octusb_host_alloc_endpoint(td))
+		return (1);		/* busy */
+
+	/* check error */
+	if (td->error_any)
+		return (0);		/* done */
+
+	if (td->qh->fixup_complete != 0) {
+		/* clear complete flag */
+		td->qh->fixup_complete = 0;
+		/* done */
+		return (0);
+	}
+	/* do control IN request */
+	if (!(td->qh->fixup_buf[0] & UE_DIR_IN)) {
+
+		struct octusb_softc *sc;
+
+		/* get softc */
+		sc = td->qh->sc;
+
+		/* flush data */
+		usb_pc_cpu_flush(td->qh->fixup_pc);
+
+		/* start USB transfer */
+		status = cvmx_usb_submit_control(
+		    &sc->sc_port[td->qh->port_index].state,
+		    td->qh->ep_handle, td->qh->fixup_phys,
+		    td->qh->fixup_phys + 8, td->qh->fixup_len,
+		    &octusb_complete_cb, td);
+
+		/* check status */
+		if (status < 0) {
+			td->error_any = 1;
+			return (0);	/* done */
+		}
+		td->qh->fixup_handle = status;
+		td->qh->fixup_pending = 1;
+		td->qh->fixup_complete = 0;
+
+		return (1);		/* busy */
+	}
+	return (0);			/* done */
+}
+
+static uint8_t
+octusb_non_control_data_tx(struct octusb_td *td)
+{
+	struct octusb_softc *sc;
+	uint32_t rem;
+	int status;
+
+	/* allocate endpoint and check pending */
+	if (octusb_host_alloc_endpoint(td))
+		return (1);		/* busy */
+
+	/* check error */
+	if (td->error_any)
+		return (0);		/* done */
+
+	if ((td->qh->fixup_complete != 0) &&
+	    ((td->qh->ep_type & UE_XFERTYPE) == UE_ISOCHRONOUS)) {
+		td->qh->fixup_complete = 0;
+		return (0);		/* done */
+	}
+	/* check complete */
+	if (td->remainder == 0) {
+		if (td->short_pkt)
+			return (0);	/* complete */
+		/* else need to send a zero length packet */
+		rem = 0;
+		td->short_pkt = 1;
+	} else {
+		/* get maximum length */
+		rem = OCTUSB_MAX_FIXUP % td->qh->max_frame_size;
+		rem = OCTUSB_MAX_FIXUP - rem;
+
+		if (rem == 0) {
+			/* should not happen */
+			DPRINTFN(1, "Fixup buffer is too small\n");
+			td->error_any = 1;
+			return (0);	/* done */
+		}
+		/* get minimum length */
+		if (rem > td->remainder) {
+			rem = td->remainder;
+			if ((rem == 0) || (rem % td->qh->max_frame_size))
+				td->short_pkt = 1;
+		}
+		/* copy data into fixup buffer */
+		usbd_copy_out(td->pc, td->offset, td->qh->fixup_buf, rem);
+
+		/* flush data */
+		usb_pc_cpu_flush(td->qh->fixup_pc);
+
+		/* pre-increment TX buffer offset */
+		td->offset += rem;
+		td->remainder -= rem;
+	}
+
+	/* get softc */
+	sc = td->qh->sc;
+
+	switch (td->qh->ep_type & UE_XFERTYPE) {
+	case UE_ISOCHRONOUS:
+		td->qh->iso_pkt.offset = 0;
+		td->qh->iso_pkt.length = rem;
+		td->qh->iso_pkt.status = 0;
+		/* start USB transfer */
+		status = cvmx_usb_submit_isochronous(&sc->sc_port[td->qh->port_index].state,
+		    td->qh->ep_handle, 1, CVMX_USB_ISOCHRONOUS_FLAGS_ALLOW_SHORT |
+		    CVMX_USB_ISOCHRONOUS_FLAGS_ASAP, 1, &td->qh->iso_pkt,
+		    td->qh->fixup_phys, rem, &octusb_complete_cb, td);
+		break;
+	case UE_BULK:
+		/* start USB transfer */
+		status = cvmx_usb_submit_bulk(&sc->sc_port[td->qh->port_index].state,
+		    td->qh->ep_handle, td->qh->fixup_phys, rem, &octusb_complete_cb, td);
+		break;
+	case UE_INTERRUPT:
+		/* start USB transfer (interrupt or interrupt) */
+		status = cvmx_usb_submit_interrupt(&sc->sc_port[td->qh->port_index].state,
+		    td->qh->ep_handle, td->qh->fixup_phys, rem, &octusb_complete_cb, td);
+		break;
+	default:
+		status = -1;
+		break;
+	}
+
+	/* check status */
+	if (status < 0) {
+		td->error_any = 1;
+		return (0);		/* done */
+	}
+	td->qh->fixup_handle = status;
+	td->qh->fixup_len = rem;
+	td->qh->fixup_pending = 1;
+	td->qh->fixup_complete = 0;
+
+	return (1);			/* busy */
+}
+
+static uint8_t
+octusb_non_control_data_rx(struct octusb_td *td)
+{
+	struct octusb_softc *sc;
+	uint32_t rem;
+	int status;
+	uint8_t got_short;
+
+	/* allocate endpoint and check pending */
+	if (octusb_host_alloc_endpoint(td))
+		return (1);		/* busy */
+
+	/* check error */
+	if (td->error_any)
+		return (0);		/* done */
+
+	got_short = 0;
+
+	if (td->qh->fixup_complete != 0) {
+
+		/* invalidate data */
+		usb_pc_cpu_invalidate(td->qh->fixup_pc);
+
+		rem = td->qh->fixup_actlen;
+
+		/* verify transfer length */
+		if (rem != td->qh->fixup_len) {
+			if (rem < td->qh->fixup_len) {
+				/* we have a short packet */
+				td->short_pkt = 1;
+				got_short = 1;
+			} else {
+				/* invalid USB packet */
+				td->error_any = 1;
+				return (0);	/* we are complete */
+			}
+		}
+		/* copy data into fixup buffer */
+		usbd_copy_in(td->pc, td->offset, td->qh->fixup_buf, rem);
+
+		/* post-increment RX buffer offset */
+		td->offset += rem;
+		td->remainder -= rem;
+
+		td->qh->fixup_complete = 0;
+
+		if ((td->qh->ep_type & UE_XFERTYPE) == UE_ISOCHRONOUS)
+			return (0);	/* done */
+	}
+	/* check if we are complete */
+	if ((td->remainder == 0) || got_short) {
+		if (td->short_pkt) {
+			/* we are complete */
+			return (0);
+		}
+		/* else need to receive a zero length packet */
+		rem = 0;
+		td->short_pkt = 1;
+	} else {
+		/* get maximum length */
+		rem = OCTUSB_MAX_FIXUP % td->qh->max_frame_size;
+		rem = OCTUSB_MAX_FIXUP - rem;
+
+		if (rem == 0) {
+			/* should not happen */
+			DPRINTFN(1, "Fixup buffer is too small\n");
+			td->error_any = 1;
+			return (0);	/* done */
+		}
+		/* get minimum length */
+		if (rem > td->remainder)
+			rem = td->remainder;
+	}
+
+	/* invalidate data */
+	usb_pc_cpu_invalidate(td->qh->fixup_pc);
+
+	/* get softc */
+	sc = td->qh->sc;
+
+	switch (td->qh->ep_type & UE_XFERTYPE) {
+	case UE_ISOCHRONOUS:
+		td->qh->iso_pkt.offset = 0;
+		td->qh->iso_pkt.length = rem;
+		td->qh->iso_pkt.status = 0;
+		/* start USB transfer */
+		status = cvmx_usb_submit_isochronous(&sc->sc_port[td->qh->port_index].state,
+		    td->qh->ep_handle, 1, CVMX_USB_ISOCHRONOUS_FLAGS_ALLOW_SHORT |
+		    CVMX_USB_ISOCHRONOUS_FLAGS_ASAP, 1, &td->qh->iso_pkt,
+		    td->qh->fixup_phys, rem, &octusb_complete_cb, td);
+		break;
+	case UE_BULK:
+		/* start USB transfer */
+		status = cvmx_usb_submit_bulk(&sc->sc_port[td->qh->port_index].state,
+		    td->qh->ep_handle, td->qh->fixup_phys, rem, &octusb_complete_cb, td);
+		break;
+	case UE_INTERRUPT:
+		/* start USB transfer */
+		status = cvmx_usb_submit_interrupt(&sc->sc_port[td->qh->port_index].state,
+		    td->qh->ep_handle, td->qh->fixup_phys, rem, &octusb_complete_cb, td);
+		break;
+	default:
+		status = -1;
+		break;
+	}
+
+	/* check status */
+	if (status < 0) {
+		td->error_any = 1;
+		return (0);		/* done */
+	}
+	td->qh->fixup_handle = status;
+	td->qh->fixup_len = rem;
+	td->qh->fixup_pending = 1;
+	td->qh->fixup_complete = 0;
+
+	return (1);			/* busy */
+}
+
+static uint8_t
+octusb_xfer_do_fifo(struct usb_xfer *xfer)
+{
+	struct octusb_td *td;
+
+	DPRINTFN(8, "\n");
+
+	td = xfer->td_transfer_cache;
+
+	while (1) {
+		if ((td->func) (td)) {
+			/* operation in progress */
+			break;
+		}
+		if (((void *)td) == xfer->td_transfer_last) {
+			goto done;
+		}
+		if (td->error_any) {
+			goto done;
+		} else if (td->remainder > 0) {
+			/*
+			 * We had a short transfer. If there is no
+			 * alternate next, stop processing !
+			 */
+			if (td->alt_next == 0)
+				goto done;
+		}
+		/*
+		 * Fetch the next transfer descriptor and transfer
+		 * some flags to the next transfer descriptor
+		 */
+		td = td->obj_next;
+		xfer->td_transfer_cache = td;
+	}
+	return (1);			/* not complete */
+
+done:
+	/* compute all actual lengths */
+
+	octusb_standard_done(xfer);
+
+	return (0);			/* complete */
+}
+
+static usb_error_t
+octusb_standard_done_sub(struct usb_xfer *xfer)
+{
+	struct octusb_td *td;
+	uint32_t len;
+	usb_error_t error;
+
+	DPRINTFN(8, "\n");
+
+	td = xfer->td_transfer_cache;
+
+	do {
+		len = td->remainder;
+
+		if (xfer->aframes != xfer->nframes) {
+			/*
+		         * Verify the length and subtract
+		         * the remainder from "frlengths[]":
+		         */
+			if (len > xfer->frlengths[xfer->aframes]) {
+				td->error_any = 1;
+			} else {
+				xfer->frlengths[xfer->aframes] -= len;
+			}
+		}
+		/* Check for transfer error */
+		if (td->error_any) {
+			/* the transfer is finished */
+			error = td->error_stall ? USB_ERR_STALLED : USB_ERR_IOERROR;
+			td = NULL;
+			break;
+		}
+		/* Check for short transfer */
+		if (len > 0) {
+			if (xfer->flags_int.short_frames_ok) {
+				/* follow alt next */
+				if (td->alt_next) {
+					td = td->obj_next;
+				} else {
+					td = NULL;
+				}
+			} else {
+				/* the transfer is finished */
+				td = NULL;
+			}
+			error = 0;
+			break;
+		}
+		td = td->obj_next;
+
+		/* this USB frame is complete */
+		error = 0;
+		break;
+
+	} while (0);
+
+	/* update transfer cache */
+
+	xfer->td_transfer_cache = td;
+
+	return (error);
+}
+
+static void
+octusb_standard_done(struct usb_xfer *xfer)
+{
+	struct octusb_softc *sc;
+	struct octusb_qh *qh;
+	usb_error_t error = 0;
+
+	DPRINTFN(12, "xfer=%p endpoint=%p transfer done\n",
+	    xfer, xfer->endpoint);
+
+	/* reset scanner */
+
+	xfer->td_transfer_cache = xfer->td_transfer_first;
+
+	if (xfer->flags_int.control_xfr) {
+
+		if (xfer->flags_int.control_hdr)
+			error = octusb_standard_done_sub(xfer);
+
+		xfer->aframes = 1;
+
+		if (xfer->td_transfer_cache == NULL)
+			goto done;
+	}
+	while (xfer->aframes != xfer->nframes) {
+
+		error = octusb_standard_done_sub(xfer);
+
+		xfer->aframes++;
+
+		if (xfer->td_transfer_cache == NULL)
+			goto done;
+	}
+
+	if (xfer->flags_int.control_xfr &&
+	    !xfer->flags_int.control_act)
+		error = octusb_standard_done_sub(xfer);
+
+done:
+	/* update data toggle */
+
+	qh = xfer->qh_start[0];
+	sc = qh->sc;
+
+	xfer->endpoint->toggle_next =
+	    cvmx_usb_get_toggle(
+	    &sc->sc_port[qh->port_index].state,
+	    qh->ep_handle) ? 1 : 0;
+
+	octusb_device_done(xfer, error);
+}
+
+static void
+octusb_interrupt_poll(struct octusb_softc *sc)
+{
+	struct usb_xfer *xfer;
+	uint8_t x;
+
+	/* poll all ports */
+	for (x = 0; x != sc->sc_noport; x++)
+		cvmx_usb_poll(&sc->sc_port[x].state);
+
+repeat:
+	TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
+		if (!octusb_xfer_do_fifo(xfer)) {
+			/* queue has been modified */
+			goto repeat;
+		}
+	}
+}
+
+static void
+octusb_start_standard_chain(struct usb_xfer *xfer)
+{
+	DPRINTFN(8, "\n");
+
+	/* poll one time */
+	if (octusb_xfer_do_fifo(xfer)) {
+
+		/* put transfer on interrupt queue */
+		usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
+
+		/* start timeout, if any */
+		if (xfer->timeout != 0) {
+			usbd_transfer_timeout_ms(xfer,
+			    &octusb_timeout, xfer->timeout);
+		}
+	}
+}
+
+void
+octusb_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
+{
+
+}
+
+usb_error_t
+octusb_init(struct octusb_softc *sc)
+{
+	cvmx_usb_initialize_flags_t flags;
+	int status;
+	uint8_t x;
+
+	/* flush all cache into memory */
+
+	usb_bus_mem_flush_all(&sc->sc_bus, &octusb_iterate_hw_softc);
+
+	/* set up the bus struct */
+	sc->sc_bus.methods = &octusb_bus_methods;
+
+	/* get number of ports */
+	sc->sc_noport = cvmx_usb_get_num_ports();
+
+	/* check number of ports */
+	if (sc->sc_noport > OCTUSB_MAX_PORTS)
+		sc->sc_noport = OCTUSB_MAX_PORTS;
+
+	/* set USB revision */
+	sc->sc_bus.usbrev = USB_REV_2_0;
+
+	/* flags for port initialization */
+	flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_AUTO;
+#ifdef USB_DEBUG
+	if (octusbdebug > 100)
+		flags |= CVMX_USB_INITIALIZE_FLAGS_DEBUG_ALL;
+#endif
+
+	USB_BUS_LOCK(&sc->sc_bus);
+
+	/* setup all ports */
+	for (x = 0; x != sc->sc_noport; x++) {
+		status = cvmx_usb_initialize(&sc->sc_port[x].state, x, flags);
+		if (status < 0)
+			sc->sc_port[x].disabled = 1;
+	}
+
+	USB_BUS_UNLOCK(&sc->sc_bus);
+
+	/* catch lost interrupts */
+	octusb_do_poll(&sc->sc_bus);
+
+	return (0);
+}
+
+usb_error_t
+octusb_uninit(struct octusb_softc *sc)
+{
+	uint8_t x;
+
+	USB_BUS_LOCK(&sc->sc_bus);
+
+	for (x = 0; x != sc->sc_noport; x++) {
+		if (sc->sc_port[x].disabled == 0)
+			cvmx_usb_shutdown(&sc->sc_port[x].state);
+	}
+	USB_BUS_UNLOCK(&sc->sc_bus);
+
+	return (0);
+
+}
+
+void
+octusb_suspend(struct octusb_softc *sc)
+{
+
+}
+
+void
+octusb_resume(struct octusb_softc *sc)
+{
+
+}
+
+/*------------------------------------------------------------------------*
+ *	octusb_interrupt - OCTUSB interrupt handler
+ *------------------------------------------------------------------------*/
+void
+octusb_interrupt(struct octusb_softc *sc)
+{
+	USB_BUS_LOCK(&sc->sc_bus);
+
+	DPRINTFN(16, "real interrupt\n");
+
+	/* poll all the USB transfers */
+	octusb_interrupt_poll(sc);
+
+	USB_BUS_UNLOCK(&sc->sc_bus);
+}
+
+/*------------------------------------------------------------------------*
+ *	octusb_timeout - OCTUSB transfer timeout handler
+ *------------------------------------------------------------------------*/
+static void
+octusb_timeout(void *arg)
+{
+	struct usb_xfer *xfer = arg;
+
+	DPRINTF("xfer=%p\n", xfer);
+
+	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
+
+	/* transfer is transferred */
+	octusb_device_done(xfer, USB_ERR_TIMEOUT);
+}
+
+/*------------------------------------------------------------------------*
+ *	octusb_do_poll - OCTUSB poll transfers
+ *------------------------------------------------------------------------*/
+static void
+octusb_do_poll(struct usb_bus *bus)
+{
+	struct octusb_softc *sc = OCTUSB_BUS2SC(bus);
+
+	USB_BUS_LOCK(&sc->sc_bus);
+	octusb_interrupt_poll(sc);
+	USB_BUS_UNLOCK(&sc->sc_bus);
+}
+
+static void
+octusb_setup_standard_chain_sub(struct octusb_std_temp *temp)
+{
+	struct octusb_td *td;
+
+	/* get current Transfer Descriptor */
+	td = temp->td_next;
+	temp->td = td;
+
+	/* prepare for next TD */
+	temp->td_next = td->obj_next;
+
+	/* fill out the Transfer Descriptor */
+	td->func = temp->func;
+	td->pc = temp->pc;
+	td->offset = temp->offset;
+	td->remainder = temp->len;
+	td->error_any = 0;
+	td->error_stall = 0;
+	td->short_pkt = temp->short_pkt;
+	td->alt_next = temp->setup_alt_next;
+}
+
+static void
+octusb_setup_standard_chain(struct usb_xfer *xfer)
+{
+	struct octusb_std_temp temp;
+	struct octusb_td *td;
+	uint32_t x;
+
+	DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
+	    xfer->address, UE_GET_ADDR(xfer->endpointno),
+	    xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
+
+	/* setup starting point */
+	td = xfer->td_start[0];
+	xfer->td_transfer_first = td;
+	xfer->td_transfer_cache = td;
+
+	temp.td = NULL;
+	temp.td_next = td;
+	temp.setup_alt_next = xfer->flags_int.short_frames_ok;
+	temp.offset = 0;
+
+	/* check if we should prepend a setup message */
+
+	if (xfer->flags_int.control_xfr) {
+
+		if (xfer->flags_int.control_hdr) {
+
+			temp.func = &octusb_host_control_header_tx;
+			temp.len = xfer->frlengths[0];
+			temp.pc = xfer->frbuffers + 0;
+			temp.short_pkt = temp.len ? 1 : 0;
+
+			/* check for last frame */
+			if (xfer->nframes == 1) {
+				/*
+				 * no STATUS stage yet, SETUP is
+				 * last
+				 */
+				if (xfer->flags_int.control_act)
+					temp.setup_alt_next = 0;
+			}
+			octusb_setup_standard_chain_sub(&temp);
+		}
+		x = 1;
+	} else {
+		x = 0;
+	}
+
+	if (x != xfer->nframes) {
+		if (xfer->endpointno & UE_DIR_IN) {
+			if (xfer->flags_int.control_xfr)
+				temp.func = &octusb_host_control_data_rx;
+			else
+				temp.func = &octusb_non_control_data_rx;
+		} else {
+			if (xfer->flags_int.control_xfr)
+				temp.func = &octusb_host_control_data_tx;
+			else
+				temp.func = &octusb_non_control_data_tx;
+		}
+
+		/* setup "pc" pointer */
+		temp.pc = xfer->frbuffers + x;
+	}
+	while (x != xfer->nframes) {
+
+		/* DATA0 or DATA1 message */
+
+		temp.len = xfer->frlengths[x];
+
+		x++;
+
+		if (x == xfer->nframes) {
+			if (xfer->flags_int.control_xfr) {
+				/* no STATUS stage yet, DATA is last */
+				if (xfer->flags_int.control_act)
+					temp.setup_alt_next = 0;
+			} else {
+				temp.setup_alt_next = 0;
+			}
+		}
+		if (temp.len == 0) {
+
+			/* make sure that we send an USB packet */
+
+			temp.short_pkt = 0;
+
+		} else {
+
+			/* regular data transfer */
+
+			temp.short_pkt = (xfer->flags.force_short_xfer) ? 0 : 1;
+		}
+
+		octusb_setup_standard_chain_sub(&temp);
+
+		if (xfer->flags_int.isochronous_xfr) {
+			/* get next data offset */
+			temp.offset += temp.len;
+		} else {
+			/* get next Page Cache pointer */
+			temp.pc = xfer->frbuffers + x;
+		}
+	}
+
+	/* check if we should append a status stage */
+
+	if (xfer->flags_int.control_xfr &&
+	    !xfer->flags_int.control_act) {
+
+		temp.func = &octusb_host_control_status_tx;
+		temp.len = 0;
+		temp.pc = NULL;
+		temp.short_pkt = 0;
+		temp.setup_alt_next = 0;
+
+		octusb_setup_standard_chain_sub(&temp);
+	}
+	/* must have at least one frame! */
+	td = temp.td;
+	xfer->td_transfer_last = td;
+
+	/* properly setup QH */
+
+	td->qh->ep_allocated = 0;
+	td->qh->ep_toggle_next = xfer->endpoint->toggle_next ? 1 : 0;
+}
+
+/*------------------------------------------------------------------------*
+ *	octusb_device_done - OCTUSB transfers done code
+ *
+ * NOTE: This function can be called more than one time in a row.
+ *------------------------------------------------------------------------*/
+static void
+octusb_device_done(struct usb_xfer *xfer, usb_error_t error)
+{
+	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
+
+	DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
+	    xfer, xfer->endpoint, error);
+
+	/*
+	 * 1) Free any endpoints.
+	 * 2) Control transfers can be split and we should not re-open
+	 *    the data pipe between transactions unless there is an error.
+	 */
+	if ((xfer->flags_int.control_act == 0) || (error != 0)) {
+		struct octusb_td *td;
+
+		td = xfer->td_start[0];
+
+		octusb_host_free_endpoint(td);
+	}
+	/* dequeue transfer and start next transfer */
+	usbd_transfer_done(xfer, error);
+}
+
+/*------------------------------------------------------------------------*
+ * octusb bulk support
+ *------------------------------------------------------------------------*/
+static void
+octusb_device_bulk_open(struct usb_xfer *xfer)
+{
+	return;
+}
+
+static void
+octusb_device_bulk_close(struct usb_xfer *xfer)
+{
+	octusb_device_done(xfer, USB_ERR_CANCELLED);
+}
+
+static void
+octusb_device_bulk_enter(struct usb_xfer *xfer)
+{
+	return;
+}
+
+static void
+octusb_device_bulk_start(struct usb_xfer *xfer)
+{
+	/* setup TDs */
+	octusb_setup_standard_chain(xfer);
+	octusb_start_standard_chain(xfer);
+}
+
+struct usb_pipe_methods octusb_device_bulk_methods =
+{
+	.open = octusb_device_bulk_open,
+	.close = octusb_device_bulk_close,
+	.enter = octusb_device_bulk_enter,
+	.start = octusb_device_bulk_start,
+};
+
+/*------------------------------------------------------------------------*
+ * octusb control support
+ *------------------------------------------------------------------------*/
+static void
+octusb_device_ctrl_open(struct usb_xfer *xfer)
+{
+	return;
+}
+
+static void
+octusb_device_ctrl_close(struct usb_xfer *xfer)
+{
+	octusb_device_done(xfer, USB_ERR_CANCELLED);
+}
+
+static void
+octusb_device_ctrl_enter(struct usb_xfer *xfer)
+{
+	return;
+}
+
+static void
+octusb_device_ctrl_start(struct usb_xfer *xfer)
+{
+	/* setup TDs */
+	octusb_setup_standard_chain(xfer);
+	octusb_start_standard_chain(xfer);
+}
+
+struct usb_pipe_methods octusb_device_ctrl_methods =
+{
+	.open = octusb_device_ctrl_open,
+	.close = octusb_device_ctrl_close,
+	.enter = octusb_device_ctrl_enter,
+	.start = octusb_device_ctrl_start,
+};
+
+/*------------------------------------------------------------------------*
+ * octusb interrupt support
+ *------------------------------------------------------------------------*/
+static void
+octusb_device_intr_open(struct usb_xfer *xfer)
+{
+	return;
+}
+
+static void
+octusb_device_intr_close(struct usb_xfer *xfer)
+{
+	octusb_device_done(xfer, USB_ERR_CANCELLED);
+}
+
+static void
+octusb_device_intr_enter(struct usb_xfer *xfer)
+{
+	return;
+}
+
+static void
+octusb_device_intr_start(struct usb_xfer *xfer)
+{
+	/* setup TDs */
+	octusb_setup_standard_chain(xfer);
+	octusb_start_standard_chain(xfer);
+}
+
+struct usb_pipe_methods octusb_device_intr_methods =
+{
+	.open = octusb_device_intr_open,
+	.close = octusb_device_intr_close,
+	.enter = octusb_device_intr_enter,
+	.start = octusb_device_intr_start,
+};
+
+/*------------------------------------------------------------------------*
+ * octusb isochronous support
+ *------------------------------------------------------------------------*/
+static void
+octusb_device_isoc_open(struct usb_xfer *xfer)
+{
+	return;
+}
+
+static void
+octusb_device_isoc_close(struct usb_xfer *xfer)
+{
+	octusb_device_done(xfer, USB_ERR_CANCELLED);
+}
+
+static void
+octusb_device_isoc_enter(struct usb_xfer *xfer)
+{
+	struct octusb_softc *sc = OCTUSB_BUS2SC(xfer->xroot->bus);
+	uint32_t temp;
+	uint32_t frame_count;
+	uint32_t fs_frames;
+
+	DPRINTFN(5, "xfer=%p next=%d nframes=%d\n",
+	    xfer, xfer->endpoint->isoc_next, xfer->nframes);
+
+	/* get the current frame index */
+
+	frame_count = cvmx_usb_get_frame_number(
+	    &sc->sc_port[xfer->xroot->udev->port_index].state);
+
+	/*
+	 * check if the frame index is within the window where the frames
+	 * will be inserted
+	 */
+	temp = (frame_count - xfer->endpoint->isoc_next) & 0x7FF;
+
+	if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
+		fs_frames = (xfer->nframes + 7) / 8;
+	} else {
+		fs_frames = xfer->nframes;
+	}
+
+	if ((xfer->endpoint->is_synced == 0) || (temp < fs_frames)) {
+		/*
+		 * If there is data underflow or the pipe queue is
+		 * empty we schedule the transfer a few frames ahead
+		 * of the current frame position. Else two isochronous
+		 * transfers might overlap.
+		 */
+		xfer->endpoint->isoc_next = (frame_count + 3) & 0x7FF;
+		xfer->endpoint->is_synced = 1;
+		DPRINTFN(2, "start next=%d\n", xfer->endpoint->isoc_next);
+	}
+	/*
+	 * compute how many milliseconds the insertion is ahead of the
+	 * current frame position:
+	 */
+	temp = (xfer->endpoint->isoc_next - frame_count) & 0x7FF;
+
+	/*
+	 * pre-compute when the isochronous transfer will be finished:
+	 */
+	xfer->isoc_time_complete =
+	    usb_isoc_time_expand(&sc->sc_bus, frame_count) + temp +
+	    fs_frames;
+
+	/* compute frame number for next insertion */
+	xfer->endpoint->isoc_next += fs_frames;
+}
+
+static void
+octusb_device_isoc_start(struct usb_xfer *xfer)
+{
+	/* setup TDs */
+	octusb_setup_standard_chain(xfer);
+	octusb_start_standard_chain(xfer);
+}
+
+struct usb_pipe_methods octusb_device_isoc_methods =
+{
+	.open = octusb_device_isoc_open,
+	.close = octusb_device_isoc_close,
+	.enter = octusb_device_isoc_enter,
+	.start = octusb_device_isoc_start,
+};
+
+/*------------------------------------------------------------------------*
+ * OCTUSB root HUB support
+ *------------------------------------------------------------------------*
+ * Simulate a hardware HUB by handling all the necessary requests.
+ *------------------------------------------------------------------------*/
+static const
+struct usb_device_descriptor octusb_devd = {
+	.bLength = sizeof(octusb_devd),
+	.bDescriptorType = UDESC_DEVICE,
+	.bcdUSB = {0x00, 0x02},
+	.bDeviceClass = UDCLASS_HUB,
+	.bDeviceSubClass = UDSUBCLASS_HUB,
+	.bDeviceProtocol = UDPROTO_FSHUB,
+	.bMaxPacketSize = 64,
+	.idVendor = {0},
+	.idProduct = {0},
+	.bcdDevice = {0x00, 0x01},
+	.iManufacturer = 1,
+	.iProduct = 2,
+	.iSerialNumber = 0,
+	.bNumConfigurations = 1,
+};
+
+static const
+struct usb_device_qualifier octusb_odevd = {
+	.bLength = sizeof(octusb_odevd),
+	.bDescriptorType = UDESC_DEVICE_QUALIFIER,
+	.bcdUSB = {0x00, 0x02},
+	.bDeviceClass = UDCLASS_HUB,
+	.bDeviceSubClass = UDSUBCLASS_HUB,
+	.bDeviceProtocol = UDPROTO_FSHUB,
+	.bMaxPacketSize0 = 0,
+	.bNumConfigurations = 0,
+	.bReserved = 0,
+};
+
+static const
+struct octusb_config_desc octusb_confd = {
+	.confd = {
+		.bLength = sizeof(struct usb_config_descriptor),
+		.bDescriptorType = UDESC_CONFIG,
+		.wTotalLength[0] = sizeof(octusb_confd),
+		.bNumInterface = 1,
+		.bConfigurationValue = 1,
+		.iConfiguration = 0,
+		.bmAttributes = UC_SELF_POWERED,
+		.bMaxPower = 0		/* max power */
+	},
+	.ifcd = {
+		.bLength = sizeof(struct usb_interface_descriptor),
+		.bDescriptorType = UDESC_INTERFACE,
+		.bNumEndpoints = 1,
+		.bInterfaceClass = UICLASS_HUB,
+		.bInterfaceSubClass = UISUBCLASS_HUB,
+		.bInterfaceProtocol = UIPROTO_FSHUB,
+	},
+	.endpd = {
+		.bLength = sizeof(struct usb_endpoint_descriptor),
+		.bDescriptorType = UDESC_ENDPOINT,
+		.bEndpointAddress = UE_DIR_IN | OCTUSB_INTR_ENDPT,
+		.bmAttributes = UE_INTERRUPT,
+		.wMaxPacketSize[0] = 8,	/* max packet (63 ports) */
+		.bInterval = 255,
+	},
+};
+
+static const
+struct usb_hub_descriptor_min octusb_hubd =
+{
+	.bDescLength = sizeof(octusb_hubd),
+	.bDescriptorType = UDESC_HUB,
+	.bNbrPorts = 2,
+	.wHubCharacteristics = {UHD_PWR_NO_SWITCH | UHD_OC_INDIVIDUAL, 0},
+	.bPwrOn2PwrGood = 50,
+	.bHubContrCurrent = 0,
+	.DeviceRemovable = {0x00},	/* all ports are removable */
+};
+
+static usb_error_t
+octusb_roothub_exec(struct usb_device *udev,
+    struct usb_device_request *req, const void **pptr, uint16_t *plength)
+{
+	struct octusb_softc *sc = OCTUSB_BUS2SC(udev->bus);
+	const void *ptr;
+	const char *str_ptr;
+	uint16_t value;
+	uint16_t index;
+	uint16_t status;
+	uint16_t change;
+	uint16_t len;
+	usb_error_t err;
+	cvmx_usb_port_status_t usb_port_status;
+
+	USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
+
+	/* XXX disable power save mode, hence it is not supported */
+	udev->power_mode = USB_POWER_MODE_ON;
+
+	/* buffer reset */
+	ptr = (const void *)&sc->sc_hub_desc.temp;
+	len = 0;
+	err = 0;
+
+	value = UGETW(req->wValue);
+	index = UGETW(req->wIndex);
+
+	DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
+	    "wValue=0x%04x wIndex=0x%04x\n",
+	    req->bmRequestType, req->bRequest,
+	    UGETW(req->wLength), value, index);
+
+#define	C(x,y) ((x) | ((y) << 8))
+	switch (C(req->bRequest, req->bmRequestType)) {
+	case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
+	case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
+	case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
+		break;
+	case C(UR_GET_CONFIG, UT_READ_DEVICE):
+		len = 1;
+		sc->sc_hub_desc.temp[0] = sc->sc_conf;
+		break;
+	case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
+		switch (value >> 8) {
+		case UDESC_DEVICE:
+			if ((value & 0xff) != 0) {
+				err = USB_ERR_IOERROR;
+				goto done;
+			}
+			len = sizeof(octusb_devd);
+
+			ptr = (const void *)&octusb_devd;
+			break;
+
+		case UDESC_DEVICE_QUALIFIER:
+			if ((value & 0xff) != 0) {
+				err = USB_ERR_IOERROR;
+				goto done;
+			}
+			len = sizeof(octusb_odevd);
+			ptr = (const void *)&octusb_odevd;
+			break;
+
+		case UDESC_CONFIG:
+			if ((value & 0xff) != 0) {
+				err = USB_ERR_IOERROR;
+				goto done;
+			}
+			len = sizeof(octusb_confd);
+			ptr = (const void *)&octusb_confd;
+			break;
+
+		case UDESC_STRING:
+			switch (value & 0xff) {
+			case 0:	/* Language table */
+				str_ptr = "\001";
+				break;
+
+			case 1:	/* Vendor */
+				str_ptr = "Cavium Networks";
+				break;
+
+			case 2:	/* Product */
+				str_ptr = "OCTUSB Root HUB";
+				break;
+
+			default:
+				str_ptr = "";
+				break;
+			}
+
+			len = usb_make_str_desc(sc->sc_hub_desc.temp,
+			    sizeof(sc->sc_hub_desc.temp), str_ptr);
+			break;
+
+		default:
+			err = USB_ERR_IOERROR;
+			goto done;
+		}
+		break;
+	case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
+		len = 1;
+		sc->sc_hub_desc.temp[0] = 0;
+		break;
+	case C(UR_GET_STATUS, UT_READ_DEVICE):
+		len = 2;
+		USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
+		break;
+	case C(UR_GET_STATUS, UT_READ_INTERFACE):
+	case C(UR_GET_STATUS, UT_READ_ENDPOINT):
+		len = 2;
+		USETW(sc->sc_hub_desc.stat.wStatus, 0);
+		break;
+	case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
+		if (value >= OCTUSB_MAX_DEVICES) {
+			err = USB_ERR_IOERROR;
+			goto done;
+		}
+		sc->sc_addr = value;
+		break;
+	case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
+		if ((value != 0) && (value != 1)) {
+			err = USB_ERR_IOERROR;
+			goto done;
+		}
+		sc->sc_conf = value;
+		break;
+	case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
+		break;
+	case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
+	case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
+	case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
+		err = USB_ERR_IOERROR;
+		goto done;
+	case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
+		break;
+	case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
+		break;
+		/* Hub requests */
+	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
+		break;
+	case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
+		DPRINTFN(4, "UR_CLEAR_PORT_FEATURE "
+		    "port=%d feature=%d\n",
+		    index, value);
+		if ((index < 1) ||
+		    (index > sc->sc_noport) ||
+		    sc->sc_port[index - 1].disabled) {
+			err = USB_ERR_IOERROR;
+			goto done;
+		}
+		index--;
+
+		switch (value) {
+		case UHF_PORT_ENABLE:
+			cvmx_usb_disable(&sc->sc_port[index].state);
+			break;
+		case UHF_PORT_SUSPEND:
+		case UHF_PORT_RESET:
+			break;
+		case UHF_C_PORT_CONNECTION:
+			cvmx_usb_set_status(&sc->sc_port[index].state,
+			    cvmx_usb_get_status(&sc->sc_port[index].state));
+			break;
+		case UHF_C_PORT_ENABLE:
+			cvmx_usb_set_status(&sc->sc_port[index].state,
+			    cvmx_usb_get_status(&sc->sc_port[index].state));
+			break;
+		case UHF_C_PORT_OVER_CURRENT:
+			cvmx_usb_set_status(&sc->sc_port[index].state,
+			    cvmx_usb_get_status(&sc->sc_port[index].state));
+			break;
+		case UHF_C_PORT_RESET:
+			sc->sc_isreset = 0;
+			goto done;
+		case UHF_C_PORT_SUSPEND:
+			break;
+		case UHF_PORT_CONNECTION:
+		case UHF_PORT_OVER_CURRENT:
+		case UHF_PORT_POWER:
+		case UHF_PORT_LOW_SPEED:
+		default:
+			err = USB_ERR_IOERROR;
+			goto done;
+		}
+		break;
+	case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
+		if ((value & 0xff) != 0) {
+			err = USB_ERR_IOERROR;
+			goto done;
+		}
+		sc->sc_hubd = octusb_hubd;
+		sc->sc_hubd.bNbrPorts = sc->sc_noport;
+		len = sizeof(sc->sc_hubd);
+		ptr = (const void *)&sc->sc_hubd;
+		break;
+	case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
+		len = 16;
+		memset(sc->sc_hub_desc.temp, 0, 16);
+		break;
+	case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
+		if ((index < 1) ||
+		    (index > sc->sc_noport) ||
+		    sc->sc_port[index - 1].disabled) {
+			err = USB_ERR_IOERROR;
+			goto done;
+		}
+		index--;
+
+		usb_port_status = cvmx_usb_get_status(&sc->sc_port[index].state);
+
+		status = change = 0;
+		if (usb_port_status.connected)
+			status |= UPS_CURRENT_CONNECT_STATUS;
+		if (usb_port_status.port_enabled)
+			status |= UPS_PORT_ENABLED;
+		if (usb_port_status.port_over_current)
+			status |= UPS_OVERCURRENT_INDICATOR;
+		if (usb_port_status.port_powered)
+			status |= UPS_PORT_POWER;
+
+		switch (usb_port_status.port_speed) {
+		case CVMX_USB_SPEED_HIGH:
+			status |= UPS_HIGH_SPEED;
+			break;
+		case CVMX_USB_SPEED_FULL:
+			break;
+		default:
+			status |= UPS_LOW_SPEED;
+			break;
+		}
+
+		if (usb_port_status.connect_change)
+			change |= UPS_C_CONNECT_STATUS;
+		if (sc->sc_isreset)
+			change |= UPS_C_PORT_RESET;
+
+		USETW(sc->sc_hub_desc.ps.wPortStatus, status);
+		USETW(sc->sc_hub_desc.ps.wPortChange, change);
+
+		len = sizeof(sc->sc_hub_desc.ps);
+		break;
+	case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
+		err = USB_ERR_IOERROR;
+		goto done;
+	case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
+		break;
+	case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
+		if ((index < 1) ||
+		    (index > sc->sc_noport) ||
+		    sc->sc_port[index - 1].disabled) {
+			err = USB_ERR_IOERROR;
+			goto done;
+		}
+		index--;
+
+		switch (value) {
+		case UHF_PORT_ENABLE:
+			break;
+		case UHF_PORT_RESET:
+			cvmx_usb_disable(&sc->sc_port[index].state);
+			if (cvmx_usb_enable(&sc->sc_port[index].state)) {
+				err = USB_ERR_IOERROR;
+				goto done;
+			}
+			sc->sc_isreset = 1;
+			goto done;
+		case UHF_PORT_POWER:
+			/* pretend we turned on power */
+			goto done;
+		case UHF_PORT_SUSPEND:
+		case UHF_C_PORT_CONNECTION:
+		case UHF_C_PORT_ENABLE:
+		case UHF_C_PORT_OVER_CURRENT:
+		case UHF_PORT_CONNECTION:
+		case UHF_PORT_OVER_CURRENT:
+		case UHF_PORT_LOW_SPEED:
+		case UHF_C_PORT_SUSPEND:
+		case UHF_C_PORT_RESET:
+		default:
+			err = USB_ERR_IOERROR;
+			goto done;
+		}
+		break;
+	default:
+		err = USB_ERR_IOERROR;
+		goto done;
+	}
+done:
+	*plength = len;
+	*pptr = ptr;
+	return (err);
+}
+
+static void
+octusb_xfer_setup(struct usb_setup_params *parm)
+{
+	struct usb_page_search page_info;
+	struct usb_page_cache *pc;
+	struct octusb_softc *sc;
+	struct octusb_qh *qh;
+	struct usb_xfer *xfer;
+	void *last_obj;
+	uint32_t n;
+	uint32_t ntd;
+
+	sc = OCTUSB_BUS2SC(parm->udev->bus);
+	xfer = parm->curr_xfer;
+	qh = NULL;
+
+	/*
+	 * NOTE: This driver does not use any of the parameters that
+	 * are computed from the following values. Just set some
+	 * reasonable dummies:
+	 */
+
+	parm->hc_max_packet_size = 0x400;
+	parm->hc_max_packet_count = 3;
+	parm->hc_max_frame_size = 0xC00;
+
+	usbd_transfer_setup_sub(parm);
+
+	if (parm->err)
+		return;
+
+	/* Allocate a queue head */
+
+	if (usbd_transfer_setup_sub_malloc(
+	    parm, &pc, sizeof(struct octusb_qh),
+	    USB_HOST_ALIGN, 1)) {
+		parm->err = USB_ERR_NOMEM;
+		return;
+	}
+	if (parm->buf) {
+		usbd_get_page(pc, 0, &page_info);
+
+		qh = page_info.buffer;
+
+		/* fill out QH */
+
+		qh->sc = OCTUSB_BUS2SC(xfer->xroot->bus);
+		qh->max_frame_size = xfer->max_frame_size;
+		qh->max_packet_size = xfer->max_packet_size;
+		qh->ep_num = xfer->endpointno;
+		qh->ep_type = xfer->endpoint->edesc->bmAttributes;
+		qh->dev_addr = xfer->address;
+		qh->dev_speed = usbd_get_speed(xfer->xroot->udev);
+		qh->port_index = xfer->xroot->udev->port_index;
+
+		switch (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) {
+		case UE_INTERRUPT:
+			if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH)
+				qh->ep_interval = xfer->interval * 8;
+			else
+				qh->ep_interval = xfer->interval * 1;
+			break;
+		case UE_ISOCHRONOUS:
+			qh->ep_interval = 1 << xfer->fps_shift;
+			break;
+		default:
+			qh->ep_interval = 0;
+			break;
+		}
+
+		qh->ep_mult = xfer->max_packet_count & 3;
+		qh->hs_hub_addr = xfer->xroot->udev->hs_hub_addr;
+		qh->hs_hub_port = xfer->xroot->udev->hs_port_no;
+	}
+	xfer->qh_start[0] = qh;
+
+	/* Allocate a fixup buffer */
+
+	if (usbd_transfer_setup_sub_malloc(
+	    parm, &pc, OCTUSB_MAX_FIXUP,
+	    OCTUSB_MAX_FIXUP, 1)) {
+		parm->err = USB_ERR_NOMEM;
+		return;
+	}
+	if (parm->buf) {
+		usbd_get_page(pc, 0, &page_info);
+
+		qh->fixup_phys = page_info.physaddr;
+		qh->fixup_pc = pc;
+		qh->fixup_buf = page_info.buffer;
+	}
+	/* Allocate transfer descriptors */
+
+	last_obj = NULL;
+
+	ntd = xfer->nframes + 1 /* STATUS */ + 1 /* SYNC */ ;
+
+	if (usbd_transfer_setup_sub_malloc(
+	    parm, &pc, sizeof(struct octusb_td),
+	    USB_HOST_ALIGN, ntd)) {
+		parm->err = USB_ERR_NOMEM;
+		return;
+	}
+	if (parm->buf) {
+		for (n = 0; n != ntd; n++) {
+			struct octusb_td *td;
+
+			usbd_get_page(pc + n, 0, &page_info);
+
+			td = page_info.buffer;
+
+			td->qh = qh;
+			td->obj_next = last_obj;
+
+			last_obj = td;
+		}
+	}
+	xfer->td_start[0] = last_obj;
+}
+
+static void
+octusb_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
+    struct usb_endpoint *ep)
+{
+	struct octusb_softc *sc = OCTUSB_BUS2SC(udev->bus);
+
+	DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
+	    ep, udev->address, edesc->bEndpointAddress,
+	    udev->flags.usb_mode, sc->sc_addr);
+
+	if (udev->flags.usb_mode != USB_MODE_HOST) {
+		/* not supported */
+		return;
+	}
+	if (udev->device_index != sc->sc_addr) {
+		switch (edesc->bmAttributes & UE_XFERTYPE) {
+		case UE_CONTROL:
+			ep->methods = &octusb_device_ctrl_methods;
+			break;
+		case UE_INTERRUPT:
+			ep->methods = &octusb_device_intr_methods;
+			break;
+		case UE_ISOCHRONOUS:
+			if (udev->speed != USB_SPEED_LOW)
+				ep->methods = &octusb_device_isoc_methods;
+			break;
+		case UE_BULK:
+			ep->methods = &octusb_device_bulk_methods;
+			break;
+		default:
+			/* do nothing */
+			break;
+		}
+	}
+}
+
+static void
+octusb_xfer_unsetup(struct usb_xfer *xfer)
+{
+	DPRINTF("Nothing to do.\n");
+}
+
+static void
+octusb_get_dma_delay(struct usb_bus *bus, uint32_t *pus)
+{
+	/* DMA delay - wait until any use of memory is finished */
+	*pus = (2125);			/* microseconds */
+}
+
+static void
+octusb_device_resume(struct usb_device *udev)
+{
+	DPRINTF("Nothing to do.\n");
+}
+
+static void
+octusb_device_suspend(struct usb_device *udev)
+{
+	DPRINTF("Nothing to do.\n");
+}
+
+static void
+octusb_set_hw_power(struct usb_bus *bus)
+{
+	DPRINTF("Nothing to do.\n");
+}
+
+struct usb_bus_methods octusb_bus_methods = {
+	.endpoint_init = octusb_ep_init,
+	.xfer_setup = octusb_xfer_setup,
+	.xfer_unsetup = octusb_xfer_unsetup,
+	.get_dma_delay = octusb_get_dma_delay,
+	.device_resume = octusb_device_resume,
+	.device_suspend = octusb_device_suspend,
+	.set_hw_power = octusb_set_hw_power,
+	.roothub_exec = octusb_roothub_exec,
+	.xfer_poll = octusb_do_poll,
+};
diff --git a/sys/mips/cavium/usb/octusb.h b/sys/mips/cavium/usb/octusb.h
new file mode 100644
index 0000000..31f4fc0
--- /dev/null
+++ b/sys/mips/cavium/usb/octusb.h
@@ -0,0 +1,147 @@
+/* $FreeBSD$ */
+
+/*-
+ * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _OCTUSB_H_
+#define	_OCTUSB_H_
+
+#define	OCTUSB_MAX_DEVICES MIN(USB_MAX_DEVICES, 64)
+/*
+ * The second port is on a different IRQ and so we disable it for now.
+ */
+#if 1
+#define	OCTUSB_MAX_PORTS	1	/* hardcoded */
+#else
+#define	OCTUSB_MAX_PORTS	2	/* hardcoded */
+#endif
+#define	OCTUSB_MAX_FIXUP	4096	/* bytes */
+#define	OCTUSB_INTR_ENDPT	0x01
+
+struct octusb_qh;
+struct octusb_td;
+struct octusb_softc;
+
+typedef uint8_t (octusb_cmd_t)(struct octusb_td *td);
+
+struct octusb_td {
+	struct octusb_qh *qh;
+	struct octusb_td *obj_next;
+	struct usb_page_cache *pc;
+	octusb_cmd_t *func;
+
+	uint32_t remainder;
+	uint32_t offset;
+
+	uint8_t	error_any:1;
+	uint8_t	error_stall:1;
+	uint8_t	short_pkt:1;
+	uint8_t	alt_next:1;
+	uint8_t	reserved:4;
+};
+
+struct octusb_qh {
+
+	uint64_t fixup_phys;
+
+	struct octusb_softc *sc;
+	struct usb_page_cache *fixup_pc;
+	uint8_t *fixup_buf;
+
+	cvmx_usb_iso_packet_t iso_pkt;
+
+	uint32_t fixup_off;
+
+	uint16_t max_frame_size;
+	uint16_t max_packet_size;
+	uint16_t fixup_actlen;
+	uint16_t fixup_len;
+	uint16_t ep_interval;
+
+	uint8_t	dev_addr;
+	uint8_t	dev_speed;
+	uint8_t	ep_allocated;
+	uint8_t	ep_mult;
+	uint8_t	ep_num;
+	uint8_t	ep_type;
+	uint8_t	ep_toggle_next;
+	uint8_t	port_index;
+	uint8_t	fixup_complete;
+	uint8_t	fixup_pending;
+	uint8_t	hs_hub_addr;
+	uint8_t	hs_hub_port;
+
+	int	fixup_handle;
+	int	ep_handle;
+};
+
+struct octusb_config_desc {
+	struct usb_config_descriptor confd;
+	struct usb_interface_descriptor ifcd;
+	struct usb_endpoint_descriptor endpd;
+} __packed;
+
+union octusb_hub_desc {
+	struct usb_status stat;
+	struct usb_port_status ps;
+	uint8_t	temp[128];
+};
+
+struct octusb_port {
+	cvmx_usb_state_t state;
+	uint8_t	disabled;
+};
+
+struct octusb_softc {
+
+	struct usb_bus sc_bus;		/* base device */
+	union octusb_hub_desc sc_hub_desc;
+
+	struct usb_device *sc_devices[OCTUSB_MAX_DEVICES];
+
+	struct resource *sc_irq_res;
+	void   *sc_intr_hdl;
+
+	struct octusb_port sc_port[OCTUSB_MAX_PORTS];
+	device_t sc_dev;
+
+	struct usb_hub_descriptor_min sc_hubd;
+
+	uint8_t	sc_noport;		/* number of ports */
+	uint8_t	sc_addr;		/* device address */
+	uint8_t	sc_conf;		/* device configuration */
+	uint8_t	sc_isreset;		/* set if current port is reset */
+
+	uint8_t	sc_hub_idata[1];
+};
+
+usb_bus_mem_cb_t octusb_iterate_hw_softc;
+usb_error_t octusb_init(struct octusb_softc *);
+usb_error_t octusb_uninit(struct octusb_softc *);
+void	octusb_suspend(struct octusb_softc *);
+void	octusb_resume(struct octusb_softc *);
+void	octusb_interrupt(struct octusb_softc *);
+
+#endif					/* _OCTUSB_H_ */
diff --git a/sys/mips/cavium/usb/octusb_octeon.c b/sys/mips/cavium/usb/octusb_octeon.c
new file mode 100644
index 0000000..edd8290
--- /dev/null
+++ b/sys/mips/cavium/usb/octusb_octeon.c
@@ -0,0 +1,223 @@
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*-
+ * Copyright (c) 2007-2008 Hans Petter Selasky. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/stdint.h>
+#include <sys/stddef.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/types.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/linker_set.h>
+#include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/condvar.h>
+#include <sys/sysctl.h>
+#include <sys/sx.h>
+#include <sys/unistd.h>
+#include <sys/callout.h>
+#include <sys/malloc.h>
+#include <sys/priv.h>
+#include <sys/rman.h>
+
+#include <dev/usb/usb.h>
+#include <dev/usb/usbdi.h>
+
+#include <dev/usb/usb_core.h>
+#include <dev/usb/usb_busdma.h>
+#include <dev/usb/usb_process.h>
+#include <dev/usb/usb_util.h>
+
+#include <dev/usb/usb_controller.h>
+#include <dev/usb/usb_bus.h>
+
+#include <contrib/octeon-sdk/cvmx.h>
+#include <contrib/octeon-sdk/cvmx-interrupt.h>
+#include <contrib/octeon-sdk/cvmx-usb.h>
+
+#include <mips/cavium/usb/octusb.h>
+
+#define	MEM_RID	0
+
+static device_identify_t octusb_octeon_identify;
+static device_probe_t octusb_octeon_probe;
+static device_attach_t octusb_octeon_attach;
+static device_detach_t octusb_octeon_detach;
+static device_shutdown_t octusb_octeon_shutdown;
+
+struct octusb_octeon_softc {
+	struct octusb_softc sc_dci;	/* must be first */
+};
+
+static void
+octusb_octeon_identify(driver_t *drv, device_t parent)
+{
+	if (octeon_has_feature(OCTEON_FEATURE_USB))
+		BUS_ADD_CHILD(parent, 0, "octusb", 0);
+}
+
+static int
+octusb_octeon_probe(device_t dev)
+{
+	device_set_desc(dev, "Cavium Octeon USB controller");
+	return (0);
+}
+
+static int
+octusb_octeon_attach(device_t dev)
+{
+	struct octusb_octeon_softc *sc = device_get_softc(dev);
+	int err;
+	int rid;
+
+	/* setup controller interface softc */
+
+	/* initialise some bus fields */
+	sc->sc_dci.sc_bus.parent = dev;
+	sc->sc_dci.sc_bus.devices = sc->sc_dci.sc_devices;
+	sc->sc_dci.sc_bus.devices_max = OCTUSB_MAX_DEVICES;
+
+	/* get all DMA memory */
+	if (usb_bus_mem_alloc_all(&sc->sc_dci.sc_bus,
+	    USB_GET_DMA_TAG(dev), NULL)) {
+		return (ENOMEM);
+	}
+	rid = 0;
+	sc->sc_dci.sc_irq_res =
+	    bus_alloc_resource(dev, SYS_RES_IRQ, &rid,
+			       CVMX_IRQ_USB, CVMX_IRQ_USB, 1, RF_ACTIVE);
+	if (!(sc->sc_dci.sc_irq_res)) {
+		goto error;
+	}
+
+	sc->sc_dci.sc_bus.bdev = device_add_child(dev, "usbus", -1);
+	if (!(sc->sc_dci.sc_bus.bdev)) {
+		goto error;
+	}
+	device_set_ivars(sc->sc_dci.sc_bus.bdev, &sc->sc_dci.sc_bus);
+
+#if (__FreeBSD_version >= 700031)
+	err = bus_setup_intr(dev, sc->sc_dci.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
+	    NULL, (driver_intr_t *)octusb_interrupt, sc, &sc->sc_dci.sc_intr_hdl);
+#else
+	err = bus_setup_intr(dev, sc->sc_dci.sc_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
+	    (driver_intr_t *)octusb_interrupt, sc, &sc->sc_dci.sc_intr_hdl);
+#endif
+	if (err) {
+		sc->sc_dci.sc_intr_hdl = NULL;
+		goto error;
+	}
+	err = octusb_init(&sc->sc_dci);
+	if (!err) {
+		err = device_probe_and_attach(sc->sc_dci.sc_bus.bdev);
+	}
+	if (err) {
+		goto error;
+	}
+	return (0);
+
+error:
+	octusb_octeon_detach(dev);
+	return (ENXIO);
+}
+
+static int
+octusb_octeon_detach(device_t dev)
+{
+	struct octusb_octeon_softc *sc = device_get_softc(dev);
+	device_t bdev;
+	int err;
+
+	if (sc->sc_dci.sc_bus.bdev) {
+		bdev = sc->sc_dci.sc_bus.bdev;
+		device_detach(bdev);
+		device_delete_child(dev, bdev);
+	}
+	/* during module unload there are lots of children leftover */
+	device_delete_all_children(dev);
+
+	if (sc->sc_dci.sc_irq_res && sc->sc_dci.sc_intr_hdl) {
+		/*
+		 * only call octusb_octeon_uninit() after octusb_octeon_init()
+		 */
+		octusb_uninit(&sc->sc_dci);
+
+		err = bus_teardown_intr(dev, sc->sc_dci.sc_irq_res,
+		    sc->sc_dci.sc_intr_hdl);
+		sc->sc_dci.sc_intr_hdl = NULL;
+	}
+	if (sc->sc_dci.sc_irq_res) {
+		bus_release_resource(dev, SYS_RES_IRQ, 0,
+		    sc->sc_dci.sc_irq_res);
+		sc->sc_dci.sc_irq_res = NULL;
+	}
+	usb_bus_mem_free_all(&sc->sc_dci.sc_bus, NULL);
+
+	return (0);
+}
+
+static int
+octusb_octeon_shutdown(device_t dev)
+{
+	struct octusb_octeon_softc *sc = device_get_softc(dev);
+	int err;
+
+	err = bus_generic_shutdown(dev);
+	if (err)
+		return (err);
+
+	octusb_uninit(&sc->sc_dci);
+
+	return (0);
+}
+
+static device_method_t octusb_octeon_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_identify, octusb_octeon_identify),
+	DEVMETHOD(device_probe, octusb_octeon_probe),
+	DEVMETHOD(device_attach, octusb_octeon_attach),
+	DEVMETHOD(device_detach, octusb_octeon_detach),
+	DEVMETHOD(device_shutdown, octusb_octeon_shutdown),
+
+	/* Bus interface */
+	DEVMETHOD(bus_print_child, bus_generic_print_child),
+
+	{0, 0}
+};
+
+static driver_t octusb_octeon_driver = {
+	"octusb",
+	octusb_octeon_methods,
+	sizeof(struct octusb_octeon_softc),
+};
+
+static devclass_t octusb_octeon_devclass;
+
+DRIVER_MODULE(octusb, ciu, octusb_octeon_driver, octusb_octeon_devclass, 0, 0);