Implement support for HMAC/SHA1 and HMAC/SHA256 acceleration found in

new VIA CPUs.
For older CPUs HMAC/SHA1 and HMAC/SHA256 (and others) will still be done
in software.

Move symmetric cryptography (currently only AES-CBC 128/192/256) to
padlock_cipher.c file. Move HMAC cryptography to padlock_hash.c file.

Hardware from:	Centaur Technologies

4 files changed, 766 insertions, 335 deletions

diff --git a/sys/crypto/via/padlock.c b/sys/crypto/via/padlock.c
index 9a221d7..4304c49 100644
--- a/sys/crypto/via/padlock.c
+++ b/sys/crypto/via/padlock.c
@@ -1,6 +1,5 @@
 /*-
- * Copyright (c) 2005 Pawel Jakub Dawidek <pjd@FreeBSD.org>
- * Copyright (c) 2004 Mark R V Murray
+ * Copyright (c) 2005-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -25,25 +24,6 @@
  * SUCH DAMAGE.
  */
 
-/*	$OpenBSD: via.c,v 1.3 2004/06/15 23:36:55 deraadt Exp $	*/
-/*-
- * Copyright (c) 2003 Jason Wright
- * Copyright (c) 2003, 2004 Theo de Raadt
- * All rights reserved.
- *
- * Permission to use, copy, modify, and distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
@@ -63,66 +43,14 @@ __FBSDID("$FreeBSD$");
 #endif
 
 #include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */
-#include <opencrypto/xform.h>
-#include <crypto/rijndael/rijndael.h>
-
-
-#define	PADLOCK_ROUND_COUNT_AES128	10
-#define	PADLOCK_ROUND_COUNT_AES192	12
-#define	PADLOCK_ROUND_COUNT_AES256	14
-
-#define	PADLOCK_ALGORITHM_TYPE_AES	0
-
-#define	PADLOCK_KEY_GENERATION_HW	0
-#define	PADLOCK_KEY_GENERATION_SW	1
-
-#define	PADLOCK_DIRECTION_ENCRYPT	0
-#define	PADLOCK_DIRECTION_DECRYPT	1
-
-#define	PADLOCK_KEY_SIZE_128	0
-#define	PADLOCK_KEY_SIZE_192	1
-#define	PADLOCK_KEY_SIZE_256	2
-
-union padlock_cw {
-	uint64_t raw;
-	struct {
-		u_int round_count : 4;
-		u_int algorithm_type : 3;
-		u_int key_generation : 1;
-		u_int intermediate : 1;
-		u_int direction : 1;
-		u_int key_size : 2;
-		u_int filler0 : 20;
-		u_int filler1 : 32;
-		u_int filler2 : 32;
-		u_int filler3 : 32;
-	} __field;
-};
-#define	cw_round_count		__field.round_count
-#define	cw_algorithm_type	__field.algorithm_type
-#define	cw_key_generation	__field.key_generation
-#define	cw_intermediate		__field.intermediate
-#define	cw_direction		__field.direction
-#define	cw_key_size		__field.key_size
-#define	cw_filler0		__field.filler0
-#define	cw_filler1		__field.filler1
-#define	cw_filler2		__field.filler2
-#define	cw_filler3		__field.filler3
-
-struct padlock_session {
-	union padlock_cw ses_cw __aligned(16);
-	uint32_t	ses_ekey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16);	/* 128 bit aligned */
-	uint32_t	ses_dkey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16);	/* 128 bit aligned */
-	uint8_t		ses_iv[16] __aligned(16);			/* 128 bit aligned */
-	struct auth_hash *ses_axf;
-	uint8_t		*ses_ictx;
-	uint8_t		*ses_octx;
-	int		ses_mlen;
-	int		ses_used;
-	uint32_t	ses_id;
-	TAILQ_ENTRY(padlock_session) ses_next;
-};
+
+#include <crypto/via/padlock.h>
+
+/*
+ * Technical documentation about the PadLock engine can be found here:
+ *
+ * http://www.via.com.tw/en/downloads/whitepapers/initiatives/padlock/programming_guide.pdf
+ */
 
 struct padlock_softc {
 	int32_t		sc_cid;
@@ -139,47 +67,50 @@ static int padlock_freesession(void *arg __unused, uint64_t tid);
 static int padlock_process(void *arg __unused, struct cryptop *crp,
     int hint __unused);
 
-static __inline void
-padlock_cbc(void *in, void *out, size_t count, void *key, union padlock_cw *cw,
-    void *iv)
-{
-#ifdef __GNUCLIKE_ASM
-	/* The .byte line is really VIA C3 "xcrypt-cbc" instruction */
-	__asm __volatile(
-		"pushf				\n\t"
-		"popf				\n\t"
-		"rep				\n\t"
-		".byte	0x0f, 0xa7, 0xd0"
-			: "+a" (iv), "+c" (count), "+D" (out), "+S" (in)
-			: "b" (key), "d" (cw)
-			: "cc", "memory"
-		);
-#endif
-}
+MALLOC_DEFINE(M_PADLOCK, "padlock_data", "PadLock Data");
 
 static int
 padlock_init(void)
 {
 	struct padlock_softc *sc;
+	char capp[256];
+
 #if defined(__i386__) && !defined(PC98)
+	/* If there is no AES support, we has nothing to do here. */
 	if (!(via_feature_xcrypt & VIA_HAS_AES)) {
-		printf("PADLOCK: No ACE support.\n");
+		printf("PadLock: No ACE support.\n");
 		return (EINVAL);
-	} else
-		printf("PADLOCK: HW support loaded.\n");
+	}
+	strlcpy(capp, "AES-CBC", sizeof(capp));
+#if 0
+	strlcat(capp, ",AES-EBC", sizeof(capp));
+	strlcat(capp, ",AES-CFB", sizeof(capp));
+	strlcat(capp, ",AES-OFB", sizeof(capp));
+#endif
+	if (via_feature_xcrypt & VIA_HAS_SHA) {
+		strlcat(capp, ",SHA1", sizeof(capp));
+		strlcat(capp, ",SHA256", sizeof(capp));
+	}
+#if 0
+	if (via_feature_xcrypt & VIA_HAS_AESCTR)
+		strlcat(capp, ",AES-CTR", sizeof(capp));
+	if (via_feature_xcrypt & VIA_HAS_MM)
+		strlcat(capp, ",RSA", sizeof(capp));
+#endif
+	printf("PadLock: HW support loaded for %s.\n", capp);
 #else
 	return (EINVAL);
 #endif
 
-	padlock_sc = sc = malloc(sizeof(*padlock_sc), M_DEVBUF,
+	padlock_sc = sc = malloc(sizeof(*padlock_sc), M_PADLOCK,
 	    M_WAITOK | M_ZERO);
 	TAILQ_INIT(&sc->sc_sessions);
 	sc->sc_sid = 1;
 
 	sc->sc_cid = crypto_get_driverid(0);
 	if (sc->sc_cid < 0) {
-		printf("PADLOCK: Could not get crypto driver id.\n");
-		free(padlock_sc, M_DEVBUF);
+		printf("PadLock: Could not get crypto driver id.\n");
+		free(padlock_sc, M_PADLOCK);
 		padlock_sc = NULL;
 		return (ENOMEM);
 	}       
@@ -218,7 +149,7 @@ padlock_destroy(void)
 	}
 	if (active > 0) {
 		mtx_unlock(&sc->sc_sessions_mtx);
-		printf("PADLOCK: Cannot destroy, %u sessions active.\n",
+		printf("PadLock: Cannot destroy, %u sessions active.\n",
 		    active);
 		return (EBUSY);
 	}
@@ -226,102 +157,21 @@ padlock_destroy(void)
 	for (ses = TAILQ_FIRST(&sc->sc_sessions); ses != NULL;
 	    ses = TAILQ_FIRST(&sc->sc_sessions)) {
 		TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
-		free(ses, M_DEVBUF);
+		free(ses, M_PADLOCK);
 	}
 	mtx_destroy(&sc->sc_sessions_mtx);
 	crypto_unregister_all(sc->sc_cid);
-	free(sc, M_DEVBUF);
+	free(sc, M_PADLOCK);
 	return (0);
 }
 
-static void
-padlock_setup_enckey(struct padlock_session *ses, caddr_t key, int klen)
-{
-	union padlock_cw *cw;
-	int i;
-
-	cw = &ses->ses_cw;
-	if (cw->cw_key_generation == PADLOCK_KEY_GENERATION_SW) {
-		/* Build expanded keys for both directions */
-		rijndaelKeySetupEnc(ses->ses_ekey, key, klen);
-		rijndaelKeySetupDec(ses->ses_dkey, key, klen);
-		for (i = 0; i < 4 * (RIJNDAEL_MAXNR + 1); i++) {
-			ses->ses_ekey[i] = ntohl(ses->ses_ekey[i]);
-			ses->ses_dkey[i] = ntohl(ses->ses_dkey[i]);
-		}
-	} else {
-		bcopy(key, ses->ses_ekey, klen);
-		bcopy(key, ses->ses_dkey, klen);
-	}
-}
-
-static void
-padlock_setup_mackey(struct padlock_session *ses, caddr_t key, int klen)
-{
-	struct auth_hash *axf;
-	int i;
-
-	klen /= 8;
-	axf = ses->ses_axf;
-
-	for (i = 0; i < klen; i++)
-		key[i] ^= HMAC_IPAD_VAL;
-
-	axf->Init(ses->ses_ictx);
-	axf->Update(ses->ses_ictx, key, klen);
-	axf->Update(ses->ses_ictx, hmac_ipad_buffer, axf->blocksize - klen);
-
-	for (i = 0; i < klen; i++)
-		key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-
-	axf->Init(ses->ses_octx); 
-	axf->Update(ses->ses_octx, key, klen);
-	axf->Update(ses->ses_octx, hmac_opad_buffer, axf->blocksize - klen);
-
-	for (i = 0; i < klen; i++)
-		key[i] ^= HMAC_OPAD_VAL;
-}
-
-/*
- * Compute keyed-hash authenticator.
- */
-static int
-padlock_authcompute(struct padlock_session *ses, struct cryptodesc *crd,
-    caddr_t buf, int flags)
-{
-	u_char hash[HASH_MAX_LEN];
-	struct auth_hash *axf;
-	union authctx ctx;
-	int error;
-
-	axf = ses->ses_axf;
-
-	bcopy(ses->ses_ictx, &ctx, axf->ctxsize);
-
-	error = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
-	    (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
-	if (error != 0)
-		return (error);
-
-	axf->Final(hash, &ctx);
-	bcopy(ses->ses_octx, &ctx, axf->ctxsize);
-	axf->Update(&ctx, hash, axf->hashsize);
-	axf->Final(hash, &ctx);
-
-	/* Inject the authentication data */
-	crypto_copyback(flags, buf, crd->crd_inject,
-	    ses->ses_mlen == 0 ? axf->hashsize : ses->ses_mlen, hash);
-	return (0);
-}
-
-
 static int
 padlock_newsession(void *arg __unused, uint32_t *sidp, struct cryptoini *cri)
 {
 	struct padlock_softc *sc = padlock_sc;
 	struct padlock_session *ses = NULL;
 	struct cryptoini *encini, *macini;
-	union padlock_cw *cw;
+	int error;
 
 	if (sc == NULL || sidp == NULL || cri == NULL)
 		return (EINVAL);
@@ -357,10 +207,6 @@ padlock_newsession(void *arg __unused, uint32_t *sidp, struct cryptoini *cri)
 	 */
 	if (encini == NULL)
 		return (EINVAL);
-	if (encini->cri_klen != 128 && encini->cri_klen != 192 &&
-	    encini->cri_klen != 256) {
-		return (EINVAL);
-	}
 
 	/*
 	 * Let's look for a free session structure.
@@ -380,7 +226,7 @@ padlock_newsession(void *arg __unused, uint32_t *sidp, struct cryptoini *cri)
 	}
 	mtx_unlock(&sc->sc_sessions_mtx);
 	if (ses == NULL) {
-		ses = malloc(sizeof(*ses), M_DEVBUF, M_NOWAIT | M_ZERO);
+		ses = malloc(sizeof(*ses), M_PADLOCK, M_NOWAIT | M_ZERO);
 		if (ses == NULL)
 			return (ENOMEM);
 		ses->ses_used = 1;
@@ -390,76 +236,17 @@ padlock_newsession(void *arg __unused, uint32_t *sidp, struct cryptoini *cri)
 		mtx_unlock(&sc->sc_sessions_mtx);
 	}
 
-	cw = &ses->ses_cw;
-	bzero(cw, sizeof(*cw));
-	cw->cw_algorithm_type = PADLOCK_ALGORITHM_TYPE_AES;
-	cw->cw_key_generation = PADLOCK_KEY_GENERATION_SW;
-	cw->cw_intermediate = 0;
-	switch (encini->cri_klen) {
-	case 128:
-		cw->cw_round_count = PADLOCK_ROUND_COUNT_AES128;
-		cw->cw_key_size = PADLOCK_KEY_SIZE_128;
-#ifdef HW_KEY_GENERATION
-		/* This doesn't buy us much, that's why it is commented out. */
-		cw->cw_key_generation = PADLOCK_KEY_GENERATION_HW;
-#endif
-		break;
-	case 192:
-		cw->cw_round_count = PADLOCK_ROUND_COUNT_AES192;
-		cw->cw_key_size = PADLOCK_KEY_SIZE_192;
-		break;
-	case 256:
-		cw->cw_round_count = PADLOCK_ROUND_COUNT_AES256;
-		cw->cw_key_size = PADLOCK_KEY_SIZE_256;
-		break;
+	error = padlock_cipher_setup(ses, encini);
+	if (error != 0) {
+		padlock_freesession(NULL, ses->ses_id);
+		return (error);
 	}
-	if (encini->cri_key != NULL)
-		padlock_setup_enckey(ses, encini->cri_key, encini->cri_klen);
-
-	arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0);
 
 	if (macini != NULL) {
-		ses->ses_mlen = macini->cri_mlen;
-
-		/* Find software structure which describes HMAC algorithm. */
-		switch (macini->cri_alg) {
-		case CRYPTO_NULL_HMAC:
-			ses->ses_axf = &auth_hash_null;
-			break;
-		case CRYPTO_MD5_HMAC:
-			ses->ses_axf = &auth_hash_hmac_md5;
-			break;
-		case CRYPTO_SHA1_HMAC:
-			ses->ses_axf = &auth_hash_hmac_sha1;
-			break;
-		case CRYPTO_RIPEMD160_HMAC:
-			ses->ses_axf = &auth_hash_hmac_ripemd_160;
-			break;
-		case CRYPTO_SHA2_256_HMAC:
-			ses->ses_axf = &auth_hash_hmac_sha2_256;
-			break;
-		case CRYPTO_SHA2_384_HMAC:
-			ses->ses_axf = &auth_hash_hmac_sha2_384;
-			break;
-		case CRYPTO_SHA2_512_HMAC:
-			ses->ses_axf = &auth_hash_hmac_sha2_512;
-			break;
-		}
-
-		/* Allocate memory for HMAC inner and outer contexts. */
-		ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_CRYPTO_DATA,
-		    M_NOWAIT);
-		ses->ses_octx = malloc(ses->ses_axf->ctxsize, M_CRYPTO_DATA,
-		    M_NOWAIT);
-		if (ses->ses_ictx == NULL || ses->ses_octx == NULL) {
+		error = padlock_hash_setup(ses, macini);
+		if (error != 0) {
 			padlock_freesession(NULL, ses->ses_id);
-			return (ENOMEM);
-		}
-
-		/* Setup key if given. */
-		if (macini->cri_key != NULL) {
-			padlock_setup_mackey(ses, macini->cri_key,
-			    macini->cri_klen);
+			return (error);
 		}
 	}
 
@@ -486,16 +273,7 @@ padlock_freesession(void *arg __unused, uint64_t tid)
 		return (EINVAL);
 	}
 	TAILQ_REMOVE(&sc->sc_sessions, ses, ses_next);
-	if (ses->ses_ictx != NULL) {
-		bzero(ses->ses_ictx, ses->ses_axf->ctxsize);
-		free(ses->ses_ictx, M_CRYPTO_DATA);
-		ses->ses_ictx = NULL;
-	}
-	if (ses->ses_octx != NULL) {
-		bzero(ses->ses_ictx, ses->ses_axf->ctxsize);
-		free(ses->ses_octx, M_CRYPTO_DATA);
-		ses->ses_octx = NULL;
-	}
+	padlock_hash_free(ses);
 	bzero(ses, sizeof(*ses));
 	ses->ses_used = 0;
 	TAILQ_INSERT_TAIL(&sc->sc_sessions, ses, ses_next);
@@ -507,15 +285,11 @@ static int
 padlock_process(void *arg __unused, struct cryptop *crp, int hint __unused)
 {
 	struct padlock_softc *sc = padlock_sc;
-	struct padlock_session *ses;
-	union padlock_cw *cw;
+	struct padlock_session *ses = NULL;
 	struct cryptodesc *crd, *enccrd, *maccrd;
-	uint32_t *key;
-	u_char *buf, *abuf;
 	int error = 0;
 
 	enccrd = maccrd = NULL;
-	buf = NULL;
 
 	if (crp == NULL || crp->crp_callback == NULL || crp->crp_desc == NULL) {
 		error = EINVAL;
@@ -564,81 +338,36 @@ padlock_process(void *arg __unused, struct cryptop *crp, int hint __unused)
 		goto out;
 	}
 
-	buf = malloc(enccrd->crd_len + 16, M_DEVBUF, M_NOWAIT);
-	if (buf == NULL) {
-		error = ENOMEM;
-		goto out;
-	}
-	/* Buffer has to be 16 bytes aligned. */
-	abuf = buf + 16 - ((uintptr_t)buf % 16);
-
-	if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0)
-		padlock_setup_enckey(ses, enccrd->crd_key, enccrd->crd_klen);
-	if (maccrd != NULL && (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0)
-		padlock_setup_mackey(ses, maccrd->crd_key, maccrd->crd_klen);
-
-	cw = &ses->ses_cw;
-	cw->cw_filler0 = 0;
-	cw->cw_filler1 = 0;
-	cw->cw_filler2 = 0;
-	cw->cw_filler3 = 0;
-	if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
-		cw->cw_direction = PADLOCK_DIRECTION_ENCRYPT;
-		key = ses->ses_ekey;
-		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
-			bcopy(enccrd->crd_iv, ses->ses_iv, 16);
-
-		if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    enccrd->crd_inject, AES_BLOCK_LEN, ses->ses_iv);
-		}
-	} else {
-		cw->cw_direction = PADLOCK_DIRECTION_DECRYPT;
-		key = ses->ses_dkey;
-		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
-			bcopy(enccrd->crd_iv, ses->ses_iv, AES_BLOCK_LEN);
-		else {
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    enccrd->crd_inject, AES_BLOCK_LEN, ses->ses_iv);
-		}
-	}
-
 	/* Perform data authentication if requested before encryption. */
 	if (maccrd != NULL && maccrd->crd_next == enccrd) {
-		error = padlock_authcompute(ses, maccrd, crp->crp_buf,
-		    crp->crp_flags);
+		error = padlock_hash_process(ses, maccrd, crp);
 		if (error != 0)
 			goto out;
 	}
 
-	crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
-	    enccrd->crd_len, abuf);
-
-	padlock_cbc(abuf, abuf, enccrd->crd_len / 16, key, cw, ses->ses_iv);
-
-	crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
-	    enccrd->crd_len, abuf);
+	error = padlock_cipher_process(ses, enccrd, crp);
+	if (error != 0)
+		goto out;
 
 	/* Perform data authentication if requested after encryption. */
 	if (maccrd != NULL && enccrd->crd_next == maccrd) {
-		error = padlock_authcompute(ses, maccrd, crp->crp_buf,
-		    crp->crp_flags);
+		error = padlock_hash_process(ses, maccrd, crp);
 		if (error != 0)
 			goto out;
 	}
 
-	/* copy out last block for use as next session IV */
-	if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
-		crypto_copydata(crp->crp_flags, crp->crp_buf,
-		    enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN,
-		    AES_BLOCK_LEN, ses->ses_iv);
-	}
-
 out:
-	if (buf != NULL) {
-		bzero(buf, enccrd->crd_len + 16);
-		free(buf, M_DEVBUF);
+#if 0
+	/*
+	 * This code is not necessary, because contexts will be freed on next
+	 * padlock_setup_mackey() call or at padlock_freesession() call.
+	 */
+	if (ses != NULL && maccrd != NULL &&
+	    (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
+		padlock_free_ctx(ses->ses_axf, ses->ses_ictx);
+		padlock_free_ctx(ses->ses_axf, ses->ses_octx);
 	}
+#endif
 	crp->crp_etype = error;
 	crypto_done(crp);
 	return (error);
diff --git a/sys/crypto/via/padlock.h b/sys/crypto/via/padlock.h
new file mode 100644
index 0000000..29eed52
--- /dev/null
+++ b/sys/crypto/via/padlock.h
@@ -0,0 +1,87 @@
+/*-
+ * Copyright (c) 2005-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _PADLOCK_H_
+#define _PADLOCK_H_
+
+#include <opencrypto/cryptodev.h>
+#include <crypto/rijndael/rijndael.h>
+
+union padlock_cw {
+	uint64_t raw;
+	struct {
+		u_int round_count : 4;
+		u_int algorithm_type : 3;
+		u_int key_generation : 1;
+		u_int intermediate : 1;
+		u_int direction : 1;
+		u_int key_size : 2;
+		u_int filler0 : 20;
+		u_int filler1 : 32;
+		u_int filler2 : 32;
+		u_int filler3 : 32;
+	} __field;
+};
+#define	cw_round_count		__field.round_count
+#define	cw_algorithm_type	__field.algorithm_type
+#define	cw_key_generation	__field.key_generation
+#define	cw_intermediate		__field.intermediate
+#define	cw_direction		__field.direction
+#define	cw_key_size		__field.key_size
+#define	cw_filler0		__field.filler0
+#define	cw_filler1		__field.filler1
+#define	cw_filler2		__field.filler2
+#define	cw_filler3		__field.filler3
+
+struct padlock_session {
+	union padlock_cw ses_cw __aligned(16);
+	uint32_t	ses_ekey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16);	/* 128 bit aligned */
+	uint32_t	ses_dkey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16);	/* 128 bit aligned */
+	uint8_t		ses_iv[16] __aligned(16);			/* 128 bit aligned */
+	struct auth_hash *ses_axf;
+	uint8_t		*ses_ictx;
+	uint8_t		*ses_octx;
+	int		ses_mlen;
+	int		ses_used;
+	uint32_t	ses_id;
+	TAILQ_ENTRY(padlock_session) ses_next;
+};
+
+#define PADLOCK_ALIGN(p)	(void *)((char *)(p) - ((uintptr_t)(p) % 16) + 16)
+
+int	padlock_cipher_setup(struct padlock_session *ses,
+	    struct cryptoini *encini);
+int	padlock_cipher_process(struct padlock_session *ses,
+	    struct cryptodesc *enccrd, struct cryptop *crp);
+int	padlock_hash_setup(struct padlock_session *ses,
+	    struct cryptoini *macini);
+int	padlock_hash_process(struct padlock_session *ses,
+	    struct cryptodesc *maccrd, struct cryptop *crp);
+void	padlock_hash_free(struct padlock_session *ses);
+
+#endif	/* !_PADLOCK_H_ */
diff --git a/sys/crypto/via/padlock_cipher.c b/sys/crypto/via/padlock_cipher.c
new file mode 100644
index 0000000..d881a8b
--- /dev/null
+++ b/sys/crypto/via/padlock_cipher.c
@@ -0,0 +1,226 @@
+/*-
+ * Copyright (c) 2005-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
+ * Copyright (c) 2004 Mark R V Murray
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*	$OpenBSD: via.c,v 1.3 2004/06/15 23:36:55 deraadt Exp $	*/
+/*-
+ * Copyright (c) 2003 Jason Wright
+ * Copyright (c) 2003, 2004 Theo de Raadt
+ * All rights reserved.
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/libkern.h>
+
+#include <opencrypto/cryptodev.h>
+#include <crypto/rijndael/rijndael.h>
+
+#include <crypto/via/padlock.h>
+
+#define	PADLOCK_ROUND_COUNT_AES128	10
+#define	PADLOCK_ROUND_COUNT_AES192	12
+#define	PADLOCK_ROUND_COUNT_AES256	14
+
+#define	PADLOCK_ALGORITHM_TYPE_AES	0
+
+#define	PADLOCK_KEY_GENERATION_HW	0
+#define	PADLOCK_KEY_GENERATION_SW	1
+
+#define	PADLOCK_DIRECTION_ENCRYPT	0
+#define	PADLOCK_DIRECTION_DECRYPT	1
+
+#define	PADLOCK_KEY_SIZE_128	0
+#define	PADLOCK_KEY_SIZE_192	1
+#define	PADLOCK_KEY_SIZE_256	2
+
+MALLOC_DECLARE(M_PADLOCK);
+
+static __inline void
+padlock_cbc(void *in, void *out, size_t count, void *key, union padlock_cw *cw,
+    void *iv)
+{
+#ifdef __GNUCLIKE_ASM
+	/* The .byte line is really VIA C3 "xcrypt-cbc" instruction */
+	__asm __volatile(
+		"pushf				\n\t"
+		"popf				\n\t"
+		"rep				\n\t"
+		".byte	0x0f, 0xa7, 0xd0"
+			: "+a" (iv), "+c" (count), "+D" (out), "+S" (in)
+			: "b" (key), "d" (cw)
+			: "cc", "memory"
+		);
+#endif
+}
+
+static void
+padlock_cipher_key_setup(struct padlock_session *ses, caddr_t key, int klen)
+{
+	union padlock_cw *cw;
+	int i;
+
+	cw = &ses->ses_cw;
+	if (cw->cw_key_generation == PADLOCK_KEY_GENERATION_SW) {
+		/* Build expanded keys for both directions */
+		rijndaelKeySetupEnc(ses->ses_ekey, key, klen);
+		rijndaelKeySetupDec(ses->ses_dkey, key, klen);
+		for (i = 0; i < 4 * (RIJNDAEL_MAXNR + 1); i++) {
+			ses->ses_ekey[i] = ntohl(ses->ses_ekey[i]);
+			ses->ses_dkey[i] = ntohl(ses->ses_dkey[i]);
+		}
+	} else {
+		bcopy(key, ses->ses_ekey, klen);
+		bcopy(key, ses->ses_dkey, klen);
+	}
+}
+
+int
+padlock_cipher_setup(struct padlock_session *ses, struct cryptoini *encini)
+{
+	union padlock_cw *cw;
+
+	if (encini->cri_klen != 128 && encini->cri_klen != 192 &&
+	    encini->cri_klen != 256) {
+		return (EINVAL);
+	}
+
+	cw = &ses->ses_cw;
+	bzero(cw, sizeof(*cw));
+	cw->cw_algorithm_type = PADLOCK_ALGORITHM_TYPE_AES;
+	cw->cw_key_generation = PADLOCK_KEY_GENERATION_SW;
+	cw->cw_intermediate = 0;
+	switch (encini->cri_klen) {
+	case 128:
+		cw->cw_round_count = PADLOCK_ROUND_COUNT_AES128;
+		cw->cw_key_size = PADLOCK_KEY_SIZE_128;
+#ifdef HW_KEY_GENERATION
+		/* This doesn't buy us much, that's why it is commented out. */
+		cw->cw_key_generation = PADLOCK_KEY_GENERATION_HW;
+#endif
+		break;
+	case 192:
+		cw->cw_round_count = PADLOCK_ROUND_COUNT_AES192;
+		cw->cw_key_size = PADLOCK_KEY_SIZE_192;
+		break;
+	case 256:
+		cw->cw_round_count = PADLOCK_ROUND_COUNT_AES256;
+		cw->cw_key_size = PADLOCK_KEY_SIZE_256;
+		break;
+	}
+	if (encini->cri_key != NULL) {
+		padlock_cipher_key_setup(ses, encini->cri_key,
+		    encini->cri_klen);
+	}
+
+	arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0);
+	return (0);
+}
+
+int
+padlock_cipher_process(struct padlock_session *ses, struct cryptodesc *enccrd,
+    struct cryptop *crp)
+{
+	union padlock_cw *cw;
+	u_char *buf, *abuf;
+	uint32_t *key;
+
+	buf = malloc(enccrd->crd_len + 16, M_PADLOCK, M_NOWAIT);
+	if (buf == NULL)
+		return (ENOMEM);
+	/* Buffer has to be 16 bytes aligned. */
+	abuf = PADLOCK_ALIGN(buf);
+
+	if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
+		padlock_cipher_key_setup(ses, enccrd->crd_key,
+		    enccrd->crd_klen);
+	}
+
+	cw = &ses->ses_cw;
+	cw->cw_filler0 = 0;
+	cw->cw_filler1 = 0;
+	cw->cw_filler2 = 0;
+	cw->cw_filler3 = 0;
+	if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
+		cw->cw_direction = PADLOCK_DIRECTION_ENCRYPT;
+		key = ses->ses_ekey;
+		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
+			bcopy(enccrd->crd_iv, ses->ses_iv, 16);
+
+		if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
+			crypto_copyback(crp->crp_flags, crp->crp_buf,
+			    enccrd->crd_inject, AES_BLOCK_LEN, ses->ses_iv);
+		}
+	} else {
+		cw->cw_direction = PADLOCK_DIRECTION_DECRYPT;
+		key = ses->ses_dkey;
+		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
+			bcopy(enccrd->crd_iv, ses->ses_iv, AES_BLOCK_LEN);
+		else {
+			crypto_copydata(crp->crp_flags, crp->crp_buf,
+			    enccrd->crd_inject, AES_BLOCK_LEN, ses->ses_iv);
+		}
+	}
+
+	crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
+	    enccrd->crd_len, abuf);
+
+	padlock_cbc(abuf, abuf, enccrd->crd_len / 16, key, cw, ses->ses_iv);
+
+	crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
+	    enccrd->crd_len, abuf);
+
+	/* copy out last block for use as next session IV */
+	if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
+		crypto_copydata(crp->crp_flags, crp->crp_buf,
+		    enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN,
+		    AES_BLOCK_LEN, ses->ses_iv);
+	}
+
+	if (buf != NULL) {
+		bzero(buf, enccrd->crd_len + 16);
+		free(buf, M_PADLOCK);
+	}
+	return (0);
+}
diff --git a/sys/crypto/via/padlock_hash.c b/sys/crypto/via/padlock_hash.c
new file mode 100644
index 0000000..8f666c2
--- /dev/null
+++ b/sys/crypto/via/padlock_hash.c
@@ -0,0 +1,389 @@
+/*-
+ * Copyright (c) 2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/libkern.h>
+#include <sys/endian.h>
+#if defined(__i386__) && !defined(PC98)
+#include <machine/cpufunc.h>
+#include <machine/cputypes.h>
+#include <machine/md_var.h>
+#include <machine/specialreg.h>
+#endif
+
+#include <opencrypto/cryptodev.h>
+#include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */
+#include <opencrypto/xform.h>
+
+#include <crypto/via/padlock.h>
+
+/*
+ * Implementation notes.
+ *
+ * Some VIA CPUs provides SHA1 and SHA256 acceleration.
+ * We implement all HMAC algorithms provided by crypto(9) framework, but we do
+ * the crypto work in software unless this is HMAC/SHA1 or HMAC/SHA256 and
+ * our CPU can accelerate it.
+ *
+ * Additional CPU instructions, which preform SHA1 and SHA256 are one-shot
+ * functions - we have only one chance to give the data, CPU itself will add
+ * the padding and calculate hash automatically.
+ * This means, it is not possible to implement common init(), update(), final()
+ * methods.
+ * The way I've choosen is to keep adding data to the buffer on update()
+ * (reallocating the buffer if necessary) and call XSHA{1,256} instruction on
+ * final().
+ */
+
+struct padlock_sha_ctx {
+	uint8_t	*psc_buf;
+	int	 psc_offset;
+	int	 psc_size;
+};
+CTASSERT(sizeof(struct padlock_sha_ctx) <= sizeof(union authctx));
+
+static void	padlock_sha_init(struct padlock_sha_ctx *ctx);
+static int	padlock_sha_update(struct padlock_sha_ctx *ctx, uint8_t *buf,
+		    uint16_t bufsize);
+static void	padlock_sha1_final(uint8_t *hash, struct padlock_sha_ctx *ctx);
+static void	padlock_sha256_final(uint8_t *hash, struct padlock_sha_ctx *ctx);
+
+static struct auth_hash padlock_hmac_sha1 = {
+	CRYPTO_SHA1_HMAC, "HMAC-SHA1",
+	20, SHA1_HASH_LEN, SHA1_HMAC_BLOCK_LEN, sizeof(struct padlock_sha_ctx),
+        (void (*)(void *))padlock_sha_init,
+	(int (*)(void *, uint8_t *, uint16_t))padlock_sha_update,
+	(void (*)(uint8_t *, void *))padlock_sha1_final
+};
+
+static struct auth_hash padlock_hmac_sha256 = {
+	CRYPTO_SHA2_256_HMAC, "HMAC-SHA2-256",
+	32, SHA2_256_HASH_LEN, SHA2_256_HMAC_BLOCK_LEN, sizeof(struct padlock_sha_ctx),
+        (void (*)(void *))padlock_sha_init,
+	(int (*)(void *, uint8_t *, uint16_t))padlock_sha_update,
+	(void (*)(uint8_t *, void *))padlock_sha256_final
+};
+
+MALLOC_DECLARE(M_PADLOCK);
+
+static __inline void
+padlock_output_block(uint32_t *src, uint32_t *dst, size_t count)
+{
+
+	while (count-- > 0)
+		*dst++ = bswap32(*src++);
+}
+
+static void
+padlock_do_sha1(const u_char *in, u_char *out, int count)
+{
+	u_char buf[128+16];	/* PadLock needs at least 128 bytes buffer. */
+	u_char *result = PADLOCK_ALIGN(buf);
+
+	((uint32_t *)result)[0] = 0x67452301;
+	((uint32_t *)result)[1] = 0xEFCDAB89;
+	((uint32_t *)result)[2] = 0x98BADCFE;
+	((uint32_t *)result)[3] = 0x10325476;
+	((uint32_t *)result)[4] = 0xC3D2E1F0;
+
+#ifdef __GNUCLIKE_ASM
+	__asm __volatile(
+		".byte  0xf3, 0x0f, 0xa6, 0xc8" /* rep xsha1 */
+			: "+S"(in), "+D"(result)
+			: "c"(count), "a"(0)
+		);
+#endif
+
+	padlock_output_block((uint32_t *)result, (uint32_t *)out,
+	    SHA1_HASH_LEN / sizeof(uint32_t));
+}
+
+static void
+padlock_do_sha256(const char *in, char *out, int count)
+{
+	char buf[128+16];	/* PadLock needs at least 128 bytes buffer. */
+	char *result = PADLOCK_ALIGN(buf);
+
+	((uint32_t *)result)[0] = 0x6A09E667;
+	((uint32_t *)result)[1] = 0xBB67AE85;
+	((uint32_t *)result)[2] = 0x3C6EF372;
+	((uint32_t *)result)[3] = 0xA54FF53A;
+	((uint32_t *)result)[4] = 0x510E527F;
+	((uint32_t *)result)[5] = 0x9B05688C;
+	((uint32_t *)result)[6] = 0x1F83D9AB;
+	((uint32_t *)result)[7] = 0x5BE0CD19;
+
+#ifdef __GNUCLIKE_ASM
+	__asm __volatile(
+		".byte  0xf3, 0x0f, 0xa6, 0xd0" /* rep xsha256 */
+			: "+S"(in), "+D"(result)
+			: "c"(count), "a"(0)
+		);
+#endif
+
+	padlock_output_block((uint32_t *)result, (uint32_t *)out,
+	    SHA2_256_HASH_LEN / sizeof(uint32_t));
+}
+
+static void
+padlock_sha_init(struct padlock_sha_ctx *ctx)
+{
+
+	ctx->psc_buf = NULL;
+	ctx->psc_offset = 0;
+	ctx->psc_size = 0;
+}
+
+static int
+padlock_sha_update(struct padlock_sha_ctx *ctx, uint8_t *buf, uint16_t bufsize)
+{
+
+	if (ctx->psc_size - ctx->psc_offset < bufsize) {
+		ctx->psc_size = MAX(ctx->psc_size * 2, ctx->psc_size + bufsize);
+		ctx->psc_buf = realloc(ctx->psc_buf, ctx->psc_size, M_PADLOCK,
+		    M_WAITOK);
+	}
+	bcopy(buf, ctx->psc_buf + ctx->psc_offset, bufsize);
+	ctx->psc_offset += bufsize;
+	return (0);
+}
+
+static void
+padlock_sha_free(struct padlock_sha_ctx *ctx)
+{
+
+	if (ctx->psc_buf != NULL) {
+		//bzero(ctx->psc_buf, ctx->psc_size);
+		free(ctx->psc_buf, M_PADLOCK);
+		ctx->psc_buf = NULL;
+		ctx->psc_offset = 0;
+		ctx->psc_size = 0;
+	}
+}
+
+static void
+padlock_sha1_final(uint8_t *hash, struct padlock_sha_ctx *ctx)
+{
+
+	padlock_do_sha1(ctx->psc_buf, hash, ctx->psc_offset);
+	padlock_sha_free(ctx);
+}
+
+static void
+padlock_sha256_final(uint8_t *hash, struct padlock_sha_ctx *ctx)
+{
+
+	padlock_do_sha256(ctx->psc_buf, hash, ctx->psc_offset);
+	padlock_sha_free(ctx);
+}
+
+static void
+padlock_copy_ctx(struct auth_hash *axf, void *sctx, void *dctx)
+{
+
+	if ((via_feature_xcrypt & VIA_HAS_SHA) != 0 &&
+	    (axf->type == CRYPTO_SHA1_HMAC ||
+	     axf->type == CRYPTO_SHA2_256_HMAC)) {
+		struct padlock_sha_ctx *spctx = sctx, *dpctx = dctx;
+
+		dpctx->psc_offset = spctx->psc_offset;
+		dpctx->psc_size = spctx->psc_size;
+		dpctx->psc_buf = malloc(dpctx->psc_size, M_PADLOCK, M_WAITOK);
+		bcopy(spctx->psc_buf, dpctx->psc_buf, dpctx->psc_size);
+	} else {
+		bcopy(sctx, dctx, axf->ctxsize);
+	}
+}
+
+static void
+padlock_free_ctx(struct auth_hash *axf, void *ctx)
+{
+
+	if ((via_feature_xcrypt & VIA_HAS_SHA) != 0 &&
+	    (axf->type == CRYPTO_SHA1_HMAC ||
+	     axf->type == CRYPTO_SHA2_256_HMAC)) {
+		padlock_sha_free(ctx);
+	}
+}
+
+static void
+padlock_hash_key_setup(struct padlock_session *ses, caddr_t key, int klen)
+{
+	struct auth_hash *axf;
+	int i;
+
+	klen /= 8;
+	axf = ses->ses_axf;
+
+	/*
+	 * Try to free contexts before using them, because
+	 * padlock_hash_key_setup() can be called twice - once from
+	 * padlock_newsession() and again from padlock_process().
+	 */
+	padlock_free_ctx(axf, ses->ses_ictx);
+	padlock_free_ctx(axf, ses->ses_octx);
+
+	for (i = 0; i < klen; i++)
+		key[i] ^= HMAC_IPAD_VAL;
+
+	axf->Init(ses->ses_ictx);
+	axf->Update(ses->ses_ictx, key, klen);
+	axf->Update(ses->ses_ictx, hmac_ipad_buffer, axf->blocksize - klen);
+
+	for (i = 0; i < klen; i++)
+		key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
+
+	axf->Init(ses->ses_octx); 
+	axf->Update(ses->ses_octx, key, klen);
+	axf->Update(ses->ses_octx, hmac_opad_buffer, axf->blocksize - klen);
+
+	for (i = 0; i < klen; i++)
+		key[i] ^= HMAC_OPAD_VAL;
+}
+
+/*
+ * Compute keyed-hash authenticator.
+ */
+static int
+padlock_authcompute(struct padlock_session *ses, struct cryptodesc *crd,
+    caddr_t buf, int flags)
+{
+	u_char hash[HASH_MAX_LEN];
+	struct auth_hash *axf;
+	union authctx ctx;
+	int error;
+
+	axf = ses->ses_axf;
+
+	padlock_copy_ctx(axf, ses->ses_ictx, &ctx);
+	error = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
+	    (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
+	if (error != 0) {
+		padlock_free_ctx(axf, &ctx);
+		return (error);
+	}
+	axf->Final(hash, &ctx);
+
+	padlock_copy_ctx(axf, ses->ses_octx, &ctx);
+	axf->Update(&ctx, hash, axf->hashsize);
+	axf->Final(hash, &ctx);
+
+	/* Inject the authentication data */
+	crypto_copyback(flags, buf, crd->crd_inject,
+	    ses->ses_mlen == 0 ? axf->hashsize : ses->ses_mlen, hash);
+	return (0);
+}
+
+int
+padlock_hash_setup(struct padlock_session *ses, struct cryptoini *macini)
+{
+
+	ses->ses_mlen = macini->cri_mlen;
+
+	/* Find software structure which describes HMAC algorithm. */
+	switch (macini->cri_alg) {
+	case CRYPTO_NULL_HMAC:
+		ses->ses_axf = &auth_hash_null;
+		break;
+	case CRYPTO_MD5_HMAC:
+		ses->ses_axf = &auth_hash_hmac_md5;
+		break;
+	case CRYPTO_SHA1_HMAC:
+		if ((via_feature_xcrypt & VIA_HAS_SHA) != 0)
+			ses->ses_axf = &padlock_hmac_sha1;
+		else
+			ses->ses_axf = &auth_hash_hmac_sha1;
+		break;
+	case CRYPTO_RIPEMD160_HMAC:
+		ses->ses_axf = &auth_hash_hmac_ripemd_160;
+		break;
+	case CRYPTO_SHA2_256_HMAC:
+		if ((via_feature_xcrypt & VIA_HAS_SHA) != 0)
+			ses->ses_axf = &padlock_hmac_sha256;
+		else
+			ses->ses_axf = &auth_hash_hmac_sha2_256;
+		break;
+	case CRYPTO_SHA2_384_HMAC:
+		ses->ses_axf = &auth_hash_hmac_sha2_384;
+		break;
+	case CRYPTO_SHA2_512_HMAC:
+		ses->ses_axf = &auth_hash_hmac_sha2_512;
+		break;
+	}
+
+	/* Allocate memory for HMAC inner and outer contexts. */
+	ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_PADLOCK,
+	    M_ZERO | M_NOWAIT);
+	ses->ses_octx = malloc(ses->ses_axf->ctxsize, M_PADLOCK,
+	    M_ZERO | M_NOWAIT);
+	if (ses->ses_ictx == NULL || ses->ses_octx == NULL)
+		return (ENOMEM);
+
+	/* Setup key if given. */
+	if (macini->cri_key != NULL) {
+		padlock_hash_key_setup(ses, macini->cri_key,
+		    macini->cri_klen);
+	}
+	return (0);
+}
+
+int
+padlock_hash_process(struct padlock_session *ses, struct cryptodesc *maccrd,
+    struct cryptop *crp)
+{
+	int error;
+
+	if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0)
+		padlock_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen);
+
+	error = padlock_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags);
+	return (error);
+}
+
+void
+padlock_hash_free(struct padlock_session *ses)
+{
+
+	if (ses->ses_ictx != NULL) {
+		padlock_free_ctx(ses->ses_axf, ses->ses_ictx);
+		bzero(ses->ses_ictx, ses->ses_axf->ctxsize);
+		free(ses->ses_ictx, M_PADLOCK);
+		ses->ses_ictx = NULL;
+	}
+	if (ses->ses_octx != NULL) {
+		padlock_free_ctx(ses->ses_axf, ses->ses_octx);
+		bzero(ses->ses_octx, ses->ses_axf->ctxsize);
+		free(ses->ses_octx, M_PADLOCK);
+		ses->ses_octx = NULL;
+	}
+}