replace the kernel's version w/ cperciva's implementation... In all

my tests, it is faster ~20%, even on an old IXP425 533MHz it is ~45%
faster...  This is partly due to loop unrolling, so the code size does
significantly increase...  I do plan on committing a version that
rolls up the loops again for smaller code size for embedded systems
where size is more important than absolute performance (it'll save ~6k
code)...

The kernel implementation is now shared w/ userland's libcrypt and
libmd...

We drop support for sha256 from sha2.c, so now sha2.c only contains
sha384 and sha512...

Reviewed by:	secteam@

4 files changed, 15 insertions, 404 deletions

diff --git a/sys/crypto/sha2/sha2.c b/sys/crypto/sha2/sha2.c
index 66ec6e9..5bb52f3 100644
--- a/sys/crypto/sha2/sha2.c
+++ b/sys/crypto/sha2/sha2.c
@@ -121,20 +121,10 @@ __FBSDID("$FreeBSD$");
  * Thank you, Jun-ichiro itojun Hagino, for suggesting using u_intXX_t
  * types and pointing out recent ANSI C support for uintXX_t in inttypes.h.
  */
-#if 0 /*def SHA2_USE_INTTYPES_H*/
-
 typedef uint8_t  sha2_byte;	/* Exactly 1 byte */
 typedef uint32_t sha2_word32;	/* Exactly 4 bytes */
 typedef uint64_t sha2_word64;	/* Exactly 8 bytes */
 
-#else /* SHA2_USE_INTTYPES_H */
-
-typedef u_int8_t  sha2_byte;	/* Exactly 1 byte */
-typedef u_int32_t sha2_word32;	/* Exactly 4 bytes */
-typedef u_int64_t sha2_word64;	/* Exactly 8 bytes */
-
-#endif /* SHA2_USE_INTTYPES_H */
-
 
 /*** SHA-256/384/512 Various Length Definitions ***********************/
 /* NOTE: Most of these are in sha2.h */
@@ -183,8 +173,6 @@ typedef u_int64_t sha2_word64;	/* Exactly 8 bytes */
  */
 /* Shift-right (used in SHA-256, SHA-384, and SHA-512): */
 #define R(b,x) 		((x) >> (b))
-/* 32-bit Rotate-right (used in SHA-256): */
-#define S32(b,x)	(((x) >> (b)) | ((x) << (32 - (b))))
 /* 64-bit Rotate-right (used in SHA-384 and SHA-512): */
 #define S64(b,x)	(((x) >> (b)) | ((x) << (64 - (b))))
 
@@ -192,12 +180,6 @@ typedef u_int64_t sha2_word64;	/* Exactly 8 bytes */
 #define Ch(x,y,z)	(((x) & (y)) ^ ((~(x)) & (z)))
 #define Maj(x,y,z)	(((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
 
-/* Four of six logical functions used in SHA-256: */
-#define Sigma0_256(x)	(S32(2,  (x)) ^ S32(13, (x)) ^ S32(22, (x)))
-#define Sigma1_256(x)	(S32(6,  (x)) ^ S32(11, (x)) ^ S32(25, (x)))
-#define sigma0_256(x)	(S32(7,  (x)) ^ S32(18, (x)) ^ R(3 ,   (x)))
-#define sigma1_256(x)	(S32(17, (x)) ^ S32(19, (x)) ^ R(10,   (x)))
-
 /* Four of six logical functions used in SHA-384 and SHA-512: */
 #define Sigma0_512(x)	(S64(28, (x)) ^ S64(34, (x)) ^ S64(39, (x)))
 #define Sigma1_512(x)	(S64(14, (x)) ^ S64(18, (x)) ^ S64(41, (x)))
@@ -210,43 +192,10 @@ typedef u_int64_t sha2_word64;	/* Exactly 8 bytes */
  * only.
  */
 static void SHA512_Last(SHA512_CTX*);
-static void SHA256_Transform(SHA256_CTX*, const sha2_word32*);
 static void SHA512_Transform(SHA512_CTX*, const sha2_word64*);
 
 
 /*** SHA-XYZ INITIAL HASH VALUES AND CONSTANTS ************************/
-/* Hash constant words K for SHA-256: */
-static const sha2_word32 K256[64] = {
-	0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL,
-	0x3956c25bUL, 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL,
-	0xd807aa98UL, 0x12835b01UL, 0x243185beUL, 0x550c7dc3UL,
-	0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, 0xc19bf174UL,
-	0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL,
-	0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL,
-	0x983e5152UL, 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL,
-	0xc6e00bf3UL, 0xd5a79147UL, 0x06ca6351UL, 0x14292967UL,
-	0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, 0x53380d13UL,
-	0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL,
-	0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL,
-	0xd192e819UL, 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL,
-	0x19a4c116UL, 0x1e376c08UL, 0x2748774cUL, 0x34b0bcb5UL,
-	0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, 0x682e6ff3UL,
-	0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL,
-	0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL
-};
-
-/* Initial hash value H for SHA-256: */
-static const sha2_word32 sha256_initial_hash_value[8] = {
-	0x6a09e667UL,
-	0xbb67ae85UL,
-	0x3c6ef372UL,
-	0xa54ff53aUL,
-	0x510e527fUL,
-	0x9b05688cUL,
-	0x1f83d9abUL,
-	0x5be0cd19UL
-};
-
 /* Hash constant words K for SHA-384 and SHA-512: */
 static const sha2_word64 K512[80] = {
 	0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL,
@@ -323,301 +272,6 @@ static const char *sha2_hex_digits = "0123456789abcdef";
 
 
 /*** SHA-256: *********************************************************/
-void SHA256_Init(SHA256_CTX* context) {
-	if (context == (SHA256_CTX*)0) {
-		return;
-	}
-	bcopy(sha256_initial_hash_value, context->state, SHA256_DIGEST_LENGTH);
-	bzero(context->buffer, SHA256_BLOCK_LENGTH);
-	context->bitcount = 0;
-}
-
-#ifdef SHA2_UNROLL_TRANSFORM
-
-/* Unrolled SHA-256 round macros: */
-
-#if BYTE_ORDER == LITTLE_ENDIAN
-
-#define ROUND256_0_TO_15(a,b,c,d,e,f,g,h)	\
-	REVERSE32(*data++, W256[j]); \
-	T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + \
-             K256[j] + W256[j]; \
-	(d) += T1; \
-	(h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \
-	j++
-
-
-#else /* BYTE_ORDER == LITTLE_ENDIAN */
-
-#define ROUND256_0_TO_15(a,b,c,d,e,f,g,h)	\
-	T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + \
-	     K256[j] + (W256[j] = *data++); \
-	(d) += T1; \
-	(h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \
-	j++
-
-#endif /* BYTE_ORDER == LITTLE_ENDIAN */
-
-#define ROUND256(a,b,c,d,e,f,g,h)	\
-	s0 = W256[(j+1)&0x0f]; \
-	s0 = sigma0_256(s0); \
-	s1 = W256[(j+14)&0x0f]; \
-	s1 = sigma1_256(s1); \
-	T1 = (h) + Sigma1_256(e) + Ch((e), (f), (g)) + K256[j] + \
-	     (W256[j&0x0f] += s1 + W256[(j+9)&0x0f] + s0); \
-	(d) += T1; \
-	(h) = T1 + Sigma0_256(a) + Maj((a), (b), (c)); \
-	j++
-
-static void SHA256_Transform(SHA256_CTX* context, const sha2_word32* data) {
-	sha2_word32	a, b, c, d, e, f, g, h, s0, s1;
-	sha2_word32	T1, *W256;
-	int		j;
-
-	W256 = (sha2_word32*)context->buffer;
-
-	/* Initialize registers with the prev. intermediate value */
-	a = context->state[0];
-	b = context->state[1];
-	c = context->state[2];
-	d = context->state[3];
-	e = context->state[4];
-	f = context->state[5];
-	g = context->state[6];
-	h = context->state[7];
-
-	j = 0;
-	do {
-		/* Rounds 0 to 15 (unrolled): */
-		ROUND256_0_TO_15(a,b,c,d,e,f,g,h);
-		ROUND256_0_TO_15(h,a,b,c,d,e,f,g);
-		ROUND256_0_TO_15(g,h,a,b,c,d,e,f);
-		ROUND256_0_TO_15(f,g,h,a,b,c,d,e);
-		ROUND256_0_TO_15(e,f,g,h,a,b,c,d);
-		ROUND256_0_TO_15(d,e,f,g,h,a,b,c);
-		ROUND256_0_TO_15(c,d,e,f,g,h,a,b);
-		ROUND256_0_TO_15(b,c,d,e,f,g,h,a);
-	} while (j < 16);
-
-	/* Now for the remaining rounds to 64: */
-	do {
-		ROUND256(a,b,c,d,e,f,g,h);
-		ROUND256(h,a,b,c,d,e,f,g);
-		ROUND256(g,h,a,b,c,d,e,f);
-		ROUND256(f,g,h,a,b,c,d,e);
-		ROUND256(e,f,g,h,a,b,c,d);
-		ROUND256(d,e,f,g,h,a,b,c);
-		ROUND256(c,d,e,f,g,h,a,b);
-		ROUND256(b,c,d,e,f,g,h,a);
-	} while (j < 64);
-
-	/* Compute the current intermediate hash value */
-	context->state[0] += a;
-	context->state[1] += b;
-	context->state[2] += c;
-	context->state[3] += d;
-	context->state[4] += e;
-	context->state[5] += f;
-	context->state[6] += g;
-	context->state[7] += h;
-
-	/* Clean up */
-	a = b = c = d = e = f = g = h = T1 = 0;
-}
-
-#else /* SHA2_UNROLL_TRANSFORM */
-
-static void SHA256_Transform(SHA256_CTX* context, const sha2_word32* data) {
-	sha2_word32	a, b, c, d, e, f, g, h, s0, s1;
-	sha2_word32	T1, T2, *W256;
-	int		j;
-
-	W256 = (sha2_word32*)context->buffer;
-
-	/* Initialize registers with the prev. intermediate value */
-	a = context->state[0];
-	b = context->state[1];
-	c = context->state[2];
-	d = context->state[3];
-	e = context->state[4];
-	f = context->state[5];
-	g = context->state[6];
-	h = context->state[7];
-
-	j = 0;
-	do {
-#if BYTE_ORDER == LITTLE_ENDIAN
-		/* Copy data while converting to host byte order */
-		REVERSE32(*data++,W256[j]);
-		/* Apply the SHA-256 compression function to update a..h */
-		T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + W256[j];
-#else /* BYTE_ORDER == LITTLE_ENDIAN */
-		/* Apply the SHA-256 compression function to update a..h with copy */
-		T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + (W256[j] = *data++);
-#endif /* BYTE_ORDER == LITTLE_ENDIAN */
-		T2 = Sigma0_256(a) + Maj(a, b, c);
-		h = g;
-		g = f;
-		f = e;
-		e = d + T1;
-		d = c;
-		c = b;
-		b = a;
-		a = T1 + T2;
-
-		j++;
-	} while (j < 16);
-
-	do {
-		/* Part of the message block expansion: */
-		s0 = W256[(j+1)&0x0f];
-		s0 = sigma0_256(s0);
-		s1 = W256[(j+14)&0x0f];	
-		s1 = sigma1_256(s1);
-
-		/* Apply the SHA-256 compression function to update a..h */
-		T1 = h + Sigma1_256(e) + Ch(e, f, g) + K256[j] + 
-		     (W256[j&0x0f] += s1 + W256[(j+9)&0x0f] + s0);
-		T2 = Sigma0_256(a) + Maj(a, b, c);
-		h = g;
-		g = f;
-		f = e;
-		e = d + T1;
-		d = c;
-		c = b;
-		b = a;
-		a = T1 + T2;
-
-		j++;
-	} while (j < 64);
-
-	/* Compute the current intermediate hash value */
-	context->state[0] += a;
-	context->state[1] += b;
-	context->state[2] += c;
-	context->state[3] += d;
-	context->state[4] += e;
-	context->state[5] += f;
-	context->state[6] += g;
-	context->state[7] += h;
-
-	/* Clean up */
-	a = b = c = d = e = f = g = h = T1 = T2 = 0;
-}
-
-#endif /* SHA2_UNROLL_TRANSFORM */
-
-void SHA256_Update(SHA256_CTX* context, const sha2_byte *data, size_t len) {
-	unsigned int	freespace, usedspace;
-
-	if (len == 0) {
-		/* Calling with no data is valid - we do nothing */
-		return;
-	}
-
-	/* Sanity check: */
-	assert(context != (SHA256_CTX*)0 && data != (sha2_byte*)0);
-
-	usedspace = (context->bitcount >> 3) % SHA256_BLOCK_LENGTH;
-	if (usedspace > 0) {
-		/* Calculate how much free space is available in the buffer */
-		freespace = SHA256_BLOCK_LENGTH - usedspace;
-
-		if (len >= freespace) {
-			/* Fill the buffer completely and process it */
-			bcopy(data, &context->buffer[usedspace], freespace);
-			context->bitcount += freespace << 3;
-			len -= freespace;
-			data += freespace;
-			SHA256_Transform(context, (sha2_word32*)context->buffer);
-		} else {
-			/* The buffer is not yet full */
-			bcopy(data, &context->buffer[usedspace], len);
-			context->bitcount += len << 3;
-			/* Clean up: */
-			usedspace = freespace = 0;
-			return;
-		}
-	}
-	while (len >= SHA256_BLOCK_LENGTH) {
-		/* Process as many complete blocks as we can */
-		SHA256_Transform(context, (const sha2_word32*)data);
-		context->bitcount += SHA256_BLOCK_LENGTH << 3;
-		len -= SHA256_BLOCK_LENGTH;
-		data += SHA256_BLOCK_LENGTH;
-	}
-	if (len > 0) {
-		/* There's left-overs, so save 'em */
-		bcopy(data, context->buffer, len);
-		context->bitcount += len << 3;
-	}
-	/* Clean up: */
-	usedspace = freespace = 0;
-}
-
-void SHA256_Final(sha2_byte digest[], SHA256_CTX* context) {
-	sha2_word32	*d = (sha2_word32*)digest;
-	unsigned int	usedspace;
-
-	/* Sanity check: */
-	assert(context != (SHA256_CTX*)0);
-
-	/* If no digest buffer is passed, we don't bother doing this: */
-	if (digest != (sha2_byte*)0) {
-		usedspace = (context->bitcount >> 3) % SHA256_BLOCK_LENGTH;
-#if BYTE_ORDER == LITTLE_ENDIAN
-		/* Convert FROM host byte order */
-		REVERSE64(context->bitcount,context->bitcount);
-#endif
-		if (usedspace > 0) {
-			/* Begin padding with a 1 bit: */
-			context->buffer[usedspace++] = 0x80;
-
-			if (usedspace <= SHA256_SHORT_BLOCK_LENGTH) {
-				/* Set-up for the last transform: */
-				bzero(&context->buffer[usedspace], SHA256_SHORT_BLOCK_LENGTH - usedspace);
-			} else {
-				if (usedspace < SHA256_BLOCK_LENGTH) {
-					bzero(&context->buffer[usedspace], SHA256_BLOCK_LENGTH - usedspace);
-				}
-				/* Do second-to-last transform: */
-				SHA256_Transform(context, (sha2_word32*)context->buffer);
-
-				/* And set-up for the last transform: */
-				bzero(context->buffer, SHA256_SHORT_BLOCK_LENGTH);
-			}
-		} else {
-			/* Set-up for the last transform: */
-			bzero(context->buffer, SHA256_SHORT_BLOCK_LENGTH);
-
-			/* Begin padding with a 1 bit: */
-			*context->buffer = 0x80;
-		}
-		/* Set the bit count: */
-		*(sha2_word64*)&context->buffer[SHA256_SHORT_BLOCK_LENGTH] = context->bitcount;
-
-		/* Final transform: */
-		SHA256_Transform(context, (sha2_word32*)context->buffer);
-
-#if BYTE_ORDER == LITTLE_ENDIAN
-		{
-			/* Convert TO host byte order */
-			int	j;
-			for (j = 0; j < 8; j++) {
-				REVERSE32(context->state[j],context->state[j]);
-				*d++ = context->state[j];
-			}
-		}
-#else
-		bcopy(context->state, d, SHA256_DIGEST_LENGTH);
-#endif
-	}
-
-	/* Clean up state data: */
-	bzero(context, sizeof(*context));
-	usedspace = 0;
-}
-
 char *SHA256_End(SHA256_CTX* context, char buffer[]) {
 	sha2_byte	digest[SHA256_DIGEST_LENGTH], *d = digest;
 	int		i;
@@ -641,7 +295,7 @@ char *SHA256_End(SHA256_CTX* context, char buffer[]) {
 	return buffer;
 }
 
-char* SHA256_Data(const sha2_byte* data, size_t len, char digest[SHA256_DIGEST_STRING_LENGTH]) {
+char* SHA256_Data(const void *data, unsigned int len, char *digest) {
 	SHA256_CTX	context;
 
 	SHA256_Init(&context);
diff --git a/sys/crypto/sha2/sha2.h b/sys/crypto/sha2/sha2.h
index 639d58b..59e91f6 100644
--- a/sys/crypto/sha2/sha2.h
+++ b/sys/crypto/sha2/sha2.h
@@ -56,70 +56,17 @@ extern "C" {
 #define SHA512_DIGEST_STRING_LENGTH	(SHA512_DIGEST_LENGTH * 2 + 1)
 
 
-/*** SHA-256/384/512 Context Structures *******************************/
-/* NOTE: If your architecture does not define either u_intXX_t types or
- * uintXX_t (from inttypes.h), you may need to define things by hand
- * for your system:
- */
-#if 0
-typedef unsigned char u_int8_t;		/* 1-byte  (8-bits)  */
-typedef unsigned int u_int32_t;		/* 4-bytes (32-bits) */
-typedef unsigned long long u_int64_t;	/* 8-bytes (64-bits) */
-#endif
-/*
- * Most BSD systems already define u_intXX_t types, as does Linux.
- * Some systems, however, like Compaq's Tru64 Unix instead can use
- * uintXX_t types defined by very recent ANSI C standards and included
- * in the file:
- *
- *   #include <inttypes.h>
- *
- * If you choose to use <inttypes.h> then please define: 
- *
- *   #define SHA2_USE_INTTYPES_H
- *
- * Or on the command line during compile:
- *
- *   cc -DSHA2_USE_INTTYPES_H ...
- */
-#if 0 /*def SHA2_USE_INTTYPES_H*/
-
-typedef struct _SHA256_CTX {
-	uint32_t	state[8];
-	uint64_t	bitcount;
-	uint8_t	buffer[SHA256_BLOCK_LENGTH];
-} SHA256_CTX;
+/*** SHA-384/512 Context Structures *******************************/
 typedef struct _SHA512_CTX {
 	uint64_t	state[8];
 	uint64_t	bitcount[2];
 	uint8_t	buffer[SHA512_BLOCK_LENGTH];
 } SHA512_CTX;
 
-#else /* SHA2_USE_INTTYPES_H */
-
-typedef struct _SHA256_CTX {
-	u_int32_t	state[8];
-	u_int64_t	bitcount;
-	u_int8_t	buffer[SHA256_BLOCK_LENGTH];
-} SHA256_CTX;
-typedef struct _SHA512_CTX {
-	u_int64_t	state[8];
-	u_int64_t	bitcount[2];
-	u_int8_t	buffer[SHA512_BLOCK_LENGTH];
-} SHA512_CTX;
-
-#endif /* SHA2_USE_INTTYPES_H */
-
 typedef SHA512_CTX SHA384_CTX;
 
 
-/*** SHA-256/384/512 Function Prototypes ******************************/
-
-void SHA256_Init(SHA256_CTX *);
-void SHA256_Update(SHA256_CTX*, const u_int8_t*, size_t);
-void SHA256_Final(u_int8_t[SHA256_DIGEST_LENGTH], SHA256_CTX*);
-char* SHA256_End(SHA256_CTX*, char[SHA256_DIGEST_STRING_LENGTH]);
-char* SHA256_Data(const u_int8_t*, size_t, char[SHA256_DIGEST_STRING_LENGTH]);
+/*** SHA-384/512 Function Prototypes ******************************/
 
 void SHA384_Init(SHA384_CTX*);
 void SHA384_Update(SHA384_CTX*, const u_int8_t*, size_t);
@@ -137,4 +84,6 @@ char* SHA512_Data(const u_int8_t*, size_t, char[SHA512_DIGEST_STRING_LENGTH]);
 }
 #endif /* __cplusplus */
 
+#include "sha256.h"
+
 #endif /* __SHA2_H__ */
diff --git a/sys/crypto/sha2/sha256.h b/sys/crypto/sha2/sha256.h
index 1b6a4f4..80765ef 100644
--- a/sys/crypto/sha2/sha256.h
+++ b/sys/crypto/sha2/sha256.h
@@ -29,12 +29,14 @@
 #ifndef _SHA256_H_
 #define _SHA256_H_
 
+#ifndef _KERNEL
 #include <sys/types.h>
+#endif
 
 typedef struct SHA256Context {
 	uint32_t state[8];
 	uint64_t count;
-	unsigned char buf[64];
+	uint8_t buf[64];
 } SHA256_CTX;
 
 __BEGIN_DECLS
@@ -42,9 +44,11 @@ void	SHA256_Init(SHA256_CTX *);
 void	SHA256_Update(SHA256_CTX *, const void *, size_t);
 void	SHA256_Final(unsigned char [32], SHA256_CTX *);
 char   *SHA256_End(SHA256_CTX *, char *);
+char   *SHA256_Data(const void *, unsigned int, char *);
+#ifndef _KERNEL
 char   *SHA256_File(const char *, char *);
 char   *SHA256_FileChunk(const char *, char *, off_t, off_t);
-char   *SHA256_Data(const void *, unsigned int, char *);
+#endif
 __END_DECLS
 
 #endif /* !_SHA256_H_ */
diff --git a/sys/crypto/sha2/sha256c.c b/sys/crypto/sha2/sha256c.c
index 76cec8e..9c666df 100644
--- a/sys/crypto/sha2/sha256c.c
+++ b/sys/crypto/sha2/sha256c.c
@@ -30,7 +30,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/endian.h>
 #include <sys/types.h>
 
+#ifdef _KERNEL
+#include <sys/systm.h>
+#else
 #include <string.h>
+#endif
 
 #include "sha256.h"