RFC4868 section 2.3 requires that the output be half... This fixes

problems that was introduced in r285336...  I have verified that
HMAC-SHA2-256 both ah only and w/ AES-CBC interoperate w/ a NetBSD
6.1.5 vm...

Reviewed by:	gnn

3 files changed, 38 insertions, 26 deletions

diff --git a/sys/netipsec/xform.h b/sys/netipsec/xform.h
index 132717f..fee457b 100644
--- a/sys/netipsec/xform.h
+++ b/sys/netipsec/xform.h
@@ -105,6 +105,7 @@ struct xformsw {
 #ifdef _KERNEL
 extern void xform_register(struct xformsw*);
 extern int xform_init(struct secasvar *sav, int xftype);
+extern int xform_ah_authsize(struct auth_hash *esph);
 
 struct cryptoini;
 
diff --git a/sys/netipsec/xform_ah.c b/sys/netipsec/xform_ah.c
index 0710578..ae0feb9 100644
--- a/sys/netipsec/xform_ah.c
+++ b/sys/netipsec/xform_ah.c
@@ -85,8 +85,8 @@
  * Return authenticator size in bytes, based on a field in the
  * algorithm descriptor.
  */
-#define	AUTHSIZE(sav)	\
-	((sav->flags & SADB_X_EXT_OLD) ? 16 : (sav)->tdb_authalgxform->hashsize)
+#define	AUTHSIZE(sav)	((sav->flags & SADB_X_EXT_OLD) ? 16 :	\
+			 xform_ah_authsize((sav)->tdb_authalgxform))
 
 VNET_DEFINE(int, ah_enable) = 1;	/* control flow of packets with AH */
 VNET_DEFINE(int, ah_cleartos) = 1;	/* clear ip_tos when doing AH calc */
@@ -112,6 +112,35 @@ static unsigned char ipseczeroes[256];	/* larger than an ip6 extension hdr */
 static int ah_input_cb(struct cryptop*);
 static int ah_output_cb(struct cryptop*);
 
+int
+xform_ah_authsize(struct auth_hash *esph)
+{
+	int alen;
+
+	if (esph == NULL)
+		return 0;
+
+	switch (esph->type) {
+	case CRYPTO_SHA2_256_HMAC:
+	case CRYPTO_SHA2_384_HMAC:
+	case CRYPTO_SHA2_512_HMAC:
+		alen = esph->hashsize / 2;	/* RFC4868 2.3 */
+		break;
+
+	case CRYPTO_AES_128_NIST_GMAC:
+	case CRYPTO_AES_192_NIST_GMAC:
+	case CRYPTO_AES_256_NIST_GMAC:
+		alen = esph->hashsize;
+		break;
+
+	default:
+		alen = AH_HMAC_HASHLEN;
+		break;
+	}
+
+	return alen;
+}
+
 /*
  * NB: this is public for use by the PF_KEY support.
  */
diff --git a/sys/netipsec/xform_esp.c b/sys/netipsec/xform_esp.c
index e0dc2b2..dbb1a27 100644
--- a/sys/netipsec/xform_esp.c
+++ b/sys/netipsec/xform_esp.c
@@ -320,7 +320,6 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform"));
 
-	alen = 0;
 	/* Valid IP Packet length ? */
 	if ( (skip&3) || (m->m_pkthdr.len&3) ){
 		DPRINTF(("%s: misaligned packet, skip %u pkt len %u",
@@ -335,13 +334,13 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	esph = sav->tdb_authalgxform;
 	espx = sav->tdb_encalgxform;
 
-	/* Determine the ESP header length */
+	/* Determine the ESP header and auth length */
 	if (sav->flags & SADB_X_EXT_OLD)
 		hlen = sizeof (struct esp) + sav->ivlen;
 	else
 		hlen = sizeof (struct newesp) + sav->ivlen;
-	/* Authenticator hash size */
-	alen = esph ? esph->hashsize : 0;
+
+	alen = xform_ah_authsize(esph);
 
 	/*
 	 * Verify payload length is multiple of encryption algorithm
@@ -530,7 +529,7 @@ esp_input_cb(struct cryptop *crp)
 
 	/* If authentication was performed, check now. */
 	if (esph != NULL) {
-		alen = esph->hashsize;
+		alen = xform_ah_authsize(esph);
 		AHSTAT_INC(ahs_hist[sav->alg_auth]);
 		/* Copy the authenticator from the packet */
 		m_copydata(m, m->m_pkthdr.len - alen, alen, aalg);
@@ -700,10 +699,7 @@ esp_output(struct mbuf *m, struct ipsecrequest *isr, struct mbuf **mp,
 	/* XXX clamp padding length a la KAME??? */
 	padding = ((blks - ((rlen + 2) % blks)) % blks) + 2;
 
-	if (esph)
-		alen = esph->hashsize;
-	else
-		alen = 0;
+	alen = xform_ah_authsize(esph);
 
 	ESPSTAT_INC(esps_output);
 
@@ -983,21 +979,7 @@ esp_output_cb(struct cryptop *crp)
 		if (esph !=  NULL) {
 			int alen;
 
-			switch (esph->type) {
-			case CRYPTO_SHA2_256_HMAC:
-			case CRYPTO_SHA2_384_HMAC:
-			case CRYPTO_SHA2_512_HMAC:
-				alen = esph->hashsize/2;
-				break;
-			case CRYPTO_AES_128_NIST_GMAC:
-			case CRYPTO_AES_192_NIST_GMAC:
-			case CRYPTO_AES_256_NIST_GMAC:
-				alen = esph->hashsize;
-				break;
-			default:
-				alen = AH_HMAC_HASHLEN;
-				break;
-			}
+			alen = xform_ah_authsize(esph);
 			m_copyback(m, m->m_pkthdr.len - alen,
 			    alen, ipseczeroes);
 		}