RFC 2289 requires all hashes be stored in little endian format before

folding to 64 bits, while SHA1 code is big endian.  Therefore, a bswap32
is required before using the value.

Without this change, the implementation does not conform to test vector
found in RFC 2289.

PR:		bin/170519
Submitted by:	Arthur Mesh <arthurmesh gmail com> (with changes)
MFC after:	1 week

2 files changed, 18 insertions, 0 deletions

diff --git a/contrib/opie/libopie/hash.c b/contrib/opie/libopie/hash.c
index babcbfa..8a59f41 100644
--- a/contrib/opie/libopie/hash.c
+++ b/contrib/opie/libopie/hash.c
@@ -17,6 +17,8 @@ you didn't get a copy, you may request one from <license@inner.net>.
 $FreeBSD$
 */
 
+#include <sys/endian.h>
+
 #include "opie_cfg.h"
 #include "opie.h"
 
@@ -39,6 +41,13 @@ unsigned algorithm)
       SHA1_Final((unsigned char *)digest, &sha);
       results[0] = digest[0] ^ digest[2] ^ digest[4];
       results[1] = digest[1] ^ digest[3];
+
+      /*
+       * RFC2289 mandates that we convert SHA1 digest from big-endian to little
+       * see Appendix A.
+       */
+      results[0] = bswap32(results[0]);
+      results[1] = bswap32(results[1]);
       };
       break;
     case 4:
diff --git a/contrib/opie/libopie/hashlen.c b/contrib/opie/libopie/hashlen.c
index 29d855d..0d5808c 100644
--- a/contrib/opie/libopie/hashlen.c
+++ b/contrib/opie/libopie/hashlen.c
@@ -14,6 +14,8 @@ you didn't get a copy, you may request one from <license@inner.net>.
 $FreeBSD$
 */
 
+#include <sys/endian.h>
+
 #include "opie_cfg.h"
 #include "opie.h"
 
@@ -36,6 +38,13 @@ VOIDPTR in AND struct opie_otpkey *out AND int n)
       SHA1_Final((unsigned char *)digest, &sha);
       results[0] = digest[0] ^ digest[2] ^ digest[4];
       results[1] = digest[1] ^ digest[3];
+
+      /*
+       * RFC2289 mandates that we convert SHA1 digest from big-endian to little
+       * see Appendix A.
+       */
+      results[0] = bswap32(results[0]);
+      results[1] = bswap32(results[1]);
       break;
     }
     case 4: {