Add a new function stub to libcrypto() which resolves to a symbol in

the librsa* library and reports which version of the library (OpenSSL/RSAREF)
is being used.

This is then used in openssh to detect the failure case of RSAREF and a RSA key
>1024 bits, to print a more helpful error message than 'rsa_public_encrypt() fai
led.'

This is a 4.0-RELEASE candidate.

5 files changed, 71 insertions, 2 deletions

diff --git a/crypto/openssh/rsa.c b/crypto/openssh/rsa.c
index 5cab804..6f9c7cd 100644
--- a/crypto/openssh/rsa.c
+++ b/crypto/openssh/rsa.c
@@ -31,6 +31,8 @@
  *   below:
  * 
  *     [gone - had to be deleted - what a pity]
+ *
+ * $FreeBSD$
  * 
 */
 
@@ -125,7 +127,10 @@ rsa_public_encrypt(BIGNUM *out, BIGNUM *in, RSA *key)
 
 	if ((len = RSA_public_encrypt(ilen, inbuf, outbuf, key,
 	    RSA_PKCS1_PADDING)) <= 0)
-		fatal("rsa_public_encrypt() failed");
+	    if (BN_num_bits(key->n) > 1024 && RSA_libversion() == RSALIB_RSAREF)
+		fatal("rsa_private_encrypt() failed: RSAREF cannot handle keys larger than 1024 bits.");
+	    else
+		fatal("rsa_private_encrypt() failed.");
 
 	BN_bin2bn(outbuf, len, out);
 
@@ -150,7 +155,10 @@ rsa_private_decrypt(BIGNUM *out, BIGNUM *in, RSA *key)
 
 	if ((len = RSA_private_decrypt(ilen, inbuf, outbuf, key,
 	    RSA_PKCS1_PADDING)) <= 0)
-		fatal("rsa_private_decrypt() failed");
+	    if (BN_num_bits(key->n) > 1024 && RSA_libversion() == RSALIB_RSAREF)
+		fatal("rsa_private_decrypt() failed: RSAREF cannot handle keys larger than 1024 bits.");
+	    else
+		fatal("rsa_private_decrypt() failed.");
 
 	BN_bin2bn(outbuf, len, out);
 
diff --git a/crypto/openssl/crypto/rsa/rsa.h b/crypto/openssl/crypto/rsa/rsa.h
index 6c17ccc..3040d032 100644
--- a/crypto/openssl/crypto/rsa/rsa.h
+++ b/crypto/openssl/crypto/rsa/rsa.h
@@ -244,6 +244,8 @@ int RSA_get_ex_new_index(long argl, char *argp, int (*new_func)(),
 int RSA_set_ex_data(RSA *r,int idx,char *arg);
 char *RSA_get_ex_data(RSA *r, int idx);
 
+int RSA_libversion();
+
 /* BEGIN ERROR CODES */
 /* The following lines are auto generated by the script mkerr.pl. Any changes
  * made after this point may be overwritten when the script is next run.
@@ -308,6 +310,9 @@ char *RSA_get_ex_data(RSA *r, int idx);
 #define RSA_R_UNKNOWN_PADDING_TYPE			 118
 #define RSA_R_WRONG_SIGNATURE_LENGTH			 119
 
+#define RSALIB_OPENSSL	1
+#define RSALIB_RSAREF	2
+
 #ifdef  __cplusplus
 }
 #endif
diff --git a/crypto/openssl/crypto/rsa/rsa_intlstubs.c b/crypto/openssl/crypto/rsa/rsa_intlstubs.c
new file mode 100644
index 0000000..e9a0316
--- /dev/null
+++ b/crypto/openssl/crypto/rsa/rsa_intlstubs.c
@@ -0,0 +1,39 @@
+/*-
+ * Copyright (c) 2000 Kris Kennaway <kris@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE. SO THERE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef NO_RSA
+#ifdef PIC
+#include <openssl/rsa.h>
+
+int RSA_libversion()
+{
+	return RSALIB_OPENSSL;
+}
+
+#endif /* PIC */
+#endif /* NO_RSA */
diff --git a/crypto/openssl/crypto/rsa/rsa_stubs.c b/crypto/openssl/crypto/rsa/rsa_stubs.c
index 2da9e42..f164ce0 100644
--- a/crypto/openssl/crypto/rsa/rsa_stubs.c
+++ b/crypto/openssl/crypto/rsa/rsa_stubs.c
@@ -87,6 +87,16 @@ ERR_load_RSA_strings_stub(void)
 }
 __weak_reference(ERR_load_RSA_strings_stub, ERR_load_RSA_strings);
 
+int
+RSA_libversion_stub(void)
+{
+    static void (*sym)(void);
+
+    if (sym || (sym = getsym("RSA_libversion")))
+	sym();
+}
+__weak_reference(RSA_libversion_stub, RSA_libversion);
+
 #else	/* !PIC */
 
 /* Sigh, just get your own libs, ld(1) doesn't deal with weaks here */
diff --git a/crypto/openssl/rsaref/rsaref_stubs.c b/crypto/openssl/rsaref/rsaref_stubs.c
index 6296ebb..f1f7063 100644
--- a/crypto/openssl/rsaref/rsaref_stubs.c
+++ b/crypto/openssl/rsaref/rsaref_stubs.c
@@ -40,6 +40,7 @@
 #ifndef NO_RSA
 
 #include <stdio.h>
+#include <openssl/rsa.h>
 
 #define VERBOSE_STUBS	/* undef if you don't want missing rsaref reported */
 
@@ -165,6 +166,12 @@ R_RandomUpdate_stub(void *randomStruct,
 }
 __weak_reference(R_RandomUpdate_stub, R_RandomUpdate);
 
+int
+RSA_libversion()
+{
+	return RSALIB_RSAREF;
+}
+
 #else	/* !PIC */
 
 /* Failsafe glue for static linking.  Link but complain like hell. */