If the size passed to {,v}s{w,n}printf is larger than INT_MAX+1

(i.e., the return value would overflow), set errno to EOVERFLOW
and return an error.  This improves the chances that buggy
applications -- for instance, ones that pass in a negative integer
as the size due to a bogus calculation -- will fail in safe ways.
Returning an error in these situations is specified by POSIX, but
POSIX appears to have an off-by-one error that isn't duplicated in
this change.

Previously, some of these functions would silently cap the size at
INT_MAX+1, and others would exit with an error after writing more
than INT_MAX characters.

PR:		39256
MFC after:	2 weeks

1 files changed, 11 insertions, 4 deletions

diff --git a/lib/libc/stdio/snprintf.c b/lib/libc/stdio/snprintf.c
index 74af42d..d0ae5a6 100644
--- a/lib/libc/stdio/snprintf.c
+++ b/lib/libc/stdio/snprintf.c
@@ -41,6 +41,7 @@ static char sccsid[] = "@(#)snprintf.c	8.1 (Berkeley) 6/4/93";
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <errno.h>
 #include <limits.h>
 #include <stdio.h>
 #include <stdarg.h>
@@ -59,8 +60,11 @@ snprintf(char * __restrict str, size_t n, char const * __restrict fmt, ...)
 	on = n;
 	if (n != 0)
 		n--;
-	if (n > INT_MAX)
-		n = INT_MAX;
+	if (n > INT_MAX) {
+		errno = EOVERFLOW;
+		*str = '\0';
+		return (EOF);
+	}
 	va_start(ap, fmt);
 	f._flags = __SWR | __SSTR;
 	f._bf._base = f._p = (unsigned char *)str;
@@ -84,8 +88,11 @@ snprintf_l(char * __restrict str, size_t n, locale_t locale,
 	on = n;
 	if (n != 0)
 		n--;
-	if (n > INT_MAX)
-		n = INT_MAX;
+	if (n > INT_MAX) {
+		errno = EOVERFLOW;
+		*str = '\0';
+		return (EOF);
+	}
 	va_start(ap, fmt);
 	f._flags = __SWR | __SSTR;
 	f._bf._base = f._p = (unsigned char *)str;