If the size passed to {,v}s{w,n}printf is larger than INT_MAX+1

(i.e., the return value would overflow), set errno to EOVERFLOW
and return an error.  This improves the chances that buggy
applications -- for instance, ones that pass in a negative integer
as the size due to a bogus calculation -- will fail in safe ways.
Returning an error in these situations is specified by POSIX, but
POSIX appears to have an off-by-one error that isn't duplicated in
this change.

Previously, some of these functions would silently cap the size at
INT_MAX+1, and others would exit with an error after writing more
than INT_MAX characters.

PR:		39256
MFC after:	2 weeks

1 files changed, 3 insertions, 0 deletions

diff --git a/lib/libc/stdio/vfprintf.c b/lib/libc/stdio/vfprintf.c
index bfbdc13..4ad0ee1 100644
--- a/lib/libc/stdio/vfprintf.c
+++ b/lib/libc/stdio/vfprintf.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 
 #include <ctype.h>
+#include <errno.h>
 #include <limits.h>
 #include <locale.h>
 #include <stddef.h>
@@ -480,6 +481,7 @@ __vfprintf(FILE *fp, locale_t locale, const char *fmt0, va_list ap)
 		if ((n = fmt - cp) != 0) {
 			if ((unsigned)ret + n > INT_MAX) {
 				ret = EOF;
+				errno = EOVERFLOW;
 				goto error;
 			}
 			PRINT(cp, n);
@@ -935,6 +937,7 @@ number:			if ((dprec = prec) >= 0)
 		prsize = width > realsz ? width : realsz;
 		if ((unsigned)ret + prsize > INT_MAX) {
 			ret = EOF;
+			errno = EOVERFLOW;
 			goto error;
 		}