From 4c65fa8afdeb144b1a2be9476db19538f56c60ba Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Wed, 27 Feb 2008 19:02:02 +0000
Subject: File descriptors are an int, but our stdio FILE object uses a short
 to hold them.  Thus, any fd whose value is greater than SHRT_MAX is handled
 incorrectly (the short value is sign-extended when converted to an int). An
 unpleasant side effect is that if fopen() opens a file and gets a backing fd
 that is greater than SHRT_MAX, fclose() will fail and the file descriptor
 will be leaked.  Better handle this by fixing fopen(), fdopen(), and
 freopen() to fail attempts to use a fd greater than SHRT_MAX with EMFILE.

At some point in the future we should look at expanding the file descriptor
in FILE to an int, but that is a bit complicated due to ABI issues.

MFC after:	1 week
Discussed on:	arch
Reviewed by:	wollman
---
 lib/libc/stdio/fdopen.c  | 12 ++++++++++++
 lib/libc/stdio/fopen.c   | 13 +++++++++++++
 lib/libc/stdio/freopen.c | 14 ++++++++++++++
 3 files changed, 39 insertions(+)

(limited to 'lib/libc')

diff --git a/lib/libc/stdio/fdopen.c b/lib/libc/stdio/fdopen.c
index c199871..02378ed 100644
--- a/lib/libc/stdio/fdopen.c
+++ b/lib/libc/stdio/fdopen.c
@@ -57,6 +57,18 @@ fdopen(fd, mode)
 	if (nofile == 0)
 		nofile = getdtablesize();
 
+	/*
+	 * File descriptors are a full int, but _file is only a short.
+	 * If we get a valid file descriptor that is greater than
+	 * SHRT_MAX, then the fd will get sign-extended into an
+	 * invalid file descriptor.  Handle this case by failing the
+	 * open.
+	 */
+	if (fd > SHRT_MAX) {
+		errno = EMFILE;
+		return (NULL);
+	}
+
 	if ((flags = __sflags(mode, &oflags)) == 0)
 		return (NULL);
 
diff --git a/lib/libc/stdio/fopen.c b/lib/libc/stdio/fopen.c
index 9cedcda..a6c0028 100644
--- a/lib/libc/stdio/fopen.c
+++ b/lib/libc/stdio/fopen.c
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <unistd.h>
 #include <stdio.h>
 #include <errno.h>
 #include "un-namespace.h"
@@ -63,6 +64,18 @@ fopen(file, mode)
 		fp->_flags = 0;			/* release */
 		return (NULL);
 	}
+	/*
+	 * File descriptors are a full int, but _file is only a short.
+	 * If we get a valid file descriptor that is greater than
+	 * SHRT_MAX, then the fd will get sign-extended into an
+	 * invalid file descriptor.  Handle this case by failing the
+	 * open.
+	 */
+	if (f > SHRT_MAX) {
+		_close(f);
+		errno = EMFILE;
+		return (NULL);
+	}
 	fp->_file = f;
 	fp->_flags = flags;
 	fp->_cookie = fp;
diff --git a/lib/libc/stdio/freopen.c b/lib/libc/stdio/freopen.c
index b5bc884..a566386 100644
--- a/lib/libc/stdio/freopen.c
+++ b/lib/libc/stdio/freopen.c
@@ -203,6 +203,20 @@ finish:
 		}
 	}
 
+	/*
+	 * File descriptors are a full int, but _file is only a short.
+	 * If we get a valid file descriptor that is greater than
+	 * SHRT_MAX, then the fd will get sign-extended into an
+	 * invalid file descriptor.  Handle this case by failing the
+	 * open.
+	 */
+	if (f > SHRT_MAX) {
+		fp->_flags = 0;		/* set it free */
+		FUNLOCKFILE(fp);
+		errno = EMFILE;
+		return (NULL);
+	}
+
 	fp->_flags = flags;
 	fp->_file = f;
 	fp->_cookie = fp;
-- 
cgit v1.1