If vn_open() fails during kern_open(), don't fdrop() the new file object

until after the call to fdclose().  This closes an obscure race that
could result in the later call to fdclose() actually closing a different
file descriptor if another thread close()'s the file descriptor being
opened before fdrop() is called, so the fdrop() in kern_open() frees the
file object, then the second thread (or a third) creates a new file
descriptor which reuses both the same index and the same file pointer
thus tricking fdclose() in the first thread into thinking that the
original file was still open.

MFC after:	1 week

1 files changed, 2 insertions, 5 deletions

diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index 2cca22e..3a4ff54 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -996,11 +996,6 @@ kern_open(struct thread *td, char *path, enum uio_seg pathseg, int flags,
 		}
 
 		/*
-		 * release our own reference
-		 */
-		fdrop(fp, td);
-
-		/*
 		 * handle special fdopen() case.  bleh.  dupfdopen() is
 		 * responsible for dropping the old contents of ofiles[indx]
 		 * if it succeeds.
@@ -1010,6 +1005,7 @@ kern_open(struct thread *td, char *path, enum uio_seg pathseg, int flags,
 		    (error =
 			dupfdopen(td, fdp, indx, td->td_dupfd, flags, error)) == 0) {
 			td->td_retval[0] = indx;
+			fdrop(fp, td);
 			return (0);
 		}
 		/*
@@ -1017,6 +1013,7 @@ kern_open(struct thread *td, char *path, enum uio_seg pathseg, int flags,
 		 * replaced or closed it.
 		 */
 		fdclose(fdp, fp, indx, td);
+		fdrop(fp, td);
 
 		if (error == ERESTART)
 			error = EINTR;