Optimize the O_NOCTTY handling hack in linux_common_open().

Differential Revision:	https://reviews.freebsd.org/D2323
Reviewed by:	kib@
MFC after:	1 month
Sponsored by:	The FreeBSD Foundation

1 files changed, 28 insertions, 29 deletions

diff --git a/sys/compat/linux/linux_file.c b/sys/compat/linux/linux_file.c
index c5a9237..03b96d2 100644
--- a/sys/compat/linux/linux_file.c
+++ b/sys/compat/linux/linux_file.c
@@ -132,39 +132,38 @@ linux_common_open(struct thread *td, int dirfd, char *path, int l_flags, int mod
     /* XXX LINUX_O_NOATIME: unable to be easily implemented. */
 
     error = kern_openat(td, dirfd, path, UIO_SYSSPACE, bsd_flags, mode);
-
-    if (!error) {
-	    fd = td->td_retval[0];
-	    /*
-	     * XXX In between kern_open() and fget(), another process
-	     * having the same filedesc could use that fd without
-	     * checking below.
-	     */
-	    error = fget(td, fd, cap_rights_init(&rights, CAP_IOCTL), &fp);
-	    if (!error) {
-		    sx_slock(&proctree_lock);
-		    PROC_LOCK(p);
-		    if (!(bsd_flags & O_NOCTTY) &&
-			SESS_LEADER(p) && !(p->p_flag & P_CONTROLT)) {
-			    PROC_UNLOCK(p);
-			    sx_unlock(&proctree_lock);
-			    /* XXXPJD: Verify if TIOCSCTTY is allowed. */
-			    if (fp->f_type == DTYPE_VNODE)
-				    (void) fo_ioctl(fp, TIOCSCTTY, (caddr_t) 0,
-					     td->td_ucred, td);
-		    } else {
-			    PROC_UNLOCK(p);
-			    sx_sunlock(&proctree_lock);
-		    }
+    if (error != 0)
+	    goto done;
+
+    if (bsd_flags & O_NOCTTY)
+	    goto done;
+
+    /*
+     * XXX In between kern_open() and fget(), another process
+     * having the same filedesc could use that fd without
+     * checking below.
+     */
+    fd = td->td_retval[0];
+    if (fget(td, fd, cap_rights_init(&rights, CAP_IOCTL), &fp) == 0) {
+	    if (fp->f_type != DTYPE_VNODE) {
 		    fdrop(fp, td);
-		    /*
-		     * XXX as above, fdrop()/kern_close() pair is racy.
-		     */
-		    if (error)
-			    kern_close(td, fd);
+		    goto done;
+	    }
+	    sx_slock(&proctree_lock);
+	    PROC_LOCK(p);
+	    if (SESS_LEADER(p) && !(p->p_flag & P_CONTROLT)) {
+		    PROC_UNLOCK(p);
+		    sx_sunlock(&proctree_lock);
+		    /* XXXPJD: Verify if TIOCSCTTY is allowed. */
+		    (void) fo_ioctl(fp, TIOCSCTTY, (caddr_t) 0,
+			td->td_ucred, td);
+	    } else {
+		    PROC_UNLOCK(p);
+		    sx_sunlock(&proctree_lock);
 	    }
     }
 
+done:
 #ifdef DEBUG
     if (ldebug(open))
 	    printf(LMSG("open returns error %d"), error);