SMP Lock struct file, filedesc and the global file list.

Seigo Tanimura (tanimura) posted the initial delta.

I've polished it quite a bit reducing the need for locking and
adapting it for KSE.

Locks:

1 mutex in each filedesc
   protects all the fields.
   protects "struct file" initialization, while a struct file
     is being changed from &badfileops -> &pipeops or something
     the filedesc should be locked.

1 mutex in each struct file
   protects the refcount fields.
   doesn't protect anything else.
   the flags used for garbage collection have been moved to
     f_gcflag which was the FILLER short, this doesn't need
     locking because the garbage collection is a single threaded
     container.
  could likely be made to use a pool mutex.

1 sx lock for the global filelist.

struct file *	fhold(struct file *fp);
        /* increments reference count on a file */

struct file *	fhold_locked(struct file *fp);
        /* like fhold but expects file to locked */

struct file *	ffind_hold(struct thread *, int fd);
        /* finds the struct file in thread, adds one reference and
                returns it unlocked */

struct file *	ffind_lock(struct thread *, int fd);
        /* ffind_hold, but returns file locked */

I still have to smp-safe the fget cruft, I'll get to that asap.

1 files changed, 29 insertions, 7 deletions

diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 6860d76..19d62fd 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -134,16 +134,20 @@ socket(td, uap)
 	fhold(fp);
 	error = socreate(uap->domain, &so, uap->type, uap->protocol,
 	    td->td_proc->p_ucred, td);
+	FILEDESC_LOCK(fdp);
 	if (error) {
 		if (fdp->fd_ofiles[fd] == fp) {
 			fdp->fd_ofiles[fd] = NULL;
+			FILEDESC_UNLOCK(fdp);
 			fdrop(fp, td);
-		}
+		} else
+			FILEDESC_UNLOCK(fdp);
 	} else {
 		fp->f_data = (caddr_t)so;	/* already has ref count */
 		fp->f_flag = FREAD|FWRITE;
 		fp->f_ops = &socketops;
 		fp->f_type = DTYPE_SOCKET;
+		FILEDESC_UNLOCK(fdp);
 		td->td_retval[0] = fd;
 	}
 	fdrop(fp, td);
@@ -306,11 +310,13 @@ accept1(td, uap, compat)
 	if (head->so_sigio != NULL)
 		fsetown(fgetown(head->so_sigio), &so->so_sigio);
 
+	FILE_LOCK(nfp);
 	soref(so);			/* file descriptor reference */
 	nfp->f_data = (caddr_t)so;	/* nfp has ref count from falloc */
 	nfp->f_flag = fflag;
 	nfp->f_ops = &socketops;
 	nfp->f_type = DTYPE_SOCKET;
+	FILE_UNLOCK(nfp);
 	sa = 0;
 	error = soaccept(so, &sa);
 	if (error) {
@@ -357,9 +363,13 @@ noconnection:
 	 * out from under us.
 	 */
 	if (error) {
+		FILEDESC_LOCK(fdp);
 		if (fdp->fd_ofiles[fd] == nfp) {
 			fdp->fd_ofiles[fd] = NULL;
+			FILEDESC_UNLOCK(fdp);
 			fdrop(nfp, td);
+		} else {
+			FILEDESC_UNLOCK(fdp);
 		}
 	}
 	splx(s);
@@ -510,24 +520,37 @@ socketpair(td, uap)
 		 if (error)
 			goto free4;
 	}
-	fp1->f_flag = fp2->f_flag = FREAD|FWRITE;
-	fp1->f_ops = fp2->f_ops = &socketops;
-	fp1->f_type = fp2->f_type = DTYPE_SOCKET;
+	FILE_LOCK(fp1);
+	fp1->f_flag = FREAD|FWRITE;
+	fp1->f_ops = &socketops;
+	fp1->f_type = DTYPE_SOCKET;
+	FILE_UNLOCK(fp1);
+	FILE_LOCK(fp2);
+	fp2->f_flag = FREAD|FWRITE;
+	fp2->f_ops = &socketops;
+	fp2->f_type = DTYPE_SOCKET;
+	FILE_UNLOCK(fp2);
 	error = copyout((caddr_t)sv, (caddr_t)uap->rsv, 2 * sizeof (int));
 	fdrop(fp1, td);
 	fdrop(fp2, td);
 	goto done2;
 free4:
+	FILEDESC_LOCK(fdp);
 	if (fdp->fd_ofiles[sv[1]] == fp2) {
 		fdp->fd_ofiles[sv[1]] = NULL;
+		FILEDESC_UNLOCK(fdp);
 		fdrop(fp2, td);
-	}
+	} else
+		FILEDESC_UNLOCK(fdp);
 	fdrop(fp2, td);
 free3:
+	FILEDESC_LOCK(fdp);
 	if (fdp->fd_ofiles[sv[0]] == fp1) {
 		fdp->fd_ofiles[sv[0]] = NULL;
+		FILEDESC_UNLOCK(fdp);
 		fdrop(fp1, td);
-	}
+	} else
+		FILEDESC_UNLOCK(fdp);
 	fdrop(fp1, td);
 free2:
 	(void)soclose(so2);
@@ -1932,4 +1955,3 @@ done:
 	mtx_unlock(&Giant);
 	return (error);
 }
-