SMP Lock struct file, filedesc and the global file list.

Seigo Tanimura (tanimura) posted the initial delta.

I've polished it quite a bit reducing the need for locking and
adapting it for KSE.

Locks:

1 mutex in each filedesc
   protects all the fields.
   protects "struct file" initialization, while a struct file
     is being changed from &badfileops -> &pipeops or something
     the filedesc should be locked.

1 mutex in each struct file
   protects the refcount fields.
   doesn't protect anything else.
   the flags used for garbage collection have been moved to
     f_gcflag which was the FILLER short, this doesn't need
     locking because the garbage collection is a single threaded
     container.
  could likely be made to use a pool mutex.

1 sx lock for the global filelist.

struct file *	fhold(struct file *fp);
        /* increments reference count on a file */

struct file *	fhold_locked(struct file *fp);
        /* like fhold but expects file to locked */

struct file *	ffind_hold(struct thread *, int fd);
        /* finds the struct file in thread, adds one reference and
                returns it unlocked */

struct file *	ffind_lock(struct thread *, int fd);
        /* ffind_hold, but returns file locked */

I still have to smp-safe the fget cruft, I'll get to that asap.

1 files changed, 22 insertions, 4 deletions

diff --git a/sys/dev/streams/streams.c b/sys/dev/streams/streams.c
index 38d7fee..a5c2b4d 100644
--- a/sys/dev/streams/streams.c
+++ b/sys/dev/streams/streams.c
@@ -266,15 +266,19 @@ streamsopen(dev_t dev, int oflags, int devtype, struct thread *td)
 
 	if ((error = socreate(family, &so, type, protocol,
 	    td->td_proc->p_ucred, td)) != 0) {
+	  FILEDESC_LOCK(p->p_fd);
 	  p->p_fd->fd_ofiles[fd] = 0;
+	  FILEDESC_UNLOCK(p->p_fd);
 	  ffree(fp);
 	  return error;
 	}
 
+	FILEDESC_LOCK(p->p_fd);
 	fp->f_data = (caddr_t)so;
 	fp->f_flag = FREAD|FWRITE;
 	fp->f_ops = &svr4_netops;
 	fp->f_type = DTYPE_SOCKET;
+	FILEDESC_UNLOCK(p->p_fd);
 
 	(void)svr4_stream_get(fp);
 	PROC_LOCK(p);
@@ -355,8 +359,12 @@ svr4_stream_get(fp)
 
 	so = (struct socket *) fp->f_data;
 
-       	if (so->so_emuldata)
+	/*
+	 * mpfixme: lock socketbuffer here
+	 */
+	if (so->so_emuldata) {
 		return so->so_emuldata;
+	}
 
 	/* Allocate a new one. */
 	st = malloc(sizeof(struct svr4_strm), M_TEMP, M_WAITOK);
@@ -364,8 +372,19 @@ svr4_stream_get(fp)
 	st->s_cmd = ~0;
 	st->s_afd = -1;
 	st->s_eventmask = 0;
-	so->so_emuldata = st;
-	fp->f_ops = &svr4_netops;
+	/*
+	 * avoid a race where we loose due to concurrancy issues
+	 * of two threads trying to allocate the so_emuldata.
+	 */
+	if (so->so_emuldata) {
+		/* lost the race, use the existing emuldata */
+		FREE(st, M_TEMP);
+		st = so->so_emuldata;
+	} else {
+		/* we won, or there was no race, use our copy */
+		so->so_emuldata = st;
+		fp->f_ops = &svr4_netops;
+	}
 
 	return st;
 }
@@ -406,5 +425,4 @@ svr4_soo_close(struct file *fp, struct thread *td)
 	svr4_delete_socket(td->td_proc, fp);
 	free(so->so_emuldata, M_TEMP);
 	return soo_close(fp, td);
-	return (0);
 }