Redo the sigio locking.

Turn the sigio sx into a mutex.

Sigio lock is really only needed to protect interrupts from dereferencing
the sigio pointer in an object when the sigio itself is being destroyed.

In order to do this in the most unintrusive manner change pgsigio's
sigio * argument into a **, that way we can lock internally to the
function.

8 files changed, 30 insertions, 62 deletions

diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index 13b91ad..425cb39 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -63,6 +63,7 @@
 #include <sys/event.h>
 #include <sys/sx.h>
 #include <sys/socketvar.h>
+#include <sys/signalvar.h>
 
 #include <machine/limits.h>
 
@@ -113,7 +114,7 @@ struct filelist filehead;	/* head of list of open files */
 int nfiles;			/* actual number of open files */
 extern int cmask;	
 struct sx filelist_lock;	/* sx to protect filelist */
-struct sx sigio_lock;		/* sx to protect pointers to sigio */
+struct mtx sigio_lock;		/* mtx to protect pointers to sigio */
 
 /*
  * System calls on descriptors.
@@ -536,14 +537,14 @@ void
 funsetown(sigio)
 	struct sigio *sigio;
 {
-	int s;
 
-	if (sigio == NULL)
+	SIGIO_LOCK();
+	if (sigio == NULL) {
+		SIGIO_UNLOCK();
 		return;
-
-	s = splhigh();
+	}
 	*(sigio->sio_myref) = NULL;
-	splx(s);
+	SIGIO_UNLOCK();
 	if ((sigio)->sio_pgid < 0) {
 		struct pgrp *pg = (sigio)->sio_pgrp;
 		PGRP_LOCK(pg);
@@ -566,7 +567,6 @@ void
 funsetownlst(sigiolst)
 	struct sigiolst *sigiolst;
 {
-	int s;
 	struct sigio *sigio;
 	struct proc *p;
 	struct pgrp *pg;
@@ -591,9 +591,9 @@ funsetownlst(sigiolst)
 	}
 
 	while ((sigio = SLIST_FIRST(sigiolst)) != NULL) {
-		s = splhigh();
+		SIGIO_LOCK();
 		*(sigio->sio_myref) = NULL;
-		splx(s);
+		SIGIO_UNLOCK();
 		if (pg != NULL) {
 			KASSERT(sigio->sio_pgid < 0, ("Proc sigio in pgrp sigio list"));
 			KASSERT(sigio->sio_pgrp == pg, ("Bogus pgrp in sigio list"));
@@ -632,7 +632,7 @@ fsetown(pgid, sigiop)
 	struct proc *proc;
 	struct pgrp *pgrp;
 	struct sigio *sigio;
-	int s, ret;
+	int ret;
 
 	if (pgid == 0) {
 		funsetown(*sigiop);
@@ -706,9 +706,9 @@ fsetown(pgid, sigiop)
 		PGRP_UNLOCK(pgrp);
 	}
 	sx_sunlock(&proctree_lock);
-	s = splhigh();
+	SIGIO_LOCK();
 	*sigiop = sigio;
-	splx(s);
+	SIGIO_UNLOCK();
 	return (0);
 
 fail:
@@ -2187,5 +2187,5 @@ filelistinit(dummy)
 	    NULL, NULL, UMA_ALIGN_PTR, 0);
 
 	sx_init(&filelist_lock, "filelist lock");
-	sx_init(&sigio_lock, "sigio lock");
+	mtx_init(&sigio_lock, "sigio lock", NULL, MTX_DEF);
 }
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 16c6fed..4c1b683 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -2087,13 +2087,18 @@ nosys(td, args)
  * stored credentials rather than those of the current process.
  */
 void
-pgsigio(sigio, sig, checkctty)
-	struct sigio *sigio;
+pgsigio(sigiop, sig, checkctty)
+	struct sigio **sigiop;
 	int sig, checkctty;
 {
-	if (sigio == NULL)
-		return;
+	struct sigio *sigio;
 
+	SIGIO_LOCK();
+	sigio = *sigiop;
+	if (sigio == NULL) {
+		SIGIO_UNLOCK();
+		return;
+	}
 	if (sigio->sio_pgid > 0) {
 		PROC_LOCK(sigio->sio_proc);
 		if (CANSIGIO(sigio->sio_ucred, sigio->sio_proc->p_ucred))
@@ -2112,6 +2117,7 @@ pgsigio(sigio, sig, checkctty)
 		}
 		PGRP_UNLOCK(sigio->sio_pgrp);
 	}
+	SIGIO_UNLOCK();
 }
 
 static int
diff --git a/sys/kern/subr_log.c b/sys/kern/subr_log.c
index b495a79..568b3fe 100644
--- a/sys/kern/subr_log.c
+++ b/sys/kern/subr_log.c
@@ -198,7 +198,7 @@ logtimeout(void *arg)
 	msgbuftrigger = 0;
 	selwakeup(&logsoftc.sc_selp);
 	if ((logsoftc.sc_state & LOG_ASYNC) && logsoftc.sc_sigio != NULL)
-		pgsigio(logsoftc.sc_sigio, SIGIO, 0);
+		pgsigio(&logsoftc.sc_sigio, SIGIO, 0);
 	if (logsoftc.sc_state & LOG_RDWAIT) {
 		wakeup((caddr_t)msgbufp);
 		logsoftc.sc_state &= ~LOG_RDWAIT;
diff --git a/sys/kern/sys_pipe.c b/sys/kern/sys_pipe.c
index c671c94..d0682fa 100644
--- a/sys/kern/sys_pipe.c
+++ b/sys/kern/sys_pipe.c
@@ -429,7 +429,7 @@ pipeselwakeup(cpipe)
 		selwakeup(&cpipe->pipe_sel);
 	}
 	if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio)
-		pgsigio(cpipe->pipe_sigio, SIGIO, 0);
+		pgsigio(&cpipe->pipe_sigio, SIGIO, 0);
 	KNOTE(&cpipe->pipe_sel.si_note, 0);
 }
 
diff --git a/sys/kern/tty.c b/sys/kern/tty.c
index 6e4ca18..7b3459a 100644
--- a/sys/kern/tty.c
+++ b/sys/kern/tty.c
@@ -2289,7 +2289,7 @@ ttwakeup(tp)
 	if (SEL_WAITING(&tp->t_rsel))
 		selwakeup(&tp->t_rsel);
 	if (ISSET(tp->t_state, TS_ASYNC) && tp->t_sigio != NULL)
-		pgsigio(tp->t_sigio, SIGIO, (tp->t_session != NULL));
+		pgsigio(&tp->t_sigio, SIGIO, (tp->t_session != NULL));
 	wakeup(TSA_HUP_OR_INPUT(tp));
 	KNOTE(&tp->t_rsel.si_note, 0);
 }
@@ -2305,7 +2305,7 @@ ttwwakeup(tp)
 	if (SEL_WAITING(&tp->t_wsel) && tp->t_outq.c_cc <= tp->t_olowat)
 		selwakeup(&tp->t_wsel);
 	if (ISSET(tp->t_state, TS_ASYNC) && tp->t_sigio != NULL)
-		pgsigio(tp->t_sigio, SIGIO, (tp->t_session != NULL));
+		pgsigio(&tp->t_sigio, SIGIO, (tp->t_session != NULL));
 	if (ISSET(tp->t_state, TS_BUSY | TS_SO_OCOMPLETE) ==
 	    TS_SO_OCOMPLETE && tp->t_outq.c_cc == 0) {
 		CLR(tp->t_state, TS_SO_OCOMPLETE);
diff --git a/sys/kern/uipc_sockbuf.c b/sys/kern/uipc_sockbuf.c
index cb0a682..91cb57b 100644
--- a/sys/kern/uipc_sockbuf.c
+++ b/sys/kern/uipc_sockbuf.c
@@ -52,7 +52,6 @@
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/stat.h>
-#include <sys/sx.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 
@@ -113,7 +112,6 @@ soisconnected_locked(so)
 {
 	struct socket *head = so->so_head;
 
-	SIGIO_ASSERT(SX_SLOCKED); /* XXX */
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING);
 	so->so_state |= SS_ISCONNECTED;
 	if (head && (so->so_state & SS_INCOMP)) {
@@ -122,9 +120,7 @@ soisconnected_locked(so)
 			so->so_upcallarg = head->so_accf->so_accept_filter_arg;
 			so->so_rcv.sb_flags |= SB_UPCALL;
 			so->so_options &= ~SO_ACCEPTFILTER;
-			SIGIO_SUNLOCK(); /* XXX */
 			so->so_upcall(so, so->so_upcallarg, 0);
-			SIGIO_SLOCK();
 			return;
 		}
 		TAILQ_REMOVE(&head->so_incomp, so, so_list);
@@ -147,7 +143,6 @@ soisconnected(so)
 {
 	struct socket *head = so->so_head;
 
-	SIGIO_SLOCK();
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING);
 	so->so_state |= SS_ISCONNECTED;
 	if (head && (so->so_state & SS_INCOMP)) {
@@ -156,7 +151,6 @@ soisconnected(so)
 			so->so_upcallarg = head->so_accf->so_accept_filter_arg;
 			so->so_rcv.sb_flags |= SB_UPCALL;
 			so->so_options &= ~SO_ACCEPTFILTER;
-			SIGIO_SUNLOCK();
 			so->so_upcall(so, so->so_upcallarg, 0);
 			return;
 		}
@@ -173,7 +167,6 @@ soisconnected(so)
 		sorwakeup_locked(so);
 		sowwakeup_locked(so);
 	}
-	SIGIO_SUNLOCK();
 }
 
 void
@@ -181,13 +174,11 @@ soisdisconnecting(so)
 	register struct socket *so;
 {
 
-	SIGIO_SLOCK();
 	so->so_state &= ~SS_ISCONNECTING;
 	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE);
 	wakeup((caddr_t)&so->so_timeo);
 	sowwakeup_locked(so);
 	sorwakeup_locked(so);
-	SIGIO_SUNLOCK();
 }
 
 void
@@ -195,7 +186,6 @@ soisdisconnected_locked(so)
 	register struct socket *so;
 {
 
-	SIGIO_ASSERT(SX_LOCKED);
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
 	so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED);
 	wakeup((caddr_t)&so->so_timeo);
@@ -208,9 +198,7 @@ soisdisconnected(so)
 	register struct socket *so;
 {
 
-	SIGIO_SLOCK();
 	soisdisconnected_locked(so);
-	SIGIO_SUNLOCK();
 }
 
 /*
@@ -266,9 +254,7 @@ sonewconn(head, connstatus)
 		head->so_incqlen++;
 	}
 	if (connstatus) {
-		SIGIO_SLOCK();
 		sorwakeup_locked(head);
-		SIGIO_SUNLOCK();
 		wakeup((caddr_t)&head->so_timeo);
 		so->so_state |= connstatus;
 	}
@@ -290,10 +276,8 @@ socantsendmore(so)
 	struct socket *so;
 {
 
-	SIGIO_SLOCK();
 	so->so_state |= SS_CANTSENDMORE;
 	sowwakeup_locked(so);
-	SIGIO_SUNLOCK();
 }
 
 void
@@ -301,10 +285,8 @@ socantrcvmore(so)
 	struct socket *so;
 {
 
-	SIGIO_SLOCK();
 	so->so_state |= SS_CANTRCVMORE;
 	sorwakeup_locked(so);
-	SIGIO_SUNLOCK();
 }
 
 /*
@@ -353,7 +335,6 @@ sowakeup(so, sb)
 	register struct socket *so;
 	register struct sockbuf *sb;
 {
-	SIGIO_ASSERT(SX_LOCKED);
 
 	selwakeup(&sb->sb_sel);
 	sb->sb_flags &= ~SB_SEL;
@@ -362,7 +343,7 @@ sowakeup(so, sb)
 		wakeup((caddr_t)&sb->sb_cc);
 	}
 	if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL)
-		pgsigio(so->so_sigio, SIGIO, 0);
+		pgsigio(&so->so_sigio, SIGIO, 0);
 	if (sb->sb_flags & SB_UPCALL)
 		(*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT);
 	if (sb->sb_flags & SB_AIO)
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 0e0fbd9..33f8a4d 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -1527,7 +1527,7 @@ sohasoutofband(so)
 	register struct socket *so;
 {
 	if (so->so_sigio != NULL)
-		pgsigio(so->so_sigio, SIGURG, 0);
+		pgsigio(&so->so_sigio, SIGURG, 0);
 	selwakeup(&so->so_rcv.sb_sel);
 }
 
diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index cb0a682..91cb57b 100644
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -52,7 +52,6 @@
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/stat.h>
-#include <sys/sx.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 
@@ -113,7 +112,6 @@ soisconnected_locked(so)
 {
 	struct socket *head = so->so_head;
 
-	SIGIO_ASSERT(SX_SLOCKED); /* XXX */
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING);
 	so->so_state |= SS_ISCONNECTED;
 	if (head && (so->so_state & SS_INCOMP)) {
@@ -122,9 +120,7 @@ soisconnected_locked(so)
 			so->so_upcallarg = head->so_accf->so_accept_filter_arg;
 			so->so_rcv.sb_flags |= SB_UPCALL;
 			so->so_options &= ~SO_ACCEPTFILTER;
-			SIGIO_SUNLOCK(); /* XXX */
 			so->so_upcall(so, so->so_upcallarg, 0);
-			SIGIO_SLOCK();
 			return;
 		}
 		TAILQ_REMOVE(&head->so_incomp, so, so_list);
@@ -147,7 +143,6 @@ soisconnected(so)
 {
 	struct socket *head = so->so_head;
 
-	SIGIO_SLOCK();
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING);
 	so->so_state |= SS_ISCONNECTED;
 	if (head && (so->so_state & SS_INCOMP)) {
@@ -156,7 +151,6 @@ soisconnected(so)
 			so->so_upcallarg = head->so_accf->so_accept_filter_arg;
 			so->so_rcv.sb_flags |= SB_UPCALL;
 			so->so_options &= ~SO_ACCEPTFILTER;
-			SIGIO_SUNLOCK();
 			so->so_upcall(so, so->so_upcallarg, 0);
 			return;
 		}
@@ -173,7 +167,6 @@ soisconnected(so)
 		sorwakeup_locked(so);
 		sowwakeup_locked(so);
 	}
-	SIGIO_SUNLOCK();
 }
 
 void
@@ -181,13 +174,11 @@ soisdisconnecting(so)
 	register struct socket *so;
 {
 
-	SIGIO_SLOCK();
 	so->so_state &= ~SS_ISCONNECTING;
 	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE);
 	wakeup((caddr_t)&so->so_timeo);
 	sowwakeup_locked(so);
 	sorwakeup_locked(so);
-	SIGIO_SUNLOCK();
 }
 
 void
@@ -195,7 +186,6 @@ soisdisconnected_locked(so)
 	register struct socket *so;
 {
 
-	SIGIO_ASSERT(SX_LOCKED);
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
 	so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED);
 	wakeup((caddr_t)&so->so_timeo);
@@ -208,9 +198,7 @@ soisdisconnected(so)
 	register struct socket *so;
 {
 
-	SIGIO_SLOCK();
 	soisdisconnected_locked(so);
-	SIGIO_SUNLOCK();
 }
 
 /*
@@ -266,9 +254,7 @@ sonewconn(head, connstatus)
 		head->so_incqlen++;
 	}
 	if (connstatus) {
-		SIGIO_SLOCK();
 		sorwakeup_locked(head);
-		SIGIO_SUNLOCK();
 		wakeup((caddr_t)&head->so_timeo);
 		so->so_state |= connstatus;
 	}
@@ -290,10 +276,8 @@ socantsendmore(so)
 	struct socket *so;
 {
 
-	SIGIO_SLOCK();
 	so->so_state |= SS_CANTSENDMORE;
 	sowwakeup_locked(so);
-	SIGIO_SUNLOCK();
 }
 
 void
@@ -301,10 +285,8 @@ socantrcvmore(so)
 	struct socket *so;
 {
 
-	SIGIO_SLOCK();
 	so->so_state |= SS_CANTRCVMORE;
 	sorwakeup_locked(so);
-	SIGIO_SUNLOCK();
 }
 
 /*
@@ -353,7 +335,6 @@ sowakeup(so, sb)
 	register struct socket *so;
 	register struct sockbuf *sb;
 {
-	SIGIO_ASSERT(SX_LOCKED);
 
 	selwakeup(&sb->sb_sel);
 	sb->sb_flags &= ~SB_SEL;
@@ -362,7 +343,7 @@ sowakeup(so, sb)
 		wakeup((caddr_t)&sb->sb_cc);
 	}
 	if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL)
-		pgsigio(so->so_sigio, SIGIO, 0);
+		pgsigio(&so->so_sigio, SIGIO, 0);
 	if (sb->sb_flags & SB_UPCALL)
 		(*so->so_upcall)(so, so->so_upcallarg, M_DONTWAIT);
 	if (sb->sb_flags & SB_AIO)