Refactor select to reduce contention and hide internal implementation

details from consumers.

 - Track individual selecters on a per-descriptor basis such that there
   are no longer collisions and after sleeping for events only those
   descriptors which triggered events must be rescaned.
 - Protect the selinfo (per descriptor) structure with a mtx pool mutex.
   mtx pool mutexes were chosen to preserve api compatibility with
   existing code which does nothing but bzero() to setup selinfo
   structures.
 - Use a per-thread wait channel rather than a global wait channel.
 - Hide select implementation details in a seltd structure which is
   opaque to the rest of the kernel.
 - Provide a 'selsocket' interface for those kernel consumers who wish to
   select on a socket when they have no fd so they no longer have to
   be aware of select implementation details.

Tested by:	kris
Reviewed on:	arch

3 files changed, 9 insertions, 111 deletions

diff --git a/sys/netncp/ncp_rq.c b/sys/netncp/ncp_rq.c
index b637c9a..38e3e54 100644
--- a/sys/netncp/ncp_rq.c
+++ b/sys/netncp/ncp_rq.c
@@ -43,6 +43,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/mbuf.h>
 #include <sys/poll.h>
 #include <sys/proc.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
 #include <sys/uio.h>
 
 #include <netncp/ncp.h>
@@ -274,7 +276,9 @@ ncp_request_int(struct ncp_rq *rqp)
 	/*
 	 * Flush out replies on previous reqs
 	 */
-	while (ncp_poll(so, POLLIN) != 0) {
+	tv.tv_sec = 0;
+	tv.tv_usec = 0;
+	while (selsocket(so, POLLIN, &tv, td) == 0) {
 		if (ncp_sock_recv(so, &m, &len) != 0)
 			break;
 		m_freem(m);
@@ -319,7 +323,7 @@ ncp_request_int(struct ncp_rq *rqp)
 		}
 		tv.tv_sec = conn->li.timeout;
 		tv.tv_usec = 0;
-		error = ncp_sock_rselect(so, td, &tv, POLLIN);
+		error = selsocket(so, POLLIN, &tv, td);
 		if (error == EWOULDBLOCK )	/* timeout expired */
 			continue;
 		error = ncp_chkintr(conn, td);
@@ -335,7 +339,9 @@ ncp_request_int(struct ncp_rq *rqp)
 		dosend = 1;	/* resend rq if error */
 		for (;;) {
 			error = 0;
-			if (ncp_poll(so, POLLIN) == 0)
+			tv.tv_sec = 0;
+			tv.tv_usec = 0;
+			if (selsocket(so, POLLIN, &tv, td) != 0)
 				break;
 /*			if (so->so_rcv.sb_cc == 0) {
 				break;
diff --git a/sys/netncp/ncp_sock.c b/sys/netncp/ncp_sock.c
index 0fe320f..426f40c 100644
--- a/sys/netncp/ncp_sock.c
+++ b/sys/netncp/ncp_sock.c
@@ -65,7 +65,6 @@ __FBSDID("$FreeBSD$");
 #define ipx_setnullhost(x) ((x).x_host.s_host[0] = 0); \
 	((x).x_host.s_host[1] = 0); ((x).x_host.s_host[2] = 0);
 
-/*int ncp_poll(struct socket *so, int events);*/
 /*static int ncp_getsockname(struct socket *so, caddr_t asa, int *alen);*/
 static int ncp_soconnect(struct socket *so, struct sockaddr *target,
 			 struct thread *td);
@@ -181,110 +180,6 @@ ncp_sock_send(struct socket *so, struct mbuf *top, struct ncp_rq *rqp)
 	return error;
 }
 
-int
-ncp_poll(struct socket *so, int events)
-{
-	struct thread *td = curthread;
-	int revents;
-
-	/* Fake up enough state to look like we are in poll(2). */
-	mtx_lock(&sellock);
-	thread_lock(td);
-	td->td_flags |= TDF_SELECT;
-	thread_unlock(td);
-	mtx_unlock(&sellock);
-	TAILQ_INIT(&td->td_selq);
-
-	revents = sopoll(so, events, NULL, td);
-
-	/* Tear down the fake poll(2) state. */
-	mtx_lock(&sellock);
-	clear_selinfo_list(td);
-	thread_lock(td);
-	td->td_flags &= ~TDF_SELECT;
-	thread_unlock(td);
-	mtx_unlock(&sellock);
-
-	return (revents);
-}
-
-int
-ncp_sock_rselect(struct socket *so, struct thread *td, struct timeval *tv,
-		 int events)
-{
-	struct timeval atv, rtv, ttv;
-	int ncoll, timo, error, revents;
-
-	if (tv) {
-		atv = *tv;
-		if (itimerfix(&atv)) {
-			error = EINVAL;
-			goto done_noproclock;
-		}
-		getmicrouptime(&rtv);
-		timevaladd(&atv, &rtv);
-	}
-	timo = 0;
-	mtx_lock(&sellock);
-
-retry:
-	ncoll = nselcoll;
-	thread_lock(td);
-	td->td_flags |= TDF_SELECT;
-	thread_unlock(td);
-	mtx_unlock(&sellock);
-
-	TAILQ_INIT(&td->td_selq);
-	revents = sopoll(so, events, NULL, td);
-	mtx_lock(&sellock);
-	if (revents) {
-		error = 0;
-		goto done;
-	}
-	if (tv) {
-		getmicrouptime(&rtv);
-		if (timevalcmp(&rtv, &atv, >=)) {
-			error = EWOULDBLOCK;
-			goto done;
-		}
-		ttv = atv;
-		timevalsub(&ttv, &rtv);
-		timo = tvtohz(&ttv);
-	}
-	/*
-	 * An event of our interest may occur during locking a thread.
-	 * In order to avoid missing the event that occurred during locking
-	 * the process, test TDF_SELECT and rescan file descriptors if
-	 * necessary.
-	 */
-	thread_lock(td);
-	if ((td->td_flags & TDF_SELECT) == 0 || nselcoll != ncoll) {
-		thread_unlock(td);
-		goto retry;
-	}
-	thread_unlock(td);
-
-	if (timo > 0)
-		error = cv_timedwait(&selwait, &sellock, timo);
-	else {
-		cv_wait(&selwait, &sellock);
-		error = 0;
-	}
-
-done:
-	clear_selinfo_list(td);
-
-	thread_lock(td);
-	td->td_flags &= ~TDF_SELECT;
-	thread_unlock(td);
-	mtx_unlock(&sellock);
-
-done_noproclock:
-	if (error == ERESTART)
-		error = 0;
-	return (error);
-}
-
 /*
  * Connect to specified server via IPX
  */
diff --git a/sys/netncp/ncp_sock.h b/sys/netncp/ncp_sock.h
index 7833760..a3998a4 100644
--- a/sys/netncp/ncp_sock.h
+++ b/sys/netncp/ncp_sock.h
@@ -45,9 +45,6 @@ int  ncp_sock_connect(struct ncp_conn *ncp);
 int  ncp_sock_recv(struct socket *so, struct mbuf **mp, int *rlen);
 int  ncp_sock_send(struct socket *so, struct mbuf *data, struct ncp_rq *rqp);
 int  ncp_sock_disconnect(struct ncp_conn *conn);
-int  ncp_poll(struct socket *so, int events);
-int  ncp_sock_rselect(struct socket *so, struct thread *td, struct timeval *tv,
-		      int events);
 int  ncp_sock_checksum(struct ncp_conn *conn, int enable);
 
 void ncp_check_rq(struct ncp_conn *conn);