Rework socket upcalls to close some races with setup/teardown of upcalls.

- Each socket upcall is now invoked with the appropriate socket buffer
  locked.  It is not permissible to call soisconnected() with this lock
  held; however, so socket upcalls now return an integer value.  The two
  possible values are SU_OK and SU_ISCONNECTED.  If an upcall returns
  SU_ISCONNECTED, then the soisconnected() will be invoked on the
  socket after the socket buffer lock is dropped.
- A new API is provided for setting and clearing socket upcalls.  The
  API consists of soupcall_set() and soupcall_clear().
- To simplify locking, each socket buffer now has a separate upcall.
- When a socket upcall returns SU_ISCONNECTED, the upcall is cleared from
  the receive socket buffer automatically.  Note that a SO_SND upcall
  should never return SU_ISCONNECTED.
- All this means that accept filters should now return SU_ISCONNECTED
  instead of calling soisconnected() directly.  They also no longer need
  to explicitly clear the upcall on the new socket.
- The HTTP accept filter still uses soupcall_set() to manage its internal
  state machine, but other accept filters no longer have any explicit
  knowlege of socket upcall internals aside from their return value.
- The various RPC client upcalls currently drop the socket buffer lock
  while invoking soreceive() as a temporary band-aid.  The plan for
  the future is to add a new flag to allow soreceive() to be called with
  the socket buffer locked.
- The AIO callback for socket I/O is now also invoked with the socket
  buffer locked.  Previously sowakeup() would drop the socket buffer
  lock only to call aio_swake() which immediately re-acquired the socket
  buffer lock for the duration of the function call.

Discussed with:	rwatson, rmacklem

1 files changed, 10 insertions, 13 deletions

diff --git a/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c b/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c
index 613d62e..83b1b0d 100644
--- a/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c
+++ b/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c
@@ -141,7 +141,7 @@ SYSCTL_UINT(_hw_cxgb, OID_AUTO, cong_flavor, CTLFLAG_RDTUN, &cong_flavor, 0,
 
 static void ep_timeout(void *arg);
 static void connect_reply_upcall(struct iwch_ep *ep, int status);
-static void iwch_so_upcall(struct socket *so, void *arg, int waitflag);
+static int iwch_so_upcall(struct socket *so, void *arg, int waitflag);
 
 /*
  * Cruft to offload socket upcalls onto thread.
@@ -335,9 +335,7 @@ close_socket(struct iwch_ep_common *epc)
 {
 	CTR4(KTR_IW_CXGB, "%s ep %p so %p state %s", __FUNCTION__, epc, epc->so, states[epc->state]);
 	SOCK_LOCK(epc->so);
-	epc->so->so_upcall = NULL;
-	epc->so->so_upcallarg = NULL;
-	epc->so->so_rcv.sb_flags &= ~SB_UPCALL;
+	soupcall_clear(epc->so, SO_RCV);
 	SOCK_UNLOCK(epc->so);
 	soshutdown(epc->so, SHUT_WR|SHUT_RD);
 	epc->so = NULL;
@@ -1108,7 +1106,7 @@ terminate(struct t3cdev *tdev, struct mbuf *m, void *ctx)
 {
 	struct toepcb *toep = (struct toepcb *)ctx;
 	struct socket *so = toeptoso(toep);
-	struct iwch_ep *ep = so->so_upcallarg;
+	struct iwch_ep *ep = so->so_rcv.sb_upcallarg;
 
 	CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep);
 	m_adj(m, sizeof(struct cpl_rdma_terminate));
@@ -1129,7 +1127,7 @@ ec_status(struct t3cdev *tdev, struct mbuf *m, void *ctx)
 	struct iwch_qp_attributes attrs;
 	int release = 0;
 
-	ep = so->so_upcallarg;
+	ep = so->so_rcv.sb_upcallarg;
 	CTR5(KTR_IW_CXGB, "%s ep %p so %p state %s ec_status %d", __FUNCTION__, ep, ep->com.so, states[ep->com.state], rep->status);
 	if (!so || !ep) {
 		panic("bogosity ep %p state %d, so %p state %x\n", ep, ep ? ep->com.state : -1, so, so ? so->so_state : -1); 
@@ -1309,10 +1307,10 @@ static int init_sock(struct iwch_ep_common *epc)
 	struct sockopt sopt;
 	int on=1;
 
-	epc->so->so_upcall = iwch_so_upcall;
-	epc->so->so_upcallarg = epc;
-	epc->so->so_rcv.sb_flags |= SB_UPCALL;
+	SOCK_LOCK(epc->so);
+	soupcall_set(epc->so, SO_RCV, iwch_so_upcall, epc);
 	epc->so->so_state |= SS_NBIO;
+	SOCK_UNLOCK(epc->so);
 	sopt.sopt_dir = SOPT_SET;
 	sopt.sopt_level = SOL_SOCKET;
 	sopt.sopt_name = SO_NO_DDP;
@@ -1611,10 +1609,8 @@ dequeue_socket(struct socket *head, struct sockaddr_in **remote, struct iwch_ep
 	so->so_qstate &= ~SQ_COMP;
 	so->so_head = NULL;
 	soref(so);
-	so->so_rcv.sb_flags |= SB_UPCALL;
+	soupcall_set(so, SO_RCV, iwch_so_upcall, child_ep);
 	so->so_state |= SS_NBIO;
-	so->so_upcall = iwch_so_upcall;
-	so->so_upcallarg = child_ep;
 	PANIC_IF(!(so->so_state & SS_ISCONNECTED));
 	PANIC_IF(so->so_error);
 	SOCK_UNLOCK(so);
@@ -1661,7 +1657,7 @@ process_newconn(struct iwch_ep *parent_ep)
 	process_mpa_request(child_ep);
 }
 
-static void
+static int
 iwch_so_upcall(struct socket *so, void *arg, int waitflag)
 {
 	struct iwch_ep *ep = arg;
@@ -1674,6 +1670,7 @@ iwch_so_upcall(struct socket *so, void *arg, int waitflag)
 		taskqueue_enqueue(iw_cxgb_taskq, &iw_cxgb_task);
 	}
 	mtx_unlock(&req_lock);
+	return (SU_OK);
 }
 
 static void