Create a taskqueue for each wireless interface which provides a serialised

sleepable context for net80211 driver callbacks. This removes the need for USB
and firmware based drivers to roll their own code to defer the chip programming
for state changes, scan requests, channel changes and mcast/promisc updates.
When a driver callback completes the hardware state is now guaranteed to have
been updated and is in sync with net80211 layer.

This nukes around 1300 lines of code from the wireless device drivers making
them more readable and less race prone.

The net80211 layer has been updated as follows
 - all state/channel changes are serialised on the taskqueue.
 - ieee80211_new_state() always queues and can now be called from any context
 - scanning runs from a single taskq function and executes to completion. driver
   callbacks are synchronous so the channel, phy mode and rx filters are
   guaranteed to be set in hardware before probe request frames are
   transmitted.

Help and contributions from Sam Leffler.

Reviewed by:	sam

1 files changed, 171 insertions, 44 deletions

diff --git a/sys/net80211/ieee80211_proto.c b/sys/net80211/ieee80211_proto.c
index 572580f..98888d8 100644
--- a/sys/net80211/ieee80211_proto.c
+++ b/sys/net80211/ieee80211_proto.c
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/systm.h>
-#include <sys/taskqueue.h>
 
 #include <sys/socket.h>
 #include <sys/sockio.h>
@@ -96,7 +95,13 @@ const char *ieee80211_wme_acnames[] = {
 	"WME_UPSD",
 };
 
+static void beacon_miss(void *, int);
+static void beacon_swmiss(void *, int);
 static void parent_updown(void *, int);
+static void update_mcast(void *, int);
+static void update_promisc(void *, int);
+static void update_channel(void *, int);
+static void ieee80211_newstate_cb(void *, int);
 static int ieee80211_new_state_locked(struct ieee80211vap *,
 	enum ieee80211_state, int);
 
@@ -131,6 +136,10 @@ ieee80211_proto_attach(struct ieee80211com *ic)
 	ic->ic_protmode = IEEE80211_PROT_CTSONLY;
 
 	TASK_INIT(&ic->ic_parent_task, 0, parent_updown, ifp);
+	TASK_INIT(&ic->ic_mcast_task, 0, update_mcast, ic);
+	TASK_INIT(&ic->ic_promisc_task, 0, update_promisc, ic);
+	TASK_INIT(&ic->ic_chan_task, 0, update_channel, ic);
+	TASK_INIT(&ic->ic_bmiss_task, 0, beacon_miss, ic);
 
 	ic->ic_wme.wme_hipri_switch_hysteresis =
 		AGGRESSIVE_MODE_SWITCH_HYSTERESIS;
@@ -176,6 +185,8 @@ ieee80211_proto_vattach(struct ieee80211vap *vap)
 	vap->iv_bmiss_max = IEEE80211_BMISS_MAX;
 	callout_init(&vap->iv_swbmiss, CALLOUT_MPSAFE);
 	callout_init(&vap->iv_mgtsend, CALLOUT_MPSAFE);
+	TASK_INIT(&vap->iv_nstate_task, 0, ieee80211_newstate_cb, vap);
+	TASK_INIT(&vap->iv_swbmiss_task, 0, beacon_swmiss, vap);
 	/*
 	 * Install default tx rate handling: no fixed rate, lowest
 	 * supported rate for mgmt and multicast frames.  Default
@@ -1072,6 +1083,32 @@ parent_updown(void *arg, int npending)
 	parent->if_ioctl(parent, SIOCSIFFLAGS, NULL);
 }
 
+static void
+update_mcast(void *arg, int npending)
+{
+	struct ieee80211com *ic = arg;
+	struct ifnet *parent = ic->ic_ifp;
+
+	ic->ic_update_mcast(parent);
+}
+
+static void
+update_promisc(void *arg, int npending)
+{
+	struct ieee80211com *ic = arg;
+	struct ifnet *parent = ic->ic_ifp;
+
+	ic->ic_update_promisc(parent);
+}
+
+static void
+update_channel(void *arg, int npending)
+{
+	struct ieee80211com *ic = arg;
+
+	ic->ic_set_channel(ic);
+}
+
 /*
  * Block until the parent is in a known state.  This is
  * used after any operations that dispatch a task (e.g.
@@ -1080,7 +1117,13 @@ parent_updown(void *arg, int npending)
 void
 ieee80211_waitfor_parent(struct ieee80211com *ic)
 {
-	taskqueue_drain(taskqueue_thread, &ic->ic_parent_task);
+	taskqueue_block(ic->ic_tq);
+	ieee80211_draintask(ic, &ic->ic_parent_task);
+	ieee80211_draintask(ic, &ic->ic_mcast_task);
+	ieee80211_draintask(ic, &ic->ic_promisc_task);
+	ieee80211_draintask(ic, &ic->ic_chan_task);
+	ieee80211_draintask(ic, &ic->ic_bmiss_task);
+	taskqueue_unblock(ic->ic_tq);
 }
 
 /*
@@ -1121,7 +1164,7 @@ ieee80211_start_locked(struct ieee80211vap *vap)
 			    IEEE80211_MSG_STATE | IEEE80211_MSG_DEBUG,
 			    "%s: up parent %s\n", __func__, parent->if_xname);
 			parent->if_flags |= IFF_UP;
-			taskqueue_enqueue(taskqueue_thread, &ic->ic_parent_task);
+			ieee80211_runtask(ic, &ic->ic_parent_task);
 			return;
 		}
 	}
@@ -1156,8 +1199,7 @@ ieee80211_start_locked(struct ieee80211vap *vap)
 			 * preempted if the station is locked to a particular
 			 * channel.
 			 */
-			/* XXX needed? */
-			ieee80211_new_state_locked(vap, IEEE80211_S_INIT, 0);
+			vap->iv_flags_ext |= IEEE80211_FEXT_REINIT;
 			if (vap->iv_opmode == IEEE80211_M_MONITOR ||
 			    vap->iv_opmode == IEEE80211_M_WDS)
 				ieee80211_new_state_locked(vap,
@@ -1240,7 +1282,7 @@ ieee80211_stop_locked(struct ieee80211vap *vap)
 			    IEEE80211_MSG_STATE | IEEE80211_MSG_DEBUG,
 			    "down parent %s\n", parent->if_xname);
 			parent->if_flags &= ~IFF_UP;
-			taskqueue_enqueue(taskqueue_thread, &ic->ic_parent_task);
+			ieee80211_runtask(ic, &ic->ic_parent_task);
 		}
 	}
 }
@@ -1319,10 +1361,20 @@ ieee80211_resume_all(struct ieee80211com *ic)
 void
 ieee80211_beacon_miss(struct ieee80211com *ic)
 {
+	IEEE80211_LOCK(ic);
+	if ((ic->ic_flags & IEEE80211_F_SCAN) == 0) {
+		/* Process in a taskq, the handler may reenter the driver */
+		ieee80211_runtask(ic, &ic->ic_bmiss_task);
+	}
+	IEEE80211_UNLOCK(ic);
+}
+
+static void
+beacon_miss(void *arg, int npending)
+{
+	struct ieee80211com *ic = arg;
 	struct ieee80211vap *vap;
 
-	if (ic->ic_flags & IEEE80211_F_SCAN)
-		return;
 	/* XXX locking */
 	TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
 		/*
@@ -1337,6 +1389,18 @@ ieee80211_beacon_miss(struct ieee80211com *ic)
 	}
 }
 
+static void
+beacon_swmiss(void *arg, int npending)
+{
+	struct ieee80211vap *vap = arg;
+
+	if (vap->iv_state != IEEE80211_S_RUN)
+		return;
+
+	/* XXX Call multiple times if npending > zero? */
+	vap->iv_bmiss(vap);
+}
+
 /*
  * Software beacon miss handling.  Check if any beacons
  * were received in the last period.  If not post a
@@ -1366,7 +1430,7 @@ ieee80211_swbmiss(void *arg)
 		vap->iv_swbmiss_count = 0;
 	} else if (vap->iv_swbmiss_count == 0) {
 		if (vap->iv_bmiss != NULL)
-			vap->iv_bmiss(vap);
+			ieee80211_runtask(ic, &vap->iv_swbmiss_task);
 		if (vap->iv_bmiss_count == 0)	/* don't re-arm timer */
 			return;
 	} else
@@ -1463,7 +1527,6 @@ ieee80211_cac_completeswitch(struct ieee80211vap *vap0)
  * and mark them as waiting for a scan to complete.  These vaps
  * will be brought up when the scan completes and the scanning vap
  * reaches RUN state by wakeupwaiting.
- * XXX if we do this in threads we can use sleep/wakeup.
  */
 static void
 markwaiting(struct ieee80211vap *vap0)
@@ -1473,10 +1536,16 @@ markwaiting(struct ieee80211vap *vap0)
 
 	IEEE80211_LOCK_ASSERT(ic);
 
+	/*
+	 * A vap list entry can not disappear since we are running on the
+	 * taskqueue and a vap destroy will queue and drain another state
+	 * change task.
+	 */
 	TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
 		if (vap == vap0)
 			continue;
 		if (vap->iv_state != IEEE80211_S_INIT) {
+			/* NB: iv_newstate may drop the lock */
 			vap->iv_newstate(vap, IEEE80211_S_INIT, 0);
 			vap->iv_flags_ext |= IEEE80211_FEXT_SCANWAIT;
 		}
@@ -1487,6 +1556,7 @@ markwaiting(struct ieee80211vap *vap0)
  * Wakeup all vap's waiting for a scan to complete.  This is the
  * companion to markwaiting (above) and is used to coordinate
  * multiple vaps scanning.
+ * This is called from the state taskqueue.
  */
 static void
 wakeupwaiting(struct ieee80211vap *vap0)
@@ -1496,12 +1566,18 @@ wakeupwaiting(struct ieee80211vap *vap0)
 
 	IEEE80211_LOCK_ASSERT(ic);
 
+	/*
+	 * A vap list entry can not disappear since we are running on the
+	 * taskqueue and a vap destroy will queue and drain another state
+	 * change task.
+	 */
 	TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
 		if (vap == vap0)
 			continue;
 		if (vap->iv_flags_ext & IEEE80211_FEXT_SCANWAIT) {
 			vap->iv_flags_ext &= ~IEEE80211_FEXT_SCANWAIT;
 			/* NB: sta's cannot go INIT->RUN */
+			/* NB: iv_newstate may drop the lock */
 			vap->iv_newstate(vap,
 			    vap->iv_opmode == IEEE80211_M_STA ?
 			        IEEE80211_S_SCAN : IEEE80211_S_RUN, 0);
@@ -1513,15 +1589,63 @@ wakeupwaiting(struct ieee80211vap *vap0)
  * Handle post state change work common to all operating modes.
  */
 static void
-ieee80211_newstate_cb(struct ieee80211vap *vap, 
-	enum ieee80211_state nstate, int arg)
+ieee80211_newstate_cb(void *xvap, int npending)
 {
+	struct ieee80211vap *vap = xvap;
 	struct ieee80211com *ic = vap->iv_ic;
+	enum ieee80211_state nstate, ostate;
+	int arg, rc;
 
-	IEEE80211_LOCK_ASSERT(ic);
+	IEEE80211_LOCK(ic);
+	nstate = vap->iv_nstate;
+	arg = vap->iv_nstate_arg;
 
+	if (vap->iv_flags_ext & IEEE80211_FEXT_REINIT) {
+		/*
+		 * We have been requested to drop back to the INIT before
+		 * proceeding to the new state.
+		 */
+		IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
+		    "%s: %s -> %s arg %d\n", __func__,
+		    ieee80211_state_name[vap->iv_state],
+		    ieee80211_state_name[IEEE80211_S_INIT], arg);
+		vap->iv_newstate(vap, IEEE80211_S_INIT, arg);
+		vap->iv_flags_ext &= ~IEEE80211_FEXT_REINIT;
+	}
+
+	ostate = vap->iv_state;
+	if (nstate == IEEE80211_S_SCAN && ostate != IEEE80211_S_INIT) {
+		/*
+		 * SCAN was forced; e.g. on beacon miss.  Force other running
+		 * vap's to INIT state and mark them as waiting for the scan to
+		 * complete.  This insures they don't interfere with our
+		 * scanning.  Since we are single threaded the vaps can not
+		 * transition again while we are executing.
+		 *
+		 * XXX not always right, assumes ap follows sta
+		 */
+		markwaiting(vap);
+	}
 	IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
-	    "%s: %s arg %d\n", __func__, ieee80211_state_name[nstate], arg);
+	    "%s: %s -> %s arg %d\n", __func__,
+	    ieee80211_state_name[ostate], ieee80211_state_name[nstate], arg);
+
+	rc = vap->iv_newstate(vap, nstate, arg);
+	vap->iv_flags_ext &= ~IEEE80211_FEXT_STATEWAIT;
+	if (rc != 0) {
+		/* State transition failed */
+		KASSERT(rc != EINPROGRESS, ("iv_newstate was deferred"));
+		KASSERT(nstate != IEEE80211_S_INIT,
+		    ("INIT state change failed"));
+		IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
+		    "%s: %s returned error %d\n", __func__,
+		    ieee80211_state_name[nstate], rc);
+		goto done;
+	}
+
+	/* No actual transition, skip post processing */
+	if (ostate == nstate)
+		goto done;
 
 	if (nstate == IEEE80211_S_RUN) {
 		/*
@@ -1549,7 +1673,8 @@ ieee80211_newstate_cb(struct ieee80211vap *vap,
 		/* XXX NB: cast for altq */
 		ieee80211_flush_ifq((struct ifqueue *)&ic->ic_ifp->if_snd, vap);
 	}
-	vap->iv_newstate_cb = NULL;
+done:
+	IEEE80211_UNLOCK(ic);
 }
 
 /*
@@ -1586,10 +1711,32 @@ ieee80211_new_state_locked(struct ieee80211vap *vap,
 	struct ieee80211com *ic = vap->iv_ic;
 	struct ieee80211vap *vp;
 	enum ieee80211_state ostate;
-	int nrunning, nscanning, rc;
+	int nrunning, nscanning;
 
 	IEEE80211_LOCK_ASSERT(ic);
 
+	if (vap->iv_flags_ext & IEEE80211_FEXT_STATEWAIT) {
+		if (vap->iv_nstate == IEEE80211_S_INIT) {
+			/*
+			 * XXX The vap is being stopped, do no allow any other
+			 * state changes until this is completed.
+			 */
+			return -1;
+		}
+#if 0
+		/* Warn if the previous state hasn't completed. */
+		IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE,
+		    "%s: pending %s -> %s transition lost\n", __func__,
+		    ieee80211_state_name[vap->iv_state],
+		    ieee80211_state_name[vap->iv_nstate]);
+#else
+		/* XXX temporarily enable to identify issues */
+		if_printf(vap->iv_ifp, "%s: pending %s -> %s transition lost\n",
+		    __func__, ieee80211_state_name[vap->iv_state],
+		    ieee80211_state_name[vap->iv_nstate]);
+#endif
+	}
+
 	nrunning = nscanning = 0;
 	/* XXX can track this state instead of calculating */
 	TAILQ_FOREACH(vp, &ic->ic_vaps, iv_next) {
@@ -1625,8 +1772,7 @@ ieee80211_new_state_locked(struct ieee80211vap *vap,
 				    __func__, ieee80211_state_name[ostate],
 				    ieee80211_state_name[nstate]);
 				vap->iv_flags_ext |= IEEE80211_FEXT_SCANWAIT;
-				rc = 0;
-				goto done;
+				return 0;
 			}
 			if (nrunning) {
 				/*
@@ -1650,16 +1796,6 @@ ieee80211_new_state_locked(struct ieee80211vap *vap,
 				}
 #endif
 			}
-		} else {
-			/*
-			 * SCAN was forced; e.g. on beacon miss.  Force
-			 * other running vap's to INIT state and mark
-			 * them as waiting for the scan to complete.  This
-			 * insures they don't interfere with our scanning.
-			 *
-			 * XXX not always right, assumes ap follows sta
-			 */
-			markwaiting(vap);
 		}
 		break;
 	case IEEE80211_S_RUN:
@@ -1676,8 +1812,7 @@ ieee80211_new_state_locked(struct ieee80211vap *vap,
 			     ieee80211_state_name[ostate],
 			     ieee80211_state_name[nstate]);
 			vap->iv_flags_ext |= IEEE80211_FEXT_SCANWAIT;
-			rc = 0;
-			goto done;
+			return 0;
 		}
 		if (vap->iv_opmode == IEEE80211_M_HOSTAP &&
 		    IEEE80211_IS_CHAN_DFS(ic->ic_bsschan) &&
@@ -1706,20 +1841,12 @@ ieee80211_new_state_locked(struct ieee80211vap *vap,
 	default:
 		break;
 	}
-	/* XXX on transition RUN->CAC do we need to set nstate = iv_state? */
-	if (ostate != nstate) {
-		/*
-		 * Arrange for work to happen after state change completes.
-		 * If this happens asynchronously the caller must arrange
-		 * for the com lock to be held.
-		 */
-		vap->iv_newstate_cb = ieee80211_newstate_cb;
-	}
-	rc = vap->iv_newstate(vap, nstate, arg);
-	if (rc == 0 && vap->iv_newstate_cb != NULL)
-		vap->iv_newstate_cb(vap, nstate, arg);
-done:
-	return rc;
+	/* defer the state change to a thread */
+	vap->iv_nstate = nstate;
+	vap->iv_nstate_arg = arg;
+	vap->iv_flags_ext |= IEEE80211_FEXT_STATEWAIT;
+	ieee80211_runtask(ic, &vap->iv_nstate_task);
+	return EINPROGRESS;
 }
 
 int