Flesh out some slightly dirty reset/channel change serialisation code

for the ath(4) driver.

Currently, there's nothing stopping reset, channel change and general
TX/RX from overlapping with each other. This wasn't a big deal with
pre-11n traffic as it just results in some dropped frames.
It's possible this may have also caused some inconsistencies and
badly-setup hardware.

Since locks can't be held across all of this (the Linux solution)
due to LORs with the network stack locks, some state counter
variables are used to track what parts of the code the driver is
currently in.

When the hardware is being reset, it disables the taskqueue and
waits for pending interrupts, tx, rx and tx completion before
it begins the reset or channel change.

TX and RX both abort if called during an active reset or channel
change.

Finally, the reset path now doesn't flush frames if ATH_RESET_NOLOSS
is set. Instead, completed TX and RX frames are passed back up to
net80211 before the reset occurs.

This is not without problems:

* Raw frame xmit are just dropped, rather than placed on a queue.
  The net80211 stack should be the one which queues these frames
  rather than the driver.

* It's all very messy. It'd be better if these hardware operations
  were serialised on some kind of work queue, rather than hoping
  they can be run in parallel.

* The taskqueue block/unblock may occur in parallel with the
  newstate() function - which shuts down the taskqueue and restarts
  it once the new state is known. It's likely these operations should
  be refcounted so the taskqueue is restored once no other areas
  in the code wish to suspend operations.

* .. interrupt disable/enable should likely be refcounted as well.

With this work, the driver does not drop frames during stuck beacon
or fatal errors and thus 11n traffic continues to run correctly.
Default and full resets however do still drop frames and it's possible
this may occur, causing traffic loss and session stalls.

Sponsored by:	Hobnob, Inc.

1 files changed, 50 insertions, 2 deletions

diff --git a/sys/dev/ath/if_ath_tx.c b/sys/dev/ath/if_ath_tx.c
index b8c4c80..5625bf2 100644
--- a/sys/dev/ath/if_ath_tx.c
+++ b/sys/dev/ath/if_ath_tx.c
@@ -477,8 +477,6 @@ ath_tx_handoff_mcast(struct ath_softc *sc, struct ath_txq *txq,
 	txq->axq_link = &bf->bf_lastds->ds_link;
 }
 
-
-
 /*
  * Hand-off packet to a hardware queue.
  */
@@ -501,6 +499,36 @@ ath_tx_handoff_hw(struct ath_softc *sc, struct ath_txq *txq, struct ath_buf *bf)
 	KASSERT(txq->axq_qnum != ATH_TXQ_SWQ,
 	     ("ath_tx_handoff_hw called for mcast queue"));
 
+#if 0
+	/*
+	 * This causes a LOR. Find out where the PCU lock is being
+	 * held whilst the TXQ lock is grabbed - that shouldn't
+	 * be occuring.
+	 */
+	ATH_PCU_LOCK(sc);
+	if (sc->sc_inreset_cnt) {
+		ATH_PCU_UNLOCK(sc);
+		DPRINTF(sc, ATH_DEBUG_RESET,
+		    "%s: called with sc_in_reset != 0\n",
+		    __func__);
+		DPRINTF(sc, ATH_DEBUG_XMIT,
+		    "%s: queued: TXDP[%u] = %p (%p) depth %d\n",
+		    __func__, txq->axq_qnum,
+		    (caddr_t)bf->bf_daddr, bf->bf_desc,
+		    txq->axq_depth);
+		ATH_TXQ_INSERT_TAIL(txq, bf, bf_list);
+		if (bf->bf_state.bfs_aggr)
+			txq->axq_aggr_depth++;
+		/*
+		 * There's no need to update axq_link; the hardware
+		 * is in reset and once the reset is complete, any
+		 * non-empty queues will simply have DMA restarted.
+		 */
+		return;
+		}
+	ATH_PCU_UNLOCK(sc);
+#endif
+
 	/* For now, so not to generate whitespace diffs */
 	if (1) {
 #ifdef IEEE80211_SUPPORT_TDMA
@@ -1688,6 +1716,17 @@ ath_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
 	struct ath_buf *bf;
 	int error;
 
+	ATH_PCU_LOCK(sc);
+	if (sc->sc_inreset_cnt > 0) {
+		device_printf(sc->sc_dev, "%s: sc_inreset_cnt > 0; bailing\n",
+		    __func__);
+		error = EIO;
+		ATH_PCU_UNLOCK(sc);
+		goto bad0;
+	}
+	sc->sc_txstart_cnt++;
+	ATH_PCU_UNLOCK(sc);
+
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid) {
 		DPRINTF(sc, ATH_DEBUG_XMIT, "%s: discard frame, %s", __func__,
 		    (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ?
@@ -1730,15 +1769,24 @@ ath_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
 	ifp->if_opackets++;
 	sc->sc_stats.ast_tx_raw++;
 
+	ATH_PCU_LOCK(sc);
+	sc->sc_txstart_cnt--;
+	ATH_PCU_UNLOCK(sc);
+
 	return 0;
 bad2:
 	ATH_TXBUF_LOCK(sc);
 	TAILQ_INSERT_HEAD(&sc->sc_txbuf, bf, bf_list);
 	ATH_TXBUF_UNLOCK(sc);
 bad:
+	ATH_PCU_LOCK(sc);
+	sc->sc_txstart_cnt--;
+	ATH_PCU_UNLOCK(sc);
+bad0:
 	ifp->if_oerrors++;
 	sc->sc_stats.ast_tx_raw_fail++;
 	ieee80211_free_node(ni);
+
 	return error;
 }