MFC r267600:

cxgbe(4):  Fix bug in the fast rx buffer recycle path.  In some cases rx
buffers were getting recycled when they should have been left alone.

2 files changed, 18 insertions, 10 deletions

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 9673834..a4f729c 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -254,7 +254,8 @@ struct cluster_metadata {
 
 struct fl_sdesc {
 	caddr_t cl;
-	uint8_t nmbuf;
+	uint8_t nimbuf;		/* # of inline mbufs with ref on the cluster */
+	uint8_t nembuf;		/* # of allocated mbufs with ref */
 	struct cluster_layout cll;
 };
 
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index a9d250a..f3e7b83 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -1498,22 +1498,22 @@ get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int total, int flags)
 		/* copy data to mbuf */
 		bcopy(payload, mtod(m, caddr_t), len);
 
-	} else if (sd->nmbuf * MSIZE < cll->region1) {
+	} else if (sd->nimbuf * MSIZE < cll->region1) {
 
 		/*
 		 * There's spare room in the cluster for an mbuf.  Create one
-		 * and associate it with the payload that's in the cluster too.
+		 * and associate it with the payload that's in the cluster.
 		 */
 
 		MPASS(clm != NULL);
-		m = (struct mbuf *)(sd->cl + sd->nmbuf * MSIZE);
+		m = (struct mbuf *)(sd->cl + sd->nimbuf * MSIZE);
 		/* No bzero required */
 		if (m_init(m, NULL, 0, M_NOWAIT, MT_DATA, flags | M_NOFREE))
 			return (NULL);
 		fl->mbuf_inlined++;
 		m_extaddref(m, payload, padded_len, &clm->refcount, rxb_free,
 		    swz->zone, sd->cl);
-		sd->nmbuf++;
+		sd->nimbuf++;
 
 	} else {
 
@@ -1527,10 +1527,11 @@ get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int total, int flags)
 		if (m == NULL)
 			return (NULL);
 		fl->mbuf_allocated++;
-		if (clm != NULL)
+		if (clm != NULL) {
 			m_extaddref(m, payload, padded_len, &clm->refcount,
 			    rxb_free, swz->zone, sd->cl);
-		else {
+			sd->nembuf++;
+		} else {
 			m_cljset(m, sd->cl, swz->type);
 			sd->cl = NULL;	/* consumed, not a recycle candidate */
 		}
@@ -3053,7 +3054,7 @@ refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
 
 		if (sd->cl != NULL) {
 
-			if (sd->nmbuf == 0) {
+			if (sd->nimbuf + sd->nembuf == 0) {
 				/*
 				 * Fast recycle without involving any atomics on
 				 * the cluster's metadata (if the cluster has
@@ -3062,6 +3063,11 @@ refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
 				 * fit within a single mbuf each.
 				 */
 				fl->cl_fast_recycled++;
+#ifdef INVARIANTS
+				clm = cl_metadata(sc, fl, &sd->cll, sd->cl);
+				if (clm != NULL)
+					MPASS(clm->refcount == 1);
+#endif
 				goto recycled_fast;
 			}
 
@@ -3107,7 +3113,8 @@ recycled:
 #endif
 			clm->refcount = 1;
 		}
-		sd->nmbuf = 0;
+		sd->nimbuf = 0;
+		sd->nembuf = 0;
 recycled_fast:
 		fl->pending++;
 		fl->needed--;
@@ -3176,7 +3183,7 @@ free_fl_sdesc(struct adapter *sc, struct sge_fl *fl)
 
 		cll = &sd->cll;
 		clm = cl_metadata(sc, fl, cll, sd->cl);
-		if (sd->nmbuf == 0 ||
+		if (sd->nimbuf + sd->nembuf == 0 ||
 		    (clm && atomic_fetchadd_int(&clm->refcount, -1) == 1)) {
 			uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl);
 		}