- make 9k clusters the default unless a tunable is set

- return the error from cxgb_tx_common so that when an error is hit we dont
  spin forever in the taskq thread
- remove unused rxsd_ref
- simplify header_offset calculation for embedded mbuf headers
- fix memory leak by making sure that mbuf header initialization took place
- disable printf's for stalled queue, don't do offload/ctrl queue restart
  when tunnel queue is restarted
- add more diagnostic information about the txq state
- add facility to dump the actual contents of the hardware queue using sysctl

4 files changed, 224 insertions, 125 deletions

diff --git a/sys/dev/cxgb/common/cxgb_t3_cpl.h b/sys/dev/cxgb/common/cxgb_t3_cpl.h
index 672823c..1f0eb3f 100644
--- a/sys/dev/cxgb/common/cxgb_t3_cpl.h
+++ b/sys/dev/cxgb/common/cxgb_t3_cpl.h
@@ -306,6 +306,7 @@ struct work_request_hdr {
 #define S_WR_GEN    31
 #define V_WR_GEN(x) ((x) << S_WR_GEN)
 #define F_WR_GEN    V_WR_GEN(1U)
+#define G_WR_GEN(x) ((x) >> S_WR_GEN)
 
 # define WR_HDR struct work_request_hdr wr
 # define RSS_HDR
diff --git a/sys/dev/cxgb/cxgb_main.c b/sys/dev/cxgb/cxgb_main.c
index 810d843..f1a1dd7 100644
--- a/sys/dev/cxgb/cxgb_main.c
+++ b/sys/dev/cxgb/cxgb_main.c
@@ -218,6 +218,13 @@ TUNABLE_INT("hw.cxgb.singleq", &singleq);
 SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0,
     "use a single queue-set per port");
 
+
+
+int cxgb_use_16k_clusters = 0;
+TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters);
+SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
+    &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
+
 #ifndef IFNET_MULTIQUEUE
 int cxgb_txq_buf_ring_size = 0;
 #endif
@@ -1921,6 +1928,7 @@ cxgb_tx_common(struct ifnet *ifp, struct sge_qset *qs, uint32_t txmax)
 	}
 	else if ((err == 0) &&  (txq->size <= txq->in_use + TX_MAX_DESC) &&
 	    (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
+		setbit(&qs->txq_stopped, TXQ_ETH);
 		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 		err = ENOSPC;
 	}
@@ -1953,20 +1961,17 @@ cxgb_start_tx(struct ifnet *ifp, uint32_t txmax)
 	if (!p->link_config.link_ok)
 		return (ENXIO);
 
-	if (IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
+	if (IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 		return (ENOBUFS);
-	}
 	
 	qs = &p->adapter->sge.qs[p->first_qset];
 	txq = &qs->txq[TXQ_ETH];
-	err = 0;
-
 	if (txq->flags & TXQ_TRANSMITTING)
 		return (EINPROGRESS);
 
 	mtx_lock(&txq->lock);
 	txq->flags |= TXQ_TRANSMITTING;
-	cxgb_tx_common(ifp, qs, txmax);
+	err = cxgb_tx_common(ifp, qs, txmax);
 	txq->flags &= ~TXQ_TRANSMITTING;
 	mtx_unlock(&txq->lock);
 
diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c
index a079679..ab242ce 100644
--- a/sys/dev/cxgb/cxgb_sge.c
+++ b/sys/dev/cxgb/cxgb_sge.c
@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/taskqueue.h>
 
 #include <sys/proc.h>
+#include <sys/sbuf.h>
 #include <sys/sched.h>
 #include <sys/smp.h>
 #include <sys/systm.h>
@@ -71,13 +72,13 @@ __FBSDID("$FreeBSD$");
 
 int      txq_fills = 0;
 static int bogus_imm = 0;
-#ifndef DISABLE_MBUF_IOVEC
 static int recycle_enable = 1;
-#endif
 extern int cxgb_txq_buf_ring_size;
 int cxgb_cached_allocations;
 int cxgb_cached;
 int cxgb_ext_freed;
+extern int cxgb_use_16k_clusters;
+
 
 #define USE_GTS 0
 
@@ -141,7 +142,6 @@ struct tx_sw_desc {                /* SW state per Tx descriptor */
 
 struct rx_sw_desc {                /* SW state per Rx descriptor */
 	caddr_t	         rxsd_cl;
-	uint32_t         *rxsd_ref;
 	caddr_t	         data;
 	bus_dmamap_t	  map;
 	int		  flags;
@@ -296,16 +296,11 @@ sgl_len(unsigned int n)
 static __inline int
 get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct t3_mbuf_hdr *mh)
 {
-		return (0);
-	
-	m = m_gethdr(M_DONTWAIT, MT_DATA);
-	len = IMMED_PKT_SIZE; 
-	
-	if (m) {
-		memcpy(m->m_data, resp->imm_data, IMMED_PKT_SIZE);
-		m->m_pkthdr.len = m->m_len = len;		
-	}
-	return (m != NULL);
+	struct mbuf *m = mh->m_head;
+		
+	memcpy(mtod(m, uint8_t *), resp->imm_data, IMMED_PKT_SIZE);
+	m->m_pkthdr.len = m->m_len = len;
+	return (0);
 }
 
 #else
@@ -356,7 +351,7 @@ t3_sge_prep(adapter_t *adap, struct sge_params *p)
 	int i;
 
 	/* XXX Does ETHER_ALIGN need to be accounted for here? */
-	p->max_pkt_size = MJUM16BYTES - sizeof(struct cpl_rx_data);
+	p->max_pkt_size = adap->sge.qs[0].fl[1].buf_size - sizeof(struct cpl_rx_data);
 
 	for (i = 0; i < SGE_QSETS; ++i) {
 		struct qset_params *q = p->qset + i;
@@ -367,7 +362,7 @@ t3_sge_prep(adapter_t *adap, struct sge_params *p)
 			q->coalesce_nsecs = 50000;
 		} else {
 #ifdef INVARIANTS			
-			q->coalesce_nsecs = 20000;
+			q->coalesce_nsecs = 10000;
 #else
 			q->coalesce_nsecs = 5000;
 #endif			
@@ -415,8 +410,8 @@ t3_sge_alloc(adapter_t *sc)
 	/* 
 	 * DMA tag for jumbo sized RX frames.
 	 */
-	if (bus_dma_tag_create(sc->parent_dmat, MJUMPAGESIZE, 0, BUS_SPACE_MAXADDR,
-		BUS_SPACE_MAXADDR, NULL, NULL, MJUMPAGESIZE, 1, MJUMPAGESIZE,
+	if (bus_dma_tag_create(sc->parent_dmat, MJUM16BYTES, 0, BUS_SPACE_MAXADDR,
+		BUS_SPACE_MAXADDR, NULL, NULL, MJUM16BYTES, 1, MJUM16BYTES,
 		BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_jumbo_dmat)) {
 		device_printf(sc->dev, "Cannot allocate RX jumbo DMA tag\n");
 		return (ENOMEM);
@@ -492,7 +487,8 @@ refill_fl(adapter_t *sc, struct sge_fl *q, int n)
 	struct refill_fl_cb_arg cb_arg;
 	caddr_t cl;
 	int err;
-
+	int header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t);
+	
 	cb_arg.error = 0;
 	while (n--) {
 		/*
@@ -513,7 +509,7 @@ refill_fl(adapter_t *sc, struct sge_fl *q, int n)
 		}
 #if !defined(__i386__) && !defined(__amd64__)
 		err = bus_dmamap_load(q->entry_tag, sd->map,
-		    cl + sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t), q->buf_size,
+		    cl + header_size, q->buf_size,
 		    refill_fl_cb, &cb_arg, 0);
 		
 		if (err != 0 || cb_arg.error) {
@@ -524,13 +520,11 @@ refill_fl(adapter_t *sc, struct sge_fl *q, int n)
 			return;
 		}
 #else
-		cb_arg.seg.ds_addr = pmap_kextract((vm_offset_t)(cl + sizeof(struct m_hdr) +
-			sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t)));
+		cb_arg.seg.ds_addr = pmap_kextract((vm_offset_t)(cl + header_size));
 #endif		
 		sd->flags |= RX_SW_DESC_INUSE;
 		sd->rxsd_cl = cl;
-		sd->rxsd_ref = (uint32_t *)(cl + sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_));
-		sd->data = cl + sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t);
+		sd->data = cl + header_size;
 		d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff);
 		d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff);
 		d->len_gen = htobe32(V_FLD_GEN1(q->gen));
@@ -599,7 +593,6 @@ refill_fl_service(adapter_t *adap, struct sge_fl *fl)
 	__refill_fl_lt(adap, fl, 512);
 }
 
-#ifndef DISABLE_MBUF_IOVEC
 /**
  *	recycle_rx_buf - recycle a receive buffer
  *	@adapter: the adapter
@@ -629,7 +622,6 @@ recycle_rx_buf(adapter_t *adap, struct sge_fl *q, unsigned int idx)
 	}
 	t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
 }
-#endif
 
 static void
 alloc_ring_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
@@ -838,8 +830,11 @@ reclaim_more:
 	}
 	if (pi && pi->ifp->if_drv_flags & IFF_DRV_OACTIVE &&
 	    txq->size - txq->in_use >= TX_START_MAX_DESC) {
+		struct sge_qset *qs = txq_to_qset(txq, TXQ_ETH);
+
 		txq_fills++;
 		pi->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+		clrbit(&qs->txq_stopped, TXQ_ETH);		
 		taskqueue_enqueue(pi->tq, &pi->start_task);
 	}
 
@@ -1052,8 +1047,10 @@ make_sgl(struct sg_ent *sgp, bus_dma_segment_t *segs, int nsegs)
 		idx ^= 1;
 	}
 	
-	if (idx)
+	if (idx) {
 		sgp->len[idx] = 0;
+		sgp->addr[idx] = 0;
+	}
 }
 	
 /**
@@ -1263,6 +1260,7 @@ t3_encap(struct sge_qset *qs, struct mbuf **m, int count)
 #endif
 	
 	if (count > 1) {
+		panic("count > 1 not support in CVS\n");
 		if ((err = busdma_map_sg_vec(m, &m0, segs, count)))
 			return (err);
 		nsegs = count;
@@ -1803,7 +1801,6 @@ t3_sge_stop(adapter_t *sc)
 	}
 }
 
-
 /**
  *	t3_free_tx_desc - reclaims Tx descriptors and their buffers
  *	@adapter: the adapter
@@ -1842,7 +1839,8 @@ t3_free_tx_desc(struct sge_txq *q, int reclaimable)
 
 #if defined(DIAGNOSTIC) && 0
 			if (m_get_priority(txsd->m[0]) != cidx) 
-				printf("pri=%d cidx=%d\n", (int)m_get_priority(txsd->m[0]), cidx);
+				printf("pri=%d cidx=%d\n",
+				    (int)m_get_priority(txsd->m[0]), cidx);
 #endif			
 
 		} else
@@ -1948,8 +1946,6 @@ write_ofld_wr(adapter_t *adap, struct mbuf *m,
 	    from->wr_hi, from->wr_lo);
 }
 
-
-
 /**
  *	calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet
  *	@m: the packet
@@ -1969,7 +1965,8 @@ calc_tx_descs_ofld(struct mbuf *m, unsigned int nsegs)
 	if (m->m_flags & M_IOVEC)
 		cnt = mtomv(m)->mv_count;
 
-	flits = ((uint8_t *)m->m_pkthdr.header - mtod(m, uint8_t *)) / 8;   /* headers */
+	/* headers */
+	flits = ((uint8_t *)m->m_pkthdr.header - mtod(m, uint8_t *)) / 8;
 
 	return flits_to_desc(flits + sgl_len(cnt));
 }
@@ -2093,12 +2090,6 @@ again:	cleaned = reclaim_completed_tx(q);
 #endif
 	t3_write_reg(adap, A_SG_KDOORBELL,
 		     F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
-#if 0
-	
-	for (i = 0; i < cleaned; i++) {
-		m_freem_vec(m_vec[i]);
-	}
-#endif
 }
 
 /**
@@ -2195,10 +2186,10 @@ restart_tx(struct sge_qset *qs)
 	    should_restart_tx(&qs->txq[TXQ_OFLD]) &&
 	    test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
 		qs->txq[TXQ_OFLD].restarts++;
-		printf("restarting TXQ_OFLD\n");
+		DPRINTF("restarting TXQ_OFLD\n");
 		taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_task);
 	}
-	printf("stopped=0x%x restart=%d processed=%d cleaned=%d in_use=%d\n",
+	DPRINTF("stopped=0x%x restart=%d processed=%d cleaned=%d in_use=%d\n",
 	    qs->txq_stopped, should_restart_tx(&qs->txq[TXQ_CTRL]),
 	    qs->txq[TXQ_CTRL].processed, qs->txq[TXQ_CTRL].cleaned,
 	    qs->txq[TXQ_CTRL].in_use);
@@ -2207,7 +2198,7 @@ restart_tx(struct sge_qset *qs)
 	    should_restart_tx(&qs->txq[TXQ_CTRL]) &&
 	    test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
 		qs->txq[TXQ_CTRL].restarts++;
-		printf("restarting TXQ_CTRL\n");
+		DPRINTF("restarting TXQ_CTRL\n");
 		taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_task);
 	}
 }
@@ -2232,7 +2223,7 @@ t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
 		  const struct qset_params *p, int ntxq, struct port_info *pi)
 {
 	struct sge_qset *q = &sc->sge.qs[id];
-	int i, ret = 0;
+	int i, header_size, ret = 0;
 
 	for (i = 0; i < SGE_TXQ_PER_SET; i++) {
 		if ((q->txq[i].txq_mr.br_ring = malloc(cxgb_txq_buf_ring_size*sizeof(struct mbuf *),
@@ -2313,20 +2304,28 @@ t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
 	q->rspq.cidx = 0;
 	q->rspq.size = p->rspq_size;
 
+
+	header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t);
 	q->txq[TXQ_ETH].stop_thres = nports *
 	    flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3);
 
-	q->fl[0].buf_size = (MCLBYTES - sizeof(uint32_t) - sizeof(struct m_hdr) - sizeof(struct pkthdr) - sizeof(struct m_ext_));
+	q->fl[0].buf_size = (MCLBYTES - header_size);
 	q->fl[0].zone = zone_clust;
 	q->fl[0].type = EXT_CLUSTER;
 #if __FreeBSD_version > 800000
-		q->fl[1].buf_size = MJUM16BYTES - sizeof(uint32_t) - sizeof(struct m_hdr) - sizeof(struct pkthdr) - sizeof(struct m_ext_);
+	if (cxgb_use_16k_clusters) {		
+		q->fl[1].buf_size = MJUM16BYTES - header_size;
 		q->fl[1].zone = zone_jumbo16;
 		q->fl[1].type = EXT_JUMBO16;
+	} else {
+		q->fl[1].buf_size = MJUM9BYTES - header_size;
+		q->fl[1].zone = zone_jumbo9;
+		q->fl[1].type = EXT_JUMBO9;		
+	}
 #else
-		q->fl[1].buf_size = MJUMPAGESIZE - sizeof(uint32_t) - sizeof(struct m_hdr) - sizeof(struct pkthdr) - sizeof(struct m_ext_);
-		q->fl[1].zone = zone_jumbop;
-		q->fl[1].type = EXT_JUMBOP;
+	q->fl[1].buf_size = MJUMPAGESIZE - header_size;
+	q->fl[1].zone = zone_jumbop;
+	q->fl[1].type = EXT_JUMBOP;
 #endif
 	q->lro.enabled = lro_default;
 
@@ -2448,6 +2447,46 @@ t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad)
 	(*ifp->if_input)(ifp, m);
 }
 
+static void
+ext_free_handler(void *cl, void * arg)
+{
+	uintptr_t type = (uintptr_t)arg;
+	uma_zone_t zone;
+	struct mbuf *m;
+
+	m = cl;
+	zone = m_getzonefromtype(type);
+	m->m_ext.ext_type = (int)type;
+	cxgb_ext_freed++;
+	cxgb_cache_put(zone, cl);
+}
+
+static void
+init_cluster_mbuf(caddr_t cl, int flags, int type, uma_zone_t zone)
+{
+	struct mbuf *m;
+	int header_size;
+	
+	header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t);
+	
+	bzero(cl, header_size);
+	m = (struct mbuf *)cl;
+
+	SLIST_INIT(&m->m_pkthdr.tags);
+	m->m_type = MT_DATA;
+	m->m_flags = flags | M_NOFREE | M_EXT;
+	m->m_data = cl + header_size;
+	m->m_ext.ext_buf = cl;
+	m->m_ext.ref_cnt = (uint32_t *)(cl + header_size - sizeof(uint32_t));
+	m->m_ext.ext_size = m_getsizefromtype(type);
+	m->m_ext.ext_free = ext_free_handler;
+	m->m_ext.ext_args = (void *)(uintptr_t)type;
+	m->m_ext.ext_type = EXT_EXTREF;
+	*(m->m_ext.ref_cnt) = 1;
+	DPRINTF("data=%p ref_cnt=%p\n", m->m_data, m->m_ext.ref_cnt); 
+}
+
+
 /**
  *	get_packet - return the next ingress packet buffer from a free list
  *	@adap: the adapter that received the packet
@@ -2484,28 +2523,37 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
 
 	fl->credits--;
 	bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD);
-	bus_dmamap_unload(fl->entry_tag, sd->map);
+	
+	if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) {
+		if ((m0 = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL)
+			goto skip_recycle;
+		cl = mtod(m0, void *);
+		memcpy(cl, sd->data, len);
+		recycle_rx_buf(adap, fl, fl->cidx);
+		*m = m0;
+	} else {
+	skip_recycle:
+		int flags = 0;
+		bus_dmamap_unload(fl->entry_tag, sd->map);
+		cl = sd->rxsd_cl;
+		*m = m0 = (struct mbuf *)cl;
 
-	ref = sd->rxsd_ref;
-	m_cljset(m, sd->rxsd_cl, fl->type, sd->rxsd_ref);
-	*ref = 1;
-	m->m_len = len;
-	/*
-	 * bump past the refcnt address
-	 */
-	m->m_data = sd->data;
+		m0->m_len = len;
+		if ((sopeop == RSPQ_SOP_EOP) ||
+		    (sopeop == RSPQ_SOP))
+			flags = M_PKTHDR;
+		init_cluster_mbuf(cl, flags, fl->type, fl->zone);
+	}		
 	
 	switch(sopeop) {
 	case RSPQ_SOP_EOP:
 		DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m));
 		mh->mh_head = mh->mh_tail = m;
 		m->m_pkthdr.len = len;
-		m->m_flags |= M_PKTHDR;
 		ret = 1;
 		break;
 	case RSPQ_NSOP_NEOP:
 		DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m));
-		m->m_flags &= ~M_PKTHDR;
 		if (mh->mh_tail == NULL) {
 			if (cxgb_debug)
 				printf("discarding intermediate descriptor entry\n");
@@ -2521,12 +2569,10 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
 		DBG(DBG_RX, ("get_packet: SOP m %p\n", m));
 		m->m_pkthdr.len = len;
 		mh->mh_head = mh->mh_tail = m;
-		m->m_flags |= M_PKTHDR;
 		ret = 0;
 		break;
 	case RSPQ_EOP:
 		DBG(DBG_RX, ("get_packet: EOP m %p\n", m));
-		m->m_flags &= ~M_PKTHDR;
 		mh->mh_head->m_pkthdr.len += len;
 		mh->mh_tail->m_next = m;
 		mh->mh_tail = m;
@@ -2540,44 +2586,6 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
 }
 
 #else
-static void
-ext_free_handler(void *cl, void * arg)
-{
-	uintptr_t type = (uintptr_t)arg;
-	uma_zone_t zone;
-	struct mbuf *m;
-
-	m = cl;
-	zone = m_getzonefromtype(type);
-	m->m_ext.ext_type = (int)type;
-	cxgb_ext_freed++;
-	cxgb_cache_put(zone, cl);
-}
-
-static void
-init_cluster_mbuf(caddr_t cl, int flags, int type, uma_zone_t zone)
-{
-	struct mbuf *m;
-	int header_size;
-	
-	header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t);
-	
-	bzero(cl, header_size);
-	m = (struct mbuf *)cl;
-
-	SLIST_INIT(&m->m_pkthdr.tags);
-	m->m_type = MT_DATA;
-	m->m_flags = flags | M_NOFREE | M_EXT;
-	m->m_data = cl + header_size;
-	m->m_ext.ext_buf = cl;
-	m->m_ext.ref_cnt = (uint32_t *)(cl + header_size - sizeof(uint32_t));
-	m->m_ext.ext_size = m_getsizefromtype(type);
-	m->m_ext.ext_free = ext_free_handler;
-	m->m_ext.ext_args = (void *)(uintptr_t)type;
-	m->m_ext.ext_type = EXT_EXTREF;
-	*(m->m_ext.ref_cnt) = 1;
-	DPRINTF("data=%p ref_cnt=%p\n", m->m_data, m->m_ext.ref_cnt); 
-}
 
 static int
 get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
@@ -2673,14 +2681,8 @@ handle_rsp_cntrl_info(struct sge_qset *qs, uint32_t flags)
 		clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);
 #endif
 	credits = G_RSPD_TXQ0_CR(flags);
-	if (credits) {
+	if (credits) 
 		qs->txq[TXQ_ETH].processed += credits;
-#ifndef	IFNET_MULTIQUEUE
-		if (desc_reclaimable(&qs->txq[TXQ_ETH]) > TX_START_MAX_DESC)
-			taskqueue_enqueue(qs->port->adapter->tq,
-			    &qs->port->timer_reclaim_task);
-#endif		
-	}
 	
 	credits = G_RSPD_TXQ2_CR(flags);
 	if (credits) 
@@ -2750,21 +2752,24 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 			printf("async notification\n");
 
 		} else if  (flags & F_RSPD_IMM_DATA_VALID) {
+			struct mbuf *m = NULL;
+			
 #ifdef DISABLE_MBUF_IOVEC
-
-			DPRINTF("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", r->rss_hdr.opcode, rspq->cidx);
-
-			if(get_imm_packet(adap, r, &rspq->rspq_mh) == 0) {
+			DPRINTF("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n",
+			    r->rss_hdr.opcode, rspq->cidx);
+			
+			m = rspq->rspq_mh.mh_head = m_gethdr(M_DONTWAIT, MT_DATA);
+			if (m == NULL) {
 				rspq->next_holdoff = NOMEM_INTR_DELAY;
 				budget_left--;
 				break;
-			} else {
-				eop = 1;
 			}
+
+			get_imm_packet(adap, r, &rspq->rspq_mh);
+			eop = 1;
 #else
-			struct mbuf *m = NULL;
-			
-			DPRINTF("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", r->rss_hdr.opcode, rspq->cidx);
+			DPRINTF("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n",
+			    r->rss_hdr.opcode, rspq->cidx);
 			if (rspq->rspq_mbuf == NULL)
 				rspq->rspq_mbuf = m_gethdr(M_DONTWAIT, MT_DATA);
                         else 
@@ -2809,6 +2814,7 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 			DPRINTF("pure response\n");
 			rspq->pure_rsps++;
 		}
+		
 		if (flags & RSPD_CTRL_MASK) {
 			sleeping |= flags & RSPD_GTS_MASK;
 			handle_rsp_cntrl_info(qs, flags);
@@ -2861,7 +2867,7 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 		check_ring_db(adap, qs, sleeping);
 
 	smp_mb();  /* commit Tx queue processed updates */
-	if (__predict_false(qs->txq_stopped != 0)) {
+	if (__predict_false(qs->txq_stopped > 1)) {
 		printf("restarting tx on %p\n", qs);
 		
 		restart_tx(qs);
@@ -2962,10 +2968,74 @@ t3_intr_msix(void *data)
 	}
 }
 
+#define QDUMP_SBUF_SIZE		32 * 400
 /* 
  * broken by recent mbuf changes 
  */ 
 static int
+t3_dump_queue(SYSCTL_HANDLER_ARGS)
+{
+	struct sge_txq *txq;
+	struct sge_qset *qs;
+	int i, j, err, dump_end;
+	static int multiplier = 1;
+	struct sbuf *sb;
+	struct tx_desc *txd;
+	uint32_t *WR, wr_hi, wr_lo, gen;
+	
+	txq = arg1;
+	qs = txq_to_qset(txq, TXQ_ETH);
+	if (txq->txq_dump_count == 0) {
+		return (0);
+	}
+	if (txq->txq_dump_count > TX_ETH_Q_SIZE) {
+		log(LOG_WARNING,
+		    "dump count is too large %d\n", txq->txq_dump_count);
+		txq->txq_dump_count = 1;
+		return (EINVAL);
+	}
+	if (txq->txq_dump_start > (TX_ETH_Q_SIZE-1)) {
+		log(LOG_WARNING,
+		    "dump start of %d is greater than queue size\n",
+		    txq->txq_dump_start);
+		txq->txq_dump_start = 0;
+		return (EINVAL);
+	}
+retry_sbufops:
+	sb = sbuf_new(NULL, NULL, QDUMP_SBUF_SIZE*multiplier, SBUF_FIXEDLEN);
+	
+	sbuf_printf(sb, "\n qid=%d start=%d -> end=%d\n", qs->idx,
+	    txq->txq_dump_start,
+	    (txq->txq_dump_start + txq->txq_dump_count) & (TX_ETH_Q_SIZE-1));
+
+	dump_end = txq->txq_dump_start + txq->txq_dump_count;
+	for (i = txq->txq_dump_start; i < dump_end; i++) {
+		txd = &txq->desc[i & (TX_ETH_Q_SIZE-1)];
+		WR = (uint32_t *)txd->flit;
+		wr_hi = ntohl(WR[0]);
+		wr_lo = ntohl(WR[1]);		
+		gen = G_WR_GEN(wr_lo);
+		
+		sbuf_printf(sb," wr_hi %08x wr_lo %08x gen %d\n",
+		    wr_hi, wr_lo, gen);
+		for (j = 2; j < 30; j += 4) 
+			sbuf_printf(sb, "\t%08x %08x %08x %08x \n",
+			    WR[j], WR[j + 1], WR[j + 2], WR[j + 3]);
+
+	}
+	if (sbuf_overflowed(sb)) {
+		sbuf_delete(sb);
+		multiplier++;
+		goto retry_sbufops;
+	}
+	sbuf_finish(sb);
+	err = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1);
+	sbuf_delete(sb);
+	return (err);
+}
+
+
+static int
 t3_lro_enable(SYSCTL_HANDLER_ARGS)
 {
 	adapter_t *sc;
@@ -2974,7 +3044,6 @@ t3_lro_enable(SYSCTL_HANDLER_ARGS)
 #ifndef LRO_WORKING
 	return (0);
 #endif	
-	
 	sc = arg1;
 	enabled = sc->sge.qs[0].lro.enabled;
         err = sysctl_handle_int(oidp, &enabled, arg2, req);
@@ -3088,8 +3157,7 @@ t3_add_attach_sysctls(adapter_t *sc)
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, 
 	    "ext_freed",
 	    CTLFLAG_RD, &cxgb_ext_freed,
-	    0, "#times a cluster was freed through ext_free");		
-
+	    0, "#times a cluster was freed through ext_free"); 
 }
 
 void
@@ -3169,7 +3237,27 @@ t3_add_configured_sysctls(adapter_t *sc)
 			SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "stopped_flags",
 			    CTLFLAG_RD, &qs->txq_stopped,
 			    0, "tx queues stopped");
-			
+			SYSCTL_ADD_XLONG(ctx, qspoidlist, OID_AUTO, "phys_addr",
+			    CTLFLAG_RD, &txq->phys_addr,
+			    "physical_address_of the queue");
+			SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "qgen",
+			    CTLFLAG_RW, &qs->txq[TXQ_ETH].gen,
+			    0, "txq generation");
+			SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "hw_cidx",
+			    CTLFLAG_RD, &txq->cidx,
+			    0, "hardware queue cidx");			
+			SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "hw_pidx",
+			    CTLFLAG_RD, &txq->pidx,
+			    0, "hardware queue pidx");
+			SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "dump_start",
+			    CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_start,
+			    0, "txq start idx for dump");
+			SYSCTL_ADD_UINT(ctx, qspoidlist, OID_AUTO, "dump_count",
+			    CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_count,
+			    0, "txq #entries to dump");			
+			SYSCTL_ADD_PROC(ctx, qspoidlist, OID_AUTO, "qdump",
+			    CTLTYPE_STRING | CTLFLAG_RD, &qs->txq[TXQ_ETH],
+			    0, t3_dump_queue, "A", "dump of the transmit queue");
 		}
 	}
 }
diff --git a/sys/dev/cxgb/sys/cxgb_support.c b/sys/dev/cxgb/sys/cxgb_support.c
index 176206c..757a34f 100644
--- a/sys/dev/cxgb/sys/cxgb_support.c
+++ b/sys/dev/cxgb/sys/cxgb_support.c
@@ -52,6 +52,8 @@ __FBSDID("$FreeBSD$");
 #include <dev/cxgb/sys/mvec.h>
 #endif
 
+extern int cxgb_use_16k_clusters;
+
 struct buf_stack {
 	caddr_t            *bs_stack;
 	volatile int        bs_head;
@@ -120,14 +122,17 @@ cxgb_cache_pcpu_init(struct cxgb_cache_pcpu *ccp)
 {
 	int err;
 	
-	if ((err = buf_stack_init(&ccp->ccp_jumbo_free, (JUMBO_Q_SIZE >> 1))))
+	if ((err = buf_stack_init(&ccp->ccp_jumbo_free, (JUMBO_Q_SIZE >> 2))))
 		return (err);
 	
-	if ((err = buf_stack_init(&ccp->ccp_cluster_free, (FL_Q_SIZE >> 1))))
+	if ((err = buf_stack_init(&ccp->ccp_cluster_free, (FL_Q_SIZE >> 2))))
 		return (err);
 
 #if __FreeBSD_version > 800000		
+	if (cxgb_use_16k_clusters) 
 		ccp->ccp_jumbo_zone = zone_jumbo16;
+	else
+		ccp->ccp_jumbo_zone = zone_jumbo9;
 #else
 		ccp->ccp_jumbo_zone = zone_jumbop;
 #endif