Implement support for rx buffer packing. Enable it by default for T5

cards.

This is a T4 and T5 chip feature which lets the chip deliver multiple
Ethernet frames in a single buffer.  This is more efficient within the
chip, in the driver, and reduces wastage of space in rx buffers.

- Always allocate rx buffers from the jumbop zone, no matter what the
  MTU is.  Do not use the normal cluster refcounting mechanism.
- Reserve space for an mbuf and a refcount in the cluster itself and let
  the chip DMA multiple frames in the rest.
- Use the embedded mbuf for the first frame and allocate mbufs on the
  fly for any additional frames delivered in the cluster.  Each of these
  mbufs has a reference on the underlying cluster.

2 files changed, 601 insertions, 142 deletions

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index ba956c4..89358d2 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -128,9 +128,9 @@ enum {
 
 	RX_FL_ESIZE = EQ_ESIZE,	/* 8 64bit addresses */
 #if MJUMPAGESIZE != MCLBYTES
-	FL_BUF_SIZES = 4,	/* cluster, jumbop, jumbo9k, jumbo16k */
+	FL_BUF_SIZES_MAX = 5,	/* cluster, jumbop, jumbo9k, jumbo16k, extra */
 #else
-	FL_BUF_SIZES = 3,	/* cluster, jumbo9k, jumbo16k */
+	FL_BUF_SIZES_MAX = 4,	/* cluster, jumbo9k, jumbo16k, extra */
 #endif
 
 	CTRL_EQ_QSIZE = 128,
@@ -165,6 +165,7 @@ enum {
 	MASTER_PF	= (1 << 3),
 	ADAP_SYSCTL_CTX	= (1 << 4),
 	TOM_INIT_DONE	= (1 << 5),
+	BUF_PACKING_OK	= (1 << 6),
 
 	CXGBE_BUSY	= (1 << 9),
 
@@ -232,12 +233,11 @@ struct port_info {
 };
 
 struct fl_sdesc {
-	struct mbuf *m;
 	bus_dmamap_t map;
 	caddr_t cl;
-	uint8_t tag_idx;	/* the sc->fl_tag this map comes from */
+	uint8_t tag_idx;	/* the fl->tag entry this map comes from */
 #ifdef INVARIANTS
-	__be64 ba_tag;
+	__be64 ba_hwtag;
 #endif
 };
 
@@ -359,9 +359,22 @@ struct sge_eq {
 	uint32_t unstalled;	/* recovered from stall */
 };
 
+struct fl_buf_info {
+	u_int size;
+	int type;
+	int hwtag:4;	/* tag in low 4 bits of the pa. */
+	uma_zone_t zone;
+};
+#define FL_BUF_SIZES(sc)	(sc->sge.fl_buf_sizes)
+#define FL_BUF_SIZE(sc, x)	(sc->sge.fl_buf_info[x].size)
+#define FL_BUF_TYPE(sc, x)	(sc->sge.fl_buf_info[x].type)
+#define FL_BUF_HWTAG(sc, x)	(sc->sge.fl_buf_info[x].hwtag)
+#define FL_BUF_ZONE(sc, x)	(sc->sge.fl_buf_info[x].zone)
+
 enum {
 	FL_STARVING	= (1 << 0), /* on the adapter's list of starving fl's */
 	FL_DOOMED	= (1 << 1), /* about to be destroyed */
+	FL_BUF_PACKING	= (1 << 2), /* buffer packing enabled */
 };
 
 #define FL_RUNNING_LOW(fl)	(fl->cap - fl->needed <= fl->lowat)
@@ -370,7 +383,8 @@ enum {
 struct sge_fl {
 	bus_dma_tag_t desc_tag;
 	bus_dmamap_t desc_map;
-	bus_dma_tag_t tag[FL_BUF_SIZES];
+	bus_dma_tag_t tag[FL_BUF_SIZES_MAX]; /* only first FL_BUF_SIZES(sc) are
+						valid */
 	uint8_t tag_idx;
 	struct mtx fl_lock;
 	char lockname[16];
@@ -383,11 +397,13 @@ struct sge_fl {
 	uint16_t qsize;		/* size (# of entries) of the queue */
 	uint16_t cntxt_id;	/* SGE context id for the freelist */
 	uint32_t cidx;		/* consumer idx (buffer idx, NOT hw desc idx) */
+	uint32_t rx_offset;	/* offset in fl buf (when buffer packing) */
 	uint32_t pidx;		/* producer idx (buffer idx, NOT hw desc idx) */
 	uint32_t needed;	/* # of buffers needed to fill up fl. */
 	uint32_t lowat;		/* # of buffers <= this means fl needs help */
 	uint32_t pending;	/* # of bufs allocated since last doorbell */
-	unsigned int dmamap_failed;
+	u_int dmamap_failed;
+	struct mbuf *mstash[8];
 	TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
 };
 
@@ -519,6 +535,9 @@ struct sge {
 	int eq_start;
 	struct sge_iq **iqmap;	/* iq->cntxt_id to iq mapping */
 	struct sge_eq **eqmap;	/* eq->cntxt_id to eq mapping */
+
+	u_int fl_buf_sizes __aligned(CACHE_LINE_SIZE);
+	struct fl_buf_info fl_buf_info[FL_BUF_SIZES_MAX];
 };
 
 struct rss_header;
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index 34bd27a..b46631e 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -56,19 +56,6 @@ __FBSDID("$FreeBSD$");
 #include "common/t4_regs_values.h"
 #include "common/t4_msg.h"
 
-struct fl_buf_info {
-	int size;
-	int type;
-	uma_zone_t zone;
-};
-
-/* Filled up by t4_sge_modload */
-static struct fl_buf_info fl_buf_info[FL_BUF_SIZES];
-
-#define FL_BUF_SIZE(x)	(fl_buf_info[x].size)
-#define FL_BUF_TYPE(x)	(fl_buf_info[x].type)
-#define FL_BUF_ZONE(x)	(fl_buf_info[x].zone)
-
 #ifdef T4_PKT_TIMESTAMP
 #define RX_COPY_THRESHOLD (MINCLSIZE - 8)
 #else
@@ -85,7 +72,8 @@ TUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift);
 /*
  * Pad ethernet payload up to this boundary.
  * -1: driver should figure out a good value.
- *  Any power of 2, from 32 to 4096 (both inclusive) is a valid value.
+ *  0: disable padding.
+ *  Any power of 2 from 32 to 4096 (both inclusive) is also a valid value.
  */
 static int fl_pad = -1;
 TUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad);
@@ -107,6 +95,33 @@ TUNABLE_INT("hw.cxgbe.spg_len", &spg_len);
 static int cong_drop = 0;
 TUNABLE_INT("hw.cxgbe.cong_drop", &cong_drop);
 
+/*
+ * Deliver multiple frames in the same free list buffer if they fit.
+ * -1: let the driver decide whether to enable buffer packing or not.
+ *  0: disable buffer packing.
+ *  1: enable buffer packing.
+ */
+static int buffer_packing = -1;
+TUNABLE_INT("hw.cxgbe.buffer_packing", &buffer_packing);
+
+/*
+ * Start next frame in a packed buffer at this boundary.
+ * -1: driver should figure out a good value.
+ * T4:
+ * ---
+ * if fl_pad != 0
+ * 	value specified here will be overridden by fl_pad.
+ * else
+ * 	power of 2 from 32 to 4096 (both inclusive) is a valid value here.
+ * T5:
+ * ---
+ * 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value.
+ */
+static int fl_pack = -1;
+static int t4_fl_pack;
+static int t5_fl_pack;
+TUNABLE_INT("hw.cxgbe.fl_pack", &fl_pack);
+
 /* Used to track coalesced tx work request */
 struct txpkts {
 	uint64_t *flitp;	/* ptr to flit where next pkt should start */
@@ -123,12 +138,15 @@ struct sgl {
 };
 
 static int service_iq(struct sge_iq *, int);
-static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t,
+static struct mbuf *get_fl_payload1(struct adapter *, struct sge_fl *, uint32_t,
+    int *);
+static struct mbuf *get_fl_payload2(struct adapter *, struct sge_fl *, uint32_t,
     int *);
 static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
 static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int,
     int);
-static inline void init_fl(struct sge_fl *, int, int, char *);
+static inline void init_fl(struct adapter *, struct sge_fl *, int, int, int,
+    char *);
 static inline void init_eq(struct sge_eq *, int, int, uint8_t, uint16_t,
     char *);
 static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *,
@@ -170,8 +188,8 @@ static inline void ring_fl_db(struct adapter *, struct sge_fl *);
 static int refill_fl(struct adapter *, struct sge_fl *, int);
 static void refill_sfl(void *);
 static int alloc_fl_sdesc(struct sge_fl *);
-static void free_fl_sdesc(struct sge_fl *);
-static void set_fl_tag_idx(struct sge_fl *, int);
+static void free_fl_sdesc(struct adapter *, struct sge_fl *);
+static void set_fl_tag_idx(struct adapter *, struct sge_fl *, int);
 static void add_fl_to_sfl(struct adapter *, struct sge_fl *);
 
 static int get_pkt_sgl(struct sge_txq *, struct mbuf **, struct sgl *, int);
@@ -202,27 +220,20 @@ extern u_int cpu_clflush_line_size;
 #endif
 
 /*
- * Called on MOD_LOAD.  Fills up fl_buf_info[] and validates/calculates the SGE
- * tunables.
+ * Called on MOD_LOAD.  Validates and calculates the SGE tunables.
  */
 void
 t4_sge_modload(void)
 {
-	int i;
-	int bufsize[FL_BUF_SIZES] = {
-		MCLBYTES,
-#if MJUMPAGESIZE != MCLBYTES
-		MJUMPAGESIZE,
-#endif
-		MJUM9BYTES,
-		MJUM16BYTES
-	};
+	int pad;
 
-	for (i = 0; i < FL_BUF_SIZES; i++) {
-		FL_BUF_SIZE(i) = bufsize[i];
-		FL_BUF_TYPE(i) = m_gettype(bufsize[i]);
-		FL_BUF_ZONE(i) = m_getzone(bufsize[i]);
-	}
+	/* set pad to a reasonable powerof2 between 16 and 4096 (inclusive) */
+#if defined(__i386__) || defined(__amd64__)
+	pad = max(cpu_clflush_line_size, 16);
+#else
+	pad = max(CACHE_LINE_SIZE, 16);
+#endif
+	pad = min(pad, 4096);
 
 	if (fl_pktshift < 0 || fl_pktshift > 7) {
 		printf("Invalid hw.cxgbe.fl_pktshift value (%d),"
@@ -230,23 +241,35 @@ t4_sge_modload(void)
 		fl_pktshift = 2;
 	}
 
-	if (fl_pad < 32 || fl_pad > 4096 || !powerof2(fl_pad)) {
-		int pad;
-
-#if defined(__i386__) || defined(__amd64__)
-		pad = max(cpu_clflush_line_size, 32);
-#else
-		pad = max(CACHE_LINE_SIZE, 32);
-#endif
-		pad = min(pad, 4096);
+	if (fl_pad != 0 &&
+	    (fl_pad < 32 || fl_pad > 4096 || !powerof2(fl_pad))) {
 
 		if (fl_pad != -1) {
 			printf("Invalid hw.cxgbe.fl_pad value (%d),"
-			    " using %d instead.\n", fl_pad, pad);
+			    " using %d instead.\n", fl_pad, max(pad, 32));
 		}
-		fl_pad = pad;
+		fl_pad = max(pad, 32);
 	}
 
+	/*
+	 * T4 has the same pad and pack boundary.  If a pad boundary is set,
+	 * pack boundary must be set to the same value.  Otherwise take the
+	 * specified value or auto-calculate something reasonable.
+	 */
+	if (fl_pad)
+		t4_fl_pack = fl_pad;
+	else if (fl_pack < 32 || fl_pack > 4096 || !powerof2(fl_pack))
+		t4_fl_pack = max(pad, 32);
+	else
+		t4_fl_pack = fl_pack;
+
+	/* T5's pack boundary is independent of the pad boundary. */
+	if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 ||
+	    !powerof2(fl_pack))
+	       t5_fl_pack = max(pad, 64);
+	else
+	       t5_fl_pack = fl_pack;
+
 	if (spg_len != 64 && spg_len != 128) {
 		int len;
 
@@ -292,17 +315,41 @@ t4_tweak_chip_settings(struct adapter *sc)
 	int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk;
 	int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
 	uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
+	int sw_flbuf_sizes[] = {
+		MCLBYTES,
+#if MJUMPAGESIZE != MCLBYTES
+		MJUMPAGESIZE,
+#endif
+		MJUM9BYTES,
+		MJUM16BYTES,
+		MJUMPAGESIZE - MSIZE
+	};
 
 	KASSERT(sc->flags & MASTER_PF,
 	    ("%s: trying to change chip settings when not master.", __func__));
 
-	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE |
-	    V_INGPADBOUNDARY(M_INGPADBOUNDARY) | F_EGRSTATUSPAGESIZE;
+	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE;
 	v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE |
-	    V_INGPADBOUNDARY(ilog2(fl_pad) - 5) |
 	    V_EGRSTATUSPAGESIZE(spg_len == 128);
+	if (is_t4(sc) && (fl_pad || buffer_packing)) {
+		/* t4_fl_pack has the correct value even when fl_pad = 0 */
+		m |= V_INGPADBOUNDARY(M_INGPADBOUNDARY);
+		v |= V_INGPADBOUNDARY(ilog2(t4_fl_pack) - 5);
+	} else if (is_t5(sc) && fl_pad) {
+		m |= V_INGPADBOUNDARY(M_INGPADBOUNDARY);
+		v |= V_INGPADBOUNDARY(ilog2(fl_pad) - 5);
+	}
 	t4_set_reg_field(sc, A_SGE_CONTROL, m, v);
 
+	if (is_t5(sc) && buffer_packing) {
+		m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY);
+		if (t5_fl_pack == 16)
+			v = V_INGPACKBOUNDARY(0);
+		else
+			v = V_INGPACKBOUNDARY(ilog2(t5_fl_pack) - 5);
+		t4_set_reg_field(sc, A_SGE_CONTROL2, m, v);
+	}
+
 	v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
@@ -313,9 +360,9 @@ t4_tweak_chip_settings(struct adapter *sc)
 	    V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10);
 	t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v);
 
-	for (i = 0; i < FL_BUF_SIZES; i++) {
+	for (i = 0; i < min(nitems(sw_flbuf_sizes), 16); i++) {
 		t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i),
-		    FL_BUF_SIZE(i));
+		    sw_flbuf_sizes[i]);
 	}
 
 	v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) |
@@ -376,21 +423,48 @@ int
 t4_read_chip_settings(struct adapter *sc)
 {
 	struct sge *s = &sc->sge;
-	int i, rc = 0;
+	int i, j, n, rc = 0;
 	uint32_t m, v, r;
 	uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
+	uint32_t sge_flbuf_sizes[16], sw_flbuf_sizes[] = {
+		MCLBYTES,
+#if MJUMPAGESIZE != MCLBYTES
+		MJUMPAGESIZE,
+#endif
+		MJUM9BYTES,
+		MJUM16BYTES
+	};
 
-	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE |
-	    V_INGPADBOUNDARY(M_INGPADBOUNDARY) | F_EGRSTATUSPAGESIZE;
+	m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE;
 	v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE |
-	    V_INGPADBOUNDARY(ilog2(fl_pad) - 5) |
 	    V_EGRSTATUSPAGESIZE(spg_len == 128);
+	if (is_t4(sc) && (fl_pad || buffer_packing)) {
+		m |= V_INGPADBOUNDARY(M_INGPADBOUNDARY);
+		v |= V_INGPADBOUNDARY(ilog2(t4_fl_pack) - 5);
+	} else if (is_t5(sc) && fl_pad) {
+		m |= V_INGPADBOUNDARY(M_INGPADBOUNDARY);
+		v |= V_INGPADBOUNDARY(ilog2(fl_pad) - 5);
+	}
 	r = t4_read_reg(sc, A_SGE_CONTROL);
 	if ((r & m) != v) {
 		device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r);
 		rc = EINVAL;
 	}
 
+	if (is_t5(sc) && buffer_packing) {
+		m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY);
+		if (t5_fl_pack == 16)
+			v = V_INGPACKBOUNDARY(0);
+		else
+			v = V_INGPACKBOUNDARY(ilog2(t5_fl_pack) - 5);
+		r = t4_read_reg(sc, A_SGE_CONTROL2);
+		if ((r & m) != v) {
+			device_printf(sc->dev,
+			    "invalid SGE_CONTROL2(0x%x)\n", r);
+			rc = EINVAL;
+		}
+	}
+
 	v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
 	    V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
@@ -405,14 +479,45 @@ t4_read_chip_settings(struct adapter *sc)
 		rc = EINVAL;
 	}
 
-	for (i = 0; i < FL_BUF_SIZES; i++) {
-		v = t4_read_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i));
-		if (v != FL_BUF_SIZE(i)) {
-			device_printf(sc->dev,
-			    "invalid SGE_FL_BUFFER_SIZE[%d](0x%x)\n", i, v);
-			rc = EINVAL;
+	/*
+	 * Make a list of SGE FL buffer sizes programmed in the chip and tally
+	 * it with the FL buffer sizes that we'd like to use.
+	 */
+	n = 0;
+	for (i = 0; i < nitems(sge_flbuf_sizes); i++) {
+		r = t4_read_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i));
+		sge_flbuf_sizes[i] = r;
+		if (r == MJUMPAGESIZE - MSIZE &&
+		    (sc->flags & BUF_PACKING_OK) == 0) {
+			sc->flags |= BUF_PACKING_OK;
+			FL_BUF_HWTAG(sc, n) = i;
+			FL_BUF_SIZE(sc, n) = MJUMPAGESIZE - MSIZE;
+			FL_BUF_TYPE(sc, n) = m_gettype(MJUMPAGESIZE);
+			FL_BUF_ZONE(sc, n) = m_getzone(MJUMPAGESIZE);
+			n++;
+		}
+	}
+	for (i = 0; i < nitems(sw_flbuf_sizes); i++) {
+		for (j = 0; j < nitems(sge_flbuf_sizes); j++) {
+			if (sw_flbuf_sizes[i] != sge_flbuf_sizes[j])
+				continue;
+			FL_BUF_HWTAG(sc, n) = j;
+			FL_BUF_SIZE(sc, n) = sw_flbuf_sizes[i];
+			FL_BUF_TYPE(sc, n) = m_gettype(sw_flbuf_sizes[i]);
+			FL_BUF_ZONE(sc, n) = m_getzone(sw_flbuf_sizes[i]);
+			n++;
+			break;
 		}
 	}
+	if (n == 0) {
+		device_printf(sc->dev, "no usable SGE FL buffer size.\n");
+		rc = EINVAL;
+	} else if (n == 1 && (sc->flags & BUF_PACKING_OK)) {
+		device_printf(sc->dev,
+		    "no usable SGE FL buffer size when not packing buffers.\n");
+		rc = EINVAL;
+	}
+	FL_BUF_SIZES(sc) = n;
 
 	r = t4_read_reg(sc, A_SGE_INGRESS_RX_THRESHOLD);
 	s->counter_val[0] = G_THRESHOLD_0(r);
@@ -515,6 +620,14 @@ t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx,
 
 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD,
 	    NULL, cong_drop, "congestion drop setting");
+
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "buffer_packing", CTLFLAG_RD,
+	    NULL, sc->flags & BUF_PACKING_OK ? 1 : 0,
+	    "pack multiple frames in one fl buffer");
+
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD,
+	    NULL, is_t5(sc) ? t5_fl_pack : t4_fl_pack,
+	    "payload pack boundary (bytes)");
 }
 
 int
@@ -706,7 +819,7 @@ t4_setup_port_queues(struct port_info *pi)
 	struct ifnet *ifp = pi->ifp;
 	struct sysctl_oid *oid = device_get_sysctl_tree(pi->dev);
 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
-	int bufsize;
+	int bufsize, pack;
 
 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD,
 	    NULL, "rx queues");
@@ -728,6 +841,12 @@ t4_setup_port_queues(struct port_info *pi)
 	 * b) allocate queue iff it will take direct interrupts.
 	 */
 	bufsize = mtu_to_bufsize(ifp->if_mtu);
+	if (sc->flags & BUF_PACKING_OK &&
+	    ((is_t5(sc) && buffer_packing) ||	/* 1 or -1 both ok for T5 */
+	    (is_t4(sc) && buffer_packing == 1)))
+		pack = 1;
+	else
+		pack = 0;
 	for_each_rxq(pi, i, rxq) {
 
 		init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq,
@@ -735,7 +854,7 @@ t4_setup_port_queues(struct port_info *pi)
 
 		snprintf(name, sizeof(name), "%s rxq%d-fl",
 		    device_get_nameunit(pi->dev), i);
-		init_fl(&rxq->fl, pi->qsize_rxq / 8, bufsize, name);
+		init_fl(sc, &rxq->fl, pi->qsize_rxq / 8, bufsize, pack, name);
 
 		if (sc->flags & INTR_DIRECT
 #ifdef TCP_OFFLOAD
@@ -752,6 +871,7 @@ t4_setup_port_queues(struct port_info *pi)
 
 #ifdef TCP_OFFLOAD
 	bufsize = mtu_to_bufsize_toe(sc, ifp->if_mtu);
+	pack = 0;	/* XXX: think about this some more */
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 
 		init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
@@ -759,7 +879,8 @@ t4_setup_port_queues(struct port_info *pi)
 
 		snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
 		    device_get_nameunit(pi->dev), i);
-		init_fl(&ofld_rxq->fl, pi->qsize_rxq / 8, bufsize, name);
+		init_fl(sc, &ofld_rxq->fl, pi->qsize_rxq / 8, bufsize, pack,
+		    name);
 
 		if (sc->flags & INTR_DIRECT ||
 		    (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
@@ -1036,7 +1157,12 @@ service_iq(struct sge_iq *iq, int budget)
 				    ("%s: data for an iq (%p) with no freelist",
 				    __func__, iq));
 
-				m0 = get_fl_payload(sc, fl, lq, &fl_bufs_used);
+				m0 = fl->flags & FL_BUF_PACKING ?
+				    get_fl_payload1(sc, fl, lq, &fl_bufs_used) :
+				    get_fl_payload2(sc, fl, lq, &fl_bufs_used);
+
+				if (__predict_false(m0 == NULL))
+					goto process_iql;
 #ifdef T4_PKT_TIMESTAMP
 				/*
 				 * 60 bit timestamp for the payload is
@@ -1136,6 +1262,7 @@ service_iq(struct sge_iq *iq, int budget)
 			}
 		}
 
+process_iql:
 		if (STAILQ_EMPTY(&iql))
 			break;
 
@@ -1181,13 +1308,102 @@ service_iq(struct sge_iq *iq, int budget)
 	return (0);
 }
 
+static int
+fill_mbuf_stash(struct sge_fl *fl)
+{
+	int i;
+
+	for (i = 0; i < nitems(fl->mstash); i++) {
+		if (fl->mstash[i] == NULL) {
+			struct mbuf *m;
+			if ((m = m_get(M_NOWAIT, MT_NOINIT)) == NULL)
+				return (ENOBUFS);
+			fl->mstash[i] = m;
+		}
+	}
+	return (0);
+}
+
 static struct mbuf *
-get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
+get_mbuf_from_stash(struct sge_fl *fl)
+{
+	int i;
+
+	for (i = 0; i < nitems(fl->mstash); i++) {
+		if (fl->mstash[i] != NULL) {
+			struct mbuf *m;
+
+			m = fl->mstash[i];
+			fl->mstash[i] = NULL;
+			return (m);
+		} else
+			fl->mstash[i] = m_get(M_NOWAIT, MT_NOINIT);
+	}
+
+	return (m_get(M_NOWAIT, MT_NOINIT));
+}
+
+static void
+return_mbuf_to_stash(struct sge_fl *fl, struct mbuf *m)
+{
+	int i;
+
+	if (m == NULL)
+		return;
+
+	for (i = 0; i < nitems(fl->mstash); i++) {
+		if (fl->mstash[i] == NULL) {
+			fl->mstash[i] = m;
+			return;
+		}
+	}
+	m_init(m, NULL, 0, M_NOWAIT, MT_DATA, 0);
+	m_free(m);
+}
+
+/* buf can be any address within the buffer */
+static inline u_int *
+find_buf_refcnt(caddr_t buf)
+{
+	uintptr_t ptr = (uintptr_t)buf;
+
+	return ((u_int *)((ptr & ~(MJUMPAGESIZE - 1)) + MSIZE - sizeof(u_int)));
+}
+
+static inline struct mbuf *
+find_buf_mbuf(caddr_t buf)
+{
+	uintptr_t ptr = (uintptr_t)buf;
+
+	return ((struct mbuf *)(ptr & ~(MJUMPAGESIZE - 1)));
+}
+
+static int
+rxb_free(struct mbuf *m, void *arg1, void *arg2)
+{
+	uma_zone_t zone = arg1;
+	caddr_t cl = arg2;
+#ifdef INVARIANTS
+	u_int refcount;
+
+	refcount = *find_buf_refcnt(cl);
+	KASSERT(refcount == 0, ("%s: cl %p refcount is %u", __func__,
+	    cl - MSIZE, refcount));
+#endif
+	cl -= MSIZE;
+	uma_zfree(zone, cl);
+
+	return (EXT_FREE_OK);
+}
+
+static struct mbuf *
+get_fl_payload1(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
     int *fl_bufs_used)
 {
 	struct mbuf *m0, *m;
 	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
 	unsigned int nbuf, len;
+	int pack_boundary = is_t4(sc) ? t4_fl_pack : t5_fl_pack;
 
 	/*
 	 * No assertion for the fl lock because we don't need it.  This routine
@@ -1198,29 +1414,194 @@ get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
 	 * lock but this routine does not).
 	 */
 
+	KASSERT(fl->flags & FL_BUF_PACKING,
+	    ("%s: buffer packing disabled for fl %p", __func__, fl));
+
+	len = G_RSPD_LEN(len_newbuf);
+
+	if ((len_newbuf & F_RSPD_NEWBUF) == 0) {
+		KASSERT(fl->rx_offset > 0,
+		    ("%s: packed frame but driver at offset=0", __func__));
+
+		/* A packed frame is guaranteed to fit entirely in this buf. */
+		KASSERT(FL_BUF_SIZE(sc, sd->tag_idx) - fl->rx_offset >= len,
+		    ("%s: packing error.  bufsz=%u, offset=%u, len=%u",
+		    __func__, FL_BUF_SIZE(sc, sd->tag_idx), fl->rx_offset,
+		    len));
+
+		m0 = get_mbuf_from_stash(fl);
+		if (m0 == NULL ||
+		    m_init(m0, NULL, 0, M_NOWAIT, MT_DATA, M_PKTHDR) != 0) {
+			return_mbuf_to_stash(fl, m0);
+			return (NULL);
+		}
+
+		bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map,
+		    BUS_DMASYNC_POSTREAD);
+		if (len < RX_COPY_THRESHOLD) {
+#ifdef T4_PKT_TIMESTAMP
+			/* Leave room for a timestamp */
+			m0->m_data += 8;
+#endif
+			bcopy(sd->cl + fl->rx_offset, mtod(m0, caddr_t), len);
+			m0->m_pkthdr.len = len;
+			m0->m_len = len;
+		} else {
+			m0->m_pkthdr.len = len;
+			m0->m_len = len;
+			m_extaddref(m0, sd->cl + fl->rx_offset,
+			    roundup2(m0->m_len, fl_pad),
+			    find_buf_refcnt(sd->cl), rxb_free,
+			    FL_BUF_ZONE(sc, sd->tag_idx), sd->cl);
+		}
+		fl->rx_offset += len;
+		fl->rx_offset = roundup2(fl->rx_offset, fl_pad);
+		fl->rx_offset = roundup2(fl->rx_offset, pack_boundary);
+		if (fl->rx_offset >= FL_BUF_SIZE(sc, sd->tag_idx)) {
+			fl->rx_offset = 0;
+			(*fl_bufs_used) += 1;
+			if (__predict_false(++fl->cidx == fl->cap))
+				fl->cidx = 0;
+		}
+
+		return (m0);
+	}
+
+	KASSERT(len_newbuf & F_RSPD_NEWBUF,
+	    ("%s: only new buffer handled here", __func__));
+
+	nbuf = 0;
+
+	/*
+	 * Move to the start of the next buffer if we are still in the middle of
+	 * some buffer.  This is the case where there was some room left in the
+	 * previous buffer but not enough to fit this frame in its entirety.
+	 */
+	if (fl->rx_offset > 0) {
+		KASSERT(roundup2(len, fl_pad) > FL_BUF_SIZE(sc, sd->tag_idx) -
+		    fl->rx_offset, ("%s: frame (%u bytes) should have fit at "
+		    "cidx %u offset %u bufsize %u", __func__, len, fl->cidx,
+		    fl->rx_offset, FL_BUF_SIZE(sc, sd->tag_idx)));
+		nbuf++;
+		fl->rx_offset = 0;
+		sd++;
+		if (__predict_false(++fl->cidx == fl->cap)) {
+			sd = fl->sdesc;
+			fl->cidx = 0;
+		}
+	}
+
+	m0 = find_buf_mbuf(sd->cl);
+	if (m_init(m0, NULL, 0, M_NOWAIT, MT_DATA, M_PKTHDR | M_NOFREE))
+		goto done;
+	bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map, BUS_DMASYNC_POSTREAD);
+	m0->m_len = min(len, FL_BUF_SIZE(sc, sd->tag_idx));
+	m_extaddref(m0, sd->cl, roundup2(m0->m_len, fl_pad),
+	    find_buf_refcnt(sd->cl), rxb_free, FL_BUF_ZONE(sc, sd->tag_idx),
+	    sd->cl);
+	m0->m_pkthdr.len = len;
+
+	fl->rx_offset = roundup2(m0->m_len, fl_pad);
+	fl->rx_offset = roundup2(fl->rx_offset, pack_boundary);
+	if (fl->rx_offset >= FL_BUF_SIZE(sc, sd->tag_idx)) {
+		fl->rx_offset = 0;
+		nbuf++;
+		sd++;
+		if (__predict_false(++fl->cidx == fl->cap)) {
+			sd = fl->sdesc;
+			fl->cidx = 0;
+		}
+	}
+
+	m = m0;
+	len -= m->m_len;
+
+	while (len > 0) {
+		m->m_next = find_buf_mbuf(sd->cl);
+		m = m->m_next;
+
+		bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map,
+		    BUS_DMASYNC_POSTREAD);
+
+		/* m_init for !M_PKTHDR can't fail so don't bother */
+		m_init(m, NULL, 0, M_NOWAIT, MT_DATA, M_NOFREE);
+		m->m_len = min(len, FL_BUF_SIZE(sc, sd->tag_idx));
+		m_extaddref(m, sd->cl, roundup2(m->m_len, fl_pad),
+		    find_buf_refcnt(sd->cl), rxb_free,
+		    FL_BUF_ZONE(sc, sd->tag_idx), sd->cl);
+
+		fl->rx_offset = roundup2(m->m_len, fl_pad);
+		fl->rx_offset = roundup2(fl->rx_offset, pack_boundary);
+		if (fl->rx_offset >= FL_BUF_SIZE(sc, sd->tag_idx)) {
+			fl->rx_offset = 0;
+			nbuf++;
+			sd++;
+			if (__predict_false(++fl->cidx == fl->cap)) {
+				sd = fl->sdesc;
+				fl->cidx = 0;
+			}
+		}
+
+		len -= m->m_len;
+	}
+done:
+	(*fl_bufs_used) += nbuf;
+	return (m0);
+}
+
+static struct mbuf *
+get_fl_payload2(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
+    int *fl_bufs_used)
+{
+	struct mbuf *m0, *m;
+	struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
+	unsigned int nbuf, len;
+
+	/*
+	 * No assertion for the fl lock because we don't need it.  This routine
+	 * is called only from the rx interrupt handler and it only updates
+	 * fl->cidx.  (Contrast that with fl->pidx/fl->needed which could be
+	 * updated in the rx interrupt handler or the starvation helper routine.
+	 * That's why code that manipulates fl->pidx/fl->needed needs the fl
+	 * lock but this routine does not).
+	 */
+
+	KASSERT((fl->flags & FL_BUF_PACKING) == 0,
+	    ("%s: buffer packing enabled for fl %p", __func__, fl));
 	if (__predict_false((len_newbuf & F_RSPD_NEWBUF) == 0))
 		panic("%s: cannot handle packed frames", __func__);
 	len = G_RSPD_LEN(len_newbuf);
 
-	m0 = sd->m;
-	sd->m = NULL;	/* consumed */
+	/*
+	 * We never want to run out of mbufs in between a frame when a frame
+	 * spans multiple fl buffers.  If the fl's mbuf stash isn't full and
+	 * can't be filled up to the brim then fail early.
+	 */
+	if (len > FL_BUF_SIZE(sc, sd->tag_idx) && fill_mbuf_stash(fl) != 0)
+		return (NULL);
+
+	m0 = get_mbuf_from_stash(fl);
+	if (m0 == NULL ||
+	    m_init(m0, NULL, 0, M_NOWAIT, MT_DATA, M_PKTHDR) != 0) {
+		return_mbuf_to_stash(fl, m0);
+		return (NULL);
+	}
 
 	bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map, BUS_DMASYNC_POSTREAD);
-	m_init(m0, NULL, 0, M_NOWAIT, MT_DATA, M_PKTHDR);
-#ifdef T4_PKT_TIMESTAMP
-	/* Leave room for a timestamp */
-	m0->m_data += 8;
-#endif
 
 	if (len < RX_COPY_THRESHOLD) {
+#ifdef T4_PKT_TIMESTAMP
+		/* Leave room for a timestamp */
+		m0->m_data += 8;
+#endif
 		/* copy data to mbuf, buffer will be recycled */
 		bcopy(sd->cl, mtod(m0, caddr_t), len);
 		m0->m_len = len;
 	} else {
 		bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map);
-		m_cljset(m0, sd->cl, FL_BUF_TYPE(sd->tag_idx));
+		m_cljset(m0, sd->cl, FL_BUF_TYPE(sc, sd->tag_idx));
 		sd->cl = NULL;	/* consumed */
-		m0->m_len = min(len, FL_BUF_SIZE(sd->tag_idx));
+		m0->m_len = min(len, FL_BUF_SIZE(sc, sd->tag_idx));
 	}
 	m0->m_pkthdr.len = len;
 
@@ -1235,23 +1616,23 @@ get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
 	nbuf = 1;	/* # of fl buffers used */
 
 	while (len > 0) {
-		m->m_next = sd->m;
-		sd->m = NULL;	/* consumed */
+		/* Can't fail, we checked earlier that the stash was full. */
+		m->m_next = get_mbuf_from_stash(fl);
 		m = m->m_next;
 
 		bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map,
 		    BUS_DMASYNC_POSTREAD);
 
+		/* m_init for !M_PKTHDR can't fail so don't bother */
 		m_init(m, NULL, 0, M_NOWAIT, MT_DATA, 0);
 		if (len <= MLEN) {
 			bcopy(sd->cl, mtod(m, caddr_t), len);
 			m->m_len = len;
 		} else {
-			bus_dmamap_unload(fl->tag[sd->tag_idx],
-			    sd->map);
-			m_cljset(m, sd->cl, FL_BUF_TYPE(sd->tag_idx));
+			bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map);
+			m_cljset(m, sd->cl, FL_BUF_TYPE(sc, sd->tag_idx));
 			sd->cl = NULL;	/* consumed */
-			m->m_len = min(len, FL_BUF_SIZE(sd->tag_idx));
+			m->m_len = min(len, FL_BUF_SIZE(sc, sd->tag_idx));
 		}
 
 		sd++;
@@ -1616,6 +1997,7 @@ void
 t4_update_fl_bufsize(struct ifnet *ifp)
 {
 	struct port_info *pi = ifp->if_softc;
+	struct adapter *sc = pi->adapter;
 	struct sge_rxq *rxq;
 #ifdef TCP_OFFLOAD
 	struct sge_ofld_rxq *ofld_rxq;
@@ -1628,7 +2010,7 @@ t4_update_fl_bufsize(struct ifnet *ifp)
 		fl = &rxq->fl;
 
 		FL_LOCK(fl);
-		set_fl_tag_idx(fl, bufsize);
+		set_fl_tag_idx(sc, fl, bufsize);
 		FL_UNLOCK(fl);
 	}
 #ifdef TCP_OFFLOAD
@@ -1637,7 +2019,7 @@ t4_update_fl_bufsize(struct ifnet *ifp)
 		fl = &ofld_rxq->fl;
 
 		FL_LOCK(fl);
-		set_fl_tag_idx(fl, bufsize);
+		set_fl_tag_idx(sc, fl, bufsize);
 		FL_UNLOCK(fl);
 	}
 #endif
@@ -1671,11 +2053,15 @@ init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
 }
 
 static inline void
-init_fl(struct sge_fl *fl, int qsize, int bufsize, char *name)
+init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int bufsize, int pack,
+    char *name)
 {
+
 	fl->qsize = qsize;
 	strlcpy(fl->lockname, name, sizeof(fl->lockname));
-	set_fl_tag_idx(fl, bufsize);
+	if (pack)
+		fl->flags |= FL_BUF_PACKING;
+	set_fl_tag_idx(sc, fl, bufsize);
 }
 
 static inline void
@@ -1804,7 +2190,7 @@ alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
 	if (fl) {
 		mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF);
 
-		for (i = 0; i < FL_BUF_SIZES; i++) {
+		for (i = 0; i < FL_BUF_SIZES(sc); i++) {
 
 			/*
 			 * A freelist buffer must be 16 byte aligned as the SGE
@@ -1813,8 +2199,8 @@ alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
 			 */
 			rc = bus_dma_tag_create(sc->dmat, 16, 0,
 			    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
-			    FL_BUF_SIZE(i), 1, FL_BUF_SIZE(i), BUS_DMA_ALLOCNOW,
-			    NULL, NULL, &fl->tag[i]);
+			    FL_BUF_SIZE(sc, i), 1, FL_BUF_SIZE(sc, i),
+			    BUS_DMA_ALLOCNOW, NULL, NULL, &fl->tag[i]);
 			if (rc != 0) {
 				device_printf(sc->dev,
 				    "failed to create fl DMA tag[%d]: %d\n",
@@ -1843,7 +2229,9 @@ alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
 		c.iqns_to_fl0congen |=
 		    htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
 			F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
-			F_FW_IQ_CMD_FL0PADEN);
+			(fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) |
+			(fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN :
+			    0));
 		if (cong >= 0) {
 			c.iqns_to_fl0congen |=
 				htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
@@ -1964,12 +2352,21 @@ free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
 		    fl->desc);
 
 		if (fl->sdesc)
-			free_fl_sdesc(fl);
+			free_fl_sdesc(sc, fl);
+
+		for (i = 0; i < nitems(fl->mstash); i++) {
+			struct mbuf *m = fl->mstash[i];
+
+			if (m != NULL) {
+				m_init(m, NULL, 0, M_NOWAIT, MT_DATA, 0);
+				m_free(m);
+			}
+		}
 
 		if (mtx_initialized(&fl->fl_lock))
 			mtx_destroy(&fl->fl_lock);
 
-		for (i = 0; i < FL_BUF_SIZES; i++) {
+		for (i = 0; i < FL_BUF_SIZES(sc); i++) {
 			if (fl->tag[i])
 				bus_dma_tag_destroy(fl->tag[i]);
 		}
@@ -2130,6 +2527,10 @@ alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int idx,
 	    "SGE context id of the queue");
 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cidx", CTLFLAG_RD,
 	    &rxq->fl.cidx, 0, "consumer index");
+	if (rxq->fl.flags & FL_BUF_PACKING) {
+		SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "rx_offset",
+		    CTLFLAG_RD, &rxq->fl.rx_offset, 0, "packing rx offset");
+	}
 	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "pidx", CTLFLAG_RD,
 	    &rxq->fl.pidx, 0, "producer index");
 
@@ -2691,6 +3092,12 @@ refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
 	int rc;
 
 	FL_LOCK_ASSERT_OWNED(fl);
+#ifdef INVARIANTS
+	if (fl->flags & FL_BUF_PACKING)
+		KASSERT(sd->tag_idx == 0,
+		    ("%s: expected tag 0 but found tag %d at pidx %u instead",
+		    __func__, sd->tag_idx, fl->pidx));
+#endif
 
 	if (nbufs > fl->needed)
 		nbufs = fl->needed;
@@ -2699,24 +3106,34 @@ refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
 
 		if (sd->cl != NULL) {
 
-			/*
-			 * This happens when a frame small enough to fit
-			 * entirely in an mbuf was received in cl last time.
-			 * We'd held on to cl and can reuse it now.  Note that
-			 * we reuse a cluster of the old size if fl->tag_idx is
-			 * no longer the same as sd->tag_idx.
-			 */
-
-			KASSERT(*d == sd->ba_tag,
+			KASSERT(*d == sd->ba_hwtag,
 			    ("%s: recyling problem at pidx %d",
 			    __func__, fl->pidx));
 
-			d++;
-			goto recycled;
-		}
+			if (fl->flags & FL_BUF_PACKING) {
+				u_int *refcount = find_buf_refcnt(sd->cl);
 
+				if (atomic_fetchadd_int(refcount, -1) == 1) {
+					*refcount = 1;	/* reinstate */
+					d++;
+					goto recycled;
+				}
+				sd->cl = NULL;	/* gave up my reference */
+			} else {
+				/*
+				 * This happens when a frame small enough to fit
+				 * entirely in an mbuf was received in cl last
+				 * time.  We'd held on to cl and can reuse it
+				 * now.  Note that we reuse a cluster of the old
+				 * size if fl->tag_idx is no longer the same as
+				 * sd->tag_idx.
+				 */
+				d++;
+				goto recycled;
+			}
+		}
 
-		if (fl->tag_idx != sd->tag_idx) {
+		if (__predict_false(fl->tag_idx != sd->tag_idx)) {
 			bus_dmamap_t map;
 			bus_dma_tag_t newtag = fl->tag[fl->tag_idx];
 			bus_dma_tag_t oldtag = fl->tag[sd->tag_idx];
@@ -2736,33 +3153,32 @@ refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
 
 		tag = fl->tag[sd->tag_idx];
 
-		cl = m_cljget(NULL, M_NOWAIT, FL_BUF_SIZE(sd->tag_idx));
+		cl = uma_zalloc(FL_BUF_ZONE(sc, sd->tag_idx), M_NOWAIT);
 		if (cl == NULL)
 			break;
+		if (fl->flags & FL_BUF_PACKING) {
+			*find_buf_refcnt(cl) = 1;
+			cl += MSIZE;
+		}
 
-		rc = bus_dmamap_load(tag, sd->map, cl, FL_BUF_SIZE(sd->tag_idx),
-		    oneseg_dma_callback, &pa, 0);
+		rc = bus_dmamap_load(tag, sd->map, cl,
+		    FL_BUF_SIZE(sc, sd->tag_idx), oneseg_dma_callback, &pa, 0);
 		if (rc != 0 || pa == 0) {
 			fl->dmamap_failed++;
-			uma_zfree(FL_BUF_ZONE(sd->tag_idx), cl);
+			if (fl->flags & FL_BUF_PACKING)
+				cl -= MSIZE;
+			uma_zfree(FL_BUF_ZONE(sc, sd->tag_idx), cl);
 			break;
 		}
 
 		sd->cl = cl;
-		*d++ = htobe64(pa | sd->tag_idx);
+		*d++ = htobe64(pa | FL_BUF_HWTAG(sc, sd->tag_idx));
 
 #ifdef INVARIANTS
-		sd->ba_tag = htobe64(pa | sd->tag_idx);
+		sd->ba_hwtag = htobe64(pa | FL_BUF_HWTAG(sc, sd->tag_idx));
 #endif
 
 recycled:
-		/* sd->m is never recycled, should always be NULL */
-		KASSERT(sd->m == NULL, ("%s: stray mbuf", __func__));
-
-		sd->m = m_gethdr(M_NOWAIT, MT_NOINIT);
-		if (sd->m == NULL)
-			break;
-
 		fl->pending++;
 		fl->needed--;
 		sd++;
@@ -2829,11 +3245,6 @@ failed:
 	while (--i >= 0) {
 		sd--;
 		bus_dmamap_destroy(tag, sd->map);
-		if (sd->m) {
-			m_init(sd->m, NULL, 0, M_NOWAIT, MT_DATA, 0);
-			m_free(sd->m);
-			sd->m = NULL;
-		}
 	}
 	KASSERT(sd == fl->sdesc, ("%s: EDOOFUS", __func__));
 
@@ -2844,7 +3255,7 @@ failed:
 }
 
 static void
-free_fl_sdesc(struct sge_fl *fl)
+free_fl_sdesc(struct adapter *sc, struct sge_fl *fl)
 {
 	struct fl_sdesc *sd;
 	int i;
@@ -2852,15 +3263,9 @@ free_fl_sdesc(struct sge_fl *fl)
 	sd = fl->sdesc;
 	for (i = 0; i < fl->cap; i++, sd++) {
 
-		if (sd->m) {
-			m_init(sd->m, NULL, 0, M_NOWAIT, MT_DATA, 0);
-			m_free(sd->m);
-			sd->m = NULL;
-		}
-
 		if (sd->cl) {
 			bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map);
-			uma_zfree(FL_BUF_ZONE(sd->tag_idx), sd->cl);
+			uma_zfree(FL_BUF_ZONE(sc, sd->tag_idx), sd->cl);
 			sd->cl = NULL;
 		}
 
@@ -3715,17 +4120,52 @@ get_flit(bus_dma_segment_t *sgl, int nsegs, int idx)
 	return (0);
 }
 
+/*
+ * Find an SGE FL buffer size to use for the given bufsize.  Look for the the
+ * smallest size that is large enough to hold bufsize or pick the largest size
+ * if all sizes are less than bufsize.
+ */
 static void
-set_fl_tag_idx(struct sge_fl *fl, int bufsize)
+set_fl_tag_idx(struct adapter *sc, struct sge_fl *fl, int bufsize)
 {
-	int i;
+	int i, largest, best, delta, start;
 
-	for (i = 0; i < FL_BUF_SIZES - 1; i++) {
-		if (FL_BUF_SIZE(i) >= bufsize)
-			break;
+	if (fl->flags & FL_BUF_PACKING) {
+		fl->tag_idx = 0;	/* first tag is the one for packing */
+		return;
 	}
 
-	fl->tag_idx = i;
+	start = sc->flags & BUF_PACKING_OK ? 1 : 0;
+	delta = FL_BUF_SIZE(sc, start) - bufsize;
+	if (delta == 0) {
+		fl->tag_idx = start;	/* ideal fit, look no further */
+		return;
+	}
+	best = start;
+	largest = start;
+
+	for (i = start + 1; i < FL_BUF_SIZES(sc); i++) {
+		int d, fl_buf_size;
+
+		fl_buf_size = FL_BUF_SIZE(sc, i);
+		d = fl_buf_size - bufsize;
+
+		if (d == 0) {
+			fl->tag_idx = i;	/* ideal fit, look no further */
+			return;
+		}
+		if (fl_buf_size > FL_BUF_SIZE(sc, largest))
+			largest = i;
+		if (d > 0 && (delta < 0 || delta > d)) {
+			delta = d;
+			best = i;
+		}
+	}
+
+	if (delta > 0)
+		fl->tag_idx = best;	/* Found a buf bigger than bufsize */
+	else
+		fl->tag_idx = largest;	/* No buf large enough for bufsize */
 }
 
 static void