6 files changed, 44 insertions, 8 deletions

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index df0976a..38af49f 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -132,7 +132,6 @@ enum {
 #else
 	FL_BUF_SIZES = 3,	/* cluster, jumbo9k, jumbo16k */
 #endif
-	OFLD_BUF_SIZE = MJUM16BYTES,	/* size of fl buffer for TOE rxq */
 
 	CTRL_EQ_QSIZE = 128,
 
diff --git a/sys/dev/cxgbe/offload.h b/sys/dev/cxgbe/offload.h
index 6090775..7f4b38e 100644
--- a/sys/dev/cxgbe/offload.h
+++ b/sys/dev/cxgbe/offload.h
@@ -140,6 +140,7 @@ struct tom_tunables {
 	int ddp;
 	int indsz;
 	int ddp_thres;
+	int rx_coalesce;
 };
 
 int t4_register_uld(struct uld_info *);
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 6f96390..fde79c4 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -1072,9 +1072,10 @@ cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 		if (rc)
 			return (rc);
 		ifp->if_mtu = mtu;
-		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		if (pi->flags & PORT_INIT_DONE) {
 			t4_update_fl_bufsize(ifp);
-			rc = update_mac_settings(pi, XGMAC_MTU);
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+				rc = update_mac_settings(pi, XGMAC_MTU);
 		}
 		end_synchronized_op(sc, 0);
 		break;
@@ -4381,6 +4382,10 @@ t4_sysctls(struct adapter *sc)
 		    G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
 		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
 		    &sc->tt.ddp_thres, 0, "DDP threshold");
+
+		sc->tt.rx_coalesce = 1;
+		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
+		    CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
 	}
 #endif
 
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index 89eee2d..586137a 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -659,6 +659,18 @@ mtu_to_bufsize(int mtu)
 	return (bufsize);
 }
 
+#ifdef TCP_OFFLOAD
+static inline int
+mtu_to_bufsize_toe(struct adapter *sc, int mtu)
+{
+
+	if (sc->tt.rx_coalesce)
+		return (G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)));
+
+	return (mtu);
+}
+#endif
+
 int
 t4_setup_port_queues(struct port_info *pi)
 {
@@ -673,9 +685,10 @@ t4_setup_port_queues(struct port_info *pi)
 #endif
 	char name[16];
 	struct adapter *sc = pi->adapter;
+	struct ifnet *ifp = pi->ifp;
 	struct sysctl_oid *oid = device_get_sysctl_tree(pi->dev);
 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
-	int bufsize = mtu_to_bufsize(pi->ifp->if_mtu);
+	int bufsize;
 
 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD,
 	    NULL, "rx queues");
@@ -696,6 +709,7 @@ t4_setup_port_queues(struct port_info *pi)
 	 * a) initialize iq and fl
 	 * b) allocate queue iff it will take direct interrupts.
 	 */
+	bufsize = mtu_to_bufsize(ifp->if_mtu);
 	for_each_rxq(pi, i, rxq) {
 
 		init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq,
@@ -719,6 +733,7 @@ t4_setup_port_queues(struct port_info *pi)
 	}
 
 #ifdef TCP_OFFLOAD
+	bufsize = mtu_to_bufsize_toe(sc, ifp->if_mtu);
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 
 		init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
@@ -726,7 +741,7 @@ t4_setup_port_queues(struct port_info *pi)
 
 		snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
 		    device_get_nameunit(pi->dev), i);
-		init_fl(&ofld_rxq->fl, pi->qsize_rxq / 8, OFLD_BUF_SIZE, name);
+		init_fl(&ofld_rxq->fl, pi->qsize_rxq / 8, bufsize, name);
 
 		if (sc->flags & INTR_DIRECT ||
 		    (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
@@ -1557,9 +1572,13 @@ t4_update_fl_bufsize(struct ifnet *ifp)
 {
 	struct port_info *pi = ifp->if_softc;
 	struct sge_rxq *rxq;
+#ifdef TCP_OFFLOAD
+	struct sge_ofld_rxq *ofld_rxq;
+#endif
 	struct sge_fl *fl;
-	int i, bufsize = mtu_to_bufsize(ifp->if_mtu);
+	int i, bufsize;
 
+	bufsize = mtu_to_bufsize(ifp->if_mtu);
 	for_each_rxq(pi, i, rxq) {
 		fl = &rxq->fl;
 
@@ -1567,6 +1586,16 @@ t4_update_fl_bufsize(struct ifnet *ifp)
 		set_fl_tag_idx(fl, bufsize);
 		FL_UNLOCK(fl);
 	}
+#ifdef TCP_OFFLOAD
+	bufsize = mtu_to_bufsize_toe(pi->adapter, ifp->if_mtu);
+	for_each_ofld_rxq(pi, i, ofld_rxq) {
+		fl = &ofld_rxq->fl;
+
+		FL_LOCK(fl);
+		set_fl_tag_idx(fl, bufsize);
+		FL_UNLOCK(fl);
+	}
+#endif
 }
 
 int
diff --git a/sys/dev/cxgbe/tom/t4_connect.c b/sys/dev/cxgbe/tom/t4_connect.c
index 11cb4fc..08267cd 100644
--- a/sys/dev/cxgbe/tom/t4_connect.c
+++ b/sys/dev/cxgbe/tom/t4_connect.c
@@ -251,7 +251,8 @@ calc_opt2a(struct socket *so, struct toepcb *toep)
 		opt2 |= F_T5_OPT_2_VALID;
 		opt2 |= F_CONG_CNTRL_VALID; /* OPT_2_ISS really, for T5 */
 	}
-	opt2 |= V_RX_COALESCE(M_RX_COALESCE);
+	if (sc->tt.rx_coalesce)
+		opt2 |= V_RX_COALESCE(M_RX_COALESCE);
 
 #ifdef USE_DDP_RX_FLOW_CONTROL
 	if (toep->ulp_mode == ULP_MODE_TCPDDP)
diff --git a/sys/dev/cxgbe/tom/t4_listen.c b/sys/dev/cxgbe/tom/t4_listen.c
index 7259a29..9e1dc80 100644
--- a/sys/dev/cxgbe/tom/t4_listen.c
+++ b/sys/dev/cxgbe/tom/t4_listen.c
@@ -1021,7 +1021,8 @@ calc_opt2p(struct adapter *sc, struct port_info *pi, int rxqid,
 		opt2 |= F_T5_OPT_2_VALID;
 		opt2 |= F_CONG_CNTRL_VALID; /* OPT_2_ISS really, for T5 */
 	}
-	opt2 |= V_RX_COALESCE(M_RX_COALESCE);
+	if (sc->tt.rx_coalesce)
+		opt2 |= V_RX_COALESCE(M_RX_COALESCE);
 
 #ifdef USE_DDP_RX_FLOW_CONTROL
 	if (ulp_mode == ULP_MODE_TCPDDP)