(MFp4)

        - upgrade to reflect state of 1.0.0.86
        - move from firmware rev 3.2 to 4.0.0
        - import driver bits for offload functionality
	- remove binary distribution clause from top level files as it
	  runs counter to the intent of purely supporting the hardware

MFC after: 3 days

23 files changed, 4913 insertions, 1117 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 89b6198..18552dc 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -514,6 +514,8 @@ dev/cs/if_cs.c			optional cs
 dev/cs/if_cs_isa.c		optional cs isa
 dev/cs/if_cs_pccard.c		optional cs pccard
 dev/cxgb/cxgb_main.c		optional cxgb pci
+dev/cxgb/cxgb_offload.c		optional cxgb pci
+dev/cxgb/cxgb_l2t.c		optional cxgb pci
 dev/cxgb/cxgb_lro.c		optional cxgb pci
 dev/cxgb/cxgb_sge.c		optional cxgb pci
 dev/cxgb/common/cxgb_mc5.c	optional cxgb pci
diff --git a/sys/dev/cxgb/common/cxgb_common.h b/sys/dev/cxgb/common/cxgb_common.h
index 3c49818..ae73bff 100644
--- a/sys/dev/cxgb/common/cxgb_common.h
+++ b/sys/dev/cxgb/common/cxgb_common.h
@@ -46,6 +46,7 @@ enum {
 	NMTUS          = 16,    /* size of MTU table */
 	NCCTRL_WIN     = 32,    /* # of congestion control windows */
 	NTX_SCHED      = 8,     /* # of HW Tx scheduling queues */
+	TP_TMR_RES     = 200,   /* TP timer resolution in usec */
 };
 
 #define MAX_RX_COALESCING_LEN 16224U
@@ -57,7 +58,6 @@ enum {
 };
 
 enum {
-	SUPPORTED_OFFLOAD  = 1 << 24,
 	SUPPORTED_IRQ      = 1 << 25
 };
 
@@ -70,8 +70,8 @@ enum {                            /* adapter interrupt-maintained statistics */
 };
 
 enum {
-	FW_VERSION_MAJOR = 3,
-	FW_VERSION_MINOR = 2,
+	FW_VERSION_MAJOR = 4,
+	FW_VERSION_MINOR = 0,
 	FW_VERSION_MICRO = 0
 };
 
@@ -309,6 +309,9 @@ enum {
 	MC5_MODE_72_BIT  = 2
 };
 
+/* MC5 min active region size */
+enum { MC5_MIN_TIDS = 16 };
+
 struct vpd_params {
 	unsigned int cclk;
 	unsigned int mclk;
@@ -354,6 +357,7 @@ struct adapter_params {
 	unsigned int   stats_update_period; /* MAC stats accumulation period */
 	unsigned int   linkpoll_period;     /* link poll period in 0.1s */
 	unsigned int   rev;                 /* chip revision */
+	unsigned int   offload;
 };
 
 enum {					    /* chip revisions */
@@ -427,8 +431,11 @@ struct cmac {
 	adapter_t *adapter;
 	unsigned int offset;
 	unsigned int nucast;    /* # of address filters for unicast MACs */
-	unsigned int tcnt;
-	unsigned int xcnt;
+	unsigned int tx_tcnt;
+	unsigned int tx_xcnt;
+	u64 tx_mcnt;
+	unsigned int rx_xcnt;
+	u64 rx_mcnt;
 	unsigned int toggle_cnt;
 	unsigned int txen;
 	struct mac_stats stats;
@@ -555,7 +562,7 @@ static inline int is_10G(const adapter_t *adap)
 static inline int is_offload(const adapter_t *adap)
 {
 #ifdef CONFIG_CHELSIO_T3_CORE
-	return adapter_info(adap)->caps & SUPPORTED_OFFLOAD;
+	return adap->params.offload;
 #else
 	return 0;
 #endif
diff --git a/sys/dev/cxgb/common/cxgb_mc5.c b/sys/dev/cxgb/common/cxgb_mc5.c
index 191d77d..abbf71f 100644
--- a/sys/dev/cxgb/common/cxgb_mc5.c
+++ b/sys/dev/cxgb/common/cxgb_mc5.c
@@ -329,6 +329,9 @@ int t3_mc5_init(struct mc5 *mc5, unsigned int nservers, unsigned int nfilters,
 	unsigned int tcam_size = mc5->tcam_size;
 	adapter_t *adap = mc5->adapter;
 
+	if (tcam_size == 0)
+		return 0;
+	
 	if (nroutes > MAX_ROUTES || nroutes + nservers + nfilters > tcam_size)
 		return -EINVAL;
 
diff --git a/sys/dev/cxgb/common/cxgb_t3_cpl.h b/sys/dev/cxgb/common/cxgb_t3_cpl.h
index 26c4b05..c737b05 100644
--- a/sys/dev/cxgb/common/cxgb_t3_cpl.h
+++ b/sys/dev/cxgb/common/cxgb_t3_cpl.h
@@ -260,6 +260,20 @@ struct work_request_hdr {
 #define V_WR_BCNTLFLT(x) ((x) << S_WR_BCNTLFLT)
 #define G_WR_BCNTLFLT(x) (((x) >> S_WR_BCNTLFLT) & M_WR_BCNTLFLT)
 
+/* Applicable to BYPASS WRs only: the uP will added a CPL_BARRIER before
+ * and after the BYPASS WR if the ATOMIC bit is set.
+ */
+#define S_WR_ATOMIC      16
+#define V_WR_ATOMIC(x)   ((x) << S_WR_ATOMIC)
+#define F_WR_ATOMIC      V_WR_ATOMIC(1U)
+
+/* Applicable to BYPASS WRs only: the uP will flush buffered non abort
+ * related WRs.
+ */
+#define S_WR_FLUSH       17
+#define V_WR_FLUSH(x)    ((x) << S_WR_FLUSH)
+#define F_WR_FLUSH       V_WR_FLUSH(1U)
+
 #define S_WR_DATATYPE    20
 #define V_WR_DATATYPE(x) ((x) << S_WR_DATATYPE)
 #define F_WR_DATATYPE    V_WR_DATATYPE(1U)
@@ -1487,4 +1501,46 @@ struct cpl_rdma_terminate {
 #define M_TERM_TID    0xFFFFF
 #define V_TERM_TID(x) ((x) << S_TERM_TID)
 #define G_TERM_TID(x) (((x) >> S_TERM_TID) & M_TERM_TID)
+
+/* ULP_TX opcodes */
+enum { ULP_MEM_READ = 2, ULP_MEM_WRITE = 3, ULP_TXPKT = 4 };
+
+#define S_ULPTX_CMD    28
+#define M_ULPTX_CMD    0xF
+#define V_ULPTX_CMD(x) ((x) << S_ULPTX_CMD)
+
+#define S_ULPTX_NFLITS    0
+#define M_ULPTX_NFLITS    0xFF
+#define V_ULPTX_NFLITS(x) ((x) << S_ULPTX_NFLITS)
+
+struct ulp_mem_io {
+	WR_HDR;
+	__be32 cmd_lock_addr;
+	__be32 len;
+};
+
+    /* ulp_mem_io.cmd_lock_addr fields */
+#define S_ULP_MEMIO_ADDR    0
+#define M_ULP_MEMIO_ADDR    0x7FFFFFF
+#define V_ULP_MEMIO_ADDR(x) ((x) << S_ULP_MEMIO_ADDR)
+
+#define S_ULP_MEMIO_LOCK    27
+#define V_ULP_MEMIO_LOCK(x) ((x) << S_ULP_MEMIO_LOCK)
+#define F_ULP_MEMIO_LOCK    V_ULP_MEMIO_LOCK(1U)
+
+    /* ulp_mem_io.len fields */
+#define S_ULP_MEMIO_DATA_LEN    28
+#define M_ULP_MEMIO_DATA_LEN    0xF
+#define V_ULP_MEMIO_DATA_LEN(x) ((x) << S_ULP_MEMIO_DATA_LEN)
+
+struct ulp_txpkt {
+	__be32 cmd_dest;
+	__be32 len;
+};
+
+    /* ulp_txpkt.cmd_dest fields */
+#define S_ULP_TXPKT_DEST    24
+#define M_ULP_TXPKT_DEST    0xF
+#define V_ULP_TXPKT_DEST(x) ((x) << S_ULP_TXPKT_DEST)
+
 #endif  /* T3_CPL_H */
diff --git a/sys/dev/cxgb/common/cxgb_t3_hw.c b/sys/dev/cxgb/common/cxgb_t3_hw.c
index fd467e2..c99008e 100644
--- a/sys/dev/cxgb/common/cxgb_t3_hw.c
+++ b/sys/dev/cxgb/common/cxgb_t3_hw.c
@@ -441,23 +441,23 @@ static struct adapter_info t3_adap_info[] = {
 	{ 2, 0, 0, 0,
 	  F_GPIO2_OEN | F_GPIO4_OEN |
 	  F_GPIO2_OUT_VAL | F_GPIO4_OUT_VAL, F_GPIO3 | F_GPIO5,
-	  SUPPORTED_OFFLOAD,
+	  0,
 	  &mi1_mdio_ops, "Chelsio PE9000" },
 	{ 2, 0, 0, 0,
 	  F_GPIO2_OEN | F_GPIO4_OEN |
 	  F_GPIO2_OUT_VAL | F_GPIO4_OUT_VAL, F_GPIO3 | F_GPIO5,
-	  SUPPORTED_OFFLOAD,
+	  0,
 	  &mi1_mdio_ops, "Chelsio T302" },
 	{ 1, 0, 0, 0,
 	  F_GPIO1_OEN | F_GPIO6_OEN | F_GPIO7_OEN | F_GPIO10_OEN |
 	  F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL, 0,
-	  SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_OFFLOAD,
+	  SUPPORTED_10000baseT_Full | SUPPORTED_AUI,
 	  &mi1_mdio_ext_ops, "Chelsio T310" },
 	{ 2, 0, 0, 0,
 	  F_GPIO1_OEN | F_GPIO2_OEN | F_GPIO4_OEN | F_GPIO5_OEN | F_GPIO6_OEN |
 	  F_GPIO7_OEN | F_GPIO10_OEN | F_GPIO11_OEN | F_GPIO1_OUT_VAL |
 	  F_GPIO5_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL, 0,
-	  SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_OFFLOAD,
+	  SUPPORTED_10000baseT_Full | SUPPORTED_AUI,
 	  &mi1_mdio_ext_ops, "Chelsio T320" },
 };
 
@@ -2387,9 +2387,6 @@ static void tp_config(adapter_t *adap, const struct tp_params *p)
 	t3_write_reg(adap, A_TP_MOD_RATE_LIMIT, 0);
 }
 
-/* Desired TP timer resolution in usec */
-#define TP_TMR_RES 200
-
 /* TCP timer values in ms */
 #define TP_DACK_TIMER 50
 #define TP_RTO_MIN    250
@@ -3005,6 +3002,9 @@ static int mc7_init(struct mc7 *mc7, unsigned int mc7_clock, int mem_type)
 	adapter_t *adapter = mc7->adapter;
 	const struct mc7_timing_params *p = &mc7_timings[mem_type];
 
+	if (mc7->size == 0)
+		return 0;
+	
 	val = t3_read_reg(adapter, mc7->offset + A_MC7_CFG);
 	slow = val & F_SLOW;
 	width = G_WIDTH(val);
@@ -3209,9 +3209,10 @@ int t3_init_hw(adapter_t *adapter, u32 fw_params)
 	do {                          /* wait for uP to initialize */
 		t3_os_sleep(20);
 	} while (t3_read_reg(adapter, A_CIM_HOST_ACC_DATA) && --attempts);
-	if (!attempts)
+	if (!attempts) {
+		CH_ERR(adapter, "uP initialization timed out\n");
 		goto out_err;
-
+	}
 	err = 0;
  out_err:
 	return err;
@@ -3309,7 +3310,7 @@ static void __devinit mc7_prep(adapter_t *adapter, struct mc7 *mc7,
 	mc7->name = name;
 	mc7->offset = base_addr - MC7_PMRX_BASE_ADDR;
 	cfg = t3_read_reg(adapter, mc7->offset + A_MC7_CFG);
-	mc7->size = mc7_calc_size(cfg);
+	mc7->size = G_DEN(cfg) == M_DEN ? 0 : mc7_calc_size(cfg);
 	mc7->width = G_WIDTH(cfg);
 }
 
@@ -3336,7 +3337,8 @@ void early_hw_init(adapter_t *adapter, const struct adapter_info *ai)
 		     V_I2C_CLKDIV(adapter->params.vpd.cclk / 80 - 1));
 	t3_write_reg(adapter, A_T3DBG_GPIO_EN,
 		     ai->gpio_out | F_GPIO0_OEN | F_GPIO0_OUT_VAL);
-
+	t3_write_reg(adapter, A_MC5_DB_SERVER_INDEX, 0);
+	
 	if (adapter->params.rev == 0 || !uses_xaui(adapter))
 		val |= F_ENRGMII;
 
@@ -3435,7 +3437,12 @@ int __devinit t3_prep_adapter(adapter_t *adapter,
 		p->ntimer_qs = p->cm_size >= (128 << 20) ||
 			       adapter->params.rev > 0 ? 12 : 6;
 		p->dack_re = fls(adapter->params.vpd.cclk / 10) - 1; /* 100us */
+	}
+	adapter->params.offload = t3_mc7_size(&adapter->pmrx) &&
+	    t3_mc7_size(&adapter->pmtx) &&
+	    t3_mc7_size(&adapter->cm);
 
+	if (is_offload(adapter)) {
 		adapter->params.mc5.nservers = DEFAULT_NSERVERS;
 		adapter->params.mc5.nfilters = adapter->params.rev > 0 ?
 					       DEFAULT_NFILTERS : 0;
diff --git a/sys/dev/cxgb/common/cxgb_version.h b/sys/dev/cxgb/common/cxgb_version.h
index 4f671cf..2e48b52 100644
--- a/sys/dev/cxgb/common/cxgb_version.h
+++ b/sys/dev/cxgb/common/cxgb_version.h
@@ -41,5 +41,5 @@ $FreeBSD$
 #define __CHELSIO_VERSION_H
 #define DRV_DESC "Chelsio T3 Network Driver"
 #define DRV_NAME "cxgb"
-#define DRV_VERSION "1.0.071"
-#endif 
+#define DRV_VERSION "1.0.086"
+#endif
diff --git a/sys/dev/cxgb/common/cxgb_xgmac.c b/sys/dev/cxgb/common/cxgb_xgmac.c
index aeea758..68c388e 100644
--- a/sys/dev/cxgb/common/cxgb_xgmac.c
+++ b/sys/dev/cxgb/common/cxgb_xgmac.c
@@ -329,8 +329,8 @@ int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu)
 	thres = mtu > thres ? (mtu - thres + 7) / 8 : 0;
 	thres = max(thres, 8U);                          /* need at least 8 */
 	t3_set_reg_field(adap, A_XGM_TXFIFO_CFG + mac->offset,
-	                 V_TXFIFOTHRESH(M_TXFIFOTHRESH) | V_TXIPG(M_TXIPG),
-	                 V_TXFIFOTHRESH(thres) | V_TXIPG(1));
+			 V_TXFIFOTHRESH(M_TXFIFOTHRESH) | V_TXIPG(M_TXIPG),
+			 V_TXFIFOTHRESH(thres) | V_TXIPG(1));
 
 	/* Assuming a minimum drain rate of 2.5Gbps...
 	 */
@@ -365,7 +365,7 @@ int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc)
 		t3_set_reg_field(adap, A_XGM_PORT_CFG + oft,
 				 V_PORTSPEED(M_PORTSPEED), val);
 	}
-#if 0 
+#if 0
 	val = t3_read_reg(adap, A_XGM_RXFIFO_CFG + oft);
 	val &= ~V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM);
 	if (fc & PAUSE_TX)
@@ -382,6 +382,7 @@ int t3_mac_enable(struct cmac *mac, int which)
 	int idx = macidx(mac);
 	adapter_t *adap = mac->adapter;
 	unsigned int oft = mac->offset;
+	struct mac_stats *s = &mac->stats;
 
 	if (which & MAC_DIRECTION_TX) {
 		t3_write_reg(adap, A_XGM_TX_CTRL + oft, F_TXEN);
@@ -391,12 +392,20 @@ int t3_mac_enable(struct cmac *mac, int which)
 		t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx, 1 << idx);
 
 		t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CNT_CH0 + idx);
-		mac->tcnt = (G_TXDROPCNTCH0RCVD(t3_read_reg(adap, A_TP_PIO_DATA)));
-		mac->xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, A_XGM_TX_SPI4_SOP_EOP_CNT)));
+		mac->tx_mcnt = s->tx_frames;
+		mac->tx_tcnt = (G_TXDROPCNTCH0RCVD(t3_read_reg(adap,
+							       A_TP_PIO_DATA)));
+		mac->tx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap,
+						A_XGM_TX_SPI4_SOP_EOP_CNT +
+						oft)));
+		mac->rx_mcnt = s->rx_frames;
+		mac->rx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap,
+						A_XGM_RX_SPI4_SOP_EOP_CNT +
+						oft)));
 		mac->txen = F_TXEN;
 		mac->toggle_cnt = 0;
 	}
-	if (which & MAC_DIRECTION_RX)
+	if (which & MAC_DIRECTION_RX) 
 		t3_write_reg(adap, A_XGM_RX_CTRL + oft, F_RXEN);
 	return 0;
 }
@@ -405,6 +414,7 @@ int t3_mac_disable(struct cmac *mac, int which)
 {
 	int idx = macidx(mac);
 	adapter_t *adap = mac->adapter;
+	int val;
 
 	if (which & MAC_DIRECTION_TX) {
 		t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, 0);
@@ -414,39 +424,99 @@ int t3_mac_disable(struct cmac *mac, int which)
 		t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx, 1 << idx);
 		mac->txen = 0;
 	}
-	if (which & MAC_DIRECTION_RX)
+	if (which & MAC_DIRECTION_RX) {
+		t3_set_reg_field(mac->adapter, A_XGM_RESET_CTRL + mac->offset,
+				 F_PCS_RESET_, 0);
+		t3_os_sleep(100);
 		t3_write_reg(adap, A_XGM_RX_CTRL + mac->offset, 0);
+		val = F_MAC_RESET_;
+		if (is_10G(adap))
+			val |= F_PCS_RESET_;
+		else if (uses_xaui(adap))
+			val |= F_PCS_RESET_ | F_XG2G_RESET_;
+		else
+			val |= F_RGMII_RESET_ | F_XG2G_RESET_;
+		t3_write_reg(mac->adapter, A_XGM_RESET_CTRL + mac->offset, val);
+	}
 	return 0;
 }
 
 int t3b2_mac_watchdog_task(struct cmac *mac)
 {
 	int status;
-	unsigned int tcnt, xcnt;
+	unsigned int tx_tcnt, tx_xcnt;
 	adapter_t *adap = mac->adapter;
-	t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CNT_CH0 + macidx(mac));
-	tcnt = (G_TXDROPCNTCH0RCVD(t3_read_reg(adap, A_TP_PIO_DATA)));
-	xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap, A_XGM_TX_SPI4_SOP_EOP_CNT + mac->offset)));
+	struct mac_stats *s = &mac->stats;
+	unsigned int tx_mcnt = (unsigned int)s->tx_frames;
+	unsigned int rx_mcnt = (unsigned int)s->rx_frames;
+	unsigned int rx_xcnt;
+
+	status = 0;
+	tx_xcnt = 1; /* By default tx_xcnt is making progress*/
+	tx_tcnt = mac->tx_tcnt; /* If tx_mcnt is progressing ignore tx_tcnt*/
+	rx_xcnt = 1; /* By default rx_xcnt is making progress*/
+	if (tx_mcnt == mac->tx_mcnt) {
+		tx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap,
+						A_XGM_TX_SPI4_SOP_EOP_CNT +
+					       	mac->offset)));
+		if (tx_xcnt == 0) {
+			t3_write_reg(adap, A_TP_PIO_ADDR,
+			     	A_TP_TX_DROP_CNT_CH0 + macidx(mac));
+			tx_tcnt = (G_TXDROPCNTCH0RCVD(t3_read_reg(adap,
+			      	A_TP_PIO_DATA)));
+		} else {
+			goto rxcheck;
+		}
+	} else {
+		mac->toggle_cnt = 0;
+		goto rxcheck;
+	}
 
-	if ((tcnt != mac->tcnt) && (xcnt == 0) && (mac->xcnt == 0)) {
+	if (((tx_tcnt != mac->tx_tcnt) &&
+	     (tx_xcnt == 0) && (mac->tx_xcnt == 0)) ||
+	    ((mac->tx_mcnt == tx_mcnt) &&
+	     (tx_xcnt != 0) && (mac->tx_xcnt != 0))) {
 		if (mac->toggle_cnt > 4) {
-			t3b2_mac_reset(mac);
-			mac->toggle_cnt = 0;
 			status = 2;
+			goto out;
 		} else {
-			t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, 0);
-			t3_read_reg(adap, A_XGM_TX_CTRL + mac->offset);  /* flush */
-			t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, mac->txen);
-			t3_read_reg(adap, A_XGM_TX_CTRL + mac->offset);  /* flush */
-			mac->toggle_cnt++;
 			status = 1;
-		}	
+			goto out;
+		}
 	} else {
 		mac->toggle_cnt = 0;
-		status = 0;
+		goto rxcheck;
+	}
+
+rxcheck:
+	if (rx_mcnt != mac->rx_mcnt)
+		rx_xcnt = (G_TXSPI4SOPCNT(t3_read_reg(adap,
+						A_XGM_RX_SPI4_SOP_EOP_CNT +
+						mac->offset)));
+	else 
+		goto out;
+
+	if (mac->rx_mcnt != s->rx_frames && rx_xcnt == 0 && mac->rx_xcnt == 0) {
+		status = 2;
+		goto out;
+	}
+	
+out:	
+	mac->tx_tcnt = tx_tcnt;
+	mac->tx_xcnt = tx_xcnt;
+	mac->tx_mcnt = s->tx_frames;
+	mac->rx_xcnt = rx_xcnt;
+	mac->rx_mcnt = s->rx_frames;
+	if (status == 1) {
+		t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, 0);
+		t3_read_reg(adap, A_XGM_TX_CTRL + mac->offset);  /* flush */
+		t3_write_reg(adap, A_XGM_TX_CTRL + mac->offset, mac->txen);
+		t3_read_reg(adap, A_XGM_TX_CTRL + mac->offset);  /* flush */
+		mac->toggle_cnt++;
+	} else if (status == 2) {
+		t3b2_mac_reset(mac);
+		mac->toggle_cnt = 0;
 	}
-	mac->tcnt = tcnt;
-	mac->xcnt = xcnt;
 	return status;
 }
 
diff --git a/sys/dev/cxgb/common/jhash.h b/sys/dev/cxgb/common/jhash.h
new file mode 100644
index 0000000..4546b7b
--- /dev/null
+++ b/sys/dev/cxgb/common/jhash.h
@@ -0,0 +1,140 @@
+#ifndef _JHASH_H
+#define _JHASH_H
+
+/* jhash.h: Jenkins hash support.
+ *
+ * Copyright (C) 1996 Bob Jenkins (bob_jenkins@burtleburtle.net)
+ *
+ * http://burtleburtle.net/bob/hash/
+ *
+ * These are the credits from Bob's sources:
+ *
+ * lookup2.c, by Bob Jenkins, December 1996, Public Domain.
+ * hash(), hash2(), hash3, and mix() are externally useful functions.
+ * Routines to test the hash are included if SELF_TEST is defined.
+ * You can use this free for any purpose.  It has no warranty.
+ *
+ * $FreeBSD$
+ */
+
+/* NOTE: Arguments are modified. */
+#define __jhash_mix(a, b, c) \
+{ \
+  a -= b; a -= c; a ^= (c>>13); \
+  b -= c; b -= a; b ^= (a<<8); \
+  c -= a; c -= b; c ^= (b>>13); \
+  a -= b; a -= c; a ^= (c>>12);  \
+  b -= c; b -= a; b ^= (a<<16); \
+  c -= a; c -= b; c ^= (b>>5); \
+  a -= b; a -= c; a ^= (c>>3);  \
+  b -= c; b -= a; b ^= (a<<10); \
+  c -= a; c -= b; c ^= (b>>15); \
+}
+
+/* The golden ration: an arbitrary value */
+#define JHASH_GOLDEN_RATIO	0x9e3779b9
+
+/* The most generic version, hashes an arbitrary sequence
+ * of bytes.  No alignment or length assumptions are made about
+ * the input key.
+ */
+static inline u32 jhash(const void *key, u32 length, u32 initval)
+{
+	u32 a, b, c, len;
+	const u8 *k = key;
+
+	len = length;
+	a = b = JHASH_GOLDEN_RATIO;
+	c = initval;
+
+	while (len >= 12) {
+		a += (k[0] +((u32)k[1]<<8) +((u32)k[2]<<16) +((u32)k[3]<<24));
+		b += (k[4] +((u32)k[5]<<8) +((u32)k[6]<<16) +((u32)k[7]<<24));
+		c += (k[8] +((u32)k[9]<<8) +((u32)k[10]<<16)+((u32)k[11]<<24));
+
+		__jhash_mix(a,b,c);
+
+		k += 12;
+		len -= 12;
+	}
+
+	c += length;
+	switch (len) {
+	case 11: c += ((u32)k[10]<<24);
+	case 10: c += ((u32)k[9]<<16);
+	case 9 : c += ((u32)k[8]<<8);
+	case 8 : b += ((u32)k[7]<<24);
+	case 7 : b += ((u32)k[6]<<16);
+	case 6 : b += ((u32)k[5]<<8);
+	case 5 : b += k[4];
+	case 4 : a += ((u32)k[3]<<24);
+	case 3 : a += ((u32)k[2]<<16);
+	case 2 : a += ((u32)k[1]<<8);
+	case 1 : a += k[0];
+	};
+
+	__jhash_mix(a,b,c);
+
+	return c;
+}
+
+/* A special optimized version that handles 1 or more of u32s.
+ * The length parameter here is the number of u32s in the key.
+ */
+static inline u32 jhash2(u32 *k, u32 length, u32 initval)
+{
+	u32 a, b, c, len;
+
+	a = b = JHASH_GOLDEN_RATIO;
+	c = initval;
+	len = length;
+
+	while (len >= 3) {
+		a += k[0];
+		b += k[1];
+		c += k[2];
+		__jhash_mix(a, b, c);
+		k += 3; len -= 3;
+	}
+
+	c += length * 4;
+
+	switch (len) {
+	case 2 : b += k[1];
+	case 1 : a += k[0];
+	};
+
+	__jhash_mix(a,b,c);
+
+	return c;
+}
+
+
+/* A special ultra-optimized versions that knows they are hashing exactly
+ * 3, 2 or 1 word(s).
+ *
+ * NOTE: In partilar the "c += length; __jhash_mix(a,b,c);" normally
+ *       done at the end is not done here.
+ */
+static inline u32 jhash_3words(u32 a, u32 b, u32 c, u32 initval)
+{
+	a += JHASH_GOLDEN_RATIO;
+	b += JHASH_GOLDEN_RATIO;
+	c += initval;
+
+	__jhash_mix(a, b, c);
+
+	return c;
+}
+
+static inline u32 jhash_2words(u32 a, u32 b, u32 initval)
+{
+	return jhash_3words(a, b, 0, initval);
+}
+
+static inline u32 jhash_1word(u32 a, u32 initval)
+{
+	return jhash_3words(a, 0, 0, initval);
+}
+
+#endif /* _JHASH_H */
diff --git a/sys/dev/cxgb/cxgb_adapter.h b/sys/dev/cxgb/cxgb_adapter.h
index 7d8a787..f5c8a4c 100644
--- a/sys/dev/cxgb/cxgb_adapter.h
+++ b/sys/dev/cxgb/cxgb_adapter.h
@@ -9,11 +9,7 @@ modification, are permitted provided that the following conditions are met:
  1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
 
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Chelsio Corporation nor the names of its
+ 2. Neither the name of the Chelsio Corporation nor the names of its
     contributors may be used to endorse or promote products derived from
     this software without specific prior written permission.
 
@@ -59,6 +55,9 @@ __FBSDID("$FreeBSD$");
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 
+#include <dev/cxgb/ulp/toecore/toedev.h>
+#include <dev/cxgb/sys/mbufq.h>
+
 struct adapter;
 struct sge_qset;
 extern int cxgb_debug;
@@ -91,9 +90,6 @@ enum {				/* adapter flags */
 	FW_UPTODATE     = (1 << 4),
 };
 
-/* Max active LRO sessions per queue set */
-#define MAX_LRO_PER_QSET 8
-
 
 #define FL_Q_SIZE	4096
 #define JUMBO_Q_SIZE	512
@@ -114,25 +110,23 @@ enum {
 	LRO_ACTIVE = (1 << 8),
 };
 
-struct sge_lro_session {
-	struct mbuf *m;
+/* Max concurrent LRO sessions per queue set */
+#define MAX_LRO_SES 8
+
+struct t3_lro_session {
+	struct mbuf *head;
+	struct mbuf *tail;
 	uint32_t seq;
 	uint16_t ip_len;
+	uint16_t vtag;
+	uint8_t npkts;
 };
 
-struct sge_lro {
-	unsigned int enabled;
-	unsigned int num_active;
-	struct sge_lro_session *last_s;
-	struct sge_lro_session s[MAX_LRO_PER_QSET];
-};
-
-/* has its own header on linux XXX
- * but I don't even know what it is :-/
- */
-
-struct t3cdev {
-	int foo; /* XXX fill in */
+struct lro_state {
+	unsigned short enabled;
+	unsigned short active_idx;
+	unsigned int nactive;
+	struct t3_lro_session sess[MAX_LRO_SES];
 };
 
 #define RX_BUNDLE_SIZE 8
@@ -148,14 +142,21 @@ struct sge_rspq {
 	uint32_t	holdoff_tmr;
 	uint32_t	next_holdoff;
 	uint32_t        imm_data;
-	uint32_t        pure_rsps;
 	struct rsp_desc	*desc;
-	bus_addr_t	phys_addr;
 	uint32_t	cntxt_id;
+	struct mtx      lock;
+	struct mbuf     *rx_head;    /* offload packet receive queue head */
+	struct mbuf     *rx_tail;    /* offload packet receive queue tail */
+
+	uint32_t        offload_pkts;
+	uint32_t        offload_bundles;
+	uint32_t        pure_rsps;
+	uint32_t        unhandled_irqs;
+
+	bus_addr_t	phys_addr;
 	bus_dma_tag_t	desc_tag;
 	bus_dmamap_t	desc_map;
 	struct mbuf	*m;
-	struct mtx      lock;
 };
 
 struct rx_desc;
@@ -198,12 +199,14 @@ struct sge_txq {
 	struct tx_sw_desc *sdesc;
 	uint32_t	token;
 	bus_addr_t	phys_addr;
+	struct task     qresume_tsk;
 	uint32_t	cntxt_id;
 	uint64_t	stops;
 	uint64_t	restarts;
 	bus_dma_tag_t	desc_tag;
 	bus_dmamap_t	desc_map;
 	bus_dma_tag_t   entry_tag;
+	struct mbuf_head sendq;
 	struct mtx      lock;
 };
      	
@@ -224,9 +227,9 @@ enum {
 struct sge_qset {
 	struct sge_rspq		rspq;
 	struct sge_fl		fl[SGE_RXQ_PER_SET];
-	struct sge_lro          lro;
+	struct lro_state        lro;
 	struct sge_txq		txq[SGE_TXQ_PER_SET];
-       	unsigned long           txq_stopped;       /* which Tx queues are stopped */
+	uint32_t                txq_stopped;       /* which Tx queues are stopped */
 	uint64_t                port_stats[SGE_PSTAT_MAX];
 	struct port_info        *port;
 	int                     idx; /* qset # */
@@ -240,6 +243,7 @@ struct sge {
 struct adapter {
 	device_t		dev;
 	int			flags;
+	TAILQ_ENTRY(adapter)    adapter_entry;
 	
 	/* PCI register resources */
 	uint32_t		regs_rid;
@@ -248,6 +252,7 @@ struct adapter {
 	bus_space_tag_t		bt;
 	bus_size_t              mmio_len;
 	uint32_t                link_width;
+				
 	
 	/* DMA resources */
 	bus_dma_tag_t		parent_dmat;
@@ -293,10 +298,13 @@ struct adapter {
 
 	struct port_info	port[MAX_NPORTS];
 	device_t		portdev[MAX_NPORTS];
-	struct t3cdev           tdev;
+	struct toedev           tdev;
 	char                    fw_version[64];
 	uint32_t                open_device_map;
+	uint32_t                registered_device_map;
 	struct mtx              lock;
+	driver_intr_t           *cxgb_intr;
+	int                     msi_count;
 };
 
 struct t3_rx_mode {
@@ -384,6 +392,7 @@ int t3_os_pci_restore_state(struct adapter *adapter);
 void t3_os_link_changed(adapter_t *adapter, int port_id, int link_status,
 			int speed, int duplex, int fc);
 void t3_sge_err_intr_handler(adapter_t *adapter);
+int t3_offload_tx(struct toedev *, struct mbuf *);
 void t3_os_ext_intr_handler(adapter_t *adapter);
 void t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[]);
 int t3_mgmt_tx(adapter_t *adap, struct mbuf *m);
@@ -395,6 +404,7 @@ int t3_sge_alloc_qset(adapter_t *, uint32_t, int, int, const struct qset_params
     int, struct port_info *);
 void t3_free_sge_resources(adapter_t *);
 void t3_sge_start(adapter_t *);
+void t3_sge_stop(adapter_t *);
 void t3b_intr(void *data);
 void t3_intr_msi(void *data);
 void t3_intr_msix(void *data);
@@ -406,7 +416,7 @@ void t3_sge_deinit_sw(adapter_t *);
 void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
     int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro);
 void t3_rx_eth(struct port_info *p, struct sge_rspq *rq, struct mbuf *m, int ethpad);
-void t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs);
+void t3_lro_flush(adapter_t *adap, struct sge_qset *qs, struct lro_state *state);
 
 void t3_add_sysctls(adapter_t *sc);
 int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx,
@@ -437,6 +447,19 @@ txq_to_qset(struct sge_txq *q, int qidx)
 	return container_of(q, struct sge_qset, txq[qidx]);
 }
 
+static __inline struct adapter *
+tdev2adap(struct toedev *d)
+{
+	return container_of(d, struct adapter, tdev);
+}
+
 #undef container_of
 
+#define OFFLOAD_DEVMAP_BIT 15
+static inline int offload_running(adapter_t *adapter)
+{
+        return isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
+}
+
+
 #endif
diff --git a/sys/dev/cxgb/cxgb_config.h b/sys/dev/cxgb/cxgb_config.h
index e5168ad..446c743 100644
--- a/sys/dev/cxgb/cxgb_config.h
+++ b/sys/dev/cxgb/cxgb_config.h
@@ -9,11 +9,7 @@ modification, are permitted provided that the following conditions are met:
  1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
 
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Chelsio Corporation nor the names of its
+ 2. Neither the name of the Chelsio Corporation nor the names of its
     contributors may be used to endorse or promote products derived from
     this software without specific prior written permission.
 
diff --git a/sys/dev/cxgb/cxgb_ioctl.h b/sys/dev/cxgb/cxgb_ioctl.h
index 79e5fa6..d725a9a 100644
--- a/sys/dev/cxgb/cxgb_ioctl.h
+++ b/sys/dev/cxgb/cxgb_ioctl.h
@@ -9,11 +9,7 @@ modification, are permitted provided that the following conditions are met:
  1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
 
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Chelsio Corporation nor the names of its
+ 2. Neither the name of the Chelsio Corporation nor the names of its
     contributors may be used to endorse or promote products derived from
     this software without specific prior written permission.
 
@@ -68,7 +64,8 @@ enum {
 	CH_IFCONF_GETREGS,
 	CH_GETMIIREGS,
 	CH_SETMIIREGS,
-	
+	CH_SET_FILTER,
+	CH_SET_HW_SCHED,
 };
 
 struct ch_reg {
@@ -217,6 +214,7 @@ struct mii_data {
 
 #define CHELSIO_SETREG              _IOW('f', CH_SETREG, struct ch_reg)
 #define CHELSIO_GETREG              _IOWR('f', CH_GETREG, struct ch_reg)
+#define CHELSIO_READ_TCAM_WORD      _IOR('f', CH_READ_TCAM_WORD, struct ch_tcam)
 #define CHELSIO_GET_MEM             _IOWR('f', CH_GET_MEM, struct ch_mem_range)
 #define CHELSIO_GET_SGE_CONTEXT     _IOWR('f', CH_GET_SGE_CONTEXT, struct ch_cntxt)
 #define CHELSIO_GET_SGE_DESC        _IOWR('f', CH_GET_SGE_DESC, struct ch_desc)
@@ -224,6 +222,8 @@ struct mii_data {
 #define CHELSIO_SET_QSET_PARAMS     _IOW('f', CH_SET_QSET_PARAMS, struct ch_qset_params)
 #define CHELSIO_GET_QSET_NUM        _IOWR('f', CH_GET_QSET_NUM, struct ch_reg)
 #define CHELSIO_SET_QSET_NUM        _IOW('f', CH_SET_QSET_NUM, struct ch_reg)
+#define CHELSIO_GETMTUTAB           _IOR('f', CH_GET_QSET_NUM, struct ch_mtus)
+#define CHELSIO_SETMTUTAB           _IOW('f', CH_SET_QSET_NUM, struct ch_mtus)
 
 
 #define CHELSIO_SET_TRACE_FILTER    _IOW('f', CH_SET_TRACE_FILTER, struct ch_trace)
@@ -231,4 +231,6 @@ struct mii_data {
 #define CHELSIO_IFCONF_GETREGS      _IOWR('f', CH_IFCONF_GETREGS, struct ifconf_regs)
 #define SIOCGMIIREG                 _IOWR('f', CH_GETMIIREGS, struct mii_data)
 #define SIOCSMIIREG                 _IOWR('f', CH_SETMIIREGS, struct mii_data)
+#define CHELSIO_SET_HW_SCHED        _IOWR('f', CH_SET_HW_SCHED, struct ch_hw_sched)
+#define CHELSIO_DEVUP               _IO('f', CH_DEVUP)
 #endif
diff --git a/sys/dev/cxgb/cxgb_l2t.c b/sys/dev/cxgb/cxgb_l2t.c
new file mode 100644
index 0000000..8cc8211
--- /dev/null
+++ b/sys/dev/cxgb/cxgb_l2t.c
@@ -0,0 +1,670 @@
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+ 
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+***************************************************************************/
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/bus.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/rwlock.h>
+
+
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <net/if.h>
+#include <net/ethernet.h>
+#include <net/if_vlan_var.h>
+#include <net/if_dl.h>
+#include <net/route.h>
+#include <netinet/in.h>
+#include <netinet/if_ether.h>
+
+
+#include <dev/cxgb/common/cxgb_common.h>
+#include <dev/cxgb/common/cxgb_regs.h>
+#include <dev/cxgb/common/cxgb_sge_defs.h>
+#include <dev/cxgb/common/cxgb_t3_cpl.h>
+#include <dev/cxgb/common/cxgb_firmware_exports.h>
+#include <dev/cxgb/common/jhash.h>
+#include <dev/cxgb/cxgb_offload.h>
+
+#define VLAN_NONE 0xfff
+#define SDL(s) ((struct sockaddr_dl *)s) 
+#define RT_ENADDR(rt)  ((char *)LLADDR(SDL((rt))))
+#define rt_expire rt_rmx.rmx_expire 
+
+struct llinfo_arp { 
+        struct  callout la_timer; 
+        struct  rtentry *la_rt; 
+        struct  mbuf *la_hold;  /* last packet until resolved/timeout */ 
+        u_short la_preempt;     /* countdown for pre-expiry arps */ 
+        u_short la_asked;       /* # requests sent */ 
+}; 
+
+/*
+ * Module locking notes:  There is a RW lock protecting the L2 table as a
+ * whole plus a spinlock per L2T entry.  Entry lookups and allocations happen
+ * under the protection of the table lock, individual entry changes happen
+ * while holding that entry's spinlock.  The table lock nests outside the
+ * entry locks.  Allocations of new entries take the table lock as writers so
+ * no other lookups can happen while allocating new entries.  Entry updates
+ * take the table lock as readers so multiple entries can be updated in
+ * parallel.  An L2T entry can be dropped by decrementing its reference count
+ * and therefore can happen in parallel with entry allocation but no entry
+ * can change state or increment its ref count during allocation as both of
+ * these perform lookups.
+ */
+
+static inline unsigned int
+vlan_prio(const struct l2t_entry *e)
+{
+	return e->vlan >> 13;
+}
+
+static inline unsigned int
+arp_hash(u32 key, int ifindex, const struct l2t_data *d)
+{
+	return jhash_2words(key, ifindex, 0) & (d->nentries - 1);
+}
+
+static inline void
+neigh_replace(struct l2t_entry *e, struct rtentry *rt)
+{
+	RT_LOCK(rt);
+	RT_ADDREF(rt);
+	RT_UNLOCK(rt);
+	
+	if (e->neigh) {
+		RT_LOCK(e->neigh);
+		RT_REMREF(e->neigh);
+		RT_UNLOCK(e->neigh);
+	}
+	e->neigh = rt;
+}
+
+/*
+ * Set up an L2T entry and send any packets waiting in the arp queue.  The
+ * supplied mbuf is used for the CPL_L2T_WRITE_REQ.  Must be called with the
+ * entry locked.
+ */
+static int
+setup_l2e_send_pending(struct toedev *dev, struct mbuf *m,
+			struct l2t_entry *e)
+{
+	struct cpl_l2t_write_req *req;
+
+	if (!m) {
+		if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
+		    return (ENOMEM);
+	}
+	/*
+	 * XXX MH_ALIGN
+	 */
+	req = mtod(m, struct cpl_l2t_write_req *);
+	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, e->idx));
+	req->params = htonl(V_L2T_W_IDX(e->idx) | V_L2T_W_IFF(e->smt_idx) |
+			    V_L2T_W_VLAN(e->vlan & EVL_VLID_MASK) |
+			    V_L2T_W_PRIO(vlan_prio(e)));
+
+	memcpy(e->dmac, RT_ENADDR(e->neigh), sizeof(e->dmac));
+	memcpy(req->dst_mac, e->dmac, sizeof(req->dst_mac));
+	m_set_priority(m, CPL_PRIORITY_CONTROL);
+	cxgb_ofld_send(dev, m);
+	while (e->arpq_head) {
+		m = e->arpq_head;
+		e->arpq_head = m->m_next;
+		m->m_next = NULL;
+		cxgb_ofld_send(dev, m);
+	}
+	e->arpq_tail = NULL;
+	e->state = L2T_STATE_VALID;
+
+	return 0;
+}
+
+/*
+ * Add a packet to the an L2T entry's queue of packets awaiting resolution.
+ * Must be called with the entry's lock held.
+ */
+static inline void
+arpq_enqueue(struct l2t_entry *e, struct mbuf *m)
+{
+	m->m_next = NULL;
+	if (e->arpq_head)
+		e->arpq_tail->m_next = m;
+	else
+		e->arpq_head = m;
+	e->arpq_tail = m;
+}
+
+int
+t3_l2t_send_slow(struct toedev *dev, struct mbuf *m,
+		     struct l2t_entry *e)
+{
+	struct rtentry *rt;
+	struct mbuf *m0;
+	
+	if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
+		return (ENOMEM);
+
+	rt = e->neigh;
+	
+again:
+	switch (e->state) {
+	case L2T_STATE_STALE:     /* entry is stale, kick off revalidation */
+		arpresolve(rt->rt_ifp, rt, m0, rt->rt_gateway, RT_ENADDR(rt));
+		mtx_lock(&e->lock);
+		if (e->state == L2T_STATE_STALE)
+			e->state = L2T_STATE_VALID;
+		mtx_unlock(&e->lock);
+	case L2T_STATE_VALID:     /* fast-path, send the packet on */
+		return cxgb_ofld_send(dev, m);
+	case L2T_STATE_RESOLVING:
+		mtx_lock(&e->lock);
+		if (e->state != L2T_STATE_RESOLVING) { // ARP already completed
+			mtx_unlock(&e->lock);
+			goto again;
+		}
+		arpq_enqueue(e, m);
+		mtx_unlock(&e->lock);
+
+		if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
+			return (ENOMEM);
+		/*
+		 * Only the first packet added to the arpq should kick off
+		 * resolution.  However, because the m_gethdr below can fail,
+		 * we allow each packet added to the arpq to retry resolution
+		 * as a way of recovering from transient memory exhaustion.
+		 * A better way would be to use a work request to retry L2T
+		 * entries when there's no memory.
+		 */
+		if (arpresolve(rt->rt_ifp, rt, m0, rt->rt_gateway, RT_ENADDR(rt)) == 0) {
+
+			mtx_lock(&e->lock);
+			if (e->arpq_head) 
+				setup_l2e_send_pending(dev, m, e);
+			else
+				m_freem(m);
+			mtx_unlock(&e->lock);
+		}
+	}
+	return 0;
+}
+
+void
+t3_l2t_send_event(struct toedev *dev, struct l2t_entry *e)
+{
+	struct rtentry *rt;
+	struct mbuf *m0;
+	
+	if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
+		return;
+
+	rt = e->neigh;
+again:
+	switch (e->state) {
+	case L2T_STATE_STALE:     /* entry is stale, kick off revalidation */
+		arpresolve(rt->rt_ifp, rt, m0, rt->rt_gateway, RT_ENADDR(rt));
+		mtx_lock(&e->lock);
+		if (e->state == L2T_STATE_STALE) {
+			e->state = L2T_STATE_VALID;
+		}
+		mtx_unlock(&e->lock);
+		return;
+	case L2T_STATE_VALID:     /* fast-path, send the packet on */
+		return;
+	case L2T_STATE_RESOLVING:
+		mtx_lock(&e->lock);
+		if (e->state != L2T_STATE_RESOLVING) { // ARP already completed
+			mtx_unlock(&e->lock);
+			goto again;
+		}
+		mtx_unlock(&e->lock);
+		
+		if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
+			return;
+		/*
+		 * Only the first packet added to the arpq should kick off
+		 * resolution.  However, because the alloc_skb below can fail,
+		 * we allow each packet added to the arpq to retry resolution
+		 * as a way of recovering from transient memory exhaustion.
+		 * A better way would be to use a work request to retry L2T
+		 * entries when there's no memory.
+		 */
+		arpresolve(rt->rt_ifp, rt, m0, rt->rt_gateway, RT_ENADDR(rt));
+
+	}
+	return;
+}
+/*
+ * Allocate a free L2T entry.  Must be called with l2t_data.lock held.
+ */
+static struct l2t_entry *
+alloc_l2e(struct l2t_data *d)
+{
+	struct l2t_entry *end, *e, **p;
+
+	if (!atomic_load_acq_int(&d->nfree))
+		return NULL;
+
+	/* there's definitely a free entry */
+	for (e = d->rover, end = &d->l2tab[d->nentries]; e != end; ++e)
+		if (atomic_load_acq_int(&e->refcnt) == 0)
+			goto found;
+
+	for (e = &d->l2tab[1]; atomic_load_acq_int(&e->refcnt); ++e) ;
+found:
+	d->rover = e + 1;
+	atomic_add_int(&d->nfree, -1);
+
+	/*
+	 * The entry we found may be an inactive entry that is
+	 * presently in the hash table.  We need to remove it.
+	 */
+	if (e->state != L2T_STATE_UNUSED) {
+		int hash = arp_hash(e->addr, e->ifindex, d);
+
+		for (p = &d->l2tab[hash].first; *p; p = &(*p)->next)
+			if (*p == e) {
+				*p = e->next;
+				break;
+			}
+		e->state = L2T_STATE_UNUSED;
+	}
+	return e;
+}
+
+/*
+ * Called when an L2T entry has no more users.  The entry is left in the hash
+ * table since it is likely to be reused but we also bump nfree to indicate
+ * that the entry can be reallocated for a different neighbor.  We also drop
+ * the existing neighbor reference in case the neighbor is going away and is
+ * waiting on our reference.
+ *
+ * Because entries can be reallocated to other neighbors once their ref count
+ * drops to 0 we need to take the entry's lock to avoid races with a new
+ * incarnation.
+ */
+void
+t3_l2e_free(struct l2t_data *d, struct l2t_entry *e)
+{
+	mtx_lock(&e->lock);
+	if (atomic_load_acq_int(&e->refcnt) == 0) {  /* hasn't been recycled */
+		if (e->neigh) {
+			RT_LOCK(e->neigh);
+			RT_REMREF(e->neigh);
+			RT_UNLOCK(e->neigh);
+			e->neigh = NULL;
+		}
+	}
+	mtx_unlock(&e->lock);
+	atomic_add_int(&d->nfree, 1);
+}
+
+/*
+ * Update an L2T entry that was previously used for the same next hop as neigh.
+ * Must be called with softirqs disabled.
+ */
+static inline void
+reuse_entry(struct l2t_entry *e, struct rtentry *neigh)
+{
+	struct llinfo_arp *la;
+
+	la = (struct llinfo_arp *)neigh->rt_llinfo; 
+
+	mtx_lock(&e->lock);                /* avoid race with t3_l2t_free */
+	if (neigh != e->neigh)
+		neigh_replace(e, neigh);
+	
+	if (memcmp(e->dmac, RT_ENADDR(neigh), sizeof(e->dmac)) ||
+	    (neigh->rt_expire > time_uptime))
+		e->state = L2T_STATE_RESOLVING;
+	else if (la->la_hold == NULL)
+		e->state = L2T_STATE_VALID;
+	else
+		e->state = L2T_STATE_STALE;
+	mtx_unlock(&e->lock);
+}
+
+struct l2t_entry *
+t3_l2t_get(struct toedev *dev, struct rtentry *neigh,
+			     unsigned int smt_idx)
+{
+	struct l2t_entry *e;
+	struct l2t_data *d = L2DATA(dev);
+	u32 addr = *(u32 *) rt_key(neigh);
+	int ifidx = neigh->rt_ifp->if_index;
+	int hash = arp_hash(addr, ifidx, d);
+
+	rw_wlock(&d->lock);
+	for (e = d->l2tab[hash].first; e; e = e->next)
+		if (e->addr == addr && e->ifindex == ifidx &&
+		    e->smt_idx == smt_idx) {
+			l2t_hold(d, e);
+			if (atomic_load_acq_int(&e->refcnt) == 1)
+				reuse_entry(e, neigh);
+			goto done;
+		}
+
+	/* Need to allocate a new entry */
+	e = alloc_l2e(d);
+	if (e) {
+		mtx_lock(&e->lock);          /* avoid race with t3_l2t_free */
+		e->next = d->l2tab[hash].first;
+		d->l2tab[hash].first = e;
+		e->state = L2T_STATE_RESOLVING;
+		e->addr = addr;
+		e->ifindex = ifidx;
+		e->smt_idx = smt_idx;
+		atomic_store_rel_int(&e->refcnt, 1);
+		neigh_replace(e, neigh);
+#ifdef notyet
+		/* 
+		 * XXX need to add accessor function for vlan tag
+		 */
+		if (neigh->rt_ifp->if_vlantrunk)
+			e->vlan = VLAN_DEV_INFO(neigh->dev)->vlan_id;
+		else
+#endif			    
+			e->vlan = VLAN_NONE;
+		mtx_unlock(&e->lock);
+	}
+done:
+	rw_wunlock(&d->lock);
+	return e;
+}
+
+/*
+ * Called when address resolution fails for an L2T entry to handle packets
+ * on the arpq head.  If a packet specifies a failure handler it is invoked,
+ * otherwise the packets is sent to the TOE.
+ *
+ * XXX: maybe we should abandon the latter behavior and just require a failure
+ * handler.
+ */
+static void
+handle_failed_resolution(struct toedev *dev, struct mbuf *arpq)
+{
+
+	while (arpq) {
+		struct mbuf *m = arpq;
+#ifdef notyet		
+		struct l2t_mbuf_cb *cb = L2T_MBUF_CB(m);
+#endif
+		arpq = m->m_next;
+		m->m_next = NULL;
+#ifdef notyet		
+		if (cb->arp_failure_handler)
+			cb->arp_failure_handler(dev, m);
+		else
+#endif			
+			cxgb_ofld_send(dev, m);
+	}
+
+}
+
+#if defined(NETEVENT) || !defined(CONFIG_CHELSIO_T3_MODULE)
+/*
+ * Called when the host's ARP layer makes a change to some entry that is
+ * loaded into the HW L2 table.
+ */
+void
+t3_l2t_update(struct toedev *dev, struct rtentry *neigh)
+{
+	struct l2t_entry *e;
+	struct mbuf *arpq = NULL;
+	struct l2t_data *d = L2DATA(dev);
+	u32 addr = *(u32 *) rt_key(neigh);
+	int ifidx = neigh->rt_ifp->if_index;
+	int hash = arp_hash(addr, ifidx, d);
+	struct llinfo_arp *la;
+	
+	rw_rlock(&d->lock);
+	for (e = d->l2tab[hash].first; e; e = e->next)
+		if (e->addr == addr && e->ifindex == ifidx) {
+			mtx_lock(&e->lock);
+			goto found;
+		}
+	rw_runlock(&d->lock);
+	return;
+
+found:
+	rw_runlock(&d->lock);
+	if (atomic_load_acq_int(&e->refcnt)) {
+		if (neigh != e->neigh)
+			neigh_replace(e, neigh);
+		
+		la = (struct llinfo_arp *)neigh->rt_llinfo; 
+		if (e->state == L2T_STATE_RESOLVING) {
+			
+			if (la->la_asked >= 5 /* arp_maxtries */) {
+				arpq = e->arpq_head;
+				e->arpq_head = e->arpq_tail = NULL;
+			} else if (la->la_hold == NULL)
+				setup_l2e_send_pending(dev, NULL, e);
+		} else {
+			e->state = (la->la_hold == NULL) ?
+				L2T_STATE_VALID : L2T_STATE_STALE;
+			if (memcmp(e->dmac, RT_ENADDR(neigh), 6))
+				setup_l2e_send_pending(dev, NULL, e);
+		}
+	}
+	mtx_unlock(&e->lock);
+
+	if (arpq)
+		handle_failed_resolution(dev, arpq);
+}
+#else
+/*
+ * Called from a kprobe, interrupts are off.
+ */
+void
+t3_l2t_update(struct toedev *dev, struct rtentry *neigh)
+{
+	struct l2t_entry *e;
+	struct l2t_data *d = L2DATA(dev);
+	u32 addr = *(u32 *) rt_key(neigh);
+	int ifidx = neigh->dev->ifindex;
+	int hash = arp_hash(addr, ifidx, d);
+
+	rw_rlock(&d->lock);
+	for (e = d->l2tab[hash].first; e; e = e->next)
+		if (e->addr == addr && e->ifindex == ifidx) {
+			mtx_lock(&e->lock);
+			if (atomic_load_acq_int(&e->refcnt)) {
+				if (neigh != e->neigh)
+					neigh_replace(e, neigh);
+				e->tdev = dev;
+				mod_timer(&e->update_timer, jiffies + 1);
+			}
+			mtx_unlock(&e->lock);
+			break;
+		}
+	rw_runlock(&d->lock);
+}
+
+static void
+update_timer_cb(unsigned long data)
+{
+	struct mbuf *arpq = NULL;
+	struct l2t_entry *e = (struct l2t_entry *)data;
+	struct rtentry *neigh = e->neigh;
+	struct toedev *dev = e->tdev;
+
+	barrier();
+	if (!atomic_load_acq_int(&e->refcnt))
+		return;
+
+	rw_rlock(&neigh->lock);
+	mtx_lock(&e->lock);
+
+	if (atomic_load_acq_int(&e->refcnt)) {
+		if (e->state == L2T_STATE_RESOLVING) {
+			if (neigh->nud_state & NUD_FAILED) {
+				arpq = e->arpq_head;
+				e->arpq_head = e->arpq_tail = NULL;
+			} else if (neigh_is_connected(neigh) && e->arpq_head)
+				setup_l2e_send_pending(dev, NULL, e);
+		} else {
+			e->state = neigh_is_connected(neigh) ?
+				L2T_STATE_VALID : L2T_STATE_STALE;
+			if (memcmp(e->dmac, RT_ENADDR(neigh), sizeof(e->dmac)))
+				setup_l2e_send_pending(dev, NULL, e);
+		}
+	}
+	mtx_unlock(&e->lock);
+	rw_runlock(&neigh->lock);
+
+	if (arpq)
+		handle_failed_resolution(dev, arpq);
+}
+#endif
+
+struct l2t_data *
+t3_init_l2t(unsigned int l2t_capacity)
+{
+	struct l2t_data *d;
+	int i, size = sizeof(*d) + l2t_capacity * sizeof(struct l2t_entry);
+
+	d = cxgb_alloc_mem(size);
+	if (!d)
+		return NULL;
+
+	d->nentries = l2t_capacity;
+	d->rover = &d->l2tab[1];	/* entry 0 is not used */
+	atomic_store_rel_int(&d->nfree, l2t_capacity - 1);
+	rw_init(&d->lock, "L2T");
+
+	for (i = 0; i < l2t_capacity; ++i) {
+		d->l2tab[i].idx = i;
+		d->l2tab[i].state = L2T_STATE_UNUSED;
+		mtx_init(&d->l2tab[i].lock, "L2TAB", NULL, MTX_DEF);
+		atomic_store_rel_int(&d->l2tab[i].refcnt, 0);
+#ifndef NETEVENT
+#ifdef CONFIG_CHELSIO_T3_MODULE
+		setup_timer(&d->l2tab[i].update_timer, update_timer_cb,
+			    (unsigned long)&d->l2tab[i]);
+#endif
+#endif
+	}
+	return d;
+}
+
+void
+t3_free_l2t(struct l2t_data *d)
+{
+#ifndef NETEVENT
+#ifdef CONFIG_CHELSIO_T3_MODULE
+	int i;
+
+	/* Stop all L2T timers */
+	for (i = 0; i < d->nentries; ++i)
+		del_timer_sync(&d->l2tab[i].update_timer);
+#endif
+#endif
+	cxgb_free_mem(d);
+}
+
+#ifdef CONFIG_PROC_FS
+#include <linux/module.h>
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+
+static inline void *
+l2t_get_idx(struct seq_file *seq, loff_t pos)
+{
+	struct l2t_data *d = seq->private;
+
+	return pos >= d->nentries ? NULL : &d->l2tab[pos];
+}
+
+static void *
+l2t_seq_start(struct seq_file *seq, loff_t *pos)
+{
+	return *pos ? l2t_get_idx(seq, *pos) : SEQ_START_TOKEN;
+}
+
+static void *
+l2t_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+	v = l2t_get_idx(seq, *pos + 1);
+	if (v)
+		++*pos;
+	return v;
+}
+
+static void
+l2t_seq_stop(struct seq_file *seq, void *v)
+{
+}
+
+static char
+l2e_state(const struct l2t_entry *e)
+{
+	switch (e->state) {
+	case L2T_STATE_VALID: return 'V';  /* valid, fast-path entry */
+	case L2T_STATE_STALE: return 'S';  /* needs revalidation, but usable */
+	case L2T_STATE_RESOLVING:
+		return e->arpq_head ? 'A' : 'R';
+	default:
+		return 'U';
+	}
+}
+
+static int
+l2t_seq_show(struct seq_file *seq, void *v)
+{
+	if (v == SEQ_START_TOKEN)
+		seq_puts(seq, "Index IP address      Ethernet address   VLAN  "
+			 "Prio  State   Users SMTIDX  Port\n");
+	else {
+		char ip[20];
+		struct l2t_entry *e = v;
+
+		mtx_lock(&e->lock);
+		sprintf(ip, "%u.%u.%u.%u", NIPQUAD(e->addr));
+		seq_printf(seq, "%-5u %-15s %02x:%02x:%02x:%02x:%02x:%02x  %4d"
+			   "  %3u     %c   %7u   %4u %s\n",
+			   e->idx, ip, e->dmac[0], e->dmac[1], e->dmac[2],
+			   e->dmac[3], e->dmac[4], e->dmac[5],
+			   e->vlan & EVL_VLID_MASK, vlan_prio(e),
+			   l2e_state(e), atomic_load_acq_int(&e->refcnt), e->smt_idx,
+			   e->neigh ? e->neigh->dev->name : "");
+		mtx_unlock(&e->lock);
+	}
+	return 0;
+}
+
+#endif
diff --git a/sys/dev/cxgb/cxgb_l2t.h b/sys/dev/cxgb/cxgb_l2t.h
new file mode 100644
index 0000000..a690c77
--- /dev/null
+++ b/sys/dev/cxgb/cxgb_l2t.h
@@ -0,0 +1,154 @@
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+$FreeBSD$
+
+***************************************************************************/
+#ifndef _CHELSIO_L2T_H
+#define _CHELSIO_L2T_H
+
+#include <dev/cxgb/ulp/toecore/toedev.h>
+#include <sys/lock.h>
+#include <sys/rwlock.h>
+
+enum {
+	L2T_STATE_VALID,      /* entry is up to date */
+	L2T_STATE_STALE,      /* entry may be used but needs revalidation */
+	L2T_STATE_RESOLVING,  /* entry needs address resolution */
+	L2T_STATE_UNUSED      /* entry not in use */
+};
+
+/*
+ * Each L2T entry plays multiple roles.  First of all, it keeps state for the
+ * corresponding entry of the HW L2 table and maintains a queue of offload
+ * packets awaiting address resolution.  Second, it is a node of a hash table
+ * chain, where the nodes of the chain are linked together through their next
+ * pointer.  Finally, each node is a bucket of a hash table, pointing to the
+ * first element in its chain through its first pointer.
+ */
+struct l2t_entry {
+	uint16_t state;               /* entry state */
+	uint16_t idx;                 /* entry index */
+	uint32_t addr;                /* dest IP address */
+	int ifindex;                  /* neighbor's net_device's ifindex */
+	uint16_t smt_idx;             /* SMT index */
+	uint16_t vlan;                /* VLAN TCI (id: bits 0-11, prio: 13-15 */
+	struct rtentry *neigh;        /* associated neighbour */
+	struct l2t_entry *first;      /* start of hash chain */
+	struct l2t_entry *next;       /* next l2t_entry on chain */
+	struct mbuf *arpq_head;       /* queue of packets awaiting resolution */
+	struct mbuf *arpq_tail;
+	struct mtx lock;
+	volatile uint32_t refcnt;     /* entry reference count */
+	uint8_t dmac[6];              /* neighbour's MAC address */
+#ifndef NETEVENT
+#ifdef CONFIG_CHELSIO_T3_MODULE
+	struct timer_list update_timer;
+	struct toedev *tdev;
+#endif
+#endif
+};
+
+struct l2t_data {
+	unsigned int nentries;      /* number of entries */
+	struct l2t_entry *rover;    /* starting point for next allocation */
+	volatile uint32_t nfree;    /* number of free entries */
+	struct rwlock lock;
+	struct l2t_entry l2tab[0];
+};
+
+typedef void (*arp_failure_handler_func)(struct toedev *dev,
+					 struct mbuf *m);
+
+/*
+ * Callback stored in an skb to handle address resolution failure.
+ */
+struct l2t_mbuf_cb {
+	arp_failure_handler_func arp_failure_handler;
+};
+
+/*
+ * XXX 
+ */
+#define L2T_MBUF_CB(skb) ((struct l2t_mbuf_cb *)(skb)->cb)
+
+
+static __inline void set_arp_failure_handler(struct mbuf *m,
+					   arp_failure_handler_func hnd)
+{
+#if 0
+	L2T_SKB_CB(skb)->arp_failure_handler = hnd;
+#endif
+	panic("implement me");
+}
+
+/*
+ * Getting to the L2 data from an offload device.
+ */
+#define L2DATA(dev) ((dev)->l2opt)
+
+void t3_l2e_free(struct l2t_data *d, struct l2t_entry *e);
+void t3_l2t_update(struct toedev *dev, struct rtentry *ifp);
+struct l2t_entry *t3_l2t_get(struct toedev *dev, struct rtentry *neigh,
+			     unsigned int smt_idx);
+int t3_l2t_send_slow(struct toedev *dev, struct mbuf *m,
+		     struct l2t_entry *e);
+void t3_l2t_send_event(struct toedev *dev, struct l2t_entry *e);
+struct l2t_data *t3_init_l2t(unsigned int l2t_capacity);
+void t3_free_l2t(struct l2t_data *d);
+
+#ifdef CONFIG_PROC_FS
+int t3_l2t_proc_setup(struct proc_dir_entry *dir, struct l2t_data *d);
+void t3_l2t_proc_free(struct proc_dir_entry *dir);
+#else
+#define l2t_proc_setup(dir, d) 0
+#define l2t_proc_free(dir)
+#endif
+
+int cxgb_ofld_send(struct toedev *dev, struct mbuf *m);
+
+static inline int l2t_send(struct toedev *dev, struct mbuf *m,
+			   struct l2t_entry *e)
+{
+	if (__predict_true(e->state == L2T_STATE_VALID))
+		return cxgb_ofld_send(dev, m);
+	return t3_l2t_send_slow(dev, m, e);
+}
+
+static inline void l2t_release(struct l2t_data *d, struct l2t_entry *e)
+{
+	if (atomic_fetchadd_int(&e->refcnt, -1) == 1)
+		t3_l2e_free(d, e);
+}
+
+static inline void l2t_hold(struct l2t_data *d, struct l2t_entry *e)
+{
+	if (atomic_fetchadd_int(&e->refcnt, 1) == 1)  /* 0 -> 1 transition */
+		atomic_add_int(&d->nfree, 1);
+}
+
+#endif
diff --git a/sys/dev/cxgb/cxgb_lro.c b/sys/dev/cxgb/cxgb_lro.c
index 39651e0..8b24298 100644
--- a/sys/dev/cxgb/cxgb_lro.c
+++ b/sys/dev/cxgb/cxgb_lro.c
@@ -9,11 +9,7 @@ modification, are permitted provided that the following conditions are met:
  1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
 
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Chelsio Corporation nor the names of its
+2. Neither the name of the Chelsio Corporation nor the names of its
     contributors may be used to endorse or promote products derived from
     this software without specific prior written permission.
 
@@ -82,51 +78,40 @@ __FBSDID("$FreeBSD$");
 #endif	  
 
 #define IPH_OFFSET (2 + sizeof (struct cpl_rx_pkt) + ETHER_HDR_LEN)
-#define LRO_SESSION_IDX_HINT_HASH(hash) (hash & (MAX_LRO_PER_QSET - 1))
-#define LRO_IDX_INC(idx) idx = (idx + 1) & (MAX_LRO_PER_QSET - 1)
-
-static __inline struct sge_lro_session *
-lro_session(struct sge_lro *l, int idx)
-{
-	return l->s + idx;
-}
+#define LRO_SESSION_IDX_HINT_HASH(hash) (hash & (MAX_LRO_SES - 1))
+#define LRO_IDX_INC(idx) idx = (idx + 1) & (MAX_LRO_SES - 1)
 
 static __inline int
-lro_match_session(struct sge_lro_session *s, 
-    struct ip *ih, struct tcphdr *th)
+lro_match(struct mbuf *m, struct ip *ih, struct tcphdr *th)
 {
-	struct ip *sih = (struct ip *)(s->m->m_data + IPH_OFFSET);
+	struct ip *sih = (struct ip *)(m->m_data + IPH_OFFSET);
 	struct tcphdr *sth = (struct tcphdr *) (sih + 1);
 
 	/*
-	 * Linux driver doesn't include destination port check --
-	 * need to find out why XXX
+	 * Why don't we check dest ports?
 	 */
 	return (*(uint32_t *)&th->th_sport == *(uint32_t *)&sth->th_sport &&
-	    *(uint32_t *)&th->th_dport == *(uint32_t *)&sth->th_dport &&
 	    ih->ip_src.s_addr == ih->ip_src.s_addr &&
 	    ih->ip_dst.s_addr == sih->ip_dst.s_addr);
 }
 
-static __inline struct sge_lro_session *
-lro_find_session(struct sge_lro *l, int idx, struct ip *ih, struct tcphdr *th)
+static __inline struct t3_lro_session *
+lro_lookup(struct lro_state *l, int idx, struct ip *ih, struct tcphdr *th)
 {
-	struct sge_lro_session *s;
-	int active = 0;
-
-	while (active < l->num_active) {
-		s = lro_session(l, idx); 
-		if (s->m) {
-			if (lro_match_session(s, ih, th)) {
-				l->last_s = s;
-				return s;
-			}
-			active++;
+	struct t3_lro_session *s = NULL;
+	int active = l->nactive;
+
+	while (active) {
+		s = &l->sess[idx];
+		if (s->head) {
+			if (lro_match(s->head, ih, th)) 
+				break;
+			active--;
 		}
 		LRO_IDX_INC(idx);
 	}
 
-	return NULL;
+	return (s);
 }
 
 static __inline int
@@ -174,7 +159,7 @@ can_lro_tcpsegment(struct tcphdr *th)
 }
 
 static __inline void
-lro_new_session_init(struct sge_lro_session *s, struct mbuf *m)
+lro_new_session_init(struct t3_lro_session *s, struct mbuf *m)
 {
 	struct ip *ih = (struct ip *)(m->m_data + IPH_OFFSET);
 	struct tcphdr *th = (struct tcphdr *) (ih + 1);
@@ -182,7 +167,7 @@ lro_new_session_init(struct sge_lro_session *s, struct mbuf *m)
 
 	DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m);
 	
-	s->m = m;
+	s->head = m;
 	
 	MBUF_HEADER_CHECK(m);
 	s->ip_len = ip_len;
@@ -191,10 +176,10 @@ lro_new_session_init(struct sge_lro_session *s, struct mbuf *m)
 } 
 
 static void
-lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct mbuf *m)
+lro_flush_session(struct sge_qset *qs, struct t3_lro_session *s, struct mbuf *m)
 {
-	struct sge_lro *l = &qs->lro;
-	struct mbuf *sm = s->m;
+	struct lro_state *l = &qs->lro;
+	struct mbuf *sm = s->head;
 	struct ip *ih = (struct ip *)(sm->m_data + IPH_OFFSET);
 
 	
@@ -216,33 +201,33 @@ lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct mbuf *m
 	t3_rx_eth(qs->port, &qs->rspq, sm, 2);
 	
 	if (m) {
-		s->m = m;
+		s->head = m;
 		lro_new_session_init(s, m);
 	} else {
-		s->m = NULL;
-		l->num_active--;
+		s->head = NULL;
+		l->nactive--;
 	}
 
 	qs->port_stats[SGE_PSTATS_LRO_FLUSHED]++;
 }
 
-static __inline struct sge_lro_session *
+static __inline struct t3_lro_session *
 lro_new_session(struct sge_qset *qs, struct mbuf *m, uint32_t rss_hash)
 {
-	struct sge_lro *l = &qs->lro;
+	struct lro_state *l = &qs->lro;
 	int idx = LRO_SESSION_IDX_HINT_HASH(rss_hash); 
-	struct sge_lro_session *s = lro_session(l, idx);
+	struct t3_lro_session *s = &l->sess[idx];
 
 	DPRINTF("%s(qs=%p,  m=%p, rss_hash=0x%x)\n", __FUNCTION__,
 	    qs, m, rss_hash);
 	
-	if (__predict_true(!s->m))
+	if (__predict_true(!s->head))
 		goto done;
 
-	if (l->num_active > MAX_LRO_PER_QSET)
+	if (l->nactive > MAX_LRO_SES)
 		panic("MAX_LRO_PER_QSET exceeded");
 	
-	if (l->num_active == MAX_LRO_PER_QSET) {
+	if (l->nactive == MAX_LRO_SES) {
 		lro_flush_session(qs, s, m);
 		qs->port_stats[SGE_PSTATS_LRO_X_STREAMS]++;
 		return s;
@@ -250,21 +235,21 @@ lro_new_session(struct sge_qset *qs, struct mbuf *m, uint32_t rss_hash)
 
 	while (1) {
 		LRO_IDX_INC(idx);
-		s = lro_session(l, idx);
-		if (!s->m)
+		s = &l->sess[idx];
+		if (!s->head)
 			break;
 	}
 done:
 	lro_new_session_init(s, m);
-	l->num_active++;
+	l->nactive++;
 
 	return s;
 }
 
 static __inline int
-lro_update_session(struct sge_lro_session *s, struct mbuf *m)
+lro_update_session(struct t3_lro_session *s, struct mbuf *m)
 {
-	struct mbuf *sm = s->m;
+	struct mbuf *sm = s->head;
 	struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(sm->m_data + 2);
 	struct cpl_rx_pkt *ncpl = (struct cpl_rx_pkt *)(m->m_data + 2);
 	struct ip *nih = (struct ip *)(m->m_data + IPH_OFFSET);
@@ -354,7 +339,7 @@ t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
 	struct ether_header *eh = (struct ether_header *)(cpl + 1);
 	struct ip *ih;
 	struct tcphdr *th; 
-	struct sge_lro_session *s = NULL;
+	struct t3_lro_session *s = NULL;
 	struct port_info *pi = qs->port;
 	
 	if (lro == 0)
@@ -369,7 +354,7 @@ t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
 	ih = (struct ip *)(eh + 1);
 	th = (struct tcphdr *)(ih + 1);
 	
-	s = lro_find_session(&qs->lro,
+	s = lro_lookup(&qs->lro,
 	    LRO_SESSION_IDX_HINT_HASH(rss_hash), ih, th);
 	
 	if (__predict_false(!can_lro_tcpsegment(th))) {
@@ -380,7 +365,7 @@ t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
 		if (lro_update_session(s, m)) {
 			lro_flush_session(qs, s, m);
 		}
-		if (__predict_false(s->m->m_pkthdr.len + pi->ifp->if_mtu > 65535)) {
+		if (__predict_false(s->head->m_pkthdr.len + pi->ifp->if_mtu > 65535)) {
 			lro_flush_session(qs, s, NULL);
 		}		
 	}
@@ -398,21 +383,15 @@ no_lro:
 }
 
 void
-t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs)
+t3_lro_flush(adapter_t *adap, struct sge_qset *qs, struct lro_state *state)
 {
-	struct sge_lro *l = &qs->lro;
-	struct sge_lro_session *s = l->last_s; 
-	int active = 0, idx = 0, num_active = l->num_active;
+	unsigned int idx = state->active_idx;
 
-	if (__predict_false(!s))
-		s = lro_session(l, idx);
-
-	while (active < num_active) {
-		if (s->m) {
+	while (state->nactive) {
+		struct t3_lro_session *s = &state->sess[idx];
+		
+		if (s->head) 
 			lro_flush_session(qs, s, NULL);
-			active++;
-		}
 		LRO_IDX_INC(idx);
-		s = lro_session(l, idx);
 	}
 }
diff --git a/sys/dev/cxgb/cxgb_main.c b/sys/dev/cxgb/cxgb_main.c
index b43e4d9..3a49146 100644
--- a/sys/dev/cxgb/cxgb_main.c
+++ b/sys/dev/cxgb/cxgb_main.c
@@ -9,11 +9,7 @@ modification, are permitted provided that the following conditions are met:
  1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
 
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Chelsio Corporation nor the names of its
+2. Neither the name of the Chelsio Corporation nor the names of its
     contributors may be used to endorse or promote products derived from
     this software without specific prior written permission.
 
@@ -79,6 +75,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/cxgb/cxgb_osdep.h>
 #include <dev/cxgb/common/cxgb_common.h>
 #include <dev/cxgb/cxgb_ioctl.h>
+#include <dev/cxgb/cxgb_offload.h>
 #include <dev/cxgb/common/cxgb_regs.h>
 #include <dev/cxgb/common/cxgb_t3_cpl.h>
 #include <dev/cxgb/common/cxgb_firmware_exports.h>
@@ -103,6 +100,7 @@ static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
 static int setup_sge_qsets(adapter_t *);
 static void cxgb_async_intr(void *);
 static void cxgb_ext_intr_handler(void *, int);
+static void cxgb_down(struct adapter *sc);
 static void cxgb_tick(void *);
 static void setup_rss(adapter_t *sc);
 
@@ -117,6 +115,10 @@ static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned i
     unsigned int end);
 static void cxgb_get_regs(adapter_t *sc, struct ifconf_regs *regs, uint8_t *buf);
 static int cxgb_get_regs_len(void);
+static int offload_open(struct port_info *pi);
+static int offload_close(struct toedev *tdev);
+
+
 
 static device_method_t cxgb_controller_methods[] = {
 	DEVMETHOD(device_probe,		cxgb_controller_probe),
@@ -179,14 +181,27 @@ extern int collapse_mbufs;
  */
 static int msi_allowed = 2;
 TUNABLE_INT("hw.cxgb.msi_allowed", &msi_allowed);
-
 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
 SYSCTL_UINT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
     "MSI-X, MSI, INTx selector");
+
 /*
- * Multiple queues need further tuning
+ * The driver enables offload as a default.
+ * To disable it, use ofld_disable = 1.
+ */
+static int ofld_disable = 0;
+TUNABLE_INT("hw.cxgb.ofld_disable", &ofld_disable);
+SYSCTL_UINT(_hw_cxgb, OID_AUTO, ofld_disable, CTLFLAG_RDTUN, &ofld_disable, 0,
+    "disable ULP offload");
+
+/*
+ * The driver uses an auto-queue algorithm by default.
+ * To disable it and force a single queue-set per port, use singleq = 1.
  */
 static int singleq = 1;
+TUNABLE_INT("hw.cxgb.singleq", &singleq);
+SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0,
+    "use a single queue-set per port");
 
 enum {
 	MAX_TXQ_ENTRIES      = 16384,
@@ -272,7 +287,7 @@ cxgb_controller_probe(device_t dev)
 }
 
 static int
-cxgb_fw_download(adapter_t *sc, device_t dev)
+upgrade_fw(adapter_t *sc)
 {
 	char buf[32];
 #ifdef FIRMWARE_LATEST
@@ -282,15 +297,14 @@ cxgb_fw_download(adapter_t *sc, device_t dev)
 #endif	
 	int status;
 	
-	snprintf(&buf[0], sizeof(buf), "t3fw%d%d", FW_VERSION_MAJOR,
-	    FW_VERSION_MINOR);
+	snprintf(&buf[0], sizeof(buf), "t3fw%d%d%d", FW_VERSION_MAJOR,
+	    FW_VERSION_MINOR, FW_VERSION_MICRO);
 	
 	fw = firmware_get(buf);
-
 	
 	if (fw == NULL) {
-		device_printf(dev, "Could not find firmware image %s\n", buf);
-		return ENOENT;
+		device_printf(sc->dev, "Could not find firmware image %s\n", buf);
+		return (ENOENT);
 	}
 	
 	status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
@@ -300,7 +314,6 @@ cxgb_fw_download(adapter_t *sc, device_t dev)
 	return (status);	
 }
 
-
 static int
 cxgb_controller_attach(device_t dev)
 {
@@ -308,13 +321,14 @@ cxgb_controller_attach(device_t dev)
 	device_t child;
 	const struct adapter_info *ai;
 	struct adapter *sc;
-	int i, reg, msi_needed, msi_count = 0, error = 0;
+	int i, reg, msi_needed, error = 0;
 	uint32_t vers;
 	int port_qsets = 1;
 	    
 	sc = device_get_softc(dev);
 	sc->dev = dev;
-
+	sc->msi_count = 0;
+	
 	/* find the PCIe link width and set max read request to 4KB*/
 	if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
 		uint16_t lnk, pectl;
@@ -370,13 +384,14 @@ cxgb_controller_attach(device_t dev)
 	    (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 	    &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
 
-		msi_needed = msi_count = SGE_MSIX_COUNT;
+		msi_needed = sc->msi_count = SGE_MSIX_COUNT;
 
-		if ((pci_alloc_msix(dev, &msi_count) != 0) ||
-		    (msi_count != msi_needed)) {
-			device_printf(dev, "msix allocation failed"
-			    " will try msi\n");
-			msi_count = 0;
+		if (((error = pci_alloc_msix(dev, &sc->msi_count)) != 0) ||
+		    (sc->msi_count != msi_needed)) {
+			device_printf(dev, "msix allocation failed - msi_count = %d"
+			    " msi_needed=%d will try msi err=%d\n", sc->msi_count,
+			    msi_needed, error);
+			sc->msi_count = 0;
 			pci_release_msi(dev);
 			bus_release_resource(dev, SYS_RES_MEMORY,
 			    sc->msix_regs_rid, sc->msix_regs_res);
@@ -387,11 +402,11 @@ cxgb_controller_attach(device_t dev)
 		}
 	}
 
-	if ((msi_allowed >= 1) && (msi_count == 0)) {
-		msi_count = 1;
-		if (pci_alloc_msi(dev, &msi_count)) {
+	if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
+		sc->msi_count = 1;
+		if (pci_alloc_msi(dev, &sc->msi_count)) {
 			device_printf(dev, "alloc msi failed - will try INTx\n");
-			msi_count = 0;
+			sc->msi_count = 0;
 			pci_release_msi(dev);
 		} else {
 			sc->flags |= USING_MSI;
@@ -400,7 +415,7 @@ cxgb_controller_attach(device_t dev)
 		}
 	}
 #endif
-	if (msi_count == 0) {
+	if (sc->msi_count == 0) {
 		device_printf(dev, "using line interrupts\n");
 		sc->irq_rid = 0;
 		cxgb_intr = t3b_intr;
@@ -432,22 +447,14 @@ cxgb_controller_attach(device_t dev)
 		/*
 		 * Warn user that a firmware update will be attempted in init.
 		 */
-		device_printf(dev, "firmware needs to be updated to version %d.%d\n",
-		    FW_VERSION_MAJOR, FW_VERSION_MINOR);
+		device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
+		    FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 		sc->flags &= ~FW_UPTODATE;
 	} else {
 		sc->flags |= FW_UPTODATE;
 	}
 	
-	if (t3_init_hw(sc, 0) != 0) {
-		device_printf(dev, "hw initialization failed\n");
-		error = ENXIO;
-		goto out;
-	}
-	t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
-
-
-	if ((singleq == 0) && (sc->flags & USING_MSIX))
+	if ((sc->flags & USING_MSIX) && !singleq)
 		port_qsets = min((SGE_QSETS/(sc)->params.nports), mp_ncpus);
 
 	/*
@@ -468,52 +475,32 @@ cxgb_controller_attach(device_t dev)
 		device_set_softc(child, &sc->port[i]);
 	}
 	if ((error = bus_generic_attach(dev)) != 0)
-		goto out;;
-
-	if ((error = setup_sge_qsets(sc)) != 0)
 		goto out;
-	
-	setup_rss(sc);
-	
-	/* If it's MSI or INTx, allocate a single interrupt for everything */
-	if ((sc->flags & USING_MSIX) == 0) {
-		if ((sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
-		   &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
-			device_printf(dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid);
-			error = EINVAL;
-			goto out;
-		}
-		device_printf(dev, "allocated irq_res=%p\n", sc->irq_res);
-
-		if (bus_setup_intr(dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
-#ifdef INTR_FILTERS
-			NULL,
-#endif			
-			cxgb_intr, sc, &sc->intr_tag)) {
-			device_printf(dev, "Cannot set up interrupt\n");
-			error = EINVAL;
-			goto out;
-		}
-	} else {
-		cxgb_setup_msix(sc, msi_count);
-	}
 
+	/*
+	 * XXX need to poll for link status
+	 */
 	sc->params.stats_update_period = 1;
 
 	/* initialize sge private state */
 	t3_sge_init_sw(sc);
 
 	t3_led_ready(sc);
-
+	
+	cxgb_offload_init();
+	if (is_offload(sc)) {
+		setbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT);
+		cxgb_adapter_ofld(sc);
+        }
 	error = t3_get_fw_version(sc, &vers);
 	if (error)
 		goto out;
-	
-	snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d", G_FW_VERSION_MAJOR(vers),
-	    G_FW_VERSION_MINOR(vers));
+
+	snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
+	    G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
+	    G_FW_VERSION_MICRO(vers));
 
 	t3_add_sysctls(sc);
-	
 out:
 	if (error)
 		cxgb_free(sc);
@@ -538,11 +525,25 @@ cxgb_free(struct adapter *sc)
 {
 	int i;
 
+	cxgb_down(sc);
+	
+#ifdef MSI_SUPPORTED
+	if (sc->flags & (USING_MSI | USING_MSIX)) {
+		device_printf(sc->dev, "releasing msi message(s)\n");
+		pci_release_msi(sc->dev);
+	} else {
+		device_printf(sc->dev, "no msi message to release\n");
+	}
+#endif
+	if (sc->msix_regs_res != NULL) {
+		bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
+		    sc->msix_regs_res);
+	}
+	
 	/*
 	 * XXX need to drain the ifq by hand until
 	 * it is taught about mbuf iovecs
 	 */ 
-
 	callout_drain(&sc->cxgb_tick_ch);
 
 	t3_sge_deinit_sw(sc);
@@ -559,41 +560,14 @@ cxgb_free(struct adapter *sc)
 		
 	bus_generic_detach(sc->dev);
 
+	if (is_offload(sc)) {
+		cxgb_adapter_unofld(sc);
+		if (isset(&sc->open_device_map,	OFFLOAD_DEVMAP_BIT))
+			offload_close(&sc->tdev);
+	}
 	t3_free_sge_resources(sc);
 	t3_sge_free(sc);
 
-	for (i = 0; i < SGE_QSETS; i++) {
-		if (sc->msix_intr_tag[i] != NULL) {
-			bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
-			    sc->msix_intr_tag[i]);
-		}
-		if (sc->msix_irq_res[i] != NULL) {
-			bus_release_resource(sc->dev, SYS_RES_IRQ,
-			    sc->msix_irq_rid[i], sc->msix_irq_res[i]);
-		}
-	}
-
-	if (sc->intr_tag != NULL) {
-		bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
-	}
-
-	if (sc->irq_res != NULL) {
-		device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n",
-		    sc->irq_rid, sc->irq_res);
-		bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
-		    sc->irq_res);
-	}
-#ifdef MSI_SUPPORTED
-	if (sc->flags & (USING_MSI | USING_MSIX)) {
-		device_printf(sc->dev, "releasing msi message(s)\n");
-		pci_release_msi(sc->dev);
-	}
-#endif
-	if (sc->msix_regs_res != NULL) {
-		bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
-		    sc->msix_regs_res);
-	}
-
 	if (sc->regs_res != NULL)
 		bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
 		    sc->regs_res);
@@ -617,7 +591,7 @@ static int
 setup_sge_qsets(adapter_t *sc)
 {
 	int i, j, err, irq_idx, qset_idx;
-	u_int ntxq = 3;
+	u_int ntxq = SGE_TXQ_PER_SET;
 
 	if ((err = t3_sge_alloc(sc)) != 0) {
 		device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
@@ -821,7 +795,7 @@ cxgb_port_attach(device_t dev)
 	ifmedia_add(&p->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 	ifmedia_set(&p->media, media_flags);
 
-	snprintf(buf, sizeof(buf), "cxgb_port_taskq%d", p->port);  
+	snprintf(buf, sizeof(buf), "cxgb_port_taskq%d", p->port);
 #ifdef TASKQUEUE_CURRENT
 	/* Create a port for handling TX without starvation */
 	p->tq = taskqueue_create(buf, M_NOWAIT,
@@ -851,6 +825,11 @@ cxgb_port_detach(device_t dev)
 	struct port_info *p;
 
 	p = device_get_softc(dev);
+
+	PORT_LOCK(p);
+	cxgb_stop_locked(p);
+	PORT_UNLOCK(p);
+	
 	mtx_destroy(&p->lock);
 	if (p->tq != NULL) {
 		taskqueue_drain(p->tq, &p->start_task);
@@ -962,15 +941,20 @@ t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
      int duplex, int fc)
 {
 	struct port_info *pi = &adapter->port[port_id];
+	struct cmac *mac = &adapter->port[port_id].mac;
 
 	if ((pi->ifp->if_flags & IFF_UP) == 0)
 		return;
-	
-	if (link_status)
+
+	if (link_status) {
+		t3_mac_enable(mac, MAC_DIRECTION_RX);
 		if_link_state_change(pi->ifp, LINK_STATE_UP);
-	else
+	} else {
 		if_link_state_change(pi->ifp, LINK_STATE_DOWN);
-	
+		pi->phy.ops->power_down(&pi->phy, 1);
+		t3_mac_disable(mac, MAC_DIRECTION_RX);
+		t3_link_start(&pi->phy, mac, &pi->link_config);
+	}
 }
 
 
@@ -988,11 +972,13 @@ t3_os_ext_intr_handler(adapter_t *sc)
 	 * interrupts in the meantime and let the task reenable them when
 	 * it's done.
 	 */
+	ADAPTER_LOCK(sc);
 	if (sc->slow_intr_mask) {
 		sc->slow_intr_mask &= ~F_T3DBG;
 		t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
 		taskqueue_enqueue(sc->tq, &sc->ext_intr_task);
 	}
+	ADAPTER_UNLOCK(sc);
 }
 
 void
@@ -1067,6 +1053,66 @@ setup_rss(adapter_t *adap)
 	    V_RRCPLCPUSIZE(6), cpus, rspq_map);
 }
 
+/*
+ * Sends an mbuf to an offload queue driver
+ * after dealing with any active network taps.
+ */
+static inline int
+offload_tx(struct toedev *tdev, struct mbuf *m)
+{
+	int ret;
+
+	critical_enter();
+	ret = t3_offload_tx(tdev, m);
+	critical_exit();
+	return ret;
+}
+
+static int
+write_smt_entry(struct adapter *adapter, int idx)
+{
+	struct port_info *pi = &adapter->port[idx];
+	struct cpl_smt_write_req *req;
+	struct mbuf *m;
+
+	if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
+		return (ENOMEM);
+
+	req = mtod(m, struct cpl_smt_write_req *);
+	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
+	req->mtu_idx = NMTUS - 1;  /* should be 0 but there's a T3 bug */
+	req->iff = idx;
+	memset(req->src_mac1, 0, sizeof(req->src_mac1));
+	memcpy(req->src_mac0, pi->hw_addr, ETHER_ADDR_LEN);
+
+	m_set_priority(m, 1);
+
+	offload_tx(&adapter->tdev, m);
+
+	return (0);
+}
+
+static int
+init_smt(struct adapter *adapter)
+{
+	int i;
+
+	for_each_port(adapter, i)
+		write_smt_entry(adapter, i);
+	return 0;
+}
+
+static void
+init_port_mtus(adapter_t *adapter)
+{
+	unsigned int mtus = adapter->port[0].ifp->if_mtu;
+
+	if (adapter->port[1].ifp)
+		mtus |= adapter->port[1].ifp->if_mtu << 16;
+	t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
+}
+
 static void
 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 			      int hi, int port)
@@ -1076,7 +1122,7 @@ send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 
 	m = m_gethdr(M_NOWAIT, MT_DATA);
 	if (m) {	
-		req = (struct mngt_pktsched_wr *)m->m_data;
+		req = mtod(m, struct mngt_pktsched_wr *);
 		req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 		req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 		req->sched = sched;
@@ -1103,6 +1149,192 @@ bind_qsets(adapter_t *sc)
 	}
 }
 
+/**
+ *	cxgb_up - enable the adapter
+ *	@adap: adapter being enabled
+ *
+ *	Called when the first port is enabled, this function performs the
+ *	actions necessary to make an adapter operational, such as completing
+ *	the initialization of HW modules, and enabling interrupts.
+ *
+ */
+static int
+cxgb_up(struct adapter *sc)
+{
+	int err = 0;
+
+	if ((sc->flags & FULL_INIT_DONE) == 0) {
+
+		if ((sc->flags & FW_UPTODATE) == 0)
+			err = upgrade_fw(sc);
+
+		if (err)
+			goto out;
+
+		err = t3_init_hw(sc, 0);
+		if (err)
+			goto out;
+
+		t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
+
+		err = setup_sge_qsets(sc);
+		if (err)
+			goto out;
+
+		setup_rss(sc);
+		sc->flags |= FULL_INIT_DONE;
+	}
+
+	t3_intr_clear(sc);
+
+	/* If it's MSI or INTx, allocate a single interrupt for everything */
+	if ((sc->flags & USING_MSIX) == 0) {
+		if ((sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
+		   &sc->irq_rid, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
+			device_printf(sc->dev, "Cannot allocate interrupt rid=%d\n", sc->irq_rid);
+			err = EINVAL;
+			goto out;
+		}
+		device_printf(sc->dev, "allocated irq_res=%p\n", sc->irq_res);
+
+		if (bus_setup_intr(sc->dev, sc->irq_res, INTR_MPSAFE|INTR_TYPE_NET,
+#ifdef INTR_FILTERS
+			NULL,
+#endif			
+			sc->cxgb_intr, sc, &sc->intr_tag)) {
+			device_printf(sc->dev, "Cannot set up interrupt\n");
+			err = EINVAL;
+			goto irq_err;
+		}
+	} else {
+		cxgb_setup_msix(sc, sc->msi_count);
+	}
+
+	t3_sge_start(sc);
+	t3_intr_enable(sc);
+
+	if ((sc->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX)
+		bind_qsets(sc);
+	sc->flags |= QUEUES_BOUND;
+out:
+	return (err);
+irq_err:
+	CH_ERR(sc, "request_irq failed, err %d\n", err);
+	goto out;
+}
+
+
+/*
+ * Release resources when all the ports and offloading have been stopped.
+ */
+static void
+cxgb_down(struct adapter *sc)
+{
+	int i;
+
+	t3_sge_stop(sc);
+	t3_intr_disable(sc);
+
+	for (i = 0; i < SGE_QSETS; i++) {
+		if (sc->msix_intr_tag[i] != NULL) {
+			bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
+			    sc->msix_intr_tag[i]);
+			sc->msix_intr_tag[i] = NULL;
+		}
+		if (sc->msix_irq_res[i] != NULL) {
+			bus_release_resource(sc->dev, SYS_RES_IRQ,
+			    sc->msix_irq_rid[i], sc->msix_irq_res[i]);
+			sc->msix_irq_res[i] = NULL;
+		}
+	}
+
+	if (sc->intr_tag != NULL) {
+		bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
+		sc->intr_tag = NULL;
+	}
+	if (sc->irq_res != NULL) {
+		device_printf(sc->dev, "de-allocating interrupt irq_rid=%d irq_res=%p\n",
+		    sc->irq_rid, sc->irq_res);
+		bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
+		    sc->irq_res);
+		sc->irq_res = NULL;
+	}
+
+	callout_drain(&sc->sge_timer_ch);
+	taskqueue_drain(sc->tq, &sc->slow_intr_task);
+	taskqueue_drain(sc->tq, &sc->timer_reclaim_task);
+}
+
+static int
+offload_open(struct port_info *pi)
+{
+	struct adapter *adapter = pi->adapter;
+	struct toedev *tdev = TOEDEV(pi->ifp);
+	int adap_up = adapter->open_device_map & PORT_MASK;
+	int err = 0;
+
+	if (atomic_cmpset_int(&adapter->open_device_map,
+		(adapter->open_device_map & ~OFFLOAD_DEVMAP_BIT),
+		(adapter->open_device_map | OFFLOAD_DEVMAP_BIT)) == 0)
+		return (0);
+
+	ADAPTER_LOCK(pi->adapter); 
+	if (!adap_up)
+		err = cxgb_up(adapter);
+	ADAPTER_UNLOCK(pi->adapter);
+	if (err < 0)
+		return (err);
+
+	t3_tp_set_offload_mode(adapter, 1);
+	tdev->lldev = adapter->port[0].ifp;
+	err = cxgb_offload_activate(adapter);
+	if (err)
+		goto out;
+
+	init_port_mtus(adapter);
+	t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
+		     adapter->params.b_wnd,
+		     adapter->params.rev == 0 ?
+		       adapter->port[0].ifp->if_mtu : 0xffff);
+	init_smt(adapter);
+
+	/* Call back all registered clients */
+	cxgb_add_clients(tdev);
+
+out:
+	/* restore them in case the offload module has changed them */
+	if (err) {
+		t3_tp_set_offload_mode(adapter, 0);
+		clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
+		cxgb_set_dummy_ops(tdev);
+	}
+	return (err);
+}
+
+static int
+offload_close(struct toedev *tdev)
+{
+	struct adapter *adapter = tdev2adap(tdev);
+
+	if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT))
+		return 0;
+
+	/* Call back all registered clients */
+	cxgb_remove_clients(tdev);
+	tdev->lldev = NULL;
+	cxgb_set_dummy_ops(tdev);
+	t3_tp_set_offload_mode(adapter, 0);
+	clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT);
+
+	ADAPTER_LOCK(adapter);
+	if (!adapter->open_device_map)
+		cxgb_down(adapter);
+	ADAPTER_UNLOCK(adapter);
+
+	cxgb_offload_deactivate(adapter);
+	return 0;
+}
+
 static void
 cxgb_init(void *arg)
 {
@@ -1118,43 +1350,39 @@ cxgb_init_locked(struct port_info *p)
 {
 	struct ifnet *ifp;
 	adapter_t *sc = p->adapter;
-	int error;
+	int err;
 
 	mtx_assert(&p->lock, MA_OWNED);
-	
 	ifp = p->ifp;
-	if ((sc->flags & FW_UPTODATE) == 0) {
-		device_printf(sc->dev, "updating firmware to version %d.%d\n",
-		    FW_VERSION_MAJOR, FW_VERSION_MINOR);
-		if ((error = cxgb_fw_download(sc, sc->dev)) != 0) {
-			device_printf(sc->dev, "firmware download failed err: %d"
-			    "interface will be unavailable\n", error);
-			return;
-		}
-		sc->flags |= FW_UPTODATE;
-	}
 
-	cxgb_link_start(p);
 	ADAPTER_LOCK(p->adapter);
+	if ((sc->open_device_map == 0) && ((err = cxgb_up(sc)) < 0)) {
+		ADAPTER_UNLOCK(p->adapter);
+		cxgb_stop_locked(p);
+		return;
+	}
 	if (p->adapter->open_device_map == 0)
 		t3_intr_clear(sc);
-	t3_sge_start(sc);
 
-	p->adapter->open_device_map |= (1 << p->port);
+	setbit(&p->adapter->open_device_map, p->port);
+
 	ADAPTER_UNLOCK(p->adapter);
-	t3_intr_enable(sc);
+	if (is_offload(sc) && !ofld_disable) {
+		err = offload_open(p);
+		if (err)
+			log(LOG_WARNING,
+			    "Could not initialize offload capabilities\n");
+	}
+	cxgb_link_start(p);
 	t3_port_intr_enable(sc, p->port);
 
-	if ((p->adapter->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX)
-		bind_qsets(sc);
-	p->adapter->flags |= QUEUES_BOUND;
-
 	callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz,
 	    cxgb_tick, sc);
 	
-	
+	PORT_LOCK(p);
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+	PORT_UNLOCK(p);
 }
 
 static void
@@ -1179,15 +1407,21 @@ cxgb_stop_locked(struct port_info *p)
 		
 	ifp = p->ifp;
 
-	ADAPTER_LOCK(p->adapter);
+	t3_port_intr_disable(p->adapter, p->port);
+	PORT_LOCK(p);
 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
-	p->adapter->open_device_map &= ~(1 << p->port);
+	PORT_UNLOCK(p);
+	p->phy.ops->power_down(&p->phy, 1);
+	t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
+
+	ADAPTER_LOCK(p->adapter);
+	clrbit(&p->adapter->open_device_map, p->port);
+	/*
+	 * XXX cancel check_task
+	 */
 	if (p->adapter->open_device_map == 0)
-		t3_intr_disable(p->adapter);
+		cxgb_down(p->adapter);
 	ADAPTER_UNLOCK(p->adapter);
-	t3_port_intr_disable(p->adapter, p->port);
-	t3_mac_disable(&p->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
-	
 }
 
 static int
@@ -1357,7 +1591,7 @@ cxgb_start_tx(struct ifnet *ifp, uint32_t txmax)
 		m = m0;
 		if ((err = t3_encap(p, &m)) != 0)
 			break;
-		BPF_MTAP(ifp, m); 
+		BPF_MTAP(ifp, m);
 	}
 	mtx_unlock(&txq->lock);
 
@@ -1468,11 +1702,13 @@ cxgb_ext_intr_handler(void *arg, int count)
 	t3_phy_intr_handler(sc);
 
 	/* Now reenable external interrupts */
+	ADAPTER_LOCK(sc);
 	if (sc->slow_intr_mask) {
 		sc->slow_intr_mask |= F_T3DBG;
 		t3_write_reg(sc, A_PL_INT_CAUSE0, F_T3DBG);
 		t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask);
 	}
+	ADAPTER_UNLOCK(sc);
 }
 
 static void
@@ -1750,9 +1986,9 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 		if (edata->val + sc->port[!port_idx].nqsets > SGE_QSETS)
 			return (EINVAL);
 		sc->port[port_idx].nqsets = edata->val;
+		sc->port[0].first_qset = 0;
 		/*
-		 * XXX we're hardcoding ourselves to 2 ports
-		 * just like the LEENUX
+		 * XXX hardcode ourselves to 2 ports just like LEEENUX
 		 */
 		sc->port[1].first_qset = sc->port[0].nqsets;
 		break;
@@ -1762,20 +1998,52 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 		edata->val = pi->nqsets;
 		break;
 	}
-#ifdef notyet
-		/*
-		 * XXX FreeBSD driver does not currently support any
-		 * offload functionality 
-		 */
+#ifdef notyet		
 	case CHELSIO_LOAD_FW:
-	case CHELSIO_DEVUP:
-	case CHELSIO_SETMTUTAB:
 	case CHELSIO_GET_PM:
 	case CHELSIO_SET_PM:
-	case CHELSIO_READ_TCAM_WORD:
 		return (EOPNOTSUPP);
 		break;
 #endif		
+	case CHELSIO_SETMTUTAB: {
+		struct ch_mtus *m = (struct ch_mtus *)data;
+		int i;
+		
+		if (!is_offload(sc))
+			return (EOPNOTSUPP);
+		if (offload_running(sc))
+			return (EBUSY);
+		if (m->nmtus != NMTUS)
+			return (EINVAL);
+		if (m->mtus[0] < 81)         /* accommodate SACK */
+			return (EINVAL);
+		
+		/*
+		 * MTUs must be in ascending order
+		 */
+		for (i = 1; i < NMTUS; ++i)
+			if (m->mtus[i] < m->mtus[i - 1])
+				return (EINVAL);
+
+		memcpy(sc->params.mtus, m->mtus,
+		       sizeof(sc->params.mtus));
+		break;
+	}
+	case CHELSIO_GETMTUTAB: {
+		struct ch_mtus *m = (struct ch_mtus *)data;
+
+		if (!is_offload(sc))
+			return (EOPNOTSUPP);
+
+		memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
+		m->nmtus = NMTUS;
+		break;
+	}		
+	case CHELSIO_DEVUP:
+		if (!is_offload(sc))
+			return (EOPNOTSUPP);
+		return offload_open(pi);
+		break;
 	case CHELSIO_GET_MEM: {
 		struct ch_mem_range *t = (struct ch_mem_range *)data;
 		struct mc7 *mem;
@@ -1823,6 +2091,14 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 		}
 		break;
 	}
+	case CHELSIO_READ_TCAM_WORD: {
+		struct ch_tcam_word *t = (struct ch_tcam_word *)data;
+
+		if (!is_offload(sc))
+			return (EOPNOTSUPP);
+		return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
+		break;
+	}
 	case CHELSIO_SET_TRACE_FILTER: {
 		struct ch_trace *t = (struct ch_trace *)data;
 		const struct trace_params *tp;
@@ -1864,6 +2140,42 @@ cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
 
 		break;
 	}
+	case CHELSIO_SET_HW_SCHED: {
+		struct ch_hw_sched *t = (struct ch_hw_sched *)data;
+		unsigned int ticks_per_usec = core_ticks_per_usec(sc);
+
+		if ((sc->flags & FULL_INIT_DONE) == 0)
+			return (EAGAIN);       /* need TP to be initialized */
+		if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
+		    !in_range(t->channel, 0, 1) ||
+		    !in_range(t->kbps, 0, 10000000) ||
+		    !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
+		    !in_range(t->flow_ipg, 0,
+			      dack_ticks_to_usec(sc, 0x7ff)))
+			return (EINVAL);
+
+		if (t->kbps >= 0) {
+			error = t3_config_sched(sc, t->kbps, t->sched);
+			if (error < 0)
+				return (-error);
+		}
+		if (t->class_ipg >= 0)
+			t3_set_sched_ipg(sc, t->sched, t->class_ipg);
+		if (t->flow_ipg >= 0) {
+			t->flow_ipg *= 1000;     /* us -> ns */
+			t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
+		}
+		if (t->mode >= 0) {
+			int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
+
+			t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
+					 bit, t->mode ? bit : 0);
+		}
+		if (t->channel >= 0)
+			t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
+					 1 << t->sched, t->channel << t->sched);
+		break;
+	}	
 	default:
 		return (EOPNOTSUPP);
 		break;
diff --git a/sys/dev/cxgb/cxgb_offload.c b/sys/dev/cxgb/cxgb_offload.c
new file mode 100644
index 0000000..a2aea98
--- /dev/null
+++ b/sys/dev/cxgb/cxgb_offload.c
@@ -0,0 +1,1634 @@
+
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+
+***************************************************************************/
+
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/module.h>
+#include <sys/pciio.h>
+#include <sys/conf.h>
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/bus_dma.h>
+#include <sys/rman.h>
+#include <sys/ioccom.h>
+#include <sys/mbuf.h>
+#include <sys/linker.h>
+#include <sys/firmware.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/queue.h>
+#include <sys/taskqueue.h>
+
+#include <dev/cxgb/cxgb_osdep.h>
+#include <dev/cxgb/common/cxgb_common.h>
+#include <dev/cxgb/cxgb_ioctl.h>
+#include <dev/cxgb/common/cxgb_regs.h>
+#include <dev/cxgb/common/cxgb_t3_cpl.h>
+#include <dev/cxgb/common/cxgb_ctl_defs.h>
+#include <dev/cxgb/common/cxgb_firmware_exports.h>
+#include <dev/cxgb/cxgb_offload.h>
+
+#include <net/if_vlan_var.h>
+#include <net/route.h>
+
+/*
+ * XXX 
+ */
+#define LOG_ERR 1
+#define LOG_NOTICE 2
+#define BUG_ON(...)
+#define VALIDATE_TID 0
+
+
+TAILQ_HEAD(, cxgb_client) client_list;
+TAILQ_HEAD(, toedev) ofld_dev_list;
+TAILQ_HEAD(, adapter) adapter_list;
+
+static struct mtx cxgb_db_lock;
+static struct rwlock adapter_list_lock;
+
+
+static const unsigned int MAX_ATIDS = 64 * 1024;
+static const unsigned int ATID_BASE = 0x100000;
+static int inited = 0;
+
+static inline int
+offload_activated(struct toedev *tdev)
+{
+	struct adapter *adapter = tdev2adap(tdev);
+	
+	return (isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT));
+}
+
+/**
+ *	cxgb_register_client - register an offload client
+ *	@client: the client
+ *
+ *	Add the client to the client list,
+ *	and call backs the client for each activated offload device
+ */
+void
+cxgb_register_client(struct cxgb_client *client)
+{
+	struct toedev *tdev;
+
+	mtx_lock(&cxgb_db_lock);
+	TAILQ_INSERT_TAIL(&client_list, client, client_entry);
+
+	if (client->add) {
+		TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) {
+			if (offload_activated(tdev))
+				client->add(tdev);
+		}
+	}
+	mtx_unlock(&cxgb_db_lock);
+}
+
+/**
+ *	cxgb_unregister_client - unregister an offload client
+ *	@client: the client
+ *
+ *	Remove the client to the client list,
+ *	and call backs the client for each activated offload device.
+ */
+void
+cxgb_unregister_client(struct cxgb_client *client)
+{
+	struct toedev *tdev;
+
+	mtx_lock(&cxgb_db_lock);
+	TAILQ_REMOVE(&client_list, client, client_entry);
+
+	if (client->remove) {
+		TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) {
+			if (offload_activated(tdev))
+				client->remove(tdev);
+		}
+	}
+	mtx_unlock(&cxgb_db_lock);
+}
+
+/**
+ *	cxgb_add_clients - activate register clients for an offload device
+ *	@tdev: the offload device
+ *
+ *	Call backs all registered clients once a offload device is activated 
+ */
+void
+cxgb_add_clients(struct toedev *tdev)
+{
+	struct cxgb_client *client;
+
+	mtx_lock(&cxgb_db_lock);
+	TAILQ_FOREACH(client, &client_list, client_entry) {
+		if (client->add)
+			client->add(tdev);
+	}
+	mtx_unlock(&cxgb_db_lock);
+}
+
+/**
+ *	cxgb_remove_clients - activate register clients for an offload device
+ *	@tdev: the offload device
+ *
+ *	Call backs all registered clients once a offload device is deactivated 
+ */
+void
+cxgb_remove_clients(struct toedev *tdev)
+{
+	struct cxgb_client *client;
+
+	mtx_lock(&cxgb_db_lock);
+	TAILQ_FOREACH(client, &client_list, client_entry) {
+		if (client->remove)
+			client->remove(tdev);
+	}
+	mtx_unlock(&cxgb_db_lock);
+}
+
+static int
+is_offloading(struct ifnet *ifp)
+{
+	struct adapter *adapter;
+	int port;
+
+	rw_rlock(&adapter_list_lock);
+	TAILQ_FOREACH(adapter, &adapter_list, adapter_entry) {
+		for_each_port(adapter, port) {
+			if (ifp == adapter->port[port].ifp) {
+				rw_runlock(&adapter_list_lock);
+				return 1;
+			}
+		}
+	}
+	rw_runlock(&adapter_list_lock);
+	return 0;
+}
+
+static struct ifnet *
+get_iff_from_mac(adapter_t *adapter, const uint8_t *mac, unsigned int vlan)
+{
+#ifdef notyet	
+	int i;
+
+	for_each_port(adapter, i) {
+		const struct vlan_group *grp;
+		const struct port_info *p = &adapter->port[i];
+		struct ifnet *ifnet = p->ifp;
+
+		if (!memcmp(p->hw_addr, mac, ETHER_ADDR_LEN)) {
+			if (vlan && vlan != EVL_VLID_MASK) {
+				grp = p->vlan_grp;
+				dev = grp ? grp->vlan_devices[vlan] : NULL;
+			} else
+				while (dev->master)
+					dev = dev->master;
+			return dev;
+		}
+	}
+#endif	
+	return NULL;
+}
+
+static inline void
+failover_fixup(adapter_t *adapter, int port)
+{
+	if (adapter->params.rev == 0) {
+		struct ifnet *ifp = adapter->port[port].ifp;
+		struct cmac *mac = &adapter->port[port].mac;
+		if (!(ifp->if_flags & IFF_UP)) {
+			/* Failover triggered by the interface ifdown */
+			t3_write_reg(adapter, A_XGM_TX_CTRL + mac->offset,
+				     F_TXEN);
+			t3_read_reg(adapter, A_XGM_TX_CTRL + mac->offset);
+		} else {
+			/* Failover triggered by the interface link down */
+			t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, 0);
+			t3_read_reg(adapter, A_XGM_RX_CTRL + mac->offset);
+			t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset,
+				     F_RXEN);
+		}
+	}
+}
+
+static int
+cxgb_ulp_iscsi_ctl(adapter_t *adapter, unsigned int req, void *data)
+{
+	int ret = 0;
+	struct ulp_iscsi_info *uiip = data;
+
+	switch (req) {
+	case ULP_ISCSI_GET_PARAMS:
+		uiip->llimit = t3_read_reg(adapter, A_ULPRX_ISCSI_LLIMIT);
+		uiip->ulimit = t3_read_reg(adapter, A_ULPRX_ISCSI_ULIMIT);
+		uiip->tagmask = t3_read_reg(adapter, A_ULPRX_ISCSI_TAGMASK);
+		/*
+		 * On tx, the iscsi pdu has to be <= tx page size and has to
+		 * fit into the Tx PM FIFO.
+		 */
+		uiip->max_txsz = min(adapter->params.tp.tx_pg_size,
+				     t3_read_reg(adapter, A_PM1_TX_CFG) >> 17);
+		/* on rx, the iscsi pdu has to be < rx page size and the
+		   whole pdu + cpl headers has to fit into one sge buffer */
+		uiip->max_rxsz =
+		    (unsigned int)min(adapter->params.tp.rx_pg_size,
+			(adapter->sge.qs[0].fl[1].buf_size -
+			    sizeof(struct cpl_rx_data) * 2 -
+			    sizeof(struct cpl_rx_data_ddp)) );
+		break;
+	case ULP_ISCSI_SET_PARAMS:
+		t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask);
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+	}
+	return ret;
+}
+
+/* Response queue used for RDMA events. */
+#define ASYNC_NOTIF_RSPQ 0
+
+static int
+cxgb_rdma_ctl(adapter_t *adapter, unsigned int req, void *data)
+{
+	int ret = 0;
+
+	switch (req) {
+	case RDMA_GET_PARAMS: {
+		struct rdma_info *req = data;
+
+		req->udbell_physbase = rman_get_start(adapter->regs_res);
+		req->udbell_len = rman_get_size(adapter->regs_res);
+		req->tpt_base = t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT);
+		req->tpt_top  = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT);
+		req->pbl_base = t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT);
+		req->pbl_top  = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT);
+		req->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT);
+		req->rqt_top  = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT);
+		req->kdb_addr = (void *)(rman_get_start(adapter->regs_res) + A_SG_KDOORBELL);
+		break;
+	}
+	case RDMA_CQ_OP: {
+		struct rdma_cq_op *req = data;
+
+		/* may be called in any context */
+		mtx_lock(&adapter->sge.reg_lock);
+		ret = t3_sge_cqcntxt_op(adapter, req->id, req->op,
+					req->credits);
+		mtx_unlock(&adapter->sge.reg_lock);
+		break;
+	}
+	case RDMA_GET_MEM: {
+		struct ch_mem_range *t = data;
+		struct mc7 *mem;
+
+		if ((t->addr & 7) || (t->len & 7))
+			return -EINVAL;
+		if (t->mem_id == MEM_CM)
+			mem = &adapter->cm;
+		else if (t->mem_id == MEM_PMRX)
+			mem = &adapter->pmrx;
+		else if (t->mem_id == MEM_PMTX)
+			mem = &adapter->pmtx;
+		else
+			return -EINVAL;
+
+		ret = t3_mc7_bd_read(mem, t->addr/8, t->len/8, (u64 *)t->buf);
+		if (ret)
+			return ret;
+		break;
+	}
+	case RDMA_CQ_SETUP: {
+		struct rdma_cq_setup *req = data;
+
+		mtx_lock(&adapter->sge.reg_lock);
+		ret = t3_sge_init_cqcntxt(adapter, req->id, req->base_addr,
+					  req->size, ASYNC_NOTIF_RSPQ,
+					  req->ovfl_mode, req->credits,
+					  req->credit_thres);
+		mtx_unlock(&adapter->sge.reg_lock);
+		break;
+	}
+	case RDMA_CQ_DISABLE:
+		mtx_lock(&adapter->sge.reg_lock);
+		ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data);
+		mtx_unlock(&adapter->sge.reg_lock);
+		break;
+	case RDMA_CTRL_QP_SETUP: {
+		struct rdma_ctrlqp_setup *req = data;
+
+		mtx_lock(&adapter->sge.reg_lock);
+		ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0,
+					 SGE_CNTXT_RDMA, ASYNC_NOTIF_RSPQ,
+					 req->base_addr, req->size,
+					 FW_RI_TID_START, 1, 0);
+		mtx_unlock(&adapter->sge.reg_lock);
+		break;
+	}
+	default:
+		ret = -EOPNOTSUPP;
+	}
+	return ret;
+}
+
+static int
+cxgb_offload_ctl(struct toedev *tdev, unsigned int req, void *data)
+{
+	struct adapter *adapter = tdev2adap(tdev);
+	struct tid_range *tid;
+	struct mtutab *mtup;
+	struct iff_mac *iffmacp;
+	struct ddp_params *ddpp;
+	struct adap_ports *ports;
+	int port;
+
+	switch (req) {
+	case GET_MAX_OUTSTANDING_WR:
+		*(unsigned int *)data = FW_WR_NUM;
+		break;
+	case GET_WR_LEN:
+		*(unsigned int *)data = WR_FLITS;
+		break;
+	case GET_TX_MAX_CHUNK:
+		*(unsigned int *)data = 1 << 20;  /* 1MB */
+		break;
+	case GET_TID_RANGE:
+		tid = data;
+		tid->num = t3_mc5_size(&adapter->mc5) -
+			adapter->params.mc5.nroutes -
+			adapter->params.mc5.nfilters -
+			adapter->params.mc5.nservers;
+		tid->base = 0;
+		break;
+	case GET_STID_RANGE:
+		tid = data;
+		tid->num = adapter->params.mc5.nservers;
+		tid->base = t3_mc5_size(&adapter->mc5) - tid->num -
+			adapter->params.mc5.nfilters -
+			adapter->params.mc5.nroutes;
+		break;
+	case GET_L2T_CAPACITY:
+		*(unsigned int *)data = 2048;
+		break;
+	case GET_MTUS:
+		mtup = data;
+		mtup->size = NMTUS;
+		mtup->mtus = adapter->params.mtus;
+		break;
+	case GET_IFF_FROM_MAC:
+		iffmacp = data;
+		iffmacp->dev = get_iff_from_mac(adapter, iffmacp->mac_addr,
+					  iffmacp->vlan_tag & EVL_VLID_MASK);
+		break;
+	case GET_DDP_PARAMS:
+		ddpp = data;
+		ddpp->llimit = t3_read_reg(adapter, A_ULPRX_TDDP_LLIMIT);
+		ddpp->ulimit = t3_read_reg(adapter, A_ULPRX_TDDP_ULIMIT);
+		ddpp->tag_mask = t3_read_reg(adapter, A_ULPRX_TDDP_TAGMASK);
+		break;
+	case GET_PORTS:
+		ports = data;
+		ports->nports   = adapter->params.nports;
+		for_each_port(adapter, port)
+			ports->lldevs[port] = adapter->port[port].ifp;
+		break;
+	case FAILOVER:
+		port = *(int *)data;
+		t3_port_failover(adapter, port);
+		failover_fixup(adapter, port);
+		break;
+	case FAILOVER_DONE:
+		port = *(int *)data;
+		t3_failover_done(adapter, port);
+		break;
+	case FAILOVER_CLEAR:
+		t3_failover_clear(adapter);
+		break;
+	case ULP_ISCSI_GET_PARAMS:
+	case ULP_ISCSI_SET_PARAMS:
+		if (!offload_running(adapter))
+			return -EAGAIN;
+		return cxgb_ulp_iscsi_ctl(adapter, req, data);
+	case RDMA_GET_PARAMS:
+	case RDMA_CQ_OP:
+	case RDMA_CQ_SETUP:
+	case RDMA_CQ_DISABLE:
+	case RDMA_CTRL_QP_SETUP:
+	case RDMA_GET_MEM:
+		if (!offload_running(adapter))
+			return -EAGAIN;
+		return cxgb_rdma_ctl(adapter, req, data);
+	default:
+		return -EOPNOTSUPP;
+	}
+	return 0;
+}
+
+/*
+ * Dummy handler for Rx offload packets in case we get an offload packet before
+ * proper processing is setup.  This complains and drops the packet as it isn't
+ * normal to get offload packets at this stage.
+ */
+static int
+rx_offload_blackhole(struct toedev *dev, struct mbuf **m, int n)
+{
+	CH_ERR(tdev2adap(dev), "%d unexpected offload packets, first data %u\n",
+	    n, ntohl(*mtod(m[0], uint32_t *)));
+	while (n--)
+		m_freem(m[n]);
+	return 0;
+}
+
+static void
+dummy_neigh_update(struct toedev *dev, struct rtentry *neigh)
+{
+}
+
+void
+cxgb_set_dummy_ops(struct toedev *dev)
+{
+	dev->recv         = rx_offload_blackhole;
+	dev->neigh_update = dummy_neigh_update;
+}
+
+/*
+ * Free an active-open TID.
+ */
+void *
+cxgb_free_atid(struct toedev *tdev, int atid)
+{
+	struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
+	union active_open_entry *p = atid2entry(t, atid);
+	void *ctx = p->toe_tid.ctx;
+
+	mtx_lock(&t->atid_lock);
+	p->next = t->afree;
+	t->afree = p;
+	t->atids_in_use--;
+	mtx_lock(&t->atid_lock);
+
+	return ctx;
+}
+
+/*
+ * Free a server TID and return it to the free pool.
+ */
+void
+cxgb_free_stid(struct toedev *tdev, int stid)
+{
+	struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
+	union listen_entry *p = stid2entry(t, stid);
+
+	mtx_lock(&t->stid_lock);
+	p->next = t->sfree;
+	t->sfree = p;
+	t->stids_in_use--;
+	mtx_unlock(&t->stid_lock);
+}
+
+void
+cxgb_insert_tid(struct toedev *tdev, struct cxgb_client *client,
+	void *ctx, unsigned int tid)
+{
+	struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
+
+	t->tid_tab[tid].client = client;
+	t->tid_tab[tid].ctx = ctx;
+	atomic_add_int(&t->tids_in_use, 1);
+}
+
+/*
+ * Populate a TID_RELEASE WR.  The mbuf must be already propely sized.
+ */
+static inline void
+mk_tid_release(struct mbuf *m, unsigned int tid)
+{
+	struct cpl_tid_release *req;
+
+	m_set_priority(m, CPL_PRIORITY_SETUP);
+	req = mtod(m, struct cpl_tid_release *);
+	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
+}
+
+static void
+t3_process_tid_release_list(void *data, int pending)
+{
+	struct mbuf *m;
+	struct toedev *tdev = data;
+	struct toe_data *td = TOE_DATA(tdev);
+
+	mtx_lock(&td->tid_release_lock);
+	while (td->tid_release_list) {
+		struct toe_tid_entry *p = td->tid_release_list;
+
+		td->tid_release_list = (struct toe_tid_entry *)p->ctx;
+		mtx_unlock(&td->tid_release_lock);
+		m = m_get(M_WAIT, MT_DATA);
+		mk_tid_release(m, p - td->tid_maps.tid_tab);
+		cxgb_ofld_send(tdev, m);
+		p->ctx = NULL;
+		mtx_lock(&td->tid_release_lock);
+	}
+	mtx_unlock(&td->tid_release_lock);
+}
+
+/* use ctx as a next pointer in the tid release list */
+void
+cxgb_queue_tid_release(struct toedev *tdev, unsigned int tid)
+{
+	struct toe_data *td = TOE_DATA(tdev);
+	struct toe_tid_entry *p = &td->tid_maps.tid_tab[tid];
+
+	mtx_lock(&td->tid_release_lock);
+	p->ctx = td->tid_release_list;
+	td->tid_release_list = p;
+
+	if (!p->ctx)
+		taskqueue_enqueue(tdev->adapter->tq, &td->tid_release_task);
+
+	mtx_unlock(&td->tid_release_lock);
+}
+
+/*
+ * Remove a tid from the TID table.  A client may defer processing its last
+ * CPL message if it is locked at the time it arrives, and while the message
+ * sits in the client's backlog the TID may be reused for another connection.
+ * To handle this we atomically switch the TID association if it still points
+ * to the original client context.
+ */
+void
+cxgb_remove_tid(struct toedev *tdev, void *ctx, unsigned int tid)
+{
+	struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
+
+	BUG_ON(tid >= t->ntids);
+	if (tdev->type == T3A)
+		atomic_cmpset_ptr((long *)&t->tid_tab[tid].ctx, (long)NULL, (long)ctx);
+	else {
+		struct mbuf *m;
+
+		m = m_get(M_NOWAIT, MT_DATA);
+		if (__predict_true(m != NULL)) {
+			mk_tid_release(m, tid);
+			cxgb_ofld_send(tdev, m);
+			t->tid_tab[tid].ctx = NULL;
+		} else
+			cxgb_queue_tid_release(tdev, tid);
+	}
+	atomic_add_int(&t->tids_in_use, -1);
+}
+
+int
+cxgb_alloc_atid(struct toedev *tdev, struct cxgb_client *client,
+		     void *ctx)
+{
+	int atid = -1;
+	struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
+
+	mtx_lock(&t->atid_lock);
+	if (t->afree) {
+		union active_open_entry *p = t->afree;
+
+		atid = (p - t->atid_tab) + t->atid_base;
+		t->afree = p->next;
+		p->toe_tid.ctx = ctx;
+		p->toe_tid.client = client;
+		t->atids_in_use++;
+	}
+	mtx_unlock(&t->atid_lock);
+	return atid;
+}
+
+int
+cxgb_alloc_stid(struct toedev *tdev, struct cxgb_client *client,
+		     void *ctx)
+{
+	int stid = -1;
+	struct tid_info *t = &(TOE_DATA(tdev))->tid_maps;
+
+	mtx_lock(&t->stid_lock);
+	if (t->sfree) {
+		union listen_entry *p = t->sfree;
+
+		stid = (p - t->stid_tab) + t->stid_base;
+		t->sfree = p->next;
+		p->toe_tid.ctx = ctx;
+		p->toe_tid.client = client;
+		t->stids_in_use++;
+	}
+	mtx_unlock(&t->stid_lock);
+	return stid;
+}
+
+static int
+do_smt_write_rpl(struct toedev *dev, struct mbuf *m)
+{
+	struct cpl_smt_write_rpl *rpl = cplhdr(m);
+
+	if (rpl->status != CPL_ERR_NONE)
+		log(LOG_ERR,
+		       "Unexpected SMT_WRITE_RPL status %u for entry %u\n",
+		       rpl->status, GET_TID(rpl));
+
+	return CPL_RET_BUF_DONE;
+}
+
+static int
+do_l2t_write_rpl(struct toedev *dev, struct mbuf *m)
+{
+	struct cpl_l2t_write_rpl *rpl = cplhdr(m);
+
+	if (rpl->status != CPL_ERR_NONE)
+		log(LOG_ERR,
+		       "Unexpected L2T_WRITE_RPL status %u for entry %u\n",
+		       rpl->status, GET_TID(rpl));
+
+	return CPL_RET_BUF_DONE;
+}
+
+static int
+do_act_open_rpl(struct toedev *dev, struct mbuf *m)
+{
+	struct cpl_act_open_rpl *rpl = cplhdr(m);
+	unsigned int atid = G_TID(ntohl(rpl->atid));
+	struct toe_tid_entry *toe_tid;
+
+	toe_tid = lookup_atid(&(TOE_DATA(dev))->tid_maps, atid);
+	if (toe_tid->ctx && toe_tid->client && toe_tid->client->handlers &&
+		toe_tid->client->handlers[CPL_ACT_OPEN_RPL]) {
+		return toe_tid->client->handlers[CPL_ACT_OPEN_RPL] (dev, m,
+			toe_tid->ctx);
+	} else {
+		log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
+			dev->name, CPL_ACT_OPEN_RPL);
+		return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
+	}
+}
+
+static int
+do_stid_rpl(struct toedev *dev, struct mbuf *m)
+{
+	union opcode_tid *p = cplhdr(m);
+	unsigned int stid = G_TID(ntohl(p->opcode_tid));
+	struct toe_tid_entry *toe_tid;
+
+	toe_tid = lookup_stid(&(TOE_DATA(dev))->tid_maps, stid);
+	if (toe_tid->ctx && toe_tid->client->handlers &&
+		toe_tid->client->handlers[p->opcode]) {
+		return toe_tid->client->handlers[p->opcode] (dev, m, toe_tid->ctx);
+	} else {
+		log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
+			dev->name, p->opcode);
+		return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
+	}
+}
+
+static int
+do_hwtid_rpl(struct toedev *dev, struct mbuf *m)
+{
+	union opcode_tid *p = cplhdr(m);
+	unsigned int hwtid = G_TID(ntohl(p->opcode_tid));
+	struct toe_tid_entry *toe_tid;
+
+	toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid);
+	if (toe_tid->ctx && toe_tid->client->handlers &&
+		toe_tid->client->handlers[p->opcode]) {
+		return toe_tid->client->handlers[p->opcode]
+						(dev, m, toe_tid->ctx);
+	} else {
+		log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
+			dev->name, p->opcode);
+		return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
+	}
+}
+
+static int
+do_cr(struct toedev *dev, struct mbuf *m)
+{
+	struct cpl_pass_accept_req *req = cplhdr(m);
+	unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
+	struct toe_tid_entry *toe_tid;
+
+	toe_tid = lookup_stid(&(TOE_DATA(dev))->tid_maps, stid);
+	if (toe_tid->ctx && toe_tid->client->handlers &&
+		toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ]) {
+		return toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ]
+						(dev, m, toe_tid->ctx);
+	} else {
+		log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
+			dev->name, CPL_PASS_ACCEPT_REQ);
+		return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
+	}
+}
+
+static int
+do_abort_req_rss(struct toedev *dev, struct mbuf *m)
+{
+	union opcode_tid *p = cplhdr(m);
+	unsigned int hwtid = G_TID(ntohl(p->opcode_tid));
+	struct toe_tid_entry *toe_tid;
+
+	toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid);
+	if (toe_tid->ctx && toe_tid->client->handlers &&
+		toe_tid->client->handlers[p->opcode]) {
+		return toe_tid->client->handlers[p->opcode]
+						(dev, m, toe_tid->ctx);
+	} else {
+		struct cpl_abort_req_rss *req = cplhdr(m);
+		struct cpl_abort_rpl *rpl;
+		
+		struct mbuf *m = m_get(M_NOWAIT, MT_DATA);
+		if (!m) {
+			log(LOG_NOTICE, "do_abort_req_rss: couldn't get mbuf!\n");
+			goto out;
+		}
+
+		m_set_priority(m, CPL_PRIORITY_DATA);
+#if 0	
+		__skb_put(skb, sizeof(struct cpl_abort_rpl));
+#endif		
+		rpl = cplhdr(m);
+		rpl->wr.wr_hi = 
+			htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
+		rpl->wr.wr_lo = htonl(V_WR_TID(GET_TID(req)));
+		OPCODE_TID(rpl) =
+			htonl(MK_OPCODE_TID(CPL_ABORT_RPL, GET_TID(req)));
+		rpl->cmd = req->status;
+		cxgb_ofld_send(dev, m);
+ out:
+		return CPL_RET_BUF_DONE;
+	}
+}
+
+static int
+do_act_establish(struct toedev *dev, struct mbuf *m)
+{
+	struct cpl_act_establish *req = cplhdr(m);
+	unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
+	struct toe_tid_entry *toe_tid;
+
+	toe_tid = lookup_atid(&(TOE_DATA(dev))->tid_maps, atid);
+	if (toe_tid->ctx && toe_tid->client->handlers &&
+		toe_tid->client->handlers[CPL_ACT_ESTABLISH]) {
+		return toe_tid->client->handlers[CPL_ACT_ESTABLISH]
+						(dev, m, toe_tid->ctx);
+	} else {
+		log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
+			dev->name, CPL_PASS_ACCEPT_REQ);
+		return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
+	}
+}
+
+static int
+do_set_tcb_rpl(struct toedev *dev, struct mbuf *m)
+{
+	struct cpl_set_tcb_rpl *rpl = cplhdr(m);
+
+	if (rpl->status != CPL_ERR_NONE)
+		log(LOG_ERR,
+		    "Unexpected SET_TCB_RPL status %u for tid %u\n",
+			rpl->status, GET_TID(rpl));
+	return CPL_RET_BUF_DONE;
+}
+
+static int
+do_trace(struct toedev *dev, struct mbuf *m)
+{
+#if 0
+	struct cpl_trace_pkt *p = cplhdr(m);
+
+
+	skb->protocol = 0xffff;
+	skb->dev = dev->lldev;
+	skb_pull(skb, sizeof(*p));
+	skb->mac.raw = mtod(m, (char *));
+	netif_receive_skb(skb);
+#endif	
+	return 0;
+}
+
+static int
+do_term(struct toedev *dev, struct mbuf *m)
+{
+	unsigned int hwtid = ntohl(m_get_priority(m)) >> 8 & 0xfffff;
+	unsigned int opcode = G_OPCODE(ntohl(m->m_pkthdr.csum_data));
+	struct toe_tid_entry *toe_tid;
+
+	toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid);
+	if (toe_tid->ctx && toe_tid->client->handlers &&
+		toe_tid->client->handlers[opcode]) {
+		return toe_tid->client->handlers[opcode](dev, m, toe_tid->ctx);
+	} else {
+		log(LOG_ERR, "%s: received clientless CPL command 0x%x\n",
+			dev->name, opcode);
+		return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG;
+	}
+	return (0);
+}
+
+#if defined(FOO)
+#include <linux/config.h> 
+#include <linux/kallsyms.h> 
+#include <linux/kprobes.h> 
+#include <net/arp.h> 
+
+static int (*orig_arp_constructor)(struct ifnet *);
+
+static void
+neigh_suspect(struct ifnet *neigh)
+{
+	struct hh_cache *hh;
+
+	neigh->output = neigh->ops->output;
+
+	for (hh = neigh->hh; hh; hh = hh->hh_next)
+		hh->hh_output = neigh->ops->output;
+}
+
+static void
+neigh_connect(struct ifnet *neigh)
+{
+	struct hh_cache *hh;
+
+	neigh->output = neigh->ops->connected_output;
+
+	for (hh = neigh->hh; hh; hh = hh->hh_next)
+		hh->hh_output = neigh->ops->hh_output;
+}
+
+static inline int
+neigh_max_probes(const struct neighbour *n)
+{
+	const struct neigh_parms *p = n->parms;
+	return (n->nud_state & NUD_PROBE ?
+		p->ucast_probes :
+		p->ucast_probes + p->app_probes + p->mcast_probes);
+}
+
+static void
+neigh_timer_handler_offload(unsigned long arg)
+{
+	unsigned long now, next;
+	struct neighbour *neigh = (struct neighbour *)arg;
+	unsigned state;
+	int notify = 0;
+
+	write_lock(&neigh->lock);
+
+	state = neigh->nud_state;
+	now = jiffies;
+	next = now + HZ;
+
+	if (!(state & NUD_IN_TIMER)) {
+#ifndef CONFIG_SMP
+		log(LOG_WARNING, "neigh: timer & !nud_in_timer\n");
+#endif
+		goto out;
+	}
+
+	if (state & NUD_REACHABLE) {
+		if (time_before_eq(now,
+				   neigh->confirmed + 
+				   neigh->parms->reachable_time)) {
+			next = neigh->confirmed + neigh->parms->reachable_time;
+		} else if (time_before_eq(now,
+					  neigh->used + 
+					  neigh->parms->delay_probe_time)) {
+			neigh->nud_state = NUD_DELAY;
+			neigh->updated = jiffies;
+			neigh_suspect(neigh);
+			next = now + neigh->parms->delay_probe_time;
+		} else {
+			neigh->nud_state = NUD_STALE;
+			neigh->updated = jiffies;
+			neigh_suspect(neigh);
+			cxgb_neigh_update(neigh);
+		}
+	} else if (state & NUD_DELAY) {
+		if (time_before_eq(now,
+				   neigh->confirmed +
+				   neigh->parms->delay_probe_time)) {
+			neigh->nud_state = NUD_REACHABLE;
+			neigh->updated = jiffies;
+			neigh_connect(neigh);
+			cxgb_neigh_update(neigh);
+			next = neigh->confirmed + neigh->parms->reachable_time;
+		} else {
+			neigh->nud_state = NUD_PROBE;
+			neigh->updated = jiffies;
+			atomic_set_int(&neigh->probes, 0);
+			next = now + neigh->parms->retrans_time;
+		}
+	} else {
+		/* NUD_PROBE|NUD_INCOMPLETE */
+		next = now + neigh->parms->retrans_time;
+	}
+	/*
+	 * Needed for read of probes
+	 */
+	mb();
+	if ((neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) &&
+	    neigh->probes >= neigh_max_probes(neigh)) {
+		struct mbuf *m;
+
+		neigh->nud_state = NUD_FAILED;
+		neigh->updated = jiffies;
+		notify = 1;
+		cxgb_neigh_update(neigh);
+		NEIGH_CACHE_STAT_INC(neigh->tbl, res_failed);
+
+		/* It is very thin place. report_unreachable is very 
+		   complicated routine. Particularly, it can hit the same 
+		   neighbour entry!
+		   So that, we try to be accurate and avoid dead loop. --ANK
+		 */
+		while (neigh->nud_state == NUD_FAILED &&
+		       (skb = __skb_dequeue(&neigh->arp_queue)) != NULL) {
+			write_unlock(&neigh->lock);
+			neigh->ops->error_report(neigh, skb);
+			write_lock(&neigh->lock);
+		}
+		skb_queue_purge(&neigh->arp_queue);
+	}
+
+	if (neigh->nud_state & NUD_IN_TIMER) {
+		if (time_before(next, jiffies + HZ/2))
+			next = jiffies + HZ/2;
+		if (!mod_timer(&neigh->timer, next))
+			neigh_hold(neigh);
+	}
+	if (neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) {
+		struct mbuf *m = skb_peek(&neigh->arp_queue);
+
+		write_unlock(&neigh->lock);
+		neigh->ops->solicit(neigh, skb);
+		atomic_add_int(&neigh->probes, 1);
+		if (m)
+			m_free(m);
+	} else {
+out:
+		write_unlock(&neigh->lock);
+	}
+
+#ifdef CONFIG_ARPD
+	if (notify && neigh->parms->app_probes)
+		neigh_app_notify(neigh);
+#endif
+	neigh_release(neigh);
+}
+
+static int
+arp_constructor_offload(struct neighbour *neigh)
+{
+	if (neigh->ifp && is_offloading(neigh->ifp))
+		neigh->timer.function = neigh_timer_handler_offload;
+	return orig_arp_constructor(neigh);
+}
+
+/*
+ * This must match exactly the signature of neigh_update for jprobes to work.
+ * It runs from a trap handler with interrupts off so don't disable BH.
+ */
+static int
+neigh_update_offload(struct neighbour *neigh, const u8 *lladdr,
+				u8 new, u32 flags)
+{
+	write_lock(&neigh->lock);
+	cxgb_neigh_update(neigh);
+	write_unlock(&neigh->lock);
+	jprobe_return();
+	/* NOTREACHED */
+	return 0;
+}
+
+static struct jprobe neigh_update_jprobe = {
+	.entry = (kprobe_opcode_t *) neigh_update_offload,
+	.kp.addr = (kprobe_opcode_t *) neigh_update
+};
+
+#ifdef MODULE_SUPPORT
+static int
+prepare_arp_with_t3core(void)
+{
+	int err;
+
+	err = register_jprobe(&neigh_update_jprobe);
+	if (err) {
+		log(LOG_ERR, "Could not install neigh_update jprobe, "
+				"error %d\n", err);
+		return err;
+	}
+
+	orig_arp_constructor = arp_tbl.constructor;
+	arp_tbl.constructor  = arp_constructor_offload;
+
+	return 0;
+}
+
+static void
+restore_arp_sans_t3core(void)
+{
+	arp_tbl.constructor = orig_arp_constructor;
+	unregister_jprobe(&neigh_update_jprobe);
+}
+
+#else /* Module suport */
+static inline int
+prepare_arp_with_t3core(void)
+{
+	return 0;
+}
+
+static inline void
+restore_arp_sans_t3core(void)
+{}
+#endif
+#endif
+/*
+ * Process a received packet with an unknown/unexpected CPL opcode.
+ */
+static int
+do_bad_cpl(struct toedev *dev, struct mbuf *m)
+{
+	log(LOG_ERR, "%s: received bad CPL command 0x%x\n", dev->name,
+	    *mtod(m, uint32_t *));
+	return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG);
+}
+
+/*
+ * Handlers for each CPL opcode
+ */
+static cpl_handler_func cpl_handlers[NUM_CPL_CMDS];
+
+/*
+ * Add a new handler to the CPL dispatch table.  A NULL handler may be supplied
+ * to unregister an existing handler.
+ */
+void
+t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h)
+{
+	if (opcode < NUM_CPL_CMDS)
+		cpl_handlers[opcode] = h ? h : do_bad_cpl;
+	else
+		log(LOG_ERR, "T3C: handler registration for "
+		       "opcode %x failed\n", opcode);
+}
+
+/*
+ * TOEDEV's receive method.
+ */
+int
+process_rx(struct toedev *dev, struct mbuf **m, int n)
+{
+	while (n--) {
+		struct mbuf *m0 = *m++;
+		unsigned int opcode = G_OPCODE(ntohl(m0->m_pkthdr.csum_data));
+		int ret = cpl_handlers[opcode] (dev, m0);
+
+#if VALIDATE_TID
+		if (ret & CPL_RET_UNKNOWN_TID) {
+			union opcode_tid *p = cplhdr(m0);
+
+			log(LOG_ERR, "%s: CPL message (opcode %u) had "
+			       "unknown TID %u\n", dev->name, opcode,
+			       G_TID(ntohl(p->opcode_tid)));
+		}
+#endif
+		if (ret & CPL_RET_BUF_DONE)
+			m_freem(m0);
+	}
+	return 0;
+}
+
+/*
+ * Sends an sk_buff to a T3C driver after dealing with any active network taps.
+ */
+int
+cxgb_ofld_send(struct toedev *dev, struct mbuf *m)
+{
+	int r;
+
+	critical_enter();
+	r = dev->send(dev, m);
+	critical_exit();
+	return r;
+}
+
+
+/**
+ * cxgb_ofld_recv - process n received offload packets
+ * @dev: the offload device
+ * @m: an array of offload packets
+ * @n: the number of offload packets
+ *
+ * Process an array of ingress offload packets.  Each packet is forwarded
+ * to any active network taps and then passed to the offload device's receive
+ * method.  We optimize passing packets to the receive method by passing
+ * it the whole array at once except when there are active taps.
+ */
+int
+cxgb_ofld_recv(struct toedev *dev, struct mbuf **m, int n)
+{
+
+#if defined(CONFIG_CHELSIO_T3)
+	if (likely(!netdev_nit))
+		return dev->recv(dev, skb, n);
+
+	for ( ; n; n--, skb++) {
+		skb[0]->dev = dev->lldev;
+		dev_queue_xmit_nit(skb[0], dev->lldev);
+		skb[0]->dev = NULL;
+		dev->recv(dev, skb, 1);
+	}
+	return 0;
+#else
+	return dev->recv(dev, m, n);
+#endif
+}
+
+void
+cxgb_neigh_update(struct rtentry *rt)
+{
+
+	if (is_offloading(rt->rt_ifp)) {
+		struct toedev *tdev = TOEDEV(rt->rt_ifp);
+
+		BUG_ON(!tdev);
+		t3_l2t_update(tdev, rt);
+	}
+}
+
+static void
+set_l2t_ix(struct toedev *tdev, u32 tid, struct l2t_entry *e)
+{
+	struct mbuf *m;
+	struct cpl_set_tcb_field *req;
+
+	m = m_gethdr(M_NOWAIT, MT_DATA);
+	if (!m) {
+		log(LOG_ERR, "%s: cannot allocate mbuf!\n", __FUNCTION__);
+		return;
+	}
+	
+	m_set_priority(m, CPL_PRIORITY_CONTROL);
+	req = mtod(m, struct cpl_set_tcb_field *);
+	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
+	req->reply = 0;
+	req->cpu_idx = 0;
+	req->word = htons(W_TCB_L2T_IX);
+	req->mask = htobe64(V_TCB_L2T_IX(M_TCB_L2T_IX));
+	req->val = htobe64(V_TCB_L2T_IX(e->idx));
+	tdev->send(tdev, m);
+}
+
+void
+cxgb_redirect(struct rtentry *old, struct rtentry *new)
+{
+	struct ifnet *olddev, *newdev;
+	struct tid_info *ti;
+	struct toedev *tdev;
+	u32 tid;
+	int update_tcb;
+	struct l2t_entry *e;
+	struct toe_tid_entry *te;
+
+	olddev = old->rt_ifp;
+	newdev = new->rt_ifp;
+	if (!is_offloading(olddev))
+		return;
+	if (!is_offloading(newdev)) {
+		log(LOG_WARNING, "%s: Redirect to non-offload"
+		    "device ignored.\n", __FUNCTION__);
+		return;
+	}
+	tdev = TOEDEV(olddev);
+	BUG_ON(!tdev);
+	if (tdev != TOEDEV(newdev)) {
+		log(LOG_WARNING, "%s: Redirect to different "
+		    "offload device ignored.\n", __FUNCTION__);
+		return;
+	}
+
+	/* Add new L2T entry */
+	e = t3_l2t_get(tdev, new, ((struct port_info *)new->rt_ifp->if_softc)->port);
+	if (!e) {
+		log(LOG_ERR, "%s: couldn't allocate new l2t entry!\n",
+		       __FUNCTION__);
+		return;
+	}
+
+	/* Walk tid table and notify clients of dst change. */
+	ti = &(TOE_DATA(tdev))->tid_maps;
+	for (tid=0; tid < ti->ntids; tid++) {
+		te = lookup_tid(ti, tid);
+		BUG_ON(!te);
+		if (te->ctx && te->client && te->client->redirect) {
+			update_tcb = te->client->redirect(te->ctx, old, new,
+							  e);
+			if (update_tcb)  {
+				l2t_hold(L2DATA(tdev), e);
+				set_l2t_ix(tdev, tid, e);
+			}
+		}
+	}
+	l2t_release(L2DATA(tdev), e);
+}
+
+/*
+ * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
+ * The allocated memory is cleared.
+ */
+void *
+cxgb_alloc_mem(unsigned long size)
+{
+
+	return malloc(size, M_DEVBUF, M_ZERO);
+}
+
+/*
+ * Free memory allocated through t3_alloc_mem().
+ */
+void
+cxgb_free_mem(void *addr)
+{
+	free(addr, M_DEVBUF);
+}
+
+
+/*
+ * Allocate and initialize the TID tables.  Returns 0 on success.
+ */
+static int
+init_tid_tabs(struct tid_info *t, unsigned int ntids,
+			 unsigned int natids, unsigned int nstids,
+			 unsigned int atid_base, unsigned int stid_base)
+{
+	unsigned long size = ntids * sizeof(*t->tid_tab) +
+	    natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab);
+
+	t->tid_tab = cxgb_alloc_mem(size);
+	if (!t->tid_tab)
+		return -ENOMEM;
+
+	t->stid_tab = (union listen_entry *)&t->tid_tab[ntids];
+	t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids];
+	t->ntids = ntids;
+	t->nstids = nstids;
+	t->stid_base = stid_base;
+	t->sfree = NULL;
+	t->natids = natids;
+	t->atid_base = atid_base;
+	t->afree = NULL;
+	t->stids_in_use = t->atids_in_use = 0;
+	atomic_set_int(&t->tids_in_use, 0);
+	mtx_init(&t->stid_lock, "stid", NULL, MTX_DEF);
+	mtx_init(&t->atid_lock, "atid", NULL, MTX_DEF);
+
+	/*
+	 * Setup the free lists for stid_tab and atid_tab.
+	 */
+	if (nstids) {
+		while (--nstids)
+			t->stid_tab[nstids - 1].next = &t->stid_tab[nstids];
+		t->sfree = t->stid_tab;
+	}
+	if (natids) {
+		while (--natids)
+			t->atid_tab[natids - 1].next = &t->atid_tab[natids];
+		t->afree = t->atid_tab;
+	}
+	return 0;
+}
+
+static void
+free_tid_maps(struct tid_info *t)
+{
+	cxgb_free_mem(t->tid_tab);
+}
+
+static inline void
+add_adapter(adapter_t *adap)
+{
+	rw_wlock(&adapter_list_lock);
+	TAILQ_INSERT_TAIL(&adapter_list, adap, adapter_entry);
+	rw_wunlock(&adapter_list_lock);
+}
+
+static inline void
+remove_adapter(adapter_t *adap)
+{
+	rw_wlock(&adapter_list_lock);
+	TAILQ_REMOVE(&adapter_list, adap, adapter_entry);
+	rw_wunlock(&adapter_list_lock);
+}
+
+/*
+ * XXX
+ */
+#define t3_free_l2t(...)
+
+int
+cxgb_offload_activate(struct adapter *adapter)
+{
+	struct toedev *dev = &adapter->tdev;
+	int natids, err;
+	struct toe_data *t;
+	struct tid_range stid_range, tid_range;
+	struct mtutab mtutab;
+	unsigned int l2t_capacity;
+
+	t = malloc(sizeof(*t), M_DEVBUF, M_WAITOK);
+	if (!t)
+		return (ENOMEM);
+
+	err = (EOPNOTSUPP);
+	if (dev->ctl(dev, GET_TX_MAX_CHUNK, &t->tx_max_chunk) < 0 ||
+	    dev->ctl(dev, GET_MAX_OUTSTANDING_WR, &t->max_wrs) < 0 ||
+	    dev->ctl(dev, GET_L2T_CAPACITY, &l2t_capacity) < 0 ||
+	    dev->ctl(dev, GET_MTUS, &mtutab) < 0 ||
+	    dev->ctl(dev, GET_TID_RANGE, &tid_range) < 0 ||
+	    dev->ctl(dev, GET_STID_RANGE, &stid_range) < 0)
+		goto out_free;
+
+	err = (ENOMEM);
+	L2DATA(dev) = t3_init_l2t(l2t_capacity);
+	if (!L2DATA(dev))
+		goto out_free;
+
+	natids = min(tid_range.num / 2, MAX_ATIDS);
+	err = init_tid_tabs(&t->tid_maps, tid_range.num, natids,
+			    stid_range.num, ATID_BASE, stid_range.base);
+	if (err)
+		goto out_free_l2t;
+
+	t->mtus = mtutab.mtus;
+	t->nmtus = mtutab.size;
+
+	TASK_INIT(&t->tid_release_task, 0 /* XXX? */, t3_process_tid_release_list, dev);
+	mtx_init(&t->tid_release_lock, "tid release", NULL, MTX_DEF);
+	t->dev = dev;
+
+	TOE_DATA(dev) = t;
+	dev->recv = process_rx;
+	dev->neigh_update = t3_l2t_update;
+#if 0
+	offload_proc_dev_setup(dev);
+#endif	
+	/* Register netevent handler once */
+	if (TAILQ_EMPTY(&adapter_list)) {
+#if defined(CONFIG_CHELSIO_T3_MODULE)
+		if (prepare_arp_with_t3core())
+			log(LOG_ERR, "Unable to set offload capabilities\n");
+#endif
+	}
+	add_adapter(adapter);
+	return 0;
+
+out_free_l2t:
+	t3_free_l2t(L2DATA(dev));
+	L2DATA(dev) = NULL;
+out_free:
+	free(t, M_DEVBUF);
+	return err;
+
+}
+
+void
+cxgb_offload_deactivate(struct adapter *adapter)
+{
+	struct toedev *tdev = &adapter->tdev;
+	struct toe_data *t = TOE_DATA(tdev);
+
+	remove_adapter(adapter);
+	if (TAILQ_EMPTY(&adapter_list)) {
+#if defined(CONFIG_CHELSIO_T3_MODULE)
+		restore_arp_sans_t3core();
+#endif
+	}
+	free_tid_maps(&t->tid_maps);
+	TOE_DATA(tdev) = NULL;
+	t3_free_l2t(L2DATA(tdev));
+	L2DATA(tdev) = NULL;
+	free(t, M_DEVBUF);
+}
+
+
+static inline void
+register_tdev(struct toedev *tdev)
+{
+	static int unit;
+
+	mtx_lock(&cxgb_db_lock);
+	snprintf(tdev->name, sizeof(tdev->name), "ofld_dev%d", unit++);
+	TAILQ_INSERT_TAIL(&ofld_dev_list, tdev, ofld_entry);
+	mtx_unlock(&cxgb_db_lock);
+}
+
+static inline void
+unregister_tdev(struct toedev *tdev)
+{
+	mtx_lock(&cxgb_db_lock);
+	TAILQ_REMOVE(&ofld_dev_list, tdev, ofld_entry);
+	mtx_unlock(&cxgb_db_lock);	
+}
+
+void
+cxgb_adapter_ofld(struct adapter *adapter)
+{
+	struct toedev *tdev = &adapter->tdev;
+
+	cxgb_set_dummy_ops(tdev);
+	tdev->send = t3_offload_tx;
+	tdev->ctl = cxgb_offload_ctl;
+	tdev->type = adapter->params.rev == 0 ?
+		     T3A : T3B;
+
+	register_tdev(tdev);
+#if 0	
+	offload_proc_dev_init(tdev);
+#endif	
+}
+
+void
+cxgb_adapter_unofld(struct adapter *adapter)
+{
+	struct toedev *tdev = &adapter->tdev;
+#if 0
+	offload_proc_dev_cleanup(tdev);
+	offload_proc_dev_exit(tdev);
+#endif	
+	tdev->recv = NULL;
+	tdev->neigh_update = NULL;
+
+	unregister_tdev(tdev);
+}
+
+void
+cxgb_offload_init(void)
+{
+	int i;
+
+	if (inited)
+		return;
+	else
+		inited = 1;
+
+	mtx_init(&cxgb_db_lock, "ofld db", NULL, MTX_DEF);
+	rw_init(&adapter_list_lock, "ofld adap list");
+	TAILQ_INIT(&client_list);
+	TAILQ_INIT(&ofld_dev_list);
+	TAILQ_INIT(&adapter_list);
+	
+	for (i = 0; i < NUM_CPL_CMDS; ++i)
+		cpl_handlers[i] = do_bad_cpl;
+	
+	t3_register_cpl_handler(CPL_SMT_WRITE_RPL, do_smt_write_rpl);
+	t3_register_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl);
+	t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl);
+	t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl);
+	t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_cr);
+	t3_register_cpl_handler(CPL_PASS_ESTABLISH, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_ABORT_RPL_RSS, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_ABORT_RPL, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_RX_URG_NOTIFY, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_RX_DATA, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_TX_DATA_ACK, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_TX_DMA_ACK, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl);
+	t3_register_cpl_handler(CPL_PEER_CLOSE, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_CLOSE_CON_RPL, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req_rss);
+	t3_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish);
+	t3_register_cpl_handler(CPL_SET_TCB_RPL, do_set_tcb_rpl);
+	t3_register_cpl_handler(CPL_RDMA_TERMINATE, do_term);
+	t3_register_cpl_handler(CPL_RDMA_EC_STATUS, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_TRACE_PKT, do_trace);
+	t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl);
+	t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl);
+#if 0
+       if (offload_proc_init())
+	       log(LOG_WARNING, "Unable to create /proc/net/cxgb3 dir\n");
+#endif
+}
+
+void 
+cxgb_offload_exit(void)
+{
+#if 0	
+	offload_proc_cleanup();
+#endif	
+}
+
+#if 0
+static int
+offload_info_read_proc(char *buf, char **start, off_t offset,
+				  int length, int *eof, void *data)
+{
+	struct toe_data *d = data;
+	struct tid_info *t = &d->tid_maps;
+	int len;
+
+	len = sprintf(buf, "TID range: 0..%d, in use: %u\n"
+		      "STID range: %d..%d, in use: %u\n"
+		      "ATID range: %d..%d, in use: %u\n"
+		      "MSS: %u\n",
+		      t->ntids - 1, atomic_read(&t->tids_in_use), t->stid_base,
+		      t->stid_base + t->nstids - 1, t->stids_in_use,
+		      t->atid_base, t->atid_base + t->natids - 1,
+		      t->atids_in_use, d->tx_max_chunk);
+	if (len > length)
+		len = length;
+	*eof = 1;
+	return len;
+}
+
+static int
+offload_info_proc_setup(struct proc_dir_entry *dir,
+				   struct toe_data *d)
+{
+	struct proc_dir_entry *p;
+
+	if (!dir)
+		return -EINVAL;
+
+	p = create_proc_read_entry("info", 0, dir, offload_info_read_proc, d);
+	if (!p)
+		return -ENOMEM;
+
+	p->owner = THIS_MODULE;
+	return 0;
+}
+
+
+static int
+offload_devices_read_proc(char *buf, char **start, off_t offset,
+				     int length, int *eof, void *data)
+{
+	int len;
+	struct toedev *dev;
+	struct net_device *ndev;
+	
+	len = sprintf(buf, "Device           Interfaces\n");
+	
+	mtx_lock(&cxgb_db_lock);
+	TAILQ_FOREACH(dev, &ofld_dev_list, ofld_entry) {	
+		len += sprintf(buf + len, "%-16s", dev->name);
+		read_lock(&dev_base_lock);
+		for (ndev = dev_base; ndev; ndev = ndev->next) {
+			if (TOEDEV(ndev) == dev)
+				len += sprintf(buf + len, " %s", ndev->name);
+		}
+		read_unlock(&dev_base_lock);
+		len += sprintf(buf + len, "\n");
+		if (len >= length)
+			break;
+	}
+	mtx_unlock(&cxgb_db_lock);
+	
+	if (len > length)
+		len = length;
+	*eof = 1;
+	return len;
+}
+
+#endif
+
diff --git a/sys/dev/cxgb/cxgb_offload.h b/sys/dev/cxgb/cxgb_offload.h
new file mode 100644
index 0000000..2e4b44b
--- /dev/null
+++ b/sys/dev/cxgb/cxgb_offload.h
@@ -0,0 +1,260 @@
+
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+
+ 3. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+$FreeBSD$
+
+***************************************************************************/
+
+#ifndef _CXGB_OFFLOAD_H
+#define _CXGB_OFFLOAD_H
+
+
+#include <dev/cxgb/common/cxgb_tcb.h>
+#include <dev/cxgb/cxgb_l2t.h>
+
+#include <dev/cxgb/ulp/toecore/toedev.h>
+#include <dev/cxgb/common/cxgb_t3_cpl.h>
+
+struct adapter;
+struct cxgb_client;
+
+void cxgb_offload_init(void);
+void cxgb_offload_exit(void);
+
+void cxgb_adapter_ofld(struct adapter *adapter);
+void cxgb_adapter_unofld(struct adapter *adapter);
+int cxgb_offload_activate(struct adapter *adapter);
+void cxgb_offload_deactivate(struct adapter *adapter);
+int cxgb_ofld_recv(struct toedev *dev, struct mbuf **m, int n);
+
+void cxgb_set_dummy_ops(struct toedev *dev);
+
+
+/*
+ * Client registration.  Users of T3 driver must register themselves.
+ * The T3 driver will call the add function of every client for each T3
+ * adapter activated, passing up the toedev ptr.  Each client fills out an
+ * array of callback functions to process CPL messages.
+ */
+
+void cxgb_register_client(struct cxgb_client *client);
+void cxgb_unregister_client(struct cxgb_client *client);
+void cxgb_add_clients(struct toedev *tdev);
+void cxgb_remove_clients(struct toedev *tdev);
+
+typedef int (*cxgb_cpl_handler_func)(struct toedev *dev,
+				      struct mbuf *m, void *ctx);
+
+struct cxgb_client {
+	char 			*name;
+	void 			(*add) (struct toedev *);
+	void 			(*remove) (struct toedev *);
+	cxgb_cpl_handler_func 	*handlers;
+	int			(*redirect)(void *ctx, struct rtentry *old,
+					    struct rtentry *new,
+					    struct l2t_entry *l2t);
+	TAILQ_ENTRY(cxgb_client)         client_entry;
+};
+
+/*
+ * TID allocation services.
+ */
+int cxgb_alloc_atid(struct toedev *dev, struct cxgb_client *client,
+		     void *ctx);
+int cxgb_alloc_stid(struct toedev *dev, struct cxgb_client *client,
+		     void *ctx);
+void *cxgb_free_atid(struct toedev *dev, int atid);
+void cxgb_free_stid(struct toedev *dev, int stid);
+void cxgb_insert_tid(struct toedev *dev, struct cxgb_client *client,
+		      void *ctx,
+	unsigned int tid);
+void cxgb_queue_tid_release(struct toedev *dev, unsigned int tid);
+void cxgb_remove_tid(struct toedev *dev, void *ctx, unsigned int tid);
+
+struct toe_tid_entry {
+	struct cxgb_client 	*client;
+	void 			*ctx;
+};
+
+/* CPL message priority levels */
+enum {
+	CPL_PRIORITY_DATA = 0,     /* data messages */
+	CPL_PRIORITY_SETUP = 1,	   /* connection setup messages */
+	CPL_PRIORITY_TEARDOWN = 0, /* connection teardown messages */
+	CPL_PRIORITY_LISTEN = 1,   /* listen start/stop messages */
+	CPL_PRIORITY_ACK = 1,      /* RX ACK messages */
+	CPL_PRIORITY_CONTROL = 1   /* offload control messages */
+};
+
+/* Flags for return value of CPL message handlers */
+enum {
+	CPL_RET_BUF_DONE = 1,   // buffer processing done, buffer may be freed
+	CPL_RET_BAD_MSG = 2,    // bad CPL message (e.g., unknown opcode)
+	CPL_RET_UNKNOWN_TID = 4	// unexpected unknown TID
+};
+
+typedef int (*cpl_handler_func)(struct toedev *dev, struct mbuf *m);
+
+/*
+ * Returns a pointer to the first byte of the CPL header in an sk_buff that
+ * contains a CPL message.
+ */
+static inline void *cplhdr(struct mbuf *m)
+{
+	return m->m_data;
+}
+
+void t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h);
+
+union listen_entry {
+	struct toe_tid_entry toe_tid;
+	union listen_entry *next;
+};
+
+union active_open_entry {
+	struct toe_tid_entry toe_tid;
+	union active_open_entry *next;
+};
+
+/*
+ * Holds the size, base address, free list start, etc of the TID, server TID,
+ * and active-open TID tables for a offload device.
+ * The tables themselves are allocated dynamically.
+ */
+struct tid_info {
+	struct toe_tid_entry *tid_tab;
+	unsigned int ntids;
+	volatile int tids_in_use;
+
+	union listen_entry *stid_tab;
+	unsigned int nstids;
+	unsigned int stid_base;
+
+	union active_open_entry *atid_tab;
+	unsigned int natids;
+	unsigned int atid_base;
+
+	/*
+	 * The following members are accessed R/W so we put them in their own
+	 * cache lines.
+	 *
+	 * XXX We could combine the atid fields above with the lock here since
+	 * atids are use once (unlike other tids).  OTOH the above fields are
+	 * usually in cache due to tid_tab.
+	 */
+	struct mtx atid_lock /* ____cacheline_aligned_in_smp */;
+	union active_open_entry *afree;
+	unsigned int atids_in_use;
+
+	struct mtx stid_lock /*____cacheline_aligned */;
+	union listen_entry *sfree;
+	unsigned int stids_in_use;
+};
+
+struct toe_data {
+#ifdef notyet	
+	struct list_head list_node;
+#endif	
+	struct toedev *dev;
+	unsigned int tx_max_chunk;  /* max payload for TX_DATA */
+	unsigned int max_wrs;       /* max in-flight WRs per connection */
+	unsigned int nmtus;
+	const unsigned short *mtus;
+	struct tid_info tid_maps;
+
+	struct toe_tid_entry *tid_release_list;
+	struct mtx tid_release_lock;
+	struct task tid_release_task;
+};
+
+/*
+ * toedev -> toe_data accessor
+ */
+#define TOE_DATA(dev) (*(struct toe_data **)&(dev)->l4opt)
+
+/*
+ * Map an ATID or STID to their entries in the corresponding TID tables.
+ */
+static inline union active_open_entry *atid2entry(const struct tid_info *t,
+                                                  unsigned int atid)
+{
+        return &t->atid_tab[atid - t->atid_base];
+}
+
+
+static inline union listen_entry *stid2entry(const struct tid_info *t,
+                                             unsigned int stid)
+{
+        return &t->stid_tab[stid - t->stid_base];
+}
+
+/*
+ * Find the connection corresponding to a TID.
+ */
+static inline struct toe_tid_entry *lookup_tid(const struct tid_info *t,
+                                               unsigned int tid)
+{
+        return tid < t->ntids ? &(t->tid_tab[tid]) : NULL;
+}
+
+/*
+ * Find the connection corresponding to a server TID.
+ */
+static inline struct toe_tid_entry *lookup_stid(const struct tid_info *t,
+                                                unsigned int tid)
+{
+        if (tid < t->stid_base || tid >= t->stid_base + t->nstids)
+                return NULL;
+        return &(stid2entry(t, tid)->toe_tid);
+}
+
+/*
+ * Find the connection corresponding to an active-open TID.
+ */
+static inline struct toe_tid_entry *lookup_atid(const struct tid_info *t,
+                                                unsigned int tid)
+{
+        if (tid < t->atid_base || tid >= t->atid_base + t->natids)
+                return NULL;
+        return &(atid2entry(t, tid)->toe_tid);
+}
+
+void *cxgb_alloc_mem(unsigned long size);
+void cxgb_free_mem(void *addr);
+void cxgb_neigh_update(struct rtentry *rt);
+void cxgb_redirect(struct rtentry *old, struct rtentry *new);
+int process_rx(struct toedev *dev, struct mbuf **m, int n);
+int attach_toedev(struct toedev *dev);
+void detach_toedev(struct toedev *dev);
+
+
+#endif
diff --git a/sys/dev/cxgb/cxgb_osdep.h b/sys/dev/cxgb/cxgb_osdep.h
index 42aa7f7..bd3804b 100644
--- a/sys/dev/cxgb/cxgb_osdep.h
+++ b/sys/dev/cxgb/cxgb_osdep.h
@@ -9,11 +9,7 @@ modification, are permitted provided that the following conditions are met:
  1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
 
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Chelsio Corporation nor the names of its
+ 2. Neither the name of the Chelsio Corporation nor the names of its
     contributors may be used to endorse or promote products derived from
     this software without specific prior written permission.
 
@@ -49,9 +45,17 @@ $FreeBSD$
 #define _CXGB_OSDEP_H_
 
 typedef struct adapter adapter_t;
-
 struct sge_rspq;
 
+#define PANIC_IF(exp) do {                  \
+	if (exp)                            \
+		panic("BUG: %s", exp);      \
+} while (0)
+
+
+#define m_get_priority(m) ((uintptr_t)(m)->m_pkthdr.rcvif)
+#define m_set_priority(m, pri) ((m)->m_pkthdr.rcvif = (struct ifnet *)(pri))
+
 #if __FreeBSD_version > 700030
 #define INTR_FILTERS
 #define FIRMWARE_LATEST
@@ -68,6 +72,8 @@ struct sge_rspq;
 #define TASKQUEUE_CURRENT
 #endif
 
+#define __read_mostly __attribute__((__section__(".data.read_mostly")))
+
 /*
  * Workaround for weird Chelsio issue
  */
@@ -75,9 +81,10 @@ struct sge_rspq;
 #define PRIV_SUPPORTED
 #endif
 
-#define CXGB_TX_CLEANUP_THRESHOLD 32
+#define CXGB_TX_CLEANUP_THRESHOLD        32
 
-#define LOG_WARNING 1
+#define LOG_WARNING                       1
+#define LOG_ERR                           2
 
 #ifdef DEBUG_PRINT
 #define DPRINTF printf
@@ -107,6 +114,16 @@ void prefetch(void *x)
         __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
 } 
 
+extern void kdb_backtrace(void);
+
+#define WARN_ON(condition) do { \
+        if (unlikely((condition)!=0)) { \
+                log(LOG_WARNING, "BUG: warning at %s:%d/%s()\n", __FILE__, __LINE__, __FUNCTION__); \
+                kdb_backtrace(); \
+        } \
+} while (0)
+
+
 #else /* !i386 && !amd64 */
 #define mb()
 #define rmb()
@@ -137,7 +154,13 @@ static const int debug_flags = DBG_RX;
 
 #define t3_os_sleep(x) DELAY((x) * 1000)
 
+#define test_and_clear_bit(bit, p) atomic_cmpset_int((p), ((*(p)) | bit), ((*(p)) & ~bit)) 
+
+
 #define max_t(type, a, b) (type)max((a), (b))
+#define net_device ifnet
+
+
 
 /* Standard PHY definitions */
 #define BMCR_LOOPBACK		BMCR_LOOP
diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c
index 28b95b34..aaced45 100644
--- a/sys/dev/cxgb/cxgb_sge.c
+++ b/sys/dev/cxgb/cxgb_sge.c
@@ -9,14 +9,10 @@ modification, are permitted provided that the following conditions are met:
  1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.
 
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Chelsio Corporation nor the names of its
+ 2. Neither the name of the Chelsio Corporation nor the names of its
     contributors may be used to endorse or promote products derived from
     this software without specific prior written permission.
-
+ 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -66,17 +62,27 @@ __FBSDID("$FreeBSD$");
 #include <dev/cxgb/common/cxgb_sge_defs.h>
 #include <dev/cxgb/common/cxgb_t3_cpl.h>
 #include <dev/cxgb/common/cxgb_firmware_exports.h>
+#include <dev/cxgb/cxgb_offload.h>
 
 #include <dev/cxgb/sys/mvec.h>
 
 uint32_t collapse_free = 0;
 uint32_t mb_free_vec_free = 0;
 int      collapse_mbufs = 0;
+static int recycle_enable = 1;
+
+
+/*
+ * XXX GC
+ */
+#define NET_XMIT_CN 2
+#define NET_XMIT_SUCCESS 0 
 
 #define USE_GTS 0
 
 #define SGE_RX_SM_BUF_SIZE	1536
 #define SGE_RX_DROP_THRES	16
+#define SGE_RX_COPY_THRES	128
 
 /*
  * Period of the Tx buffer reclaim timer.  This timer does not need to run
@@ -200,18 +206,30 @@ reclaim_completed_tx(adapter_t *adapter, struct sge_txq *q, int nbufs, struct mb
 	int n = 0;
 
 	mtx_assert(&q->lock, MA_OWNED);
-	
 	if (reclaim > 0) {
 		n = free_tx_desc(adapter, q, min(reclaim, nbufs), mvec);
 		reclaimed = min(reclaim, nbufs);
 		q->cleaned += reclaimed;
 		q->in_use -= reclaimed;
 	} 
-
 	return (n);
 }
 
 /**
+ *	should_restart_tx - are there enough resources to restart a Tx queue?
+ *	@q: the Tx queue
+ *
+ *	Checks if there are enough descriptors to restart a suspended Tx queue.
+ */
+static __inline int
+should_restart_tx(const struct sge_txq *q)
+{
+	unsigned int r = q->processed - q->cleaned;
+
+	return q->in_use - r < (q->size >> 1);
+}
+
+/**
  *	t3_sge_init - initialize SGE
  *	@adap: the adapter
  *	@p: the SGE parameters
@@ -554,6 +572,36 @@ __refill_fl(adapter_t *adap, struct sge_fl *fl)
 	refill_fl(adap, fl, min(16U, fl->size - fl->credits));
 }
 
+/**
+ *	recycle_rx_buf - recycle a receive buffer
+ *	@adapter: the adapter
+ *	@q: the SGE free list
+ *	@idx: index of buffer to recycle
+ *
+ *	Recycles the specified buffer on the given free list by adding it at
+ *	the next available slot on the list.
+ */
+static void
+recycle_rx_buf(adapter_t *adap, struct sge_fl *q, unsigned int idx)
+{
+	struct rx_desc *from = &q->desc[idx];
+	struct rx_desc *to   = &q->desc[q->pidx];
+
+	q->sdesc[q->pidx] = q->sdesc[idx];
+	to->addr_lo = from->addr_lo;        // already big endian
+	to->addr_hi = from->addr_hi;        // likewise
+	wmb();
+	to->len_gen = htobe32(V_FLD_GEN1(q->gen));
+	to->gen2 = htobe32(V_FLD_GEN2(q->gen));
+	q->credits++;
+
+	if (++q->pidx == q->size) {
+		q->pidx = 0;
+		q->gen ^= 1;
+	}
+	t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
+}
+
 static void
 alloc_ring_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 {
@@ -714,7 +762,7 @@ sge_timer_reclaim(void *arg, int ncount)
 			mtx_lock(&txq->lock);			
 			n = reclaim_completed_tx(sc, txq, TX_CLEAN_MAX_DESC, m_vec);
 			mtx_unlock(&txq->lock);
-			
+
 			for (i = 0; i < n; i++) {
 				m_freem_vec(m_vec[i]);
 			}
@@ -946,6 +994,95 @@ wr_gen2(struct tx_desc *d, unsigned int gen)
 #endif
 }
 
+
+
+/**
+ *	write_wr_hdr_sgl - write a WR header and, optionally, SGL
+ *	@ndesc: number of Tx descriptors spanned by the SGL
+ *	@txd: first Tx descriptor to be written
+ *	@txqs: txq state (generation and producer index)
+ *	@txq: the SGE Tx queue
+ *	@sgl: the SGL
+ *	@flits: number of flits to the start of the SGL in the first descriptor
+ *	@sgl_flits: the SGL size in flits
+ *	@wr_hi: top 32 bits of WR header based on WR type (big endian)
+ *	@wr_lo: low 32 bits of WR header based on WR type (big endian)
+ *
+ *	Write a work request header and an associated SGL.  If the SGL is
+ *	small enough to fit into one Tx descriptor it has already been written
+ *	and we just need to write the WR header.  Otherwise we distribute the
+ *	SGL across the number of descriptors it spans.
+ */
+
+static void
+write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs,
+    const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits,
+    unsigned int sgl_flits, unsigned int wr_hi, unsigned int wr_lo)
+{
+
+	struct work_request_hdr *wrp = (struct work_request_hdr *)txd;
+	struct tx_sw_desc *txsd = &txq->sdesc[txqs->pidx];
+	
+	if (__predict_true(ndesc == 1)) {
+		wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
+		    V_WR_SGLSFLT(flits)) | wr_hi;
+		wmb();
+		wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) |
+		    V_WR_GEN(txqs->gen)) | wr_lo;
+		/* XXX gen? */
+		wr_gen2(txd, txqs->gen);
+	} else {
+		unsigned int ogen = txqs->gen;
+		const uint64_t *fp = (const uint64_t *)sgl;
+		struct work_request_hdr *wp = wrp;
+		
+		wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
+		    V_WR_SGLSFLT(flits)) | wr_hi;
+		
+		while (sgl_flits) {
+			unsigned int avail = WR_FLITS - flits;
+
+			if (avail > sgl_flits)
+				avail = sgl_flits;
+			memcpy(&txd->flit[flits], fp, avail * sizeof(*fp));
+			sgl_flits -= avail;
+			ndesc--;
+			if (!sgl_flits)
+				break;
+			
+			fp += avail;
+			txd++;
+			txsd++;
+			if (++txqs->pidx == txq->size) {
+				txqs->pidx = 0;
+				txqs->gen ^= 1;
+				txd = txq->desc;
+				txsd = txq->sdesc;
+			}
+			
+			/*
+			 * when the head of the mbuf chain
+			 * is freed all clusters will be freed
+			 * with it
+			 */
+			txsd->m = NULL;
+			wrp = (struct work_request_hdr *)txd;
+			wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
+			    V_WR_SGLSFLT(1)) | wr_hi;
+			wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS,
+				    sgl_flits + 1)) |
+			    V_WR_GEN(txqs->gen)) | wr_lo;
+			wr_gen2(txd, txqs->gen);
+			flits = 1;
+		}
+		wrp->wr_hi |= htonl(F_WR_EOP);
+		wmb();
+		wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
+		wr_gen2((struct tx_desc *)wp, ogen);
+	}
+}
+
+	
 /* sizeof(*eh) + sizeof(*vhdr) + sizeof(*ip) + sizeof(*tcp) */
 #define TCPPKTHDRSIZE (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + 20 + 20)
 
@@ -1038,6 +1175,7 @@ t3_encap(struct port_info *p, struct mbuf **m)
 		if (mlen <= WR_LEN - sizeof(*cpl)) {
 			txq_prod(txq, 1, &txqs);
 			txq->sdesc[txqs.pidx].m = m0;
+			m_set_priority(m0, txqs.pidx);
 			
 			if (m0->m_len == m0->m_pkthdr.len)
 				memcpy(&txd->flit[2], m0->m_data, mlen);
@@ -1067,7 +1205,7 @@ t3_encap(struct port_info *p, struct mbuf **m)
 	m0 = *m;
 	ndesc = calc_tx_descs(m0, nsegs);
 	
-	sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : &sgl[0];
+	sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl;
 	make_sgl(sgp, segs, nsegs);
 
 	sgl_flits = sgl_len(nsegs);
@@ -1078,68 +1216,9 @@ t3_encap(struct port_info *p, struct mbuf **m)
 	wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl);
 	wr_lo = htonl(V_WR_TID(txq->token));
 	txsd->m = m0;
-	
-	if (__predict_true(ndesc == 1)) {
-		wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
-		    V_WR_SGLSFLT(flits)) | wr_hi;
-		wmb();
-		wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) |
-		    V_WR_GEN(txqs.gen)) | wr_lo;
-		/* XXX gen? */
-		wr_gen2(txd, txqs.gen);
-	} else {
-		unsigned int ogen = txqs.gen;
-		const uint64_t *fp = (const uint64_t *)sgl;
-		struct work_request_hdr *wp = wrp;
-		
-		/* XXX - CHECK ME */
-		wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
-		    V_WR_SGLSFLT(flits)) | wr_hi;
-		
-		while (sgl_flits) {
-			unsigned int avail = WR_FLITS - flits;
+	m_set_priority(m0, txqs.pidx); 
 
-			if (avail > sgl_flits)
-				avail = sgl_flits;
-			memcpy(&txd->flit[flits], fp, avail * sizeof(*fp));
-			sgl_flits -= avail;
-			ndesc--;
-			if (!sgl_flits)
-				break;
-			
-			fp += avail;
-			txd++;
-			txsd++;
-			if (++txqs.pidx == txq->size) {
-				txqs.pidx = 0;
-				txqs.gen ^= 1;
-				txd = txq->desc;
-				txsd = txq->sdesc;
-			}
-			
-			/*
-			 * when the head of the mbuf chain
-			 * is freed all clusters will be freed
-			 * with it
-			 */
-			txsd->m = NULL;
-			wrp = (struct work_request_hdr *)txd;
-			wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
-			    V_WR_SGLSFLT(1)) | wr_hi;
-			wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS,
-				    sgl_flits + 1)) |
-			    V_WR_GEN(txqs.gen)) | wr_lo;
-			wr_gen2(txd, txqs.gen);
-			flits = 1;
-		}
-#ifdef WHY			
-		skb->priority = pidx;
-#endif
-		wrp->wr_hi |= htonl(F_WR_EOP);
-		wmb();
-		wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
-		wr_gen2((struct tx_desc *)wp, ogen);
-	}
+	write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo);
 	check_ring_tx_db(p->adapter, txq);
 
 	return (0);
@@ -1158,10 +1237,11 @@ t3_encap(struct port_info *p, struct mbuf **m)
  *	carefully so the SGE doesn't read accidentally before it's written in
  *	its entirety.
  */
-static __inline void write_imm(struct tx_desc *d, struct mbuf *m,
-			     unsigned int len, unsigned int gen)
+static __inline void
+write_imm(struct tx_desc *d, struct mbuf *m,
+	  unsigned int len, unsigned int gen)
 {
-	struct work_request_hdr *from = (struct work_request_hdr *)m->m_data;
+	struct work_request_hdr *from = mtod(m, struct work_request_hdr *);
 	struct work_request_hdr *to = (struct work_request_hdr *)d;
 
 	memcpy(&to[1], &from[1], len - sizeof(*from));
@@ -1194,25 +1274,24 @@ static __inline void write_imm(struct tx_desc *d, struct mbuf *m,
  */
 static __inline int
 check_desc_avail(adapter_t *adap, struct sge_txq *q,
-				   struct mbuf *m, unsigned int ndesc,
-				   unsigned int qid)
+		 struct mbuf *m, unsigned int ndesc,
+		 unsigned int qid)
 {
 	/* 
 	 * XXX We currently only use this for checking the control queue
 	 * the control queue is only used for binding qsets which happens
 	 * at init time so we are guaranteed enough descriptors
 	 */
-#if 0	
-	if (__predict_false(!skb_queue_empty(&q->sendq))) {
-addq_exit:	__skb_queue_tail(&q->sendq, skb);
+	if (__predict_false(!mbufq_empty(&q->sendq))) {
+addq_exit:	mbufq_tail(&q->sendq, m);
 		return 1;
 	}
 	if (__predict_false(q->size - q->in_use < ndesc)) {
 
 		struct sge_qset *qs = txq_to_qset(q, qid);
 
-		set_bit(qid, &qs->txq_stopped);
-		smp_mb__after_clear_bit();
+		setbit(&qs->txq_stopped, qid);
+		smp_mb();
 
 		if (should_restart_tx(q) &&
 		    test_and_clear_bit(qid, &qs->txq_stopped))
@@ -1221,7 +1300,6 @@ addq_exit:	__skb_queue_tail(&q->sendq, skb);
 		q->stops++;
 		goto addq_exit;
 	}
-#endif	
 	return 0;
 }
 
@@ -1232,7 +1310,7 @@ addq_exit:	__skb_queue_tail(&q->sendq, skb);
  *
  *	This is a variant of reclaim_completed_tx() that is used for Tx queues
  *	that send only immediate data (presently just the control queues) and
- *	thus do not have any sk_buffs to release.
+ *	thus do not have any mbufs
  */
 static __inline void
 reclaim_completed_tx_imm(struct sge_txq *q)
@@ -1301,7 +1379,7 @@ again:	reclaim_completed_tx_imm(q);
 	return (0);
 }
 
-#ifdef RESTART_CTRLQ
+
 /**
  *	restart_ctrlq - restart a suspended control queue
  *	@qs: the queue set cotaining the control queue
@@ -1309,7 +1387,7 @@ again:	reclaim_completed_tx_imm(q);
  *	Resumes transmission on a suspended Tx control queue.
  */
 static void
-restart_ctrlq(unsigned long data)
+restart_ctrlq(void *data, int npending)
 {
 	struct mbuf *m;
 	struct sge_qset *qs = (struct sge_qset *)data;
@@ -1318,11 +1396,11 @@ restart_ctrlq(unsigned long data)
 
 	mtx_lock(&q->lock);
 again:	reclaim_completed_tx_imm(q);
-	
+
 	while (q->in_use < q->size &&
-	       (skb = __skb_dequeue(&q->sendq)) != NULL) {
+	       (m = mbufq_dequeue(&q->sendq)) != NULL) {
 
-		write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
+		write_imm(&q->desc[q->pidx], m, m->m_len, q->gen);
 
 		if (++q->pidx >= q->size) {
 			q->pidx = 0;
@@ -1330,21 +1408,20 @@ again:	reclaim_completed_tx_imm(q);
 		}
 		q->in_use++;
 	}
-	if (!skb_queue_empty(&q->sendq)) {
-		set_bit(TXQ_CTRL, &qs->txq_stopped);
-		smp_mb__after_clear_bit();
+	if (!mbufq_empty(&q->sendq)) {
+		setbit(&qs->txq_stopped, TXQ_CTRL);
+		smp_mb();
 
 		if (should_restart_tx(q) &&
 		    test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))
 			goto again;
 		q->stops++;
 	}
-
 	mtx_unlock(&q->lock);
 	t3_write_reg(adap, A_SG_KDOORBELL,
 		     F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
 }
-#endif
+
 
 /*
  * Send a management message through control queue 0
@@ -1356,172 +1433,6 @@ t3_mgmt_tx(struct adapter *adap, struct mbuf *m)
 }
 
 /**
- *	t3_sge_alloc_qset - initialize an SGE queue set
- *	@sc: the controller softc
- *	@id: the queue set id
- *	@nports: how many Ethernet ports will be using this queue set
- *	@irq_vec_idx: the IRQ vector index for response queue interrupts
- *	@p: configuration parameters for this queue set
- *	@ntxq: number of Tx queues for the queue set
- *	@pi: port info for queue set
- *
- *	Allocate resources and initialize an SGE queue set.  A queue set
- *	comprises a response queue, two Rx free-buffer queues, and up to 3
- *	Tx queues.  The Tx queues are assigned roles in the order Ethernet
- *	queue, offload queue, and control queue.
- */
-int
-t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
-		  const struct qset_params *p, int ntxq, struct port_info *pi)
-{
-	struct sge_qset *q = &sc->sge.qs[id];
-	int i, ret = 0;
-
-	init_qset_cntxt(q, id);
-	
-	if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc),
-		    sizeof(struct rx_sw_desc), &q->fl[0].phys_addr,
-		    &q->fl[0].desc, &q->fl[0].sdesc,
-		    &q->fl[0].desc_tag, &q->fl[0].desc_map,
-		    sc->rx_dmat, &q->fl[0].entry_tag)) != 0) {
-		printf("error %d from alloc ring fl0\n", ret);
-		goto err;
-	}
-
-	if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc),
-		    sizeof(struct rx_sw_desc), &q->fl[1].phys_addr,
-		    &q->fl[1].desc, &q->fl[1].sdesc,
-		    &q->fl[1].desc_tag, &q->fl[1].desc_map,
-		    sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) {
-		printf("error %d from alloc ring fl1\n", ret);
-		goto err;
-	}
-
-	if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0,
-		    &q->rspq.phys_addr, &q->rspq.desc, NULL,
-		    &q->rspq.desc_tag, &q->rspq.desc_map,
-		    NULL, NULL)) != 0) {
-		printf("error %d from alloc ring rspq\n", ret);
-		goto err;
-	}
-
-	for (i = 0; i < ntxq; ++i) {
-		/*
-		 * The control queue always uses immediate data so does not
-		 * need to keep track of any mbufs.
-		 * XXX Placeholder for future TOE support.
-		 */
-		size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
-
-		if ((ret = alloc_ring(sc, p->txq_size[i],
-			    sizeof(struct tx_desc), sz,
-			    &q->txq[i].phys_addr, &q->txq[i].desc,
-			    &q->txq[i].sdesc, &q->txq[i].desc_tag,
-			    &q->txq[i].desc_map,
-			    sc->tx_dmat, &q->txq[i].entry_tag)) != 0) {
-			printf("error %d from alloc ring tx %i\n", ret, i);
-			goto err;
-		}
-		q->txq[i].gen = 1;
-		q->txq[i].size = p->txq_size[i];
-		mtx_init(&q->txq[i].lock, "t3 txq lock", NULL, MTX_DEF);
-	}
-
-	q->fl[0].gen = q->fl[1].gen = 1;
-	q->fl[0].size = p->fl_size;
-	q->fl[1].size = p->jumbo_size;
-
-	q->rspq.gen = 1;
-	q->rspq.size = p->rspq_size;
-	mtx_init(&q->rspq.lock, "t3 rspq lock", NULL, MTX_DEF);
-	
-	q->txq[TXQ_ETH].stop_thres = nports *
-	    flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3);
-
-	q->fl[0].buf_size = MCLBYTES;
-	q->fl[0].zone = zone_clust;
-	q->fl[0].type = EXT_CLUSTER;
-	q->fl[1].buf_size = MJUMPAGESIZE;
-	q->fl[1].zone = zone_jumbop;
-	q->fl[1].type = EXT_JUMBOP;
-	
-	q->lro.enabled = lro_default;
-	
-	mtx_lock(&sc->sge.reg_lock);
-	ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx,
-				   q->rspq.phys_addr, q->rspq.size,
-				   q->fl[0].buf_size, 1, 0);
-	if (ret) {
-		printf("error %d from t3_sge_init_rspcntxt\n", ret);
-		goto err_unlock;
-	}
-
-	for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
-		ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0,
-					  q->fl[i].phys_addr, q->fl[i].size,
-					  q->fl[i].buf_size, p->cong_thres, 1,
-					  0);
-		if (ret) {
-			printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i);
-			goto err_unlock;
-		}
-	}
-
-	ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
-				 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
-				 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
-				 1, 0);
-	if (ret) {
-		printf("error %d from t3_sge_init_ecntxt\n", ret);
-		goto err_unlock;
-	}
-
-	if (ntxq > 1) {
-		ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id,
-					 USE_GTS, SGE_CNTXT_OFLD, id,
-					 q->txq[TXQ_OFLD].phys_addr,
-					 q->txq[TXQ_OFLD].size, 0, 1, 0);
-		if (ret) {
-			printf("error %d from t3_sge_init_ecntxt\n", ret);
-			goto err_unlock;
-		}
-	}
-
-	if (ntxq > 2) {
-		ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0,
-					 SGE_CNTXT_CTRL, id,
-					 q->txq[TXQ_CTRL].phys_addr,
-					 q->txq[TXQ_CTRL].size,
-					 q->txq[TXQ_CTRL].token, 1, 0);
-		if (ret) {
-			printf("error %d from t3_sge_init_ecntxt\n", ret);
-			goto err_unlock;
-		}
-	}
-	
-	mtx_unlock(&sc->sge.reg_lock);
-	t3_update_qset_coalesce(q, p);
-	q->port = pi;
-	
-	refill_fl(sc, &q->fl[0], q->fl[0].size);
-	refill_fl(sc, &q->fl[1], q->fl[1].size);
-	refill_rspq(sc, &q->rspq, q->rspq.size - 1);
-
-	t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
-		     V_NEWTIMER(q->rspq.holdoff_tmr));
-
-	return (0);
-
-err_unlock:
-	mtx_unlock(&sc->sge.reg_lock);
-err:	
-	t3_free_qset(sc, q);
-
-	return (ret);
-}
-
-
-/**
  *	free_qset - free the resources of an SGE queue set
  *	@sc: the controller owning the queue set
  *	@q: the queue set
@@ -1617,6 +1528,33 @@ t3_sge_start(adapter_t *sc)
 	t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE);
 }
 
+/**
+ *	t3_sge_stop - disable SGE operation
+ *	@sc: the adapter
+ *
+ *	Disables the DMA engine.  This can be called in emeregencies (e.g.,
+ *	from error interrupts) or from normal process context.  In the latter
+ *	case it also disables any pending queue restart tasklets.  Note that
+ *	if it is called in interrupt context it cannot disable the restart
+ *	tasklets as it cannot wait, however the tasklets will have no effect
+ *	since the doorbells are disabled and the driver will call this again
+ *	later from process context, at which time the tasklets will be stopped
+ *	if they are still running.
+ */
+void
+t3_sge_stop(adapter_t *sc)
+{
+	int i;
+	t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, 0);
+
+	for (i = 0; i < SGE_QSETS; ++i) {
+		struct sge_qset *qs = &sc->sge.qs[i];
+		
+		taskqueue_drain(sc->tq, &qs->txq[TXQ_OFLD].qresume_tsk);
+		taskqueue_drain(sc->tq, &qs->txq[TXQ_CTRL].qresume_tsk);
+	}
+}
+
 
 /**
  *	free_tx_desc - reclaims Tx descriptors and their buffers
@@ -1648,9 +1586,13 @@ free_tx_desc(adapter_t *sc, struct sge_txq *q, int n, struct mbuf **m_vec)
 				bus_dmamap_destroy(q->entry_tag, d->map);
 				d->flags &= ~TX_SW_DESC_MAPPED;
 			}
-			m_vec[nbufs] = d->m;
-			d->m = NULL;
-			nbufs++;
+			if (m_get_priority(d->m) == cidx) {
+				m_vec[nbufs] = d->m;
+				d->m = NULL;
+				nbufs++;
+			} else {
+				printf("pri=%d cidx=%d\n", m_get_priority(d->m), cidx);
+			}
 		}
 		++d;
 		if (++cidx == q->size) {
@@ -1687,37 +1629,477 @@ is_new_response(const struct rsp_desc *r,
 /* How long to delay the next interrupt in case of memory shortage, in 0.1us. */
 #define NOMEM_INTR_DELAY 2500
 
-static __inline void
-deliver_partial_bundle(struct t3cdev *tdev, struct sge_rspq *q)
+/**
+ *	write_ofld_wr - write an offload work request
+ *	@adap: the adapter
+ *	@m: the packet to send
+ *	@q: the Tx queue
+ *	@pidx: index of the first Tx descriptor to write
+ *	@gen: the generation value to use
+ *	@ndesc: number of descriptors the packet will occupy
+ *
+ *	Write an offload work request to send the supplied packet.  The packet
+ *	data already carry the work request with most fields populated.
+ */
+static void
+write_ofld_wr(adapter_t *adap, struct mbuf *m,
+    struct sge_txq *q, unsigned int pidx,
+    unsigned int gen, unsigned int ndesc,
+    bus_dma_segment_t *segs, unsigned int nsegs)
 {
-	;
+	unsigned int sgl_flits, flits;
+	struct work_request_hdr *from;
+	struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1];
+	struct tx_desc *d = &q->desc[pidx];
+	struct txq_state txqs;
+	
+	if (immediate(m)) {
+		q->sdesc[pidx].m = NULL;
+		write_imm(d, m, m->m_len, gen);
+		return;
+	}
+
+	/* Only TX_DATA builds SGLs */
+
+	from = mtod(m, struct work_request_hdr *);
+	memcpy(&d->flit[1], &from[1],
+	    (uint8_t *)m->m_pkthdr.header - mtod(m, uint8_t *) - sizeof(*from));
+
+	flits = ((uint8_t *)m->m_pkthdr.header - mtod(m, uint8_t *)) / 8;
+	sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : sgl;
+
+	make_sgl(sgp, segs, nsegs);
+	sgl_flits = sgl_len(nsegs);
+
+	txqs.gen = q->gen;
+	txqs.pidx = q->pidx;
+	txqs.compl = (q->unacked & 8) << (S_WR_COMPL - 3);
+	write_wr_hdr_sgl(ndesc, d, &txqs, q, sgl, flits, sgl_flits,
+	    from->wr_hi, from->wr_lo);
 }
 
-static __inline void
-rx_offload(struct t3cdev *tdev, struct sge_rspq *rq,
-    struct mbuf *m)
-{
-#ifdef notyet
-	if (rq->polling) {
-		rq->offload_skbs[rq->offload_skbs_idx++] = skb;
-		if (rq->offload_skbs_idx == RX_BUNDLE_SIZE) {
-			cxgb_ofld_recv(tdev, rq->offload_skbs, RX_BUNDLE_SIZE);
-			rq->offload_skbs_idx = 0;
-			rq->offload_bundles++;
+/**
+ *	calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet
+ *	@m: the packet
+ *
+ * 	Returns the number of Tx descriptors needed for the given offload
+ * 	packet.  These packets are already fully constructed.
+ */
+static __inline unsigned int
+calc_tx_descs_ofld(struct mbuf *m, unsigned int nsegs)
+{
+	unsigned int flits, cnt = 0;
+
+
+	if (m->m_len <= WR_LEN)
+		return 1;                 /* packet fits as immediate data */
+
+	if (m->m_flags & M_IOVEC)
+		cnt = mtomv(m)->mv_count;
+
+	flits = ((uint8_t *)m->m_pkthdr.header - mtod(m, uint8_t *)) / 8;   /* headers */
+
+	return flits_to_desc(flits + sgl_len(cnt));
+}
+
+/**
+ *	ofld_xmit - send a packet through an offload queue
+ *	@adap: the adapter
+ *	@q: the Tx offload queue
+ *	@m: the packet
+ *
+ *	Send an offload packet through an SGE offload queue.
+ */
+static int
+ofld_xmit(adapter_t *adap, struct sge_txq *q, struct mbuf *m)
+{
+	int ret;
+	unsigned int pidx, gen, nsegs;
+	unsigned int ndesc;
+	struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
+	bus_dma_segment_t segs[TX_MAX_SEGS];
+	int i, cleaned;
+	struct tx_sw_desc *stx = &q->sdesc[q->pidx];
+
+	mtx_lock(&q->lock);
+	if ((ret = busdma_map_mbufs(&m, q, stx, segs, &nsegs)) != 0) {
+		mtx_unlock(&q->lock);
+		return (ret);
+	}
+	ndesc = calc_tx_descs_ofld(m, nsegs);
+again:	cleaned = reclaim_completed_tx(adap, q, TX_CLEAN_MAX_DESC, m_vec);
+
+	ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD);
+	if (__predict_false(ret)) {
+		if (ret == 1) {
+			m_set_priority(m, ndesc);     /* save for restart */
+			mtx_unlock(&q->lock);
+			return NET_XMIT_CN;
 		}
-	} else
+		goto again;
+	}
+
+	gen = q->gen;
+	q->in_use += ndesc;
+	pidx = q->pidx;
+	q->pidx += ndesc;
+	if (q->pidx >= q->size) {
+		q->pidx -= q->size;
+		q->gen ^= 1;
+	}
+#ifdef T3_TRACE
+	T3_TRACE5(adap->tb[q->cntxt_id & 7],
+		  "ofld_xmit: ndesc %u, pidx %u, len %u, main %u, frags %u",
+		  ndesc, pidx, skb->len, skb->len - skb->data_len,
+		  skb_shinfo(skb)->nr_frags);
 #endif
-	{
-		/* XXX */
-		panic("implement offload enqueue\n");
+	mtx_unlock(&q->lock);
+
+	write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs);
+	check_ring_tx_db(adap, q);
+	
+	for (i = 0; i < cleaned; i++) {
+		m_freem_vec(m_vec[i]);
 	}
+	return NET_XMIT_SUCCESS;
+}
+
+/**
+ *	restart_offloadq - restart a suspended offload queue
+ *	@qs: the queue set cotaining the offload queue
+ *
+ *	Resumes transmission on a suspended Tx offload queue.
+ */
+static void
+restart_offloadq(void *data, int npending)
+{
+
+	struct mbuf *m;
+	struct sge_qset *qs = data;
+	struct sge_txq *q = &qs->txq[TXQ_OFLD];
+	adapter_t *adap = qs->port->adapter;
+	struct mbuf *m_vec[TX_CLEAN_MAX_DESC];
+	bus_dma_segment_t segs[TX_MAX_SEGS];
+	int nsegs, i, cleaned;
+	struct tx_sw_desc *stx = &q->sdesc[q->pidx];
+		
+	mtx_lock(&q->lock);
+again:	cleaned = reclaim_completed_tx(adap, q, TX_CLEAN_MAX_DESC, m_vec);
+
+	while ((m = mbufq_peek(&q->sendq)) != NULL) {
+		unsigned int gen, pidx;
+		unsigned int ndesc = m_get_priority(m);
+
+		if (__predict_false(q->size - q->in_use < ndesc)) {
+			setbit(&qs->txq_stopped, TXQ_OFLD);
+			smp_mb();
+
+			if (should_restart_tx(q) &&
+			    test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))
+				goto again;
+			q->stops++;
+			break;
+		}
 
+		gen = q->gen;
+		q->in_use += ndesc;
+		pidx = q->pidx;
+		q->pidx += ndesc;
+		if (q->pidx >= q->size) {
+			q->pidx -= q->size;
+			q->gen ^= 1;
+		}
+		
+		(void)mbufq_dequeue(&q->sendq);
+		busdma_map_mbufs(&m, q, stx, segs, &nsegs);
+		mtx_unlock(&q->lock);
+		write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs);
+		mtx_lock(&q->lock);
+	}
+	mtx_unlock(&q->lock);
+	
+#if USE_GTS
+	set_bit(TXQ_RUNNING, &q->flags);
+	set_bit(TXQ_LAST_PKT_DB, &q->flags);
+#endif
+	t3_write_reg(adap, A_SG_KDOORBELL,
+		     F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
+	
+	for (i = 0; i < cleaned; i++) {
+		m_freem_vec(m_vec[i]);
+	}
+}
+
+/**
+ *	queue_set - return the queue set a packet should use
+ *	@m: the packet
+ *
+ *	Maps a packet to the SGE queue set it should use.  The desired queue
+ *	set is carried in bits 1-3 in the packet's priority.
+ */
+static __inline int
+queue_set(const struct mbuf *m)
+{
+	return m_get_priority(m) >> 1;
+}
+
+/**
+ *	is_ctrl_pkt - return whether an offload packet is a control packet
+ *	@m: the packet
+ *
+ *	Determines whether an offload packet should use an OFLD or a CTRL
+ *	Tx queue.  This is indicated by bit 0 in the packet's priority.
+ */
+static __inline int
+is_ctrl_pkt(const struct mbuf *m)
+{
+	return m_get_priority(m) & 1;
+}
+
+/**
+ *	t3_offload_tx - send an offload packet
+ *	@tdev: the offload device to send to
+ *	@m: the packet
+ *
+ *	Sends an offload packet.  We use the packet priority to select the
+ *	appropriate Tx queue as follows: bit 0 indicates whether the packet
+ *	should be sent as regular or control, bits 1-3 select the queue set.
+ */
+int
+t3_offload_tx(struct toedev *tdev, struct mbuf *m)
+{
+	adapter_t *adap = tdev2adap(tdev);
+	struct sge_qset *qs = &adap->sge.qs[queue_set(m)];
+
+	if (__predict_false(is_ctrl_pkt(m)))
+		return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], m);
+
+	return ofld_xmit(adap, &qs->txq[TXQ_OFLD], m);
+}
+
+/**
+ *	deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts
+ *	@tdev: the offload device that will be receiving the packets
+ *	@q: the SGE response queue that assembled the bundle
+ *	@m: the partial bundle
+ *	@n: the number of packets in the bundle
+ *
+ *	Delivers a (partial) bundle of Rx offload packets to an offload device.
+ */
+static __inline void
+deliver_partial_bundle(struct toedev *tdev,
+			struct sge_rspq *q,
+			struct mbuf *mbufs[], int n)
+{
+	if (n) {
+		q->offload_bundles++;
+		cxgb_ofld_recv(tdev, mbufs, n);
+	}
+}
+
+static __inline int
+rx_offload(struct toedev *tdev, struct sge_rspq *rq,
+    struct mbuf *m, struct mbuf *rx_gather[],
+    unsigned int gather_idx)
+{
+	rq->offload_pkts++;
+	m->m_pkthdr.header = mtod(m, void *);
+	    
+	rx_gather[gather_idx++] = m;
+	if (gather_idx == RX_BUNDLE_SIZE) {
+		cxgb_ofld_recv(tdev, rx_gather, RX_BUNDLE_SIZE);
+		gather_idx = 0;
+		rq->offload_bundles++;
+	}
+	return (gather_idx);
 }
 
 static void
 restart_tx(struct sge_qset *qs)
 {
-	;
+	struct adapter *sc = qs->port->adapter;
+	
+	if (isset(&qs->txq_stopped, TXQ_OFLD) &&
+	    should_restart_tx(&qs->txq[TXQ_OFLD]) &&
+	    test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
+		qs->txq[TXQ_OFLD].restarts++;
+		taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_tsk);
+	}
+	if (isset(&qs->txq_stopped, TXQ_CTRL) &&
+	    should_restart_tx(&qs->txq[TXQ_CTRL]) &&
+	    test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
+		qs->txq[TXQ_CTRL].restarts++;
+		taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_tsk);
+	}
+}
+
+/**
+ *	t3_sge_alloc_qset - initialize an SGE queue set
+ *	@sc: the controller softc
+ *	@id: the queue set id
+ *	@nports: how many Ethernet ports will be using this queue set
+ *	@irq_vec_idx: the IRQ vector index for response queue interrupts
+ *	@p: configuration parameters for this queue set
+ *	@ntxq: number of Tx queues for the queue set
+ *	@pi: port info for queue set
+ *
+ *	Allocate resources and initialize an SGE queue set.  A queue set
+ *	comprises a response queue, two Rx free-buffer queues, and up to 3
+ *	Tx queues.  The Tx queues are assigned roles in the order Ethernet
+ *	queue, offload queue, and control queue.
+ */
+int
+t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx,
+		  const struct qset_params *p, int ntxq, struct port_info *pi)
+{
+	struct sge_qset *q = &sc->sge.qs[id];
+	int i, ret = 0;
+
+	init_qset_cntxt(q, id);
+	
+	if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc),
+		    sizeof(struct rx_sw_desc), &q->fl[0].phys_addr,
+		    &q->fl[0].desc, &q->fl[0].sdesc,
+		    &q->fl[0].desc_tag, &q->fl[0].desc_map,
+		    sc->rx_dmat, &q->fl[0].entry_tag)) != 0) {
+		printf("error %d from alloc ring fl0\n", ret);
+		goto err;
+	}
+
+	if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc),
+		    sizeof(struct rx_sw_desc), &q->fl[1].phys_addr,
+		    &q->fl[1].desc, &q->fl[1].sdesc,
+		    &q->fl[1].desc_tag, &q->fl[1].desc_map,
+		    sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) {
+		printf("error %d from alloc ring fl1\n", ret);
+		goto err;
+	}
+
+	if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0,
+		    &q->rspq.phys_addr, &q->rspq.desc, NULL,
+		    &q->rspq.desc_tag, &q->rspq.desc_map,
+		    NULL, NULL)) != 0) {
+		printf("error %d from alloc ring rspq\n", ret);
+		goto err;
+	}
+
+	for (i = 0; i < ntxq; ++i) {
+		/*
+		 * The control queue always uses immediate data so does not
+		 * need to keep track of any mbufs.
+		 * XXX Placeholder for future TOE support.
+		 */
+		size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
+
+		if ((ret = alloc_ring(sc, p->txq_size[i],
+			    sizeof(struct tx_desc), sz,
+			    &q->txq[i].phys_addr, &q->txq[i].desc,
+			    &q->txq[i].sdesc, &q->txq[i].desc_tag,
+			    &q->txq[i].desc_map,
+			    sc->tx_dmat, &q->txq[i].entry_tag)) != 0) {
+			printf("error %d from alloc ring tx %i\n", ret, i);
+			goto err;
+		}
+		mbufq_init(&q->txq[i].sendq);
+		q->txq[i].gen = 1;
+		q->txq[i].size = p->txq_size[i];
+		mtx_init(&q->txq[i].lock, "t3 txq lock", NULL, MTX_DEF);
+	}
+
+	TASK_INIT(&q->txq[TXQ_OFLD].qresume_tsk, 0, restart_offloadq, q);
+	TASK_INIT(&q->txq[TXQ_CTRL].qresume_tsk, 0, restart_ctrlq, q);
+	
+	q->fl[0].gen = q->fl[1].gen = 1;
+	q->fl[0].size = p->fl_size;
+	q->fl[1].size = p->jumbo_size;
+
+	q->rspq.gen = 1;
+	q->rspq.size = p->rspq_size;
+	mtx_init(&q->rspq.lock, "t3 rspq lock", NULL, MTX_DEF);
+	
+	q->txq[TXQ_ETH].stop_thres = nports *
+	    flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3);
+
+	q->fl[0].buf_size = MCLBYTES;
+	q->fl[0].zone = zone_clust;
+	q->fl[0].type = EXT_CLUSTER;
+	q->fl[1].buf_size = MJUMPAGESIZE;
+	q->fl[1].zone = zone_jumbop;
+	q->fl[1].type = EXT_JUMBOP;
+	
+	q->lro.enabled = lro_default;
+	
+	mtx_lock(&sc->sge.reg_lock);
+	ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx,
+				   q->rspq.phys_addr, q->rspq.size,
+				   q->fl[0].buf_size, 1, 0);
+	if (ret) {
+		printf("error %d from t3_sge_init_rspcntxt\n", ret);
+		goto err_unlock;
+	}
+
+	for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
+		ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0,
+					  q->fl[i].phys_addr, q->fl[i].size,
+					  q->fl[i].buf_size, p->cong_thres, 1,
+					  0);
+		if (ret) {
+			printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i);
+			goto err_unlock;
+		}
+	}
+
+	ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
+				 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
+				 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
+				 1, 0);
+	if (ret) {
+		printf("error %d from t3_sge_init_ecntxt\n", ret);
+		goto err_unlock;
+	}
+
+	if (ntxq > 1) {
+		ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id,
+					 USE_GTS, SGE_CNTXT_OFLD, id,
+					 q->txq[TXQ_OFLD].phys_addr,
+					 q->txq[TXQ_OFLD].size, 0, 1, 0);
+		if (ret) {
+			printf("error %d from t3_sge_init_ecntxt\n", ret);
+			goto err_unlock;
+		}
+	}
+
+	if (ntxq > 2) {
+		ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0,
+					 SGE_CNTXT_CTRL, id,
+					 q->txq[TXQ_CTRL].phys_addr,
+					 q->txq[TXQ_CTRL].size,
+					 q->txq[TXQ_CTRL].token, 1, 0);
+		if (ret) {
+			printf("error %d from t3_sge_init_ecntxt\n", ret);
+			goto err_unlock;
+		}
+	}
+	
+	mtx_unlock(&sc->sge.reg_lock);
+	t3_update_qset_coalesce(q, p);
+	q->port = pi;
+	
+	refill_fl(sc, &q->fl[0], q->fl[0].size);
+	refill_fl(sc, &q->fl[1], q->fl[1].size);
+	refill_rspq(sc, &q->rspq, q->rspq.size - 1);
+
+	t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
+		     V_NEWTIMER(q->rspq.holdoff_tmr));
+
+	return (0);
+
+err_unlock:
+	mtx_unlock(&sc->sge.reg_lock);
+err:	
+	t3_free_qset(sc, q);
+
+	return (ret);
 }
 
 void
@@ -1774,7 +2156,6 @@ t3_rx_eth(struct port_info *pi, struct sge_rspq *rq, struct mbuf *m, int ethpad)
  *	threshold and the packet is too big to copy, or (b) the packet should
  *	be copied but there is no memory for the copy.
  */
-
 static int
 get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
     struct mbuf *m, struct rsp_desc *r)
@@ -1786,19 +2167,27 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
 	uint32_t len = G_RSPD_LEN(len_cq);
 	uint32_t flags = ntohl(r->flags);
 	uint8_t sopeop = G_RSPD_SOP_EOP(flags);
+	void *cl;
 	int ret = 0;
 	
 	prefetch(sd->cl);
 	
 	fl->credits--;
 	bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD);
-	bus_dmamap_unload(fl->entry_tag, sd->map);
 
-	
+	if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) {
+		cl = mtod(m, void *);
+		memcpy(cl, sd->cl, len);
+		recycle_rx_buf(adap, fl, fl->cidx);
+	} else {
+		cl = sd->cl;
+		bus_dmamap_unload(fl->entry_tag, sd->map);
+	}
 	switch(sopeop) {
 	case RSPQ_SOP_EOP:
 		DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m));
-		m_cljset(m, sd->cl, fl->type);
+		if (cl == sd->cl)
+			m_cljset(m, cl, fl->type);
 		m->m_len = m->m_pkthdr.len = len;
 		ret = 1;
 		goto done;
@@ -1817,7 +2206,7 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
 		ret = 1;
 		break;
 	}
-	m_iovappend(m, sd->cl, fl->buf_size, len, 0);
+	m_iovappend(m, cl, fl->buf_size, len, 0);
 
 done:	
 	if (++fl->cidx == fl->size)
@@ -1873,23 +2262,6 @@ check_ring_db(adapter_t *adap, struct sge_qset *qs,
 	;
 }
 
-/*
- * This is an awful hack to bind the ithread to CPU 1
- * to work around lack of ithread affinity
- */
-static void
-bind_ithread(int cpu)
-{
-#if 0	
-	KASSERT(cpu < mp_ncpus, ("invalid cpu identifier"));
-	if (mp_ncpus > 1) {
-		mtx_lock_spin(&sched_lock);
-		sched_bind(curthread, cpu);
-		mtx_unlock_spin(&sched_lock);
-	}
-#endif
-}
-
 /**
  *	process_responses - process responses from an SGE response queue
  *	@adap: the adapter
@@ -1913,23 +2285,15 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 	int budget_left = budget;
 	unsigned int sleeping = 0;
 	int lro = qs->lro.enabled;
-		
-	static uint8_t pinned[MAXCPU];
-
+	struct mbuf *offload_mbufs[RX_BUNDLE_SIZE];
+	int ngathered = 0;
 #ifdef DEBUG	
 	static int last_holdoff = 0;
 	if (rspq->holdoff_tmr != last_holdoff) {
 		printf("next_holdoff=%d\n", rspq->holdoff_tmr);
 		last_holdoff = rspq->holdoff_tmr;
 	}
-#endif	
-	if (pinned[qs->rspq.cntxt_id * adap->params.nports] == 0) {
-		/*
-		 * Assumes that cntxt_id < mp_ncpus
-		 */
-		bind_ithread(qs->rspq.cntxt_id);
-		pinned[qs->rspq.cntxt_id * adap->params.nports] = 1;
-	}
+#endif
 	rspq->next_holdoff = rspq->holdoff_tmr;
 
 	while (__predict_true(budget_left && is_new_response(r, rspq))) {
@@ -2006,12 +2370,14 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 
 				rspq->m = NULL;
 			} else {
-#ifdef notyet
-				if (__predict_false(r->rss_hdr.opcode == CPL_TRACE_PKT))
-					m_adj(m, 2);
-
-				rx_offload(&adap->tdev, rspq, m);
-#endif
+				rspq->m->m_pkthdr.csum_data = rss_csum;
+				/*
+				 * XXX size mismatch
+				 */
+				m_set_priority(rspq->m, rss_hash);
+				
+				ngathered = rx_offload(&adap->tdev, rspq, rspq->m,
+				    offload_mbufs, ngathered);
 			}
 #ifdef notyet			
 			taskqueue_enqueue(adap->tq, &adap->timer_reclaim_task);
@@ -2022,9 +2388,11 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
 		}
 		--budget_left;
 	}
-	t3_sge_lro_flush_all(adap, qs);
-	deliver_partial_bundle(&adap->tdev, rspq);
+	
 
+	deliver_partial_bundle(&adap->tdev, rspq, offload_mbufs, ngathered);
+	t3_lro_flush(adap, qs, &qs->lro);
+	
 	if (sleeping)
 		check_ring_db(adap, qs, sleeping);
 
@@ -2130,11 +2498,8 @@ t3_intr_msix(void *data)
 	struct sge_rspq *rspq = &qs->rspq;
 
 	mtx_lock(&rspq->lock);
-	if (process_responses_gts(adap, rspq) == 0) {
-#ifdef notyet
+	if (process_responses_gts(adap, rspq) == 0)
 		rspq->unhandled_irqs++;
-#endif
-	}
 	mtx_unlock(&rspq->lock);
 }
 
diff --git a/sys/dev/cxgb/sys/mbufq.h b/sys/dev/cxgb/sys/mbufq.h
new file mode 100644
index 0000000..114ac85
--- /dev/null
+++ b/sys/dev/cxgb/sys/mbufq.h
@@ -0,0 +1,86 @@
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+$FreeBSD$
+
+***************************************************************************/
+
+struct mbuf_head {
+	struct mbuf *head;
+	struct mbuf *tail;
+	uint32_t     qlen;
+	struct mtx   lock;
+};
+
+static __inline void
+mbufq_init(struct mbuf_head *l)
+{
+	l->head = l->tail = NULL;
+}
+
+static __inline int
+mbufq_empty(struct mbuf_head *l)
+{
+	return (l->head == NULL);
+}
+
+static __inline int
+mbufq_len(struct mbuf_head *l)
+{
+	return (l->qlen);
+}
+
+
+static __inline void
+mbufq_tail(struct mbuf_head *l, struct mbuf *m)
+{
+	l->qlen++;
+	l->tail->m_nextpkt = m;
+	l->tail = m;
+}
+
+static __inline struct mbuf *
+mbufq_dequeue(struct mbuf_head *l)
+{
+	struct mbuf *m;
+
+	m = l->head;
+	if (m) {
+		if (m == l->tail) 
+			l->tail = NULL;
+		l->head = m->m_nextpkt;
+		l->qlen--;
+	}
+
+	return (m);
+}
+
+static __inline struct mbuf *
+mbufq_peek(struct mbuf_head *l)
+{
+	return (l->head);
+}
diff --git a/sys/dev/cxgb/t3fw-3.2.bin.gz.uu b/sys/dev/cxgb/t3fw-3.2.bin.gz.uu
deleted file mode 100644
index cb56875..0000000
--- a/sys/dev/cxgb/t3fw-3.2.bin.gz.uu
+++ /dev/null
@@ -1,478 +0,0 @@
-
-/*******************************************************************************
-
-Copyright (c) 2007, Chelsio Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of the Chelsio Inc, nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-$FreeBSD$
-***************************************************************************/
-
-begin 644 t3fw
-M'XL("")[[T4``W0S9G<M,RXR+F)I;@#LO`M<5-7>/[QGS\#,[-ES@V$8+@,S
-M,,">^Z#EI6.&E[+2GK"RPW/H%#G@T6Y:Q_*:$`(-=U34&6"X9:B(.I"9A=4&
-M/8A*!5B4PG!10DTE,[/.2=GO;^T!M<YYGO?Y_S__S__]O^_[C*Z]UUY[W=?O
-M\OW]UMJD8#SL'(;)`AB&T&`8EH(]C`UA>!;&Q]#=@7$P;!!C0S4FP["Y<^<F
-MH.=S&"?H3IR7H.$,WLD'%6FP/TII#!N.Q[Z+QZ[$8]7Q6$T\AF.T%A^)Y^!8
-MJR::P%ZF).:I-O/4>\Q3IYFGSHB^GP,Y6C2T#5Y8IV+6J3+K5(UUJLWW(HJ0
-M^<K$3<7BILKBIFKBV%=<'`M@3@4RIT@<VZ()8+KOBM+Q6S29$.D,9+I00F<0
-MTZ5DNFC;UQJ_)9P43,&;X@=5<^0X_'\]7L&?I_";3/)Y<]Z('X97KT^_@J':
-MVX*8-MJ&Q^$IF-_M++R9\/XJVSI]0K-1DZ7-CEHYV;_C@8X_IV#RG,0#B[;9
-MHF?P5D[^,<I$VAEO`/-!-)'Q\G0^-+5%$V5*8*M^CY[$M?BQM30&,HU;-'*F
-MD8")_N_??__^-_R`3^?^IQEZXF7,S\1_^#KRIB^<P_QW_(=QAL90^-?QUG9?
-MO+7=%Q>^!/?.2.;$88BOU+!]Y&2<P^1+T1U"UCDLX((&F`=".,0OHSL$S3DL
-M4(#N$&9`G$1W"`D0I]`=P@*(F]`=0B+$IZ$[A&<@CO(^`R$%X@^C.X27(([R
-MO@1A!<23T!U"`<11WH+Q^&JXET!H&(\WC,=+X=ZIP;@>7YSK\<45*__/C#];
-MY8L_6^6++T_]?RK^6QI]IC22^8(&5:4!NJ`AOMK7WR=ED!_>'?U2@ST!Z_@D
-MK/&S.LASU$=#K8,01_<#D3?',(8YSV!2AL&A3C\("1`R,"RDGV&P_KV??#\+
-MGN]%#=(84G5^-O0^!<-6L'W`A!DLP?^"RD2(?64YF`##5!`7^/*P!4FXA4/@
-M^1XQ+IN7A_FSHTG`!C#4`P')?,HPS`"$03Z#:O@%XS,9D=`9#:M'^5QXQ:#$
-MJWR&&>,RZ?!X!,)$L4CFR"#T(1*Z"$H7_009B=`<RH&A@3)7L7=1EYFC*R%D
-M^'HKTFBL3`8S!C5#[R*9=@'J.:H1PYCO.9@<Y=\$X2V&^1ZE_1K)?`+SV.IA
-MF)_1\ZU(YF0BS&L[QOD5\6E')'.L8PA#,B).-@2CU-A\F`'JV`CO@*^/0?EC
-MUX>@^2$T=FYGB3'1GF%,7`.A!$(=#,,&W9-+$=[06,=0'1G03Z@%YH/#_&),
-M;(0\S,]P1[,N,R8V8<;$CRFX0_F/JU%A8^)'D'8X8PCF>@@`S""T-<2N=`;T
-MY1C0Q+'.R%N8`.[#$'[QP@_&3T*?2R!<AJQ:FNUW.\B312`#%J7`^$!V+`+9
-M\]0!"%]&,L?#(0WH\(L4:$8+[T`><`_#>*NAW"[F5^:J!DN^%\)6H-LY$&`.
-M3H`\X"VX+3.9$X?NIF_.[V2J1H8EW`X*F`<@J-MA(FTBJ#!:$\[!;@<-)X%-
-MFP@H[??/*,^=D/*;H.-4:[`%@M^&1V_>%6:,AWLTV"-P?QADXD,V#3;W\AU^
-MG?\-A!WC=PB/?ODO@LEW?PCF[T&0X7.K[Y3G'=9@EKEPAWF;!?EX,/?3V^$.
-M=3[@T&!:E0:;ZM1@]X!L-4,P7==@!I#W.J@O'G2$&M+"0`\$/J7!`G[18#+H
-MIPSZ*SZDP40@RX4@B^>N]@4_G@;C0Y@)\GP*W'E'41\FC4L@=$_]W;__%3_@
-MC4[?O>TPT*("PDO_2RK^[]]___X/_`&]?^/3AO_J]R_2YT1G_%?J]<GFIQ6^
-M^Z*EOTW__T/PZ:7Q.X0_@CQ^YJ[G_V^'B?5&4`GIZ_'X`W#+N.OY4[@EW/6,
-M@(7F=\^RWST+?O?,^]TS_KMG#L($X_&)M#N_9THY?R[%GRWE/E>*S?FO4/;O
-M?L^4</Y<@C];PGVN!'OR?Z)\12FF+N&X2_'*4FY5*4]0ZB<L]2=*^:+_8G<J
-MH'`)7EG"5Y=PJTIXJTO\UI3XKRWAK_LO=J>/;=];BO>7<@=*>;Q2/[]2?_]2
-M/K]4("@5"DL)HE3TG_6E#PJ7X/TE(FA_H(1WH,3O_1+_@R7\#TH$ATJ$'Y80
-M'Y6(FO_CO@!/'->&?U,1$,[T$_0_OT>X3AM>LEFQ)2B0N85?L)4JN='$1MO6
-M8,+F*_]D%M8-MO`8,5[?DQW8J;N?GR*Q+W_SO!3[ZL[S_\C/3X'9F;__#Q?[
-M?\WO)9D?1XXG*:=V29'GC<9H+8;'XX_$8P\CUR"-R;\G`[\7<UE'H>I[4=CW
-M)%TB^(A\<Z=`GX-1!3Q#P\WU.PE7ACZ'0Q4(#`V_$(\J7MXM<F6XWG)EZG.X
-M5(%,\3W&3\`4HXQRE($(G4*O4(V.*4;'J$S?BUMP9?N"*T9_17$>.^DOR7BT
-M+6#TYTS-7[5X"O8'.B%P]*>-MHVVE?%QRM$?@T:O92<>6+0U13%Z-2M!]QBC
-MF_V`"*I?H1R]'+L8@_^KEGVM&KU`OQJ7@\'P4'WHQM:K@=A=S[^_T]5Z@F>,
-MBJ"BU'J=OR$&7Y>+$WOPX\V!HSVGI"8/W^+!Z"[N*&$Y._>,1C2(]5:G[3_/
-MUHK*"PQ:/([`7M\I<]QCG.27_K8@_5,__EH<*B+(^>V?=?SQC*;7UI>0).Y$
-M9<)&CUB:*M6C1Y:?QU,X.RQ-;O$HIV<P=:C2JB%INDI&,O(J^1Z&NI9#N];D
-M0YY[J6O9XF+.FPZ(ZW/CU:/O"W,58A=G^:88NHO>&#+:9-'RW'97BF!4+AC%
-M+4LX)(Z9F]R6EU3F/I7YH#OW'C/!2]\^536Z.^?>OD[UZ*Z*']Q7"S7;KPN\
-MN/L:W>W^236Z@_Z2NL'U%R1('7+J!C_]+P+J44RP"Z=B\`W;_0SQ8D-,U)O;
-M_:AX*16C7;]]JGZ&AJZA"$RR:YI`,#LU8X-JM#1TM-20,Q,K<6"#!7=?H3_Z
-M)K>^0J6'SMA6;9\:J^45379,DAX3A8[F'NAPOUJV(L]8N2IB-(>W1"ZIQ\M>
-M<[Y!9SA?%SH3_!URP1*\I0`NVU8>IRT:DAR5BT8%YK/9XL]$YI'LM,17T+Q"
-M5X1L5]**5]J9M:>DZM&E!7+5Z!*ZIDBVMP,;XF'UH9:H"+8OF=1;N'`WKC^8
-M"8^"X?ABNO)J(9W;>L#KIM]T<8M;JFA*B_M&;Q^;00_:QY+I@V<T29+5]C&S
-MP;2R%Y;S>'=UE_0E&=%CBS:MW!(8./I8"O9"Z.@C1S/4HP\;V9:LDS+-7;B`
-MP:U34$OBX?B"O>5T[E['_@,N?9UX32ZW>%_57KT6-Q`)A`O7Q^"?/Y6;(ERV
-M<.D66.PG@'!2$.',ICT]GO8/Z(59C?2K/-X\_@8\=4A+KX`^6@A,()B?5MS&
-M%ENVY1/#I,KP4=/R';B=><TXR?W>(^2GG/6'X"G5H"&-"\;*`LAZ^3L-5&N.
-MU\.25TK(J(9JS186<-8@"EM(UR":4@--E?WHNNJCJ<H?H/>&26[]2S)#@\HP
-MQ9VV\+4S-#UD)3#R^VE2Z>S4J]LB1A6J404U,O.%QM#T#_S0RE<7B"(5+_]=
-MY(O[K@8"RTT)&B57;9_OL)NUO+<7DY^(Z+F1H_S2H:K!+./V$>=YGE?N-Q<O
-M/4M?K?I66)7@S\BE7KSE%ES<PWH-V5U-NN3ON*C-V0<^%GPCHK9E=TDA>2*Q
-MNWHB$=$%=%'(=C&-><;.+';)EFY^.E=GTDB-&DECATDG]2S8T>QI-NHDAUZD
-M2OB&$NX.1V.]H93K<5"E_+S`).*!0MG2+I!)Y?1@OLZ@D:JN]!L.</4:2;%'
-M?X`/BUOEL<3R@$+&EXOL*=/I=!*ZT:B3TA[[V)SP*VUT(Q4541C0T.`C"JR+
-MAS6'JJ]\*OT4!P*Y31VYM+.DF"YL/<!4T>FGN84M;MJJQ2&3%>CPUMF\0+>B
-M0EX>1#<5R+U`%UGY084*4P#'/K8'&KIUAI5'_I3&'RAJF4,A%,X_ONFV^"$]
-MM,8?0\ZPT"OOJ*]4&VLK(Z_4+#\`:RXTU;IIM_1S3GH[C//76)C+O\CKUU%-
-M.70;2R,4U90M+>2D(P()I]^B?U1>V1JGY3G/;1L4K)%+7;AKJ&B*)%<AD<R+
-MO)*KO+*]^/JV+N65;;N9D"O;_&&&.G,,PE&99`DN!#;NMM@"]#:3U19(NF0\
-M1N8_*I..XL(UN`0J.I5SW[8OBZ=6?54TS5KKKNS)G6Z]J7)]G7.OLU=U)6O;
-M-_XY"B0*SA39K'7N5>>G1EY9$W)E=<XDQV1Z1YG7U5=@C.T)H'H"Z0RJ1RXI
-M2_"OE_/6R(&YI!UXRUZX5_33)_0],IAP5DRR4N0@2)$2=GHBKOS[45K/,B]5
-MFVGMPDD'3M6AY9$.Q^=?<7D<5XJ_/^!P7EG]=U[NJ'Y$#-*?K$?"WCZV!91J
-M^)7Y$5<>P9[GL2O>L!Z6&2UY;:9EB%U'MBK)Q$H7P4I;V[#TB[S"%G,;9M'B
-MDD]P"ZIK'M3EDS/D?I^<$1@T_D8MOG07;^E.3I=4))C?7H@6MR^AUY9$3D>T
-M'G'%I#];&7S%M+P4%NHYTUFWF1#SO^"L@F7E),*R;I.12^3;Y/N64(MS](2"
-M7=L'J<79IVW\`LXJM+I3*2TO^$I$:2`]:_O5TA_I.8*WY;Q5^/8?LLQ6K3KX
-MBI9W7<@'WJ055[1Q9]VE+3J;+&Z]BIPKXSEPQ15UW(@;(L[6M^_-"DQ=\7"_
-MS1W@I=-7!1V87C2=FJ2L/.9L*YQJF2$SMPG,'21?I7@Y5PM:ZI`K>K/"<7%+
-MI[/+<7[C%6<WM3EHRY?.4QLO62YKMGQ5&$#?0P\V,E+IXR^NG$K?PU\C)%T$
-M?RK)Y\XZ/JG2>=CEF=Q^'WN?UIZ)[I"'"G@Y3U19:=DFKW16ENLK;*K+/T>W
-M!09?OJ#7AL>U!5ACU'%16+K+#T2$)49M:0NFVI0AE[_W8^36-E7D9:^^+4B:
-MCEO;%$MWOHCH87$V$BQ+@%;>96F%/H@;\"3R1_O8&Z>D(9=/TC7JRR>+9?ER
-M[`>D;/8V6UDZ,I_-U&_"Q7MQ\PA:?.%PO,-3E5'HR6T"=>,!=>-H='J`8Y%"
-M0DNO=U"&*/_5-0M:3EG;PBUM$98VK:4M1G7Y([J6+C6T18=>/D:U15%MH:K+
-M?R,`:+3I]&UA(9>/4&UJ?5LDO4GX(5L1\^@X`44B`J)W&73^=F:*#[/H-?Z@
-M/)8Z>4NW`RT))FB)%12B.D1+1@VIO.PB/Y+[UPOTM3DF0K2^'.CCJ/IRJ:4V
-M6WR9\^8%>'S?NKAR^76(?$`M=K>L%^?[8,IN2JL1K$C1:V8<^QZ3SN00,QOW
-M2L],)^J%XKUR<U^X6#Q+XN4V>N$NKA<9&L*_KMZP/=RLC;&&WES;&RC>1TJ'
-M4C#A3`Y_9N-JX=O3B55$;BP5Q1<ZR%6(9AWT-P9MM$6K;:PGB,<-,=%K+\E-
-M#0RPARDF6MR,&]'PS70)G4G_J+[\0FF`0<NC9[D]I>\)ZN72_7A5HU4S@QZ6
-MC'$$#$?ZR+1BF]21(*U*X#,RNE/RV#2!5\9WR<0NO-`H>'BZR"$3.G#`2C"J
-MB,LO"BB%Q(M;M#%ZFTQB40CK<3==J.&Y9`*O7.C%Z3*XN%MR#;Q1N>NH63,#
-M]*?X_@0_KPST%Q1TM\&T"*>#OD=2R74,!%>U\TCQI++VTN-5K>8H14M%8[\E
-M5N'Z6]E)]PG'-&='UGVEGQ7_H>KSHNF57Q3>"Q-M;I/E3:$*5(8V>>Y4:HE;
-MWQ:@`YD]5UXWUSH[VWHST?I0=E9`>WIQ;)%^)U,5*QTE)9*'+3$Q:8G+6<IE
-MZ?9!^QAQ2AI^.?RH)^)RN,$'6!8C024"P+)D7%`5?E;6F?N9XXL#+M=G"*]\
-M7O494([(P4*C6\<KY&<TY3+^.CS)_\$">;Z,+C$3'`*6^O2TM,HD^ZU=/AH4
-M;4`T^!L\3G*GX)1-!I2X[&U8TIH)8@1.H%<(B?G'ZSY+1!J5MP!/$G'B=G!]
-M/`>%\VVAEZZM_$!FTD8#L&T_)G;C9K3@GUMJ*\,O75D^#-6]`,J#_)6S_A>(
-M/VM4$O0U(.CP2\-`T`WU5"U(/RDK_1X+N31(U=Y&/S.-6FO1$M&E*;!NJ8WO
-M&K0:PL,2HG1FXR_"XNG'JJ2,4,2(I()90@>WT2$0S-K%G+E*$?CJ3D2_Z;U*
-M*O2F^!^DH(8M!H2_2Y@W75I/%,;JH_AB%[D6<5%&H=U])'>QJ]61YFRWILB6
-M_0QZ]^3VXU24RAH56/XWY\FJ$QN:45I9R*GJ<@7]5N2E#^ESA:$50290NZ>J
-M.@4=<DD'7M$5?LFC3Y$96]V&Q7SC896QS7UR?\2E(VCB"H**%>84KDM&>N7B
-MEP7F/DT2YQ^6J,"X6%7CJ"4VL/T!1VRQ_AV',U;`D%+IPU:@CX4XFF#S2YJ(
-M2TTNF:\6:P`OB;/3SAQ9OA6DTW-'!WURPN_/>)(PD!4MK!P4-=K'%K"1_8!Y
-MRO]&;Z([Z6M5Q]TG72<VO`?+,[DBJ%Q1%I*:6$\/CM><(H,.5LF*%6X5='+O
-MRVP?NW(51:K"D'W[+*G<@MA\_=X.LID4B1ZNB,U;8HR)$=:394M@HNAKNQU4
-MK,H^%G.LV'!85:8DB%F">BZ2"+/*@O?4`U!-_P8ZG`W+LO841-8[E,7!NQC[
-MV#.@6WU].:,Q!@B3_+:G[=V<%Y*O*`@J6F(?6^PC6:(+C6U<)A+/]U33"X$(
-M[<PL1,,(LFNC`6`9&AB?!%I:>O\RY[U&K7;='BM@7,#_IB@U,B:/<WJKP9:\
-M-.<_LR6!LM/V?V)GCKXD4_38Z&$P*$,OW0>\H;XT;:G3;^EV/`6O-FIUJ4^(
-M*6W4.J!<?#52&<+Y[56LX$;`DY@612!\9XH*RI^TOA'R9!9/HYVIUS8"[=/7
-M"J;2.:1SNC.RL4`,-IY2`KP@KA=8:G-"+D6NOP39'T&<4,1R`GZ_46O)^XMH
-MSQ1*,R-.&^E4IS;-Y7ON%LK'JOBK"-$J@G\7+ZQRJ..4_/0,*Z6-<:I6Y<=%
-M*15OJQS!4:$W^6^3^*LIT<H@C(NJB%K/X<9-%ZPBH@(4T0%!N;&.N''ICD?0
-M):X+CE3GQ=PECK3R;_4C(FJ$6`OL@)OTVD@Z'NK.2Q'J%2"(>0ZY,YPH2"#R
-M%-)Z#G'?='Z]K"P,I+WJ$A=$/3ECNJA#5A&Z45,0*9@L)R?C!0;!9'Q+2'24
-M*-7S>9Q2I5,J8Y3!V4OIE5LOTPO+KSA>R%KF_+YT5`?Z(8,_CR.=R^$E3:,3
-M^$]-"_KN>[^_X5N'L\.%BW"^0R9@,4]6R-NAI4."5;CJNPO\J3A$M@_"E+T=
-M)IB6X`>95N'\17A<@&KK"%09$Q#,NR>!F"OG[<"E@*+.ZP*4!1&P.GJMCFR6
-M[VTV-&5[7`WUAH/95%20/D;7""0>='Q%86R!?J?7'.LO[B!)\F$3<.H322S5
-M&`]S1*+'C3$Z^ZUOZ0P8#9UJA*[/\C_(D39S1`]/HSW^\Z=EA50,EPY!GW-"
-M!<TR$-X`H57??>P_!44<8=L&<\*A%&!ST1\2_"8R;$,]_J?NVIG'6@JI&).%
-MR+@C!P3R[FKHS#($.S];NNTLXILD8J_]UN)3TA:G/L8$PIFN,:![V['BN,.*
-M[7(^?Y9@%=>A;EP%L>T!>U;%K5>`?6&_=1V6EU(*5FV$^`6''`@H,V!75)22
-M#X1BO[4/>->DE%N4`<"[RL#>%4F\G<:`0/5WKU@"`NAKI@!Y6L.5O+_8;WUT
-M2FI0RE(7[J8S<I<Z`\NON"[G+G.\X!IU?N]89@B0.66Y+SB5NS]T*=:5^^7*
-M&ETN)9T%I``4YB,+@S)$I^4!O0KFRN.T,4!O_#@%;Y5<N`K?J,Q3;%E&QY>]
-M$`-S72U(XD@7<?SNFP;$1R<*[IWF2!$LDOG5XX+L!+][IV\TBA;)B$6XWV0\
-MZ+OG!<X$B&P9S--D1P*%0GKH=TF!WR5!9.M0G@'6B%N/EPU#Q8:`$+][$_P6
-MR02+$`UM'8$T("1((Q8A^QB(:6)1-J85!3J6O#.ZZQ"E]`-N`@%AN<P1[2$=
-M095ISB7`7/J=+F`O@>!A*B;&$>&*+?\+4!WYH7RO6]^=#8)`X!+H>[+5WX7;
-M?_W.)_^$F]@U1=D^E8L9@?Y:#BB`%"R'NI9-;.2LS7B3WJ+^+L2D-1FU/%&'
-MG-<L)YMQTZ,!<.U;41GOMA4:R^_I3=A^[_:I$=_)MD]!V&<2\B-<R^8]AAL:
-M-(8;V<CA9EJ9)&1E+;NHL+BPKMP_X$CO"[EH;2_^/+&V/=6.G?1"2BFSC\4A
-M54,H?+X^`?"J:#;_/H6^@2,JFZY?8#,2'.$AGV?J6_7%T?"+HZ:>F2_4A*?O
-M],,2'5A)`1&A."5]^:+(]^2[TH-T=X$-%-=5Q$-9@D,<<3-'??$G0=4T`2,3
-M54[+-_+!BGT%C,($@1/A0K#^P725-.,`.D'/'*P7FQ4\KUP?("M/<&AXY3*!
-M0P[<9`0XO`MWS@+CJM@`255SR:H$BV8&X$W)S`0_UOY]?0EAU$:*C(H4C'=*
-M"G?QQRQ<?`B@JL")2Q]((!@Y\C;,LVC(R(M-Y*A\_RCR@NU%7C#0PE?]/O7Y
-M.21V1O&2C*!INLY(8&DKV@K1F#*A/?JZ]'N.8)1#9TB<TX#4Z$%IY31Q,4#?
-MZ0(8F`,O,HH8U!GZ*EDU31^@`.M<NASWC;(RH5C#<\*042\0B)YEU<:0G\ND
-M5H7#P&/D4%Q<F>"<"](/9LDG_B#-HN6]OD1@TD:2)AB:GTFK@8A@5"[]`M5;
-M]1!DEU3BD)UPH"J<\ZP:,N#B6I*1US*6:]G='LD^D>5&-I"B-Z%HX8FS3?6-
-M'>8^F63-%!?P(EF<Z#I=]8WCR>(GG`%5,F=OU9D-6\2-S3'/R%+3P=[I`'E1
-M&."0[RT47"1="U?W<HMEC8S[R:HG[&/G?7@TU?:8GRT%(V9RA#,;G43N="0*
-M%U#']C=,Y@IF$0XN5<`!W2:8+-CP0:1!&V,*O?G7/4KB#R39R*I#?&8C)<R<
-MSHTBH(_=-EB)37;F([9N$C0XP-R[H"TH=J%@_O&<.]A6\.C=V#8/*)>W<B=,
-M4W1:ZWVA%Z>9:RN7?PEB5::^.,52ZZ:9<=,+8^A&:I(@XN+DU+?("A5"MQ?-
-M++H%:\U,2-\$G(E]%WE1#]::]`+'&9J."ITV:"VY2UN.$JXI9FTD=*'$%9;,
-M?`HP*ZXL!&`6TK*<1QVA!?:*C?F+MSO*,_/2RG)S_U(8YLHO7N+.MN80EL6"
-M#;V0:W+H106]R:&R:GEE&4`4KHWT%B$8#CF<R(LB8'/+-;=YL=!R666YX3[Y
-M=<1%[1E-06AQ&.T!R%<NZTT9!WQ4,K.H**3]>X1%]_T6BUI]6)2*N!@`:!&5
-MK38HD>V2S``..VXD_)$,$`;:QSRPRFP<!,,.2D,Z@TB'O-[A5%"9.>D`*;"_
-M%ROTU[+%3HY3^:83GB_JM;Q*NOP(_9[`B1C4V8(,]5IWO@VYS.O<JSZXOW"R
-M8Q)]M$@9>F&P8BC\PF#98)YQ^[<D".>S[O,\M]PY(O)YI[UX2P9RI`]#%:9K
-MV:8.C>G&;7\2]`I!3'PL;9_(/O9N:F)VQ`4?^@VEK_K`;Y@[^,Y<!!3^I2C8
-M$<:"WZ(E>PO[!E?WXJ;+'$=8?DC%$O=?\F)S]7OJ"1<I%#Y<%EN\5!\3`S-6
-MM=1^ZR8H/%9P"Z*1,!55@"T,*G3K&4U^4)+@A/U6_OCKL+MP+7_/;5R[A/4=
-M:G$!D&?^9T^>T8S+8'XME*,]+:U)O%L^?!EZ88OZ0HF^MC+BPB;6TY1K!,)\
-M7G*8L^$0/*T#!B9_]OD/EV^$A'I]4S;G*$_BYFR`N>>4TW_-?:!XIB5>2$V2
-M66)X(`OXHWCPA8KM=&7+VQKAF*KR"'VA\F_T6>PH3_AWE?2+)S%@=0?NMT;N
-M/*JZ@*@,28[6(TQEE_4K?Q7JBQN`%MI@J'.'7EBQO;NLD_7S9^LK9/J#V8Y[
-MG8:WIVPW%D^M,A5-,QDPR;.*2K,^/KBE4RA,H&R4,3Z0TLTPV/3Z&+4Q)MR@
-MN]\0'R00S"-$LT")&@]?-L9$;GA;25;?;0M*&+FTF80W5FUDR(45TDTI,%&6
-MPS?I$DN,!A/,Y)`S&X\)UD\7=TRAXD/BXL/T,=$`Z6#%`!#I8U7D:#C)A%,Q
-MVKB8*&N,Q:Q16&*L8O$<LT[Q65)+J[`<3^)5TSN-8&KH_/U9IP9BB-X$;X*P
-M`N0&[[?O?&L3<4'_/^V_%"+_Y84PU840[`>>GO5>OHB\EY9:M%^"/)-UX_LE
-M1;1KT(&\EPXGO?HBZZ>^X[N\U>*3JV<TX1>,Z@NR?&WD!8/H$=%^:LW&>?2@
-MZ$,<*>=/IDDDLT^<MWYVRWQCY@NS0M/G^6$VI(1Y*N2W\\5]U\P:&)B%P+AL
-MF;3O3]B9S_IMHBB2WD01F#1*@#\0('+ZE+V=@@IS9AXK22^+U-^4I6_WPP8=
-MF*=`&!D`ZOX?(M^3[_J._O`>>HC.$%U`94_F6*#H"/2E"/6E!.7@10:@,FS<
-M=V7[`JURG;Z]IF([LYF^.K'#<-B.3CP`'J$T_DO?!CGCB"4X(/+3GJA`S&<#
-M?DIF/J0]$>?/%^P(/-]?K*&(#&J'F,Z4='#HP;@=I%D;S@7:KHW6W$__P.4F
-M<,>F$:"9P=3?@9(&!9,YD!1Y_AL1@(8.3N3YK^%=Q/F>BAWPQ&9[I5OYLH!H
-M;,8:>=BRT#?+!79F%!86NAUX_O.(\Y^#O/9\K*\.IZK#I-+'`8P@WY9##H0G
-M!."2ERX\SA&[<'.UPK5#*)PG<,G9IZ")B-(7D;@XZO.'+=41$-?O4`IS%?H=
-M0>Q5P5X#]=5J]?GW?76#6L]]E^N`>CF%YHCS'K%:(?%RQ%Y4D>O=7(/Z?(-0
-M^+@?R^34C@B!0T'M4+/7\**=8D8N'N4`_K!6:RIW2B3S$'9`3]J)2)0OPAWE
-M1-<&LL7"BDW<'Q^&X?DQZ$WQ;LN.*$F1PK)#RUXUCEWL/3+7*'9PH&DQVW3(
-M^0)SK5;(<$+.Y_E-E@M/<\1['W+N@O6(KHUJ*^9*'N&^HX`%\)\LYXX",T0=
-M'02`)A8_Y,^.1'U^0W0>E\M`+R*/%4+[$L@':[E;P'`<]="0ODX;K9FA/O\Z
-M-SI!RKX[-K-J-\`R*".)2$`0:[=>,R/P_"L<%_(X.NOA":9-.#T!9Q-"SC^!
-M^HLBML(=SOIBJBB6ZX6Y(>D?)&/3<2`4T$,[-D)R[E;N*`EID>>?X0*E>#GL
-MF]C8P-A816QL4%P,[H@M-#OU8L#]L>+`\Q'Z295(EF/G0\^'&R>YUX(DQWH1
-M^#\LYW8(S*TYE@5\I/>Q+G-K-M'/6?L-Q/]6*2L=0NJA\=#ER/,!WH2JP23_
-M?DJK)G;BJ1D/&:!XO9R+',C9PD=$^KKLV!CUB36?_]'G'5EW'"0!=OS#P)&;
-M_Y%GI'L%T*V#LD;YK_[A0?JJZCP1<5Y`M])?YLM"1FZ(YHL.SG5YW(UYFLKW
-MJ@XXWZ_Z0'T>JVH6[\7='[D.`=RJ^#!\Y)*_"XFXLB;6*?X"W6Y"CO#%['XN
-MG_5&H@CNV]#W\:YA$O>5S!5T9^#(5R9-4'%LT,B7.HT(1)S?7,(X6[GT[((R
-M:NT/3^8^6?C$\L,P#PU8%\_3CCE"!;L$@-6I6#Z,4`\C=/E&^'O7#Z""\1&.
-MH&;[.Y'<9.I1G!XRZX*JXLA#K(_^(>6RS:`R>5:"%SYR,&SDP++MT%S>LBV5
-M^JC(M"JG7O.GI9LVHSS8Q="1L^9)6L->9G.G:9+6."G:U,$SU6G7?^AGG!1E
-MK--:='\"+0^B`.ODT0N)&^2Q8O^GIPN<7,$NCN`7L`U#=BWQ]YL7HPOR;R/]
-M#PIB'U*F)J*M2D\RUARZ9X^H'"^/2_M>1[>2EUELP81`9;Z-5U&Y`&G[;$A<
-M135EDWF<]6#+,R^J1CZ)&/DD?"0O9"2_JL95G:MQ[ZQ\U[W#^0X)K%^+M%13
-M-GV!*M!0![/1EMQW-HO.WQR#5Z`-NER`<^^?DE;%V9D<:,K,OK&/U?Y3FUS4
-M9A;;YMC/(2/O1(S\M6I698))0[CG"$$HSG9H$#_-+8BU:/AE\RH>LCQ(F'7\
-M"8R@`8Q@AMD9>QA#=E-);TG:BA\-"U3&&2JZ!82]J-VG7#Y7CR2'CR2#)?EU
-MX@M/19J>N95^S`^K1GJ!C$#ZY3N1[\EWS4\P0-FB>1+I/!'YI.CA1%Z'C%R/
-M5\2'C_SY@(/'6H]5ML(Y#HW[7E#3SDG%L76,-=:_,"%RY+6U/?.XZT3TE^$C
-M">[&"@^E(5SO22%34Y$&"8H#A;$F#=]]T$&9WB><'P#DTNOX`2,FUFR*NI9]
-M8`,^BLPFA/600342>]N@JI9L&#>H8&A$NT^/Z>R,@060!0GE`>L^CZ;?,\U0
-M>1._3DSR^RX_@"XIEP%APFBN3$L=>#8_H4AF9^(0@O0F]J>4R1'VQGB]B5]7
-MY\F3_#;2)70CZ&BP?^)@!CWTD<B1!XH2]ELE1?-X5T1G.M-/X0;O+W3GZY=6
-M506E8!&N:JI!AF4XG`%T!G:UH"K069.:\K0>4(*7^_JE)P_M+J[F?T-(ET_9
-MWRQ=0E9E?%.",LOXWX@..;",`J><JL66H2WZ[*8.O9+G5*[^\;G<FF)%6M6.
-M]T:M#:K<H#-7U_R(&V[]DK9\EWDV9.06SN[M#!T)@Z6QSH45*;,99G.M4=BZ
-M+C_5B%)0SU%]^U/9O6A])A7-%HV2E;,+8_=^[([-G=M-[_[0-;=+FOV.-(=\
-M]Z=MU3FU6]_Q?QE/3^8TONRN+9QM9^H:.^+6JUP*.[,286J#)DBO409]>P10
-M>VK);)-&Y/<Y83ZK%`Y,D?:2QA'E_GJ"F&?0!9VHSG\RB\I]PK,+V\0#8;)K
-M+@^072P_*Y9WB%S]AV!V[P-X811MD=3ZSHO06C"11I32>M)71VKU*_E/%E+%
-M3WCVH5W(T=!]7G$S\"U=$_;MC@\^#_EVAQG`X?=RWWF8_<V(`S=G6A>RAUVV
-MC>/$(H][,-?C:#K@<GG6Y,H<C="@T^/;GT0-WMZL1`^ET*-J9.L.V(R@">C3
-M"7DRAZU"6:YPR06I"\N"Z1I`TGZ[ICF#Z!5FZ'F*6#P_M?.UM(6E%HUB_7D\
-M!0]>-@>$7'*5:AFL#6=:GB)?5:33:Z0%<DHC:73I==(&A\=!Z2360>ZAYD;&
-M/,BW#G,;O)X.<Q]C'N8G<7J6SD)6'EV=J[-HI.'?OFXIY9HUDOP2<RF?A/&7
-ME.E2.()E<W)S?WQS&[01ET^MW^9':?Z4^E9V7JRHGDRMS@_[-KT\F*;#ODUV
-MR.ERH6.J64-6*,E^N>B2P-240Q7XK3X)15>HOGW*T)0M:."LW@6/SQ<%%UZ3
-M]$_)0_L[L,(;RH,K`JM48%\-5@8D,QG%JK3]@44!A=<*`BL"W[R$[".PR`(+
-MY94!CJ`J58'*JN$7!>QBK#K^0&+RV"!1BZ":\(5Y:?M?5G][+U"+N$-D^HQC
-MI#>;Z"T"ULD6_NTK#<UP(9LYY<%VYE%TF*\D!:.@T:L%\B3>E#(=L."0-8!C
-MT4FM.DD*)F9[=-44P$GB6>B2@9+CI?1"^E7'IJP26!P>,:\LB)>ZD%)R>E;D
-M*],^^27ME2?&]Q(`R!YY@>ZDSX%<SK\.0K,\KIPRQ@J-L?Z&J\$$,=N@$QVH
-M-^A(ZJI4('B0TA%%/ZT8O9]N47W+EXB?=61RO'+L+1[6$8IE\/9VF(DQ3P=)
-MAH!11'>3Y#QQQU233FD?6T'7&I`$?C6?LC/OTD?8Q&1?'T`%J8:OAPW_9(B*
-M"!_^\?8)KH;+]!!0N/#BG8-<^729)Y=VM`+ETFMR_<RM'',;AHS:"[[]RMQ>
-MNDKUV5.5`<C^A:GB7BH*H!?F!YZNUBL%9S0%JD)Y\EAW^/`9A'R:Y0W-H!^*
-M5<+=(M`0]EM#K%VC&.ZD:?5P)T7$ZVU<@TV@UQ&K:]*,Q*1U-4@&I25F0N*Z
-MW>N-&O]E36\@PT@R?]G0RP`[0&L",+!&J?4:$:4)$KL(PV:E21=$.`34-N6Z
-MO^/D23S5,V<"+/!^AX9@B4_\NZ\"R(N0Q0>0E6*S.@@JD\45&7=P!2MY%$:-
-M")K05S``Y\7-A*5)J=<I)/4"PT%E>RFKQ_>PB.A'D%4^?]7RK4^>>&II:91O
-M7\F;F(0I$<L+YK=GLA"EABA#%B,;;T$SF<1Y)W5_Y[A/@;M-7RWOJ=ZP76`D
-M;HG*.<8=\AB"8V<\]/C.(52(^]W>#Q90II4@"^((#I!?"F;Z_.E<V\0)..%$
-ME6_1MO890+'/9\6_;4,GX/CS>E-2/3UT"BPN[6&/P/U-/PD+'7Z)/0;WH:&V
-M,GQXV?(=4,F<B.&EIE9WX\.2SS@;T$F828!+PX<7D_OE#3<MK3DAPRFK$5_:
-MT2DXW^X7YX]AP\_1&T.&D\%:VG[*U>ES>E9UE0^&#3]YZ+*AR0VXMT%E..A.
-M6_C70]/I&MI#[IDF$LU.]3P4,?R$>O@)9$G.#O<\BAS.8#G2!<1M6Y(N\%U#
-MAF=10&8.^6Y'5&8V78W/%D7EL!J;/8!!8$*VRK1/&^Q,$SJX]L2A/Q3J*`TH
-M=$DCB$/I#L8#<D-BZ>3FR1M'39U\RY?<'1V>#M.7?,^")&Q3KFS9IBET]496
-M)@[;+/>P,M%FOH>5B;8RG7ULU\0,ZWV'UFST#U:=U#Y60MOL8_EOVV"B[6-I
-MIZ2APX%'2R*&`U3#"I/OI$`F.BG`.OPF3@KLK>CT'6UT[07>L]9*K'5BH)NC
-MU3Y591^+/B55#>-8#8]N40_CEJ@(D`J(F[O9\YC'<7U/IOK<K\#&>V\Y.BLS
-M<CL=W0=<^AYL32ZWJ*NR$QV%^\)W%.XTW9B'CL!QCX2<^_ZN2;PSA;ZS<`;V
-M6._:7`Y=3:?0%UNZ6X[R^`DAPV%T$WU=^CA'-)]3+A/J!*G5C[TLF(>ROC77
-M6(VM;USUYH?`RUEK%VHWH,C&I1\&T>5K&F5T\>I&40O$_%L^:2FB=F`;&O]B
-MK.83S2+11_C*]W##Q[^TU]B91I,6__QS8%4SL&J7WU\_]$_!-,?OISU@[9RC
-M_T-K)Z5+2E7+<F5+WP:N7`_PQ<Z\<4J:QE0`E#@E34TOI9]O^<2X`[,S>3X?
-M2X^&7G'RZ3,I/K9*9DSZY^_PJ2%%06O8#>IYAE1%.Y@7=".`>&,:WB6-2_7O
-MDGI7H-4WV;A@#IIT1-^*9.8]<XK"0OP,YKDY%0#,#IC#R',?!9VK%"^0*\]]
-M6*C)BQ+WXOAIDGA49(C%]177V^]97@D<P]]SHLZR]MRBTYZ34PP$YS;UGCQ`
-MW;H5DS/SA5GA=UP\H0@LU__&R9.GJ3^Q]B),]<$R37<GL("'.T[]G]B9EC,V
-MD86L&Y5TD*FVY9(I`?0F.L-_+CGA]5FBAS:VS3R6\4:>6O\2Z_5A$;C0YTNZ
-M"Y._<_&P@6ZD,PAV<^ADGA$*'ISP/[$'5'D1`0C'WW5DM?:+PTR5KU,9$WZ?
-M0CM30@\*QSU2A^Q,\V&K+\\9&YTAG*DP$C)),RF:/T7O%5@`R;(^IA-OCT^&
-M[XAL-1JX*$*!MJ'8N.^:K[&/>:"I"[ZF3MJ9SUDYJ<7Q6-8]F]&19'AI)5KS
-M9,;ITY&@KP+.F<%8KSV!_/\+<-;_OR'R'/+_J\Y9I=]QT@>7.ZL/[6R<7UZ3
-MEMAPVG9F!=U"=_(_FR86ST[MG!-Y3A=Q3F>Y\=O)0`?T?C,9=+75)K"NQ$PI
-M)-DAK^W0MP*["3\4Z=O0'E0GEZTN;<!C9PZ>DJK/^=,UU.((K)$7<0X!SGTG
-M+5'H2+1D'VZ)G3CTZG&6%'N*F@XP59[TBW[6)H[U($8]SR+2-.`#.=@E+ZWL
-M6^%-`"/$DX2WY<O^DR[X^`+4'ICQ"#HJEKX513<:M+A1YT\0<]IWF`@,E![-
-M:BODCQN[,2%QD?ZS<5L/@#XWV.04$;VZQK:NQIR:^#&HQ5?0WN<_4A.[T"8W
-M=O+VWG<RXT*ZD9R?2N>QO@0;+(7Z[`EV.]_2E-/T>#HH%ZP[XFR;"4S=JY*3
-MK!+"6NBAQ@6JL\<K&IV>0\VG/:+Q<]2YD6=;(\ZVWEF':I]3E%V'NX@$B6"8
-MA0.8S\FRMUY?FTUKA,>1EP7M);&</JZCKYHT01%GWS%K1%9=D,1+6+J55;I]
-M':8>I9T!F4S3:#3)3),59HT0H./J.N#],/HJ=]RO^8*=64%?N+/?CP7=F3/>
-MR44@&XQ1D:JS;2!N=U$@&<AFTA@;:8B-6+U'W=Z$\NJ)C-UZ?6R$/E9-E^AC
-M([M+?.4->W^!;';FF]OG$N,([)7M"70=6D+6O2T0S/$'!:)3M+\%<@J:,(*8
-M;A2#/&/[S;T?UC`[-7$IB%R2G'_\2`IV&@#Q,K1>!\'>`&LCK?-(Z-DEEMI*
-MU=FT5\X_::UU"W[EK+[ZJ/JL8[LG\NQZ2LL#:,UZ0NFM50>=C<X/6+>W6Z_S
-MMURV0<66.K>X`SD]4)_U-FRI:\WM#1+.G^Q,IOKL_*.>D+./^HY\FFK1D4\1
-ME*B;./+IK<HH\A8,'!BU](@W')?E]4.M95[1[4KA\<Y#Q[-F:.-T[#B-)G%F
-MH=3N%>-S9EKI:SIYS&EG^+Y]'Q\ME_K61$!7]VCT*?Y1A(:EV[[@L[%A9V/I
-M+EK#DN]EQ%Z@)HH[_NC3'4D<_3C>!Y[Z7+[S.Q`BC8^GH[-J,51MMBB/LP[!
-MHI#2FI"SJD,-KNJTA8F'IOOD_`3P28D\JX@XJT"4^UO@XQ/"=P$?`S'.N_P.
-M@;$U6S1?9&S+ANXNW11L20DZ\6UW-9UI?1ZG4OW%0(MIR"#RG46X`XDVVT'R
-MI?JSR=;%$:;%ZO3CN)7`)E1ARZ#4,CV9>9'U?Z20[&<U.V]2K3G+LV#\.P#J
-MB8K8,6%.U5EUR-!Y^D=KJG_5D&M0GT)L/B=PX:ZSQ2D(\PWK4Q1KD=?M@ZCA
-M8,1UK=G6FRIK6S:[^T`XIZ45>>QC[X<-G<1:>"#K(H=.8&="]X]:7\LT-J)]
-M<NL;B`9$(.@&0=`-YI\]P%A'L/0/N8XAYR#2UV!*@YP;>PVD'-T(H(939`:\
-M/-8;,O1!@0QJ1&<F7D,$!0U@?:'$"=SP1F;HD`?(JOX?N=7.DMQJ1RV@KNHU
-MN;RB&DL=!G,'2X.P$I*?3YO9?L.0J0(5U9;=_B>D/*"9$#0Y[%E,HL?&HG`O
-M'>]4I&"V4]+/D_[Y.Q2LFOT.Y3"R'+,:-WH`AW.Y\WI7I#:JV6]1C./?HK06
-MIH0.Y8D1#O\`<'C$T-LL#G=$#N58SKH//"*]S$&;T9Q5GL=!>*F&TMEO4LH"
-M!?4"JC9'.H:S)S.K54-K0:8)7)PS):F;RNCW@H=6`Z-N.;*=#AYZS7>N@MVV
-MC7[-'3U#%?V&&Y);IJWK$6TOB=XAH#>[:O0-LK2%#8=>Z"VA6\Q`.^.J[J_A
-M0\M40R]0;4"H$7U7TW^>\.&I$2PY\QN\0%<[4NC6=3G1!\:$OXJV>RH:HP\(
-MD$-MZ`G6H69&#C7Q%9'Y1K;/T3:TX)_3Q[]/FO"UI=N9;!;.Y^G,&JD)P+S7
-M#&`>`7>=Q%C-]2QH;#;NX.JK^8=>W.'RN/0[^'F*Y%ON(MFRH2GTU7ST]4K(
-MT/VF`UR#1N+P&`[PD7/34P5P_KF)I;K7!^<]=)<9P?EYZ$.4V1MML%[VL;#0
-MH4@`\T.1JB'-_SV8_\UW2K>Q_"TOZ(T2^]AB$+6A0V*Z)7)('#%$%@9241'[
-MO$4!R+_4$6JYEFFHP26?XY8;J%X">*`3>*"SJ!MXH`=+O\@M[JKJ1/9YN\\^
-M?[LJJ%+ID@.Z5_25`"M@)W/E1<KB(/O8*A;/1PP%1P[^"OHF>,@(:L`0%=GC
-MP3_T@=%(?47&.!B])X;(J#NQT[SVU+.G4TY.!6:`Z?=G/P\[66"X<LM:]UL\
-M.N&\O0N/&J+4`$A/LV<F0&>UP&QW>\;W_L:_,WO?SAPZ8_,WDSN]J;8W];Y&
-MQA%IIAY:V393/#6`^L?557F1^O43J#0%H5)HSH=+4PI\UW<-=`KWM(#Z1X:`
-M(J,)CF#*-'__V:DI\RFH)V<F_]_(%X9"T\^/PS+);5MR`I:)V\E=U+L."KJJ
-MC1KOZAVT^BK8#[!0Z!NY:;Z4G7:F'E9PT[L&&)T>%1A'KO<J6CK]O*3XL2GZ
-MCP7T0F$2BUL=OFZ\T!6:_O4X)!'_$R2)C5+;Q[*@U21?&R=8'S!//$.>'Z4:
-MK,_3`C:X#V&#\J@R[>H]@3YD0'<>UKNT+HT["J%GP`-E6CMS[O99!%9':7Q8
-M-V/\_,'@7R,&5^AK*]6#K[)[5BWH^Z4?Q"<Y;R)D]5[5>_01]>#+>BW/[1&X
-MY"10[$$.NR'N!LB$A&"=^_A5$(/H/$0R\_4$ED&GT6URHRT0<TW"#D]*S7C0
-M8%.H![=C]9-6?^HG+A>:#F!A@UN75/N)*X2&HYSPP2WJP2U[RD4?XJ\T^NG/
-M8L(/2;(B_,`><8?`<#3?>+0@B9-I9[::;0%A@]G8F4FK!_Q$E2+#,+:$]A-5
-MB4S7.8`SH78[\Y5Z\`]'/92/&6LS3;XS^"QX((%IO,`T"#PP5=[TXUQ'O],+
-MO"@:MZK[[_K&E?MG4OR`7)PL#Q[\MZ*H,YJ=#:)1,E-;68M/$FVN61\-O"&6
-M-I.IGD<#!_61@W-K#X$P'^0]B\R8U4<6G3P/]DGT[776.+"$`G%(P,ME(E_<
-M=\VLP9L$KQV'J3Z^N<92BQ\?]/$%/K[N']F9CVG-X=VN&E?UYMHNZ>D$O($$
-M54].#>![R=3!919H9&3F"Z^&IJ]B&RDIX/LP`AOW70_OWO%Y%,M2$LK'4G-^
-MTS?VPU+Q;3ZX_:EI$UE;>SAJ,]C@J<@MFC'1J3P[4SC.`I0O9;^=:3HEC3+_
-M`IG'+AZN165L2,8,G,;B>72-:N`T^0``VPCU0`_RE7M]4A)I]`DI6=!93ON\
-M%*[.-;F2W"Z@65>G[[O1+JD1UF@/`N$^'4LAE/:US:DX-'E+T"E/VA,!(#+.
-ME)0I4[#IT+,O$N_XO=#1VA5`\6-':,_7GN,'3GGHY[,:#[V?Y]D:1"\$O0LF
-M?9FR-R6U<?;='K#F(IL$:=WWZ!KZ:LC`%8<F?*!NMR-D8`=90)A;E:96A=`A
-M0F>(]T\3,$1!D&J@R]3*,5][2=#!6=^+I^#JT($*\AL2M#9UXR56:[\>/K!=
-M/;#=U.K6+^"*OV!9C)."/&@#FUB]W<!^2YK`ZNR2D(%"Y$7+Y;@"0P;R6%?:
-M^CRE7A-$N`1"X3R]+NB5,F@&E^3/*PZ4")^3/)QXZ";/)1.>QET_^/;$FO'R
-MJVE/B)!3[2YP^5SDP/J(@3<1N.R.\)RY&USZ9.E=\)*NSH<F^7M<;(M\XR0L
-M8F#)VH/H&S@S&$M>^3ZOJ1O4,[E`9.IAU;/O?(EPG\^:WV)G7*Q^]BQPZ(P:
-M*6A94,@ZZ:%U.^H]]0:=1)_!S5,TNJP9?'T6EW7"9?&3?WV_0+:L90K=F8^V
-M+B('%H%4I322(@]U@(]V[#QNT,_Q$PO[&*N=RW1TC0'IYQ"DGQ5`!;D>Z+3]
-MUGGUP-2CM&I@2NC`M/&/EME#.$B631S"*?967BWPY@T<Z*CPKM_MI^^6Z'O$
-M8$8<[?29*O9;>\`"'XBC6T(&8D&T%`7LW8O$R^9,GW*.&(@$B`@4;=TV\5'B
-M;>5<U0FZ^7<^MS^?DI[1E`<;"2*9F9(?;+^U`#%8`GTM8D`5/O!D?B"E(42]
-MHDI/Q(`(;6<V536ZWM/I^$U?_NM9KZ:/J`:X1BW/]:/SJJI_S+>W48X^-39E
-MN@VZ(%.'RI3CMM_J<K.`('FLO5!N'XNZ^YO^_,ETM8G@I&"XH8&#U?"P]E!!
-M$V>UYZIO+Q\=$`13*9GY2=T_'#9@/%;E\`AW<X5YG#*/L0M?DQ$<WC^H'H@U
-MM&:*ZW%#VSA0!W)H&91\@DLDCUN^PF%B6#O7\$*):B`T<B`L?"`\HK\[I/\+
-M\]G,J`J5U,LQ;F(_[FMFS^)OQ:$*1T:9,W0@T.DJVS[NDR"4`[+P_I;@_A:I
-M6<[O)RJ5I$G4Y.5["7>`.;1T[V?N(--G'`-=8J0W">KEHGK\[BV?Z,,"47VX
-MD4A-\K^W+-#ZJFZ+K$K^Y8K`_IMY0?F!AFHL9L:`J)[*ER7YF_/EHCU4@=*H
-MY!G;-.B\O`U[[7A0@<*0J#'>IR'KJ21_'=U%>*>:'[0=F-NTJ&GR&]//-*TJ
-M#BA2.H(.C#8ZG`#"TB]6C_\=!;3)A^8RM'\SM)/$*68MWV3F[V']Q?11?2/>
-MWD!6@.G]N.E]G$1S%=&?'\E.CM1[YYM'WWJ%]V>&]6]$!DU3IKX%_^M^?WB%
-M%C.@OUW=?T+5?T*\EV,^F"EP<@2"QZF_X4#GF[^(ZI@VOI[T=3HQHG^ELO\U
-MWAQ<K_%7]+_"GHA@EH'8Q#:%]S>3]\O%R*K\R=!ZW=+Z\[J:.88&GJ'M^H;=
-M?H;6&Q`QS\+7[8X_5@4R'A(H@D/LP3%'J-#!-;3]!.]]7\$)&([O$SCQ<#Q0
-M%Q2*T?UI;;/2VGI#W_JS:!\I=9$2%RG4*]K+4K!;Q"%20"G2,J[3FT&4"86/
-MZQ_$V4'G4<8H_[6-8(8M-D5%IGWQC$F##%]_]DC-\J\0O`2!&=+_AJ20LP&9
-MHS`'<ZNZG)T6#;'UE!#D1K?O[,*7%@W?T,,W]OA;OB)B?&<76MFS"VW9=L;O
-M=K7Z5M\Q?;>Y-5MRAK,!'=/91#M"^I^)Z']&W4^%]5-5->75^9JM.RO?W;K#
-M]0[Z7+(.MT2IH3IT:H6MD=+YQ\3@ZR[B?LEXZB<[@_L#)WS6_#L^:YBXM/TC
-M]C$']$75/X6=-?SVK`$3A?7;]+-PM-J+,X&>#4O&:0&=4^PWF-A7YL69$B"3
-M)>-09WR=P[W_0,5:,]7>&ROW"Z.Z<-Q')S`[I%X..JU_3GZ&XRWA;HZ^+7/=
-M6]Q<3\=F]#Z\?R8P]C@G?H4X,:\DJH\"ABKK]=5-A/=/"_(.!O2;%=Y^VJ'L
-MGQP=+@OMCS]MPZISRF1829Y+I5^,O9SWN+1*CE<*>;<(RR#FQPBEHW+N*&'H
-M?LGX@XUH)T6.*?F#C2ZXU><)FDGJA_CR06/WRX:>ET0G2$']%,>04#@;[H2#
-M=`[MJ3?TO-Q81A"SRU(F]B84_<$PE*!^96"_Q)&X>_*)HF3F&YHV3+ING/13
-M12(0&S7IAF72SZ+]I&"41-]P6L`V!W5'?.*+\HL2?)GIA6IOP['EYBX\TML`
-M4[^N9A8UZ6=3+89(?\KUM;E^ADDW(+)N]V,0P;0\1/))H?Y`\E-^@G1$2]<R
-MP=+6W[A#\B8"6[=;[?M,'MY"*>Q/H?[U7.*N;)Y+Y0D=!]N/DJW(1P-I41\Q
-M8//AHYQNVC*1Z9_^WHI/=OLKO.ET/(\BJ$RI]1_,*UGKU=[/2`NHB?RNO,X(
-M[]IU-7]PG"JH/E!?UKDVEYO77=:Y;O=3>=W0D=YJ;'JHU,4MZS+TH%:ILYE"
-M!P=(C9J0-A/]U_GZ;SR+^M]-&V;AV/VA4B]7W,PQCHR;I%W2;MIG`/CZUVM3
-M>/](9P"$`YK7LYNK+D*_&=%\B6^?QM?_".]"TBC?]Q'[->DLA'^P7+7W,5-M
-M-AW_)KU![74IO`_M_1B-!L:WKN9^WVB@RVMS>7G=</_?,AY_9#W1G3$V/F7#
-M0KQ:P<,!0G:SN"]1E(&%>37DH_(]]?K:+.&_B_1U6;ZQ^<J$>(.@#)2\JT"X
-M-Y"<+V]H-C1E$7\2&0YFL;HQS)L<XA63%'"F(3.+RLP&0T]4SUG[%C_`*\!K
-M.7?W!U!=JN89I???LV6!?;=T8-)])##N'3,V60Q[QT0ALY9NF;UT\_QEFYY*
-MP>:#]D)RH-9@O)]C:+AF\I28/9L$'2#B\=>/SS!Y-IL]6WR/:X^;C0;LT.3&
-MN:\]LA>(,?W3>4'>V7;F4Z@BYKX+=N9@#)%J9SQF@A!^-<W.[%SQG,BO2&&]
-MGQ-S?9/E>LF*_2+"H=!#,Z5;J-+-O$?QY+%`,)(GBK/^!Y[*:PGUFO%'%/HZ
-MN6\IJ,P`7#A'ZI*C,SN30^EHT"Q1H<(H+I[)47FCN%%X5$X`N[=`:V@/G9@C
-M"^X[<NS?2N6*OB,I6+RW^I4:OKKO<&&G>"_9EQ+F#0[MNQ71]P%QAI*\`I+V
-MGCWUO4C6=MM2;3D]B48;W^<-+)"O;7]!W;=7_&?"W*WS^Q,NZL1.5:=:>PQ1
-M'")/P:OGI#Y0:YK$(0L4O`X.>1)?V[(E!7ODE#2]2AKS#$<8HDBK^I%./%V=
-MOO-[=5\%[?%5=":1K><8@RWDL8<KI*/<2KF=>1?*8(D\0AAB9PX#5F56O]+"
-M#^LKSO>(3I+0H_"^SHB^"^1A2MUW85]'19>=B?'Q^JR`OFQ]:"M%F$/[,L/Z
-MO@WL&T$NNTEF1,6PXM682)0@<DZG8G3+:M'79<G,JJ"^TZ%]>XSW72#NXR1Q
-M_V[4ZHAFZE1U>6(2]TQ?=7XB74TV"Y*XIX/Z.BU:7-W7SD-?V%.6&-S\-);$
-M[6+SU+!Y.A5][?0U-M=QZ40N=T(2MQTR)="U#<U)W&,&,*`.@'YB/^3%D[B]
-M0#XA?1_07ZGZ#D?V?1S1]W%XW\=A?=NS-;I23%?*TY4*1(LH72GIY^??\,R^
-M9_<_MVN1[JD]YJ<:=-<OF:]?U@T;S<,F\]&7=$=?M#-F-`\&`GV1ELPL#^BK
-M.%)#OT??I^HK#^\K/U*B;U.B"<D)3F:6'JF%-PO,YIM'ABP]*%G7`\F+(=*Z
-MGVZT0GHF=8--OP'ISR#J(VJ.O4>WMW;3HP;SS5;&6H=>6^J"(_HVF]FXJ2[X
-M2%O+M>@)>M77!HA<'!@OLB0"6K88"2[%9K36(2^Q<2*?N38`Z>6Z`+8.E$%?
-M%VP@:I.9F:J^]?3.UN:6O^G--\U$K7D;V\XVMC@;-[!Q:IN,E0V1?0\']JT(
-M[5L1UO>JNN_5\+Y7P6A(C>B;&]3+4(0@Y@G0M9.[KZ:?Q/T8,KAO)O]G:M_J
-MO6OJT_>LBKJ1$;4U*SHK+WH&)S:K0)>5'W>C,NZ&6W>CRFKB1/8E!_8]1[].
-M.U[.F]I]5=%G]=7!NT[M>V3OP_6/[YD+%41G9S5.CBK-BRLMH-^B2O-U-RIU
-M-]S0A1+]C:I&:V3?XZJ^1'5?8EA?9$3?XZ)D*A2NI/_^+^I/5%;3%_=^1GV5
-M%7>]P'(]+Z+O4:CA>KZN+E+=]_"^ST5>TERG-M=%A/8]&-GWD*B>5/4]J.Y[
-M<%>[-:==]RVVMR/V:(/NZ![=T7I#SH@AYUM=SOEQ?*/JNY^5'8$A?:'IQ8$O
-M"]`74P;WF*$V,'T_S\Y<.24-Z+L7*=0HO]#>]U,U*VA/<)\L$O4Q,KQ/"V.E
-MG]I8K>[]<8N,MKVLGW_&\]Y!_Q?QUUX,4O5>>>,D+U\F>I$4'*+TCXWM=%#;
-M.+W5(;WGU[+I>TCAAU3CGKW;)$Z!U"6@MKFI;96&;1G1VZJB4_-C4@M">G\*
-M[76:W+=,K6)D@H<KVK\`(UO@W]B,XD?LS+53TO):N@MKY!G=MS!OJ,?K?@<Z
-M6_`Q7/(_0=2S&3,1^(:O0R)ZLT61"NO-,:!,H#!KG<):%V0:QKJD^L-C5&VP
-MODZIKU-0=4$^O1/0^Z6J]]N0WANAO3UAO5^%]WZI[CU%)^AZ*$3U;7IT/6A`
-MUSKC.#UO]@OJ?5GH(*FGL61F=V#O=ZQ\B8[L/<]]4K'E*439UV(B>H>KT(GB
-MHBN!O1]+GE58GO9+9OX]F0E!LJNWO\4#3'?L>TO/']`R+,1-/?>-RX-%>!+^
-M=?+89ZB:^V^&]#X%%+`M@Z7NMUA&-">/Y=[!#H&]'5FVF-E/=#P=';K@G0=C
-M'GHB:V'+`ROOXQZS=DG]YA";)S7.V;+P3GY5+^W(B,E\XD3BVPL]JY:L")*L
-MEA]R^.4('"EQZQE0M[$OW?Q+^@_9B6G/>56][^=E4+72]K<@_0PR[)(O6V2]
-MC9NPS9PM>"E7WIN?*>-\9ZN5<Z[&TPF\826&8_+>^DQ9]*,8Z*'-,GGO.EJ[
-M6<8;CL_B;@3]]!;Z7I$3C?X>GZRWYDX]:R?J^>&N>IR9LL#>&6P]=U=PRL:9
-MS);?O`E[BY,F>^0M/$TV]RWN%Y$3M?UEHK9K=]66A6I[Z+>UT;)-W$WX)E3K
-MES;.',BUH41F9WZ6];YYIV_K,F5H(-,VRL:'M#J@]Y7-LCLUOX)JCO^G?GYE
-MXZ"_49FIV!B4I<P.UH:OX"BP\;\\6:K<&HQ?L*7`^\VJ@-Y4;J,-'U%&*[$H
-M)0<R!O0^S\$QK@[GZ80OR20DUR;VQ]2]SQ2K*D`ZVB28K2*$Y-@J0RO"Q#>)
-M`IH3@;>W2&FL,*0HM#A,C,=+L'@I)[XH@%1B$FZ\NO<Q6>^_B:&SG4*6#C(7
-M;DS$1^(SG\IZ&K73&+]Q4?8?,>Y_]C<+EU/Q8;TS0WMGDB8Y64_J:\,['C#6
-MA>OJU"(W26O$FZ>##HO^3<+M_?H^P'9G-,G,J._O2ODO-S\0WFL.ZZ4D!KG(
-M2)R\)].S?KIRBX>TDO1@2"^E[M5+ST[;N]?5"77>3HY6]T:/)_MJ)H^`D7L$
-MZL_5.(SON5R:-;FB7'V>H;VT3.,I+]<7Z(X-M!]QZZ`2]%<HL6/C8\'QLS;4
-M#QP_Q_I9<$6OA*_$-B;DS,+OQ;<F^"_B^`W'\]D_2HF^KO7][<A>OSMY\$7<
-M?Y4-Y>..V(XEX/@<[EP./A?GC<1S\3F0@6WG]CN_.7XQI"^'WSWWW95IO']L
-MG:Q\PD]I%&=&H.7,6=D)6V9Q[VH3)D'.N?.G-+&K\=@/\=BU^"CB_ZKL^F*;
-MJL+XN>><6V9[N]NN:[,BVV[O;73"MI[;"U268<<V<$N,D&Z#!TA6*(4U'>NT
-M-:!$)C!\(`[VI\Y-MDAB3(S!Q,5H?-(.C/HP32<&[<V(3,@2()J-&!\,YGIN
-MNU7`#?2E.?]^WY]SOG.^<]+V^\Y3Q:RJ:E-5-"<7JRKMUR-AJC\9]4B85XA5
-M_1'_1MP.%-+N,B':5*1^F;/`(<N@5;<_=?)>^RM3O^)_(=P-TC?+:0RUQG';
-M_OF7QXH&2\0_-4[#Y6HM2&&>=_(BXJ_+I6KJPI1YRM1W\>RU\5FSQO`W=!%Y
-M`S!#P@,R[C0SY)W5XX_WI7ZVG)GDTP_:;9GZ`>\`?=?>L.ELKS*41'A^]8F2
-ML=G*(DC?=Q;U7*DZPB%YR:3-&BY3W_UPIG#*=.82U)A"C2E31\/:OG+5#M*8
-M-SO/7C)?EWD1FD6J=!@VD0D1"!@U$P2=]!)]D'R!4#O=PA`9H,(@/P8RAM`)
-MFV4Z5JPL1Q*'LA,-F\E^X94>3\$QC_-53TT0G);&YI/D37G8^Y8"'<5,`76F
-M,]),ODW:\_M)(M5>&R&C\MO><PI<6PSU,9_1D54S\_WI@>G![X<N2VOHL*H9
-MBU2;7D*.K!_=L':C8=W&57P!AL\4(QUVG,*H2/0QEG+1:S\54%]](<VG7&F>
-M%ZUB)8.:2+?')-5JM-K]Y!\(-5!E@&1%8@F$VRD"(7)1R&N4$M.+=N=06]@Y
-M\@,P!)@),6>$+=HGZ3KV'GNTJW1SI\A["O;KN8PXT8IQ`Q9AV'];,B*WL0"R
-M#?3>CEAG3[L)UA=_U!C>?CO[FR#7-&_/_)72TT$MATXIJ:;EL3G==-Z97U?B
-MK=.>6Y%V.K>'[9FKBWC?<O@K2_A_]>;QWZZ$EXRK'ZG[YRO2GZQYN.[_Q-.E
-M,EQP9&[ETUJY[HNW^U]R7-DS8U2*E3)<3?/WTK-G!AR9]/_+H67/]#Z$_N(Z
-M'M5IKBR`/1-_*(W[YJ/#D7D_+V+Y,O-QF;_,WR_M7%[:!_5M<V2&\\2*'HW7
-M]3'8,J>'+"<LH@@Z!'NOQ5T/3'7@=0N]20U;3EE.[8CLN)GU@UEFV=C4KC45
-M0(_]?!=0[TLP(A][X80\H2"X!2J<Y*6N5'_;HTLM&,F,'4:?;HBN%Z+>PJC,
-M`H9.2\*UISL(NJ(*ZPK@(`B[`FP0M$<WFJ,;:(LA"%J.[PJ"YZ,^UK"7#M^Z
-MZ@7ZZ8_6E$4WF7O\>/JU()`?&Z!MZ_3:^2"03!.T5GK,7]CC9[E)6K::]3\X
-MF6BU<)86(+\`&=3PW69Q3[?;V,W:N(12C2K[<54_6I/$I4E4W8\]_:@LB<N3
-M)UI/MO7NS-XI``;(RH6TK_7U:..D@.`.N(9:DVU20'0'I*&=R5TA[=.<W[N5
-M\\M3;C0O!T%A3&31`BVP,8E%=V2P=`9-N>$\H5Z=]L.%;$'"\`Y9NF]`T;B8
-M<Z["5U?A:ZSP-57XGLOEC\OY_=SW*[#+N^F*P,8P]N/$;C?;R($M^JR+%-ML
-MCH<P.L!-B/001;;B>`C2DU7G$`\Q*1?E^4V>GQ[C>O?-SFG>"$!#1[@S'HD)
-MVW8)C5OKVY[=3(2*^I<BG0FA96]"V!;>)\A$\)*:]4J-XA-VM+32"O$)L2[A
-M0+@S<J1Z;SP2VA^.1PYVA5^,5X=BAVH\';%#84^VUW/8<^!P>R*6>*I2V$G[
-F(Q36JAPY(A"B5!-O-2%"%:5/%$*)6JBV`T>?Z/H;`"WDR?AO````
-`
-end
diff --git a/sys/dev/cxgb/t3fw-4.0.0.bin.gz.uu b/sys/dev/cxgb/t3fw-4.0.0.bin.gz.uu
new file mode 100644
index 0000000..fae25c0
--- /dev/null
+++ b/sys/dev/cxgb/t3fw-4.0.0.bin.gz.uu
@@ -0,0 +1,483 @@
+/**************************************************************************
+
+Copyright (c) 2007, Chelsio Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+
+ 2. Neither the name of the Chelsio Corporation nor the names of its
+    contributors may be used to endorse or promote products derived from
+    this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+$FreeBSD$
+
+***************************************************************************/
+
+begin 644 t3fw-4.0.0.bin.gz
+M'XL("'+5*D8``W0S9G<M-"XP+C`N8FEN`.R\"UQ35]HOO+,32+*S0RX$#,@E
+M"0%V[E%;+ZVU:)W:5N<5V]IA:J=E3+#J:+4=6ZU:000:[A$1$R"`U"O>B&6L
+M%MMNT"(J;1%:JX9P4>NE"EYJ;S/*_IZU`];VG9G?>[[?^9WO?.>\T;7WVFNO
+MR[/6>B[_9ZVU2<5XV`4,D\H9AE!A&):*/87U87@6QL?0W8%Q,*P78T,-1+&I
+M4Z<FH><+&"?LES@O2<7IO9\/A[L*2ZFF,>SB*.S;45C_**QF%+9Y%(9CM!J_
+M-(J#8\VJ.`);3(68QEE-XQXRC1MO&C<Q[C$.Y&A2T59X81F'6<9)+>-4EG'6
+MP`L-(0V421R')8Z3)HY3);*ON#@F9SI#F4X2QS:HY$S'`U%ZU`95)D3:0YE3
+M**$]C#D5SIRBK5^K@N9Q4C$%;VP05,V1X?#_S5$*_C1%T!B2SWOBK5$7X=6;
+M$_HQ5'M+&--"6_%$/!4+NI^%-PG>WV1;IT^HUJFRU-F:96."VQYO^U,J)LM)
+M;IA=9HV;R%LVYCN-D;0Q?CGS01R1L7@"'YK:H-(8D]BJWZ='<\U!;"W>4,:[
+M025CO(04^^_??__^5_Q`3KO^;8;3HZ3,C\2_?!U[-Q`N8,%;_F6<H3$4_GF\
+M^4P@WGPF$!>NAGMO+'.B#>)9*I9&3L8%3+8"W2%D7<#D/ZE`>"!$0?PNND-0
+M7<!"E>@.82+$4=I$"$D0'X_N$&9`'*7-@)`,\:?0'<)+$$=I+T%(A7@*ND-8
+M!'&4M@C"4HC/1W<(!1!':05#<0?<G1!V#\5W#\6WP+U=A7'K`W%N?2"NR/K?
+M,YYJ"<13+8'XZ[S_K^*_YM&7M;',%S28*A7P13O$5P3H?;X&\FM[P>:@`K',
+MT8LJ[+F;D`YC_DH#Y/TRP$O--R&.[D=C[PYBF(1AD(T*@I`$(0/#(KH9!NO>
+M\_&-R?#\,&J4!J,(6:SH?2J&+67IP(09+-/_A,K$0.!@`@S]Q\00E($\;$$2
+M*HZ"P$MB'S$2`I=MBX<%L[VR8CT8HD)`,I\P#-,#H9?/H-I^POA,1BP0I&+M
+M*9\+KQB4>)//,(-<)AT>CT`8+A;+'+E9$^A';"I;MR`C&9I#.=C.,C>QK8'Q
+MR8+@#%`M4JDL3`8S"#5##@PS8E`&1]3&,JU*U!-4.X8Q-SB8#)7=#&$]P]Q`
+M:?^(93Z&,6VF&>9']'P/QO<,QOD'S,V)KECF6%<?AG1&HK0/>JNR!C`$C+T5
+MPE*H9P/D`7D_!G6T\OJ`G#[4-+?=:4BV91B2WX;@A+`M`]J6(`RBL@RB>C*`
+M9J@)Z.4P/QF2O=!EYD=X+X4X9DC>#^$C"NY0]J,:2)<9DC^$M,,9?3#N?0!J
+M>J&=/G;F,Z!/QX`_CO7&WL,$<+\#M`C\\(-[%-#MA'`=LJII=NQ:C2KLA3,0
+M$"^!/GD!=-`+4R'88YGC1DBS`E^F0C-J%?8'&#WN82ASD/D'`_SXTJ<J[$_Q
+M$#J!ET,A[V$5QGO^O@YE3K0^R.^<W^A8E11+NA\4,`;`3/?#<-IP4&*T*HJ#
+MW0\J3A*;-AQ0VF^?49Y?0NJO@I8#,C;CXJ_#]*X'0M104*BP9^#^-(S--*T*
+M^UWO+_([HP%"QM`=PO1_%LC`?9I=A3W9"^5+?RG/.ZK"K)5PA_>3X1T/QF_"
+M=;B#GGT<]+&J586-A?2'#D(^H,$,NE9O56':U2IL-(Q_]#(5%@ETAP*-<L@C
+M%4"`>L5`DPAL`+(SOUL4"$%`!Q]TT22P!P^#;>"U(1I&VP.TH+O]-__^9_Q`
+M)GH#]Y8VX$70=\=6_T^I^+]___W[W_`'_'XE8!G_V>^?I#\1E_%?J3>@E_^P
+M94@___3K]/\K`FN3ANX04D#_O?S`\__987B^$61"MGHH_CC<,AYX_@1N20\\
+M(U"A^LVS]#?/@M\\\W[SC/_FF8/PP%!\..V7WTNEG#^5XB^7<E\IQ9[XKW#V
+M;WXO.3E_<N(O.[FO.+'G_E^4KRS%HIT<3RE>5<JM+N4)2H.$I<%$*5_T7R2G
+M$@H[\2HG/]K)K7;R5CB#WG8&KW3R5_T7R>EBV_>7XMVEW)Y2'J\T**@T.+B4
+MSR\5"$J%PE*"*!7].UJZH+`3[W:*H/T>)Z_!&?0W9_`!)_\#I^"@4WC(27SH
+M%#7^:UI`)OZHCCI3*8]BN@GZ/[]'F$X=Y2Q1;`@+9>[A5ZREX=PX8IUUXPC"
+M&B@_6X%U@&\\2`S5-_LEK/-7SY78E[]Z_@G[ZI?G_Y%?D`*S,3__#Q?[_\UO
+MD32((\-3PBM.2=!*'(W1:@P?A3\]"GL*+172F.P&&7I#S&47#I4W1"-OD+13
+M\"'YSG:!+@>C"GCZW7=7;R?<&;H<#E4@T._^B7A&L7BGR)WA7NO.U.5PJ0*I
+MX@;&3\(4`TSX``,1^0`3.L!L4-&I]%+EP*!B8)#*#&2X!U>6)EPQ\`\4Y[&#
+MOTC*HZWR@1\S57]5XZG8HW12Z,#WZZSKK,M&)88/?!<V<#L[N6'VQE3%P,VL
+M).WO&>V4QT5)&+TT?.!ZPEP,_B]?\+5RX`K]>F(.!MU$]:$;6Z\*8@\\__9.
+MU^@(GD$30VFB==I@?3R^*A<G=N''&T,'3G=*C/5\<SU&G^(.$.;S4\^I1+V8
+MKR9MWV6V5E1>H%?C"03VYG:IXR'=Z*#T=P7IU4'!"W&HB!!-;_VH[0_G5#YK
+M5U**^`PJ,W+@B'E_%7UF32>>RBDW[??TM-M/N1)5)$UODI++9;O6Z,[G4`3Q
+M]CIX;]*=SXX9.&A?KZ5&<P0.18B#LV2MENZC.RDUK^H[@4,F=N#1`U[7+<]-
+M$L>H_1[37255H*0.>-Y]B+Z8OOPAY<"NXH>[>F,&ZBK75SF+5&6E@@&\JH0^
+MKQS85K71M4$B2)(X9%09/_U'@>493+(/M\3C:Y8'4:/$5+SFG>5!^E$2?;QZ
+M]?+Q^HDJ"\&A"$S\TWB!8(H]8ZUR8%/LP"9+SB2LQH&U%SQX!6+T^SWZW4K]
+M`4^>=?GV"18UKW!,T6@Z2WA,U-#F>ETYD,=C9*ZE#D/56Y[ET0/OEKT)77&]
+M0;B2@ATR`8-31!1<+6_QCM-Z%4G6R41U`EUMMO"X2+<M.RWY#5H%;CY00[#4
+MI!6_;F.61PW\13FPD-Y<*#5K8G:O!B)T^S.Q4SRL+I):BPMWXKH#F9`HN#BJ
+MF*ZZ64CG-C?X/?0[[B!S,\?<@EG4>*#_ML&G.R4F`K,-_ME`!.-Z/"5DJVWP
+M8;UQF2\I):2TH^:49)&4.&V-,R[;$!HZ,#,56Q@Y,/UH1LS`,\!'T()E=*:Y
+M#T?TCT4-AEP<5;BGHCUWCV-?@UNW3?QV+K=X;_4>G1K7$TF$&]?%XY\_GYLJ
+M7#!K_@:8]2G`,*F(86;0]:?K6S^@9V5YZ==YO&G\U;B]248OI=2XD<`$@NEI
+MQ2VZN1B46[#A(_WHJB7;<!NSS#S:T_"4_<8B&#;#C,%R.8S=>_MTMW/\]2Q/
+M_5%W.]N^_CFZB5X7.:"!>:F\4-XK8-"85_<!L<;1'OTBJ;%-:1SK29OU5I?3
+M3'`L!":\,5XBF6*_618S$*$<B*`N35KHC4S_(`C--UT@BE4L_ED4B`>N>@(K
+M2`T;D"T_-MUA,ZEY[\X5?BRBI\8.D*5]U;U9ADV77)=Y?EG05+ST/'VS^ANR
+M.BF8D4G\>-,]N'@NZE1D1PWIEKWGIDJR&SX2G!%19=FG))`\G-A1,YR(Y`I(
+M)%@2TYB7;,Q<MW1^R1]RM4:5Q*`*\;89M9+Z&5L:ZQL-VI"#?Z&<?+V3N\7A
+MK=.7<NL=5"D_+S1%*"V4SC\%NL=#]^9K]2J)LO^"OH&K4X44U^L:^&AXZJNT
+MML%%G9*A^2$OEFNUVA#::]!*Z'K;X+/1_2=I+Z6)*9+OV1/@`JR/AS5&QO0?
+ME7R"`T?<9X?<=I>SN+VHHX&I;D^_&F3NX)A/L_SW"<M_]V[DA7H4;EE%&+V_
+M0.8'5G@O/ZQ0D2NS#>Z$ENY=`;T33*F"@8,6.!1"X?3CZ^^K&?)C5C3@?V3_
+M3D-M54S_UB4'8-XQ4ZV'?MW>_I,%=$T+^8.L[B^Z_3ET$<L4<;K]V<K^:OOZ
+M2%`NMD10+IVN=D&Z+!AZ?2IGK+*_-+A8$1P\+;J_VI'E61_=7[7/$=M?A436
+M6:@7SI,&^W$T;24)5KG%:J2LH20CY3FDP?.D@GFX)!U'%6TH?,13ZACGVI@S
+M/KK?65:64.LIGI"P2%F]R?%P587+!0H.55A>:$W8YEG^]3AE?V9,?T;!F+S1
+M])UJ3U5ECL%8%DJ?T)?)Z1JHM:Q*7R83LVS#6R.3#"#.&<#-95(88E899@M:
+M1=0!4!:Y&@*-1W3_W*.TA1500VVFK@\G&W'#-C0CPHNC'/[JC'Q_;D]#8X5_
+ME9M7U&T^+88)(9F`0J@&PQG5_SQVBV<"U;((6Q6)FF!K$1S$J?NUT)Z,7-K1
+MW.!VTV_G!IF:.:86S*S&0S[&S:B:E(`.$:\+Z!"!7A5L4./S=_#F;^><DH@$
+MTUN+T41V)?FL*>2,`-DQ_>-TYZN6E,$T)5O.>VB__=938"]8:['W+=W<G*9L
+M=@X?-\[-YDX`#3&6?EW9;RR5:]4\U\U-W]%_%3PEX\/`WGK79%%'*_LM_!^%
+M@N7X)GI$OX4Z[]G4E&B5@M$@ETOY4_$1_0;JD@<BI<WO/IPEMR]]JMOJD?GI
+M].5A#1.*)E"CPZN.N5H*QYDG2DTM`E,;R5<J%N>J35W*@^ZX$H7CZH9VURG'
+MY77]K@ZJ)&S#EZ[.==?,UU4;OBJ4T0_1O5Y&(IGYEV7CZ(?X;PM)-\$?1_*Y
+MDX^/KG(==M>/:7V$O8]OS41WR$/)%^>)JJK,9;(J5U6%KM*J[`^*:PD=<?T'
+MG3HJL45NB8].U&#I[B`0?W-\M+EE!-42'G']7A`CL[0H8Z]?T[6$`?=96A3S
+MM[^*9FQN-E(:\X`IMK!B,J3?Q2;;8/;(ZUT1U\_1FX%%"J6[5@4,A^E\IFX]
+M+MZ#FRX-SW%]=49A?>Y^L!WUR';LYY@//&@['LNE*$WPVVMG-G5:6J+,+3'F
+M%C5=$GG].)VE:XF+N'Y:WZ+1MT1&7O]*UQ*M;XFE6D8JKW?2FZF6>.7U#N)#
+M7+`"IUJT@!]LS/0`OY`_(GZA78!/;,PXX!N=*AB,P'P7;_XFX!NP!*V%]Q6`
+MZ&.D#PTJ,OSZ3O)#V;8Z8VV.CB!7(]-RW%B;;6\Z8IA;Q>J$3PUS/4T%=N]'
+M.K5*F)%J4DT\QF#\21QBDG<O_]L)1)U0O%=FOAX5$C)9,L#U#L`]I$ZDWQWU
+M=<V:KZ-,ZGA+Y-V5OE#Q7E+2EXH))W'XD[PKA.].()83N0F4AB]TD,OSH9U*
+M^HQ>'6=6J[UU!#%3'Q^W\IJ\?C=POC$^3@Q"B+HZJ5-"M],WZ9)2.=VGO+YZ
+M8R@]-1X4T?NN>GJ=8+9,D(*[O)1J8IZ5E\7AW1@O8*1"!T?0/Y[OEX+J(8J3
+MB/(D?IV4LDIII[9@D'=UO.3&!!$C)=Q2B1L7N_%B@PFZR>`Z=;S8I!#J%(KK
+MRP6S<1ZH)CI+!=I*,%4FF(K3>7`I;2K6\]RRZJ/0(A05C$\*<D@)!RX$.6K1
+MJB;R'DHBIK+6ZUCQ&#JC^DCA:%=KU7%/LTZC:'K)6ZY/4&S\U'72?2)K?&E;
+M\2/5GQ4]6O5YX03/%WD/Q\_UZ%NDN6/C7U+J6F2.<?'S/%2+7`N&;:ILVU3+
+ME.SL4,O=9,N3V5GRXTOS$G)UV^OT"<&$FQ0*G]+%QZ<E+V$Y%[$MF6X;#.Z4
+M1%TW'*U17C>86=VFFYL):`?!BGG#:.<S0#N?Y7\!'/O9.X>XN9^[/X.Q)P(#
+M?^_32MDY5864OPI/"?YC@2Q?2K>#R2=@^OO'IW5/M=W;,L2$+R`F_!6F)KEC
+M<1AOX,8%[\(TEPPS)-@E>JF0F'Y\VV?)R%CR9N`I(F7B%BY+.<N>H[%E'TB-
+MZC@3P6L])O;@)L0#[1'7[II+JI9<@LI>TI=X[)M?T(<3].UC^X"=E=>^)P_+
+M=K3IF@$?2UF5]X2Q&=CY,8/:4C1/=&TL\*_=N\.B5DGJ66Z43/+^)"R>(&&$
+M(D8D$4PF'%RO0R"8O(.1_!T`)KZB'?%PNB^<BKPK_CLIV,R6`N;?(<R;(*DC
+M"A-T&K[83:ZL@+:R"FV>([ESW<V.-%>K)56ZX$?`"B<W':<T2HLFM`(FN_K$
+MFD:45A[165.AH-?&7CM)7RB,K`PS@B+NK&X7M,E"VO#*4U'7CNA2I89FCWXN
+MWW!8:6CQG-P7<ZT3#51!6+'"E,IU2TF_3+Q88.I2I7!^-FM"$Q.4W@%S0FCK
+MXXZ$8MU[#E>"@"$EDJ<LP`^S@M&`FA:I8JXUNZ7#M:1PMMB8HYV2)1M!);UV
+MM#>@'/S)*8*`+CF`^$=TPS;X$AOIMPT^5_$IO1Y$\';U<0\P[IKW84+&5(95
+M*,HC[,E[Z=Y`S=!S(+!:6JSP*('(/8M9&MMS%47*PHB]>\UV;D%"OFY/&]E(
+MBD1/52;DS3/$QPOKR/)Y,%#T;:%#0"4H;8-&L*?48:4K'*:#R.,BO3!9'\X'
+MD`].0_H9H'DM3,W*3HBL<(07C]C!@/&$,@%ZSJD,<F%*T-ZT/55Y$?F*@K"B
+M>;;!EP-L*MJ-^C>D"XE5IVOH6<!X-N8QQ+<(<JOC`##I=S,!332_]+$%KH<-
+M:O6J71;`K(#?C9IHY`0>Y_AJP`>\-OO?^8#`S6G[/H:!7B15G+::"5XB@8$T
+M1%][9KXK:/XF/!4OTZNU]F>YE%JS$C0A_AI0`,VW>EAUC5`D\2PK#YJPO-&K
+M-D..E07CZ=OVYE7`\;0K;QR=0[@FN&+W%XC!30L/`84NKA.8:W/6'(&\4\V@
+MSL]/,JC->:^*=HT%796HCJ7"Q?;]4_GU#RKD8]7\Y81H.<$73!8."\%R1W1B
+M.#\]PT*IXUW*Y?F)FG#%NTK'"$WD7?Z[)/YZ:EQX&,9%56A6<[B)$P3+"8U<
+M$2</RTUP)`YI=CR&=KJO..RNJ[GS'&D5W^@NB:A+Q$H0`]P(7$^O7Z=WI`+?
+M"_)`.T\0U4D!XTE`:[LF@!+>,#).'0MJF!NGX#ND&Z+6J5P1CEC!&)E@#*Z\
+M)@?U"Y$-D7$:D;W^\\1PI38\/#Y\1/9\>MG&Z_2LBG['PJP%KANE`Z".Z0S^
+M-(YD*H>7,IY.XC\_?L0U;O9(_FS\W8B-%[.C^$BU`QS%%=\."G:`,L<AGA7I
+MZBWM@R$+VCBDW@$309%$N7+C):@R7CXBH.1Y1W$)Z/G+6GEX00S,BDZM)1ME
+M>QKU^[/KW;OK]`>R*4V8+E[K=5`)8<>7%B84Z+;[30G!XC:2))\R@H0^F\)R
+MBGXWA^4Y+>![>BWTAFY(`-*31'_A2.9Q@K/&TZFBG/'Y(RLNYD<Y(D45N*N/
+M/Q7PLS1X*A[^[2G>%ISW$`[QTEXHEA41/#8IB'W':\0K+D%'@$#H"`'N<RLN
+M`3MUV<;\ONDS8[S13&0$Y"'H3W@*'^0?B%E0"CKML_EEYY&LI!#]MGO@V#2=
+M,<4;;<SG=)\9W5N.%2<>5FR2\_F3!<NYCFCO<HAMDNU:GKA:D>X`TW$'6(<*
+M%RQ?!_$K#CDP4*9LAT83S@=&L=W;"_)J#)>9P^4@K^&AOJ4IO":#/#3ZVW5F
+MN9R^;93+TG;_F/>J[=Z'G1)]N-0^:RN=D3O?%5K1[[Z>N\"QT#W@NN%8H)=+
+M7=+<A:[PG8?<BE450;E2K]L=3F<!*^C4L0&VT(='`-@%?@4+GJB.!Y[B)RIX
+MRV7"Y?BZ\#S%A@7TJ/*%\3#6-8(4CF0V)^B1\8`4Z&3!P^.!/6=+@^IP0792
+MT,,3UAE$LZ7$;#QH#![V[9L"5Q)$-O3FJ;)C!74R2(_\=E'HMXL@LK$O3P]L
+MQ:W#RR]"Q7IY1-##24&SI8`G@(<V7H(T8"1((V8CE`#,-,Q#Z]**0AWSWAO8
+M<9`*#P)I6I6/FZ]S1+M(1UA5FFL>")=NNQO$2R!XBHJ/=\2X$RI>!:XC#\GV
+M>'0=V:``!&Z![G1V]+>C;/^X'=!YQ!)V3@/9Q&Z!OB1GR78P19F6DFS[S=7T
+M>Z'?6DQJ))$BOXS7)A.WX:9GY'"MNA7S;:(_J>IFD6'3]YM^W/1#Y1WN`%YU
+M&[E()=E=2PV'58:R;%^27AL,+:40'*1;V0F%B84YY3Z*(]LN5+/S*AZ>U],U
+MCNWT+"I<:ALT(/-"S`ZLR0E`3D53^(\HO#M%Y1-T,ZRT5W@PL)IT(_I;;M2W
+M7./I20LW1Z9O#\*2'9BS@(A1++XJ"L0#5[H]WTJ7T&OI7C2?=R2O<"1%XP4#
+M4N%L#GTSJ'@\GY%*EN`ZN524G62L'"0+QP=E3W`8P#-$,^N7!OGQ$#\.?B:8
+MEZBK]P`(D@[\P&Z]6N5*RE;Q7%($-6?CE)H7]"F^<7*Q7N"054\552>951.A
+M[I!)24$#4JCMS7D"O3J6T"M2,1RN/+#7>U#-GB<MJHD"%RYY/(D(+*I,,ZO(
+MV*LMY(!LWX#I?+;X,Y'I$EJTN!GT26"]@F=CB$52@J;I;08"2UMZFEY*)].E
+M@&:*#/3:=5;A68[PW?&"Y5*=7,$M3I(4(U0+)-"W)7[HM#A[/'\V@#;H*L\O
+M%2_&Q;-Q4^6@N'@\P4CY#![$X)"96YT$Y%<E%:IX55*!7X8&80_NF0S#0+9*
+M81@2-01T`G0@=#WQ21YT%WK^8'>-ZEC2"-T-,JI5$!%_]N^ZFS?<W8YZ\1[4
+M86!/?U+1K!/G]]=YVTQ=TI`58UW0&EF<[#I;?2;WN>)GW?)JJ=M7?6[-.L+;
+M&/^2U+[F!QO35BAWR/84"JZ2NA<X*WS<8JF7\3Q7_:QM\"RRYW;K[X.LJ1@Q
+MB2.<Y'41N1,,JHG"&=2Q?;O'<%F\1Q5PP-8)Q@C6?!"K5\<;(^_^=5<X\2A)
+M>EGSB$_R4L+,"5P-`?1U6&%6UMN8#UF,2X(=!W@[#&GU:AQL.R&8?CSG%TPK
+ML#V(:?.M.H*W[`,8G[BTYLC(JT^;:JN6G`8Q'#36>NCO[/0/T5>?=(P'<&U?
+MRS&&!X.X1EU-`H'>'7#1I,A%P[I-M=D>I;W]=--1B]I<-/^<2C(PUJ2.]?56
+M1<QACB#C6SX"]`."$YQ'"Y6Y-G>G8^ZF,ZZ.XK3JLT6OYD54^0KFE7]5>2Y_
+M_JI=D"<Q]JJ.[BL,!^EG,6A5)WTA@$.OJI&<(QPJ',*AVV.N/L+B/&5Q!)T!
+M0,\E'0)YU!SF#T4C6F\@_+GWU_@S+H`_J9BKAJ&2-?IPY)_,80!W'0>_[@"8
+M&.'SML&C8!&0/A#.L@T>A.Y7*,"2$HT"W?YA2(]=T^W/MCN_25#S/'35$<%\
+M&;)D3>QBBZ?`BM:VMWF6'WM4>>4?>6,*1]-W8J[\6+V^REELV+2Q?!/BOQ*R
+M*BEX0.8IXRU&3-[4B[BTE%V0SM;O5NG1@I"3G3.$(ODIW*BTO4;;X-\Z)?;D
+MK)@K'8'NTS<#*#?"$_;+`(PH?+4HS!%`N47S]A1V]:[P<;S7'1'Y(RKG>5Y%
+MOMJNNH"C5IY0/!^<-1BEZOFV0048.%91"X.0`A7N!L<73.;?SJGR%2G"$-N]
+M78'7@GL/8%=^VWWL^BK2G4UO4&BG00#>_HF<SUY&N1`/\INA+.@-(R%/X?8D
+M;,$2M_#1A`06CR*O[-#55D5=V<*N'Q4:@!%OV;W9(*-DOZSN+`6&(AM>O&\L
+MR>8<Y=F;ZNA.\VBI8U+"*&&I-.OQA'B>8)Y,/`^/OO(W3WV9%[1'CK+L?7I^
+MV=^PHSQ>EE+0^AP&$`EL1D-0MRS$@<=<V15^I1@BKOUTIJLIYLK.,AIF0%OK
+M\31K9RBUVSQ']ND_#8;YT)9D:V=(M67910]7Z0O'>@QYX\J-N>-->DSX1X7;
+M1(T*-5LI[:@19NW$D)!I/%Y2@E6GC8_6RC$J/BI!^YA^5!B,!B&<+-Q)ZBJO
+MZ^)CUUP>(:Y)Q<A)',$D0P&'/#@A!,"0FX17%G6L\LJ[DK6I3306,HDCFN3]
+M*,0WP5QYUQRO(AO'TC0U:J1^5(0Q/LX8KP7,!JC)F*`4#42)F"@J7F.)-YOC
+M+2:50A^O%HN?,&D5,.B?)].CFIIAZ%.X:^E-@9$'-:'3!@,L`!<"S8(OR9]$
+MTRF",?0F>M1O7\9<>>QH.Q78,:C--#3A8@:WL$N)HHNC<OM=SN+^_!L-3'5_
+M^B&N8\#5#[Z#F-TML-T;!$E27K%@?3R=)F;D%=.N;NQ>I*$6>?XBW]"J)GC^
+M^;WN>D=O_OD&AZMWQ:$@XWF.\1*['MG-KD?>^P!TZ#E5U)7)T5?B\]6Q5Y)$
+M3XOV46^OFT;WB@[A!H(C_'A\2,B4$Y<MG]TS_3!IX>3(]&E!F!498IX2K<T%
+MXH%KYF9@.#.!<=DR:3=.V)C/NJTB#4FOIPA,HA'@C\M%KH"YMU%08<ZD8\[T
+M\EC=76GZIB"LUX'5%PACY9V2Q7\7!9X"U_=TAW?1?72&Z`HJ>S+'#$4O`2U%
+MB!8GRL&+E:,R;#QP96F!5KFNP/90L8TIH6\*AW8(#MO0"060*4H5//]=T#J.
+M!((#ZCWMV4HDB$BNYC"'Z/J8R_<*MH1>_JQ811$9U!8QG1G2QJ%[$[>0)G44
+M%Y1-;9SJ,?H6EYO$'1Q/#$B16MV"DGH%8SB0%'MY0`3FLXT3>[D?WL5<OEZY
+M!9[8;*]UA"\6@*7#O#QL0>0[%0(;,P"3"F2'7KX8<_FB1<VK_TA7$T75C)1(
+M9H(M%CIPM.$G?D+HE^GRTH7'.<`,IAJ%>XM0.$W@EK%/8<.1\$`DQ,V)OMQA
+MKHE!C+,E7)BKT&T)8Z\*]AJJJXF.OGPR4#=(;>Y6K@/JY12:8BZWB*,5(7Z.
+MV(\J<F_-U4=?;A8*9P:Y$2G4EACPY:@MT>PUJFB[F)&)!S@`%2PUJJKM(+&"
+M`1G[I!Z.:`(1[@`GKC:4+3:RV,C][BGH7A"#WA3O-&_1A!0IS%O4[%7EV,'>
+M8W,-8@<'FA:S34=<KC75JH4,)^)R3=`8&<`C\9XG73M@/N)J-2W%W)"GN>\I
+M8`*"Q\@`S)JW:8XB&R`6/QG,]B3Z<FE<'I?+`!6QQPJA_1#(!W.Y4\!P''70
+MD&Z;.DXU,?IR/C<N2<*^.S:I>B<`(B@3$I,$A%;MU*DFAE[.XKBE:$&P#IY@
+MV(03DG`V(>+RJXA>%'FR<(NKKI@J2N#Z86Q(^E;(X`0<&`5LTI9UD)R[D3M`
+M0EKLY3>XP"E^#OLF(2$T(4&1D!"6&(\[$@I-+AV`>EV"./3R0[K1[%8`UJT?
+M[;'7G$&0_T/9>W6FVIR#,]XY`B_:=;5@0$^XI:7K@9=I;_1EP\%*4QG&^SW8
+M"IE9'4ULQ^V]3U)0T"'C.@26S&S)3)$E)SLA/OK$VY__P:")T6FB5[E!^K'C
+MAT(OA_VKU8^.I<"K#LJB"5YQZRGZIO*RFFZ.ALO%?&G$Y1#1=-&!J>[Z*F^>
+MJKK!];?J#V(N*]T'08-5O1_LQBL/>1JC+X^(NLQ%(/1#$MRH_4BO,:_2K4:T
+MR/T*BQGXTY":1)'?_;+?#C*K'\U=D@E]O4.GAEZZ/N)R?-2E`;,JK#AAY*5O
+MXU4BK1K`OM`T2[M@"F0Z5DZM_!FMRQU++]B?__Z2+K!W45@3K_X*YHC<L0-M
+MWR3P`>S-WZ"'ONNA[SLY=/WQ@_]^[Y]2V>D^C.9!]_D?02\XQ,^D^!XI%D<(
+MKTZ`V7H[TZM3D2,OM9)NV2ZWN23GM<LUX%+9/RE77OJ9_C+FT@F/MZK>J"+*
+MWA>`.MA?I$(;\PTF%;_\@^H#IK\1"5H^VHTMR3:VJ8QEV3;FSU^F:M$065#S
+MM#=!&V9)X(><8?<"9FL7E$`7OU^P":YY\S=`MM$Z36Q:M5&G>G'!>GB,8S/T
+M1EXZ:QZMUN]A2GI-H]7&T7$-?M,V]3O'@XRC-<9M:H/V1=O@3]`M>CWQ$REP
+M3>`?X@IW<H0_`Z:)V.D0"*?IM&&"*^2.,]"DW;D&#:,'\T?NV2/^"/<DIOW'
+M`3.!A_Q'8+.K#M198+]V=R75G$/_?<5W0,AJ77.VO?JMD9<.1EXZI+RTCKX8
+M?6E=U7J/LUA5MJE\8UEIQ88`C,,Q<W,V?<9\765NR48V\5NK21N<$(]7H!V^
+M//`M9U8GVIB\3DD"FVP;?!PM*K)\318(+,TY2WZ$]DBVO6"Z(?K2X@IHAH(A
+MWX"VPTJ*5>`=59<6)IAAT#=YRLP;AP;=T)R-'-^6[`08CGM7D6_F]#G3EGZG
+MGZ$T3%3236!81*T!0_9Y]*7949=F@]_Z=?+"YV.-+]U+/\9NBM<7D#'(EGTK
+M"CP%KOE)>BA;-"U$,DU$/B=Z*IG7)B57XY6CHB[-:7"@/0@@R5KXA$/E>1@<
+M4]?HXH1MC"4AN#`I]M)?5IZ>QETEHK^,NO2(QUM9#SUQOX_@,3`/4DH-A0E&
+M%=]SP$$9_T:X/K"H2)V6+[^42#*R6D9S.[MA#3X@,O^0C3`FO)1?4K-OS+<A
+M)60->C-TI*(U8#.U-D;/`M>"I`KYJL_CZ/>-$Y7^Y*^34X*8?#GMK)#2?2;H
+M3?]X>\_+^4E%4AN3&%AD[DXMER&<C_%\R5_7Y,E2@J?13MH+>`#\JD08P7KZ
+M2.RE1XJ2]EE"BJ;Q^D7GVM,[<;W_)[K]S6O+J\-2L1AW#;5;BF4X7'(Z`[M9
+M4!WJVFQ/?4$'B,3/??/:<P=W%M?PSQ"2)6/W-4KFD=499YPHLY1_1G30@644
+MN&14+;8`;>=G[V_3A?-<X2N^>R5W<[$BK7K+^P.6W<K<L',WW_X.U]_[*6W)
+M#A/2$]S"*;[VR$OA,#66J3`CY5;]%*Y%@ZTZ%:2\)!/4<93??%?^,)J?T453
+M1`-DU93"A#T?>1)RIW;0.P^YIYZ29+\GR2&W?E]6DU.[\;W@Q7CZ'(YWL:>V
+M<(J-V>9M2URM="MLS#*T=Z]7A>E4X8IO/@9OP>Z<8E2)>)\+#7U:X4=CQ8VD
+MX1OMGCJ"F*;7AIVHR7^BF,J=4K\#6\]#^HJ1-.*&!'YQ@N0@N>+1$:S?"5(P
+M@+9<:@-G26CU*8FXCH3J`G7L:80*[35+\Y\HI(JGU._%;O&P@<B]?G$C7I%(
+M;Q[Y3=4'GT=\4V4"('I#ADXLU$7N:T0>5TFF93(NV(%390A.2BZ.*JKW].;6
+M._8WN-WU;^?*'5YHU56/#M;$XQJ"`^R#EK=W#A.!Q+.*]:5[K`:P1?39I#RI
+MPUH95A'NE@GLL\I'T)L%@FE!.\:[%/12$_0@52R>;F]_(VU6@5FE6'T93\5'
+M+'@"]/5SU4JDRSF6O/!\99%6IY(4R"A5B->MTTIV.^H=E#;$TLL]V.AE3+U\
+MRT7N;G]]FZF+,5WDIV#E\R?#[-Z@:W*U9I4DZIO%YE*N2162[S25\DD8`F>Y
+M-I6#+7@B-_>[=Y"3%)M/Z37!J\N"*-6+]K69HCK27K-NY#<K*D:,_.9YNK=`
+M1OO$;>-,*K(JC.R6[;I&[<\QM@6O`-O+66H$Y[5I8=&(PMLAW6//]N:!L^Z6
+M$PYR37YX96BU$GRV7G,X8,OT8F7:/F7A[8+0(GEEZ#O7H&P&.(.AA;(BN4-1
+MK;2H^`7*'8Q%R^])GC/82]0B7"A<."UMWYO1WUB!7<1M(N-G'`-=8J0W"!IE
+MT(^H;Q;L;H0+V<BI&&%CGM416"JF@Q9O%LA2>!/*M2"!?<4RLU9BT8:D8I).
+M"4O/3:.<D\(;13M[G,=+Z5GTZX[U64Z8%!XQK5S!L\^BPCFGE^:'I7W"27OM
+MQ8!/:01;>>0-NIV^8-8&Y]\Q:?D5B164(4%H2`@VWAPA?/%9DIQBU(H:VHQ:
+MDKHIT6OM`L'O*"U1]/W2@<?I4Y'?<$($+^?5<APR;#,/K`=6P]OAIXC!>C^8
+M3X%_+)TI%D\3^,>9M.&VP3<Z)72S$=FE9?F4C?'0G6SRG``MV"F>\F+_R(L#
+M>DU,U,7K@2-?X`3MO@[6>2!2>/67,U_Y='G]?SJ;`2ZUX`I.(5\HUT=7*S][
+M%OG5:$RXUXOD]*S\T+,U;ABG`J5)SIDSV!YU\4L$N1IENQMU^[.+E<*=(MV!
+M;-N]"RPV"8Z]>"/L8H=XABS\XD"A*D\C]N'X69)X1J1/P'65=UH?6E(%4RW>
+M=6*;>>6%%\[6GQRK)S@`M(2[QHM$4TXV4/?NQ>>`:Q/UBYL5B8Q(W:\<K3Q5
+MW8F55X&I#Y>K.MI/2:`"+EM!VB<?V1CZG%5D)K<-A+21=NL;(6/EK/L5/)4<
+M]KWFZZ"5LDG',M[*B]4M8GTOUC8)E:BIO%_9J_>N'M;37CJ#8)=I3^8;H.B!
+M83\0/*V:`EZ,'-DX-AZXUGYQF*EF"7O``<NS,87@&PJ'?,,#-N;084L@USDK
+MG2&<I&BJ#VDD1=/'ZOP"ND_(^GHGWAT:D/M'S9P%HL"2,!L/7/-5ML'=J*DK
+M@:9.L':&1Z>.N)@6?7'>D;4Z*U=O%>BTQ(K-\B/[5^V$D2LYE\HNF;-[H4UI
+MR3,!0.C;E0`B*>(?%I7(H%(`P!(T<L2,D+JE-6@5@@&!^7M0HQW#`)GC2SW^
+MP3!$Y+,0$6S>$$:\='^_E4,@G2B9GL:40*7UC2+1-*C-7O_IZ@_6TDUD)5H*
+M"LAWVQ_-5L&JO@A+N[+UWOWRV-\1C0%N!^B[(+,GL(6X8/WI`.X)@%:S2F14
+MA8D'A)8^+5"^K\WXC1:`K"_5_LE>(-+"$LD0EML(Q]X,X-CHBS$VQ@F@R7"8
+M"9!U\F^K/T`$!;9!5QU'0/A?]Q$9'Z`!T9>`0'LK*P.!==,E&U\Y\?S\4DU@
+MEQ/U(P+1+9C>FLGBU\U$^?`R"-UT3L5]%(>1.EXWM/;%%>MJ9*=KUFP2&(A[
+MH@J.88N,GF54*>@OQ6VX40MF=!NHA5\VM3E3[I]-$%#&96FS\A,)#JBN5,SX
+M^0NYUJ&3E=C@4/7X][2U=2)HNS]GC7K7BDY6\J?!2-6?H%-!(1B&SE8>S;5&
+M7CC#GJX\I*^MBK[PU9)M:,W5U.&)NM!A/S4:0!0Y*".O"]"1A!DX>R0A#<'-
+M]:_$7OB<?F_DA1.4FK>ILZ(=G&L"@-JI:F?LA2,'*\VW/=HRCOFZTOR#)VW6
+MRH,3Z,T`I,@]X\7B*?;VIV(OT#$7FLP_3%HX):;^&;09XD1'*HEA^62?`E>Z
+M)NK"O@<T4D>-\&FDD0*'(NEV(5MG6L\N&^/ME.C"L?DESQU\M'Y&,3KAJ`,+
+M6J?72K:XZ\&0AE`WN7DRK\-\DT_=X6X9,%]GS'?X<P9/Y$L7>,?2]#HMI9+$
+M7*BB'N):5"&%5LM#?.2L6BNUMD'O\,!6!XY$6NGU8)EM@V[::ALL?=<*XVL;
+M?`TX+?+"VJ/.F`L9R@N9QL#"5R8Z*BMD<$O._:.RE4-'9=U[0$];:D,LV\3`
+M.T=K`O#&-FCHE$1<6`HPAFZ*OK"4TL1@;>@XGKX#'=4B3N#ZTYG1%Q8(+X[:
+MTX\.6N:UYW8TU.E/8RO=W*)35>WHK-87@7.6?;0W#YVOY$Z/NO"G?SJ,@?,N
+M"01V\MG[>ZASF'G#O(S8[@&Q/NE]YX-KPU)]?RTWU6+EFHD?3:!3XG&+E@CQ
+M<TQ:!0#$%,X2&[-]2(18WE7C>`(.W'<\HRU%OV@9:G#.X,*`]`-]\@L/D8=E
+MM2?`W=?-X+Y3L1J<_9@+X^S.OU8Y#V[R3C>4<=*2]Y^UGEMJ9$_M\K\(G-K]
+M7=0%H_*"D6H9MBT!U1VM`(8Z^ROE3==8K`++,LR22C[@'7R,O(.`/A]R#O;8
+MF/>C+TB4%T+HS9B79YX;L^<D`$=D@#690Z>M$X;/']57W?SEQ!ROR(L.S/V9
+M/3"7!AZ:`O@"^MJUU)\$+D5]"GXE7_IOB&#/<:@4H`L1_%/,7ZNAO7J06FTP
+M03S1NL5(8##V=/OP^C@GG-7Q*G3>D@>6H+D!#('>*J.(N!6;QZS:;#TY"S36
+MDHUV>_+3\TN?'%8I<YA*-,?D]-9F5I).!<XHL7$4G&`JE.<_!U.!/&+62+BU
+M.P;`1-B8O:";:%31'.8KH%1/"!#G(JU50U_Y1;&'_7*>ZN1LP`P&3:SR_/<`
+M9'90@!A(\`,28O4),2MV1;?N1UEU1,9.G2XA1I<033MU";$=S@`M^CT_038;
+M<R:@_Z`WB03VVJ8D>AL:)C6NUP8+!$\$LR2TKDT$0Z>),6BB5WK%K74!.KF/
+M@:_69D^>9U3C)#G]^)%4["N8E`4;06?N!UP.J#RM_9/(\QLMM55F0I!^;::N
+MUF-W/AUU_N`F9\SY;(N:1Z\+G"&GLZO*JM=7;X*9-]5Z*&VPJ<MJ4..F;1Z2
+M/6\6.+=OQ>;G+A_2&9P2&Y,3?7[5T7KE^95#1]@""]E`\?!"=K&_ZF8!.I[;
+M9CPM7NV6YG;#E+O]HOMKZ?#XRT/;RP9HX5#"?1YX&Z5V+!T:+^.RP+[UG'L;
+M;(PHL#?#KOG>*P[P23#M/*W2IP9K"-5K6V$,OAIQ_H68\\^!VZI>L`6>OZ'^
+MC(24/:R;&NC$GX>P,7#L9[+MYRRU.?N?3D='+B/0>MSZT-+URO-/'ZQT.=-F
+M/?,;??]*[/FI,>=_]U_4]R9HP2_;[C=V9),S1,;3V;K4X/GKP\VI82<NT9D@
+M5I0]&&>09"&>84$.M#5L!Z##[DZ)P<YJ',O<&//<Z/2K'/IFH!O^Y*9>B7G"
+M'.:U0#EHJDO&/R&`WBSY`?I=8X&^W"R//I^L/!])'S':@RMON6Y2J43%=^CD
+M]NV"5&2>?^!1J8J5:%G]H.'."+2(59N-OB?8EFTA.(B6>^/3N@_;!INP)EYD
+M'V.8RT[Y&YEU?DP?2?6QJN.MX7EOK[B9V^[H:'#K3F-OYW+S3U6T&_^,D[MQ
+M(ZB.P36@-V@O*'".SX"^U^A#_D#?M[JY,5#5CC9D7][(A"J1JKE?)>V^Z:"+
+MFQL<+GK%ST&Z9HZN!8,ZZ?5#M3)_Z)1H6,)-M=FF+J5I6W;KBT@9@WUC9;:C
+M?DAN@_7L9T,K<SET#9U*7VWJ:#K*XR=%G.?2^^D[DID<T71.A52H%=AKGE\L
+MF(ZRKIUNJ,%6>]/?.816ZE;.BE^#(KGS#T7,_S!<1XC>]DKIXA5>45/%V][@
+MIH^;BJ@MV!KO`D,-GP`;\R&^['U<_]%/K5MLS!YTO$:-?_X%@#43@+5307\]
+MQ$_%U,<G!59N^[;]RY7;5%#E-=)<Z?QWT0<C:=4U-N:U-*8<KIT2>WH)_>>F
+MCPU;,!OC@#X2]EDEB:,QL.``K*AP7BKV5VAX9%_QT9K8OJ*(/F?`CJ/%@ELX
+M@)Q?%@L^JNS-^RCWDX:Z\H]6NH.H_2'4`3%2U$120%(?0&:<EX:4`5;\GY&9
+M/]5^2L0B,[J=!6:?%EDC^]X(0<#L0P!F47U+V"]?%AJ;/?4S[5_,]=W4JTAE
+MWP+VVY?R4`'Z<@@`6A`+T.:S\O@J;5/VH>]2-AT9T?>*BQ;X9<$`9M!&<\(;
+MGH1%RH2W/.%]?VI:D7Y)5.9,+!70YZO6ZRJE:;-6''RDRTF?`E,H/!/PIXIC
+M^IZ/[IN-]J2FQ/AOIO\X]%&,^`%_:OBS&+JFP*KL>R*]1=/P#^$]$=1Z*O%+
+M05F[!K"'1K9#8]%D2SX161+813IEW_A_FHZ4E,-*O[ZF"$1@/MU$MP,H!Y4V
+MAYEC&TSIE("?170'_"R;C5DPA/O^4#_#H36H)'I`?8T&K>3@HUOJZ@'^A>@R
+MN'D*K]N2P==E<;<PEKN,)8L_Y^Z,`NG\)N#-S73[.K3B$ML7JWN(2ZE"BJS4
+M0WRTX&@U)?!L@S.'IT[%(K]R+;U9C[#?.-H)V"^R+XCVQO8%Q_3Q"D(!JNUM
+M*Y('OHHQW\[4;V:_C/@!<0QQ<50N[7(6TT7-#4PUG7Z56]Q432/__&#`/_^B
+M.LPMRU-T.0$FA-/>%*P[5Y8?7AQF&]P%*/->*Y+'F#Y);.]ML*4C^N+`Q.DU
+ML:?K\4,!!SQ65YDQY(`_%$]D;#NQW;2R\^6SJ2?'Z=GOVH)WL"YN@;[_GF7;
+MKWWPX87<!WQP<+G`"3\+XW-6#_:XR=C`[:@?VG/<$8!(?[,Q!\]9@TWD=K_=
+M^HXNT,B0#YZI@U;*)HG'R:F_WUP.?OCJ83\\]5=^>&I!X+I53Z=RSPJHOV<(
+M*#*.X`C&C@\.GF)/G4Y!/3F3^/]!+NR+3+\\!.Q"[N^'#L,Z<2NY@]KJH(!4
+MM6:(5#ICV#E_';0`6CL'GAX?2-EN8^H`Q:S?JH?>Z5"!(3_]8453>Y"?%/]^
+MK.XC`3U+F,+ZZ8X`&0M/1:9_/>2GB]%W(;_RTQ,TT;;!+&@UY4$_'7W(,&Q+
+M64::PQ0-G5WH?5M76Q73^R:[X748(`?MM3L;*C;0WRE[EX'L4F48R*W$CYO*
+M.*R9\1BL`M;2>([7@\IFSU1@.X9%QF25ZJPR@S44<X_&#H^VM_].;U5$]U9C
+M=:-7?!(DKA`:&["1O9YY-4'B2J'^*">JMR*ZMV)7A>@0_IHW2'<>$QXBR<JH
+MAEWB-H'^:+[A:$$*YU,;L]%DE8_L+<3.C5[1$R2J$NDO8O/H(%&UR'B'`S@3
+M:K<Q7T7W/G&T_OY6O[$/%PUO]9/`]W[@>W]!#_"]/_TXU]'M\@/V%`UY.Y<>
+M^(;UM&K!<^3P,$U%J?.?G07]//AHKM:BDIA!M!D+.'0#]0-F;8C)R:V?X?53
+M3KZIE+O%014P5"E_SC]>6##K83HCTVY,E43VZHU:KB$U)%=ET/(!I;I5+GO`
+M:V'WP`WE=JT]A%;1ITQV26`46;S*?9D4I\C">Q\2/R8KUA2HQ7^2;#.0$T7H
+MO(N&/($;$SCV]K^&]H(T&G:?H6?I"8SWXGB"F++B?1MY'#^YPWCO7APX(_=Y
+M-ND!GDTJ"%SSU,0YP4ID'MO*U18-?OP6FDG`]\*7QI/DE+0O#MF8P[3SL,FC
+M]JC<&F$)6D19=>3K5&P+[VF<-)#!C0+16'GP`&FO64A!@SF3@A?B"_\<F?YJ
+M$$:SQPL#9QC8>.!Z6+?E0U"H&<30]LZ4WQ+ZSX1+=(ZLTQ^N*U<#=$`?6&>0
+M0QR>:V,*0(HRA$-.D]?&-'1*#H-,<6R,4?_93ZA((HRG>*(L7Z/LZ<M3`_9_
+M!&'_"DVY>L6NT`#RI]L/Z]QJM\JC0>ME@/=1N0M(W_7LQ4;QZ,W*GKWDX^!`
+MQ$3W[$++]_Z`6XU@S[!;7=!>00>0E+O][=R0W%,@=^YVM+H?CX,/"_RV"^%S
+MEL_D7=:#X])FO4>O/^.DPJ6IF/7S%\^I*B-\="%PGGCQK#G,G[IJCF_IK#FX
+MG;75F\VU&!CKD)!IYG#IN53[YF<#]KJ&M==-A58QNXJB[&F-Z5EK4(F*52&-
+M0K-7:_!JZ/4AC23=*ZH:G__Z#J8B5N1(=L4L1U\G&7U)RIXW(WH."WX2":X0
+M1;&N:"ISD7"`LP8=?1MSME[9L]2B(MV1U2/!2ZR.$C`"ZG:.[C!O!4(4E.YV
+MMK)GOGU]+)59%=LS;TD.)-HLMSVZ<)G]UHMZN?2T4Z<*JW,+V4W-U\KA=4%!
+MC+AX6EJ3]^`BL?`5\5/)/+=4^#3N_O/PUIQE'I;V[-C?X/>_*'N>C^V9C?3?
+MK_%[--+CYWZ#WPNB37(IU;Q(I^*'.,@]@>;YQ5:J9='*']"W4M$]CQK!ZK?)
+M]K3IFM%*Q"&1KB6;/6/X:RS_KHTI[)0`AA).LPW.KG]:V:.75,B$$0IW3$3/
+MTL"@4>%B&+.8GD6`?6R#)RLCZ`_UX:'LV;RE!1%Z>:AM,([V1O0`C(V)Z8DJ
+MC$(*JB1SK[]HY-"WJSUA0]^NEOWS;U<M'1S+:>P!.]UYM%W9(QC9(QSRY=CU
+M>'3NHN>![Z_!G?OE:\L@74>([K08'$WVI`;Z%.@>](Q^+ZK[[Y2:5WF3IMW?
+M!9;0(KOON&X%3@,:M6&!XX"VP6EH,9#.*HR)[1GCDE5+[>W_X4\]N$B_2%D<
+MHULM6/SSXSWM9P*[B>Q>8@]M;%.=68H2%)!`O<Y-7"VM#NMQ`L)S?6GJ4NTM
+M.E.CWZW";CJJP]%NXHBTY)?RE+;!R72SLML7VQ.9'VE2$<$?B_8WHL]>ST=W
+M?^[J=5^H[*-IHY;_P/01M?C]^3.'RT%5^Y,,X0K:.V=PKT&N,,OEMGO706<$
+MWL$;7]*<P;O#;SH>_/L%^6/H&B/!2<5P_6X.VCAIC13LYZRHOQG0R<'#&/#[
+MZ.Z=([N_.U;MJ!?NY`KS..7UAE/XVQDCHKJW1G</Z)LSQ0"_6X:.F0%=3;T`
+MPD)"9IJ_PF%ZV742_4*GLOM\;/>%J.Z+,=V;(KI+3><S-951$C_'L)[]%+*1
+M(Q+--&S$H0I'1KD[LMOG*B]W#?E%1'CWF:CNG!'=.1*3C-]-5(631M%^/]]/
+M>.2FR-(]GWG"C)]Q]+330*\7H$_[\0=WLN(."T1U40;"GA+\4GFHY?68#=)J
+MV9=+0[N/YH7EA^IKL/B)/:(Z*E^:$OQ\ODRTBRH(-X3S#"U1R"&R8F\<#RM0
+MZ)-5AD=49!V5$CR=/D7XQYE^1S5,W3][_YBW)IS;O[Q87A3N"&L8\#I<857A
+MZ5=KANP:VK5$8QG9_0=H)X63QZY,SV%^'MG]/'U4Y\5;=Y.5.$G.-/X-)]%8
+MQ70GQW9O@L%!*&3X"]'`?$5U3QO9_11[\#13UX3_=5\PO$*3*>]V1G>7*+M+
+MQ'LXI@.9`A='()A)?8J#7)1\KFE[:&@^Z8MT<GCW^,XDG2I8T?T0>[3CNP5H
+M&3LWJCN+?`P8S-#\O;[YCFDROFKSXPUUYN8?]2UWUNP,TC?_`)%5.PW'JL$Z
+MP!-%<(A=..:(%#JX^I;OX242\[F9`H9C83\2%%\<%:]]<65CN*7Y!UWSCZ*]
+MI,1-AKA)H4[16IZ*?4\<)`64(BWC.ETB/(L+A3-UO\/9CN91!DWP2B_K?6EB
+MT[ZP!W@_N$U@:LY9\C40>PJD(+I[HMW9JNP65ZUW.2TJ8N,&,3KBX5"A58-2
+MBXI?4:8OXULV$O'L^0M3,[C=*E-+=F!M,%#EMC9]<PY=OA(-@$O?G&VO*:%_
+M4G8G1G<GCO3?BO#?KMKLKLE5;=SNV;IQB^L]=O$(![^8:LY&?]RC0$6U9%NT
+MP?'Q^*JS>-`<W/[QP1%^_S_;VX"A2MOWC6W0!:0HNP/CA-\?)Q"8D?Z_ZR;C
+M:&;G9B)7=M[0O*./M?UWC.PKT]Q,I,KF#4&\H3F-\A]%Q9HSH_WTLGU"S2D<
+M#_"$LEM&ZF0['1'=TOP,QUKA3HZN)7/56FYN?5L)>A_5+0(A'I*ZKY#4Y3DU
+M7480GO*N0-U$5#=/[O]'J'\'[0COQK11TC#_]K-)^I>68C4YY5+,F>=6ZN9B
+MB_-F2JIE>)60>X\P]V)!C%`R(.,-$/J.189;%-%*BAQC\WN];KC5Y0D:2>J6
+MKJ+7T+%8?WJ1Z`0IJ!OKZ!,*I\"=<)"NOEUU^M.+O>4`[,I3A]=$%/X+T)4P
+M__E0O\^1O'/,B:(YS!F:UH^^8QC]?64R,!DU^@?SZ!]%^TC!`"D9($/,BE0,
+M3"7Q<2#*+TH*9*9G1?O?.K;$=`J/];\%0[]J\V1J](_&6DR_FZ<?>V=E;I!^
+M]`\06;7S]Q#!U#S$Y"F1P<#D8[^'=&1];F<*W!S=#T.3!\H3\..JG=&!OQ$`
+M;Z$4]F)D<!V7>"!;_;6*I+8#K4?)9@0I(4WS(0/N*C[`Z:#-PYG^T]^1">AI
+MZ/_3]"@>):361EO^SKR6M3K:7TZ:97O]^:?RVF/\4U=M?M3165#34%?>OC*7
+MF]=1WKYJY_-Y'4"(KP:;$"EQ<\M/Z4^S?\+@?*;0P0%6HX8URS#]V@#]AO.(
+M_@Y:/QG''HN4^+GB1H[ATI`W?4K200]#=D2?SZKPJ^D,L-'`]#K$]#PWH2M!
+MZT+.P+I0@/X8_TC2(-O[(5HS(::PA_*SC;79]"@[_4ZT_U6%7[;G(]258\RJ
+MS8\%>H+V5G)Y>1UP_U_2EV#DH-#M\58^9<4BNJX+GI(+'025.;4K692!C>RZ
+M1CXCVU6GJ\T2_E&DVY85Z%>@3$17+Y2!D@\4B.KJ)J?+=C?J]V<1+XKT![)8
+M&TBO)?6RJ*XS([N^V@UUY>PR&.J3N&X!.K#KB'Q[+5]3FX6/%@WO+00;",RN
+M2@WWQV=+0[N.:<&%_5!@V#-HV&_6[QD414R>O^')^24S%ZQ/2<5FHCW7QY#/
+MJJ_5F^K7$\3OC?5._>[;`O;;K#>//V:L+S'5;P@\KCQN-NBQ@V.T,SAO/+T3
+M&#+]DVEA?K&-:81JXA^Y8F/VQQ-V&[/;1!#"K\;;F/>6OB(**E)8'N/$WUEO
+MON-<ND]$.!2ZQSCZT@U4:0GO&7S.X`@;LW.X^%]9_E!V_1S9]1/^M$*W33;T
+M)S0RY;CP"8E;ALX>C8FDX\">:"*%&BZ>R5%V]7,UN"9''E@O5]'U='*.=$17
+MP;'_*)4INO)3L5'^FM<V\Z.[L@O;Q7O(KM217><CNUIBNC*(<U3(:Z!N']I5
+MYT-KDAU6NS7G=++!BG2__HW,`MG*UH7176^)_T28.K1!+^*B=JRSQFXYK==P
+MB#P%KXYC?[S6.)I#%BAX;1SR)+ZR:4,J]G2G)+U:$O\2!P!P6O5W=/+9FO3M
+M-Z*[TNCZ0$7GDMEZCC'8+!Y[.$0RP*V2V9BM4`9+YA'"".1.UMN8%:\U\4=V
+MI>37BTZ20%%4ESNFZWWR,!7=]?[>MLI3-B:>':_8KIDQ7<FA77OE7?OH=KHI
+M%1-W2O1WDBQW.>;KTC?Z7C&<SUDS.A)<W]?V7P9'0$!XZP*P;57^US9FH%.B
+M.Y\M_E!&&!3V3XX3^LG'M]37>9P-[M;=]3O8>Y:I#--5\D1ZM$VWD_0X/>M-
+M9;BIC`LPU<:4C>P:93R0@18^#JP=QB^3Y5TZ760S19@BNQ)&=A6&=A5!AKC1
+M)B1AP(DUF$B4)')-H.)C%M2"9.?,86:%=:V-[)II>.0*\0@GA<L8U#%$(]59
+M4Y&<PNWMJLE/IFO(1D$*MR>L:X59C4=WO<8;P,6/4>9XW/0"EL+]FLVSF<US
+M6M&UE+[-YGI=,IS+DY3"_1PR)=&UNQM3N)_IP3%L`+O)?@R+IW#[@*4CNEZA
+MOU)VV6.[TF*ZTJ*ZTD9V/9JMTI9BVE*>ME0@FDUI2\F@H.#=+^U]>=\K.V9K
+MG]]E>GZW]LXUTYWKVHL&TT6CZ>@B[=&_V!@3&@<]8340W#G,,_*N*4<VT^_3
+MCRB[)D=U33[BU+6$HP')&3&'>8*>$>WK-ION'KEEN8123>U*[:41]O9/C]0"
+M0&<>#=PF-.^CO13D*M'EH%Q:5/8A9#V(S<?>I\\TGZ<'C::[S<446[6E942L
+M[[J9C9M:1APYW909US(D4/IFN:B.PW9>WR)O>H]^/T`.U8)V&PS#V<S-<O27
+M;%KD;!4H@[YEA)ZHG<.$1W2IZ`^:3S9]I3?=-1.UYFUL,]O8718V;F#CNFW2
+M4Y(CM>P8"&W,PVA,Y+Y+H5VRZ"Y99)<\IDL>U26GD^CG1_K.A_L\%"%(0-A@
+M3(_JC9-XL(8<X>OB)U*[5NQ]>\\;=<LM"1F6C5EQ67EQ$SGQ607:K/S$A*K$
+M!(\VH5ISEQ/KNQO:A4$5B_-&==Q,A_(,&>%K$S+4KA_W_K3GYSI&4Y81EYWE
+M'1._K$"[+%^S+$];5FTIJ[*4>;P6^F>E[W:$[W:,[W:4[^-(WVUR$372=XL4
+M!^\X4W=NEZ^J7K<URY*5%^OKC_'UT^MU6?GT5CT0<2"6'"#-!Z+-!V)&^J[L
+MZR$;2:7O<J3O<HSO2MVYO7YJ6ZOV>RS^R]W:+W=IOZPS;+MDV/:-=MOE81NB
+M]/6P.BXTPM>87ARZ6"!"`^T9U->&IN_CV9C^3HG<]W6D;[;2]SZ;\3':*S@X
+M@=JBA*?,>+MJ*5T_PK<GUO=1C.\CA>]SVDH_7U`3[2NJE$;Y&A?KII^K?_]`
+M\%_P-_X2IO0YWCK)RY>*_D(*#E*ZWP]N=U!E'%]-A"]S)9N^BQ0>HKR[]I2%
+MN`02MX`J\U#HSQMEQ)55Q]GSX^T%$;Z22-\8H^>>L5D,AH:,4K1^@71*L+<1
+MQ8_8F-OHG)S9<Z^B%@P4O19,5+W#]1Z06MP/EZ(!N!0Z0X0SUWP=.=(7*XQ6
+MF+JNF+9J35OC35LUIJUQQ@:TYFJIO&+>'&_9JK5LU9BWQ@V/E=RW5.G+C/!M
+MB/2],=+W>I1O:;1O"9T<?YJ":N-;=.AZ0(^NVPQ#T*HD*-2'_@`,-1:;PVQ1
+M^-Y%HC,E+M:7Q7M>48KDQW([/L:WMAJ=YB[J5_C^&/*RPCPV:`XS>PXC07K7
+MMZJIW@1L_G?&?/I1-#>C<./I1X84Q\-X"MXY9_#(G$$/JNKW=^-_,,T9_-/0
+M/G^H;W&6-7[*LVTOQ$7.>.]W\4\^FS6KZ?%ECW"/64Y)@IX@2D9[G]@PZQ=L
+MI/3]V9$1G_GLB>1W9]4OG[<T+&2%[*`C*$?@2$U<S0!$2%AT]]7T6]G)::_X
+ME;X7\C*H6DGK6D@_IP(C.N=:F]3W['JLA+,!+^7*?*9,*>=;:ZV,<W,4G<2[
+M&([AF,PW/5,:]PP&-K-$*O.-I-4E4M[%45G<=6!+UZ)O3SEQZ.\B2GU3?JDG
+M:KB>6P_4,R%3&GKN'%O/@Q5T6CECV/)CUF-K.6G2I]?B:=*I:[E?Q`[7)ARN
+M[?8#M<6CVB[_NC9:NIZ['E^/:OW2RGD"<L4ZP=[\*/7%_(HVZ,BYL^ND0UV*
+MD/LD)=)?:I:@FCO^$YU?63GH;X9F*M:%985GCU!'+>4HL*&_!%H:OO'_J>SH
+M8YJXXN_>NZO8WO5:H!WE\WK79<A7WU$4)+H"Q0T2,QV"_*&9=;4JJ5HF+,H^
+M=(*ZZ401T(G"9I;,/\S,(&;+_EL5$W4CINCB=B>+^!$R9^+`;/MC<>G>M5`_
+M!NJ:IKGW>[_/=[][O_>N=[_?"_!7["7]G2F)UQ@T@.&85;("T4H1Q,1KD((`
+M.2#MF+W!9&01YG0@0_WG0$HOF2FQ$>!>&TOA3U-[T[@'^GTA*A->.,.'0+MM
+M?^J!-`[*1B#SE+P_D;4"(Y(SU'LF]7>.*!N>'?6#UM?;EL(QN;5VYW)-SH#<
+M5K>K'J"GY8X,9LMIZO54]3J;9V9/LG,^3Q]RYYY(=YS(,/2Q(8'KG$^N*>DQ
+M0/Q9E!&R'E6%%9%[L5Q@NF"^.UV]E*9>-.:8#;GZ'XI:^]^;;^WJ9YUL:-2F
+M7LQ0O^=OEIPZU1,F/./@<QGJN4EPC#,[2#;C@X3_'N&CW-,]/4++'L.>.7MS
+M+G0?%?J/'9NSSW'^^H7!/@=AHF4%!1<G;8'P)M;T@/!6]!X03%:_FF4%;66[
+MR^%<>*A,5T<QM^59T22AVIO2L1R>ZHF'.+`.38>FX:$Q?+X,0@^JI&`EI,=D
+M!#T$(2HGWL=XF!?9&`935/H(TJ1^49[1M3R\(B2K'Q+)K>6[RKK*T2,RA_E$
+M931)G=TEA,S4PQ2G8%P&$S*X+XOZX\1`L]J2I+:@,3E9;2']6F92=8M>RTQZ
+M%9N5,'T/.ZS(%WE`^0@H4?7%/+'+U&G6_%#U/NJ'F:J?OXG9V[C]!ANAB%?V
+M):T9;^E-[$P1_XZP$3I+^0F$:)ZW\2+B;Q&W>^/+(6[(T'[VP&C?#2Y"\;<U
+M%7D=X"#F`>ZS<13^++4OK3UTW;3_#!]^TG\SU6K>"MI']R5I8G^A"`O_>&IK
+M2N^-O$1(]J4FM31#+6:1/.7:7(3.5,M/C1B'#/L'880R1JA,M<0?>3-+Z0=A
+MFN=L!P:Y6S(O0DXD1OMA%1X0@4"C:HR@C2S\U^'O$%I%+F6(=-!%(3<-9!I"
+M&ZR6":Z8EX4D%D4'&E;C-<([VYT)VYRV]YVE7K!7ZAWOQH?DPX6?N*`UF4H@
+M@75$&HG#I)5_M&%IP>@1W",?+3SF@CG)4,/YEF#FCXQWA`\.=U[NNB*E$[3\
+M$9.T(#Q%>:2H9V[./%WNO%E\`@U?3D8:V0Y"1E0B&\F0G6Q5B(+:V1?"?,@>
+MYGG1+.91J`HW.@W2@@AI-K[T%T(>8@R0S$A,@7`)H4`(GQ7B%H7$\*3_694)
+M9@S_"'0UU(`8<\9ED:_#9<PC?FE13I*M$/["1;NUFE.L:*9I#RU"O_NNI$<.
+M?0)D/&2O@1C;]E4&6)'<7^E?<C?Z/)E]F+<HQT):V:[IJ$.N4-7TM#';HK(_
+MGDFVQGOGC+S#L6O9HKP[25\\'7WS%/U_>N/T@9GH)7WJ,VU?.2/_,Z7/8?MB
+MJ[(W7G;,_EC^X^>I0691%A#I,U4@BTY!\8]%D:U*X_^K<691'$_A/VE#FL9S
+M9@4L2M)3>3S,]TQX)5B5JKB*6=.,QQ7^"O^XMF-Q;9^P]^<_K<J\.+/$9]-K
+M]NB2E)PN4ZM)%,%ZP;+3Y*@`AC*PVT164H=-NTR[EC8LO1.-@U%AT5SA]O1L
+MH.7B?@!(],4TPJ<+X8`\X$*P'+I8J9"$4NT^!!I<1B.9LL!`B2=0)`0*C0&9
+M`109EF;[RD8OV!1P,?8:V@O\]AK&"U8%YG&!N02B\X)E.^J]X+5`,:-;3=`7
+MS7J+_+H#I9F!^=QV-SW\@1?(LP\26*[6.NX%DF&`M#*VN8W;W0RK/0UCYK27
+MR@RD:;Q!#B`_`2GDN;107-GHT#<R26RSJP#E==#Y'2B]F\[H1@4=M+,#97;3
+M6=VMM6UU.Y='UQ2`!LC,^B+GM?-1QTHU@J/&WE7;72?5B(X:J6MY=[TO\DTL
+M[OT6B\M##C0N>X$Q*#)H@APP08E!]V4P-?<,.>`X)E&=],.)Z(%$P_MX:KT!
+M1?UD3<#LXK+LXLKLXJKLXL6Q^GZQN!_[[P=N*IQ_56""-.VFFU<XF$H6E&NC
+M+A+::J[)1Z.U[(!()D^4E-SD@V1&U20T^:B0G<B\$)>GY1Q?<4<_S.L!\*SW
+M;VAJ"`JOU`N5BRKJ7EV(A>R*MQLV-`OU_C5">>-F02X1<$EI84DI=@E+*VN%
+M0HR+A>`F8:U_0\/6@M5-#;XU_J:&=9O\FYL*?,&-I<[UP8U^9[37N<6Y=LLJ
+M5X%K3IZPG/0W$+):U]:M`L9%!1B3KY`OR*2!R<=$`U"P[/+J?P&C)[&/F'$`
+!````
+`
+end
diff --git a/sys/modules/cxgb/Makefile b/sys/modules/cxgb/Makefile
index f6147e8..fa72d2a 100644
--- a/sys/modules/cxgb/Makefile
+++ b/sys/modules/cxgb/Makefile
@@ -6,19 +6,21 @@ CXGB = ${.CURDIR}/../../dev/cxgb
 KMOD=	if_cxgb
 SRCS=	cxgb_mc5.c cxgb_vsc8211.c cxgb_ael1002.c cxgb_mv88e1xxx.c 
 SRCS+=	cxgb_xgmac.c cxgb_t3_hw.c cxgb_main.c cxgb_sge.c cxgb_lro.c 
-SRCS+=	device_if.h bus_if.h pci_if.h
+SRCS+=  cxgb_offload.c cxgb_l2t.c
+SRCS+=	device_if.h bus_if.h pci_if.h opt_zero.h 
 SRCS+=	uipc_mvec.c
 
-CFLAGS+= -DCONFIG_CHELSIO_T3_CORE -g -DDEFAULT_JUMBO -DCONFIG_DEFINED
+CFLAGS+= -DCONFIG_CHELSIO_T3_CORE -g -DDEFAULT_JUMBO -DCONFIG_DEFINED 
+#CFLAGS+= -DINVARIANT_SUPPORT -DINVARIANTS -DDEBUG
 
 .if ${MACHINE_ARCH} != "ia64"
 # ld is broken on ia64
-t3fw-3.2.bin: ${CXGB}/t3fw-3.2.bin.gz.uu
-	uudecode -p < ${CXGB}/t3fw-3.2.bin.gz.uu \
+t3fw-4.0.0.bin: ${CXGB}/t3fw-4.0.0.bin.gz.uu
+	uudecode -p < ${CXGB}/t3fw-4.0.0.bin.gz.uu \
 	| gzip -dc > ${.TARGET}
 
-FIRMWS= t3fw-3.2.bin:t3fw32
-CLEANFILES+= t3fw-3.2.bin
+FIRMWS= t3fw-4.0.0.bin:t3fw400
+CLEANFILES+= t3fw-4.0.0.bin
 .endif