cxgbe(4): netmap support for Terminator 5 (T5) based 10G/40G cards.

Netmap gets its own hardware-assisted virtual interface and won't take
over or disrupt the "normal" interface in any way.  You can use both
simultaneously.

For kernels with DEV_NETMAP, cxgbe(4) carves out an ncxl<N> interface
(note the 'n' prefix) in the hardware to accompany each cxl<N>
interface.  These two ifnet's per port share the same wire but really
are separate interfaces in the hardware and software.  Each gets its own
L2 MAC addresses (unicast and multicast), MTU, checksum caps, etc.  You
should run netmap on the 'n' interfaces only, that's what they are for.

With this, pkt-gen is able to transmit > 45Mpps out of a single 40G port
of a T580 card.  2 port tx is at ~56Mpps total (28M + 28M) as of now.
Single port receive is at 33Mpps but this is very much a work in
progress.  I expect it to be closer to 40Mpps once done.  In any case
the current effort can already saturate multiple 10G ports of a T5 card
at the smallest legal packet size.  T4 gear is totally untested.

trantor:~# ./pkt-gen -i ncxl0 -f tx -D 00:07:43:ab:cd:ef
881.952141 main [1621] interface is ncxl0
881.952250 extract_ip_range [275] range is 10.0.0.1:0 to 10.0.0.1:0
881.952253 extract_ip_range [275] range is 10.1.0.1:0 to 10.1.0.1:0
881.962540 main [1804] mapped 334980KB at 0x801dff000
Sending on netmap:ncxl0: 4 queues, 1 threads and 1 cpus.
10.0.0.1 -> 10.1.0.1 (00:00:00:00:00:00 -> 00:07:43:ab:cd:ef)
881.962562 main [1882] Sending 512 packets every  0.000000000 s
881.962563 main [1884] Wait 2 secs for phy reset
884.088516 main [1886] Ready...
884.088535 nm_open [457] overriding ifname ncxl0 ringid 0x0 flags 0x1
884.088607 sender_body [996] start
884.093246 sender_body [1064] drop copy
885.090435 main_thread [1418] 45206353 pps (45289533 pkts in 1001840 usec)
886.091600 main_thread [1418] 45322792 pps (45375593 pkts in 1001165 usec)
887.092435 main_thread [1418] 45313992 pps (45351784 pkts in 1000834 usec)
888.094434 main_thread [1418] 45315765 pps (45406397 pkts in 1002000 usec)
889.095434 main_thread [1418] 45333218 pps (45378551 pkts in 1001000 usec)
890.097434 main_thread [1418] 45315247 pps (45405877 pkts in 1002000 usec)
891.099434 main_thread [1418] 45326515 pps (45417168 pkts in 1002000 usec)
892.101434 main_thread [1418] 45333039 pps (45423705 pkts in 1002000 usec)
893.103434 main_thread [1418] 45324105 pps (45414708 pkts in 1001999 usec)
894.105434 main_thread [1418] 45318042 pps (45408723 pkts in 1002001 usec)
895.106434 main_thread [1418] 45332430 pps (45377762 pkts in 1001000 usec)
896.107434 main_thread [1418] 45338072 pps (45383410 pkts in 1001000 usec)
...

Relnotes:	Yes
Sponsored by:	Chelsio Communications.

8 files changed, 1903 insertions, 259 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 79452a9..1c447e1 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1127,6 +1127,8 @@ dev/cxgb/cxgb_t3fw.c		optional cxgb cxgb_t3fw \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgbe/t4_main.c		optional cxgbe pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
+dev/cxgbe/t4_netmap.c		optional cxgbe pci \
+	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
 dev/cxgbe/t4_sge.c		optional cxgbe pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
 dev/cxgbe/t4_l2t.c		optional cxgbe pci \
diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 64b71ef..d22bc17 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -54,6 +54,7 @@
 #include <netinet/tcp_lro.h>
 
 #include "offload.h"
+#include "common/t4_msg.h"
 #include "firmware/t4fw_interface.h"
 
 MALLOC_DECLARE(M_CXGBE);
@@ -131,6 +132,7 @@ enum {
 	RX_IQ_ESIZE = 64,	/* At least 64 so CPL_RX_PKT will fit */
 
 	EQ_ESIZE = 64,		/* All egress queues use this entry size */
+	SGE_MAX_WR_NDESC = SGE_MAX_WR_LEN / EQ_ESIZE, /* max WR size in desc */
 
 	RX_FL_ESIZE = EQ_ESIZE,	/* 8 64bit addresses */
 #if MJUMPAGESIZE != MCLBYTES
@@ -155,6 +157,17 @@ enum {
 };
 
 enum {
+	XGMAC_MTU	= (1 << 0),
+	XGMAC_PROMISC	= (1 << 1),
+	XGMAC_ALLMULTI	= (1 << 2),
+	XGMAC_VLANEX	= (1 << 3),
+	XGMAC_UCADDR	= (1 << 4),
+	XGMAC_MCADDRS	= (1 << 5),
+
+	XGMAC_ALL	= 0xffff
+};
+
+enum {
 	/* flags understood by begin_synchronized_op */
 	HOLD_LOCK	= (1 << 0),
 	SLEEP_OK	= (1 << 1),
@@ -168,7 +181,7 @@ enum {
 	/* adapter flags */
 	FULL_INIT_DONE	= (1 << 0),
 	FW_OK		= (1 << 1),
-	INTR_DIRECT	= (1 << 2),	/* direct interrupts for everything */
+	/* INTR_DIRECT	= (1 << 2),	No longer used. */
 	MASTER_PF	= (1 << 3),
 	ADAP_SYSCTL_CTX	= (1 << 4),
 	TOM_INIT_DONE	= (1 << 5),
@@ -181,6 +194,10 @@ enum {
 	PORT_INIT_DONE	= (1 << 1),
 	PORT_SYSCTL_CTX	= (1 << 2),
 	HAS_TRACEQ	= (1 << 3),
+	INTR_RXQ	= (1 << 4),	/* All NIC rxq's take interrupts */
+	INTR_OFLD_RXQ	= (1 << 5),	/* All TOE rxq's take interrupts */
+	INTR_NM_RXQ	= (1 << 6),	/* All netmap rxq's take interrupts */
+	INTR_ALL	= (INTR_RXQ | INTR_OFLD_RXQ | INTR_NM_RXQ),
 };
 
 #define IS_DOOMED(pi)	((pi)->flags & DOOMED)
@@ -225,6 +242,19 @@ struct port_info {
 	int nofldrxq;		/* # of offload rx queues */
 	int first_ofld_rxq;	/* index of first offload rx queue */
 #endif
+#ifdef DEV_NETMAP
+	int nnmtxq;		/* # of netmap tx queues */
+	int first_nm_txq;	/* index of first netmap tx queue */
+	int nnmrxq;		/* # of netmap rx queues */
+	int first_nm_rxq;	/* index of first netmap rx queue */
+
+	struct ifnet *nm_ifp;
+	struct ifmedia nm_media;
+	int nmif_flags;
+	uint16_t nm_viid;
+	int16_t nm_xact_addr_filt;
+	uint16_t nm_rss_size;	/* size of netmap VI's RSS table slice */
+#endif
 	int tmr_idx;
 	int pktc_idx;
 	int qsize_rxq;
@@ -362,7 +392,7 @@ struct sge_eq {
 	struct tx_desc *desc;	/* KVA of descriptor ring */
 	bus_addr_t ba;		/* bus address of descriptor ring */
 	struct sge_qstat *spg;	/* status page, for convenience */
-	int doorbells;
+	uint16_t doorbells;
 	volatile uint32_t *udb;	/* KVA of doorbell (lies within BAR2) */
 	u_int udb_qid;		/* relative qid within the doorbell page */
 	uint16_t cap;		/* max # of desc, for convenience */
@@ -538,6 +568,77 @@ struct sge_wrq {
 	uint32_t no_desc;	/* out of hardware descriptors */
 } __aligned(CACHE_LINE_SIZE);
 
+
+#ifdef DEV_NETMAP
+#define CPL_PAD (RX_IQ_ESIZE - sizeof(struct rsp_ctrl) - \
+    sizeof(struct rss_header))
+struct nm_iq_desc {
+	struct rss_header rss;
+	union {
+		uint8_t cpl[CPL_PAD];
+		struct cpl_fw6_msg fw6_msg;
+		struct cpl_rx_pkt rx_pkt;
+	} u;
+	struct rsp_ctrl rsp;
+};
+CTASSERT(sizeof(struct nm_iq_desc) == RX_IQ_ESIZE);
+
+struct sge_nm_rxq {
+	struct port_info *pi;
+
+	struct nm_iq_desc *iq_desc;
+	uint16_t iq_abs_id;
+	uint16_t iq_cntxt_id;
+	uint16_t iq_cidx;
+	uint16_t iq_sidx;
+	uint8_t iq_gen;
+
+	__be64  *fl_desc;
+	uint16_t fl_cntxt_id;
+	uint32_t fl_cidx;
+	uint32_t fl_pidx;
+	uint32_t fl_sidx;
+	uint32_t fl_db_val;
+	u_int fl_hwidx:4;
+
+	u_int nid;		/* netmap ring # for this queue */
+
+	/* infrequently used items after this */
+
+	bus_dma_tag_t iq_desc_tag;
+	bus_dmamap_t iq_desc_map;
+	bus_addr_t iq_ba;
+	int intr_idx;
+
+	bus_dma_tag_t fl_desc_tag;
+	bus_dmamap_t fl_desc_map;
+	bus_addr_t fl_ba;
+} __aligned(CACHE_LINE_SIZE);
+
+struct sge_nm_txq {
+	struct tx_desc *desc;
+	uint16_t cidx;
+	uint16_t pidx;
+	uint16_t sidx;
+	uint16_t equiqidx;	/* EQUIQ last requested at this pidx */
+	uint16_t equeqidx;	/* EQUEQ last requested at this pidx */
+	uint16_t dbidx;		/* pidx of the most recent doorbell */
+	uint16_t doorbells;
+	volatile uint32_t *udb;
+	u_int udb_qid;
+	u_int cntxt_id;
+	__be32 cpl_ctrl0;	/* for convenience */
+	u_int nid;		/* netmap ring # for this queue */
+
+	/* infrequently used items after this */
+
+	bus_dma_tag_t desc_tag;
+	bus_dmamap_t desc_map;
+	bus_addr_t ba;
+	int iqidx;
+} __aligned(CACHE_LINE_SIZE);
+#endif
+
 struct sge {
 	int timer_val[SGE_NTIMERS];
 	int counter_val[SGE_NCOUNTERS];
@@ -552,6 +653,10 @@ struct sge {
 	int nofldrxq;	/* total # of TOE rx queues */
 	int nofldtxq;	/* total # of TOE tx queues */
 #endif
+#ifdef DEV_NETMAP
+	int nnmrxq;	/* total # of netmap rx queues */
+	int nnmtxq;	/* total # of netmap tx queues */
+#endif
 	int niq;	/* total # of ingress queues */
 	int neq;	/* total # of egress queues */
 
@@ -564,6 +669,10 @@ struct sge {
 	struct sge_wrq *ofld_txq;	/* TOE tx queues */
 	struct sge_ofld_rxq *ofld_rxq;	/* TOE rx queues */
 #endif
+#ifdef DEV_NETMAP
+	struct sge_nm_txq *nm_txq;	/* netmap tx queues */
+	struct sge_nm_rxq *nm_rxq;	/* netmap rx queues */
+#endif
 
 	uint16_t iq_start;
 	int eq_start;
@@ -629,7 +738,7 @@ struct adapter {
 	struct l2t_data *l2t;	/* L2 table */
 	struct tid_info tids;
 
-	int doorbells;
+	uint16_t doorbells;
 	int open_device_map;
 #ifdef TCP_OFFLOAD
 	int offload_map;
@@ -730,6 +839,12 @@ struct adapter {
 #define for_each_ofld_rxq(pi, iter, q) \
 	for (q = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq], iter = 0; \
 	    iter < pi->nofldrxq; ++iter, ++q)
+#define for_each_nm_txq(pi, iter, q) \
+	for (q = &pi->adapter->sge.nm_txq[pi->first_nm_txq], iter = 0; \
+	    iter < pi->nnmtxq; ++iter, ++q)
+#define for_each_nm_rxq(pi, iter, q) \
+	for (q = &pi->adapter->sge.nm_rxq[pi->first_nm_rxq], iter = 0; \
+	    iter < pi->nnmrxq; ++iter, ++q)
 
 /* One for errors, one for firmware events */
 #define T4_EXTRA_INTR 2
@@ -854,6 +969,18 @@ int t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t);
 int t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
 int begin_synchronized_op(struct adapter *, struct port_info *, int, char *);
 void end_synchronized_op(struct adapter *, int);
+int update_mac_settings(struct ifnet *, int);
+int adapter_full_init(struct adapter *);
+int adapter_full_uninit(struct adapter *);
+int port_full_init(struct port_info *);
+int port_full_uninit(struct port_info *);
+
+#ifdef DEV_NETMAP
+/* t4_netmap.c */
+int create_netmap_ifnet(struct port_info *);
+int destroy_netmap_ifnet(struct port_info *);
+void t4_nm_intr(void *);
+#endif
 
 /* t4_sge.c */
 void t4_sge_modload(void);
diff --git a/sys/dev/cxgbe/common/common.h b/sys/dev/cxgbe/common/common.h
index e3883db..d2e2c8f 100644
--- a/sys/dev/cxgbe/common/common.h
+++ b/sys/dev/cxgbe/common/common.h
@@ -561,11 +561,11 @@ int t4_cfg_pfvf(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		unsigned int exactf, unsigned int rcaps, unsigned int wxcaps);
 int t4_alloc_vi_func(struct adapter *adap, unsigned int mbox,
 		     unsigned int port, unsigned int pf, unsigned int vf,
-		     unsigned int nmac, u8 *mac, unsigned int *rss_size,
+		     unsigned int nmac, u8 *mac, u16 *rss_size,
 		     unsigned int portfunc, unsigned int idstype);
 int t4_alloc_vi(struct adapter *adap, unsigned int mbox, unsigned int port,
 		unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac,
-		unsigned int *rss_size);
+		u16 *rss_size);
 int t4_free_vi(struct adapter *adap, unsigned int mbox,
 	       unsigned int pf, unsigned int vf,
 	       unsigned int viid);
@@ -581,6 +581,8 @@ int t4_set_addr_hash(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		     bool ucast, u64 vec, bool sleep_ok);
 int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		 bool rx_en, bool tx_en);
+int t4_enable_vi_ns(struct adapter *adap, unsigned int mbox, unsigned int viid,
+		 bool rx_en, bool tx_en);
 int t4_identify_port(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		     unsigned int nblinks);
 int t4_i2c_rd(struct adapter *adap, unsigned int mbox, unsigned int port_id,
diff --git a/sys/dev/cxgbe/common/t4_hw.c b/sys/dev/cxgbe/common/t4_hw.c
index 442d3f0..8b32bbb 100644
--- a/sys/dev/cxgbe/common/t4_hw.c
+++ b/sys/dev/cxgbe/common/t4_hw.c
@@ -4826,7 +4826,7 @@ int t4_cfg_pfvf(struct adapter *adap, unsigned int mbox, unsigned int pf,
  */
 int t4_alloc_vi_func(struct adapter *adap, unsigned int mbox,
 		     unsigned int port, unsigned int pf, unsigned int vf,
-		     unsigned int nmac, u8 *mac, unsigned int *rss_size,
+		     unsigned int nmac, u8 *mac, u16 *rss_size,
 		     unsigned int portfunc, unsigned int idstype)
 {
 	int ret;
@@ -4881,7 +4881,7 @@ int t4_alloc_vi_func(struct adapter *adap, unsigned int mbox,
  */
 int t4_alloc_vi(struct adapter *adap, unsigned int mbox, unsigned int port,
 		unsigned int pf, unsigned int vf, unsigned int nmac, u8 *mac,
-		unsigned int *rss_size)
+		u16 *rss_size)
 {
 	return t4_alloc_vi_func(adap, mbox, port, pf, vf, nmac, mac, rss_size,
 				FW_VI_FUNC_ETH, 0);
@@ -5155,6 +5155,19 @@ int t4_enable_vi(struct adapter *adap, unsigned int mbox, unsigned int viid,
 	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
 }
 
+int t4_enable_vi_ns(struct adapter *adap, unsigned int mbox, unsigned int viid,
+		 bool rx_en, bool tx_en)
+{
+	struct fw_vi_enable_cmd c;
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_viid = htonl(V_FW_CMD_OP(FW_VI_ENABLE_CMD) | F_FW_CMD_REQUEST |
+			     F_FW_CMD_EXEC | V_FW_VI_ENABLE_CMD_VIID(viid));
+	c.ien_to_len16 = htonl(V_FW_VI_ENABLE_CMD_IEN(rx_en) |
+			       V_FW_VI_ENABLE_CMD_EEN(tx_en) | FW_LEN16(c));
+	return t4_wr_mbox_ns(adap, mbox, &c, sizeof(c), NULL);
+}
+
 /**
  *	t4_identify_port - identify a VI's port by blinking its LED
  *	@adap: the adapter
@@ -5623,7 +5636,7 @@ int __devinit t4_port_init(struct port_info *p, int mbox, int pf, int vf)
 	u8 addr[6];
 	int ret, i, j;
 	struct fw_port_cmd c;
-	unsigned int rss_size;
+	u16 rss_size;
 	adapter_t *adap = p->adapter;
 
 	memset(&c, 0, sizeof(c));
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index dec9010..8b637f4 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -218,6 +218,24 @@ static int t4_nofldrxq1g = -1;
 TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
 #endif
 
+#ifdef DEV_NETMAP
+#define NNMTXQ_10G 2
+static int t4_nnmtxq10g = -1;
+TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g);
+
+#define NNMRXQ_10G 2
+static int t4_nnmrxq10g = -1;
+TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g);
+
+#define NNMTXQ_1G 1
+static int t4_nnmtxq1g = -1;
+TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g);
+
+#define NNMRXQ_1G 1
+static int t4_nnmrxq1g = -1;
+TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g);
+#endif
+
 /*
  * Holdoff parameters for 10G and 1G ports.
  */
@@ -295,19 +313,26 @@ static int t5_write_combine = 0;
 TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
 
 struct intrs_and_queues {
-	int intr_type;		/* INTx, MSI, or MSI-X */
-	int nirq;		/* Number of vectors */
-	int intr_flags;
-	int ntxq10g;		/* # of NIC txq's for each 10G port */
-	int nrxq10g;		/* # of NIC rxq's for each 10G port */
-	int ntxq1g;		/* # of NIC txq's for each 1G port */
-	int nrxq1g;		/* # of NIC rxq's for each 1G port */
-	int rsrv_noflowq;	/* Flag whether to reserve queue 0 */
+	uint16_t intr_type;	/* INTx, MSI, or MSI-X */
+	uint16_t nirq;		/* Total # of vectors */
+	uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */
+	uint16_t intr_flags_1g;	/* Interrupt flags for each 1G port */
+	uint16_t ntxq10g;	/* # of NIC txq's for each 10G port */
+	uint16_t nrxq10g;	/* # of NIC rxq's for each 10G port */
+	uint16_t ntxq1g;	/* # of NIC txq's for each 1G port */
+	uint16_t nrxq1g;	/* # of NIC rxq's for each 1G port */
+	uint16_t rsrv_noflowq;	/* Flag whether to reserve queue 0 */
 #ifdef TCP_OFFLOAD
-	int nofldtxq10g;	/* # of TOE txq's for each 10G port */
-	int nofldrxq10g;	/* # of TOE rxq's for each 10G port */
-	int nofldtxq1g;		/* # of TOE txq's for each 1G port */
-	int nofldrxq1g;		/* # of TOE rxq's for each 1G port */
+	uint16_t nofldtxq10g;	/* # of TOE txq's for each 10G port */
+	uint16_t nofldrxq10g;	/* # of TOE rxq's for each 10G port */
+	uint16_t nofldtxq1g;	/* # of TOE txq's for each 1G port */
+	uint16_t nofldrxq1g;	/* # of TOE rxq's for each 1G port */
+#endif
+#ifdef DEV_NETMAP
+	uint16_t nnmtxq10g;	/* # of netmap txq's for each 10G port */
+	uint16_t nnmrxq10g;	/* # of netmap rxq's for each 10G port */
+	uint16_t nnmtxq1g;	/* # of netmap txq's for each 1G port */
+	uint16_t nnmrxq1g;	/* # of netmap rxq's for each 1G port */
 #endif
 };
 
@@ -321,17 +346,6 @@ struct filter_entry {
         struct t4_filter_specification fs;
 };
 
-enum {
-	XGMAC_MTU	= (1 << 0),
-	XGMAC_PROMISC	= (1 << 1),
-	XGMAC_ALLMULTI	= (1 << 2),
-	XGMAC_VLANEX	= (1 << 3),
-	XGMAC_UCADDR	= (1 << 4),
-	XGMAC_MCADDRS	= (1 << 5),
-
-	XGMAC_ALL	= 0xffff
-};
-
 static int map_bars_0_and_4(struct adapter *);
 static int map_bar_2(struct adapter *);
 static void setup_memwin(struct adapter *);
@@ -350,15 +364,10 @@ static int get_params__pre_init(struct adapter *);
 static int get_params__post_init(struct adapter *);
 static int set_params__post_init(struct adapter *);
 static void t4_set_desc(struct adapter *);
-static void build_medialist(struct port_info *);
-static int update_mac_settings(struct port_info *, int);
+static void build_medialist(struct port_info *, struct ifmedia *);
 static int cxgbe_init_synchronized(struct port_info *);
 static int cxgbe_uninit_synchronized(struct port_info *);
 static int setup_intr_handlers(struct adapter *);
-static int adapter_full_init(struct adapter *);
-static int adapter_full_uninit(struct adapter *);
-static int port_full_init(struct port_info *);
-static int port_full_uninit(struct port_info *);
 static void quiesce_eq(struct adapter *, struct sge_eq *);
 static void quiesce_iq(struct adapter *, struct sge_iq *);
 static void quiesce_fl(struct adapter *, struct sge_fl *);
@@ -556,6 +565,9 @@ t4_attach(device_t dev)
 #ifdef TCP_OFFLOAD
 	int ofld_rqidx, ofld_tqidx;
 #endif
+#ifdef DEV_NETMAP
+	int nm_rqidx, nm_tqidx;
+#endif
 	const char *pcie_ts;
 
 	sc = device_get_softc(dev);
@@ -685,6 +697,13 @@ t4_attach(device_t dev)
 			sc->port[i] = NULL;
 			goto done;
 		}
+		rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
+		if (rc != 0) {
+			device_printf(dev, "port %d l1cfg failed: %d\n", i, rc);
+			free(pi, M_CXGBE);
+			sc->port[i] = NULL;
+			goto done;
+		}
 
 		snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
 		    device_get_nameunit(dev), i);
@@ -726,7 +745,6 @@ t4_attach(device_t dev)
 
 	sc->intr_type = iaq.intr_type;
 	sc->intr_count = iaq.nirq;
-	sc->flags |= iaq.intr_flags;
 
 	s = &sc->sge;
 	s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
@@ -734,10 +752,8 @@ t4_attach(device_t dev)
 	s->neq = s->ntxq + s->nrxq;	/* the free list in an rxq is an eq */
 	s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
 	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
-
 #ifdef TCP_OFFLOAD
 	if (is_offload(sc)) {
-
 		s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
 		s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
 		s->neq += s->nofldtxq + s->nofldrxq;
@@ -749,6 +765,17 @@ t4_attach(device_t dev)
 		    M_CXGBE, M_ZERO | M_WAITOK);
 	}
 #endif
+#ifdef DEV_NETMAP
+	s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g;
+	s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g;
+	s->neq += s->nnmtxq + s->nnmrxq;
+	s->niq += s->nnmrxq;
+
+	s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
+	    M_CXGBE, M_ZERO | M_WAITOK);
+	s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
+	    M_CXGBE, M_ZERO | M_WAITOK);
+#endif
 
 	s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
 	    M_ZERO | M_WAITOK);
@@ -774,6 +801,9 @@ t4_attach(device_t dev)
 #ifdef TCP_OFFLOAD
 	ofld_rqidx = ofld_tqidx = 0;
 #endif
+#ifdef DEV_NETMAP
+	nm_rqidx = nm_tqidx = 0;
+#endif
 	for_each_port(sc, i) {
 		struct port_info *pi = sc->port[i];
 
@@ -783,9 +813,11 @@ t4_attach(device_t dev)
 		pi->first_rxq = rqidx;
 		pi->first_txq = tqidx;
 		if (is_10G_port(pi) || is_40G_port(pi)) {
+			pi->flags |= iaq.intr_flags_10g;
 			pi->nrxq = iaq.nrxq10g;
 			pi->ntxq = iaq.ntxq10g;
 		} else {
+			pi->flags |= iaq.intr_flags_1g;
 			pi->nrxq = iaq.nrxq1g;
 			pi->ntxq = iaq.ntxq1g;
 		}
@@ -797,7 +829,6 @@ t4_attach(device_t dev)
 
 		rqidx += pi->nrxq;
 		tqidx += pi->ntxq;
-
 #ifdef TCP_OFFLOAD
 		if (is_offload(sc)) {
 			pi->first_ofld_rxq = ofld_rqidx;
@@ -813,6 +844,19 @@ t4_attach(device_t dev)
 			ofld_tqidx += pi->nofldtxq;
 		}
 #endif
+#ifdef DEV_NETMAP
+		pi->first_nm_rxq = nm_rqidx;
+		pi->first_nm_txq = nm_tqidx;
+		if (is_10G_port(pi) || is_40G_port(pi)) {
+			pi->nnmrxq = iaq.nnmrxq10g;
+			pi->nnmtxq = iaq.nnmtxq10g;
+		} else {
+			pi->nnmrxq = iaq.nnmrxq1g;
+			pi->nnmtxq = iaq.nnmtxq1g;
+		}
+		nm_rqidx += pi->nnmrxq;
+		nm_tqidx += pi->nnmtxq;
+#endif
 	}
 
 	rc = setup_intr_handlers(sc);
@@ -902,7 +946,7 @@ t4_detach(device_t dev)
 	for (i = 0; i < MAX_NPORTS; i++) {
 		pi = sc->port[i];
 		if (pi) {
-			t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
+			t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid);
 			if (pi->dev)
 				device_delete_child(dev, pi->dev);
 
@@ -939,6 +983,10 @@ t4_detach(device_t dev)
 	free(sc->sge.ofld_rxq, M_CXGBE);
 	free(sc->sge.ofld_txq, M_CXGBE);
 #endif
+#ifdef DEV_NETMAP
+	free(sc->sge.nm_rxq, M_CXGBE);
+	free(sc->sge.nm_txq, M_CXGBE);
+#endif
 	free(sc->irq, M_CXGBE);
 	free(sc->sge.rxq, M_CXGBE);
 	free(sc->sge.txq, M_CXGBE);
@@ -966,7 +1014,6 @@ t4_detach(device_t dev)
 	return (0);
 }
 
-
 static int
 cxgbe_probe(device_t dev)
 {
@@ -989,6 +1036,8 @@ cxgbe_attach(device_t dev)
 {
 	struct port_info *pi = device_get_softc(dev);
 	struct ifnet *ifp;
+	char *s;
+	int n, o;
 
 	/* Allocate an ifnet and set it up */
 	ifp = if_alloc(IFT_ETHER);
@@ -1021,22 +1070,39 @@ cxgbe_attach(device_t dev)
 	/* Initialize ifmedia for this port */
 	ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
 	    cxgbe_media_status);
-	build_medialist(pi);
+	build_medialist(pi, &pi->media);
 
 	pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
 	    EVENTHANDLER_PRI_ANY);
 
 	ether_ifattach(ifp, pi->hw_addr);
 
+	n = 128;
+	s = malloc(n, M_CXGBE, M_WAITOK);
+	o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq);
+	MPASS(n > o);
 #ifdef TCP_OFFLOAD
 	if (is_offload(pi->adapter)) {
-		device_printf(dev,
-		    "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
-		    pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
-	} else
+		o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)",
+		    pi->nofldtxq, pi->nofldrxq);
+		MPASS(n > o);
+	}
 #endif
-		device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
+#ifdef DEV_NETMAP
+	o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq,
+	    pi->nnmrxq);
+	MPASS(n > o);
+#endif
+	device_printf(dev, "%s\n", s);
+	free(s, M_CXGBE);
 
+#ifdef DEV_NETMAP
+	/* nm_media handled here to keep implementation private to this file */
+	ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change,
+	    cxgbe_media_status);
+	build_medialist(pi, &pi->nm_media);
+	create_netmap_ifnet(pi);	/* logs errors it something fails */
+#endif
 	cxgbe_sysctls(pi);
 
 	return (0);
@@ -1084,6 +1150,11 @@ cxgbe_detach(device_t dev)
 	ether_ifdetach(pi->ifp);
 	if_free(pi->ifp);
 
+#ifdef DEV_NETMAP
+	/* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */
+	destroy_netmap_ifnet(pi);
+#endif
+
 	ADAPTER_LOCK(sc);
 	CLR_BUSY(sc);
 	wakeup(&sc->flags);
@@ -1126,7 +1197,7 @@ cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 		if (pi->flags & PORT_INIT_DONE) {
 			t4_update_fl_bufsize(ifp);
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
-				rc = update_mac_settings(pi, XGMAC_MTU);
+				rc = update_mac_settings(ifp, XGMAC_MTU);
 		}
 		end_synchronized_op(sc, 0);
 		break;
@@ -1141,7 +1212,7 @@ cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 				flags = pi->if_flags;
 				if ((ifp->if_flags ^ flags) &
 				    (IFF_PROMISC | IFF_ALLMULTI)) {
-					rc = update_mac_settings(pi,
+					rc = update_mac_settings(ifp,
 					    XGMAC_PROMISC | XGMAC_ALLMULTI);
 				}
 			} else
@@ -1152,13 +1223,13 @@ cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 		end_synchronized_op(sc, 0);
 		break;
 
-	case SIOCADDMULTI:	
+	case SIOCADDMULTI:
 	case SIOCDELMULTI: /* these two are called with a mutex held :-( */
 		rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
 		if (rc)
 			return (rc);
 		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
-			rc = update_mac_settings(pi, XGMAC_MCADDRS);
+			rc = update_mac_settings(ifp, XGMAC_MCADDRS);
 		end_synchronized_op(sc, LOCK_HELD);
 		break;
 
@@ -1247,7 +1318,7 @@ cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
 		if (mask & IFCAP_VLAN_HWTAGGING) {
 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
-				rc = update_mac_settings(pi, XGMAC_VLANEX);
+				rc = update_mac_settings(ifp, XGMAC_VLANEX);
 		}
 		if (mask & IFCAP_VLAN_MTU) {
 			ifp->if_capenable ^= IFCAP_VLAN_MTU;
@@ -1382,13 +1453,23 @@ static void
 cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 {
 	struct port_info *pi = ifp->if_softc;
-	struct ifmedia_entry *cur = pi->media.ifm_cur;
+	struct ifmedia *media = NULL;
+	struct ifmedia_entry *cur;
 	int speed = pi->link_cfg.speed;
 	int data = (pi->port_type << 8) | pi->mod_type;
 
+	if (ifp == pi->ifp)
+		media = &pi->media;
+#ifdef DEV_NETMAP
+	else if (ifp == pi->nm_ifp)
+		media = &pi->nm_media;
+#endif
+	MPASS(media != NULL);
+
+	cur = media->ifm_cur;
 	if (cur->ifm_data != data) {
-		build_medialist(pi);
-		cur = pi->media.ifm_cur;
+		build_medialist(pi, media);
+		cur = media->ifm_cur;
 	}
 
 	ifmr->ifm_status = IFM_AVALID;
@@ -1741,6 +1822,7 @@ cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
 {
 	int rc, itype, navail, nrxq10g, nrxq1g, n;
 	int nofldrxq10g = 0, nofldrxq1g = 0;
+	int nnmrxq10g = 0, nnmrxq1g = 0;
 
 	bzero(iaq, sizeof(*iaq));
 
@@ -1757,6 +1839,12 @@ cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
 		iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
 	}
 #endif
+#ifdef DEV_NETMAP
+	iaq->nnmtxq10g = t4_nnmtxq10g;
+	iaq->nnmtxq1g = t4_nnmtxq1g;
+	iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g;
+	iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g;
+#endif
 
 	for (itype = INTR_MSIX; itype; itype >>= 1) {
 
@@ -1774,30 +1862,60 @@ restart:
 			continue;
 
 		iaq->intr_type = itype;
-		iaq->intr_flags = 0;
+		iaq->intr_flags_10g = 0;
+		iaq->intr_flags_1g = 0;
 
 		/*
 		 * Best option: an interrupt vector for errors, one for the
-		 * firmware event queue, and one each for each rxq (NIC as well
-		 * as offload).
+		 * firmware event queue, and one for every rxq (NIC, TOE, and
+		 * netmap).
 		 */
 		iaq->nirq = T4_EXTRA_INTR;
-		iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
-		iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
+		iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g);
+		iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g);
 		if (iaq->nirq <= navail &&
 		    (itype != INTR_MSI || powerof2(iaq->nirq))) {
-			iaq->intr_flags |= INTR_DIRECT;
+			iaq->intr_flags_10g = INTR_ALL;
+			iaq->intr_flags_1g = INTR_ALL;
 			goto allocate;
 		}
 
 		/*
-		 * Second best option: an interrupt vector for errors, one for
-		 * the firmware event queue, and one each for either NIC or
-		 * offload rxq's.
+		 * Second best option: a vector for errors, one for the firmware
+		 * event queue, and vectors for either all the NIC rx queues or
+		 * all the TOE rx queues.  The queues that don't get vectors
+		 * will forward their interrupts to those that do.
+		 *
+		 * Note: netmap rx queues cannot be created early and so they
+		 * can't be setup to receive forwarded interrupts for others.
 		 */
 		iaq->nirq = T4_EXTRA_INTR;
-		iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
-		iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
+		if (nrxq10g >= nofldrxq10g) {
+			iaq->intr_flags_10g = INTR_RXQ;
+			iaq->nirq += n10g * nrxq10g;
+#ifdef DEV_NETMAP
+			iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g);
+#endif
+		} else {
+			iaq->intr_flags_10g = INTR_OFLD_RXQ;
+			iaq->nirq += n10g * nofldrxq10g;
+#ifdef DEV_NETMAP
+			iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g);
+#endif
+		}
+		if (nrxq1g >= nofldrxq1g) {
+			iaq->intr_flags_1g = INTR_RXQ;
+			iaq->nirq += n1g * nrxq1g;
+#ifdef DEV_NETMAP
+			iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g);
+#endif
+		} else {
+			iaq->intr_flags_1g = INTR_OFLD_RXQ;
+			iaq->nirq += n1g * nofldrxq1g;
+#ifdef DEV_NETMAP
+			iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g);
+#endif
+		}
 		if (iaq->nirq <= navail &&
 		    (itype != INTR_MSI || powerof2(iaq->nirq)))
 			goto allocate;
@@ -1805,8 +1923,8 @@ restart:
 		/*
 		 * Next best option: an interrupt vector for errors, one for the
 		 * firmware event queue, and at least one per port.  At this
-		 * point we know we'll have to downsize nrxq or nofldrxq to fit
-		 * what's available to us.
+		 * point we know we'll have to downsize nrxq and/or nofldrxq
+		 * and/or nnmrxq to fit what's available to us.
 		 */
 		iaq->nirq = T4_EXTRA_INTR;
 		iaq->nirq += n10g + n1g;
@@ -1816,6 +1934,9 @@ restart:
 			if (n10g > 0) {
 				int target = max(nrxq10g, nofldrxq10g);
 
+				iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ?
+				    INTR_RXQ : INTR_OFLD_RXQ;
+
 				n = 1;
 				while (n < target && leftover >= n10g) {
 					leftover -= n10g;
@@ -1824,14 +1945,19 @@ restart:
 				}
 				iaq->nrxq10g = min(n, nrxq10g);
 #ifdef TCP_OFFLOAD
-				if (is_offload(sc))
-					iaq->nofldrxq10g = min(n, nofldrxq10g);
+				iaq->nofldrxq10g = min(n, nofldrxq10g);
+#endif
+#ifdef DEV_NETMAP
+				iaq->nnmrxq10g = min(n, nnmrxq10g);
 #endif
 			}
 
 			if (n1g > 0) {
 				int target = max(nrxq1g, nofldrxq1g);
 
+				iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ?
+				    INTR_RXQ : INTR_OFLD_RXQ;
+
 				n = 1;
 				while (n < target && leftover >= n1g) {
 					leftover -= n1g;
@@ -1840,8 +1966,10 @@ restart:
 				}
 				iaq->nrxq1g = min(n, nrxq1g);
 #ifdef TCP_OFFLOAD
-				if (is_offload(sc))
-					iaq->nofldrxq1g = min(n, nofldrxq1g);
+				iaq->nofldrxq1g = min(n, nofldrxq1g);
+#endif
+#ifdef DEV_NETMAP
+				iaq->nnmrxq1g = min(n, nnmrxq1g);
 #endif
 			}
 
@@ -1853,10 +1981,14 @@ restart:
 		 * Least desirable option: one interrupt vector for everything.
 		 */
 		iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
+		iaq->intr_flags_10g = iaq->intr_flags_1g = 0;
 #ifdef TCP_OFFLOAD
 		if (is_offload(sc))
 			iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
 #endif
+#ifdef DEV_NETMAP
+		iaq->nnmrxq10g = iaq->nnmrxq1g = 1;
+#endif
 
 allocate:
 		navail = iaq->nirq;
@@ -2636,9 +2768,8 @@ t4_set_desc(struct adapter *sc)
 }
 
 static void
-build_medialist(struct port_info *pi)
+build_medialist(struct port_info *pi, struct ifmedia *media)
 {
-	struct ifmedia *media = &pi->media;
 	int data, m;
 
 	PORT_LOCK(pi);
@@ -2767,17 +2898,29 @@ build_medialist(struct port_info *pi)
  * Program the port's XGMAC based on parameters in ifnet.  The caller also
  * indicates which parameters should be programmed (the rest are left alone).
  */
-static int
-update_mac_settings(struct port_info *pi, int flags)
+int
+update_mac_settings(struct ifnet *ifp, int flags)
 {
-	int rc;
-	struct ifnet *ifp = pi->ifp;
+	int rc = 0;
+	struct port_info *pi = ifp->if_softc;
 	struct adapter *sc = pi->adapter;
 	int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
+	uint16_t viid = 0xffff;
+	int16_t *xact_addr_filt = NULL;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
 	KASSERT(flags, ("%s: not told what to update.", __func__));
 
+	if (ifp == pi->ifp) {
+		viid = pi->viid;
+		xact_addr_filt = &pi->xact_addr_filt;
+	}
+#ifdef DEV_NETMAP
+	else if (ifp == pi->nm_ifp) {
+		viid = pi->nm_viid;
+		xact_addr_filt = &pi->nm_xact_addr_filt;
+	}
+#endif
 	if (flags & XGMAC_MTU)
 		mtu = ifp->if_mtu;
 
@@ -2790,25 +2933,28 @@ update_mac_settings(struct port_info *pi, int flags)
 	if (flags & XGMAC_VLANEX)
 		vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
 
-	rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
-	    vlanex, false);
-	if (rc) {
-		if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
-		return (rc);
+	if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
+		rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti,
+		    1, vlanex, false);
+		if (rc) {
+			if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
+			    rc);
+			return (rc);
+		}
 	}
 
 	if (flags & XGMAC_UCADDR) {
 		uint8_t ucaddr[ETHER_ADDR_LEN];
 
 		bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
-		rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
-		    ucaddr, true, true);
+		rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr,
+		    true, true);
 		if (rc < 0) {
 			rc = -rc;
 			if_printf(ifp, "change_mac failed: %d\n", rc);
 			return (rc);
 		} else {
-			pi->xact_addr_filt = rc;
+			*xact_addr_filt = rc;
 			rc = 0;
 		}
 	}
@@ -2828,8 +2974,8 @@ update_mac_settings(struct port_info *pi, int flags)
 			    LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
 
 			if (i == FW_MAC_EXACT_CHUNK) {
-				rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
-				    del, i, mcaddr, NULL, &hash, 0);
+				rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del,
+				    i, mcaddr, NULL, &hash, 0);
 				if (rc < 0) {
 					rc = -rc;
 					for (j = 0; j < i; j++) {
@@ -2849,8 +2995,8 @@ update_mac_settings(struct port_info *pi, int flags)
 			}
 		}
 		if (i > 0) {
-			rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
-			    del, i, mcaddr, NULL, &hash, 0);
+			rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i,
+			    mcaddr, NULL, &hash, 0);
 			if (rc < 0) {
 				rc = -rc;
 				for (j = 0; j < i; j++) {
@@ -2867,7 +3013,7 @@ update_mac_settings(struct port_info *pi, int flags)
 			}
 		}
 
-		rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
+		rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0);
 		if (rc != 0)
 			if_printf(ifp, "failed to set mc address hash: %d", rc);
 mcfail:
@@ -2970,16 +3116,10 @@ cxgbe_init_synchronized(struct port_info *pi)
 	    ((rc = port_full_init(pi)) != 0))
 		return (rc); /* error message displayed already */
 
-	rc = update_mac_settings(pi, XGMAC_ALL);
+	rc = update_mac_settings(ifp, XGMAC_ALL);
 	if (rc)
 		goto done;	/* error message displayed already */
 
-	rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
-	if (rc != 0) {
-		if_printf(ifp, "start_link failed: %d\n", rc);
-		goto done;
-	}
-
 	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
 	if (rc != 0) {
 		if_printf(ifp, "enable_vi failed: %d\n", rc);
@@ -3064,61 +3204,41 @@ setup_intr_handlers(struct adapter *sc)
 #ifdef TCP_OFFLOAD
 	struct sge_ofld_rxq *ofld_rxq;
 #endif
+#ifdef DEV_NETMAP
+	struct sge_nm_rxq *nm_rxq;
+#endif
 
 	/*
 	 * Setup interrupts.
 	 */
 	irq = &sc->irq[0];
 	rid = sc->intr_type == INTR_INTX ? 0 : 1;
-	if (sc->intr_count == 1) {
-		KASSERT(!(sc->flags & INTR_DIRECT),
-		    ("%s: single interrupt && INTR_DIRECT?", __func__));
+	if (sc->intr_count == 1)
+		return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
 
-		rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all");
-		if (rc != 0)
-			return (rc);
-	} else {
-		/* Multiple interrupts. */
-		KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
-		    ("%s: too few intr.", __func__));
+	/* Multiple interrupts. */
+	KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
+	    ("%s: too few intr.", __func__));
 
-		/* The first one is always error intr */
-		rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
-		if (rc != 0)
-			return (rc);
-		irq++;
-		rid++;
+	/* The first one is always error intr */
+	rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
+	if (rc != 0)
+		return (rc);
+	irq++;
+	rid++;
 
-		/* The second one is always the firmware event queue */
-		rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
-		    "evt");
-		if (rc != 0)
-			return (rc);
-		irq++;
-		rid++;
+	/* The second one is always the firmware event queue */
+	rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
+	if (rc != 0)
+		return (rc);
+	irq++;
+	rid++;
 
-		/*
-		 * Note that if INTR_DIRECT is not set then either the NIC rx
-		 * queues or (exclusive or) the TOE rx queueus will be taking
-		 * direct interrupts.
-		 *
-		 * There is no need to check for is_offload(sc) as nofldrxq
-		 * will be 0 if offload is disabled.
-		 */
-		for_each_port(sc, p) {
-			pi = sc->port[p];
+	for_each_port(sc, p) {
+		pi = sc->port[p];
 
-#ifdef TCP_OFFLOAD
-			/*
-			 * Skip over the NIC queues if they aren't taking direct
-			 * interrupts.
-			 */
-			if (!(sc->flags & INTR_DIRECT) &&
-			    pi->nofldrxq > pi->nrxq)
-				goto ofld_queues;
-#endif
-			rxq = &sc->sge.rxq[pi->first_rxq];
-			for (q = 0; q < pi->nrxq; q++, rxq++) {
+		if (pi->flags & INTR_RXQ) {
+			for_each_rxq(pi, q, rxq) {
 				snprintf(s, sizeof(s), "%d.%d", p, q);
 				rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
 				    s);
@@ -3127,17 +3247,10 @@ setup_intr_handlers(struct adapter *sc)
 				irq++;
 				rid++;
 			}
-
+		}
 #ifdef TCP_OFFLOAD
-			/*
-			 * Skip over the offload queues if they aren't taking
-			 * direct interrupts.
-			 */
-			if (!(sc->flags & INTR_DIRECT))
-				continue;
-ofld_queues:
-			ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
-			for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
+		if (pi->flags & INTR_OFLD_RXQ) {
+			for_each_ofld_rxq(pi, q, ofld_rxq) {
 				snprintf(s, sizeof(s), "%d,%d", p, q);
 				rc = t4_alloc_irq(sc, irq, rid, t4_intr,
 				    ofld_rxq, s);
@@ -3146,14 +3259,28 @@ ofld_queues:
 				irq++;
 				rid++;
 			}
+		}
 #endif
+#ifdef DEV_NETMAP
+		if (pi->flags & INTR_NM_RXQ) {
+			for_each_nm_rxq(pi, q, nm_rxq) {
+				snprintf(s, sizeof(s), "%d-%d", p, q);
+				rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr,
+				    nm_rxq, s);
+				if (rc != 0)
+					return (rc);
+				irq++;
+				rid++;
+			}
 		}
+#endif
 	}
+	MPASS(irq == &sc->irq[sc->intr_count]);
 
 	return (0);
 }
 
-static int
+int
 adapter_full_init(struct adapter *sc)
 {
 	int rc, i;
@@ -3191,7 +3318,7 @@ done:
 	return (rc);
 }
 
-static int
+int
 adapter_full_uninit(struct adapter *sc)
 {
 	int i;
@@ -3210,7 +3337,7 @@ adapter_full_uninit(struct adapter *sc)
 	return (0);
 }
 
-static int
+int
 port_full_init(struct port_info *pi)
 {
 	struct adapter *sc = pi->adapter;
@@ -3264,7 +3391,7 @@ done:
 /*
  * Idempotent.
  */
-static int
+int
 port_full_uninit(struct port_info *pi)
 {
 	struct adapter *sc = pi->adapter;
@@ -4597,6 +4724,18 @@ cxgbe_sysctls(struct port_info *pi)
 		    "index of first TOE tx queue");
 	}
 #endif
+#ifdef DEV_NETMAP
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
+	    &pi->nnmrxq, 0, "# of rx queues for netmap");
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
+	    &pi->nnmtxq, 0, "# of tx queues for netmap");
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
+	    CTLFLAG_RD, &pi->first_nm_rxq, 0,
+	    "index of first netmap rx queue");
+	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
+	    CTLFLAG_RD, &pi->first_nm_txq, 0,
+	    "index of first netmap tx queue");
+#endif
 
 	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
 	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
@@ -8084,6 +8223,20 @@ tweak_tunables(void)
 		t4_toecaps_allowed = 0;
 #endif
 
+#ifdef DEV_NETMAP
+	if (t4_nnmtxq10g < 1)
+		t4_nnmtxq10g = min(nc, NNMTXQ_10G);
+
+	if (t4_nnmtxq1g < 1)
+		t4_nnmtxq1g = min(nc, NNMTXQ_1G);
+
+	if (t4_nnmrxq10g < 1)
+		t4_nnmrxq10g = min(nc, NNMRXQ_10G);
+
+	if (t4_nnmrxq1g < 1)
+		t4_nnmrxq1g = min(nc, NNMRXQ_1G);
+#endif
+
 	if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
 		t4_tmr_idx_10g = TMR_IDX_10G;
 
diff --git a/sys/dev/cxgbe/t4_netmap.c b/sys/dev/cxgbe/t4_netmap.c
new file mode 100644
index 0000000..7a7785c
--- /dev/null
+++ b/sys/dev/cxgbe/t4_netmap.c
@@ -0,0 +1,1148 @@
+/*-
+ * Copyright (c) 2014 Chelsio Communications, Inc.
+ * All rights reserved.
+ * Written by: Navdeep Parhar <np@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_inet.h"
+#include "opt_inet6.h"
+
+#ifdef DEV_NETMAP
+#include <sys/param.h>
+#include <sys/eventhandler.h>
+#include <sys/lock.h>
+#include <sys/types.h>
+#include <sys/mbuf.h>
+#include <sys/selinfo.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <machine/bus.h>
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_media.h>
+#include <net/if_var.h>
+#include <net/if_clone.h>
+#include <net/if_types.h>
+#include <net/netmap.h>
+#include <dev/netmap/netmap_kern.h>
+
+#include "common/common.h"
+#include "common/t4_regs.h"
+#include "common/t4_regs_values.h"
+
+extern int fl_pad;	/* XXXNM */
+extern int spg_len;	/* XXXNM */
+extern int fl_pktshift;	/* XXXNM */
+
+/* netmap ifnet routines */
+static void cxgbe_nm_init(void *);
+static int cxgbe_nm_ioctl(struct ifnet *, unsigned long, caddr_t);
+static int cxgbe_nm_transmit(struct ifnet *, struct mbuf *);
+static void cxgbe_nm_qflush(struct ifnet *);
+
+static int cxgbe_nm_init_synchronized(struct port_info *);
+static int cxgbe_nm_uninit_synchronized(struct port_info *);
+
+static void
+cxgbe_nm_init(void *arg)
+{
+	struct port_info *pi = arg;
+	struct adapter *sc = pi->adapter;
+
+	if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4nminit") != 0)
+		return;
+	cxgbe_nm_init_synchronized(pi);
+	end_synchronized_op(sc, 0);
+
+	return;
+}
+
+static int
+cxgbe_nm_init_synchronized(struct port_info *pi)
+{
+	struct adapter *sc = pi->adapter;
+	struct ifnet *ifp = pi->nm_ifp;
+	int rc = 0;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+		return (0);	/* already running */
+
+	if (!(sc->flags & FULL_INIT_DONE) &&
+	    ((rc = adapter_full_init(sc)) != 0))
+		return (rc);	/* error message displayed already */
+
+	if (!(pi->flags & PORT_INIT_DONE) &&
+	    ((rc = port_full_init(pi)) != 0))
+		return (rc);	/* error message displayed already */
+
+	rc = update_mac_settings(ifp, XGMAC_ALL);
+	if (rc)
+		return (rc);	/* error message displayed already */
+
+	ifp->if_drv_flags |= IFF_DRV_RUNNING;
+
+	return (rc);
+}
+
+static int
+cxgbe_nm_uninit_synchronized(struct port_info *pi)
+{
+#ifdef INVARIANTS
+	struct adapter *sc = pi->adapter;
+#endif
+	struct ifnet *ifp = pi->nm_ifp;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+
+	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+
+	return (0);
+}
+
+static int
+cxgbe_nm_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
+{
+	int rc = 0, mtu, flags;
+	struct port_info *pi = ifp->if_softc;
+	struct adapter *sc = pi->adapter;
+	struct ifreq *ifr = (struct ifreq *)data;
+	uint32_t mask;
+
+	MPASS(pi->nm_ifp == ifp);
+
+	switch (cmd) {
+	case SIOCSIFMTU:
+		mtu = ifr->ifr_mtu;
+		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
+			return (EINVAL);
+
+		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4nmtu");
+		if (rc)
+			return (rc);
+		ifp->if_mtu = mtu;
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+			rc = update_mac_settings(ifp, XGMAC_MTU);
+		end_synchronized_op(sc, 0);
+		break;
+
+	case SIOCSIFFLAGS:
+		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4nflg");
+		if (rc)
+			return (rc);
+
+		if (ifp->if_flags & IFF_UP) {
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+				flags = pi->nmif_flags;
+				if ((ifp->if_flags ^ flags) &
+				    (IFF_PROMISC | IFF_ALLMULTI)) {
+					rc = update_mac_settings(ifp,
+					    XGMAC_PROMISC | XGMAC_ALLMULTI);
+				}
+			} else
+				rc = cxgbe_nm_init_synchronized(pi);
+			pi->nmif_flags = ifp->if_flags;
+		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+			rc = cxgbe_nm_uninit_synchronized(pi);
+		end_synchronized_op(sc, 0);
+		break;
+
+	case SIOCADDMULTI:
+	case SIOCDELMULTI: /* these two are called with a mutex held :-( */
+		rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4nmulti");
+		if (rc)
+			return (rc);
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+			rc = update_mac_settings(ifp, XGMAC_MCADDRS);
+		end_synchronized_op(sc, LOCK_HELD);
+		break;
+
+	case SIOCSIFCAP:
+		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
+		if (mask & IFCAP_TXCSUM) {
+			ifp->if_capenable ^= IFCAP_TXCSUM;
+			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
+		}
+		if (mask & IFCAP_TXCSUM_IPV6) {
+			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
+			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
+		}
+		if (mask & IFCAP_RXCSUM)
+			ifp->if_capenable ^= IFCAP_RXCSUM;
+		if (mask & IFCAP_RXCSUM_IPV6)
+			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
+		break;
+
+	case SIOCSIFMEDIA:
+	case SIOCGIFMEDIA:
+		ifmedia_ioctl(ifp, ifr, &pi->nm_media, cmd);
+		break;
+
+	default:
+		rc = ether_ioctl(ifp, cmd, data);
+	}
+
+	return (rc);
+}
+
+static int
+cxgbe_nm_transmit(struct ifnet *ifp, struct mbuf *m)
+{
+
+	m_freem(m);
+	return (0);
+}
+
+static void
+cxgbe_nm_qflush(struct ifnet *ifp)
+{
+
+	return;
+}
+
+static int
+alloc_nm_rxq_hwq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
+{
+	int rc, cntxt_id;
+	__be32 v;
+	struct adapter *sc = pi->adapter;
+	struct netmap_adapter *na = NA(pi->nm_ifp);
+	struct fw_iq_cmd c;
+
+	MPASS(na != NULL);
+	MPASS(nm_rxq->iq_desc != NULL);
+	MPASS(nm_rxq->fl_desc != NULL);
+
+	bzero(nm_rxq->iq_desc, pi->qsize_rxq * RX_IQ_ESIZE);
+	bzero(nm_rxq->fl_desc, na->num_rx_desc * RX_FL_ESIZE + spg_len);
+
+	bzero(&c, sizeof(c));
+	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
+	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
+	    V_FW_IQ_CMD_VFN(0));
+	c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
+	    FW_LEN16(c));
+	if (pi->flags & INTR_NM_RXQ) {
+		KASSERT(nm_rxq->intr_idx < sc->intr_count,
+		    ("%s: invalid direct intr_idx %d", __func__,
+		    nm_rxq->intr_idx));
+		v = V_FW_IQ_CMD_IQANDSTINDEX(nm_rxq->intr_idx);
+	} else {
+		CXGBE_UNIMPLEMENTED(__func__);	/* XXXNM: needs review */
+		v = V_FW_IQ_CMD_IQANDSTINDEX(nm_rxq->intr_idx) |
+		    F_FW_IQ_CMD_IQANDST;
+	}
+	c.type_to_iqandstindex = htobe32(v |
+	    V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
+	    V_FW_IQ_CMD_VIID(pi->nm_viid) |
+	    V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
+	c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
+	    F_FW_IQ_CMD_IQGTSMODE |
+	    V_FW_IQ_CMD_IQINTCNTTHRESH(0) |
+	    V_FW_IQ_CMD_IQESIZE(ilog2(RX_IQ_ESIZE) - 4));
+	c.iqsize = htobe16(pi->qsize_rxq);
+	c.iqaddr = htobe64(nm_rxq->iq_ba);
+	c.iqns_to_fl0congen |=
+	    htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
+		F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
+		(fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0));
+	c.fl0dcaen_to_fl0cidxfthresh =
+	    htobe16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
+		V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
+	c.fl0size = htobe16(na->num_rx_desc + spg_len / RX_FL_ESIZE);
+	c.fl0addr = htobe64(nm_rxq->fl_ba);
+
+	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
+	if (rc != 0) {
+		device_printf(sc->dev,
+		    "failed to create netmap ingress queue: %d\n", rc);
+		return (rc);
+	}
+
+	nm_rxq->iq_cidx = 0;
+	MPASS(nm_rxq->iq_sidx == pi->qsize_rxq - spg_len / RX_IQ_ESIZE);
+	nm_rxq->iq_gen = F_RSPD_GEN;
+	nm_rxq->iq_cntxt_id = be16toh(c.iqid);
+	nm_rxq->iq_abs_id = be16toh(c.physiqid);
+	cntxt_id = nm_rxq->iq_cntxt_id - sc->sge.iq_start;
+	if (cntxt_id >= sc->sge.niq) {
+		panic ("%s: nm_rxq->iq_cntxt_id (%d) more than the max (%d)",
+		    __func__, cntxt_id, sc->sge.niq - 1);
+	}
+	sc->sge.iqmap[cntxt_id] = (void *)nm_rxq;
+
+	nm_rxq->fl_cntxt_id = be16toh(c.fl0id);
+	nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;
+	MPASS(nm_rxq->fl_sidx == na->num_rx_desc);
+	cntxt_id = nm_rxq->fl_cntxt_id - sc->sge.eq_start;
+	if (cntxt_id >= sc->sge.neq) {
+		panic("%s: nm_rxq->fl_cntxt_id (%d) more than the max (%d)",
+		    __func__, cntxt_id, sc->sge.neq - 1);
+	}
+	sc->sge.eqmap[cntxt_id] = (void *)nm_rxq;
+
+	nm_rxq->fl_db_val = F_DBPRIO | V_QID(nm_rxq->fl_cntxt_id) | V_PIDX(0);
+	if (is_t5(sc))
+		nm_rxq->fl_db_val |= F_DBTYPE;
+
+	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(F_QINTR_CNT_EN) |
+	    V_INGRESSQID(nm_rxq->iq_cntxt_id));
+
+	return (rc);
+}
+
+static int
+free_nm_rxq_hwq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
+{
+	struct adapter *sc = pi->adapter;
+	int rc;
+
+	rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
+	    nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, 0xffff);
+	if (rc != 0)
+		device_printf(sc->dev, "%s: failed for iq %d, fl %d: %d\n",
+		    __func__, nm_rxq->iq_cntxt_id, nm_rxq->fl_cntxt_id, rc);
+	return (rc);
+}
+
+static int
+alloc_nm_txq_hwq(struct port_info *pi, struct sge_nm_txq *nm_txq)
+{
+	int rc, cntxt_id;
+	size_t len;
+	struct adapter *sc = pi->adapter;
+	struct netmap_adapter *na = NA(pi->nm_ifp);
+	struct fw_eq_eth_cmd c;
+
+	MPASS(na != NULL);
+	MPASS(nm_txq->desc != NULL);
+
+	len = na->num_tx_desc * EQ_ESIZE + spg_len;
+	bzero(nm_txq->desc, len);
+
+	bzero(&c, sizeof(c));
+	c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
+	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
+	    V_FW_EQ_ETH_CMD_VFN(0));
+	c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
+	    F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
+	c.viid_pkd = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->nm_viid));
+	c.fetchszm_to_iqid =
+	    htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
+		V_FW_EQ_ETH_CMD_PCIECHN(pi->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
+		V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id));
+	c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
+		      V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
+		      V_FW_EQ_ETH_CMD_EQSIZE(len / EQ_ESIZE));
+	c.eqaddr = htobe64(nm_txq->ba);
+
+	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
+	if (rc != 0) {
+		device_printf(pi->dev,
+		    "failed to create netmap egress queue: %d\n", rc);
+		return (rc);
+	}
+
+	nm_txq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
+	cntxt_id = nm_txq->cntxt_id - sc->sge.eq_start;
+	if (cntxt_id >= sc->sge.neq)
+	    panic("%s: nm_txq->cntxt_id (%d) more than the max (%d)", __func__,
+		cntxt_id, sc->sge.neq - 1);
+	sc->sge.eqmap[cntxt_id] = (void *)nm_txq;
+
+	nm_txq->pidx = nm_txq->cidx = 0;
+	MPASS(nm_txq->sidx == na->num_tx_desc);
+	nm_txq->equiqidx = nm_txq-> equeqidx = nm_txq->dbidx = 0;
+
+	nm_txq->doorbells = sc->doorbells;
+	if (isset(&nm_txq->doorbells, DOORBELL_UDB) ||
+	    isset(&nm_txq->doorbells, DOORBELL_UDBWC) ||
+	    isset(&nm_txq->doorbells, DOORBELL_WCWR)) {
+		uint32_t s_qpp = sc->sge.eq_s_qpp;
+		uint32_t mask = (1 << s_qpp) - 1;
+		volatile uint8_t *udb;
+
+		udb = sc->udbs_base + UDBS_DB_OFFSET;
+		udb += (nm_txq->cntxt_id >> s_qpp) << PAGE_SHIFT;
+		nm_txq->udb_qid = nm_txq->cntxt_id & mask;
+		if (nm_txq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
+	    		clrbit(&nm_txq->doorbells, DOORBELL_WCWR);
+		else {
+			udb += nm_txq->udb_qid << UDBS_SEG_SHIFT;
+			nm_txq->udb_qid = 0;
+		}
+		nm_txq->udb = (volatile void *)udb;
+	}
+
+	return (rc);
+}
+
+static int
+free_nm_txq_hwq(struct port_info *pi, struct sge_nm_txq *nm_txq)
+{
+	struct adapter *sc = pi->adapter;
+	int rc;
+
+	rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, nm_txq->cntxt_id);
+	if (rc != 0)
+		device_printf(sc->dev, "%s: failed for eq %d: %d\n", __func__,
+		    nm_txq->cntxt_id, rc);
+	return (rc);
+}
+
+static int
+cxgbe_netmap_on(struct adapter *sc, struct port_info *pi, struct ifnet *ifp,
+    struct netmap_adapter *na)
+{
+	struct netmap_slot *slot;
+	struct sge_nm_rxq *nm_rxq;
+	struct sge_nm_txq *nm_txq;
+	int rc, i, j, hwidx;
+	struct hw_buf_info *hwb;
+	uint16_t *rss;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+
+	if ((pi->flags & PORT_INIT_DONE) == 0 ||
+	    (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
+		return (EAGAIN);
+
+	hwb = &sc->sge.hw_buf_info[0];
+	for (i = 0; i < SGE_FLBUF_SIZES; i++, hwb++) {
+		if (hwb->size == NETMAP_BUF_SIZE)
+			break;
+	}
+	if (i >= SGE_FLBUF_SIZES) {
+		if_printf(ifp, "no hwidx for netmap buffer size %d.\n",
+		    NETMAP_BUF_SIZE);
+		return (ENXIO);
+	}
+	hwidx = i;
+
+	/* Must set caps before calling netmap_reset */
+	na->na_flags |= (NAF_NATIVE_ON | NAF_NETMAP_ON);
+	ifp->if_capenable |= IFCAP_NETMAP;
+
+	for_each_nm_rxq(pi, i, nm_rxq) {
+		alloc_nm_rxq_hwq(pi, nm_rxq);
+		nm_rxq->fl_hwidx = hwidx;
+		slot = netmap_reset(na, NR_RX, i, 0);
+		MPASS(slot != NULL);	/* XXXNM: error check, not assert */
+
+		/* We deal with 8 bufs at a time */
+		MPASS((na->num_rx_desc & 7) == 0);
+		MPASS(na->num_rx_desc == nm_rxq->fl_sidx);
+		for (j = 0; j < nm_rxq->fl_sidx - 8; j++) {
+			uint64_t ba;
+
+			PNMB(&slot[j], &ba);
+			nm_rxq->fl_desc[j] = htobe64(ba | hwidx);
+		}
+		nm_rxq->fl_pidx = j;
+		MPASS((j & 7) == 0);
+		j /= 8;	/* driver pidx to hardware pidx */
+		wmb();
+		t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+		    nm_rxq->fl_db_val | V_PIDX(j));
+	}
+
+	for_each_nm_txq(pi, i, nm_txq) {
+		alloc_nm_txq_hwq(pi, nm_txq);
+		slot = netmap_reset(na, NR_TX, i, 0);
+		MPASS(slot != NULL);	/* XXXNM: error check, not assert */
+	}
+
+	rss = malloc(pi->nm_rss_size * sizeof (*rss), M_CXGBE, M_ZERO |
+	    M_WAITOK);
+	for (i = 0; i < pi->nm_rss_size;) {
+		for_each_nm_rxq(pi, j, nm_rxq) {
+			rss[i++] = nm_rxq->iq_abs_id;
+			if (i == pi->nm_rss_size)
+				break;
+		}
+	}
+	rc = -t4_config_rss_range(sc, sc->mbox, pi->nm_viid, 0, pi->nm_rss_size,
+	    rss, pi->nm_rss_size);
+	if (rc != 0)
+		if_printf(ifp, "netmap rss_config failed: %d\n", rc);
+	free(rss, M_CXGBE);
+
+	rc = -t4_enable_vi(sc, sc->mbox, pi->nm_viid, true, true);
+	if (rc != 0)
+		if_printf(ifp, "netmap enable_vi failed: %d\n", rc);
+
+	return (rc);
+}
+
+static int
+cxgbe_netmap_off(struct adapter *sc, struct port_info *pi, struct ifnet *ifp,
+    struct netmap_adapter *na)
+{
+	int rc, i;
+	struct sge_nm_txq *nm_txq;
+	struct sge_nm_rxq *nm_rxq;
+
+	ASSERT_SYNCHRONIZED_OP(sc);
+
+	rc = -t4_enable_vi(sc, sc->mbox, pi->nm_viid, false, false);
+	if (rc != 0)
+		if_printf(ifp, "netmap disable_vi failed: %d\n", rc);
+	na->na_flags &= ~(NAF_NATIVE_ON | NAF_NETMAP_ON);
+	ifp->if_capenable &= ~IFCAP_NETMAP;
+
+	/*
+	 * XXXNM: We need to make sure that the tx queues are quiet and won't
+	 * request any more SGE_EGR_UPDATEs.
+	 */
+
+	for_each_nm_txq(pi, i, nm_txq) {
+		free_nm_txq_hwq(pi, nm_txq);
+	}
+	for_each_nm_rxq(pi, i, nm_rxq) {
+		free_nm_rxq_hwq(pi, nm_rxq);
+	}
+
+	return (rc);
+}
+
+static int
+cxgbe_netmap_reg(struct netmap_adapter *na, int on)
+{
+	struct ifnet *ifp = na->ifp;
+	struct port_info *pi = ifp->if_softc;
+	struct adapter *sc = pi->adapter;
+	int rc;
+
+	rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4nmreg");
+	if (rc != 0)
+		return (rc);
+	if (on)
+		rc = cxgbe_netmap_on(sc, pi, ifp, na);
+	else
+		rc = cxgbe_netmap_off(sc, pi, ifp, na);
+	end_synchronized_op(sc, 0);
+
+	return (rc);
+}
+
+/* How many packets can a single type1 WR carry in n descriptors */
+static inline int
+ndesc_to_npkt(const int n)
+{
+
+	MPASS(n > 0 && n <= SGE_MAX_WR_NDESC);
+
+	return (n * 2 - 1);
+}
+#define MAX_NPKT_IN_TYPE1_WR	(ndesc_to_npkt(SGE_MAX_WR_NDESC))
+
+/* Space (in descriptors) needed for a type1 WR that carries n packets */
+static inline int
+npkt_to_ndesc(const int n)
+{
+
+	MPASS(n > 0 && n <= MAX_NPKT_IN_TYPE1_WR);
+
+	return ((n + 2) / 2);
+}
+
+/* Space (in 16B units) needed for a type1 WR that carries n packets */
+static inline int
+npkt_to_len16(const int n)
+{
+
+	MPASS(n > 0 && n <= MAX_NPKT_IN_TYPE1_WR);
+
+	return (n * 2 + 1);
+}
+
+static inline uint16_t
+idxdiff(uint16_t head, uint16_t tail, uint16_t wrap)
+{
+	MPASS(wrap > head);
+	MPASS(wrap > tail);
+
+	if (head >= tail)
+		return (head - tail);
+	else
+		return (wrap - tail + head);
+}
+#define IDXDIFF(q, idx) idxdiff((q)->pidx, (q)->idx, (q)->sidx)
+
+static void
+ring_nm_txq_db(struct adapter *sc, struct sge_nm_txq *nm_txq)
+{
+	int n;
+	u_int db = nm_txq->doorbells;
+
+	MPASS(nm_txq->pidx != nm_txq->dbidx);
+
+	n = IDXDIFF(nm_txq, dbidx);
+	if (n > 1)
+		clrbit(&db, DOORBELL_WCWR);
+	wmb();
+
+	switch (ffs(db) - 1) {
+	case DOORBELL_UDB:
+		*nm_txq->udb = htole32(V_QID(nm_txq->udb_qid) | V_PIDX(n));
+		break;
+
+	case DOORBELL_WCWR: {
+		volatile uint64_t *dst, *src;
+
+		/*
+		 * Queues whose 128B doorbell segment fits in the page do not
+		 * use relative qid (udb_qid is always 0).  Only queues with
+		 * doorbell segments can do WCWR.
+		 */
+		KASSERT(nm_txq->udb_qid == 0 && n == 1,
+		    ("%s: inappropriate doorbell (0x%x, %d, %d) for nm_txq %p",
+		    __func__, nm_txq->doorbells, n, nm_txq->pidx, nm_txq));
+
+		dst = (volatile void *)((uintptr_t)nm_txq->udb +
+		    UDBS_WR_OFFSET - UDBS_DB_OFFSET);
+		src = (void *)&nm_txq->desc[nm_txq->dbidx];
+		while (src != (void *)&nm_txq->desc[nm_txq->dbidx + 1])
+			*dst++ = *src++;
+		wmb();
+		break;
+	}
+
+	case DOORBELL_UDBWC:
+		*nm_txq->udb = htole32(V_QID(nm_txq->udb_qid) | V_PIDX(n));
+		wmb();
+		break;
+
+	case DOORBELL_KDB:
+		t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+		    V_QID(nm_txq->cntxt_id) | V_PIDX(n));
+		break;
+	}
+	nm_txq->dbidx = nm_txq->pidx;
+}
+
+int lazy_tx_credit_flush = 1;
+
+/*
+ * Write work requests to send 'npkt' frames and ring the doorbell to send them
+ * on their way.  No need to check for wraparound.
+ */
+static void
+cxgbe_nm_tx(struct adapter *sc, struct sge_nm_txq *nm_txq,
+    struct netmap_kring *kring, int npkt, int npkt_remaining)
+{
+	struct netmap_ring *ring = kring->ring;
+	struct netmap_slot *slot;
+	const u_int lim = kring->nkr_num_slots - 1;
+	struct fw_eth_tx_pkts_wr *wr = (void *)&nm_txq->desc[nm_txq->pidx];
+	uint16_t len;
+	uint64_t ba;
+	struct cpl_tx_pkt_core *cpl;
+	struct ulptx_sgl *usgl;
+	int i, n;
+
+	while (npkt) {
+		n = min(npkt, MAX_NPKT_IN_TYPE1_WR);
+		len = 0;
+
+		wr = (void *)&nm_txq->desc[nm_txq->pidx];
+		wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
+		wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(npkt_to_len16(n)));
+		wr->npkt = n;
+		wr->r3 = 0;
+		wr->type = 1;
+		cpl = (void *)(wr + 1);
+
+		for (i = 0; i < n; i++) {
+			slot = &ring->slot[kring->nr_hwcur];
+			PNMB(slot, &ba);
+
+			cpl->ctrl0 = nm_txq->cpl_ctrl0;
+			cpl->pack = 0;
+			cpl->len = htobe16(slot->len);
+			/*
+			 * netmap(4) says "netmap does not use features such as
+			 * checksum offloading, TCP segmentation offloading,
+			 * encryption, VLAN encapsulation/decapsulation, etc."
+			 *
+			 * XXXNM: it makes sense to enable checksum offload.
+			 */
+			cpl->ctrl1 = htobe64(F_TXPKT_IPCSUM_DIS |
+			    F_TXPKT_L4CSUM_DIS);
+
+			usgl = (void *)(cpl + 1);
+			usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
+			    V_ULPTX_NSGE(1));
+			usgl->len0 = htobe32(slot->len);
+			usgl->addr0 = htobe64(ba);
+
+			slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
+			cpl = (void *)(usgl + 1);
+			MPASS(slot->len + len <= UINT16_MAX);
+			len += slot->len;
+			kring->nr_hwcur = nm_next(kring->nr_hwcur, lim);
+		}
+		wr->plen = htobe16(len);
+
+		npkt -= n;
+		nm_txq->pidx += npkt_to_ndesc(n);
+		MPASS(nm_txq->pidx <= nm_txq->sidx);
+		if (__predict_false(nm_txq->pidx == nm_txq->sidx)) {
+			/*
+			 * This routine doesn't know how to write WRs that wrap
+			 * around.  Make sure it wasn't asked to.
+			 */
+			MPASS(npkt == 0);
+			nm_txq->pidx = 0;
+		}
+
+		if (npkt == 0 && npkt_remaining == 0) {
+			/* All done. */
+			if (lazy_tx_credit_flush == 0) {
+				wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ |
+				    F_FW_WR_EQUIQ);
+				nm_txq->equeqidx = nm_txq->pidx;
+				nm_txq->equiqidx = nm_txq->pidx;
+			}
+			ring_nm_txq_db(sc, nm_txq);
+			return;
+		}
+
+		if (IDXDIFF(nm_txq, equiqidx) >= nm_txq->sidx / 2) {
+			wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ |
+			    F_FW_WR_EQUIQ);
+			nm_txq->equeqidx = nm_txq->pidx;
+			nm_txq->equiqidx = nm_txq->pidx;
+		} else if (IDXDIFF(nm_txq, equeqidx) >= 64) {
+			wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ);
+			nm_txq->equeqidx = nm_txq->pidx;
+		}
+		if (IDXDIFF(nm_txq, dbidx) >= 2 * SGE_MAX_WR_NDESC)
+			ring_nm_txq_db(sc, nm_txq);
+	}
+
+	/* Will get called again. */
+	MPASS(npkt_remaining);
+}
+
+/* How many contiguous free descriptors starting at pidx */
+static inline int
+contiguous_ndesc_available(struct sge_nm_txq *nm_txq)
+{
+
+	if (nm_txq->cidx > nm_txq->pidx)
+		return (nm_txq->cidx - nm_txq->pidx - 1);
+	else if (nm_txq->cidx > 0)
+		return (nm_txq->sidx - nm_txq->pidx);
+	else
+		return (nm_txq->sidx - nm_txq->pidx - 1);
+}
+
+static int
+reclaim_nm_tx_desc(struct sge_nm_txq *nm_txq)
+{
+	struct sge_qstat *spg = (void *)&nm_txq->desc[nm_txq->sidx];
+	uint16_t hw_cidx = spg->cidx;	/* snapshot */
+	struct fw_eth_tx_pkts_wr *wr;
+	int n = 0;
+
+	hw_cidx = be16toh(hw_cidx);
+
+	while (nm_txq->cidx != hw_cidx) {
+		wr = (void *)&nm_txq->desc[nm_txq->cidx];
+
+		MPASS(wr->op_pkd == htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)));
+		MPASS(wr->type == 1);
+		MPASS(wr->npkt > 0 && wr->npkt <= MAX_NPKT_IN_TYPE1_WR);
+
+		n += wr->npkt;
+		nm_txq->cidx += npkt_to_ndesc(wr->npkt);
+		if (__predict_false(nm_txq->cidx >= nm_txq->sidx))
+			nm_txq->cidx -= nm_txq->sidx;
+	}
+
+	return (n);
+}
+
+static int
+cxgbe_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags)
+{
+	struct netmap_kring *kring = &na->tx_rings[ring_nr];
+	struct ifnet *ifp = na->ifp;
+	struct port_info *pi = ifp->if_softc;
+	struct adapter *sc = pi->adapter;
+	struct sge_nm_txq *nm_txq = &sc->sge.nm_txq[pi->first_nm_txq + ring_nr];
+	const u_int head = kring->rhead;
+	u_int reclaimed = 0;
+	int n, d, npkt_remaining, ndesc_remaining;
+
+	/*
+	 * Tx was at kring->nr_hwcur last time around and now we need to advance
+	 * to kring->rhead.  Note that the driver's pidx moves independent of
+	 * netmap's kring->nr_hwcur (pidx counts descriptors and the relation
+	 * between descriptors and frames isn't 1:1).
+	 */
+
+	npkt_remaining = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
+	    kring->nkr_num_slots - kring->nr_hwcur + head;
+	while (npkt_remaining) {
+		reclaimed += reclaim_nm_tx_desc(nm_txq);
+		ndesc_remaining = contiguous_ndesc_available(nm_txq);
+		/* Can't run out of descriptors with packets still remaining */
+		MPASS(ndesc_remaining > 0);
+
+		/* # of desc needed to tx all remaining packets */
+		d = (npkt_remaining / MAX_NPKT_IN_TYPE1_WR) * SGE_MAX_WR_NDESC;
+		if (npkt_remaining % MAX_NPKT_IN_TYPE1_WR)
+			d += npkt_to_ndesc(npkt_remaining % MAX_NPKT_IN_TYPE1_WR);
+
+		if (d <= ndesc_remaining)
+			n = npkt_remaining;
+		else {
+			/* Can't send all, calculate how many can be sent */
+			n = (ndesc_remaining / SGE_MAX_WR_NDESC) *
+			    MAX_NPKT_IN_TYPE1_WR;
+			if (ndesc_remaining % SGE_MAX_WR_NDESC)
+				n += ndesc_to_npkt(ndesc_remaining % SGE_MAX_WR_NDESC);
+		}
+
+		/* Send n packets and update nm_txq->pidx and kring->nr_hwcur */
+		npkt_remaining -= n;
+		cxgbe_nm_tx(sc, nm_txq, kring, n, npkt_remaining);
+	}
+	MPASS(npkt_remaining == 0);
+	MPASS(kring->nr_hwcur == head);
+	MPASS(nm_txq->dbidx == nm_txq->pidx);
+
+	/*
+	 * Second part: reclaim buffers for completed transmissions.
+	 */
+	if (reclaimed || flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring)) {
+		reclaimed += reclaim_nm_tx_desc(nm_txq);
+		kring->nr_hwtail += reclaimed;
+		if (kring->nr_hwtail >= kring->nkr_num_slots)
+			kring->nr_hwtail -= kring->nkr_num_slots;
+	}
+
+	nm_txsync_finalize(kring);
+
+	return (0);
+}
+
+static int
+cxgbe_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags)
+{
+	struct netmap_kring *kring = &na->rx_rings[ring_nr];
+	struct netmap_ring *ring = kring->ring;
+	struct ifnet *ifp = na->ifp;
+	struct port_info *pi = ifp->if_softc;
+	struct adapter *sc = pi->adapter;
+	struct sge_nm_rxq *nm_rxq = &sc->sge.nm_rxq[pi->first_nm_rxq + ring_nr];
+	u_int const head = nm_rxsync_prologue(kring);
+	u_int n;
+	int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
+
+	if (netmap_no_pendintr || force_update) {
+		kring->nr_hwtail = atomic_load_acq_32(&nm_rxq->fl_cidx);
+		kring->nr_kflags &= ~NKR_PENDINTR;
+	}
+
+	/* Userspace done with buffers from kring->nr_hwcur to head */
+	n = head >= kring->nr_hwcur ? head - kring->nr_hwcur :
+	    kring->nkr_num_slots - kring->nr_hwcur + head;
+	n &= ~7U;
+	if (n > 0) {
+		u_int fl_pidx = nm_rxq->fl_pidx;
+		struct netmap_slot *slot = &ring->slot[fl_pidx];
+		uint64_t ba;
+		int i, dbinc = 0, hwidx = nm_rxq->fl_hwidx;
+
+		/*
+		 * We always deal with 8 buffers at a time.  We must have
+		 * stopped at an 8B boundary (fl_pidx) last time around and we
+		 * must have a multiple of 8B buffers to give to the freelist.
+		 */
+		MPASS((fl_pidx & 7) == 0);
+		MPASS((n & 7) == 0);
+
+		kring->nr_hwcur += n;
+		if (kring->nr_hwcur >= kring->nkr_num_slots)
+			kring->nr_hwcur -= kring->nkr_num_slots;
+
+		nm_rxq->fl_pidx += n;
+		if (nm_rxq->fl_pidx >= nm_rxq->fl_sidx)
+			nm_rxq->fl_pidx -= nm_rxq->fl_sidx;
+
+		while (n > 0) {
+			for (i = 0; i < 8; i++, fl_pidx++, slot++) {
+				PNMB(slot, &ba);
+				nm_rxq->fl_desc[fl_pidx] = htobe64(ba | hwidx);
+				slot->flags &= ~NS_BUF_CHANGED;
+				MPASS(fl_pidx <= nm_rxq->fl_sidx);
+			}
+			n -= 8;
+			if (fl_pidx == nm_rxq->fl_sidx) {
+				fl_pidx = 0;
+				slot = &ring->slot[0];
+			}
+			if (++dbinc == 8 && n >= 32) {
+				wmb();
+				t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+				    nm_rxq->fl_db_val | V_PIDX(dbinc));
+				dbinc = 0;
+			}
+		}
+		MPASS(nm_rxq->fl_pidx == fl_pidx);
+
+		if (dbinc > 0) {
+			wmb();
+			t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
+			    nm_rxq->fl_db_val | V_PIDX(dbinc));
+		}
+	}
+
+	nm_rxsync_finalize(kring);
+
+	return (0);
+}
+
+/*
+ * Create an ifnet solely for netmap use and register it with the kernel.
+ */
+int
+create_netmap_ifnet(struct port_info *pi)
+{
+	struct adapter *sc = pi->adapter;
+	struct netmap_adapter na;
+	struct ifnet *ifp;
+	device_t dev = pi->dev;
+	uint8_t mac[ETHER_ADDR_LEN];
+	int rc;
+
+	if (pi->nnmtxq <= 0 || pi->nnmrxq <= 0)
+		return (0);
+	MPASS(pi->nm_ifp == NULL);
+
+	/*
+	 * Allocate a virtual interface exclusively for netmap use.  Give it the
+	 * MAC address normally reserved for use by a TOE interface.  (The TOE
+	 * driver on FreeBSD doesn't use it).
+	 */
+	rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, &mac[0],
+	    &pi->nm_rss_size, FW_VI_FUNC_OFLD, 0);
+	if (rc < 0) {
+		device_printf(dev, "unable to allocate netmap virtual "
+		    "interface for port %d: %d\n", pi->port_id, -rc);
+		return (-rc);
+	}
+	pi->nm_viid = rc;
+	pi->nm_xact_addr_filt = -1;
+
+	ifp = if_alloc(IFT_ETHER);
+	if (ifp == NULL) {
+		device_printf(dev, "Cannot allocate netmap ifnet\n");
+		return (ENOMEM);
+	}
+	pi->nm_ifp = ifp;
+	ifp->if_softc = pi;
+
+	if_initname(ifp, is_t4(pi->adapter) ? "ncxgbe" : "ncxl",
+	    device_get_unit(dev));
+	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+
+	ifp->if_init = cxgbe_nm_init;
+	ifp->if_ioctl = cxgbe_nm_ioctl;
+	ifp->if_transmit = cxgbe_nm_transmit;
+	ifp->if_qflush = cxgbe_nm_qflush;
+
+	/*
+	 * netmap(4) says "netmap does not use features such as checksum
+	 * offloading, TCP segmentation offloading, encryption, VLAN
+	 * encapsulation/decapsulation, etc."
+	 *
+	 * By default we comply with the statement above.  But we do declare the
+	 * ifnet capable of L3/L4 checksumming so that a user can override
+	 * netmap and have the hardware do the L3/L4 checksums.
+	 */
+	ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU |
+	    IFCAP_HWCSUM_IPV6;
+	ifp->if_capenable = 0;
+	ifp->if_hwassist = 0;
+
+	/* nm_media has already been setup by the caller */
+
+	ether_ifattach(ifp, mac);
+
+	/*
+	 * Register with netmap in the kernel.
+	 */
+	bzero(&na, sizeof(na));
+
+	na.ifp = pi->nm_ifp;
+	na.na_flags = NAF_BDG_MAYSLEEP;
+
+	/* Netmap doesn't know about the space reserved for the status page. */
+	na.num_tx_desc = pi->qsize_txq - spg_len / EQ_ESIZE;
+
+	/*
+	 * The freelist's cidx/pidx drives netmap's rx cidx/pidx.  So
+	 * num_rx_desc is based on the number of buffers that can be held in the
+	 * freelist, and not the number of entries in the iq.  (These two are
+	 * not exactly the same due to the space taken up by the status page).
+	 */
+	na.num_rx_desc = (pi->qsize_rxq / 8) * 8;
+	na.nm_txsync = cxgbe_netmap_txsync;
+	na.nm_rxsync = cxgbe_netmap_rxsync;
+	na.nm_register = cxgbe_netmap_reg;
+	na.num_tx_rings = pi->nnmtxq;
+	na.num_rx_rings = pi->nnmrxq;
+	netmap_attach(&na);	/* This adds IFCAP_NETMAP to if_capabilities */
+
+	return (0);
+}
+
+int
+destroy_netmap_ifnet(struct port_info *pi)
+{
+	struct adapter *sc = pi->adapter;
+
+	if (pi->nm_ifp == NULL)
+		return (0);
+
+	netmap_detach(pi->nm_ifp);
+	ifmedia_removeall(&pi->nm_media);
+	ether_ifdetach(pi->nm_ifp);
+	if_free(pi->nm_ifp);
+	t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->nm_viid);
+
+	return (0);
+}
+
+static void
+handle_nm_fw6_msg(struct adapter *sc, struct ifnet *ifp,
+    const struct cpl_fw6_msg *cpl)
+{
+	const struct cpl_sge_egr_update *egr;
+	uint32_t oq;
+	struct sge_nm_txq *nm_txq;
+
+	if (cpl->type != FW_TYPE_RSSCPL && cpl->type != FW6_TYPE_RSSCPL)
+		panic("%s: FW_TYPE 0x%x on nm_rxq.", __func__, cpl->type);
+
+	/* data[0] is RSS header */
+	egr = (const void *)&cpl->data[1];
+	oq = be32toh(egr->opcode_qid);
+	MPASS(G_CPL_OPCODE(oq) == CPL_SGE_EGR_UPDATE);
+	nm_txq = (void *)sc->sge.eqmap[G_EGR_QID(oq) - sc->sge.eq_start];
+
+	netmap_tx_irq(ifp, nm_txq->nid);
+}
+
+void
+t4_nm_intr(void *arg)
+{
+	struct sge_nm_rxq *nm_rxq = arg;
+	struct port_info *pi = nm_rxq->pi;
+	struct adapter *sc = pi->adapter;
+	struct ifnet *ifp = pi->nm_ifp;
+	struct netmap_adapter *na = NA(ifp);
+	struct netmap_kring *kring = &na->rx_rings[nm_rxq->nid];
+	struct netmap_ring *ring = kring->ring;
+	struct nm_iq_desc *d = &nm_rxq->iq_desc[nm_rxq->iq_cidx];
+	uint32_t lq;
+	u_int n = 0;
+	int processed = 0;
+	uint8_t opcode;
+	uint32_t fl_cidx = atomic_load_acq_32(&nm_rxq->fl_cidx);
+
+	while ((d->rsp.u.type_gen & F_RSPD_GEN) == nm_rxq->iq_gen) {
+
+		rmb();
+
+		lq = be32toh(d->rsp.pldbuflen_qid);
+		opcode = d->rss.opcode;
+
+		switch (G_RSPD_TYPE(d->rsp.u.type_gen)) {
+		case X_RSPD_TYPE_FLBUF:
+			/* No buffer packing so new buf every time */
+			MPASS(lq & F_RSPD_NEWBUF);
+
+			/* fall through */
+
+		case X_RSPD_TYPE_CPL:
+			MPASS(opcode < NUM_CPL_CMDS);
+
+			switch (opcode) {
+			case CPL_FW4_MSG:
+			case CPL_FW6_MSG:
+				handle_nm_fw6_msg(sc, ifp, &d->u.fw6_msg);
+				break;
+			case CPL_RX_PKT:
+				ring->slot[fl_cidx].len = G_RSPD_LEN(lq) - fl_pktshift;
+				ring->slot[fl_cidx].flags = kring->nkr_slot_flags;
+				if (__predict_false(++fl_cidx == nm_rxq->fl_sidx))
+					fl_cidx = 0;
+				break;
+			default:
+				panic("%s: unexpected opcode 0x%x on nm_rxq %p",
+				    __func__, opcode, nm_rxq);
+			}
+			break;
+
+		case X_RSPD_TYPE_INTR:
+			/* Not equipped to handle forwarded interrupts. */
+			panic("%s: netmap queue received interrupt for iq %u\n",
+			    __func__, lq);
+
+		default:
+			panic("%s: illegal response type %d on nm_rxq %p",
+			    __func__, G_RSPD_TYPE(d->rsp.u.type_gen), nm_rxq);
+		}
+
+		d++;
+		if (__predict_false(++nm_rxq->iq_cidx == nm_rxq->iq_sidx)) {
+			nm_rxq->iq_cidx = 0;
+			d = &nm_rxq->iq_desc[0];
+			nm_rxq->iq_gen ^= F_RSPD_GEN;
+		}
+
+		if (__predict_false(++n == 64)) {	/* XXXNM: tune */
+			t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
+			    V_CIDXINC(n) | V_INGRESSQID(nm_rxq->iq_cntxt_id) |
+			    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
+			n = 0;
+		}
+	}
+	if (fl_cidx != nm_rxq->fl_cidx) {
+		atomic_store_rel_32(&nm_rxq->fl_cidx, fl_cidx);
+		netmap_rx_irq(ifp, nm_rxq->nid, &processed);
+	}
+	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(n) |
+	    V_INGRESSQID((u32)nm_rxq->iq_cntxt_id) | V_SEINTARM(F_QINTR_CNT_EN));
+}
+#endif
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index d461723..0fd3f93 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -55,6 +55,13 @@ __FBSDID("$FreeBSD$");
 #include <machine/md_var.h>
 #include <vm/vm.h>
 #include <vm/pmap.h>
+#ifdef DEV_NETMAP
+#include <machine/bus.h>
+#include <sys/selinfo.h>
+#include <net/if_var.h>
+#include <net/netmap.h>
+#include <dev/netmap/netmap_kern.h>
+#endif
 
 #include "common/common.h"
 #include "common/t4_regs.h"
@@ -71,7 +78,7 @@ __FBSDID("$FreeBSD$");
  * Ethernet frames are DMA'd at this byte offset into the freelist buffer.
  * 0-7 are valid values.
  */
-static int fl_pktshift = 2;
+int fl_pktshift = 2;
 TUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift);
 
 /*
@@ -80,7 +87,7 @@ TUNABLE_INT("hw.cxgbe.fl_pktshift", &fl_pktshift);
  *  0: disable padding.
  *  Any power of 2 from 32 to 4096 (both inclusive) is also a valid value.
  */
-static int fl_pad = -1;
+int fl_pad = -1;
 TUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad);
 
 /*
@@ -88,7 +95,7 @@ TUNABLE_INT("hw.cxgbe.fl_pad", &fl_pad);
  * -1: driver should figure out a good value.
  *  64 or 128 are the only other valid values.
  */
-static int spg_len = -1;
+int spg_len = -1;
 TUNABLE_INT("hw.cxgbe.spg_len", &spg_len);
 
 /*
@@ -194,6 +201,14 @@ static int alloc_ofld_rxq(struct port_info *, struct sge_ofld_rxq *, int, int,
     struct sysctl_oid *);
 static int free_ofld_rxq(struct port_info *, struct sge_ofld_rxq *);
 #endif
+#ifdef DEV_NETMAP
+static int alloc_nm_rxq(struct port_info *, struct sge_nm_rxq *, int, int,
+    struct sysctl_oid *);
+static int free_nm_rxq(struct port_info *, struct sge_nm_rxq *);
+static int alloc_nm_txq(struct port_info *, struct sge_nm_txq *, int, int,
+    struct sysctl_oid *);
+static int free_nm_txq(struct port_info *, struct sge_nm_txq *);
+#endif
 static int ctrl_eq_alloc(struct adapter *, struct sge_eq *);
 static int eth_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
 #ifdef TCP_OFFLOAD
@@ -809,6 +824,24 @@ t4_teardown_adapter_queues(struct adapter *sc)
 }
 
 static inline int
+port_intr_count(struct port_info *pi)
+{
+	int rc = 0;
+
+	if (pi->flags & INTR_RXQ)
+		rc += pi->nrxq;
+#ifdef TCP_OFFLOAD
+	if (pi->flags & INTR_OFLD_RXQ)
+		rc += pi->nofldrxq;
+#endif
+#ifdef DEV_NETMAP
+	if (pi->flags & INTR_NM_RXQ)
+		rc += pi->nnmrxq;
+#endif
+	return (rc);
+}
+
+static inline int
 first_vector(struct port_info *pi)
 {
 	struct adapter *sc = pi->adapter;
@@ -818,28 +851,10 @@ first_vector(struct port_info *pi)
 		return (0);
 
 	for_each_port(sc, i) {
-		struct port_info *p = sc->port[i];
-
 		if (i == pi->port_id)
 			break;
 
-#ifdef TCP_OFFLOAD
-		if (sc->flags & INTR_DIRECT)
-			rc += p->nrxq + p->nofldrxq;
-		else
-			rc += max(p->nrxq, p->nofldrxq);
-#else
-		/*
-		 * Not compiled with offload support and intr_count > 1.  Only
-		 * NIC queues exist and they'd better be taking direct
-		 * interrupts.
-		 */
-		KASSERT(sc->flags & INTR_DIRECT,
-		    ("%s: intr_count %d, !INTR_DIRECT", __func__,
-		    sc->intr_count));
-
-		rc += p->nrxq;
-#endif
+		rc += port_intr_count(sc->port[i]);
 	}
 
 	return (rc);
@@ -856,41 +871,48 @@ port_intr_iq(struct port_info *pi, int idx)
 	struct adapter *sc = pi->adapter;
 	struct sge *s = &sc->sge;
 	struct sge_iq *iq = NULL;
+	int nintr, i;
 
 	if (sc->intr_count == 1)
 		return (&sc->sge.fwq);
 
-#ifdef TCP_OFFLOAD
-	if (sc->flags & INTR_DIRECT) {
-		idx %= pi->nrxq + pi->nofldrxq;
-
-		if (idx >= pi->nrxq) {
-			idx -= pi->nrxq;
-			iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
-		} else
-			iq = &s->rxq[pi->first_rxq + idx].iq;
-
-	} else {
-		idx %= max(pi->nrxq, pi->nofldrxq);
+	nintr = port_intr_count(pi);
+	KASSERT(nintr != 0,
+	    ("%s: pi %p has no exclusive interrupts, total interrupts = %d",
+	    __func__, pi, sc->intr_count));
+#ifdef DEV_NETMAP
+	/* Exclude netmap queues as they can't take anyone else's interrupts */
+	if (pi->flags & INTR_NM_RXQ)
+		nintr -= pi->nnmrxq;
+	KASSERT(nintr > 0,
+	    ("%s: pi %p has nintr %d after netmap adjustment of %d", __func__,
+	    pi, nintr, pi->nnmrxq));
+#endif
+	i = idx % nintr;
 
-		if (pi->nrxq >= pi->nofldrxq)
-			iq = &s->rxq[pi->first_rxq + idx].iq;
-		else
-			iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
+	if (pi->flags & INTR_RXQ) {
+	       	if (i < pi->nrxq) {
+			iq = &s->rxq[pi->first_rxq + i].iq;
+			goto done;
+		}
+		i -= pi->nrxq;
+	}
+#ifdef TCP_OFFLOAD
+	if (pi->flags & INTR_OFLD_RXQ) {
+	       	if (i < pi->nofldrxq) {
+			iq = &s->ofld_rxq[pi->first_ofld_rxq + i].iq;
+			goto done;
+		}
+		i -= pi->nofldrxq;
 	}
-#else
-	/*
-	 * Not compiled with offload support and intr_count > 1.  Only NIC
-	 * queues exist and they'd better be taking direct interrupts.
-	 */
-	KASSERT(sc->flags & INTR_DIRECT,
-	    ("%s: intr_count %d, !INTR_DIRECT", __func__, sc->intr_count));
-
-	idx %= pi->nrxq;
-	iq = &s->rxq[pi->first_rxq + idx].iq;
 #endif
-
-	KASSERT(iq->flags & IQ_INTR, ("%s: EDOOFUS", __func__));
+	panic("%s: pi %p, intr_flags 0x%lx, idx %d, total intr %d\n", __func__,
+	    pi, pi->flags & INTR_ALL, idx, nintr);
+done:
+	MPASS(iq != NULL);
+	KASSERT(iq->flags & IQ_INTR,
+	    ("%s: iq %p (port %p, intr_flags 0x%lx, idx %d)", __func__, iq, pi,
+	    pi->flags & INTR_ALL, idx));
 	return (iq);
 }
 
@@ -927,7 +949,10 @@ t4_setup_port_queues(struct port_info *pi)
 #ifdef TCP_OFFLOAD
 	struct sge_ofld_rxq *ofld_rxq;
 	struct sge_wrq *ofld_txq;
-	struct sysctl_oid *oid2 = NULL;
+#endif
+#ifdef DEV_NETMAP
+	struct sge_nm_rxq *nm_rxq;
+	struct sge_nm_txq *nm_txq;
 #endif
 	char name[16];
 	struct adapter *sc = pi->adapter;
@@ -936,27 +961,20 @@ t4_setup_port_queues(struct port_info *pi)
 	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
 	int maxp, pack, mtu = ifp->if_mtu;
 
-	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD,
-	    NULL, "rx queues");
-
-#ifdef TCP_OFFLOAD
-	if (is_offload(sc)) {
-		oid2 = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
-		    CTLFLAG_RD, NULL,
-		    "rx queues for offloaded TCP connections");
-	}
-#endif
-
 	/* Interrupt vector to start from (when using multiple vectors) */
 	intr_idx = first_vector(pi);
 
 	/*
-	 * First pass over all rx queues (NIC and TOE):
+	 * First pass over all NIC and TOE rx queues:
 	 * a) initialize iq and fl
 	 * b) allocate queue iff it will take direct interrupts.
 	 */
 	maxp = mtu_to_max_payload(sc, mtu, 0);
 	pack = enable_buffer_packing(sc);
+	if (pi->flags & INTR_RXQ) {
+		oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq",
+		    CTLFLAG_RD, NULL, "rx queues");
+	}
 	for_each_rxq(pi, i, rxq) {
 
 		init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq,
@@ -966,11 +984,7 @@ t4_setup_port_queues(struct port_info *pi)
 		    device_get_nameunit(pi->dev), i);
 		init_fl(sc, &rxq->fl, pi->qsize_rxq / 8, maxp, pack, name);
 
-		if (sc->flags & INTR_DIRECT
-#ifdef TCP_OFFLOAD
-		    || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)
-#endif
-		   ) {
+		if (pi->flags & INTR_RXQ) {
 			rxq->iq.flags |= IQ_INTR;
 			rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
 			if (rc != 0)
@@ -978,9 +992,13 @@ t4_setup_port_queues(struct port_info *pi)
 			intr_idx++;
 		}
 	}
-
 #ifdef TCP_OFFLOAD
 	maxp = mtu_to_max_payload(sc, mtu, 1);
+	if (is_offload(sc) && pi->flags & INTR_OFLD_RXQ) {
+		oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
+		    CTLFLAG_RD, NULL,
+		    "rx queues for offloaded TCP connections");
+	}
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 
 		init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
@@ -990,10 +1008,26 @@ t4_setup_port_queues(struct port_info *pi)
 		    device_get_nameunit(pi->dev), i);
 		init_fl(sc, &ofld_rxq->fl, pi->qsize_rxq / 8, maxp, pack, name);
 
-		if (sc->flags & INTR_DIRECT ||
-		    (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
+		if (pi->flags & INTR_OFLD_RXQ) {
 			ofld_rxq->iq.flags |= IQ_INTR;
-			rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid2);
+			rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid);
+			if (rc != 0)
+				goto done;
+			intr_idx++;
+		}
+	}
+#endif
+#ifdef DEV_NETMAP
+	/*
+	 * We don't have buffers to back the netmap rx queues right now so we
+	 * create the queues in a way that doesn't set off any congestion signal
+	 * in the chip.
+	 */
+	if (pi->flags & INTR_NM_RXQ) {
+		oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "nm_rxq",
+		    CTLFLAG_RD, NULL, "rx queues for netmap");
+		for_each_nm_rxq(pi, i, nm_rxq) {
+			rc = alloc_nm_rxq(pi, nm_rxq, intr_idx, i, oid);
 			if (rc != 0)
 				goto done;
 			intr_idx++;
@@ -1002,35 +1036,45 @@ t4_setup_port_queues(struct port_info *pi)
 #endif
 
 	/*
-	 * Second pass over all rx queues (NIC and TOE).  The queues forwarding
+	 * Second pass over all NIC and TOE rx queues.  The queues forwarding
 	 * their interrupts are allocated now.
 	 */
 	j = 0;
-	for_each_rxq(pi, i, rxq) {
-		if (rxq->iq.flags & IQ_INTR)
-			continue;
+	if (!(pi->flags & INTR_RXQ)) {
+		oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq",
+		    CTLFLAG_RD, NULL, "rx queues");
+		for_each_rxq(pi, i, rxq) {
+			MPASS(!(rxq->iq.flags & IQ_INTR));
 
-		intr_idx = port_intr_iq(pi, j)->abs_id;
+			intr_idx = port_intr_iq(pi, j)->abs_id;
 
-		rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
-		if (rc != 0)
-			goto done;
-		j++;
+			rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
+			if (rc != 0)
+				goto done;
+			j++;
+		}
 	}
-
 #ifdef TCP_OFFLOAD
-	for_each_ofld_rxq(pi, i, ofld_rxq) {
-		if (ofld_rxq->iq.flags & IQ_INTR)
-			continue;
+	if (is_offload(sc) && !(pi->flags & INTR_OFLD_RXQ)) {
+		oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
+		    CTLFLAG_RD, NULL,
+		    "rx queues for offloaded TCP connections");
+		for_each_ofld_rxq(pi, i, ofld_rxq) {
+			MPASS(!(ofld_rxq->iq.flags & IQ_INTR));
 
-		intr_idx = port_intr_iq(pi, j)->abs_id;
+			intr_idx = port_intr_iq(pi, j)->abs_id;
 
-		rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid2);
-		if (rc != 0)
-			goto done;
-		j++;
+			rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid);
+			if (rc != 0)
+				goto done;
+			j++;
+		}
 	}
 #endif
+#ifdef DEV_NETMAP
+	if (!(pi->flags & INTR_NM_RXQ))
+		CXGBE_UNIMPLEMENTED(__func__);
+#endif
 
 	/*
 	 * Now the tx queues.  Only one pass needed.
@@ -1039,10 +1083,7 @@ t4_setup_port_queues(struct port_info *pi)
 	    NULL, "tx queues");
 	j = 0;
 	for_each_txq(pi, i, txq) {
-		uint16_t iqid;
-
 		iqid = port_intr_iq(pi, j)->cntxt_id;
-
 		snprintf(name, sizeof(name), "%s txq%d",
 		    device_get_nameunit(pi->dev), i);
 		init_eq(&txq->eq, EQ_ETH, pi->qsize_txq, pi->tx_chan, iqid,
@@ -1053,15 +1094,13 @@ t4_setup_port_queues(struct port_info *pi)
 			goto done;
 		j++;
 	}
-
 #ifdef TCP_OFFLOAD
 	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_txq",
 	    CTLFLAG_RD, NULL, "tx queues for offloaded TCP connections");
 	for_each_ofld_txq(pi, i, ofld_txq) {
-		uint16_t iqid;
+		struct sysctl_oid *oid2;
 
 		iqid = port_intr_iq(pi, j)->cntxt_id;
-
 		snprintf(name, sizeof(name), "%s ofld_txq%d",
 		    device_get_nameunit(pi->dev), i);
 		init_eq(&ofld_txq->eq, EQ_OFLD, pi->qsize_txq, pi->tx_chan,
@@ -1077,6 +1116,17 @@ t4_setup_port_queues(struct port_info *pi)
 		j++;
 	}
 #endif
+#ifdef DEV_NETMAP
+	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "nm_txq",
+	    CTLFLAG_RD, NULL, "tx queues for netmap use");
+	for_each_nm_txq(pi, i, nm_txq) {
+		iqid = pi->first_nm_rxq + (j % pi->nnmrxq);
+		rc = alloc_nm_txq(pi, nm_txq, iqid, i, oid);
+		if (rc != 0)
+			goto done;
+		j++;
+	}
+#endif
 
 	/*
 	 * Finally, the control queue.
@@ -1110,6 +1160,10 @@ t4_teardown_port_queues(struct port_info *pi)
 	struct sge_ofld_rxq *ofld_rxq;
 	struct sge_wrq *ofld_txq;
 #endif
+#ifdef DEV_NETMAP
+	struct sge_nm_rxq *nm_rxq;
+	struct sge_nm_txq *nm_txq;
+#endif
 
 	/* Do this before freeing the queues */
 	if (pi->flags & PORT_SYSCTL_CTX) {
@@ -1127,12 +1181,15 @@ t4_teardown_port_queues(struct port_info *pi)
 	for_each_txq(pi, i, txq) {
 		free_txq(pi, txq);
 	}
-
 #ifdef TCP_OFFLOAD
 	for_each_ofld_txq(pi, i, ofld_txq) {
 		free_wrq(sc, ofld_txq);
 	}
 #endif
+#ifdef DEV_NETMAP
+	for_each_nm_txq(pi, i, nm_txq)
+	    free_nm_txq(pi, nm_txq);
+#endif
 
 	/*
 	 * Then take down the rx queues that forward their interrupts, as they
@@ -1143,13 +1200,16 @@ t4_teardown_port_queues(struct port_info *pi)
 		if ((rxq->iq.flags & IQ_INTR) == 0)
 			free_rxq(pi, rxq);
 	}
-
 #ifdef TCP_OFFLOAD
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 		if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
 			free_ofld_rxq(pi, ofld_rxq);
 	}
 #endif
+#ifdef DEV_NETMAP
+	for_each_nm_rxq(pi, i, nm_rxq)
+	    free_nm_rxq(pi, nm_rxq);
+#endif
 
 	/*
 	 * Then take down the rx queues that take direct interrupts.
@@ -1159,13 +1219,15 @@ t4_teardown_port_queues(struct port_info *pi)
 		if (rxq->iq.flags & IQ_INTR)
 			free_rxq(pi, rxq);
 	}
-
 #ifdef TCP_OFFLOAD
 	for_each_ofld_rxq(pi, i, ofld_rxq) {
 		if (ofld_rxq->iq.flags & IQ_INTR)
 			free_ofld_rxq(pi, ofld_rxq);
 	}
 #endif
+#ifdef DEV_NETMAP
+	CXGBE_UNIMPLEMENTED(__func__);
+#endif
 
 	return (0);
 }
@@ -2558,6 +2620,143 @@ free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq)
 }
 #endif
 
+#ifdef DEV_NETMAP
+static int
+alloc_nm_rxq(struct port_info *pi, struct sge_nm_rxq *nm_rxq, int intr_idx,
+    int idx, struct sysctl_oid *oid)
+{
+	int rc;
+	struct sysctl_oid_list *children;
+	struct sysctl_ctx_list *ctx;
+	char name[16];
+	size_t len;
+	struct adapter *sc = pi->adapter;
+	struct netmap_adapter *na = NA(pi->nm_ifp);
+
+	MPASS(na != NULL);
+
+	len = pi->qsize_rxq * RX_IQ_ESIZE;
+	rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map,
+	    &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc);
+	if (rc != 0)
+		return (rc);
+
+	len = na->num_rx_desc * RX_FL_ESIZE + spg_len;
+	rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map,
+	    &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc);
+	if (rc != 0)
+		return (rc);
+
+	nm_rxq->pi = pi;
+	nm_rxq->nid = idx;
+	nm_rxq->iq_cidx = 0;
+	nm_rxq->iq_sidx = pi->qsize_rxq - spg_len / RX_IQ_ESIZE;
+	nm_rxq->iq_gen = F_RSPD_GEN;
+	nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0;
+	nm_rxq->fl_sidx = na->num_rx_desc;
+	nm_rxq->intr_idx = intr_idx;
+
+	ctx = &pi->ctx;
+	children = SYSCTL_CHILDREN(oid);
+
+	snprintf(name, sizeof(name), "%d", idx);
+	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name, CTLFLAG_RD, NULL,
+	    "rx queue");
+	children = SYSCTL_CHILDREN(oid);
+
+	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_abs_id, 0, sysctl_uint16,
+	    "I", "absolute id of the queue");
+	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cntxt_id, 0, sysctl_uint16,
+	    "I", "SGE context id of the queue");
+	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->iq_cidx, 0, sysctl_uint16, "I",
+	    "consumer index");
+
+	children = SYSCTL_CHILDREN(oid);
+	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", CTLFLAG_RD, NULL,
+	    "freelist");
+	children = SYSCTL_CHILDREN(oid);
+
+	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_rxq->fl_cntxt_id, 0, sysctl_uint16,
+	    "I", "SGE context id of the freelist");
+	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD,
+	    &nm_rxq->fl_cidx, 0, "consumer index");
+	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD,
+	    &nm_rxq->fl_pidx, 0, "producer index");
+
+	return (rc);
+}
+
+
+static int
+free_nm_rxq(struct port_info *pi, struct sge_nm_rxq *nm_rxq)
+{
+	struct adapter *sc = pi->adapter;
+
+	free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba,
+	    nm_rxq->iq_desc);
+	free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba,
+	    nm_rxq->fl_desc);
+
+	return (0);
+}
+
+static int
+alloc_nm_txq(struct port_info *pi, struct sge_nm_txq *nm_txq, int iqidx, int idx,
+    struct sysctl_oid *oid)
+{
+	int rc;
+	size_t len;
+	struct adapter *sc = pi->adapter;
+	struct netmap_adapter *na = NA(pi->nm_ifp);
+	char name[16];
+	struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
+
+	len = na->num_tx_desc * EQ_ESIZE + spg_len;
+	rc = alloc_ring(sc, len, &nm_txq->desc_tag, &nm_txq->desc_map,
+	    &nm_txq->ba, (void **)&nm_txq->desc);
+	if (rc)
+		return (rc);
+
+	nm_txq->pidx = nm_txq->cidx = 0;
+	nm_txq->sidx = na->num_tx_desc;
+	nm_txq->nid = idx;
+	nm_txq->iqidx = iqidx;
+	nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) |
+	    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf));
+
+	snprintf(name, sizeof(name), "%d", idx);
+	oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
+	    NULL, "netmap tx queue");
+	children = SYSCTL_CHILDREN(oid);
+
+	SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
+	    &nm_txq->cntxt_id, 0, "SGE context id of the queue");
+	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_txq->cidx, 0, sysctl_uint16, "I",
+	    "consumer index");
+	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "pidx",
+	    CTLTYPE_INT | CTLFLAG_RD, &nm_txq->pidx, 0, sysctl_uint16, "I",
+	    "producer index");
+
+	return (rc);
+}
+
+static int
+free_nm_txq(struct port_info *pi, struct sge_nm_txq *nm_txq)
+{
+	struct adapter *sc = pi->adapter;
+
+	free_ring(sc, nm_txq->desc_tag, nm_txq->desc_map, nm_txq->ba,
+	    nm_txq->desc);
+
+	return (0);
+}
+#endif
+
 static int
 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
 {
@@ -2986,7 +3185,7 @@ static inline void
 iq_next(struct sge_iq *iq)
 {
 	iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
-	if (__predict_false(++iq->cidx == iq->qsize - 1)) {
+	if (__predict_false(++iq->cidx == iq->qsize - spg_len / iq->esize)) {
 		iq->cidx = 0;
 		iq->gen ^= 1;
 		iq->cdesc = iq->desc;
diff --git a/sys/modules/cxgbe/if_cxgbe/Makefile b/sys/modules/cxgbe/if_cxgbe/Makefile
index 665feb9..cc2ba6e 100644
--- a/sys/modules/cxgbe/if_cxgbe/Makefile
+++ b/sys/modules/cxgbe/if_cxgbe/Makefile
@@ -8,7 +8,7 @@ CXGBE = ${.CURDIR}/../../../dev/cxgbe
 .PATH: ${CXGBE} ${CXGBE}/common
 
 KMOD = if_cxgbe
-SRCS = t4_main.c t4_sge.c t4_l2t.c t4_tracer.c
+SRCS = t4_main.c t4_sge.c t4_l2t.c t4_tracer.c t4_netmap.c
 SRCS+= t4_hw.c
 SRCS+= device_if.h bus_if.h pci_if.h
 SRCS+= opt_inet.h opt_inet6.h