A bunch of netmap fixes:

USERSPACE:
1. add support for devices with different number of rx and tx queues;

2. add better support for zero-copy operation, adding an extra field
   to the netmap ring to indicate how many buffers we have already processed
   but not yet released (with help from Eddie Kohler);

3. The two changes above unfortunately require an API change, so while
   at it add a version field and some spares to the ioctl() argument
   to help detect mismatches.

4. update the manual page for the two changes above;

5. update sample applications in tools/tools/netmap

KERNEL:

1. simplify the internal structures moving the global wait queues
   to the 'struct netmap_adapter';

2. simplify the functions that map kring<->nic ring indexes

3. normalize device-specific code, helps mainteinance;

4. start exploring the impact of micro-optimizations (prefetch etc.)
   in the ixgbe driver.
   Use 'legacy' descriptors on the tx ring and prefetch slots gives
   about 20% speedup at 900 MHz. Another 7-10% would come from removing
   the explict calls to bus_dmamap* in the core (they are effectively
   NOPs in this case, but it takes expensive load of the per-buffer
   dma maps to figure out that they are all NULL.

   Rx performance not investigated.

I am postponing the MFC so i can import a few more improvements
before merging.

1 files changed, 72 insertions, 85 deletions

diff --git a/sys/dev/netmap/ixgbe_netmap.h b/sys/dev/netmap/ixgbe_netmap.h
index b0a203a..c711110 100644
--- a/sys/dev/netmap/ixgbe_netmap.h
+++ b/sys/dev/netmap/ixgbe_netmap.h
@@ -25,7 +25,7 @@
 
 /*
  * $FreeBSD$
- * $Id: ixgbe_netmap.h 9802 2011-12-02 18:42:37Z luigi $
+ * $Id: ixgbe_netmap.h 10627 2012-02-23 19:37:15Z luigi $
  *
  * netmap modifications for ixgbe
  *
@@ -47,44 +47,8 @@
 #include <vm/pmap.h>
 
  */
-
 #include <dev/netmap/netmap_kern.h>
 
-/*
- * prototypes for the new API calls that are used by the
- * *_netmap_attach() routine.
- */
-static int	ixgbe_netmap_reg(struct ifnet *, int onoff);
-static int	ixgbe_netmap_txsync(struct ifnet *, u_int, int);
-static int	ixgbe_netmap_rxsync(struct ifnet *, u_int, int);
-static void	ixgbe_netmap_lock_wrapper(struct ifnet *, int, u_int);
-
-
-/*
- * The attach routine, called near the end of ixgbe_attach(),
- * fills the parameters for netmap_attach() and calls it.
- * It cannot fail, in the worst case (such as no memory)
- * netmap mode will be disabled and the driver will only
- * operate in standard mode.
- */
-static void
-ixgbe_netmap_attach(struct adapter *adapter)
-{
-	struct netmap_adapter na;
-
-	bzero(&na, sizeof(na));
-
-	na.ifp = adapter->ifp;
-	na.separate_locks = 1;	/* this card has separate rx/tx locks */
-	na.num_tx_desc = adapter->num_tx_desc;
-	na.num_rx_desc = adapter->num_rx_desc;
-	na.nm_txsync = ixgbe_netmap_txsync;
-	na.nm_rxsync = ixgbe_netmap_rxsync;
-	na.nm_lock = ixgbe_netmap_lock_wrapper;
-	na.nm_register = ixgbe_netmap_reg;
-	netmap_attach(&na, adapter->num_queues);
-}	
-
 
 /*
  * wrapper to export locks to the generic netmap code.
@@ -119,7 +83,7 @@ ixgbe_netmap_lock_wrapper(struct ifnet *_a, int what, u_int queueid)
 
 
 /*
- * Netmap register/unregister. We are already under core lock.
+ * Register/unregister. We are already under core lock.
  * Only called on the first register or the last unregister.
  */
 static int
@@ -129,8 +93,8 @@ ixgbe_netmap_reg(struct ifnet *ifp, int onoff)
 	struct netmap_adapter *na = NA(ifp);
 	int error = 0;
 
-	if (!na) /* probably, netmap_attach() failed */
-		return EINVAL;
+	if (na == NULL)
+		return EINVAL; /* no netmap support here */
 
 	ixgbe_disable_intr(adapter);
 
@@ -197,7 +161,7 @@ ixgbe_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	struct netmap_adapter *na = NA(adapter->ifp);
 	struct netmap_kring *kring = &na->tx_rings[ring_nr];
 	struct netmap_ring *ring = kring->ring;
-	int j, k, l, n = 0, lim = kring->nkr_num_slots - 1;
+	u_int j, k = ring->cur, l, n = 0, lim = kring->nkr_num_slots - 1;
 
 	/*
 	 * ixgbe can generate an interrupt on every tx packet, but it
@@ -206,20 +170,10 @@ ixgbe_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	 */
 	int report_frequency = kring->nkr_num_slots >> 1;
 
+	if (k > lim)
+		return netmap_ring_reinit(kring);
 	if (do_lock)
 		IXGBE_TX_LOCK(txr);
-	/* take a copy of ring->cur now, and never read it again */
-	k = ring->cur;
-	/* do a sanity check on cur - hwcur XXX verify */
-	l = k - kring->nr_hwcur;
-	if (l < 0)
-		l += lim + 1;
-	/* if cur is invalid reinitialize the ring. */
-	if (k > lim || l > kring->nr_hwavail) {
-		if (do_lock)
-			IXGBE_TX_UNLOCK(txr);
-		return netmap_ring_reinit(kring);
-	}
 
 	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
 			BUS_DMASYNC_POSTREAD);
@@ -241,7 +195,9 @@ ixgbe_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	 */
 	j = kring->nr_hwcur;
 	if (j != k) {	/* we have new packets to send */
-		l = netmap_tidx_k2n(na, ring_nr, j); /* NIC index */
+		prefetch(&ring->slot[j]);
+		l = netmap_idx_k2n(kring, j); /* NIC index */
+		prefetch(&txr->tx_buffers[l]);
 		for (n = 0; j != k; n++) {
 			/*
 			 * Collect per-slot info.
@@ -253,17 +209,25 @@ ixgbe_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 			 * Many other drivers preserve the address, so
 			 * we only need to access it if NS_BUF_CHANGED
 			 * is set.
+			 * XXX note, on this device the dmamap* calls are
+			 * not necessary because tag is 0, however just accessing
+			 * the per-packet tag kills 1Mpps at 900 MHz.
 			 */
 			struct netmap_slot *slot = &ring->slot[j];
-			struct ixgbe_tx_buf *txbuf = &txr->tx_buffers[l];
 			union ixgbe_adv_tx_desc *curr = &txr->tx_base[l];
+			struct ixgbe_tx_buf *txbuf = &txr->tx_buffers[l];
 			uint64_t paddr;
-			void *addr = PNMB(slot, &paddr);
 			// XXX type for flags and len ?
 			int flags = ((slot->flags & NS_REPORT) ||
 				j == 0 || j == report_frequency) ?
 					IXGBE_TXD_CMD_RS : 0;
-			int len = slot->len;
+			u_int len = slot->len;
+			void *addr = PNMB(slot, &paddr);
+
+			j = (j == lim) ? 0 : j + 1;
+			l = (l == lim) ? 0 : l + 1;
+			prefetch(&ring->slot[j]);
+			prefetch(&txr->tx_buffers[l]);
 
 			/*
 			 * Quick check for valid addr and len.
@@ -279,35 +243,29 @@ ring_reset:
 				return netmap_ring_reinit(kring);
 			}
 
-			slot->flags &= ~NS_REPORT;
 			if (slot->flags & NS_BUF_CHANGED) {
 				/* buffer has changed, unload and reload map */
 				netmap_reload_map(txr->txtag, txbuf->map, addr);
 				slot->flags &= ~NS_BUF_CHANGED;
 			}
+			slot->flags &= ~NS_REPORT;
 			/*
 			 * Fill the slot in the NIC ring.
 			 * In this driver we need to rewrite the buffer
 			 * address in the NIC ring. Other drivers do not
 			 * need this.
+			 * Use legacy descriptor, it is faster.
 			 */
 			curr->read.buffer_addr = htole64(paddr);
-			curr->read.olinfo_status = htole32(len << IXGBE_ADVTXD_PAYLEN_SHIFT);
-			curr->read.cmd_type_len =
-			    htole32(txr->txd_cmd | len |
-				(IXGBE_ADVTXD_DTYP_DATA |
-				    IXGBE_ADVTXD_DCMD_DEXT |
-				    IXGBE_ADVTXD_DCMD_IFCS |
-				    IXGBE_TXD_CMD_EOP | flags) );
+			curr->read.olinfo_status = 0;
+			curr->read.cmd_type_len = htole32(len | flags |
+				IXGBE_ADVTXD_DCMD_IFCS | IXGBE_TXD_CMD_EOP);
 
 			/* make sure changes to the buffer are synced */
-			bus_dmamap_sync(txr->txtag, txbuf->map,
-				BUS_DMASYNC_PREWRITE);
-			j = (j == lim) ? 0 : j + 1;
-			l = (l == lim) ? 0 : l + 1;
+			bus_dmamap_sync(txr->txtag, txbuf->map, BUS_DMASYNC_PREWRITE);
 		}
 		kring->nr_hwcur = k; /* the saved ring->cur */
-		/* decrease avail by number of sent packets */
+		/* decrease avail by number of packets  sent */
 		kring->nr_hwavail -= n;
 
 		/* synchronize the NIC ring */
@@ -416,20 +374,15 @@ ixgbe_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	struct netmap_adapter *na = NA(adapter->ifp);
 	struct netmap_kring *kring = &na->rx_rings[ring_nr];
 	struct netmap_ring *ring = kring->ring;
-	int j, k, l, n, lim = kring->nkr_num_slots - 1;
+	u_int j, l, n, lim = kring->nkr_num_slots - 1;
 	int force_update = do_lock || kring->nr_kflags & NKR_PENDINTR;
+	u_int k = ring->cur, resvd = ring->reserved;
 
-	k = ring->cur;	/* cache and check value, same as in txsync */
-	n = k - kring->nr_hwcur;
-	if (n < 0)
-		n += lim + 1;
-	if (k > lim || n > kring->nr_hwavail) /* userspace is cheating */
+	if (k > lim)
 		return netmap_ring_reinit(kring);
 
 	if (do_lock)
 		IXGBE_RX_LOCK(rxr);
-	if (n < 0)
-		n += lim + 1;
 	/* XXX check sync modes */
 	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
 			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
@@ -450,7 +403,7 @@ ixgbe_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	 * rxr->next_to_check is set to 0 on a ring reinit
 	 */
 	l = rxr->next_to_check;
-	j = netmap_ridx_n2k(na, ring_nr, l);
+	j = netmap_idx_n2k(kring, l);
 
 	if (netmap_no_pendintr || force_update) {
 		for (n = 0; ; n++) {
@@ -473,15 +426,22 @@ ixgbe_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 	}
 
 	/*
-	 * Skip past packets that userspace has already processed
-	 * (from kring->nr_hwcur to ring->cur excluded), and make
-	 * the buffers available for reception.
+	 * Skip past packets that userspace has released
+	 * (from kring->nr_hwcur to ring->cur - ring->reserved excluded),
+	 * and make the buffers available for reception.
 	 * As usual j is the index in the netmap ring, l is the index
 	 * in the NIC ring, and j == (l + kring->nkr_hwofs) % ring_size
 	 */
 	j = kring->nr_hwcur;
-	if (j != k) {	/* userspace has read some packets. */
-		l = netmap_ridx_k2n(na, ring_nr, j);
+	if (resvd > 0) {
+		if (resvd + ring->avail >= lim + 1) {
+			D("XXX invalid reserve/avail %d %d", resvd, ring->avail);
+			ring->reserved = resvd = 0; // XXX panic...
+		}
+		k = (k >= resvd) ? k - resvd : k + lim + 1 - resvd;
+	}
+	if (j != k) { /* userspace has released some packets. */
+		l = netmap_idx_k2n(kring, j);
 		for (n = 0; j != k; n++) {
 			/* collect per-slot info, with similar validations
 			 * and flag handling as in the txsync code.
@@ -522,7 +482,7 @@ ixgbe_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock)
 		IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDT(rxr->me), l);
 	}
 	/* tell userspace that there are new packets */
-	ring->avail = kring->nr_hwavail ;
+	ring->avail = kring->nr_hwavail - resvd;
 
 	if (do_lock)
 		IXGBE_RX_UNLOCK(rxr);
@@ -533,4 +493,31 @@ ring_reset:
 		IXGBE_RX_UNLOCK(rxr);
 	return netmap_ring_reinit(kring);
 }
+
+
+/*
+ * The attach routine, called near the end of ixgbe_attach(),
+ * fills the parameters for netmap_attach() and calls it.
+ * It cannot fail, in the worst case (such as no memory)
+ * netmap mode will be disabled and the driver will only
+ * operate in standard mode.
+ */
+static void
+ixgbe_netmap_attach(struct adapter *adapter)
+{
+	struct netmap_adapter na;
+
+	bzero(&na, sizeof(na));
+
+	na.ifp = adapter->ifp;
+	na.separate_locks = 1;	/* this card has separate rx/tx locks */
+	na.num_tx_desc = adapter->num_tx_desc;
+	na.num_rx_desc = adapter->num_rx_desc;
+	na.nm_txsync = ixgbe_netmap_txsync;
+	na.nm_rxsync = ixgbe_netmap_rxsync;
+	na.nm_lock = ixgbe_netmap_lock_wrapper;
+	na.nm_register = ixgbe_netmap_reg;
+	netmap_attach(&na, adapter->num_queues);
+}	
+
 /* end of file */