Adjust the register layout to allow for 64bit registers in the

future for nf10bmac(4).  Also, add support for and enable RX interrupts.

MFC after:	2 weeks

3 files changed, 182 insertions, 40 deletions

diff --git a/sys/dev/netfpga10g/nf10bmac/if_nf10bmac.c b/sys/dev/netfpga10g/nf10bmac/if_nf10bmac.c
index f6130b7..7076b73 100644
--- a/sys/dev/netfpga10g/nf10bmac/if_nf10bmac.c
+++ b/sys/dev/netfpga10g/nf10bmac/if_nf10bmac.c
@@ -92,13 +92,15 @@ static poll_handler_t nf10bmac_poll;
 #define	NF10BMAC_LOCK_ASSERT(_sc)	\
 	mtx_assert(&(_sc)->nf10bmac_mtx, MA_OWNED)
 
-#define	NF10BMAC_TX_LEN			0x08
-#define	NF10BMAC_TX_META		0x04
+#define	NF10BMAC_CTRL0			0x00
 #define	NF10BMAC_TX_DATA		0x00
-#define	NF10BMAC_RX_LEN			0x08
-#define	NF10BMAC_RX_META		0x04
+#define	NF10BMAC_TX_META		0x08
+#define	NF10BMAC_TX_LEN			0x10
 #define	NF10BMAC_RX_DATA		0x00
-#define	NF10BMAC_CTRL0			0x00
+#define	NF10BMAC_RX_META		0x08
+#define	NF10BMAC_RX_LEN			0x10
+#define	NF10BMAC_INTR_CLEAR_DIS		0x00
+#define	NF10BMAC_INTR_CTRL		0x08
 
 #define NF10BMAC_TUSER_MAC0		(1 << 0)
 #define NF10BMAC_TUSER_CPU0		(1 << 1)
@@ -109,11 +111,12 @@ static poll_handler_t nf10bmac_poll;
 #define NF10BMAC_TUSER_MAC3		(1 << 6)
 #define NF10BMAC_TUSER_CPU3		(1 << 7)
 
+#define	NF10BMAC_DATA_LEN_MASK		0x0000ffff
 #define	NF10BMAC_DATA_DPORT_MASK	0xff000000
 #define	NF10BMAC_DATA_DPORT_SHIFT	24
 #define	NF10BMAC_DATA_SPORT_MASK	0x00ff0000
 #define	NF10BMAC_DATA_SPORT_SHIFT	16
-#define	NF10BMAC_DATA_LAST		0x00000080
+#define	NF10BMAC_DATA_LAST		0x00008000
 #define	NF10BMAC_DATA_STRB		0x0000000f
 
 
@@ -151,7 +154,7 @@ nf10bmac_read_4_be(struct resource *res, uint32_t reg,
 }
 
 #define	NF10BMAC_WRITE_CTRL_4(sc, reg, val)				\
-	nf10bmac_write_4((sc)->nf10bmac_mem_res, (reg), (val),		\
+	nf10bmac_write_4((sc)->nf10bmac_ctrl_res, (reg), (val),		\
 	    __func__, __LINE__)
 #define	NF10BMAC_WRITE_4(sc, reg, val)					\
 	nf10bmac_write_4((sc)->nf10bmac_tx_mem_res, (reg), (val),	\
@@ -166,6 +169,21 @@ nf10bmac_read_4_be(struct resource *res, uint32_t reg,
 	nf10bmac_read_4_be((sc)->nf10bmac_rx_mem_res, (reg),		\
 	    __func__, __LINE__)
 
+#define	NF10BMAC_WRITE_INTR_4(sc, reg, val, _f, _l)			\
+	nf10bmac_write_4((sc)->nf10bmac_intr_res, (reg), (val),		\
+	    (_f), (_l))
+
+#define	NF10BMAC_RX_INTR_CLEAR_DIS(sc)					\
+	NF10BMAC_WRITE_INTR_4((sc), NF10BMAC_INTR_CLEAR_DIS, 1,		\
+	__func__, __LINE__)
+#define	NF10BMAC_RX_INTR_ENABLE(sc)					\
+	NF10BMAC_WRITE_INTR_4((sc), NF10BMAC_INTR_CTRL, 1,		\
+	__func__, __LINE__)
+#define	NF10BMAC_RX_INTR_DISABLE(sc)					\
+	NF10BMAC_WRITE_INTR_4((sc), NF10BMAC_INTR_CTRL, 0,		\
+	__func__, __LINE__)
+
+
 #ifdef ENABLE_WATCHDOG
 static void nf10bmac_tick(void *);
 #endif
@@ -318,7 +336,7 @@ nf10bmac_rx_locked(struct nf10bmac_softc *sc)
 	 * skip to tlast).
 	 */
 
-	len = NF10BMAC_READ_4(sc, NF10BMAC_RX_LEN);
+	len = NF10BMAC_READ_4(sc, NF10BMAC_RX_LEN) & NF10BMAC_DATA_LEN_MASK;
 	if (len > (MCLBYTES - ETHER_ALIGN)) {
 		nf10bmac_eat_packet_munch_munch(sc);
 		return (0);
@@ -435,6 +453,7 @@ nf10bmac_stop_locked(struct nf10bmac_softc *sc)
 
 	ifp = sc->nf10bmac_ifp;
 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
+	NF10BMAC_RX_INTR_CLEAR_DIS(sc);
 
 	sc->nf10bmac_flags &= ~NF10BMAC_FLAGS_LINK;
 	if_link_state_change(ifp, LINK_STATE_DOWN);
@@ -498,6 +517,16 @@ nf10bmac_init_locked(struct nf10bmac_softc *sc)
 	/* Instead drain the FIFO; or at least a possible first packet.. */
 	nf10bmac_eat_packet_munch_munch(sc);
 
+#ifdef DEVICE_POLLING
+	/* Only enable interrupts if we are not polling. */
+	if (ifp->if_capenable & IFCAP_POLLING) {
+		NF10BMAC_RX_INTR_CLEAR_DIS(sc);
+	} else
+#endif
+	{
+		NF10BMAC_RX_INTR_ENABLE(sc);
+	}
+
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
@@ -556,6 +585,49 @@ nf10bmac_tick(void *xsc)
 }
 #endif
 
+static void
+nf10bmac_intr(void *arg)
+{
+	struct nf10bmac_softc *sc;
+	struct ifnet *ifp;
+	int rx_npkts;
+
+	sc = (struct nf10bmac_softc *)arg;
+	ifp = sc->nf10bmac_ifp;
+
+	NF10BMAC_LOCK(sc);
+#ifdef DEVICE_POLLING
+	if (ifp->if_capenable & IFCAP_POLLING) {
+		NF10BMAC_UNLOCK(sc);
+		return;
+	} 
+#endif
+
+	/* NF10BMAC_RX_INTR_DISABLE(sc); */
+	NF10BMAC_RX_INTR_CLEAR_DIS(sc);
+
+	/* We only have an RX interrupt and no status information. */
+	rx_npkts = 0;
+	while (rx_npkts < NF10BMAC_MAX_PKTS) {
+		int c;
+
+		c = nf10bmac_rx_locked(sc);
+		rx_npkts += c;
+		if (c == 0)
+			break;
+	}
+
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		/* Re-enable interrupts. */
+		NF10BMAC_RX_INTR_ENABLE(sc);
+
+		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
+			nf10bmac_start_locked(ifp);
+	}
+	NF10BMAC_UNLOCK(sc);
+}
+
+
 #ifdef DEVICE_POLLING
 static int
 nf10bmac_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
@@ -649,10 +721,16 @@ nf10bmac_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 					break;
 				}
 
+				NF10BMAC_RX_INTR_CLEAR_DIS(sc);
+
 			/*
 			 * Do not allow disabling of polling if we do
 			 * not have interrupts.
 			 */
+			} else if (sc->nf10bmac_rx_irq_res != NULL) {
+				error = ether_poll_deregister(ifp);
+				/* Enable interrupts. */
+				NF10BMAC_RX_INTR_ENABLE(sc);
 			} else {
 				ifp->if_capenable ^= IFCAP_POLLING;
 				error = EINVAL;
@@ -673,7 +751,6 @@ nf10bmac_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 	return (error);
 }
 
-
 /*
  * Generic device handling routines.
  */
@@ -733,18 +810,40 @@ nf10bmac_attach(device_t dev)
         ifmedia_add(&sc->nf10bmac_media, IFM_ETHER | IFM_10G_T, 0, NULL);
         ifmedia_set(&sc->nf10bmac_media, IFM_ETHER | IFM_10G_T);
 
-	/* Interrupts would go here. */
+	/* Initialise. */
+	error = 0;
+
+	/* Hook up interrupts. Well the one. */
+	if (sc->nf10bmac_rx_irq_res != NULL) {
+		error = bus_setup_intr(dev, sc->nf10bmac_rx_irq_res,
+		    INTR_TYPE_NET | INTR_MPSAFE, NULL, nf10bmac_intr,
+		    sc, &sc->nf10bmac_rx_intrhand);
+		if (error != 0) {
+			device_printf(dev, "enabling RX IRQ failed\n");
+			ether_ifdetach(ifp);
+			goto err;
+		}
+	}
 
+	if ((ifp->if_capenable & IFCAP_POLLING) != 0 ||
+	    sc->nf10bmac_rx_irq_res == NULL) {
 #ifdef DEVICE_POLLING
-	ifp->if_capenable |= IFCAP_POLLING;
-	device_printf(dev, "forcing to polling due to no interrupts\n");
-	error = ether_poll_register(nf10bmac_poll, ifp);
-	if (error != 0)
-		goto err;
+		/* If not on and no IRQs force it on. */
+		if (sc->nf10bmac_rx_irq_res == NULL) {
+			ifp->if_capenable |= IFCAP_POLLING;
+			device_printf(dev,
+			    "forcing to polling due to no interrupts\n");
+		}
+		error = ether_poll_register(nf10bmac_poll, ifp);
+		if (error != 0)
+			goto err;
 #else
-	device_printf(dev, "no DEVICE_POLLING in kernel and no IRQs\n");
-	error = ENXIO;
+		device_printf(dev, "no DEVICE_POLLING in kernel and no IRQs\n");
+		error = ENXIO;
 #endif
+	} else {
+		NF10BMAC_RX_INTR_ENABLE(sc);
+	}
 
 err:
 	if (error != 0)
@@ -780,6 +879,10 @@ nf10bmac_detach(device_t dev)
 		ether_ifdetach(ifp);
 	}
 
+	if (sc->nf10bmac_rx_intrhand)
+		bus_teardown_intr(dev, sc->nf10bmac_rx_irq_res,
+		    sc->nf10bmac_rx_intrhand);
+
 	if (ifp != NULL)
 		if_free(ifp);
 	ifmedia_removeall(&sc->nf10bmac_media);
@@ -797,10 +900,15 @@ nf10bmac_detach_resources(device_t dev)
 
 	sc = device_get_softc(dev);
 
-	if (sc->nf10bmac_mem_res != NULL) {
+	if (sc->nf10bmac_rx_irq_res != NULL) {
+		bus_release_resource(dev, SYS_RES_IRQ, sc->nf10bmac_rx_irq_rid,
+		    sc->nf10bmac_rx_irq_res);
+		sc->nf10bmac_rx_irq_res = NULL;
+	}
+	if (sc->nf10bmac_intr_res != NULL) {
 		bus_release_resource(dev, SYS_RES_MEMORY,
-		    sc->nf10bmac_mem_rid, sc->nf10bmac_mem_res);
-		sc->nf10bmac_mem_res = NULL;
+		    sc->nf10bmac_intr_rid, sc->nf10bmac_intr_res);
+		sc->nf10bmac_intr_res = NULL;
 	}
 	if (sc->nf10bmac_rx_mem_res != NULL) {
 		bus_release_resource(dev, SYS_RES_MEMORY,
@@ -812,6 +920,11 @@ nf10bmac_detach_resources(device_t dev)
 		    sc->nf10bmac_tx_mem_rid, sc->nf10bmac_tx_mem_res);
 		sc->nf10bmac_tx_mem_res = NULL;
 	}
+	if (sc->nf10bmac_ctrl_res != NULL) {
+		bus_release_resource(dev, SYS_RES_MEMORY,
+		    sc->nf10bmac_ctrl_rid, sc->nf10bmac_ctrl_res);
+		sc->nf10bmac_ctrl_res = NULL;
+	}
 }
 
 int
diff --git a/sys/dev/netfpga10g/nf10bmac/if_nf10bmac_fdt.c b/sys/dev/netfpga10g/nf10bmac/if_nf10bmac_fdt.c
index 3eff847..4245da4 100644
--- a/sys/dev/netfpga10g/nf10bmac/if_nf10bmac_fdt.c
+++ b/sys/dev/netfpga10g/nf10bmac/if_nf10bmac_fdt.c
@@ -85,16 +85,34 @@ nf10bmac_attach_fdt(device_t dev)
 	/*
 	 * FDT lists our resources.  For convenience we use three different
 	 * mappings.  We need to attach them in the oder specified in .dts:
-	 * TX (size 0xc), RX (size 0xc), LOOP (size 0x4).
+	 * LOOP (size 0x1f), TX (0x2f), RX (0x2f), INTR (0xf).
 	 */
 
+	/*
+	 * LOOP memory region (this could be a general control region).
+	 * 0x00: 32bit register to enable a Y-"lopback".
+	 */
+        sc->nf10bmac_ctrl_rid = 0;
+        sc->nf10bmac_ctrl_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+            &sc->nf10bmac_ctrl_rid, RF_ACTIVE);
+        if (sc->nf10bmac_ctrl_res == NULL) {
+                device_printf(dev, "failed to map memory for CTRL region\n");
+                error = ENXIO;
+                goto err;
+        } 
+        if (bootverbose)
+                device_printf(sc->nf10bmac_dev, "CTRL region at mem %p-%p\n",
+                    (void *)rman_get_start(sc->nf10bmac_ctrl_res),
+                    (void *)(rman_get_start(sc->nf10bmac_ctrl_res) + 
+                    rman_get_size(sc->nf10bmac_ctrl_res)));
+
         /*
          * TX and TX metadata FIFO memory region.
          * 0x00: 32bit FIFO data,
-	 * 0x04: 32bit FIFO metadata,
-         * 0x08: 32bit packet length.
+	 * 0x08: 32bit FIFO metadata,
+         * 0x10: 32bit packet length.
          */
-        sc->nf10bmac_tx_mem_rid = 0;
+        sc->nf10bmac_tx_mem_rid = 1;
         sc->nf10bmac_tx_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
             &sc->nf10bmac_tx_mem_rid, RF_ACTIVE);
         if (sc->nf10bmac_tx_mem_res == NULL) {
@@ -111,10 +129,10 @@ nf10bmac_attach_fdt(device_t dev)
         /*
          * RX and RXC metadata FIFO memory region.
          * 0x00: 32bit FIFO data,
-	 * 0x04: 32bit FIFO metadata,
-         * 0x08: 32bit packet length.
+	 * 0x08: 32bit FIFO metadata,
+         * 0x10: 32bit packet length.
          */
-        sc->nf10bmac_rx_mem_rid = 1;
+        sc->nf10bmac_rx_mem_rid = 2;
         sc->nf10bmac_rx_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
             &sc->nf10bmac_rx_mem_rid, RF_ACTIVE);
         if (sc->nf10bmac_rx_mem_res == NULL) {
@@ -129,22 +147,28 @@ nf10bmac_attach_fdt(device_t dev)
                     rman_get_size(sc->nf10bmac_rx_mem_res)));
 
 	/*
-	 * LOOP memory region (this could be a general control region).
-	 * 0x00: 32bit register to enable a Y-"lopback".
+	 * Interrupt handling registers.
+	 * 0x00: 32bit register to clear (and disable) the RX interrupt.
+	 * 0x08: 32bit register to enable or disable the RX interrupt.
 	 */
-        sc->nf10bmac_mem_rid = 2;
-        sc->nf10bmac_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
-            &sc->nf10bmac_mem_rid, RF_ACTIVE);
-        if (sc->nf10bmac_mem_res == NULL) {
-                device_printf(dev, "failed to map memory for CTRL region\n");
+        sc->nf10bmac_intr_rid = 3;
+        sc->nf10bmac_intr_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+            &sc->nf10bmac_intr_rid, RF_ACTIVE);
+        if (sc->nf10bmac_intr_res == NULL) {
+                device_printf(dev, "failed to map memory for INTR region\n");
                 error = ENXIO;
                 goto err;
         } 
         if (bootverbose)
-                device_printf(sc->nf10bmac_dev, "CTRL region at mem %p-%p\n",
-                    (void *)rman_get_start(sc->nf10bmac_mem_res),
-                    (void *)(rman_get_start(sc->nf10bmac_mem_res) + 
-                    rman_get_size(sc->nf10bmac_mem_res)));
+                device_printf(sc->nf10bmac_dev, "INTR region at mem %p-%p\n",
+                    (void *)rman_get_start(sc->nf10bmac_intr_res),
+                    (void *)(rman_get_start(sc->nf10bmac_intr_res) + 
+                    rman_get_size(sc->nf10bmac_intr_res)));
+
+	/* (Optional) RX and TX IRQ. */
+	sc->nf10bmac_rx_irq_rid = 0;
+	sc->nf10bmac_rx_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+	    &sc->nf10bmac_rx_irq_rid, RF_ACTIVE | RF_SHAREABLE);
 
 	error = nf10bmac_attach(dev);
 	if (error)
diff --git a/sys/dev/netfpga10g/nf10bmac/if_nf10bmacreg.h b/sys/dev/netfpga10g/nf10bmac/if_nf10bmacreg.h
index 725cf3b..fa58116 100644
--- a/sys/dev/netfpga10g/nf10bmac/if_nf10bmacreg.h
+++ b/sys/dev/netfpga10g/nf10bmac/if_nf10bmacreg.h
@@ -35,15 +35,20 @@
 
 struct nf10bmac_softc {
 	struct ifnet		*nf10bmac_ifp;
+	struct resource		*nf10bmac_ctrl_res;
 	struct resource		*nf10bmac_tx_mem_res;
 	struct resource		*nf10bmac_rx_mem_res;
-	struct resource		*nf10bmac_mem_res;
+	struct resource		*nf10bmac_intr_res;
+	struct resource		*nf10bmac_rx_irq_res;
+	void			*nf10bmac_rx_intrhand;
 	uint8_t			*nf10bmac_tx_buf;
 	device_t		nf10bmac_dev;
 	int			nf10bmac_unit;
+	int			nf10bmac_ctrl_rid;
 	int			nf10bmac_tx_mem_rid;
 	int			nf10bmac_rx_mem_rid;
-	int			nf10bmac_mem_rid;
+	int			nf10bmac_intr_rid;
+	int			nf10bmac_rx_irq_rid;
 	int			nf10bmac_if_flags;
 	uint32_t		nf10bmac_flags;
 #define	NF10BMAC_FLAGS_LINK		0x00000001