diff options
author | yongari <yongari@FreeBSD.org> | 2007-06-12 02:16:02 +0000 |
---|---|---|
committer | yongari <yongari@FreeBSD.org> | 2007-06-12 02:16:02 +0000 |
commit | 36a63ea429a4b9cac4f1b260823bd736efac33a0 (patch) | |
tree | efdb60dcf728b71f67aaee74a5079daa410a2d45 /sys/dev/nfe | |
parent | ec6a039b6248e377ab0e0a872c4725b320c268b7 (diff) | |
download | FreeBSD-src-36a63ea429a4b9cac4f1b260823bd736efac33a0.zip FreeBSD-src-36a63ea429a4b9cac4f1b260823bd736efac33a0.tar.gz |
Bring overhauled nfe(4) into tree.
o s/printf/device_printf/g
o Nuke OpenBSDism.
o Nuke NetBSD/OpenBSD specific DMA sync operations.(we don't have a way
to sync a single descriptor within a DMA map.)
o Remove recursive mutex.
o bus_dma(9) clean up.
o 40bit DMA address support.
o Add protection for Rx map load failure.
o Fix a long standing bug for watchdog timeout. [1]
o Add additional protections, missing Tx completion interrupt, losing
start Tx command, for watchdog timeout.
o Switch to taskqueue(9) API to handle interrupts.
o Use our own timer for watchdog instead of if_watchdog/if_timer
interface.
o Advertise VLAN header length/capability correctly to upper layer.
o Remove excessive kernel stack consumption in nfe_encap().
o Handle highly fragmented mbuf chains correctly.
o Enable etherenet address reprogramming with ifconfig(8).
o Add ALTQ/TSO, MSI/MSIX support.
o Increased Rx ring to 256 descriptors from 128.
o Align Tx/Rx descriptor ring on sizeof(struct nfe_desc64) boundary.
o Remove alignment restrictions on Tx/Rx buffers.
o Rewritten jumbo frame support code.
o Add support for hardware assistend VLAN tag insertion/stripping.
o Add support for Tx/Rx flow control based on patches from Peer Chen. [2]
o Add a routine that detects whether ethernet address swap routines is
required. [3]
o Add a workaround that take MAC/PHY out of power down mode.
o Add suspend/resume support.
o style(9) and code clean up.
Special thanks to Shigeaki Tagashira, the original porter of nfe(4),
who submitted lots of patches, performed uncountable number of
regression tests and maintained nfe(4) for a long time. Without his
enthusiastic help and support I could never have completed this
overhauling task.
The only weak point of nfe(4) compared to nve(4) is instability of
manual half-duplex media selection on certain hardwares(auto sensing
media type should work for all cases, though). This was a long
standing bug of nfe(4) and I still have no idea why it doesn't work
on some hardwares.
Obtained from: OpenBSD [1]
Submitted by: Peer Chen < pchen at nvidia dot com > [2], [3]
Reviewed by: Shigeaki Tagashira < shigeaki AT se DOT hiroshima-u DOT ac DOT jp >
Tested by: Shigeaki Tagashira, current
Discussed with: current
Silence from: obrien
Diffstat (limited to 'sys/dev/nfe')
-rw-r--r-- | sys/dev/nfe/if_nfe.c | 2792 | ||||
-rw-r--r-- | sys/dev/nfe/if_nfereg.h | 100 | ||||
-rw-r--r-- | sys/dev/nfe/if_nfevar.h | 120 |
3 files changed, 2103 insertions, 909 deletions
diff --git a/sys/dev/nfe/if_nfe.c b/sys/dev/nfe/if_nfe.c index 0c54827..9765a36 100644 --- a/sys/dev/nfe/if_nfe.c +++ b/sys/dev/nfe/if_nfe.c @@ -23,13 +23,6 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -/* Uncomment the following line to enable polling. */ -/* #define DEVICE_POLLING */ - -#define NFE_JUMBO -#define NFE_CSUM -#define NVLAN 0 - #ifdef HAVE_KERNEL_OPTION_HEADERS #include "opt_device_polling.h" #endif @@ -42,7 +35,9 @@ __FBSDID("$FreeBSD$"); #include <sys/malloc.h> #include <sys/module.h> #include <sys/kernel.h> +#include <sys/queue.h> #include <sys/socket.h> +#include <sys/sysctl.h> #include <sys/taskqueue.h> #include <net/if.h> @@ -72,73 +67,100 @@ __FBSDID("$FreeBSD$"); MODULE_DEPEND(nfe, pci, 1, 1, 1); MODULE_DEPEND(nfe, ether, 1, 1, 1); MODULE_DEPEND(nfe, miibus, 1, 1, 1); + +/* "device miibus" required. See GENERIC if you get errors here. */ #include "miibus_if.h" static int nfe_probe(device_t); static int nfe_attach(device_t); static int nfe_detach(device_t); +static int nfe_suspend(device_t); +static int nfe_resume(device_t); static void nfe_shutdown(device_t); +static void nfe_power(struct nfe_softc *); static int nfe_miibus_readreg(device_t, int, int); static int nfe_miibus_writereg(device_t, int, int, int); static void nfe_miibus_statchg(device_t); +static void nfe_link_task(void *, int); +static void nfe_set_intr(struct nfe_softc *); +static __inline void nfe_enable_intr(struct nfe_softc *); +static __inline void nfe_disable_intr(struct nfe_softc *); static int nfe_ioctl(struct ifnet *, u_long, caddr_t); -static void nfe_intr(void *); -static void nfe_txdesc32_sync(struct nfe_softc *, struct nfe_desc32 *, int); -static void nfe_txdesc64_sync(struct nfe_softc *, struct nfe_desc64 *, int); -static void nfe_txdesc32_rsync(struct nfe_softc *, int, int, int); -static void nfe_txdesc64_rsync(struct nfe_softc *, int, int, int); -static void nfe_rxdesc32_sync(struct nfe_softc *, struct nfe_desc32 *, int); -static void nfe_rxdesc64_sync(struct nfe_softc *, struct nfe_desc64 *, int); -static void nfe_rxeof(struct nfe_softc *); +static void nfe_alloc_msix(struct nfe_softc *, int); +static int nfe_intr(void *); +static void nfe_int_task(void *, int); +static void *nfe_jalloc(struct nfe_softc *); +static void nfe_jfree(void *, void *); +static __inline void nfe_discard_rxbuf(struct nfe_softc *, int); +static __inline void nfe_discard_jrxbuf(struct nfe_softc *, int); +static int nfe_newbuf(struct nfe_softc *, int); +static int nfe_jnewbuf(struct nfe_softc *, int); +static int nfe_rxeof(struct nfe_softc *, int); +static int nfe_jrxeof(struct nfe_softc *, int); static void nfe_txeof(struct nfe_softc *); -static int nfe_encap(struct nfe_softc *, struct mbuf *); +static struct mbuf *nfe_defrag(struct mbuf *, int, int); +static int nfe_encap(struct nfe_softc *, struct mbuf **); static void nfe_setmulti(struct nfe_softc *); +static void nfe_tx_task(void *, int); static void nfe_start(struct ifnet *); -static void nfe_start_locked(struct ifnet *); static void nfe_watchdog(struct ifnet *); static void nfe_init(void *); static void nfe_init_locked(void *); -static void nfe_stop(struct ifnet *, int); +static void nfe_stop(struct ifnet *); static int nfe_alloc_rx_ring(struct nfe_softc *, struct nfe_rx_ring *); -static void nfe_reset_rx_ring(struct nfe_softc *, struct nfe_rx_ring *); +static int nfe_alloc_jrx_ring(struct nfe_softc *, struct nfe_jrx_ring *); +static int nfe_init_rx_ring(struct nfe_softc *, struct nfe_rx_ring *); +static int nfe_init_jrx_ring(struct nfe_softc *, struct nfe_jrx_ring *); static void nfe_free_rx_ring(struct nfe_softc *, struct nfe_rx_ring *); +static void nfe_free_jrx_ring(struct nfe_softc *, struct nfe_jrx_ring *); static int nfe_alloc_tx_ring(struct nfe_softc *, struct nfe_tx_ring *); -static void nfe_reset_tx_ring(struct nfe_softc *, struct nfe_tx_ring *); +static void nfe_init_tx_ring(struct nfe_softc *, struct nfe_tx_ring *); static void nfe_free_tx_ring(struct nfe_softc *, struct nfe_tx_ring *); static int nfe_ifmedia_upd(struct ifnet *); -static int nfe_ifmedia_upd_locked(struct ifnet *); static void nfe_ifmedia_sts(struct ifnet *, struct ifmediareq *); static void nfe_tick(void *); -static void nfe_tick_locked(struct nfe_softc *); -static void nfe_get_macaddr(struct nfe_softc *, u_char *); -static void nfe_set_macaddr(struct nfe_softc *, u_char *); -static void nfe_dma_map_segs (void *, bus_dma_segment_t *, int, int); -#ifdef DEVICE_POLLING -static void nfe_poll_locked(struct ifnet *, enum poll_cmd, int); -#endif +static void nfe_get_macaddr(struct nfe_softc *, uint8_t *); +static void nfe_set_macaddr(struct nfe_softc *, uint8_t *); +static void nfe_dma_map_segs(void *, bus_dma_segment_t *, int, int); + +static int sysctl_int_range(SYSCTL_HANDLER_ARGS, int, int); +static int sysctl_hw_nfe_proc_limit(SYSCTL_HANDLER_ARGS); #ifdef NFE_DEBUG -int nfedebug = 0; -#define DPRINTF(x) do { if (nfedebug) printf x; } while (0) -#define DPRINTFN(n,x) do { if (nfedebug >= (n)) printf x; } while (0) +static int nfedebug = 0; +#define DPRINTF(sc, ...) do { \ + if (nfedebug) \ + device_printf((sc)->nfe_dev, __VA_ARGS__); \ +} while (0) +#define DPRINTFN(sc, n, ...) do { \ + if (nfedebug >= (n)) \ + device_printf((sc)->nfe_dev, __VA_ARGS__); \ +} while (0) #else -#define DPRINTF(x) -#define DPRINTFN(n,x) +#define DPRINTF(sc, ...) +#define DPRINTFN(sc, n, ...) #endif #define NFE_LOCK(_sc) mtx_lock(&(_sc)->nfe_mtx) #define NFE_UNLOCK(_sc) mtx_unlock(&(_sc)->nfe_mtx) #define NFE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->nfe_mtx, MA_OWNED) -#define letoh16(x) le16toh(x) +#define NFE_JLIST_LOCK(_sc) mtx_lock(&(_sc)->nfe_jlist_mtx) +#define NFE_JLIST_UNLOCK(_sc) mtx_unlock(&(_sc)->nfe_jlist_mtx) -#define NV_RID 0x10 +/* Tunables. */ +static int msi_disable = 0; +static int msix_disable = 0; +TUNABLE_INT("hw.nfe.msi_disable", &msi_disable); +TUNABLE_INT("hw.nfe.msix_disable", &msix_disable); static device_method_t nfe_methods[] = { /* Device interface */ DEVMETHOD(device_probe, nfe_probe), DEVMETHOD(device_attach, nfe_attach), DEVMETHOD(device_detach, nfe_detach), + DEVMETHOD(device_suspend, nfe_suspend), + DEVMETHOD(device_resume, nfe_resume), DEVMETHOD(device_shutdown, nfe_shutdown), /* bus interface */ @@ -150,7 +172,7 @@ static device_method_t nfe_methods[] = { DEVMETHOD(miibus_writereg, nfe_miibus_writereg), DEVMETHOD(miibus_statchg, nfe_miibus_statchg), - { 0, 0 } + { NULL, NULL } }; static driver_t nfe_driver = { @@ -184,9 +206,9 @@ static struct nfe_type nfe_devs[] = { {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_NFORCE4_LAN2, "NVIDIA nForce4 CK804 MCP9 Networking Adapter"}, {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP04_LAN1, - "NVIDIA nForce MCP04 Networking Adapter"}, // MCP10 + "NVIDIA nForce MCP04 Networking Adapter"}, /* MCP10 */ {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP04_LAN2, - "NVIDIA nForce MCP04 Networking Adapter"}, // MCP11 + "NVIDIA nForce MCP04 Networking Adapter"}, /* MCP11 */ {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_NFORCE430_LAN1, "NVIDIA nForce 430 MCP12 Networking Adapter"}, {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_NFORCE430_LAN2, @@ -201,7 +223,7 @@ static struct nfe_type nfe_devs[] = { "NVIDIA nForce MCP61 Networking Adapter"}, {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP61_LAN3, "NVIDIA nForce MCP61 Networking Adapter"}, - {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP61_LAN2, + {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP61_LAN4, "NVIDIA nForce MCP61 Networking Adapter"}, {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP65_LAN1, "NVIDIA nForce MCP65 Networking Adapter"}, @@ -209,8 +231,16 @@ static struct nfe_type nfe_devs[] = { "NVIDIA nForce MCP65 Networking Adapter"}, {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP65_LAN3, "NVIDIA nForce MCP65 Networking Adapter"}, - {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP65_LAN2, + {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP65_LAN4, "NVIDIA nForce MCP65 Networking Adapter"}, + {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP67_LAN1, + "NVIDIA nForce MCP67 Networking Adapter"}, + {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP67_LAN2, + "NVIDIA nForce MCP67 Networking Adapter"}, + {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP67_LAN3, + "NVIDIA nForce MCP67 Networking Adapter"}, + {PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP67_LAN4, + "NVIDIA nForce MCP67 Networking Adapter"}, {0, 0, NULL} }; @@ -227,7 +257,7 @@ nfe_probe(device_t dev) if ((pci_get_vendor(dev) == t->vid_id) && (pci_get_device(dev) == t->dev_id)) { device_set_desc(dev, t->name); - return (0); + return (BUS_PROBE_DEFAULT); } t++; } @@ -235,54 +265,170 @@ nfe_probe(device_t dev) return (ENXIO); } +static void +nfe_alloc_msix(struct nfe_softc *sc, int count) +{ + int rid; + + rid = PCIR_BAR(2); + sc->nfe_msix_res = bus_alloc_resource_any(sc->nfe_dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); + if (sc->nfe_msix_res == NULL) { + device_printf(sc->nfe_dev, + "couldn't allocate MSIX table resource\n"); + return; + } + rid = PCIR_BAR(3); + sc->nfe_msix_pba_res = bus_alloc_resource_any(sc->nfe_dev, + SYS_RES_MEMORY, &rid, RF_ACTIVE); + if (sc->nfe_msix_pba_res == NULL) { + device_printf(sc->nfe_dev, + "couldn't allocate MSIX PBA resource\n"); + bus_release_resource(sc->nfe_dev, SYS_RES_MEMORY, PCIR_BAR(2), + sc->nfe_msix_res); + sc->nfe_msix_res = NULL; + return; + } + + if (pci_alloc_msix(sc->nfe_dev, &count) == 0) { + if (count == NFE_MSI_MESSAGES) { + if (bootverbose) + device_printf(sc->nfe_dev, + "Using %d MSIX messages\n", count); + sc->nfe_msix = 1; + } else { + if (bootverbose) + device_printf(sc->nfe_dev, + "couldn't allocate MSIX\n"); + pci_release_msi(sc->nfe_dev); + bus_release_resource(sc->nfe_dev, SYS_RES_MEMORY, + PCIR_BAR(3), sc->nfe_msix_pba_res); + bus_release_resource(sc->nfe_dev, SYS_RES_MEMORY, + PCIR_BAR(2), sc->nfe_msix_res); + sc->nfe_msix_pba_res = NULL; + sc->nfe_msix_res = NULL; + } + } +} static int nfe_attach(device_t dev) { struct nfe_softc *sc; struct ifnet *ifp; - int unit, error = 0, rid; + bus_addr_t dma_addr_max; + int error = 0, i, msic, reg, rid; sc = device_get_softc(dev); - unit = device_get_unit(dev); sc->nfe_dev = dev; - sc->nfe_unit = unit; mtx_init(&sc->nfe_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK, - MTX_DEF | MTX_RECURSE); + MTX_DEF); + mtx_init(&sc->nfe_jlist_mtx, "nfe_jlist_mtx", NULL, MTX_DEF); callout_init_mtx(&sc->nfe_stat_ch, &sc->nfe_mtx, 0); + TASK_INIT(&sc->nfe_link_task, 0, nfe_link_task, sc); + SLIST_INIT(&sc->nfe_jfree_listhead); + SLIST_INIT(&sc->nfe_jinuse_listhead); pci_enable_busmaster(dev); - rid = NV_RID; - sc->nfe_res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, - 0, ~0, 1, RF_ACTIVE); - - if (sc->nfe_res == NULL) { - printf ("nfe%d: couldn't map ports/memory\n", unit); - error = ENXIO; - goto fail; + rid = PCIR_BAR(0); + sc->nfe_res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + RF_ACTIVE); + if (sc->nfe_res[0] == NULL) { + device_printf(dev, "couldn't map memory resources\n"); + mtx_destroy(&sc->nfe_mtx); + return (ENXIO); } - sc->nfe_memt = rman_get_bustag(sc->nfe_res); - sc->nfe_memh = rman_get_bushandle(sc->nfe_res); + if (pci_find_extcap(dev, PCIY_EXPRESS, ®) == 0) { + uint16_t v, width; + + v = pci_read_config(dev, reg + 0x08, 2); + /* Change max. read request size to 4096. */ + v &= ~(7 << 12); + v |= (5 << 12); + pci_write_config(dev, reg + 0x08, v, 2); + + v = pci_read_config(dev, reg + 0x0c, 2); + /* link capability */ + v = (v >> 4) & 0x0f; + width = pci_read_config(dev, reg + 0x12, 2); + /* negotiated link width */ + width = (width >> 4) & 0x3f; + if (v != width) + device_printf(sc->nfe_dev, + "warning, negotiated width of link(x%d) != " + "max. width of link(x%d)\n", width, v); + } /* Allocate interrupt */ - rid = 0; - sc->nfe_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, - 0, ~0, 1, RF_SHAREABLE | RF_ACTIVE); + if (msix_disable == 0 || msi_disable == 0) { + if (msix_disable == 0 && + (msic = pci_msix_count(dev)) == NFE_MSI_MESSAGES) + nfe_alloc_msix(sc, msic); + if (msi_disable == 0 && sc->nfe_msix == 0 && + (msic = pci_msi_count(dev)) == NFE_MSI_MESSAGES && + pci_alloc_msi(dev, &msic) == 0) { + if (msic == NFE_MSI_MESSAGES) { + if (bootverbose) + device_printf(dev, + "Using %d MSI messages\n", msic); + sc->nfe_msi = 1; + } else + pci_release_msi(dev); + } + } - if (sc->nfe_irq == NULL) { - printf("nfe%d: couldn't map interrupt\n", unit); - error = ENXIO; - goto fail; + if (sc->nfe_msix == 0 && sc->nfe_msi == 0) { + rid = 0; + sc->nfe_irq[0] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_SHAREABLE | RF_ACTIVE); + if (sc->nfe_irq[0] == NULL) { + device_printf(dev, "couldn't allocate IRQ resources\n"); + error = ENXIO; + goto fail; + } + } else { + for (i = 0, rid = 1; i < NFE_MSI_MESSAGES; i++, rid++) { + sc->nfe_irq[i] = bus_alloc_resource_any(dev, + SYS_RES_IRQ, &rid, RF_ACTIVE); + if (sc->nfe_irq[i] == NULL) { + device_printf(dev, + "couldn't allocate IRQ resources for " + "message %d\n", rid); + error = ENXIO; + goto fail; + } + } + /* Map interrupts to vector 0. */ + if (sc->nfe_msix != 0) { + NFE_WRITE(sc, NFE_MSIX_MAP0, 0); + NFE_WRITE(sc, NFE_MSIX_MAP1, 0); + } else if (sc->nfe_msi != 0) { + NFE_WRITE(sc, NFE_MSI_MAP0, 0); + NFE_WRITE(sc, NFE_MSI_MAP1, 0); + } } - nfe_get_macaddr(sc, sc->eaddr); + /* Set IRQ status/mask register. */ + sc->nfe_irq_status = NFE_IRQ_STATUS; + sc->nfe_irq_mask = NFE_IRQ_MASK; + sc->nfe_intrs = NFE_IRQ_WANTED; + sc->nfe_nointrs = 0; + if (sc->nfe_msix != 0) { + sc->nfe_irq_status = NFE_MSIX_IRQ_STATUS; + sc->nfe_nointrs = NFE_IRQ_WANTED; + } else if (sc->nfe_msi != 0) { + sc->nfe_irq_mask = NFE_MSI_IRQ_MASK; + sc->nfe_intrs = NFE_MSI_VECTOR_0_ENABLED; + } + sc->nfe_devid = pci_get_device(dev); + sc->nfe_revid = pci_get_revid(dev); sc->nfe_flags = 0; - switch (pci_get_device(dev)) { + switch (sc->nfe_devid) { case PCI_PRODUCT_NVIDIA_NFORCE3_LAN2: case PCI_PRODUCT_NVIDIA_NFORCE3_LAN3: case PCI_PRODUCT_NVIDIA_NFORCE3_LAN4: @@ -291,7 +437,7 @@ nfe_attach(device_t dev) break; case PCI_PRODUCT_NVIDIA_MCP51_LAN1: case PCI_PRODUCT_NVIDIA_MCP51_LAN2: - sc->nfe_flags |= NFE_40BIT_ADDR; + sc->nfe_flags |= NFE_40BIT_ADDR | NFE_PWR_MGMT; break; case PCI_PRODUCT_NVIDIA_CK804_LAN1: case PCI_PRODUCT_NVIDIA_CK804_LAN2: @@ -302,34 +448,49 @@ nfe_attach(device_t dev) case PCI_PRODUCT_NVIDIA_MCP55_LAN1: case PCI_PRODUCT_NVIDIA_MCP55_LAN2: sc->nfe_flags |= NFE_JUMBO_SUP | NFE_40BIT_ADDR | NFE_HW_CSUM | - NFE_HW_VLAN; + NFE_HW_VLAN | NFE_PWR_MGMT | NFE_TX_FLOW_CTRL; break; + case PCI_PRODUCT_NVIDIA_MCP61_LAN1: case PCI_PRODUCT_NVIDIA_MCP61_LAN2: case PCI_PRODUCT_NVIDIA_MCP61_LAN3: case PCI_PRODUCT_NVIDIA_MCP61_LAN4: - sc->nfe_flags |= NFE_40BIT_ADDR; + case PCI_PRODUCT_NVIDIA_MCP67_LAN1: + case PCI_PRODUCT_NVIDIA_MCP67_LAN2: + case PCI_PRODUCT_NVIDIA_MCP67_LAN3: + case PCI_PRODUCT_NVIDIA_MCP67_LAN4: + sc->nfe_flags |= NFE_40BIT_ADDR | NFE_PWR_MGMT | + NFE_TX_FLOW_CTRL; break; case PCI_PRODUCT_NVIDIA_MCP65_LAN1: case PCI_PRODUCT_NVIDIA_MCP65_LAN2: case PCI_PRODUCT_NVIDIA_MCP65_LAN3: case PCI_PRODUCT_NVIDIA_MCP65_LAN4: - sc->nfe_flags |= NFE_JUMBO_SUP | NFE_40BIT_ADDR | NFE_HW_CSUM; + sc->nfe_flags |= NFE_JUMBO_SUP | NFE_40BIT_ADDR | + NFE_PWR_MGMT | NFE_TX_FLOW_CTRL; break; } + nfe_power(sc); + /* Check for reversed ethernet address */ + if ((NFE_READ(sc, NFE_TX_UNK) & NFE_MAC_ADDR_INORDER) != 0) + sc->nfe_flags |= NFE_CORRECT_MACADDR; + nfe_get_macaddr(sc, sc->eaddr); /* * Allocate the parent bus DMA tag appropriate for PCI. */ -#define NFE_NSEG_NEW 32 - error = bus_dma_tag_create(NULL, /* parent */ + dma_addr_max = BUS_SPACE_MAXADDR_32BIT; + if ((sc->nfe_flags & NFE_40BIT_ADDR) != 0) + dma_addr_max = NFE_DMA_MAXADDR; + error = bus_dma_tag_create( + bus_get_dma_tag(sc->nfe_dev), /* parent */ 1, 0, /* alignment, boundary */ - BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + dma_addr_max, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ - MAXBSIZE, NFE_NSEG_NEW, /* maxsize, nsegments */ + BUS_SPACE_MAXSIZE_32BIT, 0, /* maxsize, nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ - BUS_DMA_ALLOCNOW, /* flags */ + 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->nfe_parent_tag); if (error) @@ -337,63 +498,79 @@ nfe_attach(device_t dev) ifp = sc->nfe_ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { - printf("nfe%d: can not if_alloc()\n", unit); + device_printf(dev, "can not if_alloc()\n"); error = ENOSPC; goto fail; } - sc->nfe_mtu = ifp->if_mtu = ETHERMTU; + TASK_INIT(&sc->nfe_tx_task, 1, nfe_tx_task, ifp); /* * Allocate Tx and Rx rings. */ - if (nfe_alloc_tx_ring(sc, &sc->txq) != 0) { - printf("nfe%d: could not allocate Tx ring\n", unit); - error = ENXIO; + if ((error = nfe_alloc_tx_ring(sc, &sc->txq)) != 0) goto fail; - } - if (nfe_alloc_rx_ring(sc, &sc->rxq) != 0) { - printf("nfe%d: could not allocate Rx ring\n", unit); - nfe_free_tx_ring(sc, &sc->txq); - error = ENXIO; + if ((error = nfe_alloc_rx_ring(sc, &sc->rxq)) != 0) + goto fail; + + if ((error = nfe_alloc_jrx_ring(sc, &sc->jrxq)) != 0) goto fail; + + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "process_limit", CTLTYPE_INT | CTLFLAG_RW, + &sc->nfe_process_limit, 0, sysctl_hw_nfe_proc_limit, "I", + "max number of Rx events to process"); + + sc->nfe_process_limit = NFE_PROC_DEFAULT; + error = resource_int_value(device_get_name(dev), device_get_unit(dev), + "process_limit", &sc->nfe_process_limit); + if (error == 0) { + if (sc->nfe_process_limit < NFE_PROC_MIN || + sc->nfe_process_limit > NFE_PROC_MAX) { + device_printf(dev, "process_limit value out of range; " + "using default: %d\n", NFE_PROC_DEFAULT); + sc->nfe_process_limit = NFE_PROC_DEFAULT; + } } ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); + ifp->if_mtu = ETHERMTU; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = nfe_ioctl; ifp->if_start = nfe_start; - /* ifp->if_hwassist = NFE_CSUM_FEATURES; */ - ifp->if_watchdog = nfe_watchdog; + ifp->if_hwassist = 0; + ifp->if_capabilities = 0; + ifp->if_watchdog = NULL; ifp->if_init = nfe_init; - ifp->if_baudrate = IF_Gbps(1); - ifp->if_snd.ifq_maxlen = NFE_IFQ_MAXLEN; - - ifp->if_capabilities = IFCAP_VLAN_MTU; + IFQ_SET_MAXLEN(&ifp->if_snd, NFE_TX_RING_COUNT - 1); + ifp->if_snd.ifq_drv_maxlen = NFE_TX_RING_COUNT - 1; + IFQ_SET_READY(&ifp->if_snd); -#ifdef NFE_JUMBO - ifp->if_capabilities |= IFCAP_JUMBO_MTU; -#else - ifp->if_capabilities &= ~IFCAP_JUMBO_MTU; - sc->nfe_flags &= ~NFE_JUMBO_SUP; -#endif + if (sc->nfe_flags & NFE_HW_CSUM) { + ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4; + ifp->if_hwassist |= NFE_CSUM_FEATURES | CSUM_TSO; + } + ifp->if_capenable = ifp->if_capabilities; -#if NVLAN > 0 - if (sc->nfe_flags & NFE_HW_VLAN) + sc->nfe_framesize = ifp->if_mtu + NFE_RX_HEADERS; + /* VLAN capability setup. */ + ifp->if_capabilities |= IFCAP_VLAN_MTU; + if ((sc->nfe_flags & NFE_HW_VLAN) != 0) { ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; -#endif -#ifdef NFE_CSUM - if (sc->nfe_flags & NFE_HW_CSUM) { - ifp->if_capabilities |= IFCAP_HWCSUM; - ifp->if_capenable |= IFCAP_HWCSUM; - ifp->if_hwassist = NFE_CSUM_FEATURES; + if ((ifp->if_capabilities & IFCAP_HWCSUM) != 0) + ifp->if_capabilities |= IFCAP_VLAN_HWCSUM; } -#else - sc->nfe_flags &= ~NFE_HW_CSUM; -#endif ifp->if_capenable = ifp->if_capabilities; + /* + * Tell the upper layer(s) we support long frames. + * Must appear after the call to ether_ifattach() because + * ether_ifattach() sets ifi_hdrlen to the default value. + */ + ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); + #ifdef DEVICE_POLLING ifp->if_capabilities |= IFCAP_POLLING; #endif @@ -401,18 +578,35 @@ nfe_attach(device_t dev) /* Do MII setup */ if (mii_phy_probe(dev, &sc->nfe_miibus, nfe_ifmedia_upd, nfe_ifmedia_sts)) { - printf("nfe%d: MII without any phy!\n", unit); + device_printf(dev, "MII without any phy!\n"); error = ENXIO; goto fail; } - ether_ifattach(ifp, sc->eaddr); - error = bus_setup_intr(dev, sc->nfe_irq, INTR_TYPE_NET | INTR_MPSAFE, - NULL, nfe_intr, sc, &sc->nfe_intrhand); - + TASK_INIT(&sc->nfe_int_task, 0, nfe_int_task, sc); + sc->nfe_tq = taskqueue_create_fast("nfe_taskq", M_WAITOK, + taskqueue_thread_enqueue, &sc->nfe_tq); + taskqueue_start_threads(&sc->nfe_tq, 1, PI_NET, "%s taskq", + device_get_nameunit(sc->nfe_dev)); + error = 0; + if (sc->nfe_msi == 0 && sc->nfe_msix == 0) { + error = bus_setup_intr(dev, sc->nfe_irq[0], + INTR_TYPE_NET | INTR_MPSAFE, nfe_intr, NULL, sc, + &sc->nfe_intrhand[0]); + } else { + for (i = 0; i < NFE_MSI_MESSAGES; i++) { + error = bus_setup_intr(dev, sc->nfe_irq[i], + INTR_TYPE_NET | INTR_MPSAFE, nfe_intr, NULL, sc, + &sc->nfe_intrhand[i]); + if (error != 0) + break; + } + } if (error) { - printf("nfe%d: couldn't set up irq\n", unit); + device_printf(dev, "couldn't set up irq\n"); + taskqueue_free(sc->nfe_tq); + sc->nfe_tq = NULL; ether_ifdetach(ifp); goto fail; } @@ -430,66 +624,196 @@ nfe_detach(device_t dev) { struct nfe_softc *sc; struct ifnet *ifp; - u_char eaddr[ETHER_ADDR_LEN]; - int i; + uint8_t eaddr[ETHER_ADDR_LEN]; + int i, rid; sc = device_get_softc(dev); KASSERT(mtx_initialized(&sc->nfe_mtx), ("nfe mutex not initialized")); ifp = sc->nfe_ifp; #ifdef DEVICE_POLLING - if (ifp->if_capenable & IFCAP_POLLING) + if (ifp != NULL && ifp->if_capenable & IFCAP_POLLING) ether_poll_deregister(ifp); #endif - - for (i = 0; i < ETHER_ADDR_LEN; i++) { - eaddr[i] = sc->eaddr[5 - i]; - } - nfe_set_macaddr(sc, eaddr); - if (device_is_attached(dev)) { NFE_LOCK(sc); - nfe_stop(ifp, 1); + nfe_stop(ifp); ifp->if_flags &= ~IFF_UP; NFE_UNLOCK(sc); callout_drain(&sc->nfe_stat_ch); + taskqueue_drain(taskqueue_fast, &sc->nfe_tx_task); + taskqueue_drain(taskqueue_swi, &sc->nfe_link_task); ether_ifdetach(ifp); } - if (ifp) + if (ifp) { + /* restore ethernet address */ + if ((sc->nfe_flags & NFE_CORRECT_MACADDR) == 0) { + for (i = 0; i < ETHER_ADDR_LEN; i++) { + eaddr[i] = sc->eaddr[5 - i]; + } + } else + bcopy(sc->eaddr, eaddr, ETHER_ADDR_LEN); + nfe_set_macaddr(sc, eaddr); if_free(ifp); + } if (sc->nfe_miibus) device_delete_child(dev, sc->nfe_miibus); bus_generic_detach(dev); + if (sc->nfe_tq != NULL) { + taskqueue_drain(sc->nfe_tq, &sc->nfe_int_task); + taskqueue_free(sc->nfe_tq); + sc->nfe_tq = NULL; + } - if (sc->nfe_intrhand) - bus_teardown_intr(dev, sc->nfe_irq, sc->nfe_intrhand); - if (sc->nfe_irq) - bus_release_resource(dev, SYS_RES_IRQ, 0, sc->nfe_irq); - if (sc->nfe_res) - bus_release_resource(dev, SYS_RES_MEMORY, NV_RID, sc->nfe_res); + for (i = 0; i < NFE_MSI_MESSAGES; i++) { + if (sc->nfe_intrhand[i] != NULL) { + bus_teardown_intr(dev, sc->nfe_irq[i], + sc->nfe_intrhand[i]); + sc->nfe_intrhand[i] = NULL; + } + } + + if (sc->nfe_msi == 0 && sc->nfe_msix == 0) { + if (sc->nfe_irq[0] != NULL) + bus_release_resource(dev, SYS_RES_IRQ, 0, + sc->nfe_irq[0]); + } else { + for (i = 0, rid = 1; i < NFE_MSI_MESSAGES; i++, rid++) { + if (sc->nfe_irq[i] != NULL) { + bus_release_resource(dev, SYS_RES_IRQ, rid, + sc->nfe_irq[i]); + sc->nfe_irq[i] = NULL; + } + } + pci_release_msi(dev); + } + if (sc->nfe_msix_pba_res != NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(3), + sc->nfe_msix_pba_res); + sc->nfe_msix_pba_res = NULL; + } + if (sc->nfe_msix_res != NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(2), + sc->nfe_msix_res); + sc->nfe_msix_res = NULL; + } + if (sc->nfe_res[0] != NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0), + sc->nfe_res[0]); + sc->nfe_res[0] = NULL; + } nfe_free_tx_ring(sc, &sc->txq); nfe_free_rx_ring(sc, &sc->rxq); + nfe_free_jrx_ring(sc, &sc->jrxq); - if (sc->nfe_parent_tag) + if (sc->nfe_parent_tag) { bus_dma_tag_destroy(sc->nfe_parent_tag); + sc->nfe_parent_tag = NULL; + } + mtx_destroy(&sc->nfe_jlist_mtx); mtx_destroy(&sc->nfe_mtx); return (0); } +static int +nfe_suspend(device_t dev) +{ + struct nfe_softc *sc; + + sc = device_get_softc(dev); + + NFE_LOCK(sc); + nfe_stop(sc->nfe_ifp); + sc->nfe_suspended = 1; + NFE_UNLOCK(sc); + + return (0); +} + + +static int +nfe_resume(device_t dev) +{ + struct nfe_softc *sc; + struct ifnet *ifp; + + sc = device_get_softc(dev); + + NFE_LOCK(sc); + ifp = sc->nfe_ifp; + if (ifp->if_flags & IFF_UP) + nfe_init_locked(sc); + sc->nfe_suspended = 0; + NFE_UNLOCK(sc); + + return (0); +} + + +/* Take PHY/NIC out of powerdown, from Linux */ +static void +nfe_power(struct nfe_softc *sc) +{ + uint32_t pwr; + + if ((sc->nfe_flags & NFE_PWR_MGMT) == 0) + return; + NFE_WRITE(sc, NFE_RXTX_CTL, NFE_RXTX_RESET | NFE_RXTX_BIT2); + NFE_WRITE(sc, NFE_MAC_RESET, NFE_MAC_RESET_MAGIC); + DELAY(100); + NFE_WRITE(sc, NFE_MAC_RESET, 0); + DELAY(100); + NFE_WRITE(sc, NFE_RXTX_CTL, NFE_RXTX_BIT2); + pwr = NFE_READ(sc, NFE_PWR2_CTL); + pwr &= ~NFE_PWR2_WAKEUP_MASK; + if (sc->nfe_revid >= 0xa3 && + (sc->nfe_devid == PCI_PRODUCT_NVIDIA_NFORCE430_LAN1 || + sc->nfe_devid == PCI_PRODUCT_NVIDIA_NFORCE430_LAN2)) + pwr |= NFE_PWR2_REVA3; + NFE_WRITE(sc, NFE_PWR2_CTL, pwr); +} + + static void nfe_miibus_statchg(device_t dev) { struct nfe_softc *sc; - struct mii_data *mii; - u_int32_t phy, seed, misc = NFE_MISC1_MAGIC, link = NFE_MEDIA_SET; sc = device_get_softc(dev); + taskqueue_enqueue(taskqueue_swi, &sc->nfe_link_task); +} + + +static void +nfe_link_task(void *arg, int pending) +{ + struct nfe_softc *sc; + struct mii_data *mii; + struct ifnet *ifp; + uint32_t phy, seed, misc = NFE_MISC1_MAGIC, link = NFE_MEDIA_SET; + uint32_t gmask, rxctl, txctl, val; + + sc = (struct nfe_softc *)arg; + + NFE_LOCK(sc); + mii = device_get_softc(sc->nfe_miibus); + ifp = sc->nfe_ifp; + if (mii == NULL || ifp == NULL) { + NFE_UNLOCK(sc); + return; + } + + if (mii->mii_media_status & IFM_ACTIVE) { + if (IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) + sc->nfe_link = 1; + } else + sc->nfe_link = 0; phy = NFE_READ(sc, NFE_PHY_IFACE); phy &= ~(NFE_PHY_HDX | NFE_PHY_100TX | NFE_PHY_1000T); @@ -497,7 +821,7 @@ nfe_miibus_statchg(device_t dev) seed = NFE_READ(sc, NFE_RNDSEED); seed &= ~NFE_SEED_MASK; - if ((mii->mii_media_active & IFM_GMASK) == IFM_HDX) { + if (((mii->mii_media_active & IFM_GMASK) & IFM_FDX) == 0) { phy |= NFE_PHY_HDX; /* half-duplex */ misc |= NFE_MISC1_HDX; } @@ -519,11 +843,70 @@ nfe_miibus_statchg(device_t dev) break; } + if ((phy & 0x10000000) != 0) { + if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T) + val = NFE_R1_MAGIC_1000; + else + val = NFE_R1_MAGIC_10_100; + } else + val = NFE_R1_MAGIC_DEFAULT; + NFE_WRITE(sc, NFE_SETUP_R1, val); + NFE_WRITE(sc, NFE_RNDSEED, seed); /* XXX: gigabit NICs only? */ NFE_WRITE(sc, NFE_PHY_IFACE, phy); NFE_WRITE(sc, NFE_MISC1, misc); NFE_WRITE(sc, NFE_LINKSPEED, link); + + gmask = mii->mii_media_active & IFM_GMASK; + if ((gmask & IFM_FDX) != 0) { + /* It seems all hardwares supports Rx pause frames. */ + val = NFE_READ(sc, NFE_RXFILTER); + if ((gmask & IFM_FLAG0) != 0) + val |= NFE_PFF_RX_PAUSE; + else + val &= ~NFE_PFF_RX_PAUSE; + NFE_WRITE(sc, NFE_RXFILTER, val); + if ((sc->nfe_flags & NFE_TX_FLOW_CTRL) != 0) { + val = NFE_READ(sc, NFE_MISC1); + if ((gmask & IFM_FLAG1) != 0) { + NFE_WRITE(sc, NFE_TX_PAUSE_FRAME, + NFE_TX_PAUSE_FRAME_ENABLE); + val |= NFE_MISC1_TX_PAUSE; + } else { + val &= ~NFE_MISC1_TX_PAUSE; + NFE_WRITE(sc, NFE_TX_PAUSE_FRAME, + NFE_TX_PAUSE_FRAME_DISABLE); + } + NFE_WRITE(sc, NFE_MISC1, val); + } + } else { + /* disable rx/tx pause frames */ + val = NFE_READ(sc, NFE_RXFILTER); + val &= ~NFE_PFF_RX_PAUSE; + NFE_WRITE(sc, NFE_RXFILTER, val); + if ((sc->nfe_flags & NFE_TX_FLOW_CTRL) != 0) { + NFE_WRITE(sc, NFE_TX_PAUSE_FRAME, + NFE_TX_PAUSE_FRAME_DISABLE); + val = NFE_READ(sc, NFE_MISC1); + val &= ~NFE_MISC1_TX_PAUSE; + NFE_WRITE(sc, NFE_MISC1, val); + } + } + + txctl = NFE_READ(sc, NFE_TX_CTL); + rxctl = NFE_READ(sc, NFE_RX_CTL); + if (sc->nfe_link != 0 && (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { + txctl |= NFE_TX_START; + rxctl |= NFE_RX_START; + } else { + txctl &= ~NFE_TX_START; + rxctl &= ~NFE_RX_START; + } + NFE_WRITE(sc, NFE_TX_CTL, NFE_TX_START); + NFE_WRITE(sc, NFE_RX_CTL, NFE_RX_START); + + NFE_UNLOCK(sc); } @@ -531,7 +914,7 @@ static int nfe_miibus_readreg(device_t dev, int phy, int reg) { struct nfe_softc *sc = device_get_softc(dev); - u_int32_t val; + uint32_t val; int ntries; NFE_WRITE(sc, NFE_PHY_STATUS, 0xf); @@ -543,18 +926,18 @@ nfe_miibus_readreg(device_t dev, int phy, int reg) NFE_WRITE(sc, NFE_PHY_CTL, (phy << NFE_PHYADD_SHIFT) | reg); - for (ntries = 0; ntries < 1000; ntries++) { + for (ntries = 0; ntries < NFE_TIMEOUT; ntries++) { DELAY(100); if (!(NFE_READ(sc, NFE_PHY_CTL) & NFE_PHY_BUSY)) break; } - if (ntries == 1000) { - DPRINTFN(2, ("nfe%d: timeout waiting for PHY\n", sc->nfe_unit)); + if (ntries == NFE_TIMEOUT) { + DPRINTFN(sc, 2, "timeout waiting for PHY\n"); return 0; } if (NFE_READ(sc, NFE_PHY_STATUS) & NFE_PHY_ERROR) { - DPRINTFN(2, ("nfe%d: could not read PHY\n", sc->nfe_unit)); + DPRINTFN(sc, 2, "could not read PHY\n"); return 0; } @@ -562,10 +945,9 @@ nfe_miibus_readreg(device_t dev, int phy, int reg) if (val != 0xffffffff && val != 0) sc->mii_phyaddr = phy; - DPRINTFN(2, ("nfe%d: mii read phy %d reg 0x%x ret 0x%x\n", - sc->nfe_unit, phy, reg, val)); + DPRINTFN(sc, 2, "mii read phy %d reg 0x%x ret 0x%x\n", phy, reg, val); - return val; + return (val); } @@ -573,7 +955,7 @@ static int nfe_miibus_writereg(device_t dev, int phy, int reg, int val) { struct nfe_softc *sc = device_get_softc(dev); - u_int32_t ctl; + uint32_t ctl; int ntries; NFE_WRITE(sc, NFE_PHY_STATUS, 0xf); @@ -587,182 +969,399 @@ nfe_miibus_writereg(device_t dev, int phy, int reg, int val) ctl = NFE_PHY_WRITE | (phy << NFE_PHYADD_SHIFT) | reg; NFE_WRITE(sc, NFE_PHY_CTL, ctl); - for (ntries = 0; ntries < 1000; ntries++) { + for (ntries = 0; ntries < NFE_TIMEOUT; ntries++) { DELAY(100); if (!(NFE_READ(sc, NFE_PHY_CTL) & NFE_PHY_BUSY)) break; } #ifdef NFE_DEBUG - if (nfedebug >= 2 && ntries == 1000) - printf("could not write to PHY\n"); + if (nfedebug >= 2 && ntries == NFE_TIMEOUT) + device_printf(sc->nfe_dev, "could not write to PHY\n"); #endif - return 0; + return (0); } +/* + * Allocate a jumbo buffer. + */ +static void * +nfe_jalloc(struct nfe_softc *sc) +{ + struct nfe_jpool_entry *entry; + + NFE_JLIST_LOCK(sc); + + entry = SLIST_FIRST(&sc->nfe_jfree_listhead); + + if (entry == NULL) { + NFE_JLIST_UNLOCK(sc); + return (NULL); + } + + SLIST_REMOVE_HEAD(&sc->nfe_jfree_listhead, jpool_entries); + SLIST_INSERT_HEAD(&sc->nfe_jinuse_listhead, entry, jpool_entries); + + NFE_JLIST_UNLOCK(sc); + + return (sc->jrxq.jslots[entry->slot]); +} + +/* + * Release a jumbo buffer. + */ +static void +nfe_jfree(void *buf, void *args) +{ + struct nfe_softc *sc; + struct nfe_jpool_entry *entry; + int i; + + /* Extract the softc struct pointer. */ + sc = (struct nfe_softc *)args; + KASSERT(sc != NULL, ("%s: can't find softc pointer!", __func__)); + + NFE_JLIST_LOCK(sc); + /* Calculate the slot this buffer belongs to. */ + i = ((vm_offset_t)buf + - (vm_offset_t)sc->jrxq.jpool) / NFE_JLEN; + KASSERT(i >= 0 && i < NFE_JSLOTS, + ("%s: asked to free buffer that we don't manage!", __func__)); + + entry = SLIST_FIRST(&sc->nfe_jinuse_listhead); + KASSERT(entry != NULL, ("%s: buffer not in use!", __func__)); + entry->slot = i; + SLIST_REMOVE_HEAD(&sc->nfe_jinuse_listhead, jpool_entries); + SLIST_INSERT_HEAD(&sc->nfe_jfree_listhead, entry, jpool_entries); + if (SLIST_EMPTY(&sc->nfe_jinuse_listhead)) + wakeup(sc); + + NFE_JLIST_UNLOCK(sc); +} + +struct nfe_dmamap_arg { + bus_addr_t nfe_busaddr; +}; static int nfe_alloc_rx_ring(struct nfe_softc *sc, struct nfe_rx_ring *ring) { - struct nfe_desc32 *desc32; - struct nfe_desc64 *desc64; + struct nfe_dmamap_arg ctx; struct nfe_rx_data *data; - void **desc; - bus_addr_t physaddr; + void *desc; int i, error, descsize; if (sc->nfe_flags & NFE_40BIT_ADDR) { - desc = (void **)&ring->desc64; + desc = ring->desc64; descsize = sizeof (struct nfe_desc64); } else { - desc = (void **)&ring->desc32; + desc = ring->desc32; descsize = sizeof (struct nfe_desc32); } ring->cur = ring->next = 0; - ring->bufsz = (sc->nfe_mtu + NFE_RX_HEADERS <= MCLBYTES) ? - MCLBYTES : MJUM9BYTES; error = bus_dma_tag_create(sc->nfe_parent_tag, - PAGE_SIZE, 0, /* alignment, boundary */ - BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - NFE_RX_RING_COUNT * descsize, 1, /* maxsize, nsegments */ - NFE_RX_RING_COUNT * descsize, /* maxsegsize */ - BUS_DMA_ALLOCNOW, /* flags */ - NULL, NULL, /* lockfunc, lockarg */ - &ring->rx_desc_tag); + NFE_RING_ALIGN, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + NFE_RX_RING_COUNT * descsize, 1, /* maxsize, nsegments */ + NFE_RX_RING_COUNT * descsize, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &ring->rx_desc_tag); if (error != 0) { - printf("nfe%d: could not create desc DMA tag\n", sc->nfe_unit); + device_printf(sc->nfe_dev, "could not create desc DMA tag\n"); goto fail; } /* allocate memory to desc */ - error = bus_dmamem_alloc(ring->rx_desc_tag, (void **)desc, - BUS_DMA_NOWAIT, &ring->rx_desc_map); + error = bus_dmamem_alloc(ring->rx_desc_tag, &desc, BUS_DMA_WAITOK | + BUS_DMA_COHERENT | BUS_DMA_ZERO, &ring->rx_desc_map); if (error != 0) { - printf("nfe%d: could not create desc DMA map\n", sc->nfe_unit); + device_printf(sc->nfe_dev, "could not create desc DMA map\n"); goto fail; } + if (sc->nfe_flags & NFE_40BIT_ADDR) + ring->desc64 = desc; + else + ring->desc32 = desc; /* map desc to device visible address space */ - error = bus_dmamap_load(ring->rx_desc_tag, ring->rx_desc_map, *desc, - NFE_RX_RING_COUNT * descsize, nfe_dma_map_segs, - &ring->rx_desc_segs, BUS_DMA_NOWAIT); + ctx.nfe_busaddr = 0; + error = bus_dmamap_load(ring->rx_desc_tag, ring->rx_desc_map, desc, + NFE_RX_RING_COUNT * descsize, nfe_dma_map_segs, &ctx, 0); if (error != 0) { - printf("nfe%d: could not load desc DMA map\n", sc->nfe_unit); + device_printf(sc->nfe_dev, "could not load desc DMA map\n"); goto fail; } + ring->physaddr = ctx.nfe_busaddr; - bzero(*desc, NFE_RX_RING_COUNT * descsize); - ring->rx_desc_addr = ring->rx_desc_segs.ds_addr; - ring->physaddr = ring->rx_desc_addr; + error = bus_dma_tag_create(sc->nfe_parent_tag, + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MCLBYTES, 1, /* maxsize, nsegments */ + MCLBYTES, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &ring->rx_data_tag); + if (error != 0) { + device_printf(sc->nfe_dev, "could not create Rx DMA tag\n"); + goto fail; + } + + error = bus_dmamap_create(ring->rx_data_tag, 0, &ring->rx_spare_map); + if (error != 0) { + device_printf(sc->nfe_dev, + "could not create Rx DMA spare map\n"); + goto fail; + } /* * Pre-allocate Rx buffers and populate Rx ring. */ for (i = 0; i < NFE_RX_RING_COUNT; i++) { data = &sc->rxq.data[i]; - - MGETHDR(data->m, M_DONTWAIT, MT_DATA); - if (data->m == NULL) { - printf("nfe%d: could not allocate rx mbuf\n", - sc->nfe_unit); - error = ENOMEM; - goto fail; - } - - error = bus_dma_tag_create(sc->nfe_parent_tag, - ETHER_ALIGN, 0, /* alignment, boundary */ - BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MCLBYTES, 1, /* maxsize, nsegments */ - MCLBYTES, /* maxsegsize */ - BUS_DMA_ALLOCNOW, /* flags */ - NULL, NULL, /* lockfunc, lockarg */ - &data->rx_data_tag); + data->rx_data_map = NULL; + data->m = NULL; + error = bus_dmamap_create(ring->rx_data_tag, 0, + &data->rx_data_map); if (error != 0) { - printf("nfe%d: could not create DMA map\n", - sc->nfe_unit); + device_printf(sc->nfe_dev, + "could not create Rx DMA map\n"); goto fail; } + } + +fail: + return (error); +} - error = bus_dmamap_create(data->rx_data_tag, 0, + +static int +nfe_alloc_jrx_ring(struct nfe_softc *sc, struct nfe_jrx_ring *ring) +{ + struct nfe_dmamap_arg ctx; + struct nfe_rx_data *data; + void *desc; + struct nfe_jpool_entry *entry; + uint8_t *ptr; + int i, error, descsize; + + if ((sc->nfe_flags & NFE_JUMBO_SUP) == 0) + return (0); + + if (sc->nfe_flags & NFE_40BIT_ADDR) { + desc = ring->jdesc64; + descsize = sizeof (struct nfe_desc64); + } else { + desc = ring->jdesc32; + descsize = sizeof (struct nfe_desc32); + } + + ring->jcur = ring->jnext = 0; + + /* Create DMA tag for jumbo Rx ring. */ + error = bus_dma_tag_create(sc->nfe_parent_tag, + NFE_RING_ALIGN, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + NFE_JUMBO_RX_RING_COUNT * descsize, /* maxsize */ + 1, /* nsegments */ + NFE_JUMBO_RX_RING_COUNT * descsize, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &ring->jrx_desc_tag); + if (error != 0) { + device_printf(sc->nfe_dev, + "could not create jumbo ring DMA tag\n"); + goto fail; + } + + /* Create DMA tag for jumbo buffer blocks. */ + error = bus_dma_tag_create(sc->nfe_parent_tag, + PAGE_SIZE, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + NFE_JMEM, /* maxsize */ + 1, /* nsegments */ + NFE_JMEM, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &ring->jrx_jumbo_tag); + if (error != 0) { + device_printf(sc->nfe_dev, + "could not create jumbo Rx buffer block DMA tag\n"); + goto fail; + } + + /* Create DMA tag for jumbo Rx buffers. */ + error = bus_dma_tag_create(sc->nfe_parent_tag, + PAGE_SIZE, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + NFE_JLEN, /* maxsize */ + 1, /* nsegments */ + NFE_JLEN, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &ring->jrx_data_tag); + if (error != 0) { + device_printf(sc->nfe_dev, + "could not create jumbo Rx buffer DMA tag\n"); + goto fail; + } + + /* Allocate DMA'able memory and load the DMA map for jumbo Rx ring. */ + error = bus_dmamem_alloc(ring->jrx_desc_tag, &desc, BUS_DMA_WAITOK | + BUS_DMA_COHERENT | BUS_DMA_ZERO, &ring->jrx_desc_map); + if (error != 0) { + device_printf(sc->nfe_dev, + "could not allocate DMA'able memory for jumbo Rx ring\n"); + goto fail; + } + if (sc->nfe_flags & NFE_40BIT_ADDR) + ring->jdesc64 = desc; + else + ring->jdesc32 = desc; + + ctx.nfe_busaddr = 0; + error = bus_dmamap_load(ring->jrx_desc_tag, ring->jrx_desc_map, desc, + NFE_JUMBO_RX_RING_COUNT * descsize, nfe_dma_map_segs, &ctx, 0); + if (error != 0) { + device_printf(sc->nfe_dev, + "could not load DMA'able memory for jumbo Rx ring\n"); + goto fail; + } + ring->jphysaddr = ctx.nfe_busaddr; + + /* Create DMA maps for jumbo Rx buffers. */ + error = bus_dmamap_create(ring->jrx_data_tag, 0, &ring->jrx_spare_map); + if (error != 0) { + device_printf(sc->nfe_dev, + "could not create jumbo Rx DMA spare map\n"); + goto fail; + } + + for (i = 0; i < NFE_JUMBO_RX_RING_COUNT; i++) { + data = &sc->jrxq.jdata[i]; + data->rx_data_map = NULL; + data->m = NULL; + error = bus_dmamap_create(ring->jrx_data_tag, 0, &data->rx_data_map); if (error != 0) { - printf("nfe%d: could not allocate mbuf cluster\n", - sc->nfe_unit); + device_printf(sc->nfe_dev, + "could not create jumbo Rx DMA map\n"); goto fail; } + } - MCLGET(data->m, M_DONTWAIT); - if (!(data->m->m_flags & M_EXT)) { + /* Allocate DMA'able memory and load the DMA map for jumbo buf. */ + error = bus_dmamem_alloc(ring->jrx_jumbo_tag, (void **)&ring->jpool, + BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, + &ring->jrx_jumbo_map); + if (error != 0) { + device_printf(sc->nfe_dev, + "could not allocate DMA'able memory for jumbo pool\n"); + goto fail; + } + + ctx.nfe_busaddr = 0; + error = bus_dmamap_load(ring->jrx_jumbo_tag, ring->jrx_jumbo_map, + ring->jpool, NFE_JMEM, nfe_dma_map_segs, &ctx, 0); + if (error != 0) { + device_printf(sc->nfe_dev, + "could not load DMA'able memory for jumbo pool\n"); + goto fail; + } + + /* + * Now divide it up into 9K pieces and save the addresses + * in an array. + */ + ptr = ring->jpool; + for (i = 0; i < NFE_JSLOTS; i++) { + ring->jslots[i] = ptr; + ptr += NFE_JLEN; + entry = malloc(sizeof(struct nfe_jpool_entry), M_DEVBUF, + M_WAITOK); + if (entry == NULL) { + device_printf(sc->nfe_dev, + "no memory for jumbo buffers!\n"); error = ENOMEM; goto fail; } + entry->slot = i; + SLIST_INSERT_HEAD(&sc->nfe_jfree_listhead, entry, + jpool_entries); + } - error = bus_dmamap_load(data->rx_data_tag, - data->rx_data_map, mtod(data->m, void *), - ring->bufsz, nfe_dma_map_segs, &data->rx_data_segs, - BUS_DMA_NOWAIT); - if (error != 0) { - printf("nfe%d: could not load rx buf DMA map\n", - sc->nfe_unit); - goto fail; - } + return (0); - data->rx_data_addr = data->rx_data_segs.ds_addr; - physaddr = data->rx_data_addr; +fail: + nfe_free_jrx_ring(sc, ring); + return (error); +} - if (sc->nfe_flags & NFE_40BIT_ADDR) { - desc64 = &sc->rxq.desc64[i]; -#if defined(__LP64__) - desc64->physaddr[0] = htole32(physaddr >> 32); -#endif - desc64->physaddr[1] = htole32(physaddr & 0xffffffff); - desc64->length = htole16(sc->rxq.bufsz); - desc64->flags = htole16(NFE_RX_READY); - } else { - desc32 = &sc->rxq.desc32[i]; - desc32->physaddr = htole32(physaddr); - desc32->length = htole16(sc->rxq.bufsz); - desc32->flags = htole16(NFE_RX_READY); - } +static int +nfe_init_rx_ring(struct nfe_softc *sc, struct nfe_rx_ring *ring) +{ + void *desc; + size_t descsize; + int i; + ring->cur = ring->next = 0; + if (sc->nfe_flags & NFE_40BIT_ADDR) { + desc = ring->desc64; + descsize = sizeof (struct nfe_desc64); + } else { + desc = ring->desc32; + descsize = sizeof (struct nfe_desc32); + } + bzero(desc, descsize * NFE_RX_RING_COUNT); + for (i = 0; i < NFE_RX_RING_COUNT; i++) { + if (nfe_newbuf(sc, i) != 0) + return (ENOBUFS); } bus_dmamap_sync(ring->rx_desc_tag, ring->rx_desc_map, - BUS_DMASYNC_PREWRITE); - - return 0; - -fail: nfe_free_rx_ring(sc, ring); + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - return error; + return (0); } -static void -nfe_reset_rx_ring(struct nfe_softc *sc, struct nfe_rx_ring *ring) +static int +nfe_init_jrx_ring(struct nfe_softc *sc, struct nfe_jrx_ring *ring) { + void *desc; + size_t descsize; int i; - for (i = 0; i < NFE_RX_RING_COUNT; i++) { - if (sc->nfe_flags & NFE_40BIT_ADDR) { - ring->desc64[i].length = htole16(ring->bufsz); - ring->desc64[i].flags = htole16(NFE_RX_READY); - } else { - ring->desc32[i].length = htole16(ring->bufsz); - ring->desc32[i].flags = htole16(NFE_RX_READY); - } + ring->jcur = ring->jnext = 0; + if (sc->nfe_flags & NFE_40BIT_ADDR) { + desc = ring->jdesc64; + descsize = sizeof (struct nfe_desc64); + } else { + desc = ring->jdesc32; + descsize = sizeof (struct nfe_desc32); + } + bzero(desc, descsize * NFE_RX_RING_COUNT); + for (i = 0; i < NFE_JUMBO_RX_RING_COUNT; i++) { + if (nfe_jnewbuf(sc, i) != 0) + return (ENOBUFS); } - bus_dmamap_sync(ring->rx_desc_tag, ring->rx_desc_map, - BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(ring->jrx_desc_tag, ring->jrx_desc_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - ring->cur = ring->next = 0; + return (0); } @@ -781,29 +1380,118 @@ nfe_free_rx_ring(struct nfe_softc *sc, struct nfe_rx_ring *ring) descsize = sizeof (struct nfe_desc32); } + for (i = 0; i < NFE_RX_RING_COUNT; i++) { + data = &ring->data[i]; + if (data->rx_data_map != NULL) { + bus_dmamap_destroy(ring->rx_data_tag, + data->rx_data_map); + data->rx_data_map = NULL; + } + if (data->m != NULL) { + m_freem(data->m); + data->m = NULL; + } + } + if (ring->rx_data_tag != NULL) { + if (ring->rx_spare_map != NULL) { + bus_dmamap_destroy(ring->rx_data_tag, + ring->rx_spare_map); + ring->rx_spare_map = NULL; + } + bus_dma_tag_destroy(ring->rx_data_tag); + ring->rx_data_tag = NULL; + } + if (desc != NULL) { - bus_dmamap_sync(ring->rx_desc_tag, ring->rx_desc_map, - BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->rx_desc_tag, ring->rx_desc_map); bus_dmamem_free(ring->rx_desc_tag, desc, ring->rx_desc_map); + ring->desc64 = NULL; + ring->desc32 = NULL; + ring->rx_desc_map = NULL; + } + if (ring->rx_desc_tag != NULL) { bus_dma_tag_destroy(ring->rx_desc_tag); + ring->rx_desc_tag = NULL; } +} - for (i = 0; i < NFE_RX_RING_COUNT; i++) { - data = &ring->data[i]; +static void +nfe_free_jrx_ring(struct nfe_softc *sc, struct nfe_jrx_ring *ring) +{ + struct nfe_jpool_entry *entry; + struct nfe_rx_data *data; + void *desc; + int i, descsize; + + if ((sc->nfe_flags & NFE_JUMBO_SUP) == 0) + return; + + NFE_JLIST_LOCK(sc); + while ((entry = SLIST_FIRST(&sc->nfe_jinuse_listhead))) { + device_printf(sc->nfe_dev, + "asked to free buffer that is in use!\n"); + SLIST_REMOVE_HEAD(&sc->nfe_jinuse_listhead, jpool_entries); + SLIST_INSERT_HEAD(&sc->nfe_jfree_listhead, entry, + jpool_entries); + } + + while (!SLIST_EMPTY(&sc->nfe_jfree_listhead)) { + entry = SLIST_FIRST(&sc->nfe_jfree_listhead); + SLIST_REMOVE_HEAD(&sc->nfe_jfree_listhead, jpool_entries); + free(entry, M_DEVBUF); + } + NFE_JLIST_UNLOCK(sc); + + if (sc->nfe_flags & NFE_40BIT_ADDR) { + desc = ring->jdesc64; + descsize = sizeof (struct nfe_desc64); + } else { + desc = ring->jdesc32; + descsize = sizeof (struct nfe_desc32); + } + + for (i = 0; i < NFE_JUMBO_RX_RING_COUNT; i++) { + data = &ring->jdata[i]; if (data->rx_data_map != NULL) { - bus_dmamap_sync(data->rx_data_tag, - data->rx_data_map, BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(data->rx_data_tag, - data->rx_data_map); - bus_dmamap_destroy(data->rx_data_tag, + bus_dmamap_destroy(ring->jrx_data_tag, data->rx_data_map); - bus_dma_tag_destroy(data->rx_data_tag); + data->rx_data_map = NULL; } - - if (data->m != NULL) + if (data->m != NULL) { m_freem(data->m); + data->m = NULL; + } + } + if (ring->jrx_data_tag != NULL) { + if (ring->jrx_spare_map != NULL) { + bus_dmamap_destroy(ring->jrx_data_tag, + ring->jrx_spare_map); + ring->jrx_spare_map = NULL; + } + bus_dma_tag_destroy(ring->jrx_data_tag); + ring->jrx_data_tag = NULL; + } + + if (desc != NULL) { + bus_dmamap_unload(ring->jrx_desc_tag, ring->jrx_desc_map); + bus_dmamem_free(ring->jrx_desc_tag, desc, ring->jrx_desc_map); + ring->jdesc64 = NULL; + ring->jdesc32 = NULL; + ring->jrx_desc_map = NULL; + } + /* Destroy jumbo buffer block. */ + if (ring->jrx_jumbo_map != NULL) + bus_dmamap_unload(ring->jrx_jumbo_tag, ring->jrx_jumbo_map); + if (ring->jrx_jumbo_map != NULL) { + bus_dmamem_free(ring->jrx_jumbo_tag, ring->jpool, + ring->jrx_jumbo_map); + ring->jpool = NULL; + ring->jrx_jumbo_map = NULL; + } + if (ring->jrx_desc_tag != NULL) { + bus_dma_tag_destroy(ring->jrx_desc_tag); + ring->jrx_desc_tag = NULL; } } @@ -811,15 +1499,16 @@ nfe_free_rx_ring(struct nfe_softc *sc, struct nfe_rx_ring *ring) static int nfe_alloc_tx_ring(struct nfe_softc *sc, struct nfe_tx_ring *ring) { + struct nfe_dmamap_arg ctx; int i, error; - void **desc; + void *desc; int descsize; if (sc->nfe_flags & NFE_40BIT_ADDR) { - desc = (void **)&ring->desc64; + desc = ring->desc64; descsize = sizeof (struct nfe_desc64); } else { - desc = (void **)&ring->desc32; + desc = ring->desc32; descsize = sizeof (struct nfe_desc32); } @@ -827,100 +1516,91 @@ nfe_alloc_tx_ring(struct nfe_softc *sc, struct nfe_tx_ring *ring) ring->cur = ring->next = 0; error = bus_dma_tag_create(sc->nfe_parent_tag, - PAGE_SIZE, 0, /* alignment, boundary */ - BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - NFE_TX_RING_COUNT * descsize, 1, /* maxsize, nsegments */ - NFE_TX_RING_COUNT * descsize, /* maxsegsize */ - BUS_DMA_ALLOCNOW, /* flags */ - NULL, NULL, /* lockfunc, lockarg */ - &ring->tx_desc_tag); + NFE_RING_ALIGN, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + NFE_TX_RING_COUNT * descsize, 1, /* maxsize, nsegments */ + NFE_TX_RING_COUNT * descsize, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &ring->tx_desc_tag); if (error != 0) { - printf("nfe%d: could not create desc DMA tag\n", sc->nfe_unit); + device_printf(sc->nfe_dev, "could not create desc DMA tag\n"); goto fail; } - error = bus_dmamem_alloc(ring->tx_desc_tag, (void **)desc, - BUS_DMA_NOWAIT, &ring->tx_desc_map); + error = bus_dmamem_alloc(ring->tx_desc_tag, &desc, BUS_DMA_WAITOK | + BUS_DMA_COHERENT | BUS_DMA_ZERO, &ring->tx_desc_map); if (error != 0) { - printf("nfe%d: could not create desc DMA map\n", sc->nfe_unit); + device_printf(sc->nfe_dev, "could not create desc DMA map\n"); goto fail; } + if (sc->nfe_flags & NFE_40BIT_ADDR) + ring->desc64 = desc; + else + ring->desc32 = desc; - error = bus_dmamap_load(ring->tx_desc_tag, ring->tx_desc_map, *desc, - NFE_TX_RING_COUNT * descsize, nfe_dma_map_segs, &ring->tx_desc_segs, - BUS_DMA_NOWAIT); + ctx.nfe_busaddr = 0; + error = bus_dmamap_load(ring->tx_desc_tag, ring->tx_desc_map, desc, + NFE_TX_RING_COUNT * descsize, nfe_dma_map_segs, &ctx, 0); if (error != 0) { - printf("nfe%d: could not load desc DMA map\n", sc->nfe_unit); + device_printf(sc->nfe_dev, "could not load desc DMA map\n"); goto fail; } - - bzero(*desc, NFE_TX_RING_COUNT * descsize); - - ring->tx_desc_addr = ring->tx_desc_segs.ds_addr; - ring->physaddr = ring->tx_desc_addr; + ring->physaddr = ctx.nfe_busaddr; error = bus_dma_tag_create(sc->nfe_parent_tag, - ETHER_ALIGN, 0, - BUS_SPACE_MAXADDR_32BIT, - BUS_SPACE_MAXADDR, - NULL, NULL, - NFE_JBYTES, NFE_MAX_SCATTER, - NFE_JBYTES, - BUS_DMA_ALLOCNOW, - NULL, NULL, - &ring->tx_data_tag); + 1, 0, + BUS_SPACE_MAXADDR, + BUS_SPACE_MAXADDR, + NULL, NULL, + MCLBYTES * NFE_MAX_SCATTER, + NFE_MAX_SCATTER, + MCLBYTES, + 0, + NULL, NULL, + &ring->tx_data_tag); if (error != 0) { - printf("nfe%d: could not create DMA tag\n", sc->nfe_unit); - goto fail; + device_printf(sc->nfe_dev, "could not create Tx DMA tag\n"); + goto fail; } for (i = 0; i < NFE_TX_RING_COUNT; i++) { error = bus_dmamap_create(ring->tx_data_tag, 0, &ring->data[i].tx_data_map); if (error != 0) { - printf("nfe%d: could not create DMA map\n", - sc->nfe_unit); + device_printf(sc->nfe_dev, + "could not create Tx DMA map\n"); goto fail; } } - return 0; - -fail: nfe_free_tx_ring(sc, ring); - return error; +fail: + return (error); } static void -nfe_reset_tx_ring(struct nfe_softc *sc, struct nfe_tx_ring *ring) +nfe_init_tx_ring(struct nfe_softc *sc, struct nfe_tx_ring *ring) { - struct nfe_tx_data *data; - int i; - - for (i = 0; i < NFE_TX_RING_COUNT; i++) { - if (sc->nfe_flags & NFE_40BIT_ADDR) - ring->desc64[i].flags = 0; - else - ring->desc32[i].flags = 0; - - data = &ring->data[i]; + void *desc; + size_t descsize; - if (data->m != NULL) { - bus_dmamap_sync(ring->tx_data_tag, data->active, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(ring->tx_data_tag, data->active); - m_freem(data->m); - data->m = NULL; - } + sc->nfe_force_tx = 0; + ring->queued = 0; + ring->cur = ring->next = 0; + if (sc->nfe_flags & NFE_40BIT_ADDR) { + desc = ring->desc64; + descsize = sizeof (struct nfe_desc64); + } else { + desc = ring->desc32; + descsize = sizeof (struct nfe_desc32); } + bzero(desc, descsize * NFE_TX_RING_COUNT); bus_dmamap_sync(ring->tx_desc_tag, ring->tx_desc_map, - BUS_DMASYNC_PREWRITE); - - ring->queued = 0; - ring->cur = ring->next = 0; + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); } @@ -939,34 +1619,39 @@ nfe_free_tx_ring(struct nfe_softc *sc, struct nfe_tx_ring *ring) descsize = sizeof (struct nfe_desc32); } - if (desc != NULL) { - bus_dmamap_sync(ring->tx_desc_tag, ring->tx_desc_map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(ring->tx_desc_tag, ring->tx_desc_map); - bus_dmamem_free(ring->tx_desc_tag, desc, ring->tx_desc_map); - bus_dma_tag_destroy(ring->tx_desc_tag); - } - for (i = 0; i < NFE_TX_RING_COUNT; i++) { data = &ring->data[i]; if (data->m != NULL) { - bus_dmamap_sync(ring->tx_data_tag, data->active, + bus_dmamap_sync(ring->tx_data_tag, data->tx_data_map, BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(ring->tx_data_tag, data->active); + bus_dmamap_unload(ring->tx_data_tag, data->tx_data_map); m_freem(data->m); + data->m = NULL; + } + if (data->tx_data_map != NULL) { + bus_dmamap_destroy(ring->tx_data_tag, + data->tx_data_map); + data->tx_data_map = NULL; } } - /* ..and now actually destroy the DMA mappings */ - for (i = 0; i < NFE_TX_RING_COUNT; i++) { - data = &ring->data[i]; - if (data->tx_data_map == NULL) - continue; - bus_dmamap_destroy(ring->tx_data_tag, data->tx_data_map); + if (ring->tx_data_tag != NULL) { + bus_dma_tag_destroy(ring->tx_data_tag); + ring->tx_data_tag = NULL; } - bus_dma_tag_destroy(ring->tx_data_tag); + if (desc != NULL) { + bus_dmamap_sync(ring->tx_desc_tag, ring->tx_desc_map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(ring->tx_desc_tag, ring->tx_desc_map); + bus_dmamem_free(ring->tx_desc_tag, desc, ring->tx_desc_map); + ring->desc64 = NULL; + ring->desc32 = NULL; + ring->tx_desc_map = NULL; + bus_dma_tag_destroy(ring->tx_desc_tag); + ring->tx_desc_tag = NULL; + } } #ifdef DEVICE_POLLING @@ -977,91 +1662,100 @@ static void nfe_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) { struct nfe_softc *sc = ifp->if_softc; + uint32_t r; NFE_LOCK(sc); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) - nfe_poll_locked(ifp, cmd, count); - NFE_UNLOCK(sc); -} - - -static void -nfe_poll_locked(struct ifnet *ifp, enum poll_cmd cmd, int count) -{ - struct nfe_softc *sc = ifp->if_softc; - u_int32_t r; - - NFE_LOCK_ASSERT(sc); if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { + NFE_UNLOCK(sc); return; } - sc->rxcycles = count; - nfe_rxeof(sc); + nfe_rxeof(sc, count); nfe_txeof(sc); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - nfe_start_locked(ifp); + taskqueue_enqueue_fast(taskqueue_fast, &sc->nfe_tx_task); if (cmd == POLL_AND_CHECK_STATUS) { - if ((r = NFE_READ(sc, NFE_IRQ_STATUS)) == 0) { + if ((r = NFE_READ(sc, sc->nfe_irq_status)) == 0) { + NFE_UNLOCK(sc); return; } - NFE_WRITE(sc, NFE_IRQ_STATUS, r); + NFE_WRITE(sc, sc->nfe_irq_status, r); if (r & NFE_IRQ_LINK) { NFE_READ(sc, NFE_PHY_STATUS); NFE_WRITE(sc, NFE_PHY_STATUS, 0xf); - DPRINTF(("nfe%d: link state changed\n", sc->nfe_unit)); + DPRINTF(sc, "link state changed\n"); } } + NFE_UNLOCK(sc); } #endif /* DEVICE_POLLING */ +static void +nfe_set_intr(struct nfe_softc *sc) +{ + + if (sc->nfe_msi != 0) + NFE_WRITE(sc, NFE_IRQ_MASK, NFE_IRQ_WANTED); +} + + +/* In MSIX, a write to mask reegisters behaves as XOR. */ +static __inline void +nfe_enable_intr(struct nfe_softc *sc) +{ + + if (sc->nfe_msix != 0) { + /* XXX Should have a better way to enable interrupts! */ + if (NFE_READ(sc, sc->nfe_irq_mask) == 0) + NFE_WRITE(sc, sc->nfe_irq_mask, sc->nfe_intrs); + } else + NFE_WRITE(sc, sc->nfe_irq_mask, sc->nfe_intrs); +} + + +static __inline void +nfe_disable_intr(struct nfe_softc *sc) +{ + + if (sc->nfe_msix != 0) { + /* XXX Should have a better way to disable interrupts! */ + if (NFE_READ(sc, sc->nfe_irq_mask) != 0) + NFE_WRITE(sc, sc->nfe_irq_mask, sc->nfe_nointrs); + } else + NFE_WRITE(sc, sc->nfe_irq_mask, sc->nfe_nointrs); +} + static int nfe_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { - struct nfe_softc *sc = ifp->if_softc; - struct ifreq *ifr = (struct ifreq *) data; + struct nfe_softc *sc; + struct ifreq *ifr; struct mii_data *mii; - int error = 0; + int error, init, mask; + sc = ifp->if_softc; + ifr = (struct ifreq *) data; + error = 0; + init = 0; switch (cmd) { case SIOCSIFMTU: - if (ifr->ifr_mtu == ifp->if_mtu) { + if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > NFE_JUMBO_MTU) error = EINVAL; - break; - } - if ((sc->nfe_flags & NFE_JUMBO_SUP) && (ifr->ifr_mtu >= - ETHERMIN && ifr->ifr_mtu <= NV_PKTLIMIT_2)) { - NFE_LOCK(sc); - sc->nfe_mtu = ifp->if_mtu = ifr->ifr_mtu; - nfe_stop(ifp, 1); - nfe_free_tx_ring(sc, &sc->txq); - nfe_free_rx_ring(sc, &sc->rxq); - NFE_UNLOCK(sc); - - /* Reallocate Tx and Rx rings. */ - if (nfe_alloc_tx_ring(sc, &sc->txq) != 0) { - printf("nfe%d: could not allocate Tx ring\n", - sc->nfe_unit); - error = ENXIO; - break; - } - - if (nfe_alloc_rx_ring(sc, &sc->rxq) != 0) { - printf("nfe%d: could not allocate Rx ring\n", - sc->nfe_unit); - nfe_free_tx_ring(sc, &sc->txq); - error = ENXIO; - break; + else if (ifp->if_mtu != ifr->ifr_mtu) { + if ((sc->nfe_flags & NFE_JUMBO_SUP) == 0 && + ifr->ifr_mtu > ETHERMTU) + error = EINVAL; + else { + NFE_LOCK(sc); + ifp->if_mtu = ifr->ifr_mtu; + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) + nfe_init_locked(sc); + NFE_UNLOCK(sc); } - NFE_LOCK(sc); - nfe_init_locked(sc); - NFE_UNLOCK(sc); - } else { - error = EINVAL; } break; case SIOCSIFFLAGS: @@ -1080,7 +1774,7 @@ nfe_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) nfe_init_locked(sc); } else { if (ifp->if_drv_flags & IFF_DRV_RUNNING) - nfe_stop(ifp, 1); + nfe_stop(ifp); } sc->nfe_if_flags = ifp->if_flags; NFE_UNLOCK(sc); @@ -1088,7 +1782,7 @@ nfe_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) break; case SIOCADDMULTI: case SIOCDELMULTI: - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { NFE_LOCK(sc); nfe_setmulti(sc); NFE_UNLOCK(sc); @@ -1101,64 +1795,118 @@ nfe_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd); break; case SIOCSIFCAP: - { - int init = 0; - int mask = ifr->ifr_reqcap ^ ifp->if_capenable; + mask = ifr->ifr_reqcap ^ ifp->if_capenable; #ifdef DEVICE_POLLING - if (mask & IFCAP_POLLING) { - if (ifr->ifr_reqcap & IFCAP_POLLING) { + if ((mask & IFCAP_POLLING) != 0) { + if ((ifr->ifr_reqcap & IFCAP_POLLING) != 0) { error = ether_poll_register(nfe_poll, ifp); if (error) - return(error); + break; NFE_LOCK(sc); - NFE_WRITE(sc, NFE_IRQ_MASK, 0); + nfe_disable_intr(sc); ifp->if_capenable |= IFCAP_POLLING; NFE_UNLOCK(sc); } else { error = ether_poll_deregister(ifp); /* Enable interrupt even in error case */ NFE_LOCK(sc); - NFE_WRITE(sc, NFE_IRQ_MASK, NFE_IRQ_WANTED); + nfe_enable_intr(sc); ifp->if_capenable &= ~IFCAP_POLLING; NFE_UNLOCK(sc); } } #endif /* DEVICE_POLLING */ -#ifdef NFE_CSUM - if (mask & IFCAP_HWCSUM) { + if ((sc->nfe_flags & NFE_HW_CSUM) != 0 && + (mask & IFCAP_HWCSUM) != 0) { ifp->if_capenable ^= IFCAP_HWCSUM; - if (IFCAP_HWCSUM & ifp->if_capenable && - IFCAP_HWCSUM & ifp->if_capabilities) - ifp->if_hwassist = NFE_CSUM_FEATURES; + if ((IFCAP_TXCSUM & ifp->if_capenable) != 0 && + (IFCAP_TXCSUM & ifp->if_capabilities) != 0) + ifp->if_hwassist |= NFE_CSUM_FEATURES; else - ifp->if_hwassist = 0; - sc->nfe_flags ^= NFE_HW_CSUM; - init = 1; + ifp->if_hwassist &= ~NFE_CSUM_FEATURES; + init++; } -#endif - if (init && ifp->if_drv_flags & IFF_DRV_RUNNING) + if ((sc->nfe_flags & NFE_HW_VLAN) != 0 && + (mask & IFCAP_VLAN_HWTAGGING) != 0) { + ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + init++; + } + /* + * XXX + * It seems that VLAN stripping requires Rx checksum offload. + * Unfortunately FreeBSD has no way to disable only Rx side + * VLAN stripping. So when we know Rx checksum offload is + * disabled turn entire hardware VLAN assist off. + */ + if ((sc->nfe_flags & (NFE_HW_CSUM | NFE_HW_VLAN)) == + (NFE_HW_CSUM | NFE_HW_VLAN)) { + if ((ifp->if_capenable & IFCAP_RXCSUM) == 0) + ifp->if_capenable &= ~IFCAP_VLAN_HWTAGGING; + } + + if ((sc->nfe_flags & NFE_HW_CSUM) != 0 && + (mask & IFCAP_TSO4) != 0) { + ifp->if_capenable ^= IFCAP_TSO4; + if ((IFCAP_TSO4 & ifp->if_capenable) != 0 && + (IFCAP_TSO4 & ifp->if_capabilities) != 0) + ifp->if_hwassist |= CSUM_TSO; + else + ifp->if_hwassist &= ~CSUM_TSO; + } + + if (init > 0 && (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { + ifp->if_drv_flags &= ~IFF_DRV_RUNNING; nfe_init(sc); - } + } + if ((sc->nfe_flags & NFE_HW_VLAN) != 0) + VLAN_CAPABILITIES(ifp); break; - default: error = ether_ioctl(ifp, cmd, data); break; } - return error; + return (error); } -static void +static int nfe_intr(void *arg) { + struct nfe_softc *sc; + uint32_t status; + + sc = (struct nfe_softc *)arg; + + status = NFE_READ(sc, sc->nfe_irq_status); + if (status == 0 || status == 0xffffffff) + return (FILTER_STRAY); + nfe_disable_intr(sc); + taskqueue_enqueue_fast(taskqueue_fast, &sc->nfe_int_task); + + return (FILTER_HANDLED); +} + + +static void +nfe_int_task(void *arg, int pending) +{ struct nfe_softc *sc = arg; struct ifnet *ifp = sc->nfe_ifp; - u_int32_t r; + uint32_t r; + int domore; NFE_LOCK(sc); + if ((r = NFE_READ(sc, sc->nfe_irq_status)) == 0) { + nfe_enable_intr(sc); + NFE_UNLOCK(sc); + return; /* not for us */ + } + NFE_WRITE(sc, sc->nfe_irq_status, r); + + DPRINTFN(sc, 5, "nfe_intr: interrupt register %x\n", r); + #ifdef DEVICE_POLLING if (ifp->if_capenable & IFCAP_POLLING) { NFE_UNLOCK(sc); @@ -1166,154 +1914,272 @@ nfe_intr(void *arg) } #endif - if ((r = NFE_READ(sc, NFE_IRQ_STATUS)) == 0) { + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { NFE_UNLOCK(sc); - return; /* not for us */ + nfe_enable_intr(sc); + return; } - NFE_WRITE(sc, NFE_IRQ_STATUS, r); - - DPRINTFN(5, ("nfe_intr: interrupt register %x\n", r)); - - NFE_WRITE(sc, NFE_IRQ_MASK, 0); if (r & NFE_IRQ_LINK) { NFE_READ(sc, NFE_PHY_STATUS); NFE_WRITE(sc, NFE_PHY_STATUS, 0xf); - DPRINTF(("nfe%d: link state changed\n", sc->nfe_unit)); + DPRINTF(sc, "link state changed\n"); } - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - /* check Rx ring */ - nfe_rxeof(sc); - /* check Tx ring */ - nfe_txeof(sc); - } - - NFE_WRITE(sc, NFE_IRQ_MASK, NFE_IRQ_WANTED); + domore = 0; + /* check Rx ring */ + if (sc->nfe_framesize > MCLBYTES - ETHER_HDR_LEN) + domore = nfe_jrxeof(sc, sc->nfe_process_limit); + else + domore = nfe_rxeof(sc, sc->nfe_process_limit); + /* check Tx ring */ + nfe_txeof(sc); - if (ifp->if_drv_flags & IFF_DRV_RUNNING && - !IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - nfe_start_locked(ifp); + if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + taskqueue_enqueue_fast(taskqueue_fast, &sc->nfe_tx_task); NFE_UNLOCK(sc); - return; -} - - -static void -nfe_txdesc32_sync(struct nfe_softc *sc, struct nfe_desc32 *desc32, int ops) -{ + if (domore || (NFE_READ(sc, sc->nfe_irq_status) != 0)) { + taskqueue_enqueue_fast(taskqueue_fast, &sc->nfe_int_task); + return; + } - bus_dmamap_sync(sc->txq.tx_desc_tag, sc->txq.tx_desc_map, ops); + /* Reenable interrupts. */ + nfe_enable_intr(sc); } -static void -nfe_txdesc64_sync(struct nfe_softc *sc, struct nfe_desc64 *desc64, int ops) +static __inline void +nfe_discard_rxbuf(struct nfe_softc *sc, int idx) { + struct nfe_desc32 *desc32; + struct nfe_desc64 *desc64; + struct nfe_rx_data *data; + struct mbuf *m; + + data = &sc->rxq.data[idx]; + m = data->m; - bus_dmamap_sync(sc->txq.tx_desc_tag, sc->txq.tx_desc_map, ops); + if (sc->nfe_flags & NFE_40BIT_ADDR) { + desc64 = &sc->rxq.desc64[idx]; + /* VLAN packet may have overwritten it. */ + desc64->physaddr[0] = htole32(NFE_ADDR_HI(data->paddr)); + desc64->physaddr[1] = htole32(NFE_ADDR_LO(data->paddr)); + desc64->length = htole16(m->m_len); + desc64->flags = htole16(NFE_RX_READY); + } else { + desc32 = &sc->rxq.desc32[idx]; + desc32->length = htole16(m->m_len); + desc32->flags = htole16(NFE_RX_READY); + } } -static void -nfe_txdesc32_rsync(struct nfe_softc *sc, int start, int end, int ops) +static __inline void +nfe_discard_jrxbuf(struct nfe_softc *sc, int idx) { + struct nfe_desc32 *desc32; + struct nfe_desc64 *desc64; + struct nfe_rx_data *data; + struct mbuf *m; + + data = &sc->jrxq.jdata[idx]; + m = data->m; - bus_dmamap_sync(sc->txq.tx_desc_tag, sc->txq.tx_desc_map, ops); + if (sc->nfe_flags & NFE_40BIT_ADDR) { + desc64 = &sc->jrxq.jdesc64[idx]; + /* VLAN packet may have overwritten it. */ + desc64->physaddr[0] = htole32(NFE_ADDR_HI(data->paddr)); + desc64->physaddr[1] = htole32(NFE_ADDR_LO(data->paddr)); + desc64->length = htole16(m->m_len); + desc64->flags = htole16(NFE_RX_READY); + } else { + desc32 = &sc->jrxq.jdesc32[idx]; + desc32->length = htole16(m->m_len); + desc32->flags = htole16(NFE_RX_READY); + } } -static void -nfe_txdesc64_rsync(struct nfe_softc *sc, int start, int end, int ops) +static int +nfe_newbuf(struct nfe_softc *sc, int idx) { + struct nfe_rx_data *data; + struct nfe_desc32 *desc32; + struct nfe_desc64 *desc64; + struct mbuf *m; + bus_dma_segment_t segs[1]; + bus_dmamap_t map; + int nsegs; - bus_dmamap_sync(sc->txq.tx_desc_tag, sc->txq.tx_desc_map, ops); -} + m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); + if (m == NULL) + return (ENOBUFS); + m->m_len = m->m_pkthdr.len = MCLBYTES; + m_adj(m, ETHER_ALIGN); -static void -nfe_rxdesc32_sync(struct nfe_softc *sc, struct nfe_desc32 *desc32, int ops) -{ + if (bus_dmamap_load_mbuf_sg(sc->rxq.rx_data_tag, sc->rxq.rx_spare_map, + m, segs, &nsegs, BUS_DMA_NOWAIT) != 0) { + m_freem(m); + return (ENOBUFS); + } + KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs)); - bus_dmamap_sync(sc->rxq.rx_desc_tag, sc->rxq.rx_desc_map, ops); + data = &sc->rxq.data[idx]; + if (data->m != NULL) { + bus_dmamap_sync(sc->rxq.rx_data_tag, data->rx_data_map, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(sc->rxq.rx_data_tag, data->rx_data_map); + } + map = data->rx_data_map; + data->rx_data_map = sc->rxq.rx_spare_map; + sc->rxq.rx_spare_map = map; + bus_dmamap_sync(sc->rxq.rx_data_tag, data->rx_data_map, + BUS_DMASYNC_PREREAD); + data->paddr = segs[0].ds_addr; + data->m = m; + /* update mapping address in h/w descriptor */ + if (sc->nfe_flags & NFE_40BIT_ADDR) { + desc64 = &sc->rxq.desc64[idx]; + desc64->physaddr[0] = htole32(NFE_ADDR_HI(segs[0].ds_addr)); + desc64->physaddr[1] = htole32(NFE_ADDR_LO(segs[0].ds_addr)); + desc64->length = htole16(segs[0].ds_len); + desc64->flags = htole16(NFE_RX_READY); + } else { + desc32 = &sc->rxq.desc32[idx]; + desc32->physaddr = htole32(NFE_ADDR_LO(segs[0].ds_addr)); + desc32->length = htole16(segs[0].ds_len); + desc32->flags = htole16(NFE_RX_READY); + } + + return (0); } -static void -nfe_rxdesc64_sync(struct nfe_softc *sc, struct nfe_desc64 *desc64, int ops) +static int +nfe_jnewbuf(struct nfe_softc *sc, int idx) { + struct nfe_rx_data *data; + struct nfe_desc32 *desc32; + struct nfe_desc64 *desc64; + struct mbuf *m; + bus_dma_segment_t segs[1]; + bus_dmamap_t map; + int nsegs; + void *buf; + + MGETHDR(m, M_DONTWAIT, MT_DATA); + if (m == NULL) + return (ENOBUFS); + buf = nfe_jalloc(sc); + if (buf == NULL) { + m_freem(m); + return (ENOBUFS); + } + /* Attach the buffer to the mbuf. */ + MEXTADD(m, buf, NFE_JLEN, nfe_jfree, (struct nfe_softc *)sc, 0, + EXT_NET_DRV); + if ((m->m_flags & M_EXT) == 0) { + m_freem(m); + return (ENOBUFS); + } + m->m_pkthdr.len = m->m_len = NFE_JLEN; + m_adj(m, ETHER_ALIGN); + + if (bus_dmamap_load_mbuf_sg(sc->jrxq.jrx_data_tag, + sc->jrxq.jrx_spare_map, m, segs, &nsegs, BUS_DMA_NOWAIT) != 0) { + m_freem(m); + return (ENOBUFS); + } + KASSERT(nsegs == 1, ("%s: %d segments returned!", __func__, nsegs)); + + data = &sc->jrxq.jdata[idx]; + if (data->m != NULL) { + bus_dmamap_sync(sc->jrxq.jrx_data_tag, data->rx_data_map, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(sc->jrxq.jrx_data_tag, data->rx_data_map); + } + map = data->rx_data_map; + data->rx_data_map = sc->jrxq.jrx_spare_map; + sc->jrxq.jrx_spare_map = map; + bus_dmamap_sync(sc->jrxq.jrx_data_tag, data->rx_data_map, + BUS_DMASYNC_PREREAD); + data->paddr = segs[0].ds_addr; + data->m = m; + /* update mapping address in h/w descriptor */ + if (sc->nfe_flags & NFE_40BIT_ADDR) { + desc64 = &sc->jrxq.jdesc64[idx]; + desc64->physaddr[0] = htole32(NFE_ADDR_HI(segs[0].ds_addr)); + desc64->physaddr[1] = htole32(NFE_ADDR_LO(segs[0].ds_addr)); + desc64->length = htole16(segs[0].ds_len); + desc64->flags = htole16(NFE_RX_READY); + } else { + desc32 = &sc->jrxq.jdesc32[idx]; + desc32->physaddr = htole32(NFE_ADDR_LO(segs[0].ds_addr)); + desc32->length = htole16(segs[0].ds_len); + desc32->flags = htole16(NFE_RX_READY); + } - bus_dmamap_sync(sc->rxq.rx_desc_tag, sc->rxq.rx_desc_map, ops); + return (0); } -static void -nfe_rxeof(struct nfe_softc *sc) +static int +nfe_rxeof(struct nfe_softc *sc, int count) { struct ifnet *ifp = sc->nfe_ifp; - struct nfe_desc32 *desc32=NULL; - struct nfe_desc64 *desc64=NULL; + struct nfe_desc32 *desc32; + struct nfe_desc64 *desc64; struct nfe_rx_data *data; - struct mbuf *m, *mnew; - bus_addr_t physaddr; - u_int16_t flags; - int error, len; -#if NVLAN > 1 - u_int16_t vlan_tag = 0; - int have_tag = 0; -#endif + struct mbuf *m; + uint16_t flags; + int len, prog; + uint32_t vtag = 0; NFE_LOCK_ASSERT(sc); - for (;;) { + bus_dmamap_sync(sc->rxq.rx_desc_tag, sc->rxq.rx_desc_map, + BUS_DMASYNC_POSTREAD); -#ifdef DEVICE_POLLING - if (ifp->if_capenable & IFCAP_POLLING) { - if (sc->rxcycles <= 0) - break; - sc->rxcycles--; - } -#endif + for (prog = 0;;NFE_INC(sc->rxq.cur, NFE_RX_RING_COUNT), vtag = 0) { + if (count <= 0) + break; + count--; data = &sc->rxq.data[sc->rxq.cur]; if (sc->nfe_flags & NFE_40BIT_ADDR) { desc64 = &sc->rxq.desc64[sc->rxq.cur]; - nfe_rxdesc64_sync(sc, desc64, BUS_DMASYNC_POSTREAD); - - flags = letoh16(desc64->flags); - len = letoh16(desc64->length) & 0x3fff; - -#if NVLAN > 1 - if (flags & NFE_TX_VLAN_TAG) { - have_tag = 1; - vlan_tag = desc64->vtag; - } -#endif - + vtag = le32toh(desc64->physaddr[1]); + flags = le16toh(desc64->flags); + len = le16toh(desc64->length) & NFE_RX_LEN_MASK; } else { desc32 = &sc->rxq.desc32[sc->rxq.cur]; - nfe_rxdesc32_sync(sc, desc32, BUS_DMASYNC_POSTREAD); - - flags = letoh16(desc32->flags); - len = letoh16(desc32->length) & 0x3fff; + flags = le16toh(desc32->flags); + len = le16toh(desc32->length) & NFE_RX_LEN_MASK; } if (flags & NFE_RX_READY) break; - + prog++; if ((sc->nfe_flags & (NFE_JUMBO_SUP | NFE_40BIT_ADDR)) == 0) { - if (!(flags & NFE_RX_VALID_V1)) - goto skip; + if (!(flags & NFE_RX_VALID_V1)) { + ifp->if_ierrors++; + nfe_discard_rxbuf(sc, sc->rxq.cur); + continue; + } if ((flags & NFE_RX_FIXME_V1) == NFE_RX_FIXME_V1) { flags &= ~NFE_RX_ERROR; len--; /* fix buffer length */ } } else { - if (!(flags & NFE_RX_VALID_V2)) - goto skip; + if (!(flags & NFE_RX_VALID_V2)) { + ifp->if_ierrors++; + nfe_discard_rxbuf(sc, sc->rxq.cur); + continue; + } if ((flags & NFE_RX_FIXME_V2) == NFE_RX_FIXME_V2) { flags &= ~NFE_RX_ERROR; @@ -1323,113 +2189,163 @@ nfe_rxeof(struct nfe_softc *sc) if (flags & NFE_RX_ERROR) { ifp->if_ierrors++; - goto skip; - } - - /* - * Try to allocate a new mbuf for this ring element and load - * it before processing the current mbuf. If the ring element - * cannot be loaded, drop the received packet and reuse the - * old mbuf. In the unlikely case that the old mbuf can't be - * reloaded either, explicitly panic. - */ - MGETHDR(mnew, M_DONTWAIT, MT_DATA); - if (mnew == NULL) { - ifp->if_ierrors++; - goto skip; + nfe_discard_rxbuf(sc, sc->rxq.cur); + continue; } - MCLGET(mnew, M_DONTWAIT); - if (!(mnew->m_flags & M_EXT)) { - m_freem(mnew); - ifp->if_ierrors++; - goto skip; + m = data->m; + if (nfe_newbuf(sc, sc->rxq.cur) != 0) { + ifp->if_iqdrops++; + nfe_discard_rxbuf(sc, sc->rxq.cur); + continue; } - bus_dmamap_sync(data->rx_data_tag, data->rx_data_map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(data->rx_data_tag, data->rx_data_map); - error = bus_dmamap_load(data->rx_data_tag, - data->rx_data_map, mtod(mnew, void *), MCLBYTES, - nfe_dma_map_segs, &data->rx_data_segs, - BUS_DMA_NOWAIT); - if (error != 0) { - m_freem(mnew); - - /* try to reload the old mbuf */ - error = bus_dmamap_load(data->rx_data_tag, - data->rx_data_map, mtod(data->m, void *), - MCLBYTES, nfe_dma_map_segs, - &data->rx_data_segs, BUS_DMA_NOWAIT); - if (error != 0) { - /* very unlikely that it will fail.. */ - panic("nfe%d: could not load old rx mbuf", - sc->nfe_unit); - } - ifp->if_ierrors++; - goto skip; + if ((vtag & NFE_RX_VTAG) != 0 && + (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0) { + m->m_pkthdr.ether_vtag = vtag & 0xffff; + m->m_flags |= M_VLANTAG; } - data->rx_data_addr = data->rx_data_segs.ds_addr; - physaddr = data->rx_data_addr; - /* - * New mbuf successfully loaded, update Rx ring and continue - * processing. - */ - m = data->m; - data->m = mnew; - - /* finalize mbuf */ m->m_pkthdr.len = m->m_len = len; m->m_pkthdr.rcvif = ifp; - if ((sc->nfe_flags & NFE_HW_CSUM) && (flags & NFE_RX_CSUMOK)) { - m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; - if (flags & NFE_RX_IP_CSUMOK_V2) { + if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { + if ((flags & NFE_RX_IP_CSUMOK) != 0) { + m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; m->m_pkthdr.csum_flags |= CSUM_IP_VALID; - } - if (flags & NFE_RX_UDP_CSUMOK_V2 || - flags & NFE_RX_TCP_CSUMOK_V2) { - m->m_pkthdr.csum_flags |= - CSUM_DATA_VALID | CSUM_PSEUDO_HDR; - m->m_pkthdr.csum_data = 0xffff; + if ((flags & NFE_RX_TCP_CSUMOK) != 0 || + (flags & NFE_RX_UDP_CSUMOK) != 0) { + m->m_pkthdr.csum_flags |= + CSUM_DATA_VALID | CSUM_PSEUDO_HDR; + m->m_pkthdr.csum_data = 0xffff; + } } } -#if NVLAN > 1 - if (have_tag) { - m->m_pkthdr.ether_vtag = vlan_tag; - m->m_flags |= M_VLANTAG; - } -#endif ifp->if_ipackets++; NFE_UNLOCK(sc); (*ifp->if_input)(ifp, m); NFE_LOCK(sc); + } + + if (prog > 0) + bus_dmamap_sync(sc->rxq.rx_desc_tag, sc->rxq.rx_desc_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + return (count > 0 ? 0 : EAGAIN); +} + + +static int +nfe_jrxeof(struct nfe_softc *sc, int count) +{ + struct ifnet *ifp = sc->nfe_ifp; + struct nfe_desc32 *desc32; + struct nfe_desc64 *desc64; + struct nfe_rx_data *data; + struct mbuf *m; + uint16_t flags; + int len, prog; + uint32_t vtag = 0; + + NFE_LOCK_ASSERT(sc); + + bus_dmamap_sync(sc->jrxq.jrx_desc_tag, sc->jrxq.jrx_desc_map, + BUS_DMASYNC_POSTREAD); + + for (prog = 0;;NFE_INC(sc->jrxq.jcur, NFE_JUMBO_RX_RING_COUNT), + vtag = 0) { + if (count <= 0) + break; + count--; + + data = &sc->jrxq.jdata[sc->jrxq.jcur]; - /* update mapping address in h/w descriptor */ if (sc->nfe_flags & NFE_40BIT_ADDR) { -#if defined(__LP64__) - desc64->physaddr[0] = htole32(physaddr >> 32); -#endif - desc64->physaddr[1] = htole32(physaddr & 0xffffffff); + desc64 = &sc->jrxq.jdesc64[sc->jrxq.jcur]; + vtag = le32toh(desc64->physaddr[1]); + flags = le16toh(desc64->flags); + len = le16toh(desc64->length) & NFE_RX_LEN_MASK; } else { - desc32->physaddr = htole32(physaddr); + desc32 = &sc->jrxq.jdesc32[sc->jrxq.jcur]; + flags = le16toh(desc32->flags); + len = le16toh(desc32->length) & NFE_RX_LEN_MASK; } -skip: if (sc->nfe_flags & NFE_40BIT_ADDR) { - desc64->length = htole16(sc->rxq.bufsz); - desc64->flags = htole16(NFE_RX_READY); - nfe_rxdesc64_sync(sc, desc64, BUS_DMASYNC_PREWRITE); + if (flags & NFE_RX_READY) + break; + prog++; + if ((sc->nfe_flags & (NFE_JUMBO_SUP | NFE_40BIT_ADDR)) == 0) { + if (!(flags & NFE_RX_VALID_V1)) { + ifp->if_ierrors++; + nfe_discard_jrxbuf(sc, sc->jrxq.jcur); + continue; + } + if ((flags & NFE_RX_FIXME_V1) == NFE_RX_FIXME_V1) { + flags &= ~NFE_RX_ERROR; + len--; /* fix buffer length */ + } } else { - desc32->length = htole16(sc->rxq.bufsz); - desc32->flags = htole16(NFE_RX_READY); - nfe_rxdesc32_sync(sc, desc32, BUS_DMASYNC_PREWRITE); + if (!(flags & NFE_RX_VALID_V2)) { + ifp->if_ierrors++; + nfe_discard_jrxbuf(sc, sc->jrxq.jcur); + continue; + } + + if ((flags & NFE_RX_FIXME_V2) == NFE_RX_FIXME_V2) { + flags &= ~NFE_RX_ERROR; + len--; /* fix buffer length */ + } + } + + if (flags & NFE_RX_ERROR) { + ifp->if_ierrors++; + nfe_discard_jrxbuf(sc, sc->jrxq.jcur); + continue; + } + + m = data->m; + if (nfe_jnewbuf(sc, sc->jrxq.jcur) != 0) { + ifp->if_iqdrops++; + nfe_discard_jrxbuf(sc, sc->jrxq.jcur); + continue; + } + + if ((vtag & NFE_RX_VTAG) != 0 && + (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0) { + m->m_pkthdr.ether_vtag = vtag & 0xffff; + m->m_flags |= M_VLANTAG; } - sc->rxq.cur = (sc->rxq.cur + 1) % NFE_RX_RING_COUNT; - } //end for(;;) + m->m_pkthdr.len = m->m_len = len; + m->m_pkthdr.rcvif = ifp; + + if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { + if ((flags & NFE_RX_IP_CSUMOK) != 0) { + m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; + m->m_pkthdr.csum_flags |= CSUM_IP_VALID; + if ((flags & NFE_RX_TCP_CSUMOK) != 0 || + (flags & NFE_RX_UDP_CSUMOK) != 0) { + m->m_pkthdr.csum_flags |= + CSUM_DATA_VALID | CSUM_PSEUDO_HDR; + m->m_pkthdr.csum_data = 0xffff; + } + } + } + + ifp->if_ipackets++; + + NFE_UNLOCK(sc); + (*ifp->if_input)(ifp, m); + NFE_LOCK(sc); + } + + if (prog > 0) + bus_dmamap_sync(sc->jrxq.jrx_desc_tag, sc->jrxq.jrx_desc_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + return (count > 0 ? 0 : EAGAIN); } @@ -1440,167 +2356,304 @@ nfe_txeof(struct nfe_softc *sc) struct nfe_desc32 *desc32; struct nfe_desc64 *desc64; struct nfe_tx_data *data = NULL; - u_int16_t flags; + uint16_t flags; + int cons, prog; NFE_LOCK_ASSERT(sc); - while (sc->txq.next != sc->txq.cur) { - if (sc->nfe_flags & NFE_40BIT_ADDR) { - desc64 = &sc->txq.desc64[sc->txq.next]; - nfe_txdesc64_sync(sc, desc64, BUS_DMASYNC_POSTREAD); + bus_dmamap_sync(sc->txq.tx_desc_tag, sc->txq.tx_desc_map, + BUS_DMASYNC_POSTREAD); - flags = letoh16(desc64->flags); + prog = 0; + for (cons = sc->txq.next; cons != sc->txq.cur; + NFE_INC(cons, NFE_TX_RING_COUNT)) { + if (sc->nfe_flags & NFE_40BIT_ADDR) { + desc64 = &sc->txq.desc64[cons]; + flags = le16toh(desc64->flags); } else { - desc32 = &sc->txq.desc32[sc->txq.next]; - nfe_txdesc32_sync(sc, desc32, BUS_DMASYNC_POSTREAD); - - flags = letoh16(desc32->flags); + desc32 = &sc->txq.desc32[cons]; + flags = le16toh(desc32->flags); } if (flags & NFE_TX_VALID) break; - data = &sc->txq.data[sc->txq.next]; + prog++; + sc->txq.queued--; + data = &sc->txq.data[cons]; if ((sc->nfe_flags & (NFE_JUMBO_SUP | NFE_40BIT_ADDR)) == 0) { - if (!(flags & NFE_TX_LASTFRAG_V1) && data->m == NULL) - goto skip; - + if ((flags & NFE_TX_LASTFRAG_V1) == 0) + continue; if ((flags & NFE_TX_ERROR_V1) != 0) { - printf("nfe%d: tx v1 error 0x%4b\n", - sc->nfe_unit, flags, NFE_V1_TXERR); + device_printf(sc->nfe_dev, + "tx v1 error 0x%4b\n", flags, NFE_V1_TXERR); ifp->if_oerrors++; } else ifp->if_opackets++; } else { - if (!(flags & NFE_TX_LASTFRAG_V2) && data->m == NULL) - goto skip; - + if ((flags & NFE_TX_LASTFRAG_V2) == 0) + continue; if ((flags & NFE_TX_ERROR_V2) != 0) { - printf("nfe%d: tx v1 error 0x%4b\n", - sc->nfe_unit, flags, NFE_V2_TXERR); - + device_printf(sc->nfe_dev, + "tx v2 error 0x%4b\n", flags, NFE_V2_TXERR); ifp->if_oerrors++; } else ifp->if_opackets++; } - if (data->m == NULL) { /* should not get there */ - printf("nfe%d: last fragment bit w/o associated mbuf!\n", - sc->nfe_unit); - goto skip; - } - /* last fragment of the mbuf chain transmitted */ - bus_dmamap_sync(sc->txq.tx_data_tag, data->active, + KASSERT(data->m != NULL, ("%s: freeing NULL mbuf!", __func__)); + bus_dmamap_sync(sc->txq.tx_data_tag, data->tx_data_map, BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(sc->txq.tx_data_tag, data->active); + bus_dmamap_unload(sc->txq.tx_data_tag, data->tx_data_map); m_freem(data->m); data->m = NULL; - - ifp->if_timer = 0; - -skip: sc->txq.queued--; - sc->txq.next = (sc->txq.next + 1) % NFE_TX_RING_COUNT; } - if (data != NULL) { /* at least one slot freed */ + if (prog > 0) { + sc->nfe_force_tx = 0; + sc->txq.next = cons; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; - nfe_start_locked(ifp); + if (sc->txq.queued == 0) + sc->nfe_watchdog_timer = 0; } } +/* + * It's copy of ath_defrag(ath(4)). + * + * Defragment an mbuf chain, returning at most maxfrags separate + * mbufs+clusters. If this is not possible NULL is returned and + * the original mbuf chain is left in it's present (potentially + * modified) state. We use two techniques: collapsing consecutive + * mbufs and replacing consecutive mbufs by a cluster. + */ +static struct mbuf * +nfe_defrag(struct mbuf *m0, int how, int maxfrags) +{ + struct mbuf *m, *n, *n2, **prev; + u_int curfrags; + + /* + * Calculate the current number of frags. + */ + curfrags = 0; + for (m = m0; m != NULL; m = m->m_next) + curfrags++; + /* + * First, try to collapse mbufs. Note that we always collapse + * towards the front so we don't need to deal with moving the + * pkthdr. This may be suboptimal if the first mbuf has much + * less data than the following. + */ + m = m0; +again: + for (;;) { + n = m->m_next; + if (n == NULL) + break; + if ((m->m_flags & M_RDONLY) == 0 && + n->m_len < M_TRAILINGSPACE(m)) { + bcopy(mtod(n, void *), mtod(m, char *) + m->m_len, + n->m_len); + m->m_len += n->m_len; + m->m_next = n->m_next; + m_free(n); + if (--curfrags <= maxfrags) + return (m0); + } else + m = n; + } + KASSERT(maxfrags > 1, + ("maxfrags %u, but normal collapse failed", maxfrags)); + /* + * Collapse consecutive mbufs to a cluster. + */ + prev = &m0->m_next; /* NB: not the first mbuf */ + while ((n = *prev) != NULL) { + if ((n2 = n->m_next) != NULL && + n->m_len + n2->m_len < MCLBYTES) { + m = m_getcl(how, MT_DATA, 0); + if (m == NULL) + goto bad; + bcopy(mtod(n, void *), mtod(m, void *), n->m_len); + bcopy(mtod(n2, void *), mtod(m, char *) + n->m_len, + n2->m_len); + m->m_len = n->m_len + n2->m_len; + m->m_next = n2->m_next; + *prev = m; + m_free(n); + m_free(n2); + if (--curfrags <= maxfrags) /* +1 cl -2 mbufs */ + return m0; + /* + * Still not there, try the normal collapse + * again before we allocate another cluster. + */ + goto again; + } + prev = &n->m_next; + } + /* + * No place where we can collapse to a cluster; punt. + * This can occur if, for example, you request 2 frags + * but the packet requires that both be clusters (we + * never reallocate the first mbuf to avoid moving the + * packet header). + */ +bad: + return (NULL); +} + static int -nfe_encap(struct nfe_softc *sc, struct mbuf *m0) +nfe_encap(struct nfe_softc *sc, struct mbuf **m_head) { - struct nfe_desc32 *desc32=NULL; - struct nfe_desc64 *desc64=NULL; - struct nfe_tx_data *data=NULL; + struct nfe_desc32 *desc32 = NULL; + struct nfe_desc64 *desc64 = NULL; bus_dmamap_t map; bus_dma_segment_t segs[NFE_MAX_SCATTER]; - int error, i, nsegs; - u_int16_t flags = NFE_TX_VALID; + int error, i, nsegs, prod, si; + uint32_t tso_segsz; + uint16_t cflags, flags; + struct mbuf *m; - map = sc->txq.data[sc->txq.cur].tx_data_map; + prod = si = sc->txq.cur; + map = sc->txq.data[prod].tx_data_map; - error = bus_dmamap_load_mbuf_sg(sc->txq.tx_data_tag, map, m0, segs, + error = bus_dmamap_load_mbuf_sg(sc->txq.tx_data_tag, map, *m_head, segs, &nsegs, BUS_DMA_NOWAIT); - - if (error != 0) { - printf("nfe%d: could not map mbuf (error %d)\n", sc->nfe_unit, - error); - return error; + if (error == EFBIG) { + m = nfe_defrag(*m_head, M_DONTWAIT, NFE_MAX_SCATTER); + if (m == NULL) { + m_freem(*m_head); + *m_head = NULL; + return (ENOBUFS); + } + *m_head = m; + error = bus_dmamap_load_mbuf_sg(sc->txq.tx_data_tag, map, + *m_head, segs, &nsegs, BUS_DMA_NOWAIT); + if (error != 0) { + m_freem(*m_head); + *m_head = NULL; + return (ENOBUFS); + } + } else if (error != 0) + return (error); + if (nsegs == 0) { + m_freem(*m_head); + *m_head = NULL; + return (EIO); } - if (sc->txq.queued + nsegs >= NFE_TX_RING_COUNT - 1) { + if (sc->txq.queued + nsegs >= NFE_TX_RING_COUNT - 2) { bus_dmamap_unload(sc->txq.tx_data_tag, map); - return ENOBUFS; + return (ENOBUFS); } - if(sc->nfe_flags & NFE_HW_CSUM){ - if (m0->m_pkthdr.csum_flags & CSUM_IP) - flags |= NFE_TX_IP_CSUM; - if (m0->m_pkthdr.csum_flags & CSUM_TCP) - flags |= NFE_TX_TCP_CSUM; - if (m0->m_pkthdr.csum_flags & CSUM_UDP) - flags |= NFE_TX_TCP_CSUM; + m = *m_head; + cflags = flags = 0; + tso_segsz = 0; + if ((m->m_pkthdr.csum_flags & NFE_CSUM_FEATURES) != 0) { + if ((m->m_pkthdr.csum_flags & CSUM_IP) != 0) + cflags |= NFE_TX_IP_CSUM; + if ((m->m_pkthdr.csum_flags & CSUM_TCP) != 0) + cflags |= NFE_TX_TCP_UDP_CSUM; + if ((m->m_pkthdr.csum_flags & CSUM_UDP) != 0) + cflags |= NFE_TX_TCP_UDP_CSUM; + } + if ((m->m_pkthdr.csum_flags & CSUM_TSO) != 0) { + tso_segsz = (uint32_t)m->m_pkthdr.tso_segsz << + NFE_TX_TSO_SHIFT; + cflags &= ~(NFE_TX_IP_CSUM | NFE_TX_TCP_UDP_CSUM); + cflags |= NFE_TX_TSO; } for (i = 0; i < nsegs; i++) { - data = &sc->txq.data[sc->txq.cur]; - if (sc->nfe_flags & NFE_40BIT_ADDR) { - desc64 = &sc->txq.desc64[sc->txq.cur]; -#if defined(__LP64__) - desc64->physaddr[0] = htole32(segs[i].ds_addr >> 32); -#endif - desc64->physaddr[1] = htole32(segs[i].ds_addr & - 0xffffffff); + desc64 = &sc->txq.desc64[prod]; + desc64->physaddr[0] = + htole32(NFE_ADDR_HI(segs[i].ds_addr)); + desc64->physaddr[1] = + htole32(NFE_ADDR_LO(segs[i].ds_addr)); + desc64->vtag = 0; desc64->length = htole16(segs[i].ds_len - 1); desc64->flags = htole16(flags); -#if NVLAN > 0 - if (m0->m_flags & M_VLANTAG) - desc64->vtag = htole32(NFE_TX_VTAG | - m0->m_pkthdr.ether_vtag); -#endif } else { - desc32 = &sc->txq.desc32[sc->txq.cur]; - - desc32->physaddr = htole32(segs[i].ds_addr); + desc32 = &sc->txq.desc32[prod]; + desc32->physaddr = + htole32(NFE_ADDR_LO(segs[i].ds_addr)); desc32->length = htole16(segs[i].ds_len - 1); desc32->flags = htole16(flags); } - /* csum flags and vtag belong to the first fragment only */ - if (nsegs > 1) { - flags &= ~(NFE_TX_IP_CSUM | NFE_TX_TCP_CSUM); - } + /* + * Setting of the valid bit in the first descriptor is + * deferred until the whole chain is fully setup. + */ + flags |= NFE_TX_VALID; sc->txq.queued++; - sc->txq.cur = (sc->txq.cur + 1) % NFE_TX_RING_COUNT; + NFE_INC(prod, NFE_TX_RING_COUNT); } - /* the whole mbuf chain has been DMA mapped, fix last descriptor */ + /* + * the whole mbuf chain has been DMA mapped, fix last/first descriptor. + * csum flags, vtag and TSO belong to the first fragment only. + */ if (sc->nfe_flags & NFE_40BIT_ADDR) { - flags |= NFE_TX_LASTFRAG_V2; - desc64->flags = htole16(flags); + desc64->flags |= htole16(NFE_TX_LASTFRAG_V2); + desc64 = &sc->txq.desc64[si]; + if ((m->m_flags & M_VLANTAG) != 0) + desc64->vtag = htole32(NFE_TX_VTAG | + m->m_pkthdr.ether_vtag); + if (tso_segsz != 0) { + /* + * XXX + * The following indicates the descriptor element + * is a 32bit quantity. + */ + desc64->length |= htole16((uint16_t)tso_segsz); + desc64->flags |= htole16(tso_segsz >> 16); + } + /* + * finally, set the valid/checksum/TSO bit in the first + * descriptor. + */ + desc64->flags |= htole16(NFE_TX_VALID | cflags); } else { if (sc->nfe_flags & NFE_JUMBO_SUP) - flags |= NFE_TX_LASTFRAG_V2; + desc32->flags |= htole16(NFE_TX_LASTFRAG_V2); else - flags |= NFE_TX_LASTFRAG_V1; - desc32->flags = htole16(flags); + desc32->flags |= htole16(NFE_TX_LASTFRAG_V1); + desc32 = &sc->txq.desc32[si]; + if (tso_segsz != 0) { + /* + * XXX + * The following indicates the descriptor element + * is a 32bit quantity. + */ + desc32->length |= htole16((uint16_t)tso_segsz); + desc32->flags |= htole16(tso_segsz >> 16); + } + /* + * finally, set the valid/checksum/TSO bit in the first + * descriptor. + */ + desc32->flags |= htole16(NFE_TX_VALID | cflags); } - data->m = m0; - data->active = map; - data->nsegs = nsegs; + sc->txq.cur = prod; + prod = (prod + NFE_TX_RING_COUNT - 1) % NFE_TX_RING_COUNT; + sc->txq.data[si].tx_data_map = sc->txq.data[prod].tx_data_map; + sc->txq.data[prod].tx_data_map = map; + sc->txq.data[prod].m = m; bus_dmamap_sync(sc->txq.tx_data_tag, map, BUS_DMASYNC_PREWRITE); - return 0; + return (0); } @@ -1610,9 +2663,9 @@ nfe_setmulti(struct nfe_softc *sc) struct ifnet *ifp = sc->nfe_ifp; struct ifmultiaddr *ifma; int i; - u_int32_t filter = NFE_RXFILTER_MAGIC; - u_int8_t addr[ETHER_ADDR_LEN], mask[ETHER_ADDR_LEN]; - u_int8_t etherbroadcastaddr[ETHER_ADDR_LEN] = { + uint32_t filter; + uint8_t addr[ETHER_ADDR_LEN], mask[ETHER_ADDR_LEN]; + uint8_t etherbroadcastaddr[ETHER_ADDR_LEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; @@ -1659,67 +2712,69 @@ done: NFE_WRITE(sc, NFE_MULTIMASK_LO, mask[5] << 8 | mask[4]); - filter |= (ifp->if_flags & IFF_PROMISC) ? NFE_PROMISC : NFE_U2M; + filter = NFE_READ(sc, NFE_RXFILTER); + filter &= NFE_PFF_RX_PAUSE; + filter |= NFE_RXFILTER_MAGIC; + filter |= (ifp->if_flags & IFF_PROMISC) ? NFE_PFF_PROMISC : NFE_PFF_U2M; NFE_WRITE(sc, NFE_RXFILTER, filter); } static void -nfe_start(struct ifnet *ifp) +nfe_tx_task(void *arg, int pending) { - struct nfe_softc *sc; + struct ifnet *ifp; - sc = ifp->if_softc; - NFE_LOCK(sc); - nfe_start_locked(ifp); - NFE_UNLOCK(sc); + ifp = (struct ifnet *)arg; + nfe_start(ifp); } static void -nfe_start_locked(struct ifnet *ifp) +nfe_start(struct ifnet *ifp) { struct nfe_softc *sc = ifp->if_softc; struct mbuf *m0; - int old = sc->txq.cur; + int enq; + + NFE_LOCK(sc); - if (!sc->nfe_link || ifp->if_drv_flags & IFF_DRV_OACTIVE) { + if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING || sc->nfe_link == 0) { + NFE_UNLOCK(sc); return; } - for (;;) { - IFQ_POLL(&ifp->if_snd, m0); + for (enq = 0; !IFQ_DRV_IS_EMPTY(&ifp->if_snd);) { + IFQ_DRV_DEQUEUE(&ifp->if_snd, m0); if (m0 == NULL) break; - if (nfe_encap(sc, m0) != 0) { + if (nfe_encap(sc, &m0) != 0) { + if (m0 == NULL) + break; + IFQ_DRV_PREPEND(&ifp->if_snd, m0); ifp->if_drv_flags |= IFF_DRV_OACTIVE; break; } - - /* packet put in h/w queue, remove from s/w queue */ - IFQ_DEQUEUE(&ifp->if_snd, m0); - + enq++; ETHER_BPF_MTAP(ifp, m0); } - if (sc->txq.cur == old) { /* nothing sent */ - return; - } - if (sc->nfe_flags & NFE_40BIT_ADDR) - nfe_txdesc64_rsync(sc, old, sc->txq.cur, BUS_DMASYNC_PREWRITE); - else - nfe_txdesc32_rsync(sc, old, sc->txq.cur, BUS_DMASYNC_PREWRITE); + if (enq > 0) { + bus_dmamap_sync(sc->txq.tx_desc_tag, sc->txq.tx_desc_map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - /* kick Tx */ - NFE_WRITE(sc, NFE_RXTX_CTL, NFE_RXTX_KICKTX | sc->rxtxctl); + /* kick Tx */ + NFE_WRITE(sc, NFE_RXTX_CTL, NFE_RXTX_KICKTX | sc->rxtxctl); - /* - * Set a timeout in case the chip goes out to lunch. - */ - ifp->if_timer = 5; + /* + * Set a timeout in case the chip goes out to lunch. + */ + sc->nfe_watchdog_timer = 5; + } - return; + NFE_UNLOCK(sc); } @@ -1728,13 +2783,36 @@ nfe_watchdog(struct ifnet *ifp) { struct nfe_softc *sc = ifp->if_softc; - printf("nfe%d: watchdog timeout\n", sc->nfe_unit); + if (sc->nfe_watchdog_timer == 0 || --sc->nfe_watchdog_timer) + return; + + /* Check if we've lost Tx completion interrupt. */ + nfe_txeof(sc); + if (sc->txq.queued == 0) { + if_printf(ifp, "watchdog timeout (missed Tx interrupts) " + "-- recovering\n"); + if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + taskqueue_enqueue_fast(taskqueue_fast, + &sc->nfe_tx_task); + return; + } + /* Check if we've lost start Tx command. */ + sc->nfe_force_tx++; + if (sc->nfe_force_tx <= 3) { + /* + * If this is the case for watchdog timeout, the following + * code should go to nfe_txeof(). + */ + NFE_WRITE(sc, NFE_RXTX_CTL, NFE_RXTX_KICKTX | sc->rxtxctl); + return; + } + sc->nfe_force_tx = 0; + + if_printf(ifp, "watchdog timeout\n"); ifp->if_drv_flags &= ~IFF_DRV_RUNNING; - nfe_init(sc); ifp->if_oerrors++; - - return; + nfe_init_locked(sc); } @@ -1746,8 +2824,6 @@ nfe_init(void *xsc) NFE_LOCK(sc); nfe_init_locked(sc); NFE_UNLOCK(sc); - - return; } @@ -1757,76 +2833,94 @@ nfe_init_locked(void *xsc) struct nfe_softc *sc = xsc; struct ifnet *ifp = sc->nfe_ifp; struct mii_data *mii; - u_int32_t tmp; + uint32_t val; + int error; NFE_LOCK_ASSERT(sc); mii = device_get_softc(sc->nfe_miibus); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if (ifp->if_drv_flags & IFF_DRV_RUNNING) return; - } - nfe_stop(ifp, 0); + nfe_stop(ifp); + + sc->nfe_framesize = ifp->if_mtu + NFE_RX_HEADERS; - NFE_WRITE(sc, NFE_TX_UNK, 0); + nfe_init_tx_ring(sc, &sc->txq); + if (sc->nfe_framesize > (MCLBYTES - ETHER_HDR_LEN)) + error = nfe_init_jrx_ring(sc, &sc->jrxq); + else + error = nfe_init_rx_ring(sc, &sc->rxq); + if (error != 0) { + device_printf(sc->nfe_dev, + "initialization failed: no memory for rx buffers\n"); + nfe_stop(ifp); + return; + } + + val = 0; + if ((sc->nfe_flags & NFE_CORRECT_MACADDR) != 0) + val |= NFE_MAC_ADDR_INORDER; + NFE_WRITE(sc, NFE_TX_UNK, val); NFE_WRITE(sc, NFE_STATUS, 0); + if ((sc->nfe_flags & NFE_TX_FLOW_CTRL) != 0) + NFE_WRITE(sc, NFE_TX_PAUSE_FRAME, NFE_TX_PAUSE_FRAME_DISABLE); + sc->rxtxctl = NFE_RXTX_BIT2; if (sc->nfe_flags & NFE_40BIT_ADDR) sc->rxtxctl |= NFE_RXTX_V3MAGIC; else if (sc->nfe_flags & NFE_JUMBO_SUP) sc->rxtxctl |= NFE_RXTX_V2MAGIC; - if (sc->nfe_flags & NFE_HW_CSUM) + if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) sc->rxtxctl |= NFE_RXTX_RXCSUM; - -#if NVLAN > 0 - /* - * Although the adapter is capable of stripping VLAN tags from received - * frames (NFE_RXTX_VTAG_STRIP), we do not enable this functionality on - * purpose. This will be done in software by our network stack. - */ - if (sc->nfe_flags & NFE_HW_VLAN) - sc->rxtxctl |= NFE_RXTX_VTAG_INSERT; -#endif + if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0) + sc->rxtxctl |= NFE_RXTX_VTAG_INSERT | NFE_RXTX_VTAG_STRIP; NFE_WRITE(sc, NFE_RXTX_CTL, NFE_RXTX_RESET | sc->rxtxctl); DELAY(10); NFE_WRITE(sc, NFE_RXTX_CTL, sc->rxtxctl); -#if NVLAN - if (sc->nfe_flags & NFE_HW_VLAN) + if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0) NFE_WRITE(sc, NFE_VTAG_CTL, NFE_VTAG_ENABLE); -#endif + else + NFE_WRITE(sc, NFE_VTAG_CTL, 0); NFE_WRITE(sc, NFE_SETUP_R6, 0); /* set MAC address */ - nfe_set_macaddr(sc, sc->eaddr); + nfe_set_macaddr(sc, IF_LLADDR(ifp)); /* tell MAC where rings are in memory */ -#ifdef __LP64__ - NFE_WRITE(sc, NFE_RX_RING_ADDR_HI, sc->rxq.physaddr >> 32); -#endif - NFE_WRITE(sc, NFE_RX_RING_ADDR_LO, sc->rxq.physaddr & 0xffffffff); -#ifdef __LP64__ - NFE_WRITE(sc, NFE_TX_RING_ADDR_HI, sc->txq.physaddr >> 32); -#endif - NFE_WRITE(sc, NFE_TX_RING_ADDR_LO, sc->txq.physaddr & 0xffffffff); + if (sc->nfe_framesize > MCLBYTES - ETHER_HDR_LEN) { + NFE_WRITE(sc, NFE_RX_RING_ADDR_HI, + NFE_ADDR_HI(sc->jrxq.jphysaddr)); + NFE_WRITE(sc, NFE_RX_RING_ADDR_LO, + NFE_ADDR_LO(sc->jrxq.jphysaddr)); + } else { + NFE_WRITE(sc, NFE_RX_RING_ADDR_HI, + NFE_ADDR_HI(sc->rxq.physaddr)); + NFE_WRITE(sc, NFE_RX_RING_ADDR_LO, + NFE_ADDR_LO(sc->rxq.physaddr)); + } + NFE_WRITE(sc, NFE_TX_RING_ADDR_HI, NFE_ADDR_HI(sc->txq.physaddr)); + NFE_WRITE(sc, NFE_TX_RING_ADDR_LO, NFE_ADDR_LO(sc->txq.physaddr)); NFE_WRITE(sc, NFE_RING_SIZE, (NFE_RX_RING_COUNT - 1) << 16 | (NFE_TX_RING_COUNT - 1)); - NFE_WRITE(sc, NFE_RXBUFSZ, sc->rxq.bufsz); + NFE_WRITE(sc, NFE_RXBUFSZ, sc->nfe_framesize); /* force MAC to wakeup */ - tmp = NFE_READ(sc, NFE_PWR_STATE); - NFE_WRITE(sc, NFE_PWR_STATE, tmp | NFE_PWR_WAKEUP); + val = NFE_READ(sc, NFE_PWR_STATE); + if ((val & NFE_PWR_WAKEUP) == 0) + NFE_WRITE(sc, NFE_PWR_STATE, val | NFE_PWR_WAKEUP); DELAY(10); - tmp = NFE_READ(sc, NFE_PWR_STATE); - NFE_WRITE(sc, NFE_PWR_STATE, tmp | NFE_PWR_VALID); + val = NFE_READ(sc, NFE_PWR_STATE); + NFE_WRITE(sc, NFE_PWR_STATE, val | NFE_PWR_VALID); #if 1 /* configure interrupts coalescing/mitigation */ @@ -1836,7 +2930,7 @@ nfe_init_locked(void *xsc) NFE_WRITE(sc, NFE_IMTIMER, 970); #endif - NFE_WRITE(sc, NFE_SETUP_R1, NFE_R1_MAGIC); + NFE_WRITE(sc, NFE_SETUP_R1, NFE_R1_MAGIC_10_100); NFE_WRITE(sc, NFE_SETUP_R2, NFE_R2_MAGIC); NFE_WRITE(sc, NFE_SETUP_R6, NFE_R6_MAGIC); @@ -1854,10 +2948,6 @@ nfe_init_locked(void *xsc) /* set Rx filter */ nfe_setmulti(sc); - nfe_ifmedia_upd(ifp); - - nfe_tick_locked(sc); - /* enable Rx */ NFE_WRITE(sc, NFE_RX_CTL, NFE_RX_START); @@ -1868,33 +2958,38 @@ nfe_init_locked(void *xsc) #ifdef DEVICE_POLLING if (ifp->if_capenable & IFCAP_POLLING) - NFE_WRITE(sc, NFE_IRQ_MASK, 0); + nfe_disable_intr(sc); else #endif - NFE_WRITE(sc, NFE_IRQ_MASK, NFE_IRQ_WANTED); /* enable interrupts */ + nfe_set_intr(sc); + nfe_enable_intr(sc); /* enable interrupts */ ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; sc->nfe_link = 0; + mii_mediachg(mii); - return; + callout_reset(&sc->nfe_stat_ch, hz, nfe_tick, sc); } static void -nfe_stop(struct ifnet *ifp, int disable) +nfe_stop(struct ifnet *ifp) { struct nfe_softc *sc = ifp->if_softc; - struct mii_data *mii; + struct nfe_rx_ring *rx_ring; + struct nfe_jrx_ring *jrx_ring; + struct nfe_tx_ring *tx_ring; + struct nfe_rx_data *rdata; + struct nfe_tx_data *tdata; + int i; NFE_LOCK_ASSERT(sc); - ifp->if_timer = 0; + sc->nfe_watchdog_timer = 0; ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); - mii = device_get_softc(sc->nfe_miibus); - callout_stop(&sc->nfe_stat_ch); /* abort Tx */ @@ -1904,48 +2999,64 @@ nfe_stop(struct ifnet *ifp, int disable) NFE_WRITE(sc, NFE_RX_CTL, 0); /* disable interrupts */ - NFE_WRITE(sc, NFE_IRQ_MASK, 0); + nfe_disable_intr(sc); sc->nfe_link = 0; - /* reset Tx and Rx rings */ - nfe_reset_tx_ring(sc, &sc->txq); - nfe_reset_rx_ring(sc, &sc->rxq); - - return; -} - + /* free Rx and Tx mbufs still in the queues. */ + rx_ring = &sc->rxq; + for (i = 0; i < NFE_RX_RING_COUNT; i++) { + rdata = &rx_ring->data[i]; + if (rdata->m != NULL) { + bus_dmamap_sync(rx_ring->rx_data_tag, + rdata->rx_data_map, BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(rx_ring->rx_data_tag, + rdata->rx_data_map); + m_freem(rdata->m); + rdata->m = NULL; + } + } -static int -nfe_ifmedia_upd(struct ifnet *ifp) -{ - struct nfe_softc *sc = ifp->if_softc; + if ((sc->nfe_flags & NFE_JUMBO_SUP) != 0) { + jrx_ring = &sc->jrxq; + for (i = 0; i < NFE_JUMBO_RX_RING_COUNT; i++) { + rdata = &jrx_ring->jdata[i]; + if (rdata->m != NULL) { + bus_dmamap_sync(jrx_ring->jrx_data_tag, + rdata->rx_data_map, BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(jrx_ring->jrx_data_tag, + rdata->rx_data_map); + m_freem(rdata->m); + rdata->m = NULL; + } + } + } - NFE_LOCK(sc); - nfe_ifmedia_upd_locked(ifp); - NFE_UNLOCK(sc); - return (0); + tx_ring = &sc->txq; + for (i = 0; i < NFE_RX_RING_COUNT; i++) { + tdata = &tx_ring->data[i]; + if (tdata->m != NULL) { + bus_dmamap_sync(tx_ring->tx_data_tag, + tdata->tx_data_map, BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(tx_ring->tx_data_tag, + tdata->tx_data_map); + m_freem(tdata->m); + tdata->m = NULL; + } + } } static int -nfe_ifmedia_upd_locked(struct ifnet *ifp) +nfe_ifmedia_upd(struct ifnet *ifp) { struct nfe_softc *sc = ifp->if_softc; struct mii_data *mii; - NFE_LOCK_ASSERT(sc); - + NFE_LOCK(sc); mii = device_get_softc(sc->nfe_miibus); - - if (mii->mii_instance) { - struct mii_softc *miisc; - for (miisc = LIST_FIRST(&mii->mii_phys); miisc != NULL; - miisc = LIST_NEXT(miisc, mii_list)) { - mii_phy_reset(miisc); - } - } mii_mediachg(mii); + NFE_UNLOCK(sc); return (0); } @@ -1966,32 +3077,17 @@ nfe_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) ifmr->ifm_active = mii->mii_media_active; ifmr->ifm_status = mii->mii_media_status; - - return; -} - - -static void -nfe_tick(void *xsc) -{ - struct nfe_softc *sc; - - sc = xsc; - - NFE_LOCK(sc); - nfe_tick_locked(sc); - NFE_UNLOCK(sc); } void -nfe_tick_locked(struct nfe_softc *arg) +nfe_tick(void *xsc) { struct nfe_softc *sc; struct mii_data *mii; struct ifnet *ifp; - sc = arg; + sc = (struct nfe_softc *)xsc; NFE_LOCK_ASSERT(sc); @@ -1999,21 +3095,8 @@ nfe_tick_locked(struct nfe_softc *arg) mii = device_get_softc(sc->nfe_miibus); mii_tick(mii); - - if (!sc->nfe_link) { - if (mii->mii_media_status & IFM_ACTIVE && - IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) { - sc->nfe_link++; - if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T - && bootverbose) - if_printf(sc->nfe_ifp, "gigabit link up\n"); - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - nfe_start_locked(ifp); - } - } + nfe_watchdog(ifp); callout_reset(&sc->nfe_stat_ch, hz, nfe_tick, sc); - - return; } @@ -2027,33 +3110,43 @@ nfe_shutdown(device_t dev) NFE_LOCK(sc); ifp = sc->nfe_ifp; - nfe_stop(ifp,0); + nfe_stop(ifp); /* nfe_reset(sc); */ NFE_UNLOCK(sc); - - return; } static void -nfe_get_macaddr(struct nfe_softc *sc, u_char *addr) +nfe_get_macaddr(struct nfe_softc *sc, uint8_t *addr) { - uint32_t tmp; - - tmp = NFE_READ(sc, NFE_MACADDR_LO); - addr[0] = (tmp >> 8) & 0xff; - addr[1] = (tmp & 0xff); + uint32_t val; + + if ((sc->nfe_flags & NFE_CORRECT_MACADDR) == 0) { + val = NFE_READ(sc, NFE_MACADDR_LO); + addr[0] = (val >> 8) & 0xff; + addr[1] = (val & 0xff); + + val = NFE_READ(sc, NFE_MACADDR_HI); + addr[2] = (val >> 24) & 0xff; + addr[3] = (val >> 16) & 0xff; + addr[4] = (val >> 8) & 0xff; + addr[5] = (val & 0xff); + } else { + val = NFE_READ(sc, NFE_MACADDR_LO); + addr[5] = (val >> 8) & 0xff; + addr[4] = (val & 0xff); - tmp = NFE_READ(sc, NFE_MACADDR_HI); - addr[2] = (tmp >> 24) & 0xff; - addr[3] = (tmp >> 16) & 0xff; - addr[4] = (tmp >> 8) & 0xff; - addr[5] = (tmp & 0xff); + val = NFE_READ(sc, NFE_MACADDR_HI); + addr[3] = (val >> 24) & 0xff; + addr[2] = (val >> 16) & 0xff; + addr[1] = (val >> 8) & 0xff; + addr[0] = (val & 0xff); + } } static void -nfe_set_macaddr(struct nfe_softc *sc, u_char *addr) +nfe_set_macaddr(struct nfe_softc *sc, uint8_t *addr) { NFE_WRITE(sc, NFE_MACADDR_LO, addr[5] << 8 | addr[4]); @@ -2067,18 +3160,43 @@ nfe_set_macaddr(struct nfe_softc *sc, u_char *addr) */ static void -nfe_dma_map_segs(arg, segs, nseg, error) - void *arg; - bus_dma_segment_t *segs; - int error, nseg; +nfe_dma_map_segs(void *arg, bus_dma_segment_t *segs, int nseg, int error) { + struct nfe_dmamap_arg *ctx; - if (error) + if (error != 0) return; KASSERT(nseg == 1, ("too many DMA segments, %d should be 1", nseg)); - *(bus_dma_segment_t *)arg = *segs; + ctx = (struct nfe_dmamap_arg *)arg; + ctx->nfe_busaddr = segs[0].ds_addr; +} + + +static int +sysctl_int_range(SYSCTL_HANDLER_ARGS, int low, int high) +{ + int error, value; + + if (!arg1) + return (EINVAL); + value = *(int *)arg1; + error = sysctl_handle_int(oidp, &value, 0, req); + if (error || !req->newptr) + return (error); + if (value < low || value > high) + return (EINVAL); + *(int *)arg1 = value; + + return (0); +} + + +static int +sysctl_hw_nfe_proc_limit(SYSCTL_HANDLER_ARGS) +{ - return; + return (sysctl_int_range(oidp, arg1, arg2, req, NFE_PROC_MIN, + NFE_PROC_MAX)); } diff --git a/sys/dev/nfe/if_nfereg.h b/sys/dev/nfe/if_nfereg.h index cd56b55..ad82d2e 100644 --- a/sys/dev/nfe/if_nfereg.h +++ b/sys/dev/nfe/if_nfereg.h @@ -18,26 +18,47 @@ * $FreeBSD$ */ -#define NFE_PCI_BA 0x10 - -#define NFE_RX_RING_COUNT 128 +#define NFE_RX_RING_COUNT 256 +#define NFE_JUMBO_RX_RING_COUNT NFE_RX_RING_COUNT #define NFE_TX_RING_COUNT 256 +#define NFE_PROC_DEFAULT ((NFE_RX_RING_COUNT * 3) / 4) +#define NFE_PROC_MIN 50 +#define NFE_PROC_MAX (NFE_RX_RING_COUNT - 1) + +#define NFE_INC(x, y) (x) = ((x) + 1) % y + /* RX/TX MAC addr + type + VLAN + align + slack */ #define NFE_RX_HEADERS 64 /* Maximum MTU size. */ #define NV_PKTLIMIT_1 ETH_DATA_LEN /* Hard limit not known. */ #define NV_PKTLIMIT_2 9100 /* Actual limit according to NVidia:9202 */ -#define NFE_JBYTES (ETHER_MAX_LEN_JUMBO + ETHER_ALIGN) -#define NFE_JPOOL_COUNT (NFE_RX_RING_COUNT + NFE_RX_HEADERS) -#define NFE_MAX_SCATTER (NFE_TX_RING_COUNT - 2) +#define NFE_JUMBO_FRAMELEN NV_PKTLIMIT_2 +#define NFE_JUMBO_MTU \ + (NFE_JUMBO_FRAMELEN - NFE_RX_HEADERS) +#define NFE_MIN_FRAMELEN (ETHER_MIN_LEN - ETHER_CRC_LEN) +#define NFE_JSLOTS ((NFE_JUMBO_RX_RING_COUNT * 3) / 2) + +#define NFE_JRAWLEN (NFE_JUMBO_FRAMELEN + ETHER_ALIGN) +#define NFE_JLEN \ + (NFE_JRAWLEN + (sizeof(uint64_t) - (NFE_JRAWLEN % sizeof(uint64_t)))) +#define NFE_JPAGESZ PAGE_SIZE +#define NFE_RESID \ + (NFE_JPAGESZ - (NFE_JLEN * NFE_JSLOTS) % NFE_JPAGESZ) +#define NFE_JMEM ((NFE_JLEN * NFE_JSLOTS) + NFE_RESID) + +#define NFE_MAX_SCATTER 32 #define NFE_IRQ_STATUS 0x000 #define NFE_IRQ_MASK 0x004 #define NFE_SETUP_R6 0x008 #define NFE_IMTIMER 0x00c +#define NFE_MSI_MAP0 0x020 +#define NFE_MSI_MAP1 0x024 +#define NFE_MSI_IRQ_MASK 0x030 +#define NFE_MAC_RESET 0x03c #define NFE_MISC1 0x080 #define NFE_TX_CTL 0x084 #define NFE_TX_STATUS 0x088 @@ -66,6 +87,7 @@ #define NFE_RXTX_CTL 0x144 #define NFE_TX_RING_ADDR_HI 0x148 #define NFE_RX_RING_ADDR_HI 0x14c +#define NFE_TX_PAUSE_FRAME 0x170 #define NFE_PHY_STATUS 0x180 #define NFE_SETUP_R4 0x184 #define NFE_STATUS 0x188 @@ -78,6 +100,14 @@ #define NFE_PWR_CAP 0x268 #define NFE_PWR_STATE 0x26c #define NFE_VTAG_CTL 0x300 +#define NFE_MSIX_MAP0 0x3e0 +#define NFE_MSIX_MAP1 0x3e4 +#define NFE_MSIX_IRQ_STATUS 0x3f0 +#define NFE_PWR2_CTL 0x600 + +#define NFE_MAC_RESET_MAGIC 0x00f3 + +#define NFE_MAC_ADDR_INORDER 0x8000 #define NFE_PHY_ERROR 0x00001 #define NFE_PHY_WRITE 0x00400 @@ -86,7 +116,9 @@ #define NFE_STATUS_MAGIC 0x140000 -#define NFE_R1_MAGIC 0x16070f +#define NFE_R1_MAGIC_1000 0x14050f +#define NFE_R1_MAGIC_10_100 0x16070f +#define NFE_R1_MAGIC_DEFAULT 0x15050f #define NFE_R2_MAGIC 0x16 #define NFE_R4_MAGIC 0x08 #define NFE_R6_MAGIC 0x03 @@ -118,9 +150,11 @@ #define NFE_RXTX_RXCSUM 0x0400 #define NFE_RXTX_V2MAGIC 0x2100 #define NFE_RXTX_V3MAGIC 0x2200 -#define NFE_RXFILTER_MAGIC 0x007f0008 -#define NFE_U2M (1 << 5) -#define NFE_PROMISC (1 << 7) +#define NFE_RXFILTER_MAGIC 0x007f0000 +#define NFE_PFF_RX_PAUSE (1 << 3) +#define NFE_PFF_LOOPBACK (1 << 4) +#define NFE_PFF_U2M (1 << 5) +#define NFE_PFF_PROMISC (1 << 7) #define NFE_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP) /* default interrupt moderation timer of 128us */ @@ -131,6 +165,9 @@ #define NFE_PWR_VALID (1 << 8) #define NFE_PWR_WAKEUP (1 << 15) +#define NFE_PWR2_WAKEUP_MASK 0x0f11 +#define NFE_PWR2_REVA3 (1 << 0) + #define NFE_MEDIA_SET 0x10000 #define NFE_MEDIA_1000T 0x00032 #define NFE_MEDIA_100TX 0x00064 @@ -141,13 +178,32 @@ #define NFE_PHY_HDX (1 << 8) #define NFE_MISC1_MAGIC 0x003b0f3c +#define NFE_MISC1_TX_PAUSE (1 << 0) #define NFE_MISC1_HDX (1 << 1) +#define NFE_TX_PAUSE_FRAME_DISABLE 0x1ff0080 +#define NFE_TX_PAUSE_FRAME_ENABLE 0x0c00030 + #define NFE_SEED_MASK 0x0003ff00 #define NFE_SEED_10T 0x00007f00 #define NFE_SEED_100TX 0x00002d00 #define NFE_SEED_1000T 0x00007400 +#define NFE_MSI_MESSAGES 8 +#define NFE_MSI_VECTOR_0_ENABLED 0x01 + +/* + * It seems that nForce supports only the lower 40 bits of a DMA address. + */ +#if (BUS_SPACE_MAXADDR < 0xFFFFFFFFFF) +#define NFE_DMA_MAXADDR BUS_SPACE_MAXADDR +#else +#define NFE_DMA_MAXADDR 0xFFFFFFFFFF +#endif + +#define NFE_ADDR_LO(x) ((u_int64_t) (x) & 0xffffffff) +#define NFE_ADDR_HI(x) ((u_int64_t) (x) >> 32) + /* Rx/Tx descriptor */ struct nfe_desc32 { uint32_t physaddr; @@ -179,9 +235,6 @@ struct nfe_desc64 { #define NFE_RX_VALID_V2 (1 << 13) #define NFE_TX_ERROR_V2 0x5c04 #define NFE_TX_LASTFRAG_V2 (1 << 13) -#define NFE_RX_IP_CSUMOK_V2 0x1000 -#define NFE_RX_UDP_CSUMOK_V2 0x1400 -#define NFE_RX_TCP_CSUMOK_V2 0x1800 #define NFE_RX_ERROR1_V2 (1<<2) #define NFE_RX_ERROR2_V2 (1<<3) #define NFE_RX_ERROR3_V2 (1<<4) @@ -191,19 +244,28 @@ struct nfe_desc64 { #define NFE_V2_TXERR "\020" \ "\14FORCEDINT\13LASTPACKET\12UNDERFLOW\10LOSTCARRIER\09DEFERRED\02RETRY" +#define NFE_RING_ALIGN (sizeof(struct nfe_desc64)) + /* flags common to V1/V2 descriptors */ -#define NFE_RX_CSUMOK 0x1c00 +#define NFE_RX_UDP_CSUMOK (1 << 10) +#define NFE_RX_TCP_CSUMOK (1 << 11) +#define NFE_RX_IP_CSUMOK (1 << 12) #define NFE_RX_ERROR (1 << 14) #define NFE_RX_READY (1 << 15) -#define NFE_TX_TCP_CSUM (1 << 10) +#define NFE_RX_LEN_MASK 0x3fff +#define NFE_TX_TCP_UDP_CSUM (1 << 10) #define NFE_TX_IP_CSUM (1 << 11) +#define NFE_TX_TSO (1 << 12) +#define NFE_TX_TSO_SHIFT 14 #define NFE_TX_VALID (1 << 15) #define NFE_READ(sc, reg) \ - bus_space_read_4((sc)->nfe_memt, (sc)->nfe_memh, (reg)) + bus_read_4((sc)->nfe_res[0], (reg)) #define NFE_WRITE(sc, reg, val) \ - bus_space_write_4((sc)->nfe_memt, (sc)->nfe_memh, (reg), (val)) + bus_write_4((sc)->nfe_res[0], (reg), (val)) + +#define NFE_TIMEOUT 1000 #ifndef PCI_VENDOR_NVIDIA #define PCI_VENDOR_NVIDIA 0x10DE @@ -232,6 +294,10 @@ struct nfe_desc64 { #define PCI_PRODUCT_NVIDIA_MCP65_LAN2 0x0451 #define PCI_PRODUCT_NVIDIA_MCP65_LAN3 0x0452 #define PCI_PRODUCT_NVIDIA_MCP65_LAN4 0x0453 +#define PCI_PRODUCT_NVIDIA_MCP67_LAN1 0x054c +#define PCI_PRODUCT_NVIDIA_MCP67_LAN2 0x054d +#define PCI_PRODUCT_NVIDIA_MCP67_LAN3 0x054e +#define PCI_PRODUCT_NVIDIA_MCP67_LAN4 0x054f #define PCI_PRODUCT_NVIDIA_NFORCE3_LAN2 PCI_PRODUCT_NVIDIA_NFORCE2_400_LAN1 #define PCI_PRODUCT_NVIDIA_NFORCE3_LAN3 PCI_PRODUCT_NVIDIA_NFORCE2_400_LAN2 diff --git a/sys/dev/nfe/if_nfevar.h b/sys/dev/nfe/if_nfevar.h index d1929e0..7a320e3 100644 --- a/sys/dev/nfe/if_nfevar.h +++ b/sys/dev/nfe/if_nfevar.h @@ -18,110 +18,120 @@ * $FreeBSD$ */ -#define NFE_IFQ_MAXLEN 64 - struct nfe_tx_data { bus_dmamap_t tx_data_map; - bus_dmamap_t active; - int nsegs; struct mbuf *m; }; struct nfe_tx_ring { + bus_dma_tag_t tx_desc_tag; bus_dmamap_t tx_desc_map; - bus_dma_segment_t tx_desc_segs; bus_addr_t physaddr; struct nfe_desc32 *desc32; struct nfe_desc64 *desc64; + bus_dma_tag_t tx_data_tag; struct nfe_tx_data data[NFE_TX_RING_COUNT]; int queued; int cur; int next; - bus_addr_t tx_desc_addr; - bus_addr_t tx_data_addr; - bus_dma_tag_t tx_desc_tag; - bus_dma_tag_t tx_data_tag; }; -struct nfe_jbuf { - caddr_t buf; - bus_addr_t physaddr; - SLIST_ENTRY(nfe_jbuf) jnext; +struct nfe_jpool_entry { + int slot; + SLIST_ENTRY(nfe_jpool_entry) jpool_entries; }; struct nfe_rx_data { bus_dmamap_t rx_data_map; - bus_dma_tag_t rx_data_tag; - bus_addr_t rx_data_addr; - bus_dma_segment_t rx_data_segs; + bus_addr_t paddr; struct mbuf *m; }; struct nfe_rx_ring { + bus_dma_tag_t rx_desc_tag; bus_dmamap_t rx_desc_map; - bus_dma_segment_t rx_desc_segs; - bus_dma_tag_t rx_desc_tag; - bus_addr_t rx_desc_addr; -#ifndef JMBUF - bus_dmamap_t rx_jumbo_map; - bus_dma_segment_t rx_jumbo_segs; - bus_dma_tag_t rx_jumbo_tag; - bus_addr_t rx_jumbo_addr; - caddr_t jpool; - struct nfe_jbuf jbuf[NFE_JPOOL_COUNT]; - SLIST_HEAD(, nfe_jbuf) jfreelist; -#endif bus_addr_t physaddr; struct nfe_desc32 *desc32; struct nfe_desc64 *desc64; + bus_dma_tag_t rx_data_tag; + bus_dmamap_t rx_spare_map; struct nfe_rx_data data[NFE_RX_RING_COUNT]; - int bufsz; int cur; int next; }; +struct nfe_jrx_ring { + bus_dma_tag_t jrx_desc_tag; + bus_dmamap_t jrx_desc_map; + bus_dma_tag_t jrx_jumbo_tag; + bus_dmamap_t jrx_jumbo_map; + void *jpool; + caddr_t jslots[NFE_JSLOTS]; + bus_addr_t jphysaddr; + struct nfe_desc32 *jdesc32; + struct nfe_desc64 *jdesc64; + bus_dma_tag_t jrx_data_tag; + bus_dmamap_t jrx_spare_map; + struct nfe_rx_data jdata[NFE_JUMBO_RX_RING_COUNT]; + int jcur; + int jnext; +}; + struct nfe_softc { struct ifnet *nfe_ifp; device_t nfe_dev; + uint16_t nfe_devid; + uint16_t nfe_revid; device_t nfe_miibus; struct mtx nfe_mtx; - bus_space_handle_t nfe_memh; - bus_space_tag_t nfe_memt; - struct resource *nfe_res; - struct resource *nfe_irq; - void *nfe_intrhand; - struct mii_data nfe_mii; - u_int8_t nfe_unit; + struct resource *nfe_res[1]; + struct resource *nfe_msix_res; + struct resource *nfe_msix_pba_res; + struct resource *nfe_irq[NFE_MSI_MESSAGES]; + void *nfe_intrhand[NFE_MSI_MESSAGES]; struct callout nfe_stat_ch; + int nfe_watchdog_timer; - struct arpcom nfe_arpcom; bus_dma_tag_t nfe_parent_tag; - /* struct timeout nfe_tick_ch; */ - void *nfe_powerhook; int nfe_if_flags; - u_int nfe_flags; -#define NFE_JUMBO_SUP 0x01 -#define NFE_40BIT_ADDR 0x02 -#define NFE_HW_CSUM 0x04 -#define NFE_HW_VLAN 0x08 - u_int32_t rxtxctl; - u_int32_t nfe_mtu; - u_int8_t mii_phyaddr; - u_char eaddr[ETHER_ADDR_LEN]; - struct task nfe_txtask; + uint32_t nfe_flags; +#define NFE_JUMBO_SUP 0x0001 +#define NFE_40BIT_ADDR 0x0002 +#define NFE_HW_CSUM 0x0004 +#define NFE_HW_VLAN 0x0008 +#define NFE_PWR_MGMT 0x0010 +#define NFE_CORRECT_MACADDR 0x0020 +#define NFE_TX_FLOW_CTRL 0x0040 + uint32_t rxtxctl; + uint8_t mii_phyaddr; + uint8_t eaddr[ETHER_ADDR_LEN]; + struct taskqueue *nfe_tq; + struct task nfe_int_task; + struct task nfe_tx_task; + struct task nfe_link_task; int nfe_link; + int nfe_suspended; + int nfe_framesize; + int nfe_process_limit; + int nfe_force_tx; + uint32_t nfe_irq_status; + uint32_t nfe_irq_mask; + uint32_t nfe_intrs; + uint32_t nfe_nointrs; + uint32_t nfe_msi; + uint32_t nfe_msix; struct nfe_tx_ring txq; struct nfe_rx_ring rxq; - -#ifdef DEVICE_POLLING - int rxcycles; -#endif + struct nfe_jrx_ring jrxq; + SLIST_HEAD(__nfe_jfreehead, nfe_jpool_entry) nfe_jfree_listhead; + SLIST_HEAD(__nfe_jinusehead, nfe_jpool_entry) nfe_jinuse_listhead; + struct mtx nfe_jlist_mtx; }; struct nfe_type { - u_int16_t vid_id; - u_int16_t dev_id; + uint16_t vid_id; + uint16_t dev_id; char *name; }; |