diff options
Diffstat (limited to 'sys/dev/ixgbe/ixgbe.c')
| -rw-r--r-- | sys/dev/ixgbe/ixgbe.c | 975 |
1 files changed, 362 insertions, 613 deletions
diff --git a/sys/dev/ixgbe/ixgbe.c b/sys/dev/ixgbe/ixgbe.c index 33d13b8..27d3d0a 100644 --- a/sys/dev/ixgbe/ixgbe.c +++ b/sys/dev/ixgbe/ixgbe.c @@ -47,7 +47,7 @@ int ixgbe_display_debug_stats = 0; /********************************************************************* * Driver version *********************************************************************/ -char ixgbe_driver_version[] = "2.4.11"; +char ixgbe_driver_version[] = "2.5.0"; /********************************************************************* * PCI Device ID Table @@ -104,16 +104,16 @@ static int ixgbe_probe(device_t); static int ixgbe_attach(device_t); static int ixgbe_detach(device_t); static int ixgbe_shutdown(device_t); -#if __FreeBSD_version >= 800000 +#ifdef IXGBE_LEGACY_TX +static void ixgbe_start(struct ifnet *); +static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); +#else /* ! IXGBE_LEGACY_TX */ static int ixgbe_mq_start(struct ifnet *, struct mbuf *); static int ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *, struct mbuf *); static void ixgbe_qflush(struct ifnet *); static void ixgbe_deferred_mq_start(void *, int); -#else -static void ixgbe_start(struct ifnet *); -static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); -#endif +#endif /* IXGBE_LEGACY_TX */ static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); static void ixgbe_init(void *); static void ixgbe_init_locked(struct adapter *); @@ -150,7 +150,7 @@ static void ixgbe_enable_intr(struct adapter *); static void ixgbe_disable_intr(struct adapter *); static void ixgbe_update_stats_counters(struct adapter *); static bool ixgbe_txeof(struct tx_ring *); -static bool ixgbe_rxeof(struct ix_queue *, int); +static bool ixgbe_rxeof(struct ix_queue *); static void ixgbe_rx_checksum(u32, struct mbuf *, u32); static void ixgbe_set_promisc(struct adapter *); static void ixgbe_set_multi(struct adapter *); @@ -163,10 +163,10 @@ static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS); static int ixgbe_dma_malloc(struct adapter *, bus_size_t, struct ixgbe_dma_alloc *, int); static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); -static void ixgbe_add_rx_process_limit(struct adapter *, const char *, - const char *, int *, int); -static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); -static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *); +static int ixgbe_tx_ctx_setup(struct tx_ring *, + struct mbuf *, u32 *, u32 *); +static int ixgbe_tso_setup(struct tx_ring *, + struct mbuf *, u32 *, u32 *); static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); static void ixgbe_configure_ivars(struct adapter *); static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); @@ -246,9 +246,13 @@ static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate); /* How many packets rxeof tries to clean at a time */ -static int ixgbe_rx_process_limit = 128; +static int ixgbe_rx_process_limit = 256; TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); +/* How many packets txeof tries to clean at a time */ +static int ixgbe_tx_process_limit = 256; +TUNABLE_INT("hw.ixgbe.tx_process_limit", &ixgbe_tx_process_limit); + /* ** Smart speed setting, default to on ** this only works as a compile option @@ -266,15 +270,6 @@ static int ixgbe_enable_msix = 1; TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); /* - * Header split: this causes the hardware to DMA - * the header into a separate mbuf from the payload, - * it can be a performance win in some workloads, but - * in others it actually hurts, its off by default. - */ -static int ixgbe_header_split = FALSE; -TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split); - -/* * Number of Queues, can be set to 0, * it then autoconfigures based on the * number of cpus with a max of 8. This @@ -550,7 +545,6 @@ ixgbe_attach(device_t dev) case IXGBE_ERR_SFP_NOT_SUPPORTED: device_printf(dev,"Unsupported SFP+ Module\n"); error = EIO; - device_printf(dev,"Hardware Initialization Failure\n"); goto err_late; case IXGBE_ERR_SFP_NOT_PRESENT: device_printf(dev,"No SFP+ Module found\n"); @@ -573,11 +567,6 @@ ixgbe_attach(device_t dev) if (ixgbe_setup_interface(dev, adapter) != 0) goto err_late; - /* Sysctl for limiting the amount of work done in the taskqueue */ - ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", - "max number of rx packets to process", &adapter->rx_process_limit, - ixgbe_rx_process_limit); - /* Initialize statistics */ ixgbe_update_stats_counters(adapter); @@ -662,7 +651,7 @@ ixgbe_detach(device_t dev) for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { if (que->tq) { -#if __FreeBSD_version >= 800000 +#ifdef IXGBE_LEGACY_TX taskqueue_drain(que->tq, &txr->txq_task); #endif taskqueue_drain(que->tq, &que->que_task); @@ -726,7 +715,7 @@ ixgbe_shutdown(device_t dev) } -#if __FreeBSD_version < 800000 +#ifdef IXGBE_LEGACY_TX /********************************************************************* * Transmit entry point * @@ -745,17 +734,14 @@ ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) IXGBE_TX_LOCK_ASSERT(txr); - if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING) + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) return; if (!adapter->link_active) return; while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { - if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) { - txr->queue_status |= IXGBE_QUEUE_DEPLETED; + if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) break; - } IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); if (m_head == NULL) @@ -764,8 +750,6 @@ ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) if (ixgbe_xmit(txr, &m_head)) { if (m_head != NULL) IFQ_DRV_PREPEND(&ifp->if_snd, m_head); - if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) - txr->queue_status |= IXGBE_QUEUE_DEPLETED; break; } /* Send a copy of the frame to the BPF listener */ @@ -798,7 +782,8 @@ ixgbe_start(struct ifnet *ifp) return; } -#else +#else /* ! IXGBE_LEGACY_TX */ + /* ** Multiqueue Transmit driver ** @@ -820,8 +805,7 @@ ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) txr = &adapter->tx_rings[i]; que = &adapter->queues[i]; - if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) && - IXGBE_TX_TRYLOCK(txr)) { + if (IXGBE_TX_TRYLOCK(txr)) { err = ixgbe_mq_start_locked(ifp, txr, m); IXGBE_TX_UNLOCK(txr); } else { @@ -840,7 +824,6 @@ ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) int enqueued, err = 0; if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || - (txr->queue_status == IXGBE_QUEUE_DEPLETED) || adapter->link_active == 0) { if (m != NULL) err = drbr_enqueue(ifp, txr->br, m); @@ -871,16 +854,12 @@ ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) break; if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) ixgbe_txeof(txr); - if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) { - txr->queue_status |= IXGBE_QUEUE_DEPLETED; - break; - } next = drbr_dequeue(ifp, txr->br); } if (enqueued > 0) { /* Set watchdog on */ - txr->queue_status |= IXGBE_QUEUE_WORKING; + txr->queue_status = IXGBE_QUEUE_WORKING; txr->watchdog_time = ticks; } @@ -924,7 +903,7 @@ ixgbe_qflush(struct ifnet *ifp) } if_qflush(ifp); } -#endif /* __FreeBSD_version >= 800000 */ +#endif /* IXGBE_LEGACY_TX */ /********************************************************************* * Ioctl entry point @@ -939,6 +918,7 @@ static int ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) { struct adapter *adapter = ifp->if_softc; + struct ixgbe_hw *hw = &adapter->hw; struct ifreq *ifr = (struct ifreq *) data; #if defined(INET) || defined(INET6) struct ifaddr *ifa = (struct ifaddr *)data; @@ -1044,7 +1024,22 @@ ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) VLAN_CAPABILITIES(ifp); break; } - + case SIOCGI2C: + { + struct ixgbe_i2c_req i2c; + IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)"); + error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); + if (error) + break; + if ((i2c.dev_addr != 0xA0) || (i2c.dev_addr != 0xA2)){ + error = EINVAL; + break; + } + hw->phy.ops.read_i2c_byte(hw, i2c.offset, + i2c.dev_addr, i2c.data); + error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); + break; + } default: IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); error = ether_ioctl(ifp, command, data); @@ -1117,7 +1112,7 @@ ixgbe_init_locked(struct adapter *adapter) /* ** Determine the correct mbuf pool - ** for doing jumbo/headersplit + ** for doing jumbo frames */ if (adapter->max_frame_size <= 2048) adapter->rx_mbuf_sz = MCLBYTES; @@ -1325,7 +1320,6 @@ ixgbe_init_locked(struct adapter *adapter) /* Now inform the stack we're ready */ ifp->if_drv_flags |= IFF_DRV_RUNNING; - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; return; } @@ -1415,10 +1409,10 @@ ixgbe_handle_que(void *context, int pending) bool more; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - more = ixgbe_rxeof(que, adapter->rx_process_limit); + more = ixgbe_rxeof(que); IXGBE_TX_LOCK(txr); ixgbe_txeof(txr); -#if __FreeBSD_version >= 800000 +#ifndef IXGBE_LEGACY_TX if (!drbr_empty(ifp, txr->br)) ixgbe_mq_start_locked(ifp, txr, NULL); #else @@ -1463,7 +1457,7 @@ ixgbe_legacy_irq(void *arg) return; } - more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); + more_rx = ixgbe_rxeof(que); IXGBE_TX_LOCK(txr); do { @@ -1509,7 +1503,7 @@ ixgbe_msix_que(void *arg) ixgbe_disable_queue(adapter, que->msix); ++que->irqs; - more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); + more_rx = ixgbe_rxeof(que); IXGBE_TX_LOCK(txr); more_tx = ixgbe_txeof(txr); @@ -1518,7 +1512,7 @@ ixgbe_msix_que(void *arg) ** has anything queued the task gets ** scheduled to handle it. */ -#if __FreeBSD_version < 800000 +#ifdef IXGBE_LEGACY_TX if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd)) #else if (!drbr_empty(adapter->ifp, txr->br)) @@ -1743,9 +1737,9 @@ ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) { struct adapter *adapter = txr->adapter; u32 olinfo_status = 0, cmd_type_len; - u32 paylen = 0; int i, j, error, nsegs; - int first, last = 0; + int first; + bool remap = TRUE; struct mbuf *m_head; bus_dma_segment_t segs[adapter->num_segs]; bus_dmamap_t map; @@ -1773,73 +1767,58 @@ ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) /* * Map the packet for DMA. */ +retry: error = bus_dmamap_load_mbuf_sg(txr->txtag, map, *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - if (error == EFBIG) { + if (__predict_false(error)) { struct mbuf *m; - m = m_defrag(*m_headp, M_DONTWAIT); - if (m == NULL) { - adapter->mbuf_defrag_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - - /* Try it again */ - error = bus_dmamap_load_mbuf_sg(txr->txtag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - if (error == ENOMEM) { - adapter->no_tx_dma_setup++; + switch (error) { + case EFBIG: + /* Try it again? - one try */ + if (remap == TRUE) { + remap = FALSE; + m = m_defrag(*m_headp, M_NOWAIT); + if (m == NULL) { + adapter->mbuf_defrag_failed++; + m_freem(*m_headp); + *m_headp = NULL; + return (ENOBUFS); + } + *m_headp = m; + goto retry; + } else + return (error); + case ENOMEM: + txr->no_tx_dma_setup++; return (error); - } else if (error != 0) { - adapter->no_tx_dma_setup++; + default: + txr->no_tx_dma_setup++; m_freem(*m_headp); *m_headp = NULL; return (error); } - } else if (error == ENOMEM) { - adapter->no_tx_dma_setup++; - return (error); - } else if (error != 0) { - adapter->no_tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); } /* Make certain there are enough descriptors */ if (nsegs > txr->tx_avail - 2) { txr->no_desc_avail++; - error = ENOBUFS; - goto xmit_fail; + bus_dmamap_unload(txr->txtag, map); + return (ENOBUFS); } m_head = *m_headp; /* ** Set up the appropriate offload context - ** this becomes the first descriptor of - ** a packet. + ** this will consume the first descriptor */ - if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { - if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) { - cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; - olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; - olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; - ++adapter->tso_tx; - } else - return (ENXIO); - } else if (ixgbe_tx_ctx_setup(txr, m_head)) - olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; - -#ifdef IXGBE_IEEE1588 - /* This is changing soon to an mtag detection */ - if (we detect this mbuf has a TSTAMP mtag) - cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP; -#endif + error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); + if (__predict_false(error)) { + if (error == ENOBUFS) + *m_headp = NULL; + return (error); + } #ifdef IXGBE_FDIR /* Do the flow director magic */ @@ -1851,10 +1830,6 @@ ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) } } #endif - /* Record payload length */ - if (paylen == 0) - olinfo_status |= m_head->m_pkthdr.len << - IXGBE_ADVTXD_PAYLEN_SHIFT; i = txr->next_avail_desc; for (j = 0; j < nsegs; j++) { @@ -1870,13 +1845,9 @@ ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) txd->read.cmd_type_len = htole32(txr->txd_cmd | cmd_type_len |seglen); txd->read.olinfo_status = htole32(olinfo_status); - last = i; /* descriptor that will get completion IRQ */ - if (++i == adapter->num_tx_desc) + if (++i == txr->num_desc) i = 0; - - txbuf->m_head = NULL; - txbuf->eop_index = -1; } txd->read.cmd_type_len |= @@ -1885,14 +1856,19 @@ ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) txr->next_avail_desc = i; txbuf->m_head = m_head; - /* Swap the dma map between the first and last descriptor */ + /* + ** Here we swap the map so the last descriptor, + ** which gets the completion interrupt has the + ** real map, and the first descriptor gets the + ** unused map from this descriptor. + */ txr->tx_buffers[first].map = txbuf->map; txbuf->map = map; bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); - /* Set the index of the descriptor that will be marked done */ + /* Set the EOP descriptor that will be marked done */ txbuf = &txr->tx_buffers[first]; - txbuf->eop_index = last; + txbuf->eop = txd; bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); @@ -1905,10 +1881,6 @@ ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) return (0); -xmit_fail: - bus_dmamap_unload(txr->txtag, txbuf->map); - return (error); - } static void @@ -2027,13 +1999,11 @@ ixgbe_local_timer(void *arg) { struct adapter *adapter = arg; device_t dev = adapter->dev; - struct ifnet *ifp = adapter->ifp; struct ix_queue *que = adapter->queues; struct tx_ring *txr = adapter->tx_rings; - int hung, busy, paused; + int hung = 0, paused = 0; mtx_assert(&adapter->core_mtx, MA_OWNED); - hung = busy = paused = 0; /* Check for pluggable optics */ if (adapter->sfp_probe) @@ -2055,23 +2025,15 @@ ixgbe_local_timer(void *arg) ** - watchdog only if all queues show hung */ for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { - if ((txr->queue_status & IXGBE_QUEUE_HUNG) && + if ((txr->queue_status == IXGBE_QUEUE_HUNG) && (paused == 0)) ++hung; - if (txr->queue_status & IXGBE_QUEUE_DEPLETED) - ++busy; - if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0) + else if (txr->queue_status == IXGBE_QUEUE_WORKING) taskqueue_enqueue(que->tq, &que->que_task); } /* Only truely watchdog if all queues show hung */ if (hung == adapter->num_queues) goto watchdog; - /* Only turn off the stack flow when ALL are depleted */ - if (busy == adapter->num_queues) - ifp->if_drv_flags |= IFF_DRV_OACTIVE; - else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) && - (busy < adapter->num_queues)) - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; out: ixgbe_rearm_queues(adapter, adapter->que_mask); @@ -2100,7 +2062,6 @@ static void ixgbe_update_link_status(struct adapter *adapter) { struct ifnet *ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; device_t dev = adapter->dev; @@ -2121,9 +2082,6 @@ ixgbe_update_link_status(struct adapter *adapter) device_printf(dev,"Link is Down\n"); if_link_state_change(ifp, LINK_STATE_DOWN); adapter->link_active = FALSE; - for (int i = 0; i < adapter->num_queues; - i++, txr++) - txr->queue_status = IXGBE_QUEUE_IDLE; } } @@ -2154,7 +2112,6 @@ ixgbe_stop(void *arg) /* Let the stack know...*/ ifp->if_drv_flags &= ~IFF_DRV_RUNNING; - ifp->if_drv_flags |= IFF_DRV_OACTIVE; ixgbe_reset_hw(hw); hw->adapter_stopped = FALSE; @@ -2266,12 +2223,12 @@ ixgbe_setup_optics(struct adapter *adapter) static int ixgbe_allocate_legacy(struct adapter *adapter) { - device_t dev = adapter->dev; + device_t dev = adapter->dev; struct ix_queue *que = adapter->queues; -#if __FreeBSD_version >= 800000 +#ifndef IXGBE_LEGACY_TX struct tx_ring *txr = adapter->tx_rings; #endif - int error, rid = 0; + int error, rid = 0; /* MSI RID at 1 */ if (adapter->msix == 1) @@ -2290,7 +2247,7 @@ ixgbe_allocate_legacy(struct adapter *adapter) * Try allocating a fast interrupt and the associated deferred * processing contexts. */ -#if __FreeBSD_version >= 800000 +#ifndef IXGBE_LEGACY_TX TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr); #endif TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que); @@ -2372,7 +2329,7 @@ ixgbe_allocate_msix(struct adapter *adapter) if (adapter->num_queues > 1) bus_bind_intr(dev, que->res, i); -#if __FreeBSD_version >= 800000 +#ifndef IXGBE_LEGACY_TX TASK_INIT(&txr->txq_task, 0, ixgbe_deferred_mq_start, txr); #endif TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que); @@ -2618,7 +2575,7 @@ ixgbe_setup_interface(device_t dev, struct adapter *adapter) ifp->if_softc = adapter; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_ioctl = ixgbe_ioctl; -#if __FreeBSD_version >= 800000 +#ifndef IXGBE_LEGACY_TX ifp->if_transmit = ixgbe_mq_start; ifp->if_qflush = ixgbe_qflush; #else @@ -2843,6 +2800,7 @@ ixgbe_allocate_queues(struct adapter *adapter) txr = &adapter->tx_rings[i]; txr->adapter = adapter; txr->me = i; + txr->num_desc = adapter->num_tx_desc; /* Initialize the TX side lock */ snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", @@ -2866,7 +2824,7 @@ ixgbe_allocate_queues(struct adapter *adapter) error = ENOMEM; goto err_tx_desc; } -#if __FreeBSD_version >= 800000 +#ifndef IXGBE_LEGACY_TX /* Allocate a buf ring */ txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, M_WAITOK, &txr->tx_mtx); @@ -2889,6 +2847,7 @@ ixgbe_allocate_queues(struct adapter *adapter) /* Set up some basics */ rxr->adapter = adapter; rxr->me = i; + rxr->num_desc = adapter->num_rx_desc; /* Initialize the RX side lock */ snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", @@ -3034,7 +2993,7 @@ ixgbe_setup_transmit_ring(struct tx_ring *txr) /* Free any existing tx buffers. */ txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { + for (i = 0; i < txr->num_desc; i++, txbuf++) { if (txbuf->m_head != NULL) { bus_dmamap_sync(txr->txtag, txbuf->map, BUS_DMASYNC_POSTWRITE); @@ -3058,8 +3017,8 @@ ixgbe_setup_transmit_ring(struct tx_ring *txr) netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si)); } #endif /* DEV_NETMAP */ - /* Clear the EOP index */ - txbuf->eop_index = -1; + /* Clear the EOP descriptor pointer */ + txbuf->eop = NULL; } #ifdef IXGBE_FDIR @@ -3113,7 +3072,7 @@ ixgbe_initialize_transmit_units(struct adapter *adapter) (tdba & 0x00000000ffffffffULL)); IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), - adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); + adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc)); /* Setup the HW Tx Head and Tail descriptor pointers */ IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); @@ -3123,6 +3082,9 @@ ixgbe_initialize_transmit_units(struct adapter *adapter) txr->txd_cmd = IXGBE_TXD_CMD_IFCS; txr->queue_status = IXGBE_QUEUE_IDLE; + /* Set the processing limit */ + txr->process_limit = ixgbe_tx_process_limit; + /* Disable Head Writeback */ switch (hw->mac.type) { case ixgbe_mac_82598EB: @@ -3224,7 +3186,7 @@ ixgbe_free_transmit_buffers(struct tx_ring *txr) tx_buffer->map = NULL; } } -#if __FreeBSD_version >= 800000 +#ifdef IXGBE_LEGACY_TX if (txr->br != NULL) buf_ring_free(txr->br, M_DEVBUF); #endif @@ -3241,43 +3203,49 @@ ixgbe_free_transmit_buffers(struct tx_ring *txr) /********************************************************************* * - * Advanced Context Descriptor setup for VLAN or CSUM + * Advanced Context Descriptor setup for VLAN, CSUM or TSO * **********************************************************************/ -static bool -ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) +static int +ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, + u32 *cmd_type_len, u32 *olinfo_status) { - struct adapter *adapter = txr->adapter; struct ixgbe_adv_tx_context_desc *TXD; - struct ixgbe_tx_buf *tx_buffer; - u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; struct ether_vlan_header *eh; struct ip *ip; struct ip6_hdr *ip6; - int ehdrlen, ip_hlen = 0; + u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; + int ehdrlen, ip_hlen = 0; u16 etype; u8 ipproto = 0; - bool offload = TRUE; - int ctxd = txr->next_avail_desc; - u16 vtag = 0; + int offload = TRUE; + int ctxd = txr->next_avail_desc; + u16 vtag = 0; + /* First check if TSO is to be used */ + if (mp->m_pkthdr.csum_flags & CSUM_TSO) + return (ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status)); if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) offload = FALSE; - tx_buffer = &txr->tx_buffers[ctxd]; + /* Indicate the whole packet as payload when not doing TSO */ + *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; + + /* Now ready a context descriptor */ TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; /* ** In advanced descriptors the vlan tag must - ** be placed into the descriptor itself. + ** be placed into the context descriptor. Hence + ** we need to make one even if not doing offloads. */ if (mp->m_flags & M_VLANTAG) { vtag = htole16(mp->m_pkthdr.ether_vtag); vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); - } else if (offload == FALSE) - return FALSE; + } else if (offload == FALSE) /* ... no offload to do */ + return (0); /* * Determine where frame payload starts. @@ -3340,22 +3308,22 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) break; } + if (offload) /* For the TX descriptor setup */ + *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; + /* Now copy bits into descriptor */ - TXD->vlan_macip_lens |= htole32(vlan_macip_lens); - TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); + TXD->vlan_macip_lens = htole32(vlan_macip_lens); + TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); TXD->seqnum_seed = htole32(0); TXD->mss_l4len_idx = htole32(0); - tx_buffer->m_head = NULL; - tx_buffer->eop_index = -1; - /* We've consumed the first desc, adjust counters */ - if (++ctxd == adapter->num_tx_desc) + if (++ctxd == txr->num_desc) ctxd = 0; txr->next_avail_desc = ctxd; --txr->tx_avail; - return (offload); + return (0); } /********************************************************************** @@ -3364,15 +3332,13 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) * adapters using advanced tx descriptors * **********************************************************************/ -static bool -ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen, - u32 *olinfo_status) +static int +ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, + u32 *cmd_type_len, u32 *olinfo_status) { - struct adapter *adapter = txr->adapter; struct ixgbe_adv_tx_context_desc *TXD; - struct ixgbe_tx_buf *tx_buffer; u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; - u32 mss_l4len_idx = 0, len; + u32 mss_l4len_idx = 0, paylen; u16 vtag = 0, eh_type; int ctxd, ehdrlen, ip_hlen, tcp_hlen; struct ether_vlan_header *eh; @@ -3398,18 +3364,15 @@ ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen, eh_type = eh->evl_encap_proto; } - /* Ensure we have at least the IP+TCP header in the first mbuf. */ - len = ehdrlen + sizeof(struct tcphdr); switch (ntohs(eh_type)) { #ifdef INET6 case ETHERTYPE_IPV6: - if (mp->m_len < len + sizeof(struct ip6_hdr)) - return FALSE; ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); /* XXX-BZ For now we do not pretend to support ext. hdrs. */ if (ip6->ip6_nxt != IPPROTO_TCP) - return FALSE; + return (ENXIO); ip_hlen = sizeof(struct ip6_hdr); + ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; @@ -3417,11 +3380,9 @@ ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen, #endif #ifdef INET case ETHERTYPE_IP: - if (mp->m_len < len + sizeof(struct ip)) - return FALSE; ip = (struct ip *)(mp->m_data + ehdrlen); if (ip->ip_p != IPPROTO_TCP) - return FALSE; + return (ENXIO); ip->ip_sum = 0; ip_hlen = ip->ip_hl << 2; th = (struct tcphdr *)((caddr_t)ip + ip_hlen); @@ -3439,13 +3400,12 @@ ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen, } ctxd = txr->next_avail_desc; - tx_buffer = &txr->tx_buffers[ctxd]; TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; tcp_hlen = th->th_off << 2; /* This is used in the transmit desc in encap */ - *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; + paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; /* VLAN MACLEN IPLEN */ if (mp->m_flags & M_VLANTAG) { @@ -3455,12 +3415,12 @@ ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen, vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; vlan_macip_lens |= ip_hlen; - TXD->vlan_macip_lens |= htole32(vlan_macip_lens); + TXD->vlan_macip_lens = htole32(vlan_macip_lens); /* ADV DTYPE TUCMD */ type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; - TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); + TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); /* MSS L4LEN IDX */ mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); @@ -3468,15 +3428,17 @@ ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen, TXD->mss_l4len_idx = htole32(mss_l4len_idx); TXD->seqnum_seed = htole32(0); - tx_buffer->m_head = NULL; - tx_buffer->eop_index = -1; - if (++ctxd == adapter->num_tx_desc) + if (++ctxd == txr->num_desc) ctxd = 0; txr->tx_avail--; txr->next_avail_desc = ctxd; - return TRUE; + *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; + *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; + *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; + ++txr->tso_tx; + return (0); } #ifdef IXGBE_FDIR @@ -3565,11 +3527,12 @@ ixgbe_atr(struct tx_ring *txr, struct mbuf *mp) static bool ixgbe_txeof(struct tx_ring *txr) { - struct adapter *adapter = txr->adapter; - struct ifnet *ifp = adapter->ifp; - u32 first, last, done, processed; - struct ixgbe_tx_buf *tx_buffer; - struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; + struct adapter *adapter = txr->adapter; + struct ifnet *ifp = adapter->ifp; + u32 work, processed = 0; + u16 limit = txr->process_limit; + struct ixgbe_tx_buf *buf; + union ixgbe_adv_tx_desc *txd; mtx_assert(&txr->tx_mtx, MA_OWNED); @@ -3577,9 +3540,7 @@ ixgbe_txeof(struct tx_ring *txr) if (ifp->if_capenable & IFCAP_NETMAP) { struct netmap_adapter *na = NA(ifp); struct netmap_kring *kring = &na->tx_rings[txr->me]; - - tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base; - + txd = txr->tx_base; bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_POSTREAD); /* @@ -3602,7 +3563,7 @@ ixgbe_txeof(struct tx_ring *txr) */ if (!netmap_mitigate || (kring->nr_kflags < kring->nkr_num_slots && - tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) { + txd[kring->nr_kflags].wb.status & IXGBE_TXD_STAT_DD)) { kring->nr_kflags = kring->nkr_num_slots; selwakeuppri(&na->tx_rings[txr->me].si, PI_NET); IXGBE_TX_UNLOCK(txr); @@ -3615,84 +3576,92 @@ ixgbe_txeof(struct tx_ring *txr) } #endif /* DEV_NETMAP */ - if (txr->tx_avail == adapter->num_tx_desc) { + if (txr->tx_avail == txr->num_desc) { txr->queue_status = IXGBE_QUEUE_IDLE; return FALSE; } - processed = 0; - first = txr->next_to_clean; - tx_buffer = &txr->tx_buffers[first]; - /* For cleanup we just use legacy struct */ - tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; - last = tx_buffer->eop_index; - if (last == -1) - return FALSE; - eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; - - /* - ** Get the index of the first descriptor - ** BEYOND the EOP and call that 'done'. - ** I do this so the comparison in the - ** inner while loop below can be simple - */ - if (++last == adapter->num_tx_desc) last = 0; - done = last; - + /* Get work starting point */ + work = txr->next_to_clean; + buf = &txr->tx_buffers[work]; + txd = &txr->tx_base[work]; + work -= txr->num_desc; /* The distance to ring end */ bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_POSTREAD); - /* - ** Only the EOP descriptor of a packet now has the DD - ** bit set, this is what we look for... - */ - while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { - /* We clean the range of the packet */ - while (first != done) { - tx_desc->upper.data = 0; - tx_desc->lower.data = 0; - tx_desc->buffer_addr = 0; - ++txr->tx_avail; - ++processed; - if (tx_buffer->m_head) { + do { + union ixgbe_adv_tx_desc *eop= buf->eop; + if (eop == NULL) /* No work */ + break; + + if ((eop->wb.status & IXGBE_TXD_STAT_DD) == 0) + break; /* I/O not complete */ + + if (buf->m_head) { + txr->bytes += + buf->m_head->m_pkthdr.len; + bus_dmamap_sync(txr->txtag, + buf->map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(txr->txtag, + buf->map); + m_freem(buf->m_head); + buf->m_head = NULL; + buf->map = NULL; + } + buf->eop = NULL; + ++txr->tx_avail; + + /* We clean the range if multi segment */ + while (txd != eop) { + ++txd; + ++buf; + ++work; + /* wrap the ring? */ + if (__predict_false(!work)) { + work -= txr->num_desc; + buf = txr->tx_buffers; + txd = txr->tx_base; + } + if (buf->m_head) { txr->bytes += - tx_buffer->m_head->m_pkthdr.len; + buf->m_head->m_pkthdr.len; bus_dmamap_sync(txr->txtag, - tx_buffer->map, + buf->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txr->txtag, - tx_buffer->map); - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - tx_buffer->map = NULL; + buf->map); + m_freem(buf->m_head); + buf->m_head = NULL; + buf->map = NULL; } - tx_buffer->eop_index = -1; - txr->watchdog_time = ticks; - - if (++first == adapter->num_tx_desc) - first = 0; + ++txr->tx_avail; + buf->eop = NULL; - tx_buffer = &txr->tx_buffers[first]; - tx_desc = - (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; } ++txr->packets; + ++processed; ++ifp->if_opackets; - /* See if there is more work now */ - last = tx_buffer->eop_index; - if (last != -1) { - eop_desc = - (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; - /* Get next done point */ - if (++last == adapter->num_tx_desc) last = 0; - done = last; - } else - break; - } + txr->watchdog_time = ticks; + + /* Try the next packet */ + ++txd; + ++buf; + ++work; + /* reset with a wrap */ + if (__predict_false(!work)) { + work -= txr->num_desc; + buf = txr->tx_buffers; + txd = txr->tx_base; + } + prefetch(txd); + } while (__predict_true(--limit)); + bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - txr->next_to_clean = first; + work += txr->num_desc; + txr->next_to_clean = work; /* ** Watchdog calculation, we know there's @@ -3703,11 +3672,7 @@ ixgbe_txeof(struct tx_ring *txr) if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG)) txr->queue_status = IXGBE_QUEUE_HUNG; - /* With a minimum free clear the depleted state bit. */ - if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) - txr->queue_status &= ~IXGBE_QUEUE_DEPLETED; - - if (txr->tx_avail == adapter->num_tx_desc) { + if (txr->tx_avail == txr->num_desc) { txr->queue_status = IXGBE_QUEUE_IDLE; return (FALSE); } @@ -3728,81 +3693,50 @@ static void ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) { struct adapter *adapter = rxr->adapter; - bus_dma_segment_t hseg[1]; - bus_dma_segment_t pseg[1]; + bus_dma_segment_t seg[1]; struct ixgbe_rx_buf *rxbuf; - struct mbuf *mh, *mp; + struct mbuf *mp; int i, j, nsegs, error; bool refreshed = FALSE; i = j = rxr->next_to_refresh; /* Control the loop with one beyond */ - if (++j == adapter->num_rx_desc) + if (++j == rxr->num_desc) j = 0; while (j != limit) { rxbuf = &rxr->rx_buffers[i]; - if (rxr->hdr_split == FALSE) - goto no_split; - - if (rxbuf->m_head == NULL) { - mh = m_gethdr(M_DONTWAIT, MT_DATA); - if (mh == NULL) - goto update; - } else - mh = rxbuf->m_head; - - mh->m_pkthdr.len = mh->m_len = MHLEN; - mh->m_len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->htag, - rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: hdr dmamap load" - " failure - %d\n", error); - m_free(mh); - rxbuf->m_head = NULL; - goto update; - } - rxbuf->m_head = mh; - bus_dmamap_sync(rxr->htag, rxbuf->hmap, - BUS_DMASYNC_PREREAD); - rxr->rx_base[i].read.hdr_addr = - htole64(hseg[0].ds_addr); - -no_split: - if (rxbuf->m_pack == NULL) { + if (rxbuf->buf == NULL) { mp = m_getjcl(M_DONTWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); + M_PKTHDR, rxr->mbuf_sz); if (mp == NULL) goto update; } else - mp = rxbuf->m_pack; + mp = rxbuf->buf; - mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; + mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; /* If we're dealing with an mbuf that was copied rather * than replaced, there's no need to go through busdma. */ if ((rxbuf->flags & IXGBE_RX_COPY) == 0) { /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->ptag, - rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); + error = bus_dmamap_load_mbuf_sg(rxr->tag, + rxbuf->map, mp, seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { printf("Refresh mbufs: payload dmamap load" " failure - %d\n", error); m_free(mp); - rxbuf->m_pack = NULL; + rxbuf->buf = NULL; goto update; } - rxbuf->m_pack = mp; - bus_dmamap_sync(rxr->ptag, rxbuf->pmap, + rxbuf->buf = mp; + bus_dmamap_sync(rxr->tag, rxbuf->map, BUS_DMASYNC_PREREAD); - rxbuf->paddr = rxr->rx_base[i].read.pkt_addr = - htole64(pseg[0].ds_addr); + rxbuf->addr = rxr->rx_base[i].read.pkt_addr = + htole64(seg[0].ds_addr); } else { - rxr->rx_base[i].read.pkt_addr = rxbuf->paddr; + rxr->rx_base[i].read.pkt_addr = rxbuf->addr; rxbuf->flags &= ~IXGBE_RX_COPY; } @@ -3810,7 +3744,7 @@ no_split: /* Next is precalculated */ i = j; rxr->next_to_refresh = i; - if (++j == adapter->num_rx_desc) + if (++j == rxr->num_desc) j = 0; } update: @@ -3836,7 +3770,7 @@ ixgbe_allocate_receive_buffers(struct rx_ring *rxr) struct ixgbe_rx_buf *rxbuf; int i, bsize, error; - bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; + bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc; if (!(rxr->rx_buffers = (struct ixgbe_rx_buf *) malloc(bsize, M_DEVBUF, M_NOWAIT | M_ZERO))) { @@ -3850,45 +3784,23 @@ ixgbe_allocate_receive_buffers(struct rx_ring *rxr) BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ - MSIZE, /* maxsize */ - 1, /* nsegments */ - MSIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &rxr->htag))) { - device_printf(dev, "Unable to create RX DMA tag\n"); - goto fail; - } - - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ MJUM16BYTES, /* maxsize */ 1, /* nsegments */ MJUM16BYTES, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ - &rxr->ptag))) { + &rxr->tag))) { device_printf(dev, "Unable to create RX DMA tag\n"); goto fail; } - for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { + for (i = 0; i < rxr->num_desc; i++, rxbuf++) { rxbuf = &rxr->rx_buffers[i]; - error = bus_dmamap_create(rxr->htag, - BUS_DMA_NOWAIT, &rxbuf->hmap); - if (error) { - device_printf(dev, "Unable to create RX head map\n"); - goto fail; - } - error = bus_dmamap_create(rxr->ptag, - BUS_DMA_NOWAIT, &rxbuf->pmap); + error = bus_dmamap_create(rxr->tag, + BUS_DMA_NOWAIT, &rxbuf->map); if (error) { - device_printf(dev, "Unable to create RX pkt map\n"); + device_printf(dev, "Unable to create RX dma map\n"); goto fail; } } @@ -3952,11 +3864,11 @@ ixgbe_setup_hw_rsc(struct rx_ring *rxr) ** Limit the total number of descriptors that ** can be combined, so it does not exceed 64K */ - if (adapter->rx_mbuf_sz == MCLBYTES) + if (rxr->mbuf_sz == MCLBYTES) rscctrl |= IXGBE_RSCCTL_MAXDESC_16; - else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) + else if (rxr->mbuf_sz == MJUMPAGESIZE) rscctrl |= IXGBE_RSCCTL_MAXDESC_8; - else if (adapter->rx_mbuf_sz == MJUM9BYTES) + else if (rxr->mbuf_sz == MJUM9BYTES) rscctrl |= IXGBE_RSCCTL_MAXDESC_4; else /* Using 16K cluster */ rscctrl |= IXGBE_RSCCTL_MAXDESC_1; @@ -3979,29 +3891,19 @@ ixgbe_setup_hw_rsc(struct rx_ring *rxr) static void ixgbe_free_receive_ring(struct rx_ring *rxr) { - struct adapter *adapter; struct ixgbe_rx_buf *rxbuf; int i; - adapter = rxr->adapter; - for (i = 0; i < adapter->num_rx_desc; i++) { + for (i = 0; i < rxr->num_desc; i++) { rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->htag, rxbuf->hmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->htag, rxbuf->hmap); - rxbuf->m_head->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_head); - } - if (rxbuf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag, rxbuf->pmap, + if (rxbuf->buf != NULL) { + bus_dmamap_sync(rxr->tag, rxbuf->map, BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->ptag, rxbuf->pmap); - rxbuf->m_pack->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_pack); + bus_dmamap_unload(rxr->tag, rxbuf->map); + rxbuf->buf->m_flags |= M_PKTHDR; + m_freem(rxbuf->buf); + rxbuf->buf = NULL; } - rxbuf->m_head = NULL; - rxbuf->m_pack = NULL; } } @@ -4018,7 +3920,7 @@ ixgbe_setup_receive_ring(struct rx_ring *rxr) struct ifnet *ifp; device_t dev; struct ixgbe_rx_buf *rxbuf; - bus_dma_segment_t pseg[1], hseg[1]; + bus_dma_segment_t seg[1]; struct lro_ctrl *lro = &rxr->lro; int rsize, nsegs, error = 0; #ifdef DEV_NETMAP @@ -4039,17 +3941,15 @@ ixgbe_setup_receive_ring(struct rx_ring *rxr) rsize = roundup2(adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); bzero((void *)rxr->rx_base, rsize); + /* Cache the size */ + rxr->mbuf_sz = adapter->rx_mbuf_sz; /* Free current RX buffer structs and their mbufs */ ixgbe_free_receive_ring(rxr); - /* Configure header split? */ - if (ixgbe_header_split) - rxr->hdr_split = TRUE; - /* Now replenish the mbufs */ - for (int j = 0; j != adapter->num_rx_desc; ++j) { - struct mbuf *mh, *mp; + for (int j = 0; j != rxr->num_desc; ++j) { + struct mbuf *mp; rxbuf = &rxr->rx_buffers[j]; #ifdef DEV_NETMAP @@ -4066,60 +3966,30 @@ ixgbe_setup_receive_ring(struct rx_ring *rxr) void *addr; addr = PNMB(slot + sj, &paddr); - netmap_load_map(rxr->ptag, rxbuf->pmap, addr); + netmap_load_map(rxr->tag, rxbuf->map, addr); /* Update descriptor */ rxr->rx_base[j].read.pkt_addr = htole64(paddr); continue; } #endif /* DEV_NETMAP */ - /* - ** Don't allocate mbufs if not - ** doing header split, its wasteful - */ - if (rxr->hdr_split == FALSE) - goto skip_head; - - /* First the header */ - rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA); - if (rxbuf->m_head == NULL) { - error = ENOBUFS; - goto fail; - } - m_adj(rxbuf->m_head, ETHER_ALIGN); - mh = rxbuf->m_head; - mh->m_len = mh->m_pkthdr.len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->htag, - rxbuf->hmap, rxbuf->m_head, hseg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) /* Nothing elegant to do here */ - goto fail; - bus_dmamap_sync(rxr->htag, - rxbuf->hmap, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); - -skip_head: - /* Now the payload cluster */ - rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, + rxbuf->buf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); - if (rxbuf->m_pack == NULL) { + if (rxbuf->buf == NULL) { error = ENOBUFS; goto fail; } - mp = rxbuf->m_pack; - mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; + mp = rxbuf->buf; + mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->ptag, - rxbuf->pmap, mp, pseg, + error = bus_dmamap_load_mbuf_sg(rxr->tag, + rxbuf->map, mp, seg, &nsegs, BUS_DMA_NOWAIT); if (error != 0) goto fail; - bus_dmamap_sync(rxr->ptag, - rxbuf->pmap, BUS_DMASYNC_PREREAD); + bus_dmamap_sync(rxr->tag, + rxbuf->map, BUS_DMASYNC_PREREAD); /* Update descriptor */ - rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); + rxr->rx_base[j].read.pkt_addr = htole64(seg[0].ds_addr); } @@ -4127,7 +3997,6 @@ skip_head: rxr->next_to_check = 0; rxr->next_to_refresh = 0; rxr->lro_enabled = FALSE; - rxr->rx_split_packets = 0; rxr->rx_copies = 0; rxr->rx_bytes = 0; rxr->discard = FALSE; @@ -4258,19 +4127,15 @@ ixgbe_initialize_receive_units(struct adapter *adapter) srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; srrctl |= bufsz; - if (rxr->hdr_split) { - /* Use a standard mbuf for the header */ - srrctl |= ((IXGBE_RX_HDR << - IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) - & IXGBE_SRRCTL_BSIZEHDR_MASK); - srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; - } else - srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; + srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); /* Setup the HW Rx Head and Tail Descriptor Pointers */ IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); + + /* Set the processing limit */ + rxr->process_limit = ixgbe_rx_process_limit; } if (adapter->hw.mac.type != ixgbe_mac_82598EB) { @@ -4371,29 +4236,17 @@ ixgbe_free_receive_buffers(struct rx_ring *rxr) if (rxr->rx_buffers != NULL) { for (int i = 0; i < adapter->num_rx_desc; i++) { rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->htag, rxbuf->hmap, + if (rxbuf->buf != NULL) { + bus_dmamap_sync(rxr->tag, rxbuf->map, BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->htag, rxbuf->hmap); - rxbuf->m_head->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_head); - } - if (rxbuf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag, rxbuf->pmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->ptag, rxbuf->pmap); - rxbuf->m_pack->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_pack); - } - rxbuf->m_head = NULL; - rxbuf->m_pack = NULL; - if (rxbuf->hmap != NULL) { - bus_dmamap_destroy(rxr->htag, rxbuf->hmap); - rxbuf->hmap = NULL; + bus_dmamap_unload(rxr->tag, rxbuf->map); + rxbuf->buf->m_flags |= M_PKTHDR; + m_freem(rxbuf->buf); } - if (rxbuf->pmap != NULL) { - bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); - rxbuf->pmap = NULL; + rxbuf->buf = NULL; + if (rxbuf->map != NULL) { + bus_dmamap_destroy(rxr->tag, rxbuf->map); + rxbuf->map = NULL; } } if (rxr->rx_buffers != NULL) { @@ -4402,13 +4255,9 @@ ixgbe_free_receive_buffers(struct rx_ring *rxr) } } - if (rxr->htag != NULL) { - bus_dma_tag_destroy(rxr->htag); - rxr->htag = NULL; - } - if (rxr->ptag != NULL) { - bus_dma_tag_destroy(rxr->ptag); - rxr->ptag = NULL; + if (rxr->tag != NULL) { + bus_dma_tag_destroy(rxr->tag); + rxr->tag = NULL; } return; @@ -4467,16 +4316,11 @@ ixgbe_rx_discard(struct rx_ring *rxr, int i) ** the normal refresh path to get new buffers ** and mapping. */ - if (rbuf->m_head) { - m_free(rbuf->m_head); - rbuf->m_head = NULL; + if (rbuf->buf) { + m_free(rbuf->buf); + rbuf->buf = NULL; } - if (rbuf->m_pack) { - m_free(rbuf->m_pack); - rbuf->m_pack = NULL; - } - return; } @@ -4493,7 +4337,7 @@ ixgbe_rx_discard(struct rx_ring *rxr, int i) * Return TRUE for more work, FALSE for all clean. *********************************************************************/ static bool -ixgbe_rxeof(struct ix_queue *que, int count) +ixgbe_rxeof(struct ix_queue *que) { struct adapter *adapter = que->adapter; struct rx_ring *rxr = que->rxr; @@ -4502,6 +4346,7 @@ ixgbe_rxeof(struct ix_queue *que, int count) struct lro_entry *queued; int i, nextp, processed = 0; u32 staterr = 0; + u16 count = rxr->process_limit; union ixgbe_adv_rx_desc *cur; struct ixgbe_rx_buf *rbuf, *nbuf; @@ -4527,9 +4372,9 @@ ixgbe_rxeof(struct ix_queue *que, int count) } #endif /* DEV_NETMAP */ for (i = rxr->next_to_check; count != 0;) { - struct mbuf *sendmp, *mh, *mp; + struct mbuf *sendmp, *mp; u32 rsc, ptype; - u16 hlen, plen, hdr; + u16 len; u16 vtag = 0; bool eop; @@ -4551,19 +4396,13 @@ ixgbe_rxeof(struct ix_queue *que, int count) rsc = 0; cur->wb.upper.status_error = 0; rbuf = &rxr->rx_buffers[i]; - mh = rbuf->m_head; - mp = rbuf->m_pack; + mp = rbuf->buf; - plen = le16toh(cur->wb.upper.length); + len = le16toh(cur->wb.upper.length); ptype = le32toh(cur->wb.lower.lo_dword.data) & IXGBE_RXDADV_PKTTYPE_MASK; - hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); - /* Process vlan info */ - if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP)) - vtag = le16toh(cur->wb.upper.vlan); - /* Make sure bad packets are discarded */ if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || (rxr->discard)) { @@ -4611,130 +4450,71 @@ ixgbe_rxeof(struct ix_queue *que, int count) prefetch(nbuf); } /* - ** The header mbuf is ONLY used when header - ** split is enabled, otherwise we get normal - ** behavior, ie, both header and payload - ** are DMA'd into the payload buffer. - ** ** Rather than using the fmp/lmp global pointers ** we now keep the head of a packet chain in the ** buffer struct and pass this along from one ** descriptor to the next, until we get EOP. */ - if (rxr->hdr_split && (rbuf->fmp == NULL)) { - /* This must be an initial descriptor */ - hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> - IXGBE_RXDADV_HDRBUFLEN_SHIFT; - if (hlen > IXGBE_RX_HDR) - hlen = IXGBE_RX_HDR; - mh->m_len = hlen; - mh->m_flags |= M_PKTHDR; - mh->m_next = NULL; - mh->m_pkthdr.len = mh->m_len; - /* Null buf pointer so it is refreshed */ - rbuf->m_head = NULL; - /* - ** Check the payload length, this - ** could be zero if its a small - ** packet. - */ - if (plen > 0) { - mp->m_len = plen; - mp->m_next = NULL; - mp->m_flags &= ~M_PKTHDR; - mh->m_next = mp; - mh->m_pkthdr.len += mp->m_len; - /* Null buf pointer so it is refreshed */ - rbuf->m_pack = NULL; - rxr->rx_split_packets++; - } - /* - ** Now create the forward - ** chain so when complete - ** we wont have to. - */ - if (eop == 0) { - /* stash the chain head */ - nbuf->fmp = mh; - /* Make forward chain */ - if (plen) - mp->m_next = nbuf->m_pack; - else - mh->m_next = nbuf->m_pack; - } else { - /* Singlet, prepare to send */ - sendmp = mh; - /* If hardware handled vtag */ - if (vtag) { - sendmp->m_pkthdr.ether_vtag = vtag; - sendmp->m_flags |= M_VLANTAG; - } - } + mp->m_len = len; + /* + ** See if there is a stored head + ** that determines what we are + */ + sendmp = rbuf->fmp; + if (sendmp != NULL) { /* secondary frag */ + rbuf->buf = rbuf->fmp = NULL; + mp->m_flags &= ~M_PKTHDR; + sendmp->m_pkthdr.len += mp->m_len; } else { /* - ** Either no header split, or a - ** secondary piece of a fragmented - ** split packet. - */ - mp->m_len = plen; - /* - ** See if there is a stored head - ** that determines what we are - */ - sendmp = rbuf->fmp; - - if (sendmp != NULL) { /* secondary frag */ - rbuf->m_pack = rbuf->fmp = NULL; - mp->m_flags &= ~M_PKTHDR; - sendmp->m_pkthdr.len += mp->m_len; - } else { - /* - * Optimize. This might be a small packet, - * maybe just a TCP ACK. Do a fast copy that - * is cache aligned into a new mbuf, and - * leave the old mbuf+cluster for re-use. - */ - if (eop && plen <= IXGBE_RX_COPY_LEN) { - sendmp = m_gethdr(M_DONTWAIT, MT_DATA); - if (sendmp != NULL) { - sendmp->m_data += - IXGBE_RX_COPY_ALIGN; - ixgbe_bcopy(mp->m_data, - sendmp->m_data, plen); - sendmp->m_len = plen; - rxr->rx_copies++; - rbuf->flags |= IXGBE_RX_COPY; - } - } - if (sendmp == NULL) { - rbuf->m_pack = rbuf->fmp = NULL; - sendmp = mp; + * Optimize. This might be a small packet, + * maybe just a TCP ACK. Do a fast copy that + * is cache aligned into a new mbuf, and + * leave the old mbuf+cluster for re-use. + */ + if (eop && len <= IXGBE_RX_COPY_LEN) { + sendmp = m_gethdr(M_NOWAIT, MT_DATA); + if (sendmp != NULL) { + sendmp->m_data += + IXGBE_RX_COPY_ALIGN; + ixgbe_bcopy(mp->m_data, + sendmp->m_data, len); + sendmp->m_len = len; + rxr->rx_copies++; + rbuf->flags |= IXGBE_RX_COPY; } - - /* first desc of a non-ps chain */ - sendmp->m_flags |= M_PKTHDR; - sendmp->m_pkthdr.len = mp->m_len; - if (vtag) { - sendmp->m_pkthdr.ether_vtag = vtag; - sendmp->m_flags |= M_VLANTAG; - } - } - /* Pass the head pointer on */ - if (eop == 0) { - nbuf->fmp = sendmp; - sendmp = NULL; - mp->m_next = nbuf->m_pack; } + if (sendmp == NULL) { + rbuf->buf = rbuf->fmp = NULL; + sendmp = mp; + } + + /* first desc of a non-ps chain */ + sendmp->m_flags |= M_PKTHDR; + sendmp->m_pkthdr.len = mp->m_len; } ++processed; - /* Sending this frame? */ - if (eop) { + + /* Pass the head pointer on */ + if (eop == 0) { + nbuf->fmp = sendmp; + sendmp = NULL; + mp->m_next = nbuf->buf; + } else { /* Sending this frame */ sendmp->m_pkthdr.rcvif = ifp; ifp->if_ipackets++; rxr->rx_packets++; /* capture data for AIM */ rxr->bytes += sendmp->m_pkthdr.len; rxr->rx_bytes += sendmp->m_pkthdr.len; + /* Process vlan info */ + if ((rxr->vtag_strip) && + (staterr & IXGBE_RXD_STAT_VP)) + vtag = le16toh(cur->wb.upper.vlan); + if (vtag) { + sendmp->m_pkthdr.ether_vtag = vtag; + sendmp->m_flags |= M_VLANTAG; + } if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) ixgbe_rx_checksum(staterr, sendmp, ptype); #if __FreeBSD_version >= 800000 @@ -4747,7 +4527,7 @@ next_desc: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* Advance our pointers to the next descriptor. */ - if (++i == adapter->num_rx_desc) + if (++i == rxr->num_desc) i = 0; /* Now send to the stack or do LRO */ @@ -5493,15 +5273,9 @@ ixgbe_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed", CTLFLAG_RD, &adapter->mbuf_defrag_failed, "m_defrag() failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup", - CTLFLAG_RD, &adapter->no_tx_dma_setup, - "Driver tx dma failure in xmit"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", CTLFLAG_RD, &adapter->watchdog_events, "Watchdog timeouts"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx", - CTLFLAG_RD, &adapter->tso_tx, - "TSO"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", CTLFLAG_RD, &adapter->link_irq, "Link MSIX IRQ Handled"); @@ -5528,6 +5302,12 @@ ixgbe_add_hw_stats(struct adapter *adapter) CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr), ixgbe_sysctl_tdt_handler, "IU", "Transmit Descriptor Tail"); + SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx", + CTLFLAG_RD, &txr->tso_tx, + "TSO"); + SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup", + CTLFLAG_RD, &txr->no_tx_dma_setup, + "Driver tx dma failure in xmit"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", CTLFLAG_RD, &txr->no_desc_avail, "Queue No Descriptor Available"); @@ -5601,16 +5381,18 @@ ixgbe_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs", CTLFLAG_RD, &stats->rlec, "Receive Length Errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd", + + /* Flow Control stats */ + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", CTLFLAG_RD, &stats->lxontxc, "Link XON Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd", + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", CTLFLAG_RD, &stats->lxonrxc, "Link XON Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd", + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", CTLFLAG_RD, &stats->lxofftxc, "Link XOFF Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd", + SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", CTLFLAG_RD, &stats->lxoffrxc, "Link XOFF Received"); @@ -5710,29 +5492,6 @@ ixgbe_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", CTLFLAG_RD, &stats->ptc1522, "1024-1522 byte frames transmitted"); - - /* FC Stats */ - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc", - CTLFLAG_RD, &stats->fccrc, - "FC CRC Errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last", - CTLFLAG_RD, &stats->fclast, - "FC Last Error"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd", - CTLFLAG_RD, &stats->fcoerpdc, - "FCoE Packets Dropped"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd", - CTLFLAG_RD, &stats->fcoeprc, - "FCoE Packets Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd", - CTLFLAG_RD, &stats->fcoeptc, - "FCoE Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd", - CTLFLAG_RD, &stats->fcoedwrc, - "FCoE DWords Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd", - CTLFLAG_RD, &stats->fcoedwtc, - "FCoE DWords Transmitted"); } /* @@ -5778,16 +5537,6 @@ ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS) return error; } -static void -ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, - const char *description, int *limit, int value) -{ - *limit = value; - SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); -} - /* ** Control link advertise speed: ** 1 - advertise only 1G |
