diff options
Diffstat (limited to 'sys/dev/sfxge/sfxge_tx.c')
| -rw-r--r-- | sys/dev/sfxge/sfxge_tx.c | 238 |
1 files changed, 164 insertions, 74 deletions
diff --git a/sys/dev/sfxge/sfxge_tx.c b/sys/dev/sfxge/sfxge_tx.c index 780ed26..6ae5a07 100644 --- a/sys/dev/sfxge/sfxge_tx.c +++ b/sys/dev/sfxge/sfxge_tx.c @@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$"); #include <sys/socket.h> #include <sys/sysctl.h> #include <sys/syslog.h> +#include <sys/limits.h> #include <net/bpf.h> #include <net/ethernet.h> @@ -96,11 +97,11 @@ SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_put_max, CTLFLAG_RDTUN, "Maximum number of any packets in deferred packet put-list"); #define SFXGE_PARAM_TSO_FW_ASSISTED SFXGE_PARAM(tso_fw_assisted) -static int sfxge_tso_fw_assisted = 1; +static int sfxge_tso_fw_assisted = (SFXGE_FATSOV1 | SFXGE_FATSOV2); TUNABLE_INT(SFXGE_PARAM_TSO_FW_ASSISTED, &sfxge_tso_fw_assisted); SYSCTL_INT(_hw_sfxge, OID_AUTO, tso_fw_assisted, CTLFLAG_RDTUN, &sfxge_tso_fw_assisted, 0, - "Use FW-assisted TSO if supported by NIC firmware"); + "Bitmask of FW-assisted TSO allowed to use if supported by NIC firmware"); static const struct { @@ -850,6 +851,8 @@ struct sfxge_tso_state { unsigned out_len; /* Remaining length in current segment */ unsigned seqnum; /* Current sequence number */ unsigned packet_space; /* Remaining space in current packet */ + unsigned segs_space; /* Remaining number of DMA segments + for the packet (FATSOv2 only) */ /* Input position */ uint64_t dma_addr; /* DMA address of current position */ @@ -952,7 +955,7 @@ static void tso_start(struct sfxge_txq *txq, struct sfxge_tso_state *tso, struct tcphdr th_copy; #endif - tso->fw_assisted = txq->sc->tso_fw_assisted; + tso->fw_assisted = txq->tso_fw_assisted; tso->mbuf = mbuf; /* Find network protocol and header */ @@ -1059,6 +1062,8 @@ static void tso_fill_packet_with_fragment(struct sfxge_txq *txq, { efx_desc_t *desc; int n; + uint64_t dma_addr = tso->dma_addr; + boolean_t eop; if (tso->in_len == 0 || tso->packet_space == 0) return; @@ -1066,20 +1071,38 @@ static void tso_fill_packet_with_fragment(struct sfxge_txq *txq, KASSERT(tso->in_len > 0, ("TSO input length went negative")); KASSERT(tso->packet_space > 0, ("TSO packet space went negative")); - n = min(tso->in_len, tso->packet_space); + if (tso->fw_assisted & SFXGE_FATSOV2) { + n = tso->in_len; + tso->out_len -= n; + tso->seqnum += n; + tso->in_len = 0; + if (n < tso->packet_space) { + tso->packet_space -= n; + tso->segs_space--; + } else { + tso->packet_space = tso->seg_size - + (n - tso->packet_space) % tso->seg_size; + tso->segs_space = + EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1 - + (tso->packet_space != tso->seg_size); + } + } else { + n = min(tso->in_len, tso->packet_space); + tso->packet_space -= n; + tso->out_len -= n; + tso->dma_addr += n; + tso->in_len -= n; + } - tso->packet_space -= n; - tso->out_len -= n; - tso->in_len -= n; + /* + * It is OK to use binary OR below to avoid extra branching + * since all conditions may always be checked. + */ + eop = (tso->out_len == 0) | (tso->packet_space == 0) | + (tso->segs_space == 0); desc = &txq->pend_desc[txq->n_pend_desc++]; - efx_tx_qdesc_dma_create(txq->common, - tso->dma_addr, - n, - tso->out_len == 0 || tso->packet_space == 0, - desc); - - tso->dma_addr += n; + efx_tx_qdesc_dma_create(txq->common, dma_addr, n, eop, desc); } /* Callback from bus_dmamap_load() for long TSO headers. */ @@ -1112,28 +1135,47 @@ static int tso_start_new_packet(struct sfxge_txq *txq, int rc; if (tso->fw_assisted) { - uint8_t tcp_flags = tso->tcp_flags; - - if (tso->out_len > tso->seg_size) - tcp_flags &= ~(TH_FIN | TH_PUSH); - - /* TSO option descriptor */ - desc = &txq->pend_desc[txq->n_pend_desc++]; - efx_tx_qdesc_tso_create(txq->common, - tso->packet_id, - tso->seqnum, - tcp_flags, - desc++); - KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0")); - id = (id + 1) & txq->ptr_mask; + if (tso->fw_assisted & SFXGE_FATSOV2) { + /* Add 2 FATSOv2 option descriptors */ + desc = &txq->pend_desc[txq->n_pend_desc]; + efx_tx_qdesc_tso2_create(txq->common, + tso->packet_id, + tso->seqnum, + tso->seg_size, + desc, + EFX_TX_FATSOV2_OPT_NDESCS); + desc += EFX_TX_FATSOV2_OPT_NDESCS; + txq->n_pend_desc += EFX_TX_FATSOV2_OPT_NDESCS; + KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0")); + id = (id + EFX_TX_FATSOV2_OPT_NDESCS) & txq->ptr_mask; + + tso->segs_space = + EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1; + } else { + uint8_t tcp_flags = tso->tcp_flags; + + if (tso->out_len > tso->seg_size) + tcp_flags &= ~(TH_FIN | TH_PUSH); + + /* Add FATSOv1 option descriptor */ + desc = &txq->pend_desc[txq->n_pend_desc++]; + efx_tx_qdesc_tso_create(txq->common, + tso->packet_id, + tso->seqnum, + tcp_flags, + desc++); + KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0")); + id = (id + 1) & txq->ptr_mask; + + tso->seqnum += tso->seg_size; + tso->segs_space = UINT_MAX; + } /* Header DMA descriptor */ *desc = tso->header_desc; txq->n_pend_desc++; KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0")); id = (id + 1) & txq->ptr_mask; - - tso->seqnum += tso->seg_size; } else { /* Allocate a DMA-mapped header buffer. */ if (__predict_true(tso->header_len <= TSOH_STD_SIZE)) { @@ -1215,6 +1257,8 @@ static int tso_start_new_packet(struct sfxge_txq *txq, 0, desc); id = (id + 1) & txq->ptr_mask; + + tso->segs_space = UINT_MAX; } tso->packet_space = tso->seg_size; txq->tso_packets++; @@ -1264,15 +1308,19 @@ sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf, } /* End of packet? */ - if (tso.packet_space == 0) { + if ((tso.packet_space == 0) | (tso.segs_space == 0)) { + unsigned int n_fatso_opt_desc = + (tso.fw_assisted & SFXGE_FATSOV2) ? + EFX_TX_FATSOV2_OPT_NDESCS : + (tso.fw_assisted & SFXGE_FATSOV1) ? 1 : 0; + /* If the queue is now full due to tiny MSS, * or we can't create another header, discard * the remainder of the input mbuf but do not * roll back the work we have done. */ - if (txq->n_pend_desc + tso.fw_assisted + - 1 /* header */ + n_dma_seg > - txq->max_pkt_desc) { + if (txq->n_pend_desc + n_fatso_opt_desc + + 1 /* header */ + n_dma_seg > txq->max_pkt_desc) { txq->tso_pdrop_too_many++; break; } @@ -1407,12 +1455,67 @@ sfxge_tx_qstop(struct sfxge_softc *sc, unsigned int index) SFXGE_TXQ_UNLOCK(txq); } +/* + * Estimate maximum number of Tx descriptors required for TSO packet. + * With minimum MSS and maximum mbuf length we might need more (even + * than a ring-ful of descriptors), but this should not happen in + * practice except due to deliberate attack. In that case we will + * truncate the output at a packet boundary. + */ +static unsigned int +sfxge_tx_max_pkt_desc(const struct sfxge_softc *sc, enum sfxge_txq_type type, + unsigned int tso_fw_assisted) +{ + /* One descriptor for every input fragment */ + unsigned int max_descs = SFXGE_TX_MAPPING_MAX_SEG; + unsigned int sw_tso_max_descs; + unsigned int fa_tso_v1_max_descs = 0; + unsigned int fa_tso_v2_max_descs = 0; + + /* VLAN tagging Tx option descriptor may be required */ + if (efx_nic_cfg_get(sc->enp)->enc_hw_tx_insert_vlan_enabled) + max_descs++; + + if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM) { + /* + * Plus header and payload descriptor for each output segment. + * Minus one since header fragment is already counted. + * Even if FATSO is used, we should be ready to fallback + * to do it in the driver. + */ + sw_tso_max_descs = SFXGE_TSO_MAX_SEGS * 2 - 1; + + /* FW assisted TSOv1 requires one more descriptor per segment + * in comparison to SW TSO */ + if (tso_fw_assisted & SFXGE_FATSOV1) + fa_tso_v1_max_descs = + sw_tso_max_descs + SFXGE_TSO_MAX_SEGS; + + /* FW assisted TSOv2 requires 3 (2 FATSO plus header) extra + * descriptors per superframe limited by number of DMA fetches + * per packet. The first packet header is already counted. + */ + if (tso_fw_assisted & SFXGE_FATSOV2) { + fa_tso_v2_max_descs = + howmany(SFXGE_TX_MAPPING_MAX_SEG, + EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1) * + (EFX_TX_FATSOV2_OPT_NDESCS + 1) - 1; + } + + max_descs += MAX(sw_tso_max_descs, + MAX(fa_tso_v1_max_descs, fa_tso_v2_max_descs)); + } + + return (max_descs); +} + static int sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index) { struct sfxge_txq *txq; efsys_mem_t *esmp; uint16_t flags; + unsigned int tso_fw_assisted; struct sfxge_evq *evq; unsigned int desc_index; int rc; @@ -1434,6 +1537,7 @@ sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index) return (rc); /* Determine the kind of queue we are creating. */ + tso_fw_assisted = 0; switch (txq->type) { case SFXGE_TXQ_NON_CKSUM: flags = 0; @@ -1443,6 +1547,9 @@ sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index) break; case SFXGE_TXQ_IP_TCP_UDP_CKSUM: flags = EFX_TXQ_CKSUM_IPV4 | EFX_TXQ_CKSUM_TCPUDP; + tso_fw_assisted = sc->tso_fw_assisted; + if (tso_fw_assisted & SFXGE_FATSOV2) + flags |= EFX_TXQ_FATSOV2; break; default: KASSERT(0, ("Impossible TX queue")); @@ -1453,8 +1560,19 @@ sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index) /* Create the common code transmit queue. */ if ((rc = efx_tx_qcreate(sc->enp, index, txq->type, esmp, sc->txq_entries, txq->buf_base_id, flags, evq->common, - &txq->common, &desc_index)) != 0) - goto fail; + &txq->common, &desc_index)) != 0) { + /* Retry if no FATSOv2 resources, otherwise fail */ + if ((rc != ENOSPC) || (~flags & EFX_TXQ_FATSOV2)) + goto fail; + + /* Looks like all FATSOv2 contexts are used */ + flags &= ~EFX_TXQ_FATSOV2; + tso_fw_assisted &= ~SFXGE_FATSOV2; + if ((rc = efx_tx_qcreate(sc->enp, index, txq->type, esmp, + sc->txq_entries, txq->buf_base_id, flags, evq->common, + &txq->common, &desc_index)) != 0) + goto fail; + } /* Initialise queue descriptor indexes */ txq->added = txq->pending = txq->completed = txq->reaped = desc_index; @@ -1466,6 +1584,10 @@ sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index) txq->init_state = SFXGE_TXQ_STARTED; txq->flush_state = SFXGE_FLUSH_REQUIRED; + txq->tso_fw_assisted = tso_fw_assisted; + + txq->max_pkt_desc = sfxge_tx_max_pkt_desc(sc, txq->type, + tso_fw_assisted); SFXGE_TXQ_UNLOCK(txq); @@ -1574,38 +1696,6 @@ sfxge_tx_qfini(struct sfxge_softc *sc, unsigned int index) free(txq, M_SFXGE); } -/* - * Estimate maximum number of Tx descriptors required for TSO packet. - * With minimum MSS and maximum mbuf length we might need more (even - * than a ring-ful of descriptors), but this should not happen in - * practice except due to deliberate attack. In that case we will - * truncate the output at a packet boundary. - */ -static unsigned int -sfxge_tx_max_pkt_desc(const struct sfxge_softc *sc, enum sfxge_txq_type type) -{ - /* One descriptor for every input fragment */ - unsigned int max_descs = SFXGE_TX_MAPPING_MAX_SEG; - - /* VLAN tagging Tx option descriptor may be required */ - if (efx_nic_cfg_get(sc->enp)->enc_hw_tx_insert_vlan_enabled) - max_descs++; - - if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM) { - /* - * Plus header and payload descriptor for each output segment. - * Minus one since header fragment is already counted. - */ - max_descs += SFXGE_TSO_MAX_SEGS * 2 - 1; - - /* FW assisted TSO requires one more descriptor per segment */ - if (sc->tso_fw_assisted) - max_descs += SFXGE_TSO_MAX_SEGS; - } - - return (max_descs); -} - static int sfxge_tx_qinit(struct sfxge_softc *sc, unsigned int txq_index, enum sfxge_txq_type type, unsigned int evq_index) @@ -1735,8 +1825,6 @@ sfxge_tx_qinit(struct sfxge_softc *sc, unsigned int txq_index, txq->init_state = SFXGE_TXQ_INITIALIZED; txq->hw_vlan_tci = 0; - txq->max_pkt_desc = sfxge_tx_max_pkt_desc(sc, type); - return (0); fail_txq_stat_init: @@ -1852,10 +1940,12 @@ sfxge_tx_init(struct sfxge_softc *sc) sc->txq_count = SFXGE_TXQ_NTYPES - 1 + sc->intr.n_alloc; sc->tso_fw_assisted = sfxge_tso_fw_assisted; - if (sc->tso_fw_assisted) - sc->tso_fw_assisted = - (encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO) && - (encp->enc_fw_assisted_tso_enabled); + if ((~encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO) || + (!encp->enc_fw_assisted_tso_enabled)) + sc->tso_fw_assisted &= ~SFXGE_FATSOV1; + if ((~encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO_V2) || + (!encp->enc_fw_assisted_tso_v2_enabled)) + sc->tso_fw_assisted &= ~SFXGE_FATSOV2; sc->txqs_node = SYSCTL_ADD_NODE( device_get_sysctl_ctx(sc->dev), |
