MFC r294077

sfxge: support FATSOv2

Reviewed by:    gnn
Sponsored by:   Solarflare Communications, Inc.

4 files changed, 173 insertions, 77 deletions

diff --git a/share/man/man4/sfxge.4 b/share/man/man4/sfxge.4
index b924986..9057fa8 100644
--- a/share/man/man4/sfxge.4
+++ b/share/man/man4/sfxge.4
@@ -121,8 +121,10 @@ If a packet is dropped, the
 counter is incremented and the local sender receives ENOBUFS.
 The value must be greater than or equal to 0.
 .It Va hw.sfxge.tso_fw_assisted
-Enable/disable usage of FW-assisted TSO if supported by NIC firmware.
-Enabled by default.
+Bitmask to enable/disable usage of FW-assisted TSO version if supported
+by NIC firmware.
+FATSOv1 (bit 0) and FATSOv2 (bit 1) are supported.
+All enabled by default.
 .It Va hw.sfxge.N.max_rss_channels
 The maximum number of allocated RSS channels for the Nth adapter.
 If set to 0 or unset, the number of channels is determined by the number
diff --git a/sys/dev/sfxge/sfxge.h b/sys/dev/sfxge/sfxge.h
index bea1eba..9cf8f07 100644
--- a/sys/dev/sfxge/sfxge.h
+++ b/sys/dev/sfxge/sfxge.h
@@ -283,7 +283,10 @@ struct sfxge_softc {
 	unsigned int			rxq_count;
 	unsigned int			txq_count;
 
-	int				tso_fw_assisted;
+	unsigned int			tso_fw_assisted;
+#define	SFXGE_FATSOV1	(1 << 0)
+#define	SFXGE_FATSOV2	(1 << 1)
+
 #if EFSYS_OPT_MCDI_LOGGING
 	int				mcdi_logging;
 #endif
diff --git a/sys/dev/sfxge/sfxge_tx.c b/sys/dev/sfxge/sfxge_tx.c
index 780ed26..6ae5a07 100644
--- a/sys/dev/sfxge/sfxge_tx.c
+++ b/sys/dev/sfxge/sfxge_tx.c
@@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/socket.h>
 #include <sys/sysctl.h>
 #include <sys/syslog.h>
+#include <sys/limits.h>
 
 #include <net/bpf.h>
 #include <net/ethernet.h>
@@ -96,11 +97,11 @@ SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_put_max, CTLFLAG_RDTUN,
 	   "Maximum number of any packets in deferred packet put-list");
 
 #define	SFXGE_PARAM_TSO_FW_ASSISTED	SFXGE_PARAM(tso_fw_assisted)
-static int sfxge_tso_fw_assisted = 1;
+static int sfxge_tso_fw_assisted = (SFXGE_FATSOV1 | SFXGE_FATSOV2);
 TUNABLE_INT(SFXGE_PARAM_TSO_FW_ASSISTED, &sfxge_tso_fw_assisted);
 SYSCTL_INT(_hw_sfxge, OID_AUTO, tso_fw_assisted, CTLFLAG_RDTUN,
 	   &sfxge_tso_fw_assisted, 0,
-	   "Use FW-assisted TSO if supported by NIC firmware");
+	   "Bitmask of FW-assisted TSO allowed to use if supported by NIC firmware");
 
 
 static const struct {
@@ -850,6 +851,8 @@ struct sfxge_tso_state {
 	unsigned out_len;	/* Remaining length in current segment */
 	unsigned seqnum;	/* Current sequence number */
 	unsigned packet_space;	/* Remaining space in current packet */
+	unsigned segs_space;	/* Remaining number of DMA segments
+				   for the packet (FATSOv2 only) */
 
 	/* Input position */
 	uint64_t dma_addr;	/* DMA address of current position */
@@ -952,7 +955,7 @@ static void tso_start(struct sfxge_txq *txq, struct sfxge_tso_state *tso,
 	struct tcphdr th_copy;
 #endif
 
-	tso->fw_assisted = txq->sc->tso_fw_assisted;
+	tso->fw_assisted = txq->tso_fw_assisted;
 	tso->mbuf = mbuf;
 
 	/* Find network protocol and header */
@@ -1059,6 +1062,8 @@ static void tso_fill_packet_with_fragment(struct sfxge_txq *txq,
 {
 	efx_desc_t *desc;
 	int n;
+	uint64_t dma_addr = tso->dma_addr;
+	boolean_t eop;
 
 	if (tso->in_len == 0 || tso->packet_space == 0)
 		return;
@@ -1066,20 +1071,38 @@ static void tso_fill_packet_with_fragment(struct sfxge_txq *txq,
 	KASSERT(tso->in_len > 0, ("TSO input length went negative"));
 	KASSERT(tso->packet_space > 0, ("TSO packet space went negative"));
 
-	n = min(tso->in_len, tso->packet_space);
+	if (tso->fw_assisted & SFXGE_FATSOV2) {
+		n = tso->in_len;
+		tso->out_len -= n;
+		tso->seqnum += n;
+		tso->in_len = 0;
+		if (n < tso->packet_space) {
+			tso->packet_space -= n;
+			tso->segs_space--;
+		} else {
+			tso->packet_space = tso->seg_size -
+			    (n - tso->packet_space) % tso->seg_size;
+			tso->segs_space =
+			    EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1 -
+			    (tso->packet_space != tso->seg_size);
+		}
+	} else {
+		n = min(tso->in_len, tso->packet_space);
+		tso->packet_space -= n;
+		tso->out_len -= n;
+		tso->dma_addr += n;
+		tso->in_len -= n;
+	}
 
-	tso->packet_space -= n;
-	tso->out_len -= n;
-	tso->in_len -= n;
+	/*
+	 * It is OK to use binary OR below to avoid extra branching
+	 * since all conditions may always be checked.
+	 */
+	eop = (tso->out_len == 0) | (tso->packet_space == 0) |
+	    (tso->segs_space == 0);
 
 	desc = &txq->pend_desc[txq->n_pend_desc++];
-	efx_tx_qdesc_dma_create(txq->common,
-				tso->dma_addr,
-				n,
-				tso->out_len == 0 || tso->packet_space == 0,
-				desc);
-
-	tso->dma_addr += n;
+	efx_tx_qdesc_dma_create(txq->common, dma_addr, n, eop, desc);
 }
 
 /* Callback from bus_dmamap_load() for long TSO headers. */
@@ -1112,28 +1135,47 @@ static int tso_start_new_packet(struct sfxge_txq *txq,
 	int rc;
 
 	if (tso->fw_assisted) {
-		uint8_t tcp_flags = tso->tcp_flags;
-
-		if (tso->out_len > tso->seg_size)
-			tcp_flags &= ~(TH_FIN | TH_PUSH);
-
-		/* TSO option descriptor */
-		desc = &txq->pend_desc[txq->n_pend_desc++];
-		efx_tx_qdesc_tso_create(txq->common,
-					tso->packet_id,
-					tso->seqnum,
-					tcp_flags,
-					desc++);
-		KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
-		id = (id + 1) & txq->ptr_mask;
+		if (tso->fw_assisted & SFXGE_FATSOV2) {
+			/* Add 2 FATSOv2 option descriptors */
+			desc = &txq->pend_desc[txq->n_pend_desc];
+			efx_tx_qdesc_tso2_create(txq->common,
+						 tso->packet_id,
+						 tso->seqnum,
+						 tso->seg_size,
+						 desc,
+						 EFX_TX_FATSOV2_OPT_NDESCS);
+			desc += EFX_TX_FATSOV2_OPT_NDESCS;
+			txq->n_pend_desc += EFX_TX_FATSOV2_OPT_NDESCS;
+			KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
+			id = (id + EFX_TX_FATSOV2_OPT_NDESCS) & txq->ptr_mask;
+
+			tso->segs_space =
+			    EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1;
+		} else {
+			uint8_t tcp_flags = tso->tcp_flags;
+
+			if (tso->out_len > tso->seg_size)
+				tcp_flags &= ~(TH_FIN | TH_PUSH);
+
+			/* Add FATSOv1 option descriptor */
+			desc = &txq->pend_desc[txq->n_pend_desc++];
+			efx_tx_qdesc_tso_create(txq->common,
+						tso->packet_id,
+						tso->seqnum,
+						tcp_flags,
+						desc++);
+			KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
+			id = (id + 1) & txq->ptr_mask;
+
+			tso->seqnum += tso->seg_size;
+			tso->segs_space = UINT_MAX;
+		}
 
 		/* Header DMA descriptor */
 		*desc = tso->header_desc;
 		txq->n_pend_desc++;
 		KASSERT(txq->stmp[id].flags == 0, ("stmp flags are not 0"));
 		id = (id + 1) & txq->ptr_mask;
-
-		tso->seqnum += tso->seg_size;
 	} else {
 		/* Allocate a DMA-mapped header buffer. */
 		if (__predict_true(tso->header_len <= TSOH_STD_SIZE)) {
@@ -1215,6 +1257,8 @@ static int tso_start_new_packet(struct sfxge_txq *txq,
 					0,
 					desc);
 		id = (id + 1) & txq->ptr_mask;
+
+		tso->segs_space = UINT_MAX;
 	}
 	tso->packet_space = tso->seg_size;
 	txq->tso_packets++;
@@ -1264,15 +1308,19 @@ sfxge_tx_queue_tso(struct sfxge_txq *txq, struct mbuf *mbuf,
 		}
 
 		/* End of packet? */
-		if (tso.packet_space == 0) {
+		if ((tso.packet_space == 0) | (tso.segs_space == 0)) {
+			unsigned int n_fatso_opt_desc =
+			    (tso.fw_assisted & SFXGE_FATSOV2) ?
+			    EFX_TX_FATSOV2_OPT_NDESCS :
+			    (tso.fw_assisted & SFXGE_FATSOV1) ? 1 : 0;
+
 			/* If the queue is now full due to tiny MSS,
 			 * or we can't create another header, discard
 			 * the remainder of the input mbuf but do not
 			 * roll back the work we have done.
 			 */
-			if (txq->n_pend_desc + tso.fw_assisted +
-			    1 /* header */ + n_dma_seg >
-			    txq->max_pkt_desc) {
+			if (txq->n_pend_desc + n_fatso_opt_desc +
+			    1 /* header */ + n_dma_seg > txq->max_pkt_desc) {
 				txq->tso_pdrop_too_many++;
 				break;
 			}
@@ -1407,12 +1455,67 @@ sfxge_tx_qstop(struct sfxge_softc *sc, unsigned int index)
 	SFXGE_TXQ_UNLOCK(txq);
 }
 
+/*
+ * Estimate maximum number of Tx descriptors required for TSO packet.
+ * With minimum MSS and maximum mbuf length we might need more (even
+ * than a ring-ful of descriptors), but this should not happen in
+ * practice except due to deliberate attack.  In that case we will
+ * truncate the output at a packet boundary.
+ */
+static unsigned int
+sfxge_tx_max_pkt_desc(const struct sfxge_softc *sc, enum sfxge_txq_type type,
+		      unsigned int tso_fw_assisted)
+{
+	/* One descriptor for every input fragment */
+	unsigned int max_descs = SFXGE_TX_MAPPING_MAX_SEG;
+	unsigned int sw_tso_max_descs;
+	unsigned int fa_tso_v1_max_descs = 0;
+	unsigned int fa_tso_v2_max_descs = 0;
+
+	/* VLAN tagging Tx option descriptor may be required */
+	if (efx_nic_cfg_get(sc->enp)->enc_hw_tx_insert_vlan_enabled)
+		max_descs++;
+
+	if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM) {
+		/*
+		 * Plus header and payload descriptor for each output segment.
+		 * Minus one since header fragment is already counted.
+		 * Even if FATSO is used, we should be ready to fallback
+		 * to do it in the driver.
+		 */
+		sw_tso_max_descs = SFXGE_TSO_MAX_SEGS * 2 - 1;
+
+		/* FW assisted TSOv1 requires one more descriptor per segment
+		 * in comparison to SW TSO */
+		if (tso_fw_assisted & SFXGE_FATSOV1)
+			fa_tso_v1_max_descs =
+			    sw_tso_max_descs + SFXGE_TSO_MAX_SEGS;
+
+		/* FW assisted TSOv2 requires 3 (2 FATSO plus header) extra
+		 * descriptors per superframe limited by number of DMA fetches
+		 * per packet. The first packet header is already counted.
+		 */
+		if (tso_fw_assisted & SFXGE_FATSOV2) {
+			fa_tso_v2_max_descs =
+			    howmany(SFXGE_TX_MAPPING_MAX_SEG,
+				    EFX_TX_FATSOV2_DMA_SEGS_PER_PKT_MAX - 1) *
+			    (EFX_TX_FATSOV2_OPT_NDESCS + 1) - 1;
+		}
+
+		max_descs += MAX(sw_tso_max_descs,
+				 MAX(fa_tso_v1_max_descs, fa_tso_v2_max_descs));
+	}
+
+	return (max_descs);
+}
+
 static int
 sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index)
 {
 	struct sfxge_txq *txq;
 	efsys_mem_t *esmp;
 	uint16_t flags;
+	unsigned int tso_fw_assisted;
 	struct sfxge_evq *evq;
 	unsigned int desc_index;
 	int rc;
@@ -1434,6 +1537,7 @@ sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index)
 		return (rc);
 
 	/* Determine the kind of queue we are creating. */
+	tso_fw_assisted = 0;
 	switch (txq->type) {
 	case SFXGE_TXQ_NON_CKSUM:
 		flags = 0;
@@ -1443,6 +1547,9 @@ sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index)
 		break;
 	case SFXGE_TXQ_IP_TCP_UDP_CKSUM:
 		flags = EFX_TXQ_CKSUM_IPV4 | EFX_TXQ_CKSUM_TCPUDP;
+		tso_fw_assisted = sc->tso_fw_assisted;
+		if (tso_fw_assisted & SFXGE_FATSOV2)
+			flags |= EFX_TXQ_FATSOV2;
 		break;
 	default:
 		KASSERT(0, ("Impossible TX queue"));
@@ -1453,8 +1560,19 @@ sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index)
 	/* Create the common code transmit queue. */
 	if ((rc = efx_tx_qcreate(sc->enp, index, txq->type, esmp,
 	    sc->txq_entries, txq->buf_base_id, flags, evq->common,
-	    &txq->common, &desc_index)) != 0)
-		goto fail;
+	    &txq->common, &desc_index)) != 0) {
+		/* Retry if no FATSOv2 resources, otherwise fail */
+		if ((rc != ENOSPC) || (~flags & EFX_TXQ_FATSOV2))
+			goto fail;
+
+		/* Looks like all FATSOv2 contexts are used */
+		flags &= ~EFX_TXQ_FATSOV2;
+		tso_fw_assisted &= ~SFXGE_FATSOV2;
+		if ((rc = efx_tx_qcreate(sc->enp, index, txq->type, esmp,
+		    sc->txq_entries, txq->buf_base_id, flags, evq->common,
+		    &txq->common, &desc_index)) != 0)
+			goto fail;
+	}
 
 	/* Initialise queue descriptor indexes */
 	txq->added = txq->pending = txq->completed = txq->reaped = desc_index;
@@ -1466,6 +1584,10 @@ sfxge_tx_qstart(struct sfxge_softc *sc, unsigned int index)
 
 	txq->init_state = SFXGE_TXQ_STARTED;
 	txq->flush_state = SFXGE_FLUSH_REQUIRED;
+	txq->tso_fw_assisted = tso_fw_assisted;
+
+	txq->max_pkt_desc = sfxge_tx_max_pkt_desc(sc, txq->type,
+						  tso_fw_assisted);
 
 	SFXGE_TXQ_UNLOCK(txq);
 
@@ -1574,38 +1696,6 @@ sfxge_tx_qfini(struct sfxge_softc *sc, unsigned int index)
 	free(txq, M_SFXGE);
 }
 
-/*
- * Estimate maximum number of Tx descriptors required for TSO packet.
- * With minimum MSS and maximum mbuf length we might need more (even
- * than a ring-ful of descriptors), but this should not happen in
- * practice except due to deliberate attack.  In that case we will
- * truncate the output at a packet boundary.
- */
-static unsigned int
-sfxge_tx_max_pkt_desc(const struct sfxge_softc *sc, enum sfxge_txq_type type)
-{
-	/* One descriptor for every input fragment */
-	unsigned int max_descs = SFXGE_TX_MAPPING_MAX_SEG;
-
-	/* VLAN tagging Tx option descriptor may be required */
-	if (efx_nic_cfg_get(sc->enp)->enc_hw_tx_insert_vlan_enabled)
-		max_descs++;
-
-	if (type == SFXGE_TXQ_IP_TCP_UDP_CKSUM) {
-		/*
-		 * Plus header and payload descriptor for each output segment.
-		 * Minus one since header fragment is already counted.
-		 */
-		max_descs += SFXGE_TSO_MAX_SEGS * 2 - 1;
-
-		/* FW assisted TSO requires one more descriptor per segment */
-		if (sc->tso_fw_assisted)
-			max_descs += SFXGE_TSO_MAX_SEGS;
-	}
-
-	return (max_descs);
-}
-
 static int
 sfxge_tx_qinit(struct sfxge_softc *sc, unsigned int txq_index,
 	       enum sfxge_txq_type type, unsigned int evq_index)
@@ -1735,8 +1825,6 @@ sfxge_tx_qinit(struct sfxge_softc *sc, unsigned int txq_index,
 	txq->init_state = SFXGE_TXQ_INITIALIZED;
 	txq->hw_vlan_tci = 0;
 
-	txq->max_pkt_desc = sfxge_tx_max_pkt_desc(sc, type);
-
 	return (0);
 
 fail_txq_stat_init:
@@ -1852,10 +1940,12 @@ sfxge_tx_init(struct sfxge_softc *sc)
 	sc->txq_count = SFXGE_TXQ_NTYPES - 1 + sc->intr.n_alloc;
 
 	sc->tso_fw_assisted = sfxge_tso_fw_assisted;
-	if (sc->tso_fw_assisted)
-		sc->tso_fw_assisted =
-		    (encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO) &&
-		    (encp->enc_fw_assisted_tso_enabled);
+	if ((~encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO) ||
+	    (!encp->enc_fw_assisted_tso_enabled))
+		sc->tso_fw_assisted &= ~SFXGE_FATSOV1;
+	if ((~encp->enc_features & EFX_FEATURE_FW_ASSISTED_TSO_V2) ||
+	    (!encp->enc_fw_assisted_tso_v2_enabled))
+		sc->tso_fw_assisted &= ~SFXGE_FATSOV2;
 
 	sc->txqs_node = SYSCTL_ADD_NODE(
 		device_get_sysctl_ctx(sc->dev),
diff --git a/sys/dev/sfxge/sfxge_tx.h b/sys/dev/sfxge/sfxge_tx.h
index 816856a..ce5d3c4 100644
--- a/sys/dev/sfxge/sfxge_tx.h
+++ b/sys/dev/sfxge/sfxge_tx.h
@@ -170,6 +170,7 @@ struct sfxge_txq {
 	struct sfxge_softc		*sc;
 	enum sfxge_txq_state		init_state;
 	enum sfxge_flush_state		flush_state;
+	unsigned int			tso_fw_assisted;
 	enum sfxge_txq_type		type;
 	unsigned int			txq_index;
 	unsigned int			evq_index;