1 files changed, 727 insertions, 141 deletions

diff --git a/sys/dev/sfxge/common/efx_tx.c b/sys/dev/sfxge/common/efx_tx.c
index 2b0f5e5..a2e8769 100644
--- a/sys/dev/sfxge/common/efx_tx.c
+++ b/sys/dev/sfxge/common/efx_tx.c
@@ -1,26 +1,31 @@
 /*-
- * Copyright 2007-2009 Solarflare Communications Inc.  All rights reserved.
+ * Copyright (c) 2007-2015 Solarflare Communications Inc.
+ * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
+ * modification, are permitted provided that the following conditions are met:
  *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and documentation are
+ * those of the authors and should not be interpreted as representing official
+ * policies, either expressed or implied, of the FreeBSD Project.
  */
 
 #include <sys/cdefs.h>
@@ -42,11 +47,166 @@ __FBSDID("$FreeBSD$");
 #define	EFX_TX_QSTAT_INCR(_etp, _stat)
 #endif
 
+#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
+
+static	__checkReturn	int
+falconsiena_tx_init(
+	__in		efx_nic_t *enp);
+
+static			void
+falconsiena_tx_fini(
+	__in		efx_nic_t *enp);
+
+static	__checkReturn	int
+falconsiena_tx_qcreate(
+	__in		efx_nic_t *enp,
+	__in		unsigned int index,
+	__in		unsigned int label,
+	__in		efsys_mem_t *esmp,
+	__in		size_t n,
+	__in		uint32_t id,
+	__in		uint16_t flags,
+	__in		efx_evq_t *eep,
+	__in		efx_txq_t *etp,
+	__out		unsigned int *addedp);
+
+static		void
+falconsiena_tx_qdestroy(
+	__in	efx_txq_t *etp);
+
+static	__checkReturn	int
+falconsiena_tx_qpost(
+	__in		efx_txq_t *etp,
+	__in_ecount(n)	efx_buffer_t *eb,
+	__in		unsigned int n,
+	__in		unsigned int completed,
+	__inout		unsigned int *addedp);
+
+static			void
+falconsiena_tx_qpush(
+	__in	efx_txq_t *etp,
+	__in	unsigned int added,
+	__in	unsigned int pushed);
+
+static	__checkReturn	int
+falconsiena_tx_qpace(
+	__in		efx_txq_t *etp,
+	__in		unsigned int ns);
+
+static	__checkReturn	int
+falconsiena_tx_qflush(
+	__in		efx_txq_t *etp);
+
+static			void
+falconsiena_tx_qenable(
+	__in	efx_txq_t *etp);
+
+	__checkReturn	int
+falconsiena_tx_qdesc_post(
+	__in		efx_txq_t *etp,
+	__in_ecount(n)	efx_desc_t *ed,
+	__in		unsigned int n,
+	__in		unsigned int completed,
+	__inout		unsigned int *addedp);
+
+	void
+falconsiena_tx_qdesc_dma_create(
+	__in	efx_txq_t *etp,
+	__in	efsys_dma_addr_t addr,
+	__in	size_t size,
+	__in	boolean_t eop,
+	__out	efx_desc_t *edp);
+
+#if EFSYS_OPT_QSTATS
+static			void
+falconsiena_tx_qstats_update(
+	__in				efx_txq_t *etp,
+	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat);
+#endif
+
+#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
+
+
+#if EFSYS_OPT_FALCON
+static efx_tx_ops_t	__efx_tx_falcon_ops = {
+	falconsiena_tx_init,			/* etxo_init */
+	falconsiena_tx_fini,			/* etxo_fini */
+	falconsiena_tx_qcreate,			/* etxo_qcreate */
+	falconsiena_tx_qdestroy,		/* etxo_qdestroy */
+	falconsiena_tx_qpost,			/* etxo_qpost */
+	falconsiena_tx_qpush,			/* etxo_qpush */
+	falconsiena_tx_qpace,			/* etxo_qpace */
+	falconsiena_tx_qflush,			/* etxo_qflush */
+	falconsiena_tx_qenable,			/* etxo_qenable */
+	NULL,					/* etxo_qpio_enable */
+	NULL,					/* etxo_qpio_disable */
+	NULL,					/* etxo_qpio_write */
+	NULL,					/* etxo_qpio_post */
+	falconsiena_tx_qdesc_post,		/* etxo_qdesc_post */
+	falconsiena_tx_qdesc_dma_create,	/* etxo_qdesc_dma_create */
+	NULL,					/* etxo_qdesc_tso_create */
+	NULL,					/* etxo_qdesc_vlantci_create */
+#if EFSYS_OPT_QSTATS
+	falconsiena_tx_qstats_update,		/* etxo_qstats_update */
+#endif
+};
+#endif /* EFSYS_OPT_FALCON */
+
+#if EFSYS_OPT_SIENA
+static efx_tx_ops_t	__efx_tx_siena_ops = {
+	falconsiena_tx_init,			/* etxo_init */
+	falconsiena_tx_fini,			/* etxo_fini */
+	falconsiena_tx_qcreate,			/* etxo_qcreate */
+	falconsiena_tx_qdestroy,		/* etxo_qdestroy */
+	falconsiena_tx_qpost,			/* etxo_qpost */
+	falconsiena_tx_qpush,			/* etxo_qpush */
+	falconsiena_tx_qpace,			/* etxo_qpace */
+	falconsiena_tx_qflush,			/* etxo_qflush */
+	falconsiena_tx_qenable,			/* etxo_qenable */
+	NULL,					/* etxo_qpio_enable */
+	NULL,					/* etxo_qpio_disable */
+	NULL,					/* etxo_qpio_write */
+	NULL,					/* etxo_qpio_post */
+	falconsiena_tx_qdesc_post,		/* etxo_qdesc_post */
+	falconsiena_tx_qdesc_dma_create,	/* etxo_qdesc_dma_create */
+	NULL,					/* etxo_qdesc_tso_create */
+	NULL,					/* etxo_qdesc_vlantci_create */
+#if EFSYS_OPT_QSTATS
+	falconsiena_tx_qstats_update,		/* etxo_qstats_update */
+#endif
+};
+#endif /* EFSYS_OPT_SIENA */
+
+#if EFSYS_OPT_HUNTINGTON
+static efx_tx_ops_t	__efx_tx_hunt_ops = {
+	hunt_tx_init,				/* etxo_init */
+	hunt_tx_fini,				/* etxo_fini */
+	hunt_tx_qcreate,			/* etxo_qcreate */
+	hunt_tx_qdestroy,			/* etxo_qdestroy */
+	hunt_tx_qpost,				/* etxo_qpost */
+	hunt_tx_qpush,				/* etxo_qpush */
+	hunt_tx_qpace,				/* etxo_qpace */
+	hunt_tx_qflush,				/* etxo_qflush */
+	hunt_tx_qenable,			/* etxo_qenable */
+	hunt_tx_qpio_enable,			/* etxo_qpio_enable */
+	hunt_tx_qpio_disable,			/* etxo_qpio_disable */
+	hunt_tx_qpio_write,			/* etxo_qpio_write */
+	hunt_tx_qpio_post,			/* etxo_qpio_post */
+	hunt_tx_qdesc_post,			/* etxo_qdesc_post */
+	hunt_tx_qdesc_dma_create,		/* etxo_qdesc_dma_create */
+	hunt_tx_qdesc_tso_create,		/* etxo_qdesc_tso_create */
+	hunt_tx_qdesc_vlantci_create,		/* etxo_qdesc_vlantci_create */
+#if EFSYS_OPT_QSTATS
+	hunt_tx_qstats_update,			/* etxo_qstats_update */
+#endif
+};
+#endif /* EFSYS_OPT_HUNTINGTON */
+
 	__checkReturn	int
 efx_tx_init(
 	__in		efx_nic_t *enp)
 {
-	efx_oword_t oword;
+	efx_tx_ops_t *etxop;
 	int rc;
 
 	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
@@ -62,8 +222,436 @@ efx_tx_init(
 		goto fail2;
 	}
 
+	switch (enp->en_family) {
+#if EFSYS_OPT_FALCON
+	case EFX_FAMILY_FALCON:
+		etxop = (efx_tx_ops_t *)&__efx_tx_falcon_ops;
+		break;
+#endif /* EFSYS_OPT_FALCON */
+
+#if EFSYS_OPT_SIENA
+	case EFX_FAMILY_SIENA:
+		etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops;
+		break;
+#endif /* EFSYS_OPT_SIENA */
+
+#if EFSYS_OPT_HUNTINGTON
+	case EFX_FAMILY_HUNTINGTON:
+		etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops;
+		break;
+#endif /* EFSYS_OPT_HUNTINGTON */
+
+	default:
+		EFSYS_ASSERT(0);
+		rc = ENOTSUP;
+		goto fail3;
+	}
+
 	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
 
+	if ((rc = etxop->etxo_init(enp)) != 0)
+		goto fail4;
+
+	enp->en_etxop = etxop;
+	enp->en_mod_flags |= EFX_MOD_TX;
+	return (0);
+
+fail4:
+	EFSYS_PROBE(fail4);
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+
+	enp->en_etxop = NULL;
+	enp->en_mod_flags &= ~EFX_MOD_TX;
+	return (rc);
+}
+
+			void
+efx_tx_fini(
+	__in	efx_nic_t *enp)
+{
+	efx_tx_ops_t *etxop = enp->en_etxop;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
+	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
+
+	etxop->etxo_fini(enp);
+
+	enp->en_etxop = NULL;
+	enp->en_mod_flags &= ~EFX_MOD_TX;
+}
+
+	__checkReturn	int
+efx_tx_qcreate(
+	__in		efx_nic_t *enp,
+	__in		unsigned int index,
+	__in		unsigned int label,
+	__in		efsys_mem_t *esmp,
+	__in		size_t n,
+	__in		uint32_t id,
+	__in		uint16_t flags,
+	__in		efx_evq_t *eep,
+	__deref_out	efx_txq_t **etpp,
+	__out		unsigned int *addedp)
+{
+	efx_tx_ops_t *etxop = enp->en_etxop;
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	efx_txq_t *etp;
+	int rc;
+
+	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
+	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
+
+	EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
+
+	/* Allocate an TXQ object */
+	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
+
+	if (etp == NULL) {
+		rc = ENOMEM;
+		goto fail1;
+	}
+
+	etp->et_magic = EFX_TXQ_MAGIC;
+	etp->et_enp = enp;
+	etp->et_index = index;
+	etp->et_mask = n - 1;
+	etp->et_esmp = esmp;
+
+	/* Initial descriptor index may be modified by etxo_qcreate */
+	*addedp = 0;
+
+	if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
+	    n, id, flags, eep, etp, addedp)) != 0)
+			goto fail2;
+
+	enp->en_tx_qcount++;
+	*etpp = etp;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+		void
+efx_tx_qdestroy(
+	__in	efx_txq_t *etp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	EFSYS_ASSERT(enp->en_tx_qcount != 0);
+	--enp->en_tx_qcount;
+
+	etxop->etxo_qdestroy(etp);
+
+	/* Free the TXQ object */
+	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
+}
+
+	__checkReturn	int
+efx_tx_qpost(
+	__in		efx_txq_t *etp,
+	__in_ecount(n)	efx_buffer_t *eb,
+	__in		unsigned int n,
+	__in		unsigned int completed,
+	__inout		unsigned int *addedp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+	int rc;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	if ((rc = etxop->etxo_qpost(etp, eb,
+	    n, completed, addedp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+			void
+efx_tx_qpush(
+	__in	efx_txq_t *etp,
+	__in	unsigned int added,
+	__in	unsigned int pushed)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	etxop->etxo_qpush(etp, added, pushed);
+}
+
+	__checkReturn	int
+efx_tx_qpace(
+	__in		efx_txq_t *etp,
+	__in		unsigned int ns)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+	int rc;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+	__checkReturn	int
+efx_tx_qflush(
+	__in	efx_txq_t *etp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+	int rc;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	if ((rc = etxop->etxo_qflush(etp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+			void
+efx_tx_qenable(
+	__in	efx_txq_t *etp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	etxop->etxo_qenable(etp);
+}
+
+	__checkReturn	int
+efx_tx_qpio_enable(
+	__in	efx_txq_t *etp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+	int rc;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
+		rc = ENOTSUP;
+		goto fail1;
+	}
+	if (etxop->etxo_qpio_enable == NULL) {
+		rc = ENOTSUP;
+		goto fail2;
+	}
+	if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
+		goto fail3;
+
+	return (0);
+
+fail3:
+	EFSYS_PROBE(fail3);
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+		void
+efx_tx_qpio_disable(
+	__in	efx_txq_t *etp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	if (etxop->etxo_qpio_disable != NULL)
+		etxop->etxo_qpio_disable(etp);
+}
+
+	__checkReturn	int
+efx_tx_qpio_write(
+	__in			efx_txq_t *etp,
+	__in_ecount(buf_length)	uint8_t *buffer,
+	__in			size_t buf_length,
+	__in			size_t pio_buf_offset)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+	int rc;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	if (etxop->etxo_qpio_write != NULL) {
+		if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
+						pio_buf_offset)) != 0)
+			goto fail1;
+		return (0);
+	}
+
+	return (ENOTSUP);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+	__checkReturn	int
+efx_tx_qpio_post(
+	__in			efx_txq_t *etp,
+	__in			size_t pkt_length,
+	__in			unsigned int completed,
+	__inout			unsigned int *addedp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+	int rc;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	if (etxop->etxo_qpio_post != NULL) {
+		if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
+						addedp)) != 0)
+			goto fail1;
+		return (0);
+	}
+
+	return (ENOTSUP);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+	__checkReturn	int
+efx_tx_qdesc_post(
+	__in		efx_txq_t *etp,
+	__in_ecount(n)	efx_desc_t *ed,
+	__in		unsigned int n,
+	__in		unsigned int completed,
+	__inout		unsigned int *addedp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+	int rc;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	if ((rc = etxop->etxo_qdesc_post(etp, ed,
+	    n, completed, addedp)) != 0)
+		goto fail1;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+	void
+efx_tx_qdesc_dma_create(
+	__in	efx_txq_t *etp,
+	__in	efsys_dma_addr_t addr,
+	__in	size_t size,
+	__in	boolean_t eop,
+	__out	efx_desc_t *edp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+	EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
+
+	etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
+}
+
+	void
+efx_tx_qdesc_tso_create(
+	__in	efx_txq_t *etp,
+	__in	uint16_t ipv4_id,
+	__in	uint32_t tcp_seq,
+	__in	uint8_t  tcp_flags,
+	__out	efx_desc_t *edp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+	EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
+
+	etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
+}
+
+	void
+efx_tx_qdesc_vlantci_create(
+	__in	efx_txq_t *etp,
+	__in	uint16_t tci,
+	__out	efx_desc_t *edp)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+	EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
+
+	etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
+}
+
+
+#if EFSYS_OPT_QSTATS
+			void
+efx_tx_qstats_update(
+	__in				efx_txq_t *etp,
+	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat)
+{
+	efx_nic_t *enp = etp->et_enp;
+	efx_tx_ops_t *etxop = enp->en_etxop;
+
+	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
+
+	etxop->etxo_qstats_update(etp, stat);
+}
+#endif
+
+
+#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
+
+static	__checkReturn	int
+falconsiena_tx_init(
+	__in		efx_nic_t *enp)
+{
+	efx_oword_t oword;
+
 	/*
 	 * Disable the timer-based TX DMA backoff and allow TX DMA to be
 	 * controlled by the RX FIFO fill level (although always allow a
@@ -93,44 +681,8 @@ efx_tx_init(
 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
 	EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
 
-	enp->en_mod_flags |= EFX_MOD_TX;
 	return (0);
-
-fail2:
-	EFSYS_PROBE(fail2);
-fail1:
-	EFSYS_PROBE1(fail1, int, rc);
-
-	return (rc);
-}
-
-#if EFSYS_OPT_FILTER
-extern	__checkReturn	int
-efx_tx_filter_insert(
-	__in		efx_txq_t *etp,
-	__inout		efx_filter_spec_t *spec)
-{
-	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
-	EFSYS_ASSERT3P(spec, !=, NULL);
-
-	spec->efs_dmaq_id = (uint16_t)etp->et_index;
-	return (efx_filter_insert_filter(etp->et_enp, spec, B_FALSE));
-}
-#endif
-
-#if EFSYS_OPT_FILTER
-extern	__checkReturn	int
-efx_tx_filter_remove(
-	__in		efx_txq_t *etp,
-	__inout		efx_filter_spec_t *spec)
-{
-	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
-	EFSYS_ASSERT3P(spec, !=, NULL);
-
-	spec->efs_dmaq_id = (uint16_t)etp->et_index;
-	return (efx_filter_remove_filter(etp->et_enp, spec));
 }
-#endif
 
 #define	EFX_TX_DESC(_etp, _addr, _size, _eop, _added)			\
 	do {								\
@@ -157,8 +709,8 @@ efx_tx_filter_remove(
 		_NOTE(CONSTANTCONDITION)				\
 	} while (B_FALSE)
 
-	__checkReturn	int
-efx_tx_qpost(
+static	__checkReturn	int
+falconsiena_tx_qpost(
 	__in		efx_txq_t *etp,
 	__in_ecount(n)	efx_buffer_t *eb,
 	__in		unsigned int n,
@@ -169,8 +721,6 @@ efx_tx_qpost(
 	unsigned int i;
 	int rc = ENOSPC;
 
-	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
-
 	if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
 		goto fail1;
 
@@ -197,21 +747,17 @@ fail1:
 	return (rc);
 }
 
-		void
-efx_tx_qpush(
+static		void
+falconsiena_tx_qpush(
 	__in	efx_txq_t *etp,
-	__in	unsigned int added)
+	__in	unsigned int added,
+	__in	unsigned int pushed)
 {
 	efx_nic_t *enp = etp->et_enp;
 	uint32_t wptr;
 	efx_dword_t dword;
 	efx_oword_t oword;
 
-	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
-
-	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
-	EFSYS_PIO_WRITE_BARRIER();
-
 	/* Push the populated descriptors out */
 	wptr = added & etp->et_mask;
 
@@ -220,6 +766,11 @@ efx_tx_qpush(
 	/* Only write the third DWORD */
 	EFX_POPULATE_DWORD_1(dword,
 	    EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
+
+	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
+	EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
+	    wptr, pushed & etp->et_mask);
+	EFSYS_PIO_WRITE_BARRIER();
 	EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
 			    etp->et_index, &dword, B_FALSE);
 }
@@ -227,8 +778,8 @@ efx_tx_qpush(
 #define	EFX_MAX_PACE_VALUE 20
 #define	EFX_TX_PACE_CLOCK_BASE	104
 
-	__checkReturn	int
-efx_tx_qpace(
+static	__checkReturn	int
+falconsiena_tx_qpace(
 	__in		efx_txq_t *etp,
 	__in		unsigned int ns)
 {
@@ -239,8 +790,6 @@ efx_tx_qpace(
 	unsigned int timer_period;
 	int rc;
 
-	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
-
 	if (ns == 0) {
 		pace_val = 0;
 	} else {
@@ -261,7 +810,8 @@ efx_tx_qpace(
 
 	/* Update the pacing table */
 	EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
-	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, &oword);
+	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
+	    &oword, B_TRUE);
 
 	return (0);
 
@@ -271,16 +821,14 @@ fail1:
 	return (rc);
 }
 
-		void
-efx_tx_qflush(
-	__in	efx_txq_t *etp)
+static	__checkReturn	int
+falconsiena_tx_qflush(
+	__in		efx_txq_t *etp)
 {
 	efx_nic_t *enp = etp->et_enp;
 	efx_oword_t oword;
 	uint32_t label;
 
-	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
-
 	efx_tx_qpace(etp, 0);
 
 	label = etp->et_index;
@@ -289,19 +837,19 @@ efx_tx_qflush(
 	EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
 	    FRF_AZ_TX_FLUSH_DESCQ, label);
 	EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
+
+	return (0);
 }
 
-		void
-efx_tx_qenable(
+static		void
+falconsiena_tx_qenable(
 	__in	efx_txq_t *etp)
 {
 	efx_nic_t *enp = etp->et_enp;
 	efx_oword_t oword;
 
-	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
-
 	EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
-			    etp->et_index, &oword);
+			    etp->et_index, &oword, B_TRUE);
 
 	EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
 	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
@@ -314,11 +862,11 @@ efx_tx_qenable(
 	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
 
 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
-			    etp->et_index, &oword);
+			    etp->et_index, &oword, B_TRUE);
 }
 
-	__checkReturn	int
-efx_tx_qcreate(
+static	__checkReturn	int
+falconsiena_tx_qcreate(
 	__in		efx_nic_t *enp,
 	__in		unsigned int index,
 	__in		unsigned int label,
@@ -327,23 +875,22 @@ efx_tx_qcreate(
 	__in		uint32_t id,
 	__in		uint16_t flags,
 	__in		efx_evq_t *eep,
-	__deref_out	efx_txq_t **etpp)
+	__in		efx_txq_t *etp,
+	__out		unsigned int *addedp)
 {
 	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
-	efx_txq_t *etp;
 	efx_oword_t oword;
 	uint32_t size;
 	int rc;
 
-	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
-	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
-
 	EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
 	    (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
 	EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
-	EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
 
-	if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) {
+	EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
+	EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
+
+	if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
 		rc = EINVAL;
 		goto fail1;
 	}
@@ -351,7 +898,8 @@ efx_tx_qcreate(
 		rc = EINVAL;
 		goto fail2;
 	}
-	for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS);
+	for (size = 0;
+	    (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
 	    size++)
 		if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
 			break;
@@ -360,20 +908,6 @@ efx_tx_qcreate(
 		goto fail3;
 	}
 
-	/* Allocate an TXQ object */
-	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
-
-	if (etp == NULL) {
-		rc = ENOMEM;
-		goto fail4;
-	}
-
-	etp->et_magic = EFX_TXQ_MAGIC;
-	etp->et_enp = enp;
-	etp->et_index = index;
-	etp->et_mask = n - 1;
-	etp->et_esmp = esmp;
-
 	/* Set up the new descriptor queue */
 	EFX_POPULATE_OWORD_6(oword,
 	    FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
@@ -390,14 +924,10 @@ efx_tx_qcreate(
 	    (flags & EFX_CKSUM_TCPUDP) ? 0 : 1);
 
 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
-	    etp->et_index, &oword);
+	    etp->et_index, &oword, B_TRUE);
 
-	enp->en_tx_qcount++;
-	*etpp = etp;
 	return (0);
 
-fail4:
-	EFSYS_PROBE(fail4);
 fail3:
 	EFSYS_PROBE(fail3);
 fail2:
@@ -408,16 +938,83 @@ fail1:
 	return (rc);
 }
 
+	__checkReturn	int
+falconsiena_tx_qdesc_post(
+	__in		efx_txq_t *etp,
+	__in_ecount(n)	efx_desc_t *ed,
+	__in		unsigned int n,
+	__in		unsigned int completed,
+	__inout		unsigned int *addedp)
+{
+	unsigned int added = *addedp;
+	unsigned int i;
+	int rc;
+
+	if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
+		rc = ENOSPC;
+		goto fail1;
+	}
+
+	for (i = 0; i < n; i++) {
+		efx_desc_t *edp = &ed[i];
+		unsigned int id;
+		size_t offset;
+
+		id = added++ & etp->et_mask;
+		offset = id * sizeof (efx_desc_t);
+
+		EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
+	}
+
+	EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
+		    unsigned int, added, unsigned int, n);
+
+	EFX_TX_QSTAT_INCR(etp, TX_POST);
+
+	*addedp = added;
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, int, rc);
+	return (rc);
+}
+
+	void
+falconsiena_tx_qdesc_dma_create(
+	__in	efx_txq_t *etp,
+	__in	efsys_dma_addr_t addr,
+	__in	size_t size,
+	__in	boolean_t eop,
+	__out	efx_desc_t *edp)
+{
+	/* Fragments must not span 4k boundaries. */
+	EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);
+
+	EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
+		    efsys_dma_addr_t, addr,
+		    size_t, size, boolean_t, eop);
+
+	EFX_POPULATE_QWORD_4(edp->ed_eq,
+			    FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
+			    FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
+			    FSF_AZ_TX_KER_BUF_ADDR_DW0,
+			    (uint32_t)(addr & 0xffffffff),
+			    FSF_AZ_TX_KER_BUF_ADDR_DW1,
+			    (uint32_t)(addr >> 32));
+}
+
+#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */
+
 #if EFSYS_OPT_QSTATS
 #if EFSYS_OPT_NAMES
-/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */
-static const char 	__cs * __cs __efx_tx_qstat_name[] = {
+/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */
+static const char 	*__efx_tx_qstat_name[] = {
 	"post",
-	"unaligned_split",
+	"post_pio",
 };
 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
 
-		const char __cs *
+		const char *
 efx_tx_qstat_name(
 	__in	efx_nic_t *enp,
 	__in	unsigned int id)
@@ -429,18 +1026,18 @@ efx_tx_qstat_name(
 	return (__efx_tx_qstat_name[id]);
 }
 #endif	/* EFSYS_OPT_NAMES */
-#endif	/* EFSYS_OPT_QSTATS */
+#endif /* EFSYS_OPT_QSTATS */
+
+#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA
 
 #if EFSYS_OPT_QSTATS
-					void
-efx_tx_qstats_update(
+static					void
+falconsiena_tx_qstats_update(
 	__in				efx_txq_t *etp,
 	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat)
 {
 	unsigned int id;
 
-	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
-
 	for (id = 0; id < TX_NQSTATS; id++) {
 		efsys_stat_t *essp = &stat[id];
 
@@ -450,36 +1047,25 @@ efx_tx_qstats_update(
 }
 #endif	/* EFSYS_OPT_QSTATS */
 
-		void
-efx_tx_qdestroy(
+static		void
+falconsiena_tx_qdestroy(
 	__in	efx_txq_t *etp)
 {
 	efx_nic_t *enp = etp->et_enp;
 	efx_oword_t oword;
 
-	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
-
-	EFSYS_ASSERT(enp->en_tx_qcount != 0);
-	--enp->en_tx_qcount;
-
 	/* Purge descriptor queue */
 	EFX_ZERO_OWORD(oword);
 
 	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
-			    etp->et_index, &oword);
-
-	/* Free the TXQ object */
-	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
+			    etp->et_index, &oword, B_TRUE);
 }
 
-		void
-efx_tx_fini(
+static		void
+falconsiena_tx_fini(
 	__in	efx_nic_t *enp)
 {
-	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
-	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
-	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
-	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
-
-	enp->en_mod_flags &= ~EFX_MOD_TX;
+	_NOTE(ARGUNUSED(enp))
 }
+
+#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */