Add the necessary support to use both TX queues available on if_emac.

Each TX queue can hold one packet (yes, if_emac can send only two(!)
packets at a time).

Even with this change the very limited FIFO buffer (3 KiB for TX and 13 KiB
for RX) fill up too quick to sustain higher throughput.

For the TCP case it turns out that TX isn't the limiting factor, but the RX
side is (the FIFO fill up and starts to discard packets, so the sender has
to slow down).

2 files changed, 29 insertions, 12 deletions

diff --git a/sys/arm/allwinner/if_emac.c b/sys/arm/allwinner/if_emac.c
index ec3c323..af09ec5 100644
--- a/sys/arm/allwinner/if_emac.c
+++ b/sys/arm/allwinner/if_emac.c
@@ -101,6 +101,7 @@ struct emac_softc {
 	int			emac_watchdog_timer;
 	int			emac_rx_process_limit;
 	int			emac_link;
+	uint32_t		emac_fifo_mask;
 };
 
 static int	emac_probe(device_t);
@@ -121,7 +122,7 @@ static void	emac_intr(void *);
 static int	emac_ioctl(struct ifnet *, u_long, caddr_t);
 
 static void	emac_rxeof(struct emac_softc *, int);
-static void	emac_txeof(struct emac_softc *);
+static void	emac_txeof(struct emac_softc *, uint32_t);
 
 static int	emac_miibus_readreg(device_t, int, int);
 static int	emac_miibus_writereg(device_t, int, int, int);
@@ -253,14 +254,19 @@ emac_reset(struct emac_softc *sc)
 }
 
 static void
-emac_txeof(struct emac_softc *sc)
+emac_txeof(struct emac_softc *sc, uint32_t status)
 {
 	struct ifnet *ifp;
 
 	EMAC_ASSERT_LOCKED(sc);
 
 	ifp = sc->emac_ifp;
-	if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
+	status &= (EMAC_TX_FIFO0 | EMAC_TX_FIFO1);
+	sc->emac_fifo_mask &= ~status;
+	if (status == (EMAC_TX_FIFO0 | EMAC_TX_FIFO1))
+		if_inc_counter(ifp, IFCOUNTER_OPACKETS, 2);
+	else
+		if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
 	/* Unarm watchdog timer if no TX */
@@ -580,11 +586,13 @@ emac_start_locked(struct ifnet *ifp)
 {
 	struct emac_softc *sc;
 	struct mbuf *m, *m0;
-	uint32_t reg_val;
+	uint32_t fifo, reg;
 
 	sc = ifp->if_softc;
 	if (ifp->if_drv_flags & IFF_DRV_OACTIVE)
 		return;
+	if (sc->emac_fifo_mask == (EMAC_TX_FIFO0 | EMAC_TX_FIFO1))
+		return;
 	if (sc->emac_link == 0)
 		return;
 	IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
@@ -592,7 +600,14 @@ emac_start_locked(struct ifnet *ifp)
 		return;
 
 	/* Select channel */
-	EMAC_WRITE_REG(sc, EMAC_TX_INS, 0);
+	if (sc->emac_fifo_mask & EMAC_TX_FIFO0)
+		fifo = 1;
+	else
+		fifo = 0;
+	sc->emac_fifo_mask |= (1 << fifo);
+	if (sc->emac_fifo_mask == (EMAC_TX_FIFO0 | EMAC_TX_FIFO1))
+		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+	EMAC_WRITE_REG(sc, EMAC_TX_INS, fifo);
 
 	/*
 	 * Emac controller wants 4 byte aligned TX buffers.
@@ -613,17 +628,17 @@ emac_start_locked(struct ifnet *ifp)
 	    roundup2(m->m_len, 4) / 4);
 
 	/* Send the data lengh. */
-	EMAC_WRITE_REG(sc, EMAC_TX_PL0, m->m_len);
+	reg = (fifo == 0) ? EMAC_TX_PL0 : EMAC_TX_PL1;
+	EMAC_WRITE_REG(sc, reg, m->m_len);
 
 	/* Start translate from fifo to phy. */
-	reg_val = EMAC_READ_REG(sc, EMAC_TX_CTL0);
-	reg_val |= 1;
-	EMAC_WRITE_REG(sc, EMAC_TX_CTL0, reg_val);
+	reg = (fifo == 0) ? EMAC_TX_CTL0 : EMAC_TX_CTL1;
+	EMAC_WRITE_REG(sc, reg, EMAC_READ_REG(sc, reg) | 1);
 
 	/* Set timeout */
 	sc->emac_watchdog_timer = 5;
 
-	ifp->if_drv_flags |= IFF_DRV_OACTIVE;
+	/* Data have been sent to hardware, it is okay to free the mbuf now. */
 	BPF_MTAP(ifp, m);
 	m_freem(m);
 }
@@ -676,7 +691,7 @@ emac_intr(void *arg)
 
 	/* Transmit Interrupt check */
 	if (reg_val & EMAC_INT_STA_TX) {
-		emac_txeof(sc);
+		emac_txeof(sc, reg_val);
 		ifp = sc->emac_ifp;
 		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 			emac_start_locked(ifp);
diff --git a/sys/arm/allwinner/if_emacreg.h b/sys/arm/allwinner/if_emacreg.h
index ee79c30..d276c5d 100644
--- a/sys/arm/allwinner/if_emacreg.h
+++ b/sys/arm/allwinner/if_emacreg.h
@@ -51,6 +51,8 @@
 #define	EMAC_TX_TSVH0		0x30
 #define	EMAC_TX_TSVL1		0x34
 #define	EMAC_TX_TSVH1		0x38
+#define	EMAC_TX_FIFO0		(1 << 0)
+#define	EMAC_TX_FIFO1		(1 << 1)
 
 #define	EMAC_RX_CTL		0x3C
 #define	EMAC_RX_HASH0		0x40
@@ -61,7 +63,7 @@
 
 #define	EMAC_INT_CTL		0x54
 #define	EMAC_INT_STA		0x58
-#define	EMAC_INT_STA_TX		(0x01 | 0x02)
+#define	EMAC_INT_STA_TX		(EMAC_TX_FIFO0 | EMAC_TX_FIFO1)
 #define	EMAC_INT_STA_RX		0x100
 #define	EMAC_INT_EN		(0xf << 0) | (1 << 8)