1 files changed, 49 insertions, 190 deletions

diff --git a/sys/dev/cxgb/cxgb_multiq.c b/sys/dev/cxgb/cxgb_multiq.c
index bcae123..139104a 100644
--- a/sys/dev/cxgb/cxgb_multiq.c
+++ b/sys/dev/cxgb/cxgb_multiq.c
@@ -27,8 +27,6 @@ POSSIBILITY OF SUCH DAMAGE.
 
 ***************************************************************************/
 
-#define DEBUG_BUFRING
-
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
@@ -89,12 +87,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/mvec.h>
 
 extern int txq_fills;
-extern struct sysctl_oid_list sysctl__hw_cxgb_children;
-static int cxgb_pcpu_tx_coalesce = 0;
-TUNABLE_INT("hw.cxgb.tx_coalesce", &cxgb_pcpu_tx_coalesce);
-SYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_coalesce, CTLFLAG_RDTUN, &cxgb_pcpu_tx_coalesce, 0,
-    "coalesce small packets into a single work request");
+int multiq_tx_enable = 1;
+int coalesce_tx_enable = 0;
+int wakeup_tx_thread = 0;
 
+extern struct sysctl_oid_list sysctl__hw_cxgb_children;
 static int sleep_ticks = 1;
 TUNABLE_INT("hw.cxgb.sleep_ticks", &sleep_ticks);
 SYSCTL_UINT(_hw_cxgb, OID_AUTO, sleep_ticks, CTLFLAG_RDTUN, &sleep_ticks, 0,
@@ -106,12 +103,12 @@ SYSCTL_UINT(_hw_cxgb, OID_AUTO, txq_mr_size, CTLFLAG_RDTUN, &cxgb_txq_buf_ring_s
     "size of per-queue mbuf ring");
 
 
-static inline int32_t cxgb_pcpu_calc_cookie(struct ifnet *ifp, struct mbuf *immpkt);
 static void cxgb_pcpu_start_proc(void *arg);
+static int cxgb_tx(struct sge_qset *qs, uint32_t txmax);
+
 #ifdef IFNET_MULTIQUEUE
-static int cxgb_pcpu_cookie_to_qidx(struct port_info *, uint32_t cookie);
+static int cxgb_pcpu_cookie_to_qidx(struct port_info *pi, uint32_t cookie);
 #endif
-static int cxgb_tx(struct sge_qset *qs, uint32_t txmax);
 
 static inline int
 cxgb_pcpu_enqueue_packet_(struct sge_qset *qs, struct mbuf *m)
@@ -119,9 +116,6 @@ cxgb_pcpu_enqueue_packet_(struct sge_qset *qs, struct mbuf *m)
 	struct sge_txq *txq;
 	int err = 0;
 
-#ifndef IFNET_MULTIQUEUE
-	panic("not expecting enqueue without multiqueue");
-#endif	
 	KASSERT(m != NULL, ("null mbuf"));
 	KASSERT(m->m_type == MT_DATA, ("bad mbuf type %d", m->m_type));
 	if (qs->qs_flags & QS_EXITING) {
@@ -134,9 +128,9 @@ cxgb_pcpu_enqueue_packet_(struct sge_qset *qs, struct mbuf *m)
 		txq->txq_drops++;
 		m_freem(m);
 	}
-	if ((qs->txq[TXQ_ETH].flags & TXQ_TRANSMITTING) == 0)
+	if (wakeup_tx_thread && ((txq->flags & TXQ_TRANSMITTING) == 0))
 		wakeup(qs);
-
+	
 	return (err);
 }
 
@@ -149,29 +143,34 @@ cxgb_pcpu_enqueue_packet(struct ifnet *ifp, struct mbuf *m)
 #ifdef IFNET_MULTIQUEUE
 	int32_t calc_cookie;
 
-	calc_cookie = m->m_pkthdr.rss_hash;
+	calc_cookie = m->m_pkthdr.flowid;
 	qidx = cxgb_pcpu_cookie_to_qidx(pi, calc_cookie);
 #else
 	qidx = 0;
 #endif	    
 	qs = &pi->adapter->sge.qs[qidx];
-	err = cxgb_pcpu_enqueue_packet_(qs, m);
+	if (ALTQ_IS_ENABLED(&ifp->if_snd)) {
+		IFQ_ENQUEUE(qs->txq[0].txq_ifq, m, err);
+	} else {
+		err = cxgb_pcpu_enqueue_packet_(qs, m);
+	}
 	return (err);
 }
 
 static int
 cxgb_dequeue_packet(struct sge_txq *txq, struct mbuf **m_vec)
 {
-	struct mbuf *m;
+	struct mbuf *m, *m0;
 	struct sge_qset *qs;
 	int count, size, coalesced;
 	struct adapter *sc;
+
 #ifndef IFNET_MULTIQUEUE
 	struct port_info *pi = txq->port;
 
+	mtx_assert(&txq->lock, MA_OWNED);
 	if (txq->immpkt != NULL)
 		panic("immediate packet set");
-	mtx_assert(&txq->lock, MA_OWNED);
 
 	IFQ_DRV_DEQUEUE(&pi->ifp->if_snd, m);
 	if (m == NULL)
@@ -180,7 +179,16 @@ cxgb_dequeue_packet(struct sge_txq *txq, struct mbuf **m_vec)
 	m_vec[0] = m;
 	return (1);
 #endif
-
+	if (ALTQ_IS_ENABLED(txq->txq_ifq)) {
+		IFQ_DRV_DEQUEUE(txq->txq_ifq, m);
+		if (m == NULL)
+			return (0);
+	
+		m_vec[0] = m;
+		return (1);		
+	}
+	
+	mtx_assert(&txq->lock, MA_OWNED);
 	coalesced = count = size = 0;
 	qs = txq_to_qset(txq, TXQ_ETH);
 	if (qs->qs_flags & QS_EXITING)
@@ -199,9 +207,10 @@ cxgb_dequeue_packet(struct sge_txq *txq, struct mbuf **m_vec)
 		return (0);
 
 	count = 1;
+
 	m_vec[0] = m;
 	if (m->m_pkthdr.tso_segsz > 0 || m->m_pkthdr.len > TX_WR_SIZE_MAX ||
-	    m->m_next != NULL || (cxgb_pcpu_tx_coalesce == 0)) {
+	    m->m_next != NULL || (coalesce_tx_enable == 0)) {
 		return (count);
 	}
 
@@ -213,7 +222,11 @@ cxgb_dequeue_packet(struct sge_txq *txq, struct mbuf **m_vec)
 		    size + m->m_pkthdr.len > TX_WR_SIZE_MAX || m->m_next != NULL)
 			break;
 
-		buf_ring_dequeue_sc(txq->txq_mr);
+		m0 = buf_ring_dequeue_sc(txq->txq_mr);
+#ifdef DEBUG_BUFRING
+		if (m0 != m)
+			panic("peek and dequeue don't match");
+#endif		
 		size += m->m_pkthdr.len;
 		m_vec[count++] = m;
 
@@ -227,134 +240,6 @@ cxgb_dequeue_packet(struct sge_txq *txq, struct mbuf **m_vec)
 	return (count);
 }
 
-static int32_t
-cxgb_pcpu_get_cookie(struct ifnet *ifp, struct in6_addr *lip, uint16_t lport, struct in6_addr *rip, uint16_t rport, int ipv6)
-{
-	uint32_t base;
-	uint8_t buf[36];
-	int count;
-	int32_t cookie;
-
-	critical_enter();
-	/* 
-	 * Can definitely bypass bcopy XXX
-	 */
-	if (ipv6 == 0) {
-		count = 12;
-		bcopy(rip, &buf[0], 4);
-		bcopy(lip, &buf[4], 4);
-		bcopy(&rport, &buf[8], 2);
-		bcopy(&lport, &buf[10], 2);
-	} else {
-		count = 36;
-		bcopy(rip, &buf[0], 16);
-		bcopy(lip, &buf[16], 16);
-		bcopy(&rport, &buf[32], 2);
-		bcopy(&lport, &buf[34], 2);
-	}
-	
-	base = 0xffffffff;
-	base = update_crc32(base, buf, count);
-	base = sctp_csum_finalize(base);
-
-	/*
-	 * Indirection table is 128 bits
-	 * -> cookie indexes into indirection table which maps connection to queue
-	 * -> RSS map maps queue to CPU
-	 */
-	cookie = (base & (RSS_TABLE_SIZE-1));
-	critical_exit();
-	
-	return (cookie);
-}
-
-static int32_t
-cxgb_pcpu_calc_cookie(struct ifnet *ifp, struct mbuf *immpkt)
-{
-	struct in6_addr lip, rip;
-	uint16_t lport, rport;
-	struct ether_header *eh;
-	int32_t cookie;
-	struct ip *ip;
-	struct ip6_hdr *ip6;
-	struct tcphdr *th;
-	struct udphdr *uh;
-	struct sctphdr *sh;
-	uint8_t *next, proto;
-	int etype;
-
-	if (immpkt == NULL)
-		return -1;
-
-#if 1	
-	/*
-	 * XXX perf test
-	 */
-	return (0);
-#endif	
-	rport = lport = 0;
-	cookie = -1;
-	next = NULL;
-	eh = mtod(immpkt, struct ether_header *);
-	etype = ntohs(eh->ether_type);
-
-	switch (etype) {
-	case ETHERTYPE_IP:
-		ip = (struct ip *)(eh + 1);
-		next = (uint8_t *)(ip + 1);
-		bcopy(&ip->ip_src, &lip, 4);
-		bcopy(&ip->ip_dst, &rip, 4);
-		proto = ip->ip_p;
-		break;
-	case ETHERTYPE_IPV6:
-		ip6 = (struct ip6_hdr *)(eh + 1);
-		next = (uint8_t *)(ip6 + 1);
-		bcopy(&ip6->ip6_src, &lip, sizeof(struct in6_addr));
-		bcopy(&ip6->ip6_dst, &rip, sizeof(struct in6_addr));
-		if (ip6->ip6_nxt == IPPROTO_HOPOPTS) {
-			struct ip6_hbh *hbh;
-
-			hbh = (struct ip6_hbh *)(ip6 + 1);
-			proto = hbh->ip6h_nxt;
-		} else 
-			proto = ip6->ip6_nxt;
-		break;
-	case ETHERTYPE_ARP:
-	default:
-		/*
-		 * Default to queue zero
-		 */
-		proto = cookie = 0;
-	}
-	if (proto) {
-		switch (proto) {
-		case IPPROTO_TCP:
-			th = (struct tcphdr *)next;
-			lport = th->th_sport;
-			rport = th->th_dport;
-			break;
-		case IPPROTO_UDP:
-			uh = (struct udphdr *)next;
-			lport = uh->uh_sport;
-			rport = uh->uh_dport;
-			break;
-		case IPPROTO_SCTP:
-			sh = (struct sctphdr *)next;
-			lport = sh->src_port;
-			rport = sh->dest_port;
-			break;
-		default:
-			/* nothing to do */
-			break;
-		}
-	}
-	
-	if (cookie) 		
-		cookie = cxgb_pcpu_get_cookie(ifp, &lip, lport, &rip, rport, (etype == ETHERTYPE_IPV6));
-
-	return (cookie);
-}
-
 static void
 cxgb_pcpu_free(struct sge_qset *qs)
 {
@@ -377,10 +262,6 @@ cxgb_pcpu_reclaim_tx(struct sge_txq *txq)
 	int reclaimable;
 	struct sge_qset *qs = txq_to_qset(txq, TXQ_ETH);
 
-#ifdef notyet
-	KASSERT(qs->qs_cpuid == curcpu, ("cpu qset mismatch cpuid=%d curcpu=%d",
-			qs->qs_cpuid, curcpu));
-#endif
 	mtx_assert(&txq->lock, MA_OWNED);
 	
 	reclaimable = desc_reclaimable(txq);
@@ -426,7 +307,8 @@ cxgb_pcpu_start_(struct sge_qset *qs, struct mbuf *immpkt, int tx_flush)
 		initerr = ENXIO;
 	else if (immpkt) {
 
-		if (!buf_ring_empty(txq->txq_mr)) 
+		if (!buf_ring_empty(txq->txq_mr)
+		    || ALTQ_IS_ENABLED(&pi->ifp->if_snd)) 
 			initerr = cxgb_pcpu_enqueue_packet_(qs, immpkt);
 		else
 			txq->immpkt = immpkt;
@@ -491,7 +373,7 @@ cxgb_pcpu_start_(struct sge_qset *qs, struct mbuf *immpkt, int tx_flush)
 }
 
 int
-cxgb_pcpu_start(struct ifnet *ifp, struct mbuf *immpkt)
+cxgb_pcpu_transmit(struct ifnet *ifp, struct mbuf *immpkt)
 {
 	uint32_t cookie;
 	int err, qidx, locked, resid;
@@ -506,10 +388,10 @@ cxgb_pcpu_start(struct ifnet *ifp, struct mbuf *immpkt)
 	qidx = resid = err = cookie = locked = 0;
 
 #ifdef IFNET_MULTIQUEUE	
-	if (immpkt && (immpkt->m_pkthdr.rss_hash != 0)) {
-		cookie = immpkt->m_pkthdr.rss_hash;
+	if (immpkt && (immpkt->m_pkthdr.flowid != 0)) {
+		cookie = immpkt->m_pkthdr.flowid;
 		qidx = cxgb_pcpu_cookie_to_qidx(pi, cookie);
-		DPRINTF("hash=0x%x qidx=%d cpu=%d\n", immpkt->m_pkthdr.rss_hash, qidx, curcpu);
+		DPRINTF("hash=0x%x qidx=%d cpu=%d\n", immpkt->m_pkthdr.flowid, qidx, curcpu);
 		qs = &pi->adapter->sge.qs[qidx];
 	} else
 #endif		
@@ -519,7 +401,7 @@ cxgb_pcpu_start(struct ifnet *ifp, struct mbuf *immpkt)
 
 	if (((sc->tunq_coalesce == 0) ||
 		(buf_ring_count(txq->txq_mr) >= TX_WR_COUNT_MAX) ||
-		(cxgb_pcpu_tx_coalesce == 0)) && mtx_trylock(&txq->lock)) {
+		(coalesce_tx_enable == 0)) && mtx_trylock(&txq->lock)) {
 		if (cxgb_debug)
 			printf("doing immediate transmit\n");
 		
@@ -534,10 +416,6 @@ cxgb_pcpu_start(struct ifnet *ifp, struct mbuf *immpkt)
 			    sc->tunq_coalesce, buf_ring_count(txq->txq_mr), mtx_owned(&txq->lock));
 		err = cxgb_pcpu_enqueue_packet_(qs, immpkt);
 	}
-	
-	if (resid && (txq->flags & TXQ_TRANSMITTING) == 0)
-		wakeup(qs);
-
 	return ((err == ENOSPC) ? 0 : err);
 }
 
@@ -552,7 +430,7 @@ cxgb_start(struct ifnet *ifp)
 	if (IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 		return;
 
-	cxgb_pcpu_start(ifp, NULL);
+	cxgb_pcpu_transmit(ifp, NULL);
 }
 
 static void
@@ -562,11 +440,8 @@ cxgb_pcpu_start_proc(void *arg)
 	struct thread *td;
 	struct sge_txq *txq = &qs->txq[TXQ_ETH];
 	int idleticks, err = 0;
-#ifdef notyet	
-	struct adapter *sc = qs->port->adapter;
-#endif
-	td = curthread;
 
+	td = curthread;
 	sleep_ticks = max(hz/1000, 1);
 	qs->qs_flags |= QS_RUNNING;
 	thread_lock(td);
@@ -642,6 +517,9 @@ cxgb_pcpu_cookie_to_qidx(struct port_info *pi, uint32_t cookie)
 {
 	int qidx;
 	uint32_t tmp;
+
+	if (multiq_tx_enable == 0)
+		return (pi->first_qset);
 	
 	 /*
 	 * Will probably need to be changed for 4-port XXX
@@ -757,7 +635,7 @@ cxgb_tx(struct sge_qset *qs, uint32_t txmax)
 		check_pkt_coalesce(qs);
 		count = cxgb_dequeue_packet(txq, m_vec);
 		if (count == 0) {
-			err = ENOBUFS;
+			err = ENOSPC;
 			break;
 		}
 		ETHER_BPF_MTAP(ifp, m_vec[0]);
@@ -767,28 +645,10 @@ cxgb_tx(struct sge_qset *qs, uint32_t txmax)
 		txq->txq_enqueued += count;
 		m_vec[0] = NULL;
 	}
-#if 0 /* !MULTIQ */
-	if (__predict_false(err)) {
-		if (err == ENOMEM) {
-			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
-			IFQ_LOCK(&ifp->if_snd);
-			IFQ_DRV_PREPEND(&ifp->if_snd, m_vec[0]);
-			IFQ_UNLOCK(&ifp->if_snd);
-		}
-	}
-	else if ((err == 0) &&  (txq->size <= txq->in_use + TX_MAX_DESC) &&
-	    (ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) {
-		setbit(&qs->txq_stopped, TXQ_ETH);
-		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
-		txq_fills++;
-		err = ENOSPC;
-	}
-#else
 	if ((err == 0) &&  (txq->size <= txq->in_use + TX_MAX_DESC)) {
-		err = ENOSPC;
+		err = ENOBUFS;
 		txq_fills++;
 		setbit(&qs->txq_stopped, TXQ_ETH);
-		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 	}
 	if (err == ENOMEM) {
 		int i;
@@ -799,7 +659,6 @@ cxgb_tx(struct sge_qset *qs, uint32_t txmax)
 		for (i = 0; i < count; i++)
 			m_freem(m_vec[i]);
 	}
-#endif
 	return (err);
 }