diff options
author | kmacy <kmacy@FreeBSD.org> | 2007-04-14 20:40:22 +0000 |
---|---|---|
committer | kmacy <kmacy@FreeBSD.org> | 2007-04-14 20:40:22 +0000 |
commit | df7c6cd2a9ee0b08881df6dd16377e9cb099c9d5 (patch) | |
tree | 2bed777aac8345c040514b4f5f31b1051a39d130 /sys/dev | |
parent | a8a962f8f6a192cb65f331c6064b71ecbc94d697 (diff) | |
download | FreeBSD-src-df7c6cd2a9ee0b08881df6dd16377e9cb099c9d5.zip FreeBSD-src-df7c6cd2a9ee0b08881df6dd16377e9cb099c9d5.tar.gz |
Add support for mbuf iovec in the TX path
Diffstat (limited to 'sys/dev')
-rw-r--r-- | sys/dev/cxgb/cxgb_main.c | 42 | ||||
-rw-r--r-- | sys/dev/cxgb/cxgb_sge.c | 85 |
2 files changed, 87 insertions, 40 deletions
diff --git a/sys/dev/cxgb/cxgb_main.c b/sys/dev/cxgb/cxgb_main.c index 937fb10..26bfdd3 100644 --- a/sys/dev/cxgb/cxgb_main.c +++ b/sys/dev/cxgb/cxgb_main.c @@ -56,8 +56,6 @@ __FBSDID("$FreeBSD$"); #include <sys/queue.h> #include <sys/taskqueue.h> - - #include <net/bpf.h> #include <net/ethernet.h> #include <net/if.h> @@ -85,6 +83,8 @@ __FBSDID("$FreeBSD$"); #include <dev/cxgb/common/cxgb_t3_cpl.h> #include <dev/cxgb/common/cxgb_firmware_exports.h> +#include <dev/cxgb/sys/mvec.h> + #ifdef PRIV_SUPPORTED #include <sys/priv.h> @@ -532,6 +532,11 @@ cxgb_free(struct adapter *sc) { int i; + /* + * XXX need to drain the ifq by hand until + * it is taught about mbuf iovecs + */ + callout_drain(&sc->cxgb_tick_ch); t3_sge_deinit_sw(sc); @@ -1188,6 +1193,9 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) int flags, error = 0; uint32_t mask; + /* + * XXX need to check that we aren't in the middle of an unload + */ switch (command) { case SIOCSIFMTU: if ((ifr->ifr_mtu < ETHERMIN) || @@ -1212,7 +1220,6 @@ cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data) error = ether_ioctl(ifp, command, data); break; case SIOCSIFFLAGS: - if (ifp->if_flags & IFF_UP) { PORT_LOCK(p); if (ifp->if_drv_flags & IFF_DRV_RUNNING) { @@ -1294,7 +1301,7 @@ cxgb_start_tx(struct ifnet *ifp, uint32_t txmax) struct sge_qset *qs; struct sge_txq *txq; struct port_info *p = ifp->if_softc; - struct mbuf *m = NULL; + struct mbuf *m0, *m = NULL; int err, in_use_init; @@ -1315,6 +1322,33 @@ cxgb_start_tx(struct ifnet *ifp, uint32_t txmax) IFQ_DRV_DEQUEUE(&ifp->if_snd, m); if (m == NULL) break; + /* + * Convert chain to M_IOVEC + */ + KASSERT((m->m_flags & M_IOVEC) == 0, ("IOVEC set too early")); + m0 = m; +#ifdef INVARIANTS + /* + * Clean up after net stack sloppiness + * before calling m_sanity + */ + m0 = m->m_next; + while (m0) { + m0->m_flags &= ~M_PKTHDR; + m0 = m0->m_next; + } + m_sanity(m0, 0); + m0 = m; +#endif + if (m->m_pkthdr.len > MCLBYTES && + m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) { + if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) { + m = m0; + m_collapse(m, TX_MAX_SEGS, &m0); + } else + break; + } + m = m0; if ((err = t3_encap(p, &m)) != 0) break; BPF_MTAP(ifp, m); diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c index 971e1f4..8524fbf 100644 --- a/sys/dev/cxgb/cxgb_sge.c +++ b/sys/dev/cxgb/cxgb_sge.c @@ -69,6 +69,10 @@ __FBSDID("$FreeBSD$"); #include <dev/cxgb/sys/mvec.h> +uint32_t collapse_free = 0; +uint32_t mb_free_vec_free = 0; + + #define USE_GTS 0 #define SGE_RX_SM_BUF_SIZE 1536 @@ -113,6 +117,7 @@ struct rsp_desc { /* response queue descriptor */ } __packed; #define RX_SW_DESC_MAP_CREATED (1 << 0) +#define TX_SW_DESC_MAP_CREATED (1 << 1) #define RX_SW_DESC_INUSE (1 << 3) #define TX_SW_DESC_MAPPED (1 << 4) @@ -269,7 +274,7 @@ sgl_len(unsigned int n) * * Return a packet containing the immediate data of the given response. */ -static __inline int +static __inline void get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void *cl) { int len; @@ -280,28 +285,19 @@ get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void * would be a firmware bug */ if (sopeop == RSPQ_NSOP_NEOP || sopeop == RSPQ_SOP) - return (0); - - - len = G_RSPD_LEN(ntohl(resp->len_cq)); + return; - if (m) { - - - switch (sopeop) { - case RSPQ_SOP_EOP: - m = m_gethdr(M_NOWAIT, MT_DATA); - m->m_len = m->m_pkthdr.len = len; - memcpy(m->m_data, resp->imm_data, IMMED_PKT_SIZE); - MH_ALIGN(m, IMMED_PKT_SIZE); - break; - case RSPQ_EOP: - memcpy(cl, resp->imm_data, len); - m_iovappend(m, cl, MSIZE, len, 0); - break; - } + len = G_RSPD_LEN(ntohl(resp->len_cq)); + switch (sopeop) { + case RSPQ_SOP_EOP: + m->m_len = m->m_pkthdr.len = len; + memcpy(m->m_data, resp->imm_data, len); + break; + case RSPQ_EOP: + memcpy(cl, resp->imm_data, len); + m_iovappend(m, cl, MSIZE, len, 0); + break; } - return (m != NULL); } @@ -722,7 +718,7 @@ sge_timer_reclaim(void *arg, int ncount) mtx_unlock(&txq->lock); for (i = 0; i < n; i++) { - m_freem(m_vec[i]); + m_freem_vec(m_vec[i]); } } @@ -734,7 +730,7 @@ sge_timer_reclaim(void *arg, int ncount) mtx_unlock(&txq->lock); for (i = 0; i < n; i++) { - m_freem(m_vec[i]); + m_freem_vec(m_vec[i]); } } @@ -825,7 +821,7 @@ calc_tx_descs(const struct mbuf *m, int nsegs) flits = sgl_len(nsegs) + 2; #ifdef TSO_SUPPORTED - if (m->m_pkthdr.tso_segsz) + if (m->m_pkthdr.csum_flags & (CSUM_TSO)) flits++; #endif return flits_to_desc(flits); @@ -840,7 +836,16 @@ busdma_map_mbufs(struct mbuf **m, struct sge_txq *txq, m0 = *m; pktlen = m0->m_pkthdr.len; - err = bus_dmamap_load_mbuf_sg(txq->entry_tag, stx->map, m0, segs, nsegs, 0); + + if ((stx->flags & TX_SW_DESC_MAP_CREATED) == 0) { + if ((err = bus_dmamap_create(txq->entry_tag, 0, &stx->map))) { + log(LOG_WARNING, "bus_dmamap_create failed %d\n", err); + return (err); + } + stx->flags |= TX_SW_DESC_MAP_CREATED; + } + err = bus_dmamap_load_mvec_sg(txq->entry_tag, stx->map, m0, segs, nsegs, 0); +#ifdef DEBUG if (err) { int n = 0; struct mbuf *mtmp = m0; @@ -848,12 +853,10 @@ busdma_map_mbufs(struct mbuf **m, struct sge_txq *txq, n++; mtmp = mtmp->m_next; } -#ifdef DEBUG printf("map_mbufs: bus_dmamap_load_mbuf_sg failed with %d - pkthdr.len==%d nmbufs=%d\n", err, m0->m_pkthdr.len, n); -#endif } - +#endif if (err == EFBIG) { /* Too many segments, try to defrag */ m0 = m_defrag(m0, M_NOWAIT); @@ -873,7 +876,7 @@ busdma_map_mbufs(struct mbuf **m, struct sge_txq *txq, if (err) { if (cxgb_debug) printf("map failure err=%d pktlen=%d\n", err, pktlen); - m_freem(m0); + m_freem_vec(m0); *m = NULL; return (err); } @@ -1002,7 +1005,6 @@ t3_encap(struct port_info *p, struct mbuf **m) cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); if (m0->m_pkthdr.csum_flags & (CSUM_TSO)) tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); - #endif if (tso_info) { int eth_type; @@ -1010,7 +1012,7 @@ t3_encap(struct port_info *p, struct mbuf **m) struct ip *ip; struct tcphdr *tcp; uint8_t *pkthdr, tmp[TCPPKTHDRSIZE]; /* is this too large for the stack? */ - + txd->flit[2] = 0; cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO); hdr->cntrl = htonl(cntrl); @@ -1032,12 +1034,11 @@ t3_encap(struct port_info *p, struct mbuf **m) } tcp = (struct tcphdr *)((uint8_t *)ip + sizeof(*ip)); - + tso_info |= V_LSO_ETH_TYPE(eth_type) | V_LSO_IPHDR_WORDS(ip->ip_hl) | V_LSO_TCPHDR_WORDS(tcp->th_off); hdr->lso_info = htonl(tso_info); - flits = 3; } else { cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); @@ -1051,7 +1052,7 @@ t3_encap(struct port_info *p, struct mbuf **m) memcpy(&txd->flit[2], m0->m_data, mlen); else m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); - + flits = (mlen + 7) / 8 + 2; cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | @@ -1430,7 +1431,6 @@ t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx, printf("error %d from alloc ring tx %i\n", ret, i); goto err; } - q->txq[i].gen = 1; q->txq[i].size = p->txq_size[i]; mtx_init(&q->txq[i].lock, "t3 txq lock", NULL, MTX_DEF); @@ -1755,9 +1755,13 @@ t3_rx_eth(struct port_info *pi, struct sge_rspq *rq, struct mbuf *m, int ethpad) m->m_flags |= M_VLANTAG; } #endif - m->m_pkthdr.rcvif = ifp; + m->m_pkthdr.rcvif = ifp; + m->m_pkthdr.header = m->m_data + sizeof(*cpl) + ethpad; m_explode(m); + /* + * adjust after conversion to mbuf chain + */ m_adj(m, sizeof(*cpl) + ethpad); (*ifp->if_input)(ifp, m); @@ -2252,6 +2256,15 @@ t3_add_sysctls(adapter_t *sc) CTLFLAG_RW, &cxgb_debug, 0, "enable verbose debugging output"); + SYSCTL_ADD_INT(ctx, children, OID_AUTO, + "collapse_free", + CTLFLAG_RD, &collapse_free, + 0, "frees during collapse"); + SYSCTL_ADD_INT(ctx, children, OID_AUTO, + "mb_free_vec_free", + CTLFLAG_RD, &mb_free_vec_free, + 0, "frees during mb_free_vec"); + } /** |