From 89cfec3b05a77c75f70dc5a90fc7d7ae1f2a75b5 Mon Sep 17 00:00:00 2001 From: kmacy Date: Sun, 8 Apr 2007 15:04:19 +0000 Subject: Convert driver RX path over to using mbuf iovec --- sys/dev/cxgb/cxgb_adapter.h | 6 +-- sys/dev/cxgb/cxgb_lro.c | 123 ++++++++++++++++++++------------------------ sys/dev/cxgb/cxgb_osdep.h | 5 -- sys/dev/cxgb/cxgb_sge.c | 96 +++++++++++++++++----------------- 4 files changed, 107 insertions(+), 123 deletions(-) diff --git a/sys/dev/cxgb/cxgb_adapter.h b/sys/dev/cxgb/cxgb_adapter.h index 288d620..7d8a787 100644 --- a/sys/dev/cxgb/cxgb_adapter.h +++ b/sys/dev/cxgb/cxgb_adapter.h @@ -115,7 +115,7 @@ enum { }; struct sge_lro_session { - struct t3_mbuf_hdr mh; + struct mbuf *m; uint32_t seq; uint16_t ip_len; }; @@ -154,7 +154,7 @@ struct sge_rspq { uint32_t cntxt_id; bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; - struct t3_mbuf_hdr mh; + struct mbuf *m; struct mtx lock; }; @@ -403,7 +403,7 @@ int t3_encap(struct port_info *, struct mbuf **); int t3_sge_init_sw(adapter_t *); void t3_sge_deinit_sw(adapter_t *); -void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh, +void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro); void t3_rx_eth(struct port_info *p, struct sge_rspq *rq, struct mbuf *m, int ethpad); void t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs); diff --git a/sys/dev/cxgb/cxgb_lro.c b/sys/dev/cxgb/cxgb_lro.c index c888291..22b4f73 100644 --- a/sys/dev/cxgb/cxgb_lro.c +++ b/sys/dev/cxgb/cxgb_lro.c @@ -65,22 +65,17 @@ __FBSDID("$FreeBSD$"); #endif #ifdef DEBUG -#define MBUF_HEADER_CHECK(mh) do { \ - struct mbuf *head = mh->mh_head; \ - struct mbuf *tail = mh->mh_tail; \ - if (head->m_len == 0 || head->m_pkthdr.len == 0 \ - || (head->m_flags & M_PKTHDR) == 0) \ +#define MBUF_HEADER_CHECK(m) do { \ + if (m->m_len == 0 || m->m_pkthdr.len == 0 \ + || (m->m_flags & M_PKTHDR) == 0) \ panic("lro_flush_session - mbuf len=%d pktlen=%d flags=0x%x\n", \ - head->m_len, head->m_pkthdr.len, head->m_flags); \ - if ((head->m_flags & M_PKTHDR) == 0) \ + m->m_len, m->m_pkthdr.len, m->m_flags); \ + if (m->m_flags & M_PKTHDR) == 0) \ panic("first mbuf is not packet header - flags=0x%x\n", \ - head->m_flags); \ - if ((head == tail && head->m_len != head->m_pkthdr.len)) \ - panic("len=%d pktlen=%d mismatch\n", \ - head->m_len, head->m_pkthdr.len); \ - if (head->m_len < ETHER_HDR_LEN || head->m_pkthdr.len < ETHER_HDR_LEN) \ + m->m_flags); \ + if (m->m_len < ETHER_HDR_LEN || m->m_pkthdr.len < ETHER_HDR_LEN) \ panic("packet too small len=%d pktlen=%d\n", \ - head->m_len, head->m_pkthdr.len);\ + m->m_len, m->m_pkthdr.len);\ } while (0) #else #define MBUF_HEADER_CHECK(m) @@ -100,7 +95,7 @@ static __inline int lro_match_session(struct sge_lro_session *s, struct ip *ih, struct tcphdr *th) { - struct ip *sih = (struct ip *)(s->mh.mh_head->m_data + IPH_OFFSET); + struct ip *sih = (struct ip *)(s->m->m_data + IPH_OFFSET); struct tcphdr *sth = (struct tcphdr *) (sih + 1); /* @@ -121,7 +116,7 @@ lro_find_session(struct sge_lro *l, int idx, struct ip *ih, struct tcphdr *th) while (active < l->num_active) { s = lro_session(l, idx); - if (s->mh.mh_head) { + if (s->m) { if (lro_match_session(s, ih, th)) { l->last_s = s; return s; @@ -179,39 +174,37 @@ can_lro_tcpsegment(struct tcphdr *th) } static __inline void -lro_new_session_init(struct sge_lro_session *s, struct t3_mbuf_hdr *mh) +lro_new_session_init(struct sge_lro_session *s, struct mbuf *m) { - struct ip *ih = (struct ip *)(mh->mh_head->m_data + IPH_OFFSET); + struct ip *ih = (struct ip *)(m->m_data + IPH_OFFSET); struct tcphdr *th = (struct tcphdr *) (ih + 1); int ip_len = ntohs(ih->ip_len); - DPRINTF("%s(s=%p, mh->mh_head=%p, mh->mh_tail=%p)\n", __FUNCTION__, - s, mh->mh_head, mh->mh_tail); + DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m); - *&(s->mh) = *mh; - - MBUF_HEADER_CHECK(mh); + s->m = m; + + MBUF_HEADER_CHECK(m); s->ip_len = ip_len; s->seq = ntohl(th->th_seq) + ip_len - sizeof(*ih) - (th->th_off << 2); } static void -lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct t3_mbuf_hdr *mh) +lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct mbuf *m) { struct sge_lro *l = &qs->lro; - struct t3_mbuf_hdr *smh = &s->mh; - struct ip *ih = (struct ip *)(smh->mh_head->m_data + IPH_OFFSET); + struct mbuf *sm = s->m; + struct ip *ih = (struct ip *)(sm->m_data + IPH_OFFSET); DPRINTF("%s(qs=%p, s=%p, ", __FUNCTION__, qs, s); - if (mh) - DPRINTF("mh->mh_head=%p, mh->mh_tail=%p)\n", - mh->mh_head, mh->mh_tail); + if (m) + DPRINTF("m=%p)\n", m); else - DPRINTF("mh=NULL)\n"); + DPRINTF("m=NULL)\n"); ih->ip_len = htons(s->ip_len); ih->ip_sum = 0; @@ -219,15 +212,14 @@ lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct t3_mbuf MBUF_HEADER_CHECK(smh); - smh->mh_head->m_flags |= M_LRO; - t3_rx_eth(qs->port, &qs->rspq, smh->mh_head, 2); + sm->m_flags |= M_LRO; + t3_rx_eth(qs->port, &qs->rspq, sm, 2); - if (mh) { - *smh = *mh; - lro_new_session_init(s, mh); + if (m) { + s->m = m; + lro_new_session_init(s, m); } else { - smh->mh_head = NULL; - smh->mh_tail = NULL; + s->m = NULL; l->num_active--; } @@ -235,23 +227,23 @@ lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct t3_mbuf } static __inline struct sge_lro_session * -lro_new_session(struct sge_qset *qs, struct t3_mbuf_hdr *mh, uint32_t rss_hash) +lro_new_session(struct sge_qset *qs, struct mbuf *m, uint32_t rss_hash) { struct sge_lro *l = &qs->lro; int idx = LRO_SESSION_IDX_HINT_HASH(rss_hash); struct sge_lro_session *s = lro_session(l, idx); - DPRINTF("%s(qs=%p, mh->mh_head=%p, mh->mh_tail=%p, rss_hash=0x%x)\n", __FUNCTION__, - qs, mh->mh_head, mh->mh_tail, rss_hash); + DPRINTF("%s(qs=%p, m=%p, rss_hash=0x%x)\n", __FUNCTION__, + qs, m, rss_hash); - if (__predict_true(!s->mh.mh_head)) + if (__predict_true(!s->m)) goto done; if (l->num_active > MAX_LRO_PER_QSET) panic("MAX_LRO_PER_QSET exceeded"); if (l->num_active == MAX_LRO_PER_QSET) { - lro_flush_session(qs, s, mh); + lro_flush_session(qs, s, m); qs->port_stats[SGE_PSTATS_LRO_X_STREAMS]++; return s; } @@ -259,24 +251,21 @@ lro_new_session(struct sge_qset *qs, struct t3_mbuf_hdr *mh, uint32_t rss_hash) while (1) { LRO_IDX_INC(idx); s = lro_session(l, idx); - if (!s->mh.mh_head) + if (!s->m) break; } done: - lro_new_session_init(s, mh); - + lro_new_session_init(s, m); l->num_active++; - return s; - + return s; } static __inline int -lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh) +lro_update_session(struct sge_lro_session *s, struct mbuf *m) { - struct mbuf *m = mh->mh_head; - struct t3_mbuf_hdr *smh = &s->mh; - struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(smh->mh_head->m_data + 2); + struct mbuf *sm = s->m; + struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(sm->m_data + 2); struct cpl_rx_pkt *ncpl = (struct cpl_rx_pkt *)(m->m_data + 2); struct ip *nih = (struct ip *)(m->m_data + IPH_OFFSET); struct tcphdr *th, *nth = (struct tcphdr *)(nih + 1); @@ -284,8 +273,7 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh) int plen, tcpiphlen, olen = (nth->th_off << 2) - sizeof (*nth); - DPRINTF("%s(s=%p, mh->mh_head=%p, mh->mh_tail=%p)\n", __FUNCTION__, - s, mh->mh_head, mh->mh_tail); + DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m); if (cpl->vlan_valid && cpl->vlan != ncpl->vlan) { return -1; } @@ -295,7 +283,7 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh) } MBUF_HEADER_CHECK(smh); - th = (struct tcphdr *)(smh->mh_head->m_data + IPH_OFFSET + sizeof (struct ip)); + th = (struct tcphdr *)(sm->m_data + IPH_OFFSET + sizeof (struct ip)); if (olen) { uint32_t *ptr = (uint32_t *)(th + 1); @@ -315,7 +303,13 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh) plen = ntohs(nih->ip_len) - tcpiphlen; s->seq += plen; s->ip_len += plen; - smh->mh_head->m_pkthdr.len += plen; + sm->m_pkthdr.len += plen; + + /* + * XXX FIX ME + * + * + */ #if 0 /* XXX this I *do not* understand */ @@ -323,8 +317,8 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh) skb_shinfo(s->skb)->gso_size = plen; #endif #if __FreeBSD_version > 700000 - if (plen > smh->mh_head->m_pkthdr.tso_segsz) - smh->mh_head->m_pkthdr.tso_segsz = plen; + if (plen > sm->m_pkthdr.tso_segsz) + sm->m_pkthdr.tso_segsz = plen; #endif DPRINTF("m_adj(%d)\n", (int)(IPH_OFFSET + tcpiphlen)); m_adj(m, IPH_OFFSET + tcpiphlen); @@ -333,9 +327,7 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh) skb_shinfo(s->skb)->frag_list = skb; #endif - mh->mh_head->m_flags &= ~M_PKTHDR; - smh->mh_tail->m_next = mh->mh_head; - smh->mh_tail = mh->mh_tail; + #if 0 /* @@ -354,10 +346,9 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh) } void -t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh, +t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro) { - struct mbuf *m = mh->mh_head; struct sge_qset *qs = rspq_to_qset(rq); struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(m->m_data + ethpad); struct ether_header *eh = (struct ether_header *)(cpl + 1); @@ -384,12 +375,12 @@ t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh, if (__predict_false(!can_lro_tcpsegment(th))) { goto no_lro; } else if (__predict_false(!s)) { - s = lro_new_session(qs, mh, rss_hash); + s = lro_new_session(qs, m, rss_hash); } else { - if (lro_update_session(s, mh)) { - lro_flush_session(qs, s, mh); + if (lro_update_session(s, m)) { + lro_flush_session(qs, s, m); } - if (__predict_false(s->mh.mh_head->m_pkthdr.len + pi->ifp->if_mtu > 65535)) { + if (__predict_false(s->m->m_pkthdr.len + pi->ifp->if_mtu > 65535)) { lro_flush_session(qs, s, NULL); } } @@ -417,7 +408,7 @@ t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs) s = lro_session(l, idx); while (active < num_active) { - if (s->mh.mh_head) { + if (s->m) { lro_flush_session(qs, s, NULL); active++; } diff --git a/sys/dev/cxgb/cxgb_osdep.h b/sys/dev/cxgb/cxgb_osdep.h index 087cf5a..42aa7f7 100644 --- a/sys/dev/cxgb/cxgb_osdep.h +++ b/sys/dev/cxgb/cxgb_osdep.h @@ -52,11 +52,6 @@ typedef struct adapter adapter_t; struct sge_rspq; -struct t3_mbuf_hdr { - struct mbuf *mh_head; - struct mbuf *mh_tail; -}; - #if __FreeBSD_version > 700030 #define INTR_FILTERS #define FIRMWARE_LATEST diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c index 0fe7649..d8199e6 100644 --- a/sys/dev/cxgb/cxgb_sge.c +++ b/sys/dev/cxgb/cxgb_sge.c @@ -67,6 +67,8 @@ __FBSDID("$FreeBSD$"); #include #include +#include + #define USE_GTS 0 #define SGE_RX_SM_BUF_SIZE 1536 @@ -268,9 +270,8 @@ sgl_len(unsigned int n) * Return a packet containing the immediate data of the given response. */ static __inline int -get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct t3_mbuf_hdr *mh) +get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void *cl) { - struct mbuf *m; int len; uint32_t flags = ntohl(resp->flags); uint8_t sopeop = G_RSPD_SOP_EOP(flags); @@ -291,15 +292,14 @@ get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct t3_mbuf_hdr *m switch (sopeop) { case RSPQ_SOP_EOP: - mh->mh_head = mh->mh_tail = m; - m->m_pkthdr.len = len; - m->m_flags |= M_PKTHDR; + m->m_len = m->m_pkthdr.len = len; + m->m_flags |= M_PKTHDR; + memcpy(m->m_data, resp->imm_data, IMMED_PKT_SIZE); + MH_ALIGN(m, IMMED_PKT_SIZE); break; case RSPQ_EOP: - m->m_flags &= ~M_PKTHDR; - mh->mh_head->m_pkthdr.len += len; - mh->mh_tail->m_next = m; - mh->mh_tail = m; + memcpy(cl, resp->imm_data, len); + m_iovappend(m, cl, MSIZE, len, 0); break; } } @@ -1759,7 +1759,8 @@ t3_rx_eth(struct port_info *pi, struct sge_rspq *rq, struct mbuf *m, int ethpad) } #endif m->m_pkthdr.rcvif = ifp; - + + m_explode(m); m_adj(m, sizeof(*cpl) + ethpad); (*ifp->if_input)(ifp, m); @@ -1784,7 +1785,7 @@ t3_rx_eth(struct port_info *pi, struct sge_rspq *rq, struct mbuf *m, int ethpad) static int get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, - struct t3_mbuf_hdr *mh, struct rsp_desc *r, struct mbuf *m) + struct mbuf *m, struct rsp_desc *r) { unsigned int len_cq = ntohl(r->len_cq); @@ -1801,47 +1802,34 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(fl->entry_tag, sd->map); - m_cljset(m, sd->cl, fl->type); - m->m_len = len; switch(sopeop) { case RSPQ_SOP_EOP: DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); - mh->mh_head = mh->mh_tail = m; - m->m_pkthdr.len = len; + m_cljset(m, sd->cl, fl->type); + m->m_len = m->m_pkthdr.len = len; m->m_flags |= M_PKTHDR; ret = 1; + goto done; break; case RSPQ_NSOP_NEOP: DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m)); - m->m_flags &= ~M_PKTHDR; - if (mh->mh_tail == NULL) { - if (cxgb_debug) - printf("discarding intermediate descriptor entry\n"); - m_freem(m); - break; - } - mh->mh_tail->m_next = m; - mh->mh_tail = m; - mh->mh_head->m_pkthdr.len += len; ret = 0; break; case RSPQ_SOP: DBG(DBG_RX, ("get_packet: SOP m %p\n", m)); - m->m_pkthdr.len = len; - mh->mh_head = mh->mh_tail = m; m->m_flags |= M_PKTHDR; + m_iovinit(m); ret = 0; break; case RSPQ_EOP: DBG(DBG_RX, ("get_packet: EOP m %p\n", m)); - m->m_flags &= ~M_PKTHDR; - mh->mh_head->m_pkthdr.len += len; - mh->mh_tail->m_next = m; - mh->mh_tail = m; ret = 1; break; } + m_iovappend(m, sd->cl, fl->buf_size, len, 0); + +done: if (++fl->cidx == fl->size) fl->cidx = 0; @@ -1967,31 +1955,35 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget) printf("async notification\n"); } else if (flags & F_RSPD_IMM_DATA_VALID) { + struct mbuf *m = NULL; if (cxgb_debug) printf("IMM DATA VALID\n"); - - if(get_imm_packet(adap, r, &rspq->mh) == 0) { + if (rspq->m == NULL) + rspq->m = m_gethdr(M_NOWAIT, MT_DATA); + else + m = m_gethdr(M_NOWAIT, MT_DATA); + + if (rspq->m == NULL || m == NULL) { rspq->next_holdoff = NOMEM_INTR_DELAY; budget_left--; break; - } else { - eop = 1; } - + get_imm_packet(adap, r, rspq->m, m); + eop = 1; rspq->imm_data++; } else if (r->len_cq) { int drop_thresh = eth ? SGE_RX_DROP_THRES : 0; - struct mbuf *m; - - m = m_gethdr(M_NOWAIT, MT_DATA); - if (m == NULL) { - log(LOG_WARNING, "failed to get mbuf for packet\n"); - break; + if (rspq->m == NULL) { + rspq->m = m_gethdr(M_NOWAIT, MT_DATA); + rspq->m->m_flags = 0; + } else if (rspq->m == NULL) { + log(LOG_WARNING, "failed to get mbuf for packet\n"); + break; } - + ethpad = 2; - eop = get_packet(adap, drop_thresh, qs, &rspq->mh, r, m); + eop = get_packet(adap, drop_thresh, qs, rspq->m, r); } else { DPRINTF("pure response\n"); rspq->pure_rsps++; @@ -2016,14 +2008,14 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget) } if (eop) { - prefetch(rspq->mh.mh_head->m_data); - prefetch(rspq->mh.mh_head->m_data + L1_CACHE_BYTES); + prefetch(rspq->m->m_data); + prefetch(rspq->m->m_data + L1_CACHE_BYTES); if (eth) { - t3_rx_eth_lro(adap, rspq, &rspq->mh, ethpad, + t3_rx_eth_lro(adap, rspq, rspq->m, ethpad, rss_hash, rss_csum, lro); - rspq->mh.mh_tail = rspq->mh.mh_head = NULL; + rspq->m = NULL; } else { #ifdef notyet if (__predict_false(r->rss_hdr.opcode == CPL_TRACE_PKT)) @@ -2038,7 +2030,6 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget) __refill_fl(adap, &qs->fl[0]); __refill_fl(adap, &qs->fl[1]); #endif - } --budget_left; } @@ -2158,11 +2149,18 @@ t3_intr_msix(void *data) mtx_unlock(&rspq->lock); } +/* + * broken by recent mbuf changes + */ static int t3_lro_enable(SYSCTL_HANDLER_ARGS) { adapter_t *sc; int i, j, enabled, err, nqsets = 0; + +#ifndef LRO_WORKING + return (0); +#endif sc = arg1; enabled = sc->sge.qs[0].lro.enabled; -- cgit v1.1