summaryrefslogtreecommitdiffstats
path: root/sys/dev
diff options
context:
space:
mode:
authorkmacy <kmacy@FreeBSD.org>2007-04-08 15:04:19 +0000
committerkmacy <kmacy@FreeBSD.org>2007-04-08 15:04:19 +0000
commit89cfec3b05a77c75f70dc5a90fc7d7ae1f2a75b5 (patch)
treeee40cb33e98f087b7ec11e96f6f5e6e6877b717c /sys/dev
parentc2173d96894f2c438d2181f46180d0bc3ba873ba (diff)
downloadFreeBSD-src-89cfec3b05a77c75f70dc5a90fc7d7ae1f2a75b5.zip
FreeBSD-src-89cfec3b05a77c75f70dc5a90fc7d7ae1f2a75b5.tar.gz
Convert driver RX path over to using mbuf iovec
Diffstat (limited to 'sys/dev')
-rw-r--r--sys/dev/cxgb/cxgb_adapter.h6
-rw-r--r--sys/dev/cxgb/cxgb_lro.c123
-rw-r--r--sys/dev/cxgb/cxgb_osdep.h5
-rw-r--r--sys/dev/cxgb/cxgb_sge.c96
4 files changed, 107 insertions, 123 deletions
diff --git a/sys/dev/cxgb/cxgb_adapter.h b/sys/dev/cxgb/cxgb_adapter.h
index 288d620..7d8a787 100644
--- a/sys/dev/cxgb/cxgb_adapter.h
+++ b/sys/dev/cxgb/cxgb_adapter.h
@@ -115,7 +115,7 @@ enum {
};
struct sge_lro_session {
- struct t3_mbuf_hdr mh;
+ struct mbuf *m;
uint32_t seq;
uint16_t ip_len;
};
@@ -154,7 +154,7 @@ struct sge_rspq {
uint32_t cntxt_id;
bus_dma_tag_t desc_tag;
bus_dmamap_t desc_map;
- struct t3_mbuf_hdr mh;
+ struct mbuf *m;
struct mtx lock;
};
@@ -403,7 +403,7 @@ int t3_encap(struct port_info *, struct mbuf **);
int t3_sge_init_sw(adapter_t *);
void t3_sge_deinit_sw(adapter_t *);
-void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh,
+void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro);
void t3_rx_eth(struct port_info *p, struct sge_rspq *rq, struct mbuf *m, int ethpad);
void t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs);
diff --git a/sys/dev/cxgb/cxgb_lro.c b/sys/dev/cxgb/cxgb_lro.c
index c888291..22b4f73 100644
--- a/sys/dev/cxgb/cxgb_lro.c
+++ b/sys/dev/cxgb/cxgb_lro.c
@@ -65,22 +65,17 @@ __FBSDID("$FreeBSD$");
#endif
#ifdef DEBUG
-#define MBUF_HEADER_CHECK(mh) do { \
- struct mbuf *head = mh->mh_head; \
- struct mbuf *tail = mh->mh_tail; \
- if (head->m_len == 0 || head->m_pkthdr.len == 0 \
- || (head->m_flags & M_PKTHDR) == 0) \
+#define MBUF_HEADER_CHECK(m) do { \
+ if (m->m_len == 0 || m->m_pkthdr.len == 0 \
+ || (m->m_flags & M_PKTHDR) == 0) \
panic("lro_flush_session - mbuf len=%d pktlen=%d flags=0x%x\n", \
- head->m_len, head->m_pkthdr.len, head->m_flags); \
- if ((head->m_flags & M_PKTHDR) == 0) \
+ m->m_len, m->m_pkthdr.len, m->m_flags); \
+ if (m->m_flags & M_PKTHDR) == 0) \
panic("first mbuf is not packet header - flags=0x%x\n", \
- head->m_flags); \
- if ((head == tail && head->m_len != head->m_pkthdr.len)) \
- panic("len=%d pktlen=%d mismatch\n", \
- head->m_len, head->m_pkthdr.len); \
- if (head->m_len < ETHER_HDR_LEN || head->m_pkthdr.len < ETHER_HDR_LEN) \
+ m->m_flags); \
+ if (m->m_len < ETHER_HDR_LEN || m->m_pkthdr.len < ETHER_HDR_LEN) \
panic("packet too small len=%d pktlen=%d\n", \
- head->m_len, head->m_pkthdr.len);\
+ m->m_len, m->m_pkthdr.len);\
} while (0)
#else
#define MBUF_HEADER_CHECK(m)
@@ -100,7 +95,7 @@ static __inline int
lro_match_session(struct sge_lro_session *s,
struct ip *ih, struct tcphdr *th)
{
- struct ip *sih = (struct ip *)(s->mh.mh_head->m_data + IPH_OFFSET);
+ struct ip *sih = (struct ip *)(s->m->m_data + IPH_OFFSET);
struct tcphdr *sth = (struct tcphdr *) (sih + 1);
/*
@@ -121,7 +116,7 @@ lro_find_session(struct sge_lro *l, int idx, struct ip *ih, struct tcphdr *th)
while (active < l->num_active) {
s = lro_session(l, idx);
- if (s->mh.mh_head) {
+ if (s->m) {
if (lro_match_session(s, ih, th)) {
l->last_s = s;
return s;
@@ -179,39 +174,37 @@ can_lro_tcpsegment(struct tcphdr *th)
}
static __inline void
-lro_new_session_init(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
+lro_new_session_init(struct sge_lro_session *s, struct mbuf *m)
{
- struct ip *ih = (struct ip *)(mh->mh_head->m_data + IPH_OFFSET);
+ struct ip *ih = (struct ip *)(m->m_data + IPH_OFFSET);
struct tcphdr *th = (struct tcphdr *) (ih + 1);
int ip_len = ntohs(ih->ip_len);
- DPRINTF("%s(s=%p, mh->mh_head=%p, mh->mh_tail=%p)\n", __FUNCTION__,
- s, mh->mh_head, mh->mh_tail);
+ DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m);
- *&(s->mh) = *mh;
-
- MBUF_HEADER_CHECK(mh);
+ s->m = m;
+
+ MBUF_HEADER_CHECK(m);
s->ip_len = ip_len;
s->seq = ntohl(th->th_seq) + ip_len - sizeof(*ih) - (th->th_off << 2);
}
static void
-lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
+lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct mbuf *m)
{
struct sge_lro *l = &qs->lro;
- struct t3_mbuf_hdr *smh = &s->mh;
- struct ip *ih = (struct ip *)(smh->mh_head->m_data + IPH_OFFSET);
+ struct mbuf *sm = s->m;
+ struct ip *ih = (struct ip *)(sm->m_data + IPH_OFFSET);
DPRINTF("%s(qs=%p, s=%p, ", __FUNCTION__,
qs, s);
- if (mh)
- DPRINTF("mh->mh_head=%p, mh->mh_tail=%p)\n",
- mh->mh_head, mh->mh_tail);
+ if (m)
+ DPRINTF("m=%p)\n", m);
else
- DPRINTF("mh=NULL)\n");
+ DPRINTF("m=NULL)\n");
ih->ip_len = htons(s->ip_len);
ih->ip_sum = 0;
@@ -219,15 +212,14 @@ lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct t3_mbuf
MBUF_HEADER_CHECK(smh);
- smh->mh_head->m_flags |= M_LRO;
- t3_rx_eth(qs->port, &qs->rspq, smh->mh_head, 2);
+ sm->m_flags |= M_LRO;
+ t3_rx_eth(qs->port, &qs->rspq, sm, 2);
- if (mh) {
- *smh = *mh;
- lro_new_session_init(s, mh);
+ if (m) {
+ s->m = m;
+ lro_new_session_init(s, m);
} else {
- smh->mh_head = NULL;
- smh->mh_tail = NULL;
+ s->m = NULL;
l->num_active--;
}
@@ -235,23 +227,23 @@ lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct t3_mbuf
}
static __inline struct sge_lro_session *
-lro_new_session(struct sge_qset *qs, struct t3_mbuf_hdr *mh, uint32_t rss_hash)
+lro_new_session(struct sge_qset *qs, struct mbuf *m, uint32_t rss_hash)
{
struct sge_lro *l = &qs->lro;
int idx = LRO_SESSION_IDX_HINT_HASH(rss_hash);
struct sge_lro_session *s = lro_session(l, idx);
- DPRINTF("%s(qs=%p, mh->mh_head=%p, mh->mh_tail=%p, rss_hash=0x%x)\n", __FUNCTION__,
- qs, mh->mh_head, mh->mh_tail, rss_hash);
+ DPRINTF("%s(qs=%p, m=%p, rss_hash=0x%x)\n", __FUNCTION__,
+ qs, m, rss_hash);
- if (__predict_true(!s->mh.mh_head))
+ if (__predict_true(!s->m))
goto done;
if (l->num_active > MAX_LRO_PER_QSET)
panic("MAX_LRO_PER_QSET exceeded");
if (l->num_active == MAX_LRO_PER_QSET) {
- lro_flush_session(qs, s, mh);
+ lro_flush_session(qs, s, m);
qs->port_stats[SGE_PSTATS_LRO_X_STREAMS]++;
return s;
}
@@ -259,24 +251,21 @@ lro_new_session(struct sge_qset *qs, struct t3_mbuf_hdr *mh, uint32_t rss_hash)
while (1) {
LRO_IDX_INC(idx);
s = lro_session(l, idx);
- if (!s->mh.mh_head)
+ if (!s->m)
break;
}
done:
- lro_new_session_init(s, mh);
-
+ lro_new_session_init(s, m);
l->num_active++;
- return s;
-
+ return s;
}
static __inline int
-lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
+lro_update_session(struct sge_lro_session *s, struct mbuf *m)
{
- struct mbuf *m = mh->mh_head;
- struct t3_mbuf_hdr *smh = &s->mh;
- struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(smh->mh_head->m_data + 2);
+ struct mbuf *sm = s->m;
+ struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(sm->m_data + 2);
struct cpl_rx_pkt *ncpl = (struct cpl_rx_pkt *)(m->m_data + 2);
struct ip *nih = (struct ip *)(m->m_data + IPH_OFFSET);
struct tcphdr *th, *nth = (struct tcphdr *)(nih + 1);
@@ -284,8 +273,7 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
int plen, tcpiphlen, olen = (nth->th_off << 2) - sizeof (*nth);
- DPRINTF("%s(s=%p, mh->mh_head=%p, mh->mh_tail=%p)\n", __FUNCTION__,
- s, mh->mh_head, mh->mh_tail);
+ DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m);
if (cpl->vlan_valid && cpl->vlan != ncpl->vlan) {
return -1;
}
@@ -295,7 +283,7 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
}
MBUF_HEADER_CHECK(smh);
- th = (struct tcphdr *)(smh->mh_head->m_data + IPH_OFFSET + sizeof (struct ip));
+ th = (struct tcphdr *)(sm->m_data + IPH_OFFSET + sizeof (struct ip));
if (olen) {
uint32_t *ptr = (uint32_t *)(th + 1);
@@ -315,7 +303,13 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
plen = ntohs(nih->ip_len) - tcpiphlen;
s->seq += plen;
s->ip_len += plen;
- smh->mh_head->m_pkthdr.len += plen;
+ sm->m_pkthdr.len += plen;
+
+ /*
+ * XXX FIX ME
+ *
+ *
+ */
#if 0
/* XXX this I *do not* understand */
@@ -323,8 +317,8 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
skb_shinfo(s->skb)->gso_size = plen;
#endif
#if __FreeBSD_version > 700000
- if (plen > smh->mh_head->m_pkthdr.tso_segsz)
- smh->mh_head->m_pkthdr.tso_segsz = plen;
+ if (plen > sm->m_pkthdr.tso_segsz)
+ sm->m_pkthdr.tso_segsz = plen;
#endif
DPRINTF("m_adj(%d)\n", (int)(IPH_OFFSET + tcpiphlen));
m_adj(m, IPH_OFFSET + tcpiphlen);
@@ -333,9 +327,7 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
skb_shinfo(s->skb)->frag_list = skb;
#endif
- mh->mh_head->m_flags &= ~M_PKTHDR;
- smh->mh_tail->m_next = mh->mh_head;
- smh->mh_tail = mh->mh_tail;
+
#if 0
/*
@@ -354,10 +346,9 @@ lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh)
}
void
-t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh,
+t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro)
{
- struct mbuf *m = mh->mh_head;
struct sge_qset *qs = rspq_to_qset(rq);
struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(m->m_data + ethpad);
struct ether_header *eh = (struct ether_header *)(cpl + 1);
@@ -384,12 +375,12 @@ t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh,
if (__predict_false(!can_lro_tcpsegment(th))) {
goto no_lro;
} else if (__predict_false(!s)) {
- s = lro_new_session(qs, mh, rss_hash);
+ s = lro_new_session(qs, m, rss_hash);
} else {
- if (lro_update_session(s, mh)) {
- lro_flush_session(qs, s, mh);
+ if (lro_update_session(s, m)) {
+ lro_flush_session(qs, s, m);
}
- if (__predict_false(s->mh.mh_head->m_pkthdr.len + pi->ifp->if_mtu > 65535)) {
+ if (__predict_false(s->m->m_pkthdr.len + pi->ifp->if_mtu > 65535)) {
lro_flush_session(qs, s, NULL);
}
}
@@ -417,7 +408,7 @@ t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs)
s = lro_session(l, idx);
while (active < num_active) {
- if (s->mh.mh_head) {
+ if (s->m) {
lro_flush_session(qs, s, NULL);
active++;
}
diff --git a/sys/dev/cxgb/cxgb_osdep.h b/sys/dev/cxgb/cxgb_osdep.h
index 087cf5a..42aa7f7 100644
--- a/sys/dev/cxgb/cxgb_osdep.h
+++ b/sys/dev/cxgb/cxgb_osdep.h
@@ -52,11 +52,6 @@ typedef struct adapter adapter_t;
struct sge_rspq;
-struct t3_mbuf_hdr {
- struct mbuf *mh_head;
- struct mbuf *mh_tail;
-};
-
#if __FreeBSD_version > 700030
#define INTR_FILTERS
#define FIRMWARE_LATEST
diff --git a/sys/dev/cxgb/cxgb_sge.c b/sys/dev/cxgb/cxgb_sge.c
index 0fe7649..d8199e6 100644
--- a/sys/dev/cxgb/cxgb_sge.c
+++ b/sys/dev/cxgb/cxgb_sge.c
@@ -67,6 +67,8 @@ __FBSDID("$FreeBSD$");
#include <dev/cxgb/common/cxgb_t3_cpl.h>
#include <dev/cxgb/common/cxgb_firmware_exports.h>
+#include <dev/cxgb/sys/mvec.h>
+
#define USE_GTS 0
#define SGE_RX_SM_BUF_SIZE 1536
@@ -268,9 +270,8 @@ sgl_len(unsigned int n)
* Return a packet containing the immediate data of the given response.
*/
static __inline int
-get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct t3_mbuf_hdr *mh)
+get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void *cl)
{
- struct mbuf *m;
int len;
uint32_t flags = ntohl(resp->flags);
uint8_t sopeop = G_RSPD_SOP_EOP(flags);
@@ -291,15 +292,14 @@ get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct t3_mbuf_hdr *m
switch (sopeop) {
case RSPQ_SOP_EOP:
- mh->mh_head = mh->mh_tail = m;
- m->m_pkthdr.len = len;
- m->m_flags |= M_PKTHDR;
+ m->m_len = m->m_pkthdr.len = len;
+ m->m_flags |= M_PKTHDR;
+ memcpy(m->m_data, resp->imm_data, IMMED_PKT_SIZE);
+ MH_ALIGN(m, IMMED_PKT_SIZE);
break;
case RSPQ_EOP:
- m->m_flags &= ~M_PKTHDR;
- mh->mh_head->m_pkthdr.len += len;
- mh->mh_tail->m_next = m;
- mh->mh_tail = m;
+ memcpy(cl, resp->imm_data, len);
+ m_iovappend(m, cl, MSIZE, len, 0);
break;
}
}
@@ -1759,7 +1759,8 @@ t3_rx_eth(struct port_info *pi, struct sge_rspq *rq, struct mbuf *m, int ethpad)
}
#endif
m->m_pkthdr.rcvif = ifp;
-
+
+ m_explode(m);
m_adj(m, sizeof(*cpl) + ethpad);
(*ifp->if_input)(ifp, m);
@@ -1784,7 +1785,7 @@ t3_rx_eth(struct port_info *pi, struct sge_rspq *rq, struct mbuf *m, int ethpad)
static int
get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
- struct t3_mbuf_hdr *mh, struct rsp_desc *r, struct mbuf *m)
+ struct mbuf *m, struct rsp_desc *r)
{
unsigned int len_cq = ntohl(r->len_cq);
@@ -1801,47 +1802,34 @@ get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs,
bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(fl->entry_tag, sd->map);
- m_cljset(m, sd->cl, fl->type);
- m->m_len = len;
switch(sopeop) {
case RSPQ_SOP_EOP:
DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m));
- mh->mh_head = mh->mh_tail = m;
- m->m_pkthdr.len = len;
+ m_cljset(m, sd->cl, fl->type);
+ m->m_len = m->m_pkthdr.len = len;
m->m_flags |= M_PKTHDR;
ret = 1;
+ goto done;
break;
case RSPQ_NSOP_NEOP:
DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m));
- m->m_flags &= ~M_PKTHDR;
- if (mh->mh_tail == NULL) {
- if (cxgb_debug)
- printf("discarding intermediate descriptor entry\n");
- m_freem(m);
- break;
- }
- mh->mh_tail->m_next = m;
- mh->mh_tail = m;
- mh->mh_head->m_pkthdr.len += len;
ret = 0;
break;
case RSPQ_SOP:
DBG(DBG_RX, ("get_packet: SOP m %p\n", m));
- m->m_pkthdr.len = len;
- mh->mh_head = mh->mh_tail = m;
m->m_flags |= M_PKTHDR;
+ m_iovinit(m);
ret = 0;
break;
case RSPQ_EOP:
DBG(DBG_RX, ("get_packet: EOP m %p\n", m));
- m->m_flags &= ~M_PKTHDR;
- mh->mh_head->m_pkthdr.len += len;
- mh->mh_tail->m_next = m;
- mh->mh_tail = m;
ret = 1;
break;
}
+ m_iovappend(m, sd->cl, fl->buf_size, len, 0);
+
+done:
if (++fl->cidx == fl->size)
fl->cidx = 0;
@@ -1967,31 +1955,35 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
printf("async notification\n");
} else if (flags & F_RSPD_IMM_DATA_VALID) {
+ struct mbuf *m = NULL;
if (cxgb_debug)
printf("IMM DATA VALID\n");
-
- if(get_imm_packet(adap, r, &rspq->mh) == 0) {
+ if (rspq->m == NULL)
+ rspq->m = m_gethdr(M_NOWAIT, MT_DATA);
+ else
+ m = m_gethdr(M_NOWAIT, MT_DATA);
+
+ if (rspq->m == NULL || m == NULL) {
rspq->next_holdoff = NOMEM_INTR_DELAY;
budget_left--;
break;
- } else {
- eop = 1;
}
-
+ get_imm_packet(adap, r, rspq->m, m);
+ eop = 1;
rspq->imm_data++;
} else if (r->len_cq) {
int drop_thresh = eth ? SGE_RX_DROP_THRES : 0;
- struct mbuf *m;
-
- m = m_gethdr(M_NOWAIT, MT_DATA);
- if (m == NULL) {
- log(LOG_WARNING, "failed to get mbuf for packet\n");
- break;
+ if (rspq->m == NULL) {
+ rspq->m = m_gethdr(M_NOWAIT, MT_DATA);
+ rspq->m->m_flags = 0;
+ } else if (rspq->m == NULL) {
+ log(LOG_WARNING, "failed to get mbuf for packet\n");
+ break;
}
-
+
ethpad = 2;
- eop = get_packet(adap, drop_thresh, qs, &rspq->mh, r, m);
+ eop = get_packet(adap, drop_thresh, qs, rspq->m, r);
} else {
DPRINTF("pure response\n");
rspq->pure_rsps++;
@@ -2016,14 +2008,14 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
}
if (eop) {
- prefetch(rspq->mh.mh_head->m_data);
- prefetch(rspq->mh.mh_head->m_data + L1_CACHE_BYTES);
+ prefetch(rspq->m->m_data);
+ prefetch(rspq->m->m_data + L1_CACHE_BYTES);
if (eth) {
- t3_rx_eth_lro(adap, rspq, &rspq->mh, ethpad,
+ t3_rx_eth_lro(adap, rspq, rspq->m, ethpad,
rss_hash, rss_csum, lro);
- rspq->mh.mh_tail = rspq->mh.mh_head = NULL;
+ rspq->m = NULL;
} else {
#ifdef notyet
if (__predict_false(r->rss_hdr.opcode == CPL_TRACE_PKT))
@@ -2038,7 +2030,6 @@ process_responses(adapter_t *adap, struct sge_qset *qs, int budget)
__refill_fl(adap, &qs->fl[0]);
__refill_fl(adap, &qs->fl[1]);
#endif
-
}
--budget_left;
}
@@ -2158,11 +2149,18 @@ t3_intr_msix(void *data)
mtx_unlock(&rspq->lock);
}
+/*
+ * broken by recent mbuf changes
+ */
static int
t3_lro_enable(SYSCTL_HANDLER_ARGS)
{
adapter_t *sc;
int i, j, enabled, err, nqsets = 0;
+
+#ifndef LRO_WORKING
+ return (0);
+#endif
sc = arg1;
enabled = sc->sge.qs[0].lro.enabled;
OpenPOWER on IntegriCloud