From c2173d96894f2c438d2181f46180d0bc3ba873ba Mon Sep 17 00:00:00 2001 From: kmacy Date: Sun, 8 Apr 2007 14:56:16 +0000 Subject: Add driver private mbuf iovec support routines --- sys/dev/cxgb/sys/mvec.h | 135 +++++++++++++++++++++++++ sys/dev/cxgb/sys/uipc_mvec.c | 228 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 363 insertions(+) create mode 100644 sys/dev/cxgb/sys/mvec.h create mode 100644 sys/dev/cxgb/sys/uipc_mvec.c diff --git a/sys/dev/cxgb/sys/mvec.h b/sys/dev/cxgb/sys/mvec.h new file mode 100644 index 0000000..d05c380 --- /dev/null +++ b/sys/dev/cxgb/sys/mvec.h @@ -0,0 +1,135 @@ +/************************************************************************** + * + * Copyright (c) 2007, Kip Macy kmacy@freebsd.org + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + ***************************************************************************/ + +#define mtomv(m) ((struct mbuf_vec *)((m)->m_pktdat)) + +#define M_IOVEC 0x40000 /* mbuf immediate data area is used for cluster ptrs */ +#define MAX_MBUF_IOV 12 +#define MBUF_IOV_TYPE_MASK ((1<<3)-1) +#define mbuf_vec_set_type(mv, i, type) \ + (mv)->mv_vec[(i)].mi_flags = (((mv)->mv_vec[(i)].mi_flags \ + & ~MBUF_IOV_TYPE_MASK) | type) + +#define mbuf_vec_get_type(mv, i) \ + ((mv)->mv_vec[(i)].mi_flags & MBUF_IOV_TYPE_MASK) + + +struct mbuf_iovec { + uint32_t mi_flags; /* per-cluster flags */ + uint16_t mi_size; /* length of clusters */ + uint16_t mi_offset; /* data offsets of clusters */ + caddr_t mi_base; /* pointers to clusters */ +}; + +/* + * m_pktdat == 200 bytes on 64-bit arches, need to stay below that + * + * 12*16 + 8 == 200 + */ +struct mbuf_vec { + uint16_t mv_first; /* first valid cluster */ + uint16_t mv_count; /* # of clusters */ + uint32_t mv_flags; /* flags for iovec */ + struct mbuf_iovec mv_vec[MAX_MBUF_IOV]; +}; + +int _m_explode(struct mbuf *); +struct mbuf *_m_collapse(struct mbuf *, int maxbufs); +void mb_free_vec(struct mbuf *m); + + +static __inline void +m_iovinit(struct mbuf *m) +{ + struct mbuf_vec *mv = mtomv(m); + + mv->mv_first = mv->mv_count = 0; + m->m_flags |= M_IOVEC; +} + +static __inline void +m_iovappend(struct mbuf *m, void *cl, int size, int len, int offset) +{ + struct mbuf_vec *mv = mtomv(m); + struct mbuf_iovec *iov; + int idx = mv->mv_first + mv->mv_count; + + KASSERT(idx <= MAX_MBUF_IOV, ("tried to append too many clusters to mbuf iovec")); + if ((m->m_flags & M_EXT) != 0) + panic("invalid flags in %s", __func__); + + if (mv->mv_count == 0) + m->m_data = cl; + + iov = &mv->mv_vec[idx]; + iov->mi_flags = m_gettype(size); + iov->mi_base = cl; + iov->mi_size = len; + iov->mi_offset = offset; + m->m_pkthdr.len += len; + m->m_len += len; + mv->mv_count++; +} + +static __inline int +m_explode(struct mbuf *m) +{ + if ((m->m_flags & M_IOVEC) == 0) + return (0); + + return _m_explode(m); +} + +static __inline struct mbuf * +m_collapse(struct mbuf *m, int maxbufs) +{ + if (m->m_next == NULL) + return (m); + + return _m_collapse(m, maxbufs); +} + +static __inline void +m_freem_vec(struct mbuf *m) +{ + struct mbuf *n; + + while (m != NULL) { + n = m->m_next; + + if (m->m_flags & M_IOVEC) + mb_free_vec(m); + else if (m->m_flags & M_EXT) + mb_free_ext(m); + else + uma_zfree(zone_mbuf, m); + } +} diff --git a/sys/dev/cxgb/sys/uipc_mvec.c b/sys/dev/cxgb/sys/uipc_mvec.c new file mode 100644 index 0000000..42914d2 --- /dev/null +++ b/sys/dev/cxgb/sys/uipc_mvec.c @@ -0,0 +1,228 @@ +/************************************************************************** + * + * Copyright (c) 2007, Kip Macy kmacy@freebsd.org + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * + ***************************************************************************/ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include + +#include + +int +_m_explode(struct mbuf *m) +{ + int i, offset, type; + void *cl; + struct mbuf *m0, *head = NULL; + struct mbuf_vec *mv; + + mv = mtomv(m); + for (i = mv->mv_count + mv->mv_first - 1; + i > mv->mv_first; i--) { + cl = mv->mv_vec[i].mi_base; + if ((m0 = m_get(M_NOWAIT, MT_DATA)) == NULL) { + m_freem(head); + return (ENOMEM); + } + m0->m_flags = 0; + type = mbuf_vec_get_type(mv, i); + m_cljset(m0, (uint8_t *)cl, type); + m0->m_len = mv->mv_vec[i].mi_size; + + offset = mv->mv_vec[i].mi_offset; + + if (offset) + m_adj(m, offset); + + m0->m_next = head; + m->m_len -= m0->m_len; + head = m0; + } + offset = mv->mv_vec[0].mi_offset; + cl = mv->mv_vec[0].mi_base; + type = mbuf_vec_get_type(mv, 0); + m->m_flags &= ~(M_IOVEC); + m_cljset(m, cl, type); + if (offset) + m_adj(m, offset); + m->m_next = head; + + return (0); +} + +#define MAX_BUFS 36 + +struct mbuf * +_m_collapse(struct mbuf *m, int maxbufs) +{ + struct mbuf *m0, *lvec[MAX_BUFS]; + struct mbuf **mnext, **vec = &lvec[0]; + struct mbuf *mhead = NULL; + struct mbuf_vec *mv; + int i, j, max; + + if (maxbufs > MAX_BUFS) + if ((vec = malloc(maxbufs * sizeof(struct mbuf *), + M_DEVBUF, M_NOWAIT)) == NULL) + return (NULL); + + m0 = m; + for (i = 0; i < maxbufs; i++) { + if (m0 == NULL) + goto batch; + vec[i] = m0; + m0 = m0->m_next; + } + + if (i == maxbufs) + return (NULL); +batch: + max = i; + i = 0; + m0 = NULL; + mnext = NULL; + while (i < max) { + if ((vec[i]->m_flags & M_EXT) == 0) { + m0 = m_get(M_NOWAIT, MT_DATA); + } else { + m0 = vec[i]; + m0->m_flags = (vec[i]->m_flags & ~M_EXT); + } + m0->m_flags |= M_IOVEC; + if (m0 == NULL) + goto m_getfail; + if (i == 0) + mhead = m0; + if (mnext) + *mnext = m0; + mv = mtomv(m0); + mv->mv_count = mv->mv_first = 0; + for (j = 0; j < MAX_MBUF_IOV; j++, i++) { + if (vec[i]->m_flags & M_EXT) { + mv->mv_vec[j].mi_base = vec[i]->m_ext.ext_buf; + mv->mv_vec[j].mi_offset = + (vec[i]->m_ext.ext_buf - vec[i]->m_data); + mv->mv_vec[j].mi_size = vec[i]->m_ext.ext_size; + mv->mv_vec[j].mi_flags = vec[i]->m_ext.ext_type; + } else { + mv->mv_vec[j].mi_base = (caddr_t)vec[i]; + mv->mv_vec[j].mi_offset = + ((caddr_t)vec[i] - vec[i]->m_data); + mv->mv_vec[j].mi_size = MSIZE; + mv->mv_vec[j].mi_flags = EXT_MBUF; + } + } + mnext = &m0->m_next; + } + + mhead->m_flags |= (m0->m_flags & M_PKTHDR); + return (mhead); + +m_getfail: + m0 = mhead; + while (mhead) { + mhead = m0->m_next; + uma_zfree(zone_mbuf, m0); + } + return (NULL); +} + +void +mb_free_vec(struct mbuf *m) +{ + struct mbuf_vec *mv; + int i; + + KASSERT((m->m_flags & (M_EXT|M_IOVEC)) == M_IOVEC, + ("%s: M_IOVEC not set", __func__)); + + mv = mtomv(m); + KASSERT(mv->mv_count <= MAX_MBUF_IOV, + ("%s: mi_count too large %d", __func__, mv->mv_count)); + + for (i = mv->mv_first; i < mv->mv_count; i++) { + uma_zone_t zone = NULL; + int *refcnt; + int type = mbuf_vec_get_type(mv, i); + void *cl = mv->mv_vec[i].mi_base; + int size = mv->mv_vec[i].mi_size; + + zone = m_getzone(size); + refcnt = uma_find_refcnt(zone, cl); + if (*refcnt != 1 && atomic_fetchadd_int(refcnt, -1) != 1) + continue; + + switch (type) { + case EXT_PACKET: /* The packet zone is special. */ + if (*refcnt == 0) + *refcnt = 1; + uma_zfree(zone_pack, m); + return; /* Job done. */ + case EXT_CLUSTER: + case EXT_JUMBOP: + case EXT_JUMBO9: + case EXT_JUMBO16: + uma_zfree(zone, cl); + continue; + case EXT_SFBUF: + *refcnt = 0; + uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *, + refcnt)); + /* FALLTHROUGH */ + case EXT_EXTREF: +#ifdef notyet + KASSERT(m->m_ext.ext_free != NULL, + ("%s: ext_free not set", __func__)); + (*(m->m_ext.ext_free))(m->m_ext.ext_buf, + m->m_ext.ext_args); +#endif + /* + * XXX + */ + panic("unsupported mbuf_vec type: %d\n", type); + break; + default: + KASSERT(m->m_ext.ext_type == 0, + ("%s: unknown ext_type", __func__)); + + } + } + /* + * Free this mbuf back to the mbuf zone with all m_ext + * information purged. + */ + m->m_flags &= ~M_IOVEC; + uma_zfree(zone_mbuf, m); +} -- cgit v1.1