o Gather all mbuf(9) allocation functions into kern_mbuf.c, and all

  mbuf(9) manipulation functions into uipc_mbuf.c.  This looks like
  the initial intent, but had diffused in the last decade.

o Gather all declarations in mbuf.h in one place and sort them.

o Uninline m_clget() and m_cljget().

There are no functional changes in this patch.

The patch comes from a larger version, where all mbuf(9) allocation was
uninlined, which allowed to make mbuf(9) UMA zones private to kern_mbuf.c.
The performance impact of the total uninlining is still unclear, so we
are holding on now with larger version.

Together with:	melifaro, olivier

1 files changed, 22 insertions, 263 deletions

diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c
index 0b0b697..143aea7 100644
--- a/sys/kern/uipc_mbuf.c
+++ b/sys/kern/uipc_mbuf.c
@@ -48,6 +48,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/protosw.h>
 #include <sys/uio.h>
 
+#include <security/mac/mac_framework.h>
+
 int	max_linkhdr;
 int	max_protohdr;
 int	max_hdr;
@@ -132,269 +134,6 @@ CTASSERT(sizeof(m_assertbuf.m_stailqpkt) == sizeof(m_assertbuf.m_nextpkt));
 #endif
 
 /*
- * m_get2() allocates minimum mbuf that would fit "size" argument.
- */
-struct mbuf *
-m_get2(int size, int how, short type, int flags)
-{
-	struct mb_args args;
-	struct mbuf *m, *n;
-
-	args.flags = flags;
-	args.type = type;
-
-	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
-		return (uma_zalloc_arg(zone_mbuf, &args, how));
-	if (size <= MCLBYTES)
-		return (uma_zalloc_arg(zone_pack, &args, how));
-
-	if (size > MJUMPAGESIZE)
-		return (NULL);
-
-	m = uma_zalloc_arg(zone_mbuf, &args, how);
-	if (m == NULL)
-		return (NULL);
-
-	n = uma_zalloc_arg(zone_jumbop, m, how);
-	if (n == NULL) {
-		uma_zfree(zone_mbuf, m);
-		return (NULL);
-	}
-
-	return (m);
-}
-
-/*
- * m_getjcl() returns an mbuf with a cluster of the specified size attached.
- * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
- */
-struct mbuf *
-m_getjcl(int how, short type, int flags, int size)
-{
-	struct mb_args args;
-	struct mbuf *m, *n;
-	uma_zone_t zone;
-
-	if (size == MCLBYTES)
-		return m_getcl(how, type, flags);
-
-	args.flags = flags;
-	args.type = type;
-
-	m = uma_zalloc_arg(zone_mbuf, &args, how);
-	if (m == NULL)
-		return (NULL);
-
-	zone = m_getzone(size);
-	n = uma_zalloc_arg(zone, m, how);
-	if (n == NULL) {
-		uma_zfree(zone_mbuf, m);
-		return (NULL);
-	}
-	return (m);
-}
-
-/*
- * Allocate a given length worth of mbufs and/or clusters (whatever fits
- * best) and return a pointer to the top of the allocated chain.  If an
- * existing mbuf chain is provided, then we will append the new chain
- * to the existing one but still return the top of the newly allocated
- * chain.
- */
-struct mbuf *
-m_getm2(struct mbuf *m, int len, int how, short type, int flags)
-{
-	struct mbuf *mb, *nm = NULL, *mtail = NULL;
-
-	KASSERT(len >= 0, ("%s: len is < 0", __func__));
-
-	/* Validate flags. */
-	flags &= (M_PKTHDR | M_EOR);
-
-	/* Packet header mbuf must be first in chain. */
-	if ((flags & M_PKTHDR) && m != NULL)
-		flags &= ~M_PKTHDR;
-
-	/* Loop and append maximum sized mbufs to the chain tail. */
-	while (len > 0) {
-		if (len > MCLBYTES)
-			mb = m_getjcl(how, type, (flags & M_PKTHDR),
-			    MJUMPAGESIZE);
-		else if (len >= MINCLSIZE)
-			mb = m_getcl(how, type, (flags & M_PKTHDR));
-		else if (flags & M_PKTHDR)
-			mb = m_gethdr(how, type);
-		else
-			mb = m_get(how, type);
-
-		/* Fail the whole operation if one mbuf can't be allocated. */
-		if (mb == NULL) {
-			if (nm != NULL)
-				m_freem(nm);
-			return (NULL);
-		}
-
-		/* Book keeping. */
-		len -= M_SIZE(mb);
-		if (mtail != NULL)
-			mtail->m_next = mb;
-		else
-			nm = mb;
-		mtail = mb;
-		flags &= ~M_PKTHDR;	/* Only valid on the first mbuf. */
-	}
-	if (flags & M_EOR)
-		mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
-
-	/* If mbuf was supplied, append new chain to the end of it. */
-	if (m != NULL) {
-		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
-			;
-		mtail->m_next = nm;
-		mtail->m_flags &= ~M_EOR;
-	} else
-		m = nm;
-
-	return (m);
-}
-
-/*
- * Free an entire chain of mbufs and associated external buffers, if
- * applicable.
- */
-void
-m_freem(struct mbuf *mb)
-{
-
-	while (mb != NULL)
-		mb = m_free(mb);
-}
-
-/*-
- * Configure a provided mbuf to refer to the provided external storage
- * buffer and setup a reference count for said buffer.  If the setting
- * up of the reference count fails, the M_EXT bit will not be set.  If
- * successfull, the M_EXT bit is set in the mbuf's flags.
- *
- * Arguments:
- *    mb     The existing mbuf to which to attach the provided buffer.
- *    buf    The address of the provided external storage buffer.
- *    size   The size of the provided buffer.
- *    freef  A pointer to a routine that is responsible for freeing the
- *           provided external storage buffer.
- *    args   A pointer to an argument structure (of any type) to be passed
- *           to the provided freef routine (may be NULL).
- *    flags  Any other flags to be passed to the provided mbuf.
- *    type   The type that the external storage buffer should be
- *           labeled with.
- *
- * Returns:
- *    Nothing.
- */
-int
-m_extadd(struct mbuf *mb, caddr_t buf, u_int size,
-    void (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2,
-    int flags, int type, int wait)
-{
-	KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
-
-	if (type != EXT_EXTREF)
-		mb->m_ext.ext_cnt = uma_zalloc(zone_ext_refcnt, wait);
-
-	if (mb->m_ext.ext_cnt == NULL)
-		return (ENOMEM);
-
-	*(mb->m_ext.ext_cnt) = 1;
-	mb->m_flags |= (M_EXT | flags);
-	mb->m_ext.ext_buf = buf;
-	mb->m_data = mb->m_ext.ext_buf;
-	mb->m_ext.ext_size = size;
-	mb->m_ext.ext_free = freef;
-	mb->m_ext.ext_arg1 = arg1;
-	mb->m_ext.ext_arg2 = arg2;
-	mb->m_ext.ext_type = type;
-	mb->m_ext.ext_flags = 0;
-
-	return (0);
-}
-
-/*
- * Non-directly-exported function to clean up after mbufs with M_EXT
- * storage attached to them if the reference count hits 1.
- */
-void
-mb_free_ext(struct mbuf *m)
-{
-	int freembuf;
-
-	KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m));
-
-	/*
-	 * Check if the header is embedded in the cluster.
-	 */
-	freembuf = (m->m_flags & M_NOFREE) ? 0 : 1;
-
-	switch (m->m_ext.ext_type) {
-	case EXT_SFBUF:
-		sf_ext_free(m->m_ext.ext_arg1, m->m_ext.ext_arg2);
-		break;
-	case EXT_SFBUF_NOCACHE:
-		sf_ext_free_nocache(m->m_ext.ext_arg1, m->m_ext.ext_arg2);
-		break;
-	default:
-		KASSERT(m->m_ext.ext_cnt != NULL,
-		    ("%s: no refcounting pointer on %p", __func__, m));
-		/* 
-		 * Free attached storage if this mbuf is the only
-		 * reference to it.
-		 */
-		if (*(m->m_ext.ext_cnt) != 1) {
-			if (atomic_fetchadd_int(m->m_ext.ext_cnt, -1) != 1)
-				break;
-		}
-
-		switch (m->m_ext.ext_type) {
-		case EXT_PACKET:	/* The packet zone is special. */
-			if (*(m->m_ext.ext_cnt) == 0)
-				*(m->m_ext.ext_cnt) = 1;
-			uma_zfree(zone_pack, m);
-			return;		/* Job done. */
-		case EXT_CLUSTER:
-			uma_zfree(zone_clust, m->m_ext.ext_buf);
-			break;
-		case EXT_JUMBOP:
-			uma_zfree(zone_jumbop, m->m_ext.ext_buf);
-			break;
-		case EXT_JUMBO9:
-			uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
-			break;
-		case EXT_JUMBO16:
-			uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
-			break;
-		case EXT_NET_DRV:
-		case EXT_MOD_TYPE:
-		case EXT_DISPOSABLE:
-			*(m->m_ext.ext_cnt) = 0;
-			uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *,
-				m->m_ext.ext_cnt));
-			/* FALLTHROUGH */
-		case EXT_EXTREF:
-			KASSERT(m->m_ext.ext_free != NULL,
-				("%s: ext_free not set", __func__));
-			(*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1,
-			    m->m_ext.ext_arg2);
-			break;
-		default:
-			KASSERT(m->m_ext.ext_type == 0,
-				("%s: unknown ext_type", __func__));
-		}
-	}
-
-	if (freembuf)
-		uma_zfree(zone_mbuf, m);
-}
-
-/*
  * Attach the cluster from *m to *n, set up m_ext in *n
  * and bump the refcount of the cluster.
  */
@@ -534,6 +273,26 @@ m_sanity(struct mbuf *m0, int sanitize)
 #undef	M_SANITY_ACTION
 }
 
+/*
+ * Non-inlined part of m_init().
+ */
+int
+m_pkthdr_init(struct mbuf *m, int how)
+{
+#ifdef MAC
+	int error;
+#endif
+	m->m_data = m->m_pktdat;
+	bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
+#ifdef MAC
+	/* If the label init fails, fail the alloc */
+	error = mac_mbuf_init(m, how);
+	if (error)
+		return (error);
+#endif
+
+	return (0);
+}
 
 /*
  * "Move" mbuf pkthdr from "from" to "to".