Our new-old mbugf allocator. This is actually something of a blast from

the past, since it returns to the old system of allocating mbufs out of
a private area rather than using the kernel malloc().  While this may seem
like a backwards step to some, the new allocator is some 20% faster than
the old one and has much better caching properties.

Written by: John Wroclawski <jtw@lcs.mit.edu>

2 files changed, 167 insertions, 29 deletions

diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c
index 744631a..9c2d315 100644
--- a/sys/kern/uipc_mbuf.c
+++ b/sys/kern/uipc_mbuf.c
@@ -31,7 +31,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
- * $Id: uipc_mbuf.c,v 1.18 1996/05/02 14:20:29 phk Exp $
+ * $Id: uipc_mbuf.c,v 1.19 1996/05/06 17:18:12 phk Exp $
  */
 
 #include <sys/param.h>
@@ -56,6 +56,7 @@ SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbinit, NULL)
 struct mbuf *mbutl;
 char	*mclrefcnt;
 struct mbstat mbstat;
+struct mbuf *mmbfree;
 union mcluster *mclfree;
 int	max_linkhdr;
 int	max_protohdr;
@@ -71,12 +72,17 @@ mbinit(dummy)
 {
 	int s;
 
+#define NMB_INIT	16
 #if MCLBYTES < 4096
 #define NCL_INIT	(4096/MCLBYTES)
 #else
 #define NCL_INIT	1
 #endif
+
+	mmbfree = NULL; mclfree = NULL;
 	s = splimp();
+	if (m_mballoc(NMB_INIT, M_DONTWAIT) == 0)
+		goto bad;
 	if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0)
 		goto bad;
 	splx(s);
@@ -86,6 +92,47 @@ bad:
 }
 
 /*
+ * Allocate at least nmb mbufs and place on mbuf free list.
+ * Must be called at splimp.
+ */
+/* ARGSUSED */
+int
+m_mballoc(nmb, nowait)
+	register int nmb;
+	int nowait;
+{
+	register caddr_t p;
+	register int i;
+	int nbytes;
+
+	/* Once we run out of map space, it will be impossible to get
+	 * any more (nothing is ever freed back to the map) (XXX which
+	 * is dumb). (however you are not dead as m_reclaim might
+	 * still be able to free a substantial amount of space).
+	 */
+	if (mb_map_full)
+		return (0);
+
+	nbytes = round_page(nmb * MSIZE);
+	p = (caddr_t)kmem_malloc(mb_map, nbytes, nowait ? M_NOWAIT : M_WAITOK);
+	/*
+	 * Either the map is now full, or this is nowait and there
+	 * are no pages left.
+	 */
+	if (p == NULL)
+		return (0);
+
+	nmb = nbytes / MSIZE;
+	for (i = 0; i < nmb; i++) {
+		((struct mbuf *)p)->m_next = mmbfree;
+		mmbfree = (struct mbuf *)p;
+		p += MSIZE;
+	}
+	mbstat.m_mbufs += nmb;
+	return (1);
+}
+
+/*
  * Allocate some number of mbuf clusters
  * and place on cluster free list.
  * Must be called at splimp.
@@ -355,6 +402,61 @@ nospace:
 }
 
 /*
+ * Copy an entire packet, including header (which must be present).
+ * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
+ */
+struct mbuf *
+m_copypacket(m, how)
+	struct mbuf *m;
+	int how;
+{
+	struct mbuf *top, *n, *o;
+
+	MGET(n, how, m->m_type);
+	top = n;
+	if (!n)
+		goto nospace;
+
+	M_COPY_PKTHDR(n, m);
+	n->m_len = m->m_len;
+	if (m->m_flags & M_EXT) {
+		n->m_data = m->m_data;
+		mclrefcnt[mtocl(m->m_ext.ext_buf)]++;
+		n->m_ext = m->m_ext;
+		n->m_flags |= M_EXT;
+	} else {
+		bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
+	}
+
+	m = m->m_next;
+	while (m) {
+		MGET(o, how, m->m_type);
+		if (!o)
+			goto nospace;
+
+		n->m_next = o;
+		n = n->m_next;
+
+		n->m_len = m->m_len;
+		if (m->m_flags & M_EXT) {
+			n->m_data = m->m_data;
+			mclrefcnt[mtocl(m->m_ext.ext_buf)]++;
+			n->m_ext = m->m_ext;
+			n->m_flags |= M_EXT;
+		} else {
+			bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
+		}
+
+		m = m->m_next;
+	}
+	return top;
+nospace:
+	m_freem(top);
+	MCFail++;
+	return 0;
+}
+
+/*
  * Copy data from an mbuf chain starting "off" bytes from the beginning,
  * continuing for "len" bytes, into the indicated buffer.
  */
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 385eee0..8d090d5 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -31,7 +31,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)mbuf.h	8.5 (Berkeley) 2/19/95
- * $Id: mbuf.h,v 1.15 1996/02/25 23:34:03 hsu Exp $
+ * $Id: mbuf.h,v 1.14 1996/03/11 02:14:16 hsu Exp $
  */
 
 #ifndef _SYS_MBUF_H_
@@ -149,6 +149,17 @@ struct mbuf {
 #define	M_DONTWAIT	M_NOWAIT
 #define	M_WAIT		M_WAITOK
 
+/* Freelists:
+ *
+ * Normal mbuf clusters are normally treated as character arrays
+ * after allocation, but use the first word of the buffer as a free list
+ * pointer while on the free list.
+ */
+union mcluster {
+	union	mcluster *mcl_next;
+	char	mcl_buf[MCLBYTES];
+};
+
 /*
  * mbuf utility macros:
  *
@@ -173,29 +184,43 @@ struct mbuf {
  * and internal data.
  */
 #define	MGET(m, how, type) { \
-	MALLOC((m), struct mbuf *, MSIZE, mbtypes[type], (how)); \
-	if (m) { \
+	  int _ms = splimp(); \
+	  if (mmbfree == 0) \
+		(void)m_mballoc(1, (how)); \
+	  if (((m) = mmbfree) != 0) { \
+		mmbfree = (m)->m_next; \
+		mbstat.m_mtypes[MT_FREE]--; \
 		(m)->m_type = (type); \
-		MBUFLOCK(mbstat.m_mtypes[type]++;) \
+		mbstat.m_mtypes[type]++; \
 		(m)->m_next = (struct mbuf *)NULL; \
 		(m)->m_nextpkt = (struct mbuf *)NULL; \
 		(m)->m_data = (m)->m_dat; \
 		(m)->m_flags = 0; \
-	} else \
+		splx(_ms); \
+	} else { \
+		splx(_ms); \
 		(m) = m_retry((how), (type)); \
+	} \
 }
 
 #define	MGETHDR(m, how, type) { \
-	MALLOC((m), struct mbuf *, MSIZE, mbtypes[type], (how)); \
-	if (m) { \
+	  int _ms = splimp(); \
+	  if (mmbfree == 0) \
+		(void)m_mballoc(1, (how)); \
+	  if (((m) = mmbfree) != 0) { \
+		mmbfree = (m)->m_next; \
+		mbstat.m_mtypes[MT_FREE]--; \
 		(m)->m_type = (type); \
-		MBUFLOCK(mbstat.m_mtypes[type]++;) \
+		mbstat.m_mtypes[type]++; \
 		(m)->m_next = (struct mbuf *)NULL; \
 		(m)->m_nextpkt = (struct mbuf *)NULL; \
 		(m)->m_data = (m)->m_pktdat; \
 		(m)->m_flags = M_PKTHDR; \
-	} else \
+		splx(_ms); \
+	} else { \
+		splx(_ms); \
 		(m) = m_retryhdr((how), (type)); \
+	} \
 }
 
 /*
@@ -205,16 +230,7 @@ struct mbuf {
  * the flag M_EXT is set upon success.
  * MCLFREE releases a reference to a cluster allocated by MCLALLOC,
  * freeing the cluster if the reference count has reached 0.
- *
- * Normal mbuf clusters are normally treated as character arrays
- * after allocation, but use the first word of the buffer as a free list
- * pointer while on the free list.
  */
-union mcluster {
-	union	mcluster *mcl_next;
-	char	mcl_buf[MCLBYTES];
-};
-
 #define	MCLALLOC(p, how) \
 	MBUFLOCK( \
 	  if (mclfree == 0) \
@@ -256,21 +272,38 @@ union mcluster {
 		if ((m)->m_ext.ext_free) \
 			(*((m)->m_ext.ext_free))((m)->m_ext.ext_buf, \
 			    (m)->m_ext.ext_size); \
-		else \
-			MCLFREE((m)->m_ext.ext_buf); \
+		else { \
+			char *p = (m)->m_ext.ext_buf; \
+			if (--mclrefcnt[mtocl(p)] == 0) { \
+			((union mcluster *)(p))->mcl_next = mclfree; \
+			mclfree = (union mcluster *)(p); \
+			mbstat.m_clfree++; \
+		} \
 	  } \
 	  (n) = (m)->m_next; \
-	  FREE((m), mbtypes[(m)->m_type]); \
+	  (m)->m_type = MT_FREE; \
+	  mbstat.m_mtypes[MT_FREE]++; \
+	  (m)->m_next = mmbfree; \
+	  mmbfree = (m); \
 	}
 #else /* notyet */
 #define	MFREE(m, nn) \
-	{ MBUFLOCK(mbstat.m_mtypes[(m)->m_type]--;) \
-	  if ((m)->m_flags & M_EXT) { \
-		MCLFREE((m)->m_ext.ext_buf); \
-	  } \
-	  (nn) = (m)->m_next; \
-	  FREE((m), mbtypes[(m)->m_type]); \
-	}
+	MBUFLOCK ( \
+		mbstat.m_mtypes[(m)->m_type]--; \
+		if ((m)->m_flags & M_EXT) { \
+			char *p = (m)->m_ext.ext_buf; \
+			if (--mclrefcnt[mtocl(p)] == 0) { \
+				((union mcluster *)(p))->mcl_next = mclfree; \
+				mclfree = (union mcluster *)(p); \
+				mbstat.m_clfree++; \
+			} \
+		} \
+		(nn) = (m)->m_next; \
+		(m)->m_type = MT_FREE; \
+		mbstat.m_mtypes[MT_FREE]++; \
+		(m)->m_next = mmbfree; \
+		mmbfree = (m); \
+	)
 #endif
 
 /*
@@ -361,6 +394,7 @@ extern struct mbuf *mbutl;		/* virtual address of mclusters */
 extern char	*mclrefcnt;		/* cluster reference counts */
 extern struct mbstat mbstat;
 extern int	nmbclusters;
+extern struct mbuf *mmbfree;
 extern union mcluster *mclfree;
 extern int	max_linkhdr;		/* largest link-level header */
 extern int	max_protohdr;		/* largest protocol header */
@@ -369,6 +403,7 @@ extern int	max_datalen;		/* MHLEN - max_hdr */
 extern int	mbtypes[];		/* XXX */
 
 struct	mbuf *m_copym __P((struct mbuf *, int, int, int));
+struct	mbuf *m_copypacket __P((struct mbuf *, int));
 struct	mbuf *m_devget __P((char *, int, int, struct ifnet *,
 			    void (*copy)(char *, caddr_t, u_int)));
 struct	mbuf *m_free __P((struct mbuf *));
@@ -382,6 +417,7 @@ struct	mbuf *m_retryhdr __P((int, int));
 struct	mbuf *m_split __P((struct mbuf *,int,int));
 void	m_adj __P((struct mbuf *, int));
 void	m_cat __P((struct mbuf *,struct mbuf *));
+int	m_mballoc __P((int, int));
 int	m_clalloc __P((int, int));
 void	m_copyback __P((struct mbuf *, int, int, caddr_t));
 void	m_copydata __P((struct mbuf *,int,int,caddr_t));