Introduce numerous SMP friendly changes to the mbuf allocator. Namely,

introduce a modified allocation mechanism for mbufs and mbuf clusters; one
which can scale under SMP and which offers the possibility of resource
reclamation to be implemented in the future. Notable advantages:

 o Reduce contention for SMP by offering per-CPU pools and locks.
 o Better use of data cache due to per-CPU pools.
 o Much less code cache pollution due to excessively large allocation macros.
 o Framework for `grouping' objects from same page together so as to be able
   to possibly free wired-down pages back to the system if they are no longer
   needed by the network stacks.

 Additional things changed with this addition:

  - Moved some mbuf specific declarations and initializations from
    sys/conf/param.c into mbuf-specific code where they belong.
  - m_getclr() has been renamed to m_get_clrd() because the old name is really
    confusing. m_getclr() HAS been preserved though and is defined to the new
    name. No tree sweep has been done "to change the interface," as the old
    name will continue to be supported and is not depracated. The change was
    merely done because m_getclr() sounds too much like "m_get a cluster."
  - TEMPORARILY disabled mbtypes statistics displaying in netstat(1) and
    systat(1) (see TODO below).
  - Fixed systat(1) to display number of "free mbufs" based on new per-CPU
    stat structures.
  - Fixed netstat(1) to display new per-CPU stats based on sysctl-exported
    per-CPU stat structures. All infos are fetched via sysctl.

 TODO (in order of priority):

  - Re-enable mbtypes statistics in both netstat(1) and systat(1) after
    introducing an SMP friendly way to collect the mbtypes stats under the
    already introduced per-CPU locks (i.e. hopefully don't use atomic() - it
    seems too costly for a mere stat update, especially when other locks are
    already present).
  - Optionally have systat(1) display not only "total free mbufs" but also
    "total free mbufs per CPU pool."
  - Fix minor length-fetching issues in netstat(1) related to recently
    re-enabled option to read mbuf stats from a core file.
  - Move reference counters at least for mbuf clusters into an unused portion
    of the cluster itself, to save space and need to allocate a counter.
  - Look into introducing resource freeing possibly from a kproc.

Reviewed by (in parts): jlemon, jake, silby, terry
Tested by: jlemon (Intel & Alpha), mjacob (Intel & Alpha)
Preliminary performance measurements: jlemon (and me, obviously)
URL: http://people.freebsd.org/~bmilekic/mb_alloc/

1 files changed, 54 insertions, 22 deletions

diff --git a/usr.bin/systat/mbufs.c b/usr.bin/systat/mbufs.c
index c4957b9..f793053 100644
--- a/usr.bin/systat/mbufs.c
+++ b/usr.bin/systat/mbufs.c
@@ -49,7 +49,12 @@ static const char rcsid[] =
 #include "systat.h"
 #include "extern.h"
 
-static struct mbstat *mb;
+static struct mbpstat **mbpstat;
+static int num_objs, ncpu;
+#define	GENLST	(num_objs - 1)
+
+/* XXX: mbtypes stats temporarily disabled. */
+#if 0
 static u_long *m_mbtypes;
 static int nmbtypes;
 
@@ -66,6 +71,7 @@ static struct mtnames {
 };
 
 #define	NNAMES	(sizeof (mtnames) / sizeof (mtnames[0]))
+#endif
 
 WINDOW *
 openmbufs()
@@ -95,12 +101,13 @@ labelmbufs()
 void
 showmbufs()
 {
-	register int i, j, max, index;
+	int i, j, max, index;
+	u_long totfree;
 	char buf[10];
 	char *mtname;
 
-	if (mb == 0)
-		return;
+/* XXX: mbtypes stats temporarily disabled (will be back soon!) */
+#if 0
 	for (j = 0; j < wnd->_maxy; j++) {
 		max = 0, index = -1;
 		for (i = 0; i < wnd->_maxy; i++) {
@@ -135,19 +142,27 @@ showmbufs()
 			while (max--)
 				waddch(wnd, 'X');
 		wclrtoeol(wnd);
-		mb->m_mbufs -= m_mbtypes[index];
 		m_mbtypes[index] = 0;
 	}
-	if (mb->m_mbufs) {
+#endif
+
+	/*
+	 * Print total number of free mbufs.
+	 */
+	totfree = mbpstat[GENLST]->mb_mbfree; 
+	for (i = 0; i < ncpu; i++)
+		totfree += mbpstat[i]->mb_mbfree;
+	j = 0;	/* XXX */
+	if (totfree > 0) {
 		mvwprintw(wnd, 1+j, 0, "%-10.10s", "free");
-		if (mb->m_mbufs > 60) {
-			snprintf(buf, sizeof(buf), " %ld", mb->m_mbufs);
-			mb->m_mbufs = 60;
-			while (mb->m_mbufs--)
+		if (totfree > 60) {
+			snprintf(buf, sizeof(buf), " %lu", totfree);
+			totfree = 60;
+			while(totfree--)
 				waddch(wnd, 'X');
 			waddstr(wnd, buf);
 		} else {
-			while(mb->m_mbufs--)
+			while(totfree--)
 				waddch(wnd, 'X');
 		}
 		wclrtoeol(wnd);
@@ -159,7 +174,10 @@ showmbufs()
 int
 initmbufs()
 {
-	size_t len, mbtypeslen;
+	int i;
+	size_t len;
+#if 0
+	size_t mbtypeslen;
 
 	if (sysctlbyname("kern.ipc.mbtypes", NULL, &mbtypeslen, NULL, 0) < 0) {
 		error("sysctl getting mbtypes size failed");
@@ -170,15 +188,28 @@ initmbufs()
 		return 0;
 	}
 	nmbtypes = mbtypeslen / sizeof(*m_mbtypes);
-
-	len = 0;
-	if (sysctlbyname("kern.ipc.mbstat", 0, &len, 0, 0) < 0) {
-		error("sysctl getting mbstat size failed");
+#endif
+	len = sizeof(int);
+	if (sysctlbyname("kern.smp.cpus", &ncpu, &len, NULL, 0) < 0) {
+		error("sysctl getting number of cpus");
+		return 0;
+	}
+	if (sysctlbyname("kern.ipc.mb_statpcpu", NULL, &len, NULL, 0) < 0) {
+		error("sysctl getting mbpstat total size failed");
 		return 0;
 	}
+	num_objs = (int)(len / sizeof(struct mbpstat));
+	if ((mbpstat = calloc(num_objs, sizeof(struct mbpstat *))) == NULL) {
+		error("calloc mbpstat pointers failed");
+		return 0;
+	}
+	if ((mbpstat[0] = calloc(num_objs, sizeof(struct mbpstat))) == NULL) {
+		error("calloc mbpstat structures failed");
+		return 0;
+	}
+	for (i = 0; i < num_objs; i++)
+		mbpstat[i] = mbpstat[0] + i;
 
-	if (mb == 0)
-		mb = (struct mbstat *)calloc(1, sizeof *mb);
 	return 1;
 }
 
@@ -187,11 +218,12 @@ fetchmbufs()
 {
 	size_t len;
 
-	len = sizeof *mb;
-	if (sysctlbyname("kern.ipc.mbstat", mb, &len, 0, 0) < 0)
-		printw("sysctl: mbstat: %s", strerror(errno));
-
+	len = num_objs * sizeof(struct mbpstat);
+	if (sysctlbyname("kern.ipc.mb_statpcpu", mbpstat[0], &len, NULL, 0) < 0)
+		printw("sysctl: mbpstat: %s", strerror(errno));
+#if 0
 	len = nmbtypes * sizeof *m_mbtypes;
 	if (sysctlbyname("kern.ipc.mbtypes", m_mbtypes, &len, 0, 0) < 0)
 		printw("sysctl: mbtypes: %s", strerror(errno));
+#endif
 }