Re-enable mbtypes statistics in the mbuf allocator. I disabled these

when I changed the allocator bits. This implements per-CPU mbtypes
stats by keeping net number of decrements/increments of a given mbtype
per-CPU and then summing all of the per-CPU mbtypes to produce the total
net number of allocated mbufs of the given mbtype.
Counters are carefully balanced to avoid/prevent underflows/overflows.

mbtypes stats are re-enabled with the idea that we may occasionally
(although very rarely) observe slight inconsistencies in the stat
reporting. Most of the time, we should be fine, though.

Also make appropriate modifications to netstat(1) and systat(1) to do
the necessary reporting.

Submitted by: Jiangyi Liu <jyliu@163.net>

2 files changed, 62 insertions, 77 deletions

diff --git a/usr.bin/netstat/mbuf.c b/usr.bin/netstat/mbuf.c
index 61e3e2f..0c1fdd4 100644
--- a/usr.bin/netstat/mbuf.c
+++ b/usr.bin/netstat/mbuf.c
@@ -54,10 +54,8 @@ static const char rcsid[] =
 #define	YES	1
 typedef int bool;
 
-/* XXX: mbtypes stats temporarily disactivated. */
-#if 0
 static struct mbtypenames {
-	int	mt_type;
+	short	mt_type;
 	char	*mt_name;
 } mbtypenames[] = {
 	{ MT_DATA,	"data" },
@@ -92,7 +90,6 @@ static struct mbtypenames {
 #endif
 	{ 0, 0 }
 };
-#endif	/* 0 */
 
 /*
  * Print mbuf statistics.
@@ -102,42 +99,17 @@ mbpr(u_long mbaddr, u_long mbtaddr, u_long nmbcaddr, u_long nmbufaddr,
     u_long mblimaddr, u_long cllimaddr, u_long cpusaddr, u_long pgsaddr,
     u_long mbpaddr)
 {
-	int i, nmbufs, nmbclusters, page_size, num_objs;
+	int i, j, nmbufs, nmbclusters, page_size, num_objs;
 	u_int mbuf_limit, clust_limit;
 	u_long totspace[2], totused[2], totnum, totfree;
+	short nmbtypes;
 	size_t mlen;
+	long *mbtypes = NULL;
 	struct mbstat *mbstat = NULL;
 	struct mbpstat **mbpstat = NULL;
-
-/* XXX: mbtypes stats temporarily disabled. */
-#if 0
-	int nmbtypes;
-	size_t mbtypeslen;
 	struct mbtypenames *mp;
-	u_long *mbtypes = NULL;
 	bool *seen = NULL;
 
-	/*
-	 * XXX
-	 * We can't kread() mbtypeslen from a core image so we'll
-	 * bogusly assume it's the same as in the running kernel.
-	 */
-	if (sysctlbyname("kern.ipc.mbtypes", NULL, &mbtypeslen, NULL, 0) < 0) {
-		warn("sysctl: retrieving mbtypes length");
-		goto err;
-	}
-	if ((mbtypes = malloc(mbtypeslen)) == NULL) {
-		warn("malloc: %lu bytes for mbtypes", (u_long)mbtypeslen);
-		goto err;
-	}
-
-	nmbtypes = mbtypeslen / sizeof(*mbtypes);
-	if ((seen = calloc(nmbtypes, sizeof(*seen))) == NULL) {
-		warn("calloc");
-		goto err;
-	}
-#endif
-
 	mlen = sizeof *mbstat;
 	if ((mbstat = malloc(mlen)) == NULL) {
 		warn("malloc: cannot allocate memory for mbstat");
@@ -168,10 +140,6 @@ mbpr(u_long mbaddr, u_long mbtaddr, u_long nmbcaddr, u_long nmbufaddr,
 			goto err; 
 		if (kread(mbaddr, (char *)mbstat, sizeof mbstat))
 			goto err;
-#if 0
-		if (kread(mbtaddr, (char *)mbtypes, mbtypeslen))
-			goto err;
-#endif
 		if (kread(nmbcaddr, (char *)&nmbclusters, sizeof(int)))
 			goto err;
 		if (kread(nmbufaddr, (char *)&nmbufs, sizeof(int)))
@@ -194,13 +162,6 @@ mbpr(u_long mbaddr, u_long mbtaddr, u_long nmbcaddr, u_long nmbufaddr,
 			warn("sysctl: retrieving mbstat");
 			goto err;
 		}
-#if 0
-		if (sysctlbyname("kern.ipc.mbtypes", mbtypes, &mbtypeslen, NULL,
-		    0) < 0) {
-			warn("sysctl: retrieving mbtypes");
-			goto err;
-		}
-#endif
 		mlen = sizeof(int);
 		if (sysctlbyname("kern.ipc.nmbclusters", &nmbclusters, &mlen,
 		    NULL, 0) < 0) {
@@ -233,6 +194,16 @@ mbpr(u_long mbaddr, u_long mbtaddr, u_long nmbcaddr, u_long nmbufaddr,
 		}
 	}
 
+	nmbtypes = mbstat->m_numtypes;
+	if ((seen = calloc(nmbtypes, sizeof(*seen))) == NULL) {
+		warn("calloc: cannot allocate memory for mbtypes seen flag");
+		goto err;
+	}
+	if ((mbtypes = calloc(nmbtypes, sizeof(long *))) == NULL) { 
+		warn("calloc: cannot allocate memory for mbtypes");
+		goto err;
+	}
+
 	for (i = 0; i < num_objs; i++)
 		mbpstat[i] = mbpstat[0] + i;
 
@@ -250,6 +221,8 @@ mbpr(u_long mbaddr, u_long mbtaddr, u_long nmbcaddr, u_long nmbufaddr,
 	    (mbpstat[GENLST]->mb_mbpgs * MBPERPG));
 	totnum = mbpstat[GENLST]->mb_mbpgs * MBPERPG;
 	totfree = mbpstat[GENLST]->mb_mbfree;
+	for (j = 1; j < nmbtypes; j++)
+		mbtypes[j] += mbpstat[GENLST]->mb_mbtypes[j];
 	totspace[0] = mbpstat[GENLST]->mb_mbpgs * page_size;
 	for (i = 0; i < (num_objs - 1); i++) {
 		if (mbpstat[i]->mb_active == 0)
@@ -260,11 +233,26 @@ mbpr(u_long mbaddr, u_long mbtaddr, u_long nmbcaddr, u_long nmbufaddr,
 		totspace[0] += mbpstat[i]->mb_mbpgs * page_size;
 		totnum += mbpstat[i]->mb_mbpgs * MBPERPG;
 		totfree += mbpstat[i]->mb_mbfree;
+		for (j = 1; j < nmbtypes; j++)
+			mbtypes[j] += mbpstat[i]->mb_mbtypes[j]; 
 	}
 	totused[0] = totnum - totfree;
 	printf("\tTotal:\t\t%lu/%lu (in use/in pool)\n", totused[0], totnum);
 	printf("\tMaximum number allowed on each CPU list: %d\n", mbuf_limit);
 	printf("\tMaximum possible: %d\n", nmbufs);
+	printf("\tAllocated mbuf types:\n");
+	for (mp = mbtypenames; mp->mt_name; mp++) {
+		if (mbtypes[mp->mt_type]) {
+			seen[mp->mt_type] = YES;
+			printf("\t  %lu mbufs allocated to %s\n",
+				mbtypes[mp->mt_type], mp->mt_name);
+		}
+	}
+	for (i = 1; i < nmbtypes; i++) {
+		if (!seen[i] && mbtypes[i])
+			printf("\t  %lu mbufs allocated to <mbuf type: %d>\n",
+			    mbtypes[i], i);
+	}
 	printf("\t%lu%% of mbuf map consumed\n", ((totspace[0] * 100) / (nmbufs
 	    * MSIZE)));
 
@@ -300,12 +288,10 @@ mbpr(u_long mbaddr, u_long mbtaddr, u_long nmbcaddr, u_long nmbufaddr,
 	printf("%lu calls to protocol drain routines\n", mbstat->m_drain);
 
 err:
-#if 0
 	if (mbtypes != NULL)
 		free(mbtypes);
 	if (seen != NULL)
 		free(seen);
-#endif
 	if (mbstat != NULL)
 		free(mbstat);
 	if (mbpstat != NULL) {
diff --git a/usr.bin/systat/mbufs.c b/usr.bin/systat/mbufs.c
index 1983e0a..5dee078 100644
--- a/usr.bin/systat/mbufs.c
+++ b/usr.bin/systat/mbufs.c
@@ -50,16 +50,14 @@ static const char rcsid[] =
 #include "extern.h"
 
 static struct mbpstat **mbpstat;
+static struct mbstat *mbstat;
 static int num_objs;
+static long *m_mbtypes;
+static short nmbtypes;
 #define	GENLST	(num_objs - 1)
 
-/* XXX: mbtypes stats temporarily disabled. */
-#if 0
-static u_long *m_mbtypes;
-static int nmbtypes;
-
 static struct mtnames {
-	int mt_type;
+	short mt_type;
 	char *mt_name;
 } mtnames[] = {
 	{ MT_DATA, 	"data"},
@@ -69,9 +67,7 @@ static struct mtnames {
 	{ MT_CONTROL,	"control"},
 	{ MT_OOBDATA,	"oobdata"}
 };
-
 #define	NNAMES	(sizeof (mtnames) / sizeof (mtnames[0]))
-#endif
 
 WINDOW *
 openmbufs()
@@ -106,12 +102,24 @@ showmbufs()
 	char buf[10];
 	char *mtname;
 
-/* XXX: mbtypes stats temporarily disabled (will be back soon!) */
-#if 0
+	totfree = mbpstat[GENLST]->mb_mbfree; 
+	for (i = 1; i < nmbtypes; i++)
+		m_mbtypes[i] += mbpstat[GENLST]->mb_mbtypes[i];
+	for (i = 0; i < GENLST; i++) {
+		if (mbpstat[i]->mb_active == 0)
+			continue;
+		totfree += mbpstat[i]->mb_mbfree;
+		for (j = 1; j < nmbtypes; j++)
+			m_mbtypes[j] += mbpstat[i]->mb_mbtypes[j];
+	}
+
+	/*
+	 * Print totals for different mbuf types.
+	 */
 	for (j = 0; j < wnd->_maxy; j++) {
 		max = 0, index = -1;
 		for (i = 0; i < wnd->_maxy; i++) {
-			if (i == MT_FREE)
+			if (i == MT_NOTMBUF)
 				continue;
 			if (i >= nmbtypes)
 				break;
@@ -144,18 +152,10 @@ showmbufs()
 		wclrtoeol(wnd);
 		m_mbtypes[index] = 0;
 	}
-#endif
 
 	/*
 	 * Print total number of free mbufs.
 	 */
-	totfree = mbpstat[GENLST]->mb_mbfree; 
-	for (i = 0; i < (num_objs - 1); i++) {
-		if (mbpstat[i]->mb_active == 0)
-			continue;
-		totfree += mbpstat[i]->mb_mbfree;
-	}
-	j = 0;	/* XXX */
 	if (totfree > 0) {
 		mvwprintw(wnd, 1+j, 0, "%-10.10s", "free");
 		if (totfree > 60) {
@@ -179,19 +179,22 @@ initmbufs()
 {
 	int i;
 	size_t len;
-#if 0
-	size_t mbtypeslen;
 
-	if (sysctlbyname("kern.ipc.mbtypes", NULL, &mbtypeslen, NULL, 0) < 0) {
-		error("sysctl getting mbtypes size failed");
+	len = sizeof *mbstat;
+	if ((mbstat = malloc(len)) == NULL) {
+		error("malloc mbstat failed");
 		return 0;
 	}
-	if ((m_mbtypes = calloc(1, mbtypeslen)) == NULL) {
-		error("calloc mbtypes failed");
+	if (sysctlbyname("kern.ipc.mbstat", mbstat, &len, NULL, 0) < 0) {
+		error("sysctl retrieving mbstat");
 		return 0;
 	}
-	nmbtypes = mbtypeslen / sizeof(*m_mbtypes);
-#endif
+	nmbtypes = mbstat->m_numtypes;
+	if ((m_mbtypes = calloc(nmbtypes, sizeof(long *))) == NULL) {
+		error("calloc m_mbtypes failed");
+		return 0;
+	}
+
 	if (sysctlbyname("kern.ipc.mb_statpcpu", NULL, &len, NULL, 0) < 0) {
 		error("sysctl getting mbpstat total size failed");
 		return 0;
@@ -205,6 +208,7 @@ initmbufs()
 		error("calloc mbpstat structures failed");
 		return 0;
 	}
+
 	for (i = 0; i < num_objs; i++)
 		mbpstat[i] = mbpstat[0] + i;
 
@@ -219,9 +223,4 @@ fetchmbufs()
 	len = num_objs * sizeof(struct mbpstat);
 	if (sysctlbyname("kern.ipc.mb_statpcpu", mbpstat[0], &len, NULL, 0) < 0)
 		printw("sysctl: mbpstat: %s", strerror(errno));
-#if 0
-	len = nmbtypes * sizeof *m_mbtypes;
-	if (sysctlbyname("kern.ipc.mbtypes", m_mbtypes, &len, 0, 0) < 0)
-		printw("sysctl: mbtypes: %s", strerror(errno));
-#endif
 }