Add MALLOC_DEBUG_MAXZONES debug malloc(9) option to use multiple uma

zones for each malloc bucket size.  The purpose is to isolate
different malloc types into hash classes, so that any buffer overruns
or use-after-free will usually only affect memory from malloc types in
that hash class.  This is purely a debugging tool; by varying the hash
function and tracking which hash class was corrupted, the intersection
of the hash classes from each instance will point to a single malloc
type that is being misused.  At this point inspection or memguard(9)
can be used to catch the offending code.

Add MALLOC_DEBUG_MAXZONES=8 to -current GENERIC configuration files.
The suggestion to have this on by default came from Kostik Belousov on
-arch.

This code is based on work by Ron Steinke at Isilon Systems.

Reviewed by:    -arch (mostly silence)
Reviewed by:    zml
Approved by:    zml (mentor)

1 files changed, 124 insertions, 24 deletions

diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c
index cef4b26..963cf09 100644
--- a/sys/kern/kern_malloc.c
+++ b/sys/kern/kern_malloc.c
@@ -131,6 +131,11 @@ static int kmemcount;
 #define KMEM_ZSIZE	(KMEM_ZMAX >> KMEM_ZSHIFT)
 static uint8_t kmemsize[KMEM_ZSIZE + 1];
 
+#ifndef MALLOC_DEBUG_MAXZONES
+#define	MALLOC_DEBUG_MAXZONES	1
+#endif
+static int numzones = MALLOC_DEBUG_MAXZONES;
+
 /*
  * Small malloc(9) memory allocations are allocated from a set of UMA buckets
  * of various sizes.
@@ -142,25 +147,25 @@ static uint8_t kmemsize[KMEM_ZSIZE + 1];
 struct {
 	int kz_size;
 	char *kz_name;
-	uma_zone_t kz_zone;
+	uma_zone_t kz_zone[MALLOC_DEBUG_MAXZONES];
 } kmemzones[] = {
-	{16, "16", NULL},
-	{32, "32", NULL},
-	{64, "64", NULL},
-	{128, "128", NULL},
-	{256, "256", NULL},
-	{512, "512", NULL},
-	{1024, "1024", NULL},
-	{2048, "2048", NULL},
-	{4096, "4096", NULL},
+	{16, "16", },
+	{32, "32", },
+	{64, "64", },
+	{128, "128", },
+	{256, "256", },
+	{512, "512", },
+	{1024, "1024", },
+	{2048, "2048", },
+	{4096, "4096", },
 #if PAGE_SIZE > 4096
-	{8192, "8192", NULL},
+	{8192, "8192", },
 #if PAGE_SIZE > 8192
-	{16384, "16384", NULL},
+	{16384, "16384", },
 #if PAGE_SIZE > 16384
-	{32768, "32768", NULL},
+	{32768, "32768", },
 #if PAGE_SIZE > 32768
-	{65536, "65536", NULL},
+	{65536, "65536", },
 #if PAGE_SIZE > 65536
 #error	"Unsupported PAGE_SIZE"
 #endif	/* 65536 */
@@ -215,14 +220,16 @@ static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS);
  */
 static time_t t_malloc_fail;
 
+#if defined(MALLOC_MAKE_FAILURES) || (MALLOC_DEBUG_MAXZONES > 1)
+SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
+    "Kernel malloc debugging options");
+#endif
+
 /*
  * malloc(9) fault injection -- cause malloc failures every (n) mallocs when
  * the caller specifies M_NOWAIT.  If set to 0, no failures are caused.
  */
 #ifdef MALLOC_MAKE_FAILURES
-SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
-    "Kernel malloc debugging options");
-
 static int malloc_failure_rate;
 static int malloc_nowait_count;
 static int malloc_failure_count;
@@ -233,6 +240,60 @@ SYSCTL_INT(_debug_malloc, OID_AUTO, failure_count, CTLFLAG_RD,
     &malloc_failure_count, 0, "Number of imposed M_NOWAIT malloc failures");
 #endif
 
+/*
+ * malloc(9) uma zone separation -- sub-page buffer overruns in one
+ * malloc type will affect only a subset of other malloc types.
+ */
+#if MALLOC_DEBUG_MAXZONES > 1
+static void
+tunable_set_numzones(void)
+{
+
+	TUNABLE_INT_FETCH("debug.malloc.numzones",
+	    &numzones);
+
+	/* Sanity check the number of malloc uma zones. */
+	if (numzones <= 0)
+		numzones = 1;
+	if (numzones > MALLOC_DEBUG_MAXZONES)
+		numzones = MALLOC_DEBUG_MAXZONES;
+}
+SYSINIT(numzones, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_set_numzones, NULL);
+SYSCTL_INT(_debug_malloc, OID_AUTO, numzones, CTLFLAG_RDTUN,
+    &numzones, 0, "Number of malloc uma subzones");
+
+/*
+ * Any number that changes regularly is an okay choice for the
+ * offset.  Build numbers are pretty good of you have them.
+ */
+static u_int zone_offset = __FreeBSD_version;
+TUNABLE_INT("debug.malloc.zone_offset", &zone_offset);
+SYSCTL_UINT(_debug_malloc, OID_AUTO, zone_offset, CTLFLAG_RDTUN,
+    &zone_offset, 0, "Separate malloc types by examining the "
+    "Nth character in the malloc type short description.");
+
+static u_int
+mtp_get_subzone(const char *desc)
+{
+	size_t len;
+	u_int val;
+
+	if (desc == NULL || (len = strlen(desc)) == 0)
+		return (0);
+	val = desc[zone_offset % len];
+	return (val % numzones);
+}
+#elif MALLOC_DEBUG_MAXZONES == 0
+#error "MALLOC_DEBUG_MAXZONES must be positive."
+#else
+static inline u_int
+mtp_get_subzone(const char *desc)
+{
+
+	return (0);
+}
+#endif /* MALLOC_DEBUG_MAXZONES > 1 */
+
 int
 malloc_last_fail(void)
 {
@@ -327,6 +388,7 @@ void *
 malloc(unsigned long size, struct malloc_type *mtp, int flags)
 {
 	int indx;
+	struct malloc_type_internal *mtip;
 	caddr_t va;
 	uma_zone_t zone;
 #if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE)
@@ -374,10 +436,14 @@ malloc(unsigned long size, struct malloc_type *mtp, int flags)
 #endif
 
 	if (size <= KMEM_ZMAX) {
+		mtip = mtp->ks_handle;
 		if (size & KMEM_ZMASK)
 			size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
 		indx = kmemsize[size >> KMEM_ZSHIFT];
-		zone = kmemzones[indx].kz_zone;
+		KASSERT(mtip->mti_zone < numzones,
+		    ("mti_zone %u out of range %d",
+		    mtip->mti_zone, numzones));
+		zone = kmemzones[indx].kz_zone[mtip->mti_zone];
 #ifdef MALLOC_PROFILE
 		krequests[size >> KMEM_ZSHIFT]++;
 #endif
@@ -651,15 +717,18 @@ kmeminit(void *dummy)
 	for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) {
 		int size = kmemzones[indx].kz_size;
 		char *name = kmemzones[indx].kz_name;
+		int subzone;
 
-		kmemzones[indx].kz_zone = uma_zcreate(name, size,
+		for (subzone = 0; subzone < numzones; subzone++) {
+			kmemzones[indx].kz_zone[subzone] =
+			    uma_zcreate(name, size,
 #ifdef INVARIANTS
-		    mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
+			    mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
 #else
-		    NULL, NULL, NULL, NULL,
+			    NULL, NULL, NULL, NULL,
 #endif
-		    UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
-		    
+			    UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
+		}		    
 		for (;i <= size; i+= KMEM_ZBASE)
 			kmemsize[i >> KMEM_ZSHIFT] = indx;
 		
@@ -680,6 +749,7 @@ malloc_init(void *data)
 
 	mtip = uma_zalloc(mt_zone, M_WAITOK | M_ZERO);
 	mtp->ks_handle = mtip;
+	mtip->mti_zone = mtp_get_subzone(mtp->ks_shortdesc);
 
 	mtx_lock(&malloc_mtx);
 	mtp->ks_next = kmemstatistics;
@@ -902,7 +972,37 @@ DB_SHOW_COMMAND(malloc, db_show_malloc)
 		    (alloced - freed + 1023) / 1024, allocs);
 	}
 }
-#endif
+
+#if MALLOC_DEBUG_MAXZONES > 1
+DB_SHOW_COMMAND(multizone_matches, db_show_multizone_matches)
+{
+	struct malloc_type_internal *mtip;
+	struct malloc_type *mtp;
+	u_int subzone;
+
+	if (!have_addr) {
+		db_printf("Usage: show multizone_matches <malloc type/addr>\n");
+		return;
+	}
+	mtp = (void *)addr;
+	if (mtp->ks_magic != M_MAGIC) {
+		db_printf("Magic %lx does not match expected %x\n",
+		    mtp->ks_magic, M_MAGIC);
+		return;
+	}
+
+	mtip = mtp->ks_handle;
+	subzone = mtip->mti_zone;
+
+	for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) {
+		mtip = mtp->ks_handle;
+		if (mtip->mti_zone != subzone)
+			continue;
+		db_printf("%s\n", mtp->ks_shortdesc);
+	}
+}
+#endif /* MALLOC_DEBUG_MAXZONES > 1 */
+#endif /* DDB */
 
 #ifdef MALLOC_PROFILE