Add MALLOC_DEBUG_MAXZONES debug malloc(9) option to use multiple uma

zones for each malloc bucket size.  The purpose is to isolate
different malloc types into hash classes, so that any buffer overruns
or use-after-free will usually only affect memory from malloc types in
that hash class.  This is purely a debugging tool; by varying the hash
function and tracking which hash class was corrupted, the intersection
of the hash classes from each instance will point to a single malloc
type that is being misused.  At this point inspection or memguard(9)
can be used to catch the offending code.

Add MALLOC_DEBUG_MAXZONES=8 to -current GENERIC configuration files.
The suggestion to have this on by default came from Kostik Belousov on
-arch.

This code is based on work by Ron Steinke at Isilon Systems.

Reviewed by:    -arch (mostly silence)
Reviewed by:    zml
Approved by:    zml (mentor)

11 files changed, 147 insertions, 24 deletions

diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC
index 36ee673..a3d230e 100644
--- a/sys/amd64/conf/GENERIC
+++ b/sys/amd64/conf/GENERIC
@@ -76,6 +76,7 @@ options 	INVARIANTS		# Enable calls of extra sanity checking
 options 	INVARIANT_SUPPORT	# Extra sanity checks of internal structures, required by INVARIANTS
 options 	WITNESS			# Enable checks to detect deadlocks and cycles
 options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed
+options 	MALLOC_DEBUG_MAXZONES=8	# Separate malloc(9) zones
 
 # Make an SMP-capable kernel by default
 options 	SMP			# Symmetric MultiProcessor Kernel
diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index 0a872b2..2cf18f1 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -385,6 +385,20 @@ options 	SYSCTL_DEBUG
 options		NO_SYSCTL_DESCR
 
 #
+# MALLOC_DEBUG_MAXZONES enables multiple uma zones for malloc(9)
+# allocations that are smaller than a page.  The purpose is to isolate
+# different malloc types into hash classes, so that any buffer
+# overruns or use-after-free will usually only affect memory from
+# malloc types in that hash class.  This is purely a debugging tool;
+# by varying the hash function and tracking which hash class was
+# corrupted, the intersection of the hash classes from each instance
+# will point to a single malloc type that is being misused.  At this
+# point inspection or memguard(9) can be used to catch the offending
+# code.
+#
+options 	MALLOC_DEBUG_MAXZONES=8
+
+#
 # DEBUG_MEMGUARD builds and enables memguard(9), a replacement allocator
 # for the kernel used to detect modify-after-free scenarios.  See the
 # memguard(9) man page for more information on usage.
diff --git a/sys/conf/options b/sys/conf/options
index 9563cc8..b12290a 100644
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -586,6 +586,7 @@ VM_LEVEL_0_ORDER	opt_vm.h
 NO_SWAPPING		opt_vm.h
 MALLOC_MAKE_FAILURES	opt_vm.h
 MALLOC_PROFILE		opt_vm.h
+MALLOC_DEBUG_MAXZONES	opt_vm.h
 
 # The MemGuard replacement allocator used for tamper-after-free detection
 DEBUG_MEMGUARD		opt_vm.h
diff --git a/sys/i386/conf/GENERIC b/sys/i386/conf/GENERIC
index 24f5aab..0b456a2 100644
--- a/sys/i386/conf/GENERIC
+++ b/sys/i386/conf/GENERIC
@@ -76,6 +76,7 @@ options 	INVARIANTS		# Enable calls of extra sanity checking
 options 	INVARIANT_SUPPORT	# Extra sanity checks of internal structures, required by INVARIANTS
 options 	WITNESS			# Enable checks to detect deadlocks and cycles
 options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed
+options 	MALLOC_DEBUG_MAXZONES=8	# Separate malloc(9) zones
 
 # To make an SMP kernel, the next two lines are needed
 options 	SMP			# Symmetric MultiProcessor Kernel
diff --git a/sys/ia64/conf/GENERIC b/sys/ia64/conf/GENERIC
index cf06a29..71af42c 100644
--- a/sys/ia64/conf/GENERIC
+++ b/sys/ia64/conf/GENERIC
@@ -68,6 +68,7 @@ options 	UFS_GJOURNAL	# Enable gjournal-based UFS journaling
 options 	WITNESS		# Enable checks to detect deadlocks and cycles
 options 	WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed
 options 	_KPOSIX_PRIORITY_SCHEDULING	# Posix P1003_1B RT extensions
+options 	MALLOC_DEBUG_MAXZONES=8	# Separate malloc(9) zones
 
 # Various "busses"
 device		firewire	# FireWire bus code
diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c
index cef4b26..963cf09 100644
--- a/sys/kern/kern_malloc.c
+++ b/sys/kern/kern_malloc.c
@@ -131,6 +131,11 @@ static int kmemcount;
 #define KMEM_ZSIZE	(KMEM_ZMAX >> KMEM_ZSHIFT)
 static uint8_t kmemsize[KMEM_ZSIZE + 1];
 
+#ifndef MALLOC_DEBUG_MAXZONES
+#define	MALLOC_DEBUG_MAXZONES	1
+#endif
+static int numzones = MALLOC_DEBUG_MAXZONES;
+
 /*
  * Small malloc(9) memory allocations are allocated from a set of UMA buckets
  * of various sizes.
@@ -142,25 +147,25 @@ static uint8_t kmemsize[KMEM_ZSIZE + 1];
 struct {
 	int kz_size;
 	char *kz_name;
-	uma_zone_t kz_zone;
+	uma_zone_t kz_zone[MALLOC_DEBUG_MAXZONES];
 } kmemzones[] = {
-	{16, "16", NULL},
-	{32, "32", NULL},
-	{64, "64", NULL},
-	{128, "128", NULL},
-	{256, "256", NULL},
-	{512, "512", NULL},
-	{1024, "1024", NULL},
-	{2048, "2048", NULL},
-	{4096, "4096", NULL},
+	{16, "16", },
+	{32, "32", },
+	{64, "64", },
+	{128, "128", },
+	{256, "256", },
+	{512, "512", },
+	{1024, "1024", },
+	{2048, "2048", },
+	{4096, "4096", },
 #if PAGE_SIZE > 4096
-	{8192, "8192", NULL},
+	{8192, "8192", },
 #if PAGE_SIZE > 8192
-	{16384, "16384", NULL},
+	{16384, "16384", },
 #if PAGE_SIZE > 16384
-	{32768, "32768", NULL},
+	{32768, "32768", },
 #if PAGE_SIZE > 32768
-	{65536, "65536", NULL},
+	{65536, "65536", },
 #if PAGE_SIZE > 65536
 #error	"Unsupported PAGE_SIZE"
 #endif	/* 65536 */
@@ -215,14 +220,16 @@ static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS);
  */
 static time_t t_malloc_fail;
 
+#if defined(MALLOC_MAKE_FAILURES) || (MALLOC_DEBUG_MAXZONES > 1)
+SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
+    "Kernel malloc debugging options");
+#endif
+
 /*
  * malloc(9) fault injection -- cause malloc failures every (n) mallocs when
  * the caller specifies M_NOWAIT.  If set to 0, no failures are caused.
  */
 #ifdef MALLOC_MAKE_FAILURES
-SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0,
-    "Kernel malloc debugging options");
-
 static int malloc_failure_rate;
 static int malloc_nowait_count;
 static int malloc_failure_count;
@@ -233,6 +240,60 @@ SYSCTL_INT(_debug_malloc, OID_AUTO, failure_count, CTLFLAG_RD,
     &malloc_failure_count, 0, "Number of imposed M_NOWAIT malloc failures");
 #endif
 
+/*
+ * malloc(9) uma zone separation -- sub-page buffer overruns in one
+ * malloc type will affect only a subset of other malloc types.
+ */
+#if MALLOC_DEBUG_MAXZONES > 1
+static void
+tunable_set_numzones(void)
+{
+
+	TUNABLE_INT_FETCH("debug.malloc.numzones",
+	    &numzones);
+
+	/* Sanity check the number of malloc uma zones. */
+	if (numzones <= 0)
+		numzones = 1;
+	if (numzones > MALLOC_DEBUG_MAXZONES)
+		numzones = MALLOC_DEBUG_MAXZONES;
+}
+SYSINIT(numzones, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_set_numzones, NULL);
+SYSCTL_INT(_debug_malloc, OID_AUTO, numzones, CTLFLAG_RDTUN,
+    &numzones, 0, "Number of malloc uma subzones");
+
+/*
+ * Any number that changes regularly is an okay choice for the
+ * offset.  Build numbers are pretty good of you have them.
+ */
+static u_int zone_offset = __FreeBSD_version;
+TUNABLE_INT("debug.malloc.zone_offset", &zone_offset);
+SYSCTL_UINT(_debug_malloc, OID_AUTO, zone_offset, CTLFLAG_RDTUN,
+    &zone_offset, 0, "Separate malloc types by examining the "
+    "Nth character in the malloc type short description.");
+
+static u_int
+mtp_get_subzone(const char *desc)
+{
+	size_t len;
+	u_int val;
+
+	if (desc == NULL || (len = strlen(desc)) == 0)
+		return (0);
+	val = desc[zone_offset % len];
+	return (val % numzones);
+}
+#elif MALLOC_DEBUG_MAXZONES == 0
+#error "MALLOC_DEBUG_MAXZONES must be positive."
+#else
+static inline u_int
+mtp_get_subzone(const char *desc)
+{
+
+	return (0);
+}
+#endif /* MALLOC_DEBUG_MAXZONES > 1 */
+
 int
 malloc_last_fail(void)
 {
@@ -327,6 +388,7 @@ void *
 malloc(unsigned long size, struct malloc_type *mtp, int flags)
 {
 	int indx;
+	struct malloc_type_internal *mtip;
 	caddr_t va;
 	uma_zone_t zone;
 #if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE)
@@ -374,10 +436,14 @@ malloc(unsigned long size, struct malloc_type *mtp, int flags)
 #endif
 
 	if (size <= KMEM_ZMAX) {
+		mtip = mtp->ks_handle;
 		if (size & KMEM_ZMASK)
 			size = (size & ~KMEM_ZMASK) + KMEM_ZBASE;
 		indx = kmemsize[size >> KMEM_ZSHIFT];
-		zone = kmemzones[indx].kz_zone;
+		KASSERT(mtip->mti_zone < numzones,
+		    ("mti_zone %u out of range %d",
+		    mtip->mti_zone, numzones));
+		zone = kmemzones[indx].kz_zone[mtip->mti_zone];
 #ifdef MALLOC_PROFILE
 		krequests[size >> KMEM_ZSHIFT]++;
 #endif
@@ -651,15 +717,18 @@ kmeminit(void *dummy)
 	for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) {
 		int size = kmemzones[indx].kz_size;
 		char *name = kmemzones[indx].kz_name;
+		int subzone;
 
-		kmemzones[indx].kz_zone = uma_zcreate(name, size,
+		for (subzone = 0; subzone < numzones; subzone++) {
+			kmemzones[indx].kz_zone[subzone] =
+			    uma_zcreate(name, size,
 #ifdef INVARIANTS
-		    mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
+			    mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini,
 #else
-		    NULL, NULL, NULL, NULL,
+			    NULL, NULL, NULL, NULL,
 #endif
-		    UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
-		    
+			    UMA_ALIGN_PTR, UMA_ZONE_MALLOC);
+		}		    
 		for (;i <= size; i+= KMEM_ZBASE)
 			kmemsize[i >> KMEM_ZSHIFT] = indx;
 		
@@ -680,6 +749,7 @@ malloc_init(void *data)
 
 	mtip = uma_zalloc(mt_zone, M_WAITOK | M_ZERO);
 	mtp->ks_handle = mtip;
+	mtip->mti_zone = mtp_get_subzone(mtp->ks_shortdesc);
 
 	mtx_lock(&malloc_mtx);
 	mtp->ks_next = kmemstatistics;
@@ -902,7 +972,37 @@ DB_SHOW_COMMAND(malloc, db_show_malloc)
 		    (alloced - freed + 1023) / 1024, allocs);
 	}
 }
-#endif
+
+#if MALLOC_DEBUG_MAXZONES > 1
+DB_SHOW_COMMAND(multizone_matches, db_show_multizone_matches)
+{
+	struct malloc_type_internal *mtip;
+	struct malloc_type *mtp;
+	u_int subzone;
+
+	if (!have_addr) {
+		db_printf("Usage: show multizone_matches <malloc type/addr>\n");
+		return;
+	}
+	mtp = (void *)addr;
+	if (mtp->ks_magic != M_MAGIC) {
+		db_printf("Magic %lx does not match expected %x\n",
+		    mtp->ks_magic, M_MAGIC);
+		return;
+	}
+
+	mtip = mtp->ks_handle;
+	subzone = mtip->mti_zone;
+
+	for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) {
+		mtip = mtp->ks_handle;
+		if (mtip->mti_zone != subzone)
+			continue;
+		db_printf("%s\n", mtp->ks_shortdesc);
+	}
+}
+#endif /* MALLOC_DEBUG_MAXZONES > 1 */
+#endif /* DDB */
 
 #ifdef MALLOC_PROFILE
 
diff --git a/sys/pc98/conf/GENERIC b/sys/pc98/conf/GENERIC
index 699091e..7766716 100644
--- a/sys/pc98/conf/GENERIC
+++ b/sys/pc98/conf/GENERIC
@@ -76,6 +76,7 @@ options 	INVARIANTS		# Enable calls of extra sanity checking
 options 	INVARIANT_SUPPORT	# Extra sanity checks of internal structures, required by INVARIANTS
 options 	WITNESS			# Enable checks to detect deadlocks and cycles
 options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed
+options 	MALLOC_DEBUG_MAXZONES=8	# Separate malloc(9) zones
 
 # To make an SMP kernel, the next two lines are needed
 #options 	SMP			# Symmetric MultiProcessor Kernel
diff --git a/sys/powerpc/conf/GENERIC b/sys/powerpc/conf/GENERIC
index 1a0ede6..8bcb06a 100644
--- a/sys/powerpc/conf/GENERIC
+++ b/sys/powerpc/conf/GENERIC
@@ -74,6 +74,7 @@ options 	INVARIANTS		#Enable calls of extra sanity checking
 options 	INVARIANT_SUPPORT	#Extra sanity checks of internal structures, required by INVARIANTS
 options 	WITNESS			#Enable checks to detect deadlocks and cycles
 options 	WITNESS_SKIPSPIN	#Don't run witness on spinlocks for speed
+options 	MALLOC_DEBUG_MAXZONES=8	# Separate malloc(9) zones
 
 # To make an SMP kernel, the next line is needed
 #options 	SMP			# Symmetric MultiProcessor Kernel
diff --git a/sys/sparc64/conf/GENERIC b/sys/sparc64/conf/GENERIC
index 9ce2d6c..1e0a0f2 100644
--- a/sys/sparc64/conf/GENERIC
+++ b/sys/sparc64/conf/GENERIC
@@ -73,6 +73,7 @@ options 	INVARIANTS		# Enable calls of extra sanity checking
 options 	INVARIANT_SUPPORT	# Extra sanity checks of internal structures, required by INVARIANTS
 options 	WITNESS			# Enable checks to detect deadlocks and cycles
 options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed
+options 	MALLOC_DEBUG_MAXZONES=8	# Separate malloc(9) zones
 
 # Make an SMP-capable kernel by default
 options 	SMP			# Symmetric MultiProcessor Kernel
diff --git a/sys/sun4v/conf/GENERIC b/sys/sun4v/conf/GENERIC
index 26ec3af..f7a4e77 100644
--- a/sys/sun4v/conf/GENERIC
+++ b/sys/sun4v/conf/GENERIC
@@ -79,6 +79,7 @@ options 	DDB			# Support DDB.
 #options 	INVARIANT_SUPPORT	# Extra sanity checks of internal structures, required by INVARIANTS
 #options 	WITNESS			# Enable checks to detect deadlocks and cycles
 #options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed
+#options 	MALLOC_DEBUG_MAXZONES=8	# Separate malloc(9) zones
 #options 	DEBUG_LOCKS
 #options 	DEBUG_VFS_LOCKS
 
diff --git a/sys/sys/malloc.h b/sys/sys/malloc.h
index 030ad0e..76e94be 100644
--- a/sys/sys/malloc.h
+++ b/sys/sys/malloc.h
@@ -90,6 +90,7 @@ struct malloc_type_stats {
 struct malloc_type_internal {
 	uint32_t	mti_probes[DTMALLOC_PROBE_MAX];
 					/* DTrace probe ID array. */
+	u_char		mti_zone;
 	struct malloc_type_stats	mti_stats[MAXCPU];
 };