diff options
| -rw-r--r-- | sys/amd64/conf/GENERIC | 1 | ||||
| -rw-r--r-- | sys/conf/NOTES | 14 | ||||
| -rw-r--r-- | sys/conf/options | 1 | ||||
| -rw-r--r-- | sys/i386/conf/GENERIC | 1 | ||||
| -rw-r--r-- | sys/ia64/conf/GENERIC | 1 | ||||
| -rw-r--r-- | sys/kern/kern_malloc.c | 148 | ||||
| -rw-r--r-- | sys/pc98/conf/GENERIC | 1 | ||||
| -rw-r--r-- | sys/powerpc/conf/GENERIC | 1 | ||||
| -rw-r--r-- | sys/sparc64/conf/GENERIC | 1 | ||||
| -rw-r--r-- | sys/sun4v/conf/GENERIC | 1 | ||||
| -rw-r--r-- | sys/sys/malloc.h | 1 |
11 files changed, 147 insertions, 24 deletions
diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC index 36ee673..a3d230e 100644 --- a/sys/amd64/conf/GENERIC +++ b/sys/amd64/conf/GENERIC @@ -76,6 +76,7 @@ options INVARIANTS # Enable calls of extra sanity checking options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS options WITNESS # Enable checks to detect deadlocks and cycles options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # Make an SMP-capable kernel by default options SMP # Symmetric MultiProcessor Kernel diff --git a/sys/conf/NOTES b/sys/conf/NOTES index 0a872b2..2cf18f1 100644 --- a/sys/conf/NOTES +++ b/sys/conf/NOTES @@ -385,6 +385,20 @@ options SYSCTL_DEBUG options NO_SYSCTL_DESCR # +# MALLOC_DEBUG_MAXZONES enables multiple uma zones for malloc(9) +# allocations that are smaller than a page. The purpose is to isolate +# different malloc types into hash classes, so that any buffer +# overruns or use-after-free will usually only affect memory from +# malloc types in that hash class. This is purely a debugging tool; +# by varying the hash function and tracking which hash class was +# corrupted, the intersection of the hash classes from each instance +# will point to a single malloc type that is being misused. At this +# point inspection or memguard(9) can be used to catch the offending +# code. +# +options MALLOC_DEBUG_MAXZONES=8 + +# # DEBUG_MEMGUARD builds and enables memguard(9), a replacement allocator # for the kernel used to detect modify-after-free scenarios. See the # memguard(9) man page for more information on usage. diff --git a/sys/conf/options b/sys/conf/options index 9563cc8..b12290a 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -586,6 +586,7 @@ VM_LEVEL_0_ORDER opt_vm.h NO_SWAPPING opt_vm.h MALLOC_MAKE_FAILURES opt_vm.h MALLOC_PROFILE opt_vm.h +MALLOC_DEBUG_MAXZONES opt_vm.h # The MemGuard replacement allocator used for tamper-after-free detection DEBUG_MEMGUARD opt_vm.h diff --git a/sys/i386/conf/GENERIC b/sys/i386/conf/GENERIC index 24f5aab..0b456a2 100644 --- a/sys/i386/conf/GENERIC +++ b/sys/i386/conf/GENERIC @@ -76,6 +76,7 @@ options INVARIANTS # Enable calls of extra sanity checking options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS options WITNESS # Enable checks to detect deadlocks and cycles options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # To make an SMP kernel, the next two lines are needed options SMP # Symmetric MultiProcessor Kernel diff --git a/sys/ia64/conf/GENERIC b/sys/ia64/conf/GENERIC index cf06a29..71af42c 100644 --- a/sys/ia64/conf/GENERIC +++ b/sys/ia64/conf/GENERIC @@ -68,6 +68,7 @@ options UFS_GJOURNAL # Enable gjournal-based UFS journaling options WITNESS # Enable checks to detect deadlocks and cycles options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed options _KPOSIX_PRIORITY_SCHEDULING # Posix P1003_1B RT extensions +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # Various "busses" device firewire # FireWire bus code diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c index cef4b26..963cf09 100644 --- a/sys/kern/kern_malloc.c +++ b/sys/kern/kern_malloc.c @@ -131,6 +131,11 @@ static int kmemcount; #define KMEM_ZSIZE (KMEM_ZMAX >> KMEM_ZSHIFT) static uint8_t kmemsize[KMEM_ZSIZE + 1]; +#ifndef MALLOC_DEBUG_MAXZONES +#define MALLOC_DEBUG_MAXZONES 1 +#endif +static int numzones = MALLOC_DEBUG_MAXZONES; + /* * Small malloc(9) memory allocations are allocated from a set of UMA buckets * of various sizes. @@ -142,25 +147,25 @@ static uint8_t kmemsize[KMEM_ZSIZE + 1]; struct { int kz_size; char *kz_name; - uma_zone_t kz_zone; + uma_zone_t kz_zone[MALLOC_DEBUG_MAXZONES]; } kmemzones[] = { - {16, "16", NULL}, - {32, "32", NULL}, - {64, "64", NULL}, - {128, "128", NULL}, - {256, "256", NULL}, - {512, "512", NULL}, - {1024, "1024", NULL}, - {2048, "2048", NULL}, - {4096, "4096", NULL}, + {16, "16", }, + {32, "32", }, + {64, "64", }, + {128, "128", }, + {256, "256", }, + {512, "512", }, + {1024, "1024", }, + {2048, "2048", }, + {4096, "4096", }, #if PAGE_SIZE > 4096 - {8192, "8192", NULL}, + {8192, "8192", }, #if PAGE_SIZE > 8192 - {16384, "16384", NULL}, + {16384, "16384", }, #if PAGE_SIZE > 16384 - {32768, "32768", NULL}, + {32768, "32768", }, #if PAGE_SIZE > 32768 - {65536, "65536", NULL}, + {65536, "65536", }, #if PAGE_SIZE > 65536 #error "Unsupported PAGE_SIZE" #endif /* 65536 */ @@ -215,14 +220,16 @@ static int sysctl_kern_malloc_stats(SYSCTL_HANDLER_ARGS); */ static time_t t_malloc_fail; +#if defined(MALLOC_MAKE_FAILURES) || (MALLOC_DEBUG_MAXZONES > 1) +SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0, + "Kernel malloc debugging options"); +#endif + /* * malloc(9) fault injection -- cause malloc failures every (n) mallocs when * the caller specifies M_NOWAIT. If set to 0, no failures are caused. */ #ifdef MALLOC_MAKE_FAILURES -SYSCTL_NODE(_debug, OID_AUTO, malloc, CTLFLAG_RD, 0, - "Kernel malloc debugging options"); - static int malloc_failure_rate; static int malloc_nowait_count; static int malloc_failure_count; @@ -233,6 +240,60 @@ SYSCTL_INT(_debug_malloc, OID_AUTO, failure_count, CTLFLAG_RD, &malloc_failure_count, 0, "Number of imposed M_NOWAIT malloc failures"); #endif +/* + * malloc(9) uma zone separation -- sub-page buffer overruns in one + * malloc type will affect only a subset of other malloc types. + */ +#if MALLOC_DEBUG_MAXZONES > 1 +static void +tunable_set_numzones(void) +{ + + TUNABLE_INT_FETCH("debug.malloc.numzones", + &numzones); + + /* Sanity check the number of malloc uma zones. */ + if (numzones <= 0) + numzones = 1; + if (numzones > MALLOC_DEBUG_MAXZONES) + numzones = MALLOC_DEBUG_MAXZONES; +} +SYSINIT(numzones, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_set_numzones, NULL); +SYSCTL_INT(_debug_malloc, OID_AUTO, numzones, CTLFLAG_RDTUN, + &numzones, 0, "Number of malloc uma subzones"); + +/* + * Any number that changes regularly is an okay choice for the + * offset. Build numbers are pretty good of you have them. + */ +static u_int zone_offset = __FreeBSD_version; +TUNABLE_INT("debug.malloc.zone_offset", &zone_offset); +SYSCTL_UINT(_debug_malloc, OID_AUTO, zone_offset, CTLFLAG_RDTUN, + &zone_offset, 0, "Separate malloc types by examining the " + "Nth character in the malloc type short description."); + +static u_int +mtp_get_subzone(const char *desc) +{ + size_t len; + u_int val; + + if (desc == NULL || (len = strlen(desc)) == 0) + return (0); + val = desc[zone_offset % len]; + return (val % numzones); +} +#elif MALLOC_DEBUG_MAXZONES == 0 +#error "MALLOC_DEBUG_MAXZONES must be positive." +#else +static inline u_int +mtp_get_subzone(const char *desc) +{ + + return (0); +} +#endif /* MALLOC_DEBUG_MAXZONES > 1 */ + int malloc_last_fail(void) { @@ -327,6 +388,7 @@ void * malloc(unsigned long size, struct malloc_type *mtp, int flags) { int indx; + struct malloc_type_internal *mtip; caddr_t va; uma_zone_t zone; #if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE) @@ -374,10 +436,14 @@ malloc(unsigned long size, struct malloc_type *mtp, int flags) #endif if (size <= KMEM_ZMAX) { + mtip = mtp->ks_handle; if (size & KMEM_ZMASK) size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; indx = kmemsize[size >> KMEM_ZSHIFT]; - zone = kmemzones[indx].kz_zone; + KASSERT(mtip->mti_zone < numzones, + ("mti_zone %u out of range %d", + mtip->mti_zone, numzones)); + zone = kmemzones[indx].kz_zone[mtip->mti_zone]; #ifdef MALLOC_PROFILE krequests[size >> KMEM_ZSHIFT]++; #endif @@ -651,15 +717,18 @@ kmeminit(void *dummy) for (i = 0, indx = 0; kmemzones[indx].kz_size != 0; indx++) { int size = kmemzones[indx].kz_size; char *name = kmemzones[indx].kz_name; + int subzone; - kmemzones[indx].kz_zone = uma_zcreate(name, size, + for (subzone = 0; subzone < numzones; subzone++) { + kmemzones[indx].kz_zone[subzone] = + uma_zcreate(name, size, #ifdef INVARIANTS - mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini, + mtrash_ctor, mtrash_dtor, mtrash_init, mtrash_fini, #else - NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, #endif - UMA_ALIGN_PTR, UMA_ZONE_MALLOC); - + UMA_ALIGN_PTR, UMA_ZONE_MALLOC); + } for (;i <= size; i+= KMEM_ZBASE) kmemsize[i >> KMEM_ZSHIFT] = indx; @@ -680,6 +749,7 @@ malloc_init(void *data) mtip = uma_zalloc(mt_zone, M_WAITOK | M_ZERO); mtp->ks_handle = mtip; + mtip->mti_zone = mtp_get_subzone(mtp->ks_shortdesc); mtx_lock(&malloc_mtx); mtp->ks_next = kmemstatistics; @@ -902,7 +972,37 @@ DB_SHOW_COMMAND(malloc, db_show_malloc) (alloced - freed + 1023) / 1024, allocs); } } -#endif + +#if MALLOC_DEBUG_MAXZONES > 1 +DB_SHOW_COMMAND(multizone_matches, db_show_multizone_matches) +{ + struct malloc_type_internal *mtip; + struct malloc_type *mtp; + u_int subzone; + + if (!have_addr) { + db_printf("Usage: show multizone_matches <malloc type/addr>\n"); + return; + } + mtp = (void *)addr; + if (mtp->ks_magic != M_MAGIC) { + db_printf("Magic %lx does not match expected %x\n", + mtp->ks_magic, M_MAGIC); + return; + } + + mtip = mtp->ks_handle; + subzone = mtip->mti_zone; + + for (mtp = kmemstatistics; mtp != NULL; mtp = mtp->ks_next) { + mtip = mtp->ks_handle; + if (mtip->mti_zone != subzone) + continue; + db_printf("%s\n", mtp->ks_shortdesc); + } +} +#endif /* MALLOC_DEBUG_MAXZONES > 1 */ +#endif /* DDB */ #ifdef MALLOC_PROFILE diff --git a/sys/pc98/conf/GENERIC b/sys/pc98/conf/GENERIC index 699091e..7766716 100644 --- a/sys/pc98/conf/GENERIC +++ b/sys/pc98/conf/GENERIC @@ -76,6 +76,7 @@ options INVARIANTS # Enable calls of extra sanity checking options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS options WITNESS # Enable checks to detect deadlocks and cycles options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # To make an SMP kernel, the next two lines are needed #options SMP # Symmetric MultiProcessor Kernel diff --git a/sys/powerpc/conf/GENERIC b/sys/powerpc/conf/GENERIC index 1a0ede6..8bcb06a 100644 --- a/sys/powerpc/conf/GENERIC +++ b/sys/powerpc/conf/GENERIC @@ -74,6 +74,7 @@ options INVARIANTS #Enable calls of extra sanity checking options INVARIANT_SUPPORT #Extra sanity checks of internal structures, required by INVARIANTS options WITNESS #Enable checks to detect deadlocks and cycles options WITNESS_SKIPSPIN #Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # To make an SMP kernel, the next line is needed #options SMP # Symmetric MultiProcessor Kernel diff --git a/sys/sparc64/conf/GENERIC b/sys/sparc64/conf/GENERIC index 9ce2d6c..1e0a0f2 100644 --- a/sys/sparc64/conf/GENERIC +++ b/sys/sparc64/conf/GENERIC @@ -73,6 +73,7 @@ options INVARIANTS # Enable calls of extra sanity checking options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS options WITNESS # Enable checks to detect deadlocks and cycles options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones # Make an SMP-capable kernel by default options SMP # Symmetric MultiProcessor Kernel diff --git a/sys/sun4v/conf/GENERIC b/sys/sun4v/conf/GENERIC index 26ec3af..f7a4e77 100644 --- a/sys/sun4v/conf/GENERIC +++ b/sys/sun4v/conf/GENERIC @@ -79,6 +79,7 @@ options DDB # Support DDB. #options INVARIANT_SUPPORT # Extra sanity checks of internal structures, required by INVARIANTS #options WITNESS # Enable checks to detect deadlocks and cycles #options WITNESS_SKIPSPIN # Don't run witness on spinlocks for speed +#options MALLOC_DEBUG_MAXZONES=8 # Separate malloc(9) zones #options DEBUG_LOCKS #options DEBUG_VFS_LOCKS diff --git a/sys/sys/malloc.h b/sys/sys/malloc.h index 030ad0e..76e94be 100644 --- a/sys/sys/malloc.h +++ b/sys/sys/malloc.h @@ -90,6 +90,7 @@ struct malloc_type_stats { struct malloc_type_internal { uint32_t mti_probes[DTMALLOC_PROBE_MAX]; /* DTrace probe ID array. */ + u_char mti_zone; struct malloc_type_stats mti_stats[MAXCPU]; }; |
