diff options
-rw-r--r-- | sys/kern/kern_ktr.c | 112 | ||||
-rw-r--r-- | sys/kern/subr_lock.c | 29 | ||||
-rw-r--r-- | sys/kern/subr_smp.c | 50 | ||||
-rw-r--r-- | sys/sparc64/include/ktr.h | 1 | ||||
-rw-r--r-- | sys/sys/ktr.h | 2 | ||||
-rw-r--r-- | sys/sys/smp.h | 4 | ||||
-rw-r--r-- | usr.bin/ktrdump/ktrdump.c | 6 |
7 files changed, 153 insertions, 51 deletions
diff --git a/sys/kern/kern_ktr.c b/sys/kern/kern_ktr.c index 738b05a..6498498 100644 --- a/sys/kern/kern_ktr.c +++ b/sys/kern/kern_ktr.c @@ -47,7 +47,11 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/ktr.h> #include <sys/libkern.h> +#include <sys/lock.h> +#include <sys/malloc.h> +#include <sys/mutex.h> #include <sys/proc.h> +#include <sys/smp.h> #include <sys/sysctl.h> #include <sys/systm.h> #include <sys/time.h> @@ -66,6 +70,9 @@ __FBSDID("$FreeBSD$"); #define KTR_ENTRIES 1024 #endif +/* Limit the allocations to something manageable. */ +#define KTR_ENTRIES_MAX (8 * 1024 * 1024) + #ifndef KTR_MASK #define KTR_MASK (0) #endif @@ -82,30 +89,31 @@ __FBSDID("$FreeBSD$"); #define KTR_CPU PCPU_GET(cpuid) #endif -FEATURE(ktr, "Kernel support for KTR kernel tracing facility"); +static MALLOC_DEFINE(M_KTR, "KTR", "KTR"); -static SYSCTL_NODE(_debug, OID_AUTO, ktr, CTLFLAG_RD, 0, "KTR options"); +FEATURE(ktr, "Kernel support for KTR kernel tracing facility"); +volatile int ktr_idx = 0; int ktr_mask = KTR_MASK; +int ktr_compile = KTR_COMPILE; +int ktr_entries = KTR_ENTRIES; +int ktr_version = KTR_VERSION; +struct ktr_entry ktr_buf_init[KTR_ENTRIES]; +struct ktr_entry *ktr_buf = ktr_buf_init; +cpuset_t ktr_cpumask = CPUSET_T_INITIALIZER(KTR_CPUMASK); +static char ktr_cpumask_str[CPUSETBUFSIZ]; + TUNABLE_INT("debug.ktr.mask", &ktr_mask); -SYSCTL_INT(_debug_ktr, OID_AUTO, mask, CTLFLAG_RW, - &ktr_mask, 0, "Bitmask of KTR event classes for which logging is enabled"); -int ktr_compile = KTR_COMPILE; -SYSCTL_INT(_debug_ktr, OID_AUTO, compile, CTLFLAG_RD, - &ktr_compile, 0, "Bitmask of KTR event classes compiled into the kernel"); +TUNABLE_STR("debug.ktr.cpumask", ktr_cpumask_str, sizeof(ktr_cpumask_str)); -int ktr_entries = KTR_ENTRIES; -SYSCTL_INT(_debug_ktr, OID_AUTO, entries, CTLFLAG_RD, - &ktr_entries, 0, "Number of entries in the KTR buffer"); +static SYSCTL_NODE(_debug, OID_AUTO, ktr, CTLFLAG_RD, 0, "KTR options"); -int ktr_version = KTR_VERSION; SYSCTL_INT(_debug_ktr, OID_AUTO, version, CTLFLAG_RD, &ktr_version, 0, "Version of the KTR interface"); -cpuset_t ktr_cpumask = CPUSET_T_INITIALIZER(KTR_CPUMASK); -static char ktr_cpumask_str[CPUSETBUFSIZ]; -TUNABLE_STR("debug.ktr.cpumask", ktr_cpumask_str, sizeof(ktr_cpumask_str)); +SYSCTL_INT(_debug_ktr, OID_AUTO, compile, CTLFLAG_RD, + &ktr_compile, 0, "Bitmask of KTR event classes compiled into the kernel"); static void ktr_cpumask_initializer(void *dummy __unused) @@ -145,9 +153,6 @@ SYSCTL_PROC(_debug_ktr, OID_AUTO, cpumask, sysctl_debug_ktr_cpumask, "S", "Bitmask of CPUs on which KTR logging is enabled"); -volatile int ktr_idx = 0; -struct ktr_entry ktr_buf[KTR_ENTRIES]; - static int sysctl_debug_ktr_clear(SYSCTL_HANDLER_ARGS) { @@ -159,7 +164,7 @@ sysctl_debug_ktr_clear(SYSCTL_HANDLER_ARGS) return (error); if (clear) { - bzero(ktr_buf, sizeof(ktr_buf)); + bzero(ktr_buf, sizeof(*ktr_buf) * ktr_entries); ktr_idx = 0; } @@ -168,6 +173,67 @@ sysctl_debug_ktr_clear(SYSCTL_HANDLER_ARGS) SYSCTL_PROC(_debug_ktr, OID_AUTO, clear, CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_debug_ktr_clear, "I", "Clear KTR Buffer"); +/* + * This is a sysctl proc so that it is serialized as !MPSAFE along with + * the other ktr sysctl procs. + */ +static int +sysctl_debug_ktr_mask(SYSCTL_HANDLER_ARGS) +{ + int mask, error; + + mask = ktr_mask; + error = sysctl_handle_int(oidp, &mask, 0, req); + if (error || !req->newptr) + return (error); + ktr_mask = mask; + return (error); +} + +SYSCTL_PROC(_debug_ktr, OID_AUTO, mask, CTLTYPE_INT|CTLFLAG_RW, 0, 0, + sysctl_debug_ktr_mask, "I", + "Bitmask of KTR event classes for which logging is enabled"); + +static int +sysctl_debug_ktr_entries(SYSCTL_HANDLER_ARGS) +{ + int entries, error, mask; + struct ktr_entry *buf, *oldbuf; + + entries = ktr_entries; + error = sysctl_handle_int(oidp, &entries, 0, req); + if (error || !req->newptr) + return (error); + if (entries > KTR_ENTRIES_MAX) + return (ERANGE); + /* Disable ktr temporarily. */ + mask = ktr_mask; + atomic_store_rel_int(&ktr_mask, 0); + /* Wait for threads to go idle. */ + if ((error = quiesce_all_cpus("ktrent", PCATCH)) != 0) { + ktr_mask = mask; + return (error); + } + if (ktr_buf != ktr_buf_init) + oldbuf = ktr_buf; + else + oldbuf = NULL; + /* Allocate a new buffer. */ + buf = malloc(sizeof(*buf) * entries, M_KTR, M_WAITOK | M_ZERO); + /* Install the new buffer and restart ktr. */ + ktr_buf = buf; + ktr_entries = entries; + ktr_idx = 0; + atomic_store_rel_int(&ktr_mask, mask); + if (oldbuf != NULL) + free(oldbuf, M_KTR); + + return (error); +} + +SYSCTL_PROC(_debug_ktr, OID_AUTO, entries, CTLTYPE_INT|CTLFLAG_RW, 0, 0, + sysctl_debug_ktr_entries, "I", "Number of entries in the KTR buffer"); + #ifdef KTR_VERBOSE int ktr_verbose = KTR_VERBOSE; TUNABLE_INT("debug.ktr.verbose", &ktr_verbose); @@ -249,7 +315,7 @@ ktr_tracepoint(u_int mask, const char *file, int line, const char *format, if (panicstr) return; - if ((ktr_mask & mask) == 0) + if ((ktr_mask & mask) == 0 || ktr_buf == NULL) return; cpu = KTR_CPU; if (!CPU_ISSET(cpu, &ktr_cpumask)) @@ -281,7 +347,7 @@ ktr_tracepoint(u_int mask, const char *file, int line, const char *format, { do { saveindex = ktr_idx; - newindex = (saveindex + 1) % KTR_ENTRIES; + newindex = (saveindex + 1) % ktr_entries; } while (atomic_cmpset_rel_int(&ktr_idx, saveindex, newindex) == 0); entry = &ktr_buf[saveindex]; } @@ -336,7 +402,7 @@ static int db_mach_vtrace(void); DB_SHOW_COMMAND(ktr, db_ktr_all) { - tstate.cur = (ktr_idx - 1) % KTR_ENTRIES; + tstate.cur = (ktr_idx - 1) % ktr_entries; tstate.first = -1; db_ktr_verbose = 0; db_ktr_verbose |= (strchr(modif, 'v') != NULL) ? 2 : 0; @@ -358,7 +424,7 @@ db_mach_vtrace(void) { struct ktr_entry *kp; - if (tstate.cur == tstate.first) { + if (tstate.cur == tstate.first || ktr_buf == NULL) { db_printf("--- End of trace buffer ---\n"); return (0); } @@ -390,7 +456,7 @@ db_mach_vtrace(void) tstate.first = tstate.cur; if (--tstate.cur < 0) - tstate.cur = KTR_ENTRIES - 1; + tstate.cur = ktr_entries - 1; return (1); } diff --git a/sys/kern/subr_lock.c b/sys/kern/subr_lock.c index b040c7b..6f3682a 100644 --- a/sys/kern/subr_lock.c +++ b/sys/kern/subr_lock.c @@ -240,34 +240,13 @@ lock_prof_init(void *arg) } SYSINIT(lockprof, SI_SUB_SMP, SI_ORDER_ANY, lock_prof_init, NULL); -/* - * To be certain that lock profiling has idled on all cpus before we - * reset, we schedule the resetting thread on all active cpus. Since - * all operations happen within critical sections we can be sure that - * it is safe to zero the profiling structures. - */ -static void -lock_prof_idle(void) -{ - struct thread *td; - int cpu; - - td = curthread; - thread_lock(td); - CPU_FOREACH(cpu) { - sched_bind(td, cpu); - } - sched_unbind(td); - thread_unlock(td); -} - static void lock_prof_reset_wait(void) { /* - * Spin relinquishing our cpu so that lock_prof_idle may - * run on it. + * Spin relinquishing our cpu so that quiesce_all_cpus may + * complete. */ while (lock_prof_resetting) sched_relinquish(curthread); @@ -289,7 +268,7 @@ lock_prof_reset(void) atomic_store_rel_int(&lock_prof_resetting, 1); enabled = lock_prof_enable; lock_prof_enable = 0; - lock_prof_idle(); + quiesce_all_cpus("profreset", 0); /* * Some objects may have migrated between CPUs. Clear all links * before we zero the structures. Some items may still be linked @@ -401,7 +380,7 @@ dump_lock_prof_stats(SYSCTL_HANDLER_ARGS) "max", "wait_max", "total", "wait_total", "count", "avg", "wait_avg", "cnt_hold", "cnt_lock", "name"); enabled = lock_prof_enable; lock_prof_enable = 0; - lock_prof_idle(); + quiesce_all_cpus("profstat", 0); t = ticks; for (cpu = 0; cpu <= mp_maxid; cpu++) { if (lp_cpu[cpu] == NULL) diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index 4c73b61..3b27dce 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/mutex.h> #include <sys/pcpu.h> +#include <sys/sched.h> #include <sys/smp.h> #include <sys/sysctl.h> @@ -734,3 +735,52 @@ smp_no_rendevous_barrier(void *dummy) KASSERT((!smp_started),("smp_no_rendevous called and smp is started")); #endif } + +/* + * Wait specified idle threads to switch once. This ensures that even + * preempted threads have cycled through the switch function once, + * exiting their codepaths. This allows us to change global pointers + * with no other synchronization. + */ +int +quiesce_cpus(cpuset_t map, const char *wmesg, int prio) +{ + struct pcpu *pcpu; + u_int gen[MAXCPU]; + int error; + int cpu; + + error = 0; + for (cpu = 0; cpu <= mp_maxid; cpu++) { + if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu)) + continue; + pcpu = pcpu_find(cpu); + gen[cpu] = pcpu->pc_idlethread->td_generation; + } + for (cpu = 0; cpu <= mp_maxid; cpu++) { + if (!CPU_ISSET(cpu, &map) || CPU_ABSENT(cpu)) + continue; + pcpu = pcpu_find(cpu); + thread_lock(curthread); + sched_bind(curthread, cpu); + thread_unlock(curthread); + while (gen[cpu] == pcpu->pc_idlethread->td_generation) { + error = tsleep(quiesce_cpus, prio, wmesg, 1); + if (error) + goto out; + } + } +out: + thread_lock(curthread); + sched_unbind(curthread); + thread_unlock(curthread); + + return (error); +} + +int +quiesce_all_cpus(const char *wmesg, int prio) +{ + + return quiesce_cpus(all_cpus, wmesg, prio); +} diff --git a/sys/sparc64/include/ktr.h b/sys/sparc64/include/ktr.h index f13865f..0a7f0e1 100644 --- a/sys/sparc64/include/ktr.h +++ b/sys/sparc64/include/ktr.h @@ -58,6 +58,7 @@ l2: add r2, 1, r3 ; \ bne %icc, l2 ## b ; \ mov r3, r2 ; \ SET(ktr_buf, r3, r1) ; \ + ldx [r1], r1 ; \ mulx r2, KTR_SIZEOF, r2 ; \ add r1, r2, r1 ; \ rd %tick, r2 ; \ diff --git a/sys/sys/ktr.h b/sys/sys/ktr.h index 7885b22..c440a41 100644 --- a/sys/sys/ktr.h +++ b/sys/sys/ktr.h @@ -116,7 +116,7 @@ extern int ktr_entries; extern int ktr_verbose; extern volatile int ktr_idx; -extern struct ktr_entry ktr_buf[]; +extern struct ktr_entry *ktr_buf; #ifdef KTR diff --git a/sys/sys/smp.h b/sys/sys/smp.h index a506669..4abd2ce 100644 --- a/sys/sys/smp.h +++ b/sys/sys/smp.h @@ -167,10 +167,14 @@ int stop_cpus_hard(cpuset_t); #if defined(__amd64__) || defined(__i386__) int suspend_cpus(cpuset_t); #endif + void smp_rendezvous_action(void); extern struct mtx smp_ipi_mtx; #endif /* SMP */ + +int quiesce_all_cpus(const char *, int); +int quiesce_cpus(cpuset_t, const char *, int); void smp_no_rendevous_barrier(void *); void smp_rendezvous(void (*)(void *), void (*)(void *), diff --git a/usr.bin/ktrdump/ktrdump.c b/usr.bin/ktrdump/ktrdump.c index d74b979..77ae522 100644 --- a/usr.bin/ktrdump/ktrdump.c +++ b/usr.bin/ktrdump/ktrdump.c @@ -86,6 +86,7 @@ main(int ac, char **av) u_long parms[KTR_PARMS]; struct ktr_entry *buf; uintmax_t tlast, tnow; + unsigned long bufptr; struct stat sb; kvm_t *kd; FILE *out; @@ -179,8 +180,9 @@ main(int ac, char **av) if ((buf = malloc(sizeof(*buf) * entries)) == NULL) err(1, NULL); if (kvm_read(kd, nl[2].n_value, &index, sizeof(index)) == -1 || - kvm_read(kd, nl[3].n_value, buf, sizeof(*buf) * entries) - == -1) + kvm_read(kd, nl[3].n_value, &bufptr, + sizeof(bufptr)) == -1 || + kvm_read(kd, bufptr, buf, sizeof(*buf) * entries) == -1) errx(1, "%s", kvm_geterr(kd)); } |