diff options
| author | jhb <jhb@FreeBSD.org> | 2001-03-28 09:03:24 +0000 |
|---|---|---|
| committer | jhb <jhb@FreeBSD.org> | 2001-03-28 09:03:24 +0000 |
| commit | 0c490fd02e5b2a2764a94d809ef8be208b646e39 (patch) | |
| tree | 57041654950234a1d3bb416540f96407a6127e19 | |
| parent | 3aabb22260cdd728828c80d2439d1ba623be5a51 (diff) | |
| download | FreeBSD-src-0c490fd02e5b2a2764a94d809ef8be208b646e39.zip FreeBSD-src-0c490fd02e5b2a2764a94d809ef8be208b646e39.tar.gz | |
Rework the witness code to work with sx locks as well as mutexes.
- Introduce lock classes and lock objects. Each lock class specifies a
name and set of flags (or properties) shared by all locks of a given
type. Currently there are three lock classes: spin mutexes, sleep
mutexes, and sx locks. A lock object specifies properties of an
additional lock along with a lock name and all of the extra stuff needed
to make witness work with a given lock. This abstract lock stuff is
defined in sys/lock.h. The lockmgr constants, types, and prototypes have
been moved to sys/lockmgr.h. For temporary backwards compatability,
sys/lock.h includes sys/lockmgr.h.
- Replace proc->p_spinlocks with a per-CPU list, PCPU(spinlocks), of spin
locks held. By making this per-cpu, we do not have to jump through
magic hoops to deal with sched_lock changing ownership during context
switches.
- Replace proc->p_heldmtx, formerly a list of held sleep mutexes, with
proc->p_sleeplocks, which is a list of held sleep locks including sleep
mutexes and sx locks.
- Add helper macros for logging lock events via the KTR_LOCK KTR logging
level so that the log messages are consistent.
- Add some new flags that can be passed to mtx_init():
- MTX_NOWITNESS - specifies that this lock should be ignored by witness.
This is used for the mutex that blocks a sx lock for example.
- MTX_QUIET - this is not new, but you can pass this to mtx_init() now
and no events will be logged for this lock, so that one doesn't have
to change all the individual mtx_lock/unlock() operations.
- All lock objects maintain an initialized flag. Use this flag to export
a mtx_initialized() macro that can be safely called from drivers. Also,
we on longer walk the all_mtx list if MUTEX_DEBUG is defined as witness
performs the corresponding checks using the initialized flag.
- The lock order reversal messages have been improved to output slightly
more accurate file and line numbers.
33 files changed, 1279 insertions, 3852 deletions
diff --git a/sys/alpha/alpha/machdep.c b/sys/alpha/alpha/machdep.c index 20c529a..45284e6 100644 --- a/sys/alpha/alpha/machdep.c +++ b/sys/alpha/alpha/machdep.c @@ -988,7 +988,6 @@ alpha_init(pfn, ptb, bim, bip, biv) (u_int64_t)proc0paddr + USPACE - sizeof(struct trapframe); proc0.p_md.md_tf = (struct trapframe *)proc0paddr->u_pcb.pcb_hw.apcb_ksp; - PCPU_SET(curproc, &proc0); /* * Get the right value for the boot cpu's idle ptbr. @@ -1003,8 +1002,8 @@ alpha_init(pfn, ptb, bim, bip, biv) /* Setup curproc so that mutexes work */ PCPU_SET(curproc, &proc0); + PCPU_SET(spinlocks, NULL); - LIST_INIT(&proc0.p_heldmtx); LIST_INIT(&proc0.p_contested); /* diff --git a/sys/alpha/include/globaldata.h b/sys/alpha/include/globaldata.h index 8e6cfbe..15bd8c41a 100644 --- a/sys/alpha/include/globaldata.h +++ b/sys/alpha/include/globaldata.h @@ -58,7 +58,7 @@ struct globaldata { u_int32_t gd_current_asngen; /* ASN rollover check */ SLIST_ENTRY(globaldata) gd_allcpu; - int gd_witness_spin_check; + struct lock_list_entry *gd_spinlocks; #ifdef KTR_PERCPU volatile int gd_ktr_idx; /* Index into trace table */ char *gd_ktr_buf; diff --git a/sys/alpha/include/pcpu.h b/sys/alpha/include/pcpu.h index 8e6cfbe..15bd8c41a 100644 --- a/sys/alpha/include/pcpu.h +++ b/sys/alpha/include/pcpu.h @@ -58,7 +58,7 @@ struct globaldata { u_int32_t gd_current_asngen; /* ASN rollover check */ SLIST_ENTRY(globaldata) gd_allcpu; - int gd_witness_spin_check; + struct lock_list_entry *gd_spinlocks; #ifdef KTR_PERCPU volatile int gd_ktr_idx; /* Index into trace table */ char *gd_ktr_buf; diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index 6ca7c7e..dbdc61a 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -59,6 +59,7 @@ #include <sys/kernel.h> #include <sys/ktr.h> #include <sys/linker.h> +#include <sys/lock.h> #include <sys/malloc.h> #include <sys/mutex.h> #include <sys/proc.h> @@ -1895,8 +1896,8 @@ init386(first) /* setup curproc so that mutexes work */ PCPU_SET(curproc, &proc0); + PCPU_SET(spinlocks, NULL); - LIST_INIT(&proc0.p_heldmtx); LIST_INIT(&proc0.p_contested); mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 9797905..47903d0 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -2268,6 +2268,7 @@ ap_init(void) * something unique to lock with. */ PCPU_SET(curproc, PCPU_GET(idleproc)); + PCPU_SET(spinlocks, NULL); /* lock against other AP's that are waking up */ mtx_lock_spin(&ap_boot_mtx); diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c index 9797905..47903d0 100644 --- a/sys/amd64/amd64/mptable.c +++ b/sys/amd64/amd64/mptable.c @@ -2268,6 +2268,7 @@ ap_init(void) * something unique to lock with. */ PCPU_SET(curproc, PCPU_GET(idleproc)); + PCPU_SET(spinlocks, NULL); /* lock against other AP's that are waking up */ mtx_lock_spin(&ap_boot_mtx); diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h index 9797905..47903d0 100644 --- a/sys/amd64/include/mptable.h +++ b/sys/amd64/include/mptable.h @@ -2268,6 +2268,7 @@ ap_init(void) * something unique to lock with. */ PCPU_SET(curproc, PCPU_GET(idleproc)); + PCPU_SET(spinlocks, NULL); /* lock against other AP's that are waking up */ mtx_lock_spin(&ap_boot_mtx); diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h index d2873ef..c83d73c 100644 --- a/sys/amd64/include/pcpu.h +++ b/sys/amd64/include/pcpu.h @@ -65,6 +65,7 @@ struct globaldata { u_int gd_other_cpus; SLIST_ENTRY(globaldata) gd_allcpu; int gd_witness_spin_check; + struct lock_list_entry *gd_spinlocks; #ifdef KTR_PERCPU #ifdef KTR volatile int gd_ktr_idx; diff --git a/sys/conf/files b/sys/conf/files index 5ee34bd..5dddc37 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -715,6 +715,7 @@ kern/subr_rman.c standard kern/subr_sbuf.c standard kern/subr_scanf.c standard kern/subr_taskqueue.c standard +kern/subr_witness.c optional witness kern/subr_xxx.c standard kern/sys_generic.c standard kern/sys_pipe.c standard diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index 6ca7c7e..dbdc61a 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -59,6 +59,7 @@ #include <sys/kernel.h> #include <sys/ktr.h> #include <sys/linker.h> +#include <sys/lock.h> #include <sys/malloc.h> #include <sys/mutex.h> #include <sys/proc.h> @@ -1895,8 +1896,8 @@ init386(first) /* setup curproc so that mutexes work */ PCPU_SET(curproc, &proc0); + PCPU_SET(spinlocks, NULL); - LIST_INIT(&proc0.p_heldmtx); LIST_INIT(&proc0.p_contested); mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE); diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index 9797905..47903d0 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -2268,6 +2268,7 @@ ap_init(void) * something unique to lock with. */ PCPU_SET(curproc, PCPU_GET(idleproc)); + PCPU_SET(spinlocks, NULL); /* lock against other AP's that are waking up */ mtx_lock_spin(&ap_boot_mtx); diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c index 9797905..47903d0 100644 --- a/sys/i386/i386/mptable.c +++ b/sys/i386/i386/mptable.c @@ -2268,6 +2268,7 @@ ap_init(void) * something unique to lock with. */ PCPU_SET(curproc, PCPU_GET(idleproc)); + PCPU_SET(spinlocks, NULL); /* lock against other AP's that are waking up */ mtx_lock_spin(&ap_boot_mtx); diff --git a/sys/i386/include/globaldata.h b/sys/i386/include/globaldata.h index d2873ef..c83d73c 100644 --- a/sys/i386/include/globaldata.h +++ b/sys/i386/include/globaldata.h @@ -65,6 +65,7 @@ struct globaldata { u_int gd_other_cpus; SLIST_ENTRY(globaldata) gd_allcpu; int gd_witness_spin_check; + struct lock_list_entry *gd_spinlocks; #ifdef KTR_PERCPU #ifdef KTR volatile int gd_ktr_idx; diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h index 9797905..47903d0 100644 --- a/sys/i386/include/mptable.h +++ b/sys/i386/include/mptable.h @@ -2268,6 +2268,7 @@ ap_init(void) * something unique to lock with. */ PCPU_SET(curproc, PCPU_GET(idleproc)); + PCPU_SET(spinlocks, NULL); /* lock against other AP's that are waking up */ mtx_lock_spin(&ap_boot_mtx); diff --git a/sys/i386/include/pcpu.h b/sys/i386/include/pcpu.h index d2873ef..c83d73c 100644 --- a/sys/i386/include/pcpu.h +++ b/sys/i386/include/pcpu.h @@ -65,6 +65,7 @@ struct globaldata { u_int gd_other_cpus; SLIST_ENTRY(globaldata) gd_allcpu; int gd_witness_spin_check; + struct lock_list_entry *gd_spinlocks; #ifdef KTR_PERCPU #ifdef KTR volatile int gd_ktr_idx; diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index 9dde541..ee804f2 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -587,8 +587,8 @@ ia64_init() /* Setup curproc so that mutexes work */ PCPU_SET(curproc, &proc0); + PCPU_SET(spinlocks, NULL); - LIST_INIT(&proc0.p_heldmtx); LIST_INIT(&proc0.p_contested); /* diff --git a/sys/ia64/include/globaldata.h b/sys/ia64/include/globaldata.h index ce9164b..a92f543 100644 --- a/sys/ia64/include/globaldata.h +++ b/sys/ia64/include/globaldata.h @@ -58,7 +58,7 @@ struct globaldata { u_int32_t gd_current_asngen; /* ASN rollover check */ SLIST_ENTRY(globaldata) gd_allcpu; - int gd_witness_spin_check; + struct lock_list_entry *gd_spinlocks; #ifdef KTR_PERCPU volatile int gd_ktr_idx; /* Index into trace table */ char *gd_ktr_buf; diff --git a/sys/ia64/include/pcpu.h b/sys/ia64/include/pcpu.h index ce9164b..a92f543 100644 --- a/sys/ia64/include/pcpu.h +++ b/sys/ia64/include/pcpu.h @@ -58,7 +58,7 @@ struct globaldata { u_int32_t gd_current_asngen; /* ASN rollover check */ SLIST_ENTRY(globaldata) gd_allcpu; - int gd_witness_spin_check; + struct lock_list_entry *gd_spinlocks; #ifdef KTR_PERCPU volatile int gd_ktr_idx; /* Index into trace table */ char *gd_ktr_buf; diff --git a/sys/kern/kern_condvar.c b/sys/kern/kern_condvar.c index 176701d..d53e9e6 100644 --- a/sys/kern/kern_condvar.c +++ b/sys/kern/kern_condvar.c @@ -197,7 +197,7 @@ cv_wait(struct cv *cvp, struct mtx *mp) #endif CV_ASSERT(cvp, mp, p); WITNESS_SLEEP(0, mp); - WITNESS_SAVE(mp, mp); + WITNESS_SAVE(&mp->mtx_object, mp); mtx_lock_spin(&sched_lock); if (cold || panicstr) { @@ -225,7 +225,7 @@ cv_wait(struct cv *cvp, struct mtx *mp) #endif PICKUP_GIANT(); mtx_lock(mp); - WITNESS_RESTORE(mp, mp); + WITNESS_RESTORE(&mp->mtx_object, mp); } /* @@ -250,7 +250,7 @@ cv_wait_sig(struct cv *cvp, struct mtx *mp) #endif CV_ASSERT(cvp, mp, p); WITNESS_SLEEP(0, mp); - WITNESS_SAVE(mp, mp); + WITNESS_SAVE(&mp->mtx_object, mp); mtx_lock_spin(&sched_lock); if (cold || panicstr) { @@ -290,7 +290,7 @@ cv_wait_sig(struct cv *cvp, struct mtx *mp) ktrcsw(p->p_tracep, 0, 0); #endif mtx_lock(mp); - WITNESS_RESTORE(mp, mp); + WITNESS_RESTORE(&mp->mtx_object, mp); return (rval); } @@ -315,7 +315,7 @@ cv_timedwait(struct cv *cvp, struct mtx *mp, int timo) #endif CV_ASSERT(cvp, mp, p); WITNESS_SLEEP(0, mp); - WITNESS_SAVE(mp, mp); + WITNESS_SAVE(&mp->mtx_object, mp); mtx_lock_spin(&sched_lock); if (cold || panicstr) { @@ -350,7 +350,7 @@ cv_timedwait(struct cv *cvp, struct mtx *mp, int timo) #endif PICKUP_GIANT(); mtx_lock(mp); - WITNESS_RESTORE(mp, mp); + WITNESS_RESTORE(&mp->mtx_object, mp); return (rval); } @@ -377,7 +377,7 @@ cv_timedwait_sig(struct cv *cvp, struct mtx *mp, int timo) #endif CV_ASSERT(cvp, mp, p); WITNESS_SLEEP(0, mp); - WITNESS_SAVE(mp, mp); + WITNESS_SAVE(&mp->mtx_object, mp); mtx_lock_spin(&sched_lock); if (cold || panicstr) { @@ -424,7 +424,7 @@ cv_timedwait_sig(struct cv *cvp, struct mtx *mp, int timo) ktrcsw(p->p_tracep, 0, 0); #endif mtx_lock(mp); - WITNESS_RESTORE(mp, mp); + WITNESS_RESTORE(&mp->mtx_object, mp); return (rval); } diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 7955172..8ae5bcf 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -47,6 +47,7 @@ #include <sys/filedesc.h> #include <sys/kernel.h> #include <sys/sysctl.h> +#include <sys/lock.h> #include <sys/malloc.h> #include <sys/mutex.h> #include <sys/proc.h> @@ -62,7 +63,6 @@ #include <sys/sx.h> #include <vm/vm.h> -#include <sys/lock.h> #include <vm/pmap.h> #include <vm/vm_map.h> #include <vm/vm_extern.h> @@ -524,7 +524,6 @@ again: PROCTREE_LOCK(PT_RELEASE); PROC_LOCK(p2); LIST_INIT(&p2->p_children); - LIST_INIT(&p2->p_heldmtx); LIST_INIT(&p2->p_contested); callout_init(&p2->p_itcallout, 0); diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index ee285af..ba2da45 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -56,12 +56,13 @@ */ #include "opt_ddb.h" -#include "opt_witness.h" #include <sys/param.h> #include <sys/bus.h> #include <sys/kernel.h> #include <sys/malloc.h> +#include <sys/lock.h> +#include <sys/mutex.h> #include <sys/proc.h> #include <sys/sysctl.h> #include <sys/systm.h> @@ -78,25 +79,6 @@ #include <vm/vm.h> #include <vm/vm_extern.h> -#include <sys/mutex.h> - -/* - * The WITNESS-enabled mutex debug structure. - */ -#ifdef WITNESS -struct mtx_debug { - struct witness *mtxd_witness; - LIST_ENTRY(mtx) mtxd_held; - const char *mtxd_file; - int mtxd_line; -}; - -#define mtx_held mtx_debug->mtxd_held -#define mtx_file mtx_debug->mtxd_file -#define mtx_line mtx_debug->mtxd_line -#define mtx_witness mtx_debug->mtxd_witness -#endif /* WITNESS */ - /* * Internal utility macros. */ @@ -108,71 +90,19 @@ struct mtx_debug { #define SET_PRIO(p, pri) (p)->p_pri.pri_level = (pri) /* - * Early WITNESS-enabled declarations. - */ -#ifdef WITNESS - -/* - * Internal WITNESS routines which must be prototyped early. - * - * XXX: When/if witness code is cleaned up, it would be wise to place all - * witness prototyping early in this file. - */ -static void witness_init(struct mtx *, int flag); -static void witness_destroy(struct mtx *); -static void witness_display(void(*)(const char *fmt, ...)); - -MALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure"); - -/* All mutexes in system (used for debug/panic) */ -static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 }; - -/* - * This global is set to 0 once it becomes safe to use the witness code. - */ -static int witness_cold = 1; - -#else /* WITNESS */ - -/* XXX XXX XXX - * flag++ is sleazoid way of shuting up warning - */ -#define witness_init(m, flag) flag++ -#define witness_destroy(m) -#define witness_try_enter(m, t, f, l) -#endif /* WITNESS */ - -/* - * All mutex locks in system are kept on the all_mtx list. + * Lock classes for sleep and spin mutexes. */ -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head", - TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), - { NULL, NULL }, &all_mtx, &all_mtx, -#ifdef WITNESS - &all_mtx_debug -#else - NULL -#endif - }; - -/* - * Global variables for book keeping. - */ -static int mtx_cur_cnt; -static int mtx_max_cnt; - -/* - * Couple of strings for KTR_LOCK tracing in order to avoid duplicates. - */ -char STR_mtx_lock_slp[] = "GOT (sleep) %s [%p] r=%d at %s:%d"; -char STR_mtx_unlock_slp[] = "REL (sleep) %s [%p] r=%d at %s:%d"; -char STR_mtx_lock_spn[] = "GOT (spin) %s [%p] r=%d at %s:%d"; -char STR_mtx_unlock_spn[] = "REL (spin) %s [%p] r=%d at %s:%d"; +struct lock_class lock_class_mtx_sleep = { + "sleep mutex", + LC_SLEEPLOCK | LC_RECURSABLE +}; +struct lock_class lock_class_mtx_spin = { + "spin mutex", + LC_SPINLOCK | LC_RECURSABLE +}; /* * Prototypes for non-exported routines. - * - * NOTE: Prototypes for witness routines are placed at the bottom of the file. */ static void propagate_priority(struct proc *); @@ -241,7 +171,7 @@ propagate_priority(struct proc *p) KASSERT(p->p_stat == SMTX, ( "process %d(%s):%d holds %s but isn't blocked on a mutex\n", p->p_pid, p->p_comm, p->p_stat, - m->mtx_description)); + m->mtx_object.lo_name)); /* * Pick up the mutex that p is blocked on. @@ -280,7 +210,7 @@ propagate_priority(struct proc *p) TAILQ_INSERT_BEFORE(p1, p, p_procq); CTR4(KTR_LOCK, "propagate_priority: p %p moved before %p on [%p] %s", - p, p1, m, m->mtx_description); + p, p1, m, m->mtx_object.lo_name); } } @@ -336,23 +266,19 @@ _mtx_trylock(struct mtx *m, int opts, const char *file, int line) rval = _obtain_lock(m, curproc); -#ifdef WITNESS - if (rval && m->mtx_witness != NULL) { + LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); + if (rval) { /* * We do not handle recursion in _mtx_trylock; see the * note at the top of the routine. */ KASSERT(!mtx_recursed(m), ("mtx_trylock() called on a recursed mutex")); - witness_try_enter(m, (opts | m->mtx_flags), file, line); + mtx_update_flags(m, 1); + WITNESS_LOCK(&m->mtx_object, opts | LOP_TRYLOCK, file, line); } -#endif /* WITNESS */ - - if ((opts & MTX_QUIET) == 0) - CTR5(KTR_LOCK, "TRY_LOCK %s [%p] result=%d at %s:%d", - m->mtx_description, m, rval, file, line); - return rval; + return (rval); } /* @@ -369,15 +295,15 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { m->mtx_recurse++; atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); return; } - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR4(KTR_LOCK, "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", - m->mtx_description, (void *)m->mtx_lock, file, line); + m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); while (!_obtain_lock(m, p)) { uintptr_t v; @@ -434,7 +360,7 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) struct ithd *it = p->p_ithd; if (it->it_interrupted) { - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_lock_sleep: %p interrupted %p", it, it->it_interrupted); @@ -464,21 +390,21 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) * Save who we're blocked on. */ p->p_blocked = m; - p->p_mtxname = m->mtx_description; + p->p_mtxname = m->mtx_object.lo_name; p->p_stat = SMTX; propagate_priority(p); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR3(KTR_LOCK, "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m, - m->mtx_description); + m->mtx_object.lo_name); mi_switch(); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR3(KTR_LOCK, "_mtx_lock_sleep: p %p free from blocked on [%p] %s", - p, m, m->mtx_description); + p, m, m->mtx_object.lo_name); mtx_unlock_spin(&sched_lock); } @@ -498,7 +424,7 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, { int i = 0; - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); for (;;) { @@ -516,12 +442,12 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, else #endif panic("spin lock %s held by %p for > 5 seconds", - m->mtx_description, (void *)m->mtx_lock); + m->mtx_object.lo_name, (void *)m->mtx_lock); } } m->mtx_savecrit = mtx_crit; - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); return; @@ -545,13 +471,13 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) if (mtx_recursed(m)) { if (--(m->mtx_recurse) == 0) atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); return; } mtx_lock_spin(&sched_lock); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); p1 = TAILQ_FIRST(&m->mtx_blocked); @@ -563,7 +489,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) if (TAILQ_EMPTY(&m->mtx_blocked)) { LIST_REMOVE(m, mtx_contested); _release_lock_quick(m); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); } else atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); @@ -579,7 +505,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) pri = p->p_pri.pri_native; SET_PRIO(p, pri); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", m, p1); @@ -593,7 +519,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) struct ithd *it = p->p_ithd; if (it->it_interrupted) { - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p interrupted %p", it, it->it_interrupted); @@ -602,13 +528,13 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) } #endif setrunqueue(p); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p switching out lock=%p", m, (void *)m->mtx_lock); mi_switch(); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", m, (void *)m->mtx_lock); } @@ -623,6 +549,40 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) * See the _rel_spin_lock() macro for the details. */ +#ifdef WITNESS +/* + * Update the lock object flags before calling witness. Note that when we + * lock a mutex, this is called after getting the lock, but when unlocking + * a mutex, this function is called before releasing the lock. + */ +void +_mtx_update_flags(struct mtx *m, int locking) +{ + + mtx_assert(m, MA_OWNED); + if (locking) { + m->mtx_object.lo_flags |= LO_LOCKED; + if (mtx_recursed(m)) + m->mtx_object.lo_flags |= LO_RECURSED; + else + /* XXX: we shouldn't need this in theory. */ + m->mtx_object.lo_flags &= ~LO_RECURSED; + } else { + switch (m->mtx_recurse) { + case 0: + /* XXX: we shouldn't need the LO_RECURSED in theory. */ + m->mtx_object.lo_flags &= ~(LO_LOCKED | LO_RECURSED); + break; + case 1: + m->mtx_object.lo_flags &= ~(LO_RECURSED); + break; + default: + break; + } + } +} +#endif + /* * The backing function for the INVARIANTS-enabled mtx_assert() */ @@ -636,20 +596,20 @@ _mtx_assert(struct mtx *m, int what, const char *file, int line) case MA_OWNED | MA_NOTRECURSED: if (!mtx_owned(m)) panic("mutex %s not owned at %s:%d", - m->mtx_description, file, line); + m->mtx_object.lo_name, file, line); if (mtx_recursed(m)) { if ((what & MA_NOTRECURSED) != 0) panic("mutex %s recursed at %s:%d", - m->mtx_description, file, line); + m->mtx_object.lo_name, file, line); } else if ((what & MA_RECURSED) != 0) { panic("mutex %s unrecursed at %s:%d", - m->mtx_description, file, line); + m->mtx_object.lo_name, file, line); } break; case MA_NOTOWNED: if (mtx_owned(m)) panic("mutex %s owned at %s:%d", - m->mtx_description, file, line); + m->mtx_object.lo_name, file, line); break; default: panic("unknown mtx_assert at %s:%d", file, line); @@ -659,1058 +619,92 @@ _mtx_assert(struct mtx *m, int what, const char *file, int line) /* * The MUTEX_DEBUG-enabled mtx_validate() + * + * Most of these checks have been moved off into the LO_INITIALIZED flag + * maintained by the witness code. */ -#define MV_DESTROY 0 /* validate before destory */ -#define MV_INIT 1 /* validate before init */ - #ifdef MUTEX_DEBUG -int mtx_validate __P((struct mtx *, int)); +void mtx_validate __P((struct mtx *)); -int -mtx_validate(struct mtx *m, int when) +void +mtx_validate(struct mtx *m) { - struct mtx *mp; - int i; - int retval = 0; - -#ifdef WITNESS - if (witness_cold) - return 0; -#endif - if (m == &all_mtx || cold) - return 0; - mtx_lock(&all_mtx); /* * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly * we can re-enable the kernacc() checks. */ #ifndef __alpha__ - MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t), - VM_PROT_READ) == 1); + if (!kernacc((caddr_t)m, sizeof(m), VM_PROT_READ | VM_PROT_WRITE)) + panic("Can't read and write to mutex %p", m); #endif - MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx); - for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { -#ifndef __alpha__ - if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t), - VM_PROT_READ) != 1) { - panic("mtx_validate: mp=%p mp->mtx_next=%p", - mp, mp->mtx_next); - } -#endif - i++; - if (i > mtx_cur_cnt) { - panic("mtx_validate: too many in chain, known=%d\n", - mtx_cur_cnt); - } - } - MPASS(i == mtx_cur_cnt); - switch (when) { - case MV_DESTROY: - for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) - if (mp == m) - break; - MPASS(mp == m); - break; - case MV_INIT: - for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) - if (mp == m) { - /* - * Not good. This mutex already exists. - */ - printf("re-initing existing mutex %s\n", - m->mtx_description); - MPASS(m->mtx_lock == MTX_UNOWNED); - retval = 1; - } - } - mtx_unlock(&all_mtx); - return (retval); } #endif /* * Mutex initialization routine; initialize lock `m' of type contained in * `opts' with options contained in `opts' and description `description.' - * Place on "all_mtx" queue. */ void mtx_init(struct mtx *m, const char *description, int opts) { + struct lock_object *lock; - if ((opts & MTX_QUIET) == 0) - CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description); + MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | + MTX_SLEEPABLE | MTX_NOWITNESS)) == 0); #ifdef MUTEX_DEBUG /* Diagnostic and error correction */ - if (mtx_validate(m, MV_INIT)) - return; + mtx_validate(m); #endif - bzero((void *)m, sizeof *m); - TAILQ_INIT(&m->mtx_blocked); + bzero(m, sizeof(*m)); + lock = &m->mtx_object; + if (opts & MTX_SPIN) + lock->lo_class = &lock_class_mtx_spin; + else + lock->lo_class = &lock_class_mtx_sleep; + lock->lo_name = description; + if (opts & MTX_QUIET) + lock->lo_flags = LO_QUIET; + if (opts & MTX_RECURSE) + lock->lo_flags |= LO_RECURSABLE; + if (opts & MTX_SLEEPABLE) + lock->lo_flags |= LO_SLEEPABLE; + if ((opts & MTX_NOWITNESS) == 0) + lock->lo_flags |= LO_WITNESS; -#ifdef WITNESS - if (!witness_cold) { - m->mtx_debug = malloc(sizeof(struct mtx_debug), - M_WITNESS, M_NOWAIT | M_ZERO); - MPASS(m->mtx_debug != NULL); - } -#endif - - m->mtx_description = description; - m->mtx_flags = opts; m->mtx_lock = MTX_UNOWNED; + TAILQ_INIT(&m->mtx_blocked); - /* Put on all mutex queue */ - mtx_lock(&all_mtx); - m->mtx_next = &all_mtx; - m->mtx_prev = all_mtx.mtx_prev; - m->mtx_prev->mtx_next = m; - all_mtx.mtx_prev = m; - if (++mtx_cur_cnt > mtx_max_cnt) - mtx_max_cnt = mtx_cur_cnt; - mtx_unlock(&all_mtx); + LOCK_LOG_INIT(lock, opts); -#ifdef WITNESS - if (!witness_cold) - witness_init(m, opts); -#endif + WITNESS_INIT(lock); } /* - * Remove lock `m' from all_mtx queue. + * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be + * passed in as a flag here because if the corresponding mtx_init() was + * called with MTX_QUIET set, then it will already be set in the mutex's + * flags. */ void mtx_destroy(struct mtx *m) { -#ifdef WITNESS - KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", - __FUNCTION__)); -#endif - - CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description); - -#ifdef MUTEX_DEBUG - if (m->mtx_next == NULL) - panic("mtx_destroy: %p (%s) already destroyed", - m, m->mtx_description); + LOCK_LOG_DESTROY(&m->mtx_object, 0); - if (!mtx_owned(m)) { - MPASS(m->mtx_lock == MTX_UNOWNED); - } else { + if (!mtx_owned(m)) + MPASS(mtx_unowned(m)); + else { MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); - } - - /* diagnostic */ - mtx_validate(m, MV_DESTROY); -#endif - -#ifdef WITNESS - if (m->mtx_witness) - witness_destroy(m); -#endif /* WITNESS */ - - /* Remove from the all mutex queue */ - mtx_lock(&all_mtx); - m->mtx_next->mtx_prev = m->mtx_prev; - m->mtx_prev->mtx_next = m->mtx_next; - -#ifdef MUTEX_DEBUG - m->mtx_next = m->mtx_prev = NULL; -#endif - -#ifdef WITNESS - free(m->mtx_debug, M_WITNESS); - m->mtx_debug = NULL; -#endif - - mtx_cur_cnt--; - mtx_unlock(&all_mtx); -} - - -/* - * The WITNESS-enabled diagnostic code. - */ -#ifdef WITNESS -static void -witness_fixup(void *dummy __unused) -{ - struct mtx *mp; - - /* - * We have to release Giant before initializing its witness - * structure so that WITNESS doesn't get confused. - */ - mtx_unlock(&Giant); - mtx_assert(&Giant, MA_NOTOWNED); - - mtx_lock(&all_mtx); - - /* Iterate through all mutexes and finish up mutex initialization. */ - for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { - - mp->mtx_debug = malloc(sizeof(struct mtx_debug), - M_WITNESS, M_NOWAIT | M_ZERO); - MPASS(mp->mtx_debug != NULL); - - witness_init(mp, mp->mtx_flags); - } - mtx_unlock(&all_mtx); - - /* Mark the witness code as being ready for use. */ - atomic_store_rel_int(&witness_cold, 0); - - mtx_lock(&Giant); -} -SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) - -#define WITNESS_COUNT 200 -#define WITNESS_NCHILDREN 2 - -int witness_watch = 1; - -struct witness { - struct witness *w_next; - const char *w_description; - const char *w_file; - int w_line; - struct witness *w_morechildren; - u_char w_childcnt; - u_char w_Giant_squawked:1; - u_char w_other_squawked:1; - u_char w_same_squawked:1; - u_char w_spin:1; /* MTX_SPIN type mutex. */ - u_int w_level; - struct witness *w_children[WITNESS_NCHILDREN]; -}; - -struct witness_blessed { - char *b_lock1; - char *b_lock2; -}; - -#ifdef DDB -/* - * When DDB is enabled and witness_ddb is set to 1, it will cause the system to - * drop into kdebug() when: - * - a lock heirarchy violation occurs - * - locks are held when going to sleep. - */ -int witness_ddb; -#ifdef WITNESS_DDB -TUNABLE_INT_DECL("debug.witness_ddb", 1, witness_ddb); -#else -TUNABLE_INT_DECL("debug.witness_ddb", 0, witness_ddb); -#endif -SYSCTL_INT(_debug, OID_AUTO, witness_ddb, CTLFLAG_RW, &witness_ddb, 0, ""); -#endif /* DDB */ - -int witness_skipspin; -#ifdef WITNESS_SKIPSPIN -TUNABLE_INT_DECL("debug.witness_skipspin", 1, witness_skipspin); -#else -TUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin); -#endif -SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, - ""); - -/* - * Witness-enabled globals - */ -static struct mtx w_mtx; -static struct witness *w_free; -static struct witness *w_all; -static int w_inited; -static int witness_dead; /* fatal error, probably no memory */ - -static struct witness w_data[WITNESS_COUNT]; - -/* - * Internal witness routine prototypes - */ -static struct witness *enroll(const char *description, int flag); -static int itismychild(struct witness *parent, struct witness *child); -static void removechild(struct witness *parent, struct witness *child); -static int isitmychild(struct witness *parent, struct witness *child); -static int isitmydescendant(struct witness *parent, struct witness *child); -static int dup_ok(struct witness *); -static int blessed(struct witness *, struct witness *); -static void - witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *); -static void witness_leveldescendents(struct witness *parent, int level); -static void witness_levelall(void); -static struct witness * witness_get(void); -static void witness_free(struct witness *m); - -static char *ignore_list[] = { - "witness lock", - NULL -}; - -static char *spin_order_list[] = { -#if defined(__i386__) && defined (SMP) - "com", -#endif - "sio", -#ifdef __i386__ - "cy", -#endif - "ng_node", - "ng_worklist", - "ithread table lock", - "ithread list lock", - "sched lock", -#ifdef __i386__ - "clk", -#endif - "callout", - /* - * leaf locks - */ -#ifdef SMP -#ifdef __i386__ - "ap boot", - "imen", -#endif - "smp rendezvous", -#endif - NULL -}; - -static char *order_list[] = { - "Giant", "proctree", "allproc", "process lock", "uidinfo hash", - "uidinfo struct", NULL, - NULL -}; -static char *dup_list[] = { - "process lock", - NULL -}; - -static char *sleep_list[] = { - "Giant", - NULL -}; - -/* - * Pairs of locks which have been blessed - * Don't complain about order problems with blessed locks - */ -static struct witness_blessed blessed_list[] = { -}; -static int blessed_count = - sizeof(blessed_list) / sizeof(struct witness_blessed); - -static void -witness_init(struct mtx *m, int flag) -{ - m->mtx_witness = enroll(m->mtx_description, flag); -} - -static void -witness_destroy(struct mtx *m) -{ - struct mtx *m1; - struct proc *p; - p = curproc; - LIST_FOREACH(m1, &p->p_heldmtx, mtx_held) { - if (m1 == m) { - LIST_REMOVE(m, mtx_held); - break; - } + /* Tell witness this isn't locked to make it happy. */ + m->mtx_object.lo_flags &= ~LO_LOCKED; + WITNESS_UNLOCK(&m->mtx_object, MTX_NOSWITCH, __FILE__, + __LINE__); } - return; + WITNESS_DESTROY(&m->mtx_object); } - -static void -witness_display(void(*prnt)(const char *fmt, ...)) -{ - struct witness *w, *w1; - int level, found; - - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - witness_levelall(); - - /* - * First, handle sleep mutexes which have been acquired at least - * once. - */ - prnt("Sleep mutexes:\n"); - for (w = w_all; w; w = w->w_next) { - if (w->w_file == NULL || w->w_spin) - continue; - for (w1 = w_all; w1; w1 = w1->w_next) { - if (isitmychild(w1, w)) - break; - } - if (w1 != NULL) - continue; - /* - * This lock has no anscestors, display its descendants. - */ - witness_displaydescendants(prnt, w); - } - - /* - * Now do spin mutexes which have been acquired at least once. - */ - prnt("\nSpin mutexes:\n"); - level = 0; - while (level < sizeof(spin_order_list) / sizeof(char *)) { - found = 0; - for (w = w_all; w; w = w->w_next) { - if (w->w_file == NULL || !w->w_spin) - continue; - if (w->w_level == 1 << level) { - witness_displaydescendants(prnt, w); - level++; - found = 1; - } - } - if (found == 0) - level++; - } - - /* - * Finally, any mutexes which have not been acquired yet. - */ - prnt("\nMutexes which were never acquired:\n"); - for (w = w_all; w; w = w->w_next) { - if (w->w_file != NULL) - continue; - prnt("%s\n", w->w_description); - } -} - -void -witness_enter(struct mtx *m, int flags, const char *file, int line) -{ - struct witness *w, *w1; - struct mtx *m1; - struct proc *p; - int i; -#ifdef DDB - int go_into_ddb = 0; -#endif /* DDB */ - - if (witness_cold || m->mtx_witness == NULL || panicstr) - return; - w = m->mtx_witness; - p = curproc; - - if (flags & MTX_SPIN) { - if ((m->mtx_flags & MTX_SPIN) == 0) - panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @" - " %s:%d", m->mtx_description, file, line); - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_enter: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; - } - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - i = PCPU_GET(witness_spin_check); - if (i != 0 && w->w_level < i) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - panic("mutex_enter(%s:%x, MTX_SPIN) out of order @" - " %s:%d already holding %s:%x", - m->mtx_description, w->w_level, file, line, - spin_order_list[ffs(i)-1], i); - } - PCPU_SET(witness_spin_check, i | w->w_level); - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - p->p_spinlocks++; - MPASS(p->p_spinlocks > 0); - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - return; - } - if ((m->mtx_flags & MTX_SPIN) != 0) - panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", - m->mtx_description, file, line); - - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_enter: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; - } - if (witness_dead) - goto out; - if (cold) - goto out; - - if (p->p_spinlocks != 0) - panic("blockable mtx_lock() of %s when not legal @ %s:%d", - m->mtx_description, file, line); - /* - * Is this the first mutex acquired - */ - if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL) - goto out; - - if ((w1 = m1->mtx_witness) == w) { - if (w->w_same_squawked || dup_ok(w)) - goto out; - w->w_same_squawked = 1; - printf("acquring duplicate lock of same type: \"%s\"\n", - m->mtx_description); - printf(" 1st @ %s:%d\n", w->w_file, w->w_line); - printf(" 2nd @ %s:%d\n", file, line); -#ifdef DDB - go_into_ddb = 1; -#endif /* DDB */ - goto out; - } - MPASS(!mtx_owned(&w_mtx)); - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - /* - * If we have a known higher number just say ok - */ - if (witness_watch > 1 && w->w_level > w1->w_level) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - goto out; - } - if (isitmydescendant(m1->mtx_witness, w)) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - goto out; - } - for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) { - - MPASS(i < 200); - w1 = m1->mtx_witness; - if (isitmydescendant(w, w1)) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - if (blessed(w, w1)) - goto out; - if (m1 == &Giant) { - if (w1->w_Giant_squawked) - goto out; - else - w1->w_Giant_squawked = 1; - } else { - if (w1->w_other_squawked) - goto out; - else - w1->w_other_squawked = 1; - } - printf("lock order reversal\n"); - printf(" 1st %s last acquired @ %s:%d\n", - w->w_description, w->w_file, w->w_line); - printf(" 2nd %p %s @ %s:%d\n", - m1, w1->w_description, w1->w_file, w1->w_line); - printf(" 3rd %p %s @ %s:%d\n", - m, w->w_description, file, line); -#ifdef DDB - go_into_ddb = 1; -#endif /* DDB */ - goto out; - } - } - m1 = LIST_FIRST(&p->p_heldmtx); - if (!itismychild(m1->mtx_witness, w)) - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - -out: -#ifdef DDB - if (witness_ddb && go_into_ddb) - Debugger("witness_enter"); -#endif /* DDB */ - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - - /* - * If this pays off it likely means that a mutex being witnessed - * is acquired in hardclock. Put it in the ignore list. It is - * likely not the mutex this assert fails on. - */ - MPASS(m->mtx_held.le_prev == NULL); - LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); -} - -void -witness_try_enter(struct mtx *m, int flags, const char *file, int line) -{ - struct proc *p; - struct witness *w = m->mtx_witness; - - if (witness_cold) - return; - if (panicstr) - return; - if (flags & MTX_SPIN) { - if ((m->mtx_flags & MTX_SPIN) == 0) - panic("mutex_try_enter: " - "MTX_SPIN on MTX_DEF mutex %s @ %s:%d", - m->mtx_description, file, line); - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_try_enter: recursion on" - " non-recursive mutex %s @ %s:%d", - m->mtx_description, file, line); - return; - } - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - PCPU_SET(witness_spin_check, - PCPU_GET(witness_spin_check) | w->w_level); - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - return; - } - - if ((m->mtx_flags & MTX_SPIN) != 0) - panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", - m->mtx_description, file, line); - - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_try_enter: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, file, - line); - return; - } - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - p = curproc; - MPASS(m->mtx_held.le_prev == NULL); - LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); -} - -void -witness_exit(struct mtx *m, int flags, const char *file, int line) -{ - struct witness *w; - struct proc *p; - - if (witness_cold || m->mtx_witness == NULL || panicstr) - return; - w = m->mtx_witness; - p = curproc; - - if (flags & MTX_SPIN) { - if ((m->mtx_flags & MTX_SPIN) == 0) - panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @" - " %s:%d", m->mtx_description, file, line); - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_exit: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; - } - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - PCPU_SET(witness_spin_check, - PCPU_GET(witness_spin_check) & ~w->w_level); - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - MPASS(p->p_spinlocks > 0); - p->p_spinlocks--; - return; - } - if ((m->mtx_flags & MTX_SPIN) != 0) - panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", - m->mtx_description, file, line); - - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_exit: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; - } - - if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold) - panic("switchable mtx_unlock() of %s when not legal @ %s:%d", - m->mtx_description, file, line); - LIST_REMOVE(m, mtx_held); - m->mtx_held.le_prev = NULL; -} - -int -witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) -{ - struct mtx *m; - struct proc *p; - char **sleep; - int n = 0; - - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - p = curproc; - LIST_FOREACH(m, &p->p_heldmtx, mtx_held) { - if (m == mtx) - continue; - for (sleep = sleep_list; *sleep!= NULL; sleep++) - if (strcmp(m->mtx_description, *sleep) == 0) - goto next; - if (n == 0) - printf("Whee!\n"); - printf("%s:%d: %s with \"%s\" locked from %s:%d\n", - file, line, check_only ? "could sleep" : "sleeping", - m->mtx_description, - m->mtx_witness->w_file, m->mtx_witness->w_line); - n++; - next: - } -#ifdef DDB - if (witness_ddb && n) - Debugger("witness_sleep"); -#endif /* DDB */ - return (n); -} - -static struct witness * -enroll(const char *description, int flag) -{ - int i; - struct witness *w, *w1; - char **ignore; - char **order; - - if (!witness_watch) - return (NULL); - for (ignore = ignore_list; *ignore != NULL; ignore++) - if (strcmp(description, *ignore) == 0) - return (NULL); - - if (w_inited == 0) { - mtx_init(&w_mtx, "witness lock", MTX_SPIN); - for (i = 0; i < WITNESS_COUNT; i++) { - w = &w_data[i]; - witness_free(w); - } - w_inited = 1; - for (order = order_list; *order != NULL; order++) { - w = enroll(*order, MTX_DEF); - w->w_file = "order list"; - for (order++; *order != NULL; order++) { - w1 = enroll(*order, MTX_DEF); - w1->w_file = "order list"; - itismychild(w, w1); - w = w1; - } - } - } - if ((flag & MTX_SPIN) && witness_skipspin) - return (NULL); - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - for (w = w_all; w; w = w->w_next) { - if (strcmp(description, w->w_description) == 0) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - return (w); - } - } - if ((w = witness_get()) == NULL) - return (NULL); - w->w_next = w_all; - w_all = w; - w->w_description = description; - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - if (flag & MTX_SPIN) { - w->w_spin = 1; - - i = 1; - for (order = spin_order_list; *order != NULL; order++) { - if (strcmp(description, *order) == 0) - break; - i <<= 1; - } - if (*order == NULL) - panic("spin lock %s not in order list", description); - w->w_level = i; - } - - return (w); -} - -static int -itismychild(struct witness *parent, struct witness *child) -{ - static int recursed; - - /* - * Insert "child" after "parent" - */ - while (parent->w_morechildren) - parent = parent->w_morechildren; - - if (parent->w_childcnt == WITNESS_NCHILDREN) { - if ((parent->w_morechildren = witness_get()) == NULL) - return (1); - parent = parent->w_morechildren; - } - MPASS(child != NULL); - parent->w_children[parent->w_childcnt++] = child; - /* - * now prune whole tree - */ - if (recursed) - return (0); - recursed = 1; - for (child = w_all; child != NULL; child = child->w_next) { - for (parent = w_all; parent != NULL; - parent = parent->w_next) { - if (!isitmychild(parent, child)) - continue; - removechild(parent, child); - if (isitmydescendant(parent, child)) - continue; - itismychild(parent, child); - } - } - recursed = 0; - witness_levelall(); - return (0); -} - -static void -removechild(struct witness *parent, struct witness *child) -{ - struct witness *w, *w1; - int i; - - for (w = parent; w != NULL; w = w->w_morechildren) - for (i = 0; i < w->w_childcnt; i++) - if (w->w_children[i] == child) - goto found; - return; -found: - for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren) - continue; - w->w_children[i] = w1->w_children[--w1->w_childcnt]; - MPASS(w->w_children[i] != NULL); - - if (w1->w_childcnt != 0) - return; - - if (w1 == parent) - return; - for (w = parent; w->w_morechildren != w1; w = w->w_morechildren) - continue; - w->w_morechildren = 0; - witness_free(w1); -} - -static int -isitmychild(struct witness *parent, struct witness *child) -{ - struct witness *w; - int i; - - for (w = parent; w != NULL; w = w->w_morechildren) { - for (i = 0; i < w->w_childcnt; i++) { - if (w->w_children[i] == child) - return (1); - } - } - return (0); -} - -static int -isitmydescendant(struct witness *parent, struct witness *child) -{ - struct witness *w; - int i; - int j; - - for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) { - MPASS(j < 1000); - for (i = 0; i < w->w_childcnt; i++) { - if (w->w_children[i] == child) - return (1); - } - for (i = 0; i < w->w_childcnt; i++) { - if (isitmydescendant(w->w_children[i], child)) - return (1); - } - } - return (0); -} - -void -witness_levelall (void) -{ - struct witness *w, *w1; - - for (w = w_all; w; w = w->w_next) - if (!(w->w_spin)) - w->w_level = 0; - for (w = w_all; w; w = w->w_next) { - if (w->w_spin) - continue; - for (w1 = w_all; w1; w1 = w1->w_next) { - if (isitmychild(w1, w)) - break; - } - if (w1 != NULL) - continue; - witness_leveldescendents(w, 0); - } -} - -static void -witness_leveldescendents(struct witness *parent, int level) -{ - int i; - struct witness *w; - - if (parent->w_level < level) - parent->w_level = level; - level++; - for (w = parent; w != NULL; w = w->w_morechildren) - for (i = 0; i < w->w_childcnt; i++) - witness_leveldescendents(w->w_children[i], level); -} - -static void -witness_displaydescendants(void(*prnt)(const char *fmt, ...), - struct witness *parent) -{ - struct witness *w; - int i; - int level; - - level = parent->w_spin ? ffs(parent->w_level) : parent->w_level; - - prnt("%d", level); - if (level < 10) - prnt(" "); - for (i = 0; i < level; i++) - prnt(" "); - prnt("%s", parent->w_description); - if (parent->w_file != NULL) - prnt(" -- last acquired @ %s:%d\n", parent->w_file, - parent->w_line); - - for (w = parent; w != NULL; w = w->w_morechildren) - for (i = 0; i < w->w_childcnt; i++) - witness_displaydescendants(prnt, w->w_children[i]); - } - -static int -dup_ok(struct witness *w) -{ - char **dup; - - for (dup = dup_list; *dup!= NULL; dup++) - if (strcmp(w->w_description, *dup) == 0) - return (1); - return (0); -} - -static int -blessed(struct witness *w1, struct witness *w2) -{ - int i; - struct witness_blessed *b; - - for (i = 0; i < blessed_count; i++) { - b = &blessed_list[i]; - if (strcmp(w1->w_description, b->b_lock1) == 0) { - if (strcmp(w2->w_description, b->b_lock2) == 0) - return (1); - continue; - } - if (strcmp(w1->w_description, b->b_lock2) == 0) - if (strcmp(w2->w_description, b->b_lock1) == 0) - return (1); - } - return (0); -} - -static struct witness * -witness_get() -{ - struct witness *w; - - if ((w = w_free) == NULL) { - witness_dead = 1; - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - printf("witness exhausted\n"); - return (NULL); - } - w_free = w->w_next; - bzero(w, sizeof(*w)); - return (w); -} - -static void -witness_free(struct witness *w) -{ - w->w_next = w_free; - w_free = w; -} - -int -witness_list(struct proc *p) -{ - struct mtx *m; - int nheld; - - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - nheld = 0; - LIST_FOREACH(m, &p->p_heldmtx, mtx_held) { - printf("\t\"%s\" (%p) locked at %s:%d\n", - m->mtx_description, m, - m->mtx_witness->w_file, m->mtx_witness->w_line); - nheld++; - } - - return (nheld); -} - -#ifdef DDB - -DB_SHOW_COMMAND(mutexes, db_witness_list) -{ - - witness_list(curproc); -} - -DB_SHOW_COMMAND(witness, db_witness_display) -{ - - witness_display(db_printf); -} -#endif - -void -witness_save(struct mtx *m, const char **filep, int *linep) -{ - - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - if (m->mtx_witness == NULL) - return; - - *filep = m->mtx_witness->w_file; - *linep = m->mtx_witness->w_line; -} - -void -witness_restore(struct mtx *m, const char *file, int line) -{ - - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - if (m->mtx_witness == NULL) - return; - - m->mtx_witness->w_file = file; - m->mtx_witness->w_line = line; -} - -#endif /* WITNESS */ diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c index a1c576f..0f39097 100644 --- a/sys/kern/kern_sx.c +++ b/sys/kern/kern_sx.c @@ -46,43 +46,68 @@ #include <sys/systm.h> #include <sys/ktr.h> #include <sys/condvar.h> +#include <sys/lock.h> #include <sys/mutex.h> #include <sys/sx.h> +/* + * XXX: We don't implement the LO_RECURSED flag for this lock yet. + * We could do this by walking p_sleeplocks if we really wanted to. + */ +struct lock_class lock_class_sx = { + "sx", + LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE +}; + void sx_init(struct sx *sx, const char *description) { - - mtx_init(&sx->sx_lock, description, MTX_DEF); + struct lock_object *lock; + + bzero(sx, sizeof(*sx)); + lock = &sx->sx_object; + lock->lo_class = &lock_class_sx; + lock->lo_name = description; + lock->lo_flags = LO_WITNESS | LO_SLEEPABLE; + mtx_init(&sx->sx_lock, "sx backing lock", + MTX_DEF | MTX_NOWITNESS | MTX_QUIET); sx->sx_cnt = 0; cv_init(&sx->sx_shrd_cv, description); sx->sx_shrd_wcnt = 0; cv_init(&sx->sx_excl_cv, description); - sx->sx_descr = description; sx->sx_excl_wcnt = 0; sx->sx_xholder = NULL; + + LOCK_LOG_INIT(lock, 0); + + WITNESS_INIT(lock); } void sx_destroy(struct sx *sx) { + LOCK_LOG_DESTROY(&sx->sx_object, 0); + KASSERT((sx->sx_cnt == 0 && sx->sx_shrd_wcnt == 0 && sx->sx_excl_wcnt == - 0), ("%s (%s): holders or waiters\n", __FUNCTION__, sx->sx_descr)); + 0), ("%s (%s): holders or waiters\n", __FUNCTION__, + sx->sx_object.lo_name)); mtx_destroy(&sx->sx_lock); cv_destroy(&sx->sx_shrd_cv); cv_destroy(&sx->sx_excl_cv); + + WITNESS_DESTROY(&sx->sx_object); } void -sx_slock(struct sx *sx) +_sx_slock(struct sx *sx, const char *file, int line) { mtx_lock(&sx->sx_lock); KASSERT(sx->sx_xholder != curproc, ("%s (%s): trying to get slock while xlock is held\n", __FUNCTION__, - sx->sx_descr)); + sx->sx_object.lo_name)); /* * Loop in case we lose the race for lock acquisition. @@ -96,11 +121,17 @@ sx_slock(struct sx *sx) /* Acquire a shared lock. */ sx->sx_cnt++; +#ifdef WITNESS + sx->sx_object.lo_flags |= LO_LOCKED; +#endif + LOCK_LOG_LOCK("SLOCK", &sx->sx_object, 0, 0, file, line); + WITNESS_LOCK(&sx->sx_object, 0, file, line); + mtx_unlock(&sx->sx_lock); } void -sx_xlock(struct sx *sx) +_sx_xlock(struct sx *sx, const char *file, int line) { mtx_lock(&sx->sx_lock); @@ -113,7 +144,8 @@ sx_xlock(struct sx *sx) * INVARIANTS. */ KASSERT(sx->sx_xholder != curproc, - ("%s (%s): xlock already held", __FUNCTION__, sx->sx_descr)); + ("%s (%s): xlock already held @ %s:%d", __FUNCTION__, + sx->sx_object.lo_name, file, line)); /* Loop in case we lose the race for lock acquisition. */ while (sx->sx_cnt != 0) { @@ -128,16 +160,28 @@ sx_xlock(struct sx *sx) sx->sx_cnt--; sx->sx_xholder = curproc; +#ifdef WITNESS + sx->sx_object.lo_flags |= LO_LOCKED; +#endif + LOCK_LOG_LOCK("XLOCK", &sx->sx_object, 0, 0, file, line); + WITNESS_LOCK(&sx->sx_object, 0, file, line); + mtx_unlock(&sx->sx_lock); } void -sx_sunlock(struct sx *sx) +_sx_sunlock(struct sx *sx, const char *file, int line) { mtx_lock(&sx->sx_lock); _SX_ASSERT_SLOCKED(sx); +#ifdef WITNESS + if (sx->sx_cnt == 0) + sx->sx_object.lo_flags &= ~LO_LOCKED; +#endif + WITNESS_UNLOCK(&sx->sx_object, 0, file, line); + /* Release. */ sx->sx_cnt--; @@ -153,17 +197,24 @@ sx_sunlock(struct sx *sx) } else if (sx->sx_shrd_wcnt > 0) cv_broadcast(&sx->sx_shrd_cv); + LOCK_LOG_LOCK("SUNLOCK", &sx->sx_object, 0, 0, file, line); + mtx_unlock(&sx->sx_lock); } void -sx_xunlock(struct sx *sx) +_sx_xunlock(struct sx *sx, const char *file, int line) { mtx_lock(&sx->sx_lock); _SX_ASSERT_XLOCKED(sx); MPASS(sx->sx_cnt == -1); +#ifdef WITNESS + sx->sx_object.lo_flags &= ~LO_LOCKED; +#endif + WITNESS_UNLOCK(&sx->sx_object, 0, file, line); + /* Release. */ sx->sx_cnt++; sx->sx_xholder = NULL; @@ -176,5 +227,7 @@ sx_xunlock(struct sx *sx) else if (sx->sx_excl_wcnt > 0) cv_signal(&sx->sx_excl_cv); + LOCK_LOG_LOCK("XUNLOCK", &sx->sx_object, 0, 0, file, line); + mtx_unlock(&sx->sx_lock); } diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 4578879..5a3cd80 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -382,7 +382,7 @@ msleep(ident, mtx, priority, wmesg, timo) if (p && KTRPOINT(p, KTR_CSW)) ktrcsw(p->p_tracep, 1, 0); #endif - WITNESS_SLEEP(0, mtx); + WITNESS_SLEEP(0, &mtx->mtx_object); mtx_lock_spin(&sched_lock); if (cold || panicstr) { /* @@ -401,7 +401,7 @@ msleep(ident, mtx, priority, wmesg, timo) if (mtx != NULL) { mtx_assert(mtx, MA_OWNED | MA_NOTRECURSED); - WITNESS_SAVE(mtx, mtx); + WITNESS_SAVE(&mtx->mtx_object, mtx); mtx_unlock_flags(mtx, MTX_NOSWITCH); if (priority & PDROP) mtx = NULL; @@ -498,7 +498,7 @@ out: PICKUP_GIANT(); if (mtx != NULL) { mtx_lock(mtx); - WITNESS_RESTORE(mtx, mtx); + WITNESS_RESTORE(&mtx->mtx_object, mtx); } return (rval); } @@ -573,12 +573,12 @@ mawait(struct mtx *mtx, int priority, int timo) int s; WITNESS_SAVE_DECL(mtx); - WITNESS_SLEEP(0, mtx); + WITNESS_SLEEP(0, &mtx->mtx_object); mtx_lock_spin(&sched_lock); DROP_GIANT_NOSWITCH(); if (mtx != NULL) { mtx_assert(mtx, MA_OWNED | MA_NOTRECURSED); - WITNESS_SAVE(mtx, mtx); + WITNESS_SAVE(&mtx->mtx_object, mtx); mtx_unlock_flags(mtx, MTX_NOSWITCH); if (priority & PDROP) mtx = NULL; @@ -691,7 +691,7 @@ out: PICKUP_GIANT(); if (mtx != NULL) { mtx_lock(mtx); - WITNESS_RESTORE(mtx, mtx); + WITNESS_RESTORE(&mtx->mtx_object, mtx); } return (rval); } diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index 9797905..47903d0 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -2268,6 +2268,7 @@ ap_init(void) * something unique to lock with. */ PCPU_SET(curproc, PCPU_GET(idleproc)); + PCPU_SET(spinlocks, NULL); /* lock against other AP's that are waking up */ mtx_lock_spin(&ap_boot_mtx); diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c index ee285af..ba2da45 100644 --- a/sys/kern/subr_turnstile.c +++ b/sys/kern/subr_turnstile.c @@ -56,12 +56,13 @@ */ #include "opt_ddb.h" -#include "opt_witness.h" #include <sys/param.h> #include <sys/bus.h> #include <sys/kernel.h> #include <sys/malloc.h> +#include <sys/lock.h> +#include <sys/mutex.h> #include <sys/proc.h> #include <sys/sysctl.h> #include <sys/systm.h> @@ -78,25 +79,6 @@ #include <vm/vm.h> #include <vm/vm_extern.h> -#include <sys/mutex.h> - -/* - * The WITNESS-enabled mutex debug structure. - */ -#ifdef WITNESS -struct mtx_debug { - struct witness *mtxd_witness; - LIST_ENTRY(mtx) mtxd_held; - const char *mtxd_file; - int mtxd_line; -}; - -#define mtx_held mtx_debug->mtxd_held -#define mtx_file mtx_debug->mtxd_file -#define mtx_line mtx_debug->mtxd_line -#define mtx_witness mtx_debug->mtxd_witness -#endif /* WITNESS */ - /* * Internal utility macros. */ @@ -108,71 +90,19 @@ struct mtx_debug { #define SET_PRIO(p, pri) (p)->p_pri.pri_level = (pri) /* - * Early WITNESS-enabled declarations. - */ -#ifdef WITNESS - -/* - * Internal WITNESS routines which must be prototyped early. - * - * XXX: When/if witness code is cleaned up, it would be wise to place all - * witness prototyping early in this file. - */ -static void witness_init(struct mtx *, int flag); -static void witness_destroy(struct mtx *); -static void witness_display(void(*)(const char *fmt, ...)); - -MALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure"); - -/* All mutexes in system (used for debug/panic) */ -static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 }; - -/* - * This global is set to 0 once it becomes safe to use the witness code. - */ -static int witness_cold = 1; - -#else /* WITNESS */ - -/* XXX XXX XXX - * flag++ is sleazoid way of shuting up warning - */ -#define witness_init(m, flag) flag++ -#define witness_destroy(m) -#define witness_try_enter(m, t, f, l) -#endif /* WITNESS */ - -/* - * All mutex locks in system are kept on the all_mtx list. + * Lock classes for sleep and spin mutexes. */ -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head", - TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), - { NULL, NULL }, &all_mtx, &all_mtx, -#ifdef WITNESS - &all_mtx_debug -#else - NULL -#endif - }; - -/* - * Global variables for book keeping. - */ -static int mtx_cur_cnt; -static int mtx_max_cnt; - -/* - * Couple of strings for KTR_LOCK tracing in order to avoid duplicates. - */ -char STR_mtx_lock_slp[] = "GOT (sleep) %s [%p] r=%d at %s:%d"; -char STR_mtx_unlock_slp[] = "REL (sleep) %s [%p] r=%d at %s:%d"; -char STR_mtx_lock_spn[] = "GOT (spin) %s [%p] r=%d at %s:%d"; -char STR_mtx_unlock_spn[] = "REL (spin) %s [%p] r=%d at %s:%d"; +struct lock_class lock_class_mtx_sleep = { + "sleep mutex", + LC_SLEEPLOCK | LC_RECURSABLE +}; +struct lock_class lock_class_mtx_spin = { + "spin mutex", + LC_SPINLOCK | LC_RECURSABLE +}; /* * Prototypes for non-exported routines. - * - * NOTE: Prototypes for witness routines are placed at the bottom of the file. */ static void propagate_priority(struct proc *); @@ -241,7 +171,7 @@ propagate_priority(struct proc *p) KASSERT(p->p_stat == SMTX, ( "process %d(%s):%d holds %s but isn't blocked on a mutex\n", p->p_pid, p->p_comm, p->p_stat, - m->mtx_description)); + m->mtx_object.lo_name)); /* * Pick up the mutex that p is blocked on. @@ -280,7 +210,7 @@ propagate_priority(struct proc *p) TAILQ_INSERT_BEFORE(p1, p, p_procq); CTR4(KTR_LOCK, "propagate_priority: p %p moved before %p on [%p] %s", - p, p1, m, m->mtx_description); + p, p1, m, m->mtx_object.lo_name); } } @@ -336,23 +266,19 @@ _mtx_trylock(struct mtx *m, int opts, const char *file, int line) rval = _obtain_lock(m, curproc); -#ifdef WITNESS - if (rval && m->mtx_witness != NULL) { + LOCK_LOG_TRY("LOCK", &m->mtx_object, opts, rval, file, line); + if (rval) { /* * We do not handle recursion in _mtx_trylock; see the * note at the top of the routine. */ KASSERT(!mtx_recursed(m), ("mtx_trylock() called on a recursed mutex")); - witness_try_enter(m, (opts | m->mtx_flags), file, line); + mtx_update_flags(m, 1); + WITNESS_LOCK(&m->mtx_object, opts | LOP_TRYLOCK, file, line); } -#endif /* WITNESS */ - - if ((opts & MTX_QUIET) == 0) - CTR5(KTR_LOCK, "TRY_LOCK %s [%p] result=%d at %s:%d", - m->mtx_description, m, rval, file, line); - return rval; + return (rval); } /* @@ -369,15 +295,15 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { m->mtx_recurse++; atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); return; } - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR4(KTR_LOCK, "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", - m->mtx_description, (void *)m->mtx_lock, file, line); + m->mtx_object.lo_name, (void *)m->mtx_lock, file, line); while (!_obtain_lock(m, p)) { uintptr_t v; @@ -434,7 +360,7 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) struct ithd *it = p->p_ithd; if (it->it_interrupted) { - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_lock_sleep: %p interrupted %p", it, it->it_interrupted); @@ -464,21 +390,21 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) * Save who we're blocked on. */ p->p_blocked = m; - p->p_mtxname = m->mtx_description; + p->p_mtxname = m->mtx_object.lo_name; p->p_stat = SMTX; propagate_priority(p); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR3(KTR_LOCK, "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m, - m->mtx_description); + m->mtx_object.lo_name); mi_switch(); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR3(KTR_LOCK, "_mtx_lock_sleep: p %p free from blocked on [%p] %s", - p, m, m->mtx_description); + p, m, m->mtx_object.lo_name); mtx_unlock_spin(&sched_lock); } @@ -498,7 +424,7 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, { int i = 0; - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); for (;;) { @@ -516,12 +442,12 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, else #endif panic("spin lock %s held by %p for > 5 seconds", - m->mtx_description, (void *)m->mtx_lock); + m->mtx_object.lo_name, (void *)m->mtx_lock); } } m->mtx_savecrit = mtx_crit; - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); return; @@ -545,13 +471,13 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) if (mtx_recursed(m)) { if (--(m->mtx_recurse) == 0) atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); return; } mtx_lock_spin(&sched_lock); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); p1 = TAILQ_FIRST(&m->mtx_blocked); @@ -563,7 +489,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) if (TAILQ_EMPTY(&m->mtx_blocked)) { LIST_REMOVE(m, mtx_contested); _release_lock_quick(m); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); } else atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); @@ -579,7 +505,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) pri = p->p_pri.pri_native; SET_PRIO(p, pri); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", m, p1); @@ -593,7 +519,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) struct ithd *it = p->p_ithd; if (it->it_interrupted) { - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p interrupted %p", it, it->it_interrupted); @@ -602,13 +528,13 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) } #endif setrunqueue(p); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p switching out lock=%p", m, (void *)m->mtx_lock); mi_switch(); - if ((opts & MTX_QUIET) == 0) + if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", m, (void *)m->mtx_lock); } @@ -623,6 +549,40 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) * See the _rel_spin_lock() macro for the details. */ +#ifdef WITNESS +/* + * Update the lock object flags before calling witness. Note that when we + * lock a mutex, this is called after getting the lock, but when unlocking + * a mutex, this function is called before releasing the lock. + */ +void +_mtx_update_flags(struct mtx *m, int locking) +{ + + mtx_assert(m, MA_OWNED); + if (locking) { + m->mtx_object.lo_flags |= LO_LOCKED; + if (mtx_recursed(m)) + m->mtx_object.lo_flags |= LO_RECURSED; + else + /* XXX: we shouldn't need this in theory. */ + m->mtx_object.lo_flags &= ~LO_RECURSED; + } else { + switch (m->mtx_recurse) { + case 0: + /* XXX: we shouldn't need the LO_RECURSED in theory. */ + m->mtx_object.lo_flags &= ~(LO_LOCKED | LO_RECURSED); + break; + case 1: + m->mtx_object.lo_flags &= ~(LO_RECURSED); + break; + default: + break; + } + } +} +#endif + /* * The backing function for the INVARIANTS-enabled mtx_assert() */ @@ -636,20 +596,20 @@ _mtx_assert(struct mtx *m, int what, const char *file, int line) case MA_OWNED | MA_NOTRECURSED: if (!mtx_owned(m)) panic("mutex %s not owned at %s:%d", - m->mtx_description, file, line); + m->mtx_object.lo_name, file, line); if (mtx_recursed(m)) { if ((what & MA_NOTRECURSED) != 0) panic("mutex %s recursed at %s:%d", - m->mtx_description, file, line); + m->mtx_object.lo_name, file, line); } else if ((what & MA_RECURSED) != 0) { panic("mutex %s unrecursed at %s:%d", - m->mtx_description, file, line); + m->mtx_object.lo_name, file, line); } break; case MA_NOTOWNED: if (mtx_owned(m)) panic("mutex %s owned at %s:%d", - m->mtx_description, file, line); + m->mtx_object.lo_name, file, line); break; default: panic("unknown mtx_assert at %s:%d", file, line); @@ -659,1058 +619,92 @@ _mtx_assert(struct mtx *m, int what, const char *file, int line) /* * The MUTEX_DEBUG-enabled mtx_validate() + * + * Most of these checks have been moved off into the LO_INITIALIZED flag + * maintained by the witness code. */ -#define MV_DESTROY 0 /* validate before destory */ -#define MV_INIT 1 /* validate before init */ - #ifdef MUTEX_DEBUG -int mtx_validate __P((struct mtx *, int)); +void mtx_validate __P((struct mtx *)); -int -mtx_validate(struct mtx *m, int when) +void +mtx_validate(struct mtx *m) { - struct mtx *mp; - int i; - int retval = 0; - -#ifdef WITNESS - if (witness_cold) - return 0; -#endif - if (m == &all_mtx || cold) - return 0; - mtx_lock(&all_mtx); /* * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly * we can re-enable the kernacc() checks. */ #ifndef __alpha__ - MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t), - VM_PROT_READ) == 1); + if (!kernacc((caddr_t)m, sizeof(m), VM_PROT_READ | VM_PROT_WRITE)) + panic("Can't read and write to mutex %p", m); #endif - MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx); - for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { -#ifndef __alpha__ - if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t), - VM_PROT_READ) != 1) { - panic("mtx_validate: mp=%p mp->mtx_next=%p", - mp, mp->mtx_next); - } -#endif - i++; - if (i > mtx_cur_cnt) { - panic("mtx_validate: too many in chain, known=%d\n", - mtx_cur_cnt); - } - } - MPASS(i == mtx_cur_cnt); - switch (when) { - case MV_DESTROY: - for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) - if (mp == m) - break; - MPASS(mp == m); - break; - case MV_INIT: - for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) - if (mp == m) { - /* - * Not good. This mutex already exists. - */ - printf("re-initing existing mutex %s\n", - m->mtx_description); - MPASS(m->mtx_lock == MTX_UNOWNED); - retval = 1; - } - } - mtx_unlock(&all_mtx); - return (retval); } #endif /* * Mutex initialization routine; initialize lock `m' of type contained in * `opts' with options contained in `opts' and description `description.' - * Place on "all_mtx" queue. */ void mtx_init(struct mtx *m, const char *description, int opts) { + struct lock_object *lock; - if ((opts & MTX_QUIET) == 0) - CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description); + MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | + MTX_SLEEPABLE | MTX_NOWITNESS)) == 0); #ifdef MUTEX_DEBUG /* Diagnostic and error correction */ - if (mtx_validate(m, MV_INIT)) - return; + mtx_validate(m); #endif - bzero((void *)m, sizeof *m); - TAILQ_INIT(&m->mtx_blocked); + bzero(m, sizeof(*m)); + lock = &m->mtx_object; + if (opts & MTX_SPIN) + lock->lo_class = &lock_class_mtx_spin; + else + lock->lo_class = &lock_class_mtx_sleep; + lock->lo_name = description; + if (opts & MTX_QUIET) + lock->lo_flags = LO_QUIET; + if (opts & MTX_RECURSE) + lock->lo_flags |= LO_RECURSABLE; + if (opts & MTX_SLEEPABLE) + lock->lo_flags |= LO_SLEEPABLE; + if ((opts & MTX_NOWITNESS) == 0) + lock->lo_flags |= LO_WITNESS; -#ifdef WITNESS - if (!witness_cold) { - m->mtx_debug = malloc(sizeof(struct mtx_debug), - M_WITNESS, M_NOWAIT | M_ZERO); - MPASS(m->mtx_debug != NULL); - } -#endif - - m->mtx_description = description; - m->mtx_flags = opts; m->mtx_lock = MTX_UNOWNED; + TAILQ_INIT(&m->mtx_blocked); - /* Put on all mutex queue */ - mtx_lock(&all_mtx); - m->mtx_next = &all_mtx; - m->mtx_prev = all_mtx.mtx_prev; - m->mtx_prev->mtx_next = m; - all_mtx.mtx_prev = m; - if (++mtx_cur_cnt > mtx_max_cnt) - mtx_max_cnt = mtx_cur_cnt; - mtx_unlock(&all_mtx); + LOCK_LOG_INIT(lock, opts); -#ifdef WITNESS - if (!witness_cold) - witness_init(m, opts); -#endif + WITNESS_INIT(lock); } /* - * Remove lock `m' from all_mtx queue. + * Remove lock `m' from all_mtx queue. We don't allow MTX_QUIET to be + * passed in as a flag here because if the corresponding mtx_init() was + * called with MTX_QUIET set, then it will already be set in the mutex's + * flags. */ void mtx_destroy(struct mtx *m) { -#ifdef WITNESS - KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", - __FUNCTION__)); -#endif - - CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description); - -#ifdef MUTEX_DEBUG - if (m->mtx_next == NULL) - panic("mtx_destroy: %p (%s) already destroyed", - m, m->mtx_description); + LOCK_LOG_DESTROY(&m->mtx_object, 0); - if (!mtx_owned(m)) { - MPASS(m->mtx_lock == MTX_UNOWNED); - } else { + if (!mtx_owned(m)) + MPASS(mtx_unowned(m)); + else { MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); - } - - /* diagnostic */ - mtx_validate(m, MV_DESTROY); -#endif - -#ifdef WITNESS - if (m->mtx_witness) - witness_destroy(m); -#endif /* WITNESS */ - - /* Remove from the all mutex queue */ - mtx_lock(&all_mtx); - m->mtx_next->mtx_prev = m->mtx_prev; - m->mtx_prev->mtx_next = m->mtx_next; - -#ifdef MUTEX_DEBUG - m->mtx_next = m->mtx_prev = NULL; -#endif - -#ifdef WITNESS - free(m->mtx_debug, M_WITNESS); - m->mtx_debug = NULL; -#endif - - mtx_cur_cnt--; - mtx_unlock(&all_mtx); -} - - -/* - * The WITNESS-enabled diagnostic code. - */ -#ifdef WITNESS -static void -witness_fixup(void *dummy __unused) -{ - struct mtx *mp; - - /* - * We have to release Giant before initializing its witness - * structure so that WITNESS doesn't get confused. - */ - mtx_unlock(&Giant); - mtx_assert(&Giant, MA_NOTOWNED); - - mtx_lock(&all_mtx); - - /* Iterate through all mutexes and finish up mutex initialization. */ - for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { - - mp->mtx_debug = malloc(sizeof(struct mtx_debug), - M_WITNESS, M_NOWAIT | M_ZERO); - MPASS(mp->mtx_debug != NULL); - - witness_init(mp, mp->mtx_flags); - } - mtx_unlock(&all_mtx); - - /* Mark the witness code as being ready for use. */ - atomic_store_rel_int(&witness_cold, 0); - - mtx_lock(&Giant); -} -SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) - -#define WITNESS_COUNT 200 -#define WITNESS_NCHILDREN 2 - -int witness_watch = 1; - -struct witness { - struct witness *w_next; - const char *w_description; - const char *w_file; - int w_line; - struct witness *w_morechildren; - u_char w_childcnt; - u_char w_Giant_squawked:1; - u_char w_other_squawked:1; - u_char w_same_squawked:1; - u_char w_spin:1; /* MTX_SPIN type mutex. */ - u_int w_level; - struct witness *w_children[WITNESS_NCHILDREN]; -}; - -struct witness_blessed { - char *b_lock1; - char *b_lock2; -}; - -#ifdef DDB -/* - * When DDB is enabled and witness_ddb is set to 1, it will cause the system to - * drop into kdebug() when: - * - a lock heirarchy violation occurs - * - locks are held when going to sleep. - */ -int witness_ddb; -#ifdef WITNESS_DDB -TUNABLE_INT_DECL("debug.witness_ddb", 1, witness_ddb); -#else -TUNABLE_INT_DECL("debug.witness_ddb", 0, witness_ddb); -#endif -SYSCTL_INT(_debug, OID_AUTO, witness_ddb, CTLFLAG_RW, &witness_ddb, 0, ""); -#endif /* DDB */ - -int witness_skipspin; -#ifdef WITNESS_SKIPSPIN -TUNABLE_INT_DECL("debug.witness_skipspin", 1, witness_skipspin); -#else -TUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin); -#endif -SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, - ""); - -/* - * Witness-enabled globals - */ -static struct mtx w_mtx; -static struct witness *w_free; -static struct witness *w_all; -static int w_inited; -static int witness_dead; /* fatal error, probably no memory */ - -static struct witness w_data[WITNESS_COUNT]; - -/* - * Internal witness routine prototypes - */ -static struct witness *enroll(const char *description, int flag); -static int itismychild(struct witness *parent, struct witness *child); -static void removechild(struct witness *parent, struct witness *child); -static int isitmychild(struct witness *parent, struct witness *child); -static int isitmydescendant(struct witness *parent, struct witness *child); -static int dup_ok(struct witness *); -static int blessed(struct witness *, struct witness *); -static void - witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *); -static void witness_leveldescendents(struct witness *parent, int level); -static void witness_levelall(void); -static struct witness * witness_get(void); -static void witness_free(struct witness *m); - -static char *ignore_list[] = { - "witness lock", - NULL -}; - -static char *spin_order_list[] = { -#if defined(__i386__) && defined (SMP) - "com", -#endif - "sio", -#ifdef __i386__ - "cy", -#endif - "ng_node", - "ng_worklist", - "ithread table lock", - "ithread list lock", - "sched lock", -#ifdef __i386__ - "clk", -#endif - "callout", - /* - * leaf locks - */ -#ifdef SMP -#ifdef __i386__ - "ap boot", - "imen", -#endif - "smp rendezvous", -#endif - NULL -}; - -static char *order_list[] = { - "Giant", "proctree", "allproc", "process lock", "uidinfo hash", - "uidinfo struct", NULL, - NULL -}; -static char *dup_list[] = { - "process lock", - NULL -}; - -static char *sleep_list[] = { - "Giant", - NULL -}; - -/* - * Pairs of locks which have been blessed - * Don't complain about order problems with blessed locks - */ -static struct witness_blessed blessed_list[] = { -}; -static int blessed_count = - sizeof(blessed_list) / sizeof(struct witness_blessed); - -static void -witness_init(struct mtx *m, int flag) -{ - m->mtx_witness = enroll(m->mtx_description, flag); -} - -static void -witness_destroy(struct mtx *m) -{ - struct mtx *m1; - struct proc *p; - p = curproc; - LIST_FOREACH(m1, &p->p_heldmtx, mtx_held) { - if (m1 == m) { - LIST_REMOVE(m, mtx_held); - break; - } + /* Tell witness this isn't locked to make it happy. */ + m->mtx_object.lo_flags &= ~LO_LOCKED; + WITNESS_UNLOCK(&m->mtx_object, MTX_NOSWITCH, __FILE__, + __LINE__); } - return; + WITNESS_DESTROY(&m->mtx_object); } - -static void -witness_display(void(*prnt)(const char *fmt, ...)) -{ - struct witness *w, *w1; - int level, found; - - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - witness_levelall(); - - /* - * First, handle sleep mutexes which have been acquired at least - * once. - */ - prnt("Sleep mutexes:\n"); - for (w = w_all; w; w = w->w_next) { - if (w->w_file == NULL || w->w_spin) - continue; - for (w1 = w_all; w1; w1 = w1->w_next) { - if (isitmychild(w1, w)) - break; - } - if (w1 != NULL) - continue; - /* - * This lock has no anscestors, display its descendants. - */ - witness_displaydescendants(prnt, w); - } - - /* - * Now do spin mutexes which have been acquired at least once. - */ - prnt("\nSpin mutexes:\n"); - level = 0; - while (level < sizeof(spin_order_list) / sizeof(char *)) { - found = 0; - for (w = w_all; w; w = w->w_next) { - if (w->w_file == NULL || !w->w_spin) - continue; - if (w->w_level == 1 << level) { - witness_displaydescendants(prnt, w); - level++; - found = 1; - } - } - if (found == 0) - level++; - } - - /* - * Finally, any mutexes which have not been acquired yet. - */ - prnt("\nMutexes which were never acquired:\n"); - for (w = w_all; w; w = w->w_next) { - if (w->w_file != NULL) - continue; - prnt("%s\n", w->w_description); - } -} - -void -witness_enter(struct mtx *m, int flags, const char *file, int line) -{ - struct witness *w, *w1; - struct mtx *m1; - struct proc *p; - int i; -#ifdef DDB - int go_into_ddb = 0; -#endif /* DDB */ - - if (witness_cold || m->mtx_witness == NULL || panicstr) - return; - w = m->mtx_witness; - p = curproc; - - if (flags & MTX_SPIN) { - if ((m->mtx_flags & MTX_SPIN) == 0) - panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @" - " %s:%d", m->mtx_description, file, line); - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_enter: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; - } - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - i = PCPU_GET(witness_spin_check); - if (i != 0 && w->w_level < i) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - panic("mutex_enter(%s:%x, MTX_SPIN) out of order @" - " %s:%d already holding %s:%x", - m->mtx_description, w->w_level, file, line, - spin_order_list[ffs(i)-1], i); - } - PCPU_SET(witness_spin_check, i | w->w_level); - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - p->p_spinlocks++; - MPASS(p->p_spinlocks > 0); - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - return; - } - if ((m->mtx_flags & MTX_SPIN) != 0) - panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", - m->mtx_description, file, line); - - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_enter: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; - } - if (witness_dead) - goto out; - if (cold) - goto out; - - if (p->p_spinlocks != 0) - panic("blockable mtx_lock() of %s when not legal @ %s:%d", - m->mtx_description, file, line); - /* - * Is this the first mutex acquired - */ - if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL) - goto out; - - if ((w1 = m1->mtx_witness) == w) { - if (w->w_same_squawked || dup_ok(w)) - goto out; - w->w_same_squawked = 1; - printf("acquring duplicate lock of same type: \"%s\"\n", - m->mtx_description); - printf(" 1st @ %s:%d\n", w->w_file, w->w_line); - printf(" 2nd @ %s:%d\n", file, line); -#ifdef DDB - go_into_ddb = 1; -#endif /* DDB */ - goto out; - } - MPASS(!mtx_owned(&w_mtx)); - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - /* - * If we have a known higher number just say ok - */ - if (witness_watch > 1 && w->w_level > w1->w_level) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - goto out; - } - if (isitmydescendant(m1->mtx_witness, w)) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - goto out; - } - for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) { - - MPASS(i < 200); - w1 = m1->mtx_witness; - if (isitmydescendant(w, w1)) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - if (blessed(w, w1)) - goto out; - if (m1 == &Giant) { - if (w1->w_Giant_squawked) - goto out; - else - w1->w_Giant_squawked = 1; - } else { - if (w1->w_other_squawked) - goto out; - else - w1->w_other_squawked = 1; - } - printf("lock order reversal\n"); - printf(" 1st %s last acquired @ %s:%d\n", - w->w_description, w->w_file, w->w_line); - printf(" 2nd %p %s @ %s:%d\n", - m1, w1->w_description, w1->w_file, w1->w_line); - printf(" 3rd %p %s @ %s:%d\n", - m, w->w_description, file, line); -#ifdef DDB - go_into_ddb = 1; -#endif /* DDB */ - goto out; - } - } - m1 = LIST_FIRST(&p->p_heldmtx); - if (!itismychild(m1->mtx_witness, w)) - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - -out: -#ifdef DDB - if (witness_ddb && go_into_ddb) - Debugger("witness_enter"); -#endif /* DDB */ - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - - /* - * If this pays off it likely means that a mutex being witnessed - * is acquired in hardclock. Put it in the ignore list. It is - * likely not the mutex this assert fails on. - */ - MPASS(m->mtx_held.le_prev == NULL); - LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); -} - -void -witness_try_enter(struct mtx *m, int flags, const char *file, int line) -{ - struct proc *p; - struct witness *w = m->mtx_witness; - - if (witness_cold) - return; - if (panicstr) - return; - if (flags & MTX_SPIN) { - if ((m->mtx_flags & MTX_SPIN) == 0) - panic("mutex_try_enter: " - "MTX_SPIN on MTX_DEF mutex %s @ %s:%d", - m->mtx_description, file, line); - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_try_enter: recursion on" - " non-recursive mutex %s @ %s:%d", - m->mtx_description, file, line); - return; - } - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - PCPU_SET(witness_spin_check, - PCPU_GET(witness_spin_check) | w->w_level); - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - return; - } - - if ((m->mtx_flags & MTX_SPIN) != 0) - panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", - m->mtx_description, file, line); - - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_try_enter: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, file, - line); - return; - } - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - p = curproc; - MPASS(m->mtx_held.le_prev == NULL); - LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); -} - -void -witness_exit(struct mtx *m, int flags, const char *file, int line) -{ - struct witness *w; - struct proc *p; - - if (witness_cold || m->mtx_witness == NULL || panicstr) - return; - w = m->mtx_witness; - p = curproc; - - if (flags & MTX_SPIN) { - if ((m->mtx_flags & MTX_SPIN) == 0) - panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @" - " %s:%d", m->mtx_description, file, line); - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_exit: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; - } - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - PCPU_SET(witness_spin_check, - PCPU_GET(witness_spin_check) & ~w->w_level); - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - MPASS(p->p_spinlocks > 0); - p->p_spinlocks--; - return; - } - if ((m->mtx_flags & MTX_SPIN) != 0) - panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", - m->mtx_description, file, line); - - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_exit: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; - } - - if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold) - panic("switchable mtx_unlock() of %s when not legal @ %s:%d", - m->mtx_description, file, line); - LIST_REMOVE(m, mtx_held); - m->mtx_held.le_prev = NULL; -} - -int -witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) -{ - struct mtx *m; - struct proc *p; - char **sleep; - int n = 0; - - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - p = curproc; - LIST_FOREACH(m, &p->p_heldmtx, mtx_held) { - if (m == mtx) - continue; - for (sleep = sleep_list; *sleep!= NULL; sleep++) - if (strcmp(m->mtx_description, *sleep) == 0) - goto next; - if (n == 0) - printf("Whee!\n"); - printf("%s:%d: %s with \"%s\" locked from %s:%d\n", - file, line, check_only ? "could sleep" : "sleeping", - m->mtx_description, - m->mtx_witness->w_file, m->mtx_witness->w_line); - n++; - next: - } -#ifdef DDB - if (witness_ddb && n) - Debugger("witness_sleep"); -#endif /* DDB */ - return (n); -} - -static struct witness * -enroll(const char *description, int flag) -{ - int i; - struct witness *w, *w1; - char **ignore; - char **order; - - if (!witness_watch) - return (NULL); - for (ignore = ignore_list; *ignore != NULL; ignore++) - if (strcmp(description, *ignore) == 0) - return (NULL); - - if (w_inited == 0) { - mtx_init(&w_mtx, "witness lock", MTX_SPIN); - for (i = 0; i < WITNESS_COUNT; i++) { - w = &w_data[i]; - witness_free(w); - } - w_inited = 1; - for (order = order_list; *order != NULL; order++) { - w = enroll(*order, MTX_DEF); - w->w_file = "order list"; - for (order++; *order != NULL; order++) { - w1 = enroll(*order, MTX_DEF); - w1->w_file = "order list"; - itismychild(w, w1); - w = w1; - } - } - } - if ((flag & MTX_SPIN) && witness_skipspin) - return (NULL); - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - for (w = w_all; w; w = w->w_next) { - if (strcmp(description, w->w_description) == 0) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - return (w); - } - } - if ((w = witness_get()) == NULL) - return (NULL); - w->w_next = w_all; - w_all = w; - w->w_description = description; - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - if (flag & MTX_SPIN) { - w->w_spin = 1; - - i = 1; - for (order = spin_order_list; *order != NULL; order++) { - if (strcmp(description, *order) == 0) - break; - i <<= 1; - } - if (*order == NULL) - panic("spin lock %s not in order list", description); - w->w_level = i; - } - - return (w); -} - -static int -itismychild(struct witness *parent, struct witness *child) -{ - static int recursed; - - /* - * Insert "child" after "parent" - */ - while (parent->w_morechildren) - parent = parent->w_morechildren; - - if (parent->w_childcnt == WITNESS_NCHILDREN) { - if ((parent->w_morechildren = witness_get()) == NULL) - return (1); - parent = parent->w_morechildren; - } - MPASS(child != NULL); - parent->w_children[parent->w_childcnt++] = child; - /* - * now prune whole tree - */ - if (recursed) - return (0); - recursed = 1; - for (child = w_all; child != NULL; child = child->w_next) { - for (parent = w_all; parent != NULL; - parent = parent->w_next) { - if (!isitmychild(parent, child)) - continue; - removechild(parent, child); - if (isitmydescendant(parent, child)) - continue; - itismychild(parent, child); - } - } - recursed = 0; - witness_levelall(); - return (0); -} - -static void -removechild(struct witness *parent, struct witness *child) -{ - struct witness *w, *w1; - int i; - - for (w = parent; w != NULL; w = w->w_morechildren) - for (i = 0; i < w->w_childcnt; i++) - if (w->w_children[i] == child) - goto found; - return; -found: - for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren) - continue; - w->w_children[i] = w1->w_children[--w1->w_childcnt]; - MPASS(w->w_children[i] != NULL); - - if (w1->w_childcnt != 0) - return; - - if (w1 == parent) - return; - for (w = parent; w->w_morechildren != w1; w = w->w_morechildren) - continue; - w->w_morechildren = 0; - witness_free(w1); -} - -static int -isitmychild(struct witness *parent, struct witness *child) -{ - struct witness *w; - int i; - - for (w = parent; w != NULL; w = w->w_morechildren) { - for (i = 0; i < w->w_childcnt; i++) { - if (w->w_children[i] == child) - return (1); - } - } - return (0); -} - -static int -isitmydescendant(struct witness *parent, struct witness *child) -{ - struct witness *w; - int i; - int j; - - for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) { - MPASS(j < 1000); - for (i = 0; i < w->w_childcnt; i++) { - if (w->w_children[i] == child) - return (1); - } - for (i = 0; i < w->w_childcnt; i++) { - if (isitmydescendant(w->w_children[i], child)) - return (1); - } - } - return (0); -} - -void -witness_levelall (void) -{ - struct witness *w, *w1; - - for (w = w_all; w; w = w->w_next) - if (!(w->w_spin)) - w->w_level = 0; - for (w = w_all; w; w = w->w_next) { - if (w->w_spin) - continue; - for (w1 = w_all; w1; w1 = w1->w_next) { - if (isitmychild(w1, w)) - break; - } - if (w1 != NULL) - continue; - witness_leveldescendents(w, 0); - } -} - -static void -witness_leveldescendents(struct witness *parent, int level) -{ - int i; - struct witness *w; - - if (parent->w_level < level) - parent->w_level = level; - level++; - for (w = parent; w != NULL; w = w->w_morechildren) - for (i = 0; i < w->w_childcnt; i++) - witness_leveldescendents(w->w_children[i], level); -} - -static void -witness_displaydescendants(void(*prnt)(const char *fmt, ...), - struct witness *parent) -{ - struct witness *w; - int i; - int level; - - level = parent->w_spin ? ffs(parent->w_level) : parent->w_level; - - prnt("%d", level); - if (level < 10) - prnt(" "); - for (i = 0; i < level; i++) - prnt(" "); - prnt("%s", parent->w_description); - if (parent->w_file != NULL) - prnt(" -- last acquired @ %s:%d\n", parent->w_file, - parent->w_line); - - for (w = parent; w != NULL; w = w->w_morechildren) - for (i = 0; i < w->w_childcnt; i++) - witness_displaydescendants(prnt, w->w_children[i]); - } - -static int -dup_ok(struct witness *w) -{ - char **dup; - - for (dup = dup_list; *dup!= NULL; dup++) - if (strcmp(w->w_description, *dup) == 0) - return (1); - return (0); -} - -static int -blessed(struct witness *w1, struct witness *w2) -{ - int i; - struct witness_blessed *b; - - for (i = 0; i < blessed_count; i++) { - b = &blessed_list[i]; - if (strcmp(w1->w_description, b->b_lock1) == 0) { - if (strcmp(w2->w_description, b->b_lock2) == 0) - return (1); - continue; - } - if (strcmp(w1->w_description, b->b_lock2) == 0) - if (strcmp(w2->w_description, b->b_lock1) == 0) - return (1); - } - return (0); -} - -static struct witness * -witness_get() -{ - struct witness *w; - - if ((w = w_free) == NULL) { - witness_dead = 1; - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - printf("witness exhausted\n"); - return (NULL); - } - w_free = w->w_next; - bzero(w, sizeof(*w)); - return (w); -} - -static void -witness_free(struct witness *w) -{ - w->w_next = w_free; - w_free = w; -} - -int -witness_list(struct proc *p) -{ - struct mtx *m; - int nheld; - - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - nheld = 0; - LIST_FOREACH(m, &p->p_heldmtx, mtx_held) { - printf("\t\"%s\" (%p) locked at %s:%d\n", - m->mtx_description, m, - m->mtx_witness->w_file, m->mtx_witness->w_line); - nheld++; - } - - return (nheld); -} - -#ifdef DDB - -DB_SHOW_COMMAND(mutexes, db_witness_list) -{ - - witness_list(curproc); -} - -DB_SHOW_COMMAND(witness, db_witness_display) -{ - - witness_display(db_printf); -} -#endif - -void -witness_save(struct mtx *m, const char **filep, int *linep) -{ - - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - if (m->mtx_witness == NULL) - return; - - *filep = m->mtx_witness->w_file; - *linep = m->mtx_witness->w_line; -} - -void -witness_restore(struct mtx *m, const char *file, int line) -{ - - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - if (m->mtx_witness == NULL) - return; - - m->mtx_witness->w_file = file; - m->mtx_witness->w_line = line; -} - -#endif /* WITNESS */ diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index ee285af..c83f393 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -31,8 +31,9 @@ */ /* - * Machine independent bits of mutex implementation and implementation of - * `witness' structure & related debugging routines. + * Implementation of the `witness' lock verifier. Originally implemented for + * mutexes in BSD/OS. Extended to handle generic lock objects and lock + * classes in FreeBSD. */ /* @@ -61,838 +62,90 @@ #include <sys/param.h> #include <sys/bus.h> #include <sys/kernel.h> +#include <sys/ktr.h> +#include <sys/lock.h> #include <sys/malloc.h> +#include <sys/mutex.h> #include <sys/proc.h> #include <sys/sysctl.h> #include <sys/systm.h> -#include <sys/vmmeter.h> -#include <sys/ktr.h> - -#include <machine/atomic.h> -#include <machine/bus.h> -#include <machine/clock.h> -#include <machine/cpu.h> #include <ddb/ddb.h> -#include <vm/vm.h> -#include <vm/vm_extern.h> - -#include <sys/mutex.h> - -/* - * The WITNESS-enabled mutex debug structure. - */ -#ifdef WITNESS -struct mtx_debug { - struct witness *mtxd_witness; - LIST_ENTRY(mtx) mtxd_held; - const char *mtxd_file; - int mtxd_line; -}; - -#define mtx_held mtx_debug->mtxd_held -#define mtx_file mtx_debug->mtxd_file -#define mtx_line mtx_debug->mtxd_line -#define mtx_witness mtx_debug->mtxd_witness -#endif /* WITNESS */ - -/* - * Internal utility macros. - */ -#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) - -#define mtx_owner(m) (mtx_unowned((m)) ? NULL \ - : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK)) - -#define SET_PRIO(p, pri) (p)->p_pri.pri_level = (pri) - -/* - * Early WITNESS-enabled declarations. - */ -#ifdef WITNESS - -/* - * Internal WITNESS routines which must be prototyped early. - * - * XXX: When/if witness code is cleaned up, it would be wise to place all - * witness prototyping early in this file. - */ -static void witness_init(struct mtx *, int flag); -static void witness_destroy(struct mtx *); -static void witness_display(void(*)(const char *fmt, ...)); - -MALLOC_DEFINE(M_WITNESS, "witness", "witness mtx_debug structure"); - -/* All mutexes in system (used for debug/panic) */ -static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0 }; - -/* - * This global is set to 0 once it becomes safe to use the witness code. - */ -static int witness_cold = 1; - -#else /* WITNESS */ - -/* XXX XXX XXX - * flag++ is sleazoid way of shuting up warning - */ -#define witness_init(m, flag) flag++ -#define witness_destroy(m) -#define witness_try_enter(m, t, f, l) -#endif /* WITNESS */ - -/* - * All mutex locks in system are kept on the all_mtx list. - */ -static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, "All mutexes queue head", - TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), - { NULL, NULL }, &all_mtx, &all_mtx, -#ifdef WITNESS - &all_mtx_debug -#else - NULL -#endif - }; - -/* - * Global variables for book keeping. - */ -static int mtx_cur_cnt; -static int mtx_max_cnt; - -/* - * Couple of strings for KTR_LOCK tracing in order to avoid duplicates. - */ -char STR_mtx_lock_slp[] = "GOT (sleep) %s [%p] r=%d at %s:%d"; -char STR_mtx_unlock_slp[] = "REL (sleep) %s [%p] r=%d at %s:%d"; -char STR_mtx_lock_spn[] = "GOT (spin) %s [%p] r=%d at %s:%d"; -char STR_mtx_unlock_spn[] = "REL (spin) %s [%p] r=%d at %s:%d"; - -/* - * Prototypes for non-exported routines. - * - * NOTE: Prototypes for witness routines are placed at the bottom of the file. - */ -static void propagate_priority(struct proc *); - -static void -propagate_priority(struct proc *p) -{ - int pri = p->p_pri.pri_level; - struct mtx *m = p->p_blocked; - - mtx_assert(&sched_lock, MA_OWNED); - for (;;) { - struct proc *p1; - - p = mtx_owner(m); - - if (p == NULL) { - /* - * This really isn't quite right. Really - * ought to bump priority of process that - * next acquires the mutex. - */ - MPASS(m->mtx_lock == MTX_CONTESTED); - return; - } - - MPASS(p->p_magic == P_MAGIC); - KASSERT(p->p_stat != SSLEEP, ("sleeping process owns a mutex")); - if (p->p_pri.pri_level <= pri) - return; - - /* - * Bump this process' priority. - */ - SET_PRIO(p, pri); - - /* - * If lock holder is actually running, just bump priority. - */ - if (p->p_oncpu != NOCPU) { - MPASS(p->p_stat == SRUN || p->p_stat == SZOMB); - return; - } - -#ifndef SMP - /* - * For UP, we check to see if p is curproc (this shouldn't - * ever happen however as it would mean we are in a deadlock.) - */ - KASSERT(p != curproc, ("Deadlock detected")); -#endif - - /* - * If on run queue move to new run queue, and - * quit. - */ - if (p->p_stat == SRUN) { - MPASS(p->p_blocked == NULL); - remrunqueue(p); - setrunqueue(p); - return; - } - - /* - * If we aren't blocked on a mutex, we should be. - */ - KASSERT(p->p_stat == SMTX, ( - "process %d(%s):%d holds %s but isn't blocked on a mutex\n", - p->p_pid, p->p_comm, p->p_stat, - m->mtx_description)); - - /* - * Pick up the mutex that p is blocked on. - */ - m = p->p_blocked; - MPASS(m != NULL); - - /* - * Check if the proc needs to be moved up on - * the blocked chain - */ - if (p == TAILQ_FIRST(&m->mtx_blocked)) { - continue; - } - - p1 = TAILQ_PREV(p, procqueue, p_procq); - if (p1->p_pri.pri_level <= pri) { - continue; - } - - /* - * Remove proc from blocked chain and determine where - * it should be moved up to. Since we know that p1 has - * a lower priority than p, we know that at least one - * process in the chain has a lower priority and that - * p1 will thus not be NULL after the loop. - */ - TAILQ_REMOVE(&m->mtx_blocked, p, p_procq); - TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) { - MPASS(p1->p_magic == P_MAGIC); - if (p1->p_pri.pri_level > pri) - break; - } - - MPASS(p1 != NULL); - TAILQ_INSERT_BEFORE(p1, p, p_procq); - CTR4(KTR_LOCK, - "propagate_priority: p %p moved before %p on [%p] %s", - p, p1, m, m->mtx_description); - } -} - -/* - * Function versions of the inlined __mtx_* macros. These are used by - * modules and can also be called from assembly language if needed. - */ -void -_mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) -{ - - __mtx_lock_flags(m, opts, file, line); -} - -void -_mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) -{ - - __mtx_unlock_flags(m, opts, file, line); -} - -void -_mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) -{ - - __mtx_lock_spin_flags(m, opts, file, line); -} - -void -_mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) -{ - - __mtx_unlock_spin_flags(m, opts, file, line); -} - -/* - * The important part of mtx_trylock{,_flags}() - * Tries to acquire lock `m.' We do NOT handle recursion here; we assume that - * if we're called, it's because we know we don't already own this lock. - */ -int -_mtx_trylock(struct mtx *m, int opts, const char *file, int line) -{ - int rval; - - MPASS(curproc != NULL); - - /* - * _mtx_trylock does not accept MTX_NOSWITCH option. - */ - KASSERT((opts & MTX_NOSWITCH) == 0, - ("mtx_trylock() called with invalid option flag(s) %d", opts)); - - rval = _obtain_lock(m, curproc); - -#ifdef WITNESS - if (rval && m->mtx_witness != NULL) { - /* - * We do not handle recursion in _mtx_trylock; see the - * note at the top of the routine. - */ - KASSERT(!mtx_recursed(m), - ("mtx_trylock() called on a recursed mutex")); - witness_try_enter(m, (opts | m->mtx_flags), file, line); - } -#endif /* WITNESS */ - - if ((opts & MTX_QUIET) == 0) - CTR5(KTR_LOCK, "TRY_LOCK %s [%p] result=%d at %s:%d", - m->mtx_description, m, rval, file, line); - - return rval; -} - -/* - * _mtx_lock_sleep: the tougher part of acquiring an MTX_DEF lock. - * - * We call this if the lock is either contested (i.e. we need to go to - * sleep waiting for it), or if we need to recurse on it. - */ -void -_mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) -{ - struct proc *p = curproc; - - if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { - m->mtx_recurse++; - atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); - if ((opts & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "_mtx_lock_sleep: %p recursing", m); - return; - } - - if ((opts & MTX_QUIET) == 0) - CTR4(KTR_LOCK, - "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", - m->mtx_description, (void *)m->mtx_lock, file, line); - - while (!_obtain_lock(m, p)) { - uintptr_t v; - struct proc *p1; - - mtx_lock_spin(&sched_lock); - /* - * Check if the lock has been released while spinning for - * the sched_lock. - */ - if ((v = m->mtx_lock) == MTX_UNOWNED) { - mtx_unlock_spin(&sched_lock); - continue; - } - - /* - * The mutex was marked contested on release. This means that - * there are processes blocked on it. - */ - if (v == MTX_CONTESTED) { - p1 = TAILQ_FIRST(&m->mtx_blocked); - MPASS(p1 != NULL); - m->mtx_lock = (uintptr_t)p | MTX_CONTESTED; - - if (p1->p_pri.pri_level < p->p_pri.pri_level) - SET_PRIO(p, p1->p_pri.pri_level); - mtx_unlock_spin(&sched_lock); - return; - } - - /* - * If the mutex isn't already contested and a failure occurs - * setting the contested bit, the mutex was either released - * or the state of the MTX_RECURSED bit changed. - */ - if ((v & MTX_CONTESTED) == 0 && - !atomic_cmpset_ptr(&m->mtx_lock, (void *)v, - (void *)(v | MTX_CONTESTED))) { - mtx_unlock_spin(&sched_lock); - continue; - } - - /* - * We deffinately must sleep for this lock. - */ - mtx_assert(m, MA_NOTOWNED); - -#ifdef notyet - /* - * If we're borrowing an interrupted thread's VM context, we - * must clean up before going to sleep. - */ - if (p->p_ithd != NULL) { - struct ithd *it = p->p_ithd; - - if (it->it_interrupted) { - if ((opts & MTX_QUIET) == 0) - CTR2(KTR_LOCK, - "_mtx_lock_sleep: %p interrupted %p", - it, it->it_interrupted); - intr_thd_fixup(it); - } - } -#endif - - /* - * Put us on the list of threads blocked on this mutex. - */ - if (TAILQ_EMPTY(&m->mtx_blocked)) { - p1 = (struct proc *)(m->mtx_lock & MTX_FLAGMASK); - LIST_INSERT_HEAD(&p1->p_contested, m, mtx_contested); - TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); - } else { - TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) - if (p1->p_pri.pri_level > p->p_pri.pri_level) - break; - if (p1) - TAILQ_INSERT_BEFORE(p1, p, p_procq); - else - TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); - } - - /* - * Save who we're blocked on. - */ - p->p_blocked = m; - p->p_mtxname = m->mtx_description; - p->p_stat = SMTX; - propagate_priority(p); - - if ((opts & MTX_QUIET) == 0) - CTR3(KTR_LOCK, - "_mtx_lock_sleep: p %p blocked on [%p] %s", p, m, - m->mtx_description); - - mi_switch(); - - if ((opts & MTX_QUIET) == 0) - CTR3(KTR_LOCK, - "_mtx_lock_sleep: p %p free from blocked on [%p] %s", - p, m, m->mtx_description); - - mtx_unlock_spin(&sched_lock); - } - - return; -} - -/* - * _mtx_lock_spin: the tougher part of acquiring an MTX_SPIN lock. - * - * This is only called if we need to actually spin for the lock. Recursion - * is handled inline. - */ -void -_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, - int line) -{ - int i = 0; - - if ((opts & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); - - for (;;) { - if (_obtain_lock(m, curproc)) - break; - - while (m->mtx_lock != MTX_UNOWNED) { - if (i++ < 1000000) - continue; - if (i++ < 6000000) - DELAY(1); -#ifdef DDB - else if (!db_active) -#else - else -#endif - panic("spin lock %s held by %p for > 5 seconds", - m->mtx_description, (void *)m->mtx_lock); - } - } - - m->mtx_savecrit = mtx_crit; - if ((opts & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); - - return; -} - -/* - * _mtx_unlock_sleep: the tougher part of releasing an MTX_DEF lock. - * - * We are only called here if the lock is recursed or contested (i.e. we - * need to wake up a blocked thread). - */ -void -_mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) -{ - struct proc *p, *p1; - struct mtx *m1; - int pri; - - p = curproc; - - if (mtx_recursed(m)) { - if (--(m->mtx_recurse) == 0) - atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); - if ((opts & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p unrecurse", m); - return; - } - - mtx_lock_spin(&sched_lock); - if ((opts & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); - - p1 = TAILQ_FIRST(&m->mtx_blocked); - MPASS(p->p_magic == P_MAGIC); - MPASS(p1->p_magic == P_MAGIC); - - TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq); - - if (TAILQ_EMPTY(&m->mtx_blocked)) { - LIST_REMOVE(m, mtx_contested); - _release_lock_quick(m); - if ((opts & MTX_QUIET) == 0) - CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p not held", m); - } else - atomic_store_rel_ptr(&m->mtx_lock, (void *)MTX_CONTESTED); - - pri = PRI_MAX; - LIST_FOREACH(m1, &p->p_contested, mtx_contested) { - int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_pri.pri_level; - if (cp < pri) - pri = cp; - } - - if (pri > p->p_pri.pri_native) - pri = p->p_pri.pri_native; - SET_PRIO(p, pri); - - if ((opts & MTX_QUIET) == 0) - CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p contested setrunqueue %p", - m, p1); - - p1->p_blocked = NULL; - p1->p_stat = SRUN; - setrunqueue(p1); - - if ((opts & MTX_NOSWITCH) == 0 && p1->p_pri.pri_level < pri) { -#ifdef notyet - if (p->p_ithd != NULL) { - struct ithd *it = p->p_ithd; - - if (it->it_interrupted) { - if ((opts & MTX_QUIET) == 0) - CTR2(KTR_LOCK, - "_mtx_unlock_sleep: %p interrupted %p", - it, it->it_interrupted); - intr_thd_fixup(it); - } - } -#endif - setrunqueue(p); - if ((opts & MTX_QUIET) == 0) - CTR2(KTR_LOCK, - "_mtx_unlock_sleep: %p switching out lock=%p", m, - (void *)m->mtx_lock); - - mi_switch(); - if ((opts & MTX_QUIET) == 0) - CTR2(KTR_LOCK, "_mtx_unlock_sleep: %p resuming lock=%p", - m, (void *)m->mtx_lock); - } - - mtx_unlock_spin(&sched_lock); - - return; -} - -/* - * All the unlocking of MTX_SPIN locks is done inline. - * See the _rel_spin_lock() macro for the details. - */ - -/* - * The backing function for the INVARIANTS-enabled mtx_assert() - */ -#ifdef INVARIANT_SUPPORT -void -_mtx_assert(struct mtx *m, int what, const char *file, int line) -{ - switch (what) { - case MA_OWNED: - case MA_OWNED | MA_RECURSED: - case MA_OWNED | MA_NOTRECURSED: - if (!mtx_owned(m)) - panic("mutex %s not owned at %s:%d", - m->mtx_description, file, line); - if (mtx_recursed(m)) { - if ((what & MA_NOTRECURSED) != 0) - panic("mutex %s recursed at %s:%d", - m->mtx_description, file, line); - } else if ((what & MA_RECURSED) != 0) { - panic("mutex %s unrecursed at %s:%d", - m->mtx_description, file, line); - } - break; - case MA_NOTOWNED: - if (mtx_owned(m)) - panic("mutex %s owned at %s:%d", - m->mtx_description, file, line); - break; - default: - panic("unknown mtx_assert at %s:%d", file, line); - } -} -#endif - -/* - * The MUTEX_DEBUG-enabled mtx_validate() - */ -#define MV_DESTROY 0 /* validate before destory */ -#define MV_INIT 1 /* validate before init */ - -#ifdef MUTEX_DEBUG - -int mtx_validate __P((struct mtx *, int)); - -int -mtx_validate(struct mtx *m, int when) -{ - struct mtx *mp; - int i; - int retval = 0; - -#ifdef WITNESS - if (witness_cold) - return 0; -#endif - if (m == &all_mtx || cold) - return 0; - - mtx_lock(&all_mtx); -/* - * XXX - When kernacc() is fixed on the alpha to handle K0_SEG memory properly - * we can re-enable the kernacc() checks. - */ -#ifndef __alpha__ - MPASS(kernacc((caddr_t)all_mtx.mtx_next, sizeof(uintptr_t), - VM_PROT_READ) == 1); -#endif - MPASS(all_mtx.mtx_next->mtx_prev == &all_mtx); - for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { -#ifndef __alpha__ - if (kernacc((caddr_t)mp->mtx_next, sizeof(uintptr_t), - VM_PROT_READ) != 1) { - panic("mtx_validate: mp=%p mp->mtx_next=%p", - mp, mp->mtx_next); - } -#endif - i++; - if (i > mtx_cur_cnt) { - panic("mtx_validate: too many in chain, known=%d\n", - mtx_cur_cnt); - } - } - MPASS(i == mtx_cur_cnt); - switch (when) { - case MV_DESTROY: - for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) - if (mp == m) - break; - MPASS(mp == m); - break; - case MV_INIT: - for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) - if (mp == m) { - /* - * Not good. This mutex already exists. - */ - printf("re-initing existing mutex %s\n", - m->mtx_description); - MPASS(m->mtx_lock == MTX_UNOWNED); - retval = 1; - } - } - mtx_unlock(&all_mtx); - return (retval); -} -#endif - -/* - * Mutex initialization routine; initialize lock `m' of type contained in - * `opts' with options contained in `opts' and description `description.' - * Place on "all_mtx" queue. - */ -void -mtx_init(struct mtx *m, const char *description, int opts) -{ - - if ((opts & MTX_QUIET) == 0) - CTR2(KTR_LOCK, "mtx_init %p (%s)", m, description); - -#ifdef MUTEX_DEBUG - /* Diagnostic and error correction */ - if (mtx_validate(m, MV_INIT)) - return; -#endif - - bzero((void *)m, sizeof *m); - TAILQ_INIT(&m->mtx_blocked); - -#ifdef WITNESS - if (!witness_cold) { - m->mtx_debug = malloc(sizeof(struct mtx_debug), - M_WITNESS, M_NOWAIT | M_ZERO); - MPASS(m->mtx_debug != NULL); - } -#endif - - m->mtx_description = description; - m->mtx_flags = opts; - m->mtx_lock = MTX_UNOWNED; - - /* Put on all mutex queue */ - mtx_lock(&all_mtx); - m->mtx_next = &all_mtx; - m->mtx_prev = all_mtx.mtx_prev; - m->mtx_prev->mtx_next = m; - all_mtx.mtx_prev = m; - if (++mtx_cur_cnt > mtx_max_cnt) - mtx_max_cnt = mtx_cur_cnt; - mtx_unlock(&all_mtx); - -#ifdef WITNESS - if (!witness_cold) - witness_init(m, opts); -#endif -} - -/* - * Remove lock `m' from all_mtx queue. - */ -void -mtx_destroy(struct mtx *m) -{ - -#ifdef WITNESS - KASSERT(!witness_cold, ("%s: Cannot destroy while still cold\n", - __FUNCTION__)); -#endif - - CTR2(KTR_LOCK, "mtx_destroy %p (%s)", m, m->mtx_description); - -#ifdef MUTEX_DEBUG - if (m->mtx_next == NULL) - panic("mtx_destroy: %p (%s) already destroyed", - m, m->mtx_description); - - if (!mtx_owned(m)) { - MPASS(m->mtx_lock == MTX_UNOWNED); - } else { - MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); - } - - /* diagnostic */ - mtx_validate(m, MV_DESTROY); -#endif - -#ifdef WITNESS - if (m->mtx_witness) - witness_destroy(m); -#endif /* WITNESS */ - - /* Remove from the all mutex queue */ - mtx_lock(&all_mtx); - m->mtx_next->mtx_prev = m->mtx_prev; - m->mtx_prev->mtx_next = m->mtx_next; - -#ifdef MUTEX_DEBUG - m->mtx_next = m->mtx_prev = NULL; -#endif - -#ifdef WITNESS - free(m->mtx_debug, M_WITNESS); - m->mtx_debug = NULL; -#endif - - mtx_cur_cnt--; - mtx_unlock(&all_mtx); -} - - +#define WITNESS_COUNT 200 +#define WITNESS_CHILDCOUNT (WITNESS_COUNT * 4) /* - * The WITNESS-enabled diagnostic code. + * XXX: This is somewhat bogus, as we assume here that at most 1024 processes + * will hold LOCK_NCHILDREN * 2 locks. We handle failure ok, and we should + * probably be safe for the most part, but it's still a SWAG. */ -#ifdef WITNESS -static void -witness_fixup(void *dummy __unused) -{ - struct mtx *mp; - - /* - * We have to release Giant before initializing its witness - * structure so that WITNESS doesn't get confused. - */ - mtx_unlock(&Giant); - mtx_assert(&Giant, MA_NOTOWNED); - - mtx_lock(&all_mtx); +#define LOCK_CHILDCOUNT (MAXCPU + 1024) * 2 - /* Iterate through all mutexes and finish up mutex initialization. */ - for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { +#define WITNESS_NCHILDREN 6 - mp->mtx_debug = malloc(sizeof(struct mtx_debug), - M_WITNESS, M_NOWAIT | M_ZERO); - MPASS(mp->mtx_debug != NULL); +struct witness_child_list_entry; - witness_init(mp, mp->mtx_flags); - } - mtx_unlock(&all_mtx); - - /* Mark the witness code as being ready for use. */ - atomic_store_rel_int(&witness_cold, 0); - - mtx_lock(&Giant); -} -SYSINIT(wtnsfxup, SI_SUB_MUTEX, SI_ORDER_FIRST, witness_fixup, NULL) +struct witness { + const char *w_name; + struct lock_class *w_class; + STAILQ_ENTRY(witness) w_list; /* List of all witnesses. */ + STAILQ_ENTRY(witness) w_typelist; /* Witnesses of a type. */ + struct witness_child_list_entry *w_children; /* Great evilness... */ + const char *w_file; + int w_line; + u_int w_level; + u_int w_refcount; + u_char w_Giant_squawked:1; + u_char w_other_squawked:1; + u_char w_same_squawked:1; +}; -#define WITNESS_COUNT 200 -#define WITNESS_NCHILDREN 2 +struct witness_child_list_entry { + struct witness_child_list_entry *wcl_next; + struct witness *wcl_children[WITNESS_NCHILDREN]; + u_int wcl_count; +}; -int witness_watch = 1; +STAILQ_HEAD(witness_list, witness); -struct witness { - struct witness *w_next; - const char *w_description; - const char *w_file; - int w_line; - struct witness *w_morechildren; - u_char w_childcnt; - u_char w_Giant_squawked:1; - u_char w_other_squawked:1; - u_char w_same_squawked:1; - u_char w_spin:1; /* MTX_SPIN type mutex. */ - u_int w_level; - struct witness *w_children[WITNESS_NCHILDREN]; +struct witness_blessed { + const char *b_lock1; + const char *b_lock2; }; -struct witness_blessed { - char *b_lock1; - char *b_lock2; +struct witness_order_list_entry { + const char *w_name; + struct lock_class *w_class; }; +static struct witness *enroll(const char *description, + struct lock_class *lock_class); +static int itismychild(struct witness *parent, struct witness *child); +static void removechild(struct witness *parent, struct witness *child); +static int isitmychild(struct witness *parent, struct witness *child); +static int isitmydescendant(struct witness *parent, struct witness *child); +static int dup_ok(struct witness *); +static int blessed(struct witness *, struct witness *); +static void witness_display_list(void(*prnt)(const char *fmt, ...), + struct witness_list *list); +static void witness_displaydescendants(void(*)(const char *fmt, ...), + struct witness *); +static void witness_leveldescendents(struct witness *parent, int level); +static void witness_levelall(void); +static struct witness *witness_get(void); +static void witness_free(struct witness *m); +static struct witness_child_list_entry *witness_child_get(void); +static void witness_child_free(struct witness_child_list_entry *wcl); +static struct lock_list_entry *witness_lock_list_get(void); +static void witness_lock_list_free(struct lock_list_entry *lle); +static void witness_display(void(*)(const char *fmt, ...)); + +MALLOC_DEFINE(M_WITNESS, "witness", "witness structure"); + +static int witness_watch; +TUNABLE_INT_DECL("debug.witness_watch", 1, witness_watch); +SYSCTL_INT(_debug, OID_AUTO, witness_watch, CTLFLAG_RD, &witness_watch, 0, ""); + #ifdef DDB /* * When DDB is enabled and witness_ddb is set to 1, it will cause the system to @@ -918,85 +171,62 @@ TUNABLE_INT_DECL("debug.witness_skipspin", 0, witness_skipspin); SYSCTL_INT(_debug, OID_AUTO, witness_skipspin, CTLFLAG_RD, &witness_skipspin, 0, ""); -/* - * Witness-enabled globals - */ -static struct mtx w_mtx; -static struct witness *w_free; -static struct witness *w_all; -static int w_inited; -static int witness_dead; /* fatal error, probably no memory */ - -static struct witness w_data[WITNESS_COUNT]; - -/* - * Internal witness routine prototypes - */ -static struct witness *enroll(const char *description, int flag); -static int itismychild(struct witness *parent, struct witness *child); -static void removechild(struct witness *parent, struct witness *child); -static int isitmychild(struct witness *parent, struct witness *child); -static int isitmydescendant(struct witness *parent, struct witness *child); -static int dup_ok(struct witness *); -static int blessed(struct witness *, struct witness *); -static void - witness_displaydescendants(void(*)(const char *fmt, ...), struct witness *); -static void witness_leveldescendents(struct witness *parent, int level); -static void witness_levelall(void); -static struct witness * witness_get(void); -static void witness_free(struct witness *m); - -static char *ignore_list[] = { - "witness lock", - NULL -}; - -static char *spin_order_list[] = { +static struct mtx w_mtx; +static struct witness_list w_free = STAILQ_HEAD_INITIALIZER(w_free); +static struct witness_list w_all = STAILQ_HEAD_INITIALIZER(w_all); +static struct witness_list w_spin = STAILQ_HEAD_INITIALIZER(w_spin); +static struct witness_list w_sleep = STAILQ_HEAD_INITIALIZER(w_sleep); +static struct witness_child_list_entry *w_child_free = NULL; +static struct lock_list_entry *w_lock_list_free = NULL; +static int witness_dead; /* fatal error, probably no memory */ + +static struct witness w_data[WITNESS_COUNT]; +static struct witness_child_list_entry w_childdata[WITNESS_CHILDCOUNT]; +static struct lock_list_entry w_locklistdata[LOCK_CHILDCOUNT]; + +static struct witness_order_list_entry order_lists[] = { + { "Giant", &lock_class_mtx_sleep }, + { "proctree", &lock_class_sx }, + { "allproc", &lock_class_sx }, + { "process lock", &lock_class_mtx_sleep }, + { "uidinfo hash", &lock_class_mtx_sleep }, + { "uidinfo struct", &lock_class_mtx_sleep }, + { NULL, NULL }, #if defined(__i386__) && defined (SMP) - "com", + { "com", &lock_class_mtx_spin }, #endif - "sio", + { "sio", &lock_class_mtx_spin }, #ifdef __i386__ - "cy", + { "cy", &lock_class_mtx_spin }, #endif - "ng_node", - "ng_worklist", - "ithread table lock", - "ithread list lock", - "sched lock", + { "ng_node", &lock_class_mtx_spin }, + { "ng_worklist", &lock_class_mtx_spin }, + { "ithread table lock", &lock_class_mtx_spin }, + { "ithread list lock", &lock_class_mtx_spin }, + { "sched lock", &lock_class_mtx_spin }, #ifdef __i386__ - "clk", + { "clk", &lock_class_mtx_spin }, #endif - "callout", + { "callout", &lock_class_mtx_spin }, /* * leaf locks */ #ifdef SMP #ifdef __i386__ - "ap boot", - "imen", + { "ap boot", &lock_class_mtx_spin }, + { "imen", &lock_class_mtx_spin }, #endif - "smp rendezvous", + { "smp rendezvous", &lock_class_mtx_spin }, #endif - NULL -}; - -static char *order_list[] = { - "Giant", "proctree", "allproc", "process lock", "uidinfo hash", - "uidinfo struct", NULL, - NULL + { NULL, NULL }, + { NULL, NULL } }; -static char *dup_list[] = { +static const char *dup_list[] = { "process lock", NULL }; -static char *sleep_list[] = { - "Giant", - NULL -}; - /* * Pairs of locks which have been blessed * Don't complain about order problems with blessed locks @@ -1006,165 +236,272 @@ static struct witness_blessed blessed_list[] = { static int blessed_count = sizeof(blessed_list) / sizeof(struct witness_blessed); -static void -witness_init(struct mtx *m, int flag) -{ - m->mtx_witness = enroll(m->mtx_description, flag); -} +/* + * List of all locks in the system. + */ +STAILQ_HEAD(, lock_object) all_locks = STAILQ_HEAD_INITIALIZER(all_locks); + +static struct mtx all_mtx = { + { &lock_class_mtx_sleep, /* mtx_object.lo_class */ + "All locks list", /* mtx_object.lo_name */ + NULL, /* mtx_object.lo_file */ + 0, /* mtx_object.lo_line */ + LO_INITIALIZED, /* mtx_object.lo_flags */ + { NULL }, /* mtx_object.lo_list */ + NULL }, /* mtx_object.lo_witness */ + MTX_UNOWNED, 0, /* mtx_lock, mtx_recurse */ + 0, /* mtx_savecrit */ + TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), + { NULL, NULL } /* mtx_contested */ +}; + +/* + * This global is set to 0 once it becomes safe to use the witness code. + */ +static int witness_cold = 1; + +/* + * Global variables for book keeping. + */ +static int lock_cur_cnt; +static int lock_max_cnt; +/* + * The WITNESS-enabled diagnostic code. + */ static void -witness_destroy(struct mtx *m) +witness_initialize(void *dummy __unused) { - struct mtx *m1; - struct proc *p; - p = curproc; - LIST_FOREACH(m1, &p->p_heldmtx, mtx_held) { - if (m1 == m) { - LIST_REMOVE(m, mtx_held); - break; + struct lock_object *lock; + struct witness_order_list_entry *order; + struct witness *w, *w1; + int i; + + /* + * We have to release Giant before initializing its witness + * structure so that WITNESS doesn't get confused. + */ + mtx_unlock(&Giant); + mtx_assert(&Giant, MA_NOTOWNED); + + STAILQ_INSERT_HEAD(&all_locks, &all_mtx.mtx_object, lo_list); + mtx_init(&w_mtx, "witness lock", MTX_SPIN | MTX_QUIET | MTX_NOWITNESS); + for (i = 0; i < WITNESS_COUNT; i++) + witness_free(&w_data[i]); + for (i = 0; i < WITNESS_CHILDCOUNT; i++) + witness_child_free(&w_childdata[i]); + for (i = 0; i < LOCK_CHILDCOUNT; i++) + witness_lock_list_free(&w_locklistdata[i]); + + /* First add in all the specified order lists. */ + for (order = order_lists; order->w_name != NULL; order++) { + w = enroll(order->w_name, order->w_class); + w->w_file = "order list"; + for (order++; order->w_name != NULL; order++) { + w1 = enroll(order->w_name, order->w_class); + w1->w_file = "order list"; + itismychild(w, w1); + w = w1; } } - return; + /* Iterate through all locks and add them to witness. */ + mtx_lock(&all_mtx); + STAILQ_FOREACH(lock, &all_locks, lo_list) { + if (lock->lo_flags & LO_WITNESS) + lock->lo_witness = enroll(lock->lo_name, + lock->lo_class); + else + lock->lo_witness = NULL; + } + mtx_unlock(&all_mtx); + + /* Mark the witness code as being ready for use. */ + atomic_store_rel_int(&witness_cold, 0); + + mtx_lock(&Giant); +} +SYSINIT(witness_init, SI_SUB_WITNESS, SI_ORDER_FIRST, witness_initialize, NULL) + +void +witness_init(struct lock_object *lock) +{ + struct lock_class *class; + + class = lock->lo_class; + if (lock->lo_flags & LO_INITIALIZED) + panic("%s: lock (%s) %s is already initialized!\n", __func__, + class->lc_name, lock->lo_name); + + if ((lock->lo_flags & LO_RECURSABLE) != 0 && + (class->lc_flags & LC_RECURSABLE) == 0) + panic("%s: lock (%s) %s can not be recursable!\n", __func__, + class->lc_name, lock->lo_name); + + if ((lock->lo_flags & LO_SLEEPABLE) != 0 && + (class->lc_flags & LC_SLEEPABLE) == 0) + panic("%s: lock (%s) %s can not be sleepable!\n", __func__, + class->lc_name, lock->lo_name); + + mtx_lock(&all_mtx); + STAILQ_INSERT_TAIL(&all_locks, lock, lo_list); + lock->lo_flags |= LO_INITIALIZED; + lock_cur_cnt++; + if (lock_cur_cnt > lock_max_cnt) + lock_max_cnt = lock_cur_cnt; + mtx_unlock(&all_mtx); + if (!witness_cold && !witness_dead && + (lock->lo_flags & LO_WITNESS) != 0) + lock->lo_witness = enroll(lock->lo_name, class); + else + lock->lo_witness = NULL; +} + +void +witness_destroy(struct lock_object *lock) +{ + + if (witness_cold) + panic("lock (%s) %s destroyed while witness_cold", + lock->lo_class->lc_name, lock->lo_name); + + if ((lock->lo_flags & LO_INITIALIZED) == 0) + panic("%s: lock (%s) %s is not initialized!\n", __func__, + lock->lo_class->lc_name, lock->lo_name); + + if (lock->lo_flags & LO_LOCKED) + panic("lock (%s) %s destroyed while held", + lock->lo_class->lc_name, lock->lo_name); + + mtx_lock(&all_mtx); + lock_cur_cnt--; + STAILQ_REMOVE(&all_locks, lock, lock_object, lo_list); + lock->lo_flags &= LO_INITIALIZED; + mtx_unlock(&all_mtx); } static void -witness_display(void(*prnt)(const char *fmt, ...)) +witness_display_list(void(*prnt)(const char *fmt, ...), + struct witness_list *list) { struct witness *w, *w1; - int level, found; + int found; - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - witness_levelall(); - - /* - * First, handle sleep mutexes which have been acquired at least - * once. - */ - prnt("Sleep mutexes:\n"); - for (w = w_all; w; w = w->w_next) { - if (w->w_file == NULL || w->w_spin) + STAILQ_FOREACH(w, list, w_typelist) { + if (w->w_file == NULL) continue; - for (w1 = w_all; w1; w1 = w1->w_next) { - if (isitmychild(w1, w)) + found = 0; + STAILQ_FOREACH(w1, list, w_typelist) { + if (isitmychild(w1, w)) { + found++; break; + } } - if (w1 != NULL) + if (found) continue; /* * This lock has no anscestors, display its descendants. */ witness_displaydescendants(prnt, w); } +} + +static void +witness_display(void(*prnt)(const char *fmt, ...)) +{ + struct witness *w; + + KASSERT(!witness_cold, ("%s: witness_cold\n", __func__)); + witness_levelall(); + + /* + * First, handle sleep mutexes which have been acquired at least + * once. + */ + prnt("Sleep locks:\n"); + witness_display_list(prnt, &w_sleep); /* * Now do spin mutexes which have been acquired at least once. */ - prnt("\nSpin mutexes:\n"); - level = 0; - while (level < sizeof(spin_order_list) / sizeof(char *)) { - found = 0; - for (w = w_all; w; w = w->w_next) { - if (w->w_file == NULL || !w->w_spin) - continue; - if (w->w_level == 1 << level) { - witness_displaydescendants(prnt, w); - level++; - found = 1; - } - } - if (found == 0) - level++; - } + prnt("\nSpin locks:\n"); + witness_display_list(prnt, &w_spin); /* * Finally, any mutexes which have not been acquired yet. */ - prnt("\nMutexes which were never acquired:\n"); - for (w = w_all; w; w = w->w_next) { + prnt("\nLocks which were never acquired:\n"); + STAILQ_FOREACH(w, &w_all, w_list) { if (w->w_file != NULL) continue; - prnt("%s\n", w->w_description); + prnt("%s\n", w->w_name); } } void -witness_enter(struct mtx *m, int flags, const char *file, int line) +witness_lock(struct lock_object *lock, int flags, const char *file, int line) { + struct lock_list_entry **lock_list, *lle; + struct lock_object *lock1, *lock2; + struct lock_class *class; struct witness *w, *w1; - struct mtx *m1; struct proc *p; - int i; + int i, j; #ifdef DDB int go_into_ddb = 0; #endif /* DDB */ - if (witness_cold || m->mtx_witness == NULL || panicstr) + if (witness_cold || witness_dead || lock->lo_witness == NULL || + panicstr) return; - w = m->mtx_witness; + w = lock->lo_witness; + class = lock->lo_class; p = curproc; - if (flags & MTX_SPIN) { - if ((m->mtx_flags & MTX_SPIN) == 0) - panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @" - " %s:%d", m->mtx_description, file, line); - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_enter: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; - } - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - i = PCPU_GET(witness_spin_check); - if (i != 0 && w->w_level < i) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - panic("mutex_enter(%s:%x, MTX_SPIN) out of order @" - " %s:%d already holding %s:%x", - m->mtx_description, w->w_level, file, line, - spin_order_list[ffs(i)-1], i); - } - PCPU_SET(witness_spin_check, i | w->w_level); - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - p->p_spinlocks++; - MPASS(p->p_spinlocks > 0); - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; + if ((lock->lo_flags & LO_LOCKED) == 0) + panic("%s: lock (%s) %s is not locked @ %s:%d", __func__, + class->lc_name, lock->lo_name, file, line); + + if ((lock->lo_flags & LO_RECURSED) != 0) { + if ((lock->lo_flags & LO_RECURSABLE) == 0) + panic( + "%s: recursed on non-recursive lock (%s) %s @ %s:%d", + __func__, class->lc_name, lock->lo_name, file, + line); return; } - if ((m->mtx_flags & MTX_SPIN) != 0) - panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", - m->mtx_description, file, line); - - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_enter: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; + + lock_list = PCPU_PTR(spinlocks); + if (class->lc_flags & LC_SLEEPLOCK) { + if (*lock_list != NULL) + panic("blockable sleep lock (%s) %s @ %s:%d", + class->lc_name, lock->lo_name, file, line); + lock_list = &p->p_sleeplocks; } - if (witness_dead) - goto out; - if (cold) + + if (flags & LOP_TRYLOCK) goto out; - if (p->p_spinlocks != 0) - panic("blockable mtx_lock() of %s when not legal @ %s:%d", - m->mtx_description, file, line); /* - * Is this the first mutex acquired + * Is this the first lock acquired? If so, then no order checking + * is needed. */ - if ((m1 = LIST_FIRST(&p->p_heldmtx)) == NULL) + if (*lock_list == NULL) goto out; - if ((w1 = m1->mtx_witness) == w) { + /* + * Check for duplicate locks of the same type. Note that we only + * have to check for this on the last lock we just acquired. Any + * other cases will be caught as lock order violations. + */ + lock1 = (*lock_list)->ll_children[(*lock_list)->ll_count - 1]; + w1 = lock1->lo_witness; + if (w1 == w) { if (w->w_same_squawked || dup_ok(w)) goto out; w->w_same_squawked = 1; printf("acquring duplicate lock of same type: \"%s\"\n", - m->mtx_description); + lock->lo_name); printf(" 1st @ %s:%d\n", w->w_file, w->w_line); printf(" 2nd @ %s:%d\n", file, line); #ifdef DDB @@ -1173,27 +510,44 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) goto out; } MPASS(!mtx_owned(&w_mtx)); - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); + mtx_lock_spin(&w_mtx); /* * If we have a known higher number just say ok */ if (witness_watch > 1 && w->w_level > w1->w_level) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + mtx_unlock_spin(&w_mtx); goto out; } - if (isitmydescendant(m1->mtx_witness, w)) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + if (isitmydescendant(w1, w)) { + mtx_unlock_spin(&w_mtx); goto out; } - for (i = 0; m1 != NULL; m1 = LIST_NEXT(m1, mtx_held), i++) { + for (j = 0, lle = *lock_list; lle != NULL; lle = lle->ll_next) { + for (i = lle->ll_count - 1; i >= 0; i--, j++) { - MPASS(i < 200); - w1 = m1->mtx_witness; - if (isitmydescendant(w, w1)) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + MPASS(j < WITNESS_COUNT); + lock1 = lle->ll_children[i]; + w1 = lock1->lo_witness; + + /* + * If this lock doesn't undergo witness checking, + * then skip it. + */ + if (w1 == NULL) { + KASSERT((lock1->lo_flags & LO_WITNESS) == 0, + ("lock missing witness structure")); + continue; + } + if (!isitmydescendant(w, w1)) + continue; + /* + * We have a lock order violation, check to see if it + * is allowed or has already been yelled about. + */ + mtx_unlock_spin(&w_mtx); if (blessed(w, w1)) goto out; - if (m1 == &Giant) { + if (lock1 == &Giant.mtx_object) { if (w1->w_Giant_squawked) goto out; else @@ -1204,22 +558,50 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) else w1->w_other_squawked = 1; } + /* + * Ok, yell about it. + */ printf("lock order reversal\n"); - printf(" 1st %s last acquired @ %s:%d\n", - w->w_description, w->w_file, w->w_line); + /* + * Try to locate an earlier lock with + * witness w in our list. + */ + do { + lock2 = lle->ll_children[i]; + MPASS(lock2 != NULL); + if (lock2->lo_witness == w) + break; + i--; + if (i == 0 && lle->ll_next != NULL) { + lle = lle->ll_next; + i = lle->ll_count - 1; + MPASS(i != 0); + } + } while (i >= 0); + if (i < 0) + /* + * We are very likely bogus in this case. + */ + printf(" 1st %s last acquired @ %s:%d\n", + w->w_name, w->w_file, w->w_line); + else + printf(" 1st %p %s @ %s:%d\n", lock2, + lock2->lo_name, lock2->lo_file, + lock2->lo_line); printf(" 2nd %p %s @ %s:%d\n", - m1, w1->w_description, w1->w_file, w1->w_line); + lock1, lock1->lo_name, lock1->lo_file, + lock1->lo_line); printf(" 3rd %p %s @ %s:%d\n", - m, w->w_description, file, line); + lock, lock->lo_name, file, line); #ifdef DDB go_into_ddb = 1; #endif /* DDB */ goto out; } } - m1 = LIST_FIRST(&p->p_heldmtx); - if (!itismychild(m1->mtx_witness, w)) - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + lock1 = (*lock_list)->ll_children[(*lock_list)->ll_count - 1]; + if (!itismychild(lock1->lo_witness, w)) + mtx_unlock_spin(&w_mtx); out: #ifdef DDB @@ -1228,212 +610,162 @@ out: #endif /* DDB */ w->w_file = file; w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - - /* - * If this pays off it likely means that a mutex being witnessed - * is acquired in hardclock. Put it in the ignore list. It is - * likely not the mutex this assert fails on. - */ - MPASS(m->mtx_held.le_prev == NULL); - LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); -} - -void -witness_try_enter(struct mtx *m, int flags, const char *file, int line) -{ - struct proc *p; - struct witness *w = m->mtx_witness; - - if (witness_cold) - return; - if (panicstr) - return; - if (flags & MTX_SPIN) { - if ((m->mtx_flags & MTX_SPIN) == 0) - panic("mutex_try_enter: " - "MTX_SPIN on MTX_DEF mutex %s @ %s:%d", - m->mtx_description, file, line); - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_try_enter: recursion on" - " non-recursive mutex %s @ %s:%d", - m->mtx_description, file, line); + lock->lo_line = line; + lock->lo_file = file; + + lle = *lock_list; + if (lle == NULL || lle->ll_count == LOCK_CHILDCOUNT) { + *lock_list = witness_lock_list_get(); + if (*lock_list == NULL) return; - } - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - PCPU_SET(witness_spin_check, - PCPU_GET(witness_spin_check) | w->w_level); - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - return; + (*lock_list)->ll_next = lle; + lle = *lock_list; } - - if ((m->mtx_flags & MTX_SPIN) != 0) - panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", - m->mtx_description, file, line); - - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_try_enter: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, file, - line); - return; - } - w->w_file = file; - w->w_line = line; - m->mtx_line = line; - m->mtx_file = file; - p = curproc; - MPASS(m->mtx_held.le_prev == NULL); - LIST_INSERT_HEAD(&p->p_heldmtx, (struct mtx*)m, mtx_held); + lle->ll_children[lle->ll_count++] = lock; } void -witness_exit(struct mtx *m, int flags, const char *file, int line) +witness_unlock(struct lock_object *lock, int flags, const char *file, int line) { - struct witness *w; + struct lock_list_entry **lock_list, *lle; + struct lock_class *class; struct proc *p; + int i, j; - if (witness_cold || m->mtx_witness == NULL || panicstr) + if (witness_cold || witness_dead || lock->lo_witness == NULL || + panicstr) return; - w = m->mtx_witness; p = curproc; + class = lock->lo_class; - if (flags & MTX_SPIN) { - if ((m->mtx_flags & MTX_SPIN) == 0) - panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @" - " %s:%d", m->mtx_description, file, line); - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_exit: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); - return; - } - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - PCPU_SET(witness_spin_check, - PCPU_GET(witness_spin_check) & ~w->w_level); - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - MPASS(p->p_spinlocks > 0); - p->p_spinlocks--; - return; - } - if ((m->mtx_flags & MTX_SPIN) != 0) - panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", - m->mtx_description, file, line); - - if (mtx_recursed(m)) { - if ((m->mtx_flags & MTX_RECURSE) == 0) - panic("mutex_exit: recursion on non-recursive" - " mutex %s @ %s:%d", m->mtx_description, - file, line); + if (lock->lo_flags & LO_RECURSED) { + if ((lock->lo_flags & LO_LOCKED) == 0) + panic("%s: recursed lock (%s) %s is not locked @ %s:%d", + __func__, class->lc_name, lock->lo_name, file, + line); return; } - if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold) - panic("switchable mtx_unlock() of %s when not legal @ %s:%d", - m->mtx_description, file, line); - LIST_REMOVE(m, mtx_held); - m->mtx_held.le_prev = NULL; + /* + * We don't need to protect this PCPU_GET() here against preemption + * because if we hold any spinlocks then we are already protected, + * and if we don't we will get NULL if we hold no spinlocks even if + * we switch CPU's while reading it. + */ + if (class->lc_flags & LC_SLEEPLOCK) { + if ((flags & LOP_NOSWITCH) == 0 && PCPU_GET(spinlocks) != NULL) + panic("switchable sleep unlock (%s) %s @ %s:%d", + class->lc_name, lock->lo_name, file, line); + lock_list = &p->p_sleeplocks; + } else + lock_list = PCPU_PTR(spinlocks); + + for (; *lock_list != NULL; lock_list = &(*lock_list)->ll_next) + for (i = 0; i < (*lock_list)->ll_count; i++) + if ((*lock_list)->ll_children[i] == lock) { + (*lock_list)->ll_count--; + for (j = i; j < (*lock_list)->ll_count; j++) + (*lock_list)->ll_children[j] = + (*lock_list)->ll_children[j + 1]; + if ((*lock_list)->ll_count == 0) { + lle = *lock_list; + *lock_list = lle->ll_next; + witness_lock_list_free(lle); + } + return; + } } +/* + * Warn if any held locks are not sleepable. Note that Giant and the lock + * passed in are both special cases since they are both released during the + * sleep process and aren't actually held while the process is asleep. + */ int -witness_sleep(int check_only, struct mtx *mtx, const char *file, int line) +witness_sleep(int check_only, struct lock_object *lock, const char *file, + int line) { - struct mtx *m; + struct lock_list_entry **lock_list, *lle; + struct lock_object *lock1; struct proc *p; - char **sleep; - int n = 0; + critical_t savecrit; + int i, n; - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); + if (witness_dead || panicstr) + return (0); + KASSERT(!witness_cold, ("%s: witness_cold\n", __func__)); + n = 0; + /* + * Preemption bad because we need PCPU_PTR(spinlocks) to not change. + */ + savecrit = critical_enter(); p = curproc; - LIST_FOREACH(m, &p->p_heldmtx, mtx_held) { - if (m == mtx) - continue; - for (sleep = sleep_list; *sleep!= NULL; sleep++) - if (strcmp(m->mtx_description, *sleep) == 0) - goto next; - if (n == 0) - printf("Whee!\n"); - printf("%s:%d: %s with \"%s\" locked from %s:%d\n", - file, line, check_only ? "could sleep" : "sleeping", - m->mtx_description, - m->mtx_witness->w_file, m->mtx_witness->w_line); - n++; - next: + lock_list = &p->p_sleeplocks; +again: + for (lle = *lock_list; lle != NULL; lle = lle->ll_next) + for (i = lle->ll_count - 1; i >= 0; i--) { + lock1 = lle->ll_children[i]; + if (lock1 == lock || lock1 == &Giant.mtx_object || + (lock1->lo_flags & LO_SLEEPABLE)) + continue; + n++; + printf("%s:%d: %s with \"%s\" locked from %s:%d\n", + file, line, check_only ? "could sleep" : "sleeping", + lock1->lo_name, lock1->lo_file, lock1->lo_line); + } + if (lock_list == &p->p_sleeplocks) { + lock_list = PCPU_PTR(spinlocks); + goto again; } #ifdef DDB if (witness_ddb && n) Debugger("witness_sleep"); #endif /* DDB */ + critical_exit(savecrit); return (n); } static struct witness * -enroll(const char *description, int flag) +enroll(const char *description, struct lock_class *lock_class) { - int i; - struct witness *w, *w1; - char **ignore; - char **order; + struct witness *w; if (!witness_watch) return (NULL); - for (ignore = ignore_list; *ignore != NULL; ignore++) - if (strcmp(description, *ignore) == 0) - return (NULL); - - if (w_inited == 0) { - mtx_init(&w_mtx, "witness lock", MTX_SPIN); - for (i = 0; i < WITNESS_COUNT; i++) { - w = &w_data[i]; - witness_free(w); - } - w_inited = 1; - for (order = order_list; *order != NULL; order++) { - w = enroll(*order, MTX_DEF); - w->w_file = "order list"; - for (order++; *order != NULL; order++) { - w1 = enroll(*order, MTX_DEF); - w1->w_file = "order list"; - itismychild(w, w1); - w = w1; - } - } - } - if ((flag & MTX_SPIN) && witness_skipspin) + + if ((lock_class->lc_flags & LC_SPINLOCK) && witness_skipspin) return (NULL); - mtx_lock_spin_flags(&w_mtx, MTX_QUIET); - for (w = w_all; w; w = w->w_next) { - if (strcmp(description, w->w_description) == 0) { - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + mtx_lock_spin(&w_mtx); + STAILQ_FOREACH(w, &w_all, w_list) { + if (strcmp(description, w->w_name) == 0) { + mtx_unlock_spin(&w_mtx); + if (lock_class != w->w_class) + panic( + "lock (%s) %s does not match earlier (%s) lock", + description, lock_class->lc_name, + w->w_class->lc_name); return (w); } } + /* + * This isn't quite right, as witness_cold is still 0 while we + * enroll all the locks initialized before witness_initialize(). + */ + if ((lock_class->lc_flags & LC_SPINLOCK) && !witness_cold) + panic("spin lock %s not in order list", description); if ((w = witness_get()) == NULL) return (NULL); - w->w_next = w_all; - w_all = w; - w->w_description = description; - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - if (flag & MTX_SPIN) { - w->w_spin = 1; - - i = 1; - for (order = spin_order_list; *order != NULL; order++) { - if (strcmp(description, *order) == 0) - break; - i <<= 1; - } - if (*order == NULL) - panic("spin lock %s not in order list", description); - w->w_level = i; - } + w->w_name = description; + w->w_class = lock_class; + STAILQ_INSERT_HEAD(&w_all, w, w_list); + if (lock_class->lc_flags & LC_SPINLOCK) + STAILQ_INSERT_HEAD(&w_spin, w, w_typelist); + else if (lock_class->lc_flags & LC_SLEEPLOCK) + STAILQ_INSERT_HEAD(&w_sleep, w, w_typelist); + else + panic("lock class %s is not sleep or spin", + lock_class->lc_name); + mtx_unlock_spin(&w_mtx); return (w); } @@ -1442,29 +774,46 @@ static int itismychild(struct witness *parent, struct witness *child) { static int recursed; + struct witness_child_list_entry **wcl; + struct witness_list *list; + + MPASS(child != NULL && parent != NULL); + if ((parent->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK)) != + (child->w_class->lc_flags & (LC_SLEEPLOCK | LC_SPINLOCK))) + panic( + "%s: parent (%s) and child (%s) are not the same lock type", + __func__, parent->w_class->lc_name, + child->w_class->lc_name); /* * Insert "child" after "parent" */ - while (parent->w_morechildren) - parent = parent->w_morechildren; + wcl = &parent->w_children; + while (*wcl != NULL && (*wcl)->wcl_count == WITNESS_NCHILDREN) + wcl = &(*wcl)->wcl_next; - if (parent->w_childcnt == WITNESS_NCHILDREN) { - if ((parent->w_morechildren = witness_get()) == NULL) + if (*wcl == NULL) { + *wcl = witness_child_get(); + if (*wcl == NULL) return (1); - parent = parent->w_morechildren; } - MPASS(child != NULL); - parent->w_children[parent->w_childcnt++] = child; + + (*wcl)->wcl_children[(*wcl)->wcl_count++] = child; + /* - * now prune whole tree + * Now prune whole tree. We look for cases where a lock is now + * both a descendant and a direct child of a given lock. In that + * case, we want to remove the direct child link from the tree. */ if (recursed) return (0); recursed = 1; - for (child = w_all; child != NULL; child = child->w_next) { - for (parent = w_all; parent != NULL; - parent = parent->w_next) { + if (parent->w_class->lc_flags & LC_SLEEPLOCK) + list = &w_sleep; + else + list = &w_spin; + STAILQ_FOREACH(child, list, w_typelist) { + STAILQ_FOREACH(parent, list, w_typelist) { if (!isitmychild(parent, child)) continue; removechild(parent, child); @@ -1481,40 +830,38 @@ itismychild(struct witness *parent, struct witness *child) static void removechild(struct witness *parent, struct witness *child) { - struct witness *w, *w1; + struct witness_child_list_entry **wcl, *wcl1; int i; - for (w = parent; w != NULL; w = w->w_morechildren) - for (i = 0; i < w->w_childcnt; i++) - if (w->w_children[i] == child) + for (wcl = &parent->w_children; *wcl != NULL; wcl = &(*wcl)->wcl_next) + for (i = 0; i < (*wcl)->wcl_count; i++) + if ((*wcl)->wcl_children[i] == child) goto found; return; found: - for (w1 = w; w1->w_morechildren != NULL; w1 = w1->w_morechildren) - continue; - w->w_children[i] = w1->w_children[--w1->w_childcnt]; - MPASS(w->w_children[i] != NULL); + (*wcl)->wcl_count--; + if ((*wcl)->wcl_count > i) + (*wcl)->wcl_children[i] = + (*wcl)->wcl_children[(*wcl)->wcl_count]; + MPASS((*wcl)->wcl_children[i] != NULL); - if (w1->w_childcnt != 0) + if ((*wcl)->wcl_count != 0) return; - if (w1 == parent) - return; - for (w = parent; w->w_morechildren != w1; w = w->w_morechildren) - continue; - w->w_morechildren = 0; - witness_free(w1); + wcl1 = *wcl; + *wcl = wcl1->wcl_next; + witness_child_free(wcl1); } static int isitmychild(struct witness *parent, struct witness *child) { - struct witness *w; + struct witness_child_list_entry *wcl; int i; - for (w = parent; w != NULL; w = w->w_morechildren) { - for (i = 0; i < w->w_childcnt; i++) { - if (w->w_children[i] == child) + for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) { + for (i = 0; i < wcl->wcl_count; i++) { + if (wcl->wcl_children[i] == child) return (1); } } @@ -1524,20 +871,19 @@ isitmychild(struct witness *parent, struct witness *child) static int isitmydescendant(struct witness *parent, struct witness *child) { - struct witness *w; - int i; - int j; + struct witness_child_list_entry *wcl; + int i, j; - for (j = 0, w = parent; w != NULL; w = w->w_morechildren, j++) { + if (isitmychild(parent, child)) + return (1); + j = 0; + for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) { MPASS(j < 1000); - for (i = 0; i < w->w_childcnt; i++) { - if (w->w_children[i] == child) - return (1); - } - for (i = 0; i < w->w_childcnt; i++) { - if (isitmydescendant(w->w_children[i], child)) + for (i = 0; i < wcl->wcl_count; i++) { + if (isitmydescendant(wcl->wcl_children[i], child)) return (1); } + j++; } return (0); } @@ -1545,70 +891,81 @@ isitmydescendant(struct witness *parent, struct witness *child) void witness_levelall (void) { + struct witness_list *list; struct witness *w, *w1; - for (w = w_all; w; w = w->w_next) - if (!(w->w_spin)) - w->w_level = 0; - for (w = w_all; w; w = w->w_next) { - if (w->w_spin) - continue; - for (w1 = w_all; w1; w1 = w1->w_next) { + /* + * First clear all levels. + */ + STAILQ_FOREACH(w, &w_all, w_list) { + w->w_level = 0; + } + + /* + * Look for locks with no parent and level all their descendants. + */ + STAILQ_FOREACH(w, &w_all, w_list) { + /* + * This is just an optimization, technically we could get + * away just walking the all list each time. + */ + if (w->w_class->lc_flags & LC_SLEEPLOCK) + list = &w_sleep; + else + list = &w_spin; + STAILQ_FOREACH(w1, list, w_typelist) { if (isitmychild(w1, w)) - break; + goto skip; } - if (w1 != NULL) - continue; witness_leveldescendents(w, 0); + skip: } } static void witness_leveldescendents(struct witness *parent, int level) { + struct witness_child_list_entry *wcl; int i; - struct witness *w; if (parent->w_level < level) parent->w_level = level; level++; - for (w = parent; w != NULL; w = w->w_morechildren) - for (i = 0; i < w->w_childcnt; i++) - witness_leveldescendents(w->w_children[i], level); + for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) + for (i = 0; i < wcl->wcl_count; i++) + witness_leveldescendents(wcl->wcl_children[i], level); } static void witness_displaydescendants(void(*prnt)(const char *fmt, ...), struct witness *parent) { - struct witness *w; - int i; - int level; + struct witness_child_list_entry *wcl; + int i, level; - level = parent->w_spin ? ffs(parent->w_level) : parent->w_level; + level = parent->w_level; - prnt("%d", level); - if (level < 10) - prnt(" "); + prnt("%-2d", level); for (i = 0; i < level; i++) prnt(" "); - prnt("%s", parent->w_description); + prnt("%s", parent->w_name); if (parent->w_file != NULL) prnt(" -- last acquired @ %s:%d\n", parent->w_file, parent->w_line); - for (w = parent; w != NULL; w = w->w_morechildren) - for (i = 0; i < w->w_childcnt; i++) - witness_displaydescendants(prnt, w->w_children[i]); - } + for (wcl = parent->w_children; wcl != NULL; wcl = wcl->wcl_next) + for (i = 0; i < wcl->wcl_count; i++) + witness_displaydescendants(prnt, + wcl->wcl_children[i]); +} static int dup_ok(struct witness *w) { - char **dup; + const char **dup; - for (dup = dup_list; *dup!= NULL; dup++) - if (strcmp(w->w_description, *dup) == 0) + for (dup = dup_list; *dup != NULL; dup++) + if (strcmp(w->w_name, *dup) == 0) return (1); return (0); } @@ -1621,30 +978,31 @@ blessed(struct witness *w1, struct witness *w2) for (i = 0; i < blessed_count; i++) { b = &blessed_list[i]; - if (strcmp(w1->w_description, b->b_lock1) == 0) { - if (strcmp(w2->w_description, b->b_lock2) == 0) + if (strcmp(w1->w_name, b->b_lock1) == 0) { + if (strcmp(w2->w_name, b->b_lock2) == 0) return (1); continue; } - if (strcmp(w1->w_description, b->b_lock2) == 0) - if (strcmp(w2->w_description, b->b_lock1) == 0) + if (strcmp(w1->w_name, b->b_lock2) == 0) + if (strcmp(w2->w_name, b->b_lock1) == 0) return (1); } return (0); } static struct witness * -witness_get() +witness_get(void) { struct witness *w; - if ((w = w_free) == NULL) { + if (STAILQ_EMPTY(&w_free)) { witness_dead = 1; - mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); - printf("witness exhausted\n"); + mtx_unlock_spin(&w_mtx); + printf("%s: witness exhausted\n", __func__); return (NULL); } - w_free = w->w_next; + w = STAILQ_FIRST(&w_free); + STAILQ_REMOVE_HEAD(&w_free, w_list); bzero(w, sizeof(*w)); return (w); } @@ -1652,65 +1010,144 @@ witness_get() static void witness_free(struct witness *w) { - w->w_next = w_free; - w_free = w; + + STAILQ_INSERT_HEAD(&w_free, w, w_list); } -int -witness_list(struct proc *p) +static struct witness_child_list_entry * +witness_child_get(void) { - struct mtx *m; - int nheld; + struct witness_child_list_entry *wcl; - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - nheld = 0; - LIST_FOREACH(m, &p->p_heldmtx, mtx_held) { - printf("\t\"%s\" (%p) locked at %s:%d\n", - m->mtx_description, m, - m->mtx_witness->w_file, m->mtx_witness->w_line); - nheld++; + wcl = w_child_free; + if (wcl == NULL) { + witness_dead = 1; + mtx_unlock_spin(&w_mtx); + printf("%s: witness exhausted\n", __func__); + return (NULL); } + w_child_free = wcl->wcl_next; + bzero(wcl, sizeof(*wcl)); + return (wcl); +} - return (nheld); +static void +witness_child_free(struct witness_child_list_entry *wcl) +{ + + wcl->wcl_next = w_child_free; + w_child_free = wcl; } -#ifdef DDB +static struct lock_list_entry * +witness_lock_list_get(void) +{ + struct lock_list_entry *lle; -DB_SHOW_COMMAND(mutexes, db_witness_list) + mtx_lock_spin(&w_mtx); + lle = w_lock_list_free; + if (lle == NULL) { + witness_dead = 1; + mtx_unlock_spin(&w_mtx); + printf("%s: witness exhausted\n", __func__); + return (NULL); + } + w_lock_list_free = lle->ll_next; + mtx_unlock_spin(&w_mtx); + bzero(lle, sizeof(*lle)); + return (lle); +} + +static void +witness_lock_list_free(struct lock_list_entry *lle) { - witness_list(curproc); + mtx_lock_spin(&w_mtx); + lle->ll_next = w_lock_list_free; + w_lock_list_free = lle; + mtx_unlock_spin(&w_mtx); } -DB_SHOW_COMMAND(witness, db_witness_display) +/* + * Calling this on p != curproc is bad unless we are in ddb. + */ +int +witness_list(struct proc *p) { + struct lock_list_entry **lock_list, *lle; + struct lock_object *lock; + critical_t savecrit; + int i, nheld; + + KASSERT(p == curproc || db_active, + ("%s: p != curproc and we aren't in the debugger", __func__)); + KASSERT(!witness_cold, ("%s: witness_cold", __func__)); + nheld = 0; + /* + * Preemption bad because we need PCPU_PTR(spinlocks) to not change. + */ + savecrit = critical_enter(); + lock_list = &p->p_sleeplocks; +again: + for (lle = *lock_list; lle != NULL; lle = lle->ll_next) + for (i = lle->ll_count - 1; i >= 0; i--) { + lock = lle->ll_children[i]; + printf("\t(%s) %s (%p) locked at %s:%d\n", + lock->lo_class->lc_name, lock->lo_name, lock, + lock->lo_file, lock->lo_line); + nheld++; + } + /* + * We only handle spinlocks if p == curproc. This is somewhat broken + * if p is currently executing on some other CPU and holds spin locks + * as we won't display those locks. + */ + if (lock_list == &p->p_sleeplocks && p == curproc) { + lock_list = PCPU_PTR(spinlocks); + goto again; + } + critical_exit(savecrit); - witness_display(db_printf); + return (nheld); } -#endif void -witness_save(struct mtx *m, const char **filep, int *linep) +witness_save(struct lock_object *lock, const char **filep, int *linep) { - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - if (m->mtx_witness == NULL) + KASSERT(!witness_cold, ("%s: witness_cold\n", __func__)); + if (lock->lo_witness == NULL) return; - *filep = m->mtx_witness->w_file; - *linep = m->mtx_witness->w_line; + *filep = lock->lo_file; + *linep = lock->lo_line; } void -witness_restore(struct mtx *m, const char *file, int line) +witness_restore(struct lock_object *lock, const char *file, int line) { - KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__)); - if (m->mtx_witness == NULL) + KASSERT(!witness_cold, ("%s: witness_cold\n", __func__)); + if (lock->lo_witness == NULL) return; - m->mtx_witness->w_file = file; - m->mtx_witness->w_line = line; + lock->lo_witness->w_file = file; + lock->lo_witness->w_line = line; + lock->lo_file = file; + lock->lo_line = line; } -#endif /* WITNESS */ +#ifdef DDB + +DB_SHOW_COMMAND(mutexes, db_witness_list) +{ + + witness_list(curproc); +} + +DB_SHOW_COMMAND(witness, db_witness_display) +{ + + witness_display(db_printf); +} +#endif diff --git a/sys/powerpc/include/globaldata.h b/sys/powerpc/include/globaldata.h index 8e6cfbe..15bd8c41a 100644 --- a/sys/powerpc/include/globaldata.h +++ b/sys/powerpc/include/globaldata.h @@ -58,7 +58,7 @@ struct globaldata { u_int32_t gd_current_asngen; /* ASN rollover check */ SLIST_ENTRY(globaldata) gd_allcpu; - int gd_witness_spin_check; + struct lock_list_entry *gd_spinlocks; #ifdef KTR_PERCPU volatile int gd_ktr_idx; /* Index into trace table */ char *gd_ktr_buf; diff --git a/sys/powerpc/include/pcpu.h b/sys/powerpc/include/pcpu.h index 8e6cfbe..15bd8c41a 100644 --- a/sys/powerpc/include/pcpu.h +++ b/sys/powerpc/include/pcpu.h @@ -58,7 +58,7 @@ struct globaldata { u_int32_t gd_current_asngen; /* ASN rollover check */ SLIST_ENTRY(globaldata) gd_allcpu; - int gd_witness_spin_check; + struct lock_list_entry *gd_spinlocks; #ifdef KTR_PERCPU volatile int gd_ktr_idx; /* Index into trace table */ char *gd_ktr_buf; diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h index a13c25e..3bcef6f 100644 --- a/sys/sys/kernel.h +++ b/sys/sys/kernel.h @@ -119,7 +119,7 @@ enum sysinit_sub_id { SI_SUB_VM = 0x1000000, /* virtual memory system init*/ SI_SUB_KMEM = 0x1800000, /* kernel memory*/ SI_SUB_KVM_RSRC = 0x1A00000, /* kvm operational limits*/ - SI_SUB_MUTEX = 0x1A80000, /* mutex (witness) fixup */ + SI_SUB_WITNESS = 0x1A80000, /* witness initialization */ SI_SUB_LOCK = 0x1B00000, /* lockmgr locks */ SI_SUB_EVENTHANDLER = 0x1C00000, /* eventhandler init */ SI_SUB_CPU = 0x2000000, /* CPU resource(s)*/ diff --git a/sys/sys/lock.h b/sys/sys/lock.h index 80f8764..ed9cac6 100644 --- a/sys/sys/lock.h +++ b/sys/sys/lock.h @@ -1,10 +1,5 @@ -/* - * Copyright (c) 1995 - * The Regents of the University of California. All rights reserved. - * - * This code contains ideas from software contributed to Berkeley by - * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating - * System project at Carnegie-Mellon University. +/*- + * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -14,18 +9,14 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the University of - * California, Berkeley and its contributors. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. + * 3. Berkeley Software Design Inc's name may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) @@ -34,189 +25,184 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * @(#)lock.h 8.12 (Berkeley) 5/19/95 + * from BSDI $Id: mutex.h,v 2.7.2.35 2000/04/27 03:10:26 cp Exp $ * $FreeBSD$ */ -#ifndef _LOCK_H_ -#define _LOCK_H_ - -struct mtx; +#ifndef _SYS_LOCK_H_ +#define _SYS_LOCK_H_ /* - * The general lock structure. Provides for multiple shared locks, - * upgrading from shared to exclusive, and sleeping until the lock - * can be gained. - */ -struct lock { - struct mtx *lk_interlock; /* lock on remaining fields */ - u_int lk_flags; /* see below */ - int lk_sharecount; /* # of accepted shared locks */ - int lk_waitcount; /* # of processes sleeping for lock */ - short lk_exclusivecount; /* # of recursive exclusive locks */ - short lk_prio; /* priority at which to sleep */ - char *lk_wmesg; /* resource sleeping (for tsleep) */ - int lk_timo; /* maximum sleep time (for tsleep) */ - pid_t lk_lockholder; /* pid of exclusive lock holder */ -#ifdef DEBUG_LOCKS - const char *lk_filename; - const char *lk_lockername; - int lk_lineno; -#endif -}; -/* - * Lock request types: - * LK_SHARED - get one of many possible shared locks. If a process - * holding an exclusive lock requests a shared lock, the exclusive - * lock(s) will be downgraded to shared locks. - * LK_EXCLUSIVE - stop further shared locks, when they are cleared, - * grant a pending upgrade if it exists, then grant an exclusive - * lock. Only one exclusive lock may exist at a time, except that - * a process holding an exclusive lock may get additional exclusive - * locks if it explicitly sets the LK_CANRECURSE flag in the lock - * request, or if the LK_CANRECUSE flag was set when the lock was - * initialized. - * LK_UPGRADE - the process must hold a shared lock that it wants to - * have upgraded to an exclusive lock. Other processes may get - * exclusive access to the resource between the time that the upgrade - * is requested and the time that it is granted. - * LK_EXCLUPGRADE - the process must hold a shared lock that it wants to - * have upgraded to an exclusive lock. If the request succeeds, no - * other processes will have gotten exclusive access to the resource - * between the time that the upgrade is requested and the time that - * it is granted. However, if another process has already requested - * an upgrade, the request will fail (see error returns below). - * LK_DOWNGRADE - the process must hold an exclusive lock that it wants - * to have downgraded to a shared lock. If the process holds multiple - * (recursive) exclusive locks, they will all be downgraded to shared - * locks. - * LK_RELEASE - release one instance of a lock. - * LK_DRAIN - wait for all activity on the lock to end, then mark it - * decommissioned. This feature is used before freeing a lock that - * is part of a piece of memory that is about to be freed. - * LK_EXCLOTHER - return for lockstatus(). Used when another process - * holds the lock exclusively. - * - * These are flags that are passed to the lockmgr routine. + * XXX - compatability until lockmgr() goes away or all the #includes are + * updated. */ -#define LK_TYPE_MASK 0x0000000f /* type of lock sought */ -#define LK_SHARED 0x00000001 /* shared lock */ -#define LK_EXCLUSIVE 0x00000002 /* exclusive lock */ -#define LK_UPGRADE 0x00000003 /* shared-to-exclusive upgrade */ -#define LK_EXCLUPGRADE 0x00000004 /* first shared-to-exclusive upgrade */ -#define LK_DOWNGRADE 0x00000005 /* exclusive-to-shared downgrade */ -#define LK_RELEASE 0x00000006 /* release any type of lock */ -#define LK_DRAIN 0x00000007 /* wait for all lock activity to end */ -#define LK_EXCLOTHER 0x00000008 /* other process holds lock */ -/* - * External lock flags. - * - * The first three flags may be set in lock_init to set their mode permanently, - * or passed in as arguments to the lock manager. The LK_REENABLE flag may be - * set only at the release of a lock obtained by drain. - */ -#define LK_EXTFLG_MASK 0x01000070 /* mask of external flags */ -#define LK_NOWAIT 0x00000010 /* do not sleep to await lock */ -#define LK_SLEEPFAIL 0x00000020 /* sleep, then return failure */ -#define LK_CANRECURSE 0x00000040 /* allow recursive exclusive lock */ -#define LK_REENABLE 0x00000080 /* lock is be reenabled after drain */ -#define LK_NOPAUSE 0x01000000 /* no spinloop */ +#include <sys/lockmgr.h> + +#include <sys/queue.h> + /* - * Internal lock flags. + * Lock classes. Each lock has a class which describes characteristics + * common to all types of locks of a given class. * - * These flags are used internally to the lock manager. + * Spin locks in general must always protect against preemption, as it is + * an error to perform any type of context switch while holding a spin lock. + * Also, for an individual lock to be recursable, its class must allow + * recursion and the lock itself must explicitly allow recursion. */ -#define LK_WANT_UPGRADE 0x00000100 /* waiting for share-to-excl upgrade */ -#define LK_WANT_EXCL 0x00000200 /* exclusive lock sought */ -#define LK_HAVE_EXCL 0x00000400 /* exclusive lock obtained */ -#define LK_WAITDRAIN 0x00000800 /* process waiting for lock to drain */ -#define LK_DRAINING 0x00004000 /* lock is being drained */ + +struct lock_class { + const char *lc_name; + u_int lc_flags; +}; + +#define LC_SLEEPLOCK 0x00000001 /* Sleep lock. */ +#define LC_SPINLOCK 0x00000002 /* Spin lock. */ +#define LC_SLEEPABLE 0x00000004 /* Sleeping allowed with this lock. */ +#define LC_RECURSABLE 0x00000008 /* Locks of this type may recurse. */ + +struct witness; + +struct lock_object { + struct lock_class *lo_class; + const char *lo_name; + const char *lo_file; /* File and line of last acquire. */ + int lo_line; + u_int lo_flags; + STAILQ_ENTRY(lock_object) lo_list; /* List of all locks in system. */ + struct witness *lo_witness; +}; + +#define LO_CLASSFLAGS 0x0000ffff /* Class specific flags. */ +#define LO_INITIALIZED 0x00010000 /* Lock has been initialized. */ +#define LO_WITNESS 0x00020000 /* Should witness monitor this lock. */ +#define LO_QUIET 0x00040000 /* Don't log locking operations. */ +#define LO_RECURSABLE 0x00080000 /* Lock may recurse. */ +#define LO_SLEEPABLE 0x00100000 /* Lock may be held while sleeping. */ +#define LO_LOCKED 0x01000000 /* Someone holds this lock. */ +#define LO_RECURSED 0x02000000 /* Someone has recursed on this lock. */ + /* - * Control flags - * - * Non-persistent external flags. + * Option flags passed to lock operations that witness also needs to know + * about or that are generic across all locks. */ -#define LK_INTERLOCK 0x00010000 /* - * unlock passed mutex after getting - * lk_interlock - */ -#define LK_RETRY 0x00020000 /* vn_lock: retry until locked */ -#define LK_NOOBJ 0x00040000 /* vget: don't create object */ -#define LK_THISLAYER 0x00080000 /* vn_lock: lock/unlock only current layer */ +#define LOP_NOSWITCH 0x00000001 /* Lock doesn't switch on release. */ +#define LOP_QUIET 0x00000002 /* Don't log locking operations. */ +#define LOP_TRYLOCK 0x00000004 /* Don't check lock order. */ +#ifdef _KERNEL /* - * Internal state flags corresponding to lk_sharecount, and lk_waitcount + * A simple list type used to build the list of locks held by a process + * or CPU. We can't simply embed the list in struct lock_object since a + * lock may be held by more than one process if it is a shared lock. Locks + * are added to the head of the list, so we fill up each list entry from + * "the back" logically. To ease some of the arithmetic, we actually fill + * in each list entry the normal way (childer[0] then children[1], etc.) but + * when we traverse the list we read children[count-1] as the first entry + * down to children[0] as the final entry. */ -#define LK_SHARE_NONZERO 0x00100000 -#define LK_WAIT_NONZERO 0x00200000 +#define LOCK_NCHILDREN 6 + +struct lock_list_entry { + struct lock_list_entry *ll_next; + struct lock_object *ll_children[LOCK_NCHILDREN]; + u_int ll_count; +}; /* - * Lock return status. + * Macros for KTR_LOCK tracing. * - * Successfully obtained locks return 0. Locks will always succeed - * unless one of the following is true: - * LK_FORCEUPGRADE is requested and some other process has already - * requested a lock upgrade (returns EBUSY). - * LK_WAIT is set and a sleep would be required (returns EBUSY). - * LK_SLEEPFAIL is set and a sleep was done (returns ENOLCK). - * PCATCH is set in lock priority and a signal arrives (returns - * either EINTR or ERESTART if system calls is to be restarted). - * Non-null lock timeout and timeout expires (returns EWOULDBLOCK). - * A failed lock attempt always returns a non-zero error value. No lock - * is held after an error return (in particular, a failed LK_UPGRADE - * or LK_FORCEUPGRADE will have released its shared access lock). + * opname - name of this operation (LOCK/UNLOCK/SLOCK, etc.) + * lo - struct lock_object * for this lock + * flags - flags passed to the lock operation + * recurse - this locks recursion level (or 0 if class is not recursable) + * result - result of a try lock operation + * file - file name + * line - line number */ +#define LOCK_LOG_TEST(lo, flags) \ + (((flags) & LOP_QUIET) == 0 && ((lo)->lo_flags & LO_QUIET) == 0) + +#define LOCK_LOG_LOCK(opname, lo, flags, recurse, file, line) do { \ + if (LOCK_LOG_TEST((lo), (flags))) \ + CTR5(KTR_LOCK, opname " (%s) %s r = %d at %s:%d", \ + (lo)->lo_class->lc_name, (lo)->lo_name, \ + (u_int)(recurse), (file), (line)); \ +} while (0) + +#define LOCK_LOG_TRY(opname, lo, flags, result, file, line) do { \ + if (LOCK_LOG_TEST((lo), (flags))) \ + CTR5(KTR_LOCK, "TRY_" opname " (%s) %s result=%d at %s:%d",\ + (lo)->lo_class->lc_name, (lo)->lo_name, \ + (u_int)(result), (file), (line)); \ +} while (0) + +#define LOCK_LOG_INIT(lo, flags) do { \ + if (LOCK_LOG_TEST((lo), (flags))) \ + CTR3(KTR_LOCK, __func__ ": %p (%s) %s", (lo), \ + (lo)->lo_class->lc_name, (lo)->lo_name); \ +} while (0) + +#define LOCK_LOG_DESTROY(lo, flags) LOCK_LOG_INIT(lo, flags) /* - * Indicator that no process holds exclusive lock + * Helpful macros for quickly coming up with assertions with informative + * panic messages. */ -#define LK_KERNPROC ((pid_t) -2) -#define LK_NOPROC ((pid_t) -1) - -#ifdef INVARIANTS -#define LOCKMGR_ASSERT(lkp, what, p) do { \ - switch ((what)) { \ - case LK_SHARED: \ - if (lockstatus((lkp), (p)) == LK_SHARED) \ - break; \ - /* fall into exclusive */ \ - case LK_EXCLUSIVE: \ - if (lockstatus((lkp), (p)) != LK_EXCLUSIVE) \ - panic("lock %s %s not held at %s:%d", \ - (lkp)->lk_wmesg, #what, __FILE__, \ - __LINE__); \ - break; \ - default: \ - panic("unknown LOCKMGR_ASSERT at %s:%d", __FILE__, \ - __LINE__); \ - } \ -} while (0) -#else /* INVARIANTS */ -#define LOCKMGR_ASSERT(lkp, p, what) -#endif /* INVARIANTS */ - -void dumplockinfo(struct lock *lkp); -struct proc; - -void lockinit __P((struct lock *, int prio, char *wmesg, int timo, - int flags)); -void lockdestroy __P((struct lock *)); - -#ifdef DEBUG_LOCKS -int debuglockmgr __P((struct lock *, u_int flags, - struct mtx *, struct proc *p, - const char *, - const char *, - int)); -#define lockmgr(lockp, flags, slockp, proc) \ - debuglockmgr((lockp), (flags), (slockp), (proc), \ - "lockmgr", __FILE__, __LINE__) -#else -int lockmgr __P((struct lock *, u_int flags, - struct mtx *, struct proc *p)); -#endif -void lockmgr_printinfo __P((struct lock *)); -int lockstatus __P((struct lock *, struct proc *)); -int lockcount __P((struct lock *)); - -#endif /* !_LOCK_H_ */ +#define MPASS(ex) MPASS4(ex, #ex, __FILE__, __LINE__) +#define MPASS2(ex, what) MPASS4(ex, what, __FILE__, __LINE__) +#define MPASS3(ex, file, line) MPASS4(ex, #ex, file, line) +#define MPASS4(ex, what, file, line) \ + KASSERT((ex), ("Assertion %s failed at %s:%d", what, file, line)) + +extern struct lock_class lock_class_mtx_sleep; +extern struct lock_class lock_class_mtx_spin; +extern struct lock_class lock_class_sx; + +void witness_init(struct lock_object *); +void witness_destroy(struct lock_object *); +void witness_lock(struct lock_object *, int, const char *, int); +void witness_unlock(struct lock_object *, int, const char *, int); +void witness_save(struct lock_object *, const char **, int *); +void witness_restore(struct lock_object *, const char *, int); +int witness_list(struct proc *); +int witness_sleep(int, struct lock_object *, const char *, int); + +#ifdef WITNESS +#define WITNESS_INIT(lock) \ + witness_init((lock)) + +#define WITNESS_DESTROY(lock) \ + witness_destroy(lock) + +#define WITNESS_LOCK(lock, flags, file, line) \ + witness_lock((lock), (flags), (file), (line)) + +#define WITNESS_UNLOCK(lock, flags, file, line) \ + witness_unlock((lock), (flags), (file), (line)) + +#define WITNESS_SLEEP(check, lock) \ + witness_sleep((check), (lock), __FILE__, __LINE__) + +#define WITNESS_SAVE_DECL(n) \ + const char * __CONCAT(n, __wf); \ + int __CONCAT(n, __wl) + +#define WITNESS_SAVE(lock, n) \ + witness_save((lock), &__CONCAT(n, __wf), &__CONCAT(n, __wl)) + +#define WITNESS_RESTORE(lock, n) \ + witness_restore((lock), __CONCAT(n, __wf), __CONCAT(n, __wl)) + +#else /* WITNESS */ +#define WITNESS_INIT(lock) (lock)->lo_flags |= LO_INITIALIZED +#define WITNESS_DESTROY(lock) (lock)->lo_flags &= ~LO_INITIALIZED +#define WITNESS_LOCK(lock, flags, file, line) +#define WITNESS_UNLOCK(lock, flags, file, line) +#define WITNESS_SLEEP(check, lock) +#define WITNESS_SAVE_DECL(n) +#define WITNESS_SAVE(lock, n) +#define WITNESS_RESTORE(lock, n) +#endif /* WITNESS */ + +#endif /* _KERNEL */ +#endif /* _SYS_LOCK_H_ */ diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index d637cec..7a893a2 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -37,7 +37,6 @@ #ifdef _KERNEL #include <sys/ktr.h> -#include <sys/systm.h> #include <machine/atomic.h> #include <machine/cpufunc.h> #include <machine/globals.h> @@ -49,25 +48,21 @@ #ifdef _KERNEL /* - * Mutex types and options stored in mutex->mtx_flags + * Mutex types and options passed to mtx_init(). MTX_QUIET can also be + * passed in. */ #define MTX_DEF 0x00000000 /* DEFAULT (sleep) lock */ #define MTX_SPIN 0x00000001 /* Spin lock (disables interrupts) */ -#define MTX_RECURSE 0x00000002 /* Option: lock allowed to recurse */ +#define MTX_RECURSE 0x00000004 /* Option: lock allowed to recurse */ +#define MTX_NOWITNESS 0x00000008 /* Don't do any witness checking. */ +#define MTX_SLEEPABLE 0x00000010 /* We can sleep with this lock. */ /* * Option flags passed to certain lock/unlock routines, through the use * of corresponding mtx_{lock,unlock}_flags() interface macros. - * - * XXX: The only reason we make these bits not interfere with the above "types - * and options" bits is because we have to pass both to the witness - * routines right now; if/when we clean up the witness interface to - * not check for mutex type from the passed in flag, but rather from - * the mutex lock's mtx_flags field, then we can change these values to - * 0x1, 0x2, ... */ -#define MTX_NOSWITCH 0x00000004 /* Do not switch on release */ -#define MTX_QUIET 0x00000008 /* Don't log a mutex event */ +#define MTX_NOSWITCH LOP_NOSWITCH /* Do not switch on release */ +#define MTX_QUIET LOP_QUIET /* Don't log a mutex event */ /* * State bits kept in mutex->mtx_lock, for the DEFAULT lock type. None of this, @@ -82,22 +77,19 @@ #ifndef LOCORE -struct mtx_debug; - /* * Sleep/spin mutex */ -struct mtx { + +struct lock_object; + +struct mtx { + struct lock_object mtx_object; /* Common lock properties. */ volatile uintptr_t mtx_lock; /* owner (and state for sleep locks) */ - volatile u_int mtx_recurse; /* number of recursive holds */ - u_int mtx_savecrit; /* saved flags (for spin locks) */ - int mtx_flags; /* flags passed to mtx_init() */ - const char *mtx_description; + volatile u_int mtx_recurse; /* number of recursive holds */ + critical_t mtx_savecrit; /* saved flags (for spin locks) */ TAILQ_HEAD(, proc) mtx_blocked; /* threads blocked on this lock */ LIST_ENTRY(mtx) mtx_contested; /* list of all contested locks */ - struct mtx *mtx_next; /* all existing locks */ - struct mtx *mtx_prev; /* in system... */ - struct mtx_debug *mtx_debug; /* debugging information... */ }; /* @@ -109,20 +101,12 @@ struct mtx { #ifdef _KERNEL /* - * Strings for KTR_LOCK tracing. - */ -extern char STR_mtx_lock_slp[]; -extern char STR_mtx_lock_spn[]; -extern char STR_mtx_unlock_slp[]; -extern char STR_mtx_unlock_spn[]; - -/* * Prototypes * * NOTE: Functions prepended with `_' (underscore) are exported to other parts * of the kernel via macros, thus allowing us to use the cpp __FILE__ * and __LINE__. These functions should not be called directly by any - * code using the IPI. Their macros cover their functionality. + * code using the API. Their macros cover their functionality. * * [See below for descriptions] * @@ -144,6 +128,9 @@ void _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, #ifdef INVARIANT_SUPPORT void _mtx_assert(struct mtx *m, int what, const char *file, int line); #endif +#ifdef WITNESS +void _mtx_update_flags(struct mtx *m, int locking); +#endif /* * We define our machine-independent (unoptimized) mutex micro-operations @@ -231,6 +218,15 @@ void _mtx_assert(struct mtx *m, int what, const char *file, int line); #endif /* + * Update the lock object flags based on the current mutex state. + */ +#ifdef WITNESS +#define mtx_update_flags(m, locking) _mtx_update_flags((m), (locking)) +#else +#define mtx_update_flags(m, locking) +#endif + +/* * Exported lock manipulation interface. * * mtx_lock(m) locks MTX_DEF mutex `m' @@ -255,6 +251,8 @@ void _mtx_assert(struct mtx *m, int what, const char *file, int line); * mtx_trylock_flags(m, opts) is used the same way as mtx_trylock() but accepts * relevant option flags `opts.' * + * mtx_initialized(m) returns non-zero if the lock `m' has been initialized. + * * mtx_owned(m) returns non-zero if the current thread owns the lock `m' * * mtx_recursed(m) returns non-zero if the lock `m' is presently recursed. @@ -290,48 +288,46 @@ void _mtx_assert(struct mtx *m, int what, const char *file, int line); KASSERT(((opts) & MTX_NOSWITCH) == 0, \ ("MTX_NOSWITCH used at %s:%d", (file), (line))); \ _get_sleep_lock((m), curproc, (opts), (file), (line)); \ - if (((opts) & MTX_QUIET) == 0) \ - CTR5(KTR_LOCK, STR_mtx_lock_slp, \ - (m)->mtx_description, (m), (m)->mtx_recurse, \ - (file), (line)); \ - WITNESS_ENTER((m), ((m)->mtx_flags | (opts)), (file), (line)); \ + LOCK_LOG_LOCK("LOCK", &(m)->mtx_object, opts, m->mtx_recurse, \ + (file), (line)); \ + mtx_update_flags((m), 1); \ + WITNESS_LOCK(&(m)->mtx_object, (opts), (file), (line)); \ } while (0) #define __mtx_lock_spin_flags(m, opts, file, line) do { \ MPASS(curproc != NULL); \ _get_spin_lock((m), curproc, (opts), (file), (line)); \ - if (((opts) & MTX_QUIET) == 0) \ - CTR5(KTR_LOCK, STR_mtx_lock_spn, \ - (m)->mtx_description, (m), (m)->mtx_recurse, \ - (file), (line)); \ - WITNESS_ENTER((m), ((m)->mtx_flags | (opts)), (file), (line)); \ + LOCK_LOG_LOCK("LOCK", &(m)->mtx_object, opts, m->mtx_recurse, \ + (file), (line)); \ + mtx_update_flags((m), 1); \ + WITNESS_LOCK(&(m)->mtx_object, (opts), (file), (line)); \ } while (0) #define __mtx_unlock_flags(m, opts, file, line) do { \ MPASS(curproc != NULL); \ mtx_assert((m), MA_OWNED); \ - WITNESS_EXIT((m), ((m)->mtx_flags | (opts)), (file), (line)); \ + mtx_update_flags((m), 0); \ + WITNESS_UNLOCK(&(m)->mtx_object, (opts), (file), (line)); \ _rel_sleep_lock((m), curproc, (opts), (file), (line)); \ - if (((opts) & MTX_QUIET) == 0) \ - CTR5(KTR_LOCK, STR_mtx_unlock_slp, \ - (m)->mtx_description, (m), (m)->mtx_recurse, \ - (file), (line)); \ + LOCK_LOG_LOCK("UNLOCK", &(m)->mtx_object, (opts), \ + (m)->mtx_recurse, (file), (line)); \ } while (0) #define __mtx_unlock_spin_flags(m, opts, file, line) do { \ MPASS(curproc != NULL); \ mtx_assert((m), MA_OWNED); \ - WITNESS_EXIT((m), ((m)->mtx_flags | (opts)), (file), (line)); \ + mtx_update_flags((m), 0); \ + WITNESS_UNLOCK(&(m)->mtx_object, (opts), (file), (line)); \ _rel_spin_lock((m)); \ - if (((opts) & MTX_QUIET) == 0) \ - CTR5(KTR_LOCK, STR_mtx_unlock_spn, \ - (m)->mtx_description, (m), (m)->mtx_recurse, \ - (file), (line)); \ + LOCK_LOG_LOCK("UNLOCK", &(m)->mtx_object, (opts), \ + (m)->mtx_recurse, (file), (line)); \ } while (0) #define mtx_trylock_flags(m, opts) \ _mtx_trylock((m), (opts), __FILE__, __LINE__) +#define mtx_initialized(m) ((m)->mtx_object.lo_flags & LO_INITIALIZED) + #define mtx_owned(m) (((m)->mtx_lock & MTX_FLAGMASK) == (uintptr_t)curproc) #define mtx_recursed(m) ((m)->mtx_recurse != 0) @@ -354,7 +350,7 @@ do { \ WITNESS_SAVE_DECL(Giant); \ \ if (mtx_owned(&Giant)) \ - WITNESS_SAVE(&Giant, Giant); \ + WITNESS_SAVE(&Giant.mtx_object, Giant); \ for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++) \ mtx_unlock_flags(&Giant, MTX_NOSWITCH) @@ -364,7 +360,7 @@ do { \ WITNESS_SAVE_DECL(Giant); \ \ if (mtx_owned(&Giant)) \ - WITNESS_SAVE(&Giant, Giant); \ + WITNESS_SAVE(&Giant.mtx_object, Giant); \ for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++) \ mtx_unlock(&Giant) @@ -373,7 +369,7 @@ do { \ while (_giantcnt--) \ mtx_lock(&Giant); \ if (mtx_owned(&Giant)) \ - WITNESS_RESTORE(&Giant, Giant); \ + WITNESS_RESTORE(&Giant.mtx_object, Giant); \ } while (0) #define PARTIAL_PICKUP_GIANT() \ @@ -381,7 +377,7 @@ do { \ while (_giantcnt--) \ mtx_lock(&Giant); \ if (mtx_owned(&Giant)) \ - WITNESS_RESTORE(&Giant, Giant) + WITNESS_RESTORE(&Giant.mtx_object, Giant) /* * The INVARIANTS-enabled mtx_assert() functionality. @@ -405,58 +401,6 @@ do { \ #define mtx_assert(m, what) #endif /* INVARIANTS */ -#define MPASS(ex) MPASS4(ex, #ex, __FILE__, __LINE__) -#define MPASS2(ex, what) MPASS4(ex, what, __FILE__, __LINE__) -#define MPASS3(ex, file, line) MPASS4(ex, #ex, file, line) -#define MPASS4(ex, what, file, line) \ - KASSERT((ex), ("Assertion %s failed at %s:%d", what, file, line)) - -/* - * Exported WITNESS-enabled functions and corresponding wrapper macros. - */ -#ifdef WITNESS -void witness_save(struct mtx *, const char **, int *); -void witness_restore(struct mtx *, const char *, int); -void witness_enter(struct mtx *, int, const char *, int); -void witness_try_enter(struct mtx *, int, const char *, int); -void witness_exit(struct mtx *, int, const char *, int); -int witness_list(struct proc *); -int witness_sleep(int, struct mtx *, const char *, int); - -#define WITNESS_ENTER(m, t, f, l) \ - witness_enter((m), (t), (f), (l)) - -#define WITNESS_EXIT(m, t, f, l) \ - witness_exit((m), (t), (f), (l)) - -#define WITNESS_SLEEP(check, m) \ - witness_sleep(check, (m), __FILE__, __LINE__) - -#define WITNESS_SAVE_DECL(n) \ - const char * __CONCAT(n, __wf); \ - int __CONCAT(n, __wl) - -#define WITNESS_SAVE(m, n) \ - witness_save(m, &__CONCAT(n, __wf), &__CONCAT(n, __wl)) - -#define WITNESS_RESTORE(m, n) \ - witness_restore(m, __CONCAT(n, __wf), __CONCAT(n, __wl)) - -#else /* WITNESS */ -#define witness_enter(m, t, f, l) -#define witness_tryenter(m, t, f, l) -#define witness_exit(m, t, f, l) -#define witness_list(p) -#define witness_sleep(c, m, f, l) - -#define WITNESS_ENTER(m, t, f, l) -#define WITNESS_EXIT(m, t, f, l) -#define WITNESS_SLEEP(check, m) -#define WITNESS_SAVE_DECL(n) -#define WITNESS_SAVE(m, n) -#define WITNESS_RESTORE(m, n) -#endif /* WITNESS */ - #endif /* _KERNEL */ #endif /* !LOCORE */ #endif /* _SYS_MUTEX_H_ */ diff --git a/sys/sys/proc.h b/sys/sys/proc.h index c271303..34db50f 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -231,7 +231,7 @@ struct proc { int p_sig; /* (n) For core dump/debugger XXX. */ u_long p_code; /* (n) For core dump/debugger XXX. */ struct klist p_klist; /* (c) Knotes attached to this process. */ - LIST_HEAD(, mtx) p_heldmtx; /* (j) For debugging code. */ + struct lock_list_entry *p_sleeplocks; /* (k) Held sleep locks. */ struct mtx *p_blocked; /* (j) Mutex process is blocked on. */ const char *p_mtxname; /* (j) Name of mutex blocked on. */ LIST_HEAD(, mtx) p_contested; /* (j) Contested locks. */ diff --git a/sys/sys/sx.h b/sys/sys/sx.h index 2c5c981..3b4114d 100644 --- a/sys/sys/sx.h +++ b/sys/sys/sx.h @@ -31,12 +31,15 @@ #define _SYS_SX_H_ #ifndef LOCORE -#include <sys/mutex.h> -#include <sys/condvar.h> +#include <sys/lock.h> /* XXX */ +#include <sys/mutex.h> /* XXX */ +#include <sys/condvar.h> /* XXX */ + +struct lock_object; struct sx { + struct lock_object sx_object; /* Common lock properties. */ struct mtx sx_lock; /* General protection lock. */ - const char *sx_descr; /* sx lock description. */ int sx_cnt; /* -1: xlock, > 0: slock count. */ struct cv sx_shrd_cv; /* slock waiters. */ int sx_shrd_wcnt; /* Number of slock waiters. */ @@ -48,10 +51,15 @@ struct sx { #ifdef _KERNEL void sx_init(struct sx *sx, const char *description); void sx_destroy(struct sx *sx); -void sx_slock(struct sx *sx); -void sx_xlock(struct sx *sx); -void sx_sunlock(struct sx *sx); -void sx_xunlock(struct sx *sx); +void _sx_slock(struct sx *sx, const char *file, int line); +void _sx_xlock(struct sx *sx, const char *file, int line); +void _sx_sunlock(struct sx *sx, const char *file, int line); +void _sx_xunlock(struct sx *sx, const char *file, int line); + +#define sx_slock(sx) _sx_slock((sx), __FILE__, __LINE__) +#define sx_xlock(sx) _sx_xlock((sx), __FILE__, __LINE__) +#define sx_sunlock(sx) _sx_sunlock((sx), __FILE__, __LINE__) +#define sx_xunlock(sx) _sx_xunlock((sx), __FILE__, __LINE__) #ifdef INVARIANTS /* @@ -65,7 +73,7 @@ void sx_xunlock(struct sx *sx); } while (0) #define _SX_ASSERT_SLOCKED(sx) do { \ KASSERT(((sx)->sx_cnt > 0), ("%s: lacking slock %s\n", \ - __FUNCTION__, (sx)->sx_descr)); \ + __FUNCTION__, (sx)->sx_object.lo_name)); \ } while (0) /* @@ -79,7 +87,7 @@ void sx_xunlock(struct sx *sx); #define _SX_ASSERT_XLOCKED(sx) do { \ KASSERT(((sx)->sx_xholder == curproc), \ ("%s: thread %p lacking xlock %s\n", __FUNCTION__, \ - curproc, (sx)->sx_descr)); \ + curproc, (sx)->sx_object.lo_name)); \ } while (0) #else /* INVARIANTS */ |
