diff options
author | bmilekic <bmilekic@FreeBSD.org> | 2001-01-19 01:59:14 +0000 |
---|---|---|
committer | bmilekic <bmilekic@FreeBSD.org> | 2001-01-19 01:59:14 +0000 |
commit | 37decc93f5a2dfb49438d1e77245218780583ef6 (patch) | |
tree | 2e4b859fb1993ec2fe69e3e946d14a9fdc9cf441 /sys | |
parent | 5977e97236d57a51dc2ddf0634bb4a69d30dc36f (diff) | |
download | FreeBSD-src-37decc93f5a2dfb49438d1e77245218780583ef6.zip FreeBSD-src-37decc93f5a2dfb49438d1e77245218780583ef6.tar.gz |
Implement MTX_RECURSE flag for mtx_init().
All calls to mtx_init() for mutexes that recurse must now include
the MTX_RECURSE bit in the flag argument variable. This change is in
preparation for an upcoming (further) mutex API cleanup.
The witness code will call panic() if a lock is found to recurse but
the MTX_RECURSE bit was not set during the lock's initialization.
The old MTX_RECURSE "state" bit (in mtx_lock) has been renamed to
MTX_RECURSED, which is more appropriate given its meaning.
The following locks have been made "recursive," thus far:
eventhandler, Giant, callout, sched_lock, possibly some others declared
in the architecture-specific code, all of the network card driver locks
in pci/, as well as some other locks in dev/ stuff that I've found to
be recursive.
Reviewed by: jhb
Diffstat (limited to 'sys')
37 files changed, 226 insertions, 116 deletions
diff --git a/sys/alpha/alpha/machdep.c b/sys/alpha/alpha/machdep.c index 1976cd9..d220797 100644 --- a/sys/alpha/alpha/machdep.c +++ b/sys/alpha/alpha/machdep.c @@ -390,7 +390,7 @@ again: TAILQ_INIT(&callwheel[i]); } - mtx_init(&callout_lock, "callout", MTX_SPIN); + mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE); #if defined(USERCONFIG) #if defined(USERCONFIG_BOOT) @@ -1003,8 +1003,8 @@ alpha_init(pfn, ptb, bim, bip, biv) /* * Initialise mutexes. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); /* * Look at arguments passed to us and compute boothowto. diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index a6d27ea..225ec00 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -422,7 +422,7 @@ again: TAILQ_INIT(&callwheel[i]); } - mtx_init(&callout_lock, "callout", MTX_SPIN); + mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE); #if defined(USERCONFIG) userconfig(); @@ -441,7 +441,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); #ifdef SMP /* @@ -1939,7 +1939,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -1954,7 +1954,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/dev/an/if_an.c b/sys/dev/an/if_an.c index 60c7892..9fd258a 100644 --- a/sys/dev/an/if_an.c +++ b/sys/dev/an/if_an.c @@ -288,7 +288,8 @@ int an_attach(sc, unit, flags) { struct ifnet *ifp = &sc->arpcom.ac_if; - mtx_init(&sc->an_mtx, device_get_nameunit(sc->an_dev), MTX_DEF); + mtx_init(&sc->an_mtx, device_get_nameunit(sc->an_dev), MTX_DEF | + MTX_RECURSE); AN_LOCK(sc); sc->an_gone = 0; diff --git a/sys/dev/dc/if_dc.c b/sys/dev/dc/if_dc.c index 63e7adb..3b5e91a 100644 --- a/sys/dev/dc/if_dc.c +++ b/sys/dev/dc/if_dc.c @@ -1762,7 +1762,7 @@ static int dc_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct dc_softc)); - mtx_init(&sc->dc_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->dc_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); DC_LOCK(sc); /* diff --git a/sys/dev/fxp/if_fxp.c b/sys/dev/fxp/if_fxp.c index b6f66d1..e76640e 100644 --- a/sys/dev/fxp/if_fxp.c +++ b/sys/dev/fxp/if_fxp.c @@ -316,7 +316,7 @@ fxp_attach(device_t dev) u_long val; int rid; - mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); callout_handle_init(&sc->stat_ch); FXP_LOCK(sc); diff --git a/sys/dev/sf/if_sf.c b/sys/dev/sf/if_sf.c index aa33e7d..d03934f 100644 --- a/sys/dev/sf/if_sf.c +++ b/sys/dev/sf/if_sf.c @@ -679,7 +679,7 @@ static int sf_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct sf_softc)); - mtx_init(&sc->sf_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->sf_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); SF_LOCK(sc); /* * Handle power management nonsense. diff --git a/sys/dev/sk/if_sk.c b/sys/dev/sk/if_sk.c index 143c4e4..f09df99 100644 --- a/sys/dev/sk/if_sk.c +++ b/sys/dev/sk/if_sk.c @@ -1194,7 +1194,7 @@ static int sk_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct sk_softc)); - mtx_init(&sc->sk_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->sk_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); SK_LOCK(sc); /* diff --git a/sys/dev/ti/if_ti.c b/sys/dev/ti/if_ti.c index 7f6660f..9b29e09 100644 --- a/sys/dev/ti/if_ti.c +++ b/sys/dev/ti/if_ti.c @@ -1476,7 +1476,7 @@ static int ti_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct ti_softc)); - mtx_init(&sc->ti_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->ti_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); TI_LOCK(sc); /* diff --git a/sys/dev/usb/if_aue.c b/sys/dev/usb/if_aue.c index a3b844b..9804573 100644 --- a/sys/dev/usb/if_aue.c +++ b/sys/dev/usb/if_aue.c @@ -681,7 +681,8 @@ USB_ATTACH(aue) } } - mtx_init(&sc->aue_mtx, device_get_nameunit(self), MTX_DEF); + mtx_init(&sc->aue_mtx, device_get_nameunit(self), MTX_DEF | + MTX_RECURSE); AUE_LOCK(sc); /* Reset the adapter. */ diff --git a/sys/dev/usb/if_cue.c b/sys/dev/usb/if_cue.c index c62a0d7..6b71b48 100644 --- a/sys/dev/usb/if_cue.c +++ b/sys/dev/usb/if_cue.c @@ -504,7 +504,8 @@ USB_ATTACH(cue) } } - mtx_init(&sc->cue_mtx, device_get_nameunit(self), MTX_DEF); + mtx_init(&sc->cue_mtx, device_get_nameunit(self), MTX_DEF | + MTX_RECURSE); CUE_LOCK(sc); #ifdef notdef diff --git a/sys/dev/usb/if_kue.c b/sys/dev/usb/if_kue.c index f3a5581..4d89b70 100644 --- a/sys/dev/usb/if_kue.c +++ b/sys/dev/usb/if_kue.c @@ -450,7 +450,8 @@ USB_ATTACH(kue) } } - mtx_init(&sc->kue_mtx, device_get_nameunit(self), MTX_DEF); + mtx_init(&sc->kue_mtx, device_get_nameunit(self), MTX_DEF | + MTX_RECURSE); KUE_LOCK(sc); /* Load the firmware into the NIC. */ diff --git a/sys/dev/vr/if_vr.c b/sys/dev/vr/if_vr.c index a1fa0c0..7c3729f 100644 --- a/sys/dev/vr/if_vr.c +++ b/sys/dev/vr/if_vr.c @@ -643,7 +643,7 @@ static int vr_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct vr_softc *)); - mtx_init(&sc->vr_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->vr_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); VR_LOCK(sc); /* diff --git a/sys/dev/wi/if_wi.c b/sys/dev/wi/if_wi.c index 70330f1..0b2415d 100644 --- a/sys/dev/wi/if_wi.c +++ b/sys/dev/wi/if_wi.c @@ -304,7 +304,7 @@ static int wi_pccard_attach(device_t dev) return (error); } - mtx_init(&sc->wi_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->wi_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); WI_LOCK(sc); /* Reset the NIC. */ diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index a6d27ea..225ec00 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -422,7 +422,7 @@ again: TAILQ_INIT(&callwheel[i]); } - mtx_init(&callout_lock, "callout", MTX_SPIN); + mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE); #if defined(USERCONFIG) userconfig(); @@ -441,7 +441,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); #ifdef SMP /* @@ -1939,7 +1939,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -1954,7 +1954,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/i386/isa/if_el.c b/sys/i386/isa/if_el.c index b77edc1..ed53636 100644 --- a/sys/i386/isa/if_el.c +++ b/sys/i386/isa/if_el.c @@ -149,7 +149,7 @@ el_probe(device_t dev) sc->el_btag = rman_get_bustag(sc->el_res); sc->el_bhandle = rman_get_bushandle(sc->el_res); mtx_init(&sc->el_mtx, - device_get_nameunit(dev), MTX_DEF); + device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); EL_LOCK(sc); /* Now attempt to grab the station address from the PROM diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index a4ff91a..30f5ef6 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -319,7 +319,7 @@ again: TAILQ_INIT(&callwheel[i]); } - mtx_init(&callout_lock, "callout", MTX_SPIN); + mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE); #if defined(USERCONFIG) #if defined(USERCONFIG_BOOT) @@ -590,8 +590,8 @@ ia64_init() /* * Initialise mutexes. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); #if 0 /* diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index a79299c..da8f197 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -259,7 +259,7 @@ mtx_enter_hard(struct mtx *m, int type, int saveintr) case MTX_DEF: if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { m->mtx_recurse++; - atomic_set_ptr(&m->mtx_lock, MTX_RECURSE); + atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); if ((type & MTX_QUIET) == 0) CTR1(KTR_LOCK, "mtx_enter: 0x%p recurse", m); return; @@ -434,9 +434,9 @@ mtx_exit_hard(struct mtx *m, int type) switch (type) { case MTX_DEF: case MTX_DEF | MTX_NOSWITCH: - if (m->mtx_recurse != 0) { + if (mtx_recursed(m)) { if (--(m->mtx_recurse) == 0) - atomic_clear_ptr(&m->mtx_lock, MTX_RECURSE); + atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); if ((type & MTX_QUIET) == 0) CTR1(KTR_LOCK, "mtx_exit: 0x%p unrecurse", m); return; @@ -501,7 +501,7 @@ mtx_exit_hard(struct mtx *m, int type) break; case MTX_SPIN: case MTX_SPIN | MTX_FIRST: - if (m->mtx_recurse != 0) { + if (mtx_recursed(m)) { m->mtx_recurse--; return; } @@ -515,7 +515,7 @@ mtx_exit_hard(struct mtx *m, int type) } break; case MTX_SPIN | MTX_TOPHALF: - if (m->mtx_recurse != 0) { + if (mtx_recursed(m)) { m->mtx_recurse--; return; } @@ -655,7 +655,7 @@ mtx_destroy(struct mtx *m) if (!mtx_owned(m)) { MPASS(m->mtx_lock == MTX_UNOWNED); } else { - MPASS((m->mtx_lock & (MTX_RECURSE|MTX_CONTESTED)) == 0); + MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); } mtx_validate(m, MV_DESTROY); /* diagnostic */ #endif @@ -701,8 +701,9 @@ struct witness { u_char w_Giant_squawked:1; u_char w_other_squawked:1; u_char w_same_squawked:1; - u_char w_sleep:1; - u_char w_spin:1; /* this is a spin mutex */ + u_char w_sleep:1; /* MTX_DEF type mutex. */ + u_char w_spin:1; /* MTX_SPIN type mutex. */ + u_char w_recurse:1; /* MTX_RECURSE mutex option. */ u_int w_level; struct witness *w_children[WITNESS_NCHILDREN]; }; @@ -838,11 +839,16 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) p = CURPROC; if (flags & MTX_SPIN) { - if (!w->w_spin) + if (!(w->w_spin)) panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @" " %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_enter: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); i = PCPU_GET(witness_spin_check); if (i != 0 && w->w_level < i) { @@ -864,8 +870,13 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_enter: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } if (witness_dead) goto out; if (cold) @@ -971,11 +982,16 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) w = m->mtx_witness; if (flags & MTX_SPIN) { - if (!w->w_spin) + if (!(w->w_spin)) panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @" " %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_exit: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) & ~w->w_level); @@ -986,8 +1002,13 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_exit: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold) panic("switchable mtx_exit() of %s when not legal @ %s:%d", @@ -1005,12 +1026,17 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) if (panicstr) return; if (flags & MTX_SPIN) { - if (!w->w_spin) + if (!(w->w_spin)) panic("mutex_try_enter: " "MTX_SPIN on MTX_DEF mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_try_enter: recursion on" + " non-recursive mutex %s @ %s:%d", + m->mtx_description, file, line); return; + } mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) | w->w_level); @@ -1026,9 +1052,13 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_try_enter: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, file, + line); return; - + } w->w_file = file; w->w_line = line; m->mtx_line = line; @@ -1156,7 +1186,12 @@ enroll(const char *description, int flag) if (*order == NULL) panic("spin lock %s not in order list", description); w->w_level = i; - } + } else + w->w_sleep = 1; + + if (flag & MTX_RECURSE) + w->w_recurse = 1; + return (w); } @@ -1270,7 +1305,7 @@ witness_levelall (void) struct witness *w, *w1; for (w = w_all; w; w = w->w_next) - if (!w->w_spin) + if (!(w->w_spin)) w->w_level = 0; for (w = w_all; w; w = w->w_next) { if (w->w_spin) diff --git a/sys/kern/subr_eventhandler.c b/sys/kern/subr_eventhandler.c index ca35a05..12f5e3d 100644 --- a/sys/kern/subr_eventhandler.c +++ b/sys/kern/subr_eventhandler.c @@ -53,7 +53,7 @@ static void eventhandler_init(void *dummy __unused) { TAILQ_INIT(&eventhandler_lists); - mtx_init(&eventhandler_mutex, "eventhandler", MTX_DEF); + mtx_init(&eventhandler_mutex, "eventhandler", MTX_DEF | MTX_RECURSE); eventhandler_lists_initted = 1; } SYSINIT(eventhandlers, SI_SUB_EVENTHANDLER, SI_ORDER_FIRST, eventhandler_init, diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c index a79299c..da8f197 100644 --- a/sys/kern/subr_turnstile.c +++ b/sys/kern/subr_turnstile.c @@ -259,7 +259,7 @@ mtx_enter_hard(struct mtx *m, int type, int saveintr) case MTX_DEF: if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { m->mtx_recurse++; - atomic_set_ptr(&m->mtx_lock, MTX_RECURSE); + atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); if ((type & MTX_QUIET) == 0) CTR1(KTR_LOCK, "mtx_enter: 0x%p recurse", m); return; @@ -434,9 +434,9 @@ mtx_exit_hard(struct mtx *m, int type) switch (type) { case MTX_DEF: case MTX_DEF | MTX_NOSWITCH: - if (m->mtx_recurse != 0) { + if (mtx_recursed(m)) { if (--(m->mtx_recurse) == 0) - atomic_clear_ptr(&m->mtx_lock, MTX_RECURSE); + atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); if ((type & MTX_QUIET) == 0) CTR1(KTR_LOCK, "mtx_exit: 0x%p unrecurse", m); return; @@ -501,7 +501,7 @@ mtx_exit_hard(struct mtx *m, int type) break; case MTX_SPIN: case MTX_SPIN | MTX_FIRST: - if (m->mtx_recurse != 0) { + if (mtx_recursed(m)) { m->mtx_recurse--; return; } @@ -515,7 +515,7 @@ mtx_exit_hard(struct mtx *m, int type) } break; case MTX_SPIN | MTX_TOPHALF: - if (m->mtx_recurse != 0) { + if (mtx_recursed(m)) { m->mtx_recurse--; return; } @@ -655,7 +655,7 @@ mtx_destroy(struct mtx *m) if (!mtx_owned(m)) { MPASS(m->mtx_lock == MTX_UNOWNED); } else { - MPASS((m->mtx_lock & (MTX_RECURSE|MTX_CONTESTED)) == 0); + MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); } mtx_validate(m, MV_DESTROY); /* diagnostic */ #endif @@ -701,8 +701,9 @@ struct witness { u_char w_Giant_squawked:1; u_char w_other_squawked:1; u_char w_same_squawked:1; - u_char w_sleep:1; - u_char w_spin:1; /* this is a spin mutex */ + u_char w_sleep:1; /* MTX_DEF type mutex. */ + u_char w_spin:1; /* MTX_SPIN type mutex. */ + u_char w_recurse:1; /* MTX_RECURSE mutex option. */ u_int w_level; struct witness *w_children[WITNESS_NCHILDREN]; }; @@ -838,11 +839,16 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) p = CURPROC; if (flags & MTX_SPIN) { - if (!w->w_spin) + if (!(w->w_spin)) panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @" " %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_enter: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); i = PCPU_GET(witness_spin_check); if (i != 0 && w->w_level < i) { @@ -864,8 +870,13 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_enter: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } if (witness_dead) goto out; if (cold) @@ -971,11 +982,16 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) w = m->mtx_witness; if (flags & MTX_SPIN) { - if (!w->w_spin) + if (!(w->w_spin)) panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @" " %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_exit: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) & ~w->w_level); @@ -986,8 +1002,13 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_exit: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold) panic("switchable mtx_exit() of %s when not legal @ %s:%d", @@ -1005,12 +1026,17 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) if (panicstr) return; if (flags & MTX_SPIN) { - if (!w->w_spin) + if (!(w->w_spin)) panic("mutex_try_enter: " "MTX_SPIN on MTX_DEF mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_try_enter: recursion on" + " non-recursive mutex %s @ %s:%d", + m->mtx_description, file, line); return; + } mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) | w->w_level); @@ -1026,9 +1052,13 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_try_enter: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, file, + line); return; - + } w->w_file = file; w->w_line = line; m->mtx_line = line; @@ -1156,7 +1186,12 @@ enroll(const char *description, int flag) if (*order == NULL) panic("spin lock %s not in order list", description); w->w_level = i; - } + } else + w->w_sleep = 1; + + if (flag & MTX_RECURSE) + w->w_recurse = 1; + return (w); } @@ -1270,7 +1305,7 @@ witness_levelall (void) struct witness *w, *w1; for (w = w_all; w; w = w->w_next) - if (!w->w_spin) + if (!(w->w_spin)) w->w_level = 0; for (w = w_all; w; w = w->w_next) { if (w->w_spin) diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index a79299c..da8f197 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -259,7 +259,7 @@ mtx_enter_hard(struct mtx *m, int type, int saveintr) case MTX_DEF: if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)p) { m->mtx_recurse++; - atomic_set_ptr(&m->mtx_lock, MTX_RECURSE); + atomic_set_ptr(&m->mtx_lock, MTX_RECURSED); if ((type & MTX_QUIET) == 0) CTR1(KTR_LOCK, "mtx_enter: 0x%p recurse", m); return; @@ -434,9 +434,9 @@ mtx_exit_hard(struct mtx *m, int type) switch (type) { case MTX_DEF: case MTX_DEF | MTX_NOSWITCH: - if (m->mtx_recurse != 0) { + if (mtx_recursed(m)) { if (--(m->mtx_recurse) == 0) - atomic_clear_ptr(&m->mtx_lock, MTX_RECURSE); + atomic_clear_ptr(&m->mtx_lock, MTX_RECURSED); if ((type & MTX_QUIET) == 0) CTR1(KTR_LOCK, "mtx_exit: 0x%p unrecurse", m); return; @@ -501,7 +501,7 @@ mtx_exit_hard(struct mtx *m, int type) break; case MTX_SPIN: case MTX_SPIN | MTX_FIRST: - if (m->mtx_recurse != 0) { + if (mtx_recursed(m)) { m->mtx_recurse--; return; } @@ -515,7 +515,7 @@ mtx_exit_hard(struct mtx *m, int type) } break; case MTX_SPIN | MTX_TOPHALF: - if (m->mtx_recurse != 0) { + if (mtx_recursed(m)) { m->mtx_recurse--; return; } @@ -655,7 +655,7 @@ mtx_destroy(struct mtx *m) if (!mtx_owned(m)) { MPASS(m->mtx_lock == MTX_UNOWNED); } else { - MPASS((m->mtx_lock & (MTX_RECURSE|MTX_CONTESTED)) == 0); + MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); } mtx_validate(m, MV_DESTROY); /* diagnostic */ #endif @@ -701,8 +701,9 @@ struct witness { u_char w_Giant_squawked:1; u_char w_other_squawked:1; u_char w_same_squawked:1; - u_char w_sleep:1; - u_char w_spin:1; /* this is a spin mutex */ + u_char w_sleep:1; /* MTX_DEF type mutex. */ + u_char w_spin:1; /* MTX_SPIN type mutex. */ + u_char w_recurse:1; /* MTX_RECURSE mutex option. */ u_int w_level; struct witness *w_children[WITNESS_NCHILDREN]; }; @@ -838,11 +839,16 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) p = CURPROC; if (flags & MTX_SPIN) { - if (!w->w_spin) + if (!(w->w_spin)) panic("mutex_enter: MTX_SPIN on MTX_DEF mutex %s @" " %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_enter: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); i = PCPU_GET(witness_spin_check); if (i != 0 && w->w_level < i) { @@ -864,8 +870,13 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) panic("mutex_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_enter: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } if (witness_dead) goto out; if (cold) @@ -971,11 +982,16 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) w = m->mtx_witness; if (flags & MTX_SPIN) { - if (!w->w_spin) + if (!(w->w_spin)) panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @" " %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_exit: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) & ~w->w_level); @@ -986,8 +1002,13 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_exit: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, + file, line); return; + } if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold) panic("switchable mtx_exit() of %s when not legal @ %s:%d", @@ -1005,12 +1026,17 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) if (panicstr) return; if (flags & MTX_SPIN) { - if (!w->w_spin) + if (!(w->w_spin)) panic("mutex_try_enter: " "MTX_SPIN on MTX_DEF mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_try_enter: recursion on" + " non-recursive mutex %s @ %s:%d", + m->mtx_description, file, line); return; + } mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET); PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) | w->w_level); @@ -1026,9 +1052,13 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line) panic("mutex_try_enter: MTX_DEF on MTX_SPIN mutex %s @ %s:%d", m->mtx_description, file, line); - if (m->mtx_recurse != 0) + if (mtx_recursed(m)) { + if (!(w->w_recurse)) + panic("mutex_try_enter: recursion on non-recursive" + " mutex %s @ %s:%d", m->mtx_description, file, + line); return; - + } w->w_file = file; w->w_line = line; m->mtx_line = line; @@ -1156,7 +1186,12 @@ enroll(const char *description, int flag) if (*order == NULL) panic("spin lock %s not in order list", description); w->w_level = i; - } + } else + w->w_sleep = 1; + + if (flag & MTX_RECURSE) + w->w_recurse = 1; + return (w); } @@ -1270,7 +1305,7 @@ witness_levelall (void) struct witness *w, *w1; for (w = w_all; w; w = w->w_next) - if (!w->w_spin) + if (!(w->w_spin)) w->w_level = 0; for (w = w_all; w; w = w->w_next) { if (w->w_spin) diff --git a/sys/pc98/i386/machdep.c b/sys/pc98/i386/machdep.c index 2db7319..efe0efd 100644 --- a/sys/pc98/i386/machdep.c +++ b/sys/pc98/i386/machdep.c @@ -436,7 +436,7 @@ again: TAILQ_INIT(&callwheel[i]); } - mtx_init(&callout_lock, "callout", MTX_SPIN); + mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE); #if defined(USERCONFIG) userconfig(); @@ -455,7 +455,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); #ifdef SMP /* @@ -2249,7 +2249,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -2264,7 +2264,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/pc98/pc98/machdep.c b/sys/pc98/pc98/machdep.c index 2db7319..efe0efd 100644 --- a/sys/pc98/pc98/machdep.c +++ b/sys/pc98/pc98/machdep.c @@ -436,7 +436,7 @@ again: TAILQ_INIT(&callwheel[i]); } - mtx_init(&callout_lock, "callout", MTX_SPIN); + mtx_init(&callout_lock, "callout", MTX_SPIN | MTX_RECURSE); #if defined(USERCONFIG) userconfig(); @@ -455,7 +455,7 @@ again: SLIST_INIT(&cpuhead); SLIST_INSERT_HEAD(&cpuhead, GLOBALDATA, gd_allcpu); - mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD); + mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_COLD | MTX_RECURSE); #ifdef SMP /* @@ -2249,7 +2249,7 @@ init386(first) /* * We need this mutex before the console probe. */ - mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD); + mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_COLD | MTX_RECURSE); /* * Initialize the console before we print anything out. @@ -2264,7 +2264,7 @@ init386(first) /* * Giant is used early for at least debugger traps and unexpected traps. */ - mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD); + mtx_init(&Giant, "Giant", MTX_DEF | MTX_COLD | MTX_RECURSE); #ifdef DDB kdb_init(); diff --git a/sys/pci/if_dc.c b/sys/pci/if_dc.c index 63e7adb..3b5e91a 100644 --- a/sys/pci/if_dc.c +++ b/sys/pci/if_dc.c @@ -1762,7 +1762,7 @@ static int dc_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct dc_softc)); - mtx_init(&sc->dc_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->dc_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); DC_LOCK(sc); /* diff --git a/sys/pci/if_fxp.c b/sys/pci/if_fxp.c index b6f66d1..e76640e 100644 --- a/sys/pci/if_fxp.c +++ b/sys/pci/if_fxp.c @@ -316,7 +316,7 @@ fxp_attach(device_t dev) u_long val; int rid; - mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); callout_handle_init(&sc->stat_ch); FXP_LOCK(sc); diff --git a/sys/pci/if_pcn.c b/sys/pci/if_pcn.c index c5139ef..af893fd 100644 --- a/sys/pci/if_pcn.c +++ b/sys/pci/if_pcn.c @@ -497,7 +497,7 @@ static int pcn_attach(dev) unit = device_get_unit(dev); /* Initialize our mutex. */ - mtx_init(&sc->pcn_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->pcn_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); PCN_LOCK(sc); /* diff --git a/sys/pci/if_rl.c b/sys/pci/if_rl.c index 3c815b6..5463f07 100644 --- a/sys/pci/if_rl.c +++ b/sys/pci/if_rl.c @@ -802,7 +802,7 @@ static int rl_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct rl_softc)); - mtx_init(&sc->rl_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->rl_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); RL_LOCK(sc); /* diff --git a/sys/pci/if_sf.c b/sys/pci/if_sf.c index aa33e7d..d03934f 100644 --- a/sys/pci/if_sf.c +++ b/sys/pci/if_sf.c @@ -679,7 +679,7 @@ static int sf_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct sf_softc)); - mtx_init(&sc->sf_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->sf_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); SF_LOCK(sc); /* * Handle power management nonsense. diff --git a/sys/pci/if_sis.c b/sys/pci/if_sis.c index 8f9b4d6..d271e47 100644 --- a/sys/pci/if_sis.c +++ b/sys/pci/if_sis.c @@ -638,7 +638,7 @@ static int sis_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct sis_softc)); - mtx_init(&sc->sis_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->sis_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); SIS_LOCK(sc); if (pci_get_device(dev) == SIS_DEVICEID_900) diff --git a/sys/pci/if_sk.c b/sys/pci/if_sk.c index 143c4e4..f09df99 100644 --- a/sys/pci/if_sk.c +++ b/sys/pci/if_sk.c @@ -1194,7 +1194,7 @@ static int sk_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct sk_softc)); - mtx_init(&sc->sk_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->sk_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); SK_LOCK(sc); /* diff --git a/sys/pci/if_ste.c b/sys/pci/if_ste.c index 7b099e8..017ac06 100644 --- a/sys/pci/if_ste.c +++ b/sys/pci/if_ste.c @@ -917,7 +917,7 @@ static int ste_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct ste_softc)); - mtx_init(&sc->ste_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->ste_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); STE_LOCK(sc); /* diff --git a/sys/pci/if_ti.c b/sys/pci/if_ti.c index 7f6660f..9b29e09 100644 --- a/sys/pci/if_ti.c +++ b/sys/pci/if_ti.c @@ -1476,7 +1476,7 @@ static int ti_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct ti_softc)); - mtx_init(&sc->ti_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->ti_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); TI_LOCK(sc); /* diff --git a/sys/pci/if_tl.c b/sys/pci/if_tl.c index 7bf6901..ccc7069 100644 --- a/sys/pci/if_tl.c +++ b/sys/pci/if_tl.c @@ -1132,7 +1132,7 @@ static int tl_attach(dev) goto fail; } - mtx_init(&sc->tl_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->tl_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); TL_LOCK(sc); /* diff --git a/sys/pci/if_vr.c b/sys/pci/if_vr.c index a1fa0c0..7c3729f 100644 --- a/sys/pci/if_vr.c +++ b/sys/pci/if_vr.c @@ -643,7 +643,7 @@ static int vr_attach(dev) unit = device_get_unit(dev); bzero(sc, sizeof(struct vr_softc *)); - mtx_init(&sc->vr_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->vr_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); VR_LOCK(sc); /* diff --git a/sys/pci/if_wb.c b/sys/pci/if_wb.c index 19893cf..fecc20d 100644 --- a/sys/pci/if_wb.c +++ b/sys/pci/if_wb.c @@ -817,7 +817,7 @@ static int wb_attach(dev) sc = device_get_softc(dev); unit = device_get_unit(dev); - mtx_init(&sc->wb_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->wb_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); WB_LOCK(sc); /* diff --git a/sys/pci/if_wx.c b/sys/pci/if_wx.c index 51ff687..b1151e4 100644 --- a/sys/pci/if_wx.c +++ b/sys/pci/if_wx.c @@ -597,7 +597,7 @@ wx_attach(device_t dev) } #ifdef SMPNG - mtx_init(&sc->wx_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->wx_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); #endif WX_LOCK(sc); diff --git a/sys/pci/if_xl.c b/sys/pci/if_xl.c index e7a3174..c6abf6f 100644 --- a/sys/pci/if_xl.c +++ b/sys/pci/if_xl.c @@ -1239,7 +1239,7 @@ static int xl_attach(dev) sc = device_get_softc(dev); unit = device_get_unit(dev); - mtx_init(&sc->xl_mtx, device_get_nameunit(dev), MTX_DEF); + mtx_init(&sc->xl_mtx, device_get_nameunit(dev), MTX_DEF | MTX_RECURSE); XL_LOCK(sc); sc->xl_flags = 0; diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index cdb9939..dff8098 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -67,7 +67,8 @@ #define MTX_SPIN 0x1 /* Spin only lock */ /* Options */ -#define MTX_RLIKELY 0x4 /* (opt) Recursion likely */ +#define MTX_RECURSE 0x2 /* Recursive lock (for mtx_init) */ +#define MTX_RLIKELY 0x4 /* Recursion likely */ #define MTX_NORECURSE 0x8 /* No recursion possible */ #define MTX_NOSPIN 0x10 /* Don't spin before sleeping */ #define MTX_NOSWITCH 0x20 /* Do not switch on release */ @@ -80,9 +81,9 @@ #define MTX_HARDOPTS (MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH) /* Flags/value used in mtx_lock */ -#define MTX_RECURSE 0x01 /* (non-spin) lock held recursively */ +#define MTX_RECURSED 0x01 /* (non-spin) lock held recursively */ #define MTX_CONTESTED 0x02 /* (non-spin) lock contested */ -#define MTX_FLAGMASK ~(MTX_RECURSE | MTX_CONTESTED) +#define MTX_FLAGMASK ~(MTX_RECURSED | MTX_CONTESTED) #define MTX_UNOWNED 0x8 /* Cookie for free mutex */ #endif /* _KERNEL */ @@ -360,7 +361,7 @@ void witness_restore(struct mtx *, const char *, int); if (((mp)->mtx_lock & MTX_FLAGMASK) != ((uintptr_t)(tid)))\ mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0); \ else { \ - atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSE); \ + atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSED); \ (mp)->mtx_recurse++; \ } \ } \ @@ -408,10 +409,10 @@ void witness_restore(struct mtx *, const char *, int); */ #define _exitlock(mp, tid, type) do { \ if (!_release_lock(mp, tid)) { \ - if ((mp)->mtx_lock & MTX_RECURSE) { \ + if ((mp)->mtx_lock & MTX_RECURSED) { \ if (--((mp)->mtx_recurse) == 0) \ atomic_clear_ptr(&(mp)->mtx_lock, \ - MTX_RECURSE); \ + MTX_RECURSED); \ } else { \ mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ } \ @@ -422,7 +423,7 @@ void witness_restore(struct mtx *, const char *, int); #ifndef _exitlock_spin /* Release a spin lock (with possible recursion). */ #define _exitlock_spin(mp) do { \ - if ((mp)->mtx_recurse == 0) { \ + if (!mtx_recursed((mp))) { \ int _mtx_intr = (mp)->mtx_saveintr; \ \ _release_lock_quick(mp); \ |