diff options
| -rw-r--r-- | sys/conf/files | 1 | ||||
| -rw-r--r-- | sys/kern/kern_mutex.c | 84 | ||||
| -rw-r--r-- | sys/kern/kern_sx.c | 21 | ||||
| -rw-r--r-- | sys/kern/subr_lock.c | 113 | ||||
| -rw-r--r-- | sys/kern/subr_witness.c | 8 | ||||
| -rw-r--r-- | sys/sys/lock.h | 14 | ||||
| -rw-r--r-- | sys/sys/mutex.h | 2 |
7 files changed, 155 insertions, 88 deletions
diff --git a/sys/conf/files b/sys/conf/files index 246c25d..6a7497c 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -1299,6 +1299,7 @@ kern/subr_eventhandler.c standard kern/subr_hints.c standard kern/subr_kdb.c standard kern/subr_kobj.c standard +kern/subr_lock.c standard kern/subr_log.c standard kern/subr_mbpool.c optional libmbpool kern/subr_mchain.c optional libmchain diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index 37370ca..d524127 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -278,7 +278,7 @@ _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line) { MPASS(curthread != NULL); - KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SLEEP_MUTEX, + KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, ("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, @@ -303,7 +303,7 @@ _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line) { MPASS(curthread != NULL); - KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SLEEP_MUTEX, + KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, ("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); @@ -382,7 +382,7 @@ _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line) { MPASS(curthread != NULL); - KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SPIN_MUTEX, + KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin, ("mtx_lock_spin() of sleep mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, @@ -398,7 +398,7 @@ _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line) { MPASS(curthread != NULL); - KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SPIN_MUTEX, + KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin, ("mtx_unlock_spin() of sleep mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line); @@ -419,7 +419,7 @@ _mtx_trylock(struct mtx *m, int opts, const char *file, int line) int rval; MPASS(curthread != NULL); - KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SLEEP_MUTEX, + KASSERT(LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_sleep, ("mtx_trylock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name, file, line)); @@ -830,7 +830,8 @@ mtx_sysinit(void *arg) void mtx_init(struct mtx *m, const char *name, const char *type, int opts) { - struct lock_object *lock; + struct lock_class *class; + int flags; MPASS((opts & ~(MTX_SPIN | MTX_QUIET | MTX_RECURSE | MTX_NOWITNESS | MTX_DUPOK)) == 0); @@ -840,30 +841,33 @@ mtx_init(struct mtx *m, const char *name, const char *type, int opts) mtx_validate(m); #endif - lock = &m->mtx_object; - KASSERT((lock->lo_flags & LO_INITIALIZED) == 0, - ("mutex \"%s\" %p already initialized", name, m)); - bzero(m, sizeof(*m)); + /* Determine lock class and lock flags. */ if (opts & MTX_SPIN) - lock->lo_flags = LOCK_CLASS_SPIN_MUTEX << LO_CLASSSHIFT; + class = &lock_class_mtx_spin; else - lock->lo_flags = LOCK_CLASS_SLEEP_MUTEX << LO_CLASSSHIFT; - lock->lo_name = name; - lock->lo_type = type != NULL ? type : name; + class = &lock_class_mtx_sleep; + flags = 0; if (opts & MTX_QUIET) - lock->lo_flags |= LO_QUIET; + flags |= LO_QUIET; if (opts & MTX_RECURSE) - lock->lo_flags |= LO_RECURSABLE; + flags |= LO_RECURSABLE; if ((opts & MTX_NOWITNESS) == 0) - lock->lo_flags |= LO_WITNESS; + flags |= LO_WITNESS; if (opts & MTX_DUPOK) - lock->lo_flags |= LO_DUPOK; + flags |= LO_DUPOK; + /* Initialize mutex. */ m->mtx_lock = MTX_UNOWNED; + m->mtx_recurse = 0; +#ifdef MUTEX_PROFILING + m->mtx_acqtime = 0; + m->mtx_filename = NULL; + m->mtx_lineno = 0; + m->mtx_contest_holding = 0; + m->mtx_contest_locking = 0; +#endif - LOCK_LOG_INIT(lock, opts); - - WITNESS_INIT(lock); + lock_init(&m->mtx_object, class, name, type, flags); } /* @@ -876,15 +880,13 @@ void mtx_destroy(struct mtx *m) { - LOCK_LOG_DESTROY(&m->mtx_object, 0); - if (!mtx_owned(m)) MPASS(mtx_unowned(m)); else { MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0); /* Perform the non-mtx related part of mtx_unlock_spin(). */ - if (LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SPIN_MUTEX) + if (LOCK_CLASS(&m->mtx_object) == &lock_class_mtx_spin) spinlock_exit(); /* Tell witness this isn't locked to make it happy. */ @@ -892,7 +894,7 @@ mtx_destroy(struct mtx *m) __LINE__); } - WITNESS_DESTROY(&m->mtx_object); + lock_destroy(&m->mtx_object); } /* @@ -917,37 +919,7 @@ mutex_init(void) mtx_lock(&Giant); } -#if LOCK_DEBUG > 0 || defined(DDB) -/* XXX: This is not mutex-specific. */ -struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = { - &lock_class_mtx_spin, - &lock_class_mtx_sleep, - &lock_class_sx, -}; -#endif - #ifdef DDB -/* XXX: This function is not mutex-specific. */ -DB_SHOW_COMMAND(lock, db_show_lock) -{ - struct lock_object *lock; - struct lock_class *class; - - if (!have_addr) - return; - lock = (struct lock_object *)addr; - if (LO_CLASSINDEX(lock) > LOCK_CLASS_MAX) { - db_printf("Unknown lock class: %d\n", LO_CLASSINDEX(lock)); - return; - } - class = LOCK_CLASS(lock); - db_printf(" class: %s\n", class->lc_name); - db_printf(" name: %s\n", lock->lo_name); - if (lock->lo_type && lock->lo_type != lock->lo_name) - db_printf(" type: %s\n", lock->lo_type); - class->lc_ddb_show(lock); -} - void db_show_mtx(struct lock_object *lock) { @@ -957,7 +929,7 @@ db_show_mtx(struct lock_object *lock) m = (struct mtx *)lock; db_printf(" flags: {"); - if (LO_CLASSINDEX(lock) == LOCK_CLASS_SPIN_MUTEX) + if (LOCK_CLASS(lock) == &lock_class_mtx_spin) db_printf("SPIN"); else db_printf("DEF"); diff --git a/sys/kern/kern_sx.c b/sys/kern/kern_sx.c index d24e4de..a3a7da8 100644 --- a/sys/kern/kern_sx.c +++ b/sys/kern/kern_sx.c @@ -77,16 +77,7 @@ sx_sysinit(void *arg) void sx_init(struct sx *sx, const char *description) { - struct lock_object *lock; - - lock = &sx->sx_object; - KASSERT((lock->lo_flags & LO_INITIALIZED) == 0, - ("sx lock %s %p already initialized", description, sx)); - bzero(sx, sizeof(*sx)); - lock->lo_flags = LOCK_CLASS_SX << LO_CLASSSHIFT; - lock->lo_type = lock->lo_name = description; - lock->lo_flags |= LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | - LO_UPGRADABLE; + sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx); sx->sx_cnt = 0; cv_init(&sx->sx_shrd_cv, description); @@ -94,18 +85,14 @@ sx_init(struct sx *sx, const char *description) cv_init(&sx->sx_excl_cv, description); sx->sx_excl_wcnt = 0; sx->sx_xholder = NULL; - - LOCK_LOG_INIT(lock, 0); - - WITNESS_INIT(lock); + lock_init(&sx->sx_object, &lock_class_sx, description, NULL, + LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE); } void sx_destroy(struct sx *sx) { - LOCK_LOG_DESTROY(&sx->sx_object, 0); - KASSERT((sx->sx_cnt == 0 && sx->sx_shrd_wcnt == 0 && sx->sx_excl_wcnt == 0), ("%s (%s): holders or waiters\n", __func__, sx->sx_object.lo_name)); @@ -114,7 +101,7 @@ sx_destroy(struct sx *sx) cv_destroy(&sx->sx_shrd_cv); cv_destroy(&sx->sx_excl_cv); - WITNESS_DESTROY(&sx->sx_object); + lock_destroy(&sx->sx_object); } void diff --git a/sys/kern/subr_lock.c b/sys/kern/subr_lock.c new file mode 100644 index 0000000..3273d91 --- /dev/null +++ b/sys/kern/subr_lock.c @@ -0,0 +1,113 @@ +/*- + * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of any co-contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * This module holds the global variables and functions used to maintain + * lock_object structures. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/ktr.h> +#include <sys/lock.h> + +#ifdef DDB +#include <ddb/ddb.h> +#endif + +CTASSERT(LOCK_CLASS_MAX == 15); + +#if LOCK_DEBUG > 0 || defined(DDB) +struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = { + &lock_class_mtx_spin, + &lock_class_mtx_sleep, + &lock_class_sx, +}; +#endif + +void +lock_init(struct lock_object *lock, struct lock_class *class, const char *name, + const char *type, int flags) +{ + int i; + + /* Check for double-init and zero object. */ + KASSERT(!lock_initalized(lock), ("lock \"%s\" %p already initialized", + name, lock)); + + /* Look up lock class to find its index. */ + for (i = 0; i < LOCK_CLASS_MAX; i++) + if (lock_classes[i] == class) { + lock->lo_flags = i << LO_CLASSSHIFT; + break; + } + KASSERT(i < LOCK_CLASS_MAX, ("unknown lock class %p", class)); + + /* Initialize the lock object. */ + lock->lo_name = name; + lock->lo_type = type != NULL ? type : name; + lock->lo_flags |= flags | LO_INITIALIZED; + LOCK_LOG_INIT(lock, 0); + WITNESS_INIT(lock); +} + +void +lock_destroy(struct lock_object *lock) +{ + + KASSERT(lock_initalized(lock), ("lock %p is not initialized", lock)); + WITNESS_DESTROY(lock); + LOCK_LOG_DESTROY(lock, 0); + lock->lo_flags &= ~LO_INITIALIZED; +} + +#ifdef DDB +DB_SHOW_COMMAND(lock, db_show_lock) +{ + struct lock_object *lock; + struct lock_class *class; + + if (!have_addr) + return; + lock = (struct lock_object *)addr; + if (LO_CLASSINDEX(lock) > LOCK_CLASS_MAX) { + db_printf("Unknown lock class: %d\n", LO_CLASSINDEX(lock)); + return; + } + class = LOCK_CLASS(lock); + db_printf(" class: %s\n", class->lc_name); + db_printf(" name: %s\n", lock->lo_name); + if (lock->lo_type && lock->lo_type != lock->lo_name) + db_printf(" type: %s\n", lock->lo_type); + class->lc_ddb_show(lock); +} +#endif diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index db10e8f..6e6c78a 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -543,9 +543,6 @@ witness_init(struct lock_object *lock) /* Various sanity checks. */ class = LOCK_CLASS(lock); - if (lock->lo_flags & LO_INITIALIZED) - panic("%s: lock (%s) %s is already initialized", __func__, - class->lc_name, lock->lo_name); if ((lock->lo_flags & LO_RECURSABLE) != 0 && (class->lc_flags & LC_RECURSABLE) == 0) panic("%s: lock (%s) %s can not be recursable", __func__, @@ -566,7 +563,6 @@ witness_init(struct lock_object *lock) * it to the pending_locks list. If it is not too early, then enroll * the lock now. */ - lock->lo_flags |= LO_INITIALIZED; if (witness_watch == 0 || panicstr != NULL || (lock->lo_flags & LO_WITNESS) == 0) lock->lo_witness = NULL; @@ -587,9 +583,6 @@ witness_destroy(struct lock_object *lock) if (witness_cold) panic("lock (%s) %s destroyed while witness_cold", class->lc_name, lock->lo_name); - if ((lock->lo_flags & LO_INITIALIZED) == 0) - panic("%s: lock (%s) %s is not initialized", __func__, - class->lc_name, lock->lo_name); /* XXX: need to verify that no one holds the lock */ if ((lock->lo_flags & (LO_WITNESS | LO_ENROLLPEND)) == LO_WITNESS && @@ -615,7 +608,6 @@ witness_destroy(struct lock_object *lock) STAILQ_REMOVE(&pending_locks, lock, lock_object, lo_list); lock->lo_flags &= ~LO_ENROLLPEND; } - lock->lo_flags &= ~LO_INITIALIZED; } #ifdef DDB diff --git a/sys/sys/lock.h b/sys/sys/lock.h index a6f9080..03b41e3 100644 --- a/sys/sys/lock.h +++ b/sys/sys/lock.h @@ -78,10 +78,7 @@ struct lock_class { #define LO_CLASSSHIFT 24 #define LO_CLASSINDEX(lock) ((((lock)->lo_flags) & LO_CLASSMASK) >> LO_CLASSSHIFT) #define LOCK_CLASS(lock) (lock_classes[LO_CLASSINDEX((lock))]) -#define LOCK_CLASS_SPIN_MUTEX 0 -#define LOCK_CLASS_SLEEP_MUTEX 1 -#define LOCK_CLASS_SX 2 -#define LOCK_CLASS_MAX LOCK_CLASS_SX +#define LOCK_CLASS_MAX (LO_CLASSMASK >> LO_CLASSSHIFT) #define LI_RECURSEMASK 0x0000ffff /* Recursion depth of lock instance. */ #define LI_EXCLUSIVE 0x00010000 /* Exclusive lock instance. */ @@ -199,6 +196,8 @@ struct lock_list_entry { #define LOCK_LOG_DESTROY(lo, flags) LOCK_LOG_INIT(lo, flags) +#define lock_initalized(lo) ((lo)->lo_flags & LO_INITIALIZED) + /* * Helpful macros for quickly coming up with assertions with informative * panic messages. @@ -215,6 +214,9 @@ extern struct lock_class lock_class_sx; extern struct lock_class *lock_classes[]; +void lock_init(struct lock_object *lock, struct lock_class *class, + const char *name, const char *type, int flags); +void lock_destroy(struct lock_object *lock); void spinlock_enter(void); void spinlock_exit(void); void witness_init(struct lock_object *); @@ -286,8 +288,8 @@ const char *witness_file(struct lock_object *); witness_line(lock) #else /* WITNESS */ -#define WITNESS_INIT(lock) ((lock)->lo_flags |= LO_INITIALIZED) -#define WITNESS_DESTROY(lock) ((lock)->lo_flags &= ~LO_INITIALIZED) +#define WITNESS_INIT(lock) +#define WITNESS_DESTROY(lock) #define WITNESS_DEFINEORDER(lock1, lock2) 0 #define WITNESS_CHECKORDER(lock, flags, file, line) #define WITNESS_LOCK(lock, flags, file, line) diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index 46cbe68..7984a25 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -320,7 +320,7 @@ extern struct mtx_pool *mtxpool_sleep; #define mtx_trylock_flags(m, opts) \ _mtx_trylock((m), (opts), LOCK_FILE, LOCK_LINE) -#define mtx_initialized(m) ((m)->mtx_object.lo_flags & LO_INITIALIZED) +#define mtx_initialized(m) lock_initalized(&(m)->mtx_object) #define mtx_owned(m) (((m)->mtx_lock & MTX_FLAGMASK) == (uintptr_t)curthread) |
