Move most of sys/mutex.h into kern/kern_mutex.c, thereby making the mutex

inline functions non-inlined.  Hide parts of the mutex implementation that
should not be exposed.

Make sure that WITNESS code is not executed during boot until the mutexes
are fully initialized by SI_SUB_MUTEX (the original motivation for this
commit).

Submitted by:	peter

9 files changed, 1224 insertions, 595 deletions

diff --git a/sys/alpha/include/mutex.h b/sys/alpha/include/mutex.h
index 5c37f54..40717b0 100644
--- a/sys/alpha/include/mutex.h
+++ b/sys/alpha/include/mutex.h
@@ -68,6 +68,8 @@ extern char STR_SIEN[];
  *--------------------------------------------------------------------------
  */
 
+#ifdef _KERN_MUTEX_C_
+
 #define	_V(x)	__STRING(x)
 
 /*
@@ -84,6 +86,10 @@ extern char STR_SIEN[];
 	}								\
 } while (0)
 
+#undef _V
+
+#endif	/* _KERN_MUTEX_C_ */
+
 #endif	/* _KERNEL */
 
 #else	/* !LOCORE */
diff --git a/sys/amd64/include/mutex.h b/sys/amd64/include/mutex.h
index 0e58108..2f16de3 100644
--- a/sys/amd64/include/mutex.h
+++ b/sys/amd64/include/mutex.h
@@ -67,6 +67,8 @@ extern char STR_SIEN[];
  *------------------------------------------------------------------------------
  */
 
+#ifdef _KERN_MUTEX_C_
+
 #define	_V(x)	__STRING(x)
 
 #if 0
@@ -248,6 +250,10 @@ extern char STR_SIEN[];
 
 #endif	/* I386_CPU */
 
+#undef _V
+
+#endif	/* _KERN_MUTEX_C_ */
+
 #endif	/* _KERNEL */
 
 #else	/* !LOCORE */
diff --git a/sys/i386/include/mutex.h b/sys/i386/include/mutex.h
index 0e58108..2f16de3 100644
--- a/sys/i386/include/mutex.h
+++ b/sys/i386/include/mutex.h
@@ -67,6 +67,8 @@ extern char STR_SIEN[];
  *------------------------------------------------------------------------------
  */
 
+#ifdef _KERN_MUTEX_C_
+
 #define	_V(x)	__STRING(x)
 
 #if 0
@@ -248,6 +250,10 @@ extern char STR_SIEN[];
 
 #endif	/* I386_CPU */
 
+#undef _V
+
+#endif	/* _KERN_MUTEX_C_ */
+
 #endif	/* _KERNEL */
 
 #else	/* !LOCORE */
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index b3c6e7e..1127a40 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -84,8 +84,140 @@
 /*
  * Machine independent bits of the mutex implementation
  */
-/* All mutexes in system (used for debug/panic) */
+
 #ifdef WITNESS
+struct mtx_debug {
+	struct witness	*mtxd_witness;
+	LIST_ENTRY(mtx)	mtxd_held;
+	const char	*mtxd_file;
+	int		mtxd_line;
+	const char	*mtxd_description;
+};
+
+#define mtx_description	mtx_union.mtxu_debug->mtxd_description
+#define mtx_held	mtx_union.mtxu_debug->mtxd_held
+#define	mtx_file	mtx_union.mtxu_debug->mtxd_file
+#define	mtx_line	mtx_union.mtxu_debug->mtxd_line
+#define	mtx_witness	mtx_union.mtxu_debug->mtxd_witness
+#else	/* WITNESS */
+#define mtx_description	mtx_union.mtxu_description
+#endif	/* WITNESS */
+
+/*
+ * Assembly macros
+ *------------------------------------------------------------------------------
+ */
+
+#define	_V(x)	__STRING(x)
+
+/*
+ * Default, unoptimized mutex micro-operations
+ */
+
+#ifndef _obtain_lock
+/* Actually obtain mtx_lock */
+#define _obtain_lock(mp, tid)						\
+	atomic_cmpset_acq_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED, (tid))
+#endif
+
+#ifndef _release_lock
+/* Actually release mtx_lock */
+#define _release_lock(mp, tid)		       				\
+	atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), (void *)MTX_UNOWNED)
+#endif
+
+#ifndef _release_lock_quick
+/* Actually release mtx_lock quickly assuming that we own it */
+#define	_release_lock_quick(mp) 					\
+	atomic_store_rel_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED)
+#endif
+
+#ifndef _getlock_sleep
+/* Get a sleep lock, deal with recursion inline. */
+#define	_getlock_sleep(mp, tid, type) do {				\
+	if (!_obtain_lock(mp, tid)) {					\
+		if (((mp)->mtx_lock & MTX_FLAGMASK) != ((uintptr_t)(tid)))\
+			mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0);	\
+		else {							\
+			atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSED);	\
+			(mp)->mtx_recurse++;				\
+		}							\
+	}								\
+} while (0)
+#endif
+
+#ifndef _getlock_spin_block
+/* Get a spin lock, handle recursion inline (as the less common case) */
+#define	_getlock_spin_block(mp, tid, type) do {				\
+	u_int _mtx_intr = save_intr();					\
+	disable_intr();							\
+	if (!_obtain_lock(mp, tid))					\
+		mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _mtx_intr);	\
+	else								\
+		(mp)->mtx_saveintr = _mtx_intr;				\
+} while (0)
+#endif
+
+#ifndef _getlock_norecurse
+/*
+ * Get a lock without any recursion handling. Calls the hard enter function if
+ * we can't get it inline.
+ */
+#define	_getlock_norecurse(mp, tid, type) do {				\
+	if (!_obtain_lock(mp, tid))					\
+		mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0);		\
+} while (0)
+#endif
+
+#ifndef _exitlock_norecurse
+/*
+ * Release a sleep lock assuming we haven't recursed on it, recursion is handled
+ * in the hard function.
+ */
+#define	_exitlock_norecurse(mp, tid, type) do {				\
+	if (!_release_lock(mp, tid))					\
+		mtx_exit_hard((mp), (type) & MTX_HARDOPTS);		\
+} while (0)
+#endif
+
+#ifndef _exitlock
+/*
+ * Release a sleep lock when its likely we recursed (the code to
+ * deal with simple recursion is inline).
+ */
+#define	_exitlock(mp, tid, type) do {					\
+	if (!_release_lock(mp, tid)) {					\
+		if ((mp)->mtx_lock & MTX_RECURSED) {			\
+			if (--((mp)->mtx_recurse) == 0)			\
+				atomic_clear_ptr(&(mp)->mtx_lock,	\
+				    MTX_RECURSED);			\
+		} else {						\
+			mtx_exit_hard((mp), (type) & MTX_HARDOPTS);	\
+		}							\
+	}								\
+} while (0)
+#endif
+
+#ifndef _exitlock_spin
+/* Release a spin lock (with possible recursion). */
+#define	_exitlock_spin(mp) do {						\
+	if (!mtx_recursed((mp))) {					\
+		int _mtx_intr = (mp)->mtx_saveintr;			\
+									\
+		_release_lock_quick(mp);				\
+		restore_intr(_mtx_intr);				\
+	} else {							\
+		(mp)->mtx_recurse--;					\
+	}								\
+} while (0)
+#endif
+
+#ifdef WITNESS
+static void	witness_init(struct mtx *, int flag);
+static void	witness_destroy(struct mtx *);
+static void	witness_display(void(*)(const char *fmt, ...));
+
+/* All mutexes in system (used for debug/panic) */
 static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0,
 	"All mutexes queue head" };
 static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, {&all_mtx_debug},
@@ -97,15 +229,26 @@ static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, {&all_mtx_debug},
  */
 static int witness_cold = 1;
 #else	/* WITNESS */
+/* All mutexes in system (used for debug/panic) */
 static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, {"All mutexes queue head"},
 	TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
 	{ NULL, NULL }, &all_mtx, &all_mtx };
+
+/*
+ * flag++ is slezoid way of shutting up unused parameter warning
+ * in mtx_init()
+ */
+#define witness_init(m, flag) flag++
+#define witness_destroy(m)
+#define witness_try_enter(m, t, f, l)
 #endif	/* WITNESS */
 
 static int	mtx_cur_cnt;
 static int	mtx_max_cnt;
 
-static void propagate_priority(struct proc *);
+static void	propagate_priority(struct proc *);
+static void	mtx_enter_hard(struct mtx *, int type, int saveintr);
+static void	mtx_exit_hard(struct mtx *, int type);
 
 #define	mtx_unowned(m)	((m)->mtx_lock == MTX_UNOWNED)
 #define	mtx_owner(m)	(mtx_unowned(m) ? NULL \
@@ -233,6 +376,143 @@ propagate_priority(struct proc *p)
 	}
 }
 
+/*
+ * Get lock 'm', the macro handles the easy (and most common cases) and leaves
+ * the slow stuff to the mtx_enter_hard() function.
+ *
+ * Note: since type is usually a constant much of this code is optimized out.
+ */
+void
+_mtx_enter(struct mtx *mtxp, int type, const char *file, int line)
+{
+	struct mtx	*mpp = mtxp;
+
+	/* bits only valid on mtx_exit() */
+	MPASS4(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0,
+	    STR_mtx_bad_type, file, line);
+
+	if ((type) & MTX_SPIN) {
+		/*
+		 * Easy cases of spin locks:
+		 *
+		 * 1) We already own the lock and will simply recurse on it (if
+		 *    RLIKELY)
+		 *
+		 * 2) The lock is free, we just get it
+		 */
+		if ((type) & MTX_RLIKELY) {
+			/*
+			 * Check for recursion, if we already have this
+			 * lock we just bump the recursion count.
+			 */
+			if (mpp->mtx_lock == (uintptr_t)CURTHD) {
+				mpp->mtx_recurse++;
+				goto done;
+			}
+		}
+
+		if (((type) & MTX_TOPHALF) == 0) {
+			/*
+			 * If an interrupt thread uses this we must block
+			 * interrupts here.
+			 */
+			if ((type) & MTX_FIRST) {
+				ASS_IEN;
+				disable_intr();
+				_getlock_norecurse(mpp, CURTHD,
+				    (type) & MTX_HARDOPTS);
+			} else {
+				_getlock_spin_block(mpp, CURTHD,
+				    (type) & MTX_HARDOPTS);
+			}
+		} else
+			_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
+	} else {
+		/* Sleep locks */
+		if ((type) & MTX_RLIKELY)
+			_getlock_sleep(mpp, CURTHD, (type) & MTX_HARDOPTS);
+		else
+			_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
+	}
+done:
+	WITNESS_ENTER(mpp, type, file, line);
+	if (((type) & MTX_QUIET) == 0)
+		CTR5(KTR_LOCK, STR_mtx_enter_fmt,
+		    mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
+
+}
+
+/*
+ * Attempt to get MTX_DEF lock, return non-zero if lock acquired.
+ *
+ * XXX DOES NOT HANDLE RECURSION
+ */
+int
+_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line)
+{
+	struct mtx	*const mpp = mtxp;
+	int	rval;
+
+	rval = _obtain_lock(mpp, CURTHD);
+#ifdef WITNESS
+	if (rval && mpp->mtx_witness != NULL) {
+		MPASS(mpp->mtx_recurse == 0);
+		witness_try_enter(mpp, type, file, line);
+	}
+#endif	/* WITNESS */
+	if (((type) & MTX_QUIET) == 0)
+		CTR5(KTR_LOCK, STR_mtx_try_enter_fmt,
+		    mpp->mtx_description, mpp, rval, file, line);
+
+	return rval;
+}
+
+/*
+ * Release lock m.
+ */
+void
+_mtx_exit(struct mtx *mtxp, int type, const char *file, int line)
+{
+	struct mtx	*const mpp = mtxp;
+
+	MPASS4(mtx_owned(mpp), STR_mtx_owned, file, line);
+	WITNESS_EXIT(mpp, type, file, line);
+	if (((type) & MTX_QUIET) == 0)
+		CTR5(KTR_LOCK, STR_mtx_exit_fmt,
+		    mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
+	if ((type) & MTX_SPIN) {
+		if ((type) & MTX_NORECURSE) {
+			int mtx_intr = mpp->mtx_saveintr;
+
+			MPASS4(mpp->mtx_recurse == 0, STR_mtx_recurse,
+			    file, line);
+			_release_lock_quick(mpp);
+			if (((type) & MTX_TOPHALF) == 0) {
+				if ((type) & MTX_FIRST) {
+					ASS_IDIS;
+					enable_intr();
+				} else
+					restore_intr(mtx_intr);
+			}
+		} else {
+			if (((type & MTX_TOPHALF) == 0) &&
+			    (type & MTX_FIRST)) {
+				ASS_IDIS;
+				ASS_SIEN(mpp);
+			}
+			_exitlock_spin(mpp);
+		}
+	} else {
+		/* Handle sleep locks */
+		if ((type) & MTX_RLIKELY)
+			_exitlock(mpp, CURTHD, (type) & MTX_HARDOPTS);
+		else {
+			_exitlock_norecurse(mpp, CURTHD,
+			    (type) & MTX_HARDOPTS);
+		}
+	}
+}
+
 void
 mtx_enter_hard(struct mtx *m, int type, int saveintr)
 {
@@ -512,6 +792,37 @@ mtx_exit_hard(struct mtx *m, int type)
 	}
 }
 
+#ifdef INVARIANTS
+void
+mtx_assert(struct mtx *m, int what)
+{
+	switch ((what)) {
+	case MA_OWNED:
+	case MA_OWNED | MA_RECURSED:
+	case MA_OWNED | MA_NOTRECURSED:
+		if (!mtx_owned((m)))
+			panic("mutex %s not owned at %s:%d",
+			    (m)->mtx_description, __FILE__, __LINE__);
+		if (mtx_recursed((m))) {
+			if (((what) & MA_NOTRECURSED) != 0)
+				panic("mutex %s recursed at %s:%d",
+				    (m)->mtx_description, __FILE__, __LINE__);
+		} else if (((what) & MA_RECURSED) != 0) {
+			panic("mutex %s unrecursed at %s:%d",
+			    (m)->mtx_description, __FILE__, __LINE__);
+		}
+		break;
+	case MA_NOTOWNED:
+		if (mtx_owned((m)))
+			panic("mutex %s owned at %s:%d",
+			    (m)->mtx_description, __FILE__, __LINE__);
+		break;
+	default:
+		panic("unknown mtx_assert at %s:%d", __FILE__, __LINE__);
+	}
+}
+#endif
+
 #define MV_DESTROY	0	/* validate before destory */
 #define MV_INIT		1	/* validate before init */
 
@@ -818,13 +1129,13 @@ static struct witness_blessed blessed_list[] = {
 };
 static int blessed_count = sizeof(blessed_list) / sizeof(struct witness_blessed);
 
-void
+static void
 witness_init(struct mtx *m, int flag)
 {
 	m->mtx_witness = enroll(m->mtx_description, flag);
 }
 
-void
+static void
 witness_destroy(struct mtx *m)
 {
 	struct mtx *m1;
@@ -841,6 +1152,36 @@ witness_destroy(struct mtx *m)
 
 }
 
+static void
+witness_display(void(*prnt)(const char *fmt, ...))
+{
+	struct witness *w, *w1;
+
+	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
+	witness_levelall();
+
+	for (w = w_all; w; w = w->w_next) {
+		if (w->w_file == NULL)
+			continue;
+		for (w1 = w_all; w1; w1 = w1->w_next) {
+			if (isitmychild(w1, w))
+				break;
+		}
+		if (w1 != NULL)
+			continue;
+		/*
+		 * This lock has no anscestors, display its descendants. 
+		 */
+		witness_displaydescendants(prnt, w);
+	}
+	prnt("\nMutex which were never acquired\n");
+	for (w = w_all; w; w = w->w_next) {
+		if (w->w_file != NULL)
+			continue;
+		prnt("%s\n", w->w_description);
+	}
+}
+
 void
 witness_enter(struct mtx *m, int flags, const char *file, int line)
 {
@@ -852,9 +1193,7 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 	int go_into_ddb = 0;
 #endif /* DDB */
 
-	if (witness_cold)
-		return;
-	if (panicstr)
+	if (witness_cold || m->mtx_witness == NULL || panicstr)
 		return;
 	w = m->mtx_witness;
 	p = CURPROC;
@@ -994,53 +1333,6 @@ out:
 }
 
 void
-witness_exit(struct mtx *m, int flags, const char *file, int line)
-{
-	struct witness *w;
-
-	if (witness_cold)
-		return;
-	if (panicstr)
-		return;
-	w = m->mtx_witness;
-
-	if (flags & MTX_SPIN) {
-		if (!(w->w_spin))
-			panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
-			    " %s:%d", m->mtx_description, file, line);
-		if (mtx_recursed(m)) {
-			if (!(w->w_recurse))
-				panic("mutex_exit: recursion on non-recursive"
-				    " mutex %s @ %s:%d", m->mtx_description,
-				    file, line); 
-			return;
-		}
-		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
-		PCPU_SET(witness_spin_check,
-		    PCPU_GET(witness_spin_check) & ~w->w_level);
-		mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
-		return;
-	}
-	if (w->w_spin)
-		panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
-		    m->mtx_description, file, line);
-
-	if (mtx_recursed(m)) {
-		if (!(w->w_recurse))
-			panic("mutex_exit: recursion on non-recursive"
-			    " mutex %s @ %s:%d", m->mtx_description,
-			    file, line); 
-		return;
-	}
-
-	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
-		panic("switchable mtx_exit() of %s when not legal @ %s:%d",
-			    m->mtx_description, file, line);
-	LIST_REMOVE(m, mtx_held);
-	m->mtx_held.le_prev = NULL;
-}
-
-void
 witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 {
 	struct proc *p;
@@ -1094,33 +1386,48 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 }
 
 void
-witness_display(void(*prnt)(const char *fmt, ...))
+witness_exit(struct mtx *m, int flags, const char *file, int line)
 {
-	struct witness *w, *w1;
+	struct witness *w;
 
-	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
-	witness_levelall();
+	if (witness_cold || m->mtx_witness == NULL || panicstr)
+		return;
+	w = m->mtx_witness;
 
-	for (w = w_all; w; w = w->w_next) {
-		if (w->w_file == NULL)
-			continue;
-		for (w1 = w_all; w1; w1 = w1->w_next) {
-			if (isitmychild(w1, w))
-				break;
+	if (flags & MTX_SPIN) {
+		if (!(w->w_spin))
+			panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
+			    " %s:%d", m->mtx_description, file, line);
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_exit: recursion on non-recursive"
+				    " mutex %s @ %s:%d", m->mtx_description,
+				    file, line); 
+			return;
 		}
-		if (w1 != NULL)
-			continue;
-		/*
-		 * This lock has no anscestors, display its descendants. 
-		 */
-		witness_displaydescendants(prnt, w);
+		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
+		PCPU_SET(witness_spin_check,
+		    PCPU_GET(witness_spin_check) & ~w->w_level);
+		mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
+		return;
 	}
-	prnt("\nMutex which were never acquired\n");
-	for (w = w_all; w; w = w->w_next) {
-		if (w->w_file != NULL)
-			continue;
-		prnt("%s\n", w->w_description);
+	if (w->w_spin)
+		panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
+		    m->mtx_description, file, line);
+
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_exit: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description,
+			    file, line); 
+		return;
 	}
+
+	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
+		panic("switchable mtx_exit() of %s when not legal @ %s:%d",
+			    m->mtx_description, file, line);
+	LIST_REMOVE(m, mtx_held);
+	m->mtx_held.le_prev = NULL;
 }
 
 int
@@ -1466,6 +1773,9 @@ witness_save(struct mtx *m, const char **filep, int *linep)
 {
 
 	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
+	if (m->mtx_witness == NULL)
+		return;
+
 	*filep = m->mtx_witness->w_file;
 	*linep = m->mtx_witness->w_line;
 }
@@ -1475,6 +1785,9 @@ witness_restore(struct mtx *m, const char *file, int line)
 {
 
 	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
+	if (m->mtx_witness == NULL)
+		return;
+
 	m->mtx_witness->w_file = file;
 	m->mtx_witness->w_line = line;
 }
diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c
index b3c6e7e..1127a40 100644
--- a/sys/kern/subr_turnstile.c
+++ b/sys/kern/subr_turnstile.c
@@ -84,8 +84,140 @@
 /*
  * Machine independent bits of the mutex implementation
  */
-/* All mutexes in system (used for debug/panic) */
+
 #ifdef WITNESS
+struct mtx_debug {
+	struct witness	*mtxd_witness;
+	LIST_ENTRY(mtx)	mtxd_held;
+	const char	*mtxd_file;
+	int		mtxd_line;
+	const char	*mtxd_description;
+};
+
+#define mtx_description	mtx_union.mtxu_debug->mtxd_description
+#define mtx_held	mtx_union.mtxu_debug->mtxd_held
+#define	mtx_file	mtx_union.mtxu_debug->mtxd_file
+#define	mtx_line	mtx_union.mtxu_debug->mtxd_line
+#define	mtx_witness	mtx_union.mtxu_debug->mtxd_witness
+#else	/* WITNESS */
+#define mtx_description	mtx_union.mtxu_description
+#endif	/* WITNESS */
+
+/*
+ * Assembly macros
+ *------------------------------------------------------------------------------
+ */
+
+#define	_V(x)	__STRING(x)
+
+/*
+ * Default, unoptimized mutex micro-operations
+ */
+
+#ifndef _obtain_lock
+/* Actually obtain mtx_lock */
+#define _obtain_lock(mp, tid)						\
+	atomic_cmpset_acq_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED, (tid))
+#endif
+
+#ifndef _release_lock
+/* Actually release mtx_lock */
+#define _release_lock(mp, tid)		       				\
+	atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), (void *)MTX_UNOWNED)
+#endif
+
+#ifndef _release_lock_quick
+/* Actually release mtx_lock quickly assuming that we own it */
+#define	_release_lock_quick(mp) 					\
+	atomic_store_rel_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED)
+#endif
+
+#ifndef _getlock_sleep
+/* Get a sleep lock, deal with recursion inline. */
+#define	_getlock_sleep(mp, tid, type) do {				\
+	if (!_obtain_lock(mp, tid)) {					\
+		if (((mp)->mtx_lock & MTX_FLAGMASK) != ((uintptr_t)(tid)))\
+			mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0);	\
+		else {							\
+			atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSED);	\
+			(mp)->mtx_recurse++;				\
+		}							\
+	}								\
+} while (0)
+#endif
+
+#ifndef _getlock_spin_block
+/* Get a spin lock, handle recursion inline (as the less common case) */
+#define	_getlock_spin_block(mp, tid, type) do {				\
+	u_int _mtx_intr = save_intr();					\
+	disable_intr();							\
+	if (!_obtain_lock(mp, tid))					\
+		mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _mtx_intr);	\
+	else								\
+		(mp)->mtx_saveintr = _mtx_intr;				\
+} while (0)
+#endif
+
+#ifndef _getlock_norecurse
+/*
+ * Get a lock without any recursion handling. Calls the hard enter function if
+ * we can't get it inline.
+ */
+#define	_getlock_norecurse(mp, tid, type) do {				\
+	if (!_obtain_lock(mp, tid))					\
+		mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0);		\
+} while (0)
+#endif
+
+#ifndef _exitlock_norecurse
+/*
+ * Release a sleep lock assuming we haven't recursed on it, recursion is handled
+ * in the hard function.
+ */
+#define	_exitlock_norecurse(mp, tid, type) do {				\
+	if (!_release_lock(mp, tid))					\
+		mtx_exit_hard((mp), (type) & MTX_HARDOPTS);		\
+} while (0)
+#endif
+
+#ifndef _exitlock
+/*
+ * Release a sleep lock when its likely we recursed (the code to
+ * deal with simple recursion is inline).
+ */
+#define	_exitlock(mp, tid, type) do {					\
+	if (!_release_lock(mp, tid)) {					\
+		if ((mp)->mtx_lock & MTX_RECURSED) {			\
+			if (--((mp)->mtx_recurse) == 0)			\
+				atomic_clear_ptr(&(mp)->mtx_lock,	\
+				    MTX_RECURSED);			\
+		} else {						\
+			mtx_exit_hard((mp), (type) & MTX_HARDOPTS);	\
+		}							\
+	}								\
+} while (0)
+#endif
+
+#ifndef _exitlock_spin
+/* Release a spin lock (with possible recursion). */
+#define	_exitlock_spin(mp) do {						\
+	if (!mtx_recursed((mp))) {					\
+		int _mtx_intr = (mp)->mtx_saveintr;			\
+									\
+		_release_lock_quick(mp);				\
+		restore_intr(_mtx_intr);				\
+	} else {							\
+		(mp)->mtx_recurse--;					\
+	}								\
+} while (0)
+#endif
+
+#ifdef WITNESS
+static void	witness_init(struct mtx *, int flag);
+static void	witness_destroy(struct mtx *);
+static void	witness_display(void(*)(const char *fmt, ...));
+
+/* All mutexes in system (used for debug/panic) */
 static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0,
 	"All mutexes queue head" };
 static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, {&all_mtx_debug},
@@ -97,15 +229,26 @@ static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, {&all_mtx_debug},
  */
 static int witness_cold = 1;
 #else	/* WITNESS */
+/* All mutexes in system (used for debug/panic) */
 static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, {"All mutexes queue head"},
 	TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
 	{ NULL, NULL }, &all_mtx, &all_mtx };
+
+/*
+ * flag++ is slezoid way of shutting up unused parameter warning
+ * in mtx_init()
+ */
+#define witness_init(m, flag) flag++
+#define witness_destroy(m)
+#define witness_try_enter(m, t, f, l)
 #endif	/* WITNESS */
 
 static int	mtx_cur_cnt;
 static int	mtx_max_cnt;
 
-static void propagate_priority(struct proc *);
+static void	propagate_priority(struct proc *);
+static void	mtx_enter_hard(struct mtx *, int type, int saveintr);
+static void	mtx_exit_hard(struct mtx *, int type);
 
 #define	mtx_unowned(m)	((m)->mtx_lock == MTX_UNOWNED)
 #define	mtx_owner(m)	(mtx_unowned(m) ? NULL \
@@ -233,6 +376,143 @@ propagate_priority(struct proc *p)
 	}
 }
 
+/*
+ * Get lock 'm', the macro handles the easy (and most common cases) and leaves
+ * the slow stuff to the mtx_enter_hard() function.
+ *
+ * Note: since type is usually a constant much of this code is optimized out.
+ */
+void
+_mtx_enter(struct mtx *mtxp, int type, const char *file, int line)
+{
+	struct mtx	*mpp = mtxp;
+
+	/* bits only valid on mtx_exit() */
+	MPASS4(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0,
+	    STR_mtx_bad_type, file, line);
+
+	if ((type) & MTX_SPIN) {
+		/*
+		 * Easy cases of spin locks:
+		 *
+		 * 1) We already own the lock and will simply recurse on it (if
+		 *    RLIKELY)
+		 *
+		 * 2) The lock is free, we just get it
+		 */
+		if ((type) & MTX_RLIKELY) {
+			/*
+			 * Check for recursion, if we already have this
+			 * lock we just bump the recursion count.
+			 */
+			if (mpp->mtx_lock == (uintptr_t)CURTHD) {
+				mpp->mtx_recurse++;
+				goto done;
+			}
+		}
+
+		if (((type) & MTX_TOPHALF) == 0) {
+			/*
+			 * If an interrupt thread uses this we must block
+			 * interrupts here.
+			 */
+			if ((type) & MTX_FIRST) {
+				ASS_IEN;
+				disable_intr();
+				_getlock_norecurse(mpp, CURTHD,
+				    (type) & MTX_HARDOPTS);
+			} else {
+				_getlock_spin_block(mpp, CURTHD,
+				    (type) & MTX_HARDOPTS);
+			}
+		} else
+			_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
+	} else {
+		/* Sleep locks */
+		if ((type) & MTX_RLIKELY)
+			_getlock_sleep(mpp, CURTHD, (type) & MTX_HARDOPTS);
+		else
+			_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
+	}
+done:
+	WITNESS_ENTER(mpp, type, file, line);
+	if (((type) & MTX_QUIET) == 0)
+		CTR5(KTR_LOCK, STR_mtx_enter_fmt,
+		    mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
+
+}
+
+/*
+ * Attempt to get MTX_DEF lock, return non-zero if lock acquired.
+ *
+ * XXX DOES NOT HANDLE RECURSION
+ */
+int
+_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line)
+{
+	struct mtx	*const mpp = mtxp;
+	int	rval;
+
+	rval = _obtain_lock(mpp, CURTHD);
+#ifdef WITNESS
+	if (rval && mpp->mtx_witness != NULL) {
+		MPASS(mpp->mtx_recurse == 0);
+		witness_try_enter(mpp, type, file, line);
+	}
+#endif	/* WITNESS */
+	if (((type) & MTX_QUIET) == 0)
+		CTR5(KTR_LOCK, STR_mtx_try_enter_fmt,
+		    mpp->mtx_description, mpp, rval, file, line);
+
+	return rval;
+}
+
+/*
+ * Release lock m.
+ */
+void
+_mtx_exit(struct mtx *mtxp, int type, const char *file, int line)
+{
+	struct mtx	*const mpp = mtxp;
+
+	MPASS4(mtx_owned(mpp), STR_mtx_owned, file, line);
+	WITNESS_EXIT(mpp, type, file, line);
+	if (((type) & MTX_QUIET) == 0)
+		CTR5(KTR_LOCK, STR_mtx_exit_fmt,
+		    mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
+	if ((type) & MTX_SPIN) {
+		if ((type) & MTX_NORECURSE) {
+			int mtx_intr = mpp->mtx_saveintr;
+
+			MPASS4(mpp->mtx_recurse == 0, STR_mtx_recurse,
+			    file, line);
+			_release_lock_quick(mpp);
+			if (((type) & MTX_TOPHALF) == 0) {
+				if ((type) & MTX_FIRST) {
+					ASS_IDIS;
+					enable_intr();
+				} else
+					restore_intr(mtx_intr);
+			}
+		} else {
+			if (((type & MTX_TOPHALF) == 0) &&
+			    (type & MTX_FIRST)) {
+				ASS_IDIS;
+				ASS_SIEN(mpp);
+			}
+			_exitlock_spin(mpp);
+		}
+	} else {
+		/* Handle sleep locks */
+		if ((type) & MTX_RLIKELY)
+			_exitlock(mpp, CURTHD, (type) & MTX_HARDOPTS);
+		else {
+			_exitlock_norecurse(mpp, CURTHD,
+			    (type) & MTX_HARDOPTS);
+		}
+	}
+}
+
 void
 mtx_enter_hard(struct mtx *m, int type, int saveintr)
 {
@@ -512,6 +792,37 @@ mtx_exit_hard(struct mtx *m, int type)
 	}
 }
 
+#ifdef INVARIANTS
+void
+mtx_assert(struct mtx *m, int what)
+{
+	switch ((what)) {
+	case MA_OWNED:
+	case MA_OWNED | MA_RECURSED:
+	case MA_OWNED | MA_NOTRECURSED:
+		if (!mtx_owned((m)))
+			panic("mutex %s not owned at %s:%d",
+			    (m)->mtx_description, __FILE__, __LINE__);
+		if (mtx_recursed((m))) {
+			if (((what) & MA_NOTRECURSED) != 0)
+				panic("mutex %s recursed at %s:%d",
+				    (m)->mtx_description, __FILE__, __LINE__);
+		} else if (((what) & MA_RECURSED) != 0) {
+			panic("mutex %s unrecursed at %s:%d",
+			    (m)->mtx_description, __FILE__, __LINE__);
+		}
+		break;
+	case MA_NOTOWNED:
+		if (mtx_owned((m)))
+			panic("mutex %s owned at %s:%d",
+			    (m)->mtx_description, __FILE__, __LINE__);
+		break;
+	default:
+		panic("unknown mtx_assert at %s:%d", __FILE__, __LINE__);
+	}
+}
+#endif
+
 #define MV_DESTROY	0	/* validate before destory */
 #define MV_INIT		1	/* validate before init */
 
@@ -818,13 +1129,13 @@ static struct witness_blessed blessed_list[] = {
 };
 static int blessed_count = sizeof(blessed_list) / sizeof(struct witness_blessed);
 
-void
+static void
 witness_init(struct mtx *m, int flag)
 {
 	m->mtx_witness = enroll(m->mtx_description, flag);
 }
 
-void
+static void
 witness_destroy(struct mtx *m)
 {
 	struct mtx *m1;
@@ -841,6 +1152,36 @@ witness_destroy(struct mtx *m)
 
 }
 
+static void
+witness_display(void(*prnt)(const char *fmt, ...))
+{
+	struct witness *w, *w1;
+
+	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
+	witness_levelall();
+
+	for (w = w_all; w; w = w->w_next) {
+		if (w->w_file == NULL)
+			continue;
+		for (w1 = w_all; w1; w1 = w1->w_next) {
+			if (isitmychild(w1, w))
+				break;
+		}
+		if (w1 != NULL)
+			continue;
+		/*
+		 * This lock has no anscestors, display its descendants. 
+		 */
+		witness_displaydescendants(prnt, w);
+	}
+	prnt("\nMutex which were never acquired\n");
+	for (w = w_all; w; w = w->w_next) {
+		if (w->w_file != NULL)
+			continue;
+		prnt("%s\n", w->w_description);
+	}
+}
+
 void
 witness_enter(struct mtx *m, int flags, const char *file, int line)
 {
@@ -852,9 +1193,7 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 	int go_into_ddb = 0;
 #endif /* DDB */
 
-	if (witness_cold)
-		return;
-	if (panicstr)
+	if (witness_cold || m->mtx_witness == NULL || panicstr)
 		return;
 	w = m->mtx_witness;
 	p = CURPROC;
@@ -994,53 +1333,6 @@ out:
 }
 
 void
-witness_exit(struct mtx *m, int flags, const char *file, int line)
-{
-	struct witness *w;
-
-	if (witness_cold)
-		return;
-	if (panicstr)
-		return;
-	w = m->mtx_witness;
-
-	if (flags & MTX_SPIN) {
-		if (!(w->w_spin))
-			panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
-			    " %s:%d", m->mtx_description, file, line);
-		if (mtx_recursed(m)) {
-			if (!(w->w_recurse))
-				panic("mutex_exit: recursion on non-recursive"
-				    " mutex %s @ %s:%d", m->mtx_description,
-				    file, line); 
-			return;
-		}
-		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
-		PCPU_SET(witness_spin_check,
-		    PCPU_GET(witness_spin_check) & ~w->w_level);
-		mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
-		return;
-	}
-	if (w->w_spin)
-		panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
-		    m->mtx_description, file, line);
-
-	if (mtx_recursed(m)) {
-		if (!(w->w_recurse))
-			panic("mutex_exit: recursion on non-recursive"
-			    " mutex %s @ %s:%d", m->mtx_description,
-			    file, line); 
-		return;
-	}
-
-	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
-		panic("switchable mtx_exit() of %s when not legal @ %s:%d",
-			    m->mtx_description, file, line);
-	LIST_REMOVE(m, mtx_held);
-	m->mtx_held.le_prev = NULL;
-}
-
-void
 witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 {
 	struct proc *p;
@@ -1094,33 +1386,48 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 }
 
 void
-witness_display(void(*prnt)(const char *fmt, ...))
+witness_exit(struct mtx *m, int flags, const char *file, int line)
 {
-	struct witness *w, *w1;
+	struct witness *w;
 
-	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
-	witness_levelall();
+	if (witness_cold || m->mtx_witness == NULL || panicstr)
+		return;
+	w = m->mtx_witness;
 
-	for (w = w_all; w; w = w->w_next) {
-		if (w->w_file == NULL)
-			continue;
-		for (w1 = w_all; w1; w1 = w1->w_next) {
-			if (isitmychild(w1, w))
-				break;
+	if (flags & MTX_SPIN) {
+		if (!(w->w_spin))
+			panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
+			    " %s:%d", m->mtx_description, file, line);
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_exit: recursion on non-recursive"
+				    " mutex %s @ %s:%d", m->mtx_description,
+				    file, line); 
+			return;
 		}
-		if (w1 != NULL)
-			continue;
-		/*
-		 * This lock has no anscestors, display its descendants. 
-		 */
-		witness_displaydescendants(prnt, w);
+		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
+		PCPU_SET(witness_spin_check,
+		    PCPU_GET(witness_spin_check) & ~w->w_level);
+		mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
+		return;
 	}
-	prnt("\nMutex which were never acquired\n");
-	for (w = w_all; w; w = w->w_next) {
-		if (w->w_file != NULL)
-			continue;
-		prnt("%s\n", w->w_description);
+	if (w->w_spin)
+		panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
+		    m->mtx_description, file, line);
+
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_exit: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description,
+			    file, line); 
+		return;
 	}
+
+	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
+		panic("switchable mtx_exit() of %s when not legal @ %s:%d",
+			    m->mtx_description, file, line);
+	LIST_REMOVE(m, mtx_held);
+	m->mtx_held.le_prev = NULL;
 }
 
 int
@@ -1466,6 +1773,9 @@ witness_save(struct mtx *m, const char **filep, int *linep)
 {
 
 	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
+	if (m->mtx_witness == NULL)
+		return;
+
 	*filep = m->mtx_witness->w_file;
 	*linep = m->mtx_witness->w_line;
 }
@@ -1475,6 +1785,9 @@ witness_restore(struct mtx *m, const char *file, int line)
 {
 
 	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
+	if (m->mtx_witness == NULL)
+		return;
+
 	m->mtx_witness->w_file = file;
 	m->mtx_witness->w_line = line;
 }
diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c
index b3c6e7e..1127a40 100644
--- a/sys/kern/subr_witness.c
+++ b/sys/kern/subr_witness.c
@@ -84,8 +84,140 @@
 /*
  * Machine independent bits of the mutex implementation
  */
-/* All mutexes in system (used for debug/panic) */
+
 #ifdef WITNESS
+struct mtx_debug {
+	struct witness	*mtxd_witness;
+	LIST_ENTRY(mtx)	mtxd_held;
+	const char	*mtxd_file;
+	int		mtxd_line;
+	const char	*mtxd_description;
+};
+
+#define mtx_description	mtx_union.mtxu_debug->mtxd_description
+#define mtx_held	mtx_union.mtxu_debug->mtxd_held
+#define	mtx_file	mtx_union.mtxu_debug->mtxd_file
+#define	mtx_line	mtx_union.mtxu_debug->mtxd_line
+#define	mtx_witness	mtx_union.mtxu_debug->mtxd_witness
+#else	/* WITNESS */
+#define mtx_description	mtx_union.mtxu_description
+#endif	/* WITNESS */
+
+/*
+ * Assembly macros
+ *------------------------------------------------------------------------------
+ */
+
+#define	_V(x)	__STRING(x)
+
+/*
+ * Default, unoptimized mutex micro-operations
+ */
+
+#ifndef _obtain_lock
+/* Actually obtain mtx_lock */
+#define _obtain_lock(mp, tid)						\
+	atomic_cmpset_acq_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED, (tid))
+#endif
+
+#ifndef _release_lock
+/* Actually release mtx_lock */
+#define _release_lock(mp, tid)		       				\
+	atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), (void *)MTX_UNOWNED)
+#endif
+
+#ifndef _release_lock_quick
+/* Actually release mtx_lock quickly assuming that we own it */
+#define	_release_lock_quick(mp) 					\
+	atomic_store_rel_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED)
+#endif
+
+#ifndef _getlock_sleep
+/* Get a sleep lock, deal with recursion inline. */
+#define	_getlock_sleep(mp, tid, type) do {				\
+	if (!_obtain_lock(mp, tid)) {					\
+		if (((mp)->mtx_lock & MTX_FLAGMASK) != ((uintptr_t)(tid)))\
+			mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0);	\
+		else {							\
+			atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSED);	\
+			(mp)->mtx_recurse++;				\
+		}							\
+	}								\
+} while (0)
+#endif
+
+#ifndef _getlock_spin_block
+/* Get a spin lock, handle recursion inline (as the less common case) */
+#define	_getlock_spin_block(mp, tid, type) do {				\
+	u_int _mtx_intr = save_intr();					\
+	disable_intr();							\
+	if (!_obtain_lock(mp, tid))					\
+		mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _mtx_intr);	\
+	else								\
+		(mp)->mtx_saveintr = _mtx_intr;				\
+} while (0)
+#endif
+
+#ifndef _getlock_norecurse
+/*
+ * Get a lock without any recursion handling. Calls the hard enter function if
+ * we can't get it inline.
+ */
+#define	_getlock_norecurse(mp, tid, type) do {				\
+	if (!_obtain_lock(mp, tid))					\
+		mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0);		\
+} while (0)
+#endif
+
+#ifndef _exitlock_norecurse
+/*
+ * Release a sleep lock assuming we haven't recursed on it, recursion is handled
+ * in the hard function.
+ */
+#define	_exitlock_norecurse(mp, tid, type) do {				\
+	if (!_release_lock(mp, tid))					\
+		mtx_exit_hard((mp), (type) & MTX_HARDOPTS);		\
+} while (0)
+#endif
+
+#ifndef _exitlock
+/*
+ * Release a sleep lock when its likely we recursed (the code to
+ * deal with simple recursion is inline).
+ */
+#define	_exitlock(mp, tid, type) do {					\
+	if (!_release_lock(mp, tid)) {					\
+		if ((mp)->mtx_lock & MTX_RECURSED) {			\
+			if (--((mp)->mtx_recurse) == 0)			\
+				atomic_clear_ptr(&(mp)->mtx_lock,	\
+				    MTX_RECURSED);			\
+		} else {						\
+			mtx_exit_hard((mp), (type) & MTX_HARDOPTS);	\
+		}							\
+	}								\
+} while (0)
+#endif
+
+#ifndef _exitlock_spin
+/* Release a spin lock (with possible recursion). */
+#define	_exitlock_spin(mp) do {						\
+	if (!mtx_recursed((mp))) {					\
+		int _mtx_intr = (mp)->mtx_saveintr;			\
+									\
+		_release_lock_quick(mp);				\
+		restore_intr(_mtx_intr);				\
+	} else {							\
+		(mp)->mtx_recurse--;					\
+	}								\
+} while (0)
+#endif
+
+#ifdef WITNESS
+static void	witness_init(struct mtx *, int flag);
+static void	witness_destroy(struct mtx *);
+static void	witness_display(void(*)(const char *fmt, ...));
+
+/* All mutexes in system (used for debug/panic) */
 static struct mtx_debug all_mtx_debug = { NULL, {NULL, NULL}, NULL, 0,
 	"All mutexes queue head" };
 static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, {&all_mtx_debug},
@@ -97,15 +229,26 @@ static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, {&all_mtx_debug},
  */
 static int witness_cold = 1;
 #else	/* WITNESS */
+/* All mutexes in system (used for debug/panic) */
 static struct mtx all_mtx = { MTX_UNOWNED, 0, 0, 0, {"All mutexes queue head"},
 	TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked),
 	{ NULL, NULL }, &all_mtx, &all_mtx };
+
+/*
+ * flag++ is slezoid way of shutting up unused parameter warning
+ * in mtx_init()
+ */
+#define witness_init(m, flag) flag++
+#define witness_destroy(m)
+#define witness_try_enter(m, t, f, l)
 #endif	/* WITNESS */
 
 static int	mtx_cur_cnt;
 static int	mtx_max_cnt;
 
-static void propagate_priority(struct proc *);
+static void	propagate_priority(struct proc *);
+static void	mtx_enter_hard(struct mtx *, int type, int saveintr);
+static void	mtx_exit_hard(struct mtx *, int type);
 
 #define	mtx_unowned(m)	((m)->mtx_lock == MTX_UNOWNED)
 #define	mtx_owner(m)	(mtx_unowned(m) ? NULL \
@@ -233,6 +376,143 @@ propagate_priority(struct proc *p)
 	}
 }
 
+/*
+ * Get lock 'm', the macro handles the easy (and most common cases) and leaves
+ * the slow stuff to the mtx_enter_hard() function.
+ *
+ * Note: since type is usually a constant much of this code is optimized out.
+ */
+void
+_mtx_enter(struct mtx *mtxp, int type, const char *file, int line)
+{
+	struct mtx	*mpp = mtxp;
+
+	/* bits only valid on mtx_exit() */
+	MPASS4(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0,
+	    STR_mtx_bad_type, file, line);
+
+	if ((type) & MTX_SPIN) {
+		/*
+		 * Easy cases of spin locks:
+		 *
+		 * 1) We already own the lock and will simply recurse on it (if
+		 *    RLIKELY)
+		 *
+		 * 2) The lock is free, we just get it
+		 */
+		if ((type) & MTX_RLIKELY) {
+			/*
+			 * Check for recursion, if we already have this
+			 * lock we just bump the recursion count.
+			 */
+			if (mpp->mtx_lock == (uintptr_t)CURTHD) {
+				mpp->mtx_recurse++;
+				goto done;
+			}
+		}
+
+		if (((type) & MTX_TOPHALF) == 0) {
+			/*
+			 * If an interrupt thread uses this we must block
+			 * interrupts here.
+			 */
+			if ((type) & MTX_FIRST) {
+				ASS_IEN;
+				disable_intr();
+				_getlock_norecurse(mpp, CURTHD,
+				    (type) & MTX_HARDOPTS);
+			} else {
+				_getlock_spin_block(mpp, CURTHD,
+				    (type) & MTX_HARDOPTS);
+			}
+		} else
+			_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
+	} else {
+		/* Sleep locks */
+		if ((type) & MTX_RLIKELY)
+			_getlock_sleep(mpp, CURTHD, (type) & MTX_HARDOPTS);
+		else
+			_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
+	}
+done:
+	WITNESS_ENTER(mpp, type, file, line);
+	if (((type) & MTX_QUIET) == 0)
+		CTR5(KTR_LOCK, STR_mtx_enter_fmt,
+		    mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
+
+}
+
+/*
+ * Attempt to get MTX_DEF lock, return non-zero if lock acquired.
+ *
+ * XXX DOES NOT HANDLE RECURSION
+ */
+int
+_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line)
+{
+	struct mtx	*const mpp = mtxp;
+	int	rval;
+
+	rval = _obtain_lock(mpp, CURTHD);
+#ifdef WITNESS
+	if (rval && mpp->mtx_witness != NULL) {
+		MPASS(mpp->mtx_recurse == 0);
+		witness_try_enter(mpp, type, file, line);
+	}
+#endif	/* WITNESS */
+	if (((type) & MTX_QUIET) == 0)
+		CTR5(KTR_LOCK, STR_mtx_try_enter_fmt,
+		    mpp->mtx_description, mpp, rval, file, line);
+
+	return rval;
+}
+
+/*
+ * Release lock m.
+ */
+void
+_mtx_exit(struct mtx *mtxp, int type, const char *file, int line)
+{
+	struct mtx	*const mpp = mtxp;
+
+	MPASS4(mtx_owned(mpp), STR_mtx_owned, file, line);
+	WITNESS_EXIT(mpp, type, file, line);
+	if (((type) & MTX_QUIET) == 0)
+		CTR5(KTR_LOCK, STR_mtx_exit_fmt,
+		    mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
+	if ((type) & MTX_SPIN) {
+		if ((type) & MTX_NORECURSE) {
+			int mtx_intr = mpp->mtx_saveintr;
+
+			MPASS4(mpp->mtx_recurse == 0, STR_mtx_recurse,
+			    file, line);
+			_release_lock_quick(mpp);
+			if (((type) & MTX_TOPHALF) == 0) {
+				if ((type) & MTX_FIRST) {
+					ASS_IDIS;
+					enable_intr();
+				} else
+					restore_intr(mtx_intr);
+			}
+		} else {
+			if (((type & MTX_TOPHALF) == 0) &&
+			    (type & MTX_FIRST)) {
+				ASS_IDIS;
+				ASS_SIEN(mpp);
+			}
+			_exitlock_spin(mpp);
+		}
+	} else {
+		/* Handle sleep locks */
+		if ((type) & MTX_RLIKELY)
+			_exitlock(mpp, CURTHD, (type) & MTX_HARDOPTS);
+		else {
+			_exitlock_norecurse(mpp, CURTHD,
+			    (type) & MTX_HARDOPTS);
+		}
+	}
+}
+
 void
 mtx_enter_hard(struct mtx *m, int type, int saveintr)
 {
@@ -512,6 +792,37 @@ mtx_exit_hard(struct mtx *m, int type)
 	}
 }
 
+#ifdef INVARIANTS
+void
+mtx_assert(struct mtx *m, int what)
+{
+	switch ((what)) {
+	case MA_OWNED:
+	case MA_OWNED | MA_RECURSED:
+	case MA_OWNED | MA_NOTRECURSED:
+		if (!mtx_owned((m)))
+			panic("mutex %s not owned at %s:%d",
+			    (m)->mtx_description, __FILE__, __LINE__);
+		if (mtx_recursed((m))) {
+			if (((what) & MA_NOTRECURSED) != 0)
+				panic("mutex %s recursed at %s:%d",
+				    (m)->mtx_description, __FILE__, __LINE__);
+		} else if (((what) & MA_RECURSED) != 0) {
+			panic("mutex %s unrecursed at %s:%d",
+			    (m)->mtx_description, __FILE__, __LINE__);
+		}
+		break;
+	case MA_NOTOWNED:
+		if (mtx_owned((m)))
+			panic("mutex %s owned at %s:%d",
+			    (m)->mtx_description, __FILE__, __LINE__);
+		break;
+	default:
+		panic("unknown mtx_assert at %s:%d", __FILE__, __LINE__);
+	}
+}
+#endif
+
 #define MV_DESTROY	0	/* validate before destory */
 #define MV_INIT		1	/* validate before init */
 
@@ -818,13 +1129,13 @@ static struct witness_blessed blessed_list[] = {
 };
 static int blessed_count = sizeof(blessed_list) / sizeof(struct witness_blessed);
 
-void
+static void
 witness_init(struct mtx *m, int flag)
 {
 	m->mtx_witness = enroll(m->mtx_description, flag);
 }
 
-void
+static void
 witness_destroy(struct mtx *m)
 {
 	struct mtx *m1;
@@ -841,6 +1152,36 @@ witness_destroy(struct mtx *m)
 
 }
 
+static void
+witness_display(void(*prnt)(const char *fmt, ...))
+{
+	struct witness *w, *w1;
+
+	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
+	witness_levelall();
+
+	for (w = w_all; w; w = w->w_next) {
+		if (w->w_file == NULL)
+			continue;
+		for (w1 = w_all; w1; w1 = w1->w_next) {
+			if (isitmychild(w1, w))
+				break;
+		}
+		if (w1 != NULL)
+			continue;
+		/*
+		 * This lock has no anscestors, display its descendants. 
+		 */
+		witness_displaydescendants(prnt, w);
+	}
+	prnt("\nMutex which were never acquired\n");
+	for (w = w_all; w; w = w->w_next) {
+		if (w->w_file != NULL)
+			continue;
+		prnt("%s\n", w->w_description);
+	}
+}
+
 void
 witness_enter(struct mtx *m, int flags, const char *file, int line)
 {
@@ -852,9 +1193,7 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 	int go_into_ddb = 0;
 #endif /* DDB */
 
-	if (witness_cold)
-		return;
-	if (panicstr)
+	if (witness_cold || m->mtx_witness == NULL || panicstr)
 		return;
 	w = m->mtx_witness;
 	p = CURPROC;
@@ -994,53 +1333,6 @@ out:
 }
 
 void
-witness_exit(struct mtx *m, int flags, const char *file, int line)
-{
-	struct witness *w;
-
-	if (witness_cold)
-		return;
-	if (panicstr)
-		return;
-	w = m->mtx_witness;
-
-	if (flags & MTX_SPIN) {
-		if (!(w->w_spin))
-			panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
-			    " %s:%d", m->mtx_description, file, line);
-		if (mtx_recursed(m)) {
-			if (!(w->w_recurse))
-				panic("mutex_exit: recursion on non-recursive"
-				    " mutex %s @ %s:%d", m->mtx_description,
-				    file, line); 
-			return;
-		}
-		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
-		PCPU_SET(witness_spin_check,
-		    PCPU_GET(witness_spin_check) & ~w->w_level);
-		mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
-		return;
-	}
-	if (w->w_spin)
-		panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
-		    m->mtx_description, file, line);
-
-	if (mtx_recursed(m)) {
-		if (!(w->w_recurse))
-			panic("mutex_exit: recursion on non-recursive"
-			    " mutex %s @ %s:%d", m->mtx_description,
-			    file, line); 
-		return;
-	}
-
-	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
-		panic("switchable mtx_exit() of %s when not legal @ %s:%d",
-			    m->mtx_description, file, line);
-	LIST_REMOVE(m, mtx_held);
-	m->mtx_held.le_prev = NULL;
-}
-
-void
 witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 {
 	struct proc *p;
@@ -1094,33 +1386,48 @@ witness_try_enter(struct mtx *m, int flags, const char *file, int line)
 }
 
 void
-witness_display(void(*prnt)(const char *fmt, ...))
+witness_exit(struct mtx *m, int flags, const char *file, int line)
 {
-	struct witness *w, *w1;
+	struct witness *w;
 
-	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
-	witness_levelall();
+	if (witness_cold || m->mtx_witness == NULL || panicstr)
+		return;
+	w = m->mtx_witness;
 
-	for (w = w_all; w; w = w->w_next) {
-		if (w->w_file == NULL)
-			continue;
-		for (w1 = w_all; w1; w1 = w1->w_next) {
-			if (isitmychild(w1, w))
-				break;
+	if (flags & MTX_SPIN) {
+		if (!(w->w_spin))
+			panic("mutex_exit: MTX_SPIN on MTX_DEF mutex %s @"
+			    " %s:%d", m->mtx_description, file, line);
+		if (mtx_recursed(m)) {
+			if (!(w->w_recurse))
+				panic("mutex_exit: recursion on non-recursive"
+				    " mutex %s @ %s:%d", m->mtx_description,
+				    file, line); 
+			return;
 		}
-		if (w1 != NULL)
-			continue;
-		/*
-		 * This lock has no anscestors, display its descendants. 
-		 */
-		witness_displaydescendants(prnt, w);
+		mtx_enter(&w_mtx, MTX_SPIN | MTX_QUIET);
+		PCPU_SET(witness_spin_check,
+		    PCPU_GET(witness_spin_check) & ~w->w_level);
+		mtx_exit(&w_mtx, MTX_SPIN | MTX_QUIET);
+		return;
 	}
-	prnt("\nMutex which were never acquired\n");
-	for (w = w_all; w; w = w->w_next) {
-		if (w->w_file != NULL)
-			continue;
-		prnt("%s\n", w->w_description);
+	if (w->w_spin)
+		panic("mutex_exit: MTX_DEF on MTX_SPIN mutex %s @ %s:%d",
+		    m->mtx_description, file, line);
+
+	if (mtx_recursed(m)) {
+		if (!(w->w_recurse))
+			panic("mutex_exit: recursion on non-recursive"
+			    " mutex %s @ %s:%d", m->mtx_description,
+			    file, line); 
+		return;
 	}
+
+	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
+		panic("switchable mtx_exit() of %s when not legal @ %s:%d",
+			    m->mtx_description, file, line);
+	LIST_REMOVE(m, mtx_held);
+	m->mtx_held.le_prev = NULL;
 }
 
 int
@@ -1466,6 +1773,9 @@ witness_save(struct mtx *m, const char **filep, int *linep)
 {
 
 	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
+	if (m->mtx_witness == NULL)
+		return;
+
 	*filep = m->mtx_witness->w_file;
 	*linep = m->mtx_witness->w_line;
 }
@@ -1475,6 +1785,9 @@ witness_restore(struct mtx *m, const char *file, int line)
 {
 
 	KASSERT(!witness_cold, ("%s: witness_cold\n", __FUNCTION__));
+	if (m->mtx_witness == NULL)
+		return;
+
 	m->mtx_witness->w_file = file;
 	m->mtx_witness->w_line = line;
 }
diff --git a/sys/net/if.c b/sys/net/if.c
index 0c6fc84..8239b4d 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -116,7 +116,8 @@ ifinit(dummy)
 			    ifp->if_name, ifp->if_unit);
 			ifp->if_snd.ifq_maxlen = ifqmaxlen;
 		}
-		if (ifp->if_snd.ifq_mtx.mtx_description == NULL) {
+		/* XXX This is an access violation of the mutex internals. */
+		if (ifp->if_snd.ifq_mtx.mtx_flags == 0) {
 			printf("%s%d XXX: driver didn't initialize queue mtx\n",
 			    ifp->if_name, ifp->if_unit);
 			mtx_init(&ifp->if_snd.ifq_mtx, "unknown", MTX_DEF);
diff --git a/sys/powerpc/include/mutex.h b/sys/powerpc/include/mutex.h
index 5c37f54..40717b0 100644
--- a/sys/powerpc/include/mutex.h
+++ b/sys/powerpc/include/mutex.h
@@ -68,6 +68,8 @@ extern char STR_SIEN[];
  *--------------------------------------------------------------------------
  */
 
+#ifdef _KERN_MUTEX_C_
+
 #define	_V(x)	__STRING(x)
 
 /*
@@ -84,6 +86,10 @@ extern char STR_SIEN[];
 	}								\
 } while (0)
 
+#undef _V
+
+#endif	/* _KERN_MUTEX_C_ */
+
 #endif	/* _KERNEL */
 
 #else	/* !LOCORE */
diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h
index 4ce9d36..f2365f0 100644
--- a/sys/sys/mutex.h
+++ b/sys/sys/mutex.h
@@ -48,16 +48,6 @@
 #ifdef _KERNEL
 
 /*
- * If kern_mutex.c is being built, compile non-inlined versions of various
- * functions so that kernel modules can use them.
- */
-#ifndef _KERN_MUTEX_C_
-#define _MTX_INLINE	static __inline
-#else
-#define _MTX_INLINE
-#endif
-
-/*
  * Mutex flags
  *
  * Types
@@ -88,23 +78,7 @@
 
 #ifndef LOCORE
 
-#ifdef WITNESS
-struct mtx_debug {
-	struct witness	*mtxd_witness;
-	LIST_ENTRY(mtx)	mtxd_held;
-	const char	*mtxd_file;
-	int		mtxd_line;
-	const char	*mtxd_description;
-};
-
-#define mtx_description	mtx_union.mtxu_debug->mtxd_description
-#define mtx_held	mtx_union.mtxu_debug->mtxd_held
-#define	mtx_line	mtx_union.mtxu_debug->mtxd_line
-#define	mtx_file	mtx_union.mtxu_debug->mtxd_file
-#define	mtx_witness	mtx_union.mtxu_debug->mtxd_witness
-#else	/* WITNESS */
-#define mtx_description	mtx_union.mtxu_description
-#endif	/* WITNESS */
+struct mtx_debug;
 
 /*
  * Sleep/spin mutex
@@ -129,19 +103,15 @@ struct mtx {
 #ifdef _KERNEL
 /* Prototypes */
 void	mtx_init(struct mtx *m, const char *description, int flag);
-void	mtx_enter_hard(struct mtx *, int type, int saveintr);
-void	mtx_exit_hard(struct mtx *, int type);
 void	mtx_destroy(struct mtx *m);
 
 /*
  * Wrap the following functions with cpp macros so that filenames and line
  * numbers are embedded in the code correctly.
  */
-#if (defined(KLD_MODULE) || defined(_KERN_MUTEX_C_))
 void	_mtx_enter(struct mtx *mtxp, int type, const char *file, int line);
 int	_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line);
 void	_mtx_exit(struct mtx *mtxp, int type, const char *file, int line);
-#endif
 
 #define	mtx_enter(mtxp, type)						\
 	_mtx_enter((mtxp), (type), __FILE__, __LINE__)
@@ -207,7 +177,6 @@ do {									\
 	if (mtx_owned(&Giant))						\
 		WITNESS_RESTORE(&Giant, Giant)
 
-
 /*
  * Debugging
  */
@@ -216,31 +185,7 @@ do {									\
 #define MA_NOTOWNED	2
 #define MA_RECURSED	4
 #define MA_NOTRECURSED	8
-#define mtx_assert(m, what) do {					\
-	switch ((what)) {						\
-	case MA_OWNED:							\
-	case MA_OWNED | MA_RECURSED:					\
-	case MA_OWNED | MA_NOTRECURSED:					\
-		if (!mtx_owned((m)))					\
-			panic("mutex %s not owned at %s:%d",		\
-			    (m)->mtx_description, __FILE__, __LINE__);	\
-		if (mtx_recursed((m))) {				\
-			if (((what) & MA_NOTRECURSED) != 0)		\
-				panic("mutex %s recursed at %s:%d",	\
-				    (m)->mtx_description, __FILE__, __LINE__); \
-		} else if (((what) & MA_RECURSED) != 0)			\
-				panic("mutex %s unrecursed at %s:%d",	\
-				    (m)->mtx_description, __FILE__, __LINE__); \
-		break;							\
-	case MA_NOTOWNED:						\
-		if (mtx_owned((m)))					\
-			panic("mutex %s owned at %s:%d",		\
-			    (m)->mtx_description, __FILE__, __LINE__);	\
-		break;							\
-	default:							\
-		panic("unknown mtx_assert at %s:%d", __FILE__, __LINE__); \
-	}								\
-} while(0)
+void	mtx_assert(struct mtx *m, int what);
 #else	/* INVARIANTS */
 #define mtx_assert(m, what)
 #endif	/* INVARIANTS */
@@ -265,169 +210,6 @@ do {									\
 #define	MPASS4(ex, what, file, line)
 #endif	/* MUTEX_DEBUG */
 
-#ifdef	WITNESS
-#define WITNESS_ENTER(m, t, f, l)					\
-	if ((m)->mtx_witness != NULL)					\
-		witness_enter((m), (t), (f), (l))
-#define WITNESS_EXIT(m, t, f, l)					\
-	if ((m)->mtx_witness != NULL)					\
-		witness_exit((m), (t), (f), (l))
-
-#define	WITNESS_SLEEP(check, m) witness_sleep(check, (m), __FILE__, __LINE__)
-#define	WITNESS_SAVE_DECL(n)						\
-	const char * __CONCAT(n, __wf);					\
-	int __CONCAT(n, __wl)
-
-#define	WITNESS_SAVE(m, n) 						\
-do {									\
-	if ((m)->mtx_witness != NULL) 					\
-		witness_save(m, &__CONCAT(n, __wf), &__CONCAT(n, __wl));\
-} while (0)
-
-#define	WITNESS_RESTORE(m, n) 						\
-do {									\
-	if ((m)->mtx_witness != NULL)					\
-		witness_restore(m, __CONCAT(n, __wf), __CONCAT(n, __wl));\
-} while (0)
-
-void	witness_init(struct mtx *, int flag);
-void	witness_destroy(struct mtx *);
-void	witness_enter(struct mtx *, int, const char *, int);
-void	witness_try_enter(struct mtx *, int, const char *, int);
-void	witness_exit(struct mtx *, int, const char *, int);
-void	witness_display(void(*)(const char *fmt, ...));
-int	witness_list(struct proc *);
-int	witness_sleep(int, struct mtx *, const char *, int);
-void	witness_save(struct mtx *, const char **, int *);
-void	witness_restore(struct mtx *, const char *, int);
-#else	/* WITNESS */
-#define WITNESS_ENTER(m, t, f, l)
-#define WITNESS_EXIT(m, t, f, l)
-#define	WITNESS_SLEEP(check, m)
-#define	WITNESS_SAVE_DECL(n)
-#define	WITNESS_SAVE(m, n)
-#define	WITNESS_RESTORE(m, n)
-
-/*
- * flag++ is slezoid way of shutting up unused parameter warning
- * in mtx_init()
- */
-#define witness_init(m, flag) flag++
-#define witness_destroy(m)
-#define witness_enter(m, t, f, l)
-#define witness_try_enter(m, t, f, l)
-#define witness_exit(m, t, f, l)
-#endif	/* WITNESS */
-
-/*
- * Assembly macros (for internal use only)
- *------------------------------------------------------------------------------
- */
-
-#define	_V(x)	__STRING(x)
-
-/*
- * Default, unoptimized mutex micro-operations
- */
-
-#ifndef _obtain_lock
-/* Actually obtain mtx_lock */
-#define _obtain_lock(mp, tid)						\
-	atomic_cmpset_acq_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED, (tid))
-#endif
-
-#ifndef _release_lock
-/* Actually release mtx_lock */
-#define _release_lock(mp, tid)		       				\
-	atomic_cmpset_rel_ptr(&(mp)->mtx_lock, (tid), (void *)MTX_UNOWNED)
-#endif
-
-#ifndef _release_lock_quick
-/* Actually release mtx_lock quickly assuming that we own it */
-#define	_release_lock_quick(mp) 					\
-	atomic_store_rel_ptr(&(mp)->mtx_lock, (void *)MTX_UNOWNED)
-#endif
-
-#ifndef _getlock_sleep
-/* Get a sleep lock, deal with recursion inline. */
-#define	_getlock_sleep(mp, tid, type) do {				\
-	if (!_obtain_lock(mp, tid)) {					\
-		if (((mp)->mtx_lock & MTX_FLAGMASK) != ((uintptr_t)(tid)))\
-			mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0);	\
-		else {							\
-			atomic_set_ptr(&(mp)->mtx_lock, MTX_RECURSED);	\
-			(mp)->mtx_recurse++;				\
-		}							\
-	}								\
-} while (0)
-#endif
-
-#ifndef _getlock_spin_block
-/* Get a spin lock, handle recursion inline (as the less common case) */
-#define	_getlock_spin_block(mp, tid, type) do {				\
-	u_int _mtx_intr = save_intr();					\
-	disable_intr();							\
-	if (!_obtain_lock(mp, tid))					\
-		mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _mtx_intr);	\
-	else								\
-		(mp)->mtx_saveintr = _mtx_intr;				\
-} while (0)
-#endif
-
-#ifndef _getlock_norecurse
-/*
- * Get a lock without any recursion handling. Calls the hard enter function if
- * we can't get it inline.
- */
-#define	_getlock_norecurse(mp, tid, type) do {				\
-	if (!_obtain_lock(mp, tid))					\
-		mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0);		\
-} while (0)
-#endif
-
-#ifndef _exitlock_norecurse
-/*
- * Release a sleep lock assuming we haven't recursed on it, recursion is handled
- * in the hard function.
- */
-#define	_exitlock_norecurse(mp, tid, type) do {				\
-	if (!_release_lock(mp, tid))					\
-		mtx_exit_hard((mp), (type) & MTX_HARDOPTS);		\
-} while (0)
-#endif
-
-#ifndef _exitlock
-/*
- * Release a sleep lock when its likely we recursed (the code to
- * deal with simple recursion is inline).
- */
-#define	_exitlock(mp, tid, type) do {					\
-	if (!_release_lock(mp, tid)) {					\
-		if ((mp)->mtx_lock & MTX_RECURSED) {			\
-			if (--((mp)->mtx_recurse) == 0)			\
-				atomic_clear_ptr(&(mp)->mtx_lock,	\
-				    MTX_RECURSED);			\
-		} else {						\
-			mtx_exit_hard((mp), (type) & MTX_HARDOPTS);	\
-		}							\
-	}								\
-} while (0)
-#endif
-
-#ifndef _exitlock_spin
-/* Release a spin lock (with possible recursion). */
-#define	_exitlock_spin(mp) do {						\
-	if (!mtx_recursed((mp))) {					\
-		int _mtx_intr = (mp)->mtx_saveintr;			\
-									\
-		_release_lock_quick(mp);				\
-		restore_intr(_mtx_intr);				\
-	} else {							\
-		(mp)->mtx_recurse--;					\
-	}								\
-} while (0)
-#endif
-
 /*
  * Externally visible mutex functions.
  *------------------------------------------------------------------------------
@@ -460,158 +242,41 @@ extern	char STR_mtx_recurse[];
 extern	char STR_mtx_try_enter_fmt[];
 #endif	/* _KERN_MUTEX_C_ */
 
-#ifndef KLD_MODULE
-/*
- * Get lock 'm', the macro handles the easy (and most common cases) and leaves
- * the slow stuff to the mtx_enter_hard() function.
- *
- * Note: since type is usually a constant much of this code is optimized out.
- */
-_MTX_INLINE void
-_mtx_enter(struct mtx *mtxp, int type, const char *file, int line)
-{
-	struct mtx	*mpp = mtxp;
-
-	/* bits only valid on mtx_exit() */
-	MPASS4(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0,
-	    STR_mtx_bad_type, file, line);
-
-	if ((type) & MTX_SPIN) {
-		/*
-		 * Easy cases of spin locks:
-		 *
-		 * 1) We already own the lock and will simply recurse on it (if
-		 *    RLIKELY)
-		 *
-		 * 2) The lock is free, we just get it
-		 */
-		if ((type) & MTX_RLIKELY) {
-			/*
-			 * Check for recursion, if we already have this
-			 * lock we just bump the recursion count.
-			 */
-			if (mpp->mtx_lock == (uintptr_t)CURTHD) {
-				mpp->mtx_recurse++;
-				goto done;
-			}
-		}
-
-		if (((type) & MTX_TOPHALF) == 0) {
-			/*
-			 * If an interrupt thread uses this we must block
-			 * interrupts here.
-			 */
-			if ((type) & MTX_FIRST) {
-				ASS_IEN;
-				disable_intr();
-				_getlock_norecurse(mpp, CURTHD,
-				    (type) & MTX_HARDOPTS);
-			} else {
-				_getlock_spin_block(mpp, CURTHD,
-				    (type) & MTX_HARDOPTS);
-			}
-		} else
-			_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
-	} else {
-		/* Sleep locks */
-		if ((type) & MTX_RLIKELY)
-			_getlock_sleep(mpp, CURTHD, (type) & MTX_HARDOPTS);
-		else
-			_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
-	}
-done:
-	WITNESS_ENTER(mpp, type, file, line);
-	if (((type) & MTX_QUIET) == 0)
-		CTR5(KTR_LOCK, STR_mtx_enter_fmt,
-		    mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
-
-}
+#ifdef	WITNESS
+void	witness_save(struct mtx *, const char **, int *);
+void	witness_restore(struct mtx *, const char *, int);
+void	witness_enter(struct mtx *, int, const char *, int);
+void	witness_try_enter(struct mtx *, int, const char *, int);
+void	witness_exit(struct mtx *, int, const char *, int);
+int	witness_list(struct proc *);
+int	witness_sleep(int, struct mtx *, const char *, int);
 
-/*
- * Attempt to get MTX_DEF lock, return non-zero if lock acquired.
- *
- * XXX DOES NOT HANDLE RECURSION
- */
-_MTX_INLINE int
-_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line)
-{
-	struct mtx	*const mpp = mtxp;
-	int	rval;
-
-	rval = _obtain_lock(mpp, CURTHD);
-#ifdef WITNESS
-	if (rval && mpp->mtx_witness != NULL) {
-		MPASS(mpp->mtx_recurse == 0);
-		witness_try_enter(mpp, type, file, line);
-	}
-#endif	/* WITNESS */
-	if (((type) & MTX_QUIET) == 0)
-		CTR5(KTR_LOCK, STR_mtx_try_enter_fmt,
-		    mpp->mtx_description, mpp, rval, file, line);
+#define WITNESS_ENTER(m, t, f, l) witness_enter((m), (t), (f), (l))
+#define WITNESS_EXIT(m, t, f, l) witness_exit((m), (t), (f), (l))
+#define	WITNESS_SLEEP(check, m) witness_sleep(check, (m), __FILE__, __LINE__)
+#define	WITNESS_SAVE_DECL(n)						\
+	const char * __CONCAT(n, __wf);					\
+	int __CONCAT(n, __wl)
+#define	WITNESS_SAVE(m, n) 						\
+	witness_save(m, &__CONCAT(n, __wf), &__CONCAT(n, __wl))
+#define	WITNESS_RESTORE(m, n) 						\
+	witness_restore(m, __CONCAT(n, __wf), __CONCAT(n, __wl))
+#else	/* WITNESS */
+#define witness_enter(m, t, f, l)
+#define witness_tryenter(m, t, f, l)
+#define witness_exit(m, t, f, l)
+#define	witness_list(p)
+#define	witness_sleep(c, m, f, l)
 
-	return rval;
-}
+#define WITNESS_ENTER(m, t, f, l)
+#define WITNESS_EXIT(m, t, f, l)
+#define	WITNESS_SLEEP(check, m)
+#define	WITNESS_SAVE_DECL(n)
+#define	WITNESS_SAVE(m, n)
+#define	WITNESS_RESTORE(m, n)
+#endif	/* WITNESS */
 
-/*
- * Release lock m.
- */
-_MTX_INLINE void
-_mtx_exit(struct mtx *mtxp, int type, const char *file, int line)
-{
-	struct mtx	*const mpp = mtxp;
-
-	MPASS4(mtx_owned(mpp), STR_mtx_owned, file, line);
-	WITNESS_EXIT(mpp, type, file, line);
-	if (((type) & MTX_QUIET) == 0)
-		CTR5(KTR_LOCK, STR_mtx_exit_fmt,
-		    mpp->mtx_description, mpp, mpp->mtx_recurse, file, line);
-	if ((type) & MTX_SPIN) {
-		if ((type) & MTX_NORECURSE) {
-			int mtx_intr = mpp->mtx_saveintr;
-
-			MPASS4(mpp->mtx_recurse == 0, STR_mtx_recurse,
-			    file, line);
-			_release_lock_quick(mpp);
-			if (((type) & MTX_TOPHALF) == 0) {
-				if ((type) & MTX_FIRST) {
-					ASS_IDIS;
-					enable_intr();
-				} else
-					restore_intr(mtx_intr);
-			}
-		} else {
-			if (((type & MTX_TOPHALF) == 0) &&
-			    (type & MTX_FIRST)) {
-				ASS_IDIS;
-				ASS_SIEN(mpp);
-			}
-			_exitlock_spin(mpp);
-		}
-	} else {
-		/* Handle sleep locks */
-		if ((type) & MTX_RLIKELY)
-			_exitlock(mpp, CURTHD, (type) & MTX_HARDOPTS);
-		else {
-			_exitlock_norecurse(mpp, CURTHD,
-			    (type) & MTX_HARDOPTS);
-		}
-	}
-}
-
-#endif	/* KLD_MODULE */
-
-/* Avoid namespace pollution */
-#ifndef _KERN_MUTEX_C_
-#undef	_obtain_lock
-#undef	_release_lock
-#undef	_release_lock_quick
-#undef	_getlock_sleep
-#undef	_getlock_spin_block
-#undef	_getlock_norecurse
-#undef	_exitlock_norecurse
-#undef	_exitlock
-#undef	_exitlock_spin
-#endif	/* !_KERN_MUTEX_C_ */
+/* XXX jasone Move. */ 
 
 #endif	/* _KERNEL */
 #endif	/* !LOCORE */