Optimize lockmgr in order to get rid of the pool mutex interlock, of the

state transitioning flags and of msleep(9) callings.
Use, instead, an algorithm very similar to what sx(9) and rwlock(9)
alredy do and direct accesses to the sleepqueue(9) primitive.

In order to avoid writer starvation a mechanism very similar to what
rwlock(9) uses now is implemented, with the correspective per-thread
shared lockmgrs counter.

This patch also adds 2 new functions to lockmgr KPI: lockmgr_rw() and
lockmgr_args_rw().  These two are like the 2 "normal" versions, but they
both accept a rwlock as interlock.  In order to realize this, the general
lockmgr manager function "__lockmgr_args()" has been implemented through
the generic lock layer. It supports all the blocking primitives, but
currently only these 2 mappers live.

The patch drops the support for WITNESS atm, but it will be probabilly
added soon. Also, there is a little race in the draining code which is
also present in the current CVS stock implementation: if some sharers,
once they wakeup, are in the runqueue they can contend the lock with
the exclusive drainer.  This is hard to be fixed but the now committed
code mitigate this issue a lot better than the (past) CVS version.
In addition assertive KA_HELD and KA_UNHELD have been made mute
assertions because they are dangerous and they will be nomore supported
soon.

In order to avoid namespace pollution, stack.h is splitted into two
parts: one which includes only the "struct stack" definition (_stack.h)
and one defining the KPI.  In this way, newly added _lockmgr.h can
just include _stack.h.

Kernel ABI results heavilly changed by this commit (the now committed
version of "struct lock" is a lot smaller than the previous one) and
KPI results broken by lockmgr_rw() / lockmgr_args_rw() introduction,
so manpages and __FreeBSD_version will be updated accordingly.

Tested by:      kris, pho, jeff, danger
Reviewed by:    jeff
Sponsored by:   Google, Summer of Code program 2007

10 files changed, 1043 insertions, 826 deletions

diff --git a/sys/fs/unionfs/union_subr.c b/sys/fs/unionfs/union_subr.c
index 2db12a5..05e9399 100644
--- a/sys/fs/unionfs/union_subr.c
+++ b/sys/fs/unionfs/union_subr.c
@@ -538,12 +538,13 @@ static void
 unionfs_node_update(struct unionfs_node *unp, struct vnode *uvp,
 		    struct thread *td)
 {
-	int		count, lockcnt;
+	unsigned	count, lockrec;
 	struct vnode   *vp;
 	struct vnode   *lvp;
 
 	vp = UNIONFSTOV(unp);
 	lvp = unp->un_lowervp;
+	ASSERT_VOP_ELOCKED(lvp, "unionfs_node_update");
 
 	/*
 	 * lock update
@@ -551,11 +552,9 @@ unionfs_node_update(struct unionfs_node *unp, struct vnode *uvp,
 	VI_LOCK(vp);
 	unp->un_uppervp = uvp;
 	vp->v_vnlock = uvp->v_vnlock;
-	lockcnt = lvp->v_vnlock->lk_exclusivecount;
-	if (lockcnt <= 0)
-		panic("unionfs: no exclusive lock");
 	VI_UNLOCK(vp);
-	for (count = 1; count < lockcnt; count++)
+	lockrec = lvp->v_vnlock->lk_recurse;
+	for (count = 0; count < lockrec; count++)
 		vn_lock(uvp, LK_EXCLUSIVE | LK_CANRECURSE | LK_RETRY);
 }
 
diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
index 987b361..93c262b 100644
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -1,692 +1,900 @@
 /*-
- * Copyright (c) 1995
- *	The Regents of the University of California.  All rights reserved.
- *
- * Copyright (C) 1997
- *	John S. Dyson.  All rights reserved.
- *
- * This code contains ideas from software contributed to Berkeley by
- * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating
- * System project at Carnegie-Mellon University.
+ * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
+ * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
+ *    notice(s), this list of conditions and the following disclaimer as
+ *    the first lines of this file unmodified other than the possible
+ *    addition of one or more copyright notices.
  * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
+ *    notice(s), this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by the University of
- *	California, Berkeley and its contributors.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
  *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)kern_lock.c	8.18 (Berkeley) 5/21/95
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
  */
 
+#include "opt_ddb.h"
+
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include "opt_ddb.h"
-#include "opt_global.h"
-
 #include <sys/param.h>
-#include <sys/kdb.h>
-#include <sys/kernel.h>
 #include <sys/ktr.h>
 #include <sys/lock.h>
+#include <sys/lock_profile.h>
 #include <sys/lockmgr.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
-#include <sys/systm.h>
-#include <sys/lock_profile.h>
+#include <sys/sleepqueue.h>
 #ifdef DEBUG_LOCKS
 #include <sys/stack.h>
 #endif
+#include <sys/systm.h>
 
-#define	LOCKMGR_TRYOP(x)	((x) & LK_NOWAIT)
-#define	LOCKMGR_TRYW(x)		(LOCKMGR_TRYOP((x)) ? LOP_TRYLOCK : 0)
-#define	LOCKMGR_UNHELD(x)	(((x) & (LK_HAVE_EXCL | LK_SHARE_NONZERO)) == 0)
-#define	LOCKMGR_NOTOWNER(td)	((td) != curthread && (td) != LK_KERNPROC)
+#include <machine/cpu.h>
 
-static void	assert_lockmgr(struct lock_object *lock, int what);
 #ifdef DDB
 #include <ddb/ddb.h>
-static void	db_show_lockmgr(struct lock_object *lock);
 #endif
-static void	lock_lockmgr(struct lock_object *lock, int how);
-static int	unlock_lockmgr(struct lock_object *lock);
+
+CTASSERT(((LK_CANRECURSE | LK_NOSHARE) & LO_CLASSFLAGS) ==
+    (LK_CANRECURSE | LK_NOSHARE));
+
+#define	SQ_EXCLUSIVE_QUEUE	0
+#define	SQ_SHARED_QUEUE		1
+
+#ifndef INVARIANTS
+#define	_lockmgr_assert(lk, what, file, line)
+#define	TD_LOCKS_INC(td)
+#define	TD_LOCKS_DEC(td)
+#else
+#define	TD_LOCKS_INC(td)	((td)->td_locks++)
+#define	TD_LOCKS_DEC(td)	((td)->td_locks--)
+#endif
+#define	TD_SLOCKS_INC(td)	((td)->td_lk_slocks++)
+#define	TD_SLOCKS_DEC(td)	((td)->td_lk_slocks--)
+
+#ifndef DEBUG_LOCKS
+#define	STACK_PRINT(lk)
+#define	STACK_SAVE(lk)
+#define	STACK_ZERO(lk)
+#else
+#define	STACK_PRINT(lk)	stack_print_ddb(&(lk)->lk_stack)
+#define	STACK_SAVE(lk)	stack_save(&(lk)->lk_stack)
+#define	STACK_ZERO(lk)	stack_zero(&(lk)->lk_stack)
+#endif
+
+#define	LOCK_LOG2(lk, string, arg1, arg2)				\
+	if (LOCK_LOG_TEST(&(lk)->lock_object, 0))			\
+		CTR2(KTR_LOCK, (string), (arg1), (arg2))
+#define	LOCK_LOG3(lk, string, arg1, arg2, arg3)				\
+	if (LOCK_LOG_TEST(&(lk)->lock_object, 0))			\
+		CTR3(KTR_LOCK, (string), (arg1), (arg2), (arg3))
+
+#define	LK_TRYOP(x)							\
+	((x) & LK_NOWAIT)
+#define	LK_CAN_SHARE(x)							\
+	(((x) & LK_SHARE) && (((x) & LK_EXCLUSIVE_WAITERS) == 0 ||	\
+	curthread->td_lk_slocks))
+
+#define	lockmgr_disowned(lk)						\
+	(((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC)
+
+#define	lockmgr_xlocked(lk)						\
+	(((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread)
+
+static void	 assert_lockmgr(struct lock_object *lock, int how);
+#ifdef DDB
+static void	 db_show_lockmgr(struct lock_object *lock);
+#endif
+static void	 lock_lockmgr(struct lock_object *lock, int how);
+static int	 unlock_lockmgr(struct lock_object *lock);
 
 struct lock_class lock_class_lockmgr = {
 	.lc_name = "lockmgr",
-	.lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE,
+	.lc_flags = LC_RECURSABLE | LC_SLEEPABLE | LC_SLEEPLOCK | LC_UPGRADABLE,
 	.lc_assert = assert_lockmgr,
 #ifdef DDB
 	.lc_ddb_show = db_show_lockmgr,
 #endif
 	.lc_lock = lock_lockmgr,
-	.lc_unlock = unlock_lockmgr,
+	.lc_unlock = unlock_lockmgr
 };
 
-#ifndef INVARIANTS
-#define	_lockmgr_assert(lkp, what, file, line)
-#endif
+static __inline struct thread *
+lockmgr_xholder(struct lock *lk)
+{
+	uintptr_t x;
+
+	x = lk->lk_lock;
+	return ((x & LK_SHARE) ? NULL : (struct thread *)LK_HOLDER(x));
+}
 
 /*
- * Locking primitives implementation.
- * Locks provide shared/exclusive sychronization.
+ * It assumes sleepq_lock held and returns with this one unheld.
+ * It also assumes the generic interlock is sane and previously checked.
+ * If LK_INTERLOCK is specified the interlock is not reacquired after the
+ * sleep.
  */
+static __inline int
+sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
+    const char *wmesg, int pri, int timo, int queue)
+{
+	struct lock_class *class;
+	int catch, error;
 
-void
+	class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
+	catch = (pri) ? (pri & PCATCH) : 0;
+	pri &= PRIMASK;
+	error = 0;
+
+	LOCK_LOG3(lk, "%s: %p blocking on the %s sleepqueue", __func__, lk,
+	    (queue == SQ_EXCLUSIVE_QUEUE) ? "exclusive" : "shared");
+
+	if (flags & LK_INTERLOCK)
+		class->lc_unlock(ilk);
+	DROP_GIANT();
+	sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
+	    SLEEPQ_INTERRUPTIBLE : 0), queue);
+	if ((flags & LK_TIMELOCK) && timo)
+		sleepq_set_timeout(&lk->lock_object, timo);
+
+	/*
+	 * Decisional switch for real sleeping.
+	 */
+	if ((flags & LK_TIMELOCK) && timo && catch)
+		error = sleepq_timedwait_sig(&lk->lock_object, pri);
+	else if ((flags & LK_TIMELOCK) && timo)
+		error = sleepq_timedwait(&lk->lock_object, pri);
+	else if (catch)
+		error = sleepq_wait_sig(&lk->lock_object, pri);
+	else
+		sleepq_wait(&lk->lock_object, pri);
+	PICKUP_GIANT();
+	if ((flags & LK_SLEEPFAIL) && error == 0)
+		error = ENOLCK;
+
+	return (error);
+}
+
+static __inline void
+wakeupshlk(struct lock *lk, const char *file, int line)
+{
+	uintptr_t v, x;
+	int queue;
+
+	TD_LOCKS_DEC(curthread);
+	TD_SLOCKS_DEC(curthread);
+	LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
+
+	for (;;) {
+		x = lk->lk_lock;
+
+		/*
+		 * If there is more than one shared lock held, just drop one
+		 * and return.
+		 */
+		if (LK_SHARERS(x) > 1) {
+			if (atomic_cmpset_ptr(&lk->lk_lock, x,
+			    x - LK_ONE_SHARER))
+				break;
+			continue;
+		}
+
+		/*
+		 * If there are not waiters on the exclusive queue, drop the
+		 * lock quickly.
+		 */
+		if ((x & LK_ALL_WAITERS) == 0) {
+			MPASS(x == LK_SHARERS_LOCK(1));
+			if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1),
+			    LK_UNLOCKED))
+				break;
+			continue;
+		}
+
+		/*
+		 * We should have a sharer with waiters, so enter the hard
+		 * path in order to handle wakeups correctly.
+		 */
+		sleepq_lock(&lk->lock_object);
+		x = lk->lk_lock & LK_ALL_WAITERS;
+		v = LK_UNLOCKED;
+
+		/*
+		 * If the lock has exclusive waiters, give them preference in
+		 * order to avoid deadlock with shared runners up.
+		 */
+		if (x & LK_EXCLUSIVE_WAITERS) {
+			queue = SQ_EXCLUSIVE_QUEUE;
+			v |= (x & LK_SHARED_WAITERS);
+		} else {
+			MPASS(x == LK_SHARED_WAITERS);
+			queue = SQ_SHARED_QUEUE;
+		}
+
+		if (!atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x,
+		    v)) {
+			sleepq_release(&lk->lock_object);
+			continue;
+		}
+		LOCK_LOG3(lk, "%s: %p waking up threads on the %s queue",
+		    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
+		    "exclusive");
+		sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
+		sleepq_release(&lk->lock_object);
+		break;
+	}
+
+	lock_profile_release_lock(&lk->lock_object);
+}
+
+static void
 assert_lockmgr(struct lock_object *lock, int what)
 {
 
 	panic("lockmgr locks do not support assertions");
 }
 
-void
+static void
 lock_lockmgr(struct lock_object *lock, int how)
 {
 
 	panic("lockmgr locks do not support sleep interlocking");
 }
 
-int
+static int
 unlock_lockmgr(struct lock_object *lock)
 {
 
 	panic("lockmgr locks do not support sleep interlocking");
 }
 
-#define	COUNT(td, x)	((td)->td_locks += (x))
-#define LK_ALL (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | \
-	LK_SHARE_NONZERO | LK_WAIT_NONZERO)
+void
+lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
+{
+	int iflags;
 
-static int acquire(struct lock **lkpp, int extflags, int wanted,
-    const char *wmesg, int prio, int timo, int *contested, uint64_t *waittime);
-static int acquiredrain(struct lock *lkp, int extflags, const char *wmesg,
-    int prio, int timo);
+	MPASS((flags & ~LK_INIT_MASK) == 0);
 
-static __inline void
-sharelock(struct thread *td, struct lock *lkp, int incr) {
-	lkp->lk_flags |= LK_SHARE_NONZERO;
-	lkp->lk_sharecount += incr;
-	COUNT(td, incr);
+	iflags = LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE;
+	if ((flags & LK_NODUP) == 0)
+		iflags |= LO_DUPOK;
+	if (flags & LK_NOPROFILE)
+		iflags |= LO_NOPROFILE;
+	if ((flags & LK_NOWITNESS) == 0)
+		iflags |= LO_WITNESS;
+	if (flags & LK_QUIET)
+		iflags |= LO_QUIET;
+	iflags |= flags & (LK_CANRECURSE | LK_NOSHARE);
+
+	lk->lk_lock = LK_UNLOCKED;
+	lk->lk_recurse = 0;
+	lk->lk_timo = timo;
+	lk->lk_pri = pri;
+	lock_init(&lk->lock_object, &lock_class_lockmgr, wmesg, NULL, iflags);
+	STACK_ZERO(lk);
 }
 
-static __inline void
-shareunlock(struct thread *td, struct lock *lkp, int decr) {
-
-	KASSERT(lkp->lk_sharecount >= decr, ("shareunlock: count < decr"));
+void
+lockdestroy(struct lock *lk)
+{
 
-	COUNT(td, -decr);
-	if (lkp->lk_sharecount == decr) {
-		lkp->lk_flags &= ~LK_SHARE_NONZERO;
-		if (lkp->lk_flags & (LK_WANT_UPGRADE | LK_WANT_EXCL)) {
-			wakeup(lkp);
-		}
-		lkp->lk_sharecount = 0;
-	} else {
-		lkp->lk_sharecount -= decr;
-	}
+	KASSERT(lk->lk_lock == LK_UNLOCKED, ("lockmgr still held"));
+	KASSERT(lk->lk_recurse == 0, ("lockmgr still recursed"));
+	lock_destroy(&lk->lock_object);
 }
 
-static int
-acquire(struct lock **lkpp, int extflags, int wanted, const char *wmesg,
-    int prio, int timo, int *contested, uint64_t *waittime)
+int
+__lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
+    const char *wmesg, int pri, int timo, const char *file, int line)
 {
-	struct lock *lkp = *lkpp;
+	uint64_t waittime;
+	struct lock_class *class;
 	const char *iwmesg;
-	int error, iprio, itimo;
-
-	iwmesg = (wmesg != LK_WMESG_DEFAULT) ? wmesg : lkp->lk_wmesg;
-	iprio = (prio != LK_PRIO_DEFAULT) ? prio : lkp->lk_prio;
-	itimo = (timo != LK_TIMO_DEFAULT) ? timo : lkp->lk_timo;
+	uintptr_t tid, v, x;
+	u_int op;
+	int contested, error, ipri, itimo, queue;
 
-	CTR3(KTR_LOCK,
-	    "acquire(): lkp == %p, extflags == 0x%x, wanted == 0x%x",
-	    lkp, extflags, wanted);
-
-	if ((extflags & LK_NOWAIT) && (lkp->lk_flags & wanted))
-		return EBUSY;
+	contested = 0;
 	error = 0;
-	if ((lkp->lk_flags & wanted) != 0)
-		lock_profile_obtain_lock_failed(&lkp->lk_object, contested, waittime);
-	
-	while ((lkp->lk_flags & wanted) != 0) {
-		CTR2(KTR_LOCK,
-		    "acquire(): lkp == %p, lk_flags == 0x%x sleeping",
-		    lkp, lkp->lk_flags);
-		lkp->lk_flags |= LK_WAIT_NONZERO;
-		lkp->lk_waitcount++;
-		error = msleep(lkp, lkp->lk_interlock, iprio, iwmesg,
-		    ((extflags & LK_TIMELOCK) ? itimo : 0));
-		lkp->lk_waitcount--;
-		if (lkp->lk_waitcount == 0)
-			lkp->lk_flags &= ~LK_WAIT_NONZERO;
-		if (error)
-			break;
-		if (extflags & LK_SLEEPFAIL) {
-			error = ENOLCK;
-			break;
-		}
-		if (lkp->lk_newlock != NULL) {
-			mtx_lock(lkp->lk_newlock->lk_interlock);
-			mtx_unlock(lkp->lk_interlock);
-			if (lkp->lk_waitcount == 0)
-				wakeup((void *)(&lkp->lk_newlock));
-			*lkpp = lkp = lkp->lk_newlock;
-		}
+	waittime = 0;
+	tid = (uintptr_t)curthread;
+	op = (flags & LK_TYPE_MASK);
+	iwmesg = (wmesg == LK_WMESG_DEFAULT) ? lk->lock_object.lo_name : wmesg;
+	ipri = (pri == LK_PRIO_DEFAULT) ? lk->lk_pri : pri;
+	itimo = (timo == LK_TIMO_DEFAULT) ? lk->lk_timo : timo;
+
+	MPASS((flags & ~LK_TOTAL_MASK) == 0);
+	KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0 ||
+	    (op != LK_DOWNGRADE && op != LK_RELEASE),
+	    ("%s: Invalid flags in regard of the operation desired @ %s:%d",
+	    __func__, file, line));
+	KASSERT((flags & LK_INTERLOCK) == 0 || ilk != NULL,
+	    ("%s: LK_INTERLOCK passed without valid interlock @ %s:%d",
+	    __func__, file, line));
+
+	class = (flags & LK_INTERLOCK) ? LOCK_CLASS(ilk) : NULL;
+	if (panicstr != NULL) {
+		if (flags & LK_INTERLOCK)
+			class->lc_unlock(ilk);
+		return (0);
 	}
-	mtx_assert(lkp->lk_interlock, MA_OWNED);
-	return (error);
-}
 
-/*
- * Set, change, or release a lock.
- *
- * Shared requests increment the shared count. Exclusive requests set the
- * LK_WANT_EXCL flag (preventing further shared locks), and wait for already
- * accepted shared locks and shared-to-exclusive upgrades to go away.
- */
-int
-_lockmgr_args(struct lock *lkp, u_int flags, struct mtx *interlkp,
-    const char *wmesg, int prio, int timo, char *file, int line)
+	if (op == LK_SHARED && (lk->lock_object.lo_flags & LK_NOSHARE))
+		op = LK_EXCLUSIVE;
 
-{
-	struct thread *td;
-	int error;
-	int extflags, lockflags;
-	int contested = 0;
-	uint64_t waitstart = 0;
+	switch (op) {
+	case LK_SHARED:
+		for (;;) {
+			x = lk->lk_lock;
 
-	error = 0;
-	td = curthread;
+			/*
+			 * If no other thread has an exclusive lock, or
+			 * no exclusive waiter is present, bump the count of
+			 * sharers.  Since we have to preserve the state of
+			 * waiters, if we fail to acquire the shared lock
+			 * loop back and retry.
+			 */
+			if (LK_CAN_SHARE(x)) {
+				if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
+				    x + LK_ONE_SHARER))
+					break;
+				continue;
+			}
+			lock_profile_obtain_lock_failed(&lk->lock_object,
+			    &contested, &waittime);
 
-#ifdef INVARIANTS
-	if (lkp->lk_flags & LK_DESTROYED) {
-		if (flags & LK_INTERLOCK)
-			mtx_unlock(interlkp);
-		if (panicstr != NULL)
-			return (0);
-		panic("%s: %p lockmgr is destroyed", __func__, lkp);
-	}
-#endif
-	mtx_lock(lkp->lk_interlock);
-	CTR6(KTR_LOCK,
-	    "lockmgr(): lkp == %p (lk_wmesg == \"%s\"), owner == %p, exclusivecount == %d, flags == 0x%x, "
-	    "td == %p", lkp, (wmesg != LK_WMESG_DEFAULT) ? wmesg :
-	    lkp->lk_wmesg, lkp->lk_lockholder, lkp->lk_exclusivecount, flags,
-	    td);
-#ifdef DEBUG_LOCKS
-	{
-		struct stack stack; /* XXX */
-		stack_save(&stack);
-		CTRSTACK(KTR_LOCK, &stack, 0, 1);
-	}
-#endif
+			/*
+			 * If the lock is alredy held by curthread in
+			 * exclusive way avoid a deadlock.
+			 */
+			if (LK_HOLDER(x) == tid) {
+				LOCK_LOG2(lk,
+				    "%s: %p alredy held in exclusive mode",
+				    __func__, lk);
+				error = EDEADLK;
+				break;
+			}
 
-	if (flags & LK_INTERLOCK) {
-		mtx_assert(interlkp, MA_OWNED | MA_NOTRECURSED);
-		mtx_unlock(interlkp);
-	}
+			/*
+			 * If the lock is expected to not sleep just give up
+			 * and return.
+			 */
+			if (LK_TRYOP(flags)) {
+				LOCK_LOG2(lk, "%s: %p fails the try operation",
+				    __func__, lk);
+				error = EBUSY;
+				break;
+			}
 
-	if ((flags & (LK_NOWAIT|LK_RELEASE)) == 0)
-		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK,
-		    &lkp->lk_interlock->lock_object,
-		    "Acquiring lockmgr lock \"%s\"",
-		    (wmesg != LK_WMESG_DEFAULT) ? wmesg : lkp->lk_wmesg);
+			/*
+			 * Acquire the sleepqueue chain lock because we
+			 * probabilly will need to manipulate waiters flags.
+			 */
+			sleepq_lock(&lk->lock_object);
+			x = lk->lk_lock;
 
-	if (panicstr != NULL) {
-		mtx_unlock(lkp->lk_interlock);
-		return (0);
-	}
-	if ((lkp->lk_flags & LK_NOSHARE) &&
-	    (flags & LK_TYPE_MASK) == LK_SHARED) {
-		flags &= ~LK_TYPE_MASK;
-		flags |= LK_EXCLUSIVE;
-	}
-	extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK;
+			/*
+			 * if the lock can be acquired in shared mode, try
+			 * again.
+			 */
+			if (LK_CAN_SHARE(x)) {
+				sleepq_release(&lk->lock_object);
+				continue;
+			}
 
-	switch (flags & LK_TYPE_MASK) {
+			/*
+			 * Try to set the LK_SHARED_WAITERS flag.  If we fail,
+			 * loop back and retry.
+			 */
+			if ((x & LK_SHARED_WAITERS) == 0) {
+				if (!atomic_cmpset_acq_ptr(&lk->lk_lock, x,
+				    x | LK_SHARED_WAITERS)) {
+					sleepq_release(&lk->lock_object);
+					continue;
+				}
+				LOCK_LOG2(lk, "%s: %p set shared waiters flag",
+				    __func__, lk);
+			}
 
-	case LK_SHARED:
-		if (!LOCKMGR_TRYOP(extflags))
-			WITNESS_CHECKORDER(&lkp->lk_object, LOP_NEWORDER, file,
+			/*
+			 * As far as we have been unable to acquire the
+			 * shared lock and the shared waiters flag is set,
+			 * we will sleep.
+			 */
+			error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
+			    SQ_SHARED_QUEUE);
+			flags &= ~LK_INTERLOCK;
+			if (error) {
+				LOCK_LOG3(lk,
+				    "%s: interrupted sleep for %p with %d",
+				    __func__, lk, error);
+				break;
+			}
+			LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
+			    __func__, lk);
+		}
+		if (error == 0) {
+			lock_profile_obtain_lock_success(&lk->lock_object,
+			    contested, waittime, file, line);
+			LOCK_LOG_LOCK("SLOCK", &lk->lock_object, 0, 0, file,
 			    line);
+			TD_LOCKS_INC(curthread);
+			TD_SLOCKS_INC(curthread);
+			STACK_SAVE(lk);
+		}
+		break;
+	case LK_UPGRADE:
+		_lockmgr_assert(lk, KA_SLOCKED, file, line);
+		x = lk->lk_lock & LK_ALL_WAITERS;
+
 		/*
-		 * If we are not the exclusive lock holder, we have to block
-		 * while there is an exclusive lock holder or while an
-		 * exclusive lock request or upgrade request is in progress.
-		 *
-		 * However, if TDP_DEADLKTREAT is set, we override exclusive
-		 * lock requests or upgrade requests ( but not the exclusive
-		 * lock itself ).
+		 * Try to switch from one shared lock to an exclusive one.
+		 * We need to preserve waiters flags during the operation.
 		 */
-		if (lkp->lk_lockholder != td) {
-			lockflags = LK_HAVE_EXCL;
-			if (!(td->td_pflags & TDP_DEADLKTREAT))
-				lockflags |= LK_WANT_EXCL | LK_WANT_UPGRADE;
-			error = acquire(&lkp, extflags, lockflags, wmesg,
-			    prio, timo, &contested, &waitstart);
-			if (error)
-				break;
-			sharelock(td, lkp, 1);
-			if (lkp->lk_sharecount == 1)
-				lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line);
-			WITNESS_LOCK(&lkp->lk_object, LOCKMGR_TRYW(extflags),
-			    file, line);
-
-#if defined(DEBUG_LOCKS)
-			stack_save(&lkp->lk_stack);
-#endif
+		if (atomic_cmpset_ptr(&lk->lk_lock, LK_SHARERS_LOCK(1) | x,
+		    tid | x)) {
+			LOCK_LOG_LOCK("XUPGRADE", &lk->lock_object, 0, 0, file,
+			    line);
+			TD_SLOCKS_DEC(curthread);
 			break;
 		}
+
 		/*
-		 * We hold an exclusive lock, so downgrade it to shared.
-		 * An alternative would be to fail with EDEADLK.
+		 * We have been unable to succeed in upgrading, so just
+		 * give up the shared lock.
 		 */
-		/* FALLTHROUGH downgrade */
+		wakeupshlk(lk, file, line);
 
-	case LK_DOWNGRADE:
-		_lockmgr_assert(lkp, KA_XLOCKED, file, line);
-		sharelock(td, lkp, lkp->lk_exclusivecount);
-		WITNESS_DOWNGRADE(&lkp->lk_object, 0, file, line);
-		COUNT(td, -lkp->lk_exclusivecount);
-		lkp->lk_exclusivecount = 0;
-		lkp->lk_flags &= ~LK_HAVE_EXCL;
-		lkp->lk_lockholder = LK_NOPROC;
-		if (lkp->lk_waitcount)
-			wakeup((void *)lkp);
-		break;
+		/* FALLTHROUGH */
+	case LK_EXCLUSIVE:
 
-	case LK_UPGRADE:
-		/*
-		 * Upgrade a shared lock to an exclusive one. If another
-		 * shared lock has already requested an upgrade to an
-		 * exclusive lock, our shared lock is released and an
-		 * exclusive lock is requested (which will be granted
-		 * after the upgrade). If we return an error, the file
-		 * will always be unlocked.
-		 */
-		_lockmgr_assert(lkp, KA_SLOCKED, file, line);
-		shareunlock(td, lkp, 1);
-		if (lkp->lk_sharecount == 0)
-			lock_profile_release_lock(&lkp->lk_object);
 		/*
-		 * If we are just polling, check to see if we will block.
+		 * If curthread alredy holds the lock and this one is
+		 * allowed to recurse, simply recurse on it.
 		 */
-		if ((extflags & LK_NOWAIT) &&
-		    ((lkp->lk_flags & LK_WANT_UPGRADE) ||
-		     lkp->lk_sharecount > 1)) {
-			error = EBUSY;
-			WITNESS_UNLOCK(&lkp->lk_object, 0, file, line);
+		if (lockmgr_xlocked(lk)) {
+			if ((flags & LK_CANRECURSE) == 0 &&
+			    (lk->lock_object.lo_flags & LK_CANRECURSE) == 0) {
+
+				/*
+				 * If the lock is expected to not panic just
+				 * give up and return.
+				 */
+				if (LK_TRYOP(flags)) {
+					LOCK_LOG2(lk,
+					    "%s: %p fails the try operation",
+					    __func__, lk);
+					error = EBUSY;
+					break;
+				}
+				if (flags & LK_INTERLOCK)
+					class->lc_unlock(ilk);
+		panic("%s: recursing on non recursive lockmgr %s @ %s:%d\n",
+				    __func__, iwmesg, file, line);
+			}
+			lk->lk_recurse++;
+			LOCK_LOG2(lk, "%s: %p recursing", __func__, lk);
+			LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
+			    lk->lk_recurse, file, line);
+			TD_LOCKS_INC(curthread);
 			break;
 		}
-		if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) {
+
+		while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED,
+		    tid)) {
+			lock_profile_obtain_lock_failed(&lk->lock_object,
+			    &contested, &waittime);
+
 			/*
-			 * We are first shared lock to request an upgrade, so
-			 * request upgrade and wait for the shared count to
-			 * drop to zero, then take exclusive lock.
+			 * If the lock is expected to not sleep just give up
+			 * and return.
 			 */
-			lkp->lk_flags |= LK_WANT_UPGRADE;
-			error = acquire(&lkp, extflags, LK_SHARE_NONZERO, wmesg,
-			    prio, timo, &contested, &waitstart);
-			lkp->lk_flags &= ~LK_WANT_UPGRADE;
+			if (LK_TRYOP(flags)) {
+				LOCK_LOG2(lk, "%s: %p fails the try operation",
+				    __func__, lk);
+				error = EBUSY;
+				break;
+			}
 
-			if (error) {
-			         if ((lkp->lk_flags & ( LK_WANT_EXCL | LK_WAIT_NONZERO)) == (LK_WANT_EXCL | LK_WAIT_NONZERO))
-			                   wakeup((void *)lkp);
-				WITNESS_UNLOCK(&lkp->lk_object, 0, file, line);
-			         break;
+			/*
+			 * Acquire the sleepqueue chain lock because we
+			 * probabilly will need to manipulate waiters flags.
+			 */
+			sleepq_lock(&lk->lock_object);
+			x = lk->lk_lock;
+			v = x & LK_ALL_WAITERS;
+
+			/*
+			 * if the lock has been released while we spun on
+			 * the sleepqueue chain lock just try again.
+			 */
+			if (x == LK_UNLOCKED) {
+				sleepq_release(&lk->lock_object);
+				continue;
+			}
+
+			/*
+			 * The lock can be in the state where there is a
+			 * pending queue of waiters, but still no owner.
+			 * This happens when the lock is contested and an
+			 * owner is going to claim the lock.
+			 * If curthread is the one successfully acquiring it
+			 * claim lock ownership and return, preserving waiters
+			 * flags.
+			 */
+			if (x == (LK_UNLOCKED | v)) {
+				if (atomic_cmpset_acq_ptr(&lk->lk_lock, x,
+				    tid | v)) {
+					sleepq_release(&lk->lock_object);
+					LOCK_LOG2(lk,
+					    "%s: %p claimed by a new writer",
+					    __func__, lk);
+					break;
+				}
+				sleepq_release(&lk->lock_object);
+				continue;
 			}
-			if (lkp->lk_exclusivecount != 0)
-				panic("lockmgr: non-zero exclusive count");
-			lkp->lk_flags |= LK_HAVE_EXCL;
-			lkp->lk_lockholder = td;
-			lkp->lk_exclusivecount = 1;
-			WITNESS_UPGRADE(&lkp->lk_object, LOP_EXCLUSIVE |
-			    LOP_TRYLOCK, file, line);
-			COUNT(td, 1);
-			lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line);
-#if defined(DEBUG_LOCKS)
-			stack_save(&lkp->lk_stack);
-#endif
-			break;
-		}
-		/*
-		 * Someone else has requested upgrade. Release our shared
-		 * lock, awaken upgrade requestor if we are the last shared
-		 * lock, then request an exclusive lock.
-		 */
-		WITNESS_UNLOCK(&lkp->lk_object, 0, file, line);
-		if ( (lkp->lk_flags & (LK_SHARE_NONZERO|LK_WAIT_NONZERO)) ==
-			LK_WAIT_NONZERO)
-			wakeup((void *)lkp);
-		/* FALLTHROUGH exclusive request */
 
-	case LK_EXCLUSIVE:
-		if (!LOCKMGR_TRYOP(extflags))
-			WITNESS_CHECKORDER(&lkp->lk_object, LOP_NEWORDER |
-			    LOP_EXCLUSIVE, file, line);
-		if (lkp->lk_lockholder == td) {
 			/*
-			 *	Recursive lock.
+			 * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
+			 * fail, loop back and retry.
 			 */
-			if ((extflags & (LK_NOWAIT | LK_CANRECURSE)) == 0)
-				panic("lockmgr: locking against myself");
-			if ((extflags & LK_CANRECURSE) != 0) {
-				lkp->lk_exclusivecount++;
-				WITNESS_LOCK(&lkp->lk_object, LOP_EXCLUSIVE |
-				    LOCKMGR_TRYW(extflags), file, line);
-				COUNT(td, 1);
+			if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
+				if (!atomic_cmpset_ptr(&lk->lk_lock, x,
+				    x | LK_EXCLUSIVE_WAITERS)) {
+					sleepq_release(&lk->lock_object);
+					continue;
+				}
+				LOCK_LOG2(lk, "%s: %p set excl waiters flag",
+				    __func__, lk);
+			}
+
+			/*
+			 * As far as we have been unable to acquire the
+			 * exclusive lock and the exclusive waiters flag
+			 * is set, we will sleep.
+			 */
+			error = sleeplk(lk, flags, ilk, iwmesg, ipri, itimo,
+			    SQ_EXCLUSIVE_QUEUE);
+			flags &= ~LK_INTERLOCK;
+			if (error) {
+				LOCK_LOG3(lk,
+				    "%s: interrupted sleep for %p with %d",
+				    __func__, lk, error);
 				break;
 			}
+			LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
+			    __func__, lk);
 		}
-		/*
-		 * If we are just polling, check to see if we will sleep.
-		 */
-		if ((extflags & LK_NOWAIT) &&
-		    (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO))) {
-			error = EBUSY;
-			break;
+		if (error == 0) {
+			lock_profile_obtain_lock_success(&lk->lock_object,
+			    contested, waittime, file, line);
+			LOCK_LOG_LOCK("XLOCK", &lk->lock_object, 0,
+			    lk->lk_recurse, file, line);
+			TD_LOCKS_INC(curthread);
+			STACK_SAVE(lk);
 		}
+		break;
+	case LK_DOWNGRADE:
+		_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line);
+
 		/*
-		 * Try to acquire the want_exclusive flag.
-		 */
-		error = acquire(&lkp, extflags, (LK_HAVE_EXCL | LK_WANT_EXCL),
-		    wmesg, prio, timo, &contested, &waitstart);
-		if (error)
-			break;
-		lkp->lk_flags |= LK_WANT_EXCL;
-		/*
-		 * Wait for shared locks and upgrades to finish.
+		 * In order to preserve waiters flags, just spin.
 		 */
-		error = acquire(&lkp, extflags, LK_HAVE_EXCL | LK_WANT_UPGRADE |
-		    LK_SHARE_NONZERO, wmesg, prio, timo,
-		    &contested, &waitstart);
-		lkp->lk_flags &= ~LK_WANT_EXCL;
-		if (error) {
-			if (lkp->lk_flags & LK_WAIT_NONZERO)		
-			         wakeup((void *)lkp);
-			break;
-		}	
-		lkp->lk_flags |= LK_HAVE_EXCL;
-		lkp->lk_lockholder = td;
-		if (lkp->lk_exclusivecount != 0)
-			panic("lockmgr: non-zero exclusive count");
-		lkp->lk_exclusivecount = 1;
-		WITNESS_LOCK(&lkp->lk_object, LOP_EXCLUSIVE |
-		    LOCKMGR_TRYW(extflags), file, line);
-		COUNT(td, 1);
-		lock_profile_obtain_lock_success(&lkp->lk_object, contested, waitstart, file, line);
-#if defined(DEBUG_LOCKS)
-		stack_save(&lkp->lk_stack);
-#endif
+		for (;;) {
+			x = lk->lk_lock & LK_ALL_WAITERS;
+			if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid | x,
+			    LK_SHARERS_LOCK(1) | x)) {
+				LOCK_LOG_LOCK("XDOWNGRADE", &lk->lock_object,
+				    0, 0, file, line);
+				TD_SLOCKS_INC(curthread);
+				break;
+			}
+			cpu_spinwait();
+		}
 		break;
-
 	case LK_RELEASE:
-		_lockmgr_assert(lkp, KA_LOCKED, file, line);
-		if (lkp->lk_exclusivecount != 0) {
-			if (lkp->lk_lockholder != LK_KERNPROC) {
-				WITNESS_UNLOCK(&lkp->lk_object, LOP_EXCLUSIVE,
-				    file, line);
-				COUNT(td, -1);
+		_lockmgr_assert(lk, KA_LOCKED, file, line);
+		x = lk->lk_lock;
+
+		if ((x & LK_SHARE) == 0) {
+
+			/*
+			 * As first option, treact the lock as if it has not
+			 * any waiter.
+			 * Fix-up the tid var if the lock has been disowned.
+			 */
+			if (LK_HOLDER(x) == LK_KERNPROC)
+				tid = LK_KERNPROC;
+			else
+				TD_LOCKS_DEC(curthread);
+			LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0,
+			    lk->lk_recurse, file, line);
+
+			/*
+			 * The lock is held in exclusive mode.
+			 * If the lock is recursed also, then unrecurse it.
+			 */
+			if (lockmgr_xlocked(lk) && lockmgr_recursed(lk)) {
+				LOCK_LOG2(lk, "%s: %p unrecursing", __func__,
+				    lk);
+				lk->lk_recurse--;
+				break;
 			}
-			if (lkp->lk_exclusivecount-- == 1) {
-				lkp->lk_flags &= ~LK_HAVE_EXCL;
-				lkp->lk_lockholder = LK_NOPROC;
-				lock_profile_release_lock(&lkp->lk_object);
+			lock_profile_release_lock(&lk->lock_object);
+
+			if (atomic_cmpset_rel_ptr(&lk->lk_lock, tid,
+			    LK_UNLOCKED))
+				break;
+
+			sleepq_lock(&lk->lock_object);
+			x = lk->lk_lock & LK_ALL_WAITERS;
+			v = LK_UNLOCKED;
+
+			/*
+		 	 * If the lock has exclusive waiters, give them
+			 * preference in order to avoid deadlock with
+			 * shared runners up.
+			 */
+			if (x & LK_EXCLUSIVE_WAITERS) {
+				queue = SQ_EXCLUSIVE_QUEUE;
+				v |= (x & LK_SHARED_WAITERS);
+			} else {
+				MPASS(x == LK_SHARED_WAITERS);
+				queue = SQ_SHARED_QUEUE;
 			}
-		} else if (lkp->lk_flags & LK_SHARE_NONZERO) {
-			WITNESS_UNLOCK(&lkp->lk_object, 0, file, line);
-			shareunlock(td, lkp, 1);
-		}
 
-		if (lkp->lk_flags & LK_WAIT_NONZERO)
-			wakeup((void *)lkp);
+			LOCK_LOG3(lk,
+			    "%s: %p waking up threads on the %s queue",
+			    __func__, lk, queue == SQ_SHARED_QUEUE ? "shared" :
+			    "exclusive");
+			atomic_store_rel_ptr(&lk->lk_lock, v);
+			sleepq_broadcast(&lk->lock_object, SLEEPQ_LK, 0, queue);
+			sleepq_release(&lk->lock_object);
+			break;
+		} else
+			wakeupshlk(lk, file, line);
 		break;
-
 	case LK_DRAIN:
+
 		/*
-		 * Check that we do not already hold the lock, as it can 
-		 * never drain if we do. Unfortunately, we have no way to
-		 * check for holding a shared lock, but at least we can
-		 * check for an exclusive one.
+		 * Trying to drain a lock we alredy own will result in a
+		 * deadlock.
 		 */
-		if (!LOCKMGR_TRYOP(extflags))
-			WITNESS_CHECKORDER(&lkp->lk_object, LOP_NEWORDER |
-			    LOP_EXCLUSIVE, file, line);
-		if (lkp->lk_lockholder == td)
-			panic("lockmgr: draining against myself");
-
-		error = acquiredrain(lkp, extflags, wmesg, prio, timo);
-		if (error)
-			break;
-		lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL;
-		lkp->lk_lockholder = td;
-		lkp->lk_exclusivecount = 1;
-		WITNESS_LOCK(&lkp->lk_object, LOP_EXCLUSIVE |
-		    LOCKMGR_TRYW(extflags), file, line);
-		COUNT(td, 1);
-#if defined(DEBUG_LOCKS)
-		stack_save(&lkp->lk_stack);
-#endif
-		break;
+		if (lockmgr_xlocked(lk)) {
+			if (flags & LK_INTERLOCK)
+				class->lc_unlock(ilk);
+			panic("%s: draining %s with the lock held @ %s:%d\n",
+			    __func__, iwmesg, file, line);
+		}
 
-	default:
-		mtx_unlock(lkp->lk_interlock);
-		panic("lockmgr: unknown locktype request %d",
-		    flags & LK_TYPE_MASK);
-		/* NOTREACHED */
-	}
-	if ((lkp->lk_flags & LK_WAITDRAIN) &&
-	    (lkp->lk_flags & (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE |
-		LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0) {
-		lkp->lk_flags &= ~LK_WAITDRAIN;
-		wakeup((void *)&lkp->lk_flags);
-	}
-	mtx_unlock(lkp->lk_interlock);
-	return (error);
-}
+		while (!atomic_cmpset_acq_ptr(&lk->lk_lock, LK_UNLOCKED, tid)) {
+			lock_profile_obtain_lock_failed(&lk->lock_object,
+			    &contested, &waittime);
 
-static int
-acquiredrain(struct lock *lkp, int extflags, const char *wmesg, int prio,
-    int timo)
-{
-	const char *iwmesg;
-	int error, iprio, itimo;
+			/*
+			 * If the lock is expected to not sleep just give up
+			 * and return.
+			 */
+			if (LK_TRYOP(flags)) {
+				LOCK_LOG2(lk, "%s: %p fails the try operation",
+				    __func__, lk);
+				error = EBUSY;
+				break;
+			}
 
-	iwmesg = (wmesg != LK_WMESG_DEFAULT) ? wmesg : lkp->lk_wmesg;
-	iprio = (prio != LK_PRIO_DEFAULT) ? prio : lkp->lk_prio;
-	itimo = (timo != LK_TIMO_DEFAULT) ? timo : lkp->lk_timo;
+			/*
+			 * Acquire the sleepqueue chain lock because we
+			 * probabilly will need to manipulate waiters flags.
+			 */
+			sleepq_lock(&lk->lock_object);
+			x = lk->lk_lock;
+			v = x & LK_ALL_WAITERS;
 
-	if ((extflags & LK_NOWAIT) && (lkp->lk_flags & LK_ALL)) {
-		return EBUSY;
-	}
-	while (lkp->lk_flags & LK_ALL) {
-		lkp->lk_flags |= LK_WAITDRAIN;
-		error = msleep(&lkp->lk_flags, lkp->lk_interlock, iprio, iwmesg,
-			((extflags & LK_TIMELOCK) ? itimo : 0));
-		if (error)
-			return error;
-		if (extflags & LK_SLEEPFAIL) {
-			return ENOLCK;
+			/*
+			 * if the lock has been released while we spun on
+			 * the sleepqueue chain lock just try again.
+			 */
+			if (x == LK_UNLOCKED) {
+				sleepq_release(&lk->lock_object);
+				continue;
+			}
+
+			if (x == (LK_UNLOCKED | v)) {
+				v = x;
+				if (v & LK_EXCLUSIVE_WAITERS) {
+					queue = SQ_EXCLUSIVE_QUEUE;
+					v &= ~LK_EXCLUSIVE_WAITERS;
+				} else {
+					MPASS(v & LK_SHARED_WAITERS);
+					queue = SQ_SHARED_QUEUE;
+					v &= ~LK_SHARED_WAITERS;
+				}
+				if (!atomic_cmpset_ptr(&lk->lk_lock, x, v)) {
+					sleepq_release(&lk->lock_object);
+					continue;
+				}
+				LOCK_LOG3(lk,
+				"%s: %p waking up all threads on the %s queue",
+				    __func__, lk, queue == SQ_SHARED_QUEUE ?
+				    "shared" : "exclusive");
+				sleepq_broadcast(&lk->lock_object, SLEEPQ_LK,
+				    0, queue);
+
+				/*
+				 * If shared waiters have been woken up we need
+				 * to wait for one of them to acquire the lock
+				 * before to set the exclusive waiters in
+				 * order to avoid a deadlock.
+				 */
+				if (queue == SQ_SHARED_QUEUE) {
+					for (v = lk->lk_lock;
+					    (v & LK_SHARE) && !LK_SHARERS(v);
+					    v = lk->lk_lock)
+						cpu_spinwait();
+				}
+			}
+
+			/*
+			 * Try to set the LK_EXCLUSIVE_WAITERS flag.  If we
+			 * fail, loop back and retry.
+			 */
+			if ((x & LK_EXCLUSIVE_WAITERS) == 0) {
+				if (!atomic_cmpset_ptr(&lk->lk_lock, x,
+				    x | LK_EXCLUSIVE_WAITERS)) {
+					sleepq_release(&lk->lock_object);
+					continue;
+				}
+				LOCK_LOG2(lk, "%s: %p set drain waiters flag",
+				    __func__, lk);
+			}
+
+			/*
+			 * As far as we have been unable to acquire the
+			 * exclusive lock and the exclusive waiters flag
+			 * is set, we will sleep.
+			 */
+			if (flags & LK_INTERLOCK) {
+				class->lc_unlock(ilk);
+				flags &= ~LK_INTERLOCK;
+			}
+			DROP_GIANT();
+			sleepq_add(&lk->lock_object, NULL, iwmesg, SLEEPQ_LK,
+			    SQ_EXCLUSIVE_QUEUE);
+			sleepq_wait(&lk->lock_object, ipri & PRIMASK);
+			PICKUP_GIANT();
+			LOCK_LOG2(lk, "%s: %p resuming from the sleep queue",
+			    __func__, lk);
+		}
+
+		if (error == 0) {
+			lock_profile_obtain_lock_success(&lk->lock_object,
+			    contested, waittime, file, line);
+			LOCK_LOG_LOCK("DRAIN", &lk->lock_object, 0,
+			    lk->lk_recurse, file, line);
+			TD_LOCKS_INC(curthread);
+			STACK_SAVE(lk);
 		}
+		break;
+	default:
+		if (flags & LK_INTERLOCK)
+			class->lc_unlock(ilk);
+		panic("%s: unknown lockmgr request 0x%x\n", __func__, op);
 	}
-	return 0;
-}
 
-/*
- * Initialize a lock; required before use.
- */
-void
-lockinit(lkp, prio, wmesg, timo, flags)
-	struct lock *lkp;
-	int prio;
-	const char *wmesg;
-	int timo;
-	int flags;
-{
-	int iflags;
+	/*
+	 * We could have exited from the switch without reacquiring the
+	 * interlock, so we need to check for the interlock ownership.
+	 */
+	if (flags & LK_INTERLOCK)
+		class->lc_unlock(ilk);
 
-	KASSERT((flags & (LK_NOWAIT | LK_SLEEPFAIL)) == 0,
-	    ("%s: Invalid flags passed with mask 0x%x", __func__,
-	    flags & LK_EXTFLG_MASK));
-	CTR5(KTR_LOCK, "lockinit(): lkp == %p, prio == %d, wmesg == \"%s\", "
-	    "timo == %d, flags = 0x%x\n", lkp, prio, wmesg, timo, flags);
-
-	lkp->lk_interlock = mtx_pool_alloc(mtxpool_lockbuilder);
-	lkp->lk_flags = (flags & LK_EXTFLG_MASK) & ~(LK_FUNC_MASK);
-	lkp->lk_sharecount = 0;
-	lkp->lk_waitcount = 0;
-	lkp->lk_exclusivecount = 0;
-	lkp->lk_prio = prio;
-	lkp->lk_timo = timo;
-	lkp->lk_lockholder = LK_NOPROC;
-	lkp->lk_newlock = NULL;
-	iflags = LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE;
-	if (!(flags & LK_NODUP))
-		iflags |= LO_DUPOK;
-	if (flags & LK_NOPROFILE)
-		iflags |= LO_NOPROFILE;
-	if (!(flags & LK_NOWITNESS))
-		iflags |= LO_WITNESS;
-	if (flags & LK_QUIET)
-		iflags |= LO_QUIET;
-#ifdef DEBUG_LOCKS
-	stack_zero(&lkp->lk_stack);
-#endif
-	lock_init(&lkp->lk_object, &lock_class_lockmgr, wmesg, NULL, iflags);
+	return (error);
 }
 
-/*
- * Destroy a lock.
- */
 void
-lockdestroy(lkp)
-	struct lock *lkp;
+_lockmgr_disown(struct lock *lk, const char *file, int line)
 {
+	uintptr_t tid, x;
+
+	tid = (uintptr_t)curthread;
+	_lockmgr_assert(lk, KA_XLOCKED | KA_NOTRECURSED, file, line);
+
+	/*
+	 * If the owner is alredy LK_KERNPROC just skip the whole operation.
+	 */
+	if (LK_HOLDER(lk->lk_lock) != tid)
+		return;
 
-	CTR2(KTR_LOCK, "lockdestroy(): lkp == %p (lk_wmesg == \"%s\")",
-	    lkp, lkp->lk_wmesg);
-	KASSERT((lkp->lk_flags & (LK_HAVE_EXCL | LK_SHARE_NONZERO)) == 0,
-	    ("lockmgr still held"));
-	KASSERT(lkp->lk_exclusivecount == 0, ("lockmgr still recursed"));
-	lkp->lk_flags = LK_DESTROYED;
-	lock_destroy(&lkp->lk_object);
+	/*
+	 * In order to preserve waiters flags, just spin.
+	 */
+	for (;;) {
+		x = lk->lk_lock & LK_ALL_WAITERS;
+		if (atomic_cmpset_ptr(&lk->lk_lock, tid | x,
+		    LK_KERNPROC | x)) {
+			LOCK_LOG_LOCK("XDISOWN", &lk->lock_object, 0, 0, file,
+			    line);
+			TD_LOCKS_DEC(curthread);
+			return;
+		}
+		cpu_spinwait();
+	}
 }
 
-/*
- * Disown the lockmgr.
- */
 void
-_lockmgr_disown(struct lock *lkp, const char *file, int line)
+lockmgr_printinfo(struct lock *lk)
 {
 	struct thread *td;
+	uintptr_t x;
+
+	if (lk->lk_lock == LK_UNLOCKED)
+		printf(" lock type %s: UNLOCKED\n", lk->lock_object.lo_name);
+	else if (lk->lk_lock & LK_SHARE)
+		printf(" lock type %s: SHARED (count %ju)\n",
+		    lk->lock_object.lo_name,
+		    (uintmax_t)LK_SHARERS(lk->lk_lock));
+	else {
+		td = lockmgr_xholder(lk);
+		printf(" lock type %s: EXCL by thread %p (pid %d)\n",
+		    lk->lock_object.lo_name, td, td->td_proc->p_pid);
+	}
 
-	td = curthread;
-	KASSERT(panicstr != NULL || (lkp->lk_flags & LK_DESTROYED) == 0,
-	    ("%s: %p lockmgr is destroyed", __func__, lkp));
-	_lockmgr_assert(lkp, KA_XLOCKED | KA_NOTRECURSED, file, line);
+	x = lk->lk_lock;
+	if (x & LK_EXCLUSIVE_WAITERS)
+		printf(" with exclusive waiters pending\n");
+	if (x & LK_SHARED_WAITERS)
+		printf(" with shared waiters pending\n");
 
-	/*
-	 * Drop the lock reference and switch the owner.  This will result
-	 * in an atomic operation like td_lock is only accessed by curthread
-	 * and lk_lockholder only needs one write.  Note also that the lock
-	 * owner can be alredy KERNPROC, so in that case just skip the
-	 * decrement.
-	 */
-	if (lkp->lk_lockholder == td) {	
-		WITNESS_UNLOCK(&lkp->lk_object, LOP_EXCLUSIVE, file, line);
-		td->td_locks--;
-	}
-	lkp->lk_lockholder = LK_KERNPROC;
+	STACK_PRINT(lk);
 }
 
-/*
- * Determine the status of a lock.
- */
 int
-lockstatus(lkp)
-	struct lock *lkp;
+lockstatus(struct lock *lk)
 {
-	int lock_type = 0;
-	int interlocked;
-
-	KASSERT((lkp->lk_flags & LK_DESTROYED) == 0,
-	    ("%s: %p lockmgr is destroyed", __func__, lkp));
-
-	if (!kdb_active) {
-		interlocked = 1;
-		mtx_lock(lkp->lk_interlock);
-	} else
-		interlocked = 0;
-	if (lkp->lk_exclusivecount != 0) {
-		if (lkp->lk_lockholder == curthread)
-			lock_type = LK_EXCLUSIVE;
-		else
-			lock_type = LK_EXCLOTHER;
-	} else if (lkp->lk_sharecount != 0)
-		lock_type = LK_SHARED;
-	if (interlocked)
-		mtx_unlock(lkp->lk_interlock);
-	return (lock_type);
-}
+	uintptr_t v, x;
+	int ret;
 
-/*
- * Print out information about state of a lock. Used by VOP_PRINT
- * routines to display status about contained locks.
- */
-void
-lockmgr_printinfo(lkp)
-	struct lock *lkp;
-{
+	ret = LK_SHARED;
+	x = lk->lk_lock;
+	v = LK_HOLDER(x);
 
-	if (lkp->lk_sharecount)
-		printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg,
-		    lkp->lk_sharecount);
-	else if (lkp->lk_flags & LK_HAVE_EXCL)
-		printf(" lock type %s: EXCL (count %d) by thread %p (pid %d)",
-		    lkp->lk_wmesg, lkp->lk_exclusivecount,
-		    lkp->lk_lockholder, lkp->lk_lockholder->td_proc->p_pid);
-	if (lkp->lk_waitcount > 0)
-		printf(" with %d pending", lkp->lk_waitcount);
-#ifdef DEBUG_LOCKS
-	stack_print_ddb(&lkp->lk_stack);
-#endif
+	if ((x & LK_SHARE) == 0) {
+		if (v == (uintptr_t)curthread || v == LK_KERNPROC)
+			ret = LK_EXCLUSIVE;
+		else
+			ret = LK_EXCLOTHER;
+	} else if (x == LK_UNLOCKED)
+		ret = 0;
+
+	return (ret);
 }
 
 #ifdef INVARIANT_SUPPORT
 #ifndef INVARIANTS
-#undef _lockmgr_assert
+#undef	_lockmgr_assert
 #endif
 
 void
-_lockmgr_assert(struct lock *lkp, int what, const char *file, int line)
+_lockmgr_assert(struct lock *lk, int what, const char *file, int line)
 {
-	struct thread *td;
-	u_int x;
 	int slocked = 0;
 
-	x = lkp->lk_flags;
-	td = lkp->lk_lockholder;
 	if (panicstr != NULL)
 		return;
 	switch (what) {
@@ -697,133 +905,107 @@ _lockmgr_assert(struct lock *lkp, int what, const char *file, int line)
 	case KA_LOCKED:
 	case KA_LOCKED | KA_NOTRECURSED:
 	case KA_LOCKED | KA_RECURSED:
-#ifdef WITNESS
-		/*
-		 * We cannot trust WITNESS if the lock is held in
-		 * exclusive mode and a call to lockmgr_disown() happened.
-		 * Workaround this skipping the check if the lock is
-		 * held in exclusive mode even for the KA_LOCKED case.
-		 */
-		if (slocked || (x & LK_HAVE_EXCL) == 0) {
-			witness_assert(&lkp->lk_object, what, file, line);
-			break;
-		}
-#endif
-		if (LOCKMGR_UNHELD(x) || ((x & LK_SHARE_NONZERO) == 0 &&
-		    (slocked || LOCKMGR_NOTOWNER(td))))
+		if (lk->lk_lock == LK_UNLOCKED ||
+		    ((lk->lk_lock & LK_SHARE) == 0 && (slocked ||
+		    (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk)))))
 			panic("Lock %s not %slocked @ %s:%d\n",
-			    lkp->lk_object.lo_name, slocked ? "share " : "",
+			    lk->lock_object.lo_name, slocked ? "share" : "",
 			    file, line);
-		if ((x & LK_SHARE_NONZERO) == 0) {
-			if (lockmgr_recursed(lkp)) {
+
+		if ((lk->lk_lock & LK_SHARE) == 0) {
+			if (lockmgr_recursed(lk)) {
 				if (what & KA_NOTRECURSED)
 					panic("Lock %s recursed @ %s:%d\n",
-					    lkp->lk_object.lo_name, file, line);
+					    lk->lock_object.lo_name, file,
+					    line);
 			} else if (what & KA_RECURSED)
 				panic("Lock %s not recursed @ %s:%d\n",
-				    lkp->lk_object.lo_name, file, line);
+				    lk->lock_object.lo_name, file, line);
 		}
 		break;
 	case KA_XLOCKED:
 	case KA_XLOCKED | KA_NOTRECURSED:
 	case KA_XLOCKED | KA_RECURSED:
-		if ((x & LK_HAVE_EXCL) == 0 || LOCKMGR_NOTOWNER(td))
+		if (!lockmgr_xlocked(lk) && !lockmgr_disowned(lk))
 			panic("Lock %s not exclusively locked @ %s:%d\n",
-			    lkp->lk_object.lo_name, file, line);
-		if (lockmgr_recursed(lkp)) {
+			    lk->lock_object.lo_name, file, line);
+		if (lockmgr_recursed(lk)) {
 			if (what & KA_NOTRECURSED)
 				panic("Lock %s recursed @ %s:%d\n",
-				    lkp->lk_object.lo_name, file, line);
+				    lk->lock_object.lo_name, file, line);
 		} else if (what & KA_RECURSED)
 			panic("Lock %s not recursed @ %s:%d\n",
-			    lkp->lk_object.lo_name, file, line);
+			    lk->lock_object.lo_name, file, line);
 		break;
 	case KA_UNLOCKED:
-		if (td == curthread || td == LK_KERNPROC)
+		if (lockmgr_xlocked(lk) || lockmgr_disowned(lk))
 			panic("Lock %s exclusively locked @ %s:%d\n",
-			    lkp->lk_object.lo_name, file, line);
-		break;
-	case KA_HELD:
-	case KA_UNHELD:
-		if (LOCKMGR_UNHELD(x)) {
-			if (what & KA_HELD)
-				panic("Lock %s not locked by anyone @ %s:%d\n",
-				    lkp->lk_object.lo_name, file, line);
-		} else if (what & KA_UNHELD)
-			panic("Lock %s locked by someone @ %s:%d\n",
-			    lkp->lk_object.lo_name, file, line);
+			    lk->lock_object.lo_name, file, line);
 		break;
 	default:
-		panic("Unknown lockmgr lock assertion: 0x%x @ %s:%d", what,
-		    file, line);
+		panic("Unknown lockmgr assertion: %d @ %s:%d\n", what, file,
+		    line);
 	}
 }
-#endif	/* INVARIANT_SUPPORT */
+#endif
 
 #ifdef DDB
-/*
- * Check to see if a thread that is blocked on a sleep queue is actually
- * blocked on a 'struct lock'.  If so, output some details and return true.
- * If the lock has an exclusive owner, return that in *ownerp.
- */
 int
 lockmgr_chain(struct thread *td, struct thread **ownerp)
 {
-	struct lock *lkp;
+	struct lock *lk;
 
-	lkp = td->td_wchan;
+	lk = td->td_wchan;
 
-	/* Simple test to see if wchan points to a lockmgr lock. */
-	if (LOCK_CLASS(&lkp->lk_object) == &lock_class_lockmgr &&
-	    lkp->lk_wmesg == td->td_wmesg)
-		goto ok;
+	if (LOCK_CLASS(&lk->lock_object) != &lock_class_lockmgr)
+		return (0);
+	db_printf("blocked on lockmgr %s", lk->lock_object.lo_name);
+	if (lk->lk_lock & LK_SHARE)
+		db_printf("SHARED (count %ju)\n",
+		    (uintmax_t)LK_SHARERS(lk->lk_lock));
+	else
+		db_printf("EXCL\n");
+	*ownerp = lockmgr_xholder(lk);
 
-	/*
-	 * If this thread is doing a DRAIN, then it would be asleep on
-	 * &lkp->lk_flags rather than lkp.
-	 */
-	lkp = (struct lock *)((char *)td->td_wchan -
-	    offsetof(struct lock, lk_flags));
-	if (LOCK_CLASS(&lkp->lk_object) == &lock_class_lockmgr &&
-	    lkp->lk_wmesg == td->td_wmesg && (lkp->lk_flags & LK_WAITDRAIN))
-		goto ok;
-
-	/* Doen't seem to be a lockmgr lock. */
-	return (0);
-
-ok:
-	/* Ok, we think we have a lockmgr lock, so output some details. */
-	db_printf("blocked on lk \"%s\" ", lkp->lk_wmesg);
-	if (lkp->lk_sharecount) {
-		db_printf("SHARED (count %d)\n", lkp->lk_sharecount);
-		*ownerp = NULL;
-	} else {
-		db_printf("EXCL (count %d)\n", lkp->lk_exclusivecount);
-		*ownerp = lkp->lk_lockholder;
-	}
 	return (1);
 }
 
-void
+static void
 db_show_lockmgr(struct lock_object *lock)
 {
 	struct thread *td;
-	struct lock *lkp;
+	struct lock *lk;
 
-	lkp = (struct lock *)lock;
+	lk = (struct lock *)lock;
 
-	db_printf(" lock type: %s\n", lkp->lk_wmesg);
 	db_printf(" state: ");
-	if (lkp->lk_sharecount)
-		db_printf("SHARED (count %d)\n", lkp->lk_sharecount);
-	else if (lkp->lk_flags & LK_HAVE_EXCL) {
-		td = lkp->lk_lockholder;
-		db_printf("EXCL (count %d) %p ", lkp->lk_exclusivecount, td);
-		db_printf("(tid %d, pid %d, \"%s\")\n", td->td_tid,
-		    td->td_proc->p_pid, td->td_name);
-	} else
+	if (lk->lk_lock == LK_UNLOCKED)
 		db_printf("UNLOCKED\n");
-	if (lkp->lk_waitcount > 0)
-		db_printf(" waiters: %d\n", lkp->lk_waitcount);
+	else if (lk->lk_lock & LK_SHARE)
+		db_printf("SLOCK: %ju\n", (uintmax_t)LK_SHARERS(lk->lk_lock));
+	else {
+		td = lockmgr_xholder(lk);
+		if (td == (struct thread *)LK_KERNPROC)
+			db_printf("XLOCK: LK_KERNPROC\n");
+		else
+			db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
+			    td->td_tid, td->td_proc->p_pid,
+			    td->td_proc->p_comm);
+		if (lockmgr_recursed(lk))
+			db_printf(" recursed: %d\n", lk->lk_recurse);
+	}
+	db_printf(" waiters: ");
+	switch (lk->lk_lock & LK_ALL_WAITERS) {
+	case LK_SHARED_WAITERS:
+		db_printf("shared\n");
+	case LK_EXCLUSIVE_WAITERS:
+		db_printf("exclusive\n");
+		break;
+	case LK_ALL_WAITERS:
+		db_printf("shared and exclusive\n");
+		break;
+	default:
+		db_printf("none\n");
+	}
 }
 #endif
diff --git a/sys/sys/_lockmgr.h b/sys/sys/_lockmgr.h
new file mode 100644
index 0000000..11ddac6
--- /dev/null
+++ b/sys/sys/_lockmgr.h
@@ -0,0 +1,49 @@
+/*-
+ * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice(s), this list of conditions and the following disclaimer as
+ *    the first lines of this file unmodified other than the possible 
+ *    addition of one or more copyright notices.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice(s), this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SYS__LOCKMGR_H_
+#define	_SYS__LOCKMGR_H_
+
+#ifdef DEBUG_LOCKS
+#include <sys/_stack.h>
+#endif
+
+struct lock {
+	struct lock_object	lock_object;
+	volatile uintptr_t	lk_lock;
+	volatile unsigned	lk_recurse;
+	int			lk_timo;
+	int			lk_pri;
+#ifdef DEBUG_LOCKS
+	struct stack		lk_stack;
+#endif
+};
+
+#endif
diff --git a/sys/sys/_stack.h b/sys/sys/_stack.h
new file mode 100644
index 0000000..96273d3
--- /dev/null
+++ b/sys/sys/_stack.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2005 Antoine Brodin
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SYS__STACK_H_
+#define	_SYS__STACK_H_
+
+#define	STACK_MAX	18	/* Don't change, stack_ktr relies on this. */
+
+struct stack {
+	int		depth;
+	vm_offset_t	pcs[STACK_MAX];
+};
+
+#endif
diff --git a/sys/sys/lockmgr.h b/sys/sys/lockmgr.h
index 9e2fe59..5251f31 100644
--- a/sys/sys/lockmgr.h
+++ b/sys/sys/lockmgr.h
@@ -1,240 +1,189 @@
 /*-
- * Copyright (c) 1995
- *	The Regents of the University of California.  All rights reserved.
- *
- * This code contains ideas from software contributed to Berkeley by
- * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating
- * System project at Carnegie-Mellon University.
+ * Copyright (c) 2008 Attilio Rao <attilio@FreeBSD.org>
+ * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
+ *    notice(s), this list of conditions and the following disclaimer as
+ *    the first lines of this file unmodified other than the possible 
+ *    addition of one or more copyright notices.
  * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
+ *    notice(s), this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
  *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
  *
- *	@(#)lock.h	8.12 (Berkeley) 5/19/95
  * $FreeBSD$
  */
 
 #ifndef	_SYS_LOCKMGR_H_
 #define	_SYS_LOCKMGR_H_
 
-#ifdef	DEBUG_LOCKS
-#include <sys/stack.h> /* XXX */
-#endif
-#include <sys/queue.h>
 #include <sys/_lock.h>
+#include <sys/_lockmgr.h>
+#include <sys/_mutex.h>
+#include <sys/_rwlock.h>
+
+#define	LK_SHARE			0x01
+#define	LK_SHARED_WAITERS		0x02
+#define	LK_EXCLUSIVE_WAITERS		0x04
+#define	LK_ALL_WAITERS							\
+	(LK_SHARED_WAITERS | LK_EXCLUSIVE_WAITERS)
+#define	LK_FLAGMASK							\
+	(LK_SHARE | LK_ALL_WAITERS)
+
+#define	LK_HOLDER(x)			((x) & ~LK_FLAGMASK)
+#define	LK_SHARERS_SHIFT		3
+#define	LK_SHARERS(x)			(LK_HOLDER(x) >> LK_SHARERS_SHIFT)
+#define	LK_SHARERS_LOCK(x)		((x) << LK_SHARERS_SHIFT | LK_SHARE)
+#define	LK_ONE_SHARER			(1 << LK_SHARERS_SHIFT)
+#define	LK_UNLOCKED			LK_SHARERS_LOCK(0)
+#define	LK_KERNPROC			((uintptr_t)(-1) & ~LK_FLAGMASK)
+
+#ifdef _KERNEL
 
-struct	mtx;
+#if !defined(LOCK_FILE) || !defined(LOCK_LINE)
+#error	"LOCK_FILE and LOCK_LINE not defined, include <sys/lock.h> before"
+#endif
+
+struct thread;
 
 /*
- * The general lock structure.  Provides for multiple shared locks,
- * upgrading from shared to exclusive, and sleeping until the lock
- * can be gained.
+ * Function prototipes.  Routines that start with an underscore are not part
+ * of the public interface and might be wrappered with a macro.
  */
-struct lock {
-	struct lock_object lk_object;   /* common lock properties */
-	struct	mtx *lk_interlock;	/* lock on remaining fields */
-	u_int	lk_flags;		/* see below */
-	int	lk_sharecount;		/* # of accepted shared locks */
-	int	lk_waitcount;		/* # of processes sleeping for lock */
-	short	lk_exclusivecount;	/* # of recursive exclusive locks */
-	short	lk_prio;		/* priority at which to sleep */
-	int	lk_timo;		/* maximum sleep time (for tsleep) */
-	struct thread *lk_lockholder;	/* thread of exclusive lock holder */
-	struct	lock *lk_newlock;	/* lock taking over this lock */
-
-#ifdef	DEBUG_LOCKS
-	struct stack lk_stack;
+int	 __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
+	    const char *wmesg, int prio, int timo, const char *file, int line);
+#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
+void	 _lockmgr_assert(struct lock *lk, int what, const char *file, int line);
 #endif
-};
+void	 _lockmgr_disown(struct lock *lk, const char *file, int line);
 
-#define lk_wmesg lk_object.lo_name
-
-#ifdef _KERNEL
+void	 lockdestroy(struct lock *lk);
+void	 lockinit(struct lock *lk, int prio, const char *wmesg, int timo,
+	    int flags);
+#ifdef DDB
+int	 lockmgr_chain(struct thread *td, struct thread **ownerp);
+#endif
+void	 lockmgr_printinfo(struct lock *lk);
+int	 lockstatus(struct lock *lk);
 
 /*
- * Lock request types:
- *   LK_SHARED - get one of many possible shared locks. If a process
- *	holding an exclusive lock requests a shared lock, the exclusive
- *	lock(s) will be downgraded to shared locks.
- *   LK_EXCLUSIVE - stop further shared locks, when they are cleared,
- *	grant a pending upgrade if it exists, then grant an exclusive
- *	lock. Only one exclusive lock may exist at a time, except that
- *	a process holding an exclusive lock may get additional exclusive
- *	locks if it explicitly sets the LK_CANRECURSE flag in the lock
- *	request, or if the LK_CANRECUSE flag was set when the lock was
- *	initialized.
- *   LK_UPGRADE - the process must hold a shared lock that it wants to
- *	have upgraded to an exclusive lock. Other processes may get
- *	exclusive access to the resource between the time that the upgrade
- *	is requested and the time that it is granted.
- *   LK_DOWNGRADE - the process must hold an exclusive lock that it wants
- *	to have downgraded to a shared lock. If the process holds multiple
- *	(recursive) exclusive locks, they will all be downgraded to shared
- *	locks.
- *   LK_RELEASE - release one instance of a lock.
- *   LK_DRAIN - wait for all activity on the lock to end, then mark it
- *	decommissioned. This feature is used before freeing a lock that
- *	is part of a piece of memory that is about to be freed.
- *   LK_EXCLOTHER - return for lockstatus().  Used when another process
- *	holds the lock exclusively.
- *
- * These are flags that are passed to the lockmgr routine.
+ * As far as the ilk can be a static NULL pointer these functions need a
+ * strict prototype in order to safely use the lock_object member.
  */
-#define	LK_TYPE_MASK	0x0000000f	/* type of lock sought */
-#define	LK_SHARED	0x00000001	/* shared lock */
-#define	LK_EXCLUSIVE	0x00000002	/* exclusive lock */
-#define	LK_UPGRADE	0x00000003	/* shared-to-exclusive upgrade */
-#define	LK_DOWNGRADE	0x00000005	/* exclusive-to-shared downgrade */
-#define	LK_RELEASE	0x00000006	/* release any type of lock */
-#define	LK_DRAIN	0x00000007	/* wait for all lock activity to end */
-#define	LK_EXCLOTHER	0x00000008	/* other process holds lock */
+static __inline int
+_lockmgr_args(struct lock *lk, u_int flags, struct mtx *ilk, const char *wmesg,
+    int prio, int timo, const char *file, int line)
+{
+
+	return (__lockmgr_args(lk, flags, (ilk != NULL) ? &ilk->lock_object :
+	    NULL, wmesg, prio, timo, file, line));
+}
+
+static __inline int
+_lockmgr_args_rw(struct lock *lk, u_int flags, struct rwlock *ilk,
+    const char *wmesg, int prio, int timo, const char *file, int line)
+{
+
+	return (__lockmgr_args(lk, flags, (ilk != NULL) ? &ilk->lock_object :
+	    NULL, wmesg, prio, timo, file, line));
+}
+
 /*
- * External lock flags.
- *
- * These may be set in lock_init to set their mode permanently,
- * or passed in as arguments to the lock manager.
+ * Define aliases in order to complete lockmgr KPI.
  */
-#define	LK_EXTFLG_MASK	0x0000fff0	/* mask of external flags */
-#define	LK_NOWAIT	0x00000010	/* do not sleep to await lock */
-#define	LK_SLEEPFAIL	0x00000020	/* sleep, then return failure */
-#define	LK_CANRECURSE	0x00000040	/* allow recursive exclusive lock */
-#define	LK_NOSHARE	0x00000080	/* Only allow exclusive locks */
-#define	LK_TIMELOCK	0x00000100	/* use lk_timo, else no timeout */
-#define	LK_NOWITNESS	0x00000200	/* disable WITNESS */
-#define	LK_NODUP	0x00000400	/* enable duplication logging */
-#define	LK_NOPROFILE	0x00000800	/* disable lock profiling */
-#define	LK_QUIET	0x00001000	/* disable lock operations tracking */
-#define	LK_FUNC_MASK	(LK_NODUP | LK_NOPROFILE | LK_NOWITNESS | LK_QUIET)
+#define	lockmgr(lk, flags, ilk)						\
+	_lockmgr_args((lk), (flags), (ilk), LK_WMESG_DEFAULT,		\
+	    LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, LOCK_FILE, LOCK_LINE)
+#define	lockmgr_args(lk, flags, ilk, wmesg, prio, timo)			\
+	_lockmgr_args((lk), (flags), (ilk), (wmesg), (prio), (timo),	\
+	    LOCK_FILE, LOCK_LINE)
+#define	lockmgr_args_rw(lk, flags, ilk, wmesg, prio, timo)		\
+	_lockmgr_args_rw((lk), (flags), (ilk), (wmesg), (prio), (timo),	\
+	    LOCK_FILE, LOCK_LINE)
+#define	lockmgr_disown(lk)						\
+	_lockmgr_disown((lk), LOCK_FILE, LOCK_LINE)
+#define	lockmgr_recursed(lk)						\
+	((lk)->lk_recurse != 0)
+#define	lockmgr_rw(lk, flags, ilk)					\
+	_lockmgr_args_rw((lk), (flags), (ilk), LK_WMESG_DEFAULT,	\
+	    LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, LOCK_FILE, LOCK_LINE)
+#define	lockmgr_waiters(lk)						\
+	((lk)->lk_lock & LK_ALL_WAITERS)
+#ifdef INVARIANTS
+#define	lockmgr_assert(lk, what)					\
+	_lockmgr_assert((lk), (what), LOCK_FILE, LOCK_LINE)
+#else
+#define	lockmgr_assert(lk, what)
+#endif
+
 /*
- * Nonpersistent external flags.
+ * Flags for lockinit().
  */
-#define	LK_RETRY	0x00010000 /* vn_lock: retry until locked */
-#define	LK_INTERLOCK	0x00020000 /*
-				    * unlock passed mutex after getting
-				    * lk_interlock
-				    */
+#define	LK_INIT_MASK	0x000FF
+#define	LK_CANRECURSE	0x00001
+#define	LK_NODUP	0x00002
+#define	LK_NOPROFILE	0x00004
+#define	LK_NOSHARE	0x00008
+#define	LK_NOWITNESS	0x00010
+#define	LK_QUIET	0x00020
 
 /*
- * Default values for lockmgr_args().
+ * Additional attributes to be used in lockmgr().
  */
-#define	LK_WMESG_DEFAULT	(NULL)
-#define	LK_PRIO_DEFAULT		(-1)
-#define	LK_TIMO_DEFAULT		(0)
+#define	LK_EATTR_MASK	0x0FF00
+#define	LK_INTERLOCK	0x00100
+#define	LK_NOWAIT	0x00200
+#define	LK_RETRY	0x00400
+#define	LK_SLEEPFAIL	0x00800
+#define	LK_TIMELOCK	0x01000
 
 /*
- * Internal lock flags.
- *
- * These flags are used internally to the lock manager.
+ * Operations for lockmgr().
  */
-#define	LK_WANT_UPGRADE	0x00100000	/* waiting for share-to-excl upgrade */
-#define	LK_WANT_EXCL	0x00200000	/* exclusive lock sought */
-#define	LK_HAVE_EXCL	0x00400000	/* exclusive lock obtained */
-#define	LK_WAITDRAIN	0x00800000	/* process waiting for lock to drain */
-#define	LK_DRAINING	0x01000000	/* lock is being drained */
-#define	LK_DESTROYED	0x02000000	/* lock is destroyed */
+#define	LK_TYPE_MASK	0xF0000
+#define	LK_DOWNGRADE	0x10000
+#define	LK_DRAIN	0x20000
+#define	LK_EXCLOTHER	0x30000
+#define	LK_EXCLUSIVE	0x40000
+#define	LK_RELEASE	0x50000
+#define	LK_SHARED	0x60000
+#define	LK_UPGRADE	0x70000
+
+#define	LK_TOTAL_MASK	(LK_INIT_MASK | LK_EATTR_MASK | LK_TYPE_MASK)
+
 /*
- * Internal state flags corresponding to lk_sharecount, and lk_waitcount
+ * Default values for lockmgr_args().
  */
-#define	LK_SHARE_NONZERO 0x10000000
-#define	LK_WAIT_NONZERO  0x20000000
-
-#ifndef LOCK_FILE
-#error "LOCK_FILE not defined, include <sys/lock.h> before <sys/lockmgr.h>"
-#endif
+#define	LK_WMESG_DEFAULT	(NULL)
+#define	LK_PRIO_DEFAULT		(0)
+#define	LK_TIMO_DEFAULT		(0)
 
 /*
  * Assertion flags.
  */
 #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
-#define	KA_BASE		(LA_MASKASSERT + 1)
 #define	KA_LOCKED	LA_LOCKED
 #define	KA_SLOCKED	LA_SLOCKED
 #define	KA_XLOCKED	LA_XLOCKED
 #define	KA_UNLOCKED	LA_UNLOCKED
 #define	KA_RECURSED	LA_RECURSED
 #define	KA_NOTRECURSED	LA_NOTRECURSED
-#define	KA_HELD		(KA_BASE << 0x00)
-#define	KA_UNHELD	(KA_BASE << 0x01)
-#endif
-
-/*
- * Lock return status.
- *
- * Successfully obtained locks return 0. Locks will always succeed
- * unless one of the following is true:
- *	LK_FORCEUPGRADE is requested and some other process has already
- *	    requested a lock upgrade (returns EBUSY).
- *	LK_WAIT is set and a sleep would be required (returns EBUSY).
- *	LK_SLEEPFAIL is set and a sleep was done (returns ENOLCK).
- *	PCATCH is set in lock priority and a signal arrives (returns
- *	    either EINTR or ERESTART if system calls is to be restarted).
- *	Non-null lock timeout and timeout expires (returns EWOULDBLOCK).
- * A failed lock attempt always returns a non-zero error value. No lock
- * is held after an error return (in particular, a failed LK_UPGRADE
- * or LK_FORCEUPGRADE will have released its shared access lock).
- */
-
-/*
- * Indicator that no process holds exclusive lock
- */
-#define LK_KERNPROC ((struct thread *)-2)
-#define LK_NOPROC ((struct thread *) -1)
-
-struct thread;
-
-void	lockinit(struct lock *, int prio, const char *wmesg,
-			int timo, int flags);
-void	lockdestroy(struct lock *);
-
-int	_lockmgr_args(struct lock *, u_int flags, struct mtx *,
-	    const char *wmesg, int prio, int timo, char *file, int line);
-#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
-void	_lockmgr_assert(struct lock *, int what, const char *, int);
-#endif
-void	_lockmgr_disown(struct lock *, const char *, int);
-void	lockmgr_printinfo(struct lock *);
-int	lockstatus(struct lock *);
-
-#define lockmgr(lock, flags, mtx)					\
-	_lockmgr_args((lock), (flags), (mtx), LK_WMESG_DEFAULT,		\
-	    LK_PRIO_DEFAULT, LK_TIMO_DEFAULT, LOCK_FILE, LOCK_LINE)
-#define	lockmgr_disown(lock)						\
-	_lockmgr_disown((lock), LOCK_FILE, LOCK_LINE)
-#define	lockmgr_args(lock, flags, mtx, wmesg, prio, timo)		\
-	_lockmgr_args((lock), (flags), (mtx), (wmesg), (prio), (timo),	\
-	    LOCK_FILE, LOCK_LINE)
-#define	lockmgr_recursed(lkp)						\
-	((lkp)->lk_exclusivecount > 1)
-#define	lockmgr_waiters(lkp)						\
-	((lkp)->lk_waitcount != 0)
-#ifdef INVARIANTS
-#define	lockmgr_assert(lkp, what)					\
-	_lockmgr_assert((lkp), (what), LOCK_FILE, LOCK_LINE)
-#else
-#define	lockmgr_assert(lkp, what)
-#endif
-#ifdef DDB
-int	lockmgr_chain(struct thread *td, struct thread **ownerp);
+#define	KA_HELD
+#define	KA_UNHELD
 #endif
 
 #endif /* _KERNEL */
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index a68fbef..a3e055f 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -200,7 +200,8 @@ struct thread {
 	volatile u_char td_owepreempt;  /* (k*) Preempt on last critical_exit */
 	u_char		td_tsqueue;	/* (t) Turnstile queue blocked on. */
 	short		td_locks;	/* (k) Count of non-spin locks. */
-	short		td_rw_rlocks;	/* (k) count of rwlock read locks. */
+	short		td_rw_rlocks;	/* (k) Count of rwlock read locks. */
+	short		td_lk_slocks;	/* (k) Count of lockmgr shared locks. */
 	struct turnstile *td_blocked;	/* (t) Lock thread is blocked on. */
 	const char	*td_lockname;	/* (t) Name of lock blocked on. */
 	LIST_HEAD(, turnstile) td_contested;	/* (q) Contested locks. */
diff --git a/sys/sys/sleepqueue.h b/sys/sys/sleepqueue.h
index 170d858..57b2c43 100644
--- a/sys/sys/sleepqueue.h
+++ b/sys/sys/sleepqueue.h
@@ -87,6 +87,7 @@ struct thread;
 #define	SLEEPQ_CONDVAR		0x01		/* Used for a cv. */
 #define	SLEEPQ_PAUSE		0x02		/* Used by pause. */
 #define	SLEEPQ_SX		0x03		/* Used by an sx lock. */
+#define	SLEEPQ_LK		0x04		/* Used by a lockmgr. */
 #define	SLEEPQ_INTERRUPTIBLE	0x100		/* Sleep is interruptible. */
 
 void	init_sleepqueues(void);
diff --git a/sys/sys/stack.h b/sys/sys/stack.h
index 6bd52ab..0ebdd22 100644
--- a/sys/sys/stack.h
+++ b/sys/sys/stack.h
@@ -29,15 +29,10 @@
 #ifndef _SYS_STACK_H_
 #define	_SYS_STACK_H_
 
-#define	STACK_MAX	18	/* Don't change, stack_ktr relies on this. */
+#include <sys/_stack.h>
 
 struct sbuf;
 
-struct stack {
-	int		depth;
-	vm_offset_t	pcs[STACK_MAX];
-};
-
 /* MI Routines. */
 struct stack	*stack_create(void);
 void		 stack_destroy(struct stack *);
diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h
index b82ab13..6705b3c 100644
--- a/sys/sys/vnode.h
+++ b/sys/sys/vnode.h
@@ -395,8 +395,10 @@ extern void	(*lease_updatetime)(int deltat);
 #define	VI_UNLOCK(vp)	mtx_unlock(&(vp)->v_interlock)
 #define	VI_MTX(vp)	(&(vp)->v_interlock)
 
-#define	VN_LOCK_AREC(vp)	((vp)->v_vnlock->lk_flags |= LK_CANRECURSE)
-#define	VN_LOCK_ASHARE(vp)	((vp)->v_vnlock->lk_flags &= ~LK_NOSHARE)
+#define	VN_LOCK_AREC(vp)						\
+	((vp)->v_vnlock->lock_object.lo_flags |= LK_CANRECURSE)
+#define	VN_LOCK_ASHARE(vp)						\
+	((vp)->v_vnlock->lock_object.lo_flags &= ~LK_NOSHARE)
 
 #endif /* _KERNEL */
 
diff --git a/sys/ufs/ffs/ffs_softdep.c b/sys/ufs/ffs/ffs_softdep.c
index 255723c..e4c0def 100644
--- a/sys/ufs/ffs/ffs_softdep.c
+++ b/sys/ufs/ffs/ffs_softdep.c
@@ -553,8 +553,8 @@ MTX_SYSINIT(softdep_lock, &lk, "Softdep Lock", MTX_DEF);
 #define ACQUIRE_LOCK(lk)		mtx_lock(lk)
 #define FREE_LOCK(lk)			mtx_unlock(lk)
 
-#define	BUF_AREC(bp)	((bp)->b_lock.lk_flags |= LK_CANRECURSE)
-#define	BUF_NOREC(bp)	((bp)->b_lock.lk_flags &= ~LK_CANRECURSE)
+#define	BUF_AREC(bp)	((bp)->b_lock.lock_object.lo_flags |= LK_CANRECURSE)
+#define	BUF_NOREC(bp)	((bp)->b_lock.lock_object.lo_flags &= ~LK_CANRECURSE)
 
 /*
  * Worklist queue management.