Major update to the way synchronization is done in the kernel. Highlights

include:

* Mutual exclusion is used instead of spl*().  See mutex(9).  (Note: The
  alpha port is still in transition and currently uses both.)

* Per-CPU idle processes.

* Interrupts are run in their own separate kernel threads and can be
  preempted (i386 only).

Partially contributed by:	BSDi (BSD/OS)
Submissions by (at least):	cp, dfr, dillon, grog, jake, jhb, sheldonh

7 files changed, 910 insertions, 74 deletions

diff --git a/sys/amd64/include/cpu.h b/sys/amd64/include/cpu.h
index ffabf7f..18822b8 100644
--- a/sys/amd64/include/cpu.h
+++ b/sys/amd64/include/cpu.h
@@ -46,6 +46,7 @@
 #include <machine/psl.h>
 #include <machine/frame.h>
 #include <machine/segments.h>
+#include <machine/globals.h>
 
 /*
  * definitions of cpu-dependent requirements
@@ -86,7 +87,9 @@
  * added, we will have an atomicy problem.  The type of atomicy we need is
  * a non-locked orl.
  */
-#define	need_resched()		do { astpending = AST_RESCHED|AST_PENDING; } while (0)
+#define	need_resched() do {						\
+	PCPU_SET(astpending, AST_RESCHED|AST_PENDING);			\
+} while (0)
 #define	resched_wanted()	(astpending & AST_RESCHED)
 
 /*
@@ -109,8 +112,9 @@
  * it off (asynchronous need_resched() conflicts are not critical).
  */
 #define	signotify(p)	aston()
-
-#define	aston()			do { astpending |= AST_PENDING; } while (0)
+#define	aston() do {							\
+	PCPU_SET(astpending, astpending | AST_PENDING);			\
+} while (0)
 #define astoff()
 
 /*
@@ -135,7 +139,9 @@
 #ifdef _KERNEL
 extern char	btext[];
 extern char	etext[];
+#ifndef intr_nesting_level
 extern u_char	intr_nesting_level;
+#endif
 
 void	fork_trampoline __P((void));
 void	fork_return __P((struct proc *, struct trapframe));
diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h
index 9a4052f..39868df 100644
--- a/sys/amd64/include/cpufunc.h
+++ b/sys/amd64/include/cpufunc.h
@@ -86,20 +86,29 @@ static __inline void
 disable_intr(void)
 {
 	__asm __volatile("cli" : : : "memory");
-#ifdef SMP
-	MPINTR_LOCK();
-#endif
 }
 
 static __inline void
 enable_intr(void)
 {
-#ifdef SMP
-	MPINTR_UNLOCK();
-#endif
 	__asm __volatile("sti");
 }
 
+static __inline u_int
+save_intr(void)
+{
+	u_int	ef;
+
+	__asm __volatile("pushfl; popl %0" : "=r" (ef));
+	return (ef);
+}
+
+static __inline void
+restore_intr(u_int ef)
+{
+	__asm __volatile("pushl %0; popfl" : : "r" (ef) : "memory" );
+}
+
 #define	HAVE_INLINE_FFS
 
 static __inline int
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index 61c5ecf..95b5759 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -36,6 +36,7 @@
 #endif
 
 #include <sys/param.h>
+#include <sys/bus.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/proc.h>
@@ -65,6 +66,7 @@
 #include <machine/apic.h>
 #include <machine/atomic.h>
 #include <machine/cpufunc.h>
+#include <machine/mutex.h>
 #include <machine/mpapic.h>
 #include <machine/psl.h>
 #include <machine/segments.h>
@@ -236,6 +238,8 @@ typedef struct BASETABLE_ENTRY {
 
 #define MP_ANNOUNCE_POST	0x19
 
+/* used to hold the AP's until we are ready to release them */
+struct simplelock	ap_boot_lock;
 
 /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
 int	current_postcode;
@@ -336,6 +340,7 @@ static int	start_all_aps(u_int boot_addr);
 static void	install_ap_tramp(u_int boot_addr);
 static int	start_ap(int logicalCpu, u_int boot_addr);
 static int	apic_int_is_bus_type(int intr, int bus_type);
+static void	release_aps(void *dummy);
 
 /*
  * Calculate usable address in base memory for AP trampoline code.
@@ -403,7 +408,7 @@ found:
 
 
 /*
- * Startup the SMP processors.
+ * Initialize the SMP hardware and the APIC and start up the AP's.
  */
 void
 mp_start(void)
@@ -619,6 +624,9 @@ mp_enable(u_int boot_addr)
 	/* initialize all SMP locks */
 	init_locks();
 
+	/* obtain the ap_boot_lock */
+	s_lock(&ap_boot_lock);
+
 	/* start each Application Processor */
 	start_all_aps(boot_addr);
 }
@@ -1866,9 +1874,6 @@ struct simplelock	fast_intr_lock;
 /* critical region around INTR() routines */
 struct simplelock	intr_lock;
 
-/* lock regions protected in UP kernel via cli/sti */
-struct simplelock	mpintr_lock;
-
 /* lock region used by kernel profiling */
 struct simplelock	mcount_lock;
 
@@ -1885,26 +1890,16 @@ struct simplelock	clock_lock;
 /* lock around the MP rendezvous */
 static struct simplelock smp_rv_lock;
 
+/* only 1 CPU can panic at a time :) */
+struct simplelock	panic_lock;
+
 static void
 init_locks(void)
 {
-	/*
-	 * Get the initial mp_lock with a count of 1 for the BSP.
-	 * This uses a LOGICAL cpu ID, ie BSP == 0.
-	 */
-	mp_lock = 0x00000001;
-
-#if 0
-	/* ISR uses its own "giant lock" */
-	isr_lock = FREE_LOCK;
-#endif
-
 #if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
 	s_lock_init((struct simplelock*)&apic_itrace_debuglock);
 #endif
 
-	s_lock_init((struct simplelock*)&mpintr_lock);
-
 	s_lock_init((struct simplelock*)&mcount_lock);
 
 	s_lock_init((struct simplelock*)&fast_intr_lock);
@@ -1912,6 +1907,7 @@ init_locks(void)
 	s_lock_init((struct simplelock*)&imen_lock);
 	s_lock_init((struct simplelock*)&cpl_lock);
 	s_lock_init(&smp_rv_lock);
+	s_lock_init(&panic_lock);
 
 #ifdef USE_COMLOCK
 	s_lock_init((struct simplelock*)&com_lock);
@@ -1919,11 +1915,9 @@ init_locks(void)
 #ifdef USE_CLOCKLOCK
 	s_lock_init((struct simplelock*)&clock_lock);
 #endif /* USE_CLOCKLOCK */
-}
-
 
-/* Wait for all APs to be fully initialized */
-extern int wait_ap(unsigned int);
+	s_lock_init(&ap_boot_lock);
+}
 
 /*
  * start each AP in our list
@@ -1987,6 +1981,7 @@ start_all_aps(u_int boot_addr)
 		SMPpt[pg + 4] = 0;		/* *prv_PMAP1 */
 
 		/* prime data page for it to use */
+		SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu);
 		gd->gd_cpuid = x;
 		gd->gd_cpu_lockid = x << 24;
 		gd->gd_prv_CMAP1 = &SMPpt[pg + 1];
@@ -2211,7 +2206,6 @@ start_ap(int logical_cpu, u_int boot_addr)
 	return 0;		/* return FAILURE */
 }
 
-
 /*
  * Flush the TLB on all other CPU's
  *
@@ -2348,10 +2342,13 @@ SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
 void ap_init(void);
 
 void
-ap_init()
+ap_init(void)
 {
 	u_int	apic_id;
 
+	/* lock against other AP's that are waking up */
+	s_lock(&ap_boot_lock);
+
 	/* BSP may have changed PTD while we're waiting for the lock */
 	cpu_invltlb();
 
@@ -2397,6 +2394,30 @@ ap_init()
 		smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */
 		smp_active = 1;	 /* historic */
 	}
+
+	/* let other AP's wake up now */
+	s_unlock(&ap_boot_lock);
+
+	/* wait until all the AP's are up */
+	while (smp_started == 0)
+		; /* nothing */
+
+	/*
+	 * Set curproc to our per-cpu idleproc so that mutexes have
+	 * something unique to lock with.
+	 */
+	PCPU_SET(curproc,idleproc);
+	PCPU_SET(prevproc,idleproc);
+
+	microuptime(&switchtime);
+	switchticks = ticks;
+
+	/* ok, now grab sched_lock and enter the scheduler */
+	enable_intr();
+	mtx_enter(&sched_lock, MTX_SPIN);
+	cpu_throw();	/* doesn't return */
+
+	panic("scheduler returned us to ap_init");
 }
 
 #ifdef BETTER_CLOCK
@@ -2453,6 +2474,12 @@ forwarded_statclock(int id, int pscnt, int *astmap)
 	p = checkstate_curproc[id];
 	cpustate = checkstate_cpustate[id];
 
+	/* XXX */
+	if (p->p_ithd)
+		cpustate = CHECKSTATE_INTR;
+	else if (p == idleproc)
+		cpustate = CHECKSTATE_SYS;
+
 	switch (cpustate) {
 	case CHECKSTATE_USER:
 		if (p->p_flag & P_PROFIL)
@@ -2482,9 +2509,10 @@ forwarded_statclock(int id, int pscnt, int *astmap)
 		if (pscnt > 1)
 			return;
 
-		if (!p)
+		if (p == idleproc) {
+			p->p_sticks++;
 			cp_time[CP_IDLE]++;
-		else {
+		} else {
 			p->p_sticks++;
 			cp_time[CP_SYS]++;
 		}
@@ -2510,7 +2538,7 @@ forwarded_statclock(int id, int pscnt, int *astmap)
 			p->p_iticks++;
 		cp_time[CP_INTR]++;
 	}
-	if (p != NULL) {
+	if (p != idleproc) {
 		schedclock(p);
 		
 		/* Update resource usage integrals and maximums. */
@@ -2863,3 +2891,11 @@ smp_rendezvous(void (* setup_func)(void *),
 	/* release lock */
 	s_unlock(&smp_rv_lock);
 }
+
+void
+release_aps(void *dummy __unused)
+{
+	s_unlock(&ap_boot_lock);
+}
+
+SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
diff --git a/sys/amd64/include/mutex.h b/sys/amd64/include/mutex.h
new file mode 100644
index 0000000..ef0c963
--- /dev/null
+++ b/sys/amd64/include/mutex.h
@@ -0,0 +1,786 @@
+/*-
+ * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Berkeley Software Design Inc's name may not be used to endorse or
+ *    promote products derived from this software without specific prior
+ *    written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from BSDI $Id: mutex.h,v 2.7.2.35 2000/04/27 03:10:26 cp Exp $
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_MUTEX_H_
+#define _MACHINE_MUTEX_H_
+
+#ifndef LOCORE
+
+#include <sys/ktr.h>
+#include <sys/queue.h>
+#include <machine/atomic.h>
+#include <machine/cpufunc.h>
+#include <machine/globals.h>
+
+/*
+ * If kern_mutex.c is being built, compile non-inlined versions of various
+ * functions so that kernel modules can use them.
+ */
+#ifndef _KERN_MUTEX_C_
+#define _MTX_INLINE	static __inline
+#else
+#define _MTX_INLINE
+#endif
+
+/*
+ * Mutex flags
+ *
+ * Types
+ */
+#define	MTX_DEF		0x0		/* Default (spin/sleep) */
+#define MTX_SPIN	0x1		/* Spin only lock */
+
+/* Options */
+#define	MTX_RLIKELY	0x4		/* (opt) Recursion likely */
+#define	MTX_NORECURSE	0x8		/* No recursion possible */
+#define	MTX_NOSPIN	0x10		/* Don't spin before sleeping */
+#define	MTX_NOSWITCH	0x20		/* Do not switch on release */
+#define	MTX_FIRST	0x40		/* First spin lock holder */
+#define MTX_TOPHALF	0x80		/* Interrupts not disabled on spin */
+
+/* options that should be passed on to mtx_enter_hard, mtx_exit_hard */
+#define	MTX_HARDOPTS	(MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH)
+
+/* Flags/value used in mtx_lock */
+#define	MTX_RECURSE	0x01		/* (non-spin) lock held recursively */
+#define	MTX_CONTESTED	0x02		/* (non-spin) lock contested */
+#define	MTX_FLAGMASK	~(MTX_RECURSE | MTX_CONTESTED)
+#define MTX_UNOWNED	0x8		/* Cookie for free mutex */
+
+struct proc;	/* XXX */
+
+/*
+ * Sleep/spin mutex
+ */
+struct mtx {
+	volatile u_int	mtx_lock;	/* lock owner/gate/flags */
+	volatile u_short mtx_recurse;	/* number of recursive holds */
+	u_short		mtx_f1;
+	u_int		mtx_savefl;	/* saved flags (for spin locks) */
+	char		*mtx_description;
+	TAILQ_HEAD(, proc) mtx_blocked;
+	LIST_ENTRY(mtx)	mtx_contested;
+	struct mtx	*mtx_next;	/* all locks in system */
+	struct mtx	*mtx_prev;
+#ifdef SMP_DEBUG
+	/* If you add anything here, adjust the mtxf_t definition below */
+	struct witness	*mtx_witness;
+	LIST_ENTRY(mtx)	mtx_held;
+	char		*mtx_file;
+	int		mtx_line;
+#endif /* SMP_DEBUG */
+};
+
+typedef struct mtx mtx_t;
+
+/*
+ * Filler for structs which need to remain the same size
+ * whether or not SMP_DEBUG is turned on.
+ */
+typedef struct mtxf {
+#ifdef SMP_DEBUG
+	char	mtxf_data[0];
+#else
+	char	mtxf_data[4*sizeof(void *) + sizeof(int)];
+#endif
+}	mtxf_t;
+
+#define mp_fixme(string)
+
+#ifdef _KERNEL
+/* Misc */
+#define CURTHD	((u_int)CURPROC)	/* Current thread ID */
+
+/* Prototypes */
+void	mtx_init(mtx_t *m, char *description, int flag);
+void	mtx_enter_hard(mtx_t *, int type, int flags);
+void	mtx_exit_hard(mtx_t *, int type);
+void	mtx_destroy(mtx_t *m);
+
+#if (defined(KLD_MODULE) || defined(_KERN_MUTEX_C_))
+void	mtx_enter(mtx_t *mtxp, int type);
+int	mtx_try_enter(mtx_t *mtxp, int type);
+void	mtx_exit(mtx_t *mtxp, int type);
+#endif
+
+/* Global locks */
+extern mtx_t	sched_lock;
+extern mtx_t	Giant;
+
+/*
+ * Used to replace return with an exit Giant and return.
+ */
+
+#define EGAR(a)								\
+do {									\
+	mtx_exit(&Giant, MTX_DEF);					\
+	return (a);							\
+} while (0)
+
+#define VEGAR								\
+do {									\
+	mtx_exit(&Giant, MTX_DEF);					\
+	return;								\
+} while (0)
+
+#define DROP_GIANT()							\
+do {									\
+	int _giantcnt;							\
+	WITNESS_SAVE_DECL(Giant);					\
+									\
+	WITNESS_SAVE(&Giant, Giant);					\
+	for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++)		\
+		mtx_exit(&Giant, MTX_DEF)
+
+#define PICKUP_GIANT()							\
+	mtx_assert(&Giant, MA_NOTOWNED);				\
+	while (_giantcnt--)						\
+		mtx_enter(&Giant, MTX_DEF);				\
+	WITNESS_RESTORE(&Giant, Giant);					\
+} while (0)
+
+#define PARTIAL_PICKUP_GIANT()						\
+	mtx_assert(&Giant, MA_NOTOWNED);				\
+	while (_giantcnt--)						\
+		mtx_enter(&Giant, MTX_DEF);				\
+	WITNESS_RESTORE(&Giant, Giant)
+
+
+/*
+ * Debugging
+ */
+#ifndef SMP_DEBUG
+#define mtx_assert(m, what)
+#else /* SMP_DEBUG */
+
+#define MA_OWNED	1
+#define MA_NOTOWNED	2
+#define mtx_assert(m, what) {						\
+	switch ((what)) {						\
+	case MA_OWNED:							\
+		ASS(mtx_owned((m)));					\
+		break;							\
+	case MA_NOTOWNED:						\
+		ASS(!mtx_owned((m)));					\
+		break;							\
+	default:							\
+		panic("unknown mtx_assert at %s:%d", __FILE__, __LINE__); \
+	}								\
+}
+
+#ifdef INVARIANTS
+#define	ASS(ex) MPASS(ex)
+#define MPASS(ex) if (!(ex)) panic("Assertion %s failed at %s:%d",	\
+                #ex, __FILE__, __LINE__)
+#define MPASS2(ex, what) if (!(ex)) panic("Assertion %s failed at %s:%d", \
+                what, __FILE__, __LINE__)
+
+#ifdef MTX_STRS
+char STR_IEN[] = "fl & 0x200";
+char STR_IDIS[] = "!(fl & 0x200)";
+#else /* MTX_STRS */
+extern char STR_IEN[];
+extern char STR_IDIS[];
+#endif /* MTX_STRS */
+#define	ASS_IEN		MPASS2(read_eflags() & 0x200, STR_IEN)
+#define	ASS_IDIS	MPASS2((read_eflags() & 0x200) == 0, STR_IDIS)
+#endif /* INVARIANTS */
+
+#endif /* SMP_DEBUG */
+
+#if !defined(SMP_DEBUG) || !defined(INVARIANTS)
+#define ASS(ex)
+#define	MPASS(ex)
+#define	MPASS2(ex, where)
+#define	ASS_IEN
+#define ASS_IDIS
+#endif /* !defined(SMP_DEBUG) || !defined(INVARIANTS) */
+
+#ifdef	WITNESS
+#ifndef	SMP_DEBUG
+#error	WITNESS requires SMP_DEBUG
+#endif	/* SMP_DEBUG */
+#define WITNESS_ENTER(m, f)						\
+	if ((m)->mtx_witness != NULL)					\
+		witness_enter((m), (f), __FILE__, __LINE__)
+#define WITNESS_EXIT(m, f)						\
+	if ((m)->mtx_witness != NULL)					\
+		witness_exit((m), (f), __FILE__, __LINE__)
+
+#define	WITNESS_SLEEP(check, m) witness_sleep(check, (m), __FILE__, __LINE__)
+#define	WITNESS_SAVE_DECL(n)						\
+	char * __CONCAT(n, __wf);					\
+	int __CONCAT(n, __wl)
+
+#define	WITNESS_SAVE(m, n) 						\
+do {									\
+	if ((m)->mtx_witness != NULL) 					\
+	    witness_save(m, &__CONCAT(n, __wf), &__CONCAT(n, __wl));	\
+} while (0)
+
+#define	WITNESS_RESTORE(m, n) 						\
+do {									\
+	if ((m)->mtx_witness != NULL)					\
+	    witness_restore(m, __CONCAT(n, __wf), __CONCAT(n, __wl));	\
+} while (0)
+
+void	witness_init(mtx_t *, int flag);
+void	witness_destroy(mtx_t *);
+void	witness_enter(mtx_t *, int, char *, int);
+void	witness_try_enter(mtx_t *, int, char *, int);
+void	witness_exit(mtx_t *, int, char *, int);
+void	witness_display(void(*)(const char *fmt, ...));
+void	witness_list(struct proc *);
+int	witness_sleep(int, mtx_t *, char *, int);
+void	witness_save(mtx_t *, char **, int *);
+void	witness_restore(mtx_t *, char *, int);
+#else /* WITNESS */
+#define WITNESS_ENTER(m, flag)
+#define WITNESS_EXIT(m, flag)
+#define	WITNESS_SLEEP(check, m)
+#define	WITNESS_SAVE_DECL(n)
+#define	WITNESS_SAVE(m, n)
+#define	WITNESS_RESTORE(m, n)
+
+/*
+ * flag++ is slezoid way of shutting up unused parameter warning
+ * in mtx_init()
+ */
+#define witness_init(m, flag) flag++
+#define witness_destroy(m)
+#define witness_enter(m, flag,  f, l)
+#define witness_try_enter(m, flag, f, l )
+#define witness_exit(m, flag, f, l)
+#endif /* WITNESS */
+
+/*
+ * Assembly macros (for internal use only)
+ *------------------------------------------------------------------------------
+ */
+
+#define	_V(x)	__STRING(x)
+
+#ifndef I386_CPU
+
+/*
+ * For 486 and newer processors.
+ */
+
+/* Get a sleep lock, deal with recursion inline. */
+#define	_getlock_sleep(mtxp, tid, type) ({				\
+	int	_res;							\
+									\
+	__asm __volatile (						\
+"	movl	$" _V(MTX_UNOWNED) ",%%eax;"	/* Unowned cookie */	\
+"	" MPLOCKED ""							\
+"	cmpxchgl %3,%1;"			/* Try */		\
+"	jz	1f;"				/* Got it */		\
+"	andl	$" _V(MTX_FLAGMASK) ",%%eax;"	/* turn off spec bits */ \
+"	cmpl	%%eax,%3;"			/* already have it? */	\
+"	je	2f;"				/* yes, recurse */	\
+"	pushl	%4;"							\
+"	pushl	%5;"							\
+"	call	mtx_enter_hard;"					\
+"	addl	$8,%%esp;"						\
+"	jmp	1f;"							\
+"2:	lock; orl $" _V(MTX_RECURSE) ",%1;"				\
+"	incw	%2;"							\
+"1:"									\
+"# getlock_sleep"							\
+	: "=&a" (_res),				/* 0 (dummy output) */	\
+	  "+m" (mtxp->mtx_lock), 		/* 1 */			\
+	  "+m" (mtxp->mtx_recurse) 		/* 2 */			\
+	: "r" (tid),				/* 3 (input) */		\
+	  "gi" (type),				/* 4 */			\
+	  "g" (mtxp)				/* 5 */			\
+	: "memory", "ecx", "edx"		/* used */ );		\
+})
+
+/* Get a spin lock, handle recursion inline (as the less common case) */
+#define	_getlock_spin_block(mtxp, tid, type) ({				\
+	int	_res;							\
+									\
+	__asm __volatile (						\
+"	pushfl;"							\
+"	cli;"								\
+"	movl	$" _V(MTX_UNOWNED) ",%%eax;"	/* Unowned cookie */	\
+"	" MPLOCKED ""							\
+"	cmpxchgl %3,%1;"			/* Try */		\
+"	jz	2f;"				/* got it */		\
+"	pushl	%4;"							\
+"	pushl	%5;"							\
+"	call	mtx_enter_hard;" /* mtx_enter_hard(mtxp, type, oflags) */ \
+"	addl	$0xc,%%esp;"						\
+"	jmp	1f;"							\
+"2:	popl	%2;"				/* save flags */	\
+"1:"									\
+"# getlock_spin_block"							\
+	: "=&a" (_res),				/* 0 (dummy output) */	\
+	  "+m" (mtxp->mtx_lock),		/* 1 */			\
+	  "=m" (mtxp->mtx_savefl)		/* 2 */			\
+	: "r" (tid),				/* 3 (input) */		\
+	  "gi" (type),				/* 4 */			\
+	  "g" (mtxp)				/* 5 */			\
+	: "memory", "ecx", "edx"		/* used */ );		\
+})
+
+/*
+ * Get a lock without any recursion handling. Calls the hard enter function if
+ * we can't get it inline.
+ */
+#define	_getlock_norecurse(mtxp, tid, type) ({				\
+	int	_res;							\
+									\
+	__asm __volatile (						\
+"	movl	$" _V(MTX_UNOWNED) ",%%eax;"	/* Unowned cookie */	\
+"	" MPLOCKED ""							\
+"	cmpxchgl %2,%1;"			/* Try */		\
+"	jz	1f;"				/* got it */		\
+"	pushl	%3;"							\
+"	pushl	%4;"							\
+"	call	mtx_enter_hard;" /* mtx_enter_hard(mtxp, type) */	\
+"	addl	$8,%%esp;"						\
+"1:"									\
+"# getlock_norecurse"							\
+	: "=&a" (_res),				/* 0 (dummy output) */	\
+	  "+m" (mtxp->mtx_lock)			/* 1 */			\
+	: "r" (tid),				/* 2 (input) */		\
+	  "gi" (type),				/* 3 */			\
+	  "g" (mtxp)				/* 4 */			\
+	: "memory", "ecx", "edx"		/* used */ );		\
+})
+
+/*
+ * Release a sleep lock assuming we haven't recursed on it, recursion is handled
+ * in the hard function.
+ */
+#define	_exitlock_norecurse(mtxp, tid, type) ({				\
+	int	_tid = (int)(tid);					\
+									\
+	__asm __volatile (						\
+"	" MPLOCKED ""							\
+"	cmpxchgl %4,%0;"			/* try easy rel */	\
+"	jz	1f;"				/* released! */		\
+"	pushl	%2;"							\
+"	pushl	%3;"							\
+"	call	mtx_exit_hard;"						\
+"	addl	$8,%%esp;"						\
+"1:"									\
+"# exitlock_norecurse"							\
+	: "+m" (mtxp->mtx_lock), 		/* 0 */			\
+	  "+a" (_tid)				/* 1 */			\
+	: "gi" (type),				/* 2 (input) */		\
+	  "g" (mtxp),				/* 3 */			\
+	  "r" (MTX_UNOWNED)			/* 4 */			\
+	: "memory", "ecx", "edx"		/* used */ );		\
+})
+
+/*
+ * Release a sleep lock when its likely we recursed (the code to
+ * deal with simple recursion is inline).
+ */
+#define	_exitlock(mtxp, tid, type) ({					\
+	int	_tid = (int)(tid);					\
+									\
+	__asm __volatile (						\
+"	" MPLOCKED ""							\
+"	cmpxchgl %5,%0;"			/* try easy rel */	\
+"	jz	1f;"				/* released! */		\
+"	testl	$" _V(MTX_RECURSE) ",%%eax;"	/* recursed? */		\
+"	jnz	3f;"				/* handle recursion */	\
+	/* Lock not recursed and contested: do the hard way */		\
+"	pushl	%3;"							\
+"	pushl	%4;"							\
+"	call	mtx_exit_hard;"			/* mtx_exit_hard(mtxp,type) */ \
+"	addl	$8,%%esp;"						\
+"	jmp	1f;"							\
+	/* lock recursed, lower recursion level */			\
+"3:	decw	%1;"				/* one less level */	\
+"	jnz	1f;"				/* still recursed, done */ \
+"	lock; andl $~" _V(MTX_RECURSE) ",%0;"	/* turn off recurse flag */ \
+"1:"									\
+"# exitlock"								\
+	: "+m" (mtxp->mtx_lock),		/* 0 */			\
+	  "+m" (mtxp->mtx_recurse), 		/* 1 */ 		\
+	  "+a" (_tid)				/* 2 */			\
+	: "gi" (type),				/* 3 (input) */		\
+	  "g" (mtxp),				/* 4 */			\
+	  "r" (MTX_UNOWNED)			/* 5 */			\
+	: "memory", "ecx", "edx"		/* used */ );		\
+})
+
+/*
+ * Release a spin lock (with possible recursion).
+ *
+ * We use cmpxchgl to clear lock (instead of simple store) to flush posting
+ * buffers and make the change visible to other CPU's.
+ */
+#define	_exitlock_spin(mtxp, inten1, inten2) ({				\
+	int	_res;							\
+									\
+	__asm __volatile (						\
+"	movw	%1,%%ax;"						\
+"	decw	%%ax;"							\
+"	js	1f;"							\
+"	movw	%%ax,%1;"						\
+"	jmp	2f;"							\
+"1:	movl	%0,%%eax;"						\
+"	movl	$ " _V(MTX_UNOWNED) ",%%ecx;"				\
+"	" inten1 ";"							\
+"	" MPLOCKED ""							\
+"	cmpxchgl %%ecx,%0;"				  		\
+"	" inten2 ";"							\
+"2:"									\
+"# exitlock_spin"							\
+	: "+m" (mtxp->mtx_lock),	/* 0 */				\
+	  "+m" (mtxp->mtx_recurse),	/* 1 */ 			\
+	  "=&a" (_res)			/* 2 */				\
+	: "g"  (mtxp->mtx_savefl)	/* 3 (used in 'inten') */	\
+	: "memory", "ecx"		/* used */ );			\
+})
+
+#else	/* I386_CPU */
+
+/*
+ * For 386 processors only.
+ */
+
+/* Get a sleep lock, deal with recursion inline. */
+#define	_getlock_sleep(mp, tid, type) do {				\
+	if (atomic_cmpset_int(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) { \
+		if (((mp)->mtx_lock & MTX_FLAGMASK) != (tid))		\
+			mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0);	\
+		else {							\
+			atomic_set_int(&(mp)->mtx_lock, MTX_RECURSE);	\
+			(mp)->mtx_recurse++;				\
+		}							\
+	}								\
+} while (0)
+
+/* Get a spin lock, handle recursion inline (as the less common case) */
+#define	_getlock_spin_block(mp, tid, type) do {				\
+	u_int _mtx_fl = read_eflags();					\
+	disable_intr();							\
+	if (atomic_cmpset_int(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) \
+		mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _mtx_fl);	\
+	else								\
+		(mp)->mtx_savefl = _mtx_fl;				\
+} while (0)
+
+/*
+ * Get a lock without any recursion handling. Calls the hard enter function if
+ * we can't get it inline.
+ */
+#define	_getlock_norecurse(mp, tid, type) do {				\
+	if (atomic_cmpset_int(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) \
+		mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0);		\
+} while (0)
+
+/*
+ * Release a sleep lock assuming we haven't recursed on it, recursion is handled
+ * in the hard function.
+ */
+#define	_exitlock_norecurse(mp, tid, type) do {				\
+	if (atomic_cmpset_int(&(mp)->mtx_lock, (tid), MTX_UNOWNED) == 0) \
+		mtx_exit_hard((mp), (type) & MTX_HARDOPTS);		\
+} while (0)
+
+/*
+ * Release a sleep lock when its likely we recursed (the code to
+ * deal with simple recursion is inline).
+ */
+#define	_exitlock(mp, tid, type) do {					\
+	if (atomic_cmpset_int(&(mp)->mtx_lock, (tid), MTX_UNOWNED) == 0) { \
+		if ((mp)->mtx_lock & MTX_RECURSE) {			\
+			if (--((mp)->mtx_recurse) == 0)			\
+				atomic_clear_int(&(mp)->mtx_lock,	\
+				    MTX_RECURSE);			\
+		} else {						\
+			mtx_exit_hard((mp), (type) & MTX_HARDOPTS);	\
+		}							\
+	}								\
+} while (0)
+
+/* Release a spin lock (with possible recursion). */
+#define	_exitlock_spin(mp, inten1, inten2) do {				\
+	if ((mp)->mtx_recurse == 0) {					\
+		atomic_cmpset_int(&(mp)->mtx_lock, (mp)->mtx_lock,	\
+		    MTX_UNOWNED);					\
+		write_eflags((mp)->mtx_savefl);				\
+	} else {							\
+		(mp)->mtx_recurse--;					\
+	}								\
+} while (0)
+
+#endif	/* I386_CPU */
+
+/*
+ * Externally visible mutex functions.
+ *------------------------------------------------------------------------------
+ */
+
+/*
+ * Return non-zero if a mutex is already owned by the current thread.
+ */
+#define	mtx_owned(m)    (((m)->mtx_lock & MTX_FLAGMASK) == CURTHD)
+
+/* Common strings */
+#ifdef MTX_STRS
+#ifdef KTR_EXTEND
+
+/*
+ * KTR_EXTEND saves file name and line for all entries, so we don't need them
+ * here.  Theoretically we should also change the entries which refer to them
+ * (from CTR5 to CTR3), but since they're just passed to snprinf as the last
+ * parameters, it doesn't do any harm to leave them.
+ */
+char	STR_mtx_enter_fmt[] = "GOT %s [%x] r=%d";
+char	STR_mtx_exit_fmt[] = "REL %s [%x] r=%d";
+char	STR_mtx_try_enter_fmt[] = "TRY_ENTER %s [%x] result=%d";
+#else
+char	STR_mtx_enter_fmt[] = "GOT %s [%x] at %s:%d r=%d";
+char	STR_mtx_exit_fmt[] = "REL %s [%x] at %s:%d r=%d";
+char	STR_mtx_try_enter_fmt[] = "TRY_ENTER %s [%x] at %s:%d result=%d";
+#endif
+char	STR_mtx_bad_type[] = "((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0";
+char	STR_mtx_owned[] = "mtx_owned(_mpp)";
+char	STR_mtx_recurse[] = "_mpp->mtx_recurse == 0";
+#else /* MTX_STRS */
+extern	char STR_mtx_enter_fmt[];
+extern	char STR_mtx_bad_type[];
+extern	char STR_mtx_exit_fmt[];
+extern	char STR_mtx_owned[];
+extern	char STR_mtx_recurse[];
+extern	char STR_mtx_try_enter_fmt[];
+#endif /* MTX_STRS */
+
+#ifndef KLD_MODULE
+/*
+ * Get lock 'm', the macro handles the easy (and most common cases) and leaves
+ * the slow stuff to the mtx_enter_hard() function.
+ *
+ * Note: since type is usually a constant much of this code is optimized out.
+ */
+_MTX_INLINE void
+mtx_enter(mtx_t *mtxp, int type)
+{
+	mtx_t	*_mpp = mtxp;
+
+	/* bits only valid on mtx_exit() */
+	MPASS2(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0,
+	    STR_mtx_bad_type);
+
+	do {
+		if ((type) & MTX_SPIN) {
+			/*
+			 * Easy cases of spin locks:
+			 *
+			 * 1) We already own the lock and will simply
+			 *    recurse on it (if RLIKELY)
+			 *
+			 * 2) The lock is free, we just get it
+			 */
+			if ((type) & MTX_RLIKELY) {
+				/*
+				 * Check for recursion, if we already
+				 * have this lock we just bump the
+				 * recursion count.
+				 */
+				if (_mpp->mtx_lock == CURTHD) {
+					_mpp->mtx_recurse++;
+					break;	/* Done */
+				}
+			}
+
+			if (((type) & MTX_TOPHALF) == 0) {
+				/*
+				 * If an interrupt thread uses this
+				 * we must block interrupts here.
+				 */
+				if ((type) & MTX_FIRST) {
+					ASS_IEN;
+					disable_intr();
+					_getlock_norecurse(_mpp, CURTHD,
+					    (type) & MTX_HARDOPTS);
+				} else {
+					_getlock_spin_block(_mpp, CURTHD,
+					    (type) & MTX_HARDOPTS);
+				}
+			} else
+				_getlock_norecurse(_mpp, CURTHD,
+				    (type) & MTX_HARDOPTS);
+		} else {
+			/* Sleep locks */
+			if ((type) & MTX_RLIKELY)
+				_getlock_sleep(_mpp, CURTHD,
+				    (type) & MTX_HARDOPTS);
+			else
+				_getlock_norecurse(_mpp, CURTHD,
+				    (type) & MTX_HARDOPTS);
+		}
+	} while (0);
+	WITNESS_ENTER(_mpp, type);
+	CTR5(KTR_LOCK, STR_mtx_enter_fmt,
+	    (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__,
+	    (_mpp)->mtx_recurse);
+}
+
+/*
+ * Attempt to get MTX_DEF lock, return non-zero if lock acquired.
+ *
+ * XXX DOES NOT HANDLE RECURSION
+ */
+_MTX_INLINE int
+mtx_try_enter(mtx_t *mtxp, int type)
+{
+	mtx_t	*const _mpp = mtxp;
+	int	_rval;
+
+	_rval = atomic_cmpset_int(&_mpp->mtx_lock, MTX_UNOWNED, CURTHD);
+#ifdef SMP_DEBUG
+	if (_rval && (_mpp)->mtx_witness != NULL) {
+		ASS((_mpp)->mtx_recurse == 0);
+		witness_try_enter(_mpp, type,  __FILE__, __LINE__);
+	}
+#endif
+	CTR5(KTR_LOCK, STR_mtx_try_enter_fmt,
+	    (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, _rval);
+
+	return _rval;
+}
+
+#define	mtx_legal2block()	(read_eflags() & 0x200)
+
+/*
+ * Release lock m.
+ */
+_MTX_INLINE void
+mtx_exit(mtx_t *mtxp, int type)
+{
+	mtx_t	*const _mpp = mtxp;
+
+	MPASS2(mtx_owned(_mpp), STR_mtx_owned);
+	WITNESS_EXIT(_mpp, type);
+	CTR5(KTR_LOCK, STR_mtx_exit_fmt,
+	    (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__,
+	    (_mpp)->mtx_recurse);
+	if ((type) & MTX_SPIN) {
+		if ((type) & MTX_NORECURSE) {
+			MPASS2(_mpp->mtx_recurse == 0, STR_mtx_recurse);
+			atomic_cmpset_int(&_mpp->mtx_lock, _mpp->mtx_lock,
+			    MTX_UNOWNED);
+			if (((type) & MTX_TOPHALF) == 0) {
+				if ((type) & MTX_FIRST) {
+					ASS_IDIS;
+					enable_intr();
+				} else
+					write_eflags(_mpp->mtx_savefl);
+			}
+		} else {
+			if ((type) & MTX_TOPHALF)
+				_exitlock_spin(_mpp,,);
+			else {
+				if ((type) & MTX_FIRST) {
+					ASS_IDIS;
+					_exitlock_spin(_mpp,, "sti");
+				} else {
+					_exitlock_spin(_mpp,
+					    "pushl %3", "popfl");
+				}
+			}
+		}
+	} else {
+		/* Handle sleep locks */
+		if ((type) & MTX_RLIKELY)
+			_exitlock(_mpp, CURTHD, (type) & MTX_HARDOPTS);
+		else {
+			_exitlock_norecurse(_mpp, CURTHD,
+			    (type) & MTX_HARDOPTS);
+		}
+	}
+}
+
+#endif	/* KLD_MODULE */
+#endif /* _KERNEL */
+
+#else	/* !LOCORE */
+
+/*
+ * Simple assembly macros to get and release non-recursive spin locks
+ */
+
+#if defined(I386_CPU)
+
+#define	MTX_EXIT(lck, reg)						\
+	movl	$ MTX_UNOWNED,lck+MTX_LOCK;
+
+#else /* I386_CPU */
+
+#define MTX_ENTER(reg, lck)						\
+9:	movl	$ MTX_UNOWNED,%eax;					\
+	MPLOCKED							\
+	cmpxchgl reg,lck+MTX_LOCK;					\
+	jnz	9b	
+
+/* Must use locked bus op (cmpxchg) when setting to unowned (barrier) */
+#define	MTX_EXIT(lck,reg)						\
+	movl	lck+MTX_LOCK,%eax;					\
+	movl	$ MTX_UNOWNED,reg;					\
+	MPLOCKED							\
+	cmpxchgl reg,lck+MTX_LOCK;					\
+
+#define MTX_ENTER_WITH_RECURSION(reg, lck)				\
+	movl	lck+MTX_LOCK,%eax;					\
+	cmpl	PCPU_CURPROC,%eax;					\
+	jne	9f;							\
+	incw	lck+MTX_RECURSECNT;					\
+	jmp	8f;							\
+9:	movl	$ MTX_UNOWNED,%eax;					\
+	MPLOCKED							\
+	cmpxchgl reg,lck+MTX_LOCK;      				\
+	jnz	9b;							\
+8:
+
+#define	MTX_EXIT_WITH_RECURSION(lck,reg)				\
+	movw	lck+MTX_RECURSECNT,%ax;					\
+	decw	%ax;							\
+	js	9f;							\
+	movw	%ax,lck+MTX_RECURSECNT;					\
+	jmp	8f;							\
+9:	movl	lck+MTX_LOCK,%eax;					\
+	movl	$ MTX_UNOWNED,reg;					\
+	MPLOCKED							\
+	cmpxchgl reg,lck+MTX_LOCK;					\
+8:
+
+#endif /* I386_CPU */
+#endif /* !LOCORE */
+#endif /* __MACHINE_MUTEX_H */
diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h
index 08beb5a..1c7af85 100644
--- a/sys/amd64/include/pcb.h
+++ b/sys/amd64/include/pcb.h
@@ -72,11 +72,7 @@ struct pcb {
 #define	FP_SOFTFP	0x01	/* process using software fltng pnt emulator */
 #define	PCB_DBREGS	0x02	/* process using debug registers */
 	caddr_t	pcb_onfault;	/* copyin/out fault recovery */
-#ifdef SMP
-	u_long	pcb_mpnest;
-#else
-	u_long	pcb_mpnest_dontuse;
-#endif
+	int	pcb_schednest;
 	int	pcb_gs;
 	struct	pcb_ext	*pcb_ext;	/* optional pcb extension */
 	u_long	__pcb_spare[3];	/* adjust to avoid core dump size changes */
diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h
index 58bd9cf..440da60 100644
--- a/sys/amd64/include/pcpu.h
+++ b/sys/amd64/include/pcpu.h
@@ -26,6 +26,20 @@
  * $FreeBSD$
  */
 
+#ifndef _MACHINE_GLOBALDATA_H_
+#define _MACHINE_GLOBALDATA_H_
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <machine/pmap.h>
+#include <machine/segments.h>
+#include <machine/tss.h>
+
+/* XXX */
+#ifdef KTR_PERCPU
+#include <sys/ktr.h>
+#endif
+
 /*
  * This structure maps out the global data that needs to be kept on a
  * per-cpu basis.  genassym uses this to generate offsets for the assembler
@@ -41,11 +55,14 @@
 struct globaldata {
 	struct privatespace *gd_prvspace;	/* self-reference */
 	struct proc	*gd_curproc;
+	struct proc	*gd_prevproc;
 	struct proc	*gd_npxproc;
 	struct pcb	*gd_curpcb;
+	struct proc	*gd_idleproc;
 	struct timeval	gd_switchtime;
 	struct i386tss	gd_common_tss;
 	int		gd_switchticks;
+	int		gd_intr_nesting_level;
 	struct segment_descriptor gd_common_tssd;
 	struct segment_descriptor *gd_tss_gdt;
 #ifdef USER_LDT
@@ -67,8 +84,22 @@ struct globaldata {
 	unsigned	*gd_prv_PADDR1;
 #endif
 	u_int		gd_astpending;
+	SLIST_ENTRY(globaldata) gd_allcpu;
+	int		gd_witness_spin_check;
+#ifdef KTR_PERCPU
+#ifdef KTR
+	volatile int	gd_ktr_idx;
+	char		*gd_ktr_buf;
+	char		gd_ktr_buf_data[KTR_SIZE];
+#endif
+#endif
 };
 
+extern struct globaldata globaldata;
+
+SLIST_HEAD(cpuhead, globaldata);
+extern struct cpuhead cpuhead;
+
 #ifdef SMP
 /*
  * This is the upper (0xff800000) address space layout that is per-cpu.
@@ -93,3 +124,5 @@ struct privatespace {
 extern struct privatespace SMP_prvspace[];
 
 #endif
+
+#endif	/* ! _MACHINE_GLOBALDATA_H_ */
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index 69b716b..20d4fa3 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -15,6 +15,9 @@
 
 #ifdef _KERNEL
 
+#ifdef I386_CPU
+#error SMP not supported with I386_CPU
+#endif
 #if defined(SMP) && !defined(APIC_IO)
 # error APIC_IO required for SMP, add "options APIC_IO" to your config file.
 #endif /* SMP && !APIC_IO */
@@ -57,23 +60,6 @@ extern int			bootMP_size;
 /* functions in mpboot.s */
 void	bootMP			__P((void));
 
-/* global data in mplock.s */
-extern u_int			mp_lock;
-extern u_int			isr_lock;
-#ifdef RECURSIVE_MPINTRLOCK
-extern u_int			mpintr_lock;
-#endif /*  RECURSIVE_MPINTRLOCK */
-
-/* functions in mplock.s */
-void	get_mplock		__P((void));
-void	rel_mplock		__P((void));
-int		try_mplock		__P((void));
-#ifdef RECURSIVE_MPINTRLOCK
-void	get_mpintrlock		__P((void));
-void	rel_mpintrlock		__P((void));
-int		try_mpintrlock		__P((void));
-#endif /*  RECURSIVE_MPINTRLOCK */
-
 /* global data in apic_vector.s */
 extern volatile u_int		stopped_cpus;
 extern volatile u_int		started_cpus;
@@ -185,23 +171,7 @@ extern int			smp_started;
 extern volatile int		smp_idle_loops;
 
 #endif /* !LOCORE */
-#else	/* !SMP && !APIC_IO */
-
-/*
- * Create dummy MP lock empties
- */
-
-static __inline void
-get_mplock(void)
-{
-}
-
-static __inline void
-rel_mplock(void)
-{
-}
-
-#endif
+#endif /* SMP && !APIC_IO */
 
 #endif /* _KERNEL */
 #endif /* _MACHINE_SMP_H_ */