Clean up the SMP AP bootstrap and eliminate the wretched idle procs.

- We now have enough per-cpu idle context, the real idle loop has been
revived (cpu's halt now with nothing to do).
- Some preliminary support for running some operations outside the
global lock (eg: zeroing "free but not yet zeroed pages") is present
but appears to cause problems.  Off by default.
- the smp_active sysctl now behaves differently. It's merely a 'true/false'
option.  Setting smp_active to zero causes the AP's to halt in the idle
loop and stop scheduling processes.
- bootstrap is a lot safer.  Instead of sharing a statically compiled in
stack a number of times (which has caused lots of problems) and then
abandoning it, we use the idle context to boot the AP's directly.  This
should help >2 cpu support since the bootlock stuff was in doubt.
- print physical apic id in traps.. helps identify private pages getting
out of sync.  (You don't want to know how much hair I tore out with this!)

More cleanup to follow, this is more of a checkpoint than a
'finished' thing.

4 files changed, 147 insertions, 376 deletions

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 90c113c..4c4024a 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -39,7 +39,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)init_main.c	8.9 (Berkeley) 1/21/94
- * $Id: init_main.c,v 1.5 1997/08/15 02:13:31 smp Exp smp $
+ * $Id: init_main.c,v 1.69 1997/08/15 02:33:30 fsmp Exp $
  */
 
 #include "opt_rlimit.h"
@@ -541,12 +541,6 @@ kthread_init(dummy)
 	/* Create process 1 (init(8)). */
 	start_init(curproc);
 
-#ifdef SMP
-	/* wait for the SMP idle loops to come online */
-	while (smp_idle_loops < mp_ncpus)
-		tsleep((caddr_t *)&smp_idle_loops, PWAIT, "smpilw", 0);
-#endif /* SMP */
-
 	prepare_usermode();
 
 	/*
diff --git a/sys/kern/init_smp.c b/sys/kern/init_smp.c
deleted file mode 100644
index 6d56a60..0000000
--- a/sys/kern/init_smp.c
+++ /dev/null
@@ -1,334 +0,0 @@
-/*
- * Copyright (c) 1996, Peter Wemm <peter@freebsd.org>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $Id: init_smp.c,v 1.10 1997/08/15 02:13:31 smp Exp smp $
- */
-
-#include "opt_smp.h"
-
-#include <sys/param.h>
-#include <sys/filedesc.h>
-#include <sys/kernel.h>
-#include <sys/sysctl.h>
-#include <sys/proc.h>
-#include <sys/resourcevar.h>
-#include <sys/signalvar.h>
-#include <sys/systm.h>
-#include <sys/vnode.h>
-#include <sys/sysent.h>
-#include <sys/reboot.h>
-#include <sys/sysproto.h>
-#include <sys/vmmeter.h>
-#include <sys/lock.h>
-
-#include <machine/cpu.h>
-#include <machine/smp.h>
-#include <machine/smptests.h>	/** IGNORE_IDLEPROCS */
-#include <machine/specialreg.h>
-
-#include <vm/vm.h>
-#include <vm/vm_param.h>
-#include <vm/vm_prot.h>
-#include <vm/pmap.h>
-#include <vm/vm_map.h>
-#include <sys/user.h>
-
-int smp_active = 0;	/* are the APs allowed to run? */
-
-static int
-sysctl_smp_active SYSCTL_HANDLER_ARGS
-{
-	int error = 0;
-	int new_val;
-
-	error = SYSCTL_OUT(req, &smp_active, sizeof(int));
-
-	if (error || !req->newptr)
-		return (error);
-
-	error = SYSCTL_IN(req, &new_val, sizeof(int));
-	if (error)
-		return (error);
-	if (new_val < 1)
-		return (EBUSY);
-	if (new_val > mp_ncpus)
-		return (EINVAL);
-	smp_active = new_val;
-	return (0);
-}
-
-SYSCTL_PROC(_kern, OID_AUTO, smp_active, CTLTYPE_INT|CTLFLAG_RW,
-		0, 0, &sysctl_smp_active, "I", "");
-
-int smp_cpus = 0;	/* how many cpu's running */
-SYSCTL_INT(_kern, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
-
-int idle_debug = 0;
-SYSCTL_INT(_kern, OID_AUTO, idle_debug, CTLFLAG_RW, &idle_debug, 0, "");
-
-int invltlb_ok = 0;	/* throttle smp_invltlb() till safe */
-
-#if defined(IGNORE_IDLEPROCS)
-int ignore_idleprocs = 1;
-#else
-int ignore_idleprocs = 0;
-#endif
-SYSCTL_INT(_kern, OID_AUTO, ignore_idleprocs, CTLFLAG_RW, &ignore_idleprocs,
-	   0, "");
-
-static void smp_kickoff __P((void *dummy));
-SYSINIT(smpkick, SI_SUB_SMP, SI_ORDER_FIRST, smp_kickoff, NULL)
-
-static void smp_idleloop __P((void *));
-
-void	secondary_main __P((void));
-
-volatile int	smp_idle_loops = 0;
-void	boot_unlock __P((void));
- 
-struct proc *SMPidleproc[NCPU];
-static int cpu_starting = -1;
-
-static void
-smp_kickoff(dummy)
-	void *dummy;
-{
-	int		rval[2];	/* return from fork */
-	struct proc	*p;
-	int i;
-
-	/*
-	 * Create the appropriate number of cpu-idle-eaters
-	 */
-	for (i = 0; i < mp_ncpus; i++) {
-		/* kernel thread*/
-		if (fork(&proc0, NULL, rval))
-			panic("cannot fork idle process");
-		p = pfind(rval[0]);
-		cpu_set_fork_handler(p, smp_idleloop, NULL);
-		SMPidleproc[i] = p;
-		p->p_flag |= P_INMEM | P_SYSTEM | P_IDLEPROC;
-		sprintf(p->p_comm, "cpuidle%d", i);
-		/*
-		 * PRIO_IDLE is the last scheduled of the three
-		 * classes and we choose the lowest priority possible
-		 * for there.
-		 */
-		p->p_rtprio.type = RTP_PRIO_IDLE;
-		p->p_rtprio.prio = RTP_PRIO_MAX;
-	}
-
-}
-
-
-#define MSG_CPU_MADEIT \
-	printf("SMP: TADA! CPU #%d made it into the scheduler!.\n", \
-	       cpuid)
-#define MSG_NEXT_CPU \
-	printf("SMP: %d of %d CPU's online. Unlocking next CPU..\n", \
-	       smp_cpus, mp_ncpus)
-#define MSG_FINAL_CPU \
-	printf("SMP: All %d CPU's are online!\n", \
-	       smp_cpus)
-#define MSG_TOOMANY_CPU \
-	printf("SMP: Hey! Too many cpu's started, %d of %d running!\n", \
-	       smp_cpus, mp_ncpus)
-
-/*
- * This is run by the secondary processor to kick things off.
- * It basically drops into the switch routine to pick the first
- * available process to run, which is probably an idle process.
- */
-
-void
-secondary_main()
-{
-	u_int   temp;
-
-	get_mplock();
-
-	/*
-	 * Record our ID so we know when we've released the mp_stk.
-	 * We must remain single threaded through this.
-	 */
-	cpu_starting = cpuid;
-	smp_cpus++;
-
-	/* Build our map of 'other' CPUs. */
-	other_cpus = all_cpus & ~(1 << cpuid);
-
-	printf("SMP: AP CPU #%d LAUNCHED!!  Starting Scheduling...\n",
-		cpuid);
-
-        /* XXX FIXME: i386 specific, and redundant: Setup the FPU. */
-        load_cr0((rcr0() & ~CR0_EM) | CR0_MP | CR0_NE | CR0_TS);
-
-	curproc = NULL;			/* ensure no context to save */
-	cpu_switch(curproc);		/* start first process */
-
-	panic("switch returned!");
-}
-
-
-/*
- * The main program loop for the idle process
- */
-
-static void
-smp_idleloop(dummy)
-void *dummy;
-{
-	int dcnt = 0;
-	int apic_id;
-
-	/*
-	 * This code is executed only on startup of the idleprocs
-	 * The fact that this is executed is an indication that the
-	 * idle procs are online and it's safe to kick off the first
-	 * AP cpu.
-	 */
-	if ( ++smp_idle_loops == mp_ncpus ) {
-		printf("SMP: All idle procs online.\n");
-
-		/* let the init process finish */
-		wakeup((caddr_t *)&smp_idle_loops);
-
-#ifndef NO_AUTOSTART
-		printf("SMP: *** AUTO *** starting 1st AP!\n");
-		smp_cpus = 1;
-		smp_active = mp_ncpus;	/* XXX */
-		boot_unlock();
-#else
-		printf("You can now activate SMP processing, use: sysctl -w kern.smp_active=%d\n", mp_ncpus);
-#endif
-	}
-
-	spl0();
-	rel_mplock();
-
-	while (1) {
-		/*
-		 * make the optimiser assume nothing about the
-		 * which*qs variables
-		 */
-		__asm __volatile("" : : : "memory");
- 
-#ifdef NO_AUTOSTART
-		if (smp_cpus == 0 && smp_active != 0) {
-			get_mplock();
-			printf("SMP: Starting 1st AP!\n");
-			smp_cpus = 1;
-			smp_active = mp_ncpus;  /* XXX */
-			boot_unlock();
-			rel_mplock();
-		}
-#endif
-
-		/*
-		 * If smp_active is set to (say) 1, we want cpu id's
-		 * 1,2,etc to freeze here.
-		 */
-		if (smp_active && smp_active <= cpuid) {
-			get_mplock();
-			printf("SMP: cpu#%d freezing\n", cpuid);
-			wakeup((caddr_t)&smp_active);
-			rel_mplock();
-
-			while (smp_active <= cpuid) {
-				__asm __volatile("" : : : "memory");
-			}
-			get_mplock();
-			printf("SMP: cpu#%d waking up!\n", cpuid);
-			rel_mplock();
-		}
-
-/* XXX DEBUG */
-		apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]);
-		if (cpuid != apic_id) {
-			printf("SMP: cpuid = %d\n", cpuid);
-			printf("SMP: apic_id = %d\n", apic_id);
-			printf("PTD[MPPTDI] = %08x\n", PTD[MPPTDI]);
-			panic("cpuid mismatch! boom!!");
-		}
-/* XXX END DEBUG */
-			
-		if (whichqs || whichrtqs || (!ignore_idleprocs && whichidqs)) {
-			/* grab lock for kernel "entry" */
-			get_mplock();
-
-			/* We need to retest due to the spin lock */
-			__asm __volatile("" : : : "memory");
-
-			if (whichqs || whichrtqs ||
-					(!ignore_idleprocs && whichidqs)) {
-				splhigh();
-				if (curproc)
-					setrunqueue(curproc);
-				cnt.v_swtch++;
-				cpu_switch(curproc);
-				microtime(&runtime);
-
-				if (cpu_starting != -1 &&
-				    cpu_starting == cpuid) {
-					/*
-					 * TADA! we have arrived! unlock the
-					 * next cpu now that we have released
-					 * the single mp_stk.
-					 */
-					MSG_CPU_MADEIT;
-					cpu_starting = -1;
-
-					/* Init local apic for irq's */
-					apic_initialize();
-
-					if (smp_cpus < mp_ncpus) {
-						MSG_NEXT_CPU;
-						boot_unlock();
-					} else if (smp_cpus > mp_ncpus) {
-						MSG_TOOMANY_CPU;
-						panic("too many cpus");
-					} else {
-						MSG_FINAL_CPU;
-						/*
-						 * It's safe to send IPI's now
-						 * that all CPUs are online.
-						 */
-						invltlb_ok = 1;
-					}
-				}
-
-				(void)spl0();
-			}
-			rel_mplock();
-		} else {
-			dcnt++;
-			if (idle_debug && (dcnt % idle_debug) == 0) {
-				get_mplock();
-				printf("idleproc pid#%d on cpu#%d, lock %08x\n",
-					curproc->p_pid, cpuid, mp_lock);
-				rel_mplock();
-			}
-		}
-	}
-}
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index f647070..a4a6423 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -22,17 +22,20 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.44 1997/08/24 20:33:32 fsmp Exp $
+ *	$Id: mp_machdep.c,v 1.45 1997/08/25 21:28:08 bde Exp $
  */
 
 #include "opt_smp.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/proc.h>
+#include <sys/sysctl.h>
 
-#include <vm/vm.h>		/* for KERNBASE */
-#include <vm/vm_param.h>	/* for KERNBASE */
-#include <vm/pmap.h>		/* for KERNBASE */
+#include <vm/vm.h>
+#include <vm/vm_param.h>
+#include <vm/pmap.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_extern.h>
 
@@ -230,15 +233,24 @@ int     apic_id_to_logical[NAPICID];
 /* Bitmap of all available CPUs */
 u_int	all_cpus;
 
-/* Boot of AP uses this PTD */
-u_int *bootPTD;
+/* AP uses this PTD during bootstrap */
+pd_entry_t *bootPTD;
 
 /* Hotwire a 0->4MB V==P mapping */
-extern pt_entry_t KPTphys;
+extern pt_entry_t *KPTphys;
 
 /* Virtual address of per-cpu common_tss */
 extern struct i386tss common_tss;
 
+/* IdlePTD per cpu */
+pd_entry_t *IdlePTDS[NCPU];
+
+/* "my" private page table page, for BSP init */
+extern pt_entry_t SMP_prvpt[];
+
+/* Private page pointer to curcpu's PTD, used during BSP init */
+extern pd_entry_t *my_idlePTD;
+
 /*
  * Local data and functions.
  */
@@ -1473,9 +1485,9 @@ start_all_aps(u_int boot_addr)
 	int     x, i;
 	u_char  mpbiosreason;
 	u_long  mpbioswarmvec;
-	pd_entry_t newptd;
-	pt_entry_t newpt;
-	int *newpp;
+	pd_entry_t *newptd;
+	pt_entry_t *newpt;
+	int *newpp, *stack;
 
 	POSTCODE(START_ALL_APS_POST);
 
@@ -1498,57 +1510,71 @@ start_all_aps(u_int boot_addr)
 	/* start each AP */
 	for (x = 1; x <= mp_naps; ++x) {
 
-		/* HACK HACK HACK !!! */
+		/* This is a bit verbose, it will go away soon.  */
 
 		/* alloc new page table directory */
-		newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE));
+		newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
+
+		/* Store the virtual PTD address for this CPU */
+		IdlePTDS[x] = newptd;
 
 		/* clone currently active one (ie: IdlePTD) */
 		bcopy(PTD, newptd, PAGE_SIZE);	/* inc prv page pde */
 
 		/* set up 0 -> 4MB P==V mapping for AP boot */
-		newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
+		newptd[0] = (pd_entry_t) (PG_V | PG_RW |
+						((u_long)KPTphys & PG_FRAME));
 
-		/* store PTD for this AP */
-		bootPTD = (pd_entry_t)vtophys(newptd);
+		/* store PTD for this AP's boot sequence */
+		bootPTD = (pd_entry_t *)vtophys(newptd);
 
 		/* alloc new page table page */
-		newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE));
+		newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
 
 		/* set the new PTD's private page to point there */
-		newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt);
+		newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt));
 
 		/* install self referential entry */
-		newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd);
+		newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd));
 
-		/* get a new private data page */
+		/* allocate a new private data page */
 		newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE);
 
 		/* wire it into the private page table page */
-		newpt[0] = PG_V | PG_RW | vtophys(newpp);
+		newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpp));
 
 		/* wire the ptp into itself for access */
-		newpt[1] = PG_V | PG_RW | vtophys(newpt);
+		newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt));
 
-		/* and the local apic */
+		/* copy in the pointer to the local apic */
 		newpt[2] = SMP_prvpt[2];
 
 		/* and the IO apic mapping[s] */
 		for (i = 16; i < 32; i++)
 			newpt[i] = SMP_prvpt[i];
 
-		/* prime data page for it to use */
-		newpp[0] = x;		/* cpuid */
-		newpp[1] = 0;		/* curproc */
-		newpp[2] = 0;		/* curpcb */
-		newpp[3] = 0;		/* npxproc */
-		newpp[4] = 0;		/* runtime.tv_sec */
-		newpp[5] = 0;		/* runtime.tv_usec */
-		newpp[6] = x << 24;	/* cpu_lockid */
+		/* allocate and set up an idle stack data page */
+		stack = (int *)kmem_alloc(kernel_map, PAGE_SIZE);
+		newpt[3] = (pt_entry_t)(PG_V | PG_RW | vtophys(stack));
 
-		/* XXX NOTE: ABANDON bootPTD for now!!!! */
+		newpt[4] = 0;			/* *prv_CMAP1 */
+		newpt[5] = 0;			/* *prv_CMAP2 */
+		newpt[6] = 0;			/* *prv_CMAP3 */
 
-		/* END REVOLTING HACKERY */
+		/* prime data page for it to use */
+		newpp[0] = x;			/* cpuid */
+		newpp[1] = 0;			/* curproc */
+		newpp[2] = 0;			/* curpcb */
+		newpp[3] = 0;			/* npxproc */
+		newpp[4] = 0;			/* runtime.tv_sec */
+		newpp[5] = 0;			/* runtime.tv_usec */
+		newpp[6] = x << 24;		/* cpu_lockid */
+		newpp[7] = 0;			/* other_cpus */
+		newpp[8] = (int)bootPTD;	/* my_idlePTD */
+		newpp[9] = 0;			/* ss_tpr */
+		newpp[10] = (int)&newpt[4];	/* prv_CMAP1 */
+		newpp[11] = (int)&newpt[5];	/* prv_CMAP2 */
+		newpp[12] = (int)&newpt[6];	/* prv_CMAP3 */
 
 		/* setup a vector to our boot code */
 		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
@@ -1585,6 +1611,26 @@ start_all_aps(u_int boot_addr)
 	outb(CMOS_REG, BIOS_RESET);
 	outb(CMOS_DATA, mpbiosreason);
 
+	/*
+	 * Set up the idle context for the BSP.  Similar to above except
+	 * that some was done by locore, some by pmap.c and some is implicit
+	 * because the BSP is cpu#0 and the page is initially zero, and also
+	 * because we can refer to variables by name on the BSP..
+	 */
+	newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
+
+	bcopy(PTD, newptd, PAGE_SIZE);	/* inc prv page pde */
+	IdlePTDS[0] = newptd;
+
+	/* Point PTD[] to this page instead of IdlePTD's physical page */
+	newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd));
+
+	my_idlePTD = (pd_entry_t *)vtophys(newptd);
+
+	/* Allocate and setup BSP idle stack */
+	stack = (int *)kmem_alloc(kernel_map, PAGE_SIZE);
+	SMP_prvpt[3] = (pt_entry_t)(PG_V | PG_RW | vtophys(stack));
+
 	pmap_set_opt_bsp();
 
 	/* number of APs actually started */
@@ -1800,7 +1846,7 @@ invltlb(void)
  *            from executing at same time.
  */
 int
-stop_cpus( u_int map )
+stop_cpus(u_int map)
 {
 	if (!smp_active)
 		return 0;
@@ -1832,7 +1878,7 @@ stop_cpus( u_int map )
  *   1: ok
  */
 int
-restart_cpus( u_int map )
+restart_cpus(u_int map)
 {
 	if (!smp_active)
 		return 0;
@@ -1844,3 +1890,63 @@ restart_cpus( u_int map )
 
 	return 1;
 }
+
+int smp_active = 0;	/* are the APs allowed to run? */
+SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
+
+/* XXX maybe should be hw.ncpu */
+int smp_cpus = 1;	/* how many cpu's running */
+SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
+
+int invltlb_ok = 0;	/* throttle smp_invltlb() till safe */
+SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
+
+int do_page_zero_idle = 0; /* bzero pages for fun and profit in idleloop */
+SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW,
+	   &do_page_zero_idle, 0, "");
+
+
+/*
+ * This is called once the rest of the system is up and running and we're
+ * ready to let the AP's out of the pen.
+ */
+void ap_init(void);
+
+void
+ap_init()
+{
+	u_int   temp;
+	u_int	apic_id;
+
+	smp_cpus++;
+
+	/* Build our map of 'other' CPUs. */
+	other_cpus = all_cpus & ~(1 << cpuid);
+
+	printf("SMP: AP CPU #%d Launched!\n", cpuid);
+
+	/* XXX FIXME: i386 specific, and redundant: Setup the FPU. */
+	load_cr0((rcr0() & ~CR0_EM) | CR0_MP | CR0_NE | CR0_TS);
+
+	/* A quick check from sanity claus */
+	apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]);
+	if (cpuid != apic_id) {
+		printf("SMP: cpuid = %d\n", cpuid);
+		printf("SMP: apic_id = %d\n", apic_id);
+		printf("PTD[MPPTDI] = %08x\n", PTD[MPPTDI]);
+		panic("cpuid mismatch! boom!!");
+	}
+
+	/* Init local apic for irq's */
+	apic_initialize();
+
+	/*
+	 * Activate smp_invltlb, although strictly speaking, this isn't
+	 * quite correct yet.  We should have a bitfield for cpus willing
+	 * to accept TLB flush IPI's or something and sync them.
+	 */
+	invltlb_ok = 1;
+	smp_active = 1;		/* historic */
+
+	curproc = NULL;		/* make sure */
+}
diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c
index ac139d5..5895361 100644
--- a/sys/kern/subr_trap.c
+++ b/sys/kern/subr_trap.c
@@ -35,7 +35,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)trap.c	7.4 (Berkeley) 5/13/91
- *	$Id: trap.c,v 1.106 1997/08/20 05:25:48 fsmp Exp $
+ *	$Id: trap.c,v 1.107 1997/08/21 06:32:39 charnier Exp $
  */
 
 /*
@@ -725,6 +725,7 @@ trap_fatal(frame)
 			ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel");
 #ifdef SMP
 	printf("cpuid = %d\n", cpuid);
+	printf("lapic.id = %d\n", lapic.id);
 #endif
 	if (type == T_PAGEFLT) {
 		printf("fault virtual address	= 0x%x\n", eva);
@@ -769,6 +770,9 @@ trap_fatal(frame)
 	} else {
 		printf("Idle\n");
 	}
+#ifdef SMP
+	printf("mp_lock                 = %08x\n", mp_lock);
+#endif
 	printf("interrupt mask		= ");
 	if ((cpl & net_imask) == net_imask)
 		printf("net ");
@@ -824,6 +828,7 @@ dblfault_handler()
 	printf("ebp = 0x%x\n", common_tss.tss_ebp);
 #ifdef SMP
 	printf("cpuid = %d\n", cpuid);
+	printf("lapic.id = %d\n", lapic.id);
 #endif
 	panic("double fault");
 }