Preliminary support for per-cpu data pages.

This eliminates a lot of #ifdef SMP type code.  Things like _curproc reside
in a data page that is unique on each cpu, eliminating the expensive macros
like:    #define curproc (SMPcurproc[cpunumber()])

There are some unresolved bootstrap and address space sharing issues at
present, but Steve is waiting on this for other work.  There is still some
strictly temporary code present that isn't exactly pretty.

This is part of a larger change that has run into some bumps, this part is
standalone so it should be safe.  The temporary code goes away when the
full idle cpu support is finished.

Reviewed by: fsmp, dyson

7 files changed, 165 insertions, 100 deletions

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 949dad8..55d9256 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -39,7 +39,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)init_main.c	8.9 (Berkeley) 1/21/94
- * $Id: init_main.c,v 1.63 1997/05/29 04:52:03 peter Exp $
+ * $Id: init_main.c,v 1.64 1997/06/16 00:29:30 dyson Exp $
  */
 
 #include "opt_rlimit.h"
@@ -104,11 +104,7 @@ struct	timeval boottime;
 SYSCTL_STRUCT(_kern, KERN_BOOTTIME, boottime,
 	CTLFLAG_RW, &boottime, timeval, "");
 
-/*
- * for SMP, the runtime variable has to be per-cpu, so we use the
- * extern declaration in sys/kernel.h
- */
-#ifndef SMP
+#ifndef SMP	/* per-cpu on smp */
 struct	timeval runtime;
 #endif
 
diff --git a/sys/kern/init_smp.c b/sys/kern/init_smp.c
index 0751f15..b972872 100644
--- a/sys/kern/init_smp.c
+++ b/sys/kern/init_smp.c
@@ -22,11 +22,10 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $Id: init_smp.c,v 1.5 1997/05/04 02:08:09 peter Exp $
+ * $Id: init_smp.c,v 1.6 1997/05/06 07:10:06 fsmp Exp $
  */
 
 #include "opt_smp.h"
-#include "opt_smp_autostart.h"
 
 #include <sys/param.h>
 #include <sys/filedesc.h>
@@ -145,7 +144,7 @@ smp_kickoff(dummy)
 
 #define MSG_CPU_MADEIT \
 	printf("SMP: TADA! CPU #%d made it into the scheduler!.\n", \
-	       cpunumber())
+	       cpuid)
 #define MSG_NEXT_CPU \
 	printf("SMP: %d of %d CPU's online. Unlocking next CPU..\n", \
 	       smp_cpus, mp_ncpus)
@@ -171,11 +170,11 @@ secondary_main()
 	 * Record our ID so we know when we've released the mp_stk.
 	 * We must remain single threaded through this.
 	 */
-	cpu_starting = cpunumber();
+	cpu_starting = cpuid;
 	smp_cpus++;
 
 	printf("SMP: AP CPU #%d LAUNCHED!!  Starting Scheduling...\n",
-		cpunumber());
+		cpuid);
 
 	curproc = NULL;			/* ensure no context to save */
 	cpu_switch(curproc);		/* start first process */
@@ -192,6 +191,7 @@ smp_idleloop(dummy)
 void *dummy;
 {
 	int dcnt = 0;
+	int apic_id;
 
 	/*
 	 * This code is executed only on startup of the idleprocs
@@ -202,14 +202,14 @@ void *dummy;
 	if ( ++idle_loops == mp_ncpus ) {
 		printf("SMP: All idle procs online.\n");
 
-#if defined(SMP_AUTOSTART)
+#ifndef NO_AUTOSTART
 		printf("SMP: *** AUTO *** starting 1st AP!\n");
 		smp_cpus = 1;
 		smp_active = mp_ncpus;	/* XXX */
 		boot_unlock();
 #else
 		printf("You can now activate SMP processing, use: sysctl -w kern.smp_active=%d\n", mp_ncpus);
-#endif /* SMP_AUTOSTART */
+#endif
 	}
 
 	spl0();
@@ -222,38 +222,44 @@ void *dummy;
 		 */
 		__asm __volatile("" : : : "memory");
  
-#if !defined(SMP_AUTOSTART)
-		/*
-		 * Alternate code to enable a lockstep startup
-		 * via sysctl instead of automatically.
-		 */
+#ifdef NO_AUTOSTART
 		if (smp_cpus == 0 && smp_active != 0) {
 			get_mplock();
 			printf("SMP: Starting 1st AP!\n");
 			smp_cpus = 1;
-			smp_active = mp_ncpus;	/* XXX */
+			smp_active = mp_ncpus;  /* XXX */
 			boot_unlock();
 			rel_mplock();
 		}
-#endif /* !SMP_AUTOSTART */
+#endif
 
 		/*
 		 * If smp_active is set to (say) 1, we want cpu id's
 		 * 1,2,etc to freeze here.
 		 */
-		if (smp_active && smp_active <= cpunumber()) {
+		if (smp_active && smp_active <= cpuid) {
 			get_mplock();
-			printf("SMP: cpu#%d freezing\n", cpunumber());
+			printf("SMP: cpu#%d freezing\n", cpuid);
 			wakeup((caddr_t)&smp_active);
 			rel_mplock();
 
-			while (smp_active <= cpunumber()) {
+			while (smp_active <= cpuid) {
 				__asm __volatile("" : : : "memory");
 			}
 			get_mplock();
-			printf("SMP: cpu#%d waking up!\n", cpunumber());
+			printf("SMP: cpu#%d waking up!\n", cpuid);
 			rel_mplock();
 		}
+
+/* XXX DEBUG */
+		apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]);
+		if (cpuid != apic_id) {
+			printf("SMP: cpuid = %d\n", cpuid);
+			printf("SMP: apic_id = %d\n", apic_id);
+			printf("PTD[MPPTDI] = %08x\n", PTD[MPPTDI]);
+			panic("cpuid mismatch! boom!!");
+		}
+/* XXX END DEBUG */
 			
 		if (whichqs || whichrtqs || (!ignore_idleprocs && whichidqs)) {
 			/* grab lock for kernel "entry" */
@@ -272,7 +278,7 @@ void *dummy;
 				microtime(&runtime);
 
 				if (cpu_starting != -1 &&
-				    cpu_starting == cpunumber()) {
+				    cpu_starting == cpuid) {
 					/*
 					 * TADA! we have arrived! unlock the
 					 * next cpu now that we have released
@@ -309,7 +315,7 @@ void *dummy;
 			if (idle_debug && (dcnt % idle_debug) == 0) {
 				get_mplock();
 				printf("idleproc pid#%d on cpu#%d, lock %08x\n",
-					curproc->p_pid, cpunumber(), mp_lock);
+					curproc->p_pid, cpuid, mp_lock);
 				rel_mplock();
 			}
 		}
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 33e961f..cb50acb 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -36,7 +36,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)kern_fork.c	8.6 (Berkeley) 4/8/94
- * $Id: kern_fork.c,v 1.42 1997/05/29 04:52:04 peter Exp $
+ * $Id: kern_fork.c,v 1.43 1997/06/16 00:29:30 dyson Exp $
  */
 
 #include "opt_ktrace.h"
@@ -142,6 +142,19 @@ fork1(p1, flags, retval)
 	if ((flags & (RFFDG|RFCFDG)) == (RFFDG|RFCFDG))
 		return (EINVAL);
 
+#ifdef SMP
+	/*
+	 * FATAL now, we cannot have the same PTD on both cpus, the PTD
+	 * needs to move out of PTmap and be per-process, even for shared
+	 * page table processes.  Unfortunately, this means either removing
+	 * PTD[] as a fixed virtual address, or move it to the per-cpu map
+	 * area for SMP mode.  Both cases require seperate management of
+	 * the per-process-even-if-PTmap-is-shared PTD.
+	 */
+	if (flags & RFMEM)
+		return (EOPNOTSUPP);
+#endif
+
 	/*
 	 * Here we don't create a new process, but we divorce
 	 * certain parts of a process from itself.
diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c
index c7cebca..1353dbf 100644
--- a/sys/kern/kern_shutdown.c
+++ b/sys/kern/kern_shutdown.c
@@ -36,7 +36,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)kern_shutdown.c	8.3 (Berkeley) 1/21/94
- * $Id: kern_shutdown.c,v 1.15 1997/05/24 18:35:44 fsmp Exp $
+ * $Id: kern_shutdown.c,v 1.16 1997/06/15 02:03:03 wollman Exp $
  */
 
 #include "opt_ddb.h"
@@ -61,6 +61,9 @@
 #include <machine/clock.h>
 #include <machine/cons.h>
 #include <machine/md_var.h>
+#ifdef SMP
+#include <machine/smp.h>		/* smp_active, cpuid */
+#endif
 
 #include <sys/utsname.h>
 #include <sys/signalvar.h>
@@ -175,8 +178,8 @@ boot(howto)
 
 		spins = 100;
 
-		printf("boot() called on cpu#%d\n", cpunumber());
-		while ((c = cpunumber()) != 0) {
+		printf("boot() called on cpu#%d\n", cpuid);
+		while ((c = cpuid) != 0) {
 			if (spins-- < 1) {
 				printf("timeout waiting for cpu #0!\n");
 				break;
@@ -383,7 +386,7 @@ panic(const char *fmt, ...)
 	va_end(ap);
 	printf("\n");
 #ifdef SMP
-	printf(" cpu#%d\n", cpunumber());
+	printf(" cpuid %d\n", cpuid);
 #endif
 
 #if defined(DDB)
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index f684c05..b6d00a5 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -36,7 +36,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)kern_synch.c	8.9 (Berkeley) 5/19/95
- * $Id: kern_synch.c,v 1.30 1997/02/27 18:03:48 bde Exp $
+ * $Id: kern_synch.c,v 1.31 1997/04/26 11:46:15 peter Exp $
  */
 
 #include "opt_ktrace.h"
@@ -216,15 +216,9 @@ schedcpu(arg)
 		resetpriority(p);
 		if (p->p_priority >= PUSER) {
 #define	PPQ	(128 / NQS)		/* priorities per queue */
-#ifdef SMP
-			for (j = i = 0; i < NCPU; i++) {
-				if (p == SMPcurproc[i])
-					j++;
-			}
-			if (!j &&
-
-#else
 			if ((p != curproc) &&
+#ifdef SMP
+			    (u_char)p->p_oncpu == 0xff && 	/* idle */
 #endif
 			    p->p_stat == SRUN &&
 			    (p->p_flag & P_INMEM) &&
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index a27cf86..c817925 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -22,11 +22,10 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.16 1997/05/29 05:58:41 fsmp Exp $
+ *	$Id: mp_machdep.c,v 1.17 1997/06/02 10:44:08 dfr Exp $
  */
 
 #include "opt_smp.h"
-#include "opt_serial.h"
 
 #include <sys/param.h>		/* for KERNBASE */
 #include <sys/types.h>
@@ -38,18 +37,21 @@
 #include <vm/vm_param.h>	/* for KERNBASE */
 #include <vm/pmap.h>		/* for KERNBASE */
 #include <machine/pmap.h>	/* for KERNBASE */
+#include <vm/vm_kern.h>
+#include <vm/vm_extern.h>
 
 #include <machine/smp.h>
 #include <machine/apic.h>
 #include <machine/mpapic.h>
 #include <machine/cpufunc.h>
 #include <machine/segments.h>
-#include <machine/smptests.h>	/** TEST_DEFAULT_CONFIG, LATE_START */
+#include <machine/smptests.h>	/** TEST_DEFAULT_CONFIG */
+#include <machine/tss.h>
 
 #include <i386/i386/cons.h>	/* cngetc() */
 
 #if defined(APIC_IO)
-#include <i386/include/md_var.h>	/* setidt() */
+#include <machine/md_var.h>	/* setidt() */
 #include <i386/isa/icu.h>		/* Xinvltlb() */
 #include <i386/isa/intr_machdep.h>	/* Xinvltlb() */
 #endif	/* APIC_IO */
@@ -186,11 +188,6 @@ typedef struct BASETABLE_ENTRY {
 /** FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
-/* global data */
-struct proc *SMPcurproc[NCPU];
-struct pcb *SMPcurpcb[NCPU];
-struct timeval SMPruntime[NCPU];
-
 int	mp_ncpus;		/* # of CPUs, including BSP */
 int	mp_naps;		/* # of Applications processors */
 int	mp_nbusses;		/* # of busses */
@@ -210,6 +207,15 @@ int     cpu_num_to_apic_id[NAPICID];
 int     io_num_to_apic_id[NAPICID];
 int     apic_id_to_logical[NAPICID];
 
+/* Boot of AP uses this PTD */
+u_int *bootPTD;
+
+/* Hotwire a 0->4MB V==P mapping */
+extern pt_entry_t KPTphys;
+
+/* virtual address of per-cpu common_tss */
+extern struct i386tss common_tss;
+
 /*
  * look for MP compliant motherboard.
  */
@@ -283,11 +289,6 @@ found:				/* please forgive the 'goto'! */
 	if (mptable_pass1())
 		panic("you must reconfigure your kernel");
 
-#if defined(LATE_START)
-	/* create pages for (address common) cpu APIC and each IO APIC */
-	pmap_bootstrap_apics();
-#endif  /* LATE_START */
-
 	/* flag fact that we are running multiple processors */
 	mp_capable = 1;
 	return 1;
@@ -305,9 +306,6 @@ mp_start(void)
 		mp_enable(boot_address);
 	else
 		panic("MP hardware not found!");
-
-	/* finish pmap initialization - turn off V==P mapping at zero */
-	pmap_bootstrap2();
 }
 
 
@@ -321,23 +319,25 @@ mp_announce(void)
 
 	printf("FreeBSD/SMP: Multiprocessor motherboard\n");
 	printf(" cpu0 (BSP): apic id: %d", CPU_TO_ID(0));
-	printf(", version: 0x%08x\n", cpu_apic_versions[0]);
+	printf(", version: 0x%08x", cpu_apic_versions[0]);
+	printf(", at 0x%08x\n", cpu_apic_address);
 	for (x = 1; x <= mp_naps; ++x) {
 		printf(" cpu%d (AP):  apic id: %d", x, CPU_TO_ID(x));
-		printf(", version: 0x%08x\n", cpu_apic_versions[x]);
+		printf(", version: 0x%08x", cpu_apic_versions[x]);
+		printf(", at 0x%08x\n", cpu_apic_address);
 	}
 
 #if defined(APIC_IO)
 	for (x = 0; x < mp_napics; ++x) {
 		printf(" io%d (APIC): apic id: %d", x, IO_TO_ID(x));
-		printf(", version: 0x%08x\n", io_apic_versions[x]);
+		printf(", version: 0x%08x", io_apic_versions[x]);
+		printf(", at 0x%08x\n", io_apic_address[x]);
 	}
 #else
 	printf(" Warning: APIC I/O disabled\n");
 #endif	/* APIC_IO */
 }
 
-
 /*
  * AP cpu's call this to sync up protected mode.
  */
@@ -352,12 +352,18 @@ init_secondary(void)
 	lidt(&r_idt);
 	lldt(_default_ldt);
 
-	slot = NGDT + cpunumber();
+	slot = NGDT + cpuid;
 	gsel_tss = GSEL(slot, SEL_KPL);
 	gdt[slot].sd.sd_type = SDT_SYS386TSS;
+	common_tss.tss_esp0 = 0;	/* not used until after switch */
+	common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
+	common_tss.tss_ioopt = (sizeof common_tss) << 16;
 	ltr(gsel_tss);
 
 	load_cr0(0x8005003b);	/* XXX! */
+
+	PTD[0] = 0;
+	invltlb();
 }
 
 
@@ -375,9 +381,9 @@ configure_local_apic(void)
 		outb(0x23, byte);	/* disconnect 8259s/NMI */
 	}
 	/* mask the LVT1 */
-	temp = lapic__lvt_lint0;
+	temp = lapic.lvt_lint0;
 	temp |= APIC_LVT_M;
-	lapic__lvt_lint0 = temp;
+	lapic.lvt_lint0 = temp;
 }
 #endif	/* APIC_IO */
 
@@ -398,13 +404,15 @@ mp_enable(u_int boot_addr)
 	u_int   ux;
 #endif	/* APIC_IO */
 
-	/* examine the MP table for needed info */
+	/* Turn on 4MB of V == P addressing so we can get to MP table */
+	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
+	invltlb();
+
+	/* examine the MP table for needed info, uses physical addresses */
 	x = mptable_pass2();
 
-#if !defined(LATE_START)
-	/* create pages for (address common) cpu APIC and each IO APIC */
-	pmap_bootstrap_apics();
-#endif  /* LATE_START */
+	*(int *)PTD = 0;
+	invltlb();
 
 	/* can't process default configs till the CPU APIC is pmapped */
 	if (x)
@@ -832,7 +840,7 @@ fix_mp_table(void)
 		for (x = 0; x < mp_nbusses; ++x) {
 			if (bus_data[x].bus_type != PCI)
 				continue;
-			if (bus_data[x].bus_id >= num_pci_bus )
+			if (bus_data[x].bus_id >= num_pci_bus)
 				panic("bad PCI bus numbering");
 		}
 	}
@@ -1258,7 +1266,7 @@ default_mp_table(int type)
 	}
 #endif	/* 0 */
 
-	boot_cpu_id = (lapic__id & APIC_ID_MASK) >> 24;
+	boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24;
 	ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
 
 	/* BSP */
@@ -1351,9 +1359,12 @@ default_mp_table(int type)
 static int
 start_all_aps(u_int boot_addr)
 {
-	int     x;
+	int     x, i;
 	u_char  mpbiosreason;
 	u_long  mpbioswarmvec;
+	pd_entry_t newptd;
+	pt_entry_t newpt;
+	int *newpp;
 
 	/**
          * NOTE: this needs further thought:
@@ -1362,7 +1373,7 @@ start_all_aps(u_int boot_addr)
          *
          * get the initial mp_lock with a count of 1 for the BSP
          */
-	mp_lock = (lapic__id & APIC_ID_MASK) + 1;
+	mp_lock = (lapic.id & APIC_ID_MASK) + 1;
 
 	/* initialize BSP's local APIC */
 	apic_initialize(1);
@@ -1370,6 +1381,7 @@ start_all_aps(u_int boot_addr)
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
 
+
 	/* save the current value of the warm-start vector */
 	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
 	outb(CMOS_REG, BIOS_RESET);
@@ -1378,6 +1390,58 @@ start_all_aps(u_int boot_addr)
 	/* start each AP */
 	for (x = 1; x <= mp_naps; ++x) {
 
+		/* HACK HACK HACK !!! */
+
+		/* alloc new page table directory */
+		newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE));
+
+		/* clone currently active one (ie: IdlePTD) */
+		bcopy(PTD, newptd, PAGE_SIZE);	/* inc prv page pde */
+
+		/* set up 0 -> 4MB P==V mapping for AP boot */
+		newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
+
+		/* store PTD for this AP */
+		bootPTD = (pd_entry_t)vtophys(newptd);
+
+		/* alloc new page table page */
+		newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE));
+
+		/* set the new PTD's private page to point there */
+		newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt);
+
+		/* install self referential entry */
+		newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd);
+
+		/* get a new private data page */
+		newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE);
+
+		/* wire it into the private page table page */
+		newpt[0] = PG_V | PG_RW | vtophys(newpp);
+
+		/* wire the ptp into itself for access */
+		newpt[1] = PG_V | PG_RW | vtophys(newpt);
+
+		/* and the local apic */
+		newpt[2] = SMP_prvpt[2];
+
+		/* and the IO apic mapping[s] */
+		for (i = 16; i < 32; i++)
+			newpt[i] = SMP_prvpt[i];
+
+		/* prime data page for it to use */
+		newpp[0] = x;		/* cpuid */
+		newpp[1] = 0;		/* curproc */
+		newpp[2] = 0;		/* curpcb */
+		newpp[3] = 0;		/* npxproc */
+		newpp[4] = 0;		/* runtime.tv_sec */
+		newpp[5] = 0;		/* runtime.tv_usec */
+		newpp[6] = x << 24;	/* cpu_lockid */
+
+		/* XXX NOTE: ABANDON bootPTD for now!!!! */
+
+		/* END REVOLTING HACKERY */
+
 		/* setup a vector to our boot code */
 		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
 		*((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
@@ -1401,7 +1465,7 @@ start_all_aps(u_int boot_addr)
 	}
 
 	/* fill in our (BSP) APIC version */
-	cpu_apic_versions[0] = lapic__version;
+	cpu_apic_versions[0] = lapic.version;
 
 	/* restore the warmstart vector */
 	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
@@ -1503,24 +1567,24 @@ start_ap(int logical_cpu, u_int boot_addr)
 	 */
 
 	/* setup the address for the target AP */
-	icr_hi = lapic__icr_hi & ~APIC_ID_MASK;
+	icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
 	icr_hi |= (physical_cpu << 24);
-	lapic__icr_hi = icr_hi;
+	lapic.icr_hi = icr_hi;
 
 	/* do an INIT IPI: assert RESET */
-	icr_lo = lapic__icr_lo & 0xfff00000;
-	lapic__icr_lo = icr_lo | 0x0000c500;
+	icr_lo = lapic.icr_lo & 0xfff00000;
+	lapic.icr_lo = icr_lo | 0x0000c500;
 
 	/* wait for pending status end */
-	while (lapic__icr_lo & APIC_DELSTAT_MASK)
+	while (lapic.icr_lo & APIC_DELSTAT_MASK)
 		 /* spin */ ;
 
 	/* do an INIT IPI: deassert RESET */
-	lapic__icr_lo = icr_lo | 0x00008500;
+	lapic.icr_lo = icr_lo | 0x00008500;
 
 	/* wait for pending status end */
 	u_sleep(10000);		/* wait ~10mS */
-	while (lapic__icr_lo & APIC_DELSTAT_MASK)
+	while (lapic.icr_lo & APIC_DELSTAT_MASK)
 		 /* spin */ ;
 
 	/*
@@ -1533,8 +1597,8 @@ start_ap(int logical_cpu, u_int boot_addr)
 	 */
 
 	/* do a STARTUP IPI */
-	lapic__icr_lo = icr_lo | 0x00000600 | vector;
-	while (lapic__icr_lo & APIC_DELSTAT_MASK)
+	lapic.icr_lo = icr_lo | 0x00000600 | vector;
+	while (lapic.icr_lo & APIC_DELSTAT_MASK)
 		 /* spin */ ;
 	u_sleep(200);		/* wait ~200uS */
 
@@ -1545,8 +1609,8 @@ start_ap(int logical_cpu, u_int boot_addr)
 	 * recognized after hardware RESET or INIT IPI.
 	 */
 
-	lapic__icr_lo = icr_lo | 0x00000600 | vector;
-	while (lapic__icr_lo & APIC_DELSTAT_MASK)
+	lapic.icr_lo = icr_lo | 0x00000600 | vector;
+	while (lapic.icr_lo & APIC_DELSTAT_MASK)
 		 /* spin */ ;
 	u_sleep(200);		/* wait ~200uS */
 
diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c
index 89dd34f..8914a5b 100644
--- a/sys/kern/subr_trap.c
+++ b/sys/kern/subr_trap.c
@@ -35,7 +35,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)trap.c	7.4 (Berkeley) 5/13/91
- *	$Id: trap.c,v 1.98 1997/06/02 08:19:03 dfr Exp $
+ *	$Id: trap.c,v 1.99 1997/06/07 04:36:10 bde Exp $
  */
 
 /*
@@ -87,11 +87,7 @@
 #include "isa.h"
 #include "npx.h"
 
-#ifdef SMP
-extern struct i386tss *SMPcommon_tss_ptr[];
-#else
 extern struct i386tss common_tss;
-#endif
 
 int (*pmath_emulate) __P((struct trapframe *));
 
@@ -704,7 +700,7 @@ trap_fatal(frame)
 			type, trap_msg[type],
 			ISPL(frame->tf_cs) == SEL_UPL ? "user" : "kernel");
 #ifdef SMP
-	printf("cpunumber = %d\n", cpunumber());
+	printf("cpuid = %d\n", cpuid);
 #endif
 	if (type == T_PAGEFLT) {
 		printf("fault virtual address	= 0x%x\n", eva);
@@ -790,19 +786,12 @@ trap_fatal(frame)
 void
 dblfault_handler()
 {
-#ifdef SMP
-	int x = cpunumber();
-#endif
-
 	printf("\nFatal double fault:\n");
-#ifdef SMP
-	printf("eip = 0x%x\n", SMPcommon_tss_ptr[x]->tss_eip);
-	printf("esp = 0x%x\n", SMPcommon_tss_ptr[x]->tss_esp);
-	printf("ebp = 0x%x\n", SMPcommon_tss_ptr[x]->tss_ebp);
-#else
 	printf("eip = 0x%x\n", common_tss.tss_eip);
 	printf("esp = 0x%x\n", common_tss.tss_esp);
 	printf("ebp = 0x%x\n", common_tss.tss_ebp);
+#ifdef SMP
+	printf("cpuid = %d\n", cpuid);
 #endif
 	panic("double fault");
 }