Do not setup 4M pdir until all APs are up.

11 files changed, 97 insertions, 136 deletions

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 5afe5fe..5a9f672 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.102 1999/06/01 18:19:42 jlemon Exp $
+ *	$Id: mp_machdep.c,v 1.103 1999/06/22 20:54:25 msmith Exp $
  */
 
 #include "opt_smp.h"
@@ -300,7 +300,7 @@ u_int	all_cpus;
 
 /* AP uses this during bootstrap.  Do not staticize.  */
 char *bootSTK;
-int boot_cpuid;
+static int bootAP;
 
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t *KPTphys;
@@ -454,7 +454,7 @@ void
 init_secondary(void)
 {
 	int	gsel_tss;
-	int	x, myid = boot_cpuid;
+	int	x, myid = bootAP;
 
 	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
 	gdt_segs[GPROC0_SEL].ssd_base =
@@ -487,9 +487,7 @@ init_secondary(void)
 
 	load_cr0(0x8005003b);		/* XXX! */
 
-	pmap_set_opt((unsigned *)PTD);
-
-	invltlb();
+	pmap_set_opt();
 }
 
 
@@ -1827,7 +1825,7 @@ start_all_aps(u_int boot_addr)
 #endif
 
 		bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE];
-		boot_cpuid = x;
+		bootAP = x;
 
 		/* attempt to start the Application Processor */
 		CHECK_INIT(99);	/* setup checkpoints */
@@ -1874,7 +1872,7 @@ start_all_aps(u_int boot_addr)
 		    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
 
 	*(int *)PTD = 0;
-	pmap_set_opt_bsp();
+	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2172,6 +2170,9 @@ ap_init()
 {
 	u_int	apic_id;
 
+	/* BSP may have changed PTD while we're waiting for the lock */
+	cpu_invltlb();
+
 	smp_cpus++;
 
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index 5afe5fe..5a9f672 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.102 1999/06/01 18:19:42 jlemon Exp $
+ *	$Id: mp_machdep.c,v 1.103 1999/06/22 20:54:25 msmith Exp $
  */
 
 #include "opt_smp.h"
@@ -300,7 +300,7 @@ u_int	all_cpus;
 
 /* AP uses this during bootstrap.  Do not staticize.  */
 char *bootSTK;
-int boot_cpuid;
+static int bootAP;
 
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t *KPTphys;
@@ -454,7 +454,7 @@ void
 init_secondary(void)
 {
 	int	gsel_tss;
-	int	x, myid = boot_cpuid;
+	int	x, myid = bootAP;
 
 	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
 	gdt_segs[GPROC0_SEL].ssd_base =
@@ -487,9 +487,7 @@ init_secondary(void)
 
 	load_cr0(0x8005003b);		/* XXX! */
 
-	pmap_set_opt((unsigned *)PTD);
-
-	invltlb();
+	pmap_set_opt();
 }
 
 
@@ -1827,7 +1825,7 @@ start_all_aps(u_int boot_addr)
 #endif
 
 		bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE];
-		boot_cpuid = x;
+		bootAP = x;
 
 		/* attempt to start the Application Processor */
 		CHECK_INIT(99);	/* setup checkpoints */
@@ -1874,7 +1872,7 @@ start_all_aps(u_int boot_addr)
 		    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
 
 	*(int *)PTD = 0;
-	pmap_set_opt_bsp();
+	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2172,6 +2170,9 @@ ap_init()
 {
 	u_int	apic_id;
 
+	/* BSP may have changed PTD while we're waiting for the lock */
+	cpu_invltlb();
+
 	smp_cpus++;
 
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index f961651..82ee99e 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -39,7 +39,7 @@
  * SUCH DAMAGE.
  *
  *	from:	@(#)pmap.c	7.7 (Berkeley)	5/12/91
- *	$Id: pmap.c,v 1.238 1999/06/05 16:16:37 luoqi Exp $
+ *	$Id: pmap.c,v 1.239 1999/06/08 17:14:22 dt Exp $
  */
 
 /*
@@ -385,11 +385,6 @@ pmap_bootstrap(firstaddr, loadaddr)
 	if (cpu_feature & CPUID_PSE) {
 		unsigned ptditmp;
 		/*
-		 * Enable the PSE mode
-		 */
-		load_cr4(rcr4() | CR4_PSE);
-
-		/*
 		 * Note that we have enabled PSE mode
 		 */
 		pseflag = PG_PS;
@@ -400,6 +395,11 @@ pmap_bootstrap(firstaddr, loadaddr)
 
 #if !defined(SMP)
 		/*
+		 * Enable the PSE mode.
+		 */
+		load_cr4(rcr4() | CR4_PSE);
+
+		/*
 		 * We can do the mapping here for the single processor
 		 * case.  We simply ignore the old page table page from
 		 * now on.
@@ -460,38 +460,19 @@ pmap_bootstrap(firstaddr, loadaddr)
 
 #ifdef SMP
 /*
- * Set 4mb pdir for mp startup, and global flags
+ * Set 4mb pdir for mp startup
  */
 void
-pmap_set_opt(unsigned *pdir) {
-	int i;
-
+pmap_set_opt(void)
+{
 	if (pseflag && (cpu_feature & CPUID_PSE)) {
 		load_cr4(rcr4() | CR4_PSE);
-		if (pdir4mb) {
-			pdir[KPTDI] = pdir4mb;
+		if (pdir4mb && cpuid == 0) {	/* only on BSP */
+			kernel_pmap->pm_pdir[KPTDI] =
+			    PTD[KPTDI] = (pd_entry_t)pdir4mb;
+			cpu_invltlb();
 		}
 	}
-
-	if (pgeflag && (cpu_feature & CPUID_PGE)) {
-		load_cr4(rcr4() | CR4_PGE);
-		for(i = KPTDI; i < KPTDI + nkpt; i++) {
-			if (pdir[i]) {
-				pdir[i] |= PG_G;
-			}
-		}
-	}
-}
-
-/*
- * Setup the PTD for the boot processor
- */
-void
-pmap_set_opt_bsp(void)
-{
-	pmap_set_opt((unsigned *)kernel_pmap->pm_pdir);
-	pmap_set_opt((unsigned *)PTD);
-	invltlb();
 }
 #endif
 
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index 5afe5fe..5a9f672 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.102 1999/06/01 18:19:42 jlemon Exp $
+ *	$Id: mp_machdep.c,v 1.103 1999/06/22 20:54:25 msmith Exp $
  */
 
 #include "opt_smp.h"
@@ -300,7 +300,7 @@ u_int	all_cpus;
 
 /* AP uses this during bootstrap.  Do not staticize.  */
 char *bootSTK;
-int boot_cpuid;
+static int bootAP;
 
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t *KPTphys;
@@ -454,7 +454,7 @@ void
 init_secondary(void)
 {
 	int	gsel_tss;
-	int	x, myid = boot_cpuid;
+	int	x, myid = bootAP;
 
 	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
 	gdt_segs[GPROC0_SEL].ssd_base =
@@ -487,9 +487,7 @@ init_secondary(void)
 
 	load_cr0(0x8005003b);		/* XXX! */
 
-	pmap_set_opt((unsigned *)PTD);
-
-	invltlb();
+	pmap_set_opt();
 }
 
 
@@ -1827,7 +1825,7 @@ start_all_aps(u_int boot_addr)
 #endif
 
 		bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE];
-		boot_cpuid = x;
+		bootAP = x;
 
 		/* attempt to start the Application Processor */
 		CHECK_INIT(99);	/* setup checkpoints */
@@ -1874,7 +1872,7 @@ start_all_aps(u_int boot_addr)
 		    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
 
 	*(int *)PTD = 0;
-	pmap_set_opt_bsp();
+	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2172,6 +2170,9 @@ ap_init()
 {
 	u_int	apic_id;
 
+	/* BSP may have changed PTD while we're waiting for the lock */
+	cpu_invltlb();
+
 	smp_cpus++;
 
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h
index f6cd46f..fd55bdc 100644
--- a/sys/amd64/include/pmap.h
+++ b/sys/amd64/include/pmap.h
@@ -42,7 +42,7 @@
  *
  *	from: hp300: @(#)pmap.h	7.2 (Berkeley) 12/16/90
  *	from: @(#)pmap.h	7.4 (Berkeley) 5/12/91
- * 	$Id: pmap.h,v 1.59 1999/03/11 18:28:46 dg Exp $
+ * 	$Id: pmap.h,v 1.60 1999/04/02 17:59:49 alc Exp $
  */
 
 #ifndef _MACHINE_PMAP_H_
@@ -255,13 +255,9 @@ pmap_t	pmap_kernel __P((void));
 void	*pmap_mapdev __P((vm_offset_t, vm_size_t));
 unsigned *pmap_pte __P((pmap_t, vm_offset_t)) __pure2;
 vm_page_t pmap_use_pt __P((pmap_t, vm_offset_t));
-void	pmap_set_opt	__P((unsigned *));
-void	pmap_set_opt_bsp	__P((void));
-void 	getmtrr __P((void));
-void	putmtrr __P((void));
-void	putfmtrr __P((void));
-void	pmap_setdevram __P((unsigned long long, unsigned));
-void	pmap_setvidram __P((void));
+#ifdef SMP
+void	pmap_set_opt __P((void));
+#endif
 
 #endif /* KERNEL */
 
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 5afe5fe..5a9f672 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.102 1999/06/01 18:19:42 jlemon Exp $
+ *	$Id: mp_machdep.c,v 1.103 1999/06/22 20:54:25 msmith Exp $
  */
 
 #include "opt_smp.h"
@@ -300,7 +300,7 @@ u_int	all_cpus;
 
 /* AP uses this during bootstrap.  Do not staticize.  */
 char *bootSTK;
-int boot_cpuid;
+static int bootAP;
 
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t *KPTphys;
@@ -454,7 +454,7 @@ void
 init_secondary(void)
 {
 	int	gsel_tss;
-	int	x, myid = boot_cpuid;
+	int	x, myid = bootAP;
 
 	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
 	gdt_segs[GPROC0_SEL].ssd_base =
@@ -487,9 +487,7 @@ init_secondary(void)
 
 	load_cr0(0x8005003b);		/* XXX! */
 
-	pmap_set_opt((unsigned *)PTD);
-
-	invltlb();
+	pmap_set_opt();
 }
 
 
@@ -1827,7 +1825,7 @@ start_all_aps(u_int boot_addr)
 #endif
 
 		bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE];
-		boot_cpuid = x;
+		bootAP = x;
 
 		/* attempt to start the Application Processor */
 		CHECK_INIT(99);	/* setup checkpoints */
@@ -1874,7 +1872,7 @@ start_all_aps(u_int boot_addr)
 		    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
 
 	*(int *)PTD = 0;
-	pmap_set_opt_bsp();
+	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2172,6 +2170,9 @@ ap_init()
 {
 	u_int	apic_id;
 
+	/* BSP may have changed PTD while we're waiting for the lock */
+	cpu_invltlb();
+
 	smp_cpus++;
 
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index 5afe5fe..5a9f672 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.102 1999/06/01 18:19:42 jlemon Exp $
+ *	$Id: mp_machdep.c,v 1.103 1999/06/22 20:54:25 msmith Exp $
  */
 
 #include "opt_smp.h"
@@ -300,7 +300,7 @@ u_int	all_cpus;
 
 /* AP uses this during bootstrap.  Do not staticize.  */
 char *bootSTK;
-int boot_cpuid;
+static int bootAP;
 
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t *KPTphys;
@@ -454,7 +454,7 @@ void
 init_secondary(void)
 {
 	int	gsel_tss;
-	int	x, myid = boot_cpuid;
+	int	x, myid = bootAP;
 
 	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
 	gdt_segs[GPROC0_SEL].ssd_base =
@@ -487,9 +487,7 @@ init_secondary(void)
 
 	load_cr0(0x8005003b);		/* XXX! */
 
-	pmap_set_opt((unsigned *)PTD);
-
-	invltlb();
+	pmap_set_opt();
 }
 
 
@@ -1827,7 +1825,7 @@ start_all_aps(u_int boot_addr)
 #endif
 
 		bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE];
-		boot_cpuid = x;
+		bootAP = x;
 
 		/* attempt to start the Application Processor */
 		CHECK_INIT(99);	/* setup checkpoints */
@@ -1874,7 +1872,7 @@ start_all_aps(u_int boot_addr)
 		    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
 
 	*(int *)PTD = 0;
-	pmap_set_opt_bsp();
+	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2172,6 +2170,9 @@ ap_init()
 {
 	u_int	apic_id;
 
+	/* BSP may have changed PTD while we're waiting for the lock */
+	cpu_invltlb();
+
 	smp_cpus++;
 
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index f961651..82ee99e 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -39,7 +39,7 @@
  * SUCH DAMAGE.
  *
  *	from:	@(#)pmap.c	7.7 (Berkeley)	5/12/91
- *	$Id: pmap.c,v 1.238 1999/06/05 16:16:37 luoqi Exp $
+ *	$Id: pmap.c,v 1.239 1999/06/08 17:14:22 dt Exp $
  */
 
 /*
@@ -385,11 +385,6 @@ pmap_bootstrap(firstaddr, loadaddr)
 	if (cpu_feature & CPUID_PSE) {
 		unsigned ptditmp;
 		/*
-		 * Enable the PSE mode
-		 */
-		load_cr4(rcr4() | CR4_PSE);
-
-		/*
 		 * Note that we have enabled PSE mode
 		 */
 		pseflag = PG_PS;
@@ -400,6 +395,11 @@ pmap_bootstrap(firstaddr, loadaddr)
 
 #if !defined(SMP)
 		/*
+		 * Enable the PSE mode.
+		 */
+		load_cr4(rcr4() | CR4_PSE);
+
+		/*
 		 * We can do the mapping here for the single processor
 		 * case.  We simply ignore the old page table page from
 		 * now on.
@@ -460,38 +460,19 @@ pmap_bootstrap(firstaddr, loadaddr)
 
 #ifdef SMP
 /*
- * Set 4mb pdir for mp startup, and global flags
+ * Set 4mb pdir for mp startup
  */
 void
-pmap_set_opt(unsigned *pdir) {
-	int i;
-
+pmap_set_opt(void)
+{
 	if (pseflag && (cpu_feature & CPUID_PSE)) {
 		load_cr4(rcr4() | CR4_PSE);
-		if (pdir4mb) {
-			pdir[KPTDI] = pdir4mb;
+		if (pdir4mb && cpuid == 0) {	/* only on BSP */
+			kernel_pmap->pm_pdir[KPTDI] =
+			    PTD[KPTDI] = (pd_entry_t)pdir4mb;
+			cpu_invltlb();
 		}
 	}
-
-	if (pgeflag && (cpu_feature & CPUID_PGE)) {
-		load_cr4(rcr4() | CR4_PGE);
-		for(i = KPTDI; i < KPTDI + nkpt; i++) {
-			if (pdir[i]) {
-				pdir[i] |= PG_G;
-			}
-		}
-	}
-}
-
-/*
- * Setup the PTD for the boot processor
- */
-void
-pmap_set_opt_bsp(void)
-{
-	pmap_set_opt((unsigned *)kernel_pmap->pm_pdir);
-	pmap_set_opt((unsigned *)PTD);
-	invltlb();
 }
 #endif
 
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index 5afe5fe..5a9f672 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.102 1999/06/01 18:19:42 jlemon Exp $
+ *	$Id: mp_machdep.c,v 1.103 1999/06/22 20:54:25 msmith Exp $
  */
 
 #include "opt_smp.h"
@@ -300,7 +300,7 @@ u_int	all_cpus;
 
 /* AP uses this during bootstrap.  Do not staticize.  */
 char *bootSTK;
-int boot_cpuid;
+static int bootAP;
 
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t *KPTphys;
@@ -454,7 +454,7 @@ void
 init_secondary(void)
 {
 	int	gsel_tss;
-	int	x, myid = boot_cpuid;
+	int	x, myid = bootAP;
 
 	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
 	gdt_segs[GPROC0_SEL].ssd_base =
@@ -487,9 +487,7 @@ init_secondary(void)
 
 	load_cr0(0x8005003b);		/* XXX! */
 
-	pmap_set_opt((unsigned *)PTD);
-
-	invltlb();
+	pmap_set_opt();
 }
 
 
@@ -1827,7 +1825,7 @@ start_all_aps(u_int boot_addr)
 #endif
 
 		bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE];
-		boot_cpuid = x;
+		bootAP = x;
 
 		/* attempt to start the Application Processor */
 		CHECK_INIT(99);	/* setup checkpoints */
@@ -1874,7 +1872,7 @@ start_all_aps(u_int boot_addr)
 		    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
 
 	*(int *)PTD = 0;
-	pmap_set_opt_bsp();
+	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2172,6 +2170,9 @@ ap_init()
 {
 	u_int	apic_id;
 
+	/* BSP may have changed PTD while we're waiting for the lock */
+	cpu_invltlb();
+
 	smp_cpus++;
 
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)
diff --git a/sys/i386/include/pmap.h b/sys/i386/include/pmap.h
index f6cd46f..fd55bdc 100644
--- a/sys/i386/include/pmap.h
+++ b/sys/i386/include/pmap.h
@@ -42,7 +42,7 @@
  *
  *	from: hp300: @(#)pmap.h	7.2 (Berkeley) 12/16/90
  *	from: @(#)pmap.h	7.4 (Berkeley) 5/12/91
- * 	$Id: pmap.h,v 1.59 1999/03/11 18:28:46 dg Exp $
+ * 	$Id: pmap.h,v 1.60 1999/04/02 17:59:49 alc Exp $
  */
 
 #ifndef _MACHINE_PMAP_H_
@@ -255,13 +255,9 @@ pmap_t	pmap_kernel __P((void));
 void	*pmap_mapdev __P((vm_offset_t, vm_size_t));
 unsigned *pmap_pte __P((pmap_t, vm_offset_t)) __pure2;
 vm_page_t pmap_use_pt __P((pmap_t, vm_offset_t));
-void	pmap_set_opt	__P((unsigned *));
-void	pmap_set_opt_bsp	__P((void));
-void 	getmtrr __P((void));
-void	putmtrr __P((void));
-void	putfmtrr __P((void));
-void	pmap_setdevram __P((unsigned long long, unsigned));
-void	pmap_setvidram __P((void));
+#ifdef SMP
+void	pmap_set_opt __P((void));
+#endif
 
 #endif /* KERNEL */
 
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 5afe5fe..5a9f672 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.102 1999/06/01 18:19:42 jlemon Exp $
+ *	$Id: mp_machdep.c,v 1.103 1999/06/22 20:54:25 msmith Exp $
  */
 
 #include "opt_smp.h"
@@ -300,7 +300,7 @@ u_int	all_cpus;
 
 /* AP uses this during bootstrap.  Do not staticize.  */
 char *bootSTK;
-int boot_cpuid;
+static int bootAP;
 
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t *KPTphys;
@@ -454,7 +454,7 @@ void
 init_secondary(void)
 {
 	int	gsel_tss;
-	int	x, myid = boot_cpuid;
+	int	x, myid = bootAP;
 
 	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
 	gdt_segs[GPROC0_SEL].ssd_base =
@@ -487,9 +487,7 @@ init_secondary(void)
 
 	load_cr0(0x8005003b);		/* XXX! */
 
-	pmap_set_opt((unsigned *)PTD);
-
-	invltlb();
+	pmap_set_opt();
 }
 
 
@@ -1827,7 +1825,7 @@ start_all_aps(u_int boot_addr)
 #endif
 
 		bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE];
-		boot_cpuid = x;
+		bootAP = x;
 
 		/* attempt to start the Application Processor */
 		CHECK_INIT(99);	/* setup checkpoints */
@@ -1874,7 +1872,7 @@ start_all_aps(u_int boot_addr)
 		    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
 
 	*(int *)PTD = 0;
-	pmap_set_opt_bsp();
+	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2172,6 +2170,9 @@ ap_init()
 {
 	u_int	apic_id;
 
+	/* BSP may have changed PTD while we're waiting for the lock */
+	cpu_invltlb();
+
 	smp_cpus++;
 
 #if defined(I586_CPU) && !defined(NO_F00F_HACK)