Commit Bosko's patch to clean up the PSE/PG_G initialization to and

avoid problems with some Pentium 4 cpus and some older PPro/Pentium2
cpus.  There are several problems, some documented in Intel errata.
This patch:
1) moves the kernel to the second page in the PSE case.  There is an
errata that says that you Must Not point a 4MB page at physical
address zero on older cpus.  We avoided bugs here due to sheer luck.
2) sets up PSE page tables right from the start in locore, rather than
trying to switch from 4K to 4M (or 2M) pages part way through the boot
sequence at the same time that we're messing with PG_G.

For some reason, the pmap work over the last 18 months seems to tickle
the problems, and the PAE infrastructure changes disturb the cpu
bugs even more.

A couple of people have reported a problem with APM bios calls during
boot.  I'll work with people to get this resolved.

Obtained from:	bmilekic

17 files changed, 180 insertions, 170 deletions

diff --git a/sys/amd64/amd64/bios.c b/sys/amd64/amd64/bios.c
index 69f6ded..da862ba 100644
--- a/sys/amd64/amd64/bios.c
+++ b/sys/amd64/amd64/bios.c
@@ -396,18 +396,16 @@ bios16(struct bios_args *args, char *fmt, ...)
 	 */
 	pte = (pt_entry_t *)malloc(PAGE_SIZE, M_TEMP, M_WAITOK);
 	ptd = (pd_entry_t *)((u_int)IdlePTD + KERNBASE);
+	*pte = (vm86pa - PAGE_SIZE) | PG_RW | PG_V;
 	*ptd = vtophys(pte) | PG_RW | PG_V;
     } else {
 	/*
 	 * this is a user-level page table 
 	 */
 	pte = PTmap;
+	*pte = (vm86pa - PAGE_SIZE) | PG_RW | PG_V;
     }
-    /*
-     * install pointer to page 0.  we don't need to flush the tlb,
-     * since there should not be a previous mapping for page 0.
-     */
-    *pte = (vm86pa - PAGE_SIZE) | PG_RW | PG_V; 
+    pmap_invalidate_all(kernel_pmap);	/* XXX insurance for now */
 
     stack_top = stack;
     va_start(ap, fmt);
@@ -457,19 +455,21 @@ bios16(struct bios_args *args, char *fmt, ...)
     bioscall_vector.vec16.segment = GSEL(GBIOSCODE16_SEL, SEL_KPL);
 
     i = bios16_call(&args->r, stack_top);
-    
+
     if (pte == PTmap) {
 	*pte = 0;			/* remove entry */
+	/*
+	 * XXX only needs to be invlpg(0) but that doesn't work on the 386 
+	 */
+	pmap_invalidate_all(kernel_pmap);
     } else {
 	*ptd = 0;			/* remove page table */
+	/*
+	 * XXX only needs to be invlpg(0) but that doesn't work on the 386 
+	 */
+	pmap_invalidate_all(kernel_pmap);
 	free(pte, M_TEMP);		/* ... and free it */
     }
-
-    /*
-     * XXX only needs to be invlpg(0) but that doesn't work on the 386 
-     */
-    pmap_invalidate_all(kernel_pmap);
-
     return (i);
 }
 
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index a050d3c..fe150ff 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -536,8 +536,6 @@ init_secondary(void)
 	cr0 = rcr0();
 	cr0 &= ~(CR0_CD | CR0_NW | CR0_EM);
 	load_cr0(cr0);
-
-	pmap_set_opt();
 }
 
 
@@ -926,7 +924,6 @@ mptable_pass2(void)
 	int     type;
 	int     apic, bus, cpu, intr;
 	int	i, j;
-	int	pgeflag;
 
 	POSTCODE(MPTABLE_PASS2_POST);
 
@@ -935,8 +932,6 @@ mptable_pass2(void)
 	proc.type = 0;
 	proc.cpu_flags = PROCENTRY_FLAG_EN;
 
-	pgeflag = 0;		/* XXX - Not used under SMP yet.  */
-
 	MALLOC(io_apic_versions, u_int32_t *, sizeof(u_int32_t) * mp_napics,
 	    M_DEVBUF, M_WAITOK);
 	MALLOC(ioapic, volatile ioapic_t **, sizeof(ioapic_t *) * mp_napics,
@@ -961,7 +956,7 @@ mptable_pass2(void)
 			/* use this slot if available */
 			if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == 0) {
 				SMPpt[NPTEPG-2-j] = (pt_entry_t)(PG_V | PG_RW |
-				    pgeflag | (io_apic_address[i] & PG_FRAME));
+				    (io_apic_address[i] & PG_FRAME));
 				ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace
 					+ (NPTEPG-2-j) * PAGE_SIZE
 					+ (io_apic_address[i] & PAGE_MASK));
@@ -2177,7 +2172,6 @@ start_all_aps(u_int boot_addr)
 
 	for (x = 0; x < NKPT; x++)
 		PTD[x] = 0;
-	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2285,7 +2279,7 @@ start_ap(int logical_cpu, u_int boot_addr)
 	/* setup common fields for subsequent IPIs */
 	icr_lo = lapic.icr_lo & APIC_ICRLO_RESV_MASK;
 	icr_lo |= APIC_DESTMODE_PHY;
-	
+
 	/* do an INIT IPI: assert RESET */
 	lapic.icr_lo = icr_lo | APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
 	    APIC_LEVEL_ASSERT | APIC_DELMODE_INIT;
diff --git a/sys/amd64/amd64/mpboot.S b/sys/amd64/amd64/mpboot.S
index e245007..d77b877 100644
--- a/sys/amd64/amd64/mpboot.S
+++ b/sys/amd64/amd64/mpboot.S
@@ -86,6 +86,22 @@ NON_GPROF_ENTRY(MPentry)
 	movl	R(IdlePTD), %eax
 	movl	%eax,%cr3	
 #endif
+#ifndef DISABLE_PSE
+	cmpl	$0, R(pseflag)
+	je	1f
+	movl	%cr4, %eax
+	orl	$CR4_PSE, %eax
+	movl	%eax, %cr4
+1:
+#endif
+#ifndef DISABLE_PG_G
+	cmpl	$0, R(pgeflag)
+	je	2f
+	movl	%cr4, %eax
+	orl	$CR4_PGE, %eax
+	movl	%eax, %cr4
+2:
+#endif
 	movl	%cr0,%eax
 	orl	$CR0_PE|CR0_PG,%eax		/* enable paging */
 	movl	%eax,%cr0			/* let the games begin! */
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index a050d3c..fe150ff 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -536,8 +536,6 @@ init_secondary(void)
 	cr0 = rcr0();
 	cr0 &= ~(CR0_CD | CR0_NW | CR0_EM);
 	load_cr0(cr0);
-
-	pmap_set_opt();
 }
 
 
@@ -926,7 +924,6 @@ mptable_pass2(void)
 	int     type;
 	int     apic, bus, cpu, intr;
 	int	i, j;
-	int	pgeflag;
 
 	POSTCODE(MPTABLE_PASS2_POST);
 
@@ -935,8 +932,6 @@ mptable_pass2(void)
 	proc.type = 0;
 	proc.cpu_flags = PROCENTRY_FLAG_EN;
 
-	pgeflag = 0;		/* XXX - Not used under SMP yet.  */
-
 	MALLOC(io_apic_versions, u_int32_t *, sizeof(u_int32_t) * mp_napics,
 	    M_DEVBUF, M_WAITOK);
 	MALLOC(ioapic, volatile ioapic_t **, sizeof(ioapic_t *) * mp_napics,
@@ -961,7 +956,7 @@ mptable_pass2(void)
 			/* use this slot if available */
 			if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == 0) {
 				SMPpt[NPTEPG-2-j] = (pt_entry_t)(PG_V | PG_RW |
-				    pgeflag | (io_apic_address[i] & PG_FRAME));
+				    (io_apic_address[i] & PG_FRAME));
 				ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace
 					+ (NPTEPG-2-j) * PAGE_SIZE
 					+ (io_apic_address[i] & PAGE_MASK));
@@ -2177,7 +2172,6 @@ start_all_aps(u_int boot_addr)
 
 	for (x = 0; x < NKPT; x++)
 		PTD[x] = 0;
-	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2285,7 +2279,7 @@ start_ap(int logical_cpu, u_int boot_addr)
 	/* setup common fields for subsequent IPIs */
 	icr_lo = lapic.icr_lo & APIC_ICRLO_RESV_MASK;
 	icr_lo |= APIC_DESTMODE_PHY;
-	
+
 	/* do an INIT IPI: assert RESET */
 	lapic.icr_lo = icr_lo | APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
 	    APIC_LEVEL_ASSERT | APIC_DELMODE_INIT;
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index a050d3c..fe150ff 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -536,8 +536,6 @@ init_secondary(void)
 	cr0 = rcr0();
 	cr0 &= ~(CR0_CD | CR0_NW | CR0_EM);
 	load_cr0(cr0);
-
-	pmap_set_opt();
 }
 
 
@@ -926,7 +924,6 @@ mptable_pass2(void)
 	int     type;
 	int     apic, bus, cpu, intr;
 	int	i, j;
-	int	pgeflag;
 
 	POSTCODE(MPTABLE_PASS2_POST);
 
@@ -935,8 +932,6 @@ mptable_pass2(void)
 	proc.type = 0;
 	proc.cpu_flags = PROCENTRY_FLAG_EN;
 
-	pgeflag = 0;		/* XXX - Not used under SMP yet.  */
-
 	MALLOC(io_apic_versions, u_int32_t *, sizeof(u_int32_t) * mp_napics,
 	    M_DEVBUF, M_WAITOK);
 	MALLOC(ioapic, volatile ioapic_t **, sizeof(ioapic_t *) * mp_napics,
@@ -961,7 +956,7 @@ mptable_pass2(void)
 			/* use this slot if available */
 			if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == 0) {
 				SMPpt[NPTEPG-2-j] = (pt_entry_t)(PG_V | PG_RW |
-				    pgeflag | (io_apic_address[i] & PG_FRAME));
+				    (io_apic_address[i] & PG_FRAME));
 				ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace
 					+ (NPTEPG-2-j) * PAGE_SIZE
 					+ (io_apic_address[i] & PAGE_MASK));
@@ -2177,7 +2172,6 @@ start_all_aps(u_int boot_addr)
 
 	for (x = 0; x < NKPT; x++)
 		PTD[x] = 0;
-	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2285,7 +2279,7 @@ start_ap(int logical_cpu, u_int boot_addr)
 	/* setup common fields for subsequent IPIs */
 	icr_lo = lapic.icr_lo & APIC_ICRLO_RESV_MASK;
 	icr_lo |= APIC_DESTMODE_PHY;
-	
+
 	/* do an INIT IPI: assert RESET */
 	lapic.icr_lo = icr_lo | APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
 	    APIC_LEVEL_ASSERT | APIC_DELMODE_INIT;
diff --git a/sys/conf/ldscript.i386 b/sys/conf/ldscript.i386
index 181e366..b8f7575 100644
--- a/sys/conf/ldscript.i386
+++ b/sys/conf/ldscript.i386
@@ -6,7 +6,7 @@ SEARCH_DIR(/usr/lib);
 SECTIONS
 {
   /* Read-only sections, merged into text segment: */
-  . = kernbase + 0x00100000 + SIZEOF_HEADERS;
+  . = kernbase + kernload + SIZEOF_HEADERS;
   .interp     : { *(.interp) 	}
   .hash          : { *(.hash)		}
   .dynsym        : { *(.dynsym)		}
diff --git a/sys/i386/i386/bios.c b/sys/i386/i386/bios.c
index 69f6ded..da862ba 100644
--- a/sys/i386/i386/bios.c
+++ b/sys/i386/i386/bios.c
@@ -396,18 +396,16 @@ bios16(struct bios_args *args, char *fmt, ...)
 	 */
 	pte = (pt_entry_t *)malloc(PAGE_SIZE, M_TEMP, M_WAITOK);
 	ptd = (pd_entry_t *)((u_int)IdlePTD + KERNBASE);
+	*pte = (vm86pa - PAGE_SIZE) | PG_RW | PG_V;
 	*ptd = vtophys(pte) | PG_RW | PG_V;
     } else {
 	/*
 	 * this is a user-level page table 
 	 */
 	pte = PTmap;
+	*pte = (vm86pa - PAGE_SIZE) | PG_RW | PG_V;
     }
-    /*
-     * install pointer to page 0.  we don't need to flush the tlb,
-     * since there should not be a previous mapping for page 0.
-     */
-    *pte = (vm86pa - PAGE_SIZE) | PG_RW | PG_V; 
+    pmap_invalidate_all(kernel_pmap);	/* XXX insurance for now */
 
     stack_top = stack;
     va_start(ap, fmt);
@@ -457,19 +455,21 @@ bios16(struct bios_args *args, char *fmt, ...)
     bioscall_vector.vec16.segment = GSEL(GBIOSCODE16_SEL, SEL_KPL);
 
     i = bios16_call(&args->r, stack_top);
-    
+
     if (pte == PTmap) {
 	*pte = 0;			/* remove entry */
+	/*
+	 * XXX only needs to be invlpg(0) but that doesn't work on the 386 
+	 */
+	pmap_invalidate_all(kernel_pmap);
     } else {
 	*ptd = 0;			/* remove page table */
+	/*
+	 * XXX only needs to be invlpg(0) but that doesn't work on the 386 
+	 */
+	pmap_invalidate_all(kernel_pmap);
 	free(pte, M_TEMP);		/* ... and free it */
     }
-
-    /*
-     * XXX only needs to be invlpg(0) but that doesn't work on the 386 
-     */
-    pmap_invalidate_all(kernel_pmap);
-
     return (i);
 }
 
diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c
index 0954f19..39a926a 100644
--- a/sys/i386/i386/genassym.c
+++ b/sys/i386/i386/genassym.c
@@ -110,13 +110,16 @@ ASSYM(NPDEPTD, NPDEPTD);
 ASSYM(NPGPTD, NPGPTD);
 ASSYM(PDESIZE, sizeof(pd_entry_t));
 ASSYM(PTESIZE, sizeof(pt_entry_t));
+ASSYM(PDESHIFT, PDESHIFT);
 ASSYM(PTESHIFT, PTESHIFT);
 ASSYM(PAGE_SHIFT, PAGE_SHIFT);
 ASSYM(PAGE_MASK, PAGE_MASK);
 ASSYM(PDRSHIFT, PDRSHIFT);
+ASSYM(PDRMASK, PDRMASK);
 ASSYM(USRSTACK, USRSTACK);
 ASSYM(VM_MAXUSER_ADDRESS, VM_MAXUSER_ADDRESS);
 ASSYM(KERNBASE, KERNBASE);
+ASSYM(KERNLOAD, KERNLOAD);
 ASSYM(MCLBYTES, MCLBYTES);
 ASSYM(PCB_CR3, offsetof(struct pcb, pcb_cr3));
 ASSYM(PCB_EDI, offsetof(struct pcb, pcb_edi));
diff --git a/sys/i386/i386/locore.s b/sys/i386/i386/locore.s
index 0dd008e..ce8d602 100644
--- a/sys/i386/i386/locore.s
+++ b/sys/i386/i386/locore.s
@@ -87,10 +87,12 @@
 #endif /* SMP */
 
 /*
- * Compiled KERNBASE location
+ * Compiled KERNBASE location and the kernel load address
  */
 	.globl	kernbase
 	.set	kernbase,KERNBASE
+	.globl	kernload
+	.set	kernload,KERNLOAD
 
 /*
  * Globals
@@ -731,9 +733,9 @@ over_symalloc:
 	je	no_kernend
 	movl	%edi,%esi
 no_kernend:
-	
-	addl	$PAGE_MASK,%esi
-	andl	$~PAGE_MASK,%esi
+
+	addl	$PDRMASK,%esi		/* Play conservative for now, and */
+	andl	$~PDRMASK,%esi		/*   ... wrap to next 4M. */
 	movl	%esi,R(KERNend)		/* save end of kernel */
 	movl	%esi,R(physfree)	/* next free page is at end of kernel */
 
@@ -783,16 +785,46 @@ no_kernend:
 	movl	%esi, R(SMPpt)		/* relocated to KVM space */
 #endif	/* SMP */
 
-/* Map read-only from zero to the end of the kernel text section */
+/* Map read-only from zero to the beginning of the kernel text section */
 	xorl	%eax, %eax
 	xorl	%edx,%edx
-	movl	$R(etext),%ecx
+	movl	$R(btext),%ecx
 	addl	$PAGE_MASK,%ecx
 	shrl	$PAGE_SHIFT,%ecx
 	fillkptphys(%edx)
 
-/* Map read-write, data, bss and symbols */
-	movl	$R(etext),%eax
+/*
+ * Enable PSE and PGE.
+ */
+#ifndef DISABLE_PSE
+	testl	$CPUID_PSE, R(cpu_feature)
+	jz	1f
+	movl	$PG_PS, R(pseflag)
+	movl	%cr4, %eax
+	orl	$CR4_PSE, %eax
+	movl	%eax, %cr4
+1:
+#endif
+#ifndef DISABLE_PG_G
+	testl	$CPUID_PGE, R(cpu_feature)
+	jz	2f
+	movl	$PG_G, R(pgeflag)
+	movl	%cr4, %eax
+	orl	$CR4_PGE, %eax
+	movl	%eax, %cr4
+2:
+#endif
+
+/*
+ * Write page tables for the kernel starting at btext and
+ * until the end.  Make sure to map read+write.  We do this even
+ * if we've enabled PSE above, we'll just switch the corresponding kernel
+ * PDEs before we turn on paging.
+ *
+ * XXX: We waste some pages here in the PSE case!  DON'T BLINDLY REMOVE
+ * THIS!  SMP needs the page table to be there to map the kernel P==V.
+ */
+	movl	$R(btext),%eax
 	addl	$PAGE_MASK, %eax
 	andl	$~PAGE_MASK, %eax
 	movl	$PG_RW,%edx
@@ -881,12 +913,33 @@ no_kernend:
 	movl	$NKPT, %ecx
 	fillkpt(R(IdlePTD), $PG_RW)
 
-/* install pde's for pt's */
+/*
+ * For the non-PSE case, install PDEs for PTs covering the kernel.
+ * For the PSE case, do the same, but clobber the ones corresponding
+ * to the kernel (from btext to KERNend) with 4M ('PS') PDEs immediately
+ * after.
+ */
 	movl	R(KPTphys), %eax
 	movl	$KPTDI, %ebx
 	movl	$NKPT, %ecx
 	fillkpt(R(IdlePTD), $PG_RW)
+	cmpl	$0,R(pseflag)
+	je	done_pde
+
+	movl	R(KERNend), %ecx
+	movl	$KERNLOAD, %eax
+	subl	%eax, %ecx
+	shrl	$PDRSHIFT, %ecx
+	movl	$(KPTDI+(KERNLOAD/(1 << PDRSHIFT))), %ebx
+	shll	$PDESHIFT, %ebx
+	addl	R(IdlePTD), %ebx
+	orl	$(PG_V|PG_RW|PG_PS), %eax
+1:	movl	%eax, (%ebx)
+	addl	$(1 << PDRSHIFT), %eax
+	addl	$PDESIZE, %ebx
+	loop	1b
 
+done_pde:
 /* install a pde recursively mapping page directory as a page table */
 	movl	R(IdlePTD), %eax
 	movl	$PTDPTDI, %ebx
diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c
index dd229d8..01149ba 100644
--- a/sys/i386/i386/machdep.c
+++ b/sys/i386/i386/machdep.c
@@ -1798,7 +1798,7 @@ physmap_done:
 			/*
 			 * block out kernel memory as not available.
 			 */
-			if (pa >= 0x100000 && pa < first)
+			if (pa >= KERNLOAD && pa < first)
 				continue;
 	
 			page_bad = FALSE;
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index a050d3c..fe150ff 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -536,8 +536,6 @@ init_secondary(void)
 	cr0 = rcr0();
 	cr0 &= ~(CR0_CD | CR0_NW | CR0_EM);
 	load_cr0(cr0);
-
-	pmap_set_opt();
 }
 
 
@@ -926,7 +924,6 @@ mptable_pass2(void)
 	int     type;
 	int     apic, bus, cpu, intr;
 	int	i, j;
-	int	pgeflag;
 
 	POSTCODE(MPTABLE_PASS2_POST);
 
@@ -935,8 +932,6 @@ mptable_pass2(void)
 	proc.type = 0;
 	proc.cpu_flags = PROCENTRY_FLAG_EN;
 
-	pgeflag = 0;		/* XXX - Not used under SMP yet.  */
-
 	MALLOC(io_apic_versions, u_int32_t *, sizeof(u_int32_t) * mp_napics,
 	    M_DEVBUF, M_WAITOK);
 	MALLOC(ioapic, volatile ioapic_t **, sizeof(ioapic_t *) * mp_napics,
@@ -961,7 +956,7 @@ mptable_pass2(void)
 			/* use this slot if available */
 			if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == 0) {
 				SMPpt[NPTEPG-2-j] = (pt_entry_t)(PG_V | PG_RW |
-				    pgeflag | (io_apic_address[i] & PG_FRAME));
+				    (io_apic_address[i] & PG_FRAME));
 				ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace
 					+ (NPTEPG-2-j) * PAGE_SIZE
 					+ (io_apic_address[i] & PAGE_MASK));
@@ -2177,7 +2172,6 @@ start_all_aps(u_int boot_addr)
 
 	for (x = 0; x < NKPT; x++)
 		PTD[x] = 0;
-	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2285,7 +2279,7 @@ start_ap(int logical_cpu, u_int boot_addr)
 	/* setup common fields for subsequent IPIs */
 	icr_lo = lapic.icr_lo & APIC_ICRLO_RESV_MASK;
 	icr_lo |= APIC_DESTMODE_PHY;
-	
+
 	/* do an INIT IPI: assert RESET */
 	lapic.icr_lo = icr_lo | APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
 	    APIC_LEVEL_ASSERT | APIC_DELMODE_INIT;
diff --git a/sys/i386/i386/mpboot.s b/sys/i386/i386/mpboot.s
index e245007..d77b877 100644
--- a/sys/i386/i386/mpboot.s
+++ b/sys/i386/i386/mpboot.s
@@ -86,6 +86,22 @@ NON_GPROF_ENTRY(MPentry)
 	movl	R(IdlePTD), %eax
 	movl	%eax,%cr3	
 #endif
+#ifndef DISABLE_PSE
+	cmpl	$0, R(pseflag)
+	je	1f
+	movl	%cr4, %eax
+	orl	$CR4_PSE, %eax
+	movl	%eax, %cr4
+1:
+#endif
+#ifndef DISABLE_PG_G
+	cmpl	$0, R(pgeflag)
+	je	2f
+	movl	%cr4, %eax
+	orl	$CR4_PGE, %eax
+	movl	%eax, %cr4
+2:
+#endif
 	movl	%cr0,%eax
 	orl	$CR0_PE|CR0_PG,%eax		/* enable paging */
 	movl	%eax,%cr0			/* let the games begin! */
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index a050d3c..fe150ff 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -536,8 +536,6 @@ init_secondary(void)
 	cr0 = rcr0();
 	cr0 &= ~(CR0_CD | CR0_NW | CR0_EM);
 	load_cr0(cr0);
-
-	pmap_set_opt();
 }
 
 
@@ -926,7 +924,6 @@ mptable_pass2(void)
 	int     type;
 	int     apic, bus, cpu, intr;
 	int	i, j;
-	int	pgeflag;
 
 	POSTCODE(MPTABLE_PASS2_POST);
 
@@ -935,8 +932,6 @@ mptable_pass2(void)
 	proc.type = 0;
 	proc.cpu_flags = PROCENTRY_FLAG_EN;
 
-	pgeflag = 0;		/* XXX - Not used under SMP yet.  */
-
 	MALLOC(io_apic_versions, u_int32_t *, sizeof(u_int32_t) * mp_napics,
 	    M_DEVBUF, M_WAITOK);
 	MALLOC(ioapic, volatile ioapic_t **, sizeof(ioapic_t *) * mp_napics,
@@ -961,7 +956,7 @@ mptable_pass2(void)
 			/* use this slot if available */
 			if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == 0) {
 				SMPpt[NPTEPG-2-j] = (pt_entry_t)(PG_V | PG_RW |
-				    pgeflag | (io_apic_address[i] & PG_FRAME));
+				    (io_apic_address[i] & PG_FRAME));
 				ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace
 					+ (NPTEPG-2-j) * PAGE_SIZE
 					+ (io_apic_address[i] & PAGE_MASK));
@@ -2177,7 +2172,6 @@ start_all_aps(u_int boot_addr)
 
 	for (x = 0; x < NKPT; x++)
 		PTD[x] = 0;
-	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2285,7 +2279,7 @@ start_ap(int logical_cpu, u_int boot_addr)
 	/* setup common fields for subsequent IPIs */
 	icr_lo = lapic.icr_lo & APIC_ICRLO_RESV_MASK;
 	icr_lo |= APIC_DESTMODE_PHY;
-	
+
 	/* do an INIT IPI: assert RESET */
 	lapic.icr_lo = icr_lo | APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
 	    APIC_LEVEL_ASSERT | APIC_DELMODE_INIT;
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index 7851489..f1ce5a3 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -195,8 +195,8 @@ vm_paddr_t avail_end;	/* PA of last available physical page */
 vm_offset_t virtual_avail;	/* VA of first avail page (after kernel bss) */
 vm_offset_t virtual_end;	/* VA of last avail page (end of kernel AS) */
 static boolean_t pmap_initialized = FALSE;	/* Has pmap_init completed? */
-static int pgeflag;		/* PG_G or-in */
-static int pseflag;		/* PG_PS or-in */
+int pgeflag = 0;		/* PG_G or-in */
+int pseflag = 0;		/* PG_PS or-in */
 
 static int nkpt;
 vm_offset_t kernel_vm_end;
@@ -260,8 +260,6 @@ static void *pmap_pv_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wai
 static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
 #endif
 
-static pd_entry_t pdir4mb;
-
 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
 CTASSERT(1 << PTESHIFT == sizeof(pt_entry_t));
 
@@ -285,7 +283,7 @@ pmap_kmem_choose(vm_offset_t addr)
 #endif
 #ifndef DISABLE_PSE
 	if (cpu_feature & CPUID_PSE)
-		newaddr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
+		newaddr = (addr + PDRMASK) & ~PDRMASK;
 #endif
 	return newaddr;
 }
@@ -397,11 +395,6 @@ pmap_bootstrap(firstaddr, loadaddr)
 	for (i = 0; i < NKPT; i++)
 		PTD[i] = 0;
 
-	pgeflag = 0;
-#ifndef DISABLE_PG_G
-	if (cpu_feature & CPUID_PGE)
-		pgeflag = PG_G;
-#endif
 #ifdef I686_CPU_not	/* Problem seems to have gone away */
 	/* Deal with un-resolved Pentium4 issues */
 	if (cpu_class == CPUCLASS_686 &&
@@ -411,21 +404,7 @@ pmap_bootstrap(firstaddr, loadaddr)
 		pgeflag = 0;
 	}
 #endif
-	
-/*
- * Initialize the 4MB page size flag
- */
-	pseflag = 0;
-/*
- * The 4MB page version of the initial
- * kernel page mapping.
- */
-	pdir4mb = 0;
 
-#ifndef DISABLE_PSE
-	if (cpu_feature & CPUID_PSE)
-		pseflag = PG_PS;
-#endif
 #ifdef I686_CPU_not	/* Problem seems to have gone away */
 	/* Deal with un-resolved Pentium4 issues */
 	if (cpu_class == CPUCLASS_686 &&
@@ -435,26 +414,10 @@ pmap_bootstrap(firstaddr, loadaddr)
 		pseflag = 0;
 	}
 #endif
-#ifndef DISABLE_PSE
-	if (pseflag) {
-		pd_entry_t ptditmp;
-		/*
-		 * Note that we have enabled PSE mode
-		 */
-		ptditmp = *(PTmap + i386_btop(KERNBASE));
-		ptditmp &= ~(NBPDR - 1);
-		ptditmp |= PG_V | PG_RW | PG_PS | PG_U | pgeflag;
-		pdir4mb = ptditmp;
-	}
-#endif
-#ifndef SMP
-	/*
-	 * Turn on PGE/PSE.  SMP does this later on since the
-	 * 4K page tables are required for AP boot (for now).
-	 * XXX fixme.
-	 */
-	pmap_set_opt();
-#endif
+
+	/* Turn on PG_G on kernel page(s) */
+	pmap_set_pg();
+
 #ifdef SMP
 	if (cpu_apic_address == 0)
 		panic("pmap_bootstrap: no local apic! (non-SMP hardware?)");
@@ -467,55 +430,41 @@ pmap_bootstrap(firstaddr, loadaddr)
 }
 
 /*
- * Enable 4MB page mode for MP startup.  Turn on PG_G support.
- * BSP will run this after all the AP's have started up.
+ * Set PG_G on kernel pages.  Only the BSP calls this when SMP is turned on.
  */
 void
-pmap_set_opt(void)
+pmap_set_pg(void)
 {
+	pd_entry_t pdir;
 	pt_entry_t *pte;
 	vm_offset_t va, endva;
+	int i; 
 
-	if (pgeflag && (cpu_feature & CPUID_PGE)) {
-		load_cr4(rcr4() | CR4_PGE);
-		invltlb();		/* Insurance */
-	}
-#ifndef DISABLE_PSE
-	if (pseflag && (cpu_feature & CPUID_PSE)) {
-		load_cr4(rcr4() | CR4_PSE);
-		invltlb();		/* Insurance */
-	}
-#endif
-	if (PCPU_GET(cpuid) == 0) {
-#ifndef DISABLE_PSE
-		if (pdir4mb) {
-			kernel_pmap->pm_pdir[KPTDI] = PTD[KPTDI] = pdir4mb;
-			invltlb();	/* Insurance */
+	if (pgeflag == 0)
+		return;
+
+	i = KERNLOAD/NBPDR;
+	endva = KERNBASE + KERNend;
+
+	if (pseflag) {
+		va = KERNBASE + KERNLOAD;
+		while (va  < endva) {
+			pdir = kernel_pmap->pm_pdir[KPTDI+i];
+			pdir |= pgeflag;
+			kernel_pmap->pm_pdir[KPTDI+i] = PTD[KPTDI+i] = pdir;
+			invltlb();	/* Play it safe, invltlb() every time */
+			i++;
+			va += NBPDR;
 		}
-#endif
-		if (pgeflag) {
-			/* Turn on PG_G for text, data, bss pages. */
-			va = (vm_offset_t)btext;
-#ifndef DISABLE_PSE
-			if (pseflag && (cpu_feature & CPUID_PSE)) {
-				if (va < KERNBASE + (1 << PDRSHIFT))
-					va = KERNBASE + (1 << PDRSHIFT);
-			}
-#endif
-			endva = KERNBASE + KERNend;
-			while (va < endva) {
-				pte = vtopte(va);
-				if (*pte)
-					*pte |= pgeflag;
-				va += PAGE_SIZE;
-			}
-			invltlb();	/* Insurance */
+	} else {
+		va = (vm_offset_t)btext;
+		while (va < endva) {
+			pte = vtopte(va);
+			if (*pte)
+				*pte |= pgeflag;
+			invltlb();	/* Play it safe, invltlb() every time */
+			va += PAGE_SIZE;
 		}
-		/*
-		 * We do not need to broadcast the invltlb here, because
-		 * each AP does it the moment it is released from the boot
-		 * lock.  See ap_init().
-		 */
 	}
 }
 
@@ -3156,7 +3105,7 @@ pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size)
 		return addr;
 	}
 
-	addr = (addr + (NBPDR - 1)) & ~(NBPDR - 1);
+	addr = (addr + PDRMASK) & ~PDRMASK;
 	return addr;
 }
 
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index a050d3c..fe150ff 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -536,8 +536,6 @@ init_secondary(void)
 	cr0 = rcr0();
 	cr0 &= ~(CR0_CD | CR0_NW | CR0_EM);
 	load_cr0(cr0);
-
-	pmap_set_opt();
 }
 
 
@@ -926,7 +924,6 @@ mptable_pass2(void)
 	int     type;
 	int     apic, bus, cpu, intr;
 	int	i, j;
-	int	pgeflag;
 
 	POSTCODE(MPTABLE_PASS2_POST);
 
@@ -935,8 +932,6 @@ mptable_pass2(void)
 	proc.type = 0;
 	proc.cpu_flags = PROCENTRY_FLAG_EN;
 
-	pgeflag = 0;		/* XXX - Not used under SMP yet.  */
-
 	MALLOC(io_apic_versions, u_int32_t *, sizeof(u_int32_t) * mp_napics,
 	    M_DEVBUF, M_WAITOK);
 	MALLOC(ioapic, volatile ioapic_t **, sizeof(ioapic_t *) * mp_napics,
@@ -961,7 +956,7 @@ mptable_pass2(void)
 			/* use this slot if available */
 			if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == 0) {
 				SMPpt[NPTEPG-2-j] = (pt_entry_t)(PG_V | PG_RW |
-				    pgeflag | (io_apic_address[i] & PG_FRAME));
+				    (io_apic_address[i] & PG_FRAME));
 				ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace
 					+ (NPTEPG-2-j) * PAGE_SIZE
 					+ (io_apic_address[i] & PAGE_MASK));
@@ -2177,7 +2172,6 @@ start_all_aps(u_int boot_addr)
 
 	for (x = 0; x < NKPT; x++)
 		PTD[x] = 0;
-	pmap_set_opt();
 
 	/* number of APs actually started */
 	return mp_ncpus - 1;
@@ -2285,7 +2279,7 @@ start_ap(int logical_cpu, u_int boot_addr)
 	/* setup common fields for subsequent IPIs */
 	icr_lo = lapic.icr_lo & APIC_ICRLO_RESV_MASK;
 	icr_lo |= APIC_DESTMODE_PHY;
-	
+
 	/* do an INIT IPI: assert RESET */
 	lapic.icr_lo = icr_lo | APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE |
 	    APIC_LEVEL_ASSERT | APIC_DELMODE_INIT;
diff --git a/sys/i386/include/pmap.h b/sys/i386/include/pmap.h
index 3437a8c..62afddb 100644
--- a/sys/i386/include/pmap.h
+++ b/sys/i386/include/pmap.h
@@ -330,6 +330,8 @@ extern vm_paddr_t avail_start;
 extern vm_offset_t clean_eva;
 extern vm_offset_t clean_sva;
 extern vm_paddr_t phys_avail[];
+extern int pseflag;
+extern int pgeflag;
 extern char *ptvmmap;		/* poor name! */
 extern vm_offset_t virtual_avail;
 extern vm_offset_t virtual_end;
@@ -340,7 +342,7 @@ void	pmap_kremove(vm_offset_t);
 void	*pmap_mapdev(vm_paddr_t, vm_size_t);
 void	pmap_unmapdev(vm_offset_t, vm_size_t);
 pt_entry_t *pmap_pte_quick(pmap_t, vm_offset_t) __pure2;
-void	pmap_set_opt(void);
+void	pmap_set_pg(void);
 void	pmap_invalidate_page(pmap_t, vm_offset_t);
 void	pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t);
 void	pmap_invalidate_all(pmap_t);
diff --git a/sys/i386/include/vmparam.h b/sys/i386/include/vmparam.h
index 9465890..5943e76 100644
--- a/sys/i386/include/vmparam.h
+++ b/sys/i386/include/vmparam.h
@@ -84,6 +84,13 @@
 
 
 /*
+ * Kernel physical load address.
+ */
+#ifndef KERNLOAD
+#define	KERNLOAD		(1 << PDRSHIFT)
+#endif
+
+/*
  * Virtual addresses of things.  Derived from the page directory and
  * page table indexes from pmap.h for precision.
  * Because of the page that is both a PD and PT, it looks a little