Add the PG_NX support for i386/PAE.

Reviewed by:	alc

8 files changed, 135 insertions, 34 deletions

diff --git a/sys/i386/i386/identcpu.c b/sys/i386/i386/identcpu.c
index c2f533c..48c8b47 100644
--- a/sys/i386/i386/identcpu.c
+++ b/sys/i386/i386/identcpu.c
@@ -188,20 +188,6 @@ printcpuinfo(void)
 		}
 	}
 
-	/* Detect AMD features (PTE no-execute bit, 3dnow, 64 bit mode etc) */
-	if (strcmp(cpu_vendor, "GenuineIntel") == 0 ||
-	    strcmp(cpu_vendor, "AuthenticAMD") == 0) {
-		if (cpu_exthigh >= 0x80000001) {
-			do_cpuid(0x80000001, regs);
-			amd_feature = regs[3] & ~(cpu_feature & 0x0183f3ff);
-			amd_feature2 = regs[2];
-		}
-		if (cpu_exthigh >= 0x80000008) {
-			do_cpuid(0x80000008, regs);
-			cpu_procinfo2 = regs[2];
-		}
-	}
-
 	if (strcmp(cpu_vendor, "GenuineIntel") == 0) {
 		if ((cpu_id & 0xf00) > 0x300) {
 			u_int brand_index;
@@ -1104,7 +1090,20 @@ finishidentcpu(void)
 	u_char	ccr3;
 	u_int	regs[4];
 
-	if (strcmp(cpu_vendor, "CyrixInstead") == 0) {
+	/* Detect AMD features (PTE no-execute bit, 3dnow, 64 bit mode etc) */
+	if (strcmp(cpu_vendor, "GenuineIntel") == 0 ||
+	    strcmp(cpu_vendor, "AuthenticAMD") == 0) {
+		init_exthigh();
+		if (cpu_exthigh >= 0x80000001) {
+			do_cpuid(0x80000001, regs);
+			amd_feature = regs[3] & ~(cpu_feature & 0x0183f3ff);
+			amd_feature2 = regs[2];
+		}
+		if (cpu_exthigh >= 0x80000008) {
+			do_cpuid(0x80000008, regs);
+			cpu_procinfo2 = regs[2];
+		}
+	} else if (strcmp(cpu_vendor, "CyrixInstead") == 0) {
 		if (cpu == CPU_486) {
 			/*
 			 * These conditions are equivalent to:
diff --git a/sys/i386/i386/initcpu.c b/sys/i386/i386/initcpu.c
index 2c1ee52..d6ef32a 100644
--- a/sys/i386/i386/initcpu.c
+++ b/sys/i386/i386/initcpu.c
@@ -41,6 +41,9 @@ __FBSDID("$FreeBSD$");
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
 
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
 #if !defined(CPU_DISABLE_SSE) && defined(I686_CPU)
 #define CPU_ENABLE_SSE
 #endif
@@ -686,6 +689,15 @@ initializecpu(void)
 				break;
 			}
 		}
+#ifdef PAE
+		if ((amd_feature & AMDID_NX) != 0) {
+			uint64_t msr;
+
+			msr = rdmsr(MSR_EFER) | EFER_NXE;
+			wrmsr(MSR_EFER, msr);
+			pg_nx = PG_NX;
+		}
+#endif
 		break;
 #endif
 	default:
diff --git a/sys/i386/i386/minidump_machdep.c b/sys/i386/i386/minidump_machdep.c
index be97479..63402ed 100644
--- a/sys/i386/i386/minidump_machdep.c
+++ b/sys/i386/i386/minidump_machdep.c
@@ -211,7 +211,7 @@ minidumpsys(struct dumperinfo *di)
 		j = va >> PDRSHIFT;
 		if ((pd[j] & (PG_PS | PG_V)) == (PG_PS | PG_V))  {
 			/* This is an entire 2M page. */
-			pa = pd[j] & PG_FRAME & ~PDRMASK;
+			pa = pd[j] & PG_PS_FRAME;
 			for (k = 0; k < NPTEPG; k++) {
 				if (is_dumpable(pa))
 					dump_add_page(pa);
@@ -310,7 +310,7 @@ minidumpsys(struct dumperinfo *di)
 		j = va >> PDRSHIFT;
 		if ((pd[j] & (PG_PS | PG_V)) == (PG_PS | PG_V))  {
 			/* This is a single 2M block. Generate a fake PTP */
-			pa = pd[j] & PG_FRAME & ~PDRMASK;
+			pa = pd[j] & PG_PS_FRAME;
 			for (k = 0; k < NPTEPG; k++) {
 				fakept[k] = (pa + (k * PAGE_SIZE)) | PG_V | PG_RW | PG_A | PG_M;
 			}
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index a05d680..3f0c603 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -206,6 +206,7 @@ vm_offset_t kernel_vm_end;
 extern u_int32_t KERNend;
 
 #ifdef PAE
+pt_entry_t pg_nx;
 static uma_zone_t pdptzone;
 #endif
 
@@ -958,7 +959,7 @@ pmap_extract(pmap_t pmap, vm_offset_t va)
 	pde = pmap->pm_pdir[va >> PDRSHIFT];
 	if (pde != 0) {
 		if ((pde & PG_PS) != 0) {
-			rtval = (pde & ~PDRMASK) | (va & PDRMASK);
+			rtval = (pde & PG_PS_FRAME) | (va & PDRMASK);
 			PMAP_UNLOCK(pmap);
 			return rtval;
 		}
@@ -991,7 +992,7 @@ pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
 	if (pde != 0) {
 		if (pde & PG_PS) {
 			if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) {
-				m = PHYS_TO_VM_PAGE((pde & ~PDRMASK) |
+				m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) |
 				    (va & PDRMASK));
 				vm_page_hold(m);
 			}
@@ -1365,7 +1366,7 @@ retry:
 	 * hold count, and activate it.
 	 */
 	if (ptepa) {
-		m = PHYS_TO_VM_PAGE(ptepa);
+		m = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
 		m->wire_count++;
 	} else {
 		/*
@@ -1498,7 +1499,8 @@ pmap_release(pmap_t pmap)
 	mtx_unlock_spin(&allpmaps_lock);
 
 	for (i = 0; i < NPGPTD; i++)
-		ptdpg[i] = PHYS_TO_VM_PAGE(pmap->pm_pdir[PTDPTDI + i]);
+		ptdpg[i] = PHYS_TO_VM_PAGE(pmap->pm_pdir[PTDPTDI + i] &
+		    PG_FRAME);
 
 	bzero(pmap->pm_pdir + PTDPTDI, (nkpt + NPGPTD) *
 	    sizeof(*pmap->pm_pdir));
@@ -1946,7 +1948,7 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va)
 		pmap_invalidate_page(kernel_pmap, va);
 	pmap->pm_stats.resident_count -= 1;
 	if (oldpte & PG_MANAGED) {
-		m = PHYS_TO_VM_PAGE(oldpte);
+		m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME);
 		if (oldpte & PG_M) {
 			KASSERT((oldpte & PG_RW),
 	("pmap_remove_pte: modified page not writable: va: %#x, pte: %#jx",
@@ -2154,8 +2156,14 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 		return;
 	}
 
+#ifdef PAE
+	if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) ==
+	    (VM_PROT_WRITE|VM_PROT_EXECUTE))
+		return;
+#else
 	if (prot & VM_PROT_WRITE)
 		return;
+#endif
 
 	anychanged = 0;
 
@@ -2163,7 +2171,8 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 	sched_pin();
 	PMAP_LOCK(pmap);
 	for (; sva < eva; sva = pdnxt) {
-		unsigned obits, pbits, pdirindex;
+		pt_entry_t obits, pbits;
+		unsigned pdirindex;
 
 		pdnxt = (sva + NBPDR) & ~PDRMASK;
 
@@ -2181,7 +2190,12 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot)
 		 * Check for large page.
 		 */
 		if ((ptpaddr & PG_PS) != 0) {
-			pmap->pm_pdir[pdirindex] &= ~(PG_M|PG_RW);
+			if ((prot & VM_PROT_WRITE) == 0)
+				pmap->pm_pdir[pdirindex] &= ~(PG_M|PG_RW);
+#ifdef PAE
+			if ((prot & VM_PROT_EXECUTE) == 0)
+				pmap->pm_pdir[pdirindex] |= pg_nx;
+#endif
 			anychanged = 1;
 			continue;
 		}
@@ -2199,27 +2213,39 @@ retry:
 			 * size, PG_RW, PG_A, and PG_M are among the least
 			 * significant 32 bits.
 			 */
-			obits = pbits = *(u_int *)pte;
+			obits = pbits = *pte;
+			if ((pbits & PG_V) == 0)
+				continue;
 			if (pbits & PG_MANAGED) {
 				m = NULL;
 				if (pbits & PG_A) {
-					m = PHYS_TO_VM_PAGE(*pte);
+					m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 					vm_page_flag_set(m, PG_REFERENCED);
 					pbits &= ~PG_A;
 				}
 				if ((pbits & PG_M) != 0) {
 					if (m == NULL)
-						m = PHYS_TO_VM_PAGE(*pte);
+						m = PHYS_TO_VM_PAGE(pbits & PG_FRAME);
 					vm_page_dirty(m);
 				}
 			}
 
-			pbits &= ~(PG_RW | PG_M);
+			if ((prot & VM_PROT_WRITE) == 0)
+				pbits &= ~(PG_RW | PG_M);
+#ifdef PAE
+			if ((prot & VM_PROT_EXECUTE) == 0)
+				pbits |= pg_nx;
+#endif
 
 			if (pbits != obits) {
+#ifdef PAE
+				if (!atomic_cmpset_64(pte, obits, pbits))
+					goto retry;
+#else
 				if (!atomic_cmpset_int((u_int *)pte, obits,
 				    pbits))
 					goto retry;
+#endif
 				if (obits & PG_G)
 					pmap_invalidate_page(pmap, sva);
 				else
@@ -2384,6 +2410,10 @@ validate:
 		newpte |= PG_RW;
 		vm_page_flag_set(m, PG_WRITEABLE);
 	}
+#ifdef PAE
+	if ((prot & VM_PROT_EXECUTE) == 0)
+		newpte |= pg_nx;
+#endif
 	if (wired)
 		newpte |= PG_W;
 	if (va < VM_MAXUSER_ADDRESS)
@@ -2404,6 +2434,11 @@ validate:
 					vm_page_flag_set(om, PG_REFERENCED);
 				if (opa != VM_PAGE_TO_PHYS(m))
 					invlva = TRUE;
+#ifdef PAE
+				if ((origpte & PG_NX) == 0 &&
+				    (newpte & PG_NX) != 0)
+					invlva = TRUE;
+#endif
 			}
 			if (origpte & PG_M) {
 				KASSERT((origpte & PG_RW),
@@ -2514,7 +2549,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 			if (ptepa) {
 				if (ptepa & PG_PS)
 					panic("pmap_enter_quick: unexpected mapping into 4MB page");
-				mpte = PHYS_TO_VM_PAGE(ptepa);
+				mpte = PHYS_TO_VM_PAGE(ptepa & PG_FRAME);
 				mpte->wire_count++;
 			} else {
 				mpte = _pmap_allocpte(pmap, ptepindex,
@@ -2560,6 +2595,10 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 	pmap->pm_stats.resident_count++;
 
 	pa = VM_PAGE_TO_PHYS(m);
+#ifdef PAE
+	if ((prot & VM_PROT_EXECUTE) == 0)
+		pa |= pg_nx;
+#endif
 
 	/*
 	 * Now validate mapping with RO protection
@@ -2746,7 +2785,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 			continue;
 		}
 
-		srcmpte = PHYS_TO_VM_PAGE(srcptepaddr);
+		srcmpte = PHYS_TO_VM_PAGE(srcptepaddr & PG_FRAME);
 		if (srcmpte->wire_count == 0)
 			panic("pmap_copy: source page table page is unused");
 
@@ -2990,7 +3029,7 @@ pmap_remove_pages(pmap_t pmap)
 					continue;
 				}
 
-				m = PHYS_TO_VM_PAGE(tpte);
+				m = PHYS_TO_VM_PAGE(tpte & PG_FRAME);
 				KASSERT(m->phys_addr == (tpte & PG_FRAME),
 				    ("vm_page_t %p phys_addr mismatch %016jx %016jx",
 				    m, (uintmax_t)m->phys_addr,
@@ -3521,7 +3560,7 @@ pmap_pid_dump(int pid)
 							pt_entry_t pa;
 							vm_page_t m;
 							pa = *pte;
-							m = PHYS_TO_VM_PAGE(pa);
+							m = PHYS_TO_VM_PAGE(pa & PG_FRAME);
 							printf("va: 0x%x, pt: 0x%x, h: %d, w: %d, f: 0x%x",
 								va, pa, m->hold_count, m->wire_count, m->flags);
 							npte++;
diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c
index d0eb30d..1bb2687 100644
--- a/sys/i386/i386/trap.c
+++ b/sys/i386/i386/trap.c
@@ -736,8 +736,16 @@ trap_pfault(frame, usermode, eva)
 		map = &vm->vm_map;
 	}
 
+	/*
+	 * PGEX_I is defined only if the execute disable bit capability is
+	 * supported and enabled.
+	 */
 	if (frame->tf_err & PGEX_W)
 		ftype = VM_PROT_WRITE;
+#ifdef PAE
+	else if ((frame->tf_err & PGEX_I) && pg_nx != 0)
+		ftype = VM_PROT_EXECUTE;
+#endif
 	else
 		ftype = VM_PROT_READ;
 
diff --git a/sys/i386/include/pmap.h b/sys/i386/include/pmap.h
index 95a0c42..775c29e 100644
--- a/sys/i386/include/pmap.h
+++ b/sys/i386/include/pmap.h
@@ -63,12 +63,21 @@
 #define	PG_AVAIL2	0x400	/*   <	programmers use		*/
 #define	PG_AVAIL3	0x800	/*    \				*/
 #define	PG_PDE_PAT	0x1000	/* PAT	PAT index		*/
+#ifdef PAE
+#define	PG_NX		(1ull<<63) /* No-execute */
+#endif
 
 
 /* Our various interpretations of the above */
 #define PG_W		PG_AVAIL1	/* "Wired" pseudoflag */
 #define	PG_MANAGED	PG_AVAIL2
-#define	PG_FRAME	(~((vm_paddr_t)PAGE_MASK))
+#ifdef PAE
+#define	PG_FRAME	(0x000ffffffffff000ull)
+#define	PG_PS_FRAME	(0x000fffffffe00000ull)
+#else
+#define	PG_FRAME	(~PAGE_MASK)
+#define	PG_PS_FRAME	(0xffc00000)
+#endif
 #define	PG_PROT		(PG_RW|PG_U)	/* all protection bits . */
 #define PG_N		(PG_NC_PWT|PG_NC_PCD)	/* Non-cacheable */
 
@@ -79,6 +88,7 @@
 #define PGEX_P		0x01	/* Protection violation vs. not present */
 #define PGEX_W		0x02	/* during a Write cycle */
 #define PGEX_U		0x04	/* access from User mode (UPL) */
+#define PGEX_I		0x10	/* during an instruction fetch */
 
 /*
  * Size of Kernel address space.  This is the number of page table pages
@@ -201,7 +211,7 @@ pmap_kextract(vm_offset_t va)
 	vm_paddr_t pa;
 
 	if ((pa = PTD[va >> PDRSHIFT]) & PG_PS) {
-		pa = (pa & ~(NBPDR - 1)) | (va & (NBPDR - 1));
+		pa = (pa & PG_PS_FRAME) | (va & PDRMASK);
 	} else {
 		pa = *vtopte(va);
 		pa = (pa & PG_FRAME) | (va & PAGE_MASK);
@@ -238,10 +248,33 @@ pte_load_store(pt_entry_t *ptep, pt_entry_t v)
 	return (r);
 }
 
+/* XXXRU move to atomic.h? */
+static __inline int
+atomic_cmpset_64(volatile uint64_t *dst, uint64_t exp, uint64_t src)
+{
+	int64_t res = exp;
+
+	__asm __volatile (
+	"	lock ;			"
+	"	cmpxchg8b %2 ;		"
+	"	setz	%%al ;		"
+	"	movzbl	%%al,%0 ;	"
+	"# atomic_cmpset_64"
+	: "+A" (res),			/* 0 (result) */
+	  "=m" (*dst)			/* 1 */
+	: "m" (*dst),			/* 2 */
+	  "b" ((uint32_t)src),
+	  "c" ((uint32_t)(src >> 32)));
+
+	return (res);
+}
+
 #define	pte_load_clear(ptep)	pte_load_store((ptep), (pt_entry_t)0ULL)
 
 #define	pte_store(ptep, pte)	pte_load_store((ptep), (pt_entry_t)pte)
 
+extern pt_entry_t pg_nx;
+
 #else /* PAE */
 
 static __inline pt_entry_t
diff --git a/sys/i386/include/specialreg.h b/sys/i386/include/specialreg.h
index f89760f..9fa01ee 100644
--- a/sys/i386/include/specialreg.h
+++ b/sys/i386/include/specialreg.h
@@ -68,6 +68,11 @@
 #define	CR4_XMM	0x00000400	/* enable SIMD/MMX2 to use except 16 */
 
 /*
+ * Bits in AMD64 special registers.  EFER is 64 bits wide.
+ */
+#define	EFER_NXE 0x000000800	/* PTE No-Execute bit enable (R/W) */
+
+/*
  * CPUID instruction features register
  */
 #define	CPUID_FPU	0x00000001
@@ -420,6 +425,9 @@
 #define	AMD_WT_ALLOC_PRE	0x20000	/* programmable range enable */
 #define	AMD_WT_ALLOC_FRE	0x10000	/* fixed (A0000-FFFFF) range enable */
 
+/* AMD64 MSR's */
+#define	MSR_EFER	0xc0000080	/* extended features */
+
 /* VIA ACE crypto featureset: for via_feature_rng */
 #define	VIA_HAS_RNG		1	/* cpu has RNG */
 
diff --git a/sys/i386/include/vmparam.h b/sys/i386/include/vmparam.h
index a94408a..53c3f45 100644
--- a/sys/i386/include/vmparam.h
+++ b/sys/i386/include/vmparam.h
@@ -43,7 +43,9 @@
  * Machine dependent constants for 386.
  */
 
+#ifndef PAE
 #define VM_PROT_READ_IS_EXEC	/* if you can read -- then you can exec */
+#endif
 
 /*
  * Virtual memory related constants, all in bytes