The variable MTRR registers actually have variable-sized PhysBase and

PhysMask fields based on the number of physical address bits supported
by the current CPU.  The old code assumed 36 bits on i386 and 40 bits on
amd64.  In truth, all Intel CPUs up until recently used 36 bits (a newer
Intel CPU uses 38 bits) and all the Opteron CPUs used 40 bits.

In at least one case (the new Intel CPU) having the size of the mask field
wrong resulted in writing questionable values into the MTRR registers on
the application processors (BSP as well if you modify the MTRRs via
memcontrol or running X, etc.).  The result of the questionable physmask
was that all of memory was apparently treated as uncached rather than
write-back resulting in a very significant performance hit.

Fix this by constructing a run-time mask for the PhysBase and PhysMask
fields based on the number of physical address bits supported by the CPU.
All 64-bit capable CPUs provide a count of PA bits supported via the
0x80000008 extended CPUID feature, so use that if it is available.  If that
feature is not available, then assume 36 PA bits.

While I'm here, expand the (now-unused) macros for the PhysBase and
PhysMask fields to the current largest possible value (52 PA bits).

MFC after:	1 week
PR:		i386/120516
Reported by:	Nokia

4 files changed, 52 insertions, 16 deletions

diff --git a/sys/amd64/amd64/amd64_mem.c b/sys/amd64/amd64/amd64_mem.c
index 5874496..36c4045 100644
--- a/sys/amd64/amd64/amd64_mem.c
+++ b/sys/amd64/amd64/amd64_mem.c
@@ -82,6 +82,9 @@ static struct mem_range_ops amd64_mrops = {
 /* XXX for AP startup hook */
 static u_int64_t mtrrcap, mtrrdef;
 
+/* The bitmask for the PhysBase and PhysMask fields of the variable MTRRs. */
+static u_int64_t mtrr_physmask;
+
 static struct mem_range_desc *mem_range_match(struct mem_range_softc *sc,
 		    struct mem_range_desc *mrd);
 static void	amd64_mrfetch(struct mem_range_softc *sc);
@@ -214,15 +217,15 @@ amd64_mrfetch(struct mem_range_softc *sc)
 		msrv = rdmsr(msr);
 		mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) |
 		    amd64_mtrr2mrt(msrv & MTRR_PHYSBASE_TYPE);
-		mrd->mr_base = msrv & MTRR_PHYSBASE_PHYSBASE;
+		mrd->mr_base = msrv & mtrr_physmask;
 		msrv = rdmsr(msr + 1);
 		mrd->mr_flags = (msrv & MTRR_PHYSMASK_VALID) ?
 		    (mrd->mr_flags | MDF_ACTIVE) :
 		    (mrd->mr_flags & ~MDF_ACTIVE);
 
 		/* Compute the range from the mask. Ick. */
-		mrd->mr_len = (~(msrv & MTRR_PHYSMASK_PHYSMASK)
-		    & (MTRR_PHYSMASK_PHYSMASK | 0xfffL)) + 1;
+		mrd->mr_len = (~(msrv & mtrr_physmask) &
+		    (mtrr_physmask | 0xfffL)) + 1;
 		if (!mrvalid(mrd->mr_base, mrd->mr_len))
 			mrd->mr_flags |= MDF_BOGUS;
 
@@ -361,7 +364,7 @@ amd64_mrstoreone(void *arg)
 		/* base/type register */
 		omsrv = rdmsr(msr);
 		if (mrd->mr_flags & MDF_ACTIVE) {
-			msrv = mrd->mr_base & MTRR_PHYSBASE_PHYSBASE;
+			msrv = mrd->mr_base & mtrr_physmask;
 			msrv |= amd64_mrt2mtrr(mrd->mr_flags, omsrv);
 		} else {
 			msrv = 0;
@@ -371,7 +374,7 @@ amd64_mrstoreone(void *arg)
 		/* mask/active register */
 		if (mrd->mr_flags & MDF_ACTIVE) {
 			msrv = MTRR_PHYSMASK_VALID |
-			    (~(mrd->mr_len - 1) & MTRR_PHYSMASK_PHYSMASK);
+			    (~(mrd->mr_len - 1) & mtrr_physmask);
 		} else {
 			msrv = 0;
 		}
@@ -578,7 +581,8 @@ static void
 amd64_mrinit(struct mem_range_softc *sc)
 {
 	struct mem_range_desc *mrd;
-	int i, nmdesc = 0;
+	u_int regs[4];
+	int i, nmdesc = 0, pabits;
 
 	mtrrcap = rdmsr(MSR_MTRRcap);
 	mtrrdef = rdmsr(MSR_MTRRdefType);
@@ -591,6 +595,20 @@ amd64_mrinit(struct mem_range_softc *sc)
 	}
 	nmdesc = mtrrcap & MTRR_CAP_VCNT;
 
+	/*
+	 * Determine the size of the PhysMask and PhysBase fields in
+	 * the variable range MTRRs.  If the extended CPUID 0x80000008
+	 * is present, use that to figure out how many physical
+	 * address bits the CPU supports.  Otherwise, default to 36
+	 * address bits.
+	 */
+	if (cpu_exthigh >= 0x80000008) {
+		do_cpuid(0x80000008, regs);
+		pabits = regs[0] & 0xff;
+	} else
+		pabits = 36;
+	mtrr_physmask = ((1UL << pabits) - 1) & ~0xfffUL;
+
 	/* If fixed MTRRs supported and enabled. */
 	if ((mtrrcap & MTRR_CAP_FIXED) && (mtrrdef & MTRR_DEF_FIXED_ENABLE)) {
 		sc->mr_cap = MR686_FIXMTRR;
diff --git a/sys/amd64/include/specialreg.h b/sys/amd64/include/specialreg.h
index 94d70ad..760d2c5 100644
--- a/sys/amd64/include/specialreg.h
+++ b/sys/amd64/include/specialreg.h
@@ -275,9 +275,9 @@
 #define	MTRR_DEF_ENABLE		0x0000000000000800UL
 #define	MTRR_DEF_FIXED_ENABLE	0x0000000000000400UL
 #define	MTRR_DEF_TYPE		0x00000000000000ffUL
-#define	MTRR_PHYSBASE_PHYSBASE	0x000000fffffff000UL
+#define	MTRR_PHYSBASE_PHYSBASE	0x000ffffffffff000UL
 #define	MTRR_PHYSBASE_TYPE	0x00000000000000ffUL
-#define	MTRR_PHYSMASK_PHYSMASK	0x000000fffffff000UL
+#define	MTRR_PHYSMASK_PHYSMASK	0x000ffffffffff000UL
 #define	MTRR_PHYSMASK_VALID	0x0000000000000800UL
 
 /* Performance Control Register (5x86 only). */
diff --git a/sys/i386/i386/i686_mem.c b/sys/i386/i386/i686_mem.c
index f1240b6..b06ce46 100644
--- a/sys/i386/i386/i686_mem.c
+++ b/sys/i386/i386/i686_mem.c
@@ -82,6 +82,9 @@ static struct mem_range_ops i686_mrops = {
 /* XXX for AP startup hook */
 static u_int64_t mtrrcap, mtrrdef;
 
+/* The bitmask for the PhysBase and PhysMask fields of the variable MTRRs. */
+static u_int64_t mtrr_physmask;
+
 static struct mem_range_desc *mem_range_match(struct mem_range_softc *sc,
 		    struct mem_range_desc *mrd);
 static void	i686_mrfetch(struct mem_range_softc *sc);
@@ -212,15 +215,15 @@ i686_mrfetch(struct mem_range_softc *sc)
 		msrv = rdmsr(msr);
 		mrd->mr_flags = (mrd->mr_flags & ~MDF_ATTRMASK) |
 		    i686_mtrr2mrt(msrv & MTRR_PHYSBASE_TYPE);
-		mrd->mr_base = msrv & MTRR_PHYSBASE_PHYSBASE;
+		mrd->mr_base = msrv & mtrr_physmask;
 		msrv = rdmsr(msr + 1);
 		mrd->mr_flags = (msrv & MTRR_PHYSMASK_VALID) ?
 		    (mrd->mr_flags | MDF_ACTIVE) :
 		    (mrd->mr_flags & ~MDF_ACTIVE);
 
 		/* Compute the range from the mask. Ick. */
-		mrd->mr_len = (~(msrv & MTRR_PHYSMASK_PHYSMASK) &
-		    (MTRR_PHYSMASK_PHYSMASK | 0xfffLL)) + 1;
+		mrd->mr_len = (~(msrv & mtrr_physmask) &
+		    (mtrr_physmask | 0xfffLL)) + 1;
 		if (!mrvalid(mrd->mr_base, mrd->mr_len))
 			mrd->mr_flags |= MDF_BOGUS;
 
@@ -359,7 +362,7 @@ i686_mrstoreone(void *arg)
 		/* base/type register */
 		omsrv = rdmsr(msr);
 		if (mrd->mr_flags & MDF_ACTIVE) {
-			msrv = mrd->mr_base & MTRR_PHYSBASE_PHYSBASE;
+			msrv = mrd->mr_base & mtrr_physmask;
 			msrv |= i686_mrt2mtrr(mrd->mr_flags, omsrv);
 		} else {
 			msrv = 0;
@@ -369,7 +372,7 @@ i686_mrstoreone(void *arg)
 		/* mask/active register */
 		if (mrd->mr_flags & MDF_ACTIVE) {
 			msrv = MTRR_PHYSMASK_VALID |
-			    (~(mrd->mr_len - 1) & MTRR_PHYSMASK_PHYSMASK);
+			    (~(mrd->mr_len - 1) & mtrr_physmask);
 		} else {
 			msrv = 0;
 		}
@@ -576,7 +579,8 @@ static void
 i686_mrinit(struct mem_range_softc *sc)
 {
 	struct mem_range_desc *mrd;
-	int i, nmdesc = 0;
+	u_int regs[4];
+	int i, nmdesc = 0, pabits;
 
 	mtrrcap = rdmsr(MSR_MTRRcap);
 	mtrrdef = rdmsr(MSR_MTRRdefType);
@@ -591,6 +595,20 @@ i686_mrinit(struct mem_range_softc *sc)
 	if (bootverbose)
 		printf("Pentium Pro MTRR support enabled\n");
 
+	/*
+	 * Determine the size of the PhysMask and PhysBase fields in
+	 * the variable range MTRRs.  If the extended CPUID 0x80000008
+	 * is present, use that to figure out how many physical
+	 * address bits the CPU supports.  Otherwise, default to 36
+	 * address bits.
+	 */
+	if (cpu_exthigh >= 0x80000008) {
+		do_cpuid(0x80000008, regs);
+		pabits = regs[0] & 0xff;
+	} else
+		pabits = 36;
+	mtrr_physmask = ((1ULL << pabits) - 1) & ~0xfffULL;
+
 	/* If fixed MTRRs supported and enabled. */
 	if ((mtrrcap & MTRR_CAP_FIXED) && (mtrrdef & MTRR_DEF_FIXED_ENABLE)) {
 		sc->mr_cap = MR686_FIXMTRR;
diff --git a/sys/i386/include/specialreg.h b/sys/i386/include/specialreg.h
index 4e1f5e3..07a74ad 100644
--- a/sys/i386/include/specialreg.h
+++ b/sys/i386/include/specialreg.h
@@ -266,9 +266,9 @@
 #define	MTRR_DEF_ENABLE		0x0000000000000800ULL
 #define	MTRR_DEF_FIXED_ENABLE	0x0000000000000400ULL
 #define	MTRR_DEF_TYPE		0x00000000000000ffULL
-#define	MTRR_PHYSBASE_PHYSBASE	0x0000000ffffff000ULL
+#define	MTRR_PHYSBASE_PHYSBASE	0x000ffffffffff000ULL
 #define	MTRR_PHYSBASE_TYPE	0x00000000000000ffULL
-#define	MTRR_PHYSMASK_PHYSMASK	0x0000000ffffff000ULL
+#define	MTRR_PHYSMASK_PHYSMASK	0x000ffffffffff000ULL
 #define	MTRR_PHYSMASK_VALID	0x0000000000000800ULL
 
 /*