- Reimplement {d,i}tlb_enter() and {d,i}tlb_va_to_pa() in C. There's

  no particular reason for them to be implemented in assembler and
  having them in C allows easier extension as well as using more C
  macros and {d,i}tlb_slot_max rather than hard-coding magic (and
  actually spitfire-only) values.
- Fix the compilation of pmap_print_tte().
- Change pmap_print_tlb() to use ldxa() rather than re-rolling it
  inline as well as TLB_DAR_SLOT and {d,i}tlb_slot_max rather than
  hardcoding magic (and actually spitfire-only) values.
- While at it, suffix the above mentioned functions with "_sun4u" to
  underline they're architecture-specific.
- Use __FBSDID and macros instead of magic values in locore.S.
- Remove unused includes and smp_stack in locore.S.

4 files changed, 118 insertions, 123 deletions

diff --git a/sys/boot/sparc64/loader/locore.S b/sys/boot/sparc64/loader/locore.S
index 5eb6935..a73f5bf 100644
--- a/sys/boot/sparc64/loader/locore.S
+++ b/sys/boot/sparc64/loader/locore.S
@@ -4,35 +4,34 @@
  * All rights reserved.
  *
  * As long as the above copyright statement and this notice remain
- * unchanged, you can do what ever you want with this file. 
- *
- * $FreeBSD$
+ * unchanged, you can do what ever you want with this file.
  */
 
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
 #define	LOCORE
 
-#include <machine/asi.h>
-#include <machine/asm.h>
+#include <machine/frame.h>
+#include <machine/fsr.h>
+#include <machine/intr_machdep.h>
 #include <machine/pstate.h>
-#include <machine/smp.h>
-#include <machine/upa.h>
 
 #define	PAGE_SIZE	8192
 #define	PAGE_SHIFT	13
 
-#define	SPOFF		2047
 #define	STACK_SIZE	(2 * PAGE_SIZE)
 
 ENTRY(_start)
-	/* limit interrupts */
-	wrpr	%g0, 13, %pil
+	/* Limit interrupts. */
+	wrpr	%g0, PIL_TICK - 1, %pil
 
 	/*
 	 * PSTATE: privileged, interrupts enabled, floating point
 	 * unit enabled
 	 */
-	wrpr	%g0, PSTATE_PRIV|PSTATE_IE|PSTATE_PEF, %pstate
-	wr	%g0, 0x4, %fprs
+	wrpr	%g0, PSTATE_PRIV | PSTATE_IE | PSTATE_PEF, %pstate
+	wr	%g0, FPRS_FEF, %fprs
 
 	setx	stack + STACK_SIZE - SPOFF - CCFSZ, %l7, %l6
 	mov	%l6, %sp
@@ -40,74 +39,4 @@ ENTRY(_start)
 	 mov	%o4, %o0
 	sir
 
-/*
- * %o0 input VA constant
- * %o1 current iTLB offset
- * %o2 current iTLB TTE tag
- */
-ENTRY(itlb_va_to_pa)
-	clr	%o1
-0:	ldxa	[%o1] ASI_ITLB_TAG_READ_REG, %o2
-	cmp	%o2, %o0
-	bne,a	%xcc, 1f
-	 nop
-	/* return PA of matching entry */
-	ldxa	[%o1] ASI_ITLB_DATA_ACCESS_REG, %o0
-	sllx	%o0, 23, %o0
-	srlx	%o0, PAGE_SHIFT+23, %o0
-	sllx	%o0, PAGE_SHIFT, %o0
-	retl
-	 mov	%o0, %o1
-1:	cmp	%o1, 63<<3
-	blu	%xcc, 0b
-	 add	%o1, 8, %o1
-	clr	%o0
-	retl
-	 not	%o0
-
-ENTRY(dtlb_va_to_pa)
-	clr	%o1
-0:	ldxa	[%o1] ASI_DTLB_TAG_READ_REG, %o2
-	cmp	%o2, %o0
-	bne,a	%xcc, 1f
-	 nop
-	/* return PA of matching entry */
-	ldxa	[%o1] ASI_DTLB_DATA_ACCESS_REG, %o0
-	sllx	%o0, 23, %o0
-	srlx	%o0, PAGE_SHIFT+23, %o0
-	sllx	%o0, PAGE_SHIFT, %o0
-	retl
-	 mov	%o0, %o1
-1:	cmp	%o1, 63<<3
-	blu	%xcc, 0b
-	 add	%o1, 8, %o1
-	clr	%o0
-	retl
-	 not	%o0
-
-/*
- * %o0 = vpn
- * %o1 = tte data
- */
-ENTRY(itlb_enter)
-	rdpr	%pstate, %o4
-	wrpr	%o4, PSTATE_IE, %pstate
-	mov	AA_IMMU_TAR, %o3
-	stxa	%o0, [%o3] ASI_IMMU
-	stxa	%o1, [%g0] ASI_ITLB_DATA_IN_REG
-	membar	#Sync
-	retl
-	 wrpr	%o4, 0, %pstate
-
-ENTRY(dtlb_enter)
-	rdpr	%pstate, %o4
-	wrpr	%o4, PSTATE_IE, %pstate
-	mov	AA_DMMU_TAR, %o3
-	stxa	%o0, [%o3] ASI_DMMU
-	stxa	%o1, [%g0] ASI_DTLB_DATA_IN_REG
-	membar	#Sync
-	retl
-	 wrpr	%o4, 0, %pstate
-
 	.comm	stack, STACK_SIZE, 32
-	.comm	smp_stack, STACK_SIZE, 32
diff --git a/sys/boot/sparc64/loader/main.c b/sys/boot/sparc64/loader/main.c
index 39d9602..7a85dc2 100644
--- a/sys/boot/sparc64/loader/main.c
+++ b/sys/boot/sparc64/loader/main.c
@@ -4,7 +4,7 @@
  * All rights reserved.
  *
  * As long as the above copyright statement and this notice remain
- * unchanged, you can do what ever you want with this file. 
+ * unchanged, you can do what ever you want with this file.
  */
 
 #include <sys/cdefs.h>
@@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/lsu.h>
 #include <machine/metadata.h>
 #include <machine/tte.h>
+#include <machine/tlb.h>
 #include <machine/upa.h>
 
 #include "bootstrap.h"
@@ -56,10 +57,10 @@ static struct mmu_ops {
 typedef void kernel_entry_t(vm_offset_t mdp, u_long o1, u_long o2, u_long o3,
     void *openfirmware);
 
-extern void itlb_enter(u_long vpn, u_long data);
-extern void dtlb_enter(u_long vpn, u_long data);
-extern vm_offset_t itlb_va_to_pa(vm_offset_t);
-extern vm_offset_t dtlb_va_to_pa(vm_offset_t);
+static void dtlb_enter_sun4u(u_long vpn, u_long data);
+static vm_offset_t dtlb_va_to_pa_sun4u(vm_offset_t);
+static void itlb_enter_sun4u(u_long vpn, u_long data);
+static vm_offset_t itlb_va_to_pa_sun4u(vm_offset_t);
 extern vm_offset_t md_load(char *, vm_offset_t *);
 static int sparc64_autoload(void);
 static ssize_t sparc64_readin(const int, vm_offset_t, const size_t);
@@ -76,6 +77,13 @@ static vm_offset_t init_heap(void);
 static void tlb_init_sun4u(void);
 static void tlb_init_sun4v(void);
 
+#ifdef LOADER_DEBUG
+typedef u_int64_t tte_t;
+
+static void pmap_print_tlb_sun4u(void);
+static void pmap_print_tte_sun4u(tte_t, tte_t);
+#endif
+
 static struct mmu_ops mmu_ops_sun4u = { tlb_init_sun4u, mmu_mapin_sun4u };
 static struct mmu_ops mmu_ops_sun4v = { tlb_init_sun4v, mmu_mapin_sun4v };
 
@@ -344,9 +352,8 @@ __elfN(exec)(struct preloaded_file *fp)
 		return (error);
 
 	printf("jumping to kernel entry at %#lx.\n", e->e_entry);
-#if 0
-	pmap_print_tlb('i');
-	pmap_print_tlb('d');
+#if LOADER_DEBUG
+	pmap_print_tlb_sun4u();
 #endif
 
 	entry = e->e_entry;
@@ -358,6 +365,64 @@ __elfN(exec)(struct preloaded_file *fp)
 	panic("%s: exec returned", __func__);
 }
 
+static vm_offset_t
+dtlb_va_to_pa_sun4u(vm_offset_t va)
+{
+	u_long reg;
+	int i;
+
+	for (i = 0; i < dtlb_slot_max; i++) {
+		reg = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG);
+		if (TLB_TAR_VA(reg) != va)
+			continue;
+		reg = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG);
+		return ((reg & TD_PA_SF_MASK) >> TD_PA_SHIFT);
+	}
+	return (-1);
+}
+
+static vm_offset_t
+itlb_va_to_pa_sun4u(vm_offset_t va)
+{
+	u_long reg;
+	int i;
+
+	for (i = 0; i < itlb_slot_max; i++) {
+		reg = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG);
+		if (TLB_TAR_VA(reg) != va)
+			continue;
+		reg = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG);
+		return ((reg & TD_PA_SF_MASK) >> TD_PA_SHIFT);
+	}
+	return (-1);
+}
+
+static void
+itlb_enter_sun4u(u_long vpn, u_long data)
+{
+	u_long reg;
+
+	reg = rdpr(pstate);
+	wrpr(pstate, reg & ~PSTATE_IE, 0);
+	stxa(AA_IMMU_TAR, ASI_IMMU, vpn);
+	stxa(0, ASI_ITLB_DATA_IN_REG, data);
+	membar(Sync);
+	wrpr(pstate, reg, 0);
+}
+
+static void
+dtlb_enter_sun4u(u_long vpn, u_long data)
+{
+	u_long reg;
+
+	reg = rdpr(pstate);
+	wrpr(pstate, reg & ~PSTATE_IE, 0);
+	stxa(AA_DMMU_TAR, ASI_DMMU, vpn);
+	stxa(0, ASI_DTLB_DATA_IN_REG, data);
+	membar(Sync);
+	wrpr(pstate, reg, 0);
+}
+
 static int
 mmu_mapin_sun4u(vm_offset_t va, vm_size_t len)
 {
@@ -371,8 +436,8 @@ mmu_mapin_sun4u(vm_offset_t va, vm_size_t len)
 	len += va & PAGE_MASK_4M;
 	va &= ~PAGE_MASK_4M;
 	while (len) {
-		if (dtlb_va_to_pa(va) == (vm_offset_t)-1 ||
-		    itlb_va_to_pa(va) == (vm_offset_t)-1) {
+		if (dtlb_va_to_pa_sun4u(va) == (vm_offset_t)-1 ||
+		    itlb_va_to_pa_sun4u(va) == (vm_offset_t)-1) {
 			/* Allocate a physical page, claim the virtual area */
 			if (pa == (vm_offset_t)-1) {
 				pa = alloc_phys(PAGE_SIZE_4M, PAGE_SIZE_4M);
@@ -402,8 +467,8 @@ mmu_mapin_sun4u(vm_offset_t va, vm_size_t len)
 			itlb_store[itlb_slot].te_va = va;
 			dtlb_slot++;
 			itlb_slot++;
-			dtlb_enter(va, data);
-			itlb_enter(va, data);
+			dtlb_enter_sun4u(va, data);
+			itlb_enter_sun4u(va, data);
 			pa = (vm_offset_t)-1;
 		}
 		len -= len > PAGE_SIZE_4M ? PAGE_SIZE_4M : len;
@@ -617,14 +682,12 @@ exit(int code)
 }
 
 #ifdef LOADER_DEBUG
-typedef u_int64_t tte_t;
-
 static const char *page_sizes[] = {
 	"  8k", " 64k", "512k", "  4m"
 };
 
 static void
-pmap_print_tte(tte_t tag, tte_t tte)
+pmap_print_tte_sun4u(tte_t tag, tte_t tte)
 {
 
 	printf("%s %s ",
@@ -638,36 +701,31 @@ pmap_print_tte(tte_t tag, tte_t tte)
 	printf(tte & TD_L ? "\e[32mL\e[0m " : "  ");
 	printf(tte & TD_IE ? "IE " : "   ");
 	printf(tte & TD_NFO ? "NFO " : "    ");
-	printf("tag=0x%lx pa=0x%lx va=0x%lx ctx=%ld\n", tag, TD_PA(tte),
-	    TT_VA(tag), TT_CTX(tag));
+	printf("pa=0x%lx va=0x%lx ctx=%ld\n",
+	    TD_PA(tte), TLB_TAR_VA(tag), TLB_TAR_CTX(tag));
 }
-void
-pmap_print_tlb(char which)
+
+static void
+pmap_print_tlb_sun4u(void)
 {
+	tte_t tag, tte;
 	int i;
-	tte_t tte, tag;
-
-	for (i = 0; i < 64*8; i += 8) {
-		if (which == 'i') {
-			__asm__ __volatile__("ldxa	[%1] %2, %0\n" :
-			    "=r" (tag) : "r" (i),
-			    "i" (ASI_ITLB_TAG_READ_REG));
-			__asm__ __volatile__("ldxa	[%1] %2, %0\n" :
-			    "=r" (tte) : "r" (i),
-			    "i" (ASI_ITLB_DATA_ACCESS_REG));
-		}
-		else {
-			__asm__ __volatile__("ldxa	[%1] %2, %0\n" :
-			    "=r" (tag) : "r" (i),
-			    "i" (ASI_DTLB_TAG_READ_REG));
-			__asm__ __volatile__("ldxa	[%1] %2, %0\n" :
-			    "=r" (tte) : "r" (i),
-			    "i" (ASI_DTLB_DATA_ACCESS_REG));
-		}
+
+	for (i = 0; i < itlb_slot_max; i++) {
+		tte = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG);
+		if (!(tte & TD_V))
+			continue;
+		tag = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG);
+		printf("iTLB-%2u: ", i);
+		pmap_print_tte_sun4u(tag, tte);
+	}
+	for (i = 0; i < dtlb_slot_max; i++) {
+		tte = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG);
 		if (!(tte & TD_V))
 			continue;
-		printf("%cTLB-%2u: ", which, i>>3);
-		pmap_print_tte(tag, tte);
+		tag = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG);
+		printf("dTLB-%2u: ", i);
+		pmap_print_tte_sun4u(tag, tte);
 	}
 }
 #endif
diff --git a/sys/sparc64/include/tte.h b/sys/sparc64/include/tte.h
index 70337fe..72543fa 100644
--- a/sys/sparc64/include/tte.h
+++ b/sys/sparc64/include/tte.h
@@ -43,12 +43,16 @@
 #define	TD_SIZE_BITS	(2)
 #define	TD_SOFT2_BITS	(9)
 #define	TD_DIAG_BITS	(9)
-#define	TD_PA_BITS	(28)
+#define	TD_PA_CH_BITS	(30)
+#define	TD_PA_SF_BITS	(28)
+#define	TD_PA_BITS	TD_PA_SF_BITS
 #define	TD_SOFT_BITS	(6)
 
 #define	TD_SIZE_MASK	((1UL << TD_SIZE_BITS) - 1)
 #define	TD_SOFT2_MASK	((1UL << TD_SOFT2_BITS) - 1)
 #define	TD_DIAG_MASK	((1UL << TD_DIAG_BITS) - 1)
+#define	TD_PA_CH_MASK	((1UL << TD_PA_CH_BITS) - 1)
+#define	TD_PA_SF_MASK	((1UL << TD_PA_SF_BITS) - 1)
 #define	TD_PA_MASK	((1UL << TD_PA_BITS) - 1)
 #define	TD_SOFT_MASK	((1UL << TD_SOFT_BITS) - 1)
 
diff --git a/sys/sun4v/include/tte.h b/sys/sun4v/include/tte.h
index 4cdcd0d..9b19027 100644
--- a/sys/sun4v/include/tte.h
+++ b/sys/sun4v/include/tte.h
@@ -41,11 +41,15 @@
 
 #define	TD_SOFT2_BITS	(9)
 #define	TD_DIAG_BITS	(9)
+#define	TD_PA_CH_BITS	(30)
+#define	TD_PA_SF_BITS	(28)
 #define	TD_PA_BITS	(42)
 #define	TD_SOFT_BITS	(6)
 
 #define	TD_SOFT2_MASK	((1UL << TD_SOFT2_BITS) - 1)
 #define	TD_DIAG_MASK	((1UL << TD_DIAG_BITS) - 1)
+#define	TD_PA_CH_MASK	((1UL << TD_PA_CH_BITS) - 1)
+#define	TD_PA_SF_MASK	((1UL << TD_PA_SF_BITS) - 1)
 #define	TD_PA_MASK	((1UL << TD_PA_BITS) - 1)
 #define	TD_SOFT_MASK	((1UL << TD_SOFT_BITS) - 1)