Flesh out MMU and cache handling of cheetah-class CPUs.

6 files changed, 171 insertions, 7 deletions

diff --git a/sys/sparc64/include/cpu.h b/sys/sparc64/include/cpu.h
index ef48721..b593f13 100644
--- a/sys/sparc64/include/cpu.h
+++ b/sys/sparc64/include/cpu.h
@@ -52,6 +52,7 @@
 extern	char btext[];
 extern	char etext[];
 
+void	cheetah_init(void);
 void	cpu_halt(void);
 void	cpu_reset(void);
 void	fork_trampoline(void);
diff --git a/sys/sparc64/include/dcr.h b/sys/sparc64/include/dcr.h
new file mode 100644
index 0000000..a885d05
--- /dev/null
+++ b/sys/sparc64/include/dcr.h
@@ -0,0 +1,66 @@
+/*-
+ * Copyright (c) 2008 Marius Strobl <marius@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_MACHINE_DCR_H_
+#define	_MACHINE_DCR_H_
+
+/*
+ * Definitions for the UltraSPARC-III Depatch Control Register (ASR 18).
+ */
+#define	DCR_MS			(1UL << 0)
+#define	DCR_IFPOE		(1UL << 1)
+#define	DCR_SI			(1UL << 3)
+#define	DCR_RPE			(1UL << 4)
+#define	DCR_BPE			(1UL << 5)
+
+#define	DCR_OBSDATA_SHIFT	6
+#define	DCR_OBSDATA_CT_BITS	8
+#define	DCR_OBSDATA_CT_MASK						\
+	(((1UL << DCR_OBSDATA_CT_BITS) - 1) << DCR_OBSDATA_SHIFT)
+
+/* The following bits are valid for the UltraSPARC-III+/IV+ only. */
+#define	DCR_IPE			(1UL << 5)
+
+#define	DCR_OBSDATA_CTP_BITS	6
+#define	DCR_OBSDATA_CTP_MASK						\
+	(((1UL << DCR_OBSDATA_CTP_BITS) - 1) << DCR_OBSDATA_SHIFT)
+
+#define	DCR_DPE			(1UL << 12)
+
+/* The following bits are valid for the UltraSPARC-IV+ only. */
+#define	DCR_BPM_SHIFT		13
+#define	DCR_BPM_BITS		2
+#define	DCR_BPM_MASK							\
+	(((1UL << DCR_BPM_BITS) - 1) << DCR_BPM_SHIFT)
+
+#define	DCR_JPE			(1UL << 15)
+#define	DCR_ITPE		(1UL << 16)
+#define	DCR_DTPE		(1UL << 17)
+#define	DCR_PPE			(1UL << 18)
+
+#endif	/* _MACHINE_DCR_H_ */
diff --git a/sys/sparc64/sparc64/cheetah.c b/sys/sparc64/sparc64/cheetah.c
index f9e2332..bf42a44 100644
--- a/sys/sparc64/sparc64/cheetah.c
+++ b/sys/sparc64/sparc64/cheetah.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2003 Jake Burkholder.
+ * Copyright (c) 2005, 2008 Marius Strobl <marius@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,9 +40,60 @@ __FBSDID("$FreeBSD$");
 #include <vm/pmap.h>
 
 #include <machine/cache.h>
+#include <machine/cpu.h>
 #include <machine/cpufunc.h>
+#include <machine/dcr.h>
+#include <machine/lsu.h>
 #include <machine/smp.h>
 #include <machine/tlb.h>
+#include <machine/ver.h>
+#include <machine/vmparam.h>
+
+/* A FLUSH is required after changing LSU_IC (the address is ignored). */
+#define	CHEETAH_FLUSH_LSU_IC()	__asm __volatile("flush %%g0" : :)
+
+#define	CHEETAH_ICACHE_TAG_LOWER	0x30
+
+/*
+ * CPU-specific initialization
+ */
+void
+cheetah_init(void)
+{
+	register_t s;
+
+	/*
+	 * Disable interrupts for safety, this shouldn't be actually
+	 * necessary though.
+	 */
+	s = intr_disable();
+
+	/*
+	 * Ensure DCR_IFPOE is disabled as long as we haven't implemented
+	 * support for it (if ever) as most if not all firmware versions
+	 * apparently turn it on.  Not making use of DCR_IFPOE should also
+	 * avoid Cheetah erratum #109.
+	 */
+	wr(asr18, rd(asr18) & ~DCR_IFPOE, 0);
+
+	/* Ensure the TSB Extension Registers hold 0 as TSB_Base. */
+
+	stxa(AA_DMMU_TSB_PEXT_REG, ASI_DMMU, 0);
+	stxa(AA_IMMU_TSB_PEXT_REG, ASI_IMMU, 0);
+	membar(Sync);
+
+	stxa(AA_DMMU_TSB_SEXT_REG, ASI_DMMU, 0);
+	/*
+	 * NB: the secondary context was removed from the iMMU.
+	 */
+	membar(Sync);
+
+	stxa(AA_DMMU_TSB_NEXT_REG, ASI_DMMU, 0);
+	stxa(AA_IMMU_TSB_NEXT_REG, ASI_IMMU, 0);
+	membar(Sync);
+
+	intr_restore(s);
+}
 
 /*
  * Enable level 1 caches.
@@ -49,7 +101,15 @@ __FBSDID("$FreeBSD$");
 void
 cheetah_cache_enable(void)
 {
-
+	u_long lsu;
+
+	lsu = ldxa(0, ASI_LSU_CTL_REG);
+	if (cpu_impl == CPU_IMPL_ULTRASPARCIII) {
+		/* Disable P$ due to Cheetah erratum #18. */
+		lsu &= ~LSU_PE;
+	}
+	stxa(0, ASI_LSU_CTL_REG, lsu | LSU_IC | LSU_DC);
+	CHEETAH_FLUSH_LSU_IC();
 }
 
 /*
@@ -58,7 +118,22 @@ cheetah_cache_enable(void)
 void
 cheetah_cache_flush(void)
 {
-
+	u_long addr, lsu;
+
+	for (addr = 0; addr < PCPU_GET(cache.dc_size);
+	    addr += PCPU_GET(cache.dc_linesize))
+		stxa_sync(addr, ASI_DCACHE_TAG, 0);
+
+	/* The I$ must be disabled when flushing it so ensure it's off. */
+	lsu = ldxa(0, ASI_LSU_CTL_REG);
+	stxa(0, ASI_LSU_CTL_REG, lsu & ~(LSU_IC));
+	CHEETAH_FLUSH_LSU_IC();
+	for (addr = CHEETAH_ICACHE_TAG_LOWER;
+	    addr < PCPU_GET(cache.ic_size) * 2;
+	    addr += PCPU_GET(cache.ic_linesize) * 2)
+		stxa_sync(addr, ASI_ICACHE_TAG, 0);
+	stxa(0, ASI_LSU_CTL_REG, lsu);
+	CHEETAH_FLUSH_LSU_IC();
 }
 
 /*
@@ -87,6 +162,12 @@ cheetah_icache_page_inval(vm_paddr_t pa)
 
 }
 
+#define	cheetah_dmap_all() do {						\
+	stxa(TLB_DEMAP_ALL, ASI_DMMU_DEMAP, 0);				\
+	stxa(TLB_DEMAP_ALL, ASI_IMMU_DEMAP, 0);				\
+	flush(KERNBASE);						\
+} while (0)
+
 /*
  * Flush all non-locked mappings from the TLB.
  */
@@ -94,15 +175,20 @@ void
 cheetah_tlb_flush_nonlocked(void)
 {
 
-	panic("cheetah_tlb_flush_nonlocked");
+	cheetah_dmap_all();
 }
 
 /*
  * Flush all user mappings from the TLB.
  */
 void
-cheetah_tlb_flush_user(void)
+cheetah_tlb_flush_user()
 {
 
-	panic("cheetah_tlb_flush_user");
+	/*
+	 * Just use cheetah_dmap_all() and accept somes TLB misses
+	 * rather than searching all 1040 D-TLB and 144 I-TLB slots
+	 * for non-kernel mappings.
+	 */
+	cheetah_dmap_all();
 }
diff --git a/sys/sparc64/sparc64/genassym.c b/sys/sparc64/sparc64/genassym.c
index eb133e2..afd0bdb 100644
--- a/sys/sparc64/sparc64/genassym.c
+++ b/sys/sparc64/sparc64/genassym.c
@@ -66,10 +66,13 @@ ASSYM(PCPU_PAGES, PCPU_PAGES);
 
 ASSYM(TAR_VPN_SHIFT, TAR_VPN_SHIFT);
 
-ASSYM(TLB_DEMAP_NUCLEUS, TLB_DEMAP_NUCLEUS);
-ASSYM(TLB_DEMAP_PRIMARY, TLB_DEMAP_PRIMARY);
+#ifdef SUN4U
+ASSYM(TLB_DEMAP_ALL, TLB_DEMAP_ALL);
+#endif
 ASSYM(TLB_DEMAP_CONTEXT, TLB_DEMAP_CONTEXT);
+ASSYM(TLB_DEMAP_NUCLEUS, TLB_DEMAP_NUCLEUS);
 ASSYM(TLB_DEMAP_PAGE, TLB_DEMAP_PAGE);
+ASSYM(TLB_DEMAP_PRIMARY, TLB_DEMAP_PRIMARY);
 
 ASSYM(INT_SHIFT, INT_SHIFT);
 ASSYM(PTR_SHIFT, PTR_SHIFT);
diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c
index f0c5a16..382fbbf 100644
--- a/sys/sparc64/sparc64/machdep.c
+++ b/sys/sparc64/sparc64/machdep.c
@@ -255,6 +255,12 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
 	cpu_impl = VER_IMPL(rdpr(ver));
 
 	/*
+	 * Do CPU-specific Initialization.
+	 */
+	if (cpu_impl >= CPU_IMPL_ULTRASPARCIII)
+		cheetah_init();
+
+	/*
 	 * Clear (S)TICK timer (including NPT).
 	 */
 	tick_clear();
diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c
index e40b5ea..58921fc 100644
--- a/sys/sparc64/sparc64/mp_machdep.c
+++ b/sys/sparc64/sparc64/mp_machdep.c
@@ -384,6 +384,8 @@ cpu_mp_bootstrap(struct pcpu *pc)
 	volatile struct cpu_start_args *csa;
 
 	csa = &cpu_start_args;
+	if (cpu_impl >= CPU_IMPL_ULTRASPARCIII)
+		cheetah_init();
 	pmap_map_tsb();
 	/*
 	 * Flush all non-locked TLB entries possibly left over by the