- Do as the comment in pmap_bootstrap() suggests and flush all non-locked

  TLB entries possibly left over by the firmware and also do so while
  bootstrapping APs.
- Use __FBSDID.

MFC after:	1 month

7 files changed, 85 insertions, 19 deletions

diff --git a/sys/sparc64/include/tlb.h b/sys/sparc64/include/tlb.h
index 235a186..3d54ef8 100644
--- a/sys/sparc64/include/tlb.h
+++ b/sys/sparc64/include/tlb.h
@@ -59,6 +59,8 @@
 #define	TLB_DEMAP_TYPE_SHIFT		(6)
 #define	TLB_DEMAP_TYPE_PAGE		(0)
 #define	TLB_DEMAP_TYPE_CONTEXT		(1)
+/* US-III and greater only */
+#define	TLB_DEMAP_TYPE_ALL		(2)
 
 #define	TLB_DEMAP_VA(va)		((va) & ~PAGE_MASK)
 #define	TLB_DEMAP_ID(id)		((id) << TLB_DEMAP_ID_SHIFT)
@@ -66,6 +68,7 @@
 
 #define	TLB_DEMAP_PAGE			(TLB_DEMAP_TYPE(TLB_DEMAP_TYPE_PAGE))
 #define	TLB_DEMAP_CONTEXT		(TLB_DEMAP_TYPE(TLB_DEMAP_TYPE_CONTEXT))
+#define	TLB_DEMAP_ALL			(TLB_DEMAP_TYPE(TLB_DEMAP_TYPE_ALL))
 
 #define	TLB_DEMAP_PRIMARY		(TLB_DEMAP_ID(TLB_DEMAP_ID_PRIMARY))
 #define	TLB_DEMAP_SECONDARY		(TLB_DEMAP_ID(TLB_DEMAP_ID_SECONDARY))
@@ -93,6 +96,7 @@
 #define	MMU_SFSR_W			(1UL << MMU_SFSR_W_SHIFT)
 #define	MMU_SFSR_FV			(1UL << MMU_SFSR_FV_SHIFT)
 
+typedef void tlb_flush_nonlocked_t(void);
 typedef void tlb_flush_user_t(void);
 
 struct pmap;
@@ -105,9 +109,13 @@ void	tlb_context_demap(struct pmap *pm);
 void	tlb_page_demap(struct pmap *pm, vm_offset_t va);
 void	tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end);
 
+tlb_flush_nonlocked_t cheetah_tlb_flush_nonlocked;
 tlb_flush_user_t cheetah_tlb_flush_user;
+
+tlb_flush_nonlocked_t spitfire_tlb_flush_nonlocked;
 tlb_flush_user_t spitfire_tlb_flush_user;
 
+extern tlb_flush_nonlocked_t *tlb_flush_nonlocked;
 extern tlb_flush_user_t *tlb_flush_user;
 
 #endif /* !_MACHINE_TLB_H_ */
diff --git a/sys/sparc64/sparc64/cache.c b/sys/sparc64/sparc64/cache.c
index 48d324a..9e1309a 100644
--- a/sys/sparc64/sparc64/cache.c
+++ b/sys/sparc64/sparc64/cache.c
@@ -67,10 +67,11 @@
  *
  *	from: @(#)cache.c	8.2 (Berkeley) 10/30/93
  *	from: NetBSD: cache.c,v 1.5 2000/12/06 01:47:50 mrg Exp
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include <sys/param.h>
 #include <sys/systm.h>
 
@@ -127,12 +128,14 @@ cache_init(phandle_t node)
 		cache_flush = cheetah_cache_flush;
 		dcache_page_inval = cheetah_dcache_page_inval;
 		icache_page_inval = cheetah_icache_page_inval;
+		tlb_flush_nonlocked = cheetah_tlb_flush_nonlocked;
 		tlb_flush_user = cheetah_tlb_flush_user;
 	} else {
 		cache_enable = spitfire_cache_enable;
 		cache_flush = spitfire_cache_flush;
 		dcache_page_inval = spitfire_dcache_page_inval;
 		icache_page_inval = spitfire_icache_page_inval;
+		tlb_flush_nonlocked = spitfire_tlb_flush_nonlocked;
 		tlb_flush_user = spitfire_tlb_flush_user;
 	}
 }
diff --git a/sys/sparc64/sparc64/cheetah.c b/sys/sparc64/sparc64/cheetah.c
index 14f099d..ddf3e82 100644
--- a/sys/sparc64/sparc64/cheetah.c
+++ b/sys/sparc64/sparc64/cheetah.c
@@ -22,10 +22,11 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "opt_pmap.h"
 
 #include <sys/param.h>
@@ -51,6 +52,7 @@
 void
 cheetah_cache_enable(void)
 {
+
 }
 
 /*
@@ -59,6 +61,7 @@ cheetah_cache_enable(void)
 void
 cheetah_cache_flush(void)
 {
+
 }
 
 /*
@@ -85,10 +88,25 @@ cheetah_dcache_page_inval(vm_paddr_t spa)
 void
 cheetah_icache_page_inval(vm_paddr_t pa)
 {
+
 }
 
+/*
+ * Flush all non-locked mappings from the TLB.
+ */
+void
+cheetah_tlb_flush_nonlocked(void)
+{
+
+	panic("cheetah_tlb_flush_nonlocked");
+}
+
+/*
+ * Flush all user mappings from the TLB.
+ */
 void
 cheetah_tlb_flush_user(void)
 {
+
 	panic("cheetah_tlb_flush_user");
 }
diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c
index 1eef281..165341d 100644
--- a/sys/sparc64/sparc64/mp_machdep.c
+++ b/sys/sparc64/sparc64/mp_machdep.c
@@ -193,7 +193,7 @@ struct cpu_group *
 cpu_topo(void)
 {
 
-	return smp_topo_none();
+	return (smp_topo_none());
 }
 
 static void
@@ -371,6 +371,11 @@ cpu_mp_bootstrap(struct pcpu *pc)
 
 	csa = &cpu_start_args;
 	pmap_map_tsb();
+	/*
+	 * Flush all non-locked TLB entries possibly left over by the
+	 * firmware.
+	 */
+	tlb_flush_nonlocked();
 	cpu_setregs(pc);
 	tick_start();
 
diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c
index dc71b80..ad4d249 100644
--- a/sys/sparc64/sparc64/pmap.c
+++ b/sys/sparc64/sparc64/pmap.c
@@ -39,9 +39,11 @@
  * SUCH DAMAGE.
  *
  *      from:   @(#)pmap.c      7.7 (Berkeley)  5/12/91
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 /*
  * Manages physical address maps.
  *
@@ -81,7 +83,7 @@
 
 #include <dev/ofw/openfirm.h>
 
-#include <vm/vm.h> 
+#include <vm/vm.h>
 #include <vm/vm_param.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_page.h>
@@ -385,7 +387,7 @@ pmap_bootstrap(vm_offset_t ekva)
 	PATCH(tl1_dmmu_miss_patch_2);
 	PATCH(tl1_dmmu_prot_patch_1);
 	PATCH(tl1_dmmu_prot_patch_2);
-	
+
 	/*
 	 * Enter fake 8k pages for the 4MB kernel pages, so that
 	 * pmap_kextract() will work for them.
@@ -510,7 +512,11 @@ pmap_bootstrap(vm_offset_t ekva)
 		pm->pm_context[i] = TLB_CTX_KERNEL;
 	pm->pm_active = ~0;
 
-	/* XXX flush all non-locked tlb entries */
+	/*
+	 * Flush all non-locked TLB entries possibly left over by the
+	 * firmware.
+	 */
+	tlb_flush_nonlocked();
 }
 
 void
@@ -734,7 +740,7 @@ pmap_cache_enter(vm_page_t m, vm_offset_t va)
 
 	/*
 	 * If the mapping is already non-cacheable, just return.
-	 */	
+	 */
 	if (m->md.color == -1) {
 		PMAP_STATS_INC(pmap_ncache_enter_nc);
 		return (0);
@@ -1932,6 +1938,7 @@ pmap_remove_write(vm_page_t m)
 int
 pmap_mincore(pmap_t pm, vm_offset_t addr)
 {
+
 	/* TODO; */
 	return (0);
 }
diff --git a/sys/sparc64/sparc64/spitfire.c b/sys/sparc64/sparc64/spitfire.c
index abbcdae..7be988b 100644
--- a/sys/sparc64/sparc64/spitfire.c
+++ b/sys/sparc64/sparc64/spitfire.c
@@ -22,10 +22,11 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "opt_pmap.h"
 
 #include <sys/param.h>
@@ -139,7 +140,27 @@ spitfire_icache_page_inval(vm_paddr_t pa)
 }
 
 /*
- * Flush all user mappings from the tlb.
+ * Flush all non-locked mappings from the TLB.
+ */
+void
+spitfire_tlb_flush_nonlocked(void)
+{
+	int i;
+
+	for (i = 0; i < SPITFIRE_TLB_ENTRIES; i++) {
+		if ((ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG) &
+		    TD_L) == 0)
+			stxa_sync(TLB_DAR_SLOT(i),
+			    ASI_DTLB_DATA_ACCESS_REG, 0);
+		if ((ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG) &
+		    TD_L) == 0)
+			stxa_sync(TLB_DAR_SLOT(i),
+			    ASI_ITLB_DATA_ACCESS_REG, 0);
+	}
+}
+
+/*
+ * Flush all user mappings from the TLB.
  */
 void
 spitfire_tlb_flush_user(void)
@@ -153,11 +174,13 @@ spitfire_tlb_flush_user(void)
 		tag = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG);
 		if ((data & TD_V) != 0 && (data & TD_L) == 0 &&
 		    TLB_TAR_CTX(tag) != TLB_CTX_KERNEL)
-			stxa_sync(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG, 0);
+			stxa_sync(TLB_DAR_SLOT(i),
+			    ASI_DTLB_DATA_ACCESS_REG, 0);
 		data = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG);
 		tag = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG);
 		if ((data & TD_V) != 0 && (data & TD_L) == 0 &&
 		    TLB_TAR_CTX(tag) != TLB_CTX_KERNEL)
-			stxa_sync(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG, 0);
+			stxa_sync(TLB_DAR_SLOT(i),
+			    ASI_ITLB_DATA_ACCESS_REG, 0);
 	}
 }
diff --git a/sys/sparc64/sparc64/tlb.c b/sys/sparc64/sparc64/tlb.c
index 2226e02..56edc87 100644
--- a/sys/sparc64/sparc64/tlb.c
+++ b/sys/sparc64/sparc64/tlb.c
@@ -22,10 +22,11 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
- *
- * $FreeBSD$
  */
 
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
 #include "opt_pmap.h"
 
 #include <sys/param.h>
@@ -49,6 +50,7 @@ PMAP_STATS_VAR(tlb_ncontext_demap);
 PMAP_STATS_VAR(tlb_npage_demap);
 PMAP_STATS_VAR(tlb_nrange_demap);
 
+tlb_flush_nonlocked_t *tlb_flush_nonlocked;
 tlb_flush_user_t *tlb_flush_user;
 
 /*
@@ -105,7 +107,7 @@ tlb_page_demap(struct pmap *pm, vm_offset_t va)
 			flags = TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE;
 		else
 			flags = TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE;
-	
+
 		s = intr_disable();
 		stxa(TLB_DEMAP_VA(va) | flags, ASI_DMMU_DEMAP, 0);
 		stxa(TLB_DEMAP_VA(va) | flags, ASI_IMMU_DEMAP, 0);
@@ -132,7 +134,7 @@ tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end)
 			flags = TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE;
 		else
 			flags = TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE;
-	
+
 		s = intr_disable();
 		for (va = start; va < end; va += PAGE_SIZE) {
 			stxa(TLB_DEMAP_VA(va) | flags, ASI_DMMU_DEMAP, 0);