Fix a bug in the data access error recorvery. Before re-enabling the data

cache after a data access error we must discard all cache lines.  When
disabled existing cache lines are not invalidated by stores to memory, so
we risk reading stale data that was cached before the data access error if
we don't flush them.  This is especially fatal when the memory involved
is the active part of the kernel or user stack.  For good measure we also
flush the instruction cache.

This fixes random crashes when the X server probes the PCI bus through
/dev/pci.

6 files changed, 60 insertions, 4 deletions

diff --git a/sys/sparc64/include/cache.h b/sys/sparc64/include/cache.h
index 94c65e5..7c97705 100644
--- a/sys/sparc64/include/cache.h
+++ b/sys/sparc64/include/cache.h
@@ -97,16 +97,25 @@ struct cacheinfo {
 
 #ifdef _KERNEL
 
+typedef void cache_enable_t(void);
+typedef void cache_flush_t(void);
 typedef void dcache_page_inval_t(vm_paddr_t pa);
 typedef void icache_page_inval_t(vm_paddr_t pa);
 
 void	cache_init(phandle_t node);
 
+cache_enable_t cheetah_cache_enable;
+cache_flush_t cheetah_cache_flush;
 dcache_page_inval_t cheetah_dcache_page_inval;
 icache_page_inval_t cheetah_icache_page_inval;
+
+cache_enable_t spitfire_cache_enable;
+cache_flush_t spitfire_cache_flush;
 dcache_page_inval_t spitfire_dcache_page_inval;
 icache_page_inval_t spitfire_icache_page_inval;
 
+extern cache_enable_t *cache_enable;
+extern cache_flush_t *cache_flush;
 extern dcache_page_inval_t *dcache_page_inval;
 extern icache_page_inval_t *icache_page_inval;
 
diff --git a/sys/sparc64/sparc64/cache.c b/sys/sparc64/sparc64/cache.c
index dad13f1..fc1fd56 100644
--- a/sys/sparc64/sparc64/cache.c
+++ b/sys/sparc64/sparc64/cache.c
@@ -86,6 +86,8 @@
 
 struct cacheinfo cache;
 
+cache_enable_t *cache_enable;
+cache_flush_t *cache_flush;
 dcache_page_inval_t *dcache_page_inval;
 icache_page_inval_t *icache_page_inval;
 
@@ -125,10 +127,14 @@ cache_init(phandle_t node)
 		panic("cache_init: E$ set size not a power of 2");
 
 	if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) {
+		cache_enable = cheetah_cache_enable;
+		cache_flush = cheetah_cache_flush;
 		dcache_page_inval = cheetah_dcache_page_inval;
 		icache_page_inval = cheetah_icache_page_inval;
 		tlb_flush_user = cheetah_tlb_flush_user;
 	} else {
+		cache_enable = spitfire_cache_enable;
+		cache_flush = spitfire_cache_flush;
 		dcache_page_inval = spitfire_dcache_page_inval;
 		icache_page_inval = spitfire_icache_page_inval;
 		tlb_flush_user = spitfire_tlb_flush_user;
diff --git a/sys/sparc64/sparc64/cheetah.c b/sys/sparc64/sparc64/cheetah.c
index 6dbbc06..14f099d 100644
--- a/sys/sparc64/sparc64/cheetah.c
+++ b/sys/sparc64/sparc64/cheetah.c
@@ -46,6 +46,22 @@
 #include <machine/tlb.h>
 
 /*
+ * Enable level 1 caches.
+ */
+void
+cheetah_cache_enable(void)
+{
+}
+
+/*
+ * Flush all lines from the level 1 caches.
+ */
+void
+cheetah_cache_flush(void)
+{
+}
+
+/*
  * Flush a physical page from the data cache.
  */
 void
diff --git a/sys/sparc64/sparc64/spitfire.c b/sys/sparc64/sparc64/spitfire.c
index 429dc9a..abbcdae 100644
--- a/sys/sparc64/sparc64/spitfire.c
+++ b/sys/sparc64/sparc64/spitfire.c
@@ -42,6 +42,7 @@
 
 #include <machine/cache.h>
 #include <machine/cpufunc.h>
+#include <machine/lsu.h>
 #include <machine/smp.h>
 #include <machine/tlb.h>
 
@@ -53,6 +54,32 @@ PMAP_STATS_VAR(spitfire_icache_npage_inval);
 PMAP_STATS_VAR(spitfire_icache_npage_inval_match);
 
 /*
+ * Enable the level 1 caches.
+ */
+void
+spitfire_cache_enable(void)
+{
+	u_long lsu;
+
+	lsu = ldxa(0, ASI_LSU_CTL_REG);
+	stxa_sync(0, ASI_LSU_CTL_REG, lsu | LSU_IC | LSU_DC);
+}
+
+/*
+ * Flush all lines from the level 1 caches.
+ */
+void
+spitfire_cache_flush(void)
+{
+	u_long addr;
+
+	for (addr = 0; addr < cache.dc_size; addr += cache.dc_linesize)
+		stxa_sync(addr, ASI_DCACHE_TAG, 0);
+	for (addr = 0; addr < cache.ic_size; addr += cache.ic_linesize)
+		stxa_sync(addr, ASI_ICACHE_TAG, 0);
+}
+
+/*
  * Flush a physical page from the data cache.
  */
 void
diff --git a/sys/sparc64/sparc64/support.S b/sys/sparc64/sparc64/support.S
index 3ac52fd..57ca637 100644
--- a/sys/sparc64/sparc64/support.S
+++ b/sys/sparc64/sparc64/support.S
@@ -575,10 +575,6 @@ fas_nofault_end:
 
 	.globl	fas_fault
 ENTRY(fas_fault)
-	ldxa	[%g0] ASI_LSU_CTL_REG, %o0
-	or	%o0, LSU_IC | LSU_DC, %o0
-	stxa	%o0, [%g0] ASI_LSU_CTL_REG
-	membar	#Sync
 	retl
 	 mov	-1, %o0
 END(fas_fault)
diff --git a/sys/sparc64/sparc64/trap.c b/sys/sparc64/sparc64/trap.c
index 99aab96..0a0dbab 100644
--- a/sys/sparc64/sparc64/trap.c
+++ b/sys/sparc64/sparc64/trap.c
@@ -345,6 +345,8 @@ trap(struct trapframe *tf)
 			    tf->tf_tpc < (u_long)fas_nofault_end &&
 			    *(u_int32_t *)tf->tf_tpc == MEMBARSYNC_INST &&
 			    ((u_int32_t *)tf->tf_tpc)[-2] == MEMBARSYNC_INST) {
+				cache_flush();
+				cache_enable();
 				tf->tf_tpc = (u_long)fas_fault;
 				tf->tf_tnpc = tf->tf_tpc + 4;
 				error = 0;