- Remove unused cache flushing routines. These will not necessary work

  on future UltraSPARC cpus for which the data cache is not direct mapped.
- Move UltraSPARC I and II (spitfire, blackbird, sapphire, sabre) specific
  functions to spitfire.c, and add cheetah.c for UltraSPARC III specific
  functions.  Initially just cache flushing, but there are a few other
  functions that will need to move here.
- Add an ipi handler for data cache flushing on UltraSPARC III.
- Use function pointers to select the right cache flushing functions based
  on cpu_impl.

With this it is possible to boot single user from an mfs root on UltraSPARC
III systems, including spinning up secondary processors.  There is currently
no support for the host to pci bridge, and no documentation for it is
publically available.

Thanks to Oleg Derevenetz for providing access to a system with UltraSPARC
III+ cpus.

8 files changed, 260 insertions, 456 deletions

diff --git a/sys/conf/files.sparc64 b/sys/conf/files.sparc64
index 8f8ff74..93887df 100644
--- a/sys/conf/files.sparc64
+++ b/sys/conf/files.sparc64
@@ -36,6 +36,7 @@ sparc64/sbus/sbus.c		optional	sbus
 sparc64/sparc64/autoconf.c	standard
 sparc64/sparc64/bus_machdep.c	standard
 sparc64/sparc64/cache.c		standard
+sparc64/sparc64/cheetah.c	standard
 sparc64/sparc64/clock.c		standard
 sparc64/sparc64/counter.c	standard
 sparc64/sparc64/critical.c	standard
@@ -68,6 +69,7 @@ sparc64/sparc64/pmap.c		standard
 sparc64/sparc64/prof_machdep.c	optional	profiling-routine
 sparc64/sparc64/rwindow.c	standard
 sparc64/sparc64/sparcbus_if.m	standard
+sparc64/sparc64/spitfire.c	standard
 sparc64/sparc64/support.S	standard
 sparc64/sparc64/sys_machdep.c	standard
 sparc64/sparc64/swtch.S		standard
diff --git a/sys/sparc64/include/cache.h b/sys/sparc64/include/cache.h
index 1ebffc1..68b88c7 100644
--- a/sys/sparc64/include/cache.h
+++ b/sys/sparc64/include/cache.h
@@ -47,65 +47,14 @@
 #ifndef _MACHINE_CACHE_H_
 #define _MACHINE_CACHE_H_
 
-#include <vm/vm.h>
-#include <vm/pmap.h>
-
 #include <dev/ofw/openfirm.h>
 
-/*
- * Cache diagnostic access definitions.
- */
-/* ASI offsets for I$ diagnostic access */
-#define	ICDA_SET_SHIFT		13
-#define	ICDA_SET_MASK		(1UL << ICDA_SET_SHIFT)
-#define	ICDA_SET(a)		(((a) << ICDA_SET_SHIFT) & ICDA_SET_MASK)
-/* I$ tag/valid format */
-#define	ICDT_TAG_SHIFT		8
-#define	ICDT_TAG_BITS		28
-#define	ICDT_TAG_MASK		(((1UL << ICDT_TAG_BITS) - 1) << ICDT_TAG_SHIFT)
-#define	ICDT_TAG(x)		(((x) & ICDT_TAG_MASK) >> ICDT_TAG_SHIFT)
-#define	ICDT_VALID		(1UL << 36)
-/* D$ tag/valid format */
-#define	DCDT_TAG_SHIFT		2
-#define	DCDT_TAG_BITS		28
-#define	DCDT_TAG_MASK		(((1UL << DCDT_TAG_BITS) - 1) << DCDT_TAG_SHIFT)
-#define	DCDT_TAG(x)		(((x) & DCDT_TAG_MASK) >> DCDT_TAG_SHIFT)
-#define	DCDT_VALID_BITS		2
-#define	DCDT_VALID_MASK		((1UL << DCDT_VALID_BITS) - 1)
-/* E$ ASI_ECACHE_W/ASI_ECACHE_R address flags */
-#define	ECDA_DATA		(1UL << 39)
-#define	ECDA_TAG		(1UL << 40)
-/* E$ tag/state/parity format */
-#define	ECDT_TAG_BITS		13
-#define	ECDT_TAG_SIZE		(1UL << ECDT_TAG_BITS)
-#define	ECDT_TAG_MASK		(ECDT_TAG_SIZE - 1)
-
-/*
- * Do two virtual addresses (at which the same page is mapped) form and illegal
- * alias in D$? XXX: should use cache.dc_size here.
- */
-#define	DCACHE_BOUNDARY		0x4000
-#define	DCACHE_BMASK		(DCACHE_BOUNDARY - 1)
-#define	CACHE_BADALIAS(v1, v2) \
-	(((v1) & DCACHE_BMASK) != ((v2) & DCACHE_BMASK))
-
-/*
- * Routines for dealing with the cache.
- */
-void	cache_init(phandle_t);		/* turn it on */
-void	icache_flush(vm_offset_t, vm_offset_t);
-void	icache_inval_phys(vm_offset_t, vm_offset_t);
-void	dcache_flush(vm_offset_t, vm_offset_t);
-void	dcache_inval(pmap_t, vm_offset_t, vm_offset_t);
-void	dcache_inval_phys(vm_offset_t, vm_offset_t);
-void	dcache_blast(void);
-void	ecache_flush(vm_offset_t, vm_offset_t);
-#if 0
-void	ecache_inval_phys(vm_offset_t, vm_offset_t);
-#endif
-
-void	dcache_page_inval(vm_offset_t pa);
-void	icache_page_inval(vm_offset_t pa);
+#define	DCACHE_COLOR_BITS	(1)
+#define	DCACHE_COLORS		(1 << DCACHE_COLOR_BITS)
+#define	DCACHE_COLOR_MASK	(DCACHE_COLORS - 1)
+#define	DCACHE_COLOR(va)	(((va) >> PAGE_SHIFT) & DCACHE_COLOR_MASK)
+#define	DCACHE_OTHER_COLOR(color) \
+	((color) ^ DCACHE_COLOR_BITS)
 
 #define	DC_TAG_SHIFT	2
 #define	DC_VALID_SHIFT	0
@@ -146,6 +95,19 @@ struct cacheinfo {
 	u_int	ec_l2linesize;
 };
 
+typedef void dcache_page_inval_t(vm_offset_t pa);
+typedef void icache_page_inval_t(vm_offset_t pa);
+
+void	cache_init(phandle_t node);
+
+void	cheetah_dcache_page_inval(vm_offset_t pa);
+void	cheetah_icache_page_inval(vm_offset_t pa);
+void	spitfire_dcache_page_inval(vm_offset_t pa);
+void	spitfire_icache_page_inval(vm_offset_t pa);
+
+extern dcache_page_inval_t *dcache_page_inval;
+extern icache_page_inval_t *icache_page_inval;
+
 extern struct cacheinfo cache;
 
 #endif	/* !_MACHINE_CACHE_H_ */
diff --git a/sys/sparc64/include/pmap.h b/sys/sparc64/include/pmap.h
index b0325f9..a5fddf1 100644
--- a/sys/sparc64/include/pmap.h
+++ b/sys/sparc64/include/pmap.h
@@ -44,15 +44,9 @@
 #define	_MACHINE_PMAP_H_
 
 #include <sys/queue.h>
+#include <machine/cache.h>
 #include <machine/tte.h>
 
-#define	DCACHE_COLOR_BITS	(1)
-#define	DCACHE_COLORS		(1 << DCACHE_COLOR_BITS)
-#define	DCACHE_COLOR_MASK	(DCACHE_COLORS - 1)
-#define	DCACHE_COLOR(va)	(((va) >> PAGE_SHIFT) & DCACHE_COLOR_MASK)
-#define	DCACHE_OTHER_COLOR(color) \
-	((color) ^ DCACHE_COLOR_BITS)
-
 #define	PMAP_CONTEXT_MAX	8192
 
 #define	pmap_page_is_mapped(m)	(!TAILQ_EMPTY(&(m)->md.tte_list))
diff --git a/sys/sparc64/include/smp.h b/sys/sparc64/include/smp.h
index 270244b..5dbfc76 100644
--- a/sys/sparc64/include/smp.h
+++ b/sys/sparc64/include/smp.h
@@ -96,8 +96,10 @@ extern	u_long mp_tramp_func;
 
 extern	void mp_startup(void);
 
-extern	char tl_ipi_dcache_page_inval[];
-extern	char tl_ipi_icache_page_inval[];
+extern	char tl_ipi_cheetah_dcache_page_inval[];
+extern	char tl_ipi_spitfire_dcache_page_inval[];
+extern	char tl_ipi_spitfire_icache_page_inval[];
+
 extern	char tl_ipi_level[];
 extern	char tl_ipi_tlb_context_demap[];
 extern	char tl_ipi_tlb_page_demap[];
@@ -108,7 +110,7 @@ extern	char tl_ipi_tlb_range_demap[];
 #if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_)
 
 static __inline void *
-ipi_dcache_page_inval(vm_offset_t pa)
+ipi_dcache_page_inval(void *func, vm_offset_t pa)
 {
 	struct ipi_cache_args *ica;
 
@@ -118,13 +120,12 @@ ipi_dcache_page_inval(vm_offset_t pa)
 	mtx_lock_spin(&ipi_mtx);
 	ica->ica_mask = all_cpus;
 	ica->ica_pa = pa;
-	cpu_ipi_selected(PCPU_GET(other_cpus), 0,
-	    (u_long)tl_ipi_dcache_page_inval, (u_long)ica);
+	cpu_ipi_selected(PCPU_GET(other_cpus), 0, (u_long)func, (u_long)ica);
 	return (&ica->ica_mask);
 }
 
 static __inline void *
-ipi_icache_page_inval(vm_offset_t pa)
+ipi_icache_page_inval(void *func, vm_offset_t pa)
 {
 	struct ipi_cache_args *ica;
 
@@ -134,8 +135,7 @@ ipi_icache_page_inval(vm_offset_t pa)
 	mtx_lock_spin(&ipi_mtx);
 	ica->ica_mask = all_cpus;
 	ica->ica_pa = pa;
-	cpu_ipi_selected(PCPU_GET(other_cpus), 0,
-	    (u_long)tl_ipi_icache_page_inval, (u_long)ica);
+	cpu_ipi_selected(PCPU_GET(other_cpus), 0, (u_long)func, (u_long)ica);
 	return (&ica->ica_mask);
 }
 
@@ -215,13 +215,13 @@ ipi_wait(void *cookie)
 #else
 
 static __inline void *
-ipi_dcache_page_inval(vm_offset_t pa)
+ipi_dcache_page_inval(void *func, vm_offset_t pa)
 {
 	return (NULL);
 }
 
 static __inline void *
-ipi_icache_page_inval(vm_offset_t pa)
+ipi_icache_page_inval(void *func, vm_offset_t pa)
 {
 	return (NULL);
 }
diff --git a/sys/sparc64/sparc64/cache.c b/sys/sparc64/sparc64/cache.c
index 4ef6bb3..a3e1a1c 100644
--- a/sys/sparc64/sparc64/cache.c
+++ b/sys/sparc64/sparc64/cache.c
@@ -75,120 +75,18 @@
  * $FreeBSD$
  */
 
-/*
- * Cache routines.
- *
- * UltraSPARCs have a virtually indexed, physically tagged (VIPT) level 1 data
- * cache (D$) and physically indexed, physically tagged (PIPT) level 1
- * instruction (I$) and Level 2 (E$) caches.
- * D$ is directly mapped, I$ is pseudo 2-way associative. The Level 2 cache (E$)
- * is documented to be directly mapped on the UltraSPARC IIi, but there are
- * apparently models (using the IIe version) that have a 4-way associative E$.
- *
- * D$ uses a write-through model, while E$ uses write-back and is
- * write-allocating. The lines present in D$ are forced to be a subset of those
- * in E$.
- * This means that lines that are present in D$ always have an identical
- * corresponding (sub-) line in E$.
- *
- * The term "main memory" is used in the following to refer to the non-cache
- * memory as well as to memory-mapped device i/o space.
- *
- * There are 3 documented ways to flush the D$ and E$ caches:
- * - displacement flushing (a sequence of loads of addresses that alias to
- *   to-be-flushed ones in the caches). This only works for directly mapped
- *   caches, and is recommended to flush D$ and E$ in the IIi manual. It is not
- *   used to flush E$ because of the aforementioned models that have a
- *   multiple-associative E$. Displacement flushing invalidates the cache
- *   entries and writes modified lines back to main memory.
- * - diagnostic acceses can be used to invalidate cache pages. All lines
- *   are discarded, which means that changes in D$/E$ that have not been
- *   committed to main memory are lost.
- * - block-commit stores. Those use the block transfer ASIs to load a
- *   64-byte block to a set of FP registers and store them back using a
- *   special ASI that will cause the data to be immediately committed to main
- *   memory. This method has the same properties as the first method, but
- *   (hopefully) works regardless of the associativity of the caches. It is
- *   expected to be slow.
- *
- * I$ can be handled using the flush instruction.
- *
- * Some usage guidelines:
- *
- * The inval functions are variants of the flush ones that discard modified
- * cache lines.
- * PCI DMA transactions are cache-coherent and do not require flushing
- * before DMA reads or after DMA writes. It is unclear from the manual
- * how far this applies to UPA transactions.
- *
- * icache_flush(): needed before code that has been written to memory is
- *	executed, because I$ is not necessarily consistent with D$, E$, or
- *	main memory. An exception is that I$ snoops DMA transfers, so no
- *	flush is required after to-be-executed data has been fetched this way.
- * icache_inval_phys(): has roughly same effect as icache_flush() since there
- *	are no writes to I$.
- *
- * dcache_flush(): required when a page mapping is changed from cacheable to
- *	noncacheable, or to resolve illegal aliases. Both cases should happen
- *	seldom. Mapping address changes do not require this, since D$ is VIPT.
- * dcache_inval(): has roughly same effect as dcache_flush() since D$ is
- *	write-through.
- * dcache_blast(): discards all lines in D$.
- *
- * ecache_flush(): needed to commit modified lines to main memory, and to make
- *	sure that no stale data is used when the main memory has changed without
- *	the cache controller noticing. This is e.g. needed for device i/o space.
- *	It is usually better to use a non-cacheable mapping in this case.
- *	ecache_flush() is guaranteed to also flush the relevant addresses out of
- *	D$.
- * ecache_inval_phys():  like ecache_flush(), but invalidates a physical range
- *	in the cache. This function is usually dangerous and should not be used.
- *
- * All operations have a line size granularity!
- *
- * All flush methods tend to be expensive, so unnecessary flushes should be
- * avoided.
- */
-
-#include "opt_pmap.h"
-
 #include <sys/param.h>
-#include <sys/linker_set.h>
-#include <sys/proc.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
-#include <sys/smp.h>
-#include <sys/sysctl.h>
 #include <sys/systm.h>
 
-#include <vm/vm.h>
-#include <vm/pmap.h>
+#include <dev/ofw/openfirm.h>
 
 #include <machine/cache.h>
-#include <machine/cpufunc.h>
-#include <machine/fp.h>
-#include <machine/fsr.h>
-#include <machine/pcb.h>
-#include <machine/pmap.h>
-#include <machine/smp.h>
-#include <machine/tte.h>
 #include <machine/ver.h>
-#include <machine/vmparam.h>
 
 struct cacheinfo cache;
 
-PMAP_STATS_VAR(dcache_npage_inval);
-PMAP_STATS_VAR(dcache_npage_inval_match);
-PMAP_STATS_VAR(icache_npage_inval);
-PMAP_STATS_VAR(icache_npage_inval_match);
-
-/* Read to %g0, needed for E$ access. */
-#define	CDIAG_RDG0(asi, addr)						\
-	__asm __volatile("ldxa [%0] %1, %%g0" : : "r" (addr), "I" (asi))
-/* Sigh. I$ diagnostic registers want ldda. */
-#define	ICDIAG_RD(asi, addr, r)						\
-	__asm __volatile("ldda [%1] %2, %%o4; mov %%o5, %0" : "=r" (r) :\
-	"r" (addr), "I" (asi) :	"%o4", "%o5");
+dcache_page_inval_t *dcache_page_inval;
+icache_page_inval_t *icache_page_inval;
 
 #define	OF_GET(h, n, v)	OF_getprop((h), (n), &(v), sizeof(v))
 
@@ -224,277 +122,12 @@ cache_init(phandle_t node)
 	cache.ec_l2set = ffs(set) - 1;
 	if ((set & ~(1UL << cache.ec_l2set)) != 0)
 		panic("cache_init: E$ set size not a power of 2");
-	cache.c_enabled = 1; /* enable cache flushing */
-}
-
-void
-dcache_page_inval(vm_offset_t pa)
-{
-	u_long target;
-	void *cookie;
-	u_long addr;
-	u_long tag;
-
-	KASSERT((pa & PAGE_MASK) == 0,
-	    ("dcache_page_inval: pa not page aligned"));
-
-	if (!cache.c_enabled)
-		return;
-	PMAP_STATS_INC(dcache_npage_inval);
-	target = pa >> (PAGE_SHIFT - DC_TAG_SHIFT);
-	cookie = ipi_dcache_page_inval(pa);
-	for (addr = 0; addr < cache.dc_size; addr += cache.dc_linesize) {
-		tag = ldxa(addr, ASI_DCACHE_TAG);
-		if (((tag >> DC_VALID_SHIFT) & DC_VALID_MASK) == 0)
-			continue;
-		tag &= DC_TAG_MASK << DC_TAG_SHIFT;
-		if (tag == target) {
-			PMAP_STATS_INC(dcache_npage_inval_match);
-			stxa_sync(addr, ASI_DCACHE_TAG, tag);
-		}
-	}
-	ipi_wait(cookie);
-}
-
-void
-icache_page_inval(vm_offset_t pa)
-{
-	register u_long tag __asm("%g1");
-	u_long target;
-	void *cookie;
-	u_long addr;
-
-	KASSERT((pa & PAGE_MASK) == 0,
-	    ("icache_page_inval: pa not page aligned"));
-
-	if (!cache.c_enabled)
-		return;
-	PMAP_STATS_INC(icache_npage_inval);
-	target = pa >> (PAGE_SHIFT - IC_TAG_SHIFT);
-	cookie = ipi_icache_page_inval(pa);
-	for (addr = 0; addr < cache.ic_size; addr += cache.ic_linesize) {
-		__asm __volatile("ldda [%1] %2, %%g0" /*, %g1 */
-		    : "=r" (tag) : "r" (addr), "n" (ASI_ICACHE_TAG));
-		if (((tag >> IC_VALID_SHIFT) & IC_VALID_MASK) == 0)
-			continue;
-		tag &= IC_TAG_MASK << IC_TAG_SHIFT;
-		if (tag == target) {
-			PMAP_STATS_INC(icache_npage_inval_match);
-			stxa_sync(addr, ASI_ICACHE_TAG, tag);
-		}
-	}
-	ipi_wait(cookie);
-}
-
-
-/* Flush a range of addresses from I$ using the flush instruction. */
-void
-icache_flush(vm_offset_t start, vm_offset_t end)
-{
-	char *p, *ep;
-	int ls;
-
-	if (!cache.c_enabled)
-		return;
-
-	ls = cache.ic_linesize;
-	ep = (char *)ulmin(end, start + cache.ic_size);
-	for (p = (char *)start; p < ep; p += ls)
-		flush(p);
-}
-
-/*
- * Invalidate an I$ physical range using diagnostic accesses.
- * NOTE: there is a race between checking the tag and invalidating it. It
- * cannot be closed by disabling interrupts, since the fetch for the next
- * instruction may be in that line, so we don't even bother.
- * Since blasting a line does not discard data, this has no ill effect except
- * a minor slowdown.
- */
-void
-icache_inval_phys(vm_offset_t start, vm_offset_t end)
-{
-	vm_offset_t addr, ica;
-	u_long tag;
-	u_long j;
-
-	if (!cache.c_enabled)
-		return;
-
-	for (addr = start & ~(cache.ic_linesize - 1); addr <= end;
-	     addr += cache.ic_linesize) {
-		for (j = 0; j < 2; j++) {
-			ica = (addr & (cache.ic_set - 1)) | ICDA_SET(j);
-			ICDIAG_RD(ASI_ICACHE_TAG, ica, tag);
-			if ((tag & ICDT_VALID) == 0 ||
-			    ICDT_TAG(tag) != addr >> cache.ic_l2set)
-				continue;
-			stxa_sync(ica, ASI_ICACHE_TAG, 0);
-		}
-	}
-}
-
-/*
- * Flush a range of addresses from D$ using displacement flushes. This does
- * not necessarily flush E$, because we do not take care of flushing the
- * correct physical colors and E$ may not be directly mapped.
- */
-void
-dcache_flush(vm_offset_t start, vm_offset_t end)
-{
-	int j;
-	vm_offset_t baseoff;
-	u_long i, mask;
-	char *kp;
-
-	if (!cache.c_enabled)
-		return;
-
-	mask = cache.dc_size - 1;
-	/* No need to flush lines more than once. */
-	baseoff = start & mask;
-	/*
-	 * Use a locked page for flushing. D$ should be smaller than 4M, which
-	 * is somewhat likely...
-	 */
-	kp = (char *)KERNBASE;
-	j = 0;
-	for (i = 0; i <= ulmin((end - start), cache.dc_size);
-	     i += cache.dc_linesize)
-		j += kp[(baseoff + i) & mask];
-}
-
-/*
- * Invalidate a D$ range using diagnostic accesses.
- * This has the same (harmless) races as icache_blast().
- */
-void
-dcache_inval(pmap_t pmap, vm_offset_t start, vm_offset_t end)
-{
-	vm_offset_t va, pa, offs, dca;
-	u_long tag;
-
-	if (!cache.c_enabled)
-		return;
-	for (va = start & ~(cache.dc_linesize - 1); va <= end;
-	     va = (va + PAGE_SIZE_MIN) & ~PAGE_MASK_MIN) {
-		if ((pa = pmap_extract(pmap, va)) == 0)
-			continue;
-		for (offs = start & PAGE_MASK_MIN;
-		     offs < ulmin(PAGE_SIZE_MIN, end - va + 1);
-		     offs += cache.dc_linesize) {
-			dca = (va + offs) & (cache.dc_size - 1);
-			tag = ldxa(dca, ASI_DCACHE_TAG);
-			if (DCDT_TAG(tag) != (pa + offs) >> PAGE_SHIFT_MIN)
-				continue;
-			stxa_sync(dca, ASI_DCACHE_TAG, 0);
-		}
-	}
-}
-
-/*
- * Invalidate a physical D$ range using diagnostic accesses.
- * This has the same (harmless) races as icache_blast().
- */
-void
-dcache_inval_phys(vm_offset_t start, vm_offset_t end)
-{
-	vm_offset_t pa, dca;
-	u_long tag, color, ncolors;
-
-	if (!cache.c_enabled)
-		return;
-	ncolors = 1 << (cache.dc_l2size - PAGE_SHIFT_MIN);
-	for (pa = start & ~(cache.dc_linesize - 1); pa <= end;
-	    pa += cache.dc_linesize) {
-		for (color = 0; color < ncolors; color++) {
-			dca = (color << PAGE_SHIFT_MIN) | (pa & PAGE_MASK_MIN);
-			tag = ldxa(dca, ASI_DCACHE_TAG);
-			if (DCDT_TAG(tag) == pa >> PAGE_SHIFT_MIN) {
-				stxa_sync(dca, ASI_DCACHE_TAG, 0);
-				break;
-			}
-		}
-	}
-}
-
-/* Discard all lines in D$. */
-void
-dcache_blast()
-{
-	vm_offset_t dca;
-
-	if (!cache.c_enabled)
-		return;
-	for (dca = 0; dca < cache.dc_size; dca += cache.dc_linesize)
-		stxa_sync(dca, ASI_DCACHE_TAG, 0);
-}
-
-/* Flush an E$ physical range using block commit stores. */
-void
-ecache_flush(vm_offset_t start, vm_offset_t end)
-{
-	vm_offset_t addr;
-
-	if (!cache.c_enabled)
-		return;
-
-	/* XXX: not needed in all cases, provide a wrapper in fp.c */
-	savefpctx(&curthread->td_pcb->pcb_fpstate);
-	wr(fprs, 0, FPRS_FEF);
-
-	for (addr = start & ~(cache.ec_linesize - 1); addr <= end;
-	     addr += cache.ec_linesize) {
-		__asm __volatile("ldda [%0] %1, %%f0; membar #Sync; "
-		    "stda %%f0, [%0] %2" : : "r" (addr), "I" (ASI_BLK_S),
-		    "I" (ASI_BLK_COMMIT_S));
-	}
-	membar(Sync);
-
-	restorefpctx(&curthread->td_pcb->pcb_fpstate);
-}
-
-#if 0
-/*
- * Invalidate an E$ range using diagnostic accesses.
- * This is disabled: it suffers from the same races as dcache_blast() and
- * icache_blast_phys(), but they may be fatal here because blasting an E$ line
- * can discard modified data.
- * There is no really use for this anyway.
- */
-void
-ecache_inval_phys(vm_offset_t start, vm_offset_t end)
-{
-	vm_offset_t addr, eca;
-	u_long tag, j;
-
-	if (!cache.c_enabled)
-		return;
 
-	for (addr = start & ~(cache.ec_linesize - 1); addr <= end;
-	     addr += cache.ec_linesize) {
-		for (j = 0; j < cache.ec_assoc; j++) {
-			/* XXX: guesswork... */
-			eca = (addr & (cache.ec_size - 1)) |
-			    (j << (cache.ec_l2set));
-			/*
-			 * Retrieve the tag:
-			 * A read from the appropriate VA in ASI_ECACHE_R
-			 * will transfer the tag from the tag RAM to the
-			 * data register (ASI_ECACHE_TAG_DATA, VA 0).
-			 */
-			CDIAG_RDG0(ASI_ECACHE_R, ECDA_TAG | eca);
-			tag = ldxa(0, ASI_ECACHE_TAG_DATA);
-			if ((addr & ~cache.ec_size) >> cache.ec_l2set ==
-			    (tag & ECDT_TAG_MASK)) {
-				/*
-				 * Clear. Works like retrieving the tag, but
-				 * the other way round.
-				 */
-				stxa_sync(0, ASI_ECACHE_TAG_DATA, 0);
-				stxa_sync(ECDA_TAG | eca, ASI_ECACHE_W, 0);
-			}
-		}
+	if (cpu_impl >= CPU_IMPL_ULTRASPARCIII) {
+		dcache_page_inval = cheetah_dcache_page_inval;
+		icache_page_inval = cheetah_icache_page_inval;
+	} else {
+		dcache_page_inval = spitfire_dcache_page_inval;
+		icache_page_inval = spitfire_icache_page_inval;
 	}
 }
-#endif
diff --git a/sys/sparc64/sparc64/cheetah.c b/sys/sparc64/sparc64/cheetah.c
new file mode 100644
index 0000000..27a5a1c
--- /dev/null
+++ b/sys/sparc64/sparc64/cheetah.c
@@ -0,0 +1,71 @@
+/*-
+ * Copyright (c) 2003 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "opt_pmap.h"
+
+#include <sys/param.h>
+#include <sys/linker_set.h>
+#include <sys/proc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/cache.h>
+#include <machine/cpufunc.h>
+#include <machine/smp.h>
+
+/*
+ * Flush a physical page from the data cache.
+ */
+void
+cheetah_dcache_page_inval(vm_offset_t spa)
+{
+	vm_offset_t pa;
+	void *cookie;
+
+	KASSERT((spa & PAGE_MASK) == 0,
+	    ("dcache_page_inval: pa not page aligned"));
+	cookie = ipi_dcache_page_inval(tl_ipi_cheetah_dcache_page_inval, spa);
+	for (pa = spa; pa < spa + PAGE_SIZE; pa += cache.dc_linesize)
+		stxa_sync(pa, ASI_DCACHE_INVALIDATE, 0);
+	ipi_wait(cookie);
+}
+
+/*
+ * Flush a physical page from the intsruction cache.  Instruction cache
+ * consistency is maintained by hardware.
+ */
+void
+cheetah_icache_page_inval(vm_offset_t pa)
+{
+}
diff --git a/sys/sparc64/sparc64/mp_exception.S b/sys/sparc64/sparc64/mp_exception.S
index f6964c8..d8e977e 100644
--- a/sys/sparc64/sparc64/mp_exception.S
+++ b/sys/sparc64/sparc64/mp_exception.S
@@ -44,9 +44,9 @@
 	 nop
 
 /*
- * Invalidate a phsyical page in the data cache.
+ * Invalidate a physical page in the data cache.  For UltraSPARC I and II.
  */
-ENTRY(tl_ipi_dcache_page_inval)
+ENTRY(tl_ipi_spitfire_dcache_page_inval)
 #if KTR_COMPILE & KTR_SMP
 	CATR(KTR_SMP, "ipi_dcache_page_inval: pa=%#lx"
 	    , %g1, %g2, %g3, 7, 8, 9)
@@ -81,12 +81,13 @@ ENTRY(tl_ipi_dcache_page_inval)
 
 	IPI_WAIT(%g5, %g1, %g2, %g3)
 	retry
-END(tl_ipi_dcache_page_inval)
+END(tl_ipi_spitfire_dcache_page_inval)
 
 /*
- * Invalidate a phsyical page in the instruction cache.
+ * Invalidate a physical page in the instruction cache.  For UltraSPARC I and
+ * II.
  */
-ENTRY(tl_ipi_icache_page_inval)
+ENTRY(tl_ipi_spitfire_icache_page_inval)
 #if KTR_COMPILE & KTR_SMP
 	CATR(KTR_SMP, "ipi_icache_page_inval: pa=%#lx"
 	    , %g1, %g2, %g3, 7, 8, 9)
@@ -121,7 +122,39 @@ ENTRY(tl_ipi_icache_page_inval)
 
 	IPI_WAIT(%g5, %g1, %g2, %g3)
 	retry
-END(tl_ipi_icache_page_inval)
+END(tl_ipi_spitfire_icache_page_inval)
+
+/*
+ * Invalidate a physical page in the data cache.  For UltraSPARC III.
+ */
+ENTRY(tl_ipi_cheetah_dcache_page_inval)
+#if KTR_COMPILE & KTR_SMP
+	CATR(KTR_SMP, "ipi_dcache_page_inval: pa=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	ldx	[%g5 + ICA_PA], %g2
+	stx	%g2, [%g1 + KTR_PARM1]
+9:
+#endif
+
+	ldx	[%g5 + ICA_PA], %g1
+
+	set	PAGE_SIZE, %g2
+	add	%g1, %g2, %g3
+
+	SET(cache, %g4, %g2)
+	lduw	[%g2 + DC_LINESIZE], %g2
+
+1:	stxa	%g0, [%g1] ASI_DCACHE_INVALIDATE
+	membar	#Sync
+
+	add	%g1, %g2, %g1
+	cmp	%g1, %g3
+	blt,a,pt %xcc, 1b
+	 nop
+
+	IPI_WAIT(%g5, %g1, %g2, %g3)
+	retry
+END(tl_ipi_cheetah_dcache_page_inval)
 
 /*
  * Trigger a softint at the desired level.
diff --git a/sys/sparc64/sparc64/spitfire.c b/sys/sparc64/sparc64/spitfire.c
new file mode 100644
index 0000000..2d905d5
--- /dev/null
+++ b/sys/sparc64/sparc64/spitfire.c
@@ -0,0 +1,109 @@
+/*-
+ * Copyright (c) 2003 Jake Burkholder.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "opt_pmap.h"
+
+#include <sys/param.h>
+#include <sys/linker_set.h>
+#include <sys/proc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/cache.h>
+#include <machine/cpufunc.h>
+#include <machine/smp.h>
+
+PMAP_STATS_VAR(spitfire_dcache_npage_inval);
+PMAP_STATS_VAR(spitfire_dcache_npage_inval_match);
+PMAP_STATS_VAR(spitfire_icache_npage_inval);
+PMAP_STATS_VAR(spitfire_icache_npage_inval_match);
+
+/*
+ * Flush a physical page from the data cache.
+ */
+void
+spitfire_dcache_page_inval(vm_offset_t pa)
+{
+	u_long target;
+	void *cookie;
+	u_long addr;
+	u_long tag;
+
+	KASSERT((pa & PAGE_MASK) == 0,
+	    ("dcache_page_inval: pa not page aligned"));
+	PMAP_STATS_INC(spitfire_dcache_npage_inval);
+	target = pa >> (PAGE_SHIFT - DC_TAG_SHIFT);
+	cookie = ipi_dcache_page_inval(tl_ipi_spitfire_dcache_page_inval, pa);
+	for (addr = 0; addr < cache.dc_size; addr += cache.dc_linesize) {
+		tag = ldxa(addr, ASI_DCACHE_TAG);
+		if (((tag >> DC_VALID_SHIFT) & DC_VALID_MASK) == 0)
+			continue;
+		tag &= DC_TAG_MASK << DC_TAG_SHIFT;
+		if (tag == target) {
+			PMAP_STATS_INC(spitfire_dcache_npage_inval_match);
+			stxa_sync(addr, ASI_DCACHE_TAG, tag);
+		}
+	}
+	ipi_wait(cookie);
+}
+
+/*
+ * Flush a physical page from the instruction cache.
+ */
+void
+spitfire_icache_page_inval(vm_offset_t pa)
+{
+	register u_long tag __asm("%g1");
+	u_long target;
+	void *cookie;
+	u_long addr;
+
+	KASSERT((pa & PAGE_MASK) == 0,
+	    ("icache_page_inval: pa not page aligned"));
+	PMAP_STATS_INC(spitfire_icache_npage_inval);
+	target = pa >> (PAGE_SHIFT - IC_TAG_SHIFT);
+	cookie = ipi_icache_page_inval(tl_ipi_spitfire_icache_page_inval, pa);
+	for (addr = 0; addr < cache.ic_size; addr += cache.ic_linesize) {
+		__asm __volatile("ldda [%1] %2, %%g0" /*, %g1 */
+		    : "=r" (tag) : "r" (addr), "n" (ASI_ICACHE_TAG));
+		if (((tag >> IC_VALID_SHIFT) & IC_VALID_MASK) == 0)
+			continue;
+		tag &= IC_TAG_MASK << IC_TAG_SHIFT;
+		if (tag == target) {
+			PMAP_STATS_INC(spitfire_icache_npage_inval_match);
+			stxa_sync(addr, ASI_ICACHE_TAG, tag);
+		}
+	}
+	ipi_wait(cookie);
+}