For CPUs which ignore TD_CV and support hardware unaliasing don't

bother doing page coloring. This results in a small but measurable
performance improvement in buildworld times.

6 files changed, 80 insertions, 60 deletions

diff --git a/sys/sparc64/include/cache.h b/sys/sparc64/include/cache.h
index 29bff69..3852da4 100644
--- a/sys/sparc64/include/cache.h
+++ b/sys/sparc64/include/cache.h
@@ -49,7 +49,7 @@
 #define	DCACHE_COLORS		(1 << DCACHE_COLOR_BITS)
 #define	DCACHE_COLOR_MASK	(DCACHE_COLORS - 1)
 #define	DCACHE_COLOR(va)	(((va) >> PAGE_SHIFT) & DCACHE_COLOR_MASK)
-#define	DCACHE_OTHER_COLOR(color) \
+#define	DCACHE_OTHER_COLOR(color)					\
 	((color) ^ DCACHE_COLOR_BITS)
 
 #define	DC_TAG_SHIFT	2
@@ -89,6 +89,8 @@ struct cacheinfo {
 
 #ifdef _KERNEL
 
+extern u_int dcache_color_ignore;
+
 struct pcpu;
 
 typedef void cache_enable_t(u_int cpu_impl);
diff --git a/sys/sparc64/sparc64/cache.c b/sys/sparc64/sparc64/cache.c
index a3c6119..636c18a 100644
--- a/sys/sparc64/sparc64/cache.c
+++ b/sys/sparc64/sparc64/cache.c
@@ -88,6 +88,8 @@ cache_flush_t *cache_flush;
 dcache_page_inval_t *dcache_page_inval;
 icache_page_inval_t *icache_page_inval;
 
+u_int dcache_color_ignore;
+
 #define	OF_GET(h, n, v)	OF_getprop((h), (n), &(v), sizeof(v))
 
 static u_int cache_new_prop(u_int cpu_impl);
@@ -114,6 +116,13 @@ cache_init(struct pcpu *pcpu)
 	u_long set;
 	u_int use_new_prop;
 
+	/*
+	 * For CPUs which ignore TD_CV and support hardware unaliasing don't
+	 * bother doing page coloring.  This is equal across all CPUs.
+	 */
+	if (pcpu->pc_cpuid == 0 && pcpu->pc_impl == CPU_IMPL_SPARC64V)
+		dcache_color_ignore = 1;
+
 	use_new_prop = cache_new_prop(pcpu->pc_impl);
 	if (OF_GET(pcpu->pc_node, !use_new_prop ? "icache-size" :
 	    "l1-icache-size", pcpu->pc_cache.ic_size) == -1 ||
@@ -145,9 +154,8 @@ cache_init(struct pcpu *pcpu)
 	 * For CPUs which don't support unaliasing in hardware ensure that
 	 * the data cache doesn't have too many virtual colors.
 	 */
-	if (pcpu->pc_impl != CPU_IMPL_SPARC64V &&
-	    ((pcpu->pc_cache.dc_size / pcpu->pc_cache.dc_assoc) /
-	    PAGE_SIZE) != DCACHE_COLORS)
+	if (dcache_color_ignore == 0 && ((pcpu->pc_cache.dc_size /
+	    pcpu->pc_cache.dc_assoc) / PAGE_SIZE) != DCACHE_COLORS)
 		panic("cache_init: too many D$ colors");
 	set = pcpu->pc_cache.ec_size / pcpu->pc_cache.ec_assoc;
 	if ((set & ~(1UL << (ffs(set) - 1))) != 0)
diff --git a/sys/sparc64/sparc64/mem.c b/sys/sparc64/sparc64/mem.c
index b85f266..1707ad6 100644
--- a/sys/sparc64/sparc64/mem.c
+++ b/sys/sparc64/sparc64/mem.c
@@ -95,8 +95,10 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 	vm_page_t m;
 	int error;
 	int i;
+	uint32_t colors;
 
 	cnt = 0;
+	colors = 1;
 	error = 0;
 	ova = 0;
 
@@ -134,20 +136,20 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 			}
 
 			if (m != NULL) {
+				if (ova == 0) {
 #ifndef SUN4V
-				if (ova == 0)
+					if (dcache_color_ignore == 0)
+						colors = DCACHE_COLORS;
+#endif
 					ova = kmem_alloc_wait(kernel_map,
-					    PAGE_SIZE * DCACHE_COLORS);
-				if (m->md.color != -1)
+					    PAGE_SIZE * colors);
+				}
+#ifndef SUN4V
+				if (colors != 1 && m->md.color != -1)
 					va = ova + m->md.color * PAGE_SIZE;
 				else
-					va = ova;
-#else
-				if (ova == 0)
-					ova = kmem_alloc_wait(kernel_map,
-					    PAGE_SIZE);
-				va = ova;
 #endif
+					va = ova;
 				pmap_qenter(va, &m, 1);
 				error = uiomove((void *)(va + off), cnt,
 				    uio);
@@ -158,8 +160,7 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 				    uio);
 			}
 			break;
-		}
-		else if (dev2unit(dev) == CDEV_MINOR_KMEM) {
+		} else if (dev2unit(dev) == CDEV_MINOR_KMEM) {
 			va = trunc_page(uio->uio_offset);
 			eva = round_page(uio->uio_offset + iov->iov_len);
 
@@ -184,15 +185,12 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 		/* else panic! */
 	}
 	if (ova != 0)
-#ifndef SUN4V
-		kmem_free_wakeup(kernel_map, ova, PAGE_SIZE * DCACHE_COLORS);
-#else
-		kmem_free_wakeup(kernel_map, ova, PAGE_SIZE);
-#endif
+		kmem_free_wakeup(kernel_map, ova, PAGE_SIZE * colors);
 	return (error);
 }
 
 void
 dev_mem_md_init(void)
 {
+
 }
diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c
index e30d569..e158bf4 100644
--- a/sys/sparc64/sparc64/pmap.c
+++ b/sys/sparc64/sparc64/pmap.c
@@ -154,7 +154,7 @@ struct pmap kernel_pmap_store;
 /*
  * Allocate physical memory for use in pmap_bootstrap.
  */
-static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size);
+static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size, uint32_t colors);
 
 /*
  * Map the given physical page at the specified virtual address in the
@@ -308,6 +308,9 @@ pmap_bootstrap(u_int cpu_impl)
 	int i;
 	int j;
 	int sz;
+	uint32_t colors;
+
+	colors = dcache_color_ignore != 0 ? 1 : DCACHE_COLORS;
 
 	/*
 	 * Find out what physical memory is available from the PROM and
@@ -379,7 +382,7 @@ pmap_bootstrap(u_int cpu_impl)
 	/*
 	 * Allocate the kernel TSB and lock it in the TLB.
 	 */
-	pa = pmap_bootstrap_alloc(tsb_kernel_size);
+	pa = pmap_bootstrap_alloc(tsb_kernel_size, colors);
 	if (pa & PAGE_MASK_4M)
 		panic("pmap_bootstrap: tsb unaligned\n");
 	tsb_kernel_phys = pa;
@@ -390,13 +393,13 @@ pmap_bootstrap(u_int cpu_impl)
 	/*
 	 * Allocate and map the dynamic per-CPU area for the BSP.
 	 */
-	pa = pmap_bootstrap_alloc(DPCPU_SIZE);
+	pa = pmap_bootstrap_alloc(DPCPU_SIZE, colors);
 	dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pa);
 
 	/*
 	 * Allocate and map the message buffer.
 	 */
-	pa = pmap_bootstrap_alloc(MSGBUF_SIZE);
+	pa = pmap_bootstrap_alloc(MSGBUF_SIZE, colors);
 	msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(pa);
 
 	/*
@@ -458,26 +461,26 @@ pmap_bootstrap(u_int cpu_impl)
 	 * Allocate kva space for temporary mappings.
 	 */
 	pmap_idle_map = virtual_avail;
-	virtual_avail += PAGE_SIZE * DCACHE_COLORS;
+	virtual_avail += PAGE_SIZE * colors;
 	pmap_temp_map_1 = virtual_avail;
-	virtual_avail += PAGE_SIZE * DCACHE_COLORS;
+	virtual_avail += PAGE_SIZE * colors;
 	pmap_temp_map_2 = virtual_avail;
-	virtual_avail += PAGE_SIZE * DCACHE_COLORS;
+	virtual_avail += PAGE_SIZE * colors;
 
 	/*
 	 * Allocate a kernel stack with guard page for thread0 and map it
 	 * into the kernel TSB.  We must ensure that the virtual address is
-	 * coloured properly, since we're allocating from phys_avail so the
-	 * memory won't have an associated vm_page_t.
+	 * colored properly for corresponding CPUs, since we're allocating
+	 * from phys_avail so the memory won't have an associated vm_page_t.
 	 */
-	pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE);
+	pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, colors);
 	kstack0_phys = pa;
-	virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) *
-	    PAGE_SIZE;
+	virtual_avail += roundup(KSTACK_GUARD_PAGES, colors) * PAGE_SIZE;
 	kstack0 = virtual_avail;
-	virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE;
-	KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys),
-	    ("pmap_bootstrap: kstack0 miscoloured"));
+	virtual_avail += roundup(KSTACK_PAGES, colors) * PAGE_SIZE;
+	if (dcache_color_ignore == 0)
+		KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys),
+		    ("pmap_bootstrap: kstack0 miscolored"));
 	for (i = 0; i < KSTACK_PAGES; i++) {
 		pa = kstack0_phys + i * PAGE_SIZE;
 		va = kstack0 + i * PAGE_SIZE;
@@ -609,12 +612,12 @@ pmap_map_tsb(void)
  * calculated.
  */
 static vm_paddr_t
-pmap_bootstrap_alloc(vm_size_t size)
+pmap_bootstrap_alloc(vm_size_t size, uint32_t colors)
 {
 	vm_paddr_t pa;
 	int i;
 
-	size = roundup(size, PAGE_SIZE * DCACHE_COLORS);
+	size = roundup(size, PAGE_SIZE * colors);
 	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
 		if (phys_avail[i + 1] - phys_avail[i] < size)
 			continue;
@@ -755,6 +758,9 @@ pmap_cache_enter(vm_page_t m, vm_offset_t va)
 	    ("pmap_cache_enter: fake page"));
 	PMAP_STATS_INC(pmap_ncache_enter);
 
+	if (dcache_color_ignore != 0)
+		return (1);
+
 	/*
 	 * Find the color for this virtual address and note the added mapping.
 	 */
@@ -832,6 +838,9 @@ pmap_cache_remove(vm_page_t m, vm_offset_t va)
 	    m->md.colors[DCACHE_COLOR(va)]));
 	PMAP_STATS_INC(pmap_ncache_remove);
 
+	if (dcache_color_ignore != 0)
+		return;
+
 	/*
 	 * Find the color for this virtual address and note the removal of
 	 * the mapping.
@@ -900,7 +909,7 @@ pmap_kenter(vm_offset_t va, vm_page_t m)
 	    va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data);
 	if (DCACHE_COLOR(VM_PAGE_TO_PHYS(m)) != DCACHE_COLOR(va)) {
 		CTR5(KTR_SPARE2,
-	"pmap_kenter: off colour va=%#lx pa=%#lx o=%p ot=%d pi=%#lx",
+	"pmap_kenter: off color va=%#lx pa=%#lx o=%p ot=%d pi=%#lx",
 		    va, VM_PAGE_TO_PHYS(m), m->object,
 		    m->object ? m->object->type : -1,
 		    m->pindex);
@@ -1614,13 +1623,13 @@ pmap_zero_page(vm_page_t m)
 	    ("pmap_zero_page: fake page"));
 	PMAP_STATS_INC(pmap_nzero_page);
 	pa = VM_PAGE_TO_PHYS(m);
-	if (m->md.color == -1) {
-		PMAP_STATS_INC(pmap_nzero_page_nc);
-		aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE);
-	} else if (m->md.color == DCACHE_COLOR(pa)) {
+	if (dcache_color_ignore != 0 || m->md.color == DCACHE_COLOR(pa)) {
 		PMAP_STATS_INC(pmap_nzero_page_c);
 		va = TLB_PHYS_TO_DIRECT(pa);
 		cpu_block_zero((void *)va, PAGE_SIZE);
+	} else if (m->md.color == -1) {
+		PMAP_STATS_INC(pmap_nzero_page_nc);
+		aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE);
 	} else {
 		PMAP_STATS_INC(pmap_nzero_page_oc);
 		PMAP_LOCK(kernel_pmap);
@@ -1646,13 +1655,13 @@ pmap_zero_page_area(vm_page_t m, int off, int size)
 	KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size"));
 	PMAP_STATS_INC(pmap_nzero_page_area);
 	pa = VM_PAGE_TO_PHYS(m);
-	if (m->md.color == -1) {
-		PMAP_STATS_INC(pmap_nzero_page_area_nc);
-		aszero(ASI_PHYS_USE_EC, pa + off, size);
-	} else if (m->md.color == DCACHE_COLOR(pa)) {
+	if (dcache_color_ignore != 0 || m->md.color == DCACHE_COLOR(pa)) {
 		PMAP_STATS_INC(pmap_nzero_page_area_c);
 		va = TLB_PHYS_TO_DIRECT(pa);
 		bzero((void *)(va + off), size);
+	} else if (m->md.color == -1) {
+		PMAP_STATS_INC(pmap_nzero_page_area_nc);
+		aszero(ASI_PHYS_USE_EC, pa + off, size);
 	} else {
 		PMAP_STATS_INC(pmap_nzero_page_area_oc);
 		PMAP_LOCK(kernel_pmap);
@@ -1677,13 +1686,13 @@ pmap_zero_page_idle(vm_page_t m)
 	    ("pmap_zero_page_idle: fake page"));
 	PMAP_STATS_INC(pmap_nzero_page_idle);
 	pa = VM_PAGE_TO_PHYS(m);
-	if (m->md.color == -1) {
-		PMAP_STATS_INC(pmap_nzero_page_idle_nc);
-		aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE);
-	} else if (m->md.color == DCACHE_COLOR(pa)) {
+	if (dcache_color_ignore != 0 || m->md.color == DCACHE_COLOR(pa)) {
 		PMAP_STATS_INC(pmap_nzero_page_idle_c);
 		va = TLB_PHYS_TO_DIRECT(pa);
 		cpu_block_zero((void *)va, PAGE_SIZE);
+	} else if (m->md.color == -1) {
+		PMAP_STATS_INC(pmap_nzero_page_idle_nc);
+		aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE);
 	} else {
 		PMAP_STATS_INC(pmap_nzero_page_idle_oc);
 		va = pmap_idle_map + (m->md.color * PAGE_SIZE);
@@ -1711,15 +1720,16 @@ pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
 	PMAP_STATS_INC(pmap_ncopy_page);
 	pdst = VM_PAGE_TO_PHYS(mdst);
 	psrc = VM_PAGE_TO_PHYS(msrc);
-	if (msrc->md.color == -1 && mdst->md.color == -1) {
-		PMAP_STATS_INC(pmap_ncopy_page_nc);
-		ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE);
-	} else if (msrc->md.color == DCACHE_COLOR(psrc) &&
-	    mdst->md.color == DCACHE_COLOR(pdst)) {
+	if (dcache_color_ignore != 0 ||
+	    (msrc->md.color == DCACHE_COLOR(psrc) &&
+	    mdst->md.color == DCACHE_COLOR(pdst))) {
 		PMAP_STATS_INC(pmap_ncopy_page_c);
 		vdst = TLB_PHYS_TO_DIRECT(pdst);
 		vsrc = TLB_PHYS_TO_DIRECT(psrc);
 		cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE);
+	} else if (msrc->md.color == -1 && mdst->md.color == -1) {
+		PMAP_STATS_INC(pmap_ncopy_page_nc);
+		ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE);
 	} else if (msrc->md.color == -1) {
 		if (mdst->md.color == DCACHE_COLOR(pdst)) {
 			PMAP_STATS_INC(pmap_ncopy_page_dc);
diff --git a/sys/sparc64/sparc64/uio_machdep.c b/sys/sparc64/sparc64/uio_machdep.c
index 4a670fa..434713f 100644
--- a/sys/sparc64/sparc64/uio_machdep.c
+++ b/sys/sparc64/sparc64/uio_machdep.c
@@ -51,12 +51,13 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_page.h>
 #include <vm/vm_param.h>
 
+#include <machine/cache.h>
 #include <machine/tlb.h>
 
 /*
  * Implement uiomove(9) from physical memory using a combination
  * of the direct mapping and sf_bufs to reduce the creation and
- * destruction of ephemeral mappings.  
+ * destruction of ephemeral mappings.
  */
 int
 uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
@@ -92,7 +93,8 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
 		cnt = ulmin(cnt, PAGE_SIZE - page_offset);
 		m = ma[offset >> PAGE_SHIFT];
 		pa = VM_PAGE_TO_PHYS(m);
-		if (m->md.color != DCACHE_COLOR(pa)) {
+		if (dcache_color_ignore == 0 &&
+		    m->md.color != DCACHE_COLOR(pa)) {
 			sf = sf_buf_alloc(m, 0);
 			cp = (char *)sf_buf_kva(sf) + page_offset;
 		} else {
diff --git a/sys/sparc64/sparc64/vm_machdep.c b/sys/sparc64/sparc64/vm_machdep.c
index 009f45d..9dc55c6 100644
--- a/sys/sparc64/sparc64/vm_machdep.c
+++ b/sys/sparc64/sparc64/vm_machdep.c
@@ -227,7 +227,7 @@ cpu_set_upcall(struct thread *td, struct thread *td0)
 
 void
 cpu_set_upcall_kse(struct thread *td, void (*entry)(void *), void *arg,
-	stack_t *stack)
+    stack_t *stack)
 {
 	struct trapframe *tf;
 	uint64_t sp;
@@ -251,7 +251,7 @@ cpu_set_user_tls(struct thread *td, void *tls_base)
 
 	if (td == curthread)
 		flushw();
-	td->td_frame->tf_global[7] = (uint64_t) tls_base;
+	td->td_frame->tf_global[7] = (uint64_t)tls_base;
 	return (0);
 }
 
@@ -531,7 +531,7 @@ uma_small_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
 	}
 
 	pa = VM_PAGE_TO_PHYS(m);
-	if (m->md.color != DCACHE_COLOR(pa)) {
+	if (dcache_color_ignore == 0 && m->md.color != DCACHE_COLOR(pa)) {
 		KASSERT(m->md.colors[0] == 0 && m->md.colors[1] == 0,
 		    ("uma_small_alloc: free page still has mappings!"));
 		PMAP_STATS_INC(uma_nsmall_alloc_oc);