Add support for CPUs with cache coherent DMA. The two main changes are:

- We don't need to fall back to uncacheable memory to satisfy BUS_DMA_COHERENT
  requests on these CPUs.

- The bus_dmamap_sync() is a no-op for these CPUs.

A side-effect of this change is rename DMAMAP_COHERENT flag to
DMAMAP_UNCACHEABLE. This conveys the purpose of the flag more accurately.

Reviewed by: gonzo, imp

4 files changed, 35 insertions, 10 deletions

diff --git a/sys/mips/include/cpuinfo.h b/sys/mips/include/cpuinfo.h
index 4378c6a..106b49e 100644
--- a/sys/mips/include/cpuinfo.h
+++ b/sys/mips/include/cpuinfo.h
@@ -56,6 +56,7 @@ struct mips_cpuinfo {
 	u_int8_t	tlb_type;
 	u_int16_t	tlb_nentries;
 	u_int8_t	icache_virtual;
+	boolean_t	cache_coherent_dma;
 	struct {
 		u_int32_t	ic_size;
 		u_int8_t	ic_linesize;
@@ -68,6 +69,8 @@ struct mips_cpuinfo {
 	} l1;
 };
 
+extern struct mips_cpuinfo cpuinfo;
+
 /* TODO: Merge above structure with NetBSD's below. */
 
 struct cpu_info {
diff --git a/sys/mips/mips/busdma_machdep.c b/sys/mips/mips/busdma_machdep.c
index 4e7b45f..f2d41df 100644
--- a/sys/mips/mips/busdma_machdep.c
+++ b/sys/mips/mips/busdma_machdep.c
@@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/bus.h>
 #include <machine/cache.h>
 #include <machine/cpufunc.h>
+#include <machine/cpuinfo.h>
 #include <machine/md_var.h>
 
 #define MAX_BPAGES 64
@@ -124,7 +125,7 @@ SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
 #define DMAMAP_MBUF		0x2
 #define DMAMAP_UIO		0x4
 #define DMAMAP_TYPE_MASK	(DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO)
-#define DMAMAP_COHERENT		0x8
+#define DMAMAP_UNCACHEABLE	0x8
 #define DMAMAP_ALLOCATED	0x10
 #define DMAMAP_MALLOCUSED	0x20
 
@@ -340,6 +341,8 @@ bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
         newtag->nsegments = nsegments;
 	newtag->maxsegsz = maxsegsz;
 	newtag->flags = flags;
+	if (cpuinfo.cache_coherent_dma)
+		newtag->flags |= BUS_DMA_COHERENT;
 	newtag->ref_count = 1; /* Count ourself */
 	newtag->map_count = 0;
 	if (lockfunc != NULL) {
@@ -517,9 +520,6 @@ bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
 		bz->map_count++;
 	}
 
-	if (flags & BUS_DMA_COHERENT)
-	    newmap->flags |= DMAMAP_COHERENT;
-
 	CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 	    __func__, dmat, dmat->flags, error);
 
@@ -577,13 +577,23 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
 	*mapp = newmap;
 	newmap->dmat = dmat;
 
+	/*
+	 * If all the memory is coherent with DMA then we don't need to
+	 * do anything special for a coherent mapping request.
+	 */
+	if (dmat->flags & BUS_DMA_COHERENT)
+	    flags &= ~BUS_DMA_COHERENT;
+
+	/*
+	 * Allocate uncacheable memory if all else fails.
+	 */
 	if (flags & BUS_DMA_COHERENT)
-	    newmap->flags |= DMAMAP_COHERENT;
-	
+	    newmap->flags |= DMAMAP_UNCACHEABLE;
+
         if (dmat->maxsize <= PAGE_SIZE &&
 	   (dmat->alignment < dmat->maxsize) &&
 	   !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr) && 
-	   !(flags & BUS_DMA_COHERENT)) {
+	   !(newmap->flags & DMAMAP_UNCACHEABLE)) {
                 *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
 		newmap->flags |= DMAMAP_MALLOCUSED;
         } else {
@@ -619,7 +629,7 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
                 return (ENOMEM);
 	}
 
-	if (flags & BUS_DMA_COHERENT) {
+	if (newmap->flags & DMAMAP_UNCACHEABLE) {
 		void *tmpaddr = (void *)*vaddr;
 
 		if (tmpaddr) {
@@ -1177,8 +1187,13 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 		return;
 	if (STAILQ_FIRST(&map->bpages))
 		_bus_dmamap_sync_bp(dmat, map, op);
-	if (map->flags & DMAMAP_COHERENT)
+
+	if (dmat->flags & BUS_DMA_COHERENT)
 		return;
+
+	if (map->flags & DMAMAP_UNCACHEABLE)
+		return;
+
 	CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
 	switch(map->flags & DMAMAP_TYPE_MASK) {
 	case DMAMAP_LINEAR:
diff --git a/sys/mips/mips/cpu.c b/sys/mips/mips/cpu.c
index 1b490e6..4539d1e 100644
--- a/sys/mips/mips/cpu.c
+++ b/sys/mips/mips/cpu.c
@@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/pte.h>
 #include <machine/hwfunc.h>
 
-static struct mips_cpuinfo cpuinfo;
+struct mips_cpuinfo cpuinfo;
 
 union	cpuprid cpu_id;
 union	cpuprid fpu_id;
diff --git a/sys/mips/sibyte/sb_machdep.c b/sys/mips/sibyte/sb_machdep.c
index c6043b8..8f59815 100644
--- a/sys/mips/sibyte/sb_machdep.c
+++ b/sys/mips/sibyte/sb_machdep.c
@@ -220,6 +220,13 @@ mips_init(void)
 	mips_cpu_init();
 
 	/*
+	 * Sibyte has a L1 data cache coherent with DMA. This includes
+	 * on-chip network interfaces as well as PCI/HyperTransport bus
+	 * masters.
+	 */
+	cpuinfo.cache_coherent_dma = TRUE;
+
+	/*
 	 * XXX
 	 * The kernel is running in 32-bit mode but the CFE is running in
 	 * 64-bit mode. So the SR_KX bit in the status register is turned