Introduce support for DMA coherent ARM platforms

- Inherit BUS_DMA_COHERENT flag from parent buses
- Use cacheable memory attributes on dma coherent platform
- Disable cache synchronization on coherent platform

Changes are based on ARMv8 busdma code and commit r299683.

Submitted by: Michal Mazur <mkm@semihalf.com>
Obtained from: Semihalf
Sponsored by: Stormshield
Reviewed by: ian
Differential revision: https://reviews.freebsd.org/D11201

(cherry picked from commit dbbf5a80043fca8abfaa7ea9e855a468d0c44e88)

1 files changed, 27 insertions, 13 deletions

diff --git a/sys/arm/arm/busdma_machdep-v6.c b/sys/arm/arm/busdma_machdep-v6.c
index 58e2012..a6da87a 100644
--- a/sys/arm/arm/busdma_machdep-v6.c
+++ b/sys/arm/arm/busdma_machdep-v6.c
@@ -491,6 +491,7 @@ bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
 		newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
 		newtag->alignment = MAX(parent->alignment, newtag->alignment);
 		newtag->flags |= parent->flags & BUS_DMA_COULD_BOUNCE;
+		newtag->flags |= parent->flags & BUS_DMA_COHERENT;
 		if (newtag->boundary == 0)
 			newtag->boundary = parent->boundary;
 		else if (parent->boundary != 0)
@@ -755,11 +756,19 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags,
 	}
 	map->flags = DMAMAP_DMAMEM_ALLOC;
 
-	/* Choose a busdma buffer allocator based on memory type flags. */
-	if (flags & BUS_DMA_COHERENT) {
+	/* For coherent memory, set the map flag that disables sync ops. */
+	if (flags & BUS_DMA_COHERENT)
+		map->flags |= DMAMAP_COHERENT;
+
+	/*
+	 * Choose a busdma buffer allocator based on memory type flags.
+	 * If the tag's COHERENT flag is set, that means normal memory
+	 * is already coherent, use the normal allocator.
+	 */
+	if ((flags & BUS_DMA_COHERENT) &&
+	    ((dmat->flags & BUS_DMA_COHERENT) == 0)) {
 		memattr = VM_MEMATTR_UNCACHEABLE;
 		ba = coherent_allocator;
-		map->flags |= DMAMAP_COHERENT;
 	} else {
 		memattr = VM_MEMATTR_DEFAULT;
 		ba = standard_allocator;
@@ -829,7 +838,8 @@ bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
 	struct busdma_bufzone *bufzone;
 	busdma_bufalloc_t ba;
 
-	if (map->flags & DMAMAP_COHERENT)
+	if ((map->flags & DMAMAP_COHERENT) &&
+	    ((dmat->flags & BUS_DMA_COHERENT) == 0))
 		ba = coherent_allocator;
 	else
 		ba = standard_allocator;
@@ -1030,7 +1040,7 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
 			sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
 			curaddr = add_bounce_page(dmat, map, 0, curaddr,
 			    sgsize);
-		} else {
+		} else if ((dmat->flags & BUS_DMA_COHERENT) == 0) {
 			if (map->sync_count > 0)
 				sl_end = sl->paddr + sl->datacount;
 
@@ -1144,7 +1154,7 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
 		    sgsize)) {
 			curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
 			    sgsize);
-		} else {
+		} else if ((dmat->flags & BUS_DMA_COHERENT) == 0) {
 			if (map->sync_count > 0) {
 				sl_pend = sl->paddr + sl->datacount;
 				sl_vend = sl->vaddr + sl->datacount;
@@ -1353,8 +1363,9 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 				    bpage->datacount);
 				if (tempvaddr != 0)
 					pmap_quick_remove_page(tempvaddr);
-				dcache_wb_poc(bpage->vaddr, bpage->busaddr,
-				    bpage->datacount);
+				if ((dmat->flags & BUS_DMA_COHERENT) == 0)
+					dcache_wb_poc(bpage->vaddr,
+					    bpage->busaddr, bpage->datacount);
 				bpage = STAILQ_NEXT(bpage, links);
 			}
 			dmat->bounce_zone->total_bounced++;
@@ -1374,8 +1385,9 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 		if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) {
 			bpage = STAILQ_FIRST(&map->bpages);
 			while (bpage != NULL) {
-				dcache_inv_poc_dma(bpage->vaddr, bpage->busaddr,
-				    bpage->datacount);
+				if ((dmat->flags & BUS_DMA_COHERENT) == 0)
+					dcache_inv_poc_dma(bpage->vaddr,
+					    bpage->busaddr, bpage->datacount);
 				bpage = STAILQ_NEXT(bpage, links);
 			}
 		}
@@ -1391,8 +1403,9 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 		 */
 		if (op & BUS_DMASYNC_POSTREAD) {
 			while (bpage != NULL) {
-				dcache_inv_poc(bpage->vaddr, bpage->busaddr,
-				    bpage->datacount);
+				if ((dmat->flags & BUS_DMA_COHERENT) == 0)
+					dcache_inv_poc(bpage->vaddr,
+					    bpage->busaddr, bpage->datacount);
 				tempvaddr = 0;
 				datavaddr = bpage->datavaddr;
 				if (datavaddr == 0) {
@@ -1421,7 +1434,8 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 	if (map->flags & DMAMAP_COHERENT) {
 		if (op & BUS_DMASYNC_PREWRITE) {
 			dsb();
-			cpu_l2cache_drain_writebuf();
+			if ((dmat->flags & BUS_DMA_COHERENT) == 0)
+				cpu_l2cache_drain_writebuf();
 		}
 		return;
 	}