Introduce bus_dmamap_load_mbuf_sg(). Instead of taking a callback arg, this

cuts to the chase and fills in a provided s/g list.  This is meant to optimize
out the cost of the callback since the callback doesn't serve much purpose for
mbufs since mbuf loads will never be deferred.  This is just for amd64 and
i386 at the moment, other arches will be coming shortly.

5 files changed, 111 insertions, 26 deletions

diff --git a/sys/amd64/amd64/busdma_machdep.c b/sys/amd64/amd64/busdma_machdep.c
index 1654c1c..9d631f2 100644
--- a/sys/amd64/amd64/busdma_machdep.c
+++ b/sys/amd64/amd64/busdma_machdep.c
@@ -562,10 +562,10 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
 			pmap_t pmap,
 			int flags,
 			bus_addr_t *lastaddrp,
+			bus_dma_segment_t *segs,
 			int *segp,
 			int first)
 {
-	bus_dma_segment_t *segs;
 	bus_size_t sgsize;
 	bus_addr_t curaddr, lastaddr, baddr, bmask;
 	vm_offset_t vaddr;
@@ -573,8 +573,6 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
 	int needbounce = 0;
 	int seg;
 
-	segs = dmat->segments;
-
 	if (map == NULL)
 		map = &nobounce_dmamap;
 
@@ -714,7 +712,7 @@ bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
 	}
 
 	error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags,
-	     &lastaddr, &nsegs, 1);
+	     &lastaddr, dmat->segments, &nsegs, 1);
 
 	if (error == EINPROGRESS) {
 		CTR3(KTR_BUSDMA, "bus_dmamap_load: tag %p tag flags 0x%x "
@@ -727,8 +725,8 @@ bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
 	else
 		(*callback)(callback_arg, dmat->segments, nsegs + 1, 0);
 
-	CTR2(KTR_BUSDMA, "bus_dmamap_load: tag %p tag flags 0x%x error 0",
-	    dmat, dmat->flags);
+	CTR3(KTR_BUSDMA, "bus_dmamap_load: tag %p tag flags 0x%x error 0 "
+	    "nsegs %d", dmat, dmat->flags, nsegs + 1);
 	return (0);
 }
 
@@ -759,7 +757,7 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
 				error = _bus_dmamap_load_buffer(dmat, map,
 						m->m_data, m->m_len,
 						NULL, flags, &lastaddr,
-						&nsegs, first);
+						dmat->segments, &nsegs, first);
 				first = 0;
 			}
 		}
@@ -774,8 +772,45 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
 		(*callback)(callback_arg, dmat->segments,
 			    nsegs+1, m0->m_pkthdr.len, error);
 	}
-	CTR3(KTR_BUSDMA, "bus_dmamap_load_mbuf: tag %p tag flags 0x%x "
-	    "error %d", dmat, dmat->flags, error);
+	CTR4(KTR_BUSDMA, "bus_dmamap_load_mbuf: tag %p tag flags 0x%x "
+	    "error %d nsegs %d", dmat, dmat->flags, error, nsegs + 1);
+	return (error);
+}
+
+int
+bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
+			struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
+			int flags)
+{
+	int error;
+
+	M_ASSERTPKTHDR(m0);
+
+	flags |= BUS_DMA_NOWAIT;
+	*nsegs = 0;
+	error = 0;
+	if (m0->m_pkthdr.len <= dmat->maxsize) {
+		int first = 1;
+		bus_addr_t lastaddr = 0;
+		struct mbuf *m;
+
+		for (m = m0; m != NULL && error == 0; m = m->m_next) {
+			if (m->m_len > 0) {
+				error = _bus_dmamap_load_buffer(dmat, map,
+						m->m_data, m->m_len,
+						NULL, flags, &lastaddr,
+						segs, nsegs, first);
+				first = 0;
+			}
+		}
+	} else {
+		error = EINVAL;
+	}
+
+	/* XXX FIXME: Having to increment nsegs is really annoying */
+	++*nsegs;
+	CTR4(KTR_BUSDMA, "bus_dmamap_load_mbuf: tag %p tag flags 0x%x "
+	    "error %d nsegs %d", dmat, dmat->flags, error, *nsegs);
 	return (error);
 }
 
@@ -819,8 +854,8 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
 
 		if (minlen > 0) {
 			error = _bus_dmamap_load_buffer(dmat, map,
-					addr, minlen,
-					pmap, flags, &lastaddr, &nsegs, first);
+					addr, minlen, pmap, flags, &lastaddr,
+					dmat->segments, &nsegs, first);
 			first = 0;
 
 			resid -= minlen;
@@ -834,8 +869,8 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
 		(*callback)(callback_arg, dmat->segments,
 			    nsegs+1, uio->uio_resid, error);
 	}
-	CTR3(KTR_BUSDMA, "bus_dmamap_load_uio: tag %p tag flags 0x%x "
-	    "error %d", dmat, dmat->flags, error);
+	CTR4(KTR_BUSDMA, "bus_dmamap_load_uio: tag %p tag flags 0x%x "
+	    "error %d nsegs %d", dmat, dmat->flags, error, nsegs + 1);
 	return (error);
 }
 
diff --git a/sys/amd64/include/bus_dma.h b/sys/amd64/include/bus_dma.h
index 7b94b88..fe02855 100644
--- a/sys/amd64/include/bus_dma.h
+++ b/sys/amd64/include/bus_dma.h
@@ -230,6 +230,11 @@ int bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
 			 struct mbuf *mbuf,
 			 bus_dmamap_callback2_t *callback, void *callback_arg,
 			 int flags);
+
+int bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
+			    struct mbuf *mbuf, bus_dma_segment_t *segs,
+			    int *nsegs, int flags);
+
 /*
  * Like bus_dmamap_load but for uios.  Note the use of the
  * bus_dmamap_callback2_t interface.
diff --git a/sys/i386/i386/busdma_machdep.c b/sys/i386/i386/busdma_machdep.c
index 1654c1c..9d631f2 100644
--- a/sys/i386/i386/busdma_machdep.c
+++ b/sys/i386/i386/busdma_machdep.c
@@ -562,10 +562,10 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
 			pmap_t pmap,
 			int flags,
 			bus_addr_t *lastaddrp,
+			bus_dma_segment_t *segs,
 			int *segp,
 			int first)
 {
-	bus_dma_segment_t *segs;
 	bus_size_t sgsize;
 	bus_addr_t curaddr, lastaddr, baddr, bmask;
 	vm_offset_t vaddr;
@@ -573,8 +573,6 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
 	int needbounce = 0;
 	int seg;
 
-	segs = dmat->segments;
-
 	if (map == NULL)
 		map = &nobounce_dmamap;
 
@@ -714,7 +712,7 @@ bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
 	}
 
 	error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags,
-	     &lastaddr, &nsegs, 1);
+	     &lastaddr, dmat->segments, &nsegs, 1);
 
 	if (error == EINPROGRESS) {
 		CTR3(KTR_BUSDMA, "bus_dmamap_load: tag %p tag flags 0x%x "
@@ -727,8 +725,8 @@ bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
 	else
 		(*callback)(callback_arg, dmat->segments, nsegs + 1, 0);
 
-	CTR2(KTR_BUSDMA, "bus_dmamap_load: tag %p tag flags 0x%x error 0",
-	    dmat, dmat->flags);
+	CTR3(KTR_BUSDMA, "bus_dmamap_load: tag %p tag flags 0x%x error 0 "
+	    "nsegs %d", dmat, dmat->flags, nsegs + 1);
 	return (0);
 }
 
@@ -759,7 +757,7 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
 				error = _bus_dmamap_load_buffer(dmat, map,
 						m->m_data, m->m_len,
 						NULL, flags, &lastaddr,
-						&nsegs, first);
+						dmat->segments, &nsegs, first);
 				first = 0;
 			}
 		}
@@ -774,8 +772,45 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
 		(*callback)(callback_arg, dmat->segments,
 			    nsegs+1, m0->m_pkthdr.len, error);
 	}
-	CTR3(KTR_BUSDMA, "bus_dmamap_load_mbuf: tag %p tag flags 0x%x "
-	    "error %d", dmat, dmat->flags, error);
+	CTR4(KTR_BUSDMA, "bus_dmamap_load_mbuf: tag %p tag flags 0x%x "
+	    "error %d nsegs %d", dmat, dmat->flags, error, nsegs + 1);
+	return (error);
+}
+
+int
+bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
+			struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
+			int flags)
+{
+	int error;
+
+	M_ASSERTPKTHDR(m0);
+
+	flags |= BUS_DMA_NOWAIT;
+	*nsegs = 0;
+	error = 0;
+	if (m0->m_pkthdr.len <= dmat->maxsize) {
+		int first = 1;
+		bus_addr_t lastaddr = 0;
+		struct mbuf *m;
+
+		for (m = m0; m != NULL && error == 0; m = m->m_next) {
+			if (m->m_len > 0) {
+				error = _bus_dmamap_load_buffer(dmat, map,
+						m->m_data, m->m_len,
+						NULL, flags, &lastaddr,
+						segs, nsegs, first);
+				first = 0;
+			}
+		}
+	} else {
+		error = EINVAL;
+	}
+
+	/* XXX FIXME: Having to increment nsegs is really annoying */
+	++*nsegs;
+	CTR4(KTR_BUSDMA, "bus_dmamap_load_mbuf: tag %p tag flags 0x%x "
+	    "error %d nsegs %d", dmat, dmat->flags, error, *nsegs);
 	return (error);
 }
 
@@ -819,8 +854,8 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
 
 		if (minlen > 0) {
 			error = _bus_dmamap_load_buffer(dmat, map,
-					addr, minlen,
-					pmap, flags, &lastaddr, &nsegs, first);
+					addr, minlen, pmap, flags, &lastaddr,
+					dmat->segments, &nsegs, first);
 			first = 0;
 
 			resid -= minlen;
@@ -834,8 +869,8 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
 		(*callback)(callback_arg, dmat->segments,
 			    nsegs+1, uio->uio_resid, error);
 	}
-	CTR3(KTR_BUSDMA, "bus_dmamap_load_uio: tag %p tag flags 0x%x "
-	    "error %d", dmat, dmat->flags, error);
+	CTR4(KTR_BUSDMA, "bus_dmamap_load_uio: tag %p tag flags 0x%x "
+	    "error %d nsegs %d", dmat, dmat->flags, error, nsegs + 1);
 	return (error);
 }
 
diff --git a/sys/i386/include/bus_dma.h b/sys/i386/include/bus_dma.h
index b9c17de..03d6085 100644
--- a/sys/i386/include/bus_dma.h
+++ b/sys/i386/include/bus_dma.h
@@ -230,6 +230,11 @@ int bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
 			 struct mbuf *mbuf,
 			 bus_dmamap_callback2_t *callback, void *callback_arg,
 			 int flags);
+
+int bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
+			    struct mbuf *mbuf, bus_dma_segment_t *segs,
+			    int *nsegs, int flags);
+
 /*
  * Like bus_dmamap_load but for uios.  Note the use of the
  * bus_dmamap_callback2_t interface.
diff --git a/sys/sys/bus_dma.h b/sys/sys/bus_dma.h
index b9c17de..03d6085 100644
--- a/sys/sys/bus_dma.h
+++ b/sys/sys/bus_dma.h
@@ -230,6 +230,11 @@ int bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
 			 struct mbuf *mbuf,
 			 bus_dmamap_callback2_t *callback, void *callback_arg,
 			 int flags);
+
+int bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
+			    struct mbuf *mbuf, bus_dma_segment_t *segs,
+			    int *nsegs, int flags);
+
 /*
  * Like bus_dmamap_load but for uios.  Note the use of the
  * bus_dmamap_callback2_t interface.