Create an architecture-agnostic buffer pool manager that uses uma(9) to

manage a set of power-of-2 sized buffers for bus_dmamem_alloc().

This allows the caller to provide the back-end allocator uma allocator,
allowing full control of the memory pages backing the pool.  For
convenience, it provides an optional builtin allocator that provides pages
allocated with the VM_MEMATTR_UNCACHEABLE attribute, for managing pools of
DMA buffers for BUS_DMA_COHERENT or BUS_DMA_NOCACHE.

This also allows the caller to specify a minimum alignment, and it ensures
that all buffers start on a boundary and have a length that's a multiple of
that value, to avoid using buffers that trigger partial cache line flushes.

Submitted by:	Ian Lepore <freebsd@damnhippie.dyndns.org>

2 files changed, 292 insertions, 0 deletions

diff --git a/sys/kern/subr_busdma_bufalloc.c b/sys/kern/subr_busdma_bufalloc.c
new file mode 100644
index 0000000..9406d95
--- /dev/null
+++ b/sys/kern/subr_busdma_bufalloc.c
@@ -0,0 +1,174 @@
+/*-
+ * Copyright (c) 2012 Ian Lepore
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * Buffer allocation support routines for bus_dmamem_alloc implementations.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/busdma_bufalloc.h>
+#include <sys/malloc.h>
+
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/uma.h>
+
+/*
+ * We manage buffer zones up to a page in size.  Buffers larger than a page can
+ * be managed by one of the kernel's page-oriented memory allocation routines as
+ * efficiently as what we can do here.  Also, a page is the largest size for
+ * which we can g'tee contiguity when using uma, and contiguity is one of the
+ * requirements we have to fulfill.
+ */
+#define	MIN_ZONE_BUFSIZE	32
+#define	MAX_ZONE_BUFSIZE	PAGE_SIZE
+
+/*
+ * The static array of 12 bufzones is big enough to handle all the zones for the
+ * smallest supported allocation size of 32 through the largest supported page
+ * size of 64K.  If you up the biggest page size number, up the array size too.
+ * Basically the size of the array needs to be log2(maxsize)-log2(minsize)+1,
+ * but I don't know of an easy way to express that as a compile-time constant.
+ */
+#if PAGE_SIZE > 65536
+#error Unsupported page size
+#endif
+
+struct busdma_bufalloc {
+	bus_size_t		min_size;
+	size_t			num_zones;
+	struct busdma_bufzone	buf_zones[12];
+};
+
+busdma_bufalloc_t 
+busdma_bufalloc_create(const char *name, bus_size_t minimum_alignment,
+    uma_alloc alloc_func, uma_free free_func, u_int32_t zcreate_flags)
+{
+	struct busdma_bufalloc *ba;
+	struct busdma_bufzone *bz;
+	int i;
+	bus_size_t cursize;
+
+	ba = malloc(sizeof(struct busdma_bufalloc), M_DEVBUF, 
+	    M_ZERO | M_WAITOK);
+
+	ba->min_size = MAX(MIN_ZONE_BUFSIZE, minimum_alignment);
+
+	/*
+	 * Each uma zone is created with an alignment of size-1, meaning that
+	 * the alignment is equal to the size (I.E., 64 byte buffers are aligned
+	 * to 64 byte boundaries, etc).  This allows for a fast efficient test
+	 * when deciding whether a pool buffer meets the constraints of a given
+	 * tag used for allocation: the buffer is usable if tag->alignment <=
+	 * bufzone->size.
+	 */
+	for (i = 0, bz = ba->buf_zones, cursize = ba->min_size;
+	    i < nitems(ba->buf_zones) && cursize <= MAX_ZONE_BUFSIZE;
+	    ++i, ++bz, cursize <<= 1) {
+		snprintf(bz->name, sizeof(bz->name), "dma %.10s %lu",
+		    name, cursize);
+		bz->size = cursize;
+		bz->umazone = uma_zcreate(bz->name, bz->size,
+		    NULL, NULL, NULL, NULL, bz->size - 1, zcreate_flags);
+		if (bz->umazone == NULL) {
+			busdma_bufalloc_destroy(ba);
+			return (NULL);
+		}
+		if (alloc_func != NULL)
+			uma_zone_set_allocf(bz->umazone, alloc_func);
+		if (free_func != NULL)
+			uma_zone_set_freef(bz->umazone, free_func);
+		++ba->num_zones;
+	}
+
+	return (ba);
+}
+
+void 
+busdma_bufalloc_destroy(busdma_bufalloc_t ba)
+{
+	struct busdma_bufzone *bz;
+	int i;
+
+	if (ba == NULL)
+		return;
+
+	for (i = 0, bz = ba->buf_zones; i < ba->num_zones; ++i, ++bz) {
+		uma_zdestroy(bz->umazone);
+	}
+
+	free(ba, M_DEVBUF);
+}
+
+struct busdma_bufzone * 
+busdma_bufalloc_findzone(busdma_bufalloc_t ba, bus_size_t size)
+{
+	struct busdma_bufzone *bz;
+	int i;
+
+	if (size > MAX_ZONE_BUFSIZE)
+		return (NULL);
+
+	for (i = 0, bz = ba->buf_zones; i < ba->num_zones; ++i, ++bz) {
+		if (bz->size >= size)
+			return (bz);
+	}
+
+	panic("Didn't find a buffer zone of the right size");
+}
+
+void *
+busdma_bufalloc_alloc_uncacheable(uma_zone_t zone, int size, u_int8_t *pflag,
+    int wait)
+{
+#ifdef VM_MEMATTR_UNCACHEABLE
+
+	/* Inform UMA that this allocator uses kernel_map/object. */
+	*pflag = UMA_SLAB_KERNEL;
+
+	return ((void *)kmem_alloc_attr(kernel_map, size, wait, 0,
+	    BUS_SPACE_MAXADDR, VM_MEMATTR_UNCACHEABLE));
+
+#else
+
+	panic("VM_MEMATTR_UNCACHEABLE unavailable");
+
+#endif	/* VM_MEMATTR_UNCACHEABLE */
+}
+
+void 
+busdma_bufalloc_free_uncacheable(void *item, int size, u_int8_t pflag)
+{
+
+	kmem_free(kernel_map, (vm_offset_t)item, size);
+}
+
diff --git a/sys/sys/busdma_bufalloc.h b/sys/sys/busdma_bufalloc.h
new file mode 100644
index 0000000..f5ec32f
--- /dev/null
+++ b/sys/sys/busdma_bufalloc.h
@@ -0,0 +1,118 @@
+/*-
+ * Copyright (c) 2012 Ian Lepore
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * $FreeBSD$
+ */
+
+/*
+ * A buffer pool manager, for use by a platform's busdma implementation.
+ */
+
+#ifndef _MACHINE_BUSDMA_BUFALLOC_H_
+#define _MACHINE_BUSDMA_BUFALLOC_H_
+
+#include <machine/bus.h>
+#include <vm/uma.h>
+
+/*
+ * Information about a buffer zone, returned by busdma_bufalloc_findzone().
+ */
+struct busdma_bufzone {
+	bus_size_t	size;
+	uma_zone_t	umazone;
+	char		name[24];
+};
+
+/*
+ * Opaque handle type returned by busdma_bufalloc_create().
+ */
+struct busdma_bufalloc;
+typedef struct busdma_bufalloc *busdma_bufalloc_t;
+
+/*
+ * Create an allocator that manages a pool of DMA buffers.
+ *
+ * The allocator manages a collection of uma(9) zones of buffers in power-of-two
+ * sized increments ranging from minimum_alignment to the platform's PAGE_SIZE.
+ * The buffers within each zone are aligned on boundaries corresponding to the
+ * buffer size, and thus by implication each buffer is contiguous within a page
+ * and does not cross a power of two boundary larger than the buffer size.
+ * These rules are intended to make it easy for a busdma implementation to
+ * check whether a tag's constraints allow use of a buffer from the allocator.
+ *
+ * minimum_alignment is also the minimum buffer allocation size.  For platforms
+ * with software-assisted cache coherency, this is typically the data cache line
+ * size (and MUST not be smaller than the cache line size).
+ *
+ * name appears in zone stats as 'dma name nnnnn' where 'dma' is fixed and
+ * 'nnnnn' is the size of buffers in that zone.
+ *
+ * If if the alloc/free function pointers are NULL, the regular uma internal
+ * allocators are used (I.E., you get "plain old kernel memory").  On a platform
+ * with an exclusion zone that applies to all DMA operations, a custom allocator
+ * could be used to ensure no buffer memory is ever allocated from that zone,
+ * allowing the bus_dmamem_alloc() implementation to make the assumption that
+ * buffers provided by the allocation could never lead to the need for a bounce.
+ */
+busdma_bufalloc_t busdma_bufalloc_create(const char *name,
+    bus_size_t minimum_alignment,
+    uma_alloc uma_alloc_func, uma_free uma_free_func,
+    u_int32_t uma_zcreate_flags);
+
+/*
+ * Destroy an allocator created by busdma_bufalloc_create().
+ * Safe to call with a NULL pointer.
+ */
+void busdma_bufalloc_destroy(busdma_bufalloc_t ba);
+
+/*
+ * Return a pointer to the busdma_bufzone that should be used to allocate or
+ * free a buffer of the given size.  Returns NULL if the size is larger than the
+ * largest zone handled by the allocator.
+ */
+struct busdma_bufzone * busdma_bufalloc_findzone(busdma_bufalloc_t ba,
+    bus_size_t size);
+
+/*
+ * These built-in allocation routines are available for managing a pools of
+ * uncacheable memory on platforms that support VM_MEMATTR_UNCACHEABLE.
+ *
+ * Allocation is done using kmem_alloc_attr() with these parameters:
+ *   lowaddr  = 0
+ *   highaddr = BUS_SPACE_MAXADDR
+ *   memattr  = VM_MEMATTR_UNCACHEABLE.
+ *
+ * If your platform has no exclusion region (lowaddr/highaddr), and its pmap
+ * routines support pmap_page_set_memattr() and the VM_MEMATTR_UNCACHEABLE flag
+ * you can probably use these when you need uncacheable buffers.
+ */
+void * busdma_bufalloc_alloc_uncacheable(uma_zone_t zone, int size, 
+    u_int8_t *pflag, int wait);
+void  busdma_bufalloc_free_uncacheable(void *item, int size, u_int8_t pflag);
+
+#endif	/* _MACHINE_BUSDMA_BUFALLOC_H_ */
+